31 #include "../include/FFmpegReader.h" 36 : last_frame(0), is_seeking(0), seeking_pts(0), seeking_frame(0), seek_count(0),
37 audio_pts_offset(99999), video_pts_offset(99999), path(path), is_video_seek(
true), check_interlace(
false),
38 check_fps(
false), enable_seek(
true), is_open(
false), seek_audio_frame_found(0), seek_video_frame_found(0),
39 prev_samples(0), prev_pts(0), pts_total(0), pts_counter(0), is_duration_known(
false), largest_frame_processed(0),
40 current_video_frame(0), has_missing_frames(
false), num_packets_since_video_frame(0), num_checks_since_final(0) {
44 avcodec_register_all();
47 working_cache.SetMaxBytesFromInfo(
OPEN_MP_NUM_PROCESSORS * 30, info.width, info.height, info.sample_rate, info.channels);
48 missing_frames.SetMaxBytesFromInfo(
OPEN_MP_NUM_PROCESSORS * 2, info.width, info.height, info.sample_rate, info.channels);
49 final_cache.SetMaxBytesFromInfo(
OPEN_MP_NUM_PROCESSORS * 2, info.width, info.height, info.sample_rate, info.channels);
57 : last_frame(0), is_seeking(0), seeking_pts(0), seeking_frame(0), seek_count(0),
58 audio_pts_offset(99999), video_pts_offset(99999), path(path), is_video_seek(
true), check_interlace(
false),
59 check_fps(
false), enable_seek(
true), is_open(
false), seek_audio_frame_found(0), seek_video_frame_found(0),
60 prev_samples(0), prev_pts(0), pts_total(0), pts_counter(0), is_duration_known(
false), largest_frame_processed(0),
61 current_video_frame(0), has_missing_frames(
false), num_packets_since_video_frame(0), num_checks_since_final(0) {
65 avcodec_register_all();
68 working_cache.SetMaxBytesFromInfo(
OPEN_MP_NUM_PROCESSORS * 30, info.width, info.height, info.sample_rate, info.channels);
69 missing_frames.SetMaxBytesFromInfo(
OPEN_MP_NUM_PROCESSORS * 2, info.width, info.height, info.sample_rate, info.channels);
70 final_cache.SetMaxBytesFromInfo(
OPEN_MP_NUM_PROCESSORS * 2, info.width, info.height, info.sample_rate, info.channels);
89 if (abs(location.
frame - frame) >= 2)
93 int sample_diff = abs(location.
sample_start - sample_start);
94 if (location.
frame == frame && sample_diff >= 0 && sample_diff <= amount)
99 if (location.
frame > frame)
102 sample_diff = (samples_per_frame - sample_start) + location.
sample_start;
103 if (sample_diff >= 0 && sample_diff <= amount)
108 if (location.
frame < frame)
111 sample_diff = (samples_per_frame - location.
sample_start) + sample_start;
112 if (sample_diff >= 0 && sample_diff <= amount)
129 if (avformat_open_input(&pFormatCtx, path.c_str(), NULL, NULL) != 0)
130 throw InvalidFile(
"File could not be opened.", path);
133 if (avformat_find_stream_info(pFormatCtx, NULL) < 0)
134 throw NoStreamsFound(
"No streams found in file.", path);
139 for (
unsigned int i = 0; i < pFormatCtx->nb_streams; i++)
142 if (pFormatCtx->streams[i]->codec->codec_type == AVMEDIA_TYPE_VIDEO && videoStream < 0) {
146 if (pFormatCtx->streams[i]->codec->codec_type == AVMEDIA_TYPE_AUDIO && audioStream < 0) {
150 if (videoStream == -1 && audioStream == -1)
151 throw NoStreamsFound(
"No video or audio streams found in this file.", path);
154 if (videoStream != -1)
160 pStream = pFormatCtx->streams[videoStream];
161 pCodecCtx = pFormatCtx->streams[videoStream]->codec;
167 AVCodec *pCodec = avcodec_find_decoder(pCodecCtx->codec_id);
168 if (pCodec == NULL) {
169 throw InvalidCodec(
"A valid video codec could not be found for this file.", path);
172 if (avcodec_open2(pCodecCtx, pCodec, NULL) < 0)
173 throw InvalidCodec(
"A video codec was found, but could not be opened.", path);
180 if (audioStream != -1)
186 aStream = pFormatCtx->streams[audioStream];
187 aCodecCtx = pFormatCtx->streams[audioStream]->codec;
193 AVCodec *aCodec = avcodec_find_decoder(aCodecCtx->codec_id);
194 if (aCodec == NULL) {
195 throw InvalidCodec(
"A valid audio codec could not be found for this file.", path);
198 if (avcodec_open2(aCodecCtx, aCodec, NULL) < 0)
199 throw InvalidCodec(
"An audio codec was found, but could not be opened.", path);
206 previous_packet_location.
frame = -1;
227 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::Close",
"", -1,
"", -1,
"", -1,
"", -1,
"", -1,
"", -1);
232 avcodec_flush_buffers(pCodecCtx);
233 avcodec_close(pCodecCtx);
237 avcodec_flush_buffers(aCodecCtx);
238 avcodec_close(aCodecCtx);
243 working_cache.
Clear();
244 missing_frames.
Clear();
249 processed_video_frames.clear();
250 processed_audio_frames.clear();
251 processing_video_frames.clear();
252 processing_audio_frames.clear();
253 missing_audio_frames.clear();
254 missing_video_frames.clear();
255 missing_audio_frames_source.clear();
256 missing_video_frames_source.clear();
257 checked_frames.clear();
261 avformat_close_input(&pFormatCtx);
262 av_freep(&pFormatCtx);
266 largest_frame_processed = 0;
267 seek_audio_frame_found = 0;
268 seek_video_frame_found = 0;
269 current_video_frame = 0;
270 has_missing_frames =
false;
274 void FFmpegReader::UpdateAudioInfo()
278 info.
file_size = pFormatCtx->pb ? avio_size(pFormatCtx->pb) : -1;
281 if (aCodecCtx->channel_layout == 0)
282 aCodecCtx->channel_layout = av_get_default_channel_layout( aCodecCtx->channels );
292 if (aStream->duration > 0.0f && aStream->duration >
info.
duration)
318 void FFmpegReader::UpdateVideoInfo()
322 info.
file_size = pFormatCtx->pb ? avio_size(pFormatCtx->pb) : -1;
334 if (pStream->sample_aspect_ratio.num != 0)
339 else if (pCodecCtx->sample_aspect_ratio.num != 0)
385 is_duration_known =
false;
390 is_duration_known =
true;
416 throw ReaderClosed(
"The FFmpegReader is closed. Call Open() before calling this method.", path);
419 if (requested_frame < 1)
425 throw InvalidFile(
"Could not detect the duration of the video or audio stream.", path);
428 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::GetFrame",
"requested_frame", requested_frame,
"last_frame", last_frame,
"", -1,
"", -1,
"", -1,
"", -1);
434 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::GetFrame",
"returned cached frame", requested_frame,
"", -1,
"", -1,
"", -1,
"", -1,
"", -1);
445 if (has_missing_frames)
446 CheckMissingFrame(requested_frame);
450 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::GetFrame",
"returned cached frame on 2nd look", requested_frame,
"", -1,
"", -1,
"", -1,
"", -1,
"", -1);
461 if (last_frame == 0 && requested_frame != 1)
466 long int diff = requested_frame - last_frame;
467 if (diff >= 1 && diff <= 20)
470 return ReadStream(requested_frame);
477 Seek(requested_frame);
487 return ReadStream(requested_frame);
494 tr1::shared_ptr<Frame> FFmpegReader::ReadStream(
long int requested_frame)
497 bool end_of_stream =
false;
498 bool check_seek =
false;
499 bool frame_finished =
false;
500 int packet_error = -1;
503 int packets_processed = 0;
509 omp_set_nested(
true);
512 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ReadStream",
"requested_frame", requested_frame,
"OPEN_MP_NUM_PROCESSORS",
OPEN_MP_NUM_PROCESSORS,
"", -1,
"", -1,
"", -1,
"", -1);
522 packet_error = GetNextPacket();
525 while (processing_video_frames.size() + processing_audio_frames.size() >= minimum_packets)
529 if (packet_error < 0)
532 end_of_stream =
true;
537 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ReadStream (GetNextPacket)",
"requested_frame", requested_frame,
"processing_video_frames.size()", processing_video_frames.size(),
"processing_audio_frames.size()", processing_audio_frames.size(),
"minimum_packets", minimum_packets,
"packets_processed", packets_processed,
"", -1);
543 num_packets_since_video_frame = 0;
547 #pragma omp critical (openshot_seek) 548 check_seek = CheckSeek(
true);
554 RemoveAVPacket(packet);
561 frame_finished = GetAVFrame();
567 UpdatePTSOffset(
true);
570 ProcessVideoPacket(requested_frame);
575 else if (
info.
has_audio && packet->stream_index == audioStream)
578 num_packets_since_video_frame++;
582 #pragma omp critical (openshot_seek) 583 check_seek = CheckSeek(
false);
589 RemoveAVPacket(packet);
596 UpdatePTSOffset(
false);
606 bool is_cache_found =
false;
609 CheckMissingFrame(requested_frame);
612 CheckWorkingFrames(
false, requested_frame);
622 if ((is_cache_found && packets_processed >= minimum_packets))
631 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ReadStream (Completed)",
"packets_processed", packets_processed,
"end_of_stream", end_of_stream,
"largest_frame_processed", largest_frame_processed,
"Working Cache Count", working_cache.
Count(),
"", -1,
"", -1);
636 CheckWorkingFrames(end_of_stream, requested_frame);
653 tr1::shared_ptr<Frame> f = CreateFrame(largest_frame_processed);
662 int FFmpegReader::GetNextPacket()
664 int found_packet = 0;
665 AVPacket *next_packet =
new AVPacket();
666 found_packet = av_read_frame(pFormatCtx, next_packet);
668 if (found_packet >= 0)
671 packet = next_packet;
676 av_free_packet(next_packet);
685 bool FFmpegReader::GetAVFrame()
687 int frameFinished = -1;
691 #pragma omp critical (packet_cache) 692 avcodec_decode_video2(pCodecCtx, next_frame, &frameFinished, packet);
699 AVPicture *copyFrame =
new AVPicture();
701 av_picture_copy(copyFrame, (AVPicture *) next_frame, pCodecCtx->pix_fmt,
info.
width,
info.
height);
703 #pragma omp critical (packet_cache) 706 frames[copyFrame] = copyFrame;
707 pFrame = frames[copyFrame];
711 if (!check_interlace)
713 check_interlace =
true;
722 RemoveAVPacket(packet);
729 return frameFinished;
733 bool FFmpegReader::CheckSeek(
bool is_video)
740 if ((is_video_seek && !seek_video_frame_found) || (!is_video_seek && !seek_audio_frame_found))
748 long int max_seeked_frame = seek_audio_frame_found;
749 if (seek_video_frame_found > max_seeked_frame)
750 max_seeked_frame = seek_video_frame_found;
753 if (max_seeked_frame >= seeking_frame)
756 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::CheckSeek (Too far, seek again)",
"is_video_seek", is_video_seek,
"max_seeked_frame", max_seeked_frame,
"seeking_frame", seeking_frame,
"seeking_pts", seeking_pts,
"seek_video_frame_found", seek_video_frame_found,
"seek_audio_frame_found", seek_audio_frame_found);
759 Seek(seeking_frame - (20 * seek_count * seek_count));
764 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::CheckSeek (Successful)",
"is_video_seek", is_video_seek,
"current_pts", packet->pts,
"seeking_pts", seeking_pts,
"seeking_frame", seeking_frame,
"seek_video_frame_found", seek_video_frame_found,
"seek_audio_frame_found", seek_audio_frame_found);
778 void FFmpegReader::ProcessVideoPacket(
long int requested_frame)
781 long int current_frame = ConvertVideoPTStoFrame(GetVideoPTS());
784 if (!seek_video_frame_found && is_seeking)
785 seek_video_frame_found = current_frame;
788 if ((current_frame < (requested_frame - 20)) or (current_frame == -1))
791 RemoveAVFrame(pFrame);
792 RemoveAVPacket(packet);
795 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessVideoPacket (Skipped)",
"requested_frame", requested_frame,
"current_frame", current_frame,
"", -1,
"", -1,
"", -1,
"", -1);
802 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessVideoPacket (Before)",
"requested_frame", requested_frame,
"current_frame", current_frame,
"", -1,
"", -1,
"", -1,
"", -1);
809 AVPacket *my_packet = packet;
810 AVPicture *my_frame = frames[pFrame];
814 processing_video_frames[current_frame] = current_frame;
816 #pragma omp task firstprivate(current_frame, my_packet, my_frame, height, width, video_length, pix_fmt) 819 AVFrame *pFrameRGB = NULL;
821 uint8_t *buffer = NULL;
825 if (pFrameRGB == NULL)
826 throw OutOfBoundsFrame(
"Convert Image Broke!", current_frame, video_length);
831 int original_height = height;
834 float ratio = float(width) / float(height);
835 int possible_width = round(
max_height * ratio);
836 int possible_height = round(
max_width / ratio);
840 width = possible_width;
845 height = possible_height;
850 numBytes = avpicture_get_size(
PIX_FMT_RGBA, width, height);
851 buffer = (uint8_t *) av_malloc(numBytes *
sizeof(uint8_t));
856 avpicture_fill((AVPicture *) pFrameRGB, buffer,
PIX_FMT_RGBA, width, height);
858 SwsContext *img_convert_ctx = sws_getContext(
info.
width,
info.
height, pCodecCtx->pix_fmt, width,
862 sws_scale(img_convert_ctx, my_frame->data, my_frame->linesize, 0,
863 original_height, pFrameRGB->data, pFrameRGB->linesize);
866 tr1::shared_ptr<Frame> f = CreateFrame(current_frame);
869 f->AddImage(width, height, 4, QImage::Format_RGBA8888, buffer);
872 working_cache.
Add(f);
875 last_video_frame = f;
882 RemoveAVFrame(my_frame);
883 RemoveAVPacket(my_packet);
884 sws_freeContext(img_convert_ctx);
889 processing_video_frames.erase(current_frame);
890 processed_video_frames[current_frame] = current_frame;
894 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessVideoPacket (After)",
"requested_frame", requested_frame,
"current_frame", current_frame,
"f->number", f->number,
"", -1,
"", -1,
"", -1);
901 void FFmpegReader::ProcessAudioPacket(
long int requested_frame,
long int target_frame,
int starting_sample)
904 if (!seek_audio_frame_found && is_seeking)
905 seek_audio_frame_found = target_frame;
908 if (target_frame < (requested_frame - 20))
911 RemoveAVPacket(packet);
914 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessAudioPacket (Skipped)",
"requested_frame", requested_frame,
"target_frame", target_frame,
"starting_sample", starting_sample,
"", -1,
"", -1,
"", -1);
921 AVPacket *my_packet = packet;
924 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessAudioPacket (Before)",
"requested_frame", requested_frame,
"target_frame", target_frame,
"starting_sample", starting_sample,
"", -1,
"", -1,
"", -1);
927 int frame_finished = 0;
931 int packet_samples = 0;
936 int used = avcodec_decode_audio4(aCodecCtx, audio_frame, &frame_finished, my_packet);
938 if (frame_finished) {
941 int planar = av_sample_fmt_is_planar(aCodecCtx->sample_fmt);
943 data_size = av_samples_get_buffer_size(&plane_size,
945 audio_frame->nb_samples,
946 aCodecCtx->sample_fmt, 1);
949 packet_samples = audio_frame->nb_samples * aCodecCtx->channels;
953 int pts_remaining_samples = packet_samples /
info.
channels;
956 int adjusted_pts = packet->pts + audio_pts_offset;
961 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessAudioPacket (Decode Info A)",
"pts_counter", pts_counter,
"PTS", adjusted_pts,
"Offset", audio_pts_offset,
"PTS Diff", adjusted_pts - prev_pts,
"Samples", pts_remaining_samples,
"Sample PTS ratio",
float(adjusted_pts - prev_pts) / pts_remaining_samples);
962 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessAudioPacket (Decode Info B)",
"Sample Diff", pts_remaining_samples - prev_samples - prev_pts,
"Total", pts_total,
"PTS Seconds", audio_seconds,
"Sample Seconds", sample_seconds,
"Seconds Diff", audio_seconds - sample_seconds,
"raw samples", packet_samples);
965 prev_pts = adjusted_pts;
966 pts_total += pts_remaining_samples;
968 prev_samples = pts_remaining_samples;
973 processing_audio_frames.insert(pair<int, int>(previous_packet_location.
frame, previous_packet_location.
frame));
976 while (pts_remaining_samples)
982 int samples = samples_per_frame - previous_packet_location.
sample_start;
983 if (samples > pts_remaining_samples)
984 samples = pts_remaining_samples;
987 pts_remaining_samples -= samples;
989 if (pts_remaining_samples > 0) {
991 previous_packet_location.
frame++;
997 processing_audio_frames.insert(pair<int, int>(previous_packet_location.
frame, previous_packet_location.
frame));
1010 #pragma omp task firstprivate(requested_frame, target_frame, starting_sample, my_packet, audio_frame) 1015 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessAudioPacket (ReSample)",
"packet_samples", packet_samples,
"info.channels",
info.
channels,
"info.sample_rate",
info.
sample_rate,
"aCodecCtx->sample_fmt", aCodecCtx->sample_fmt,
"AV_SAMPLE_FMT_S16", AV_SAMPLE_FMT_S16,
"", -1);
1020 audio_converted->nb_samples = audio_frame->nb_samples;
1021 av_samples_alloc(audio_converted->data, audio_converted->linesize,
info.
channels, audio_frame->nb_samples, AV_SAMPLE_FMT_S16, 0);
1023 AVAudioResampleContext *avr = NULL;
1028 avr = avresample_alloc_context();
1029 av_opt_set_int(avr,
"in_channel_layout", aCodecCtx->channel_layout, 0);
1030 av_opt_set_int(avr,
"out_channel_layout", aCodecCtx->channel_layout, 0);
1031 av_opt_set_int(avr,
"in_sample_fmt", aCodecCtx->sample_fmt, 0);
1032 av_opt_set_int(avr,
"out_sample_fmt", AV_SAMPLE_FMT_S16, 0);
1037 int r = avresample_open(avr);
1040 nb_samples = avresample_convert(avr,
1041 audio_converted->data,
1042 audio_converted->linesize[0],
1043 audio_converted->nb_samples,
1045 audio_frame->linesize[0],
1046 audio_frame->nb_samples);
1050 memcpy(audio_buf, audio_converted->data[0], audio_converted->nb_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) *
info.
channels);
1053 avresample_close(avr);
1054 avresample_free(&avr);
1058 av_free(audio_converted->data[0]);
1061 int starting_frame_number = -1;
1062 bool partial_frame =
true;
1063 for (
int channel_filter = 0; channel_filter <
info.
channels; channel_filter++)
1066 starting_frame_number = target_frame;
1067 int channel_buffer_size = packet_samples /
info.
channels;
1068 float *channel_buffer =
new float[channel_buffer_size];
1071 for (
int z = 0; z < channel_buffer_size; z++)
1072 channel_buffer[z] = 0.0f;
1078 for (
int sample = 0; sample < packet_samples; sample++)
1081 if (channel_filter == channel)
1084 channel_buffer[position] = audio_buf[sample] * (1.0f / (1 << 15));
1100 int start = starting_sample;
1101 int remaining_samples = channel_buffer_size;
1102 float *iterate_channel_buffer = channel_buffer;
1103 while (remaining_samples > 0)
1109 int samples = samples_per_frame - start;
1110 if (samples > remaining_samples)
1111 samples = remaining_samples;
1114 tr1::shared_ptr<Frame> f = CreateFrame(starting_frame_number);
1117 if (samples_per_frame == start + samples)
1118 partial_frame =
false;
1120 partial_frame =
true;
1124 f->AddAudio(
true, channel_filter, start, iterate_channel_buffer, samples, 0.98f);
1127 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessAudioPacket (f->AddAudio)",
"frame", starting_frame_number,
"start", start,
"samples", samples,
"channel", channel_filter,
"partial_frame", partial_frame,
"samples_per_frame", samples_per_frame);
1130 working_cache.
Add(f);
1133 remaining_samples -= samples;
1136 if (remaining_samples > 0)
1137 iterate_channel_buffer += samples;
1140 starting_frame_number++;
1147 delete[] channel_buffer;
1148 channel_buffer = NULL;
1149 iterate_channel_buffer = NULL;
1160 for (
long int f = target_frame; f < starting_frame_number; f++) {
1164 processing_audio_frames.erase(processing_audio_frames.find(f));
1167 if (processing_audio_frames.count(f) == 0)
1169 processed_audio_frames[f] = f;
1172 if (target_frame == starting_frame_number) {
1174 processing_audio_frames.erase(processing_audio_frames.find(target_frame));
1179 RemoveAVPacket(my_packet);
1182 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessAudioPacket (After)",
"requested_frame", requested_frame,
"starting_frame", target_frame,
"end_frame", starting_frame_number - 1,
"", -1,
"", -1,
"", -1);
1189 #pragma omp taskwait 1198 void FFmpegReader::Seek(
long int requested_frame)
throw(
TooManySeeks)
1201 if (requested_frame < 1)
1202 requested_frame = 1;
1207 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::Seek",
"requested_frame", requested_frame,
"seek_count", seek_count,
"last_frame", last_frame,
"processing_video_frames.size()", processing_video_frames.size(),
"processing_audio_frames.size()", processing_audio_frames.size(),
"", -1);
1210 while (processing_video_frames.size() + processing_audio_frames.size() > 0)
1214 working_cache.
Clear();
1215 missing_frames.
Clear();
1220 processing_audio_frames.clear();
1221 processing_video_frames.clear();
1222 processed_video_frames.clear();
1223 processed_audio_frames.clear();
1224 duplicate_video_frames.clear();
1225 missing_audio_frames.clear();
1226 missing_video_frames.clear();
1227 missing_audio_frames_source.clear();
1228 missing_video_frames_source.clear();
1229 checked_frames.clear();
1234 current_video_frame = 0;
1235 largest_frame_processed = 0;
1236 num_checks_since_final = 0;
1237 num_packets_since_video_frame = 0;
1238 has_missing_frames =
false;
1244 int buffer_amount = 6;
1245 if (requested_frame - buffer_amount < 20)
1253 if (seek_count == 1) {
1256 seeking_pts = ConvertFrameToVideoPTS(1);
1258 seek_audio_frame_found = 0;
1259 seek_video_frame_found = 0;
1264 bool seek_worked =
false;
1265 int64_t seek_target = 0;
1270 seek_target = ConvertFrameToVideoPTS(requested_frame - buffer_amount);
1272 fprintf(stderr,
"%s: error while seeking video stream\n", pFormatCtx->filename);
1276 is_video_seek =
true;
1284 seek_target = ConvertFrameToAudioPTS(requested_frame - buffer_amount);
1286 fprintf(stderr,
"%s: error while seeking audio stream\n", pFormatCtx->filename);
1290 is_video_seek =
false;
1300 avcodec_flush_buffers(aCodecCtx);
1304 avcodec_flush_buffers(pCodecCtx);
1307 previous_packet_location.
frame = -1;
1312 if (seek_count == 1) {
1314 seeking_pts = seek_target;
1315 seeking_frame = requested_frame;
1317 seek_audio_frame_found = 0;
1318 seek_video_frame_found = 0;
1340 long int FFmpegReader::GetVideoPTS()
1342 int current_pts = 0;
1343 if(packet->dts != AV_NOPTS_VALUE)
1344 current_pts = packet->dts;
1351 void FFmpegReader::UpdatePTSOffset(
bool is_video)
1357 if (video_pts_offset == 99999)
1359 video_pts_offset = 0 - GetVideoPTS();
1364 if (audio_pts_offset == 99999)
1366 audio_pts_offset = 0 - packet->pts;
1371 long int FFmpegReader::ConvertVideoPTStoFrame(
long int pts)
1374 pts = pts + video_pts_offset;
1375 long int previous_video_frame = current_video_frame;
1384 if (current_video_frame == 0)
1385 current_video_frame = frame;
1389 if (frame == previous_video_frame) {
1390 duplicate_video_frames.insert(pair<int,int>(frame, frame));
1397 current_video_frame++;
1399 if (current_video_frame < frame)
1401 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ConvertVideoPTStoFrame (detected missing frame)",
"calculated frame", frame,
"previous_video_frame", previous_video_frame,
"current_video_frame", current_video_frame,
"", -1,
"", -1,
"", -1);
1406 while (current_video_frame < frame) {
1407 if (!missing_video_frames.count(current_video_frame)) {
1408 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ConvertVideoPTStoFrame (tracking missing frame)",
"current_video_frame", current_video_frame,
"previous_video_frame", previous_video_frame,
"", -1,
"", -1,
"", -1,
"", -1);
1409 missing_video_frames.insert(pair<long int, long int>(current_video_frame, previous_video_frame));
1410 missing_video_frames_source.insert(pair<long int, long int>(previous_video_frame, current_video_frame));
1414 has_missing_frames =
true;
1417 current_video_frame++;
1426 long int FFmpegReader::ConvertFrameToVideoPTS(
long int frame_number)
1435 return video_pts - video_pts_offset;
1439 long int FFmpegReader::ConvertFrameToAudioPTS(
long int frame_number)
1448 return audio_pts - audio_pts_offset;
1452 AudioLocation FFmpegReader::GetAudioPTSLocation(
long int pts)
1455 pts = pts + audio_pts_offset;
1464 int whole_frame = int(frame);
1467 double sample_start_percentage = frame - double(whole_frame);
1473 int sample_start = round(
double(samples_per_frame) * sample_start_percentage);
1476 if (whole_frame < 1)
1478 if (sample_start < 0)
1485 if (previous_packet_location.
frame != -1 && location.
is_near(previous_packet_location, samples_per_frame, samples_per_frame))
1487 int orig_frame = location.
frame;
1491 if (previous_packet_location.
sample_start <= samples_per_frame)
1494 location.
frame = previous_packet_location.
frame;
1504 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::GetAudioPTSLocation (Audio Gap Detected)",
"Source Frame", orig_frame,
"Source Audio Sample", orig_start,
"Target Frame", location.
frame,
"Target Audio Sample", location.
sample_start,
"pts", pts,
"", -1);
1508 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::GetAudioPTSLocation (Audio Gap Ignored - too big)",
"Previous location frame", previous_packet_location.
frame,
"Target Frame", location.
frame,
"Target Audio Sample", location.
sample_start,
"pts", pts,
"", -1,
"", -1);
1511 for (
long int audio_frame = previous_packet_location.
frame; audio_frame < location.
frame; audio_frame++) {
1512 if (!missing_audio_frames.count(audio_frame)) {
1513 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::GetAudioPTSLocation (tracking missing frame)",
"missing_audio_frame", audio_frame,
"previous_audio_frame", previous_packet_location.
frame,
"new location frame", location.
frame,
"", -1,
"", -1,
"", -1);
1514 missing_audio_frames.insert(pair<long int, long int>(previous_packet_location.
frame - 1, audio_frame));
1520 previous_packet_location = location;
1527 tr1::shared_ptr<Frame> FFmpegReader::CreateFrame(
long int requested_frame)
1530 tr1::shared_ptr<Frame> output = working_cache.
GetFrame(requested_frame);
1539 working_cache.
Add(output);
1542 if (requested_frame > largest_frame_processed)
1543 largest_frame_processed = requested_frame;
1551 bool FFmpegReader::IsPartialFrame(
long int requested_frame) {
1554 bool seek_trash =
false;
1555 long int max_seeked_frame = seek_audio_frame_found;
1556 if (seek_video_frame_found > max_seeked_frame)
1557 max_seeked_frame = seek_video_frame_found;
1558 if ((
info.
has_audio && seek_audio_frame_found && max_seeked_frame >= requested_frame) ||
1559 (
info.
has_video && seek_video_frame_found && max_seeked_frame >= requested_frame))
1566 bool FFmpegReader::CheckMissingFrame(
long int requested_frame)
1572 int checked_count = 0;
1575 if (checked_frames.count(requested_frame) == 0)
1576 checked_frames[requested_frame] = 1;
1578 checked_frames[requested_frame]++;
1579 checked_count = checked_frames[requested_frame];
1582 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::CheckMissingFrame",
"requested_frame", requested_frame,
"has_missing_frames", has_missing_frames,
"missing_video_frames.size()", missing_video_frames.size(),
"checked_count", checked_count,
"", -1,
"", -1);
1586 map<long int, long int>::iterator itr;
1587 bool found_missing_frame =
false;
1590 if (missing_video_frames.count(requested_frame) || missing_audio_frames.count(requested_frame)) {
1591 long int missing_source_frame = -1;
1592 if (missing_video_frames.count(requested_frame))
1593 missing_source_frame = missing_video_frames.find(requested_frame)->second;
1594 else if (missing_audio_frames.count(requested_frame))
1595 missing_source_frame = missing_audio_frames.find(requested_frame)->second;
1598 if (checked_frames.count(missing_source_frame) == 0)
1599 checked_frames[missing_source_frame] = 1;
1601 checked_frames[missing_source_frame]++;
1604 tr1::shared_ptr<Frame> parent_frame = missing_frames.
GetFrame(missing_source_frame);
1605 if (parent_frame == NULL) {
1607 if (parent_frame != NULL) {
1609 missing_frames.
Add(parent_frame);
1614 tr1::shared_ptr<Frame> missing_frame = CreateFrame(requested_frame);
1617 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::CheckMissingFrame (Is Previous Video Frame Final)",
"requested_frame", requested_frame,
"missing_frame->number", missing_frame->number,
"missing_source_frame", missing_source_frame,
"", -1,
"", -1,
"", -1);
1620 if (parent_frame != NULL) {
1622 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::CheckMissingFrame (AddImage from Previous Video Frame)",
"requested_frame", requested_frame,
"missing_frame->number", missing_frame->number,
"missing_source_frame", missing_source_frame,
"", -1,
"", -1,
"", -1);
1625 missing_frame->AddImage(tr1::shared_ptr<QImage>(
new QImage(*parent_frame->GetImage())));
1627 processed_video_frames[missing_frame->number] = missing_frame->number;
1628 processed_audio_frames[missing_frame->number] = missing_frame->number;
1634 working_cache.
Remove(missing_frame->number);
1637 last_frame = missing_frame->number;
1642 return found_missing_frame;
1646 void FFmpegReader::CheckWorkingFrames(
bool end_of_stream,
long int requested_frame)
1660 CheckMissingFrame(f->number);
1663 int checked_count = 0;
1665 bool is_video_ready =
false;
1666 bool is_audio_ready =
false;
1669 is_video_ready = processed_video_frames.count(f->number);
1670 is_audio_ready = processed_audio_frames.count(f->number);
1673 checked_count = checked_frames[f->number];
1676 if (previous_packet_location.
frame == f->number && !end_of_stream)
1677 is_audio_ready =
false;
1678 bool is_seek_trash = IsPartialFrame(f->number);
1685 if (checked_count > 40 && (!is_video_ready || !is_audio_ready)) {
1687 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::CheckWorkingFrames (exceeded checked_count)",
"frame_number", f->number,
"is_video_ready", is_video_ready,
"is_audio_ready", is_audio_ready,
"checked_count", checked_count,
"checked_frames.size()", checked_frames.size(),
"", -1);
1689 if (
info.
has_video && !is_video_ready && last_video_frame) {
1691 f->AddImage(tr1::shared_ptr<QImage>(
new QImage(*last_video_frame->GetImage())));
1692 is_video_ready =
true;
1698 is_audio_ready =
true;
1703 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::CheckWorkingFrames",
"frame_number", f->number,
"is_video_ready", is_video_ready,
"is_audio_ready", is_audio_ready,
"checked_count", checked_count,
"checked_frames.size()", checked_frames.size(),
"", -1);
1706 if ((!end_of_stream && is_video_ready && is_audio_ready && f->number <= requested_frame) || end_of_stream || is_seek_trash)
1709 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::CheckWorkingFrames (mark frame as final)",
"f->number", f->number,
"is_seek_trash", is_seek_trash,
"Working Cache Count", working_cache.
Count(),
"Final Cache Count",
final_cache.
Count(),
"", -1,
"", -1);
1714 num_checks_since_final = 0;
1722 if (missing_video_frames_source.count(f->number)) {
1724 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::CheckWorkingFrames (add frame to missing cache)",
"f->number", f->number,
"is_seek_trash", is_seek_trash,
"Missing Cache Count", missing_frames.
Count(),
"Working Cache Count", working_cache.
Count(),
"Final Cache Count",
final_cache.
Count(),
"", -1);
1725 missing_frames.
Add(f);
1730 working_cache.
Remove(f->number);
1733 last_frame = f->number;
1736 checked_frames.erase(f->number);
1740 working_cache.
Remove(f->number);
1750 void FFmpegReader::CheckFPS()
1755 int first_second_counter = 0;
1756 int second_second_counter = 0;
1757 int third_second_counter = 0;
1758 int forth_second_counter = 0;
1759 int fifth_second_counter = 0;
1762 int threshold = 500;
1768 if (GetNextPacket() < 0)
1773 if (packet->stream_index == videoStream)
1779 UpdatePTSOffset(
true);
1782 long int pts = GetVideoPTS();
1785 RemoveAVFrame(pFrame);
1788 RemoveAVPacket(packet);
1791 pts += video_pts_offset;
1797 if (video_seconds <= 1.0)
1798 first_second_counter++;
1799 else if (video_seconds > 1.0 && video_seconds <= 2.0)
1800 second_second_counter++;
1801 else if (video_seconds > 2.0 && video_seconds <= 3.0)
1802 third_second_counter++;
1803 else if (video_seconds > 3.0 && video_seconds <= 4.0)
1804 forth_second_counter++;
1805 else if (video_seconds > 4.0 && video_seconds <= 5.0)
1806 fifth_second_counter++;
1813 RemoveAVPacket(packet);
1817 RemoveAVPacket(packet);
1823 if (iterations > threshold)
1828 if (second_second_counter == 0 || third_second_counter == 0 || forth_second_counter == 0 || fifth_second_counter == 0)
1837 int sum_fps = second_second_counter + third_second_counter + forth_second_counter + fifth_second_counter;
1838 int avg_fps = round(sum_fps / 4.0f);
1846 double diff = fps - double(avg_fps);
1849 if (diff <= -1 || diff >= 1)
1853 diff = half_fps - double(avg_fps);
1856 if (diff <= -1 || diff >= 1)
1873 void FFmpegReader::RemoveAVFrame(AVPicture* remove_frame)
1875 #pragma omp critical (packet_cache) 1878 if (frames.count(remove_frame))
1881 avpicture_free(frames[remove_frame]);
1884 frames.erase(remove_frame);
1887 delete remove_frame;
1894 void FFmpegReader::RemoveAVPacket(AVPacket* remove_packet)
1897 av_free_packet(remove_packet);
1900 delete remove_packet;
1904 long int FFmpegReader::GetSmallestVideoFrame()
1907 map<long int, long int>::iterator itr;
1908 int smallest_frame = -1;
1909 for(itr = processing_video_frames.begin(); itr != processing_video_frames.end(); ++itr)
1911 if (itr->first < smallest_frame || smallest_frame == -1)
1912 smallest_frame = itr->first;
1916 return smallest_frame;
1920 long int FFmpegReader::GetSmallestAudioFrame()
1923 map<long int, long int>::iterator itr;
1924 int smallest_frame = -1;
1926 for(itr = processing_audio_frames.begin(); itr != processing_audio_frames.end(); ++itr)
1928 if (itr->first < smallest_frame || smallest_frame == -1)
1929 smallest_frame = itr->first;
1933 return smallest_frame;
1948 root[
"type"] =
"FFmpegReader";
1949 root[
"path"] = path;
1960 Json::Reader reader;
1961 bool success = reader.parse( value, root );
1964 throw InvalidJSON(
"JSON could not be parsed (or is invalid)",
"");
1974 throw InvalidJSON(
"JSON is invalid (missing keys or invalid data types)",
"");
1985 if (!root[
"path"].isNull())
1986 path = root[
"path"].asString();
#define AV_RESET_FRAME(av_frame)
long long file_size
Size of file (in bytes)
int max_height
The maximium image height needed by this clip (used for optimizations)
tr1::shared_ptr< Frame > GetFrame(long int requested_frame)
#define AV_FREE_FRAME(av_frame)
int num
Numerator for the fraction.
CriticalSection getFrameCriticalSection
Section lock for multiple threads.
void Remove(long int frame_number)
Remove a specific frame.
CriticalSection processingCriticalSection
ChannelLayout channel_layout
The channel layout (mono, stereo, 5 point surround, etc...)
int width
The width of the video (in pixesl)
This class represents a single frame of video (i.e. image & audio data)
float ToFloat()
Return this fraction as a float (i.e. 1/2 = 0.5)
float duration
Length of time (in seconds)
string acodec
The name of the audio codec used to encode / decode the video stream.
void Reduce()
Reduce this fraction (i.e. 640/480 = 4/3)
#define AVCODEC_MAX_AUDIO_FRAME_SIZE
#define OPEN_MP_NUM_PROCESSORS
string Json()
Get and Set JSON methods.
Json::Value JsonValue()
Generate Json::JsonValue for this object.
Exception when a reader is closed, and a frame is requested.
bool has_video
Determines if this file has a video stream.
void SetMaxBytesFromInfo(long int number_of_frames, int width, int height, int sample_rate, int channels)
Set maximum bytes to a different amount based on a ReaderInfo struct.
Fraction display_ratio
The ratio of width to height of the video stream (i.e. 640x480 has a ratio of 4/3) ...
FFmpegReader(string path)
int audio_bit_rate
The bit rate of the audio stream (in bytes)
bool has_audio
Determines if this file has an audio stream.
Exception when no valid codec is found for a file.
int audio_stream_index
The index of the audio stream.
tr1::shared_ptr< Frame > GetSmallestFrame()
Get the smallest frame number.
Exception when no streams are found in the file.
int height
The height of the video (in pixels)
void SetJson(string value)
Load JSON string into this object.
#define AV_ALLOCATE_FRAME()
Exception for files that can not be found or opened.
void AppendDebugMethod(string method_name, string arg1_name, float arg1_value, string arg2_name, float arg2_value, string arg3_name, float arg3_value, string arg4_name, float arg4_value, string arg5_name, float arg5_value, string arg6_name, float arg6_value)
Append debug information.
This class represents a fraction.
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround...
virtual Json::Value JsonValue()=0
Generate Json::JsonValue for this object.
virtual void SetJsonValue(Json::Value root)=0
Load Json::JsonValue into this object.
ReaderInfo info
Information about the current media file.
void Clear()
Clear the cache of all frames.
Fraction fps
Frames per second, as a fraction (i.e. 24/1 = 24 fps)
Fraction video_timebase
The video timebase determines how long each frame stays on the screen.
Exception for frames that are out of bounds.
int is_near(AudioLocation location, int samples_per_frame, int amount)
static ZmqLogger * Instance()
Create or get an instance of this logger singleton (invoke the class with this method) ...
Fraction pixel_ratio
The pixel ratio of the video stream as a fraction (i.e. some pixels are not square) ...
void Add(tr1::shared_ptr< Frame > frame)
Add a Frame to the cache.
This namespace is the default namespace for all code in the openshot library.
Exception for invalid JSON.
int pixel_format
The pixel format (i.e. YUV420P, RGB24, etc...)
This struct holds the associated video frame and starting sample # for an audio packet.
tr1::shared_ptr< Frame > GetFrame(long int frame_number)
Get a frame from the cache.
void Open()
Open File - which is called by the constructor automatically.
int video_bit_rate
The bit rate of the video stream (in bytes)
Fraction audio_timebase
The audio timebase determines how long each audio packet should be played.
string vcodec
The name of the video codec used to encode / decode the video stream.
CacheMemory final_cache
Final cache object used to hold final frames.
~FFmpegReader()
Destructor.
long int Count()
Count the frames in the queue.
int den
Denominator for the fraction.
int channels
The number of audio channels used in the audio stream.
void SetJsonValue(Json::Value root)
Load Json::JsonValue into this object.
int video_stream_index
The index of the video stream.
long int video_length
The number of frames in the video stream.
int max_width
The maximum image width needed by this clip (used for optimizations)
int GetSamplesPerFrame(Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
double ToDouble()
Return this fraction as a double (i.e. 1/2 = 0.5)
int sample_rate
The number of audio samples per second (44100 is a common sample rate)
Exception when too many seek attempts happen.