OpenShot Library | libopenshot  0.1.3
FFmpegReader.cpp
Go to the documentation of this file.
1 /**
2  * @file
3  * @brief Source file for FFmpegReader class
4  * @author Jonathan Thomas <jonathan@openshot.org>, Fabrice Bellard
5  *
6  * @section LICENSE
7  *
8  * Copyright (c) 2008-2013 OpenShot Studios, LLC, Fabrice Bellard
9  * (http://www.openshotstudios.com). This file is part of
10  * OpenShot Library (http://www.openshot.org), an open-source project
11  * dedicated to delivering high quality video editing and animation solutions
12  * to the world.
13  *
14  * This file is originally based on the Libavformat API example, and then modified
15  * by the libopenshot project.
16  *
17  * OpenShot Library (libopenshot) is free software: you can redistribute it
18  * and/or modify it under the terms of the GNU Lesser General Public License
19  * as published by the Free Software Foundation, either version 3 of the
20  * License, or (at your option) any later version.
21  *
22  * OpenShot Library (libopenshot) is distributed in the hope that it will be
23  * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
24  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
25  * GNU Lesser General Public License for more details.
26  *
27  * You should have received a copy of the GNU Lesser General Public License
28  * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
29  */
30 
31 #include "../include/FFmpegReader.h"
32 
33 using namespace openshot;
34 
36  : last_frame(0), is_seeking(0), seeking_pts(0), seeking_frame(0), seek_count(0),
37  audio_pts_offset(99999), video_pts_offset(99999), path(path), is_video_seek(true), check_interlace(false),
38  check_fps(false), enable_seek(true), is_open(false), seek_audio_frame_found(0), seek_video_frame_found(0),
39  prev_samples(0), prev_pts(0), pts_total(0), pts_counter(0), is_duration_known(false), largest_frame_processed(0),
40  current_video_frame(0), has_missing_frames(false), num_packets_since_video_frame(0), num_checks_since_final(0) {
41 
42  // Initialize FFMpeg, and register all formats and codecs
43  av_register_all();
44  avcodec_register_all();
45 
46  // Init cache
47  working_cache.SetMaxBytesFromInfo(OPEN_MP_NUM_PROCESSORS * 30, info.width, info.height, info.sample_rate, info.channels);
48  missing_frames.SetMaxBytesFromInfo(OPEN_MP_NUM_PROCESSORS * 2, info.width, info.height, info.sample_rate, info.channels);
49  final_cache.SetMaxBytesFromInfo(OPEN_MP_NUM_PROCESSORS * 2, info.width, info.height, info.sample_rate, info.channels);
50 
51  // Open and Close the reader, to populate it's attributes (such as height, width, etc...)
52  Open();
53  Close();
54 }
55 
56 FFmpegReader::FFmpegReader(string path, bool inspect_reader) throw(InvalidFile, NoStreamsFound, InvalidCodec)
57  : last_frame(0), is_seeking(0), seeking_pts(0), seeking_frame(0), seek_count(0),
58  audio_pts_offset(99999), video_pts_offset(99999), path(path), is_video_seek(true), check_interlace(false),
59  check_fps(false), enable_seek(true), is_open(false), seek_audio_frame_found(0), seek_video_frame_found(0),
60  prev_samples(0), prev_pts(0), pts_total(0), pts_counter(0), is_duration_known(false), largest_frame_processed(0),
61  current_video_frame(0), has_missing_frames(false), num_packets_since_video_frame(0), num_checks_since_final(0) {
62 
63  // Initialize FFMpeg, and register all formats and codecs
64  av_register_all();
65  avcodec_register_all();
66 
67  // Init cache
68  working_cache.SetMaxBytesFromInfo(OPEN_MP_NUM_PROCESSORS * 30, info.width, info.height, info.sample_rate, info.channels);
69  missing_frames.SetMaxBytesFromInfo(OPEN_MP_NUM_PROCESSORS * 2, info.width, info.height, info.sample_rate, info.channels);
70  final_cache.SetMaxBytesFromInfo(OPEN_MP_NUM_PROCESSORS * 2, info.width, info.height, info.sample_rate, info.channels);
71 
72  // Open and Close the reader, to populate it's attributes (such as height, width, etc...)
73  if (inspect_reader) {
74  Open();
75  Close();
76  }
77 }
78 
80  if (is_open)
81  // Auto close reader if not already done
82  Close();
83 }
84 
85 // This struct holds the associated video frame and starting sample # for an audio packet.
86 int AudioLocation::is_near(AudioLocation location, int samples_per_frame, int amount)
87 {
88  // Is frame even close to this one?
89  if (abs(location.frame - frame) >= 2)
90  // This is too far away to be considered
91  return false;
92 
93  int sample_diff = abs(location.sample_start - sample_start);
94  if (location.frame == frame && sample_diff >= 0 && sample_diff <= amount)
95  // close
96  return true;
97 
98  // new frame is after
99  if (location.frame > frame)
100  {
101  // remaining samples + new samples
102  sample_diff = (samples_per_frame - sample_start) + location.sample_start;
103  if (sample_diff >= 0 && sample_diff <= amount)
104  return true;
105  }
106 
107  // new frame is before
108  if (location.frame < frame)
109  {
110  // remaining new samples + old samples
111  sample_diff = (samples_per_frame - location.sample_start) + sample_start;
112  if (sample_diff >= 0 && sample_diff <= amount)
113  return true;
114  }
115 
116  // not close
117  return false;
118 }
119 
121 {
122  // Open reader if not already open
123  if (!is_open)
124  {
125  // Initialize format context
126  pFormatCtx = NULL;
127 
128  // Open video file
129  if (avformat_open_input(&pFormatCtx, path.c_str(), NULL, NULL) != 0)
130  throw InvalidFile("File could not be opened.", path);
131 
132  // Retrieve stream information
133  if (avformat_find_stream_info(pFormatCtx, NULL) < 0)
134  throw NoStreamsFound("No streams found in file.", path);
135 
136  videoStream = -1;
137  audioStream = -1;
138  // Loop through each stream, and identify the video and audio stream index
139  for (unsigned int i = 0; i < pFormatCtx->nb_streams; i++)
140  {
141  // Is this a video stream?
142  if (pFormatCtx->streams[i]->codec->codec_type == AVMEDIA_TYPE_VIDEO && videoStream < 0) {
143  videoStream = i;
144  }
145  // Is this an audio stream?
146  if (pFormatCtx->streams[i]->codec->codec_type == AVMEDIA_TYPE_AUDIO && audioStream < 0) {
147  audioStream = i;
148  }
149  }
150  if (videoStream == -1 && audioStream == -1)
151  throw NoStreamsFound("No video or audio streams found in this file.", path);
152 
153  // Is there a video stream?
154  if (videoStream != -1)
155  {
156  // Set the stream index
157  info.video_stream_index = videoStream;
158 
159  // Set the codec and codec context pointers
160  pStream = pFormatCtx->streams[videoStream];
161  pCodecCtx = pFormatCtx->streams[videoStream]->codec;
162 
163  // Set number of threads equal to number of processors + 1
164  pCodecCtx->thread_count = OPEN_MP_NUM_PROCESSORS;
165 
166  // Find the decoder for the video stream
167  AVCodec *pCodec = avcodec_find_decoder(pCodecCtx->codec_id);
168  if (pCodec == NULL) {
169  throw InvalidCodec("A valid video codec could not be found for this file.", path);
170  }
171  // Open video codec
172  if (avcodec_open2(pCodecCtx, pCodec, NULL) < 0)
173  throw InvalidCodec("A video codec was found, but could not be opened.", path);
174 
175  // Update the File Info struct with video details (if a video stream is found)
176  UpdateVideoInfo();
177  }
178 
179  // Is there an audio stream?
180  if (audioStream != -1)
181  {
182  // Set the stream index
183  info.audio_stream_index = audioStream;
184 
185  // Get a pointer to the codec context for the audio stream
186  aStream = pFormatCtx->streams[audioStream];
187  aCodecCtx = pFormatCtx->streams[audioStream]->codec;
188 
189  // Set number of threads equal to number of processors + 1
190  aCodecCtx->thread_count = OPEN_MP_NUM_PROCESSORS;
191 
192  // Find the decoder for the audio stream
193  AVCodec *aCodec = avcodec_find_decoder(aCodecCtx->codec_id);
194  if (aCodec == NULL) {
195  throw InvalidCodec("A valid audio codec could not be found for this file.", path);
196  }
197  // Open audio codec
198  if (avcodec_open2(aCodecCtx, aCodec, NULL) < 0)
199  throw InvalidCodec("An audio codec was found, but could not be opened.", path);
200 
201  // Update the File Info struct with audio details (if an audio stream is found)
202  UpdateAudioInfo();
203  }
204 
205  // Init previous audio location to zero
206  previous_packet_location.frame = -1;
207  previous_packet_location.sample_start = 0;
208 
209  // Adjust cache size based on size of frame and audio
213 
214  // Mark as "open"
215  is_open = true;
216  }
217 }
218 
220 {
221  // Close all objects, if reader is 'open'
222  if (is_open)
223  {
224  // Mark as "closed"
225  is_open = false;
226 
227  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::Close", "", -1, "", -1, "", -1, "", -1, "", -1, "", -1);
228 
229  // Close the codec
230  if (info.has_video)
231  {
232  avcodec_flush_buffers(pCodecCtx);
233  avcodec_close(pCodecCtx);
234  }
235  if (info.has_audio)
236  {
237  avcodec_flush_buffers(aCodecCtx);
238  avcodec_close(aCodecCtx);
239  }
240 
241  // Clear final cache
242  final_cache.Clear();
243  working_cache.Clear();
244  missing_frames.Clear();
245 
246  // Clear processed lists
247  {
248  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
249  processed_video_frames.clear();
250  processed_audio_frames.clear();
251  processing_video_frames.clear();
252  processing_audio_frames.clear();
253  missing_audio_frames.clear();
254  missing_video_frames.clear();
255  missing_audio_frames_source.clear();
256  missing_video_frames_source.clear();
257  checked_frames.clear();
258  }
259 
260  // Close the video file
261  avformat_close_input(&pFormatCtx);
262  av_freep(&pFormatCtx);
263 
264  // Reset some variables
265  last_frame = 0;
266  largest_frame_processed = 0;
267  seek_audio_frame_found = 0;
268  seek_video_frame_found = 0;
269  current_video_frame = 0;
270  has_missing_frames = false;
271  }
272 }
273 
274 void FFmpegReader::UpdateAudioInfo()
275 {
276  // Set values of FileInfo struct
277  info.has_audio = true;
278  info.file_size = pFormatCtx->pb ? avio_size(pFormatCtx->pb) : -1;
279  info.acodec = aCodecCtx->codec->name;
280  info.channels = aCodecCtx->channels;
281  if (aCodecCtx->channel_layout == 0)
282  aCodecCtx->channel_layout = av_get_default_channel_layout( aCodecCtx->channels );
283  info.channel_layout = (ChannelLayout) aCodecCtx->channel_layout;
284  info.sample_rate = aCodecCtx->sample_rate;
285  info.audio_bit_rate = aCodecCtx->bit_rate;
286 
287  // Set audio timebase
288  info.audio_timebase.num = aStream->time_base.num;
289  info.audio_timebase.den = aStream->time_base.den;
290 
291  // Get timebase of audio stream (if valid) and greater than the current duration
292  if (aStream->duration > 0.0f && aStream->duration > info.duration)
293  info.duration = aStream->duration * info.audio_timebase.ToDouble();
294 
295  // Check for an invalid video length
296  if (info.has_video && info.video_length <= 0)
297  {
298  // Calculate the video length from the audio duration
300  }
301 
302  // Set video timebase (if no video stream was found)
303  if (!info.has_video)
304  {
305  // Set a few important default video settings (so audio can be divided into frames)
306  info.fps.num = 24;
307  info.fps.den = 1;
308  info.video_timebase.num = 1;
309  info.video_timebase.den = 24;
311  info.width = 720;
312  info.height = 480;
313 
314  }
315 
316 }
317 
318 void FFmpegReader::UpdateVideoInfo()
319 {
320  // Set values of FileInfo struct
321  info.has_video = true;
322  info.file_size = pFormatCtx->pb ? avio_size(pFormatCtx->pb) : -1;
323  info.height = pCodecCtx->height;
324  info.width = pCodecCtx->width;
325  info.vcodec = pCodecCtx->codec->name;
326  info.video_bit_rate = pFormatCtx->bit_rate;
327  if (!check_fps)
328  {
329  // set frames per second (fps)
330  info.fps.num = pStream->avg_frame_rate.num;
331  info.fps.den = pStream->avg_frame_rate.den;
332  }
333 
334  if (pStream->sample_aspect_ratio.num != 0)
335  {
336  info.pixel_ratio.num = pStream->sample_aspect_ratio.num;
337  info.pixel_ratio.den = pStream->sample_aspect_ratio.den;
338  }
339  else if (pCodecCtx->sample_aspect_ratio.num != 0)
340  {
341  info.pixel_ratio.num = pCodecCtx->sample_aspect_ratio.num;
342  info.pixel_ratio.den = pCodecCtx->sample_aspect_ratio.den;
343  }
344  else
345  {
346  info.pixel_ratio.num = 1;
347  info.pixel_ratio.den = 1;
348  }
349 
350  info.pixel_format = pCodecCtx->pix_fmt;
351 
352  // Calculate the DAR (display aspect ratio)
354 
355  // Reduce size fraction
356  size.Reduce();
357 
358  // Set the ratio based on the reduced fraction
359  info.display_ratio.num = size.num;
360  info.display_ratio.den = size.den;
361 
362  // Set the video timebase
363  info.video_timebase.num = pStream->time_base.num;
364  info.video_timebase.den = pStream->time_base.den;
365 
366  // Set the duration in seconds, and video length (# of frames)
367  info.duration = pStream->duration * info.video_timebase.ToDouble();
368 
369  // Check for valid duration (if found)
370  if (info.duration <= 0.0f && pFormatCtx->duration >= 0)
371  // Use the format's duration
372  info.duration = pFormatCtx->duration / AV_TIME_BASE;
373 
374  // Calculate duration from filesize and bitrate (if any)
375  if (info.duration <= 0.0f && info.video_bit_rate > 0 && info.file_size > 0)
376  // Estimate from bitrate, total bytes, and framerate
378 
379  // No duration found in stream of file
380  if (info.duration <= 0.0f)
381  {
382  // No duration is found in the video stream
383  info.duration = -1;
384  info.video_length = -1;
385  is_duration_known = false;
386  }
387  else
388  {
389  // Yes, a duration was found
390  is_duration_known = true;
391 
392  // Calculate number of frames
394  }
395 
396  // Override an invalid framerate
397  if (info.fps.ToFloat() > 120.0f || (info.fps.num == 0 || info.fps.den == 0))
398  {
399  // Set a few important default video settings (so audio can be divided into frames)
400  info.fps.num = 24;
401  info.fps.den = 1;
402  info.video_timebase.num = 1;
403  info.video_timebase.den = 24;
404 
405  // Calculate number of frames
407  }
408 
409 }
410 
411 
412 tr1::shared_ptr<Frame> FFmpegReader::GetFrame(long int requested_frame) throw(OutOfBoundsFrame, ReaderClosed, TooManySeeks)
413 {
414  // Check for open reader (or throw exception)
415  if (!is_open)
416  throw ReaderClosed("The FFmpegReader is closed. Call Open() before calling this method.", path);
417 
418  // Adjust for a requested frame that is too small or too large
419  if (requested_frame < 1)
420  requested_frame = 1;
421  if (requested_frame > info.video_length && is_duration_known)
422  requested_frame = info.video_length;
423  if (info.has_video && info.video_length == 0)
424  // Invalid duration of video file
425  throw InvalidFile("Could not detect the duration of the video or audio stream.", path);
426 
427  // Debug output
428  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetFrame", "requested_frame", requested_frame, "last_frame", last_frame, "", -1, "", -1, "", -1, "", -1);
429 
430  // Check the cache for this frame
431  tr1::shared_ptr<Frame> frame = final_cache.GetFrame(requested_frame);
432  if (frame) {
433  // Debug output
434  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetFrame", "returned cached frame", requested_frame, "", -1, "", -1, "", -1, "", -1, "", -1);
435 
436  // Return the cached frame
437  return frame;
438  }
439  else
440  {
441  // Create a scoped lock, allowing only a single thread to run the following code at one time
442  const GenericScopedLock<CriticalSection> lock(getFrameCriticalSection);
443 
444  // Check the cache a 2nd time (due to a potential previous lock)
445  if (has_missing_frames)
446  CheckMissingFrame(requested_frame);
447  frame = final_cache.GetFrame(requested_frame);
448  if (frame) {
449  // Debug output
450  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetFrame", "returned cached frame on 2nd look", requested_frame, "", -1, "", -1, "", -1, "", -1, "", -1);
451 
452  // Return the cached frame
453  return frame;
454  }
455 
456  // Frame is not in cache
457  // Reset seek count
458  seek_count = 0;
459 
460  // Check for first frame (always need to get frame 1 before other frames, to correctly calculate offsets)
461  if (last_frame == 0 && requested_frame != 1)
462  // Get first frame
463  ReadStream(1);
464 
465  // Are we within X frames of the requested frame?
466  long int diff = requested_frame - last_frame;
467  if (diff >= 1 && diff <= 20)
468  {
469  // Continue walking the stream
470  return ReadStream(requested_frame);
471  }
472  else
473  {
474  // Greater than 30 frames away, or backwards, we need to seek to the nearest key frame
475  if (enable_seek)
476  // Only seek if enabled
477  Seek(requested_frame);
478 
479  else if (!enable_seek && diff < 0)
480  {
481  // Start over, since we can't seek, and the requested frame is smaller than our position
482  Close();
483  Open();
484  }
485 
486  // Then continue walking the stream
487  return ReadStream(requested_frame);
488  }
489 
490  }
491 }
492 
493 // Read the stream until we find the requested Frame
494 tr1::shared_ptr<Frame> FFmpegReader::ReadStream(long int requested_frame)
495 {
496  // Allocate video frame
497  bool end_of_stream = false;
498  bool check_seek = false;
499  bool frame_finished = false;
500  int packet_error = -1;
501 
502  // Minimum number of packets to process (for performance reasons)
503  int packets_processed = 0;
504  int minimum_packets = OPEN_MP_NUM_PROCESSORS;
505 
506  // Set the number of threads in OpenMP
507  omp_set_num_threads(OPEN_MP_NUM_PROCESSORS);
508  // Allow nested OpenMP sections
509  omp_set_nested(true);
510 
511  // Debug output
512  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream", "requested_frame", requested_frame, "OPEN_MP_NUM_PROCESSORS", OPEN_MP_NUM_PROCESSORS, "", -1, "", -1, "", -1, "", -1);
513 
514  #pragma omp parallel
515  {
516  #pragma omp single
517  {
518  // Loop through the stream until the correct frame is found
519  while (true)
520  {
521  // Get the next packet into a local variable called packet
522  packet_error = GetNextPacket();
523 
524  // Wait if too many frames are being processed
525  while (processing_video_frames.size() + processing_audio_frames.size() >= minimum_packets)
526  usleep(2500);
527 
528  // Get the next packet (if any)
529  if (packet_error < 0)
530  {
531  // Break loop when no more packets found
532  end_of_stream = true;
533  break;
534  }
535 
536  // Debug output
537  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream (GetNextPacket)", "requested_frame", requested_frame, "processing_video_frames.size()", processing_video_frames.size(), "processing_audio_frames.size()", processing_audio_frames.size(), "minimum_packets", minimum_packets, "packets_processed", packets_processed, "", -1);
538 
539  // Video packet
540  if (info.has_video && packet->stream_index == videoStream)
541  {
542  // Reset this counter, since we have a video packet
543  num_packets_since_video_frame = 0;
544 
545  // Check the status of a seek (if any)
546  if (is_seeking)
547  #pragma omp critical (openshot_seek)
548  check_seek = CheckSeek(true);
549  else
550  check_seek = false;
551 
552  if (check_seek) {
553  // Remove packet (since this packet is pointless)
554  RemoveAVPacket(packet);
555 
556  // Jump to the next iteration of this loop
557  continue;
558  }
559 
560  // Get the AVFrame from the current packet
561  frame_finished = GetAVFrame();
562 
563  // Check if the AVFrame is finished and set it
564  if (frame_finished)
565  {
566  // Update PTS / Frame Offset (if any)
567  UpdatePTSOffset(true);
568 
569  // Process Video Packet
570  ProcessVideoPacket(requested_frame);
571  }
572 
573  }
574  // Audio packet
575  else if (info.has_audio && packet->stream_index == audioStream)
576  {
577  // Increment this (to track # of packets since the last video packet)
578  num_packets_since_video_frame++;
579 
580  // Check the status of a seek (if any)
581  if (is_seeking)
582  #pragma omp critical (openshot_seek)
583  check_seek = CheckSeek(false);
584  else
585  check_seek = false;
586 
587  if (check_seek) {
588  // Remove packet (since this packet is pointless)
589  RemoveAVPacket(packet);
590 
591  // Jump to the next iteration of this loop
592  continue;
593  }
594 
595  // Update PTS / Frame Offset (if any)
596  UpdatePTSOffset(false);
597 
598  // Determine related video frame and starting sample # from audio PTS
599  AudioLocation location = GetAudioPTSLocation(packet->pts);
600 
601  // Process Audio Packet
602  ProcessAudioPacket(requested_frame, location.frame, location.sample_start);
603  }
604 
605  // Check if working frames are 'finished'
606  bool is_cache_found = false;
607  if (!is_seeking) {
608  // Check for any missing frames
609  CheckMissingFrame(requested_frame);
610 
611  // Check for final frames
612  CheckWorkingFrames(false, requested_frame);
613  }
614 
615  // Check if requested 'final' frame is available
616  is_cache_found = (final_cache.GetFrame(requested_frame) != NULL);
617 
618  // Increment frames processed
619  packets_processed++;
620 
621  // Break once the frame is found
622  if ((is_cache_found && packets_processed >= minimum_packets))
623  break;
624 
625  } // end while
626 
627  } // end omp single
628  } // end omp parallel
629 
630  // Debug output
631  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream (Completed)", "packets_processed", packets_processed, "end_of_stream", end_of_stream, "largest_frame_processed", largest_frame_processed, "Working Cache Count", working_cache.Count(), "", -1, "", -1);
632 
633  // End of stream?
634  if (end_of_stream)
635  // Mark the any other working frames as 'finished'
636  CheckWorkingFrames(end_of_stream, requested_frame);
637 
638  // Return requested frame (if found)
639  tr1::shared_ptr<Frame> frame = final_cache.GetFrame(requested_frame);
640  if (frame)
641  // Return prepared frame
642  return frame;
643  else {
644 
645  // Check if largest frame is still cached
646  frame = final_cache.GetFrame(largest_frame_processed);
647  if (frame) {
648  // return the largest processed frame (assuming it was the last in the video file)
649  return frame;
650  }
651  else {
652  // The largest processed frame is no longer in cache, return a blank frame
653  tr1::shared_ptr<Frame> f = CreateFrame(largest_frame_processed);
654  f->AddColor(info.width, info.height, "#000");
655  return f;
656  }
657  }
658 
659 }
660 
661 // Get the next packet (if any)
662 int FFmpegReader::GetNextPacket()
663 {
664  int found_packet = 0;
665  AVPacket *next_packet = new AVPacket();
666  found_packet = av_read_frame(pFormatCtx, next_packet);
667 
668  if (found_packet >= 0)
669  {
670  // Update current packet pointer
671  packet = next_packet;
672  }
673  else
674  {
675  // Free packet, since it's unused
676  av_free_packet(next_packet);
677  delete next_packet;
678  }
679 
680  // Return if packet was found (or error number)
681  return found_packet;
682 }
683 
684 // Get an AVFrame (if any)
685 bool FFmpegReader::GetAVFrame()
686 {
687  int frameFinished = -1;
688 
689  // Decode video frame
690  AVFrame *next_frame = AV_ALLOCATE_FRAME();
691  #pragma omp critical (packet_cache)
692  avcodec_decode_video2(pCodecCtx, next_frame, &frameFinished, packet);
693 
694  // is frame finished
695  if (frameFinished)
696  {
697  // AVFrames are clobbered on the each call to avcodec_decode_video, so we
698  // must make a copy of the image data before this method is called again.
699  AVPicture *copyFrame = new AVPicture();
700  avpicture_alloc(copyFrame, pCodecCtx->pix_fmt, info.width, info.height);
701  av_picture_copy(copyFrame, (AVPicture *) next_frame, pCodecCtx->pix_fmt, info.width, info.height);
702 
703  #pragma omp critical (packet_cache)
704  {
705  // add to AVFrame cache (if frame finished)
706  frames[copyFrame] = copyFrame;
707  pFrame = frames[copyFrame];
708  }
709 
710  // Detect interlaced frame (only once)
711  if (!check_interlace)
712  {
713  check_interlace = true;
714  info.interlaced_frame = next_frame->interlaced_frame;
715  info.top_field_first = next_frame->top_field_first;
716  }
717 
718  }
719  else
720  {
721  // Remove packet (since this packet is pointless)
722  RemoveAVPacket(packet);
723  }
724 
725  // deallocate the frame
726  AV_FREE_FRAME(&next_frame);
727 
728  // Did we get a video frame?
729  return frameFinished;
730 }
731 
732 // Check the current seek position and determine if we need to seek again
733 bool FFmpegReader::CheckSeek(bool is_video)
734 {
735  // Are we seeking for a specific frame?
736  if (is_seeking)
737  {
738  // Determine if both an audio and video packet have been decoded since the seek happened.
739  // If not, allow the ReadStream method to keep looping
740  if ((is_video_seek && !seek_video_frame_found) || (!is_video_seek && !seek_audio_frame_found))
741  return false;
742 
743  // Check for both streams
744  if ((info.has_video && !seek_video_frame_found) || (info.has_audio && !seek_audio_frame_found))
745  return false;
746 
747  // Determine max seeked frame
748  long int max_seeked_frame = seek_audio_frame_found; // determine max seeked frame
749  if (seek_video_frame_found > max_seeked_frame)
750  max_seeked_frame = seek_video_frame_found;
751 
752  // determine if we are "before" the requested frame
753  if (max_seeked_frame >= seeking_frame)
754  {
755  // SEEKED TOO FAR
756  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckSeek (Too far, seek again)", "is_video_seek", is_video_seek, "max_seeked_frame", max_seeked_frame, "seeking_frame", seeking_frame, "seeking_pts", seeking_pts, "seek_video_frame_found", seek_video_frame_found, "seek_audio_frame_found", seek_audio_frame_found);
757 
758  // Seek again... to the nearest Keyframe
759  Seek(seeking_frame - (20 * seek_count * seek_count));
760  }
761  else
762  {
763  // SEEK WORKED
764  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckSeek (Successful)", "is_video_seek", is_video_seek, "current_pts", packet->pts, "seeking_pts", seeking_pts, "seeking_frame", seeking_frame, "seek_video_frame_found", seek_video_frame_found, "seek_audio_frame_found", seek_audio_frame_found);
765 
766  // Seek worked, and we are "before" the requested frame
767  is_seeking = false;
768  seeking_frame = 0;
769  seeking_pts = -1;
770  }
771  }
772 
773  // return the pts to seek to (if any)
774  return is_seeking;
775 }
776 
777 // Process a video packet
778 void FFmpegReader::ProcessVideoPacket(long int requested_frame)
779 {
780  // Calculate current frame #
781  long int current_frame = ConvertVideoPTStoFrame(GetVideoPTS());
782 
783  // Track 1st video packet after a successful seek
784  if (!seek_video_frame_found && is_seeking)
785  seek_video_frame_found = current_frame;
786 
787  // Are we close enough to decode the frame? and is this frame # valid?
788  if ((current_frame < (requested_frame - 20)) or (current_frame == -1))
789  {
790  // Remove frame and packet
791  RemoveAVFrame(pFrame);
792  RemoveAVPacket(packet);
793 
794  // Debug output
795  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessVideoPacket (Skipped)", "requested_frame", requested_frame, "current_frame", current_frame, "", -1, "", -1, "", -1, "", -1);
796 
797  // Skip to next frame without decoding or caching
798  return;
799  }
800 
801  // Debug output
802  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessVideoPacket (Before)", "requested_frame", requested_frame, "current_frame", current_frame, "", -1, "", -1, "", -1, "", -1);
803 
804  // Init some things local (for OpenMP)
805  PixelFormat pix_fmt = pCodecCtx->pix_fmt;
806  int height = info.height;
807  int width = info.width;
808  long int video_length = info.video_length;
809  AVPacket *my_packet = packet;
810  AVPicture *my_frame = frames[pFrame];
811 
812  // Add video frame to list of processing video frames
813  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
814  processing_video_frames[current_frame] = current_frame;
815 
816  #pragma omp task firstprivate(current_frame, my_packet, my_frame, height, width, video_length, pix_fmt)
817  {
818  // Create variables for a RGB Frame (since most videos are not in RGB, we must convert it)
819  AVFrame *pFrameRGB = NULL;
820  int numBytes;
821  uint8_t *buffer = NULL;
822 
823  // Allocate an AVFrame structure
824  pFrameRGB = AV_ALLOCATE_FRAME();
825  if (pFrameRGB == NULL)
826  throw OutOfBoundsFrame("Convert Image Broke!", current_frame, video_length);
827 
828  // Determine if video needs to be scaled down (for performance reasons)
829  // Timelines pass their size to the clips, which pass their size to the readers (as max size)
830  // If a clip is being scaled larger, it will set max_width and max_height = 0 (which means don't down scale)
831  int original_height = height;
832  if (max_width != 0 && max_height != 0 && max_width < width && max_height < height) {
833  // Override width and height (but maintain aspect ratio)
834  float ratio = float(width) / float(height);
835  int possible_width = round(max_height * ratio);
836  int possible_height = round(max_width / ratio);
837 
838  if (possible_width <= max_width) {
839  // use calculated width, and max_height
840  width = possible_width;
841  height = max_height;
842  } else {
843  // use max_width, and calculated height
844  width = max_width;
845  height = possible_height;
846  }
847  }
848 
849  // Determine required buffer size and allocate buffer
850  numBytes = avpicture_get_size(PIX_FMT_RGBA, width, height);
851  buffer = (uint8_t *) av_malloc(numBytes * sizeof(uint8_t));
852 
853  // Assign appropriate parts of buffer to image planes in pFrameRGB
854  // Note that pFrameRGB is an AVFrame, but AVFrame is a superset
855  // of AVPicture
856  avpicture_fill((AVPicture *) pFrameRGB, buffer, PIX_FMT_RGBA, width, height);
857 
858  SwsContext *img_convert_ctx = sws_getContext(info.width, info.height, pCodecCtx->pix_fmt, width,
859  height, PIX_FMT_RGBA, SWS_BILINEAR, NULL, NULL, NULL);
860 
861  // Resize / Convert to RGB
862  sws_scale(img_convert_ctx, my_frame->data, my_frame->linesize, 0,
863  original_height, pFrameRGB->data, pFrameRGB->linesize);
864 
865  // Create or get the existing frame object
866  tr1::shared_ptr<Frame> f = CreateFrame(current_frame);
867 
868  // Add Image data to frame
869  f->AddImage(width, height, 4, QImage::Format_RGBA8888, buffer);
870 
871  // Update working cache
872  working_cache.Add(f);
873 
874  // Keep track of last last_video_frame
875  last_video_frame = f;
876 
877  // Free the RGB image
878  av_free(buffer);
879  AV_FREE_FRAME(&pFrameRGB);
880 
881  // Remove frame and packet
882  RemoveAVFrame(my_frame);
883  RemoveAVPacket(my_packet);
884  sws_freeContext(img_convert_ctx);
885 
886  // Remove video frame from list of processing video frames
887  {
888  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
889  processing_video_frames.erase(current_frame);
890  processed_video_frames[current_frame] = current_frame;
891  }
892 
893  // Debug output
894  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessVideoPacket (After)", "requested_frame", requested_frame, "current_frame", current_frame, "f->number", f->number, "", -1, "", -1, "", -1);
895 
896  } // end omp task
897 
898 }
899 
900 // Process an audio packet
901 void FFmpegReader::ProcessAudioPacket(long int requested_frame, long int target_frame, int starting_sample)
902 {
903  // Track 1st audio packet after a successful seek
904  if (!seek_audio_frame_found && is_seeking)
905  seek_audio_frame_found = target_frame;
906 
907  // Are we close enough to decode the frame's audio?
908  if (target_frame < (requested_frame - 20))
909  {
910  // Remove packet
911  RemoveAVPacket(packet);
912 
913  // Debug output
914  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (Skipped)", "requested_frame", requested_frame, "target_frame", target_frame, "starting_sample", starting_sample, "", -1, "", -1, "", -1);
915 
916  // Skip to next frame without decoding or caching
917  return;
918  }
919 
920  // Init some local variables (for OpenMP)
921  AVPacket *my_packet = packet;
922 
923  // Debug output
924  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (Before)", "requested_frame", requested_frame, "target_frame", target_frame, "starting_sample", starting_sample, "", -1, "", -1, "", -1);
925 
926  // Init an AVFrame to hold the decoded audio samples
927  int frame_finished = 0;
928  AVFrame *audio_frame = AV_ALLOCATE_FRAME();
929  AV_RESET_FRAME(audio_frame);
930 
931  int packet_samples = 0;
932  int data_size = 0;
933 
934  // re-initialize buffer size (it gets changed in the avcodec_decode_audio2 method call)
935  int buf_size = AVCODEC_MAX_AUDIO_FRAME_SIZE + FF_INPUT_BUFFER_PADDING_SIZE;
936  int used = avcodec_decode_audio4(aCodecCtx, audio_frame, &frame_finished, my_packet);
937 
938  if (frame_finished) {
939 
940  // determine how many samples were decoded
941  int planar = av_sample_fmt_is_planar(aCodecCtx->sample_fmt);
942  int plane_size = -1;
943  data_size = av_samples_get_buffer_size(&plane_size,
944  aCodecCtx->channels,
945  audio_frame->nb_samples,
946  aCodecCtx->sample_fmt, 1);
947 
948  // Calculate total number of samples
949  packet_samples = audio_frame->nb_samples * aCodecCtx->channels;
950  }
951 
952  // Estimate the # of samples and the end of this packet's location (to prevent GAPS for the next timestamp)
953  int pts_remaining_samples = packet_samples / info.channels; // Adjust for zero based array
954 
955  // DEBUG (FOR AUDIO ISSUES) - Get the audio packet start time (in seconds)
956  int adjusted_pts = packet->pts + audio_pts_offset;
957  double audio_seconds = double(adjusted_pts) * info.audio_timebase.ToDouble();
958  double sample_seconds = float(pts_total) / info.sample_rate;
959 
960  // Debug output
961  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (Decode Info A)", "pts_counter", pts_counter, "PTS", adjusted_pts, "Offset", audio_pts_offset, "PTS Diff", adjusted_pts - prev_pts, "Samples", pts_remaining_samples, "Sample PTS ratio", float(adjusted_pts - prev_pts) / pts_remaining_samples);
962  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (Decode Info B)", "Sample Diff", pts_remaining_samples - prev_samples - prev_pts, "Total", pts_total, "PTS Seconds", audio_seconds, "Sample Seconds", sample_seconds, "Seconds Diff", audio_seconds - sample_seconds, "raw samples", packet_samples);
963 
964  // DEBUG (FOR AUDIO ISSUES)
965  prev_pts = adjusted_pts;
966  pts_total += pts_remaining_samples;
967  pts_counter++;
968  prev_samples = pts_remaining_samples;
969 
970  // Add audio frame to list of processing audio frames
971  {
972  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
973  processing_audio_frames.insert(pair<int, int>(previous_packet_location.frame, previous_packet_location.frame));
974  }
975 
976  while (pts_remaining_samples)
977  {
978  // Get Samples per frame (for this frame number)
979  int samples_per_frame = Frame::GetSamplesPerFrame(previous_packet_location.frame, info.fps, info.sample_rate, info.channels);
980 
981  // Calculate # of samples to add to this frame
982  int samples = samples_per_frame - previous_packet_location.sample_start;
983  if (samples > pts_remaining_samples)
984  samples = pts_remaining_samples;
985 
986  // Decrement remaining samples
987  pts_remaining_samples -= samples;
988 
989  if (pts_remaining_samples > 0) {
990  // next frame
991  previous_packet_location.frame++;
992  previous_packet_location.sample_start = 0;
993 
994  // Add audio frame to list of processing audio frames
995  {
996  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
997  processing_audio_frames.insert(pair<int, int>(previous_packet_location.frame, previous_packet_location.frame));
998  }
999 
1000  } else {
1001  // Increment sample start
1002  previous_packet_location.sample_start += samples;
1003  }
1004  }
1005 
1006 
1007 
1008  // Process the audio samples in a separate thread (this includes resampling to 16 bit integer, and storing
1009  // in a openshot::Frame object).
1010  #pragma omp task firstprivate(requested_frame, target_frame, starting_sample, my_packet, audio_frame)
1011  {
1012  // Allocate audio buffer
1013  int16_t *audio_buf = new int16_t[AVCODEC_MAX_AUDIO_FRAME_SIZE + FF_INPUT_BUFFER_PADDING_SIZE];
1014 
1015  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (ReSample)", "packet_samples", packet_samples, "info.channels", info.channels, "info.sample_rate", info.sample_rate, "aCodecCtx->sample_fmt", aCodecCtx->sample_fmt, "AV_SAMPLE_FMT_S16", AV_SAMPLE_FMT_S16, "", -1);
1016 
1017  // Create output frame
1018  AVFrame *audio_converted = AV_ALLOCATE_FRAME();
1019  AV_RESET_FRAME(audio_converted);
1020  audio_converted->nb_samples = audio_frame->nb_samples;
1021  av_samples_alloc(audio_converted->data, audio_converted->linesize, info.channels, audio_frame->nb_samples, AV_SAMPLE_FMT_S16, 0);
1022 
1023  AVAudioResampleContext *avr = NULL;
1024  int nb_samples = 0;
1025  #pragma ordered
1026  {
1027  // setup resample context
1028  avr = avresample_alloc_context();
1029  av_opt_set_int(avr, "in_channel_layout", aCodecCtx->channel_layout, 0);
1030  av_opt_set_int(avr, "out_channel_layout", aCodecCtx->channel_layout, 0);
1031  av_opt_set_int(avr, "in_sample_fmt", aCodecCtx->sample_fmt, 0);
1032  av_opt_set_int(avr, "out_sample_fmt", AV_SAMPLE_FMT_S16, 0);
1033  av_opt_set_int(avr, "in_sample_rate", info.sample_rate, 0);
1034  av_opt_set_int(avr, "out_sample_rate", info.sample_rate, 0);
1035  av_opt_set_int(avr, "in_channels", info.channels, 0);
1036  av_opt_set_int(avr, "out_channels", info.channels, 0);
1037  int r = avresample_open(avr);
1038 
1039  // Convert audio samples
1040  nb_samples = avresample_convert(avr, // audio resample context
1041  audio_converted->data, // output data pointers
1042  audio_converted->linesize[0], // output plane size, in bytes. (0 if unknown)
1043  audio_converted->nb_samples, // maximum number of samples that the output buffer can hold
1044  audio_frame->data, // input data pointers
1045  audio_frame->linesize[0], // input plane size, in bytes (0 if unknown)
1046  audio_frame->nb_samples); // number of input samples to convert
1047  }
1048 
1049  // Copy audio samples over original samples
1050  memcpy(audio_buf, audio_converted->data[0], audio_converted->nb_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) * info.channels);
1051 
1052  // Deallocate resample buffer
1053  avresample_close(avr);
1054  avresample_free(&avr);
1055  avr = NULL;
1056 
1057  // Free AVFrames
1058  av_free(audio_converted->data[0]);
1059  AV_FREE_FRAME(&audio_converted);
1060 
1061  int starting_frame_number = -1;
1062  bool partial_frame = true;
1063  for (int channel_filter = 0; channel_filter < info.channels; channel_filter++)
1064  {
1065  // Array of floats (to hold samples for each channel)
1066  starting_frame_number = target_frame;
1067  int channel_buffer_size = packet_samples / info.channels;
1068  float *channel_buffer = new float[channel_buffer_size];
1069 
1070  // Init buffer array
1071  for (int z = 0; z < channel_buffer_size; z++)
1072  channel_buffer[z] = 0.0f;
1073 
1074  // Loop through all samples and add them to our Frame based on channel.
1075  // Toggle through each channel number, since channel data is stored like (left right left right)
1076  int channel = 0;
1077  int position = 0;
1078  for (int sample = 0; sample < packet_samples; sample++)
1079  {
1080  // Only add samples for current channel
1081  if (channel_filter == channel)
1082  {
1083  // Add sample (convert from (-32768 to 32768) to (-1.0 to 1.0))
1084  channel_buffer[position] = audio_buf[sample] * (1.0f / (1 << 15));
1085 
1086  // Increment audio position
1087  position++;
1088  }
1089 
1090  // increment channel (if needed)
1091  if ((channel + 1) < info.channels)
1092  // move to next channel
1093  channel ++;
1094  else
1095  // reset channel
1096  channel = 0;
1097  }
1098 
1099  // Loop through samples, and add them to the correct frames
1100  int start = starting_sample;
1101  int remaining_samples = channel_buffer_size;
1102  float *iterate_channel_buffer = channel_buffer; // pointer to channel buffer
1103  while (remaining_samples > 0)
1104  {
1105  // Get Samples per frame (for this frame number)
1106  int samples_per_frame = Frame::GetSamplesPerFrame(starting_frame_number, info.fps, info.sample_rate, info.channels);
1107 
1108  // Calculate # of samples to add to this frame
1109  int samples = samples_per_frame - start;
1110  if (samples > remaining_samples)
1111  samples = remaining_samples;
1112 
1113  // Create or get the existing frame object
1114  tr1::shared_ptr<Frame> f = CreateFrame(starting_frame_number);
1115 
1116  // Determine if this frame was "partially" filled in
1117  if (samples_per_frame == start + samples)
1118  partial_frame = false;
1119  else
1120  partial_frame = true;
1121 
1122  // Add samples for current channel to the frame. Reduce the volume to 98%, to prevent
1123  // some louder samples from maxing out at 1.0 (not sure why this happens)
1124  f->AddAudio(true, channel_filter, start, iterate_channel_buffer, samples, 0.98f);
1125 
1126  // Debug output
1127  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (f->AddAudio)", "frame", starting_frame_number, "start", start, "samples", samples, "channel", channel_filter, "partial_frame", partial_frame, "samples_per_frame", samples_per_frame);
1128 
1129  // Add or update cache
1130  working_cache.Add(f);
1131 
1132  // Decrement remaining samples
1133  remaining_samples -= samples;
1134 
1135  // Increment buffer (to next set of samples)
1136  if (remaining_samples > 0)
1137  iterate_channel_buffer += samples;
1138 
1139  // Increment frame number
1140  starting_frame_number++;
1141 
1142  // Reset starting sample #
1143  start = 0;
1144  }
1145 
1146  // clear channel buffer
1147  delete[] channel_buffer;
1148  channel_buffer = NULL;
1149  iterate_channel_buffer = NULL;
1150  }
1151 
1152  // Clean up some arrays
1153  delete[] audio_buf;
1154  audio_buf = NULL;
1155 
1156  // Remove audio frame from list of processing audio frames
1157  {
1158  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1159  // Update all frames as completed
1160  for (long int f = target_frame; f < starting_frame_number; f++) {
1161  // Remove the frame # from the processing list. NOTE: If more than one thread is
1162  // processing this frame, the frame # will be in this list multiple times. We are only
1163  // removing a single instance of it here.
1164  processing_audio_frames.erase(processing_audio_frames.find(f));
1165 
1166  // Check and see if this frame is also being processed by another thread
1167  if (processing_audio_frames.count(f) == 0)
1168  // No other thread is processing it. Mark the audio as processed (final)
1169  processed_audio_frames[f] = f;
1170  }
1171 
1172  if (target_frame == starting_frame_number) {
1173  // This typically never happens, but just in case, remove the currently processing number
1174  processing_audio_frames.erase(processing_audio_frames.find(target_frame));
1175  }
1176  }
1177 
1178  // Remove this packet
1179  RemoveAVPacket(my_packet);
1180 
1181  // Debug output
1182  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (After)", "requested_frame", requested_frame, "starting_frame", target_frame, "end_frame", starting_frame_number - 1, "", -1, "", -1, "", -1);
1183 
1184  } // end task
1185 
1186 
1187  // TODO: Fix this bug. Wait on the task to complete. This is not ideal, but solves an issue with the
1188  // audio_frame being modified by the next call to this method. I think this is a scope issue with OpenMP.
1189  #pragma omp taskwait
1190 
1191  // Free audio frame
1192  AV_FREE_FRAME(&audio_frame);
1193 }
1194 
1195 
1196 
1197 // Seek to a specific frame. This is not always frame accurate, it's more of an estimation on many codecs.
1198 void FFmpegReader::Seek(long int requested_frame) throw(TooManySeeks)
1199 {
1200  // Adjust for a requested frame that is too small or too large
1201  if (requested_frame < 1)
1202  requested_frame = 1;
1203  if (requested_frame > info.video_length)
1204  requested_frame = info.video_length;
1205 
1206  // Debug output
1207  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::Seek", "requested_frame", requested_frame, "seek_count", seek_count, "last_frame", last_frame, "processing_video_frames.size()", processing_video_frames.size(), "processing_audio_frames.size()", processing_audio_frames.size(), "", -1);
1208 
1209  // Wait for any processing frames to complete
1210  while (processing_video_frames.size() + processing_audio_frames.size() > 0)
1211  usleep(2500);
1212 
1213  // Clear working cache (since we are seeking to another location in the file)
1214  working_cache.Clear();
1215  missing_frames.Clear();
1216 
1217  // Clear processed lists
1218  {
1219  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1220  processing_audio_frames.clear();
1221  processing_video_frames.clear();
1222  processed_video_frames.clear();
1223  processed_audio_frames.clear();
1224  duplicate_video_frames.clear();
1225  missing_audio_frames.clear();
1226  missing_video_frames.clear();
1227  missing_audio_frames_source.clear();
1228  missing_video_frames_source.clear();
1229  checked_frames.clear();
1230  }
1231 
1232  // Reset the last frame variable
1233  last_frame = 0;
1234  current_video_frame = 0;
1235  largest_frame_processed = 0;
1236  num_checks_since_final = 0;
1237  num_packets_since_video_frame = 0;
1238  has_missing_frames = false;
1239 
1240  // Increment seek count
1241  seek_count++;
1242 
1243  // If seeking near frame 1, we need to close and re-open the file (this is more reliable than seeking)
1244  int buffer_amount = 6;
1245  if (requested_frame - buffer_amount < 20)
1246  {
1247  // Close and re-open file (basically seeking to frame 1)
1248  Close();
1249  Open();
1250 
1251  // Not actually seeking, so clear these flags
1252  is_seeking = false;
1253  if (seek_count == 1) {
1254  // Don't redefine this on multiple seek attempts for a specific frame
1255  seeking_frame = 1;
1256  seeking_pts = ConvertFrameToVideoPTS(1);
1257  }
1258  seek_audio_frame_found = 0; // used to detect which frames to throw away after a seek
1259  seek_video_frame_found = 0; // used to detect which frames to throw away after a seek
1260  }
1261  else
1262  {
1263  // Seek to nearest key-frame (aka, i-frame)
1264  bool seek_worked = false;
1265  int64_t seek_target = 0;
1266 
1267  // Seek video stream (if any)
1268  if (!seek_worked && info.has_video)
1269  {
1270  seek_target = ConvertFrameToVideoPTS(requested_frame - buffer_amount);
1271  if (av_seek_frame(pFormatCtx, info.video_stream_index, seek_target, AVSEEK_FLAG_BACKWARD) < 0) {
1272  fprintf(stderr, "%s: error while seeking video stream\n", pFormatCtx->filename);
1273  } else
1274  {
1275  // VIDEO SEEK
1276  is_video_seek = true;
1277  seek_worked = true;
1278  }
1279  }
1280 
1281  // Seek audio stream (if not already seeked... and if an audio stream is found)
1282  if (!seek_worked && info.has_audio)
1283  {
1284  seek_target = ConvertFrameToAudioPTS(requested_frame - buffer_amount);
1285  if (av_seek_frame(pFormatCtx, info.audio_stream_index, seek_target, AVSEEK_FLAG_BACKWARD) < 0) {
1286  fprintf(stderr, "%s: error while seeking audio stream\n", pFormatCtx->filename);
1287  } else
1288  {
1289  // AUDIO SEEK
1290  is_video_seek = false;
1291  seek_worked = true;
1292  }
1293  }
1294 
1295  // Was the seek successful?
1296  if (seek_worked)
1297  {
1298  // Flush audio buffer
1299  if (info.has_audio)
1300  avcodec_flush_buffers(aCodecCtx);
1301 
1302  // Flush video buffer
1303  if (info.has_video)
1304  avcodec_flush_buffers(pCodecCtx);
1305 
1306  // Reset previous audio location to zero
1307  previous_packet_location.frame = -1;
1308  previous_packet_location.sample_start = 0;
1309 
1310  // init seek flags
1311  is_seeking = true;
1312  if (seek_count == 1) {
1313  // Don't redefine this on multiple seek attempts for a specific frame
1314  seeking_pts = seek_target;
1315  seeking_frame = requested_frame;
1316  }
1317  seek_audio_frame_found = 0; // used to detect which frames to throw away after a seek
1318  seek_video_frame_found = 0; // used to detect which frames to throw away after a seek
1319 
1320  }
1321  else
1322  {
1323  // seek failed
1324  is_seeking = false;
1325  seeking_pts = 0;
1326  seeking_frame = 0;
1327 
1328  // dislable seeking for this reader (since it failed)
1329  // TODO: Find a safer way to do this... not sure how common it is for a seek to fail.
1330  enable_seek = false;
1331 
1332  // Close and re-open file (basically seeking to frame 1)
1333  Close();
1334  Open();
1335  }
1336  }
1337 }
1338 
1339 // Get the PTS for the current video packet
1340 long int FFmpegReader::GetVideoPTS()
1341 {
1342  int current_pts = 0;
1343  if(packet->dts != AV_NOPTS_VALUE)
1344  current_pts = packet->dts;
1345 
1346  // Return adjusted PTS
1347  return current_pts;
1348 }
1349 
1350 // Update PTS Offset (if any)
1351 void FFmpegReader::UpdatePTSOffset(bool is_video)
1352 {
1353  // Determine the offset between the PTS and Frame number (only for 1st frame)
1354  if (is_video)
1355  {
1356  // VIDEO PACKET
1357  if (video_pts_offset == 99999) // Has the offset been set yet?
1358  // Find the difference between PTS and frame number
1359  video_pts_offset = 0 - GetVideoPTS();
1360  }
1361  else
1362  {
1363  // AUDIO PACKET
1364  if (audio_pts_offset == 99999) // Has the offset been set yet?
1365  // Find the difference between PTS and frame number
1366  audio_pts_offset = 0 - packet->pts;
1367  }
1368 }
1369 
1370 // Convert PTS into Frame Number
1371 long int FFmpegReader::ConvertVideoPTStoFrame(long int pts)
1372 {
1373  // Apply PTS offset
1374  pts = pts + video_pts_offset;
1375  long int previous_video_frame = current_video_frame;
1376 
1377  // Get the video packet start time (in seconds)
1378  double video_seconds = double(pts) * info.video_timebase.ToDouble();
1379 
1380  // Divide by the video timebase, to get the video frame number (frame # is decimal at this point)
1381  long int frame = round(video_seconds * info.fps.ToDouble()) + 1;
1382 
1383  // Keep track of the expected video frame #
1384  if (current_video_frame == 0)
1385  current_video_frame = frame;
1386  else {
1387 
1388  // Sometimes frames are duplicated due to identical (or similar) timestamps
1389  if (frame == previous_video_frame) {
1390  duplicate_video_frames.insert(pair<int,int>(frame, frame));
1391 
1392  // return -1 frame number
1393  frame = -1;
1394  }
1395  else
1396  // Increment expected frame
1397  current_video_frame++;
1398 
1399  if (current_video_frame < frame)
1400  // has missing frames
1401  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ConvertVideoPTStoFrame (detected missing frame)", "calculated frame", frame, "previous_video_frame", previous_video_frame, "current_video_frame", current_video_frame, "", -1, "", -1, "", -1);
1402 
1403  // Sometimes frames are missing due to varying timestamps, or they were dropped. Determine
1404  // if we are missing a video frame.
1405  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1406  while (current_video_frame < frame) {
1407  if (!missing_video_frames.count(current_video_frame)) {
1408  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ConvertVideoPTStoFrame (tracking missing frame)", "current_video_frame", current_video_frame, "previous_video_frame", previous_video_frame, "", -1, "", -1, "", -1, "", -1);
1409  missing_video_frames.insert(pair<long int, long int>(current_video_frame, previous_video_frame));
1410  missing_video_frames_source.insert(pair<long int, long int>(previous_video_frame, current_video_frame));
1411  }
1412 
1413  // Mark this reader as containing missing frames
1414  has_missing_frames = true;
1415 
1416  // Increment current frame
1417  current_video_frame++;
1418  }
1419  }
1420 
1421  // Return frame #
1422  return frame;
1423 }
1424 
1425 // Convert Frame Number into Video PTS
1426 long int FFmpegReader::ConvertFrameToVideoPTS(long int frame_number)
1427 {
1428  // Get timestamp of this frame (in seconds)
1429  double seconds = double(frame_number) / info.fps.ToDouble();
1430 
1431  // Calculate the # of video packets in this timestamp
1432  long int video_pts = round(seconds / info.video_timebase.ToDouble());
1433 
1434  // Apply PTS offset (opposite)
1435  return video_pts - video_pts_offset;
1436 }
1437 
1438 // Convert Frame Number into Video PTS
1439 long int FFmpegReader::ConvertFrameToAudioPTS(long int frame_number)
1440 {
1441  // Get timestamp of this frame (in seconds)
1442  double seconds = double(frame_number) / info.fps.ToDouble();
1443 
1444  // Calculate the # of audio packets in this timestamp
1445  long int audio_pts = round(seconds / info.audio_timebase.ToDouble());
1446 
1447  // Apply PTS offset (opposite)
1448  return audio_pts - audio_pts_offset;
1449 }
1450 
1451 // Calculate Starting video frame and sample # for an audio PTS
1452 AudioLocation FFmpegReader::GetAudioPTSLocation(long int pts)
1453 {
1454  // Apply PTS offset
1455  pts = pts + audio_pts_offset;
1456 
1457  // Get the audio packet start time (in seconds)
1458  double audio_seconds = double(pts) * info.audio_timebase.ToDouble();
1459 
1460  // Divide by the video timebase, to get the video frame number (frame # is decimal at this point)
1461  double frame = (audio_seconds * info.fps.ToDouble()) + 1;
1462 
1463  // Frame # as a whole number (no more decimals)
1464  int whole_frame = int(frame);
1465 
1466  // Remove the whole number, and only get the decimal of the frame
1467  double sample_start_percentage = frame - double(whole_frame);
1468 
1469  // Get Samples per frame
1470  int samples_per_frame = Frame::GetSamplesPerFrame(whole_frame, info.fps, info.sample_rate, info.channels);
1471 
1472  // Calculate the sample # to start on
1473  int sample_start = round(double(samples_per_frame) * sample_start_percentage);
1474 
1475  // Protect against broken (i.e. negative) timestamps
1476  if (whole_frame < 1)
1477  whole_frame = 1;
1478  if (sample_start < 0)
1479  sample_start = 0;
1480 
1481  // Prepare final audio packet location
1482  AudioLocation location = {whole_frame, sample_start};
1483 
1484  // Compare to previous audio packet (and fix small gaps due to varying PTS timestamps)
1485  if (previous_packet_location.frame != -1 && location.is_near(previous_packet_location, samples_per_frame, samples_per_frame))
1486  {
1487  int orig_frame = location.frame;
1488  int orig_start = location.sample_start;
1489 
1490  // Update sample start, to prevent gaps in audio
1491  if (previous_packet_location.sample_start <= samples_per_frame)
1492  {
1493  location.sample_start = previous_packet_location.sample_start;
1494  location.frame = previous_packet_location.frame;
1495  }
1496  else
1497  {
1498  // set to next frame (since we exceeded the # of samples on a frame)
1499  location.sample_start = 0;
1500  location.frame++;
1501  }
1502 
1503  // Debug output
1504  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAudioPTSLocation (Audio Gap Detected)", "Source Frame", orig_frame, "Source Audio Sample", orig_start, "Target Frame", location.frame, "Target Audio Sample", location.sample_start, "pts", pts, "", -1);
1505 
1506  } else {
1507  // Debug output
1508  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAudioPTSLocation (Audio Gap Ignored - too big)", "Previous location frame", previous_packet_location.frame, "Target Frame", location.frame, "Target Audio Sample", location.sample_start, "pts", pts, "", -1, "", -1);
1509 
1510  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1511  for (long int audio_frame = previous_packet_location.frame; audio_frame < location.frame; audio_frame++) {
1512  if (!missing_audio_frames.count(audio_frame)) {
1513  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAudioPTSLocation (tracking missing frame)", "missing_audio_frame", audio_frame, "previous_audio_frame", previous_packet_location.frame, "new location frame", location.frame, "", -1, "", -1, "", -1);
1514  missing_audio_frames.insert(pair<long int, long int>(previous_packet_location.frame - 1, audio_frame));
1515  }
1516  }
1517  }
1518 
1519  // Set previous location
1520  previous_packet_location = location;
1521 
1522  // Return the associated video frame and starting sample #
1523  return location;
1524 }
1525 
1526 // Create a new Frame (or return an existing one) and add it to the working queue.
1527 tr1::shared_ptr<Frame> FFmpegReader::CreateFrame(long int requested_frame)
1528 {
1529  // Check working cache
1530  tr1::shared_ptr<Frame> output = working_cache.GetFrame(requested_frame);
1531  if (!output)
1532  {
1533  // Create a new frame on the working cache
1534  output = tr1::shared_ptr<Frame>(new Frame(requested_frame, info.width, info.height, "#000000", Frame::GetSamplesPerFrame(requested_frame, info.fps, info.sample_rate, info.channels), info.channels));
1535  output->SetPixelRatio(info.pixel_ratio.num, info.pixel_ratio.den); // update pixel ratio
1536  output->ChannelsLayout(info.channel_layout); // update audio channel layout from the parent reader
1537  output->SampleRate(info.sample_rate); // update the frame's sample rate of the parent reader
1538 
1539  working_cache.Add(output);
1540 
1541  // Set the largest processed frame (if this is larger)
1542  if (requested_frame > largest_frame_processed)
1543  largest_frame_processed = requested_frame;
1544  }
1545 
1546  // Return new frame
1547  return output;
1548 }
1549 
1550 // Determine if frame is partial due to seek
1551 bool FFmpegReader::IsPartialFrame(long int requested_frame) {
1552 
1553  // Sometimes a seek gets partial frames, and we need to remove them
1554  bool seek_trash = false;
1555  long int max_seeked_frame = seek_audio_frame_found; // determine max seeked frame
1556  if (seek_video_frame_found > max_seeked_frame)
1557  max_seeked_frame = seek_video_frame_found;
1558  if ((info.has_audio && seek_audio_frame_found && max_seeked_frame >= requested_frame) ||
1559  (info.has_video && seek_video_frame_found && max_seeked_frame >= requested_frame))
1560  seek_trash = true;
1561 
1562  return seek_trash;
1563 }
1564 
1565 // Check if a frame is missing and attempt to replace it's frame image (and
1566 bool FFmpegReader::CheckMissingFrame(long int requested_frame)
1567 {
1568  // Lock
1569  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1570 
1571  // Init # of times this frame has been checked so far
1572  int checked_count = 0;
1573 
1574  // Increment check count for this frame (or init to 1)
1575  if (checked_frames.count(requested_frame) == 0)
1576  checked_frames[requested_frame] = 1;
1577  else
1578  checked_frames[requested_frame]++;
1579  checked_count = checked_frames[requested_frame];
1580 
1581  // Debug output
1582  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckMissingFrame", "requested_frame", requested_frame, "has_missing_frames", has_missing_frames, "missing_video_frames.size()", missing_video_frames.size(), "checked_count", checked_count, "", -1, "", -1);
1583 
1584 
1585  // Missing frames (sometimes frame #'s are skipped due to invalid or missing timestamps)
1586  map<long int, long int>::iterator itr;
1587  bool found_missing_frame = false;
1588 
1589  // Check if requested frame is a missing frame
1590  if (missing_video_frames.count(requested_frame) || missing_audio_frames.count(requested_frame)) {
1591  long int missing_source_frame = -1;
1592  if (missing_video_frames.count(requested_frame))
1593  missing_source_frame = missing_video_frames.find(requested_frame)->second;
1594  else if (missing_audio_frames.count(requested_frame))
1595  missing_source_frame = missing_audio_frames.find(requested_frame)->second;
1596 
1597  // Increment missing source frame check count (or init to 1)
1598  if (checked_frames.count(missing_source_frame) == 0)
1599  checked_frames[missing_source_frame] = 1;
1600  else
1601  checked_frames[missing_source_frame]++;
1602 
1603  // Get the previous frame of this missing frame (if it's available in missing cache)
1604  tr1::shared_ptr<Frame> parent_frame = missing_frames.GetFrame(missing_source_frame);
1605  if (parent_frame == NULL) {
1606  parent_frame = final_cache.GetFrame(missing_source_frame);
1607  if (parent_frame != NULL) {
1608  // Add missing final frame to missing cache
1609  missing_frames.Add(parent_frame);
1610  }
1611  }
1612 
1613  // Create blank missing frame
1614  tr1::shared_ptr<Frame> missing_frame = CreateFrame(requested_frame);
1615 
1616  // Debug output
1617  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckMissingFrame (Is Previous Video Frame Final)", "requested_frame", requested_frame, "missing_frame->number", missing_frame->number, "missing_source_frame", missing_source_frame, "", -1, "", -1, "", -1);
1618 
1619  // If previous frame found, copy image from previous to missing frame (else we'll just wait a bit and try again later)
1620  if (parent_frame != NULL) {
1621  // Debug output
1622  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckMissingFrame (AddImage from Previous Video Frame)", "requested_frame", requested_frame, "missing_frame->number", missing_frame->number, "missing_source_frame", missing_source_frame, "", -1, "", -1, "", -1);
1623 
1624  // Add this frame to the processed map (since it's already done)
1625  missing_frame->AddImage(tr1::shared_ptr<QImage>(new QImage(*parent_frame->GetImage())));
1626 
1627  processed_video_frames[missing_frame->number] = missing_frame->number;
1628  processed_audio_frames[missing_frame->number] = missing_frame->number;
1629 
1630  // Move frame to final cache
1631  final_cache.Add(missing_frame);
1632 
1633  // Remove frame from working cache
1634  working_cache.Remove(missing_frame->number);
1635 
1636  // Update last_frame processed
1637  last_frame = missing_frame->number;
1638  }
1639 
1640  }
1641 
1642  return found_missing_frame;
1643 }
1644 
1645 // Check the working queue, and move finished frames to the finished queue
1646 void FFmpegReader::CheckWorkingFrames(bool end_of_stream, long int requested_frame)
1647 {
1648  // Loop through all working queue frames
1649  while (true)
1650  {
1651  // Get the front frame of working cache
1652  tr1::shared_ptr<Frame> f(working_cache.GetSmallestFrame());
1653 
1654  // Was a frame found?
1655  if (!f)
1656  // No frames found
1657  break;
1658 
1659  // Check if this frame is 'missing'
1660  CheckMissingFrame(f->number);
1661 
1662  // Init # of times this frame has been checked so far
1663  int checked_count = 0;
1664 
1665  bool is_video_ready = false;
1666  bool is_audio_ready = false;
1667  { // limit scope of next few lines
1668  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1669  is_video_ready = processed_video_frames.count(f->number);
1670  is_audio_ready = processed_audio_frames.count(f->number);
1671 
1672  // Get check count for this frame
1673  checked_count = checked_frames[f->number];
1674  }
1675 
1676  if (previous_packet_location.frame == f->number && !end_of_stream)
1677  is_audio_ready = false; // don't finalize the last processed audio frame
1678  bool is_seek_trash = IsPartialFrame(f->number);
1679 
1680  // Adjust for available streams
1681  if (!info.has_video) is_video_ready = true;
1682  if (!info.has_audio) is_audio_ready = true;
1683 
1684  // Make final any frames that get stuck (for whatever reason)
1685  if (checked_count > 40 && (!is_video_ready || !is_audio_ready)) {
1686  // Debug output
1687  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames (exceeded checked_count)", "frame_number", f->number, "is_video_ready", is_video_ready, "is_audio_ready", is_audio_ready, "checked_count", checked_count, "checked_frames.size()", checked_frames.size(), "", -1);
1688 
1689  if (info.has_video && !is_video_ready && last_video_frame) {
1690  // Copy image from last frame
1691  f->AddImage(tr1::shared_ptr<QImage>(new QImage(*last_video_frame->GetImage())));
1692  is_video_ready = true;
1693  }
1694 
1695  if (info.has_audio && !is_audio_ready) {
1696  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1697  // Mark audio as processed, and indicate the frame has audio data
1698  is_audio_ready = true;
1699  }
1700  }
1701 
1702  // Debug output
1703  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames", "frame_number", f->number, "is_video_ready", is_video_ready, "is_audio_ready", is_audio_ready, "checked_count", checked_count, "checked_frames.size()", checked_frames.size(), "", -1);
1704 
1705  // Check if working frame is final
1706  if ((!end_of_stream && is_video_ready && is_audio_ready && f->number <= requested_frame) || end_of_stream || is_seek_trash)
1707  {
1708  // Debug output
1709  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames (mark frame as final)", "f->number", f->number, "is_seek_trash", is_seek_trash, "Working Cache Count", working_cache.Count(), "Final Cache Count", final_cache.Count(), "", -1, "", -1);
1710 
1711  if (!is_seek_trash)
1712  {
1713  // Reset counter since last 'final' frame
1714  num_checks_since_final = 0;
1715 
1716  // Move frame to final cache
1717  final_cache.Add(f);
1718 
1719  // Add to missing cache (if another frame depends on it)
1720  {
1721  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1722  if (missing_video_frames_source.count(f->number)) {
1723  // Debug output
1724  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames (add frame to missing cache)", "f->number", f->number, "is_seek_trash", is_seek_trash, "Missing Cache Count", missing_frames.Count(), "Working Cache Count", working_cache.Count(), "Final Cache Count", final_cache.Count(), "", -1);
1725  missing_frames.Add(f);
1726  }
1727  }
1728 
1729  // Remove frame from working cache
1730  working_cache.Remove(f->number);
1731 
1732  // Update last frame processed
1733  last_frame = f->number;
1734 
1735  // Remove from 'checked' count
1736  checked_frames.erase(f->number);
1737 
1738  } else {
1739  // Seek trash, so delete the frame from the working cache, and never add it to the final cache.
1740  working_cache.Remove(f->number);
1741  }
1742  }
1743  else
1744  // Stop looping
1745  break;
1746  }
1747 }
1748 
1749 // Check for the correct frames per second (FPS) value by scanning the 1st few seconds of video packets.
1750 void FFmpegReader::CheckFPS()
1751 {
1752  check_fps = true;
1753  avpicture_alloc(pFrame, pCodecCtx->pix_fmt, info.width, info.height);
1754 
1755  int first_second_counter = 0;
1756  int second_second_counter = 0;
1757  int third_second_counter = 0;
1758  int forth_second_counter = 0;
1759  int fifth_second_counter = 0;
1760 
1761  int iterations = 0;
1762  int threshold = 500;
1763 
1764  // Loop through the stream
1765  while (true)
1766  {
1767  // Get the next packet (if any)
1768  if (GetNextPacket() < 0)
1769  // Break loop when no more packets found
1770  break;
1771 
1772  // Video packet
1773  if (packet->stream_index == videoStream)
1774  {
1775  // Check if the AVFrame is finished and set it
1776  if (GetAVFrame())
1777  {
1778  // Update PTS / Frame Offset (if any)
1779  UpdatePTSOffset(true);
1780 
1781  // Get PTS of this packet
1782  long int pts = GetVideoPTS();
1783 
1784  // Remove pFrame
1785  RemoveAVFrame(pFrame);
1786 
1787  // remove packet
1788  RemoveAVPacket(packet);
1789 
1790  // Apply PTS offset
1791  pts += video_pts_offset;
1792 
1793  // Get the video packet start time (in seconds)
1794  double video_seconds = double(pts) * info.video_timebase.ToDouble();
1795 
1796  // Increment the correct counter
1797  if (video_seconds <= 1.0)
1798  first_second_counter++;
1799  else if (video_seconds > 1.0 && video_seconds <= 2.0)
1800  second_second_counter++;
1801  else if (video_seconds > 2.0 && video_seconds <= 3.0)
1802  third_second_counter++;
1803  else if (video_seconds > 3.0 && video_seconds <= 4.0)
1804  forth_second_counter++;
1805  else if (video_seconds > 4.0 && video_seconds <= 5.0)
1806  fifth_second_counter++;
1807  else
1808  // Too far
1809  break;
1810  }
1811  else
1812  // remove packet
1813  RemoveAVPacket(packet);
1814  }
1815  else
1816  // remove packet
1817  RemoveAVPacket(packet);
1818 
1819  // Increment counters
1820  iterations++;
1821 
1822  // Give up (if threshold exceeded)
1823  if (iterations > threshold)
1824  break;
1825  }
1826 
1827  // Double check that all counters have greater than zero (or give up)
1828  if (second_second_counter == 0 || third_second_counter == 0 || forth_second_counter == 0 || fifth_second_counter == 0)
1829  {
1830  // Seek to frame 1
1831  Seek(1);
1832 
1833  // exit with no changes to FPS (not enough data to calculate)
1834  return;
1835  }
1836 
1837  int sum_fps = second_second_counter + third_second_counter + forth_second_counter + fifth_second_counter;
1838  int avg_fps = round(sum_fps / 4.0f);
1839 
1840  // Sometimes the FPS is incorrectly detected by FFmpeg. If the 1st and 2nd seconds counters
1841  // agree with each other, we are going to adjust the FPS of this reader instance. Otherwise, print
1842  // a warning message.
1843 
1844  // Get diff from actual frame rate
1845  double fps = info.fps.ToDouble();
1846  double diff = fps - double(avg_fps);
1847 
1848  // Is difference bigger than 1 frame?
1849  if (diff <= -1 || diff >= 1)
1850  {
1851  // Compare to half the frame rate (the most common type of issue)
1852  double half_fps = Fraction(info.fps.num / 2, info.fps.den).ToDouble();
1853  diff = half_fps - double(avg_fps);
1854 
1855  // Is difference bigger than 1 frame?
1856  if (diff <= -1 || diff >= 1)
1857  {
1858  // Update FPS for this reader instance
1859  info.fps = Fraction(avg_fps, 1);
1860  }
1861  else
1862  {
1863  // Update FPS for this reader instance (to 1/2 the original framerate)
1864  info.fps = Fraction(info.fps.num / 2, info.fps.den);
1865  }
1866  }
1867 
1868  // Seek to frame 1
1869  Seek(1);
1870 }
1871 
1872 // Remove AVFrame from cache (and deallocate it's memory)
1873 void FFmpegReader::RemoveAVFrame(AVPicture* remove_frame)
1874 {
1875  #pragma omp critical (packet_cache)
1876  {
1877  // Remove pFrame (if exists)
1878  if (frames.count(remove_frame))
1879  {
1880  // Free memory
1881  avpicture_free(frames[remove_frame]);
1882 
1883  // Remove from cache
1884  frames.erase(remove_frame);
1885 
1886  // Delete the object
1887  delete remove_frame;
1888  }
1889 
1890  } // end omp critical
1891 }
1892 
1893 // Remove AVPacket from cache (and deallocate it's memory)
1894 void FFmpegReader::RemoveAVPacket(AVPacket* remove_packet)
1895 {
1896  // deallocate memory for packet
1897  av_free_packet(remove_packet);
1898 
1899  // Delete the object
1900  delete remove_packet;
1901 }
1902 
1903 /// Get the smallest video frame that is still being processed
1904 long int FFmpegReader::GetSmallestVideoFrame()
1905 {
1906  // Loop through frame numbers
1907  map<long int, long int>::iterator itr;
1908  int smallest_frame = -1;
1909  for(itr = processing_video_frames.begin(); itr != processing_video_frames.end(); ++itr)
1910  {
1911  if (itr->first < smallest_frame || smallest_frame == -1)
1912  smallest_frame = itr->first;
1913  }
1914 
1915  // Return frame number
1916  return smallest_frame;
1917 }
1918 
1919 /// Get the smallest audio frame that is still being processed
1920 long int FFmpegReader::GetSmallestAudioFrame()
1921 {
1922  // Loop through frame numbers
1923  map<long int, long int>::iterator itr;
1924  int smallest_frame = -1;
1925  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1926  for(itr = processing_audio_frames.begin(); itr != processing_audio_frames.end(); ++itr)
1927  {
1928  if (itr->first < smallest_frame || smallest_frame == -1)
1929  smallest_frame = itr->first;
1930  }
1931 
1932  // Return frame number
1933  return smallest_frame;
1934 }
1935 
1936 // Generate JSON string of this object
1938 
1939  // Return formatted string
1940  return JsonValue().toStyledString();
1941 }
1942 
1943 // Generate Json::JsonValue for this object
1945 
1946  // Create root json object
1947  Json::Value root = ReaderBase::JsonValue(); // get parent properties
1948  root["type"] = "FFmpegReader";
1949  root["path"] = path;
1950 
1951  // return JsonValue
1952  return root;
1953 }
1954 
1955 // Load JSON string into this object
1956 void FFmpegReader::SetJson(string value) throw(InvalidJSON) {
1957 
1958  // Parse JSON string into JSON objects
1959  Json::Value root;
1960  Json::Reader reader;
1961  bool success = reader.parse( value, root );
1962  if (!success)
1963  // Raise exception
1964  throw InvalidJSON("JSON could not be parsed (or is invalid)", "");
1965 
1966  try
1967  {
1968  // Set all values that match
1969  SetJsonValue(root);
1970  }
1971  catch (exception e)
1972  {
1973  // Error parsing JSON (or missing keys)
1974  throw InvalidJSON("JSON is invalid (missing keys or invalid data types)", "");
1975  }
1976 }
1977 
1978 // Load Json::JsonValue into this object
1979 void FFmpegReader::SetJsonValue(Json::Value root) throw(InvalidFile) {
1980 
1981  // Set parent data
1983 
1984  // Set data from Json (if key is found)
1985  if (!root["path"].isNull())
1986  path = root["path"].asString();
1987 
1988  // Re-Open path, and re-init everything (if needed)
1989  if (is_open)
1990  {
1991  Close();
1992  Open();
1993  }
1994 }
#define AV_RESET_FRAME(av_frame)
long long file_size
Size of file (in bytes)
Definition: ReaderBase.h:65
int max_height
The maximium image height needed by this clip (used for optimizations)
Definition: ReaderBase.h:103
tr1::shared_ptr< Frame > GetFrame(long int requested_frame)
#define AV_FREE_FRAME(av_frame)
int num
Numerator for the fraction.
Definition: Fraction.h:44
CriticalSection getFrameCriticalSection
Section lock for multiple threads.
Definition: ReaderBase.h:99
void Remove(long int frame_number)
Remove a specific frame.
CriticalSection processingCriticalSection
Definition: ReaderBase.h:100
ChannelLayout channel_layout
The channel layout (mono, stereo, 5 point surround, etc...)
Definition: ReaderBase.h:83
int width
The width of the video (in pixesl)
Definition: ReaderBase.h:67
This class represents a single frame of video (i.e. image & audio data)
Definition: Frame.h:115
float ToFloat()
Return this fraction as a float (i.e. 1/2 = 0.5)
Definition: Fraction.cpp:41
float duration
Length of time (in seconds)
Definition: ReaderBase.h:64
string acodec
The name of the audio codec used to encode / decode the video stream.
Definition: ReaderBase.h:79
void Reduce()
Reduce this fraction (i.e. 640/480 = 4/3)
Definition: Fraction.cpp:71
#define AVCODEC_MAX_AUDIO_FRAME_SIZE
#define OPEN_MP_NUM_PROCESSORS
string Json()
Get and Set JSON methods.
Json::Value JsonValue()
Generate Json::JsonValue for this object.
Exception when a reader is closed, and a frame is requested.
Definition: Exceptions.h:234
bool has_video
Determines if this file has a video stream.
Definition: ReaderBase.h:61
void SetMaxBytesFromInfo(long int number_of_frames, int width, int height, int sample_rate, int channels)
Set maximum bytes to a different amount based on a ReaderInfo struct.
Definition: CacheBase.cpp:46
void Close()
Close File.
Fraction display_ratio
The ratio of width to height of the video stream (i.e. 640x480 has a ratio of 4/3) ...
Definition: ReaderBase.h:72
FFmpegReader(string path)
int audio_bit_rate
The bit rate of the audio stream (in bytes)
Definition: ReaderBase.h:80
bool has_audio
Determines if this file has an audio stream.
Definition: ReaderBase.h:62
Exception when no valid codec is found for a file.
Definition: Exceptions.h:122
int audio_stream_index
The index of the audio stream.
Definition: ReaderBase.h:84
tr1::shared_ptr< Frame > GetSmallestFrame()
Get the smallest frame number.
Exception when no streams are found in the file.
Definition: Exceptions.h:192
int height
The height of the video (in pixels)
Definition: ReaderBase.h:66
void SetJson(string value)
Load JSON string into this object.
#define AV_ALLOCATE_FRAME()
Exception for files that can not be found or opened.
Definition: Exceptions.h:132
void AppendDebugMethod(string method_name, string arg1_name, float arg1_value, string arg2_name, float arg2_value, string arg3_name, float arg3_value, string arg4_name, float arg4_value, string arg5_name, float arg5_value, string arg6_name, float arg6_value)
Append debug information.
Definition: ZmqLogger.cpp:162
This class represents a fraction.
Definition: Fraction.h:42
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround...
virtual Json::Value JsonValue()=0
Generate Json::JsonValue for this object.
Definition: ReaderBase.cpp:106
#define PixelFormat
virtual void SetJsonValue(Json::Value root)=0
Load Json::JsonValue into this object.
Definition: ReaderBase.cpp:155
ReaderInfo info
Information about the current media file.
Definition: ReaderBase.h:111
void Clear()
Clear the cache of all frames.
Fraction fps
Frames per second, as a fraction (i.e. 24/1 = 24 fps)
Definition: ReaderBase.h:69
Fraction video_timebase
The video timebase determines how long each frame stays on the screen.
Definition: ReaderBase.h:76
Exception for frames that are out of bounds.
Definition: Exceptions.h:202
int is_near(AudioLocation location, int samples_per_frame, int amount)
static ZmqLogger * Instance()
Create or get an instance of this logger singleton (invoke the class with this method) ...
Definition: ZmqLogger.cpp:38
Fraction pixel_ratio
The pixel ratio of the video stream as a fraction (i.e. some pixels are not square) ...
Definition: ReaderBase.h:71
void Add(tr1::shared_ptr< Frame > frame)
Add a Frame to the cache.
This namespace is the default namespace for all code in the openshot library.
Exception for invalid JSON.
Definition: Exceptions.h:152
int pixel_format
The pixel format (i.e. YUV420P, RGB24, etc...)
Definition: ReaderBase.h:68
This struct holds the associated video frame and starting sample # for an audio packet.
Definition: FFmpegReader.h:59
tr1::shared_ptr< Frame > GetFrame(long int frame_number)
Get a frame from the cache.
void Open()
Open File - which is called by the constructor automatically.
int video_bit_rate
The bit rate of the video stream (in bytes)
Definition: ReaderBase.h:70
#define PIX_FMT_RGBA
Fraction audio_timebase
The audio timebase determines how long each audio packet should be played.
Definition: ReaderBase.h:85
string vcodec
The name of the video codec used to encode / decode the video stream.
Definition: ReaderBase.h:73
CacheMemory final_cache
Final cache object used to hold final frames.
Definition: FFmpegReader.h:224
~FFmpegReader()
Destructor.
long int Count()
Count the frames in the queue.
int den
Denominator for the fraction.
Definition: Fraction.h:45
int channels
The number of audio channels used in the audio stream.
Definition: ReaderBase.h:82
void SetJsonValue(Json::Value root)
Load Json::JsonValue into this object.
int video_stream_index
The index of the video stream.
Definition: ReaderBase.h:75
long int video_length
The number of frames in the video stream.
Definition: ReaderBase.h:74
int max_width
The maximum image width needed by this clip (used for optimizations)
Definition: ReaderBase.h:102
int GetSamplesPerFrame(Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
Definition: Frame.cpp:497
double ToDouble()
Return this fraction as a double (i.e. 1/2 = 0.5)
Definition: Fraction.cpp:46
int sample_rate
The number of audio samples per second (44100 is a common sample rate)
Definition: ReaderBase.h:81
Exception when too many seek attempts happen.
Definition: Exceptions.h:254