OpenShot Library | libopenshot  0.1.3
FFmpegWriter.cpp
Go to the documentation of this file.
1 /**
2  * @file
3  * @brief Source file for FFmpegWriter class
4  * @author Jonathan Thomas <jonathan@openshot.org>, Fabrice Bellard
5  *
6  * @section LICENSE
7  *
8  * Copyright (c) 2008-2013 OpenShot Studios, LLC, Fabrice Bellard
9  * (http://www.openshotstudios.com). This file is part of
10  * OpenShot Library (http://www.openshot.org), an open-source project
11  * dedicated to delivering high quality video editing and animation solutions
12  * to the world.
13  *
14  * This file is originally based on the Libavformat API example, and then modified
15  * by the libopenshot project.
16  *
17  * OpenShot Library (libopenshot) is free software: you can redistribute it
18  * and/or modify it under the terms of the GNU Lesser General Public License
19  * as published by the Free Software Foundation, either version 3 of the
20  * License, or (at your option) any later version.
21  *
22  * OpenShot Library (libopenshot) is distributed in the hope that it will be
23  * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
24  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
25  * GNU Lesser General Public License for more details.
26  *
27  * You should have received a copy of the GNU Lesser General Public License
28  * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
29  */
30 
31 #include "../include/FFmpegWriter.h"
32 
33 using namespace openshot;
34 
36  path(path), fmt(NULL), oc(NULL), audio_st(NULL), video_st(NULL), audio_pts(0), video_pts(0), samples(NULL),
37  audio_outbuf(NULL), audio_outbuf_size(0), audio_input_frame_size(0), audio_input_position(0),
38  initial_audio_input_frame_size(0), img_convert_ctx(NULL), cache_size(8), num_of_rescalers(32),
39  rescaler_position(0), video_codec(NULL), audio_codec(NULL), is_writing(false), write_video_count(0), write_audio_count(0),
40  original_sample_rate(0), original_channels(0), avr(NULL), avr_planar(NULL), is_open(false), prepare_streams(false),
41  write_header(false), write_trailer(false), audio_encoder_buffer_size(0), audio_encoder_buffer(NULL)
42 {
43 
44  // Disable audio & video (so they can be independently enabled)
45  info.has_audio = false;
46  info.has_video = false;
47 
48  // Initialize FFMpeg, and register all formats and codecs
49  av_register_all();
50 
51  // auto detect format
52  auto_detect_format();
53 }
54 
55 // Open the writer
57 {
58  // Open the writer
59  is_open = true;
60 
61  // Prepare streams (if needed)
62  if (!prepare_streams)
64 
65  // Write header (if needed)
66  if (!write_header)
67  WriteHeader();
68 }
69 
70 // auto detect format (from path)
71 void FFmpegWriter::auto_detect_format()
72 {
73  // Auto detect the output format from the name. default is mpeg.
74  fmt = av_guess_format(NULL, path.c_str(), NULL);
75  if (!fmt)
76  throw InvalidFormat("Could not deduce output format from file extension.", path);
77 
78  // Allocate the output media context
79  oc = avformat_alloc_context();
80  if (!oc)
81  throw OutOfMemory("Could not allocate memory for AVFormatContext.", path);
82 
83  // Set the AVOutputFormat for the current AVFormatContext
84  oc->oformat = fmt;
85 
86  // Update codec names
87  if (fmt->video_codec != AV_CODEC_ID_NONE && info.has_video)
88  // Update video codec name
89  info.vcodec = avcodec_find_encoder(fmt->video_codec)->name;
90 
91  if (fmt->audio_codec != AV_CODEC_ID_NONE && info.has_audio)
92  // Update audio codec name
93  info.acodec = avcodec_find_encoder(fmt->audio_codec)->name;
94 }
95 
96 // initialize streams
97 void FFmpegWriter::initialize_streams()
98 {
99  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::initialize_streams", "fmt->video_codec", fmt->video_codec, "fmt->audio_codec", fmt->audio_codec, "AV_CODEC_ID_NONE", AV_CODEC_ID_NONE, "", -1, "", -1, "", -1);
100 
101  // Add the audio and video streams using the default format codecs and initialize the codecs
102  video_st = NULL;
103  audio_st = NULL;
104  if (fmt->video_codec != AV_CODEC_ID_NONE && info.has_video)
105  // Add video stream
106  video_st = add_video_stream();
107 
108  if (fmt->audio_codec != AV_CODEC_ID_NONE && info.has_audio)
109  // Add audio stream
110  audio_st = add_audio_stream();
111 }
112 
113 // Set video export options
114 void FFmpegWriter::SetVideoOptions(bool has_video, string codec, Fraction fps, int width, int height, Fraction pixel_ratio, bool interlaced, bool top_field_first, int bit_rate)
116 {
117  // Set the video options
118  if (codec.length() > 0)
119  {
120  AVCodec *new_codec = avcodec_find_encoder_by_name(codec.c_str());
121  if (new_codec == NULL)
122  throw InvalidCodec("A valid audio codec could not be found for this file.", path);
123  else {
124  // Set video codec
125  info.vcodec = new_codec->name;
126 
127  // Update video codec in fmt
128  fmt->video_codec = new_codec->id;
129  }
130  }
131  if (fps.num > 0)
132  {
133  // Set frames per second (if provided)
134  info.fps.num = fps.num;
135  info.fps.den = fps.den;
136 
137  // Set the timebase (inverse of fps)
140  }
141  if (width >= 1)
142  info.width = width;
143  if (height >= 1)
144  info.height = height;
145  if (pixel_ratio.num > 0)
146  {
147  info.pixel_ratio.num = pixel_ratio.num;
148  info.pixel_ratio.den = pixel_ratio.den;
149  }
150  if (bit_rate >= 1000)
151  info.video_bit_rate = bit_rate;
152 
153  info.interlaced_frame = interlaced;
154  info.top_field_first = top_field_first;
155 
156  // Calculate the DAR (display aspect ratio)
158 
159  // Reduce size fraction
160  size.Reduce();
161 
162  // Set the ratio based on the reduced fraction
163  info.display_ratio.num = size.num;
164  info.display_ratio.den = size.den;
165 
166  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::SetVideoOptions (" + codec + ")", "width", width, "height", height, "size.num", size.num, "size.den", size.den, "fps.num", fps.num, "fps.den", fps.den);
167 
168  // Enable / Disable video
169  info.has_video = has_video;
170 }
171 
172 // Set audio export options
173 void FFmpegWriter::SetAudioOptions(bool has_audio, string codec, int sample_rate, int channels, ChannelLayout channel_layout, int bit_rate)
175 {
176  // Set audio options
177  if (codec.length() > 0)
178  {
179  AVCodec *new_codec = avcodec_find_encoder_by_name(codec.c_str());
180  if (new_codec == NULL)
181  throw InvalidCodec("A valid audio codec could not be found for this file.", path);
182  else
183  {
184  // Set audio codec
185  info.acodec = new_codec->name;
186 
187  // Update audio codec in fmt
188  fmt->audio_codec = new_codec->id;
189  }
190  }
191  if (sample_rate > 7999)
192  info.sample_rate = sample_rate;
193  if (channels > 0)
194  info.channels = channels;
195  if (bit_rate > 999)
196  info.audio_bit_rate = bit_rate;
197  info.channel_layout = channel_layout;
198 
199  // init resample options (if zero)
200  if (original_sample_rate == 0)
201  original_sample_rate = info.sample_rate;
202  if (original_channels == 0)
203  original_channels = info.channels;
204 
205  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::SetAudioOptions (" + codec + ")", "sample_rate", sample_rate, "channels", channels, "bit_rate", bit_rate, "", -1, "", -1, "", -1);
206 
207  // Enable / Disable audio
208  info.has_audio = has_audio;
209 }
210 
211 // Set custom options (some codecs accept additional params)
212 void FFmpegWriter::SetOption(StreamType stream, string name, string value) throw(NoStreamsFound, InvalidOptions)
213 {
214  // Declare codec context
215  AVCodecContext *c = NULL;
216  stringstream convert(value);
217 
218  if (info.has_video && stream == VIDEO_STREAM && video_st)
219  c = video_st->codec;
220  else if (info.has_audio && stream == AUDIO_STREAM && audio_st)
221  c = audio_st->codec;
222  else
223  throw NoStreamsFound("The stream was not found. Be sure to call PrepareStreams() first.", path);
224 
225  // Init AVOption
226  const AVOption *option = NULL;
227 
228  // Was a codec / stream found?
229  if (c)
230  // Find AVOption (if it exists)
231  #if LIBAVFORMAT_VERSION_MAJOR <= 53
232  option = av_find_opt(c->priv_data, name.c_str(), NULL, NULL, NULL);
233  #else
234  option = av_opt_find(c->priv_data, name.c_str(), NULL, 0, 0);
235  #endif
236 
237  // Was option found?
238  if (option || (name == "g" || name == "qmin" || name == "qmax" || name == "max_b_frames" || name == "mb_decision" ||
239  name == "level" || name == "profile" || name == "slices" || name == "rc_min_rate" || name == "rc_max_rate"))
240  {
241  // Check for specific named options
242  if (name == "g")
243  // Set gop_size
244  convert >> c->gop_size;
245 
246  else if (name == "qmin")
247  // Minimum quantizer
248  convert >> c->qmin;
249 
250  else if (name == "qmax")
251  // Maximum quantizer
252  convert >> c->qmax;
253 
254  else if (name == "max_b_frames")
255  // Maximum number of B-frames between non-B-frames
256  convert >> c->max_b_frames;
257 
258  else if (name == "mb_decision")
259  // Macroblock decision mode
260  convert >> c->mb_decision;
261 
262  else if (name == "level")
263  // Set codec level
264  convert >> c->level;
265 
266  else if (name == "profile")
267  // Set codec profile
268  convert >> c->profile;
269 
270  else if (name == "slices")
271  // Indicates number of picture subdivisions
272  convert >> c->slices;
273 
274  else if (name == "rc_min_rate")
275  // Minimum bitrate
276  convert >> c->rc_min_rate;
277 
278  else if (name == "rc_max_rate")
279  // Maximum bitrate
280  convert >> c->rc_max_rate;
281 
282  else if (name == "rc_buffer_size")
283  // Buffer size
284  convert >> c->rc_buffer_size;
285 
286  else
287  // Set AVOption
288  #if LIBAVFORMAT_VERSION_MAJOR <= 53
289  av_set_string3 (c->priv_data, name.c_str(), value.c_str(), 0, NULL);
290  #else
291  av_opt_set (c->priv_data, name.c_str(), value.c_str(), 0);
292  #endif
293 
294  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::SetOption (" + (string)name + ")", "stream == VIDEO_STREAM", stream == VIDEO_STREAM, "", -1, "", -1, "", -1, "", -1, "", -1);
295 
296  }
297  else
298  throw InvalidOptions("The option is not valid for this codec.", path);
299 
300 }
301 
302 // Prepare & initialize streams and open codecs
304 {
305  if (!info.has_audio && !info.has_video)
306  throw InvalidOptions("No video or audio options have been set. You must set has_video or has_audio (or both).", path);
307 
308  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::PrepareStreams [" + path + "]", "info.has_audio", info.has_audio, "info.has_video", info.has_video, "", -1, "", -1, "", -1, "", -1);
309 
310  // Initialize the streams (i.e. add the streams)
311  initialize_streams();
312 
313  // Now that all the parameters are set, we can open the audio and video codecs and allocate the necessary encode buffers
314  if (info.has_video && video_st)
315  open_video(oc, video_st);
316  if (info.has_audio && audio_st)
317  open_audio(oc, audio_st);
318 
319  // Mark as 'prepared'
320  prepare_streams = true;
321 }
322 
323 // Write the file header (after the options are set)
325 {
326  if (!info.has_audio && !info.has_video)
327  throw InvalidOptions("No video or audio options have been set. You must set has_video or has_audio (or both).", path);
328 
329  // Open the output file, if needed
330  if (!(fmt->flags & AVFMT_NOFILE)) {
331  if (avio_open(&oc->pb, path.c_str(), AVIO_FLAG_WRITE) < 0)
332  throw InvalidFile("Could not open or write file.", path);
333  }
334 
335  // Write the stream header, if any
336  // TODO: add avoptions / parameters instead of NULL
337  avformat_write_header(oc, NULL);
338 
339  // Mark as 'written'
340  write_header = true;
341 
342  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::WriteHeader", "", -1, "", -1, "", -1, "", -1, "", -1, "", -1);
343 }
344 
345 // Add a frame to the queue waiting to be encoded.
346 void FFmpegWriter::WriteFrame(tr1::shared_ptr<Frame> frame) throw(ErrorEncodingVideo, WriterClosed)
347 {
348  // Check for open reader (or throw exception)
349  if (!is_open)
350  throw WriterClosed("The FFmpegWriter is closed. Call Open() before calling this method.", path);
351 
352  // Add frame pointer to "queue", waiting to be processed the next
353  // time the WriteFrames() method is called.
354  if (info.has_video && video_st)
355  spooled_video_frames.push_back(frame);
356 
357  if (info.has_audio && audio_st)
358  spooled_audio_frames.push_back(frame);
359 
360  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::WriteFrame", "frame->number", frame->number, "spooled_video_frames.size()", spooled_video_frames.size(), "spooled_audio_frames.size()", spooled_audio_frames.size(), "cache_size", cache_size, "is_writing", is_writing, "", -1);
361 
362  // Write the frames once it reaches the correct cache size
363  if (spooled_video_frames.size() == cache_size || spooled_audio_frames.size() == cache_size)
364  {
365  // Is writer currently writing?
366  if (!is_writing)
367  // Write frames to video file
368  write_queued_frames();
369 
370  else
371  {
372  // YES, WRITING... so wait until it finishes, before writing again
373  while (is_writing)
374  usleep(250000); // sleep for 250 milliseconds
375 
376  // Write frames to video file
377  write_queued_frames();
378  }
379  }
380 
381  // Keep track of the last frame added
382  last_frame = frame;
383 }
384 
385 // Write all frames in the queue to the video file.
386 void FFmpegWriter::write_queued_frames() throw (ErrorEncodingVideo)
387 {
388  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_queued_frames", "spooled_video_frames.size()", spooled_video_frames.size(), "spooled_audio_frames.size()", spooled_audio_frames.size(), "", -1, "", -1, "", -1, "", -1);
389 
390  // Flip writing flag
391  is_writing = true;
392 
393  // Transfer spool to queue
394  queued_video_frames = spooled_video_frames;
395  queued_audio_frames = spooled_audio_frames;
396 
397  // Empty spool
398  spooled_video_frames.clear();
399  spooled_audio_frames.clear();
400 
401  // Set the number of threads in OpenMP
402  omp_set_num_threads(OPEN_MP_NUM_PROCESSORS);
403  // Allow nested OpenMP sections
404  omp_set_nested(true);
405 
406  // Create blank exception
407  bool has_error_encoding_video = false;
408 
409  #pragma omp parallel
410  {
411  #pragma omp single
412  {
413  // Process all audio frames (in a separate thread)
414  if (info.has_audio && audio_st && !queued_audio_frames.empty())
415  write_audio_packets(false);
416 
417  // Loop through each queued image frame
418  while (!queued_video_frames.empty())
419  {
420  // Get front frame (from the queue)
421  tr1::shared_ptr<Frame> frame = queued_video_frames.front();
422 
423  // Add to processed queue
424  processed_frames.push_back(frame);
425 
426  // Encode and add the frame to the output file
427  if (info.has_video && video_st)
428  process_video_packet(frame);
429 
430  // Remove front item
431  queued_video_frames.pop_front();
432 
433  } // end while
434  } // end omp single
435 
436  #pragma omp single
437  {
438  // Loop back through the frames (in order), and write them to the video file
439  while (!processed_frames.empty())
440  {
441  // Get front frame (from the queue)
442  tr1::shared_ptr<Frame> frame = processed_frames.front();
443 
444  if (info.has_video && video_st)
445  {
446  // Add to deallocate queue (so we can remove the AVFrames when we are done)
447  deallocate_frames.push_back(frame);
448 
449  // Does this frame's AVFrame still exist
450  if (av_frames.count(frame))
451  {
452  // Get AVFrame
453  AVFrame *frame_final = av_frames[frame];
454 
455  // Write frame to video file
456  bool success = write_video_packet(frame, frame_final);
457  if (!success)
458  has_error_encoding_video = true;
459  }
460  }
461 
462  // Remove front item
463  processed_frames.pop_front();
464  }
465 
466  // Loop through, and deallocate AVFrames
467  while (!deallocate_frames.empty())
468  {
469  // Get front frame (from the queue)
470  tr1::shared_ptr<Frame> frame = deallocate_frames.front();
471 
472  // Does this frame's AVFrame still exist
473  if (av_frames.count(frame))
474  {
475  // Get AVFrame
476  AVFrame *av_frame = av_frames[frame];
477 
478  // Deallocate AVPicture and AVFrame
479  av_freep(&(av_frame->data[0]));
480  AV_FREE_FRAME(&av_frame);
481  av_frames.erase(frame);
482  }
483 
484  // Remove front item
485  deallocate_frames.pop_front();
486  }
487 
488  // Done writing
489  is_writing = false;
490 
491  } // end omp single
492  } // end omp parallel
493 
494  // Raise exception from main thread
495  if (has_error_encoding_video)
496  throw ErrorEncodingVideo("Error while writing raw video frame", -1);
497 }
498 
499 // Write a block of frames from a reader
500 void FFmpegWriter::WriteFrame(ReaderBase* reader, long int start, long int length) throw(ErrorEncodingVideo, WriterClosed)
501 {
502  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::WriteFrame (from Reader)", "start", start, "length", length, "", -1, "", -1, "", -1, "", -1);
503 
504  // Loop through each frame (and encoded it)
505  for (long int number = start; number <= length; number++)
506  {
507  // Get the frame
508  tr1::shared_ptr<Frame> f = reader->GetFrame(number);
509 
510  // Encode frame
511  WriteFrame(f);
512  }
513 }
514 
515 // Write the file trailer (after all frames are written)
517 {
518  // Write any remaining queued frames to video file
519  write_queued_frames();
520 
521  // Process final audio frame (if any)
522  if (info.has_audio && audio_st)
523  write_audio_packets(true);
524 
525  // Flush encoders (who sometimes hold on to frames)
526  flush_encoders();
527 
528  /* write the trailer, if any. The trailer must be written
529  * before you close the CodecContexts open when you wrote the
530  * header; otherwise write_trailer may try to use memory that
531  * was freed on av_codec_close() */
532  av_write_trailer(oc);
533 
534  // Mark as 'written'
535  write_trailer = true;
536 
537  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::WriteTrailer", "", -1, "", -1, "", -1, "", -1, "", -1, "", -1);
538 }
539 
540 // Flush encoders
541 void FFmpegWriter::flush_encoders()
542 {
543  if (info.has_audio && audio_codec && audio_st->codec->codec_type == AVMEDIA_TYPE_AUDIO && audio_codec->frame_size <= 1)
544  return;
545  if (info.has_video && video_st->codec->codec_type == AVMEDIA_TYPE_VIDEO && (oc->oformat->flags & AVFMT_RAWPICTURE) && video_codec->codec->id == AV_CODEC_ID_RAWVIDEO)
546  return;
547 
548  int error_code = 0;
549  int stop_encoding = 1;
550 
551  // FLUSH VIDEO ENCODER
552  if (info.has_video)
553  for (;;) {
554 
555  // Increment PTS (in frames and scaled to the codec's timebase)
556  write_video_count += av_rescale_q(1, (AVRational){info.fps.den, info.fps.num}, video_codec->time_base);
557 
558  AVPacket pkt;
559  av_init_packet(&pkt);
560  pkt.data = NULL;
561  pkt.size = 0;
562 
563  // Pointer for video buffer (if using old FFmpeg version)
564  uint8_t *video_outbuf = NULL;
565 
566  /* encode the image */
567  int got_packet = 0;
568  int error_code = 0;
569 
570  #if LIBAVFORMAT_VERSION_MAJOR >= 54
571  // Newer versions of FFMpeg
572  error_code = avcodec_encode_video2(video_codec, &pkt, NULL, &got_packet);
573 
574  #else
575  // Older versions of FFmpeg (much sloppier)
576 
577  // Encode Picture and Write Frame
578  int video_outbuf_size = 0;
579 
580  /* encode the image */
581  int out_size = avcodec_encode_video(video_codec, NULL, video_outbuf_size, NULL);
582 
583  /* if zero size, it means the image was buffered */
584  if (out_size > 0) {
585  if(video_codec->coded_frame->key_frame)
586  pkt.flags |= AV_PKT_FLAG_KEY;
587  pkt.data= video_outbuf;
588  pkt.size= out_size;
589 
590  // got data back (so encode this frame)
591  got_packet = 1;
592  }
593  #endif
594 
595  if (error_code < 0) {
596  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::flush_encoders ERROR [" + (string)av_err2str(error_code) + "]", "error_code", error_code, "", -1, "", -1, "", -1, "", -1, "", -1);
597  }
598  if (!got_packet) {
599  stop_encoding = 1;
600  break;
601  }
602 
603  // Override PTS (in frames and scaled to the codec's timebase)
604  //pkt.pts = write_video_count;
605 
606  // set the timestamp
607  if (pkt.pts != AV_NOPTS_VALUE)
608  pkt.pts = av_rescale_q(pkt.pts, video_codec->time_base, video_st->time_base);
609  if (pkt.dts != AV_NOPTS_VALUE)
610  pkt.dts = av_rescale_q(pkt.dts, video_codec->time_base, video_st->time_base);
611  if (pkt.duration > 0)
612  pkt.duration = av_rescale_q(pkt.duration, video_codec->time_base, video_st->time_base);
613  pkt.stream_index = video_st->index;
614 
615  // Write packet
616  error_code = av_interleaved_write_frame(oc, &pkt);
617  if (error_code < 0) {
618  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::flush_encoders ERROR [" + (string)av_err2str(error_code) + "]", "error_code", error_code, "", -1, "", -1, "", -1, "", -1, "", -1);
619  }
620 
621  // Deallocate memory (if needed)
622  if (video_outbuf)
623  av_freep(&video_outbuf);
624  }
625 
626  // FLUSH AUDIO ENCODER
627  if (info.has_audio)
628  for (;;) {
629 
630  // Increment PTS (in samples and scaled to the codec's timebase)
631 #if LIBAVFORMAT_VERSION_MAJOR >= 54
632  // for some reason, it requires me to multiply channels X 2
633  write_audio_count += av_rescale_q(audio_input_position / (audio_codec->channels * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16)), (AVRational){1, info.sample_rate}, audio_codec->time_base);
634 #else
635  write_audio_count += av_rescale_q(audio_input_position / audio_codec->channels, (AVRational){1, info.sample_rate}, audio_codec->time_base);
636 #endif
637 
638  AVPacket pkt;
639  av_init_packet(&pkt);
640  pkt.data = NULL;
641  pkt.size = 0;
642  pkt.pts = pkt.dts = write_audio_count;
643 
644  /* encode the image */
645  int got_packet = 0;
646  error_code = avcodec_encode_audio2(audio_codec, &pkt, NULL, &got_packet);
647  if (error_code < 0) {
648  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::flush_encoders ERROR [" + (string)av_err2str(error_code) + "]", "error_code", error_code, "", -1, "", -1, "", -1, "", -1, "", -1);
649  }
650  if (!got_packet) {
651  stop_encoding = 1;
652  break;
653  }
654 
655  // Since the PTS can change during encoding, set the value again. This seems like a huge hack,
656  // but it fixes lots of PTS related issues when I do this.
657  pkt.pts = pkt.dts = write_audio_count;
658 
659  // Scale the PTS to the audio stream timebase (which is sometimes different than the codec's timebase)
660  if (pkt.pts != AV_NOPTS_VALUE)
661  pkt.pts = av_rescale_q(pkt.pts, audio_codec->time_base, audio_st->time_base);
662  if (pkt.dts != AV_NOPTS_VALUE)
663  pkt.dts = av_rescale_q(pkt.dts, audio_codec->time_base, audio_st->time_base);
664  if (pkt.duration > 0)
665  pkt.duration = av_rescale_q(pkt.duration, audio_codec->time_base, audio_st->time_base);
666 
667  // set stream
668  pkt.stream_index = audio_st->index;
669  pkt.flags |= AV_PKT_FLAG_KEY;
670 
671  // Write packet
672  error_code = av_interleaved_write_frame(oc, &pkt);
673  if (error_code < 0) {
674  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::flush_encoders ERROR [" + (string)av_err2str(error_code) + "]", "error_code", error_code, "", -1, "", -1, "", -1, "", -1, "", -1);
675  }
676 
677  // deallocate memory for packet
678  av_free_packet(&pkt);
679  }
680 
681 
682 }
683 
684 // Close the video codec
685 void FFmpegWriter::close_video(AVFormatContext *oc, AVStream *st)
686 {
687  avcodec_close(st->codec);
688  video_codec = NULL;
689 }
690 
691 // Close the audio codec
692 void FFmpegWriter::close_audio(AVFormatContext *oc, AVStream *st)
693 {
694  avcodec_close(st->codec);
695  audio_codec = NULL;
696 
697  // Clear buffers
698  delete[] samples;
699  delete[] audio_outbuf;
700  delete[] audio_encoder_buffer;
701  samples = NULL;
702  audio_outbuf = NULL;
703  audio_encoder_buffer = NULL;
704 
705  // Deallocate resample buffer
706  if (avr) {
707  avresample_close(avr);
708  avresample_free(&avr);
709  avr = NULL;
710  }
711 
712  if (avr_planar) {
713  avresample_close(avr_planar);
714  avresample_free(&avr_planar);
715  avr_planar = NULL;
716  }
717 }
718 
719 // Close the writer
721 {
722  // Write trailer (if needed)
723  if (!write_trailer)
724  WriteTrailer();
725 
726  // Close each codec
727  if (video_st)
728  close_video(oc, video_st);
729  if (audio_st)
730  close_audio(oc, audio_st);
731 
732  // Deallocate image scalers
733  if (image_rescalers.size() > 0)
734  RemoveScalers();
735 
736  // Free the streams
737  for (int i = 0; i < oc->nb_streams; i++) {
738  av_freep(&oc->streams[i]->codec);
739  av_freep(&oc->streams[i]);
740  }
741 
742  if (!(fmt->flags & AVFMT_NOFILE)) {
743  /* close the output file */
744  avio_close(oc->pb);
745  }
746 
747  // Reset frame counters
748  write_video_count = 0;
749  write_audio_count = 0;
750 
751  // Free the context
752  av_freep(&oc);
753 
754  // Close writer
755  is_open = false;
756  prepare_streams = false;
757  write_header = false;
758  write_trailer = false;
759 
760  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::Close", "", -1, "", -1, "", -1, "", -1, "", -1, "", -1);
761 }
762 
763 // Add an AVFrame to the cache
764 void FFmpegWriter::add_avframe(tr1::shared_ptr<Frame> frame, AVFrame* av_frame)
765 {
766  // Add AVFrame to map (if it does not already exist)
767  if (!av_frames.count(frame))
768  {
769  // Add av_frame
770  av_frames[frame] = av_frame;
771  }
772  else
773  {
774  // Do not add, and deallocate this AVFrame
775  AV_FREE_FRAME(&av_frame);
776  }
777 }
778 
779 // Add an audio output stream
780 AVStream* FFmpegWriter::add_audio_stream()
781 {
782  AVCodecContext *c;
783  AVStream *st;
784 
785  // Find the audio codec
786  AVCodec *codec = avcodec_find_encoder_by_name(info.acodec.c_str());
787  if (codec == NULL)
788  throw InvalidCodec("A valid audio codec could not be found for this file.", path);
789 
790  // Create a new audio stream
791  st = avformat_new_stream(oc, codec);
792  if (!st)
793  throw OutOfMemory("Could not allocate memory for the audio stream.", path);
794 
795  // Set default values
796  avcodec_get_context_defaults3(st->codec, codec);
797 
798  c = st->codec;
799  c->codec_id = codec->id;
800 #if LIBAVFORMAT_VERSION_MAJOR >= 53
801  c->codec_type = AVMEDIA_TYPE_AUDIO;
802 #else
803  c->codec_type = CODEC_TYPE_AUDIO;
804 #endif
805 
806  // Set the sample parameters
807  c->bit_rate = info.audio_bit_rate;
808  c->channels = info.channels;
809 
810  // Set valid sample rate (or throw error)
811  if (codec->supported_samplerates) {
812  int i;
813  for (i = 0; codec->supported_samplerates[i] != 0; i++)
814  if (info.sample_rate == codec->supported_samplerates[i])
815  {
816  // Set the valid sample rate
817  c->sample_rate = info.sample_rate;
818  break;
819  }
820  if (codec->supported_samplerates[i] == 0)
821  throw InvalidSampleRate("An invalid sample rate was detected for this codec.", path);
822  } else
823  // Set sample rate
824  c->sample_rate = info.sample_rate;
825 
826 
827  // Set a valid number of channels (or throw error)
828  int channel_layout = info.channel_layout;
829  if (codec->channel_layouts) {
830  int i;
831  for (i = 0; codec->channel_layouts[i] != 0; i++)
832  if (channel_layout == codec->channel_layouts[i])
833  {
834  // Set valid channel layout
835  c->channel_layout = channel_layout;
836  break;
837  }
838  if (codec->channel_layouts[i] == 0)
839  throw InvalidChannels("An invalid channel layout was detected (i.e. MONO / STEREO).", path);
840  } else
841  // Set valid channel layout
842  c->channel_layout = channel_layout;
843 
844  // Choose a valid sample_fmt
845  if (codec->sample_fmts) {
846  for (int i = 0; codec->sample_fmts[i] != AV_SAMPLE_FMT_NONE; i++)
847  {
848  // Set sample format to 1st valid format (and then exit loop)
849  c->sample_fmt = codec->sample_fmts[i];
850  break;
851  }
852  }
853  if (c->sample_fmt == AV_SAMPLE_FMT_NONE) {
854  // Default if no sample formats found
855  c->sample_fmt = AV_SAMPLE_FMT_S16;
856  }
857 
858  // some formats want stream headers to be separate
859  if (oc->oformat->flags & AVFMT_GLOBALHEADER)
860  c->flags |= CODEC_FLAG_GLOBAL_HEADER;
861 
862  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::add_audio_stream", "c->codec_id", c->codec_id, "c->bit_rate", c->bit_rate, "c->channels", c->channels, "c->sample_fmt", c->sample_fmt, "c->channel_layout", c->channel_layout, "c->sample_rate", c->sample_rate);
863 
864  return st;
865 }
866 
867 // Add a video output stream
868 AVStream* FFmpegWriter::add_video_stream()
869 {
870  AVCodecContext *c;
871  AVStream *st;
872 
873  // Find the audio codec
874  AVCodec *codec = avcodec_find_encoder_by_name(info.vcodec.c_str());
875  if (codec == NULL)
876  throw InvalidCodec("A valid video codec could not be found for this file.", path);
877 
878  // Create a new stream
879  st = avformat_new_stream(oc, codec);
880  if (!st)
881  throw OutOfMemory("Could not allocate memory for the video stream.", path);
882 
883  // Set default values
884  avcodec_get_context_defaults3(st->codec, codec);
885 
886  c = st->codec;
887  c->codec_id = codec->id;
888 #if LIBAVFORMAT_VERSION_MAJOR >= 53
889  c->codec_type = AVMEDIA_TYPE_VIDEO;
890 #else
891  c->codec_type = CODEC_TYPE_VIDEO;
892 #endif
893 
894  /* Init video encoder options */
895  c->bit_rate = info.video_bit_rate;
896  c->rc_min_rate = info.video_bit_rate - (info.video_bit_rate / 6);
897  c->rc_max_rate = info.video_bit_rate;
898  c->rc_buffer_size = FFMAX(c->rc_max_rate, 15000000) * 112L / 15000000 * 16384;
899  c->qmin = 2;
900  c->qmax = 30;
901 
902  /* resolution must be a multiple of two */
903  // TODO: require /2 height and width
904  c->width = info.width;
905  c->height = info.height;
906 
907  /* time base: this is the fundamental unit of time (in seconds) in terms
908  of which frame timestamps are represented. for fixed-fps content,
909  timebase should be 1/framerate and timestamp increments should be
910  identically 1. */
911  c->time_base.num = info.video_timebase.num;
912  c->time_base.den = info.video_timebase.den;
913  c->gop_size = 12; /* TODO: add this to "info"... emit one intra frame every twelve frames at most */
914  c->max_b_frames = 10;
915  if (c->codec_id == AV_CODEC_ID_MPEG2VIDEO)
916  /* just for testing, we also add B frames */
917  c->max_b_frames = 2;
918  if (c->codec_id == AV_CODEC_ID_MPEG1VIDEO)
919  /* Needed to avoid using macroblocks in which some coeffs overflow.
920  This does not happen with normal video, it just happens here as
921  the motion of the chroma plane does not match the luma plane. */
922  c->mb_decision = 2;
923  // some formats want stream headers to be separate
924  if (oc->oformat->flags & AVFMT_GLOBALHEADER)
925  c->flags |= CODEC_FLAG_GLOBAL_HEADER;
926 
927  // Find all supported pixel formats for this codec
928  const PixelFormat* supported_pixel_formats = codec->pix_fmts;
929  while (supported_pixel_formats != NULL && *supported_pixel_formats != PIX_FMT_NONE) {
930  // Assign the 1st valid pixel format (if one is missing)
931  if (c->pix_fmt == PIX_FMT_NONE)
932  c->pix_fmt = *supported_pixel_formats;
933  ++supported_pixel_formats;
934  }
935 
936  // Codec doesn't have any pix formats?
937  if (c->pix_fmt == PIX_FMT_NONE) {
938  if(fmt->video_codec == AV_CODEC_ID_RAWVIDEO) {
939  // Raw video should use RGB24
940  c->pix_fmt = PIX_FMT_RGB24;
941 
942  if (strcmp(fmt->name, "gif") != 0)
943  // If not GIF format, skip the encoding process
944  // Set raw picture flag (so we don't encode this video)
945  oc->oformat->flags |= AVFMT_RAWPICTURE;
946  } else {
947  // Set the default codec
948  c->pix_fmt = PIX_FMT_YUV420P;
949  }
950  }
951 
952  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::add_video_stream (" + (string)fmt->name + " : " + (string)av_get_pix_fmt_name(c->pix_fmt) + ")", "c->codec_id", c->codec_id, "c->bit_rate", c->bit_rate, "c->pix_fmt", c->pix_fmt, "oc->oformat->flags", oc->oformat->flags, "AVFMT_RAWPICTURE", AVFMT_RAWPICTURE, "", -1);
953 
954  return st;
955 }
956 
957 // open audio codec
958 void FFmpegWriter::open_audio(AVFormatContext *oc, AVStream *st)
959 {
960  AVCodec *codec;
961  audio_codec = st->codec;
962 
963  // Set number of threads equal to number of processors + 1
964  audio_codec->thread_count = OPEN_MP_NUM_PROCESSORS;
965 
966  // Find the audio encoder
967  codec = avcodec_find_encoder(audio_codec->codec_id);
968  if (!codec)
969  throw InvalidCodec("Could not find codec", path);
970 
971  // Open the codec
972  if (avcodec_open2(audio_codec, codec, NULL) < 0)
973  throw InvalidCodec("Could not open codec", path);
974 
975  // Calculate the size of the input frame (i..e how many samples per packet), and the output buffer
976  // TODO: Ugly hack for PCM codecs (will be removed ASAP with new PCM support to compute the input frame size in samples
977  if (audio_codec->frame_size <= 1) {
978  // No frame size found... so calculate
979  audio_input_frame_size = 50000 / info.channels;
980 
981  switch (st->codec->codec_id) {
982  case AV_CODEC_ID_PCM_S16LE:
983  case AV_CODEC_ID_PCM_S16BE:
984  case AV_CODEC_ID_PCM_U16LE:
985  case AV_CODEC_ID_PCM_U16BE:
986  audio_input_frame_size >>= 1;
987  break;
988  default:
989  break;
990  }
991  } else {
992  // Set frame size based on the codec
993  audio_input_frame_size = audio_codec->frame_size;
994  }
995 
996  // Set the initial frame size (since it might change during resampling)
997  initial_audio_input_frame_size = audio_input_frame_size;
998 
999  // Allocate array for samples
1000  samples = new int16_t[AVCODEC_MAX_AUDIO_FRAME_SIZE];
1001 
1002  // Set audio output buffer (used to store the encoded audio)
1003  audio_outbuf_size = AVCODEC_MAX_AUDIO_FRAME_SIZE;
1004  audio_outbuf = new uint8_t[audio_outbuf_size];
1005 
1006  // Set audio packet encoding buffer
1007  audio_encoder_buffer_size = AUDIO_PACKET_ENCODING_SIZE;
1008  audio_encoder_buffer = new uint8_t[audio_encoder_buffer_size];
1009 
1010  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::open_audio", "audio_codec->thread_count", audio_codec->thread_count, "audio_input_frame_size", audio_input_frame_size, "buffer_size", AVCODEC_MAX_AUDIO_FRAME_SIZE + FF_INPUT_BUFFER_PADDING_SIZE, "", -1, "", -1, "", -1);
1011 
1012 }
1013 
1014 // open video codec
1015 void FFmpegWriter::open_video(AVFormatContext *oc, AVStream *st)
1016 {
1017  AVCodec *codec;
1018  video_codec = st->codec;
1019 
1020  // Set number of threads equal to number of processors + 1
1021  video_codec->thread_count = OPEN_MP_NUM_PROCESSORS;
1022 
1023  /* find the video encoder */
1024  codec = avcodec_find_encoder(video_codec->codec_id);
1025  if (!codec)
1026  throw InvalidCodec("Could not find codec", path);
1027 
1028  /* open the codec */
1029  if (avcodec_open2(video_codec, codec, NULL) < 0)
1030  throw InvalidCodec("Could not open codec", path);
1031 
1032  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::open_video", "video_codec->thread_count", video_codec->thread_count, "", -1, "", -1, "", -1, "", -1, "", -1);
1033 
1034 }
1035 
1036 // write all queued frames' audio to the video file
1037 void FFmpegWriter::write_audio_packets(bool final)
1038 {
1039  #pragma omp task firstprivate(final)
1040  {
1041  // Init audio buffers / variables
1042  int total_frame_samples = 0;
1043  int frame_position = 0;
1044  int channels_in_frame = 0;
1045  int sample_rate_in_frame = 0;
1046  int samples_in_frame = 0;
1047  ChannelLayout channel_layout_in_frame = LAYOUT_MONO; // default channel layout
1048 
1049  // Create a new array (to hold all S16 audio samples, for the current queued frames
1050  int16_t* all_queued_samples = (int16_t*)av_malloc((sizeof(int16_t)*(queued_audio_frames.size() * AVCODEC_MAX_AUDIO_FRAME_SIZE)));
1051  int16_t* all_resampled_samples = NULL;
1052  int16_t* final_samples_planar = NULL;
1053  int16_t* final_samples = NULL;
1054 
1055  // Loop through each queued audio frame
1056  while (!queued_audio_frames.empty())
1057  {
1058  // Get front frame (from the queue)
1059  tr1::shared_ptr<Frame> frame = queued_audio_frames.front();
1060 
1061  // Get the audio details from this frame
1062  sample_rate_in_frame = frame->SampleRate();
1063  samples_in_frame = frame->GetAudioSamplesCount();
1064  channels_in_frame = frame->GetAudioChannelsCount();
1065  channel_layout_in_frame = frame->ChannelsLayout();
1066 
1067 
1068  // Get audio sample array
1069  float* frame_samples_float = NULL;
1070  // Get samples interleaved together (c1 c2 c1 c2 c1 c2)
1071  frame_samples_float = frame->GetInterleavedAudioSamples(sample_rate_in_frame, NULL, &samples_in_frame);
1072 
1073 
1074  // Calculate total samples
1075  total_frame_samples = samples_in_frame * channels_in_frame;
1076 
1077  // Translate audio sample values back to 16 bit integers
1078  for (int s = 0; s < total_frame_samples; s++, frame_position++)
1079  // Translate sample value and copy into buffer
1080  all_queued_samples[frame_position] = int(frame_samples_float[s] * (1 << 15));
1081 
1082 
1083  // Deallocate float array
1084  delete[] frame_samples_float;
1085 
1086  // Remove front item
1087  queued_audio_frames.pop_front();
1088 
1089  } // end while
1090 
1091 
1092  // Update total samples (since we've combined all queued frames)
1093  total_frame_samples = frame_position;
1094  int remaining_frame_samples = total_frame_samples;
1095  int samples_position = 0;
1096 
1097 
1098  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_audio_packets", "final", final, "total_frame_samples", total_frame_samples, "channel_layout_in_frame", channel_layout_in_frame, "channels_in_frame", channels_in_frame, "samples_in_frame", samples_in_frame, "LAYOUT_MONO", LAYOUT_MONO);
1099 
1100  // Keep track of the original sample format
1101  AVSampleFormat output_sample_fmt = audio_codec->sample_fmt;
1102 
1103  AVFrame *audio_frame = NULL;
1104  if (!final) {
1105  // Create input frame (and allocate arrays)
1106  audio_frame = AV_ALLOCATE_FRAME();
1107  AV_RESET_FRAME(audio_frame);
1108  audio_frame->nb_samples = total_frame_samples / channels_in_frame;
1109 
1110  // Fill input frame with sample data
1111  avcodec_fill_audio_frame(audio_frame, channels_in_frame, AV_SAMPLE_FMT_S16, (uint8_t *) all_queued_samples,
1112  audio_encoder_buffer_size, 0);
1113 
1114  // Do not convert audio to planar format (yet). We need to keep everything interleaved at this point.
1115  switch (audio_codec->sample_fmt)
1116  {
1117  case AV_SAMPLE_FMT_FLTP:
1118  {
1119  output_sample_fmt = AV_SAMPLE_FMT_FLT;
1120  break;
1121  }
1122  case AV_SAMPLE_FMT_S32P:
1123  {
1124  output_sample_fmt = AV_SAMPLE_FMT_S32;
1125  break;
1126  }
1127  case AV_SAMPLE_FMT_S16P:
1128  {
1129  output_sample_fmt = AV_SAMPLE_FMT_S16;
1130  break;
1131  }
1132  case AV_SAMPLE_FMT_U8P:
1133  {
1134  output_sample_fmt = AV_SAMPLE_FMT_U8;
1135  break;
1136  }
1137  }
1138 
1139  // Update total samples & input frame size (due to bigger or smaller data types)
1140  total_frame_samples *= (float(info.sample_rate) / sample_rate_in_frame); // adjust for different byte sizes
1141  total_frame_samples *= (float(info.channels) / channels_in_frame); // adjust for different # of channels
1142 
1143  // Set remaining samples
1144  remaining_frame_samples = total_frame_samples;
1145 
1146  // Create output frame (and allocate arrays)
1147  AVFrame *audio_converted = AV_ALLOCATE_FRAME();
1148  AV_RESET_FRAME(audio_converted);
1149  audio_converted->nb_samples = total_frame_samples / channels_in_frame;
1150  av_samples_alloc(audio_converted->data, audio_converted->linesize, info.channels, audio_converted->nb_samples, output_sample_fmt, 0);
1151 
1152  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_audio_packets (1st resampling)", "in_sample_fmt", AV_SAMPLE_FMT_S16, "out_sample_fmt", output_sample_fmt, "in_sample_rate", sample_rate_in_frame, "out_sample_rate", info.sample_rate, "in_channels", channels_in_frame, "out_channels", info.channels);
1153 
1154  // setup resample context
1155  if (!avr) {
1156  avr = avresample_alloc_context();
1157  av_opt_set_int(avr, "in_channel_layout", channel_layout_in_frame, 0);
1158  av_opt_set_int(avr, "out_channel_layout", info.channel_layout, 0);
1159  av_opt_set_int(avr, "in_sample_fmt", AV_SAMPLE_FMT_S16, 0);
1160  av_opt_set_int(avr, "out_sample_fmt", output_sample_fmt, 0); // planar not allowed here
1161  av_opt_set_int(avr, "in_sample_rate", sample_rate_in_frame, 0);
1162  av_opt_set_int(avr, "out_sample_rate", info.sample_rate, 0);
1163  av_opt_set_int(avr, "in_channels", channels_in_frame, 0);
1164  av_opt_set_int(avr, "out_channels", info.channels, 0);
1165  avresample_open(avr);
1166  }
1167  int nb_samples = 0;
1168 
1169  // Convert audio samples
1170  nb_samples = avresample_convert(avr, // audio resample context
1171  audio_converted->data, // output data pointers
1172  audio_converted->linesize[0], // output plane size, in bytes. (0 if unknown)
1173  audio_converted->nb_samples, // maximum number of samples that the output buffer can hold
1174  audio_frame->data, // input data pointers
1175  audio_frame->linesize[0], // input plane size, in bytes (0 if unknown)
1176  audio_frame->nb_samples); // number of input samples to convert
1177 
1178  // Create a new array (to hold all resampled S16 audio samples)
1179  all_resampled_samples = (int16_t*)av_malloc(sizeof(int16_t) * nb_samples * info.channels * (av_get_bytes_per_sample(output_sample_fmt) / av_get_bytes_per_sample(AV_SAMPLE_FMT_S16)));
1180 
1181  // Copy audio samples over original samples
1182  memcpy(all_resampled_samples, audio_converted->data[0], nb_samples * info.channels * av_get_bytes_per_sample(output_sample_fmt));
1183 
1184  // Remove converted audio
1185  av_freep(&(audio_frame->data[0]));
1186  AV_FREE_FRAME(&audio_frame);
1187  av_freep(&audio_converted->data[0]);
1188  AV_FREE_FRAME(&audio_converted);
1189  all_queued_samples = NULL; // this array cleared with above call
1190 
1191  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_audio_packets (Successfully completed 1st resampling)", "nb_samples", nb_samples, "remaining_frame_samples", remaining_frame_samples, "", -1, "", -1, "", -1, "", -1);
1192  }
1193 
1194  // Loop until no more samples
1195  while (remaining_frame_samples > 0 || final) {
1196  // Get remaining samples needed for this packet
1197  int remaining_packet_samples = (audio_input_frame_size * info.channels) - audio_input_position;
1198 
1199  // Determine how many samples we need
1200  int diff = 0;
1201  if (remaining_frame_samples >= remaining_packet_samples)
1202  diff = remaining_packet_samples;
1203  else if (remaining_frame_samples < remaining_packet_samples)
1204  diff = remaining_frame_samples;
1205 
1206  // Copy frame samples into the packet samples array
1207  if (!final)
1208  //TODO: Make this more sane
1209  memcpy(samples + (audio_input_position * (av_get_bytes_per_sample(output_sample_fmt) / av_get_bytes_per_sample(AV_SAMPLE_FMT_S16))), all_resampled_samples + samples_position, diff * av_get_bytes_per_sample(output_sample_fmt));
1210 
1211  // Increment counters
1212  audio_input_position += diff;
1213  samples_position += diff * (av_get_bytes_per_sample(output_sample_fmt) / av_get_bytes_per_sample(AV_SAMPLE_FMT_S16));
1214  remaining_frame_samples -= diff;
1215  remaining_packet_samples -= diff;
1216 
1217  // Do we have enough samples to proceed?
1218  if (audio_input_position < (audio_input_frame_size * info.channels) && !final)
1219  // Not enough samples to encode... so wait until the next frame
1220  break;
1221 
1222  // Convert to planar (if needed by audio codec)
1223  AVFrame *frame_final = AV_ALLOCATE_FRAME();
1224  AV_RESET_FRAME(frame_final);
1225  if (av_sample_fmt_is_planar(audio_codec->sample_fmt))
1226  {
1227  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_audio_packets (2nd resampling for Planar formats)", "in_sample_fmt", output_sample_fmt, "out_sample_fmt", audio_codec->sample_fmt, "in_sample_rate", info.sample_rate, "out_sample_rate", info.sample_rate, "in_channels", info.channels, "out_channels", info.channels);
1228 
1229  // setup resample context
1230  if (!avr_planar) {
1231  avr_planar = avresample_alloc_context();
1232  av_opt_set_int(avr_planar, "in_channel_layout", info.channel_layout, 0);
1233  av_opt_set_int(avr_planar, "out_channel_layout", info.channel_layout, 0);
1234  av_opt_set_int(avr_planar, "in_sample_fmt", output_sample_fmt, 0);
1235  av_opt_set_int(avr_planar, "out_sample_fmt", audio_codec->sample_fmt, 0); // planar not allowed here
1236  av_opt_set_int(avr_planar, "in_sample_rate", info.sample_rate, 0);
1237  av_opt_set_int(avr_planar, "out_sample_rate", info.sample_rate, 0);
1238  av_opt_set_int(avr_planar, "in_channels", info.channels, 0);
1239  av_opt_set_int(avr_planar, "out_channels", info.channels, 0);
1240  avresample_open(avr_planar);
1241  }
1242 
1243  // Create input frame (and allocate arrays)
1244  audio_frame = AV_ALLOCATE_FRAME();
1245  AV_RESET_FRAME(audio_frame);
1246  audio_frame->nb_samples = audio_input_position / info.channels;
1247 
1248  // Create a new array
1249  final_samples_planar = (int16_t*)av_malloc(sizeof(int16_t) * audio_frame->nb_samples * info.channels * (av_get_bytes_per_sample(output_sample_fmt) / av_get_bytes_per_sample(AV_SAMPLE_FMT_S16)));
1250 
1251  // Copy audio into buffer for frame
1252  memcpy(final_samples_planar, samples, audio_frame->nb_samples * info.channels * av_get_bytes_per_sample(output_sample_fmt));
1253 
1254  // Fill input frame with sample data
1255  avcodec_fill_audio_frame(audio_frame, info.channels, output_sample_fmt, (uint8_t *) final_samples_planar,
1256  audio_encoder_buffer_size, 0);
1257 
1258  // Create output frame (and allocate arrays)
1259  frame_final->nb_samples = audio_input_frame_size;
1260  av_samples_alloc(frame_final->data, frame_final->linesize, info.channels, frame_final->nb_samples, audio_codec->sample_fmt, 0);
1261 
1262  // Convert audio samples
1263  int nb_samples = avresample_convert(avr_planar, // audio resample context
1264  frame_final->data, // output data pointers
1265  frame_final->linesize[0], // output plane size, in bytes. (0 if unknown)
1266  frame_final->nb_samples, // maximum number of samples that the output buffer can hold
1267  audio_frame->data, // input data pointers
1268  audio_frame->linesize[0], // input plane size, in bytes (0 if unknown)
1269  audio_frame->nb_samples); // number of input samples to convert
1270 
1271  // Copy audio samples over original samples
1272  if (nb_samples > 0)
1273  memcpy(samples, frame_final->data[0], nb_samples * av_get_bytes_per_sample(audio_codec->sample_fmt) * info.channels);
1274 
1275  // deallocate AVFrame
1276  av_freep(&(audio_frame->data[0]));
1277  AV_FREE_FRAME(&audio_frame);
1278  all_queued_samples = NULL; // this array cleared with above call
1279 
1280  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_audio_packets (Successfully completed 2nd resampling for Planar formats)", "nb_samples", nb_samples, "", -1, "", -1, "", -1, "", -1, "", -1);
1281 
1282  } else {
1283  // Create a new array
1284  final_samples = new int16_t[audio_input_position * (av_get_bytes_per_sample(audio_codec->sample_fmt) / av_get_bytes_per_sample(AV_SAMPLE_FMT_S16))];
1285 
1286  // Copy audio into buffer for frame
1287  memcpy(final_samples, samples, audio_input_position * av_get_bytes_per_sample(audio_codec->sample_fmt));
1288 
1289  // Init the nb_samples property
1290  frame_final->nb_samples = audio_input_frame_size;
1291 
1292  // Fill the final_frame AVFrame with audio (non planar)
1293  avcodec_fill_audio_frame(frame_final, audio_codec->channels, audio_codec->sample_fmt, (uint8_t *) final_samples,
1294  audio_encoder_buffer_size, 0);
1295  }
1296 
1297  // Increment PTS (in samples)
1298  write_audio_count += FFMIN(audio_input_frame_size, audio_input_position);
1299  frame_final->pts = write_audio_count; // Set the AVFrame's PTS
1300 
1301  // Init the packet
1302  AVPacket pkt;
1303  av_init_packet(&pkt);
1304  pkt.data = audio_encoder_buffer;
1305  pkt.size = audio_encoder_buffer_size;
1306 
1307  // Set the packet's PTS prior to encoding
1308  pkt.pts = pkt.dts = write_audio_count;
1309 
1310  /* encode the audio samples */
1311  int got_packet_ptr = 0;
1312  int error_code = avcodec_encode_audio2(audio_codec, &pkt, frame_final, &got_packet_ptr);
1313 
1314  /* if zero size, it means the image was buffered */
1315  if (error_code == 0 && got_packet_ptr) {
1316 
1317  // Since the PTS can change during encoding, set the value again. This seems like a huge hack,
1318  // but it fixes lots of PTS related issues when I do this.
1319  pkt.pts = pkt.dts = write_audio_count;
1320 
1321  // Scale the PTS to the audio stream timebase (which is sometimes different than the codec's timebase)
1322  if (pkt.pts != AV_NOPTS_VALUE)
1323  pkt.pts = av_rescale_q(pkt.pts, audio_codec->time_base, audio_st->time_base);
1324  if (pkt.dts != AV_NOPTS_VALUE)
1325  pkt.dts = av_rescale_q(pkt.dts, audio_codec->time_base, audio_st->time_base);
1326  if (pkt.duration > 0)
1327  pkt.duration = av_rescale_q(pkt.duration, audio_codec->time_base, audio_st->time_base);
1328 
1329  // set stream
1330  pkt.stream_index = audio_st->index;
1331  pkt.flags |= AV_PKT_FLAG_KEY;
1332 
1333  /* write the compressed frame in the media file */
1334  int error_code = av_interleaved_write_frame(oc, &pkt);
1335  if (error_code < 0)
1336  {
1337  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_audio_packets ERROR [" + (string)av_err2str(error_code) + "]", "error_code", error_code, "", -1, "", -1, "", -1, "", -1, "", -1);
1338  }
1339  }
1340 
1341  if (error_code < 0)
1342  {
1343  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_audio_packets ERROR [" + (string)av_err2str(error_code) + "]", "error_code", error_code, "", -1, "", -1, "", -1, "", -1, "", -1);
1344  }
1345 
1346  // deallocate AVFrame
1347  av_freep(&(frame_final->data[0]));
1348  AV_FREE_FRAME(&frame_final);
1349 
1350  // deallocate memory for packet
1351  av_free_packet(&pkt);
1352 
1353  // Reset position
1354  audio_input_position = 0;
1355  final = false;
1356  }
1357 
1358  // Delete arrays (if needed)
1359  if (all_resampled_samples) {
1360  av_freep(&all_resampled_samples);
1361  all_resampled_samples = NULL;
1362  }
1363  if (all_queued_samples) {
1364  av_freep(&all_queued_samples);
1365  all_queued_samples = NULL;
1366  }
1367 
1368  } // end task
1369 }
1370 
1371 // Allocate an AVFrame object
1372 AVFrame* FFmpegWriter::allocate_avframe(PixelFormat pix_fmt, int width, int height, int *buffer_size, uint8_t *new_buffer)
1373 {
1374  // Create an RGB AVFrame
1375  AVFrame *new_av_frame = NULL;
1376 
1377  // Allocate an AVFrame structure
1378  new_av_frame = AV_ALLOCATE_FRAME();
1379  if (new_av_frame == NULL)
1380  throw OutOfMemory("Could not allocate AVFrame", path);
1381 
1382  // Determine required buffer size and allocate buffer
1383  *buffer_size = avpicture_get_size(pix_fmt, width, height);
1384 
1385  // Create buffer (if not provided)
1386  if (!new_buffer)
1387  {
1388  // New Buffer
1389  new_buffer = (uint8_t*)av_malloc(*buffer_size * sizeof(uint8_t));
1390  // Attach buffer to AVFrame
1391  avpicture_fill((AVPicture *)new_av_frame, new_buffer, pix_fmt, width, height);
1392  new_av_frame->width = width;
1393  new_av_frame->height = height;
1394  new_av_frame->format = pix_fmt;
1395  }
1396 
1397  // return AVFrame
1398  return new_av_frame;
1399 }
1400 
1401 // process video frame
1402 void FFmpegWriter::process_video_packet(tr1::shared_ptr<Frame> frame)
1403 {
1404  // Determine the height & width of the source image
1405  int source_image_width = frame->GetWidth();
1406  int source_image_height = frame->GetHeight();
1407 
1408  // Do nothing if size is 1x1 (i.e. no image in this frame)
1409  if (source_image_height == 1 && source_image_width == 1)
1410  return;
1411 
1412  // Init rescalers (if not initialized yet)
1413  if (image_rescalers.size() == 0)
1414  InitScalers(source_image_width, source_image_height);
1415 
1416  // Get a unique rescaler (for this thread)
1417  SwsContext *scaler = image_rescalers[rescaler_position];
1418  rescaler_position++;
1419  if (rescaler_position == num_of_rescalers)
1420  rescaler_position = 0;
1421 
1422  #pragma omp task firstprivate(frame, scaler, source_image_width, source_image_height)
1423  {
1424  // Allocate an RGB frame & final output frame
1425  int bytes_source = 0;
1426  int bytes_final = 0;
1427  AVFrame *frame_source = NULL;
1428  const uchar *pixels = NULL;
1429 
1430  // Get a list of pixels from source image
1431  pixels = frame->GetPixels();
1432 
1433  // Init AVFrame for source image & final (converted image)
1434  frame_source = allocate_avframe(PIX_FMT_RGBA, source_image_width, source_image_height, &bytes_source, (uint8_t*) pixels);
1435  AVFrame *frame_final = allocate_avframe(video_codec->pix_fmt, info.width, info.height, &bytes_final, NULL);
1436 
1437  // Fill with data
1438  avpicture_fill((AVPicture *) frame_source, (uint8_t*)pixels, PIX_FMT_RGBA, source_image_width, source_image_height);
1439  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::process_video_packet", "frame->number", frame->number, "bytes_source", bytes_source, "bytes_final", bytes_final, "", -1, "", -1, "", -1);
1440 
1441  // Resize & convert pixel format
1442  sws_scale(scaler, frame_source->data, frame_source->linesize, 0,
1443  source_image_height, frame_final->data, frame_final->linesize);
1444 
1445  // Add resized AVFrame to av_frames map
1446  #pragma omp critical (av_frames_section)
1447  add_avframe(frame, frame_final);
1448 
1449  // Deallocate memory
1450  AV_FREE_FRAME(&frame_source);
1451 
1452  } // end task
1453 
1454 }
1455 
1456 // write video frame
1457 bool FFmpegWriter::write_video_packet(tr1::shared_ptr<Frame> frame, AVFrame* frame_final)
1458 {
1459  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_video_packet", "frame->number", frame->number, "oc->oformat->flags & AVFMT_RAWPICTURE", oc->oformat->flags & AVFMT_RAWPICTURE, "", -1, "", -1, "", -1, "", -1);
1460 
1461  if (oc->oformat->flags & AVFMT_RAWPICTURE) {
1462  // Raw video case.
1463  AVPacket pkt;
1464  av_init_packet(&pkt);
1465 
1466  pkt.flags |= AV_PKT_FLAG_KEY;
1467  pkt.stream_index= video_st->index;
1468  pkt.data= (uint8_t*)frame_final->data;
1469  pkt.size= sizeof(AVPicture);
1470 
1471  // Increment PTS (in frames and scaled to the codec's timebase)
1472  write_video_count += av_rescale_q(1, (AVRational){info.fps.den, info.fps.num}, video_codec->time_base);
1473  pkt.pts = write_video_count;
1474 
1475  /* write the compressed frame in the media file */
1476  int error_code = av_interleaved_write_frame(oc, &pkt);
1477  if (error_code < 0)
1478  {
1479  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_video_packet ERROR [" + (string)av_err2str(error_code) + "]", "error_code", error_code, "", -1, "", -1, "", -1, "", -1, "", -1);
1480  return false;
1481  }
1482 
1483  // Deallocate packet
1484  av_free_packet(&pkt);
1485 
1486  } else {
1487 
1488  AVPacket pkt;
1489  av_init_packet(&pkt);
1490  pkt.data = NULL;
1491  pkt.size = 0;
1492  pkt.pts = pkt.dts = AV_NOPTS_VALUE;
1493 
1494  // Pointer for video buffer (if using old FFmpeg version)
1495  uint8_t *video_outbuf = NULL;
1496 
1497  // Increment PTS (in frames and scaled to the codec's timebase)
1498  write_video_count += av_rescale_q(1, (AVRational){info.fps.den, info.fps.num}, video_codec->time_base);
1499 
1500  // Assign the initial AVFrame PTS from the frame counter
1501  frame_final->pts = write_video_count;
1502 
1503  /* encode the image */
1504  int got_packet_ptr = 0;
1505  int error_code = 0;
1506  #if LIBAVFORMAT_VERSION_MAJOR >= 54
1507  // Newer versions of FFMpeg
1508  error_code = avcodec_encode_video2(video_codec, &pkt, frame_final, &got_packet_ptr);
1509 
1510  #else
1511  // Older versions of FFmpeg (much sloppier)
1512 
1513  // Encode Picture and Write Frame
1514  int video_outbuf_size = 200000;
1515  video_outbuf = (uint8_t*) av_malloc(200000);
1516 
1517  /* encode the image */
1518  int out_size = avcodec_encode_video(video_codec, video_outbuf, video_outbuf_size, frame_final);
1519 
1520  /* if zero size, it means the image was buffered */
1521  if (out_size > 0) {
1522  if(video_codec->coded_frame->key_frame)
1523  pkt.flags |= AV_PKT_FLAG_KEY;
1524  pkt.data= video_outbuf;
1525  pkt.size= out_size;
1526 
1527  // got data back (so encode this frame)
1528  got_packet_ptr = 1;
1529  }
1530  #endif
1531 
1532  /* if zero size, it means the image was buffered */
1533  if (error_code == 0 && got_packet_ptr) {
1534 
1535  // Since the PTS can change during encoding, set the value again. This seems like a huge hack,
1536  // but it fixes lots of PTS related issues when I do this.
1537  //pkt.pts = pkt.dts = write_video_count;
1538 
1539  // set the timestamp
1540  if (pkt.pts != AV_NOPTS_VALUE)
1541  pkt.pts = av_rescale_q(pkt.pts, video_codec->time_base, video_st->time_base);
1542  if (pkt.dts != AV_NOPTS_VALUE)
1543  pkt.dts = av_rescale_q(pkt.dts, video_codec->time_base, video_st->time_base);
1544  if (pkt.duration > 0)
1545  pkt.duration = av_rescale_q(pkt.duration, video_codec->time_base, video_st->time_base);
1546  pkt.stream_index = video_st->index;
1547 
1548  /* write the compressed frame in the media file */
1549  int error_code = av_interleaved_write_frame(oc, &pkt);
1550  if (error_code < 0)
1551  {
1552  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_video_packet ERROR [" + (string)av_err2str(error_code) + "]", "error_code", error_code, "", -1, "", -1, "", -1, "", -1, "", -1);
1553  return false;
1554  }
1555  }
1556 
1557  // Deallocate memory (if needed)
1558  if (video_outbuf)
1559  delete[] video_outbuf;
1560 
1561  // Deallocate packet
1562  av_free_packet(&pkt);
1563  }
1564 
1565  // Success
1566  return true;
1567 }
1568 
1569 // Output the ffmpeg info about this format, streams, and codecs (i.e. dump format)
1571 {
1572  // output debug info
1573  av_dump_format(oc, 0, path.c_str(), 1);
1574 }
1575 
1576 // Init a collection of software rescalers (thread safe)
1577 void FFmpegWriter::InitScalers(int source_width, int source_height)
1578 {
1579  // Get the codec
1580  AVCodecContext *c;
1581  c = video_st->codec;
1582 
1583  // Init software rescalers vector (many of them, one for each thread)
1584  for (int x = 0; x < num_of_rescalers; x++)
1585  {
1586  // Init the software scaler from FFMpeg
1587  img_convert_ctx = sws_getContext(source_width, source_height, PIX_FMT_RGBA, info.width, info.height, c->pix_fmt, SWS_FAST_BILINEAR, NULL, NULL, NULL);
1588 
1589  // Add rescaler to vector
1590  image_rescalers.push_back(img_convert_ctx);
1591  }
1592 }
1593 
1594 // Set audio resample options
1595 void FFmpegWriter::ResampleAudio(int sample_rate, int channels) {
1596  original_sample_rate = sample_rate;
1597  original_channels = channels;
1598 }
1599 
1600 // Remove & deallocate all software scalers
1602 {
1603  // Close all rescalers
1604  for (int x = 0; x < num_of_rescalers; x++)
1605  sws_freeContext(image_rescalers[x]);
1606 
1607  // Clear vector
1608  image_rescalers.clear();
1609 }
#define AV_RESET_FRAME(av_frame)
int channels
The number of audio channels used in the audio stream.
Definition: WriterBase.h:72
A video stream (used to determine which type of stream)
Definition: FFmpegWriter.h:64
#define AV_FREE_FRAME(av_frame)
void SetOption(StreamType stream, string name, string value)
Set custom options (some codecs accept additional params). This must be called after the PrepareStrea...
int num
Numerator for the fraction.
Definition: Fraction.h:44
WriterInfo info
Information about the current media file.
Definition: WriterBase.h:92
void OutputStreamInfo()
Output the ffmpeg info about this format, streams, and codecs (i.e. dump format)
An audio stream (used to determine which type of stream)
Definition: FFmpegWriter.h:65
int video_bit_rate
The bit rate of the video stream (in bytes)
Definition: WriterBase.h:60
Fraction pixel_ratio
The pixel ratio of the video stream as a fraction (i.e. some pixels are not square) ...
Definition: WriterBase.h:61
Exception when an invalid # of audio channels are detected.
Definition: Exceptions.h:112
#define PIX_FMT_RGB24
string acodec
The name of the audio codec used to encode / decode the video stream.
Definition: WriterBase.h:69
string vcodec
The name of the video codec used to encode / decode the video stream.
Definition: WriterBase.h:63
void Reduce()
Reduce this fraction (i.e. 640/480 = 4/3)
Definition: Fraction.cpp:71
#define AVCODEC_MAX_AUDIO_FRAME_SIZE
This abstract class is the base class, used by all readers in libopenshot.
Definition: ReaderBase.h:95
int width
The width of the video (in pixels)
Definition: WriterBase.h:57
int audio_bit_rate
The bit rate of the audio stream (in bytes)
Definition: WriterBase.h:70
#define OPEN_MP_NUM_PROCESSORS
Exception when encoding audio packet.
Definition: Exceptions.h:101
Exception when invalid sample rate is detected during encoding.
Definition: Exceptions.h:172
Fraction video_timebase
The video timebase determines how long each frame stays on the screen.
Definition: WriterBase.h:66
void Open()
Open writer.
void SetVideoOptions(bool has_video, string codec, Fraction fps, int width, int height, Fraction pixel_ratio, bool interlaced, bool top_field_first, int bit_rate)
Set video export options.
void WriteFrame(tr1::shared_ptr< Frame > frame)
Add a frame to the stack waiting to be encoded.
Exception when no valid codec is found for a file.
Definition: Exceptions.h:122
Exception when memory could not be allocated.
Definition: Exceptions.h:224
Exception when invalid encoding options are used.
Definition: Exceptions.h:162
Exception when no streams are found in the file.
Definition: Exceptions.h:192
void RemoveScalers()
Remove & deallocate all software scalers.
#define AV_ALLOCATE_FRAME()
bool top_field_first
Which interlaced field should be displayed first.
Definition: WriterBase.h:68
Exception for files that can not be found or opened.
Definition: Exceptions.h:132
FFmpegWriter(string path)
Constructor for FFmpegWriter. Throws one of the following exceptions.
void AppendDebugMethod(string method_name, string arg1_name, float arg1_value, string arg2_name, float arg2_value, string arg3_name, float arg3_value, string arg4_name, float arg4_value, string arg5_name, float arg5_value, string arg6_name, float arg6_value)
Append debug information.
Definition: ZmqLogger.cpp:162
This class represents a fraction.
Definition: Fraction.h:42
void ResampleAudio(int sample_rate, int channels)
Set audio resample options.
#define PIX_FMT_YUV420P
Fraction display_ratio
The ratio of width to height of the video stream (i.e. 640x480 has a ratio of 4/3) ...
Definition: WriterBase.h:62
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround...
#define PixelFormat
void WriteTrailer()
Write the file trailer (after all frames are written). This is called automatically by the Close() me...
#define av_err2str(errnum)
void WriteHeader()
Write the file header (after the options are set). This method is called automatically by the Open() ...
void Close()
Close the writer.
bool interlaced_frame
Are the contents of this frame interlaced.
Definition: WriterBase.h:67
int sample_rate
The number of audio samples per second (44100 is a common sample rate)
Definition: WriterBase.h:71
bool has_video
Determines if this file has a video stream.
Definition: WriterBase.h:51
Fraction fps
Frames per second, as a fraction (i.e. 24/1 = 24 fps)
Definition: WriterBase.h:59
static ZmqLogger * Instance()
Create or get an instance of this logger singleton (invoke the class with this method) ...
Definition: ZmqLogger.cpp:38
This namespace is the default namespace for all code in the openshot library.
bool has_audio
Determines if this file has an audio stream.
Definition: WriterBase.h:52
#define PIX_FMT_RGBA
void SetAudioOptions(bool has_audio, string codec, int sample_rate, int channels, ChannelLayout channel_layout, int bit_rate)
Set audio export options.
Exception when a writer is closed, and a frame is requested.
Definition: Exceptions.h:264
ChannelLayout channel_layout
The channel layout (mono, stereo, 5 point surround, etc...)
Definition: WriterBase.h:73
void PrepareStreams()
Prepare & initialize streams and open codecs. This method is called automatically by the Open() metho...
int height
The height of the video (in pixels)
Definition: WriterBase.h:56
#define PIX_FMT_NONE
int den
Denominator for the fraction.
Definition: Fraction.h:45
Exception when no valid format is found for a file.
Definition: Exceptions.h:142
#define AUDIO_PACKET_ENCODING_SIZE
StreamType
This enumeration designates the type of stream when encoding (video or audio)
Definition: FFmpegWriter.h:62