OpenShot Library | libopenshot  0.1.3
FrameMapper.cpp
Go to the documentation of this file.
1 /**
2  * @file
3  * @brief Source file for the FrameMapper class
4  * @author Jonathan Thomas <jonathan@openshot.org>
5  *
6  * @section LICENSE
7  *
8  * Copyright (c) 2008-2014 OpenShot Studios, LLC
9  * <http://www.openshotstudios.com/>. This file is part of
10  * OpenShot Library (libopenshot), an open-source project dedicated to
11  * delivering high quality video editing and animation solutions to the
12  * world. For more information visit <http://www.openshot.org/>.
13  *
14  * OpenShot Library (libopenshot) is free software: you can redistribute it
15  * and/or modify it under the terms of the GNU Lesser General Public License
16  * as published by the Free Software Foundation, either version 3 of the
17  * License, or (at your option) any later version.
18  *
19  * OpenShot Library (libopenshot) is distributed in the hope that it will be
20  * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
21  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
22  * GNU Lesser General Public License for more details.
23  *
24  * You should have received a copy of the GNU Lesser General Public License
25  * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
26  */
27 
28 #include "../include/FrameMapper.h"
29 
30 using namespace std;
31 using namespace openshot;
32 
33 FrameMapper::FrameMapper(ReaderBase *reader, Fraction target, PulldownType target_pulldown, int target_sample_rate, int target_channels, ChannelLayout target_channel_layout) :
34  reader(reader), target(target), pulldown(target_pulldown), is_dirty(true), avr(NULL)
35 {
36  // Set the original frame rate from the reader
37  original = Fraction(reader->info.fps.num, reader->info.fps.den);
38 
39  // Set all info struct members equal to the internal reader
40  info = reader->info;
41  info.fps.num = target.num;
42  info.fps.den = target.den;
43  info.video_timebase.num = target.den;
44  info.video_timebase.den = target.num;
46  info.sample_rate = target_sample_rate;
47  info.channels = target_channels;
48  info.channel_layout = target_channel_layout;
49  info.width = reader->info.width;
50  info.height = reader->info.height;
51 
52  // Used to toggle odd / even fields
53  field_toggle = true;
54 
55  // Adjust cache size based on size of frame and audio
57 
58  // init mapping between original and target frames
59  Init();
60 }
61 
62 // Destructor
64  if (is_open)
65  // Auto Close if not already
66  Close();
67 }
68 
69 void FrameMapper::AddField(long int frame)
70 {
71  // Add a field, and toggle the odd / even field
72  AddField(Field(frame, field_toggle));
73 }
74 
75 void FrameMapper::AddField(Field field)
76 {
77  // Add a field to the end of the field list
78  fields.push_back(field);
79 
80  // toggle the odd / even flag
81  field_toggle = (field_toggle ? false : true);
82 }
83 
84 // Use the original and target frame rates and a pull-down technique to create
85 // a mapping between the original fields and frames or a video to a new frame rate.
86 // This might repeat or skip fields and frames of the original video, depending on
87 // whether the frame rate is increasing or decreasing.
88 void FrameMapper::Init()
89 {
90  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::Init (Calculate frame mappings)", "", -1, "", -1, "", -1, "", -1, "", -1, "", -1);
91 
92  // Do not initialize anything if just a picture with no audio
94  // Skip initialization
95  return;
96 
97  // Clear the fields & frames lists
98  fields.clear();
99  frames.clear();
100 
101  // Mark as not dirty
102  is_dirty = false;
103 
104  // Clear cache
105  final_cache.Clear();
106 
107  // Some framerates are handled special, and some use a generic Keyframe curve to
108  // map the framerates. These are the special framerates:
109  if ((fabs(original.ToFloat() - 24.0) < 1e-7 || fabs(original.ToFloat() - 25.0) < 1e-7 || fabs(original.ToFloat() - 30.0) < 1e-7) &&
110  (fabs(target.ToFloat() - 24.0) < 1e-7 || fabs(target.ToFloat() - 25.0) < 1e-7 || fabs(target.ToFloat() - 30.0) < 1e-7)) {
111 
112  // Get the difference (in frames) between the original and target frame rates
113  float difference = target.ToInt() - original.ToInt();
114 
115  // Find the number (i.e. interval) of fields that need to be skipped or repeated
116  int field_interval = 0;
117  int frame_interval = 0;
118 
119  if (difference != 0)
120  {
121  field_interval = round(fabs(original.ToInt() / difference));
122 
123  // Get frame interval (2 fields per frame)
124  frame_interval = field_interval * 2.0f;
125  }
126 
127 
128  // Calculate # of fields to map
129  long int frame = 1;
130  long int number_of_fields = reader->info.video_length * 2;
131 
132  // Loop through all fields in the original video file
133  for (long int field = 1; field <= number_of_fields; field++)
134  {
135 
136  if (difference == 0) // Same frame rate, NO pull-down or special techniques required
137  {
138  // Add fields
139  AddField(frame);
140  }
141  else if (difference > 0) // Need to ADD fake fields & frames, because original video has too few frames
142  {
143  // Add current field
144  AddField(frame);
145 
146  if (pulldown == PULLDOWN_CLASSIC && field % field_interval == 0)
147  {
148  // Add extra field for each 'field interval
149  AddField(frame);
150  }
151  else if (pulldown == PULLDOWN_ADVANCED && field % field_interval == 0 && field % frame_interval != 0)
152  {
153  // Add both extra fields in the middle 'together' (i.e. 2:3:3:2 technique)
154  AddField(frame); // add field for current frame
155 
156  if (frame + 1 <= info.video_length)
157  // add field for next frame (if the next frame exists)
158  AddField(Field(frame + 1, field_toggle));
159  }
160  else if (pulldown == PULLDOWN_NONE && field % frame_interval == 0)
161  {
162  // No pull-down technique needed, just repeat this frame
163  AddField(frame);
164  AddField(frame);
165  }
166  }
167  else if (difference < 0) // Need to SKIP fake fields & frames, because we want to return to the original film frame rate
168  {
169 
170  if (pulldown == PULLDOWN_CLASSIC && field % field_interval == 0)
171  {
172  // skip current field and toggle the odd/even flag
173  field_toggle = (field_toggle ? false : true);
174  }
175  else if (pulldown == PULLDOWN_ADVANCED && field % field_interval == 0 && field % frame_interval != 0)
176  {
177  // skip this field, plus the next field
178  field++;
179  }
180  else if (pulldown == PULLDOWN_NONE && frame % field_interval == 0)
181  {
182  // skip this field, plus the next one
183  field++;
184  }
185  else
186  {
187  // No skipping needed, so add the field
188  AddField(frame);
189  }
190  }
191 
192  // increment frame number (if field is divisible by 2)
193  if (field % 2 == 0 && field > 0)
194  frame++;
195  }
196 
197  } else {
198  // Map the remaining framerates using a simple Keyframe curve
199  // Calculate the difference (to be used as a multiplier)
200  float rate_diff = target.ToFloat() / original.ToFloat();
201  long int new_length = reader->info.video_length * rate_diff;
202 
203  // Build curve for framerate mapping
204  Keyframe rate_curve;
205  rate_curve.AddPoint(1, 1, LINEAR);
206  rate_curve.AddPoint(new_length, reader->info.video_length, LINEAR);
207 
208  // Loop through curve, and build list of frames
209  for (long int frame_num = 1; frame_num <= new_length; frame_num++)
210  {
211  // Add 2 fields per frame
212  AddField(rate_curve.GetInt(frame_num));
213  AddField(rate_curve.GetInt(frame_num));
214  }
215  }
216 
217  // Loop through the target frames again (combining fields into frames)
218  Field Odd(0, true); // temp field used to track the ODD field
219  Field Even(0, true); // temp field used to track the EVEN field
220 
221  // Variables used to remap audio samples
222  int start_samples_frame = 1;
223  int start_samples_position = 0;
224 
225  for (long int field = 1; field <= fields.size(); field++)
226  {
227  // Get the current field
228  Field f = fields[field - 1];
229 
230  // Is field divisible by 2?
231  if (field % 2 == 0 && field > 0)
232  {
233  // New frame number
234  long int frame_number = field / 2;
235 
236  // Set the bottom frame
237  if (f.isOdd)
238  Odd = f;
239  else
240  Even = f;
241 
242  // Determine the range of samples (from the original rate). Resampling happens in real-time when
243  // calling the GetFrame() method. So this method only needs to redistribute the original samples with
244  // the original sample rate.
245  int end_samples_frame = start_samples_frame;
246  int end_samples_position = start_samples_position;
247  int remaining_samples = Frame::GetSamplesPerFrame(frame_number, target, reader->info.sample_rate, reader->info.channels);
248 
249  while (remaining_samples > 0)
250  {
251  // get original samples
252  int original_samples = Frame::GetSamplesPerFrame(end_samples_frame, original, reader->info.sample_rate, reader->info.channels) - end_samples_position;
253 
254  // Enough samples
255  if (original_samples >= remaining_samples)
256  {
257  // Take all that we need, and break loop
258  end_samples_position += remaining_samples;
259  remaining_samples = 0;
260  } else
261  {
262  // Not enough samples (take them all, and keep looping)
263  end_samples_frame += 1; // next frame
264  end_samples_position = 0; // next frame, starting on 1st sample
265  remaining_samples -= original_samples; // reduce the remaining amount
266  }
267  }
268 
269 
270 
271  // Create the sample mapping struct
272  SampleRange Samples = {start_samples_frame, start_samples_position, end_samples_frame, end_samples_position, Frame::GetSamplesPerFrame(frame_number, target, reader->info.sample_rate, reader->info.channels)};
273 
274  // Reset the audio variables
275  start_samples_frame = end_samples_frame;
276  start_samples_position = end_samples_position + 1;
277  if (start_samples_position >= Frame::GetSamplesPerFrame(start_samples_frame, original, reader->info.sample_rate, reader->info.channels))
278  {
279  start_samples_frame += 1; // increment the frame (since we need to wrap onto the next one)
280  start_samples_position = 0; // reset to 0, since we wrapped
281  }
282 
283  // Create a frame and ADD it to the frames collection
284  MappedFrame frame = {Odd, Even, Samples};
285  frames.push_back(frame);
286  }
287  else
288  {
289  // Set the top field
290  if (f.isOdd)
291  Odd = f;
292  else
293  Even = f;
294  }
295  }
296 
297  // Clear the internal fields list (no longer needed)
298  fields.clear();
299 }
300 
301 MappedFrame FrameMapper::GetMappedFrame(long int TargetFrameNumber) throw(OutOfBoundsFrame)
302 {
303  // Ignore mapping on single image readers
305  // Return the same number
306  MappedFrame frame;
307  frame.Even.Frame = TargetFrameNumber;
308  frame.Odd.Frame = TargetFrameNumber;
309  frame.Samples.frame_start = 0;
310  frame.Samples.frame_end = 0;
311  frame.Samples.sample_start = 0;
312  frame.Samples.sample_end = 0;
313  frame.Samples.total = 0;
314  return frame;
315  }
316 
317  // Check if frame number is valid
318  if(TargetFrameNumber < 1 || frames.size() == 0)
319  // frame too small, return error
320  throw OutOfBoundsFrame("An invalid frame was requested.", TargetFrameNumber, frames.size());
321 
322  else if (TargetFrameNumber > frames.size())
323  // frame too large, set to end frame
324  TargetFrameNumber = frames.size();
325 
326  // Debug output
327  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::GetMappedFrame", "TargetFrameNumber", TargetFrameNumber, "frames.size()", frames.size(), "frames[...].Odd", frames[TargetFrameNumber - 1].Odd.Frame, "frames[...].Even", frames[TargetFrameNumber - 1].Even.Frame, "", -1, "", -1);
328 
329  // Return frame
330  return frames[TargetFrameNumber - 1];
331 }
332 
333 // Get or generate a blank frame
334 tr1::shared_ptr<Frame> FrameMapper::GetOrCreateFrame(long int number)
335 {
336  tr1::shared_ptr<Frame> new_frame;
337 
338  // Init some basic properties about this frame (keep sample rate and # channels the same as the original reader for now)
339  int samples_in_frame = Frame::GetSamplesPerFrame(number, target, reader->info.sample_rate, reader->info.channels);
340 
341  try {
342  // Debug output
343  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::GetOrCreateFrame (from reader)", "number", number, "samples_in_frame", samples_in_frame, "", -1, "", -1, "", -1, "", -1);
344 
345  // Set max image size (used for performance optimization)
346  reader->SetMaxSize(max_width, max_height);
347 
348  // Attempt to get a frame (but this could fail if a reader has just been closed)
349  new_frame = reader->GetFrame(number);
350 
351  // Return real frame
352  return new_frame;
353 
354  } catch (const ReaderClosed & e) {
355  // ...
356  } catch (const TooManySeeks & e) {
357  // ...
358  } catch (const OutOfBoundsFrame & e) {
359  // ...
360  }
361 
362  // Debug output
363  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::GetOrCreateFrame (create blank)", "number", number, "samples_in_frame", samples_in_frame, "", -1, "", -1, "", -1, "", -1);
364 
365  // Create blank frame
366  new_frame = tr1::shared_ptr<Frame>(new Frame(number, info.width, info.height, "#000000", samples_in_frame, reader->info.channels));
367  new_frame->SampleRate(reader->info.sample_rate);
368  new_frame->ChannelsLayout(info.channel_layout);
369  return new_frame;
370 }
371 
372 // Get an openshot::Frame object for a specific frame number of this reader.
373 tr1::shared_ptr<Frame> FrameMapper::GetFrame(long int requested_frame) throw(ReaderClosed)
374 {
375  // Check final cache, and just return the frame (if it's available)
376  tr1::shared_ptr<Frame> final_frame = final_cache.GetFrame(requested_frame);
377  if (final_frame) return final_frame;
378 
379  // Create a scoped lock, allowing only a single thread to run the following code at one time
380  const GenericScopedLock<CriticalSection> lock(getFrameCriticalSection);
381 
382  // Check if mappings are dirty (and need to be recalculated)
383  if (is_dirty)
384  // Recalculate mappings
385  Init();
386 
387  // Check final cache a 2nd time (due to potential lock already generating this frame)
388  final_frame = final_cache.GetFrame(requested_frame);
389  if (final_frame) return final_frame;
390 
391  // Minimum number of frames to process (for performance reasons)
392  int minimum_frames = OPEN_MP_NUM_PROCESSORS;
393 
394  // Set the number of threads in OpenMP
395  omp_set_num_threads(OPEN_MP_NUM_PROCESSORS);
396  // Allow nested OpenMP sections
397  omp_set_nested(true);
398 
399  // Debug output
400  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::GetFrame (Loop through frames)", "requested_frame", requested_frame, "minimum_frames", minimum_frames, "", -1, "", -1, "", -1, "", -1);
401 
402  #pragma omp parallel
403  {
404  // Loop through all requested frames, each frame gets it's own thread
405  #pragma omp for ordered firstprivate(requested_frame, minimum_frames)
406  for (long int frame_number = requested_frame; frame_number < requested_frame + minimum_frames; frame_number++)
407  {
408 
409  // Debug output
410  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::GetFrame (inside omp for loop)", "frame_number", frame_number, "minimum_frames", minimum_frames, "requested_frame", requested_frame, "", -1, "", -1, "", -1);
411 
412  // Get the mapped frame
413  MappedFrame mapped = GetMappedFrame(frame_number);
414  tr1::shared_ptr<Frame> mapped_frame;
415 
416  // Get the mapped frame (keeping the sample rate and channels the same as the original... for the moment)
417  mapped_frame = GetOrCreateFrame(mapped.Odd.Frame);
418 
419  // Get # of channels in the actual frame
420  int channels_in_frame = mapped_frame->GetAudioChannelsCount();
421  int samples_in_frame = Frame::GetSamplesPerFrame(frame_number, target, mapped_frame->SampleRate(), channels_in_frame);
422 
423  // Determine if mapped frame is identical to source frame
424  if (info.sample_rate == mapped_frame->SampleRate() &&
425  info.channels == mapped_frame->GetAudioChannelsCount() &&
426  info.channel_layout == mapped_frame->ChannelsLayout() &&
427  info.fps.num == reader->info.fps.num &&
428  info.fps.den == reader->info.fps.den) {
429  // Set frame # on mapped frame
430  mapped_frame->SetFrameNumber(frame_number);
431 
432  // Add original frame to cache, and skip the rest (for performance reasons)
433  final_cache.Add(mapped_frame);
434  continue;
435  }
436 
437  // Create a new frame
438  tr1::shared_ptr<Frame> frame = tr1::shared_ptr<Frame>(new Frame(frame_number, 1, 1, "#000000", samples_in_frame, channels_in_frame));
439  frame->SampleRate(mapped_frame->SampleRate());
440  frame->ChannelsLayout(mapped_frame->ChannelsLayout());
441 
442 
443  // Copy the image from the odd field
444  tr1::shared_ptr<Frame> odd_frame;
445  #pragma omp ordered
446  odd_frame = GetOrCreateFrame(mapped.Odd.Frame);
447 
448  if (odd_frame)
449  frame->AddImage(tr1::shared_ptr<QImage>(new QImage(*odd_frame->GetImage())), true);
450  if (mapped.Odd.Frame != mapped.Even.Frame) {
451  // Add even lines (if different than the previous image)
452  tr1::shared_ptr<Frame> even_frame;
453  #pragma omp ordered
454  even_frame = GetOrCreateFrame(mapped.Even.Frame);
455  if (even_frame)
456  frame->AddImage(tr1::shared_ptr<QImage>(new QImage(*even_frame->GetImage())), false);
457  }
458 
459  // Copy the samples
460  int samples_copied = 0;
461  int starting_frame = mapped.Samples.frame_start;
462  while (info.has_audio && samples_copied < mapped.Samples.total)
463  {
464  // Init number of samples to copy this iteration
465  int remaining_samples = mapped.Samples.total - samples_copied;
466  int number_to_copy = 0;
467 
468  // Loop through each channel
469  for (int channel = 0; channel < channels_in_frame; channel++)
470  {
471  // number of original samples on this frame
472  tr1::shared_ptr<Frame> original_frame = GetOrCreateFrame(starting_frame);
473  int original_samples = original_frame->GetAudioSamplesCount();
474 
475  if (starting_frame == mapped.Samples.frame_start)
476  {
477  // Starting frame (take the ending samples)
478  number_to_copy = original_samples - mapped.Samples.sample_start;
479  if (number_to_copy > remaining_samples)
480  number_to_copy = remaining_samples;
481 
482  // Add samples to new frame
483  frame->AddAudio(true, channel, samples_copied, original_frame->GetAudioSamples(channel) + mapped.Samples.sample_start, number_to_copy, 1.0);
484  }
485  else if (starting_frame > mapped.Samples.frame_start && starting_frame < mapped.Samples.frame_end)
486  {
487  // Middle frame (take all samples)
488  number_to_copy = original_samples;
489  if (number_to_copy > remaining_samples)
490  number_to_copy = remaining_samples;
491 
492  // Add samples to new frame
493  frame->AddAudio(true, channel, samples_copied, original_frame->GetAudioSamples(channel), number_to_copy, 1.0);
494  }
495  else
496  {
497  // Ending frame (take the beginning samples)
498  number_to_copy = mapped.Samples.sample_end;
499  if (number_to_copy > remaining_samples)
500  number_to_copy = remaining_samples;
501 
502  // Add samples to new frame
503  frame->AddAudio(false, channel, samples_copied, original_frame->GetAudioSamples(channel), number_to_copy, 1.0);
504  }
505  }
506 
507  // increment frame
508  samples_copied += number_to_copy;
509  starting_frame++;
510  }
511 
512  // Resample audio on frame (if needed)
513  if (info.has_audio &&
514  (info.sample_rate != frame->SampleRate() ||
515  info.channels != frame->GetAudioChannelsCount() ||
516  info.channel_layout != frame->ChannelsLayout()))
517  // Resample audio and correct # of channels if needed
518  ResampleMappedAudio(frame, mapped.Odd.Frame);
519 
520  // Add frame to final cache
521  final_cache.Add(frame);
522 
523  } // for loop
524  } // omp parallel
525 
526  // Return processed openshot::Frame
527  return final_cache.GetFrame(requested_frame);
528 }
529 
531 {
532  // Check if mappings are dirty (and need to be recalculated)
533  if (is_dirty)
534  // Recalculate mappings
535  Init();
536 
537  // Get the difference (in frames) between the original and target frame rates
538  float difference = target.ToInt() - original.ToInt();
539 
540  int field_interval = 0;
541  int frame_interval = 0;
542 
543  if (difference != 0)
544  {
545  // Find the number (i.e. interval) of fields that need to be skipped or repeated
546  field_interval = round(fabs(original.ToInt() / difference));
547 
548  // Get frame interval (2 fields per frame)
549  frame_interval = field_interval * 2.0f;
550  }
551 
552  // Loop through frame mappings
553  for (float map = 1; map <= frames.size(); map++)
554  {
555  MappedFrame frame = frames[map - 1];
556  cout << "Target frame #: " << map << " mapped to original frame #:\t(" << frame.Odd.Frame << " odd, " << frame.Even.Frame << " even)" << endl;
557  cout << " - Audio samples mapped to frame " << frame.Samples.frame_start << ":" << frame.Samples.sample_start << " to frame " << frame.Samples.frame_end << ":" << frame.Samples.sample_end << endl;
558  }
559 
560 }
561 
562 
563 // Determine if reader is open or closed
565  if (reader)
566  return reader->IsOpen();
567  else
568  return false;
569 }
570 
571 
572 // Open the internal reader
574 {
575  if (reader)
576  {
577  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::Open", "", -1, "", -1, "", -1, "", -1, "", -1, "", -1);
578 
579  // Open the reader
580  reader->Open();
581  }
582 }
583 
584 // Close the internal reader
586 {
587  if (reader)
588  {
589  // Create a scoped lock, allowing only a single thread to run the following code at one time
590  const GenericScopedLock<CriticalSection> lock(getFrameCriticalSection);
591 
592  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::Close", "", -1, "", -1, "", -1, "", -1, "", -1, "", -1);
593 
594  // Close internal reader
595  reader->Close();
596 
597  // Deallocate resample buffer
598  if (avr) {
599  avresample_close(avr);
600  avresample_free(&avr);
601  avr = NULL;
602  }
603  }
604 }
605 
606 
607 // Generate JSON string of this object
609 
610  // Return formatted string
611  return JsonValue().toStyledString();
612 }
613 
614 // Generate Json::JsonValue for this object
615 Json::Value FrameMapper::JsonValue() {
616 
617  // Create root json object
618  Json::Value root = ReaderBase::JsonValue(); // get parent properties
619  root["type"] = "FrameMapper";
620 
621  // return JsonValue
622  return root;
623 }
624 
625 // Load JSON string into this object
626 void FrameMapper::SetJson(string value) throw(InvalidJSON) {
627 
628  // Parse JSON string into JSON objects
629  Json::Value root;
630  Json::Reader reader;
631  bool success = reader.parse( value, root );
632  if (!success)
633  // Raise exception
634  throw InvalidJSON("JSON could not be parsed (or is invalid)", "");
635 
636  try
637  {
638  // Set all values that match
639  SetJsonValue(root);
640  }
641  catch (exception e)
642  {
643  // Error parsing JSON (or missing keys)
644  throw InvalidJSON("JSON is invalid (missing keys or invalid data types)", "");
645  }
646 }
647 
648 // Load Json::JsonValue into this object
649 void FrameMapper::SetJsonValue(Json::Value root) throw(InvalidFile) {
650 
651  // Set parent data
653 
654  // Re-Open path, and re-init everything (if needed)
655  if (reader) {
656 
657  Close();
658  Open();
659  }
660 }
661 
662 // Change frame rate or audio mapping details
663 void FrameMapper::ChangeMapping(Fraction target_fps, PulldownType target_pulldown, int target_sample_rate, int target_channels, ChannelLayout target_channel_layout)
664 {
665  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::ChangeMapping", "target_fps.num", target_fps.num, "target_fps.den", target_fps.num, "target_pulldown", target_pulldown, "target_sample_rate", target_sample_rate, "target_channels", target_channels, "target_channel_layout", target_channel_layout);
666 
667  // Mark as dirty
668  is_dirty = true;
669 
670  // Update mapping details
671  target = target_fps;
672  pulldown = target_pulldown;
673  info.sample_rate = target_sample_rate;
674  info.channels = target_channels;
675  info.channel_layout = target_channel_layout;
676 
677  // Clear cache
678  final_cache.Clear();
679 
680  // Adjust cache size based on size of frame and audio
682 
683  // Deallocate resample buffer
684  if (avr) {
685  avresample_close(avr);
686  avresample_free(&avr);
687  avr = NULL;
688  }
689 }
690 
691 // Resample audio and map channels (if needed)
692 void FrameMapper::ResampleMappedAudio(tr1::shared_ptr<Frame> frame, long int original_frame_number)
693 {
694  // Init audio buffers / variables
695  int total_frame_samples = 0;
696  int channels_in_frame = frame->GetAudioChannelsCount();
697  int sample_rate_in_frame = frame->SampleRate();
698  int samples_in_frame = frame->GetAudioSamplesCount();
699  ChannelLayout channel_layout_in_frame = frame->ChannelsLayout();
700 
701  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::ResampleMappedAudio", "frame->number", frame->number, "original_frame_number", original_frame_number, "channels_in_frame", channels_in_frame, "samples_in_frame", samples_in_frame, "sample_rate_in_frame", sample_rate_in_frame, "", -1);
702 
703  // Get audio sample array
704  float* frame_samples_float = NULL;
705  // Get samples interleaved together (c1 c2 c1 c2 c1 c2)
706  frame_samples_float = frame->GetInterleavedAudioSamples(sample_rate_in_frame, NULL, &samples_in_frame);
707 
708  // Calculate total samples
709  total_frame_samples = samples_in_frame * channels_in_frame;
710 
711  // Create a new array (to hold all S16 audio samples for the current queued frames)
712  int16_t* frame_samples = (int16_t*) av_malloc(sizeof(int16_t)*total_frame_samples);
713 
714  // Translate audio sample values back to 16 bit integers
715  for (int s = 0; s < total_frame_samples; s++)
716  // Translate sample value and copy into buffer
717  frame_samples[s] = int(frame_samples_float[s] * (1 << 15));
718 
719 
720  // Deallocate float array
721  delete[] frame_samples_float;
722  frame_samples_float = NULL;
723 
724  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::ResampleMappedAudio (got sample data from frame)", "frame->number", frame->number, "total_frame_samples", total_frame_samples, "target channels", info.channels, "channels_in_frame", channels_in_frame, "target sample_rate", info.sample_rate, "samples_in_frame", samples_in_frame);
725 
726 
727  // Create input frame (and allocate arrays)
728  AVFrame *audio_frame = AV_ALLOCATE_FRAME();
729  AV_RESET_FRAME(audio_frame);
730  audio_frame->nb_samples = total_frame_samples / channels_in_frame;
731 
732  int error_code = avcodec_fill_audio_frame(audio_frame, channels_in_frame, AV_SAMPLE_FMT_S16, (uint8_t *) frame_samples,
733  audio_frame->nb_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) * channels_in_frame, 1);
734 
735  if (error_code < 0)
736  {
737  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::ResampleMappedAudio ERROR [" + (string)av_err2str(error_code) + "]", "error_code", error_code, "", -1, "", -1, "", -1, "", -1, "", -1);
738  throw ErrorEncodingVideo("Error while resampling audio in frame mapper", frame->number);
739  }
740 
741  // Update total samples & input frame size (due to bigger or smaller data types)
742  total_frame_samples = Frame::GetSamplesPerFrame(frame->number, target, info.sample_rate, info.channels);
743 
744  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::ResampleMappedAudio (adjust # of samples)", "total_frame_samples", total_frame_samples, "info.sample_rate", info.sample_rate, "sample_rate_in_frame", sample_rate_in_frame, "info.channels", info.channels, "channels_in_frame", channels_in_frame, "original_frame_number", original_frame_number);
745 
746  // Create output frame (and allocate arrays)
747  AVFrame *audio_converted = AV_ALLOCATE_FRAME();
748  AV_RESET_FRAME(audio_converted);
749  audio_converted->nb_samples = total_frame_samples;
750  av_samples_alloc(audio_converted->data, audio_converted->linesize, info.channels, total_frame_samples, AV_SAMPLE_FMT_S16, 0);
751 
752  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::ResampleMappedAudio (preparing for resample)", "in_sample_fmt", AV_SAMPLE_FMT_S16, "out_sample_fmt", AV_SAMPLE_FMT_S16, "in_sample_rate", sample_rate_in_frame, "out_sample_rate", info.sample_rate, "in_channels", channels_in_frame, "out_channels", info.channels);
753 
754  int nb_samples = 0;
755  // Force the audio resampling to happen in order (1st thread to last thread), so the waveform
756  // is smooth and continuous.
757  #pragma omp ordered
758  {
759  // setup resample context
760  if (!avr) {
761  avr = avresample_alloc_context();
762  av_opt_set_int(avr, "in_channel_layout", channel_layout_in_frame, 0);
763  av_opt_set_int(avr, "out_channel_layout", info.channel_layout, 0);
764  av_opt_set_int(avr, "in_sample_fmt", AV_SAMPLE_FMT_S16, 0);
765  av_opt_set_int(avr, "out_sample_fmt", AV_SAMPLE_FMT_S16, 0);
766  av_opt_set_int(avr, "in_sample_rate", sample_rate_in_frame, 0);
767  av_opt_set_int(avr, "out_sample_rate", info.sample_rate, 0);
768  av_opt_set_int(avr, "in_channels", channels_in_frame, 0);
769  av_opt_set_int(avr, "out_channels", info.channels, 0);
770  avresample_open(avr);
771  }
772 
773  // Convert audio samples
774  nb_samples = avresample_convert(avr, // audio resample context
775  audio_converted->data, // output data pointers
776  audio_converted->linesize[0], // output plane size, in bytes. (0 if unknown)
777  audio_converted->nb_samples, // maximum number of samples that the output buffer can hold
778  audio_frame->data, // input data pointers
779  audio_frame->linesize[0], // input plane size, in bytes (0 if unknown)
780  audio_frame->nb_samples); // number of input samples to convert
781  }
782 
783  // Create a new array (to hold all resampled S16 audio samples)
784  int16_t* resampled_samples = new int16_t[(nb_samples * info.channels)];
785 
786  // Copy audio samples over original samples
787  memcpy(resampled_samples, audio_converted->data[0], (nb_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) * info.channels));
788 
789  // Free frames
790  av_freep(&audio_frame->data[0]);
791  AV_FREE_FRAME(&audio_frame);
792  av_freep(&audio_converted->data[0]);
793  AV_FREE_FRAME(&audio_converted);
794  frame_samples = NULL;
795 
796  // Resize the frame to hold the right # of channels and samples
797  int channel_buffer_size = nb_samples;
798  frame->ResizeAudio(info.channels, channel_buffer_size, info.sample_rate, info.channel_layout);
799 
800  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::ResampleMappedAudio (Audio successfully resampled)", "nb_samples", nb_samples, "total_frame_samples", total_frame_samples, "info.sample_rate", info.sample_rate, "channels_in_frame", channels_in_frame, "info.channels", info.channels, "info.channel_layout", info.channel_layout);
801 
802  // Array of floats (to hold samples for each channel)
803  float *channel_buffer = new float[channel_buffer_size];
804 
805  // Divide audio into channels. Loop through each channel
806  for (int channel_filter = 0; channel_filter < info.channels; channel_filter++)
807  {
808  // Init array
809  for (int z = 0; z < channel_buffer_size; z++)
810  channel_buffer[z] = 0.0f;
811 
812  // Loop through all samples and add them to our Frame based on channel.
813  // Toggle through each channel number, since channel data is stored like (left right left right)
814  int channel = 0;
815  int position = 0;
816  for (int sample = 0; sample < (nb_samples * info.channels); sample++)
817  {
818  // Only add samples for current channel
819  if (channel_filter == channel)
820  {
821  // Add sample (convert from (-32768 to 32768) to (-1.0 to 1.0))
822  channel_buffer[position] = resampled_samples[sample] * (1.0f / (1 << 15));
823 
824  // Increment audio position
825  position++;
826  }
827 
828  // increment channel (if needed)
829  if ((channel + 1) < info.channels)
830  // move to next channel
831  channel ++;
832  else
833  // reset channel
834  channel = 0;
835  }
836 
837  // Add samples to frame for this channel
838  frame->AddAudio(true, channel_filter, 0, channel_buffer, position, 1.0f);
839 
840  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::ResampleMappedAudio (Add audio to channel)", "number of samples", position, "channel_filter", channel_filter, "", -1, "", -1, "", -1, "", -1);
841  }
842 
843  // Update frame's audio meta data
844  frame->SampleRate(info.sample_rate);
845  frame->ChannelsLayout(info.channel_layout);
846 
847  // clear channel buffer
848  delete[] channel_buffer;
849  channel_buffer = NULL;
850 
851  // Delete arrays
852  delete[] resampled_samples;
853  resampled_samples = NULL;
854 }
#define AV_RESET_FRAME(av_frame)
Classic 2:3:2:3 pull-down.
Definition: FrameMapper.h:62
int max_height
The maximium image height needed by this clip (used for optimizations)
Definition: ReaderBase.h:103
#define AV_FREE_FRAME(av_frame)
int num
Numerator for the fraction.
Definition: Fraction.h:44
CriticalSection getFrameCriticalSection
Section lock for multiple threads.
Definition: ReaderBase.h:99
vector< MappedFrame > frames
Definition: FrameMapper.h:167
ChannelLayout channel_layout
The channel layout (mono, stereo, 5 point surround, etc...)
Definition: ReaderBase.h:83
int width
The width of the video (in pixesl)
Definition: ReaderBase.h:67
tr1::shared_ptr< Frame > GetFrame(long int requested_frame)
This method is required for all derived classes of ReaderBase, and return the openshot::Frame object...
This class represents a single frame of video (i.e. image & audio data)
Definition: Frame.h:115
void ResampleMappedAudio(tr1::shared_ptr< Frame > frame, long int original_frame_number)
Resample audio and map channels (if needed)
float ToFloat()
Return this fraction as a float (i.e. 1/2 = 0.5)
Definition: Fraction.cpp:41
~FrameMapper()
Destructor.
Definition: FrameMapper.cpp:63
float duration
Length of time (in seconds)
Definition: ReaderBase.h:64
void Close()
Close the openshot::FrameMapper and internal reader.
virtual void Close()=0
Close the reader (and any resources it was consuming)
This abstract class is the base class, used by all readers in libopenshot.
Definition: ReaderBase.h:95
#define OPEN_MP_NUM_PROCESSORS
Exception when a reader is closed, and a frame is requested.
Definition: Exceptions.h:234
bool has_video
Determines if this file has a video stream.
Definition: ReaderBase.h:61
void SetMaxBytesFromInfo(long int number_of_frames, int width, int height, int sample_rate, int channels)
Set maximum bytes to a different amount based on a ReaderInfo struct.
Definition: CacheBase.cpp:46
This struct holds a single field (half a frame).
Definition: FrameMapper.h:73
Exception when encoding audio packet.
Definition: Exceptions.h:101
void AddPoint(Point p)
Add a new point on the key-frame. Each point has a primary coordinate, a left handle, and a right handle.
Definition: KeyFrame.cpp:72
This struct holds a the range of samples needed by this frame.
Definition: FrameMapper.h:93
void SetMaxSize(int width, int height)
Set Max Image Size (used for performance optimization)
Definition: ReaderBase.h:143
MappedFrame GetMappedFrame(long int TargetFrameNumber)
Get a frame based on the target frame rate and the new frame number of a frame.
bool has_audio
Determines if this file has an audio stream.
Definition: ReaderBase.h:62
void ChangeMapping(Fraction target_fps, PulldownType pulldown, int target_sample_rate, int target_channels, ChannelLayout target_channel_layout)
Change frame rate or audio mapping details.
int height
The height of the video (in pixels)
Definition: ReaderBase.h:66
#define AV_ALLOCATE_FRAME()
Exception for files that can not be found or opened.
Definition: Exceptions.h:132
bool IsOpen()
Determine if reader is open or closed.
long int Frame
Definition: FrameMapper.h:75
void AppendDebugMethod(string method_name, string arg1_name, float arg1_value, string arg2_name, float arg2_value, string arg3_name, float arg3_value, string arg4_name, float arg4_value, string arg5_name, float arg5_value, string arg6_name, float arg6_value)
Append debug information.
Definition: ZmqLogger.cpp:162
This class represents a fraction.
Definition: Fraction.h:42
bool has_single_image
Determines if this file only contains a single image.
Definition: ReaderBase.h:63
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround...
virtual Json::Value JsonValue()=0
Generate Json::JsonValue for this object.
Definition: ReaderBase.cpp:106
virtual void SetJsonValue(Json::Value root)=0
Load Json::JsonValue into this object.
Definition: ReaderBase.cpp:155
#define av_err2str(errnum)
ReaderInfo info
Information about the current media file.
Definition: ReaderBase.h:111
void Clear()
Clear the cache of all frames.
int ToInt()
Return a rounded integer of the fraction (for example 30000/1001 returns 30)
Definition: Fraction.cpp:51
This struct holds two fields which together make up a complete video frame.
Definition: FrameMapper.h:110
Fraction fps
Frames per second, as a fraction (i.e. 24/1 = 24 fps)
Definition: ReaderBase.h:69
vector< Field > fields
Definition: FrameMapper.h:166
Fraction video_timebase
The video timebase determines how long each frame stays on the screen.
Definition: ReaderBase.h:76
string Json()
Get and Set JSON methods.
Exception for frames that are out of bounds.
Definition: Exceptions.h:202
static ZmqLogger * Instance()
Create or get an instance of this logger singleton (invoke the class with this method) ...
Definition: ZmqLogger.cpp:38
int GetInt(long int index)
Get the rounded INT value at a specific index.
Definition: KeyFrame.cpp:248
void Add(tr1::shared_ptr< Frame > frame)
Add a Frame to the cache.
This namespace is the default namespace for all code in the openshot library.
virtual tr1::shared_ptr< Frame > GetFrame(long int number)=0
Do not apply pull-down techniques, just repeat or skip entire frames.
Definition: FrameMapper.h:64
Linear curves are angular, straight lines between two points.
Definition: Point.h:47
void PrintMapping()
Print all of the original frames and which new frames they map to.
Exception for invalid JSON.
Definition: Exceptions.h:152
tr1::shared_ptr< Frame > GetFrame(long int frame_number)
Get a frame from the cache.
PulldownType
This enumeration determines how frame rates are increased or decreased.
Definition: FrameMapper.h:60
int den
Denominator for the fraction.
Definition: Fraction.h:45
int channels
The number of audio channels used in the audio stream.
Definition: ReaderBase.h:82
A Keyframe is a collection of Point instances, which is used to vary a number or property over time...
Definition: KeyFrame.h:64
long int video_length
The number of frames in the video stream.
Definition: ReaderBase.h:74
int max_width
The maximum image width needed by this clip (used for optimizations)
Definition: ReaderBase.h:102
int GetSamplesPerFrame(Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
Definition: Frame.cpp:497
void SetJsonValue(Json::Value root)
Load Json::JsonValue into this object.
Json::Value JsonValue()
Generate Json::JsonValue for this object.
Advanced 2:3:3:2 pull-down (minimal dirty frames)
Definition: FrameMapper.h:63
void SetJson(string value)
Load JSON string into this object.
void Open()
Open the internal reader.
double ToDouble()
Return this fraction as a double (i.e. 1/2 = 0.5)
Definition: Fraction.cpp:46
virtual void Open()=0
Open the reader (and start consuming resources, such as images or video files)
int sample_rate
The number of audio samples per second (44100 is a common sample rate)
Definition: ReaderBase.h:81
Exception when too many seek attempts happen.
Definition: Exceptions.h:254
virtual bool IsOpen()=0
Determine if reader is open or closed.