28 #include "../include/FrameMapper.h" 34 reader(reader), target(target), pulldown(target_pulldown), is_dirty(true), avr(NULL)
69 void FrameMapper::AddField(
long int frame)
72 AddField(
Field(frame, field_toggle));
75 void FrameMapper::AddField(
Field field)
81 field_toggle = (field_toggle ? false :
true);
88 void FrameMapper::Init()
90 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::Init (Calculate frame mappings)",
"", -1,
"", -1,
"", -1,
"", -1,
"", -1,
"", -1);
109 if ((fabs(original.
ToFloat() - 24.0) < 1
e-7 || fabs(original.
ToFloat() - 25.0) < 1
e-7 || fabs(original.
ToFloat() - 30.0) < 1
e-7) &&
110 (fabs(target.
ToFloat() - 24.0) < 1
e-7 || fabs(target.
ToFloat() - 25.0) < 1
e-7 || fabs(target.
ToFloat() - 30.0) < 1
e-7)) {
113 float difference = target.
ToInt() - original.
ToInt();
116 int field_interval = 0;
117 int frame_interval = 0;
121 field_interval = round(fabs(original.
ToInt() / difference));
124 frame_interval = field_interval * 2.0f;
133 for (
long int field = 1; field <= number_of_fields; field++)
141 else if (difference > 0)
151 else if (pulldown ==
PULLDOWN_ADVANCED && field % field_interval == 0 && field % frame_interval != 0)
158 AddField(
Field(frame + 1, field_toggle));
160 else if (pulldown ==
PULLDOWN_NONE && field % frame_interval == 0)
167 else if (difference < 0)
173 field_toggle = (field_toggle ? false :
true);
175 else if (pulldown ==
PULLDOWN_ADVANCED && field % field_interval == 0 && field % frame_interval != 0)
180 else if (pulldown ==
PULLDOWN_NONE && frame % field_interval == 0)
193 if (field % 2 == 0 && field > 0)
209 for (
long int frame_num = 1; frame_num <= new_length; frame_num++)
212 AddField(rate_curve.
GetInt(frame_num));
213 AddField(rate_curve.
GetInt(frame_num));
222 int start_samples_frame = 1;
223 int start_samples_position = 0;
225 for (
long int field = 1; field <=
fields.size(); field++)
231 if (field % 2 == 0 && field > 0)
234 long int frame_number = field / 2;
245 int end_samples_frame = start_samples_frame;
246 int end_samples_position = start_samples_position;
249 while (remaining_samples > 0)
255 if (original_samples >= remaining_samples)
258 end_samples_position += remaining_samples;
259 remaining_samples = 0;
263 end_samples_frame += 1;
264 end_samples_position = 0;
265 remaining_samples -= original_samples;
275 start_samples_frame = end_samples_frame;
276 start_samples_position = end_samples_position + 1;
279 start_samples_frame += 1;
280 start_samples_position = 0;
308 frame.
Odd.
Frame = TargetFrameNumber;
318 if(TargetFrameNumber < 1 ||
frames.size() == 0)
322 else if (TargetFrameNumber >
frames.size())
324 TargetFrameNumber =
frames.size();
327 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::GetMappedFrame",
"TargetFrameNumber", TargetFrameNumber,
"frames.size()",
frames.size(),
"frames[...].Odd",
frames[TargetFrameNumber - 1].Odd.Frame,
"frames[...].Even",
frames[TargetFrameNumber - 1].Even.Frame,
"", -1,
"", -1);
330 return frames[TargetFrameNumber - 1];
334 tr1::shared_ptr<Frame> FrameMapper::GetOrCreateFrame(
long int number)
336 tr1::shared_ptr<Frame> new_frame;
343 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::GetOrCreateFrame (from reader)",
"number", number,
"samples_in_frame", samples_in_frame,
"", -1,
"", -1,
"", -1,
"", -1);
349 new_frame = reader->
GetFrame(number);
363 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::GetOrCreateFrame (create blank)",
"number", number,
"samples_in_frame", samples_in_frame,
"", -1,
"", -1,
"", -1,
"", -1);
376 tr1::shared_ptr<Frame> final_frame = final_cache.
GetFrame(requested_frame);
377 if (final_frame)
return final_frame;
388 final_frame = final_cache.
GetFrame(requested_frame);
389 if (final_frame)
return final_frame;
397 omp_set_nested(
true);
400 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::GetFrame (Loop through frames)",
"requested_frame", requested_frame,
"minimum_frames", minimum_frames,
"", -1,
"", -1,
"", -1,
"", -1);
405 #pragma omp for ordered firstprivate(requested_frame, minimum_frames) 406 for (
long int frame_number = requested_frame; frame_number < requested_frame + minimum_frames; frame_number++)
410 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::GetFrame (inside omp for loop)",
"frame_number", frame_number,
"minimum_frames", minimum_frames,
"requested_frame", requested_frame,
"", -1,
"", -1,
"", -1);
414 tr1::shared_ptr<Frame> mapped_frame;
417 mapped_frame = GetOrCreateFrame(mapped.
Odd.
Frame);
420 int channels_in_frame = mapped_frame->GetAudioChannelsCount();
425 info.
channels == mapped_frame->GetAudioChannelsCount() &&
430 mapped_frame->SetFrameNumber(frame_number);
433 final_cache.
Add(mapped_frame);
438 tr1::shared_ptr<Frame> frame = tr1::shared_ptr<Frame>(
new Frame(frame_number, 1, 1,
"#000000", samples_in_frame, channels_in_frame));
439 frame->SampleRate(mapped_frame->SampleRate());
440 frame->ChannelsLayout(mapped_frame->ChannelsLayout());
444 tr1::shared_ptr<Frame> odd_frame;
446 odd_frame = GetOrCreateFrame(mapped.
Odd.
Frame);
449 frame->AddImage(tr1::shared_ptr<QImage>(
new QImage(*odd_frame->GetImage())),
true);
452 tr1::shared_ptr<Frame> even_frame;
454 even_frame = GetOrCreateFrame(mapped.
Even.
Frame);
456 frame->AddImage(tr1::shared_ptr<QImage>(
new QImage(*even_frame->GetImage())),
false);
460 int samples_copied = 0;
465 int remaining_samples = mapped.
Samples.
total - samples_copied;
466 int number_to_copy = 0;
469 for (
int channel = 0; channel < channels_in_frame; channel++)
472 tr1::shared_ptr<Frame> original_frame = GetOrCreateFrame(starting_frame);
473 int original_samples = original_frame->GetAudioSamplesCount();
479 if (number_to_copy > remaining_samples)
480 number_to_copy = remaining_samples;
483 frame->AddAudio(
true, channel, samples_copied, original_frame->GetAudioSamples(channel) + mapped.
Samples.
sample_start, number_to_copy, 1.0);
488 number_to_copy = original_samples;
489 if (number_to_copy > remaining_samples)
490 number_to_copy = remaining_samples;
493 frame->AddAudio(
true, channel, samples_copied, original_frame->GetAudioSamples(channel), number_to_copy, 1.0);
499 if (number_to_copy > remaining_samples)
500 number_to_copy = remaining_samples;
503 frame->AddAudio(
false, channel, samples_copied, original_frame->GetAudioSamples(channel), number_to_copy, 1.0);
508 samples_copied += number_to_copy;
521 final_cache.
Add(frame);
527 return final_cache.
GetFrame(requested_frame);
538 float difference = target.
ToInt() - original.
ToInt();
540 int field_interval = 0;
541 int frame_interval = 0;
546 field_interval = round(fabs(original.
ToInt() / difference));
549 frame_interval = field_interval * 2.0f;
553 for (
float map = 1; map <=
frames.size(); map++)
556 cout <<
"Target frame #: " << map <<
" mapped to original frame #:\t(" << frame.
Odd.
Frame <<
" odd, " << frame.
Even.
Frame <<
" even)" << endl;
577 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::Open",
"", -1,
"", -1,
"", -1,
"", -1,
"", -1,
"", -1);
592 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::Close",
"", -1,
"", -1,
"", -1,
"", -1,
"", -1,
"", -1);
599 avresample_close(avr);
600 avresample_free(&avr);
619 root[
"type"] =
"FrameMapper";
631 bool success = reader.parse( value, root );
634 throw InvalidJSON(
"JSON could not be parsed (or is invalid)",
"");
644 throw InvalidJSON(
"JSON is invalid (missing keys or invalid data types)",
"");
665 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::ChangeMapping",
"target_fps.num", target_fps.
num,
"target_fps.den", target_fps.
num,
"target_pulldown", target_pulldown,
"target_sample_rate", target_sample_rate,
"target_channels", target_channels,
"target_channel_layout", target_channel_layout);
672 pulldown = target_pulldown;
685 avresample_close(avr);
686 avresample_free(&avr);
695 int total_frame_samples = 0;
696 int channels_in_frame = frame->GetAudioChannelsCount();
697 int sample_rate_in_frame = frame->SampleRate();
698 int samples_in_frame = frame->GetAudioSamplesCount();
699 ChannelLayout channel_layout_in_frame = frame->ChannelsLayout();
701 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::ResampleMappedAudio",
"frame->number", frame->number,
"original_frame_number", original_frame_number,
"channels_in_frame", channels_in_frame,
"samples_in_frame", samples_in_frame,
"sample_rate_in_frame", sample_rate_in_frame,
"", -1);
704 float* frame_samples_float = NULL;
706 frame_samples_float = frame->GetInterleavedAudioSamples(sample_rate_in_frame, NULL, &samples_in_frame);
709 total_frame_samples = samples_in_frame * channels_in_frame;
712 int16_t* frame_samples = (int16_t*) av_malloc(
sizeof(int16_t)*total_frame_samples);
715 for (
int s = 0; s < total_frame_samples; s++)
717 frame_samples[s] =
int(frame_samples_float[s] * (1 << 15));
721 delete[] frame_samples_float;
722 frame_samples_float = NULL;
724 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::ResampleMappedAudio (got sample data from frame)",
"frame->number", frame->number,
"total_frame_samples", total_frame_samples,
"target channels",
info.
channels,
"channels_in_frame", channels_in_frame,
"target sample_rate",
info.
sample_rate,
"samples_in_frame", samples_in_frame);
730 audio_frame->nb_samples = total_frame_samples / channels_in_frame;
732 int error_code = avcodec_fill_audio_frame(audio_frame, channels_in_frame, AV_SAMPLE_FMT_S16, (uint8_t *) frame_samples,
733 audio_frame->nb_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) * channels_in_frame, 1);
737 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::ResampleMappedAudio ERROR [" + (
string)
av_err2str(error_code) +
"]",
"error_code", error_code,
"", -1,
"", -1,
"", -1,
"", -1,
"", -1);
738 throw ErrorEncodingVideo(
"Error while resampling audio in frame mapper", frame->number);
744 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::ResampleMappedAudio (adjust # of samples)",
"total_frame_samples", total_frame_samples,
"info.sample_rate",
info.
sample_rate,
"sample_rate_in_frame", sample_rate_in_frame,
"info.channels",
info.
channels,
"channels_in_frame", channels_in_frame,
"original_frame_number", original_frame_number);
749 audio_converted->nb_samples = total_frame_samples;
750 av_samples_alloc(audio_converted->data, audio_converted->linesize,
info.
channels, total_frame_samples, AV_SAMPLE_FMT_S16, 0);
752 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::ResampleMappedAudio (preparing for resample)",
"in_sample_fmt", AV_SAMPLE_FMT_S16,
"out_sample_fmt", AV_SAMPLE_FMT_S16,
"in_sample_rate", sample_rate_in_frame,
"out_sample_rate",
info.
sample_rate,
"in_channels", channels_in_frame,
"out_channels",
info.
channels);
761 avr = avresample_alloc_context();
762 av_opt_set_int(avr,
"in_channel_layout", channel_layout_in_frame, 0);
764 av_opt_set_int(avr,
"in_sample_fmt", AV_SAMPLE_FMT_S16, 0);
765 av_opt_set_int(avr,
"out_sample_fmt", AV_SAMPLE_FMT_S16, 0);
766 av_opt_set_int(avr,
"in_sample_rate", sample_rate_in_frame, 0);
768 av_opt_set_int(avr,
"in_channels", channels_in_frame, 0);
770 avresample_open(avr);
774 nb_samples = avresample_convert(avr,
775 audio_converted->data,
776 audio_converted->linesize[0],
777 audio_converted->nb_samples,
779 audio_frame->linesize[0],
780 audio_frame->nb_samples);
784 int16_t* resampled_samples =
new int16_t[(nb_samples *
info.
channels)];
787 memcpy(resampled_samples, audio_converted->data[0], (nb_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) *
info.
channels));
790 av_freep(&audio_frame->data[0]);
792 av_freep(&audio_converted->data[0]);
794 frame_samples = NULL;
797 int channel_buffer_size = nb_samples;
800 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::ResampleMappedAudio (Audio successfully resampled)",
"nb_samples", nb_samples,
"total_frame_samples", total_frame_samples,
"info.sample_rate",
info.
sample_rate,
"channels_in_frame", channels_in_frame,
"info.channels",
info.
channels,
"info.channel_layout",
info.
channel_layout);
803 float *channel_buffer =
new float[channel_buffer_size];
806 for (
int channel_filter = 0; channel_filter <
info.
channels; channel_filter++)
809 for (
int z = 0; z < channel_buffer_size; z++)
810 channel_buffer[z] = 0.0f;
816 for (
int sample = 0; sample < (nb_samples *
info.
channels); sample++)
819 if (channel_filter == channel)
822 channel_buffer[position] = resampled_samples[sample] * (1.0f / (1 << 15));
838 frame->AddAudio(
true, channel_filter, 0, channel_buffer, position, 1.0f);
840 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::ResampleMappedAudio (Add audio to channel)",
"number of samples", position,
"channel_filter", channel_filter,
"", -1,
"", -1,
"", -1,
"", -1);
848 delete[] channel_buffer;
849 channel_buffer = NULL;
852 delete[] resampled_samples;
853 resampled_samples = NULL;
#define AV_RESET_FRAME(av_frame)
Classic 2:3:2:3 pull-down.
int max_height
The maximium image height needed by this clip (used for optimizations)
#define AV_FREE_FRAME(av_frame)
int num
Numerator for the fraction.
CriticalSection getFrameCriticalSection
Section lock for multiple threads.
vector< MappedFrame > frames
ChannelLayout channel_layout
The channel layout (mono, stereo, 5 point surround, etc...)
int width
The width of the video (in pixesl)
tr1::shared_ptr< Frame > GetFrame(long int requested_frame)
This method is required for all derived classes of ReaderBase, and return the openshot::Frame object...
This class represents a single frame of video (i.e. image & audio data)
void ResampleMappedAudio(tr1::shared_ptr< Frame > frame, long int original_frame_number)
Resample audio and map channels (if needed)
float ToFloat()
Return this fraction as a float (i.e. 1/2 = 0.5)
~FrameMapper()
Destructor.
float duration
Length of time (in seconds)
void Close()
Close the openshot::FrameMapper and internal reader.
virtual void Close()=0
Close the reader (and any resources it was consuming)
This abstract class is the base class, used by all readers in libopenshot.
#define OPEN_MP_NUM_PROCESSORS
Exception when a reader is closed, and a frame is requested.
bool has_video
Determines if this file has a video stream.
void SetMaxBytesFromInfo(long int number_of_frames, int width, int height, int sample_rate, int channels)
Set maximum bytes to a different amount based on a ReaderInfo struct.
This struct holds a single field (half a frame).
Exception when encoding audio packet.
void AddPoint(Point p)
Add a new point on the key-frame. Each point has a primary coordinate, a left handle, and a right handle.
This struct holds a the range of samples needed by this frame.
void SetMaxSize(int width, int height)
Set Max Image Size (used for performance optimization)
MappedFrame GetMappedFrame(long int TargetFrameNumber)
Get a frame based on the target frame rate and the new frame number of a frame.
bool has_audio
Determines if this file has an audio stream.
void ChangeMapping(Fraction target_fps, PulldownType pulldown, int target_sample_rate, int target_channels, ChannelLayout target_channel_layout)
Change frame rate or audio mapping details.
int height
The height of the video (in pixels)
#define AV_ALLOCATE_FRAME()
Exception for files that can not be found or opened.
bool IsOpen()
Determine if reader is open or closed.
void AppendDebugMethod(string method_name, string arg1_name, float arg1_value, string arg2_name, float arg2_value, string arg3_name, float arg3_value, string arg4_name, float arg4_value, string arg5_name, float arg5_value, string arg6_name, float arg6_value)
Append debug information.
This class represents a fraction.
bool has_single_image
Determines if this file only contains a single image.
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround...
virtual Json::Value JsonValue()=0
Generate Json::JsonValue for this object.
virtual void SetJsonValue(Json::Value root)=0
Load Json::JsonValue into this object.
#define av_err2str(errnum)
ReaderInfo info
Information about the current media file.
void Clear()
Clear the cache of all frames.
int ToInt()
Return a rounded integer of the fraction (for example 30000/1001 returns 30)
This struct holds two fields which together make up a complete video frame.
Fraction fps
Frames per second, as a fraction (i.e. 24/1 = 24 fps)
Fraction video_timebase
The video timebase determines how long each frame stays on the screen.
string Json()
Get and Set JSON methods.
Exception for frames that are out of bounds.
static ZmqLogger * Instance()
Create or get an instance of this logger singleton (invoke the class with this method) ...
int GetInt(long int index)
Get the rounded INT value at a specific index.
void Add(tr1::shared_ptr< Frame > frame)
Add a Frame to the cache.
This namespace is the default namespace for all code in the openshot library.
virtual tr1::shared_ptr< Frame > GetFrame(long int number)=0
Do not apply pull-down techniques, just repeat or skip entire frames.
Linear curves are angular, straight lines between two points.
void PrintMapping()
Print all of the original frames and which new frames they map to.
Exception for invalid JSON.
tr1::shared_ptr< Frame > GetFrame(long int frame_number)
Get a frame from the cache.
PulldownType
This enumeration determines how frame rates are increased or decreased.
int den
Denominator for the fraction.
int channels
The number of audio channels used in the audio stream.
A Keyframe is a collection of Point instances, which is used to vary a number or property over time...
long int video_length
The number of frames in the video stream.
int max_width
The maximum image width needed by this clip (used for optimizations)
int GetSamplesPerFrame(Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
void SetJsonValue(Json::Value root)
Load Json::JsonValue into this object.
Json::Value JsonValue()
Generate Json::JsonValue for this object.
Advanced 2:3:3:2 pull-down (minimal dirty frames)
void SetJson(string value)
Load JSON string into this object.
void Open()
Open the internal reader.
double ToDouble()
Return this fraction as a double (i.e. 1/2 = 0.5)
virtual void Open()=0
Open the reader (and start consuming resources, such as images or video files)
int sample_rate
The number of audio samples per second (44100 is a common sample rate)
Exception when too many seek attempts happen.
virtual bool IsOpen()=0
Determine if reader is open or closed.