OpenShot Library | libopenshot  0.2.4
FFmpegReader.cpp
Go to the documentation of this file.
1 /**
2  * @file
3  * @brief Source file for FFmpegReader class
4  * @author Jonathan Thomas <jonathan@openshot.org>, Fabrice Bellard
5  *
6  * @ref License
7  */
8 
9 /* LICENSE
10  *
11  * Copyright (c) 2008-2019 OpenShot Studios, LLC, Fabrice Bellard
12  * (http://www.openshotstudios.com). This file is part of
13  * OpenShot Library (http://www.openshot.org), an open-source project
14  * dedicated to delivering high quality video editing and animation solutions
15  * to the world.
16  *
17  * This file is originally based on the Libavformat API example, and then modified
18  * by the libopenshot project.
19  *
20  * OpenShot Library (libopenshot) is free software: you can redistribute it
21  * and/or modify it under the terms of the GNU Lesser General Public License
22  * as published by the Free Software Foundation, either version 3 of the
23  * License, or (at your option) any later version.
24  *
25  * OpenShot Library (libopenshot) is distributed in the hope that it will be
26  * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
27  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
28  * GNU Lesser General Public License for more details.
29  *
30  * You should have received a copy of the GNU Lesser General Public License
31  * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
32  */
33 
34 #include "../include/FFmpegReader.h"
35 
36 #define ENABLE_VAAPI 0
37 
38 #if IS_FFMPEG_3_2
39 #pragma message "You are compiling with experimental hardware decode"
40 #else
41 #pragma message "You are compiling only with software decode"
42 #endif
43 
44 #if IS_FFMPEG_3_2
45 #define MAX_SUPPORTED_WIDTH 1950
46 #define MAX_SUPPORTED_HEIGHT 1100
47 
48 #if ENABLE_VAAPI
49 #include "libavutil/hwcontext_vaapi.h"
50 
51 typedef struct VAAPIDecodeContext {
52  VAProfile va_profile;
53  VAEntrypoint va_entrypoint;
54  VAConfigID va_config;
55  VAContextID va_context;
56 
57 #if FF_API_STRUCT_VAAPI_CONTEXT
58  // FF_DISABLE_DEPRECATION_WARNINGS
59  int have_old_context;
60  struct vaapi_context *old_context;
61  AVBufferRef *device_ref;
62  // FF_ENABLE_DEPRECATION_WARNINGS
63 #endif
64 
65  AVHWDeviceContext *device;
66  AVVAAPIDeviceContext *hwctx;
67 
68  AVHWFramesContext *frames;
69  AVVAAPIFramesContext *hwfc;
70 
71  enum AVPixelFormat surface_format;
72  int surface_count;
73  } VAAPIDecodeContext;
74 #endif
75 #endif
76 
77 
78 using namespace openshot;
79 
80 int hw_de_on = 0;
81 #if IS_FFMPEG_3_2
82  AVPixelFormat hw_de_av_pix_fmt_global = AV_PIX_FMT_NONE;
83  AVHWDeviceType hw_de_av_device_type_global = AV_HWDEVICE_TYPE_NONE;
84 #endif
85 
86 FFmpegReader::FFmpegReader(std::string path)
87  : last_frame(0), is_seeking(0), seeking_pts(0), seeking_frame(0), seek_count(0),
88  audio_pts_offset(99999), video_pts_offset(99999), path(path), is_video_seek(true), check_interlace(false),
89  check_fps(false), enable_seek(true), is_open(false), seek_audio_frame_found(0), seek_video_frame_found(0),
90  prev_samples(0), prev_pts(0), pts_total(0), pts_counter(0), is_duration_known(false), largest_frame_processed(0),
91  current_video_frame(0), has_missing_frames(false), num_packets_since_video_frame(0), num_checks_since_final(0),
92  packet(NULL) {
93 
94  // Initialize FFMpeg, and register all formats and codecs
97 
98  // Init cache
102 
103  // Open and Close the reader, to populate its attributes (such as height, width, etc...)
104  Open();
105  Close();
106 }
107 
108 FFmpegReader::FFmpegReader(std::string path, bool inspect_reader)
109  : last_frame(0), is_seeking(0), seeking_pts(0), seeking_frame(0), seek_count(0),
110  audio_pts_offset(99999), video_pts_offset(99999), path(path), is_video_seek(true), check_interlace(false),
111  check_fps(false), enable_seek(true), is_open(false), seek_audio_frame_found(0), seek_video_frame_found(0),
112  prev_samples(0), prev_pts(0), pts_total(0), pts_counter(0), is_duration_known(false), largest_frame_processed(0),
113  current_video_frame(0), has_missing_frames(false), num_packets_since_video_frame(0), num_checks_since_final(0),
114  packet(NULL) {
115 
116  // Initialize FFMpeg, and register all formats and codecs
119 
120  // Init cache
124 
125  // Open and Close the reader, to populate its attributes (such as height, width, etc...)
126  if (inspect_reader) {
127  Open();
128  Close();
129  }
130 }
131 
133  if (is_open)
134  // Auto close reader if not already done
135  Close();
136 }
137 
138 // This struct holds the associated video frame and starting sample # for an audio packet.
139 bool AudioLocation::is_near(AudioLocation location, int samples_per_frame, int64_t amount) {
140  // Is frame even close to this one?
141  if (abs(location.frame - frame) >= 2)
142  // This is too far away to be considered
143  return false;
144 
145  // Note that samples_per_frame can vary slightly frame to frame when the
146  // audio sampling rate is not an integer multiple of the video fps.
147  int64_t diff = samples_per_frame * (location.frame - frame) + location.sample_start - sample_start;
148  if (abs(diff) <= amount)
149  // close
150  return true;
151 
152  // not close
153  return false;
154 }
155 
156 #if IS_FFMPEG_3_2
157 
158 // Get hardware pix format
159 static enum AVPixelFormat get_hw_dec_format(AVCodecContext *ctx, const enum AVPixelFormat *pix_fmts)
160 {
161  const enum AVPixelFormat *p;
162 
163  for (p = pix_fmts; *p != AV_PIX_FMT_NONE; p++) {
164  switch (*p) {
165 #if defined(__linux__)
166  // Linux pix formats
167  case AV_PIX_FMT_VAAPI:
168  hw_de_av_pix_fmt_global = AV_PIX_FMT_VAAPI;
169  hw_de_av_device_type_global = AV_HWDEVICE_TYPE_VAAPI;
170  return *p;
171  break;
172  case AV_PIX_FMT_VDPAU:
173  hw_de_av_pix_fmt_global = AV_PIX_FMT_VDPAU;
174  hw_de_av_device_type_global = AV_HWDEVICE_TYPE_VDPAU;
175  return *p;
176  break;
177 #endif
178 #if defined(_WIN32)
179  // Windows pix formats
180  case AV_PIX_FMT_DXVA2_VLD:
181  hw_de_av_pix_fmt_global = AV_PIX_FMT_DXVA2_VLD;
182  hw_de_av_device_type_global = AV_HWDEVICE_TYPE_DXVA2;
183  return *p;
184  break;
185  case AV_PIX_FMT_D3D11:
186  hw_de_av_pix_fmt_global = AV_PIX_FMT_D3D11;
187  hw_de_av_device_type_global = AV_HWDEVICE_TYPE_D3D11VA;
188  return *p;
189  break;
190 #endif
191 #if defined(__APPLE__)
192  // Apple pix formats
193  case AV_PIX_FMT_VIDEOTOOLBOX:
194  hw_de_av_pix_fmt_global = AV_PIX_FMT_VIDEOTOOLBOX;
195  hw_de_av_device_type_global = AV_HWDEVICE_TYPE_VIDEOTOOLBOX;
196  return *p;
197  break;
198 #endif
199  // Cross-platform pix formats
200  case AV_PIX_FMT_CUDA:
201  hw_de_av_pix_fmt_global = AV_PIX_FMT_CUDA;
202  hw_de_av_device_type_global = AV_HWDEVICE_TYPE_CUDA;
203  return *p;
204  break;
205  case AV_PIX_FMT_QSV:
206  hw_de_av_pix_fmt_global = AV_PIX_FMT_QSV;
207  hw_de_av_device_type_global = AV_HWDEVICE_TYPE_QSV;
208  return *p;
209  break;
210  default:
211  // This is only here to silence unused-enum warnings
212  break;
213  }
214  }
215  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::get_hw_dec_format (Unable to decode this file using hardware decode)");
216  return AV_PIX_FMT_NONE;
217 }
218 
219 int FFmpegReader::IsHardwareDecodeSupported(int codecid)
220 {
221  int ret;
222  switch (codecid) {
223  case AV_CODEC_ID_H264:
224  case AV_CODEC_ID_MPEG2VIDEO:
225  case AV_CODEC_ID_VC1:
226  case AV_CODEC_ID_WMV1:
227  case AV_CODEC_ID_WMV2:
228  case AV_CODEC_ID_WMV3:
229  ret = 1;
230  break;
231  default :
232  ret = 0;
233  break;
234  }
235  return ret;
236 }
237 #endif
238 
240  // Open reader if not already open
241  if (!is_open) {
242  // Initialize format context
243  pFormatCtx = NULL;
244  {
246  }
247 
248  // Open video file
249  if (avformat_open_input(&pFormatCtx, path.c_str(), NULL, NULL) != 0)
250  throw InvalidFile("File could not be opened.", path);
251 
252  // Retrieve stream information
253  if (avformat_find_stream_info(pFormatCtx, NULL) < 0)
254  throw NoStreamsFound("No streams found in file.", path);
255 
256  videoStream = -1;
257  audioStream = -1;
258  // Loop through each stream, and identify the video and audio stream index
259  for (unsigned int i = 0; i < pFormatCtx->nb_streams; i++) {
260  // Is this a video stream?
261  if (AV_GET_CODEC_TYPE(pFormatCtx->streams[i]) == AVMEDIA_TYPE_VIDEO && videoStream < 0) {
262  videoStream = i;
263  }
264  // Is this an audio stream?
265  if (AV_GET_CODEC_TYPE(pFormatCtx->streams[i]) == AVMEDIA_TYPE_AUDIO && audioStream < 0) {
266  audioStream = i;
267  }
268  }
269  if (videoStream == -1 && audioStream == -1)
270  throw NoStreamsFound("No video or audio streams found in this file.", path);
271 
272  // Is there a video stream?
273  if (videoStream != -1) {
274  // Set the stream index
275  info.video_stream_index = videoStream;
276 
277  // Set the codec and codec context pointers
278  pStream = pFormatCtx->streams[videoStream];
279 
280  // Find the codec ID from stream
281  AVCodecID codecId = AV_FIND_DECODER_CODEC_ID(pStream);
282 
283  // Get codec and codec context from stream
284  AVCodec *pCodec = avcodec_find_decoder(codecId);
285  AVDictionary *opts = NULL;
286  int retry_decode_open = 2;
287  // If hw accel is selected but hardware cannot handle repeat with software decoding
288  do {
289  pCodecCtx = AV_GET_CODEC_CONTEXT(pStream, pCodec);
290 #if IS_FFMPEG_3_2
291  if (hw_de_on && (retry_decode_open==2)) {
292  // Up to here no decision is made if hardware or software decode
293  hw_de_supported = IsHardwareDecodeSupported(pCodecCtx->codec_id);
294  }
295 #endif
296  retry_decode_open = 0;
297 
298  // Set number of threads equal to number of processors (not to exceed 16)
299  pCodecCtx->thread_count = std::min(FF_NUM_PROCESSORS, 16);
300 
301  if (pCodec == NULL) {
302  throw InvalidCodec("A valid video codec could not be found for this file.", path);
303  }
304 
305  // Init options
306  av_dict_set(&opts, "strict", "experimental", 0);
307 #if IS_FFMPEG_3_2
308  if (hw_de_on && hw_de_supported) {
309  // Open Hardware Acceleration
310  int i_decoder_hw = 0;
311  char adapter[256];
312  char *adapter_ptr = NULL;
313  int adapter_num;
315  fprintf(stderr, "Hardware decoding device number: %d\n", adapter_num);
316 
317  // Set hardware pix format (callback)
318  pCodecCtx->get_format = get_hw_dec_format;
319 
320  if (adapter_num < 3 && adapter_num >=0) {
321 #if defined(__linux__)
322  snprintf(adapter,sizeof(adapter),"/dev/dri/renderD%d", adapter_num+128);
323  adapter_ptr = adapter;
325  switch (i_decoder_hw) {
326  case 1:
327  hw_de_av_device_type = AV_HWDEVICE_TYPE_VAAPI;
328  break;
329  case 2:
330  hw_de_av_device_type = AV_HWDEVICE_TYPE_CUDA;
331  break;
332  case 6:
333  hw_de_av_device_type = AV_HWDEVICE_TYPE_VDPAU;
334  break;
335  case 7:
336  hw_de_av_device_type = AV_HWDEVICE_TYPE_QSV;
337  break;
338  default:
339  hw_de_av_device_type = AV_HWDEVICE_TYPE_VAAPI;
340  break;
341  }
342 
343 #elif defined(_WIN32)
344  adapter_ptr = NULL;
346  switch (i_decoder_hw) {
347  case 2:
348  hw_de_av_device_type = AV_HWDEVICE_TYPE_CUDA;
349  break;
350  case 3:
351  hw_de_av_device_type = AV_HWDEVICE_TYPE_DXVA2;
352  break;
353  case 4:
354  hw_de_av_device_type = AV_HWDEVICE_TYPE_D3D11VA;
355  break;
356  case 7:
357  hw_de_av_device_type = AV_HWDEVICE_TYPE_QSV;
358  break;
359  default:
360  hw_de_av_device_type = AV_HWDEVICE_TYPE_DXVA2;
361  break;
362  }
363 #elif defined(__APPLE__)
364  adapter_ptr = NULL;
366  switch (i_decoder_hw) {
367  case 5:
368  hw_de_av_device_type = AV_HWDEVICE_TYPE_VIDEOTOOLBOX;
369  break;
370  case 7:
371  hw_de_av_device_type = AV_HWDEVICE_TYPE_QSV;
372  break;
373  default:
374  hw_de_av_device_type = AV_HWDEVICE_TYPE_VIDEOTOOLBOX;
375  break;
376  }
377 #endif
378 
379  } else {
380  adapter_ptr = NULL; // Just to be sure
381  }
382 
383  // Check if it is there and writable
384 #if defined(__linux__)
385  if( adapter_ptr != NULL && access( adapter_ptr, W_OK ) == 0 ) {
386 #elif defined(_WIN32)
387  if( adapter_ptr != NULL ) {
388 #elif defined(__APPLE__)
389  if( adapter_ptr != NULL ) {
390 #endif
391  ZmqLogger::Instance()->AppendDebugMethod("Decode Device present using device");
392  }
393  else {
394  adapter_ptr = NULL; // use default
395  ZmqLogger::Instance()->AppendDebugMethod("Decode Device not present using default");
396  }
397 
398  hw_device_ctx = NULL;
399  // Here the first hardware initialisations are made
400  if (av_hwdevice_ctx_create(&hw_device_ctx, hw_de_av_device_type, adapter_ptr, NULL, 0) >= 0) {
401  if (!(pCodecCtx->hw_device_ctx = av_buffer_ref(hw_device_ctx))) {
402  throw InvalidCodec("Hardware device reference create failed.", path);
403  }
404 
405  /*
406  av_buffer_unref(&ist->hw_frames_ctx);
407  ist->hw_frames_ctx = av_hwframe_ctx_alloc(hw_device_ctx);
408  if (!ist->hw_frames_ctx) {
409  av_log(avctx, AV_LOG_ERROR, "Error creating a CUDA frames context\n");
410  return AVERROR(ENOMEM);
411  }
412 
413  frames_ctx = (AVHWFramesContext*)ist->hw_frames_ctx->data;
414 
415  frames_ctx->format = AV_PIX_FMT_CUDA;
416  frames_ctx->sw_format = avctx->sw_pix_fmt;
417  frames_ctx->width = avctx->width;
418  frames_ctx->height = avctx->height;
419 
420  av_log(avctx, AV_LOG_DEBUG, "Initializing CUDA frames context: sw_format = %s, width = %d, height = %d\n",
421  av_get_pix_fmt_name(frames_ctx->sw_format), frames_ctx->width, frames_ctx->height);
422 
423 
424  ret = av_hwframe_ctx_init(pCodecCtx->hw_device_ctx);
425  ret = av_hwframe_ctx_init(ist->hw_frames_ctx);
426  if (ret < 0) {
427  av_log(avctx, AV_LOG_ERROR, "Error initializing a CUDA frame pool\n");
428  return ret;
429  }
430  */
431  }
432  else {
433  throw InvalidCodec("Hardware device create failed.", path);
434  }
435  }
436 #endif
437 
438  // Open video codec
439  if (avcodec_open2(pCodecCtx, pCodec, &opts) < 0)
440  throw InvalidCodec("A video codec was found, but could not be opened.", path);
441 
442 #if IS_FFMPEG_3_2
443  if (hw_de_on && hw_de_supported) {
444  AVHWFramesConstraints *constraints = NULL;
445  void *hwconfig = NULL;
446  hwconfig = av_hwdevice_hwconfig_alloc(hw_device_ctx);
447 
448 // TODO: needs va_config!
449 #if ENABLE_VAAPI
450  ((AVVAAPIHWConfig *)hwconfig)->config_id = ((VAAPIDecodeContext *)(pCodecCtx->priv_data))->va_config;
451  constraints = av_hwdevice_get_hwframe_constraints(hw_device_ctx,hwconfig);
452 #endif
453  if (constraints) {
454  if (pCodecCtx->coded_width < constraints->min_width ||
455  pCodecCtx->coded_height < constraints->min_height ||
456  pCodecCtx->coded_width > constraints->max_width ||
457  pCodecCtx->coded_height > constraints->max_height) {
458  ZmqLogger::Instance()->AppendDebugMethod("DIMENSIONS ARE TOO LARGE for hardware acceleration\n");
459  hw_de_supported = 0;
460  retry_decode_open = 1;
461  AV_FREE_CONTEXT(pCodecCtx);
462  if (hw_device_ctx) {
463  av_buffer_unref(&hw_device_ctx);
464  hw_device_ctx = NULL;
465  }
466  }
467  else {
468  // All is just peachy
469  ZmqLogger::Instance()->AppendDebugMethod("\nDecode hardware acceleration is used\n", "Min width :", constraints->min_width, "Min Height :", constraints->min_height, "MaxWidth :", constraints->max_width, "MaxHeight :", constraints->max_height, "Frame width :", pCodecCtx->coded_width, "Frame height :", pCodecCtx->coded_height);
470  retry_decode_open = 0;
471  }
472  av_hwframe_constraints_free(&constraints);
473  if (hwconfig) {
474  av_freep(&hwconfig);
475  }
476  }
477  else {
478  int max_h, max_w;
479  //max_h = ((getenv( "LIMIT_HEIGHT_MAX" )==NULL) ? MAX_SUPPORTED_HEIGHT : atoi(getenv( "LIMIT_HEIGHT_MAX" )));
481  //max_w = ((getenv( "LIMIT_WIDTH_MAX" )==NULL) ? MAX_SUPPORTED_WIDTH : atoi(getenv( "LIMIT_WIDTH_MAX" )));
483  ZmqLogger::Instance()->AppendDebugMethod("Constraints could not be found using default limit\n");
484  //cerr << "Constraints could not be found using default limit\n";
485  if (pCodecCtx->coded_width < 0 ||
486  pCodecCtx->coded_height < 0 ||
487  pCodecCtx->coded_width > max_w ||
488  pCodecCtx->coded_height > max_h ) {
489  ZmqLogger::Instance()->AppendDebugMethod("DIMENSIONS ARE TOO LARGE for hardware acceleration\n", "Max Width :", max_w, "Max Height :", max_h, "Frame width :", pCodecCtx->coded_width, "Frame height :", pCodecCtx->coded_height);
490  hw_de_supported = 0;
491  retry_decode_open = 1;
492  AV_FREE_CONTEXT(pCodecCtx);
493  if (hw_device_ctx) {
494  av_buffer_unref(&hw_device_ctx);
495  hw_device_ctx = NULL;
496  }
497  }
498  else {
499  ZmqLogger::Instance()->AppendDebugMethod("\nDecode hardware acceleration is used\n", "Max Width :", max_w, "Max Height :", max_h, "Frame width :", pCodecCtx->coded_width, "Frame height :", pCodecCtx->coded_height);
500  retry_decode_open = 0;
501  }
502  }
503  } // if hw_de_on && hw_de_supported
504  else {
505  ZmqLogger::Instance()->AppendDebugMethod("\nDecode in software is used\n");
506  }
507 #else
508  retry_decode_open = 0;
509 #endif
510  } while (retry_decode_open); // retry_decode_open
511  // Free options
512  av_dict_free(&opts);
513 
514  // Update the File Info struct with video details (if a video stream is found)
515  UpdateVideoInfo();
516  }
517 
518  // Is there an audio stream?
519  if (audioStream != -1) {
520  // Set the stream index
521  info.audio_stream_index = audioStream;
522 
523  // Get a pointer to the codec context for the audio stream
524  aStream = pFormatCtx->streams[audioStream];
525 
526  // Find the codec ID from stream
527  AVCodecID codecId = AV_FIND_DECODER_CODEC_ID(aStream);
528 
529  // Get codec and codec context from stream
530  AVCodec *aCodec = avcodec_find_decoder(codecId);
531  aCodecCtx = AV_GET_CODEC_CONTEXT(aStream, aCodec);
532 
533  // Set number of threads equal to number of processors (not to exceed 16)
534  aCodecCtx->thread_count = std::min(FF_NUM_PROCESSORS, 16);
535 
536  if (aCodec == NULL) {
537  throw InvalidCodec("A valid audio codec could not be found for this file.", path);
538  }
539 
540  // Init options
541  AVDictionary *opts = NULL;
542  av_dict_set(&opts, "strict", "experimental", 0);
543 
544  // Open audio codec
545  if (avcodec_open2(aCodecCtx, aCodec, &opts) < 0)
546  throw InvalidCodec("An audio codec was found, but could not be opened.", path);
547 
548  // Free options
549  av_dict_free(&opts);
550 
551  // Update the File Info struct with audio details (if an audio stream is found)
552  UpdateAudioInfo();
553  }
554 
555  // Add format metadata (if any)
556  AVDictionaryEntry *tag = NULL;
557  while ((tag = av_dict_get(pFormatCtx->metadata, "", tag, AV_DICT_IGNORE_SUFFIX))) {
558  QString str_key = tag->key;
559  QString str_value = tag->value;
560  info.metadata[str_key.toStdString()] = str_value.trimmed().toStdString();
561  }
562 
563  // Init previous audio location to zero
564  previous_packet_location.frame = -1;
565  previous_packet_location.sample_start = 0;
566 
567  // Adjust cache size based on size of frame and audio
571 
572  // Mark as "open"
573  is_open = true;
574  }
575 }
576 
578  // Close all objects, if reader is 'open'
579  if (is_open) {
580  // Mark as "closed"
581  is_open = false;
582 
583  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::Close");
584 
585  if (packet) {
586  // Remove previous packet before getting next one
587  RemoveAVPacket(packet);
588  packet = NULL;
589  }
590 
591  // Close the codec
592  if (info.has_video) {
593  avcodec_flush_buffers(pCodecCtx);
594  AV_FREE_CONTEXT(pCodecCtx);
595 #if IS_FFMPEG_3_2
596  if (hw_de_on) {
597  if (hw_device_ctx) {
598  av_buffer_unref(&hw_device_ctx);
599  hw_device_ctx = NULL;
600  }
601  }
602 #endif
603  }
604  if (info.has_audio) {
605  avcodec_flush_buffers(aCodecCtx);
606  AV_FREE_CONTEXT(aCodecCtx);
607  }
608 
609  // Clear final cache
610  final_cache.Clear();
611  working_cache.Clear();
612  missing_frames.Clear();
613 
614  // Clear processed lists
615  {
616  const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
617  processed_video_frames.clear();
618  processed_audio_frames.clear();
619  processing_video_frames.clear();
620  processing_audio_frames.clear();
621  missing_audio_frames.clear();
622  missing_video_frames.clear();
623  missing_audio_frames_source.clear();
624  missing_video_frames_source.clear();
625  checked_frames.clear();
626  }
627 
628  // Close the video file
629  avformat_close_input(&pFormatCtx);
630  av_freep(&pFormatCtx);
631 
632  // Reset some variables
633  last_frame = 0;
634  largest_frame_processed = 0;
635  seek_audio_frame_found = 0;
636  seek_video_frame_found = 0;
637  current_video_frame = 0;
638  has_missing_frames = false;
639 
640  last_video_frame.reset();
641  }
642 }
643 
644 void FFmpegReader::UpdateAudioInfo() {
645  // Set values of FileInfo struct
646  info.has_audio = true;
647  info.file_size = pFormatCtx->pb ? avio_size(pFormatCtx->pb) : -1;
648  info.acodec = aCodecCtx->codec->name;
649  info.channels = AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels;
650  if (AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout == 0)
651  AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout = av_get_default_channel_layout(AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels);
652  info.channel_layout = (ChannelLayout) AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout;
653  info.sample_rate = AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->sample_rate;
654  info.audio_bit_rate = AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->bit_rate;
655 
656  // Set audio timebase
657  info.audio_timebase.num = aStream->time_base.num;
658  info.audio_timebase.den = aStream->time_base.den;
659 
660  // Get timebase of audio stream (if valid) and greater than the current duration
661  if (aStream->duration > 0.0f && aStream->duration > info.duration)
662  info.duration = aStream->duration * info.audio_timebase.ToDouble();
663 
664  // Check for an invalid video length
665  if (info.has_video && info.video_length <= 0) {
666  // Calculate the video length from the audio duration
668  }
669 
670  // Set video timebase (if no video stream was found)
671  if (!info.has_video) {
672  // Set a few important default video settings (so audio can be divided into frames)
673  info.fps.num = 24;
674  info.fps.den = 1;
675  info.video_timebase.num = 1;
676  info.video_timebase.den = 24;
678  info.width = 720;
679  info.height = 480;
680  }
681 
682  // Fix invalid video lengths for certain types of files (MP3 for example)
683  if (info.has_video && ((info.duration * info.fps.ToDouble()) - info.video_length > 60)) {
685  }
686 
687  // Add audio metadata (if any found)
688  AVDictionaryEntry *tag = NULL;
689  while ((tag = av_dict_get(aStream->metadata, "", tag, AV_DICT_IGNORE_SUFFIX))) {
690  QString str_key = tag->key;
691  QString str_value = tag->value;
692  info.metadata[str_key.toStdString()] = str_value.trimmed().toStdString();
693  }
694 }
695 
696 void FFmpegReader::UpdateVideoInfo() {
697  if (check_fps)
698  // Already initialized all the video metadata, no reason to do it again
699  return;
700 
701  // Set values of FileInfo struct
702  info.has_video = true;
703  info.file_size = pFormatCtx->pb ? avio_size(pFormatCtx->pb) : -1;
704  info.height = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->height;
705  info.width = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->width;
706  info.vcodec = pCodecCtx->codec->name;
707  info.video_bit_rate = (pFormatCtx->bit_rate / 8);
708 
709  // set frames per second (fps)
710  info.fps.num = pStream->avg_frame_rate.num;
711  info.fps.den = pStream->avg_frame_rate.den;
712 
713  if (pStream->sample_aspect_ratio.num != 0) {
714  info.pixel_ratio.num = pStream->sample_aspect_ratio.num;
715  info.pixel_ratio.den = pStream->sample_aspect_ratio.den;
716  } else if (AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->sample_aspect_ratio.num != 0) {
717  info.pixel_ratio.num = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->sample_aspect_ratio.num;
718  info.pixel_ratio.den = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->sample_aspect_ratio.den;
719  } else {
720  info.pixel_ratio.num = 1;
721  info.pixel_ratio.den = 1;
722  }
723  info.pixel_format = AV_GET_CODEC_PIXEL_FORMAT(pStream, pCodecCtx);
724 
725  // Calculate the DAR (display aspect ratio)
727 
728  // Reduce size fraction
729  size.Reduce();
730 
731  // Set the ratio based on the reduced fraction
732  info.display_ratio.num = size.num;
733  info.display_ratio.den = size.den;
734 
735  // Get scan type and order from codec context/params
736  if (!check_interlace) {
737  check_interlace = true;
738  AVFieldOrder field_order = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->field_order;
739  switch(field_order) {
740  case AV_FIELD_PROGRESSIVE:
741  info.interlaced_frame = false;
742  break;
743  case AV_FIELD_TT:
744  case AV_FIELD_TB:
745  info.interlaced_frame = true;
746  info.top_field_first = true;
747  break;
748  case AV_FIELD_BT:
749  case AV_FIELD_BB:
750  info.interlaced_frame = true;
751  info.top_field_first = false;
752  break;
753  case AV_FIELD_UNKNOWN:
754  // Check again later?
755  check_interlace = false;
756  break;
757  }
758  // check_interlace will prevent these checks being repeated,
759  // unless it was cleared because we got an AV_FIELD_UNKNOWN response.
760  }
761 
762  // Set the video timebase
763  info.video_timebase.num = pStream->time_base.num;
764  info.video_timebase.den = pStream->time_base.den;
765 
766  // Set the duration in seconds, and video length (# of frames)
767  info.duration = pStream->duration * info.video_timebase.ToDouble();
768 
769  // Check for valid duration (if found)
770  if (info.duration <= 0.0f && pFormatCtx->duration >= 0)
771  // Use the format's duration
772  info.duration = pFormatCtx->duration / AV_TIME_BASE;
773 
774  // Calculate duration from filesize and bitrate (if any)
775  if (info.duration <= 0.0f && info.video_bit_rate > 0 && info.file_size > 0)
776  // Estimate from bitrate, total bytes, and framerate
778 
779  // No duration found in stream of file
780  if (info.duration <= 0.0f) {
781  // No duration is found in the video stream
782  info.duration = -1;
783  info.video_length = -1;
784  is_duration_known = false;
785  } else {
786  // Yes, a duration was found
787  is_duration_known = true;
788 
789  // Calculate number of frames
791  }
792 
793  // Override an invalid framerate
794  if (info.fps.ToFloat() > 240.0f || (info.fps.num <= 0 || info.fps.den <= 0) || info.video_length <= 0) {
795  // Calculate FPS, duration, video bit rate, and video length manually
796  // by scanning through all the video stream packets
797  CheckFPS();
798  }
799 
800  // Add video metadata (if any)
801  AVDictionaryEntry *tag = NULL;
802  while ((tag = av_dict_get(pStream->metadata, "", tag, AV_DICT_IGNORE_SUFFIX))) {
803  QString str_key = tag->key;
804  QString str_value = tag->value;
805  info.metadata[str_key.toStdString()] = str_value.trimmed().toStdString();
806  }
807 }
808 
809 
810 std::shared_ptr<Frame> FFmpegReader::GetFrame(int64_t requested_frame) {
811  // Check for open reader (or throw exception)
812  if (!is_open)
813  throw ReaderClosed("The FFmpegReader is closed. Call Open() before calling this method.", path);
814 
815  // Adjust for a requested frame that is too small or too large
816  if (requested_frame < 1)
817  requested_frame = 1;
818  if (requested_frame > info.video_length && is_duration_known)
819  requested_frame = info.video_length;
820  if (info.has_video && info.video_length == 0)
821  // Invalid duration of video file
822  throw InvalidFile("Could not detect the duration of the video or audio stream.", path);
823 
824  // Debug output
825  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetFrame", "requested_frame", requested_frame, "last_frame", last_frame);
826 
827  // Check the cache for this frame
828  std::shared_ptr<Frame> frame = final_cache.GetFrame(requested_frame);
829  if (frame) {
830  // Debug output
831  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetFrame", "returned cached frame", requested_frame);
832 
833  // Return the cached frame
834  return frame;
835  } else {
836 #pragma omp critical (ReadStream)
837  {
838  // Check the cache a 2nd time (due to a potential previous lock)
839  frame = final_cache.GetFrame(requested_frame);
840  if (frame) {
841  // Debug output
842  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetFrame", "returned cached frame on 2nd look", requested_frame);
843 
844  // Return the cached frame
845  } else {
846  // Frame is not in cache
847  // Reset seek count
848  seek_count = 0;
849 
850  // Check for first frame (always need to get frame 1 before other frames, to correctly calculate offsets)
851  if (last_frame == 0 && requested_frame != 1)
852  // Get first frame
853  ReadStream(1);
854 
855  // Are we within X frames of the requested frame?
856  int64_t diff = requested_frame - last_frame;
857  if (diff >= 1 && diff <= 20) {
858  // Continue walking the stream
859  frame = ReadStream(requested_frame);
860  } else {
861  // Greater than 30 frames away, or backwards, we need to seek to the nearest key frame
862  if (enable_seek)
863  // Only seek if enabled
864  Seek(requested_frame);
865 
866  else if (!enable_seek && diff < 0) {
867  // Start over, since we can't seek, and the requested frame is smaller than our position
868  Close();
869  Open();
870  }
871 
872  // Then continue walking the stream
873  frame = ReadStream(requested_frame);
874  }
875  }
876  } //omp critical
877  return frame;
878  }
879 }
880 
881 // Read the stream until we find the requested Frame
882 std::shared_ptr<Frame> FFmpegReader::ReadStream(int64_t requested_frame) {
883  // Allocate video frame
884  bool end_of_stream = false;
885  bool check_seek = false;
886  bool frame_finished = false;
887  int packet_error = -1;
888 
889  // Minimum number of packets to process (for performance reasons)
890  int packets_processed = 0;
891  int minimum_packets = OPEN_MP_NUM_PROCESSORS;
892  int max_packets = 4096;
893 
894  // Set the number of threads in OpenMP
895  omp_set_num_threads(OPEN_MP_NUM_PROCESSORS);
896  // Allow nested OpenMP sections
897  omp_set_nested(true);
898 
899  // Debug output
900  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream", "requested_frame", requested_frame, "OPEN_MP_NUM_PROCESSORS", OPEN_MP_NUM_PROCESSORS);
901 
902 #pragma omp parallel
903  {
904 #pragma omp single
905  {
906  // Loop through the stream until the correct frame is found
907  while (true) {
908  // Get the next packet into a local variable called packet
909  packet_error = GetNextPacket();
910 
911  int processing_video_frames_size = 0;
912  int processing_audio_frames_size = 0;
913  {
914  const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
915  processing_video_frames_size = processing_video_frames.size();
916  processing_audio_frames_size = processing_audio_frames.size();
917  }
918 
919  // Wait if too many frames are being processed
920  while (processing_video_frames_size + processing_audio_frames_size >= minimum_packets) {
921  usleep(2500);
922  const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
923  processing_video_frames_size = processing_video_frames.size();
924  processing_audio_frames_size = processing_audio_frames.size();
925  }
926 
927  // Get the next packet (if any)
928  if (packet_error < 0) {
929  // Break loop when no more packets found
930  end_of_stream = true;
931  break;
932  }
933 
934  // Debug output
935  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream (GetNextPacket)", "requested_frame", requested_frame, "processing_video_frames_size", processing_video_frames_size, "processing_audio_frames_size", processing_audio_frames_size, "minimum_packets", minimum_packets, "packets_processed", packets_processed, "is_seeking", is_seeking);
936 
937  // Video packet
938  if (info.has_video && packet->stream_index == videoStream) {
939  // Reset this counter, since we have a video packet
940  num_packets_since_video_frame = 0;
941 
942  // Check the status of a seek (if any)
943  if (is_seeking)
944 #pragma omp critical (openshot_seek)
945  check_seek = CheckSeek(true);
946  else
947  check_seek = false;
948 
949  if (check_seek) {
950  // Jump to the next iteration of this loop
951  continue;
952  }
953 
954  // Packet may become NULL on Close inside Seek if CheckSeek returns false
955  if (!packet)
956  // Jump to the next iteration of this loop
957  continue;
958 
959  // Get the AVFrame from the current packet
960  frame_finished = GetAVFrame();
961 
962  // Check if the AVFrame is finished and set it
963  if (frame_finished) {
964  // Update PTS / Frame Offset (if any)
965  UpdatePTSOffset(true);
966 
967  // Process Video Packet
968  ProcessVideoPacket(requested_frame);
969 
970  if (openshot::Settings::Instance()->WAIT_FOR_VIDEO_PROCESSING_TASK) {
971  // Wait on each OMP task to complete before moving on to the next one. This slows
972  // down processing considerably, but might be more stable on some systems.
973 #pragma omp taskwait
974  }
975  }
976 
977  }
978  // Audio packet
979  else if (info.has_audio && packet->stream_index == audioStream) {
980  // Increment this (to track # of packets since the last video packet)
981  num_packets_since_video_frame++;
982 
983  // Check the status of a seek (if any)
984  if (is_seeking)
985 #pragma omp critical (openshot_seek)
986  check_seek = CheckSeek(false);
987  else
988  check_seek = false;
989 
990  if (check_seek) {
991  // Jump to the next iteration of this loop
992  continue;
993  }
994 
995  // Packet may become NULL on Close inside Seek if CheckSeek returns false
996  if (!packet)
997  // Jump to the next iteration of this loop
998  continue;
999 
1000  // Update PTS / Frame Offset (if any)
1001  UpdatePTSOffset(false);
1002 
1003  // Determine related video frame and starting sample # from audio PTS
1004  AudioLocation location = GetAudioPTSLocation(packet->pts);
1005 
1006  // Process Audio Packet
1007  ProcessAudioPacket(requested_frame, location.frame, location.sample_start);
1008  }
1009 
1010  // Check if working frames are 'finished'
1011  if (!is_seeking) {
1012  // Check for final frames
1013  CheckWorkingFrames(false, requested_frame);
1014  }
1015 
1016  // Check if requested 'final' frame is available
1017  bool is_cache_found = (final_cache.GetFrame(requested_frame) != NULL);
1018 
1019  // Increment frames processed
1020  packets_processed++;
1021 
1022  // Break once the frame is found
1023  if ((is_cache_found && packets_processed >= minimum_packets) || packets_processed > max_packets)
1024  break;
1025 
1026  } // end while
1027 
1028  } // end omp single
1029 
1030  } // end omp parallel
1031 
1032  // Debug output
1033  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream (Completed)", "packets_processed", packets_processed, "end_of_stream", end_of_stream, "largest_frame_processed", largest_frame_processed, "Working Cache Count", working_cache.Count());
1034 
1035  // End of stream?
1036  if (end_of_stream)
1037  // Mark the any other working frames as 'finished'
1038  CheckWorkingFrames(end_of_stream, requested_frame);
1039 
1040  // Return requested frame (if found)
1041  std::shared_ptr<Frame> frame = final_cache.GetFrame(requested_frame);
1042  if (frame)
1043  // Return prepared frame
1044  return frame;
1045  else {
1046 
1047  // Check if largest frame is still cached
1048  frame = final_cache.GetFrame(largest_frame_processed);
1049  if (frame) {
1050  // return the largest processed frame (assuming it was the last in the video file)
1051  return frame;
1052  } else {
1053  // The largest processed frame is no longer in cache, return a blank frame
1054  std::shared_ptr<Frame> f = CreateFrame(largest_frame_processed);
1055  f->AddColor(info.width, info.height, "#000");
1056  return f;
1057  }
1058  }
1059 
1060 }
1061 
1062 // Get the next packet (if any)
1063 int FFmpegReader::GetNextPacket() {
1064  int found_packet = 0;
1065  AVPacket *next_packet;
1066 #pragma omp critical(getnextpacket)
1067  {
1068  next_packet = new AVPacket();
1069  found_packet = av_read_frame(pFormatCtx, next_packet);
1070 
1071 
1072  if (packet) {
1073  // Remove previous packet before getting next one
1074  RemoveAVPacket(packet);
1075  packet = NULL;
1076  }
1077 
1078  if (found_packet >= 0) {
1079  // Update current packet pointer
1080  packet = next_packet;
1081  }
1082  else
1083  delete next_packet;
1084  }
1085  // Return if packet was found (or error number)
1086  return found_packet;
1087 }
1088 
1089 // Get an AVFrame (if any)
1090 bool FFmpegReader::GetAVFrame() {
1091  int frameFinished = -1;
1092  int ret = 0;
1093 
1094  // Decode video frame
1095  AVFrame *next_frame = AV_ALLOCATE_FRAME();
1096 #pragma omp critical (packet_cache)
1097  {
1098 #if IS_FFMPEG_3_2
1099  frameFinished = 0;
1100 
1101  ret = avcodec_send_packet(pCodecCtx, packet);
1102 
1103  // Get the format from the variables set in get_hw_dec_format
1104  hw_de_av_pix_fmt = hw_de_av_pix_fmt_global;
1105  hw_de_av_device_type = hw_de_av_device_type_global;
1106 
1107  if (ret < 0 || ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
1108  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (Packet not sent)");
1109  }
1110  else {
1111  AVFrame *next_frame2;
1112  if (hw_de_on && hw_de_supported) {
1113  next_frame2 = AV_ALLOCATE_FRAME();
1114  }
1115  else
1116  {
1117  next_frame2 = next_frame;
1118  }
1119  pFrame = AV_ALLOCATE_FRAME();
1120  while (ret >= 0) {
1121  ret = avcodec_receive_frame(pCodecCtx, next_frame2);
1122  if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
1123  break;
1124  }
1125  if (ret != 0) {
1126  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (invalid return frame received)");
1127  }
1128  if (hw_de_on && hw_de_supported) {
1129  int err;
1130  if (next_frame2->format == hw_de_av_pix_fmt) {
1131  next_frame->format = AV_PIX_FMT_YUV420P;
1132  if ((err = av_hwframe_transfer_data(next_frame,next_frame2,0)) < 0) {
1133  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (Failed to transfer data to output frame)");
1134  }
1135  if ((err = av_frame_copy_props(next_frame,next_frame2)) < 0) {
1136  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (Failed to copy props to output frame)");
1137  }
1138  }
1139  }
1140  else
1141  { // No hardware acceleration used -> no copy from GPU memory needed
1142  next_frame = next_frame2;
1143  }
1144 
1145  // TODO also handle possible further frames
1146  // Use only the first frame like avcodec_decode_video2
1147  if (frameFinished == 0 ) {
1148  frameFinished = 1;
1149  av_image_alloc(pFrame->data, pFrame->linesize, info.width, info.height, (AVPixelFormat)(pStream->codecpar->format), 1);
1150  av_image_copy(pFrame->data, pFrame->linesize, (const uint8_t**)next_frame->data, next_frame->linesize,
1151  (AVPixelFormat)(pStream->codecpar->format), info.width, info.height);
1152  }
1153  }
1154  if (hw_de_on && hw_de_supported) {
1155  AV_FREE_FRAME(&next_frame2);
1156  }
1157  }
1158 #else
1159  avcodec_decode_video2(pCodecCtx, next_frame, &frameFinished, packet);
1160 
1161  // always allocate pFrame (because we do that in the ffmpeg >= 3.2 as well); it will always be freed later
1162  pFrame = AV_ALLOCATE_FRAME();
1163 
1164  // is frame finished
1165  if (frameFinished) {
1166  // AVFrames are clobbered on the each call to avcodec_decode_video, so we
1167  // must make a copy of the image data before this method is called again.
1168  avpicture_alloc((AVPicture *) pFrame, pCodecCtx->pix_fmt, info.width, info.height);
1169  av_picture_copy((AVPicture *) pFrame, (AVPicture *) next_frame, pCodecCtx->pix_fmt, info.width,
1170  info.height);
1171  }
1172 #endif
1173  }
1174 
1175  // deallocate the frame
1176  AV_FREE_FRAME(&next_frame);
1177 
1178  // Did we get a video frame?
1179  return frameFinished;
1180 }
1181 
1182 // Check the current seek position and determine if we need to seek again
1183 bool FFmpegReader::CheckSeek(bool is_video) {
1184  // Are we seeking for a specific frame?
1185  if (is_seeking) {
1186  // Determine if both an audio and video packet have been decoded since the seek happened.
1187  // If not, allow the ReadStream method to keep looping
1188  if ((is_video_seek && !seek_video_frame_found) || (!is_video_seek && !seek_audio_frame_found))
1189  return false;
1190 
1191  // Check for both streams
1192  if ((info.has_video && !seek_video_frame_found) || (info.has_audio && !seek_audio_frame_found))
1193  return false;
1194 
1195  // Determine max seeked frame
1196  int64_t max_seeked_frame = seek_audio_frame_found; // determine max seeked frame
1197  if (seek_video_frame_found > max_seeked_frame)
1198  max_seeked_frame = seek_video_frame_found;
1199 
1200  // determine if we are "before" the requested frame
1201  if (max_seeked_frame >= seeking_frame) {
1202  // SEEKED TOO FAR
1203  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckSeek (Too far, seek again)", "is_video_seek", is_video_seek, "max_seeked_frame", max_seeked_frame, "seeking_frame", seeking_frame, "seeking_pts", seeking_pts, "seek_video_frame_found", seek_video_frame_found, "seek_audio_frame_found", seek_audio_frame_found);
1204 
1205  // Seek again... to the nearest Keyframe
1206  Seek(seeking_frame - (10 * seek_count * seek_count));
1207  } else {
1208  // SEEK WORKED
1209  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckSeek (Successful)", "is_video_seek", is_video_seek, "current_pts", packet->pts, "seeking_pts", seeking_pts, "seeking_frame", seeking_frame, "seek_video_frame_found", seek_video_frame_found, "seek_audio_frame_found", seek_audio_frame_found);
1210 
1211  // Seek worked, and we are "before" the requested frame
1212  is_seeking = false;
1213  seeking_frame = 0;
1214  seeking_pts = -1;
1215  }
1216  }
1217 
1218  // return the pts to seek to (if any)
1219  return is_seeking;
1220 }
1221 
1222 // Process a video packet
1223 void FFmpegReader::ProcessVideoPacket(int64_t requested_frame) {
1224  // Calculate current frame #
1225  int64_t current_frame = ConvertVideoPTStoFrame(GetVideoPTS());
1226 
1227  // Track 1st video packet after a successful seek
1228  if (!seek_video_frame_found && is_seeking)
1229  seek_video_frame_found = current_frame;
1230 
1231  // Are we close enough to decode the frame? and is this frame # valid?
1232  if ((current_frame < (requested_frame - 20)) or (current_frame == -1)) {
1233  // Remove frame and packet
1234  RemoveAVFrame(pFrame);
1235 
1236  // Debug output
1237  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessVideoPacket (Skipped)", "requested_frame", requested_frame, "current_frame", current_frame);
1238 
1239  // Skip to next frame without decoding or caching
1240  return;
1241  }
1242 
1243  // Debug output
1244  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessVideoPacket (Before)", "requested_frame", requested_frame, "current_frame", current_frame);
1245 
1246  // Init some things local (for OpenMP)
1247  PixelFormat pix_fmt = AV_GET_CODEC_PIXEL_FORMAT(pStream, pCodecCtx);
1248  int height = info.height;
1249  int width = info.width;
1250  int64_t video_length = info.video_length;
1251  AVFrame *my_frame = pFrame;
1252  pFrame = NULL;
1253 
1254  // Add video frame to list of processing video frames
1255  const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
1256  processing_video_frames[current_frame] = current_frame;
1257 
1258 #pragma omp task firstprivate(current_frame, my_frame, height, width, video_length, pix_fmt)
1259  {
1260  // Create variables for a RGB Frame (since most videos are not in RGB, we must convert it)
1261  AVFrame *pFrameRGB = NULL;
1262  int numBytes;
1263  uint8_t *buffer = NULL;
1264 
1265  // Allocate an AVFrame structure
1266  pFrameRGB = AV_ALLOCATE_FRAME();
1267  if (pFrameRGB == NULL)
1268  throw OutOfBoundsFrame("Convert Image Broke!", current_frame, video_length);
1269 
1270  // Determine the max size of this source image (based on the timeline's size, the scaling mode,
1271  // and the scaling keyframes). This is a performance improvement, to keep the images as small as possible,
1272  // without losing quality. NOTE: We cannot go smaller than the timeline itself, or the add_layer timeline
1273  // method will scale it back to timeline size before scaling it smaller again. This needs to be fixed in
1274  // the future.
1275  int max_width = openshot::Settings::Instance()->MAX_WIDTH;
1276  if (max_width <= 0)
1277  max_width = info.width;
1278  int max_height = openshot::Settings::Instance()->MAX_HEIGHT;
1279  if (max_height <= 0)
1280  max_height = info.height;
1281 
1282  Clip *parent = (Clip *) GetClip();
1283  if (parent) {
1284  if (parent->scale == SCALE_FIT || parent->scale == SCALE_STRETCH) {
1285  // Best fit or Stretch scaling (based on max timeline size * scaling keyframes)
1286  float max_scale_x = parent->scale_x.GetMaxPoint().co.Y;
1287  float max_scale_y = parent->scale_y.GetMaxPoint().co.Y;
1288  max_width = std::max(float(max_width), max_width * max_scale_x);
1289  max_height = std::max(float(max_height), max_height * max_scale_y);
1290 
1291  } else if (parent->scale == SCALE_CROP) {
1292  // Cropping scale mode (based on max timeline size * cropped size * scaling keyframes)
1293  float max_scale_x = parent->scale_x.GetMaxPoint().co.Y;
1294  float max_scale_y = parent->scale_y.GetMaxPoint().co.Y;
1295  QSize width_size(max_width * max_scale_x,
1296  round(max_width / (float(info.width) / float(info.height))));
1297  QSize height_size(round(max_height / (float(info.height) / float(info.width))),
1298  max_height * max_scale_y);
1299  // respect aspect ratio
1300  if (width_size.width() >= max_width && width_size.height() >= max_height) {
1301  max_width = std::max(max_width, width_size.width());
1302  max_height = std::max(max_height, width_size.height());
1303  } else {
1304  max_width = std::max(max_width, height_size.width());
1305  max_height = std::max(max_height, height_size.height());
1306  }
1307 
1308  } else {
1309  // No scaling, use original image size (slower)
1310  max_width = info.width;
1311  max_height = info.height;
1312  }
1313  }
1314 
1315  // Determine if image needs to be scaled (for performance reasons)
1316  int original_height = height;
1317  if (max_width != 0 && max_height != 0 && max_width < width && max_height < height) {
1318  // Override width and height (but maintain aspect ratio)
1319  float ratio = float(width) / float(height);
1320  int possible_width = round(max_height * ratio);
1321  int possible_height = round(max_width / ratio);
1322 
1323  if (possible_width <= max_width) {
1324  // use calculated width, and max_height
1325  width = possible_width;
1326  height = max_height;
1327  } else {
1328  // use max_width, and calculated height
1329  width = max_width;
1330  height = possible_height;
1331  }
1332  }
1333 
1334  // Determine required buffer size and allocate buffer
1335  numBytes = AV_GET_IMAGE_SIZE(PIX_FMT_RGBA, width, height);
1336 
1337 #pragma omp critical (video_buffer)
1338  buffer = (uint8_t *) av_malloc(numBytes * sizeof(uint8_t));
1339 
1340  // Copy picture data from one AVFrame (or AVPicture) to another one.
1341  AV_COPY_PICTURE_DATA(pFrameRGB, buffer, PIX_FMT_RGBA, width, height);
1342 
1343  int scale_mode = SWS_FAST_BILINEAR;
1344  if (openshot::Settings::Instance()->HIGH_QUALITY_SCALING) {
1345  scale_mode = SWS_BICUBIC;
1346  }
1347  SwsContext *img_convert_ctx = sws_getContext(info.width, info.height, AV_GET_CODEC_PIXEL_FORMAT(pStream, pCodecCtx), width,
1348  height, PIX_FMT_RGBA, scale_mode, NULL, NULL, NULL);
1349 
1350  // Resize / Convert to RGB
1351  sws_scale(img_convert_ctx, my_frame->data, my_frame->linesize, 0,
1352  original_height, pFrameRGB->data, pFrameRGB->linesize);
1353 
1354  // Create or get the existing frame object
1355  std::shared_ptr<Frame> f = CreateFrame(current_frame);
1356 
1357  // Add Image data to frame
1358  f->AddImage(width, height, 4, QImage::Format_RGBA8888, buffer);
1359 
1360  // Update working cache
1361  working_cache.Add(f);
1362 
1363  // Keep track of last last_video_frame
1364 #pragma omp critical (video_buffer)
1365  last_video_frame = f;
1366 
1367  // Free the RGB image
1368  av_free(buffer);
1369  AV_FREE_FRAME(&pFrameRGB);
1370 
1371  // Remove frame and packet
1372  RemoveAVFrame(my_frame);
1373  sws_freeContext(img_convert_ctx);
1374 
1375  // Remove video frame from list of processing video frames
1376  {
1377  const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
1378  processing_video_frames.erase(current_frame);
1379  processed_video_frames[current_frame] = current_frame;
1380  }
1381 
1382  // Debug output
1383  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessVideoPacket (After)", "requested_frame", requested_frame, "current_frame", current_frame, "f->number", f->number);
1384 
1385  } // end omp task
1386 
1387 }
1388 
1389 // Process an audio packet
1390 void FFmpegReader::ProcessAudioPacket(int64_t requested_frame, int64_t target_frame, int starting_sample) {
1391  // Track 1st audio packet after a successful seek
1392  if (!seek_audio_frame_found && is_seeking)
1393  seek_audio_frame_found = target_frame;
1394 
1395  // Are we close enough to decode the frame's audio?
1396  if (target_frame < (requested_frame - 20)) {
1397  // Debug output
1398  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (Skipped)", "requested_frame", requested_frame, "target_frame", target_frame, "starting_sample", starting_sample);
1399 
1400  // Skip to next frame without decoding or caching
1401  return;
1402  }
1403 
1404  // Debug output
1405  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (Before)", "requested_frame", requested_frame, "target_frame", target_frame, "starting_sample", starting_sample);
1406 
1407  // Init an AVFrame to hold the decoded audio samples
1408  int frame_finished = 0;
1409  AVFrame *audio_frame = AV_ALLOCATE_FRAME();
1410  AV_RESET_FRAME(audio_frame);
1411 
1412  int packet_samples = 0;
1413  int data_size = 0;
1414 
1415  // re-initialize buffer size (it gets changed in the avcodec_decode_audio2 method call)
1417 #pragma omp critical (ProcessAudioPacket)
1418  {
1419 #if IS_FFMPEG_3_2
1420  int ret = 0;
1421  frame_finished = 1;
1422  while((packet->size > 0 || (!packet->data && frame_finished)) && ret >= 0) {
1423  frame_finished = 0;
1424  ret = avcodec_send_packet(aCodecCtx, packet);
1425  if (ret < 0 && ret != AVERROR(EINVAL) && ret != AVERROR_EOF) {
1426  avcodec_send_packet(aCodecCtx, NULL);
1427  break;
1428  }
1429  if (ret >= 0)
1430  packet->size = 0;
1431  ret = avcodec_receive_frame(aCodecCtx, audio_frame);
1432  if (ret >= 0)
1433  frame_finished = 1;
1434  if(ret == AVERROR(EINVAL) || ret == AVERROR_EOF) {
1435  avcodec_flush_buffers(aCodecCtx);
1436  ret = 0;
1437  }
1438  if (ret >= 0) {
1439  ret = frame_finished;
1440  }
1441  }
1442  if (!packet->data && !frame_finished)
1443  {
1444  ret = -1;
1445  }
1446 #else
1447  int used = avcodec_decode_audio4(aCodecCtx, audio_frame, &frame_finished, packet);
1448 #endif
1449  }
1450 
1451  if (frame_finished) {
1452 
1453  // determine how many samples were decoded
1454  int planar = av_sample_fmt_is_planar((AVSampleFormat) AV_GET_CODEC_PIXEL_FORMAT(aStream, aCodecCtx));
1455  int plane_size = -1;
1456  data_size = av_samples_get_buffer_size(&plane_size,
1457  AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels,
1458  audio_frame->nb_samples,
1459  (AVSampleFormat) (AV_GET_SAMPLE_FORMAT(aStream, aCodecCtx)), 1);
1460 
1461  // Calculate total number of samples
1462  packet_samples = audio_frame->nb_samples * AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels;
1463  }
1464 
1465  // Estimate the # of samples and the end of this packet's location (to prevent GAPS for the next timestamp)
1466  int pts_remaining_samples = packet_samples / info.channels; // Adjust for zero based array
1467 
1468  // DEBUG (FOR AUDIO ISSUES) - Get the audio packet start time (in seconds)
1469  int64_t adjusted_pts = packet->pts + audio_pts_offset;
1470  double audio_seconds = double(adjusted_pts) * info.audio_timebase.ToDouble();
1471  double sample_seconds = double(pts_total) / info.sample_rate;
1472 
1473  // Debug output
1474  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (Decode Info A)", "pts_counter", pts_counter, "PTS", adjusted_pts, "Offset", audio_pts_offset, "PTS Diff", adjusted_pts - prev_pts, "Samples", pts_remaining_samples, "Sample PTS ratio", float(adjusted_pts - prev_pts) / pts_remaining_samples);
1475  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (Decode Info B)", "Sample Diff", pts_remaining_samples - prev_samples - prev_pts, "Total", pts_total, "PTS Seconds", audio_seconds, "Sample Seconds", sample_seconds, "Seconds Diff", audio_seconds - sample_seconds, "raw samples", packet_samples);
1476 
1477  // DEBUG (FOR AUDIO ISSUES)
1478  prev_pts = adjusted_pts;
1479  pts_total += pts_remaining_samples;
1480  pts_counter++;
1481  prev_samples = pts_remaining_samples;
1482 
1483  // Add audio frame to list of processing audio frames
1484  {
1485  const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
1486  processing_audio_frames.insert(std::pair<int, int>(previous_packet_location.frame, previous_packet_location.frame));
1487  }
1488 
1489  while (pts_remaining_samples) {
1490  // Get Samples per frame (for this frame number)
1491  int samples_per_frame = Frame::GetSamplesPerFrame(previous_packet_location.frame, info.fps, info.sample_rate, info.channels);
1492 
1493  // Calculate # of samples to add to this frame
1494  int samples = samples_per_frame - previous_packet_location.sample_start;
1495  if (samples > pts_remaining_samples)
1496  samples = pts_remaining_samples;
1497 
1498  // Decrement remaining samples
1499  pts_remaining_samples -= samples;
1500 
1501  if (pts_remaining_samples > 0) {
1502  // next frame
1503  previous_packet_location.frame++;
1504  previous_packet_location.sample_start = 0;
1505 
1506  // Add audio frame to list of processing audio frames
1507  {
1508  const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
1509  processing_audio_frames.insert(std::pair<int, int>(previous_packet_location.frame, previous_packet_location.frame));
1510  }
1511 
1512  } else {
1513  // Increment sample start
1514  previous_packet_location.sample_start += samples;
1515  }
1516  }
1517 
1518 
1519  // Allocate audio buffer
1520  int16_t *audio_buf = new int16_t[AVCODEC_MAX_AUDIO_FRAME_SIZE + MY_INPUT_BUFFER_PADDING_SIZE];
1521 
1522  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (ReSample)", "packet_samples", packet_samples, "info.channels", info.channels, "info.sample_rate", info.sample_rate, "aCodecCtx->sample_fmt", AV_GET_SAMPLE_FORMAT(aStream, aCodecCtx), "AV_SAMPLE_FMT_S16", AV_SAMPLE_FMT_S16);
1523 
1524  // Create output frame
1525  AVFrame *audio_converted = AV_ALLOCATE_FRAME();
1526  AV_RESET_FRAME(audio_converted);
1527  audio_converted->nb_samples = audio_frame->nb_samples;
1528  av_samples_alloc(audio_converted->data, audio_converted->linesize, info.channels, audio_frame->nb_samples, AV_SAMPLE_FMT_S16, 0);
1529 
1530  SWRCONTEXT *avr = NULL;
1531  int nb_samples = 0;
1532 
1533  // setup resample context
1534  avr = SWR_ALLOC();
1535  av_opt_set_int(avr, "in_channel_layout", AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout, 0);
1536  av_opt_set_int(avr, "out_channel_layout", AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout, 0);
1537  av_opt_set_int(avr, "in_sample_fmt", AV_GET_SAMPLE_FORMAT(aStream, aCodecCtx), 0);
1538  av_opt_set_int(avr, "out_sample_fmt", AV_SAMPLE_FMT_S16, 0);
1539  av_opt_set_int(avr, "in_sample_rate", info.sample_rate, 0);
1540  av_opt_set_int(avr, "out_sample_rate", info.sample_rate, 0);
1541  av_opt_set_int(avr, "in_channels", info.channels, 0);
1542  av_opt_set_int(avr, "out_channels", info.channels, 0);
1543  int r = SWR_INIT(avr);
1544 
1545  // Convert audio samples
1546  nb_samples = SWR_CONVERT(avr, // audio resample context
1547  audio_converted->data, // output data pointers
1548  audio_converted->linesize[0], // output plane size, in bytes. (0 if unknown)
1549  audio_converted->nb_samples, // maximum number of samples that the output buffer can hold
1550  audio_frame->data, // input data pointers
1551  audio_frame->linesize[0], // input plane size, in bytes (0 if unknown)
1552  audio_frame->nb_samples); // number of input samples to convert
1553 
1554  // Copy audio samples over original samples
1555  memcpy(audio_buf, audio_converted->data[0], audio_converted->nb_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) * info.channels);
1556 
1557  // Deallocate resample buffer
1558  SWR_CLOSE(avr);
1559  SWR_FREE(&avr);
1560  avr = NULL;
1561 
1562  // Free AVFrames
1563  av_free(audio_converted->data[0]);
1564  AV_FREE_FRAME(&audio_converted);
1565 
1566  int64_t starting_frame_number = -1;
1567  bool partial_frame = true;
1568  for (int channel_filter = 0; channel_filter < info.channels; channel_filter++) {
1569  // Array of floats (to hold samples for each channel)
1570  starting_frame_number = target_frame;
1571  int channel_buffer_size = packet_samples / info.channels;
1572  float *channel_buffer = new float[channel_buffer_size];
1573 
1574  // Init buffer array
1575  for (int z = 0; z < channel_buffer_size; z++)
1576  channel_buffer[z] = 0.0f;
1577 
1578  // Loop through all samples and add them to our Frame based on channel.
1579  // Toggle through each channel number, since channel data is stored like (left right left right)
1580  int channel = 0;
1581  int position = 0;
1582  for (int sample = 0; sample < packet_samples; sample++) {
1583  // Only add samples for current channel
1584  if (channel_filter == channel) {
1585  // Add sample (convert from (-32768 to 32768) to (-1.0 to 1.0))
1586  channel_buffer[position] = audio_buf[sample] * (1.0f / (1 << 15));
1587 
1588  // Increment audio position
1589  position++;
1590  }
1591 
1592  // increment channel (if needed)
1593  if ((channel + 1) < info.channels)
1594  // move to next channel
1595  channel++;
1596  else
1597  // reset channel
1598  channel = 0;
1599  }
1600 
1601  // Loop through samples, and add them to the correct frames
1602  int start = starting_sample;
1603  int remaining_samples = channel_buffer_size;
1604  float *iterate_channel_buffer = channel_buffer; // pointer to channel buffer
1605  while (remaining_samples > 0) {
1606  // Get Samples per frame (for this frame number)
1607  int samples_per_frame = Frame::GetSamplesPerFrame(starting_frame_number, info.fps, info.sample_rate, info.channels);
1608 
1609  // Calculate # of samples to add to this frame
1610  int samples = samples_per_frame - start;
1611  if (samples > remaining_samples)
1612  samples = remaining_samples;
1613 
1614  // Create or get the existing frame object
1615  std::shared_ptr<Frame> f = CreateFrame(starting_frame_number);
1616 
1617  // Determine if this frame was "partially" filled in
1618  if (samples_per_frame == start + samples)
1619  partial_frame = false;
1620  else
1621  partial_frame = true;
1622 
1623  // Add samples for current channel to the frame. Reduce the volume to 98%, to prevent
1624  // some louder samples from maxing out at 1.0 (not sure why this happens)
1625  f->AddAudio(true, channel_filter, start, iterate_channel_buffer, samples, 0.98f);
1626 
1627  // Debug output
1628  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (f->AddAudio)", "frame", starting_frame_number, "start", start, "samples", samples, "channel", channel_filter, "partial_frame", partial_frame, "samples_per_frame", samples_per_frame);
1629 
1630  // Add or update cache
1631  working_cache.Add(f);
1632 
1633  // Decrement remaining samples
1634  remaining_samples -= samples;
1635 
1636  // Increment buffer (to next set of samples)
1637  if (remaining_samples > 0)
1638  iterate_channel_buffer += samples;
1639 
1640  // Increment frame number
1641  starting_frame_number++;
1642 
1643  // Reset starting sample #
1644  start = 0;
1645  }
1646 
1647  // clear channel buffer
1648  delete[] channel_buffer;
1649  channel_buffer = NULL;
1650  iterate_channel_buffer = NULL;
1651  }
1652 
1653  // Clean up some arrays
1654  delete[] audio_buf;
1655  audio_buf = NULL;
1656 
1657  // Remove audio frame from list of processing audio frames
1658  {
1659  const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
1660  // Update all frames as completed
1661  for (int64_t f = target_frame; f < starting_frame_number; f++) {
1662  // Remove the frame # from the processing list. NOTE: If more than one thread is
1663  // processing this frame, the frame # will be in this list multiple times. We are only
1664  // removing a single instance of it here.
1665  processing_audio_frames.erase(processing_audio_frames.find(f));
1666 
1667  // Check and see if this frame is also being processed by another thread
1668  if (processing_audio_frames.count(f) == 0)
1669  // No other thread is processing it. Mark the audio as processed (final)
1670  processed_audio_frames[f] = f;
1671  }
1672 
1673  if (target_frame == starting_frame_number) {
1674  // This typically never happens, but just in case, remove the currently processing number
1675  processing_audio_frames.erase(processing_audio_frames.find(target_frame));
1676  }
1677  }
1678 
1679  // Free audio frame
1680  AV_FREE_FRAME(&audio_frame);
1681 
1682  // Debug output
1683  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (After)", "requested_frame", requested_frame, "starting_frame", target_frame, "end_frame", starting_frame_number - 1);
1684 
1685 }
1686 
1687 
1688 // Seek to a specific frame. This is not always frame accurate, it's more of an estimation on many codecs.
1689 void FFmpegReader::Seek(int64_t requested_frame) {
1690  // Adjust for a requested frame that is too small or too large
1691  if (requested_frame < 1)
1692  requested_frame = 1;
1693  if (requested_frame > info.video_length)
1694  requested_frame = info.video_length;
1695 
1696  int processing_video_frames_size = 0;
1697  int processing_audio_frames_size = 0;
1698  {
1699  const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
1700  processing_video_frames_size = processing_video_frames.size();
1701  processing_audio_frames_size = processing_audio_frames.size();
1702  }
1703 
1704  // Debug output
1705  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::Seek", "requested_frame", requested_frame, "seek_count", seek_count, "last_frame", last_frame, "processing_video_frames_size", processing_video_frames_size, "processing_audio_frames_size", processing_audio_frames_size, "video_pts_offset", video_pts_offset);
1706 
1707  // Wait for any processing frames to complete
1708  while (processing_video_frames_size + processing_audio_frames_size > 0) {
1709  usleep(2500);
1710  const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
1711  processing_video_frames_size = processing_video_frames.size();
1712  processing_audio_frames_size = processing_audio_frames.size();
1713  }
1714 
1715  // Clear working cache (since we are seeking to another location in the file)
1716  working_cache.Clear();
1717  missing_frames.Clear();
1718 
1719  // Clear processed lists
1720  {
1721  const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
1722  processing_audio_frames.clear();
1723  processing_video_frames.clear();
1724  processed_video_frames.clear();
1725  processed_audio_frames.clear();
1726  missing_audio_frames.clear();
1727  missing_video_frames.clear();
1728  missing_audio_frames_source.clear();
1729  missing_video_frames_source.clear();
1730  checked_frames.clear();
1731  }
1732 
1733  // Reset the last frame variable
1734  last_frame = 0;
1735  current_video_frame = 0;
1736  largest_frame_processed = 0;
1737  num_checks_since_final = 0;
1738  num_packets_since_video_frame = 0;
1739  has_missing_frames = false;
1740  bool has_audio_override = info.has_audio;
1741  bool has_video_override = info.has_video;
1742 
1743  // Increment seek count
1744  seek_count++;
1745 
1746  // If seeking near frame 1, we need to close and re-open the file (this is more reliable than seeking)
1747  int buffer_amount = std::max(OPEN_MP_NUM_PROCESSORS, 8);
1748  if (requested_frame - buffer_amount < 20) {
1749  // Close and re-open file (basically seeking to frame 1)
1750  Close();
1751  Open();
1752 
1753  // Update overrides (since closing and re-opening might update these)
1754  info.has_audio = has_audio_override;
1755  info.has_video = has_video_override;
1756 
1757  // Not actually seeking, so clear these flags
1758  is_seeking = false;
1759  if (seek_count == 1) {
1760  // Don't redefine this on multiple seek attempts for a specific frame
1761  seeking_frame = 1;
1762  seeking_pts = ConvertFrameToVideoPTS(1);
1763  }
1764  seek_audio_frame_found = 0; // used to detect which frames to throw away after a seek
1765  seek_video_frame_found = 0; // used to detect which frames to throw away after a seek
1766 
1767  } else {
1768  // Seek to nearest key-frame (aka, i-frame)
1769  bool seek_worked = false;
1770  int64_t seek_target = 0;
1771 
1772  // Seek video stream (if any)
1773  if (!seek_worked && info.has_video) {
1774  seek_target = ConvertFrameToVideoPTS(requested_frame - buffer_amount);
1775  if (av_seek_frame(pFormatCtx, info.video_stream_index, seek_target, AVSEEK_FLAG_BACKWARD) < 0) {
1776  fprintf(stderr, "%s: error while seeking video stream\n", pFormatCtx->AV_FILENAME);
1777  } else {
1778  // VIDEO SEEK
1779  is_video_seek = true;
1780  seek_worked = true;
1781  }
1782  }
1783 
1784  // Seek audio stream (if not already seeked... and if an audio stream is found)
1785  if (!seek_worked && info.has_audio) {
1786  seek_target = ConvertFrameToAudioPTS(requested_frame - buffer_amount);
1787  if (av_seek_frame(pFormatCtx, info.audio_stream_index, seek_target, AVSEEK_FLAG_BACKWARD) < 0) {
1788  fprintf(stderr, "%s: error while seeking audio stream\n", pFormatCtx->AV_FILENAME);
1789  } else {
1790  // AUDIO SEEK
1791  is_video_seek = false;
1792  seek_worked = true;
1793  }
1794  }
1795 
1796  // Was the seek successful?
1797  if (seek_worked) {
1798  // Flush audio buffer
1799  if (info.has_audio)
1800  avcodec_flush_buffers(aCodecCtx);
1801 
1802  // Flush video buffer
1803  if (info.has_video)
1804  avcodec_flush_buffers(pCodecCtx);
1805 
1806  // Reset previous audio location to zero
1807  previous_packet_location.frame = -1;
1808  previous_packet_location.sample_start = 0;
1809 
1810  // init seek flags
1811  is_seeking = true;
1812  if (seek_count == 1) {
1813  // Don't redefine this on multiple seek attempts for a specific frame
1814  seeking_pts = seek_target;
1815  seeking_frame = requested_frame;
1816  }
1817  seek_audio_frame_found = 0; // used to detect which frames to throw away after a seek
1818  seek_video_frame_found = 0; // used to detect which frames to throw away after a seek
1819 
1820  } else {
1821  // seek failed
1822  is_seeking = false;
1823  seeking_pts = 0;
1824  seeking_frame = 0;
1825 
1826  // dislable seeking for this reader (since it failed)
1827  // TODO: Find a safer way to do this... not sure how common it is for a seek to fail.
1828  enable_seek = false;
1829 
1830  // Close and re-open file (basically seeking to frame 1)
1831  Close();
1832  Open();
1833 
1834  // Update overrides (since closing and re-opening might update these)
1835  info.has_audio = has_audio_override;
1836  info.has_video = has_video_override;
1837  }
1838  }
1839 }
1840 
1841 // Get the PTS for the current video packet
1842 int64_t FFmpegReader::GetVideoPTS() {
1843  int64_t current_pts = 0;
1844  if (packet->dts != AV_NOPTS_VALUE)
1845  current_pts = packet->dts;
1846 
1847  // Return adjusted PTS
1848  return current_pts;
1849 }
1850 
1851 // Update PTS Offset (if any)
1852 void FFmpegReader::UpdatePTSOffset(bool is_video) {
1853  // Determine the offset between the PTS and Frame number (only for 1st frame)
1854  if (is_video) {
1855  // VIDEO PACKET
1856  if (video_pts_offset == 99999) // Has the offset been set yet?
1857  {
1858  // Find the difference between PTS and frame number (no more than 10 timebase units allowed)
1859  video_pts_offset = 0 - std::max(GetVideoPTS(), (int64_t) info.video_timebase.ToInt() * 10);
1860 
1861  // debug output
1862  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::UpdatePTSOffset (Video)", "video_pts_offset", video_pts_offset, "is_video", is_video);
1863  }
1864  } else {
1865  // AUDIO PACKET
1866  if (audio_pts_offset == 99999) // Has the offset been set yet?
1867  {
1868  // Find the difference between PTS and frame number (no more than 10 timebase units allowed)
1869  audio_pts_offset = 0 - std::max(packet->pts, (int64_t) info.audio_timebase.ToInt() * 10);
1870 
1871  // debug output
1872  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::UpdatePTSOffset (Audio)", "audio_pts_offset", audio_pts_offset, "is_video", is_video);
1873  }
1874  }
1875 }
1876 
1877 // Convert PTS into Frame Number
1878 int64_t FFmpegReader::ConvertVideoPTStoFrame(int64_t pts) {
1879  // Apply PTS offset
1880  pts = pts + video_pts_offset;
1881  int64_t previous_video_frame = current_video_frame;
1882 
1883  // Get the video packet start time (in seconds)
1884  double video_seconds = double(pts) * info.video_timebase.ToDouble();
1885 
1886  // Divide by the video timebase, to get the video frame number (frame # is decimal at this point)
1887  int64_t frame = round(video_seconds * info.fps.ToDouble()) + 1;
1888 
1889  // Keep track of the expected video frame #
1890  if (current_video_frame == 0)
1891  current_video_frame = frame;
1892  else {
1893 
1894  // Sometimes frames are duplicated due to identical (or similar) timestamps
1895  if (frame == previous_video_frame) {
1896  // return -1 frame number
1897  frame = -1;
1898  } else {
1899  // Increment expected frame
1900  current_video_frame++;
1901  }
1902 
1903  if (current_video_frame < frame)
1904  // has missing frames
1905  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ConvertVideoPTStoFrame (detected missing frame)", "calculated frame", frame, "previous_video_frame", previous_video_frame, "current_video_frame", current_video_frame);
1906 
1907  // Sometimes frames are missing due to varying timestamps, or they were dropped. Determine
1908  // if we are missing a video frame.
1909  const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
1910  while (current_video_frame < frame) {
1911  if (!missing_video_frames.count(current_video_frame)) {
1912  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ConvertVideoPTStoFrame (tracking missing frame)", "current_video_frame", current_video_frame, "previous_video_frame", previous_video_frame);
1913  missing_video_frames.insert(std::pair<int64_t, int64_t>(current_video_frame, previous_video_frame));
1914  missing_video_frames_source.insert(std::pair<int64_t, int64_t>(previous_video_frame, current_video_frame));
1915  }
1916 
1917  // Mark this reader as containing missing frames
1918  has_missing_frames = true;
1919 
1920  // Increment current frame
1921  current_video_frame++;
1922  }
1923  }
1924 
1925  // Return frame #
1926  return frame;
1927 }
1928 
1929 // Convert Frame Number into Video PTS
1930 int64_t FFmpegReader::ConvertFrameToVideoPTS(int64_t frame_number) {
1931  // Get timestamp of this frame (in seconds)
1932  double seconds = double(frame_number) / info.fps.ToDouble();
1933 
1934  // Calculate the # of video packets in this timestamp
1935  int64_t video_pts = round(seconds / info.video_timebase.ToDouble());
1936 
1937  // Apply PTS offset (opposite)
1938  return video_pts - video_pts_offset;
1939 }
1940 
1941 // Convert Frame Number into Video PTS
1942 int64_t FFmpegReader::ConvertFrameToAudioPTS(int64_t frame_number) {
1943  // Get timestamp of this frame (in seconds)
1944  double seconds = double(frame_number) / info.fps.ToDouble();
1945 
1946  // Calculate the # of audio packets in this timestamp
1947  int64_t audio_pts = round(seconds / info.audio_timebase.ToDouble());
1948 
1949  // Apply PTS offset (opposite)
1950  return audio_pts - audio_pts_offset;
1951 }
1952 
1953 // Calculate Starting video frame and sample # for an audio PTS
1954 AudioLocation FFmpegReader::GetAudioPTSLocation(int64_t pts) {
1955  // Apply PTS offset
1956  pts = pts + audio_pts_offset;
1957 
1958  // Get the audio packet start time (in seconds)
1959  double audio_seconds = double(pts) * info.audio_timebase.ToDouble();
1960 
1961  // Divide by the video timebase, to get the video frame number (frame # is decimal at this point)
1962  double frame = (audio_seconds * info.fps.ToDouble()) + 1;
1963 
1964  // Frame # as a whole number (no more decimals)
1965  int64_t whole_frame = int64_t(frame);
1966 
1967  // Remove the whole number, and only get the decimal of the frame
1968  double sample_start_percentage = frame - double(whole_frame);
1969 
1970  // Get Samples per frame
1971  int samples_per_frame = Frame::GetSamplesPerFrame(whole_frame, info.fps, info.sample_rate, info.channels);
1972 
1973  // Calculate the sample # to start on
1974  int sample_start = round(double(samples_per_frame) * sample_start_percentage);
1975 
1976  // Protect against broken (i.e. negative) timestamps
1977  if (whole_frame < 1)
1978  whole_frame = 1;
1979  if (sample_start < 0)
1980  sample_start = 0;
1981 
1982  // Prepare final audio packet location
1983  AudioLocation location = {whole_frame, sample_start};
1984 
1985  // Compare to previous audio packet (and fix small gaps due to varying PTS timestamps)
1986  if (previous_packet_location.frame != -1) {
1987  if (location.is_near(previous_packet_location, samples_per_frame, samples_per_frame)) {
1988  int64_t orig_frame = location.frame;
1989  int orig_start = location.sample_start;
1990 
1991  // Update sample start, to prevent gaps in audio
1992  location.sample_start = previous_packet_location.sample_start;
1993  location.frame = previous_packet_location.frame;
1994 
1995  // Debug output
1996  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAudioPTSLocation (Audio Gap Detected)", "Source Frame", orig_frame, "Source Audio Sample", orig_start, "Target Frame", location.frame, "Target Audio Sample", location.sample_start, "pts", pts);
1997 
1998  } else {
1999  // Debug output
2000  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAudioPTSLocation (Audio Gap Ignored - too big)", "Previous location frame", previous_packet_location.frame, "Target Frame", location.frame, "Target Audio Sample", location.sample_start, "pts", pts);
2001 
2002  const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
2003  for (int64_t audio_frame = previous_packet_location.frame; audio_frame < location.frame; audio_frame++) {
2004  if (!missing_audio_frames.count(audio_frame)) {
2005  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAudioPTSLocation (tracking missing frame)", "missing_audio_frame", audio_frame, "previous_audio_frame", previous_packet_location.frame, "new location frame", location.frame);
2006  missing_audio_frames.insert(std::pair<int64_t, int64_t>(audio_frame, previous_packet_location.frame - 1));
2007  }
2008  }
2009  }
2010  }
2011 
2012  // Set previous location
2013  previous_packet_location = location;
2014 
2015  // Return the associated video frame and starting sample #
2016  return location;
2017 }
2018 
2019 // Create a new Frame (or return an existing one) and add it to the working queue.
2020 std::shared_ptr<Frame> FFmpegReader::CreateFrame(int64_t requested_frame) {
2021  // Check working cache
2022  std::shared_ptr<Frame> output = working_cache.GetFrame(requested_frame);
2023 
2024  if (!output) {
2025  // Lock
2026  const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
2027 
2028  // (re-)Check working cache
2029  output = working_cache.GetFrame(requested_frame);
2030  if(output) return output;
2031 
2032  // Create a new frame on the working cache
2033  output = std::make_shared<Frame>(requested_frame, info.width, info.height, "#000000", Frame::GetSamplesPerFrame(requested_frame, info.fps, info.sample_rate, info.channels), info.channels);
2034  output->SetPixelRatio(info.pixel_ratio.num, info.pixel_ratio.den); // update pixel ratio
2035  output->ChannelsLayout(info.channel_layout); // update audio channel layout from the parent reader
2036  output->SampleRate(info.sample_rate); // update the frame's sample rate of the parent reader
2037 
2038  working_cache.Add(output);
2039 
2040  // Set the largest processed frame (if this is larger)
2041  if (requested_frame > largest_frame_processed)
2042  largest_frame_processed = requested_frame;
2043  }
2044  // Return frame
2045  return output;
2046 }
2047 
2048 // Determine if frame is partial due to seek
2049 bool FFmpegReader::IsPartialFrame(int64_t requested_frame) {
2050 
2051  // Sometimes a seek gets partial frames, and we need to remove them
2052  bool seek_trash = false;
2053  int64_t max_seeked_frame = seek_audio_frame_found; // determine max seeked frame
2054  if (seek_video_frame_found > max_seeked_frame) {
2055  max_seeked_frame = seek_video_frame_found;
2056  }
2057  if ((info.has_audio && seek_audio_frame_found && max_seeked_frame >= requested_frame) ||
2058  (info.has_video && seek_video_frame_found && max_seeked_frame >= requested_frame)) {
2059  seek_trash = true;
2060  }
2061 
2062  return seek_trash;
2063 }
2064 
2065 // Check if a frame is missing and attempt to replace its frame image (and
2066 bool FFmpegReader::CheckMissingFrame(int64_t requested_frame) {
2067  // Lock
2068  const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
2069 
2070  // Increment check count for this frame (or init to 1)
2071  ++checked_frames[requested_frame];
2072 
2073  // Debug output
2074  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckMissingFrame", "requested_frame", requested_frame, "has_missing_frames", has_missing_frames, "missing_video_frames.size()", missing_video_frames.size(), "checked_count", checked_frames[requested_frame]);
2075 
2076  // Missing frames (sometimes frame #'s are skipped due to invalid or missing timestamps)
2077  std::map<int64_t, int64_t>::iterator itr;
2078  bool found_missing_frame = false;
2079 
2080  // Special MP3 Handling (ignore more than 1 video frame)
2081  if (info.has_audio and info.has_video) {
2082  AVCodecID aCodecId = AV_FIND_DECODER_CODEC_ID(aStream);
2083  AVCodecID vCodecId = AV_FIND_DECODER_CODEC_ID(pStream);
2084  // If MP3 with single video frame, handle this special case by copying the previously
2085  // decoded image to the new frame. Otherwise, it will spend a huge amount of
2086  // CPU time looking for missing images for all the audio-only frames.
2087  if (checked_frames[requested_frame] > 8 && !missing_video_frames.count(requested_frame) &&
2088  !processing_audio_frames.count(requested_frame) && processed_audio_frames.count(requested_frame) &&
2089  last_frame && last_video_frame && last_video_frame->has_image_data && aCodecId == AV_CODEC_ID_MP3 && (vCodecId == AV_CODEC_ID_MJPEGB || vCodecId == AV_CODEC_ID_MJPEG)) {
2090  missing_video_frames.insert(std::pair<int64_t, int64_t>(requested_frame, last_video_frame->number));
2091  missing_video_frames_source.insert(std::pair<int64_t, int64_t>(last_video_frame->number, requested_frame));
2092  missing_frames.Add(last_video_frame);
2093  }
2094  }
2095 
2096  // Check if requested video frame is a missing
2097  if (missing_video_frames.count(requested_frame)) {
2098  int64_t missing_source_frame = missing_video_frames.find(requested_frame)->second;
2099 
2100  // Increment missing source frame check count (or init to 1)
2101  ++checked_frames[missing_source_frame];
2102 
2103  // Get the previous frame of this missing frame (if it's available in missing cache)
2104  std::shared_ptr<Frame> parent_frame = missing_frames.GetFrame(missing_source_frame);
2105  if (parent_frame == NULL) {
2106  parent_frame = final_cache.GetFrame(missing_source_frame);
2107  if (parent_frame != NULL) {
2108  // Add missing final frame to missing cache
2109  missing_frames.Add(parent_frame);
2110  }
2111  }
2112 
2113  // Create blank missing frame
2114  std::shared_ptr<Frame> missing_frame = CreateFrame(requested_frame);
2115 
2116  // Debug output
2117  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckMissingFrame (Is Previous Video Frame Final)", "requested_frame", requested_frame, "missing_frame->number", missing_frame->number, "missing_source_frame", missing_source_frame);
2118 
2119  // If previous frame found, copy image from previous to missing frame (else we'll just wait a bit and try again later)
2120  if (parent_frame != NULL) {
2121  // Debug output
2122  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckMissingFrame (AddImage from Previous Video Frame)", "requested_frame", requested_frame, "missing_frame->number", missing_frame->number, "missing_source_frame", missing_source_frame);
2123 
2124  // Add this frame to the processed map (since it's already done)
2125  std::shared_ptr<QImage> parent_image = parent_frame->GetImage();
2126  if (parent_image) {
2127  missing_frame->AddImage(std::shared_ptr<QImage>(new QImage(*parent_image)));
2128  processed_video_frames[missing_frame->number] = missing_frame->number;
2129  }
2130  }
2131  }
2132 
2133  // Check if requested audio frame is a missing
2134  if (missing_audio_frames.count(requested_frame)) {
2135 
2136  // Create blank missing frame
2137  std::shared_ptr<Frame> missing_frame = CreateFrame(requested_frame);
2138 
2139  // Get Samples per frame (for this frame number)
2140  int samples_per_frame = Frame::GetSamplesPerFrame(missing_frame->number, info.fps, info.sample_rate, info.channels);
2141 
2142  // Debug output
2143  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckMissingFrame (Add Silence for Missing Audio Frame)", "requested_frame", requested_frame, "missing_frame->number", missing_frame->number, "samples_per_frame", samples_per_frame);
2144 
2145  // Add this frame to the processed map (since it's already done)
2146  missing_frame->AddAudioSilence(samples_per_frame);
2147  processed_audio_frames[missing_frame->number] = missing_frame->number;
2148  }
2149 
2150  return found_missing_frame;
2151 }
2152 
2153 // Check the working queue, and move finished frames to the finished queue
2154 void FFmpegReader::CheckWorkingFrames(bool end_of_stream, int64_t requested_frame) {
2155  // Loop through all working queue frames
2156  bool checked_count_tripped = false;
2157  int max_checked_count = 80;
2158 
2159  // Check if requested frame is 'missing'
2160  CheckMissingFrame(requested_frame);
2161 
2162  while (true) {
2163  // Get the front frame of working cache
2164  std::shared_ptr<Frame> f(working_cache.GetSmallestFrame());
2165 
2166  // Was a frame found?
2167  if (!f)
2168  // No frames found
2169  break;
2170 
2171  // Remove frames which are too old
2172  if (f && f->number < (requested_frame - (OPEN_MP_NUM_PROCESSORS * 2))) {
2173  working_cache.Remove(f->number);
2174  }
2175 
2176  // Check if this frame is 'missing'
2177  CheckMissingFrame(f->number);
2178 
2179  // Init # of times this frame has been checked so far
2180  int checked_count = 0;
2181  int checked_frames_size = 0;
2182 
2183  bool is_video_ready = false;
2184  bool is_audio_ready = false;
2185  { // limit scope of next few lines
2186  const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
2187  is_video_ready = processed_video_frames.count(f->number);
2188  is_audio_ready = processed_audio_frames.count(f->number);
2189 
2190  // Get check count for this frame
2191  checked_frames_size = checked_frames.size();
2192  if (!checked_count_tripped || f->number >= requested_frame)
2193  checked_count = checked_frames[f->number];
2194  else
2195  // Force checked count over the limit
2196  checked_count = max_checked_count;
2197  }
2198 
2199  if (previous_packet_location.frame == f->number && !end_of_stream)
2200  is_audio_ready = false; // don't finalize the last processed audio frame
2201  bool is_seek_trash = IsPartialFrame(f->number);
2202 
2203  // Adjust for available streams
2204  if (!info.has_video) is_video_ready = true;
2205  if (!info.has_audio) is_audio_ready = true;
2206 
2207  // Make final any frames that get stuck (for whatever reason)
2208  if (checked_count >= max_checked_count && (!is_video_ready || !is_audio_ready)) {
2209  // Debug output
2210  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames (exceeded checked_count)", "requested_frame", requested_frame, "frame_number", f->number, "is_video_ready", is_video_ready, "is_audio_ready", is_audio_ready, "checked_count", checked_count, "checked_frames_size", checked_frames_size);
2211 
2212  // Trigger checked count tripped mode (clear out all frames before requested frame)
2213  checked_count_tripped = true;
2214 
2215  if (info.has_video && !is_video_ready && last_video_frame) {
2216  // Copy image from last frame
2217  f->AddImage(std::shared_ptr<QImage>(new QImage(*last_video_frame->GetImage())));
2218  is_video_ready = true;
2219  }
2220 
2221  if (info.has_audio && !is_audio_ready) {
2222  // Mark audio as processed, and indicate the frame has audio data
2223  is_audio_ready = true;
2224  }
2225  }
2226 
2227  // Debug output
2228  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames", "requested_frame", requested_frame, "frame_number", f->number, "is_video_ready", is_video_ready, "is_audio_ready", is_audio_ready, "checked_count", checked_count, "checked_frames_size", checked_frames_size);
2229 
2230  // Check if working frame is final
2231  if ((!end_of_stream && is_video_ready && is_audio_ready) || end_of_stream || is_seek_trash) {
2232  // Debug output
2233  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames (mark frame as final)", "requested_frame", requested_frame, "f->number", f->number, "is_seek_trash", is_seek_trash, "Working Cache Count", working_cache.Count(), "Final Cache Count", final_cache.Count(), "end_of_stream", end_of_stream);
2234 
2235  if (!is_seek_trash) {
2236  // Add missing image (if needed - sometimes end_of_stream causes frames with only audio)
2237  if (info.has_video && !is_video_ready && last_video_frame)
2238  // Copy image from last frame
2239  f->AddImage(std::shared_ptr<QImage>(new QImage(*last_video_frame->GetImage())));
2240 
2241  // Reset counter since last 'final' frame
2242  num_checks_since_final = 0;
2243 
2244  // Move frame to final cache
2245  final_cache.Add(f);
2246 
2247  // Add to missing cache (if another frame depends on it)
2248  {
2249  const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
2250  if (missing_video_frames_source.count(f->number)) {
2251  // Debug output
2252  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames (add frame to missing cache)", "f->number", f->number, "is_seek_trash", is_seek_trash, "Missing Cache Count", missing_frames.Count(), "Working Cache Count", working_cache.Count(), "Final Cache Count", final_cache.Count());
2253  missing_frames.Add(f);
2254  }
2255 
2256  // Remove from 'checked' count
2257  checked_frames.erase(f->number);
2258  }
2259 
2260  // Remove frame from working cache
2261  working_cache.Remove(f->number);
2262 
2263  // Update last frame processed
2264  last_frame = f->number;
2265 
2266  } else {
2267  // Seek trash, so delete the frame from the working cache, and never add it to the final cache.
2268  working_cache.Remove(f->number);
2269  }
2270 
2271  } else {
2272  // Stop looping
2273  break;
2274  }
2275  }
2276 }
2277 
2278 // Check for the correct frames per second (FPS) value by scanning the 1st few seconds of video packets.
2279 void FFmpegReader::CheckFPS() {
2280  check_fps = true;
2281 
2282 
2283  int first_second_counter = 0;
2284  int second_second_counter = 0;
2285  int third_second_counter = 0;
2286  int forth_second_counter = 0;
2287  int fifth_second_counter = 0;
2288  int frames_detected = 0;
2289  int64_t pts = 0;
2290 
2291  // Loop through the stream
2292  while (true) {
2293  // Get the next packet (if any)
2294  if (GetNextPacket() < 0)
2295  // Break loop when no more packets found
2296  break;
2297 
2298  // Video packet
2299  if (packet->stream_index == videoStream) {
2300  // Check if the AVFrame is finished and set it
2301  if (GetAVFrame()) {
2302  // Update PTS / Frame Offset (if any)
2303  UpdatePTSOffset(true);
2304 
2305  // Get PTS of this packet
2306  pts = GetVideoPTS();
2307 
2308  // Remove pFrame
2309  RemoveAVFrame(pFrame);
2310 
2311  // Apply PTS offset
2312  pts += video_pts_offset;
2313 
2314  // Get the video packet start time (in seconds)
2315  double video_seconds = double(pts) * info.video_timebase.ToDouble();
2316 
2317  // Increment the correct counter
2318  if (video_seconds <= 1.0)
2319  first_second_counter++;
2320  else if (video_seconds > 1.0 && video_seconds <= 2.0)
2321  second_second_counter++;
2322  else if (video_seconds > 2.0 && video_seconds <= 3.0)
2323  third_second_counter++;
2324  else if (video_seconds > 3.0 && video_seconds <= 4.0)
2325  forth_second_counter++;
2326  else if (video_seconds > 4.0 && video_seconds <= 5.0)
2327  fifth_second_counter++;
2328 
2329  // Increment counters
2330  frames_detected++;
2331  }
2332  }
2333  }
2334 
2335  // Double check that all counters have greater than zero (or give up)
2336  if (second_second_counter != 0 && third_second_counter != 0 && forth_second_counter != 0 && fifth_second_counter != 0) {
2337  // Calculate average FPS (average of first few seconds)
2338  int sum_fps = second_second_counter + third_second_counter + forth_second_counter + fifth_second_counter;
2339  int avg_fps = round(sum_fps / 4.0f);
2340 
2341  // Update FPS
2342  info.fps = Fraction(avg_fps, 1);
2343 
2344  // Update Duration and Length
2345  info.video_length = frames_detected;
2346  info.duration = frames_detected / (sum_fps / 4.0f);
2347 
2348  // Update video bit rate
2350  } else if (second_second_counter != 0 && third_second_counter != 0) {
2351  // Calculate average FPS (only on second 2)
2352  int sum_fps = second_second_counter;
2353 
2354  // Update FPS
2355  info.fps = Fraction(sum_fps, 1);
2356 
2357  // Update Duration and Length
2358  info.video_length = frames_detected;
2359  info.duration = frames_detected / float(sum_fps);
2360 
2361  // Update video bit rate
2363  } else {
2364  // Too short to determine framerate, just default FPS
2365  // Set a few important default video settings (so audio can be divided into frames)
2366  info.fps.num = 30;
2367  info.fps.den = 1;
2368 
2369  // Calculate number of frames
2370  info.video_length = frames_detected;
2371  info.duration = frames_detected / info.fps.ToFloat();
2372  }
2373 }
2374 
2375 // Remove AVFrame from cache (and deallocate its memory)
2376 void FFmpegReader::RemoveAVFrame(AVFrame *remove_frame) {
2377  // Remove pFrame (if exists)
2378  if (remove_frame) {
2379  // Free memory
2380 #pragma omp critical (packet_cache)
2381  {
2382  av_freep(&remove_frame->data[0]);
2383 #ifndef WIN32
2384  AV_FREE_FRAME(&remove_frame);
2385 #endif
2386  }
2387  }
2388 }
2389 
2390 // Remove AVPacket from cache (and deallocate its memory)
2391 void FFmpegReader::RemoveAVPacket(AVPacket *remove_packet) {
2392  // deallocate memory for packet
2393  AV_FREE_PACKET(remove_packet);
2394 
2395  // Delete the object
2396  delete remove_packet;
2397 }
2398 
2399 /// Get the smallest video frame that is still being processed
2400 int64_t FFmpegReader::GetSmallestVideoFrame() {
2401  // Loop through frame numbers
2402  std::map<int64_t, int64_t>::iterator itr;
2403  int64_t smallest_frame = -1;
2404  const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
2405  for (itr = processing_video_frames.begin(); itr != processing_video_frames.end(); ++itr) {
2406  if (itr->first < smallest_frame || smallest_frame == -1)
2407  smallest_frame = itr->first;
2408  }
2409 
2410  // Return frame number
2411  return smallest_frame;
2412 }
2413 
2414 /// Get the smallest audio frame that is still being processed
2415 int64_t FFmpegReader::GetSmallestAudioFrame() {
2416  // Loop through frame numbers
2417  std::map<int64_t, int64_t>::iterator itr;
2418  int64_t smallest_frame = -1;
2419  const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
2420  for (itr = processing_audio_frames.begin(); itr != processing_audio_frames.end(); ++itr) {
2421  if (itr->first < smallest_frame || smallest_frame == -1)
2422  smallest_frame = itr->first;
2423  }
2424 
2425  // Return frame number
2426  return smallest_frame;
2427 }
2428 
2429 // Generate JSON string of this object
2430 std::string FFmpegReader::Json() {
2431 
2432  // Return formatted string
2433  return JsonValue().toStyledString();
2434 }
2435 
2436 // Generate Json::JsonValue for this object
2438 
2439  // Create root json object
2440  Json::Value root = ReaderBase::JsonValue(); // get parent properties
2441  root["type"] = "FFmpegReader";
2442  root["path"] = path;
2443 
2444  // return JsonValue
2445  return root;
2446 }
2447 
2448 // Load JSON string into this object
2449 void FFmpegReader::SetJson(std::string value) {
2450 
2451  // Parse JSON string into JSON objects
2452  Json::Value root;
2453  Json::CharReaderBuilder rbuilder;
2454  Json::CharReader* reader(rbuilder.newCharReader());
2455 
2456  std::string errors;
2457  bool success = reader->parse(value.c_str(), value.c_str() + value.size(),
2458  &root, &errors);
2459  delete reader;
2460 
2461  if (!success)
2462  // Raise exception
2463  throw InvalidJSON("JSON could not be parsed (or is invalid)");
2464 
2465  try {
2466  // Set all values that match
2467  SetJsonValue(root);
2468  }
2469  catch (const std::exception& e) {
2470  // Error parsing JSON (or missing keys)
2471  throw InvalidJSON("JSON is invalid (missing keys or invalid data types)");
2472  }
2473 }
2474 
2475 // Load Json::JsonValue into this object
2476 void FFmpegReader::SetJsonValue(Json::Value root) {
2477 
2478  // Set parent data
2480 
2481  // Set data from Json (if key is found)
2482  if (!root["path"].isNull())
2483  path = root["path"].asString();
2484 
2485  // Re-Open path, and re-init everything (if needed)
2486  if (is_open) {
2487  Close();
2488  Open();
2489  }
2490 }
#define AV_RESET_FRAME(av_frame)
void SetJson(std::string value)
Load JSON string into this object.
#define AV_FREE_FRAME(av_frame)
int MAX_HEIGHT
Maximum height for image data (useful for optimzing for a smaller preview or render) ...
Definition: Settings.h:101
int num
Numerator for the fraction.
Definition: Fraction.h:47
#define SWRCONTEXT
Point GetMaxPoint() const
Get max point (by Y coordinate)
Definition: KeyFrame.cpp:249
#define AV_FIND_DECODER_CODEC_ID(av_stream)
#define SWR_INIT(ctx)
int width
The width of the video (in pixesl)
Definition: ReaderBase.h:68
#define AV_COPY_PICTURE_DATA(av_frame, buffer, pix_fmt, width, height)
#define AV_REGISTER_ALL
std::shared_ptr< openshot::Frame > GetFrame(int64_t requested_frame)
float ToFloat()
Return this fraction as a float (i.e. 1/2 = 0.5)
Definition: Fraction.cpp:44
#define AV_GET_CODEC_PIXEL_FORMAT(av_stream, av_context)
float duration
Length of time (in seconds)
Definition: ReaderBase.h:65
bool is_near(AudioLocation location, int samples_per_frame, int64_t amount)
void Reduce()
Reduce this fraction (i.e. 640/480 = 4/3)
Definition: Fraction.cpp:74
void SetMaxBytesFromInfo(int64_t number_of_frames, int width, int height, int sample_rate, int channels)
Set maximum bytes to a different amount based on a ReaderInfo struct.
Definition: CacheBase.cpp:49
Scale the clip until both height and width fill the canvas (cropping the overlap) ...
Definition: Enums.h:54
#define AVCODEC_MAX_AUDIO_FRAME_SIZE
#define SWR_CLOSE(ctx)
#define SWR_FREE(ctx)
FFmpegReader(std::string path)
openshot::Keyframe scale_x
Curve representing the horizontal scaling in percent (0 to 1)
Definition: Clip.h:212
#define OPEN_MP_NUM_PROCESSORS
#define AV_GET_SAMPLE_FORMAT(av_stream, av_context)
#define AVCODEC_REGISTER_ALL
openshot::Keyframe scale_y
Curve representing the vertical scaling in percent (0 to 1)
Definition: Clip.h:213
Json::Value JsonValue()
Generate Json::JsonValue for this object.
Exception when a reader is closed, and a frame is requested.
Definition: Exceptions.h:337
bool has_video
Determines if this file has a video stream.
Definition: ReaderBase.h:62
void Close()
Close File.
int64_t file_size
Size of file (in bytes)
Definition: ReaderBase.h:66
void AppendDebugMethod(std::string method_name, std::string arg1_name="", float arg1_value=-1.0, std::string arg2_name="", float arg2_value=-1.0, std::string arg3_name="", float arg3_value=-1.0, std::string arg4_name="", float arg4_value=-1.0, std::string arg5_name="", float arg5_value=-1.0, std::string arg6_name="", float arg6_value=-1.0)
Append debug information.
Definition: ZmqLogger.cpp:179
std::shared_ptr< openshot::Frame > GetSmallestFrame()
Get the smallest frame number.
AVPixelFormat hw_de_av_pix_fmt_global
int audio_bit_rate
The bit rate of the audio stream (in bytes)
Definition: ReaderBase.h:81
int GetSamplesPerFrame(openshot::Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
Definition: Frame.cpp:547
bool has_audio
Determines if this file has an audio stream.
Definition: ReaderBase.h:63
This class represents a clip (used to arrange readers on the timeline)
Definition: Clip.h:95
#define AV_FREE_CONTEXT(av_context)
Exception when no valid codec is found for a file.
Definition: Exceptions.h:157
double Y
The Y value of the coordinate (usually representing the value of the property being animated) ...
Definition: Coordinate.h:58
int HARDWARE_DECODER
Use video codec for faster video decoding (if supported)
Definition: Settings.h:92
int MAX_WIDTH
Maximum width for image data (useful for optimzing for a smaller preview or render) ...
Definition: Settings.h:98
int audio_stream_index
The index of the audio stream.
Definition: ReaderBase.h:85
#define AV_GET_CODEC_ATTRIBUTES(av_stream, av_context)
int64_t video_length
The number of frames in the video stream.
Definition: ReaderBase.h:75
#define FF_NUM_PROCESSORS
#define AV_FREE_PACKET(av_packet)
juce::CriticalSection processingCriticalSection
Definition: ReaderBase.h:102
Exception when no streams are found in the file.
Definition: Exceptions.h:269
int height
The height of the video (in pixels)
Definition: ReaderBase.h:67
#define AV_ALLOCATE_FRAME()
openshot::Fraction video_timebase
The video timebase determines how long each frame stays on the screen.
Definition: ReaderBase.h:77
Exception for files that can not be found or opened.
Definition: Exceptions.h:173
int hw_de_on
This class represents a fraction.
Definition: Fraction.h:45
std::map< std::string, std::string > metadata
An optional map/dictionary of metadata for this reader.
Definition: ReaderBase.h:87
openshot::ChannelLayout channel_layout
The channel layout (mono, stereo, 5 point surround, etc...)
Definition: ReaderBase.h:84
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround...
std::shared_ptr< openshot::Frame > GetFrame(int64_t frame_number)
Get a frame from the cache.
AVHWDeviceType hw_de_av_device_type_global
virtual Json::Value JsonValue()=0
Generate Json::JsonValue for this object.
Definition: ReaderBase.cpp:116
int HW_DE_DEVICE_SET
Which GPU to use to decode (0 is the first)
Definition: Settings.h:119
auto AV_GET_CODEC_CONTEXT
#define PixelFormat
openshot::ClipBase * parent
Definition: ReaderBase.h:103
virtual void SetJsonValue(Json::Value root)=0
Load Json::JsonValue into this object.
Definition: ReaderBase.cpp:170
Scale the clip until both height and width fill the canvas (distort to fit)
Definition: Enums.h:56
void Clear()
Clear the cache of all frames.
int ToInt()
Return a rounded integer of the fraction (for example 30000/1001 returns 30)
Definition: Fraction.cpp:54
#define MY_INPUT_BUFFER_PADDING_SIZE
#define AV_GET_IMAGE_SIZE(pix_fmt, width, height)
openshot::ReaderInfo info
Information about the current media file.
Definition: ReaderBase.h:111
Exception for frames that are out of bounds.
Definition: Exceptions.h:285
static ZmqLogger * Instance()
Create or get an instance of this logger singleton (invoke the class with this method) ...
Definition: ZmqLogger.cpp:45
int64_t Count()
Count the frames in the queue.
openshot::Fraction audio_timebase
The audio timebase determines how long each audio packet should be played.
Definition: ReaderBase.h:86
std::string vcodec
The name of the video codec used to encode / decode the video stream.
Definition: ReaderBase.h:74
if(!codec) codec
This namespace is the default namespace for all code in the openshot library.
#define SWR_CONVERT(ctx, out, linesize, out_count, in, linesize2, in_count)
AVDictionary * opts
openshot::ClipBase * GetClip()
Parent clip object of this reader (which can be unparented and NULL)
Definition: ReaderBase.cpp:253
Coordinate co
This is the primary coordinate.
Definition: Point.h:84
Exception for invalid JSON.
Definition: Exceptions.h:205
#define AV_GET_CODEC_TYPE(av_stream)
int pixel_format
The pixel format (i.e. YUV420P, RGB24, etc...)
Definition: ReaderBase.h:69
openshot::Fraction display_ratio
The ratio of width to height of the video stream (i.e. 640x480 has a ratio of 4/3) ...
Definition: ReaderBase.h:73
This struct holds the associated video frame and starting sample # for an audio packet.
Definition: FFmpegReader.h:61
openshot::ScaleType scale
The scale determines how a clip should be resized to fit its parent.
Definition: Clip.h:144
void Open()
Open File - which is called by the constructor automatically.
static Settings * Instance()
Create or get an instance of this logger singleton (invoke the class with this method) ...
Definition: Settings.cpp:41
int video_bit_rate
The bit rate of the video stream (in bytes)
Definition: ReaderBase.h:71
#define SWR_ALLOC()
int DE_LIMIT_WIDTH_MAX
Maximum columns that hardware decode can handle.
Definition: Settings.h:116
#define PIX_FMT_RGBA
openshot::Fraction pixel_ratio
The pixel ratio of the video stream as a fraction (i.e. some pixels are not square) ...
Definition: ReaderBase.h:72
void Add(std::shared_ptr< openshot::Frame > frame)
Add a Frame to the cache.
void Remove(int64_t frame_number)
Remove a specific frame.
CacheMemory final_cache
Final cache object used to hold final frames.
Definition: FFmpegReader.h:231
virtual ~FFmpegReader()
Destructor.
int den
Denominator for the fraction.
Definition: Fraction.h:48
int channels
The number of audio channels used in the audio stream.
Definition: ReaderBase.h:83
void SetJsonValue(Json::Value root)
Load Json::JsonValue into this object.
int video_stream_index
The index of the video stream.
Definition: ReaderBase.h:76
Scale the clip until either height or width fills the canvas (with no cropping)
Definition: Enums.h:55
openshot::Fraction fps
Frames per second, as a fraction (i.e. 24/1 = 24 fps)
Definition: ReaderBase.h:70
std::string acodec
The name of the audio codec used to encode / decode the video stream.
Definition: ReaderBase.h:80
std::string Json()
Get and Set JSON methods.
double ToDouble()
Return this fraction as a double (i.e. 1/2 = 0.5)
Definition: Fraction.cpp:49
int sample_rate
The number of audio samples per second (44100 is a common sample rate)
Definition: ReaderBase.h:82
int DE_LIMIT_HEIGHT_MAX
Maximum rows that hardware decode can handle.
Definition: Settings.h:113