OpenShot Library | libopenshot  0.7.0
FFmpegReader.cpp
Go to the documentation of this file.
1 
12 // Copyright (c) 2008-2024 OpenShot Studios, LLC, Fabrice Bellard
13 //
14 // SPDX-License-Identifier: LGPL-3.0-or-later
15 
16 #include <thread> // for std::this_thread::sleep_for
17 #include <chrono> // for std::chrono::milliseconds
18 #include <algorithm>
19 #include <cmath>
20 #include <sstream>
21 #include <unistd.h>
22 
23 #include "FFmpegUtilities.h"
24 #include "effects/CropHelpers.h"
25 
26 #include "FFmpegReader.h"
27 #include "Exceptions.h"
28 #include "MemoryTrim.h"
29 #include "Timeline.h"
30 #include "ZmqLogger.h"
31 
32 #define ENABLE_VAAPI 0
33 
34 #if USE_HW_ACCEL
35 #define MAX_SUPPORTED_WIDTH 1950
36 #define MAX_SUPPORTED_HEIGHT 1100
37 
38 #if ENABLE_VAAPI
39 #include "libavutil/hwcontext_vaapi.h"
40 
41 typedef struct VAAPIDecodeContext {
42  VAProfile va_profile;
43  VAEntrypoint va_entrypoint;
44  VAConfigID va_config;
45  VAContextID va_context;
46 
47 #if FF_API_STRUCT_VAAPI_CONTEXT
48  // FF_DISABLE_DEPRECATION_WARNINGS
49  int have_old_context;
50  struct vaapi_context *old_context;
51  AVBufferRef *device_ref;
52  // FF_ENABLE_DEPRECATION_WARNINGS
53 #endif
54 
55  AVHWDeviceContext *device;
56  AVVAAPIDeviceContext *hwctx;
57 
58  AVHWFramesContext *frames;
59  AVVAAPIFramesContext *hwfc;
60 
61  enum AVPixelFormat surface_format;
62  int surface_count;
63  } VAAPIDecodeContext;
64 #endif // ENABLE_VAAPI
65 #endif // USE_HW_ACCEL
66 
67 
68 using namespace openshot;
69 
70 int hw_de_on = 0;
71 #if USE_HW_ACCEL
72  AVPixelFormat hw_de_av_pix_fmt_global = AV_PIX_FMT_NONE;
73  AVHWDeviceType hw_de_av_device_type_global = AV_HWDEVICE_TYPE_NONE;
74 #endif
75 
76 // Normalize deprecated JPEG-range YUVJ formats before creating swscale contexts.
77 // swscale expects non-YUVJ formats plus explicit color-range metadata.
78 static AVPixelFormat NormalizeDeprecatedPixFmt(AVPixelFormat pix_fmt, bool& is_full_range) {
79  switch (pix_fmt) {
80  case AV_PIX_FMT_YUVJ420P:
81  is_full_range = true;
82  return AV_PIX_FMT_YUV420P;
83  case AV_PIX_FMT_YUVJ422P:
84  is_full_range = true;
85  return AV_PIX_FMT_YUV422P;
86  case AV_PIX_FMT_YUVJ444P:
87  is_full_range = true;
88  return AV_PIX_FMT_YUV444P;
89  case AV_PIX_FMT_YUVJ440P:
90  is_full_range = true;
91  return AV_PIX_FMT_YUV440P;
92 #ifdef AV_PIX_FMT_YUVJ411P
93  case AV_PIX_FMT_YUVJ411P:
94  is_full_range = true;
95  return AV_PIX_FMT_YUV411P;
96 #endif
97  default:
98  return pix_fmt;
99  }
100 }
101 
102 FFmpegReader::FFmpegReader(const std::string &path, bool inspect_reader)
103  : FFmpegReader(path, DurationStrategy::VideoPreferred, inspect_reader) {}
104 
105 FFmpegReader::FFmpegReader(const std::string &path, DurationStrategy duration_strategy, bool inspect_reader)
106  : path(path), pFormatCtx(NULL), videoStream(-1), audioStream(-1), pCodecCtx(NULL), aCodecCtx(NULL),
107  pStream(NULL), aStream(NULL), packet(NULL), pFrame(NULL), is_open(false), is_duration_known(false),
108  check_interlace(false), check_fps(false), duration_strategy(duration_strategy), previous_packet_location{-1, 0},
109  is_seeking(false), seeking_pts(0), seeking_frame(0), is_video_seek(true), seek_count(0),
110  seek_audio_frame_found(0), seek_video_frame_found(0), last_seek_max_frame(-1), seek_stagnant_count(0),
111  last_frame(0), largest_frame_processed(0), current_video_frame(0), audio_pts(0), video_pts(0),
112  hold_packet(false), pts_offset_seconds(0.0), audio_pts_seconds(0.0), video_pts_seconds(0.0),
113  NO_PTS_OFFSET(-99999), enable_seek(true) {
114 
115  // Initialize FFMpeg, and register all formats and codecs
116  AV_REGISTER_ALL
117  AVCODEC_REGISTER_ALL
118 
119  // Init timestamp offsets
120  pts_offset_seconds = NO_PTS_OFFSET;
121  video_pts_seconds = NO_PTS_OFFSET;
122  audio_pts_seconds = NO_PTS_OFFSET;
123 
124  // Init cache
125  const int init_working_cache_frames = std::max(Settings::Instance()->CACHE_MIN_FRAMES, OPEN_MP_NUM_PROCESSORS * 4);
126  const int init_final_cache_frames = std::max(Settings::Instance()->CACHE_MIN_FRAMES, OPEN_MP_NUM_PROCESSORS * 4);
127  working_cache.SetMaxBytesFromInfo(init_working_cache_frames, info.width, info.height, info.sample_rate, info.channels);
128  final_cache.SetMaxBytesFromInfo(init_final_cache_frames, info.width, info.height, info.sample_rate, info.channels);
129 
130  // Open and Close the reader, to populate its attributes (such as height, width, etc...)
131  if (inspect_reader) {
132  Open();
133  Close();
134  }
135 }
136 
137 FFmpegReader::~FFmpegReader() {
138  if (is_open)
139  // Auto close reader if not already done
140  Close();
141 }
142 
143 // This struct holds the associated video frame and starting sample # for an audio packet.
144 bool AudioLocation::is_near(AudioLocation location, int samples_per_frame, int64_t amount) {
145  // Is frame even close to this one?
146  if (abs(location.frame - frame) >= 2)
147  // This is too far away to be considered
148  return false;
149 
150  // Note that samples_per_frame can vary slightly frame to frame when the
151  // audio sampling rate is not an integer multiple of the video fps.
152  int64_t diff = samples_per_frame * (location.frame - frame) + location.sample_start - sample_start;
153  if (abs(diff) <= amount)
154  // close
155  return true;
156 
157  // not close
158  return false;
159 }
160 
161 #if USE_HW_ACCEL
162 
163 // Get hardware pix format
164 static enum AVPixelFormat get_hw_dec_format(AVCodecContext *ctx, const enum AVPixelFormat *pix_fmts)
165 {
166  const enum AVPixelFormat *p;
167 
168  // Prefer only the format matching the selected hardware decoder
169  int selected = openshot::Settings::Instance()->HARDWARE_DECODER;
170 
171  for (p = pix_fmts; *p != AV_PIX_FMT_NONE; p++) {
172  switch (*p) {
173 #if defined(__linux__)
174  // Linux pix formats
175  case AV_PIX_FMT_VAAPI:
176  if (selected == 1) {
177  hw_de_av_pix_fmt_global = AV_PIX_FMT_VAAPI;
178  hw_de_av_device_type_global = AV_HWDEVICE_TYPE_VAAPI;
179  return *p;
180  }
181  break;
182  case AV_PIX_FMT_VDPAU:
183  if (selected == 6) {
184  hw_de_av_pix_fmt_global = AV_PIX_FMT_VDPAU;
185  hw_de_av_device_type_global = AV_HWDEVICE_TYPE_VDPAU;
186  return *p;
187  }
188  break;
189 #endif
190 #if defined(_WIN32)
191  // Windows pix formats
192  case AV_PIX_FMT_DXVA2_VLD:
193  if (selected == 3) {
194  hw_de_av_pix_fmt_global = AV_PIX_FMT_DXVA2_VLD;
195  hw_de_av_device_type_global = AV_HWDEVICE_TYPE_DXVA2;
196  return *p;
197  }
198  break;
199  case AV_PIX_FMT_D3D11:
200  if (selected == 4) {
201  hw_de_av_pix_fmt_global = AV_PIX_FMT_D3D11;
202  hw_de_av_device_type_global = AV_HWDEVICE_TYPE_D3D11VA;
203  return *p;
204  }
205  break;
206 #endif
207 #if defined(__APPLE__)
208  // Apple pix formats
209  case AV_PIX_FMT_VIDEOTOOLBOX:
210  if (selected == 5) {
211  hw_de_av_pix_fmt_global = AV_PIX_FMT_VIDEOTOOLBOX;
212  hw_de_av_device_type_global = AV_HWDEVICE_TYPE_VIDEOTOOLBOX;
213  return *p;
214  }
215  break;
216 #endif
217  // Cross-platform pix formats
218  case AV_PIX_FMT_CUDA:
219  if (selected == 2) {
220  hw_de_av_pix_fmt_global = AV_PIX_FMT_CUDA;
221  hw_de_av_device_type_global = AV_HWDEVICE_TYPE_CUDA;
222  return *p;
223  }
224  break;
225  case AV_PIX_FMT_QSV:
226  if (selected == 7) {
227  hw_de_av_pix_fmt_global = AV_PIX_FMT_QSV;
228  hw_de_av_device_type_global = AV_HWDEVICE_TYPE_QSV;
229  return *p;
230  }
231  break;
232  default:
233  // This is only here to silence unused-enum warnings
234  break;
235  }
236  }
237  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::get_hw_dec_format (Unable to decode this file using hardware decode)");
238  return AV_PIX_FMT_NONE;
239 }
240 
241 int FFmpegReader::IsHardwareDecodeSupported(int codecid)
242 {
243  int ret;
244  switch (codecid) {
245  case AV_CODEC_ID_H264:
246  case AV_CODEC_ID_MPEG2VIDEO:
247  case AV_CODEC_ID_VC1:
248  case AV_CODEC_ID_WMV1:
249  case AV_CODEC_ID_WMV2:
250  case AV_CODEC_ID_WMV3:
251  ret = 1;
252  break;
253  default :
254  ret = 0;
255  break;
256  }
257  return ret;
258 }
259 #endif // USE_HW_ACCEL
260 
261 void FFmpegReader::Open() {
262  // Open reader if not already open
263  if (!is_open) {
264  // Prevent async calls to the following code
265  const std::lock_guard<std::recursive_mutex> lock(getFrameMutex);
266 
267  // Initialize format context
268  pFormatCtx = NULL;
269  {
270  hw_de_on = (!force_sw_decode && openshot::Settings::Instance()->HARDWARE_DECODER != 0 ? 1 : 0);
271  hw_decode_failed = false;
272  hw_decode_error_count = 0;
273  hw_decode_succeeded = false;
274  ZmqLogger::Instance()->AppendDebugMethod("Decode hardware acceleration settings", "hw_de_on", hw_de_on, "HARDWARE_DECODER", openshot::Settings::Instance()->HARDWARE_DECODER);
275  }
276 
277  // Open video file
278  if (avformat_open_input(&pFormatCtx, path.c_str(), NULL, NULL) != 0)
279  throw InvalidFile("FFmpegReader could not open media file.", path);
280 
281  // Retrieve stream information
282  if (avformat_find_stream_info(pFormatCtx, NULL) < 0)
283  throw NoStreamsFound("No streams found in file.", path);
284 
285  videoStream = -1;
286  audioStream = -1;
287 
288  // Init end-of-file detection variables
289  packet_status.reset(true);
290 
291  // Loop through each stream, and identify the video and audio stream index
292  for (unsigned int i = 0; i < pFormatCtx->nb_streams; i++) {
293  // Is this a video stream?
294  if (AV_GET_CODEC_TYPE(pFormatCtx->streams[i]) == AVMEDIA_TYPE_VIDEO && videoStream < 0) {
295  videoStream = i;
296  packet_status.video_eof = false;
297  packet_status.packets_eof = false;
298  packet_status.end_of_file = false;
299  }
300  // Is this an audio stream?
301  if (AV_GET_CODEC_TYPE(pFormatCtx->streams[i]) == AVMEDIA_TYPE_AUDIO && audioStream < 0) {
302  audioStream = i;
303  packet_status.audio_eof = false;
304  packet_status.packets_eof = false;
305  packet_status.end_of_file = false;
306  }
307  }
308  if (videoStream == -1 && audioStream == -1)
309  throw NoStreamsFound("No video or audio streams found in this file.", path);
310 
311  // Is there a video stream?
312  if (videoStream != -1) {
313  // Set the stream index
314  info.video_stream_index = videoStream;
315 
316  // Set the codec and codec context pointers
317  pStream = pFormatCtx->streams[videoStream];
318 
319  // Find the codec ID from stream
320  const AVCodecID codecId = AV_FIND_DECODER_CODEC_ID(pStream);
321 
322  // Get codec and codec context from stream
323  const AVCodec *pCodec = avcodec_find_decoder(codecId);
324  AVDictionary *opts = NULL;
325  int retry_decode_open = 2;
326  // If hw accel is selected but hardware cannot handle repeat with software decoding
327  do {
328  pCodecCtx = AV_GET_CODEC_CONTEXT(pStream, pCodec);
329 #if USE_HW_ACCEL
330  if (hw_de_on && (retry_decode_open==2)) {
331  // Up to here no decision is made if hardware or software decode
332  hw_de_supported = IsHardwareDecodeSupported(pCodecCtx->codec_id);
333  }
334 #endif
335  retry_decode_open = 0;
336 
337  // Set number of threads equal to number of processors (not to exceed 16)
338  pCodecCtx->thread_count = std::min(FF_VIDEO_NUM_PROCESSORS, 16);
339 
340  if (pCodec == NULL) {
341  throw InvalidCodec("A valid video codec could not be found for this file.", path);
342  }
343 
344  // Init options
345  av_dict_set(&opts, "strict", "experimental", 0);
346 #if USE_HW_ACCEL
347  if (hw_de_on && hw_de_supported) {
348  // Open Hardware Acceleration
349  int i_decoder_hw = 0;
350  char adapter[256];
351  char *adapter_ptr = NULL;
352  int adapter_num;
353  adapter_num = openshot::Settings::Instance()->HW_DE_DEVICE_SET;
354  ZmqLogger::Instance()->AppendDebugMethod("Hardware decoding device number", "adapter_num", adapter_num);
355 
356  // Set hardware pix format (callback)
357  pCodecCtx->get_format = get_hw_dec_format;
358 
359  if (adapter_num < 3 && adapter_num >=0) {
360 #if defined(__linux__)
361  snprintf(adapter,sizeof(adapter),"/dev/dri/renderD%d", adapter_num+128);
362  adapter_ptr = adapter;
363  i_decoder_hw = openshot::Settings::Instance()->HARDWARE_DECODER;
364  switch (i_decoder_hw) {
365  case 1:
366  hw_de_av_device_type = AV_HWDEVICE_TYPE_VAAPI;
367  break;
368  case 2:
369  hw_de_av_device_type = AV_HWDEVICE_TYPE_CUDA;
370  break;
371  case 6:
372  hw_de_av_device_type = AV_HWDEVICE_TYPE_VDPAU;
373  break;
374  case 7:
375  hw_de_av_device_type = AV_HWDEVICE_TYPE_QSV;
376  break;
377  default:
378  hw_de_av_device_type = AV_HWDEVICE_TYPE_VAAPI;
379  break;
380  }
381 
382 #elif defined(_WIN32)
383  adapter_ptr = NULL;
384  i_decoder_hw = openshot::Settings::Instance()->HARDWARE_DECODER;
385  switch (i_decoder_hw) {
386  case 2:
387  hw_de_av_device_type = AV_HWDEVICE_TYPE_CUDA;
388  break;
389  case 3:
390  hw_de_av_device_type = AV_HWDEVICE_TYPE_DXVA2;
391  break;
392  case 4:
393  hw_de_av_device_type = AV_HWDEVICE_TYPE_D3D11VA;
394  break;
395  case 7:
396  hw_de_av_device_type = AV_HWDEVICE_TYPE_QSV;
397  break;
398  default:
399  hw_de_av_device_type = AV_HWDEVICE_TYPE_DXVA2;
400  break;
401  }
402 #elif defined(__APPLE__)
403  adapter_ptr = NULL;
404  i_decoder_hw = openshot::Settings::Instance()->HARDWARE_DECODER;
405  switch (i_decoder_hw) {
406  case 5:
407  hw_de_av_device_type = AV_HWDEVICE_TYPE_VIDEOTOOLBOX;
408  break;
409  case 7:
410  hw_de_av_device_type = AV_HWDEVICE_TYPE_QSV;
411  break;
412  default:
413  hw_de_av_device_type = AV_HWDEVICE_TYPE_VIDEOTOOLBOX;
414  break;
415  }
416 #endif
417 
418  } else {
419  adapter_ptr = NULL; // Just to be sure
420  }
421 
422  // Check if it is there and writable
423 #if defined(__linux__)
424  if( adapter_ptr != NULL && access( adapter_ptr, W_OK ) == 0 ) {
425 #elif defined(_WIN32)
426  if( adapter_ptr != NULL ) {
427 #elif defined(__APPLE__)
428  if( adapter_ptr != NULL ) {
429 #endif
430  ZmqLogger::Instance()->AppendDebugMethod("Decode Device present using device");
431  }
432  else {
433  adapter_ptr = NULL; // use default
434  ZmqLogger::Instance()->AppendDebugMethod("Decode Device not present using default");
435  }
436 
437  hw_device_ctx = NULL;
438  // Here the first hardware initialisations are made
439  if (av_hwdevice_ctx_create(&hw_device_ctx, hw_de_av_device_type, adapter_ptr, NULL, 0) >= 0) {
440  const char* hw_name = av_hwdevice_get_type_name(hw_de_av_device_type);
441  std::string hw_msg = "HW decode active: ";
442  hw_msg += (hw_name ? hw_name : "unknown");
443  ZmqLogger::Instance()->Log(hw_msg);
444  if (!(pCodecCtx->hw_device_ctx = av_buffer_ref(hw_device_ctx))) {
445  throw InvalidCodec("Hardware device reference create failed.", path);
446  }
447 
448  /*
449  av_buffer_unref(&ist->hw_frames_ctx);
450  ist->hw_frames_ctx = av_hwframe_ctx_alloc(hw_device_ctx);
451  if (!ist->hw_frames_ctx) {
452  av_log(avctx, AV_LOG_ERROR, "Error creating a CUDA frames context\n");
453  return AVERROR(ENOMEM);
454  }
455 
456  frames_ctx = (AVHWFramesContext*)ist->hw_frames_ctx->data;
457 
458  frames_ctx->format = AV_PIX_FMT_CUDA;
459  frames_ctx->sw_format = avctx->sw_pix_fmt;
460  frames_ctx->width = avctx->width;
461  frames_ctx->height = avctx->height;
462 
463  av_log(avctx, AV_LOG_DEBUG, "Initializing CUDA frames context: sw_format = %s, width = %d, height = %d\n",
464  av_get_pix_fmt_name(frames_ctx->sw_format), frames_ctx->width, frames_ctx->height);
465 
466 
467  ret = av_hwframe_ctx_init(pCodecCtx->hw_device_ctx);
468  ret = av_hwframe_ctx_init(ist->hw_frames_ctx);
469  if (ret < 0) {
470  av_log(avctx, AV_LOG_ERROR, "Error initializing a CUDA frame pool\n");
471  return ret;
472  }
473  */
474  }
475  else {
476  ZmqLogger::Instance()->Log("HW decode active: no (falling back to software)");
477  throw InvalidCodec("Hardware device create failed.", path);
478  }
479  }
480 #endif // USE_HW_ACCEL
481 
482  // Disable per-frame threading for album arts
483  // Using FF_THREAD_FRAME adds one frame decoding delay per thread,
484  // but there's only one frame in this case.
485  if (HasAlbumArt())
486  {
487  pCodecCtx->thread_type &= ~FF_THREAD_FRAME;
488  }
489 
490  // Open video codec
491  int avcodec_return = avcodec_open2(pCodecCtx, pCodec, &opts);
492  if (avcodec_return < 0) {
493  std::stringstream avcodec_error_msg;
494  avcodec_error_msg << "A video codec was found, but could not be opened. Error: " << av_err2string(avcodec_return);
495  throw InvalidCodec(avcodec_error_msg.str(), path);
496  }
497 
498 #if USE_HW_ACCEL
499  if (hw_de_on && hw_de_supported) {
500  AVHWFramesConstraints *constraints = NULL;
501  void *hwconfig = NULL;
502  hwconfig = av_hwdevice_hwconfig_alloc(hw_device_ctx);
503 
504 // TODO: needs va_config!
505 #if ENABLE_VAAPI
506  ((AVVAAPIHWConfig *)hwconfig)->config_id = ((VAAPIDecodeContext *)(pCodecCtx->priv_data))->va_config;
507  constraints = av_hwdevice_get_hwframe_constraints(hw_device_ctx,hwconfig);
508 #endif // ENABLE_VAAPI
509  if (constraints) {
510  if (pCodecCtx->coded_width < constraints->min_width ||
511  pCodecCtx->coded_height < constraints->min_height ||
512  pCodecCtx->coded_width > constraints->max_width ||
513  pCodecCtx->coded_height > constraints->max_height) {
514  ZmqLogger::Instance()->AppendDebugMethod("DIMENSIONS ARE TOO LARGE for hardware acceleration\n");
515  hw_de_supported = 0;
516  retry_decode_open = 1;
517  AV_FREE_CONTEXT(pCodecCtx);
518  if (hw_device_ctx) {
519  av_buffer_unref(&hw_device_ctx);
520  hw_device_ctx = NULL;
521  }
522  }
523  else {
524  // All is just peachy
525  ZmqLogger::Instance()->AppendDebugMethod("\nDecode hardware acceleration is used\n", "Min width :", constraints->min_width, "Min Height :", constraints->min_height, "MaxWidth :", constraints->max_width, "MaxHeight :", constraints->max_height, "Frame width :", pCodecCtx->coded_width, "Frame height :", pCodecCtx->coded_height);
526  retry_decode_open = 0;
527  }
528  av_hwframe_constraints_free(&constraints);
529  if (hwconfig) {
530  av_freep(&hwconfig);
531  }
532  }
533  else {
534  int max_h, max_w;
535  //max_h = ((getenv( "LIMIT_HEIGHT_MAX" )==NULL) ? MAX_SUPPORTED_HEIGHT : atoi(getenv( "LIMIT_HEIGHT_MAX" )));
536  max_h = openshot::Settings::Instance()->DE_LIMIT_HEIGHT_MAX;
537  //max_w = ((getenv( "LIMIT_WIDTH_MAX" )==NULL) ? MAX_SUPPORTED_WIDTH : atoi(getenv( "LIMIT_WIDTH_MAX" )));
538  max_w = openshot::Settings::Instance()->DE_LIMIT_WIDTH_MAX;
539  ZmqLogger::Instance()->AppendDebugMethod("Constraints could not be found using default limit\n");
540  //cerr << "Constraints could not be found using default limit\n";
541  if (pCodecCtx->coded_width < 0 ||
542  pCodecCtx->coded_height < 0 ||
543  pCodecCtx->coded_width > max_w ||
544  pCodecCtx->coded_height > max_h ) {
545  ZmqLogger::Instance()->AppendDebugMethod("DIMENSIONS ARE TOO LARGE for hardware acceleration\n", "Max Width :", max_w, "Max Height :", max_h, "Frame width :", pCodecCtx->coded_width, "Frame height :", pCodecCtx->coded_height);
546  hw_de_supported = 0;
547  retry_decode_open = 1;
548  AV_FREE_CONTEXT(pCodecCtx);
549  if (hw_device_ctx) {
550  av_buffer_unref(&hw_device_ctx);
551  hw_device_ctx = NULL;
552  }
553  }
554  else {
555  ZmqLogger::Instance()->AppendDebugMethod("\nDecode hardware acceleration is used\n", "Max Width :", max_w, "Max Height :", max_h, "Frame width :", pCodecCtx->coded_width, "Frame height :", pCodecCtx->coded_height);
556  retry_decode_open = 0;
557  }
558  }
559  } // if hw_de_on && hw_de_supported
560  else {
561  ZmqLogger::Instance()->AppendDebugMethod("\nDecode in software is used\n");
562  }
563 #else
564  retry_decode_open = 0;
565 #endif // USE_HW_ACCEL
566  } while (retry_decode_open); // retry_decode_open
567  // Free options
568  av_dict_free(&opts);
569 
570  // Update the File Info struct with video details (if a video stream is found)
571  UpdateVideoInfo();
572  }
573 
574  // Is there an audio stream?
575  if (audioStream != -1) {
576  // Set the stream index
577  info.audio_stream_index = audioStream;
578 
579  // Get a pointer to the codec context for the audio stream
580  aStream = pFormatCtx->streams[audioStream];
581 
582  // Find the codec ID from stream
583  AVCodecID codecId = AV_FIND_DECODER_CODEC_ID(aStream);
584 
585  // Get codec and codec context from stream
586  const AVCodec *aCodec = avcodec_find_decoder(codecId);
587  aCodecCtx = AV_GET_CODEC_CONTEXT(aStream, aCodec);
588 
589  // Audio encoding does not typically use more than 2 threads (most codecs use 1 thread)
590  aCodecCtx->thread_count = std::min(FF_AUDIO_NUM_PROCESSORS, 2);
591 
592  bool audio_opened = false;
593  if (aCodec != NULL) {
594  // Init options
595  AVDictionary *opts = NULL;
596  av_dict_set(&opts, "strict", "experimental", 0);
597 
598  // Open audio codec
599  audio_opened = (avcodec_open2(aCodecCtx, aCodec, &opts) >= 0);
600 
601  // Free options
602  av_dict_free(&opts);
603  }
604 
605  if (audio_opened) {
606  // Update the File Info struct with audio details (if an audio stream is found)
607  UpdateAudioInfo();
608 
609  // Disable malformed audio stream metadata (prevents divide-by-zero / invalid resampling math)
610  const bool invalid_audio_info =
611  (info.channels <= 0) ||
612  (info.sample_rate <= 0) ||
613  (info.audio_timebase.num <= 0) ||
614  (info.audio_timebase.den <= 0) ||
615  (aCodecCtx->sample_fmt == AV_SAMPLE_FMT_NONE);
616  if (invalid_audio_info) {
617  ZmqLogger::Instance()->AppendDebugMethod(
618  "FFmpegReader::Open (Disable invalid audio stream)",
619  "channels", info.channels,
620  "sample_rate", info.sample_rate,
621  "audio_timebase.num", info.audio_timebase.num,
622  "audio_timebase.den", info.audio_timebase.den,
623  "sample_fmt", static_cast<int>(aCodecCtx ? aCodecCtx->sample_fmt : AV_SAMPLE_FMT_NONE));
624  info.has_audio = false;
625  info.audio_stream_index = -1;
626  audioStream = -1;
627  packet_status.audio_eof = true;
628  if (aCodecCtx) {
629  if (avcodec_is_open(aCodecCtx)) {
630  avcodec_flush_buffers(aCodecCtx);
631  }
632  AV_FREE_CONTEXT(aCodecCtx);
633  aCodecCtx = nullptr;
634  }
635  aStream = nullptr;
636  }
637  } else {
638  // Keep decoding video, but disable bad/unsupported audio stream.
639  ZmqLogger::Instance()->AppendDebugMethod(
640  "FFmpegReader::Open (Audio codec unavailable; disabling audio)",
641  "audioStream", audioStream);
642  info.has_audio = false;
643  info.audio_stream_index = -1;
644  audioStream = -1;
645  packet_status.audio_eof = true;
646  if (aCodecCtx) {
647  AV_FREE_CONTEXT(aCodecCtx);
648  aCodecCtx = nullptr;
649  }
650  aStream = nullptr;
651  }
652  }
653 
654  // Guard invalid frame-rate / timebase values from malformed streams.
655  if (info.fps.num <= 0 || info.fps.den <= 0) {
656  ZmqLogger::Instance()->AppendDebugMethod(
657  "FFmpegReader::Open (Invalid FPS detected; applying fallback)",
658  "fps.num", info.fps.num,
659  "fps.den", info.fps.den);
660  info.fps.num = 30;
661  info.fps.den = 1;
662  }
663  if (info.video_timebase.num <= 0 || info.video_timebase.den <= 0) {
664  ZmqLogger::Instance()->AppendDebugMethod(
665  "FFmpegReader::Open (Invalid video_timebase detected; applying fallback)",
666  "video_timebase.num", info.video_timebase.num,
667  "video_timebase.den", info.video_timebase.den);
668  info.video_timebase = info.fps.Reciprocal();
669  }
670 
671  // Add format metadata (if any)
672  AVDictionaryEntry *tag = NULL;
673  while ((tag = av_dict_get(pFormatCtx->metadata, "", tag, AV_DICT_IGNORE_SUFFIX))) {
674  QString str_key = tag->key;
675  QString str_value = tag->value;
676  info.metadata[str_key.toStdString()] = str_value.trimmed().toStdString();
677  }
678 
679  // Process video stream side data (rotation, spherical metadata, etc)
680  for (unsigned int i = 0; i < pFormatCtx->nb_streams; i++) {
681  AVStream* st = pFormatCtx->streams[i];
682  if (st->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
683  size_t side_data_size = 0;
684  const uint8_t *displaymatrix = ffmpeg_stream_get_side_data(
685  st, AV_PKT_DATA_DISPLAYMATRIX, &side_data_size);
686  if (displaymatrix &&
687  side_data_size >= 9 * sizeof(int32_t) &&
688  !info.metadata.count("rotate")) {
689  double rotation = -av_display_rotation_get(
690  reinterpret_cast<const int32_t *>(displaymatrix));
691  if (std::isnan(rotation))
692  rotation = 0;
693  info.metadata["rotate"] = std::to_string(rotation);
694  }
695 
696  const uint8_t *spherical = ffmpeg_stream_get_side_data(
697  st, AV_PKT_DATA_SPHERICAL, &side_data_size);
698  if (spherical && side_data_size >= sizeof(AVSphericalMapping)) {
699  info.metadata["spherical"] = "1";
700 
701  const AVSphericalMapping *map =
702  reinterpret_cast<const AVSphericalMapping *>(spherical);
703  const char *proj_name = av_spherical_projection_name(map->projection);
704  info.metadata["spherical_projection"] = proj_name ? proj_name : "unknown";
705 
706  auto to_deg = [](int32_t v) {
707  return static_cast<double>(v) / 65536.0;
708  };
709  info.metadata["spherical_yaw"] = std::to_string(to_deg(map->yaw));
710  info.metadata["spherical_pitch"] = std::to_string(to_deg(map->pitch));
711  info.metadata["spherical_roll"] = std::to_string(to_deg(map->roll));
712  }
713  break;
714  }
715  }
716 
717  // Init previous audio location to zero
718  previous_packet_location.frame = -1;
719  previous_packet_location.sample_start = 0;
720 
721  // Adjust cache size based on size of frame and audio
722  const int working_cache_frames = std::max(Settings::Instance()->CACHE_MIN_FRAMES, int(OPEN_MP_NUM_PROCESSORS * info.fps.ToDouble() * 2));
723  const int final_cache_frames = std::max(Settings::Instance()->CACHE_MIN_FRAMES, OPEN_MP_NUM_PROCESSORS * 2);
724  working_cache.SetMaxBytesFromInfo(working_cache_frames, info.width, info.height, info.sample_rate, info.channels);
725  final_cache.SetMaxBytesFromInfo(final_cache_frames, info.width, info.height, info.sample_rate, info.channels);
726 
727  // Scan PTS for any offsets (i.e. non-zero starting streams). At least 1 stream must start at zero timestamp.
728  // This method allows us to shift timestamps to ensure at least 1 stream is starting at zero.
729  UpdatePTSOffset();
730 
731  // Override an invalid framerate
732  if (info.fps.ToFloat() > 240.0f || (info.fps.num <= 0 || info.fps.den <= 0) || info.video_length <= 0) {
733  // Calculate FPS, duration, video bit rate, and video length manually
734  // by scanning through all the video stream packets
735  CheckFPS();
736  }
737 
738  // Mark as "open"
739  is_open = true;
740 
741  // Seek back to beginning of file (if not already seeking)
742  if (!is_seeking) {
743  Seek(1);
744  }
745  }
746 }
747 
748 void FFmpegReader::Close() {
749  // Close all objects, if reader is 'open'
750  if (is_open) {
751  // Prevent async calls to the following code
752  const std::lock_guard<std::recursive_mutex> lock(getFrameMutex);
753 
754  // Mark as "closed"
755  is_open = false;
756 
757  // Keep track of most recent packet
758  AVPacket *recent_packet = packet;
759 
760  // Drain any packets from the decoder
761  packet = NULL;
762  int attempts = 0;
763  int max_attempts = 128;
764  while (packet_status.packets_decoded() < packet_status.packets_read() && attempts < max_attempts) {
765  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::Close (Drain decoder loop)",
766  "packets_read", packet_status.packets_read(),
767  "packets_decoded", packet_status.packets_decoded(),
768  "attempts", attempts);
769  if (packet_status.video_decoded < packet_status.video_read) {
770  ProcessVideoPacket(info.video_length);
771  }
772  if (packet_status.audio_decoded < packet_status.audio_read) {
773  ProcessAudioPacket(info.video_length);
774  }
775  attempts++;
776  }
777 
778  // Remove packet
779  if (recent_packet) {
780  RemoveAVPacket(recent_packet);
781  }
782 
783  // Close the video codec
784  if (info.has_video) {
785  if(avcodec_is_open(pCodecCtx)) {
786  avcodec_flush_buffers(pCodecCtx);
787  }
788  AV_FREE_CONTEXT(pCodecCtx);
789 #if USE_HW_ACCEL
790  if (hw_de_on) {
791  if (hw_device_ctx) {
792  av_buffer_unref(&hw_device_ctx);
793  hw_device_ctx = NULL;
794  }
795  }
796 #endif // USE_HW_ACCEL
797  if (img_convert_ctx) {
798  sws_freeContext(img_convert_ctx);
799  img_convert_ctx = nullptr;
800  }
801  if (pFrameRGB_cached) {
802  AV_FREE_FRAME(&pFrameRGB_cached);
803  }
804  }
805 
806  // Close the audio codec
807  if (info.has_audio) {
808  if(avcodec_is_open(aCodecCtx)) {
809  avcodec_flush_buffers(aCodecCtx);
810  }
811  AV_FREE_CONTEXT(aCodecCtx);
812  if (avr_ctx) {
813  SWR_CLOSE(avr_ctx);
814  SWR_FREE(&avr_ctx);
815  avr_ctx = nullptr;
816  }
817  }
818 
819  // Clear final cache
820  final_cache.Clear();
821  working_cache.Clear();
822 
823  // Close the video file
824  avformat_close_input(&pFormatCtx);
825  av_freep(&pFormatCtx);
826 
827  // Do not trim here; trimming is handled on explicit cache clears
828 
829  // Reset some variables
830  last_frame = 0;
831  hold_packet = false;
832  largest_frame_processed = 0;
833  seek_audio_frame_found = 0;
834  seek_video_frame_found = 0;
835  current_video_frame = 0;
836  last_video_frame.reset();
837  last_final_video_frame.reset();
838  }
839 }
840 
841 bool FFmpegReader::HasAlbumArt() {
842  // Check if the video stream we use is an attached picture
843  // This won't return true if the file has a cover image as a secondary stream
844  // like an MKV file with an attached image file
845  return pFormatCtx && videoStream >= 0 && pFormatCtx->streams[videoStream]
846  && (pFormatCtx->streams[videoStream]->disposition & AV_DISPOSITION_ATTACHED_PIC);
847 }
848 
849 double FFmpegReader::PickDurationSeconds() const {
850  auto has_value = [](double value) { return value > 0.0; };
851 
852  switch (duration_strategy) {
853  case DurationStrategy::VideoPreferred:
854  if (has_value(video_stream_duration_seconds))
855  return video_stream_duration_seconds;
856  if (has_value(audio_stream_duration_seconds))
857  return audio_stream_duration_seconds;
858  if (has_value(format_duration_seconds))
859  return format_duration_seconds;
860  break;
861  case DurationStrategy::AudioPreferred:
862  if (has_value(audio_stream_duration_seconds))
863  return audio_stream_duration_seconds;
864  if (has_value(video_stream_duration_seconds))
865  return video_stream_duration_seconds;
866  if (has_value(format_duration_seconds))
867  return format_duration_seconds;
868  break;
869  case DurationStrategy::LongestStream:
870  default:
871  {
872  double longest = 0.0;
873  if (has_value(video_stream_duration_seconds))
874  longest = std::max(longest, video_stream_duration_seconds);
875  if (has_value(audio_stream_duration_seconds))
876  longest = std::max(longest, audio_stream_duration_seconds);
877  if (has_value(format_duration_seconds))
878  longest = std::max(longest, format_duration_seconds);
879  if (has_value(longest))
880  return longest;
881  }
882  break;
883  }
884 
885  if (has_value(format_duration_seconds))
886  return format_duration_seconds;
887  if (has_value(inferred_duration_seconds))
888  return inferred_duration_seconds;
889 
890  return 0.0;
891 }
892 
893 void FFmpegReader::ApplyDurationStrategy() {
894  const double fps_value = info.fps.ToDouble();
895  const double chosen_seconds = PickDurationSeconds();
896 
897  if (chosen_seconds <= 0.0 || fps_value <= 0.0) {
898  info.duration = 0.0f;
899  info.video_length = 0;
900  is_duration_known = false;
901  return;
902  }
903 
904  const int64_t frames = static_cast<int64_t>(std::llround(chosen_seconds * fps_value));
905  if (frames <= 0) {
906  info.duration = 0.0f;
907  info.video_length = 0;
908  is_duration_known = false;
909  return;
910  }
911 
912  info.video_length = frames;
913  info.duration = static_cast<float>(static_cast<double>(frames) / fps_value);
914  is_duration_known = true;
915 }
916 
917 void FFmpegReader::UpdateAudioInfo() {
918  int codec_channels =
919 #if HAVE_CH_LAYOUT
920  AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->ch_layout.nb_channels;
921 #else
922  AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels;
923 #endif
924 
925  // Set default audio channel layout (if needed)
926 #if HAVE_CH_LAYOUT
927  AVChannelLayout audio_ch_layout = ffmpeg_get_valid_channel_layout(
928  AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->ch_layout, codec_channels);
929  if (audio_ch_layout.nb_channels > 0) {
930  av_channel_layout_uninit(&(AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->ch_layout));
931  av_channel_layout_copy(&(AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->ch_layout), &audio_ch_layout);
932  codec_channels = audio_ch_layout.nb_channels;
933  }
934 #else
935  if (codec_channels > 0 && AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout == 0)
936  AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout = av_get_default_channel_layout(AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels);
937 #endif
938 
939  if (info.sample_rate > 0) {
940  // Skip init - if info struct already populated
941  return;
942  }
943 
944  auto record_duration = [](double &target, double seconds) {
945  if (seconds > 0.0)
946  target = std::max(target, seconds);
947  };
948 
949  // Set values of FileInfo struct
950  info.has_audio = true;
951  info.file_size = pFormatCtx->pb ? avio_size(pFormatCtx->pb) : -1;
952  info.acodec = aCodecCtx->codec->name;
953 #if HAVE_CH_LAYOUT
954  info.channels = audio_ch_layout.nb_channels;
955  info.channel_layout = static_cast<ChannelLayout>(ffmpeg_channel_layout_mask(audio_ch_layout));
956 #else
957  info.channels = AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels;
958  info.channel_layout = (ChannelLayout) AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout;
959 #endif
960 
961  // If channel layout is not set, guess based on the number of channels
962  if (info.channel_layout == 0) {
963  if (info.channels == 1) {
964  info.channel_layout = openshot::LAYOUT_MONO;
965  } else if (info.channels == 2) {
966  info.channel_layout = openshot::LAYOUT_STEREO;
967  }
968  }
969 
970  info.sample_rate = AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->sample_rate;
971  info.audio_bit_rate = AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->bit_rate;
972  if (info.audio_bit_rate <= 0) {
973  // Get bitrate from format
974  info.audio_bit_rate = pFormatCtx->bit_rate;
975  }
976 
977  // Set audio timebase
978  info.audio_timebase.num = aStream->time_base.num;
979  info.audio_timebase.den = aStream->time_base.den;
980 
981  // Get timebase of audio stream (if valid) and greater than the current duration
982  if (aStream->duration > 0) {
983  record_duration(audio_stream_duration_seconds, aStream->duration * info.audio_timebase.ToDouble());
984  }
985  if (pFormatCtx->duration > 0) {
986  // Use the format's duration when stream duration is missing or shorter
987  record_duration(format_duration_seconds, static_cast<double>(pFormatCtx->duration) / AV_TIME_BASE);
988  }
989 
990  // Calculate duration from filesize and bitrate (if any)
991  if (info.duration <= 0.0f && info.video_bit_rate > 0 && info.file_size > 0) {
992  // Estimate from bitrate, total bytes, and framerate
993  record_duration(inferred_duration_seconds, static_cast<double>(info.file_size) / info.video_bit_rate);
994  }
995 
996  // Set video timebase (if no video stream was found)
997  if (!info.has_video) {
998  // Set a few important default video settings (so audio can be divided into frames)
999  info.fps.num = 30;
1000  info.fps.den = 1;
1001  info.video_timebase.num = 1;
1002  info.video_timebase.den = 30;
1003  info.width = 720;
1004  info.height = 480;
1005 
1006  // Use timeline to set correct width & height (if any)
1007  Clip *parent = static_cast<Clip *>(ParentClip());
1008  if (parent) {
1009  if (parent->ParentTimeline()) {
1010  // Set max width/height based on parent clip's timeline (if attached to a timeline)
1011  info.width = parent->ParentTimeline()->preview_width;
1012  info.height = parent->ParentTimeline()->preview_height;
1013  }
1014  }
1015  }
1016 
1017  ApplyDurationStrategy();
1018 
1019  // Add audio metadata (if any found)
1020  AVDictionaryEntry *tag = NULL;
1021  while ((tag = av_dict_get(aStream->metadata, "", tag, AV_DICT_IGNORE_SUFFIX))) {
1022  QString str_key = tag->key;
1023  QString str_value = tag->value;
1024  info.metadata[str_key.toStdString()] = str_value.trimmed().toStdString();
1025  }
1026 #if HAVE_CH_LAYOUT
1027  av_channel_layout_uninit(&audio_ch_layout);
1028 #endif
1029 }
1030 
1031 void FFmpegReader::UpdateVideoInfo() {
1032  if (info.vcodec.length() > 0) {
1033  // Skip init - if info struct already populated
1034  return;
1035  }
1036 
1037  auto record_duration = [](double &target, double seconds) {
1038  if (seconds > 0.0)
1039  target = std::max(target, seconds);
1040  };
1041 
1042  // Set values of FileInfo struct
1043  info.has_video = true;
1044  info.file_size = pFormatCtx->pb ? avio_size(pFormatCtx->pb) : -1;
1045  info.height = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->height;
1046  info.width = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->width;
1047  info.vcodec = pCodecCtx->codec->name;
1048  info.video_bit_rate = (pFormatCtx->bit_rate / 8);
1049 
1050  // Frame rate from the container and codec
1051  AVRational framerate = av_guess_frame_rate(pFormatCtx, pStream, NULL);
1052  if (!check_fps) {
1053  info.fps.num = framerate.num;
1054  info.fps.den = framerate.den;
1055  }
1056 
1057  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::UpdateVideoInfo", "info.fps.num", info.fps.num, "info.fps.den", info.fps.den);
1058 
1059  // TODO: remove excessive debug info in the next releases
1060  // The debug info below is just for comparison and troubleshooting on users side during the transition period
1061  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::UpdateVideoInfo (pStream->avg_frame_rate)", "num", pStream->avg_frame_rate.num, "den", pStream->avg_frame_rate.den);
1062 
1063  if (pStream->sample_aspect_ratio.num != 0) {
1064  info.pixel_ratio.num = pStream->sample_aspect_ratio.num;
1065  info.pixel_ratio.den = pStream->sample_aspect_ratio.den;
1066  } else if (AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->sample_aspect_ratio.num != 0) {
1067  info.pixel_ratio.num = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->sample_aspect_ratio.num;
1068  info.pixel_ratio.den = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->sample_aspect_ratio.den;
1069  } else {
1070  info.pixel_ratio.num = 1;
1071  info.pixel_ratio.den = 1;
1072  }
1073  info.pixel_format = AV_GET_CODEC_PIXEL_FORMAT(pStream, pCodecCtx);
1074 
1075  // Calculate the DAR (display aspect ratio)
1076  Fraction size(info.width * info.pixel_ratio.num, info.height * info.pixel_ratio.den);
1077 
1078  // Reduce size fraction
1079  size.Reduce();
1080 
1081  // Set the ratio based on the reduced fraction
1082  info.display_ratio.num = size.num;
1083  info.display_ratio.den = size.den;
1084 
1085  // Get scan type and order from codec context/params
1086  if (!check_interlace) {
1087  check_interlace = true;
1088  AVFieldOrder field_order = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->field_order;
1089  switch(field_order) {
1090  case AV_FIELD_PROGRESSIVE:
1091  info.interlaced_frame = false;
1092  break;
1093  case AV_FIELD_TT:
1094  case AV_FIELD_TB:
1095  info.interlaced_frame = true;
1096  info.top_field_first = true;
1097  break;
1098  case AV_FIELD_BT:
1099  case AV_FIELD_BB:
1100  info.interlaced_frame = true;
1101  info.top_field_first = false;
1102  break;
1103  case AV_FIELD_UNKNOWN:
1104  // Check again later?
1105  check_interlace = false;
1106  break;
1107  }
1108  // check_interlace will prevent these checks being repeated,
1109  // unless it was cleared because we got an AV_FIELD_UNKNOWN response.
1110  }
1111 
1112  // Set the video timebase
1113  info.video_timebase.num = pStream->time_base.num;
1114  info.video_timebase.den = pStream->time_base.den;
1115 
1116  // Set the duration in seconds, and video length (# of frames)
1117  record_duration(video_stream_duration_seconds, pStream->duration * info.video_timebase.ToDouble());
1118 
1119  // Check for valid duration (if found)
1120  if (pFormatCtx->duration >= 0) {
1121  // Use the format's duration as another candidate
1122  record_duration(format_duration_seconds, static_cast<double>(pFormatCtx->duration) / AV_TIME_BASE);
1123  }
1124 
1125  // Calculate duration from filesize and bitrate (if any)
1126  if (info.video_bit_rate > 0 && info.file_size > 0) {
1127  // Estimate from bitrate, total bytes, and framerate
1128  record_duration(inferred_duration_seconds, static_cast<double>(info.file_size) / info.video_bit_rate);
1129  }
1130 
1131  // Certain "image" formats do not have a valid duration
1132  if (video_stream_duration_seconds <= 0.0 && format_duration_seconds <= 0.0 &&
1133  pStream->duration == AV_NOPTS_VALUE && pFormatCtx->duration == AV_NOPTS_VALUE) {
1134  // Force an "image" duration
1135  record_duration(video_stream_duration_seconds, 60 * 60 * 1); // 1 hour duration
1136  info.has_single_image = true;
1137  }
1138  // Static GIFs can have no usable duration; fall back to a small default
1139  if (video_stream_duration_seconds <= 0.0 && format_duration_seconds <= 0.0 &&
1140  pFormatCtx && pFormatCtx->iformat && strcmp(pFormatCtx->iformat->name, "gif") == 0) {
1141  record_duration(video_stream_duration_seconds, 60 * 60 * 1); // 1 hour duration
1142  info.has_single_image = true;
1143  }
1144 
1145  ApplyDurationStrategy();
1146 
1147  // Normalize FFmpeg-decoded still images (e.g. JPG/JPEG) to match image-reader behavior.
1148  // This keeps timing/flags consistent regardless of which reader path was used.
1149  if (!info.has_single_image) {
1150  const AVCodecID codec_id = AV_FIND_DECODER_CODEC_ID(pStream);
1151  const bool likely_still_codec =
1152  codec_id == AV_CODEC_ID_MJPEG ||
1153  codec_id == AV_CODEC_ID_PNG ||
1154  codec_id == AV_CODEC_ID_BMP ||
1155  codec_id == AV_CODEC_ID_TIFF ||
1156  codec_id == AV_CODEC_ID_WEBP ||
1157  codec_id == AV_CODEC_ID_JPEG2000;
1158  const bool likely_image_demuxer =
1159  pFormatCtx && pFormatCtx->iformat && pFormatCtx->iformat->name &&
1160  strstr(pFormatCtx->iformat->name, "image2");
1161  const bool has_attached_pic = HasAlbumArt();
1162  const bool single_frame_stream =
1163  (pStream && pStream->nb_frames > 0 && pStream->nb_frames <= 1);
1164  const bool single_frame_clip = info.video_length <= 1;
1165 
1166  const bool is_still_image_video =
1167  has_attached_pic ||
1168  ((single_frame_stream || single_frame_clip) &&
1169  (likely_still_codec || likely_image_demuxer));
1170 
1171  if (is_still_image_video) {
1172  info.has_single_image = true;
1173 
1174  // Only force long duration for standalone images. For audio + attached-art
1175  // files, keep stream-derived duration so the cover image spans the audio.
1176  if (audioStream < 0) {
1177  record_duration(video_stream_duration_seconds, 60 * 60 * 1); // 1 hour duration
1178  }
1179 
1180  ApplyDurationStrategy();
1181  }
1182  }
1183 
1184  // Add video metadata (if any)
1185  AVDictionaryEntry *tag = NULL;
1186  while ((tag = av_dict_get(pStream->metadata, "", tag, AV_DICT_IGNORE_SUFFIX))) {
1187  QString str_key = tag->key;
1188  QString str_value = tag->value;
1189  info.metadata[str_key.toStdString()] = str_value.trimmed().toStdString();
1190  }
1191 }
1192 
1193 bool FFmpegReader::GetIsDurationKnown() {
1194  return this->is_duration_known;
1195 }
1196 
1197 std::shared_ptr<Frame> FFmpegReader::GetFrame(int64_t requested_frame) {
1198  last_seek_max_frame = -1;
1199  seek_stagnant_count = 0;
1200  // Check for open reader (or throw exception)
1201  if (!is_open)
1202  throw ReaderClosed("The FFmpegReader is closed. Call Open() before calling this method.", path);
1203 
1204  // Adjust for a requested frame that is too small or too large
1205  if (requested_frame < 1)
1206  requested_frame = 1;
1207  if (requested_frame > info.video_length && is_duration_known)
1208  requested_frame = info.video_length;
1209  if (info.has_video && info.video_length == 0)
1210  // Invalid duration of video file
1211  throw InvalidFile("Could not detect the duration of the video or audio stream.", path);
1212 
1213  // Debug output
1214  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetFrame", "requested_frame", requested_frame, "last_frame", last_frame);
1215 
1216  // Check the cache for this frame
1217  std::shared_ptr<Frame> frame = final_cache.GetFrame(requested_frame);
1218  if (frame) {
1219  // Debug output
1220  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetFrame", "returned cached frame", requested_frame);
1221  // Return the cached frame
1222  return frame;
1223  } else {
1224 
1225  // Prevent async calls to the remainder of this code
1226  const std::lock_guard<std::recursive_mutex> lock(getFrameMutex);
1227 
1228  // Check the cache a 2nd time (due to the potential previous lock)
1229  frame = final_cache.GetFrame(requested_frame);
1230  if (frame) {
1231  // Debug output
1232  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetFrame", "returned cached frame on 2nd look", requested_frame);
1233  } else {
1234  // Frame is not in cache
1235  // Reset seek count
1236  seek_count = 0;
1237 
1238  // Are we within X frames of the requested frame?
1239  int64_t diff = requested_frame - last_frame;
1240  if (diff >= 1 && diff <= 20) {
1241  // Continue walking the stream
1242  frame = ReadStream(requested_frame);
1243  } else {
1244  // Greater than 30 frames away, or backwards, we need to seek to the nearest key frame
1245  if (enable_seek) {
1246  // Only seek if enabled
1247  Seek(requested_frame);
1248 
1249  } else if (!enable_seek && diff < 0) {
1250  // Start over, since we can't seek, and the requested frame is smaller than our position
1251  // Since we are seeking to frame 1, this actually just closes/re-opens the reader
1252  Seek(1);
1253  }
1254 
1255  // Then continue walking the stream
1256  frame = ReadStream(requested_frame);
1257  }
1258  }
1259  return frame;
1260  }
1261 }
1262 
1263 // Read the stream until we find the requested Frame
1264 std::shared_ptr<Frame> FFmpegReader::ReadStream(int64_t requested_frame) {
1265  // Allocate video frame
1266  bool check_seek = false;
1267  int packet_error = -1;
1268  int64_t no_progress_count = 0;
1269  int64_t prev_packets_read = packet_status.packets_read();
1270  int64_t prev_packets_decoded = packet_status.packets_decoded();
1271  int64_t prev_video_decoded = packet_status.video_decoded;
1272  double prev_video_pts_seconds = video_pts_seconds;
1273 
1274  // Debug output
1275  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream", "requested_frame", requested_frame);
1276 
1277  // Loop through the stream until the correct frame is found
1278  while (true) {
1279  // Check if working frames are 'finished'
1280  if (!is_seeking) {
1281  // Check for final frames
1282  CheckWorkingFrames(requested_frame);
1283  }
1284 
1285  // Check if requested 'final' frame is available (and break out of loop if found)
1286  bool is_cache_found = (final_cache.GetFrame(requested_frame) != NULL);
1287  if (is_cache_found) {
1288  break;
1289  }
1290 
1291  if (!hold_packet || !packet) {
1292  // Get the next packet
1293  packet_error = GetNextPacket();
1294  if (packet_error < 0 && !packet) {
1295  // No more packets to be found
1296  packet_status.packets_eof = true;
1297  }
1298  }
1299 
1300  // Debug output
1301  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream (GetNextPacket)", "requested_frame", requested_frame,"packets_read", packet_status.packets_read(), "packets_decoded", packet_status.packets_decoded(), "is_seeking", is_seeking);
1302 
1303  // Check the status of a seek (if any)
1304  if (is_seeking) {
1305  check_seek = CheckSeek();
1306  } else {
1307  check_seek = false;
1308  }
1309 
1310  if (check_seek) {
1311  // Packet may become NULL on Close inside Seek if CheckSeek returns false
1312  // Jump to the next iteration of this loop
1313  continue;
1314  }
1315 
1316  // Video packet
1317  if ((info.has_video && packet && packet->stream_index == videoStream) ||
1318  (info.has_video && packet_status.video_decoded < packet_status.video_read) ||
1319  (info.has_video && !packet && !packet_status.video_eof)) {
1320  // Process Video Packet
1321  ProcessVideoPacket(requested_frame);
1322  if (ReopenWithoutHardwareDecode(requested_frame)) {
1323  continue;
1324  }
1325  }
1326  // Audio packet
1327  if ((info.has_audio && packet && packet->stream_index == audioStream) ||
1328  (info.has_audio && !packet && packet_status.audio_decoded < packet_status.audio_read) ||
1329  (info.has_audio && !packet && !packet_status.audio_eof)) {
1330  // Process Audio Packet
1331  ProcessAudioPacket(requested_frame);
1332  }
1333 
1334  // Remove unused packets (sometimes we purposely ignore video or audio packets,
1335  // if the has_video or has_audio properties are manually overridden)
1336  if ((!info.has_video && packet && packet->stream_index == videoStream) ||
1337  (!info.has_audio && packet && packet->stream_index == audioStream)) {
1338  // Keep track of deleted packet counts
1339  if (packet->stream_index == videoStream) {
1340  packet_status.video_decoded++;
1341  } else if (packet->stream_index == audioStream) {
1342  packet_status.audio_decoded++;
1343  }
1344 
1345  // Remove unused packets (sometimes we purposely ignore video or audio packets,
1346  // if the has_video or has_audio properties are manually overridden)
1347  RemoveAVPacket(packet);
1348  packet = NULL;
1349  }
1350 
1351  // Determine end-of-stream (waiting until final decoder threads finish)
1352  // Force end-of-stream in some situations
1353  packet_status.end_of_file = packet_status.packets_eof && packet_status.video_eof && packet_status.audio_eof;
1354  if ((packet_status.packets_eof && packet_status.packets_read() == packet_status.packets_decoded()) || packet_status.end_of_file) {
1355  // Force EOF (end of file) variables to true, if decoder does not support EOF detection.
1356  // If we have no more packets, and all known packets have been decoded
1357  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream (force EOF)", "packets_read", packet_status.packets_read(), "packets_decoded", packet_status.packets_decoded(), "packets_eof", packet_status.packets_eof, "video_eof", packet_status.video_eof, "audio_eof", packet_status.audio_eof, "end_of_file", packet_status.end_of_file);
1358  if (!packet_status.video_eof) {
1359  packet_status.video_eof = true;
1360  }
1361  if (!packet_status.audio_eof) {
1362  packet_status.audio_eof = true;
1363  }
1364  packet_status.end_of_file = true;
1365  break;
1366  }
1367 
1368  // Detect decoder stalls with no progress at EOF and force completion so
1369  // missing frames can be finalized from prior image data.
1370  const bool has_progress =
1371  (packet_status.packets_read() != prev_packets_read) ||
1372  (packet_status.packets_decoded() != prev_packets_decoded) ||
1373  (packet_status.video_decoded != prev_video_decoded) ||
1374  (video_pts_seconds != prev_video_pts_seconds);
1375 
1376  if (has_progress) {
1377  no_progress_count = 0;
1378  } else {
1379  no_progress_count++;
1380  if (no_progress_count >= 2000
1381  && packet_status.packets_eof
1382  && !packet
1383  && !hold_packet) {
1384  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream (force EOF after stall)",
1385  "requested_frame", requested_frame,
1386  "no_progress_count", no_progress_count,
1387  "packets_read", packet_status.packets_read(),
1388  "packets_decoded", packet_status.packets_decoded(),
1389  "video_decoded", packet_status.video_decoded,
1390  "audio_decoded", packet_status.audio_decoded);
1391  packet_status.video_eof = true;
1392  packet_status.audio_eof = true;
1393  packet_status.end_of_file = true;
1394  break;
1395  }
1396  }
1397  prev_packets_read = packet_status.packets_read();
1398  prev_packets_decoded = packet_status.packets_decoded();
1399  prev_video_decoded = packet_status.video_decoded;
1400  prev_video_pts_seconds = video_pts_seconds;
1401  } // end while
1402 
1403  // Debug output
1404  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream (Completed)",
1405  "packets_read", packet_status.packets_read(),
1406  "packets_decoded", packet_status.packets_decoded(),
1407  "end_of_file", packet_status.end_of_file,
1408  "largest_frame_processed", largest_frame_processed,
1409  "Working Cache Count", working_cache.Count());
1410 
1411  // Have we reached end-of-stream (or the final frame)?
1412  if (!packet_status.end_of_file && requested_frame >= info.video_length) {
1413  // Force end-of-stream
1414  packet_status.end_of_file = true;
1415  }
1416  if (packet_status.end_of_file) {
1417  // Mark any other working frames as 'finished'
1418  CheckWorkingFrames(requested_frame);
1419  }
1420 
1421  // Return requested frame (if found)
1422  std::shared_ptr<Frame> frame = final_cache.GetFrame(requested_frame);
1423  if (frame)
1424  // Return prepared frame
1425  return frame;
1426  else {
1427 
1428  // Check if largest frame is still cached
1429  frame = final_cache.GetFrame(largest_frame_processed);
1430  int samples_in_frame = Frame::GetSamplesPerFrame(requested_frame, info.fps,
1431  info.sample_rate, info.channels);
1432  if (frame) {
1433  // Copy and return the largest processed frame (assuming it was the last in the video file)
1434  std::shared_ptr<Frame> f = CreateFrame(largest_frame_processed);
1435  if (frame->has_image_data) {
1436  f->AddImage(std::make_shared<QImage>(frame->GetImage()->copy()));
1437  }
1438 
1439  // Use solid color (if no image data found)
1440  if (!frame->has_image_data) {
1441  // Use solid black frame if no image data available
1442  f->AddColor(info.width, info.height, "#000");
1443  }
1444  // Silence audio data (if any), since we are repeating the last frame
1445  f->AddAudioSilence(samples_in_frame);
1446 
1447  return f;
1448  } else {
1449  // The largest processed frame is no longer in cache. Prefer the most recent
1450  // finalized image first, then decoded image, to avoid black flashes.
1451  std::shared_ptr<Frame> f = CreateFrame(largest_frame_processed);
1452  if (last_final_video_frame && last_final_video_frame->has_image_data
1453  && last_final_video_frame->number <= requested_frame) {
1454  f->AddImage(std::make_shared<QImage>(last_final_video_frame->GetImage()->copy()));
1455  } else if (last_video_frame && last_video_frame->has_image_data
1456  && last_video_frame->number <= requested_frame) {
1457  f->AddImage(std::make_shared<QImage>(last_video_frame->GetImage()->copy()));
1458  } else {
1459  f->AddColor(info.width, info.height, "#000");
1460  }
1461  f->AddAudioSilence(samples_in_frame);
1462  return f;
1463  }
1464  }
1465 
1466 }
1467 
1468 // Get the next packet (if any)
1469 int FFmpegReader::GetNextPacket() {
1470  int found_packet = 0;
1471  AVPacket *next_packet;
1472  next_packet = new AVPacket();
1473  found_packet = av_read_frame(pFormatCtx, next_packet);
1474 
1475  if (packet) {
1476  // Remove previous packet before getting next one
1477  RemoveAVPacket(packet);
1478  packet = NULL;
1479  }
1480  if (found_packet >= 0) {
1481  // Update current packet pointer
1482  packet = next_packet;
1483 
1484  // Keep track of packet stats
1485  if (packet->stream_index == videoStream) {
1486  packet_status.video_read++;
1487  } else if (packet->stream_index == audioStream) {
1488  packet_status.audio_read++;
1489  }
1490  } else {
1491  // No more packets found
1492  delete next_packet;
1493  packet = NULL;
1494  }
1495  // Return if packet was found (or error number)
1496  return found_packet;
1497 }
1498 
1499 // Get an AVFrame (if any)
1500 bool FFmpegReader::GetAVFrame() {
1501  int frameFinished = 0;
1502  auto note_hw_decode_failure = [&](int err, const char* stage) {
1503 #if USE_HW_ACCEL
1504  if (!hw_de_on || !hw_de_supported || force_sw_decode) {
1505  return;
1506  }
1507  if (err == AVERROR_INVALIDDATA && packet_status.video_decoded == 0) {
1508  hw_decode_error_count++;
1509  ZmqLogger::Instance()->AppendDebugMethod(
1510  std::string("FFmpegReader::GetAVFrame (hardware decode failure candidate during ") + stage + ")",
1511  "error_count", hw_decode_error_count,
1512  "error", err);
1513  if (hw_decode_error_count >= 3) {
1514  hw_decode_failed = true;
1515  }
1516  }
1517 #else
1518  (void) err;
1519  (void) stage;
1520 #endif
1521  };
1522 
1523  // Decode video frame
1524  AVFrame *next_frame = AV_ALLOCATE_FRAME();
1525 
1526 #if IS_FFMPEG_3_2
1527  int send_packet_err = 0;
1528  int64_t send_packet_pts = 0;
1529  if ((packet && packet->stream_index == videoStream) || !packet) {
1530  send_packet_err = avcodec_send_packet(pCodecCtx, packet);
1531 
1532  if (packet && send_packet_err >= 0) {
1533  send_packet_pts = GetPacketPTS();
1534  hold_packet = false;
1535  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (send packet succeeded)", "send_packet_err", send_packet_err, "send_packet_pts", send_packet_pts);
1536  }
1537  }
1538 
1539  #if USE_HW_ACCEL
1540  // Get the format from the variables set in get_hw_dec_format
1541  hw_de_av_pix_fmt = hw_de_av_pix_fmt_global;
1542  hw_de_av_device_type = hw_de_av_device_type_global;
1543  #endif // USE_HW_ACCEL
1544  if (send_packet_err < 0 && send_packet_err != AVERROR_EOF) {
1545  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (send packet: Not sent [" + av_err2string(send_packet_err) + "])", "send_packet_err", send_packet_err, "send_packet_pts", send_packet_pts);
1546  note_hw_decode_failure(send_packet_err, "send_packet");
1547  if (send_packet_err == AVERROR(EAGAIN)) {
1548  hold_packet = true;
1549  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (send packet: AVERROR(EAGAIN): user must read output with avcodec_receive_frame()", "send_packet_pts", send_packet_pts);
1550  }
1551  if (send_packet_err == AVERROR(EINVAL)) {
1552  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (send packet: AVERROR(EINVAL): codec not opened, it is an encoder, or requires flush", "send_packet_pts", send_packet_pts);
1553  }
1554  if (send_packet_err == AVERROR(ENOMEM)) {
1555  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (send packet: AVERROR(ENOMEM): failed to add packet to internal queue, or legitimate decoding errors", "send_packet_pts", send_packet_pts);
1556  }
1557  }
1558 
1559  // Always try and receive a packet, if not EOF.
1560  // Even if the above avcodec_send_packet failed to send,
1561  // we might still need to receive a packet.
1562  int receive_frame_err = 0;
1563  AVFrame *decoded_frame = next_frame;
1564  AVFrame *next_frame2;
1565 #if USE_HW_ACCEL
1566  if (hw_de_on && hw_de_supported) {
1567  next_frame2 = AV_ALLOCATE_FRAME();
1568  }
1569  else
1570 #endif // USE_HW_ACCEL
1571  {
1572  next_frame2 = next_frame;
1573  }
1574  pFrame = AV_ALLOCATE_FRAME();
1575  while (receive_frame_err >= 0) {
1576  receive_frame_err = avcodec_receive_frame(pCodecCtx, next_frame2);
1577 
1578  if (receive_frame_err != 0) {
1579  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (receive frame: frame not ready yet from decoder [\" + av_err2string(receive_frame_err) + \"])", "receive_frame_err", receive_frame_err, "send_packet_pts", send_packet_pts);
1580  note_hw_decode_failure(receive_frame_err, "receive_frame");
1581 
1582  if (receive_frame_err == AVERROR_EOF) {
1583  ZmqLogger::Instance()->AppendDebugMethod(
1584  "FFmpegReader::GetAVFrame (receive frame: AVERROR_EOF: EOF detected from decoder, flushing buffers)", "send_packet_pts", send_packet_pts);
1585  avcodec_flush_buffers(pCodecCtx);
1586  packet_status.video_eof = true;
1587  }
1588  if (receive_frame_err == AVERROR(EINVAL)) {
1589  ZmqLogger::Instance()->AppendDebugMethod(
1590  "FFmpegReader::GetAVFrame (receive frame: AVERROR(EINVAL): invalid frame received, flushing buffers)", "send_packet_pts", send_packet_pts);
1591  avcodec_flush_buffers(pCodecCtx);
1592  }
1593  if (receive_frame_err == AVERROR(EAGAIN)) {
1594  ZmqLogger::Instance()->AppendDebugMethod(
1595  "FFmpegReader::GetAVFrame (receive frame: AVERROR(EAGAIN): output is not available in this state - user must try to send new input)", "send_packet_pts", send_packet_pts);
1596  }
1597  if (receive_frame_err == AVERROR_INPUT_CHANGED) {
1598  ZmqLogger::Instance()->AppendDebugMethod(
1599  "FFmpegReader::GetAVFrame (receive frame: AVERROR_INPUT_CHANGED: current decoded frame has changed parameters with respect to first decoded frame)", "send_packet_pts", send_packet_pts);
1600  }
1601 
1602  // Break out of decoding loop
1603  // Nothing ready for decoding yet
1604  break;
1605  }
1606 
1607 #if USE_HW_ACCEL
1608  if (hw_de_on && hw_de_supported) {
1609  int err;
1610  if (next_frame2->format == hw_de_av_pix_fmt) {
1611  if ((err = av_hwframe_transfer_data(next_frame, next_frame2, 0)) < 0) {
1612  ZmqLogger::Instance()->AppendDebugMethod(
1613  "FFmpegReader::GetAVFrame (Failed to transfer data to output frame)",
1614  "hw_de_on", hw_de_on,
1615  "error", err);
1616  note_hw_decode_failure(AVERROR_INVALIDDATA, "hwframe_transfer");
1617  break;
1618  }
1619  if ((err = av_frame_copy_props(next_frame, next_frame2)) < 0) {
1620  ZmqLogger::Instance()->AppendDebugMethod(
1621  "FFmpegReader::GetAVFrame (Failed to copy props to output frame)",
1622  "hw_de_on", hw_de_on,
1623  "error", err);
1624  note_hw_decode_failure(AVERROR_INVALIDDATA, "hwframe_copy_props");
1625  break;
1626  }
1627  if (next_frame->format == AV_PIX_FMT_NONE) {
1628  next_frame->format = pCodecCtx->sw_pix_fmt;
1629  }
1630  if (next_frame->width <= 0) {
1631  next_frame->width = next_frame2->width;
1632  }
1633  if (next_frame->height <= 0) {
1634  next_frame->height = next_frame2->height;
1635  }
1636  decoded_frame = next_frame;
1637  } else {
1638  // Some hardware decoders can still return software-readable frames.
1639  decoded_frame = next_frame2;
1640  }
1641  }
1642  else
1643 #endif // USE_HW_ACCEL
1644  { // No hardware acceleration used -> no copy from GPU memory needed
1645  decoded_frame = next_frame2;
1646  }
1647 
1648  if (!decoded_frame->data[0]) {
1649  ZmqLogger::Instance()->AppendDebugMethod(
1650  "FFmpegReader::GetAVFrame (Decoded frame missing image data)",
1651  "format", decoded_frame->format,
1652  "width", decoded_frame->width,
1653  "height", decoded_frame->height);
1654  note_hw_decode_failure(AVERROR_INVALIDDATA, "decoded_frame_empty");
1655  break;
1656  }
1657 
1658  // TODO also handle possible further frames
1659  // Use only the first frame like avcodec_decode_video2
1660  frameFinished = 1;
1661  hw_decode_error_count = 0;
1662 #if USE_HW_ACCEL
1663  if (hw_de_on && hw_de_supported && !force_sw_decode) {
1664  hw_decode_succeeded = true;
1665  }
1666 #endif
1667  packet_status.video_decoded++;
1668 
1669  // Allocate image (align 32 for simd)
1670  AVPixelFormat decoded_pix_fmt = (AVPixelFormat)(decoded_frame->format);
1671  if (decoded_pix_fmt == AV_PIX_FMT_NONE)
1672  decoded_pix_fmt = (AVPixelFormat)(pStream->codecpar->format);
1673  if (AV_ALLOCATE_IMAGE(pFrame, decoded_pix_fmt, info.width, info.height) <= 0) {
1674  throw OutOfMemory("Failed to allocate image buffer", path);
1675  }
1676  av_image_copy(pFrame->data, pFrame->linesize, (const uint8_t**)decoded_frame->data, decoded_frame->linesize,
1677  decoded_pix_fmt, info.width, info.height);
1678  pFrame->format = decoded_pix_fmt;
1679  pFrame->width = info.width;
1680  pFrame->height = info.height;
1681  pFrame->color_range = decoded_frame->color_range;
1682  pFrame->colorspace = decoded_frame->colorspace;
1683  pFrame->color_primaries = decoded_frame->color_primaries;
1684  pFrame->color_trc = decoded_frame->color_trc;
1685  pFrame->chroma_location = decoded_frame->chroma_location;
1686 
1687  // Get display PTS from video frame, often different than packet->pts.
1688  // Sending packets to the decoder (i.e. packet->pts) is async,
1689  // and retrieving packets from the decoder (frame->pts) is async. In most decoders
1690  // sending and retrieving are separated by multiple calls to this method.
1691  if (decoded_frame->pts != AV_NOPTS_VALUE) {
1692  // This is the current decoded frame (and should be the pts used) for
1693  // processing this data
1694  video_pts = decoded_frame->pts;
1695  } else if (decoded_frame->pkt_dts != AV_NOPTS_VALUE) {
1696  // Some videos only set this timestamp (fallback)
1697  video_pts = decoded_frame->pkt_dts;
1698  }
1699 
1700  ZmqLogger::Instance()->AppendDebugMethod(
1701  "FFmpegReader::GetAVFrame (Successful frame received)", "video_pts", video_pts, "send_packet_pts", send_packet_pts);
1702 
1703  // break out of loop after each successful image returned
1704  break;
1705  }
1706 #if USE_HW_ACCEL
1707  if (hw_de_on && hw_de_supported && next_frame2 != next_frame) {
1708  AV_FREE_FRAME(&next_frame2);
1709  }
1710  #endif // USE_HW_ACCEL
1711 #else
1712  avcodec_decode_video2(pCodecCtx, next_frame, &frameFinished, packet);
1713 
1714  // always allocate pFrame (because we do that in the ffmpeg >= 3.2 as well); it will always be freed later
1715  pFrame = AV_ALLOCATE_FRAME();
1716 
1717  // is frame finished
1718  if (frameFinished) {
1719  // AVFrames are clobbered on the each call to avcodec_decode_video, so we
1720  // must make a copy of the image data before this method is called again.
1721  avpicture_alloc((AVPicture *) pFrame, pCodecCtx->pix_fmt, info.width, info.height);
1722  av_picture_copy((AVPicture *) pFrame, (AVPicture *) next_frame, pCodecCtx->pix_fmt, info.width,
1723  info.height);
1724  }
1725 #endif // IS_FFMPEG_3_2
1726 
1727  // deallocate the frame
1728  AV_FREE_FRAME(&next_frame);
1729 
1730  // Did we get a video frame?
1731  return frameFinished;
1732 }
1733 
1734 bool FFmpegReader::ReopenWithoutHardwareDecode(int64_t requested_frame) {
1735 #if USE_HW_ACCEL
1736  if (!hw_decode_failed || force_sw_decode) {
1737  return false;
1738  }
1739 
1740  ZmqLogger::Instance()->AppendDebugMethod(
1741  "FFmpegReader::ReopenWithoutHardwareDecode (falling back to software decode)",
1742  "requested_frame", requested_frame,
1743  "video_packets_read", packet_status.video_read,
1744  "video_packets_decoded", packet_status.video_decoded,
1745  "hw_decode_error_count", hw_decode_error_count);
1746 
1747  force_sw_decode = true;
1748  hw_decode_failed = false;
1749  hw_decode_error_count = 0;
1750 
1751  Close();
1752  Open();
1753  Seek(requested_frame);
1754  return true;
1755 #else
1756  (void) requested_frame;
1757  return false;
1758 #endif
1759 }
1760 
1761 bool FFmpegReader::HardwareDecodeSuccessful() const {
1762 #if USE_HW_ACCEL
1763  return hw_decode_succeeded;
1764 #else
1765  return false;
1766 #endif
1767 }
1768 
1769 // Check the current seek position and determine if we need to seek again
1770 bool FFmpegReader::CheckSeek() {
1771  // Are we seeking for a specific frame?
1772  if (is_seeking) {
1773  const int64_t kSeekRetryMax = 5;
1774  const int kSeekStagnantMax = 2;
1775 
1776  // Determine if both an audio and video packet have been decoded since the seek happened.
1777  // If not, allow the ReadStream method to keep looping
1778  if ((is_video_seek && !seek_video_frame_found) || (!is_video_seek && !seek_audio_frame_found))
1779  return false;
1780 
1781  // Check for both streams
1782  if ((info.has_video && !seek_video_frame_found) || (info.has_audio && !seek_audio_frame_found))
1783  return false;
1784 
1785  // Determine max seeked frame
1786  int64_t max_seeked_frame = std::max(seek_audio_frame_found, seek_video_frame_found);
1787  // Track stagnant seek results (no progress between retries)
1788  if (max_seeked_frame == last_seek_max_frame) {
1789  seek_stagnant_count++;
1790  } else {
1791  last_seek_max_frame = max_seeked_frame;
1792  seek_stagnant_count = 0;
1793  }
1794 
1795  // determine if we are "before" the requested frame
1796  if (max_seeked_frame >= seeking_frame) {
1797  // SEEKED TOO FAR
1798  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckSeek (Too far, seek again)",
1799  "is_video_seek", is_video_seek,
1800  "max_seeked_frame", max_seeked_frame,
1801  "seeking_frame", seeking_frame,
1802  "seeking_pts", seeking_pts,
1803  "seek_video_frame_found", seek_video_frame_found,
1804  "seek_audio_frame_found", seek_audio_frame_found);
1805 
1806  // Seek again... to the nearest Keyframe
1807  if (seek_count < kSeekRetryMax) {
1808  Seek(seeking_frame - (10 * seek_count * seek_count));
1809  } else if (seek_stagnant_count >= kSeekStagnantMax) {
1810  // Stagnant seek: force a much earlier target and keep seeking.
1811  Seek(seeking_frame - (10 * kSeekRetryMax * kSeekRetryMax));
1812  } else {
1813  // Retry budget exhausted: keep seeking from a conservative offset.
1814  Seek(seeking_frame - (10 * seek_count * seek_count));
1815  }
1816  } else {
1817  // SEEK WORKED
1818  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckSeek (Successful)",
1819  "is_video_seek", is_video_seek,
1820  "packet->pts", GetPacketPTS(),
1821  "seeking_pts", seeking_pts,
1822  "seeking_frame", seeking_frame,
1823  "seek_video_frame_found", seek_video_frame_found,
1824  "seek_audio_frame_found", seek_audio_frame_found);
1825 
1826  // Seek worked, and we are "before" the requested frame
1827  is_seeking = false;
1828  seeking_frame = 0;
1829  seeking_pts = -1;
1830  }
1831  }
1832 
1833  // return the pts to seek to (if any)
1834  return is_seeking;
1835 }
1836 
1837 // Process a video packet
1838 void FFmpegReader::ProcessVideoPacket(int64_t requested_frame) {
1839  // Get the AVFrame from the current packet
1840  // This sets the video_pts to the correct timestamp
1841  int frame_finished = GetAVFrame();
1842 
1843  // Check if the AVFrame is finished and set it
1844  if (!frame_finished) {
1845  // No AVFrame decoded yet, bail out
1846  if (pFrame) {
1847  RemoveAVFrame(pFrame);
1848  }
1849  return;
1850  }
1851 
1852  // Calculate current frame #
1853  int64_t current_frame = ConvertVideoPTStoFrame(video_pts);
1854 
1855  // Track 1st video packet after a successful seek
1856  if (!seek_video_frame_found && is_seeking)
1857  seek_video_frame_found = current_frame;
1858 
1859  // Create or get the existing frame object. Requested frame needs to be created
1860  // in working_cache at least once. Seek can clear the working_cache, so we must
1861  // add the requested frame back to the working_cache here. If it already exists,
1862  // it will be moved to the top of the working_cache.
1863  working_cache.Add(CreateFrame(requested_frame));
1864 
1865  // Debug output
1866  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessVideoPacket (Before)", "requested_frame", requested_frame, "current_frame", current_frame);
1867 
1868  // Init some things local (for OpenMP)
1869  AVPixelFormat decoded_pix_fmt = (pFrame && pFrame->format != AV_PIX_FMT_NONE)
1870  ? static_cast<AVPixelFormat>(pFrame->format)
1871  : AV_GET_CODEC_PIXEL_FORMAT(pStream, pCodecCtx);
1872  bool src_full_range = (pFrame && pFrame->color_range == AVCOL_RANGE_JPEG);
1873  AVPixelFormat src_pix_fmt = NormalizeDeprecatedPixFmt(decoded_pix_fmt, src_full_range);
1874  int src_width = (pFrame && pFrame->width > 0) ? pFrame->width : info.width;
1875  int src_height = (pFrame && pFrame->height > 0) ? pFrame->height : info.height;
1876  int height = src_height;
1877  int width = src_width;
1878  // Create or reuse a RGB Frame (since most videos are not in RGB, we must convert it)
1879  AVFrame *pFrameRGB = pFrameRGB_cached;
1880  if (!pFrameRGB) {
1881  pFrameRGB = AV_ALLOCATE_FRAME();
1882  if (pFrameRGB == nullptr)
1883  throw OutOfMemory("Failed to allocate frame buffer", path);
1884  pFrameRGB_cached = pFrameRGB;
1885  }
1886  AV_RESET_FRAME(pFrameRGB);
1887  uint8_t *buffer = nullptr;
1888 
1889  // Determine the max size of this source image (based on the timeline's size, the scaling mode,
1890  // and the scaling keyframes). This is a performance improvement, to keep the images as small as possible,
1891  // without losing quality. NOTE: We cannot go smaller than the timeline itself, or the add_layer timeline
1892  // method will scale it back to timeline size before scaling it smaller again. This needs to be fixed in
1893  // the future.
1894  int max_width = info.width;
1895  int max_height = info.height;
1896 
1897  Clip *parent = static_cast<Clip *>(ParentClip());
1898  if (parent) {
1899  if (parent->ParentTimeline()) {
1900  // Set max width/height based on parent clip's timeline (if attached to a timeline)
1901  max_width = parent->ParentTimeline()->preview_width;
1902  max_height = parent->ParentTimeline()->preview_height;
1903  }
1904  if (parent->scale == SCALE_FIT || parent->scale == SCALE_STRETCH) {
1905  // Best fit or Stretch scaling (based on max timeline size * scaling keyframes)
1906  float max_scale_x = parent->scale_x.GetMaxPoint().co.Y;
1907  float max_scale_y = parent->scale_y.GetMaxPoint().co.Y;
1908  max_width = std::max(float(max_width), max_width * max_scale_x);
1909  max_height = std::max(float(max_height), max_height * max_scale_y);
1910 
1911  } else if (parent->scale == SCALE_CROP) {
1912  // Cropping scale mode (based on max timeline size * cropped size * scaling keyframes)
1913  float max_scale_x = parent->scale_x.GetMaxPoint().co.Y;
1914  float max_scale_y = parent->scale_y.GetMaxPoint().co.Y;
1915  QSize width_size(max_width * max_scale_x,
1916  round(max_width / (float(info.width) / float(info.height))));
1917  QSize height_size(round(max_height / (float(info.height) / float(info.width))),
1918  max_height * max_scale_y);
1919  // respect aspect ratio
1920  if (width_size.width() >= max_width && width_size.height() >= max_height) {
1921  max_width = std::max(max_width, width_size.width());
1922  max_height = std::max(max_height, width_size.height());
1923  } else {
1924  max_width = std::max(max_width, height_size.width());
1925  max_height = std::max(max_height, height_size.height());
1926  }
1927 
1928  } else {
1929  // Scale video to equivalent unscaled size
1930  // Since the preview window can change sizes, we want to always
1931  // scale against the ratio of original video size to timeline size
1932  float preview_ratio = 1.0;
1933  if (parent->ParentTimeline()) {
1934  Timeline *t = (Timeline *) parent->ParentTimeline();
1935  preview_ratio = t->preview_width / float(t->info.width);
1936  }
1937  float max_scale_x = parent->scale_x.GetMaxPoint().co.Y;
1938  float max_scale_y = parent->scale_y.GetMaxPoint().co.Y;
1939  max_width = info.width * max_scale_x * preview_ratio;
1940  max_height = info.height * max_scale_y * preview_ratio;
1941  }
1942 
1943  // If a crop effect is resizing the image, request enough pixels to preserve detail
1944  ApplyCropResizeScale(parent, info.width, info.height, max_width, max_height);
1945  }
1946 
1947  if (HasMaxDecodeSize()) {
1948  QSize bounded_size(max_width, max_height);
1949  const QSize max_decode_size(MaxDecodeWidth(), MaxDecodeHeight());
1950  if (bounded_size.width() > max_decode_size.width() ||
1951  bounded_size.height() > max_decode_size.height()) {
1952  bounded_size.scale(max_decode_size, Qt::KeepAspectRatio);
1953  max_width = bounded_size.width();
1954  max_height = bounded_size.height();
1955  }
1956  }
1957 
1958  // Determine if image needs to be scaled (for performance reasons)
1959  int original_height = src_height;
1960  if (max_width != 0 && max_height != 0 && max_width < width && max_height < height) {
1961  // Override width and height (but maintain aspect ratio)
1962  float ratio = float(width) / float(height);
1963  int possible_width = round(max_height * ratio);
1964  int possible_height = round(max_width / ratio);
1965 
1966  if (possible_width <= max_width) {
1967  // use calculated width, and max_height
1968  width = possible_width;
1969  height = max_height;
1970  } else {
1971  // use max_width, and calculated height
1972  width = max_width;
1973  height = possible_height;
1974  }
1975  }
1976 
1977  // Determine required buffer size and allocate buffer
1978  const int bytes_per_pixel = 4;
1979  int raw_buffer_size = (width * height * bytes_per_pixel) + 128;
1980 
1981  // Aligned memory allocation (for speed)
1982  constexpr size_t ALIGNMENT = 32; // AVX2
1983  int buffer_size = ((raw_buffer_size + ALIGNMENT - 1) / ALIGNMENT) * ALIGNMENT;
1984  buffer = (unsigned char*) aligned_malloc(buffer_size, ALIGNMENT);
1985 
1986  // Copy picture data from one AVFrame (or AVPicture) to another one.
1987  AV_COPY_PICTURE_DATA(pFrameRGB, buffer, PIX_FMT_RGBA, width, height);
1988 
1989  int scale_mode = SWS_FAST_BILINEAR;
1990  if (openshot::Settings::Instance()->HIGH_QUALITY_SCALING) {
1991  scale_mode = SWS_BICUBIC;
1992  }
1993  img_convert_ctx = sws_getCachedContext(img_convert_ctx, src_width, src_height, src_pix_fmt, width, height, PIX_FMT_RGBA, scale_mode, NULL, NULL, NULL);
1994  if (!img_convert_ctx)
1995  throw OutOfMemory("Failed to initialize sws context", path);
1996  const int *src_coeff = sws_getCoefficients(SWS_CS_DEFAULT);
1997  const int *dst_coeff = sws_getCoefficients(SWS_CS_DEFAULT);
1998  const int dst_full_range = 1; // RGB outputs are full-range
1999  sws_setColorspaceDetails(img_convert_ctx, src_coeff, src_full_range ? 1 : 0,
2000  dst_coeff, dst_full_range, 0, 1 << 16, 1 << 16);
2001 
2002  if (!pFrame || !pFrame->data[0] || pFrame->linesize[0] <= 0) {
2003 #if USE_HW_ACCEL
2004  if (hw_de_on && hw_de_supported && !force_sw_decode) {
2005  hw_decode_failed = true;
2006  ZmqLogger::Instance()->AppendDebugMethod(
2007  "FFmpegReader::ProcessVideoPacket (Invalid source frame; forcing software fallback)",
2008  "requested_frame", requested_frame,
2009  "current_frame", current_frame,
2010  "src_pix_fmt", src_pix_fmt,
2011  "src_width", src_width,
2012  "src_height", src_height);
2013  }
2014 #endif
2015  if (pFrame) {
2016  RemoveAVFrame(pFrame);
2017  pFrame = NULL;
2018  }
2019  return;
2020  }
2021 
2022  // Resize / Convert to RGB
2023  const int scaled_lines = sws_scale(img_convert_ctx, pFrame->data, pFrame->linesize, 0,
2024  original_height, pFrameRGB->data, pFrameRGB->linesize);
2025  if (scaled_lines <= 0) {
2026 #if USE_HW_ACCEL
2027  if (hw_de_on && hw_de_supported && !force_sw_decode) {
2028  hw_decode_failed = true;
2029  ZmqLogger::Instance()->AppendDebugMethod(
2030  "FFmpegReader::ProcessVideoPacket (sws_scale failed; forcing software fallback)",
2031  "requested_frame", requested_frame,
2032  "current_frame", current_frame,
2033  "scaled_lines", scaled_lines,
2034  "src_pix_fmt", src_pix_fmt,
2035  "src_width", src_width,
2036  "src_height", src_height);
2037  }
2038 #endif
2039  free(buffer);
2040  AV_RESET_FRAME(pFrameRGB);
2041  RemoveAVFrame(pFrame);
2042  pFrame = NULL;
2043  return;
2044  }
2045 
2046  // Create or get the existing frame object
2047  std::shared_ptr<Frame> f = CreateFrame(current_frame);
2048 
2049  // Add Image data to frame
2050  if (!ffmpeg_has_alpha(src_pix_fmt)) {
2051  // Add image with no alpha channel, Speed optimization
2052  f->AddImage(width, height, bytes_per_pixel, QImage::Format_RGBA8888_Premultiplied, buffer);
2053  } else {
2054  // Add image with alpha channel (this will be converted to premultipled when needed, but is slower)
2055  f->AddImage(width, height, bytes_per_pixel, QImage::Format_RGBA8888, buffer);
2056  }
2057 
2058  // Update working cache
2059  working_cache.Add(f);
2060 
2061  // Keep track of last last_video_frame
2062  last_video_frame = f;
2063 
2064  // Free the RGB image
2065  AV_RESET_FRAME(pFrameRGB);
2066 
2067  // Remove frame and packet
2068  RemoveAVFrame(pFrame);
2069 
2070  // Get video PTS in seconds
2071  video_pts_seconds = (double(video_pts) * info.video_timebase.ToDouble()) + pts_offset_seconds;
2072 
2073  // Debug output
2074  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessVideoPacket (After)", "requested_frame", requested_frame, "current_frame", current_frame, "f->number", f->number, "video_pts_seconds", video_pts_seconds);
2075 }
2076 
2077 // Process an audio packet
2078 void FFmpegReader::ProcessAudioPacket(int64_t requested_frame) {
2079  AudioLocation location;
2080  // Calculate location of current audio packet
2081  if (packet && packet->pts != AV_NOPTS_VALUE) {
2082  // Determine related video frame and starting sample # from audio PTS
2083  location = GetAudioPTSLocation(packet->pts);
2084 
2085  // Track 1st audio packet after a successful seek
2086  if (!seek_audio_frame_found && is_seeking)
2087  seek_audio_frame_found = location.frame;
2088  }
2089 
2090  // Create or get the existing frame object. Requested frame needs to be created
2091  // in working_cache at least once. Seek can clear the working_cache, so we must
2092  // add the requested frame back to the working_cache here. If it already exists,
2093  // it will be moved to the top of the working_cache.
2094  working_cache.Add(CreateFrame(requested_frame));
2095 
2096  // Debug output
2097  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (Before)",
2098  "requested_frame", requested_frame,
2099  "target_frame", location.frame,
2100  "starting_sample", location.sample_start);
2101 
2102  // Init an AVFrame to hold the decoded audio samples
2103  int frame_finished = 0;
2104  AVFrame *audio_frame = AV_ALLOCATE_FRAME();
2105  AV_RESET_FRAME(audio_frame);
2106 
2107  int packet_samples = 0;
2108  int data_size = 0;
2109 
2110 #if IS_FFMPEG_3_2
2111  int send_packet_err = avcodec_send_packet(aCodecCtx, packet);
2112  if (send_packet_err < 0 && send_packet_err != AVERROR_EOF) {
2113  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (Packet not sent)");
2114  }
2115  else {
2116  int receive_frame_err = avcodec_receive_frame(aCodecCtx, audio_frame);
2117  if (receive_frame_err >= 0) {
2118  frame_finished = 1;
2119  }
2120  if (receive_frame_err == AVERROR_EOF) {
2121  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (EOF detected from decoder)");
2122  packet_status.audio_eof = true;
2123  }
2124  if (receive_frame_err == AVERROR(EINVAL) || receive_frame_err == AVERROR_EOF) {
2125  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (invalid frame received or EOF from decoder)");
2126  avcodec_flush_buffers(aCodecCtx);
2127  }
2128  if (receive_frame_err != 0) {
2129  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (frame not ready yet from decoder)");
2130  }
2131  }
2132 #else
2133  int used = avcodec_decode_audio4(aCodecCtx, audio_frame, &frame_finished, packet);
2134 #endif
2135 
2136  if (frame_finished) {
2137  packet_status.audio_decoded++;
2138 
2139  // This can be different than the current packet, so we need to look
2140  // at the current AVFrame from the audio decoder. This timestamp should
2141  // be used for the remainder of this function
2142  audio_pts = audio_frame->pts;
2143 
2144  // Determine related video frame and starting sample # from audio PTS
2145  location = GetAudioPTSLocation(audio_pts);
2146 
2147  // determine how many samples were decoded
2148  int plane_size = -1;
2149 #if HAVE_CH_LAYOUT
2150  int nb_channels = AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->ch_layout.nb_channels;
2151 #else
2152  int nb_channels = AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels;
2153 #endif
2154  data_size = av_samples_get_buffer_size(&plane_size, nb_channels,
2155  audio_frame->nb_samples, (AVSampleFormat) (AV_GET_SAMPLE_FORMAT(aStream, aCodecCtx)), 1);
2156 
2157  // Calculate total number of samples
2158  packet_samples = audio_frame->nb_samples * nb_channels;
2159  } else {
2160  if (audio_frame) {
2161  // Free audio frame
2162  AV_FREE_FRAME(&audio_frame);
2163  }
2164  }
2165 
2166  // Estimate the # of samples and the end of this packet's location (to prevent GAPS for the next timestamp)
2167  int pts_remaining_samples = packet_samples / info.channels; // Adjust for zero based array
2168 
2169  // Bail if no samples found
2170  if (pts_remaining_samples == 0) {
2171  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (No samples, bailing)",
2172  "packet_samples", packet_samples,
2173  "info.channels", info.channels,
2174  "pts_remaining_samples", pts_remaining_samples);
2175  return;
2176  }
2177 
2178  while (pts_remaining_samples) {
2179  // Get Samples per frame (for this frame number)
2180  int samples_per_frame = Frame::GetSamplesPerFrame(previous_packet_location.frame, info.fps, info.sample_rate, info.channels);
2181 
2182  // Calculate # of samples to add to this frame
2183  int samples = samples_per_frame - previous_packet_location.sample_start;
2184  if (samples > pts_remaining_samples)
2185  samples = pts_remaining_samples;
2186 
2187  // Decrement remaining samples
2188  pts_remaining_samples -= samples;
2189 
2190  if (pts_remaining_samples > 0) {
2191  // next frame
2192  previous_packet_location.frame++;
2193  previous_packet_location.sample_start = 0;
2194  } else {
2195  // Increment sample start
2196  previous_packet_location.sample_start += samples;
2197  }
2198  }
2199 
2200  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (ReSample)",
2201  "packet_samples", packet_samples,
2202  "info.channels", info.channels,
2203  "info.sample_rate", info.sample_rate,
2204  "aCodecCtx->sample_fmt", AV_GET_SAMPLE_FORMAT(aStream, aCodecCtx));
2205 
2206  // Create output frame
2207  AVFrame *audio_converted = AV_ALLOCATE_FRAME();
2208  AV_RESET_FRAME(audio_converted);
2209  audio_converted->nb_samples = audio_frame->nb_samples;
2210  av_samples_alloc(audio_converted->data, audio_converted->linesize, info.channels, audio_frame->nb_samples, AV_SAMPLE_FMT_FLTP, 0);
2211 
2212  SWRCONTEXT *avr = avr_ctx;
2213  // setup resample context if needed
2214  if (!avr) {
2215  avr = SWR_ALLOC();
2216 #if HAVE_CH_LAYOUT
2217  AVChannelLayout input_layout = ffmpeg_get_valid_channel_layout(
2218  AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->ch_layout, info.channels);
2219  AVChannelLayout output_layout = ffmpeg_get_valid_channel_layout(
2220  input_layout, info.channels);
2221  int in_layout_err = av_opt_set_chlayout(avr, "in_chlayout", &input_layout, 0);
2222  int out_layout_err = av_opt_set_chlayout(avr, "out_chlayout", &output_layout, 0);
2223 #else
2224  av_opt_set_int(avr, "in_channel_layout", AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout, 0);
2225  av_opt_set_int(avr, "out_channel_layout", AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout, 0);
2226  av_opt_set_int(avr, "in_channels", info.channels, 0);
2227  av_opt_set_int(avr, "out_channels", info.channels, 0);
2228 #endif
2229  av_opt_set_int(avr, "in_sample_fmt", AV_GET_SAMPLE_FORMAT(aStream, aCodecCtx), 0);
2230  av_opt_set_int(avr, "out_sample_fmt", AV_SAMPLE_FMT_FLTP, 0);
2231  av_opt_set_int(avr, "in_sample_rate", info.sample_rate, 0);
2232  av_opt_set_int(avr, "out_sample_rate", info.sample_rate, 0);
2233  int swr_init_err = SWR_INIT(avr);
2234 #if HAVE_CH_LAYOUT
2235  av_channel_layout_uninit(&input_layout);
2236  av_channel_layout_uninit(&output_layout);
2237  if (in_layout_err < 0 || out_layout_err < 0 || swr_init_err < 0) {
2238  SWR_FREE(&avr);
2239  throw InvalidChannels("Could not initialize FFmpeg audio channel layout or resampler.", path);
2240  }
2241 #else
2242  if (swr_init_err < 0) {
2243  SWR_FREE(&avr);
2244  throw InvalidChannels("Could not initialize FFmpeg audio resampler.", path);
2245  }
2246 #endif
2247  avr_ctx = avr;
2248  }
2249 
2250  // Convert audio samples
2251  int nb_samples = SWR_CONVERT(avr, // audio resample context
2252  audio_converted->data, // output data pointers
2253  audio_converted->linesize[0], // output plane size, in bytes. (0 if unknown)
2254  audio_converted->nb_samples, // maximum number of samples that the output buffer can hold
2255  audio_frame->data, // input data pointers
2256  audio_frame->linesize[0], // input plane size, in bytes (0 if unknown)
2257  audio_frame->nb_samples); // number of input samples to convert
2258 
2259 
2260  int64_t starting_frame_number = -1;
2261  for (int channel_filter = 0; channel_filter < info.channels; channel_filter++) {
2262  // Array of floats (to hold samples for each channel)
2263  starting_frame_number = location.frame;
2264  int channel_buffer_size = nb_samples;
2265  auto *channel_buffer = (float *) (audio_converted->data[channel_filter]);
2266 
2267  // Loop through samples, and add them to the correct frames
2268  int start = location.sample_start;
2269  int remaining_samples = channel_buffer_size;
2270  while (remaining_samples > 0) {
2271  // Get Samples per frame (for this frame number)
2272  int samples_per_frame = Frame::GetSamplesPerFrame(starting_frame_number, info.fps, info.sample_rate, info.channels);
2273 
2274  // Calculate # of samples to add to this frame
2275  int samples = std::fmin(samples_per_frame - start, remaining_samples);
2276 
2277  // Create or get the existing frame object
2278  std::shared_ptr<Frame> f = CreateFrame(starting_frame_number);
2279 
2280  // Add samples for current channel to the frame.
2281  f->AddAudio(true, channel_filter, start, channel_buffer, samples, 1.0f);
2282 
2283  // Debug output
2284  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (f->AddAudio)",
2285  "frame", starting_frame_number,
2286  "start", start,
2287  "samples", samples,
2288  "channel", channel_filter,
2289  "samples_per_frame", samples_per_frame);
2290 
2291  // Add or update cache
2292  working_cache.Add(f);
2293 
2294  // Decrement remaining samples
2295  remaining_samples -= samples;
2296 
2297  // Increment buffer (to next set of samples)
2298  if (remaining_samples > 0)
2299  channel_buffer += samples;
2300 
2301  // Increment frame number
2302  starting_frame_number++;
2303 
2304  // Reset starting sample #
2305  start = 0;
2306  }
2307  }
2308 
2309  // Free AVFrames
2310  av_free(audio_converted->data[0]);
2311  AV_FREE_FRAME(&audio_converted);
2312  AV_FREE_FRAME(&audio_frame);
2313 
2314  // Get audio PTS in seconds
2315  audio_pts_seconds = (double(audio_pts) * info.audio_timebase.ToDouble()) + pts_offset_seconds;
2316 
2317  // Debug output
2318  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (After)",
2319  "requested_frame", requested_frame,
2320  "starting_frame", location.frame,
2321  "end_frame", starting_frame_number - 1,
2322  "audio_pts_seconds", audio_pts_seconds);
2323 
2324 }
2325 
2326 
2327 // Seek to a specific frame. This is not always frame accurate, it's more of an estimation on many codecs.
2328 void FFmpegReader::Seek(int64_t requested_frame) {
2329  // Adjust for a requested frame that is too small or too large
2330  if (requested_frame < 1)
2331  requested_frame = 1;
2332  if (requested_frame > info.video_length)
2333  requested_frame = info.video_length;
2334  if (requested_frame > largest_frame_processed && packet_status.end_of_file) {
2335  // Not possible to search past largest_frame once EOF is reached (no more packets)
2336  return;
2337  }
2338 
2339  // Debug output
2340  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::Seek",
2341  "requested_frame", requested_frame,
2342  "seek_count", seek_count,
2343  "last_frame", last_frame);
2344 
2345  // Clear working cache (since we are seeking to another location in the file)
2346  working_cache.Clear();
2347 
2348  // Reset the last frame variable
2349  video_pts = 0.0;
2350  video_pts_seconds = NO_PTS_OFFSET;
2351  audio_pts = 0.0;
2352  audio_pts_seconds = NO_PTS_OFFSET;
2353  hold_packet = false;
2354  last_frame = 0;
2355  current_video_frame = 0;
2356  largest_frame_processed = 0;
2357  last_final_video_frame.reset();
2358  bool has_audio_override = info.has_audio;
2359  bool has_video_override = info.has_video;
2360 
2361  // Init end-of-file detection variables
2362  packet_status.reset(false);
2363 
2364  // Increment seek count
2365  seek_count++;
2366 
2367  // If seeking near frame 1, we need to close and re-open the file (this is more reliable than seeking)
2368  int buffer_amount = 12;
2369  if (requested_frame - buffer_amount < 20) {
2370  // prevent Open() from seeking again
2371  is_seeking = true;
2372 
2373  // Close and re-open file (basically seeking to frame 1)
2374  Close();
2375  Open();
2376 
2377  // Update overrides (since closing and re-opening might update these)
2378  info.has_audio = has_audio_override;
2379  info.has_video = has_video_override;
2380 
2381  // Not actually seeking, so clear these flags
2382  is_seeking = false;
2383  if (seek_count == 1) {
2384  // Don't redefine this on multiple seek attempts for a specific frame
2385  seeking_frame = 1;
2386  seeking_pts = ConvertFrameToVideoPTS(1);
2387  }
2388  seek_audio_frame_found = 0; // used to detect which frames to throw away after a seek
2389  seek_video_frame_found = 0; // used to detect which frames to throw away after a seek
2390 
2391  } else {
2392  // Seek to nearest key-frame (aka, i-frame)
2393  bool seek_worked = false;
2394  int64_t seek_target = 0;
2395 
2396  // Seek video stream (if any), except album arts
2397  if (!seek_worked && info.has_video && !HasAlbumArt()) {
2398  seek_target = ConvertFrameToVideoPTS(requested_frame - buffer_amount);
2399  if (av_seek_frame(pFormatCtx, info.video_stream_index, seek_target, AVSEEK_FLAG_BACKWARD) < 0) {
2400  ZmqLogger::Instance()->Log(std::string(pFormatCtx->AV_FILENAME) + ": error while seeking video stream");
2401  } else {
2402  // VIDEO SEEK
2403  is_video_seek = true;
2404  seek_worked = true;
2405  }
2406  }
2407 
2408  // Seek audio stream (if not already seeked... and if an audio stream is found)
2409  if (!seek_worked && info.has_audio) {
2410  seek_target = ConvertFrameToAudioPTS(requested_frame - buffer_amount);
2411  if (av_seek_frame(pFormatCtx, info.audio_stream_index, seek_target, AVSEEK_FLAG_BACKWARD) < 0) {
2412  ZmqLogger::Instance()->Log(std::string(pFormatCtx->AV_FILENAME) + ": error while seeking audio stream");
2413  } else {
2414  // AUDIO SEEK
2415  is_video_seek = false;
2416  seek_worked = true;
2417  }
2418  }
2419 
2420  // Was the seek successful?
2421  if (seek_worked) {
2422  // Flush audio buffer
2423  if (info.has_audio)
2424  avcodec_flush_buffers(aCodecCtx);
2425 
2426  // Flush video buffer
2427  if (info.has_video)
2428  avcodec_flush_buffers(pCodecCtx);
2429 
2430  // Reset previous audio location to zero
2431  previous_packet_location.frame = -1;
2432  previous_packet_location.sample_start = 0;
2433 
2434  // init seek flags
2435  is_seeking = true;
2436  if (seek_count == 1) {
2437  // Don't redefine this on multiple seek attempts for a specific frame
2438  seeking_pts = seek_target;
2439  seeking_frame = requested_frame;
2440  }
2441  seek_audio_frame_found = 0; // used to detect which frames to throw away after a seek
2442  seek_video_frame_found = 0; // used to detect which frames to throw away after a seek
2443 
2444  } else {
2445  // seek failed
2446  seeking_pts = 0;
2447  seeking_frame = 0;
2448 
2449  // prevent Open() from seeking again
2450  is_seeking = true;
2451 
2452  // Close and re-open file (basically seeking to frame 1)
2453  Close();
2454  Open();
2455 
2456  // Not actually seeking, so clear these flags
2457  is_seeking = false;
2458 
2459  // disable seeking for this reader (since it failed)
2460  enable_seek = false;
2461 
2462  // Update overrides (since closing and re-opening might update these)
2463  info.has_audio = has_audio_override;
2464  info.has_video = has_video_override;
2465  }
2466  }
2467 }
2468 
2469 // Get the PTS for the current video packet
2470 int64_t FFmpegReader::GetPacketPTS() {
2471  if (packet) {
2472  int64_t current_pts = packet->pts;
2473  if (current_pts == AV_NOPTS_VALUE && packet->dts != AV_NOPTS_VALUE)
2474  current_pts = packet->dts;
2475 
2476  // Return adjusted PTS
2477  return current_pts;
2478  } else {
2479  // No packet, return NO PTS
2480  return AV_NOPTS_VALUE;
2481  }
2482 }
2483 
2484 // Update PTS Offset (if any)
2485 void FFmpegReader::UpdatePTSOffset() {
2486  if (pts_offset_seconds != NO_PTS_OFFSET) {
2487  // Skip this method if we have already set PTS offset
2488  return;
2489  }
2490  pts_offset_seconds = 0.0;
2491  double video_pts_offset_seconds = 0.0;
2492  double audio_pts_offset_seconds = 0.0;
2493 
2494  bool has_video_pts = false;
2495  if (!info.has_video) {
2496  // Mark as checked
2497  has_video_pts = true;
2498  }
2499  bool has_audio_pts = false;
2500  if (!info.has_audio) {
2501  // Mark as checked
2502  has_audio_pts = true;
2503  }
2504 
2505  // Loop through the stream (until a packet from all streams is found)
2506  while (!has_video_pts || !has_audio_pts) {
2507  // Get the next packet (if any)
2508  if (GetNextPacket() < 0)
2509  // Break loop when no more packets found
2510  break;
2511 
2512  // Get PTS of this packet
2513  int64_t pts = GetPacketPTS();
2514 
2515  // Video packet
2516  if (!has_video_pts && packet->stream_index == videoStream) {
2517  // Get the video packet start time (in seconds)
2518  video_pts_offset_seconds = 0.0 - (pts * info.video_timebase.ToDouble());
2519 
2520  // Is timestamp close to zero (within X seconds)
2521  // Ignore wildly invalid timestamps (i.e. -234923423423)
2522  if (std::abs(video_pts_offset_seconds) <= 10.0) {
2523  has_video_pts = true;
2524  }
2525  }
2526  else if (!has_audio_pts && packet->stream_index == audioStream) {
2527  // Get the audio packet start time (in seconds)
2528  audio_pts_offset_seconds = 0.0 - (pts * info.audio_timebase.ToDouble());
2529 
2530  // Is timestamp close to zero (within X seconds)
2531  // Ignore wildly invalid timestamps (i.e. -234923423423)
2532  if (std::abs(audio_pts_offset_seconds) <= 10.0) {
2533  has_audio_pts = true;
2534  }
2535  }
2536  }
2537 
2538  // Choose timestamp origin:
2539  // - If video exists, anchor timeline frame mapping to video start.
2540  // This avoids AAC priming / audio preroll shifting video frame 1 to frame 2.
2541  // - If no video exists (audio-only readers), use audio start.
2542  if (info.has_video && has_video_pts) {
2543  pts_offset_seconds = video_pts_offset_seconds;
2544  } else if (!info.has_video && has_audio_pts) {
2545  pts_offset_seconds = audio_pts_offset_seconds;
2546  } else if (has_video_pts && has_audio_pts) {
2547  // Fallback when stream flags are unusual but both timestamps exist.
2548  pts_offset_seconds = video_pts_offset_seconds;
2549  }
2550 }
2551 
2552 // Convert PTS into Frame Number
2553 int64_t FFmpegReader::ConvertVideoPTStoFrame(int64_t pts) {
2554  // Apply PTS offset
2555  int64_t previous_video_frame = current_video_frame;
2556  const double fps_value = (info.fps.num > 0 && info.fps.den > 0) ? info.fps.ToDouble() : 30.0;
2557  const double video_timebase_value =
2558  (info.video_timebase.num > 0 && info.video_timebase.den > 0)
2559  ? info.video_timebase.ToDouble()
2560  : (1.0 / 30.0);
2561 
2562  // Get the video packet start time (in seconds)
2563  double video_seconds = (double(pts) * video_timebase_value) + pts_offset_seconds;
2564 
2565  // Divide by the video timebase, to get the video frame number (frame # is decimal at this point)
2566  int64_t frame = round(video_seconds * fps_value) + 1;
2567 
2568  // Keep track of the expected video frame #
2569  if (current_video_frame == 0)
2570  current_video_frame = frame;
2571  else {
2572 
2573  // Sometimes frames are duplicated due to identical (or similar) timestamps
2574  if (frame == previous_video_frame) {
2575  // return -1 frame number
2576  frame = -1;
2577  } else {
2578  // Increment expected frame
2579  current_video_frame++;
2580  }
2581  }
2582 
2583  // Return frame #
2584  return frame;
2585 }
2586 
2587 // Convert Frame Number into Video PTS
2588 int64_t FFmpegReader::ConvertFrameToVideoPTS(int64_t frame_number) {
2589  const double fps_value = (info.fps.num > 0 && info.fps.den > 0) ? info.fps.ToDouble() : 30.0;
2590  const double video_timebase_value =
2591  (info.video_timebase.num > 0 && info.video_timebase.den > 0)
2592  ? info.video_timebase.ToDouble()
2593  : (1.0 / 30.0);
2594 
2595  // Get timestamp of this frame (in seconds)
2596  double seconds = (double(frame_number - 1) / fps_value) + pts_offset_seconds;
2597 
2598  // Calculate the # of video packets in this timestamp
2599  int64_t video_pts = round(seconds / video_timebase_value);
2600 
2601  // Apply PTS offset (opposite)
2602  return video_pts;
2603 }
2604 
2605 // Convert Frame Number into Video PTS
2606 int64_t FFmpegReader::ConvertFrameToAudioPTS(int64_t frame_number) {
2607  const double fps_value = (info.fps.num > 0 && info.fps.den > 0) ? info.fps.ToDouble() : 30.0;
2608  const double audio_timebase_value =
2609  (info.audio_timebase.num > 0 && info.audio_timebase.den > 0)
2610  ? info.audio_timebase.ToDouble()
2611  : (1.0 / 48000.0);
2612 
2613  // Get timestamp of this frame (in seconds)
2614  double seconds = (double(frame_number - 1) / fps_value) + pts_offset_seconds;
2615 
2616  // Calculate the # of audio packets in this timestamp
2617  int64_t audio_pts = round(seconds / audio_timebase_value);
2618 
2619  // Apply PTS offset (opposite)
2620  return audio_pts;
2621 }
2622 
2623 // Calculate Starting video frame and sample # for an audio PTS
2624 AudioLocation FFmpegReader::GetAudioPTSLocation(int64_t pts) {
2625  const double audio_timebase_value =
2626  (info.audio_timebase.num > 0 && info.audio_timebase.den > 0)
2627  ? info.audio_timebase.ToDouble()
2628  : (1.0 / 48000.0);
2629  const double fps_value = (info.fps.num > 0 && info.fps.den > 0) ? info.fps.ToDouble() : 30.0;
2630 
2631  // Get the audio packet start time (in seconds)
2632  double audio_seconds = (double(pts) * audio_timebase_value) + pts_offset_seconds;
2633 
2634  // Divide by the video timebase, to get the video frame number (frame # is decimal at this point)
2635  double frame = (audio_seconds * fps_value) + 1;
2636 
2637  // Frame # as a whole number (no more decimals)
2638  int64_t whole_frame = int64_t(frame);
2639 
2640  // Remove the whole number, and only get the decimal of the frame
2641  double sample_start_percentage = frame - double(whole_frame);
2642 
2643  // Get Samples per frame
2644  int samples_per_frame = Frame::GetSamplesPerFrame(whole_frame, info.fps, info.sample_rate, info.channels);
2645 
2646  // Calculate the sample # to start on
2647  int sample_start = round(double(samples_per_frame) * sample_start_percentage);
2648 
2649  // Protect against broken (i.e. negative) timestamps
2650  if (whole_frame < 1)
2651  whole_frame = 1;
2652  if (sample_start < 0)
2653  sample_start = 0;
2654 
2655  // Prepare final audio packet location
2656  AudioLocation location = {whole_frame, sample_start};
2657 
2658  // Compare to previous audio packet (and fix small gaps due to varying PTS timestamps)
2659  if (previous_packet_location.frame != -1) {
2660  if (location.is_near(previous_packet_location, samples_per_frame, samples_per_frame)) {
2661  int64_t orig_frame = location.frame;
2662  int orig_start = location.sample_start;
2663 
2664  // Update sample start, to prevent gaps in audio
2665  location.sample_start = previous_packet_location.sample_start;
2666  location.frame = previous_packet_location.frame;
2667 
2668  // Debug output
2669  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAudioPTSLocation (Audio Gap Detected)", "Source Frame", orig_frame, "Source Audio Sample", orig_start, "Target Frame", location.frame, "Target Audio Sample", location.sample_start, "pts", pts);
2670 
2671  } else {
2672  // Debug output
2673  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAudioPTSLocation (Audio Gap Ignored - too big)", "Previous location frame", previous_packet_location.frame, "Target Frame", location.frame, "Target Audio Sample", location.sample_start, "pts", pts);
2674  }
2675  }
2676 
2677  // Set previous location
2678  previous_packet_location = location;
2679 
2680  // Return the associated video frame and starting sample #
2681  return location;
2682 }
2683 
2684 // Create a new Frame (or return an existing one) and add it to the working queue.
2685 std::shared_ptr<Frame> FFmpegReader::CreateFrame(int64_t requested_frame) {
2686  // Check working cache
2687  std::shared_ptr<Frame> output = working_cache.GetFrame(requested_frame);
2688 
2689  if (!output) {
2690  // (re-)Check working cache
2691  output = working_cache.GetFrame(requested_frame);
2692  if(output) return output;
2693 
2694  // Create a new frame on the working cache
2695  output = std::make_shared<Frame>(requested_frame, info.width, info.height, "#000000", Frame::GetSamplesPerFrame(requested_frame, info.fps, info.sample_rate, info.channels), info.channels);
2696  output->SetPixelRatio(info.pixel_ratio.num, info.pixel_ratio.den); // update pixel ratio
2697  output->ChannelsLayout(info.channel_layout); // update audio channel layout from the parent reader
2698  output->SampleRate(info.sample_rate); // update the frame's sample rate of the parent reader
2699 
2700  working_cache.Add(output);
2701 
2702  // Set the largest processed frame (if this is larger)
2703  if (requested_frame > largest_frame_processed)
2704  largest_frame_processed = requested_frame;
2705  }
2706  // Return frame
2707  return output;
2708 }
2709 
2710 // Determine if frame is partial due to seek
2711 bool FFmpegReader::IsPartialFrame(int64_t requested_frame) {
2712 
2713  // Sometimes a seek gets partial frames, and we need to remove them
2714  bool seek_trash = false;
2715  int64_t max_seeked_frame = seek_audio_frame_found; // determine max seeked frame
2716  if (seek_video_frame_found > max_seeked_frame) {
2717  max_seeked_frame = seek_video_frame_found;
2718  }
2719  if ((info.has_audio && seek_audio_frame_found && max_seeked_frame >= requested_frame) ||
2720  (info.has_video && seek_video_frame_found && max_seeked_frame >= requested_frame)) {
2721  seek_trash = true;
2722  }
2723 
2724  return seek_trash;
2725 }
2726 
2727 // Check the working queue, and move finished frames to the finished queue
2728 void FFmpegReader::CheckWorkingFrames(int64_t requested_frame) {
2729 
2730  // Prevent async calls to the following code
2731  const std::lock_guard<std::recursive_mutex> lock(getFrameMutex);
2732 
2733  // Get a list of current working queue frames in the cache (in-progress frames)
2734  std::vector<std::shared_ptr<openshot::Frame>> working_frames = working_cache.GetFrames();
2735  std::vector<std::shared_ptr<openshot::Frame>>::iterator working_itr;
2736 
2737  // Loop through all working queue frames (sorted by frame #)
2738  for(working_itr = working_frames.begin(); working_itr != working_frames.end(); ++working_itr)
2739  {
2740  // Get working frame
2741  std::shared_ptr<Frame> f = *working_itr;
2742 
2743  // Was a frame found? Is frame requested yet?
2744  if (!f || f->number > requested_frame) {
2745  // If not, skip to next one
2746  continue;
2747  }
2748 
2749  // Calculate PTS in seconds (of working frame), and the most recent processed pts value
2750  double frame_pts_seconds = (double(f->number - 1) / info.fps.ToDouble()) + pts_offset_seconds;
2751  double recent_pts_seconds = std::max(video_pts_seconds, audio_pts_seconds);
2752 
2753  // Determine if video and audio are ready (based on timestamps)
2754  bool is_video_ready = false;
2755  bool is_audio_ready = false;
2756  double recent_pts_diff = recent_pts_seconds - frame_pts_seconds;
2757  if ((frame_pts_seconds <= video_pts_seconds)
2758  || (recent_pts_diff > 1.5)
2759  || packet_status.video_eof || packet_status.end_of_file) {
2760  // Video stream is past this frame (so it must be done)
2761  // OR video stream is too far behind, missing, or end-of-file
2762  is_video_ready = true;
2763  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames (video ready)",
2764  "frame_number", f->number,
2765  "frame_pts_seconds", frame_pts_seconds,
2766  "video_pts_seconds", video_pts_seconds,
2767  "recent_pts_diff", recent_pts_diff);
2768  if (info.has_video && !f->has_image_data &&
2769  (packet_status.video_eof || packet_status.end_of_file)) {
2770  // Frame has no image data. Prefer timeline-previous frames to preserve
2771  // visual order, especially when decode/prefetch is out-of-order.
2772  std::shared_ptr<Frame> previous_frame_instance = final_cache.GetFrame(f->number - 1);
2773  if (previous_frame_instance && previous_frame_instance->has_image_data) {
2774  f->AddImage(std::make_shared<QImage>(previous_frame_instance->GetImage()->copy()));
2775  }
2776 
2777  // Fall back to last finalized timeline image (survives cache churn).
2778  if (!f->has_image_data
2779  && last_final_video_frame
2780  && last_final_video_frame->has_image_data
2781  && last_final_video_frame->number <= f->number) {
2782  f->AddImage(std::make_shared<QImage>(last_final_video_frame->GetImage()->copy()));
2783  }
2784 
2785  // Fall back to the last decoded image only when it is not from the future.
2786  if (!f->has_image_data
2787  && last_video_frame
2788  && last_video_frame->has_image_data
2789  && last_video_frame->number <= f->number) {
2790  f->AddImage(std::make_shared<QImage>(last_video_frame->GetImage()->copy()));
2791  }
2792 
2793  // Last-resort fallback if no prior image is available.
2794  if (!f->has_image_data) {
2795  ZmqLogger::Instance()->AppendDebugMethod(
2796  "FFmpegReader::CheckWorkingFrames (no previous image found; using black frame)",
2797  "frame_number", f->number);
2798  f->AddColor("#000000");
2799  }
2800  }
2801  }
2802 
2803  double audio_pts_diff = audio_pts_seconds - frame_pts_seconds;
2804  if ((frame_pts_seconds < audio_pts_seconds && audio_pts_diff > 1.0)
2805  || (recent_pts_diff > 1.5)
2806  || packet_status.audio_eof || packet_status.end_of_file) {
2807  // Audio stream is past this frame (so it must be done)
2808  // OR audio stream is too far behind, missing, or end-of-file
2809  // Adding a bit of margin here, to allow for partial audio packets
2810  is_audio_ready = true;
2811  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames (audio ready)",
2812  "frame_number", f->number,
2813  "frame_pts_seconds", frame_pts_seconds,
2814  "audio_pts_seconds", audio_pts_seconds,
2815  "audio_pts_diff", audio_pts_diff,
2816  "recent_pts_diff", recent_pts_diff);
2817  }
2818  bool is_seek_trash = IsPartialFrame(f->number);
2819 
2820  // Adjust for available streams
2821  if (!info.has_video) is_video_ready = true;
2822  if (!info.has_audio) is_audio_ready = true;
2823 
2824  // Debug output
2825  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames",
2826  "frame_number", f->number,
2827  "is_video_ready", is_video_ready,
2828  "is_audio_ready", is_audio_ready,
2829  "video_eof", packet_status.video_eof,
2830  "audio_eof", packet_status.audio_eof,
2831  "end_of_file", packet_status.end_of_file);
2832 
2833  // Check if working frame is final
2834  if (info.has_video && !f->has_image_data
2835  && !packet_status.end_of_file && !is_seek_trash) {
2836  if (info.has_single_image) {
2837  // For still-image video (including attached cover art), reuse the most
2838  // recent image so playback does not stall waiting for video EOF.
2839  std::shared_ptr<Frame> previous_frame_instance = final_cache.GetFrame(f->number - 1);
2840  if (previous_frame_instance && previous_frame_instance->has_image_data) {
2841  f->AddImage(std::make_shared<QImage>(previous_frame_instance->GetImage()->copy()));
2842  }
2843  if (!f->has_image_data
2844  && last_final_video_frame
2845  && last_final_video_frame->has_image_data
2846  && last_final_video_frame->number <= f->number) {
2847  f->AddImage(std::make_shared<QImage>(last_final_video_frame->GetImage()->copy()));
2848  }
2849  if (!f->has_image_data
2850  && last_video_frame
2851  && last_video_frame->has_image_data
2852  && last_video_frame->number <= f->number) {
2853  f->AddImage(std::make_shared<QImage>(last_video_frame->GetImage()->copy()));
2854  }
2855  }
2856 
2857  // If both streams have advanced past this frame but the decoder never
2858  // produced image data for it, reuse the most recent non-future image.
2859  // This avoids stalling indefinitely on sparse/missing decoded frames.
2860  if (!f->has_image_data && is_video_ready && is_audio_ready) {
2861  std::shared_ptr<Frame> previous_frame_instance = final_cache.GetFrame(f->number - 1);
2862  if (previous_frame_instance && previous_frame_instance->has_image_data) {
2863  f->AddImage(std::make_shared<QImage>(previous_frame_instance->GetImage()->copy()));
2864  }
2865  if (!f->has_image_data
2866  && last_final_video_frame
2867  && last_final_video_frame->has_image_data
2868  && last_final_video_frame->number <= f->number) {
2869  f->AddImage(std::make_shared<QImage>(last_final_video_frame->GetImage()->copy()));
2870  }
2871  if (!f->has_image_data
2872  && last_video_frame
2873  && last_video_frame->has_image_data
2874  && last_video_frame->number <= f->number) {
2875  f->AddImage(std::make_shared<QImage>(last_video_frame->GetImage()->copy()));
2876  }
2877  }
2878 
2879  // Do not finalize non-EOF video frames without decoded image data.
2880  // This prevents repeated previous-frame fallbacks being cached as real frames.
2881  if (!f->has_image_data) {
2882  continue;
2883  }
2884  }
2885  if ((!packet_status.end_of_file && is_video_ready && is_audio_ready) || packet_status.end_of_file || is_seek_trash) {
2886  // Debug output
2887  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames (mark frame as final)",
2888  "requested_frame", requested_frame,
2889  "f->number", f->number,
2890  "is_seek_trash", is_seek_trash,
2891  "Working Cache Count", working_cache.Count(),
2892  "Final Cache Count", final_cache.Count(),
2893  "end_of_file", packet_status.end_of_file);
2894 
2895  if (!is_seek_trash) {
2896  // Move frame to final cache
2897  final_cache.Add(f);
2898  if (f->has_image_data) {
2899  last_final_video_frame = f;
2900  }
2901 
2902  // Remove frame from working cache
2903  working_cache.Remove(f->number);
2904 
2905  // Update last frame processed
2906  last_frame = f->number;
2907  } else {
2908  // Seek trash, so delete the frame from the working cache, and never add it to the final cache.
2909  working_cache.Remove(f->number);
2910  }
2911 
2912  }
2913  }
2914 
2915  // Clear vector of frames
2916  working_frames.clear();
2917  working_frames.shrink_to_fit();
2918 }
2919 
2920 // Check for the correct frames per second (FPS) value by scanning the 1st few seconds of video packets.
2921 void FFmpegReader::CheckFPS() {
2922  if (check_fps) {
2923  // Do not check FPS more than 1 time
2924  return;
2925  } else {
2926  check_fps = true;
2927  }
2928 
2929  int frames_per_second[3] = {0,0,0};
2930  int max_fps_index = sizeof(frames_per_second) / sizeof(frames_per_second[0]);
2931  int fps_index = 0;
2932 
2933  int all_frames_detected = 0;
2934  int starting_frames_detected = 0;
2935 
2936  // Loop through the stream
2937  while (true) {
2938  // Get the next packet (if any)
2939  if (GetNextPacket() < 0)
2940  // Break loop when no more packets found
2941  break;
2942 
2943  // Video packet
2944  if (packet->stream_index == videoStream) {
2945  // Get the video packet start time (in seconds)
2946  double video_seconds = (double(GetPacketPTS()) * info.video_timebase.ToDouble()) + pts_offset_seconds;
2947  fps_index = int(video_seconds); // truncate float timestamp to int (second 1, second 2, second 3)
2948 
2949  // Is this video packet from the first few seconds?
2950  if (fps_index >= 0 && fps_index < max_fps_index) {
2951  // Yes, keep track of how many frames per second (over the first few seconds)
2952  starting_frames_detected++;
2953  frames_per_second[fps_index]++;
2954  }
2955 
2956  // Track all video packets detected
2957  all_frames_detected++;
2958  }
2959  }
2960 
2961  // Calculate FPS (based on the first few seconds of video packets)
2962  float avg_fps = 30.0;
2963  if (starting_frames_detected > 0 && fps_index > 0) {
2964  avg_fps = float(starting_frames_detected) / std::min(fps_index, max_fps_index);
2965  }
2966 
2967  // Verify average FPS is a reasonable value
2968  if (avg_fps < 8.0) {
2969  // Invalid FPS assumed, so switching to a sane default FPS instead
2970  avg_fps = 30.0;
2971  }
2972 
2973  // Update FPS (truncate average FPS to Integer)
2974  info.fps = Fraction(int(avg_fps), 1);
2975 
2976  // Update Duration and Length
2977  if (all_frames_detected > 0) {
2978  // Use all video frames detected to calculate # of frames
2979  info.video_length = all_frames_detected;
2980  info.duration = all_frames_detected / avg_fps;
2981  } else {
2982  // Use previous duration to calculate # of frames
2983  info.video_length = info.duration * avg_fps;
2984  }
2985 
2986  // Update video bit rate
2987  info.video_bit_rate = info.file_size / info.duration;
2988 }
2989 
2990 // Remove AVFrame from cache (and deallocate its memory)
2991 void FFmpegReader::RemoveAVFrame(AVFrame *remove_frame) {
2992  // Remove pFrame (if exists)
2993  if (remove_frame) {
2994  // Free memory
2995  av_freep(&remove_frame->data[0]);
2996 #ifndef WIN32
2997  AV_FREE_FRAME(&remove_frame);
2998 #endif
2999  }
3000 }
3001 
3002 // Remove AVPacket from cache (and deallocate its memory)
3003 void FFmpegReader::RemoveAVPacket(AVPacket *remove_packet) {
3004  // deallocate memory for packet
3005  AV_FREE_PACKET(remove_packet);
3006 
3007  // Delete the object
3008  delete remove_packet;
3009 }
3010 
3011 // Generate JSON string of this object
3012 std::string FFmpegReader::Json() const {
3013 
3014  // Return formatted string
3015  return JsonValue().toStyledString();
3016 }
3017 
3018 // Generate Json::Value for this object
3019 Json::Value FFmpegReader::JsonValue() const {
3020 
3021  // Create root json object
3022  Json::Value root = ReaderBase::JsonValue(); // get parent properties
3023  root["type"] = "FFmpegReader";
3024  root["path"] = path;
3025  switch (duration_strategy) {
3026  case DurationStrategy::VideoPreferred:
3027  root["duration_strategy"] = "VideoPreferred";
3028  break;
3029  case DurationStrategy::AudioPreferred:
3030  root["duration_strategy"] = "AudioPreferred";
3031  break;
3032  case DurationStrategy::LongestStream:
3033  default:
3034  root["duration_strategy"] = "LongestStream";
3035  break;
3036  }
3037 
3038  // return JsonValue
3039  return root;
3040 }
3041 
3042 // Load JSON string into this object
3043 void FFmpegReader::SetJson(const std::string value) {
3044 
3045  // Parse JSON string into JSON objects
3046  try {
3047  const Json::Value root = openshot::stringToJson(value);
3048  // Set all values that match
3049  SetJsonValue(root);
3050  }
3051  catch (const std::exception& e) {
3052  // Error parsing JSON (or missing keys)
3053  throw InvalidJSON("JSON is invalid (missing keys or invalid data types)");
3054  }
3055 }
3056 
3057 // Load Json::Value into this object
3058 void FFmpegReader::SetJsonValue(const Json::Value root) {
3059 
3060  // Set parent data
3061  ReaderBase::SetJsonValue(root);
3062 
3063  // Set data from Json (if key is found)
3064  if (!root["path"].isNull())
3065  path = root["path"].asString();
3066  if (!root["duration_strategy"].isNull()) {
3067  const std::string strategy = root["duration_strategy"].asString();
3068  if (strategy == "VideoPreferred") {
3069  duration_strategy = DurationStrategy::VideoPreferred;
3070  } else if (strategy == "AudioPreferred") {
3071  duration_strategy = DurationStrategy::AudioPreferred;
3072  } else {
3073  duration_strategy = DurationStrategy::LongestStream;
3074  }
3075  }
3076 }
CropHelpers.h
Shared helpers for Crop effect scaling logic.
Exceptions.h
Header file for all Exception classes.
hw_de_av_pix_fmt_global
AVPixelFormat hw_de_av_pix_fmt_global
Definition: FFmpegReader.cpp:72
hw_de_av_device_type_global
AVHWDeviceType hw_de_av_device_type_global
Definition: FFmpegReader.cpp:73
hw_de_on
int hw_de_on
Definition: FFmpegReader.cpp:70
FFmpegReader.h
Header file for FFmpegReader class.
FFmpegUtilities.h
Header file for FFmpegUtilities.
AV_FREE_CONTEXT
#define AV_FREE_CONTEXT(av_context)
Definition: FFmpegUtilities.h:316
SWR_INIT
#define SWR_INIT(ctx)
Definition: FFmpegUtilities.h:264
AV_FREE_FRAME
#define AV_FREE_FRAME(av_frame)
Definition: FFmpegUtilities.h:314
SWR_CONVERT
#define SWR_CONVERT(ctx, out, linesize, out_count, in, linesize2, in_count)
Definition: FFmpegUtilities.h:259
SWR_ALLOC
#define SWR_ALLOC()
Definition: FFmpegUtilities.h:261
SWR_CLOSE
#define SWR_CLOSE(ctx)
Definition: FFmpegUtilities.h:262
AV_GET_CODEC_TYPE
#define AV_GET_CODEC_TYPE(av_stream)
Definition: FFmpegUtilities.h:317
AV_GET_CODEC_PIXEL_FORMAT
#define AV_GET_CODEC_PIXEL_FORMAT(av_stream, av_context)
Definition: FFmpegUtilities.h:323
AV_GET_CODEC_CONTEXT
#define AV_GET_CODEC_CONTEXT(av_stream, av_codec)
Definition: FFmpegUtilities.h:319
AV_FIND_DECODER_CODEC_ID
#define AV_FIND_DECODER_CODEC_ID(av_stream)
Definition: FFmpegUtilities.h:318
AV_ALLOCATE_FRAME
#define AV_ALLOCATE_FRAME()
Definition: FFmpegUtilities.h:310
AV_REGISTER_ALL
#define AV_REGISTER_ALL
Definition: FFmpegUtilities.h:305
PIX_FMT_RGBA
#define PIX_FMT_RGBA
Definition: FFmpegUtilities.h:110
SWR_FREE
#define SWR_FREE(ctx)
Definition: FFmpegUtilities.h:263
AV_COPY_PICTURE_DATA
#define AV_COPY_PICTURE_DATA(av_frame, buffer, pix_fmt, width, height)
Definition: FFmpegUtilities.h:327
AV_FREE_PACKET
#define AV_FREE_PACKET(av_packet)
Definition: FFmpegUtilities.h:315
SWRCONTEXT
#define SWRCONTEXT
Definition: FFmpegUtilities.h:265
AVCODEC_REGISTER_ALL
#define AVCODEC_REGISTER_ALL
Definition: FFmpegUtilities.h:306
AV_GET_CODEC_ATTRIBUTES
#define AV_GET_CODEC_ATTRIBUTES(av_stream, av_context)
Definition: FFmpegUtilities.h:322
AV_ALLOCATE_IMAGE
#define AV_ALLOCATE_IMAGE(av_frame, pix_fmt, width, height)
Definition: FFmpegUtilities.h:311
AV_GET_SAMPLE_FORMAT
#define AV_GET_SAMPLE_FORMAT(av_stream, av_context)
Definition: FFmpegUtilities.h:325
AV_RESET_FRAME
#define AV_RESET_FRAME(av_frame)
Definition: FFmpegUtilities.h:313
opts
AVDictionary * opts
Definition: FFmpegWriter.cpp:1485
if
if(!codec) codec
MemoryTrim.h
Cross-platform helper to encourage returning freed memory to the OS.
FF_VIDEO_NUM_PROCESSORS
#define FF_VIDEO_NUM_PROCESSORS
Definition: OpenMPUtilities.h:24
OPEN_MP_NUM_PROCESSORS
#define OPEN_MP_NUM_PROCESSORS
Definition: OpenMPUtilities.h:23
FF_AUDIO_NUM_PROCESSORS
#define FF_AUDIO_NUM_PROCESSORS
Definition: OpenMPUtilities.h:25
Timeline.h
Header file for Timeline class.
ZmqLogger.h
Header file for ZeroMQ-based Logger class.
openshot::CacheBase::SetMaxBytesFromInfo
void SetMaxBytesFromInfo(int64_t number_of_frames, int width, int height, int sample_rate, int channels)
Set maximum bytes to a different amount based on a ReaderInfo struct.
Definition: CacheBase.cpp:28
openshot::CacheMemory::Count
int64_t Count()
Count the frames in the queue.
Definition: CacheMemory.cpp:240
openshot::CacheMemory::Add
void Add(std::shared_ptr< openshot::Frame > frame)
Add a Frame to the cache.
Definition: CacheMemory.cpp:47
openshot::CacheMemory::GetFrame
std::shared_ptr< openshot::Frame > GetFrame(int64_t frame_number)
Get a frame from the cache.
Definition: CacheMemory.cpp:84
openshot::CacheMemory::GetFrames
std::vector< std::shared_ptr< openshot::Frame > > GetFrames()
Get an array of all Frames.
Definition: CacheMemory.cpp:100
openshot::CacheMemory::Remove
void Remove(int64_t frame_number)
Remove a specific frame.
Definition: CacheMemory.cpp:158
openshot::CacheMemory::Clear
void Clear()
Clear the cache of all frames.
Definition: CacheMemory.cpp:224
openshot::Clip
This class represents a clip (used to arrange readers on the timeline)
Definition: Clip.h:89
openshot::Clip::scale_x
openshot::Keyframe scale_x
Curve representing the horizontal scaling in percent (0 to 1)
Definition: Clip.h:322
openshot::Clip::ParentTimeline
openshot::TimelineBase * ParentTimeline() override
Get the associated Timeline pointer (if any)
Definition: Clip.h:300
openshot::Clip::scale_y
openshot::Keyframe scale_y
Curve representing the vertical scaling in percent (0 to 1)
Definition: Clip.h:323
openshot::Clip::scale
openshot::ScaleType scale
The scale determines how a clip should be resized to fit its parent.
Definition: Clip.h:179
openshot::Coordinate::Y
double Y
The Y value of the coordinate (usually representing the value of the property being animated)
Definition: Coordinate.h:41
openshot::FFmpegReader
This class uses the FFmpeg libraries, to open video files and audio files, and return openshot::Frame...
Definition: FFmpegReader.h:103
openshot::FFmpegReader::Open
void Open() override
Open File - which is called by the constructor automatically.
Definition: FFmpegReader.cpp:261
openshot::FFmpegReader::HardwareDecodeSuccessful
bool HardwareDecodeSuccessful() const override
Return true if hardware decode was requested and successfully produced at least one frame.
Definition: FFmpegReader.cpp:1761
openshot::FFmpegReader::FFmpegReader
FFmpegReader(const std::string &path, bool inspect_reader=true)
Constructor for FFmpegReader.
Definition: FFmpegReader.cpp:102
openshot::FFmpegReader::JsonValue
Json::Value JsonValue() const override
Generate Json::Value for this object.
Definition: FFmpegReader.cpp:3019
openshot::FFmpegReader::GetIsDurationKnown
bool GetIsDurationKnown()
Return true if frame can be read with GetFrame()
Definition: FFmpegReader.cpp:1193
openshot::FFmpegReader::SetJsonValue
void SetJsonValue(const Json::Value root) override
Load Json::Value into this object.
Definition: FFmpegReader.cpp:3058
openshot::FFmpegReader::final_cache
CacheMemory final_cache
Final cache object used to hold final frames.
Definition: FFmpegReader.h:257
openshot::FFmpegReader::~FFmpegReader
virtual ~FFmpegReader()
Destructor.
Definition: FFmpegReader.cpp:137
openshot::FFmpegReader::Json
std::string Json() const override
Generate JSON string of this object.
Definition: FFmpegReader.cpp:3012
openshot::FFmpegReader::GetFrame
std::shared_ptr< openshot::Frame > GetFrame(int64_t requested_frame) override
Definition: FFmpegReader.cpp:1197
openshot::FFmpegReader::Close
void Close() override
Close File.
Definition: FFmpegReader.cpp:748
openshot::FFmpegReader::SetJson
void SetJson(const std::string value) override
Load JSON string into this object.
Definition: FFmpegReader.cpp:3043
openshot::Fraction
This class represents a fraction.
Definition: Fraction.h:30
openshot::Fraction::num
int num
Numerator for the fraction.
Definition: Fraction.h:32
openshot::Fraction::ToFloat
float ToFloat()
Return this fraction as a float (i.e. 1/2 = 0.5)
Definition: Fraction.cpp:35
openshot::Fraction::ToDouble
double ToDouble() const
Return this fraction as a double (i.e. 1/2 = 0.5)
Definition: Fraction.cpp:40
openshot::Fraction::Reciprocal
Fraction Reciprocal() const
Return the reciprocal as a Fraction.
Definition: Fraction.cpp:78
openshot::Fraction::den
int den
Denominator for the fraction.
Definition: Fraction.h:33
openshot::Frame::GetSamplesPerFrame
int GetSamplesPerFrame(openshot::Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
Definition: Frame.cpp:484
openshot::InvalidChannels
Exception when an invalid # of audio channels are detected.
Definition: Exceptions.h:164
openshot::InvalidCodec
Exception when no valid codec is found for a file.
Definition: Exceptions.h:179
openshot::InvalidFile
Exception for files that can not be found or opened.
Definition: Exceptions.h:194
openshot::InvalidJSON
Exception for invalid JSON.
Definition: Exceptions.h:224
openshot::Keyframe::GetMaxPoint
Point GetMaxPoint() const
Get max point (by Y coordinate)
Definition: KeyFrame.cpp:245
openshot::NoStreamsFound
Exception when no streams are found in the file.
Definition: Exceptions.h:292
openshot::OutOfMemory
Exception when memory could not be allocated.
Definition: Exceptions.h:355
openshot::Point::co
Coordinate co
This is the primary coordinate.
Definition: Point.h:66
openshot::ReaderBase::info
openshot::ReaderInfo info
Information about the current media file.
Definition: ReaderBase.h:90
openshot::ReaderBase::MaxDecodeWidth
int MaxDecodeWidth() const
Return the current maximum decoded frame width (0 when unlimited).
Definition: ReaderBase.cpp:257
openshot::ReaderBase::SetJsonValue
virtual void SetJsonValue(const Json::Value root)=0
Load Json::Value into this object.
Definition: ReaderBase.cpp:160
openshot::ReaderBase::JsonValue
virtual Json::Value JsonValue() const =0
Generate Json::Value for this object.
Definition: ReaderBase.cpp:109
openshot::ReaderBase::MaxDecodeHeight
int MaxDecodeHeight() const
Return the current maximum decoded frame height (0 when unlimited).
Definition: ReaderBase.cpp:261
openshot::ReaderBase::getFrameMutex
std::recursive_mutex getFrameMutex
Mutex for multiple threads.
Definition: ReaderBase.h:79
openshot::ReaderBase::HasMaxDecodeSize
bool HasMaxDecodeSize() const
Return true when a maximum decoded frame size is active.
Definition: ReaderBase.cpp:265
openshot::ReaderBase::ParentClip
openshot::ClipBase * ParentClip()
Parent clip object of this reader (which can be unparented and NULL)
Definition: ReaderBase.cpp:243
openshot::ReaderClosed
Exception when a reader is closed, and a frame is requested.
Definition: Exceptions.h:370
openshot::Settings::DE_LIMIT_WIDTH_MAX
int DE_LIMIT_WIDTH_MAX
Maximum columns that hardware decode can handle.
Definition: Settings.h:89
openshot::Settings::HW_DE_DEVICE_SET
int HW_DE_DEVICE_SET
Which GPU to use to decode (0 is the first)
Definition: Settings.h:92
openshot::Settings::DE_LIMIT_HEIGHT_MAX
int DE_LIMIT_HEIGHT_MAX
Maximum rows that hardware decode can handle.
Definition: Settings.h:86
openshot::Settings::Instance
static Settings * Instance()
Create or get an instance of this logger singleton (invoke the class with this method)
Definition: Settings.cpp:43
openshot::Settings::HARDWARE_DECODER
int HARDWARE_DECODER
Use video codec for faster video decoding (if supported)
Definition: Settings.h:71
openshot::TimelineBase::preview_height
int preview_height
Optional preview width of timeline image. If your preview window is smaller than the timeline,...
Definition: TimelineBase.h:45
openshot::TimelineBase::preview_width
int preview_width
Optional preview width of timeline image. If your preview window is smaller than the timeline,...
Definition: TimelineBase.h:44
openshot::Timeline
This class represents a timeline.
Definition: Timeline.h:153
openshot::ZmqLogger::Log
void Log(std::string message)
Log message to all subscribers of this logger (if any)
Definition: ZmqLogger.cpp:103
openshot::ZmqLogger::AppendDebugMethod
void AppendDebugMethod(std::string method_name, std::string arg1_name="", float arg1_value=-1.0, std::string arg2_name="", float arg2_value=-1.0, std::string arg3_name="", float arg3_value=-1.0, std::string arg4_name="", float arg4_value=-1.0, std::string arg5_name="", float arg5_value=-1.0, std::string arg6_name="", float arg6_value=-1.0)
Append debug information.
Definition: ZmqLogger.cpp:178
openshot::ZmqLogger::Instance
static ZmqLogger * Instance()
Create or get an instance of this logger singleton (invoke the class with this method)
Definition: ZmqLogger.cpp:35
openshot
This namespace is the default namespace for all code in the openshot library.
Definition: Compressor.h:29
openshot::SCALE_FIT
@ SCALE_FIT
Scale the clip until either height or width fills the canvas (with no cropping)
Definition: Enums.h:38
openshot::SCALE_STRETCH
@ SCALE_STRETCH
Scale the clip until both height and width fill the canvas (distort to fit)
Definition: Enums.h:39
openshot::SCALE_CROP
@ SCALE_CROP
Scale the clip until both height and width fill the canvas (cropping the overlap)
Definition: Enums.h:37
openshot::ChannelLayout
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround,...
Definition: ChannelLayouts.h:29
openshot::DurationStrategy
DurationStrategy
This enumeration determines which duration source to favor.
Definition: Enums.h:60
openshot::DurationStrategy::VideoPreferred
@ VideoPreferred
Prefer the video stream's duration, fallback to audio then container.
openshot::DurationStrategy::LongestStream
@ LongestStream
Use the longest value from video, audio, or container.
openshot::DurationStrategy::AudioPreferred
@ AudioPreferred
Prefer the audio stream's duration, fallback to video then container.
openshot::ApplyCropResizeScale
void ApplyCropResizeScale(Clip *clip, int source_width, int source_height, int &max_width, int &max_height)
Scale the requested max_width / max_height based on the Crop resize amount, capped by source size.
Definition: CropHelpers.cpp:40
openshot::stringToJson
const Json::Value stringToJson(const std::string value)
Definition: Json.cpp:16
openshot::AudioLocation
This struct holds the associated video frame and starting sample # for an audio packet.
Definition: AudioLocation.h:25
openshot::AudioLocation::is_near
bool is_near(AudioLocation location, int samples_per_frame, int64_t amount)
Definition: FFmpegReader.cpp:144
openshot::PacketStatus::reset
void reset(bool eof)
Definition: FFmpegReader.h:70
openshot::ReaderInfo::audio_bit_rate
int audio_bit_rate
The bit rate of the audio stream (in bytes)
Definition: ReaderBase.h:59
openshot::ReaderInfo::video_bit_rate
int video_bit_rate
The bit rate of the video stream (in bytes)
Definition: ReaderBase.h:49
openshot::ReaderInfo::has_single_image
bool has_single_image
Determines if this file only contains a single image.
Definition: ReaderBase.h:42
openshot::ReaderInfo::duration
float duration
Length of time (in seconds)
Definition: ReaderBase.h:43
openshot::ReaderInfo::audio_timebase
openshot::Fraction audio_timebase
The audio timebase determines how long each audio packet should be played.
Definition: ReaderBase.h:64
openshot::ReaderInfo::width
int width
The width of the video (in pixesl)
Definition: ReaderBase.h:46
openshot::ReaderInfo::channels
int channels
The number of audio channels used in the audio stream.
Definition: ReaderBase.h:61
openshot::ReaderInfo::fps
openshot::Fraction fps
Frames per second, as a fraction (i.e. 24/1 = 24 fps)
Definition: ReaderBase.h:48
openshot::ReaderInfo::display_ratio
openshot::Fraction display_ratio
The ratio of width to height of the video stream (i.e. 640x480 has a ratio of 4/3)
Definition: ReaderBase.h:51
openshot::ReaderInfo::height
int height
The height of the video (in pixels)
Definition: ReaderBase.h:45
openshot::ReaderInfo::pixel_format
int pixel_format
The pixel format (i.e. YUV420P, RGB24, etc...)
Definition: ReaderBase.h:47
openshot::ReaderInfo::video_length
int64_t video_length
The number of frames in the video stream.
Definition: ReaderBase.h:53
openshot::ReaderInfo::acodec
std::string acodec
The name of the audio codec used to encode / decode the video stream.
Definition: ReaderBase.h:58
openshot::ReaderInfo::metadata
std::map< std::string, std::string > metadata
An optional map/dictionary of metadata for this reader.
Definition: ReaderBase.h:65
openshot::ReaderInfo::vcodec
std::string vcodec
The name of the video codec used to encode / decode the video stream.
Definition: ReaderBase.h:52
openshot::ReaderInfo::pixel_ratio
openshot::Fraction pixel_ratio
The pixel ratio of the video stream as a fraction (i.e. some pixels are not square)
Definition: ReaderBase.h:50
openshot::ReaderInfo::channel_layout
openshot::ChannelLayout channel_layout
The channel layout (mono, stereo, 5 point surround, etc...)
Definition: ReaderBase.h:62
openshot::ReaderInfo::has_video
bool has_video
Determines if this file has a video stream.
Definition: ReaderBase.h:40
openshot::ReaderInfo::has_audio
bool has_audio
Determines if this file has an audio stream.
Definition: ReaderBase.h:41
openshot::ReaderInfo::video_timebase
openshot::Fraction video_timebase
The video timebase determines how long each frame stays on the screen.
Definition: ReaderBase.h:55
openshot::ReaderInfo::video_stream_index
int video_stream_index
The index of the video stream.
Definition: ReaderBase.h:54
openshot::ReaderInfo::sample_rate
int sample_rate
The number of audio samples per second (44100 is a common sample rate)
Definition: ReaderBase.h:60
openshot::ReaderInfo::audio_stream_index
int audio_stream_index
The index of the audio stream.
Definition: ReaderBase.h:63
openshot::ReaderInfo::file_size
int64_t file_size
Size of file (in bytes)
Definition: ReaderBase.h:44
Generated by doxygen 1.9.1