OpenShot Library | libopenshot  0.2.4
FFmpegReader.h
Go to the documentation of this file.
1 /**
2  * @file
3  * @brief Header file for FFmpegReader class
4  * @author Jonathan Thomas <jonathan@openshot.org>, Fabrice Bellard
5  *
6  * @ref License
7  */
8 
9 /* LICENSE
10  *
11  * Copyright (c) 2008-2019 OpenShot Studios, LLC, Fabrice Bellard
12  * (http://www.openshotstudios.com). This file is part of
13  * OpenShot Library (http://www.openshot.org), an open-source project
14  * dedicated to delivering high quality video editing and animation solutions
15  * to the world.
16  *
17  * This file is originally based on the Libavformat API example, and then modified
18  * by the libopenshot project.
19  *
20  * OpenShot Library (libopenshot) is free software: you can redistribute it
21  * and/or modify it under the terms of the GNU Lesser General Public License
22  * as published by the Free Software Foundation, either version 3 of the
23  * License, or (at your option) any later version.
24  *
25  * OpenShot Library (libopenshot) is distributed in the hope that it will be
26  * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
27  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
28  * GNU Lesser General Public License for more details.
29  *
30  * You should have received a copy of the GNU Lesser General Public License
31  * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
32  */
33 
34 #ifndef OPENSHOT_FFMPEG_READER_H
35 #define OPENSHOT_FFMPEG_READER_H
36 
37 #include "ReaderBase.h"
38 
39 // Include FFmpeg headers and macros
40 #include "FFmpegUtilities.h"
41 
42 #include <cmath>
43 #include <ctime>
44 #include <iostream>
45 #include <stdio.h>
46 #include <memory>
47 #include "CacheMemory.h"
48 #include "Clip.h"
49 #include "Exceptions.h"
50 #include "OpenMPUtilities.h"
51 #include "Settings.h"
52 
53 
54 namespace openshot {
55  /**
56  * @brief This struct holds the associated video frame and starting sample # for an audio packet.
57  *
58  * Because audio packets do not match up with video frames, this helps determine exactly
59  * where the audio packet's samples belong.
60  */
61  struct AudioLocation {
62  int64_t frame;
64 
65  bool is_near(AudioLocation location, int samples_per_frame, int64_t amount);
66  };
67 
68  /**
69  * @brief This class uses the FFmpeg libraries, to open video files and audio files, and return
70  * openshot::Frame objects for any frame in the file.
71  *
72  * All seeking and caching is handled internally, and the primary public interface is the GetFrame()
73  * method. To use this reader, simply create an instance of this class, and call the GetFrame method
74  * to start retrieving frames. Use the <b>info</b> struct to obtain information on the file, such as the length
75  * (# of frames), height, width, bit rate, frames per second (fps), etc...
76  *
77  * @code
78  * // Create a reader for a video
79  * FFmpegReader r("MyAwesomeVideo.webm");
80  * r.Open(); // Open the reader
81  *
82  * // Get frame number 1 from the video
83  * std::shared_ptr<Frame> f = r.GetFrame(1);
84  *
85  * // Now that we have an openshot::Frame object, lets have some fun!
86  * f->Display(); // Display the frame on the screen
87  * f->DisplayWaveform(); // Display the audio waveform as an image
88  * f->Play(); // Play the audio through your speaker
89  *
90  * // Close the reader
91  * r.Close();
92  * @endcode
93  */
94  class FFmpegReader : public ReaderBase {
95  private:
96  std::string path;
97 
98  AVFormatContext *pFormatCtx;
99  int i, videoStream, audioStream;
100  AVCodecContext *pCodecCtx, *aCodecCtx;
101 #if (LIBAVFORMAT_VERSION_MAJOR >= 57)
102  AVBufferRef *hw_device_ctx = NULL; //PM
103 #endif
104  AVStream *pStream, *aStream;
105  AVPacket *packet;
106  AVFrame *pFrame;
107  bool is_open;
108  bool is_duration_known;
109  bool check_interlace;
110  bool check_fps;
111  bool has_missing_frames;
112 
113  CacheMemory working_cache;
114  CacheMemory missing_frames;
115  std::map<int64_t, int64_t> processing_video_frames;
116  std::multimap<int64_t, int64_t> processing_audio_frames;
117  std::map<int64_t, int64_t> processed_video_frames;
118  std::map<int64_t, int64_t> processed_audio_frames;
119  std::multimap<int64_t, int64_t> missing_video_frames;
120  std::multimap<int64_t, int64_t> missing_video_frames_source;
121  std::multimap<int64_t, int64_t> missing_audio_frames;
122  std::multimap<int64_t, int64_t> missing_audio_frames_source;
123  std::map<int64_t, int> checked_frames;
124  AudioLocation previous_packet_location;
125 
126  // DEBUG VARIABLES (FOR AUDIO ISSUES)
127  int prev_samples;
128  int64_t prev_pts;
129  int64_t pts_total;
130  int64_t pts_counter;
131  int64_t num_packets_since_video_frame;
132  int64_t num_checks_since_final;
133  std::shared_ptr<openshot::Frame> last_video_frame;
134 
135  bool is_seeking;
136  int64_t seeking_pts;
137  int64_t seeking_frame;
138  bool is_video_seek;
139  int seek_count;
140  int64_t seek_audio_frame_found;
141  int64_t seek_video_frame_found;
142 
143  int64_t audio_pts_offset;
144  int64_t video_pts_offset;
145  int64_t last_frame;
146  int64_t largest_frame_processed;
147  int64_t current_video_frame; // can't reliably use PTS of video to determine this
148 
149  int hw_de_supported = 0; // Is set by FFmpegReader
150 #if IS_FFMPEG_3_2
151  AVPixelFormat hw_de_av_pix_fmt = AV_PIX_FMT_NONE;
152  AVHWDeviceType hw_de_av_device_type = AV_HWDEVICE_TYPE_NONE;
153 #endif
154 
155  int IsHardwareDecodeSupported(int codecid);
156 
157  /// Check for the correct frames per second value by scanning the 1st few seconds of video packets.
158  void CheckFPS();
159 
160  /// Check the current seek position and determine if we need to seek again
161  bool CheckSeek(bool is_video);
162 
163  /// Check if a frame is missing and attempt to replace its frame image (and
164  bool CheckMissingFrame(int64_t requested_frame);
165 
166  /// Check the working queue, and move finished frames to the finished queue
167  void CheckWorkingFrames(bool end_of_stream, int64_t requested_frame);
168 
169  /// Convert Frame Number into Audio PTS
170  int64_t ConvertFrameToAudioPTS(int64_t frame_number);
171 
172  /// Convert Frame Number into Video PTS
173  int64_t ConvertFrameToVideoPTS(int64_t frame_number);
174 
175  /// Convert Video PTS into Frame Number
176  int64_t ConvertVideoPTStoFrame(int64_t pts);
177 
178  /// Create a new Frame (or return an existing one) and add it to the working queue.
179  std::shared_ptr<openshot::Frame> CreateFrame(int64_t requested_frame);
180 
181  /// Calculate Starting video frame and sample # for an audio PTS
182  AudioLocation GetAudioPTSLocation(int64_t pts);
183 
184  /// Get an AVFrame (if any)
185  bool GetAVFrame();
186 
187  /// Get the next packet (if any)
188  int GetNextPacket();
189 
190  /// Get the smallest video frame that is still being processed
191  int64_t GetSmallestVideoFrame();
192 
193  /// Get the smallest audio frame that is still being processed
194  int64_t GetSmallestAudioFrame();
195 
196  /// Get the PTS for the current video packet
197  int64_t GetVideoPTS();
198 
199  /// Remove partial frames due to seek
200  bool IsPartialFrame(int64_t requested_frame);
201 
202  /// Process a video packet
203  void ProcessVideoPacket(int64_t requested_frame);
204 
205  /// Process an audio packet
206  void ProcessAudioPacket(int64_t requested_frame, int64_t target_frame, int starting_sample);
207 
208  /// Read the stream until we find the requested Frame
209  std::shared_ptr<openshot::Frame> ReadStream(int64_t requested_frame);
210 
211  /// Remove AVFrame from cache (and deallocate its memory)
212  void RemoveAVFrame(AVFrame *);
213 
214  /// Remove AVPacket from cache (and deallocate its memory)
215  void RemoveAVPacket(AVPacket *);
216 
217  /// Seek to a specific Frame. This is not always frame accurate, it's more of an estimation on many codecs.
218  void Seek(int64_t requested_frame);
219 
220  /// Update PTS Offset (if any)
221  void UpdatePTSOffset(bool is_video);
222 
223  /// Update File Info for audio streams
224  void UpdateAudioInfo();
225 
226  /// Update File Info for video streams
227  void UpdateVideoInfo();
228 
229  public:
230  /// Final cache object used to hold final frames
232 
233  /// Enable or disable seeking. Seeking can more quickly locate the requested frame, but some
234  /// codecs have trouble seeking, and can introduce artifacts or blank images into the video.
236 
237  /// Constructor for FFmpegReader. This automatically opens the media file and loads
238  /// frame 1, or it throws one of the following exceptions.
239  FFmpegReader(std::string path);
240 
241  /// Constructor for FFmpegReader. This only opens the media file to inspect its properties
242  /// if inspect_reader=true. When not inspecting the media file, it's much faster, and useful
243  /// when you are inflating the object using JSON after instantiating it.
244  FFmpegReader(std::string path, bool inspect_reader);
245 
246  /// Destructor
247  virtual ~FFmpegReader();
248 
249  /// Close File
250  void Close();
251 
252  /// Get the cache object used by this reader
253  CacheMemory *GetCache() { return &final_cache; };
254 
255  /// Get a shared pointer to a openshot::Frame object for a specific frame number of this reader.
256  ///
257  /// @returns The requested frame of video
258  /// @param requested_frame The frame number that is requested.
259  std::shared_ptr<openshot::Frame> GetFrame(int64_t requested_frame);
260 
261  /// Determine if reader is open or closed
262  bool IsOpen() { return is_open; };
263 
264  /// Return the type name of the class
265  std::string Name() { return "FFmpegReader"; };
266 
267  /// Get and Set JSON methods
268  std::string Json(); ///< Generate JSON string of this object
269  void SetJson(std::string value); ///< Load JSON string into this object
270  Json::Value JsonValue(); ///< Generate Json::JsonValue for this object
271  void SetJsonValue(Json::Value root); ///< Load Json::JsonValue into this object
272 
273  /// Open File - which is called by the constructor automatically
274  void Open();
275  };
276 
277 }
278 
279 #endif
Header file for ReaderBase class.
Header file for OpenMPUtilities (set some common macros)
bool is_near(AudioLocation location, int samples_per_frame, int64_t amount)
CacheMemory * GetCache()
Get the cache object used by this reader.
Definition: FFmpegReader.h:253
This abstract class is the base class, used by all readers in libopenshot.
Definition: ReaderBase.h:97
Header file for CacheMemory class.
bool IsOpen()
Determine if reader is open or closed.
Definition: FFmpegReader.h:262
Header file for all Exception classes.
This class uses the FFmpeg libraries, to open video files and audio files, and return openshot::Frame...
Definition: FFmpegReader.h:94
std::string Name()
Return the type name of the class.
Definition: FFmpegReader.h:265
Header file for Clip class.
Header file for global Settings class.
This namespace is the default namespace for all code in the openshot library.
This struct holds the associated video frame and starting sample # for an audio packet.
Definition: FFmpegReader.h:61
CacheMemory final_cache
Final cache object used to hold final frames.
Definition: FFmpegReader.h:231
Header file for FFmpegUtilities.
This class is a memory-based cache manager for Frame objects.
Definition: CacheMemory.h:51