OpenShot Library | libopenshot  0.2.4
CacheDisk.cpp
Go to the documentation of this file.
1 /**
2  * @file
3  * @brief Source file for CacheDisk class
4  * @author Jonathan Thomas <jonathan@openshot.org>
5  *
6  * @ref License
7  */
8 
9 /* LICENSE
10  *
11  * Copyright (c) 2008-2019 OpenShot Studios, LLC
12  * <http://www.openshotstudios.com/>. This file is part of
13  * OpenShot Library (libopenshot), an open-source project dedicated to
14  * delivering high quality video editing and animation solutions to the
15  * world. For more information visit <http://www.openshot.org/>.
16  *
17  * OpenShot Library (libopenshot) is free software: you can redistribute it
18  * and/or modify it under the terms of the GNU Lesser General Public License
19  * as published by the Free Software Foundation, either version 3 of the
20  * License, or (at your option) any later version.
21  *
22  * OpenShot Library (libopenshot) is distributed in the hope that it will be
23  * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
24  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
25  * GNU Lesser General Public License for more details.
26  *
27  * You should have received a copy of the GNU Lesser General Public License
28  * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
29  */
30 
31 #include "../include/CacheDisk.h"
32 
33 using namespace std;
34 using namespace openshot;
35 
36 // Default constructor, no max bytes
37 CacheDisk::CacheDisk(std::string cache_path, std::string format, float quality, float scale) : CacheBase(0) {
38  // Set cache type name
39  cache_type = "CacheDisk";
40  range_version = 0;
41  needs_range_processing = false;
42  frame_size_bytes = 0;
43  image_format = format;
44  image_quality = quality;
45  image_scale = scale;
46  max_bytes = 0;
47 
48  // Init path directory
49  InitPath(cache_path);
50 }
51 
52 // Constructor that sets the max bytes to cache
53 CacheDisk::CacheDisk(std::string cache_path, std::string format, float quality, float scale, int64_t max_bytes) : CacheBase(max_bytes) {
54  // Set cache type name
55  cache_type = "CacheDisk";
56  range_version = 0;
57  needs_range_processing = false;
58  frame_size_bytes = 0;
59  image_format = format;
60  image_quality = quality;
61  image_scale = scale;
62 
63  // Init path directory
64  InitPath(cache_path);
65 }
66 
67 // Initialize cache directory
68 void CacheDisk::InitPath(std::string cache_path) {
69  QString qpath;
70 
71  if (!cache_path.empty()) {
72  // Init QDir with cache directory
73  qpath = QString(cache_path.c_str());
74 
75  } else {
76  // Init QDir with user's temp directory
77  qpath = QDir::tempPath() + QString("/preview-cache/");
78  }
79 
80  // Init QDir with cache directory
81  path = QDir(qpath);
82 
83  // Check if cache directory exists
84  if (!path.exists())
85  // Create
86  path.mkpath(qpath);
87 }
88 
89 // Calculate ranges of frames
90 void CacheDisk::CalculateRanges() {
91  // Only calculate when something has changed
92  if (needs_range_processing) {
93 
94  // Create a scoped lock, to protect the cache from multiple threads
95  const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
96 
97  // Sort ordered frame #s, and calculate JSON ranges
98  std::sort(ordered_frame_numbers.begin(), ordered_frame_numbers.end());
99 
100  // Clear existing JSON variable
101  Json::Value ranges = Json::Value(Json::arrayValue);
102 
103  // Increment range version
104  range_version++;
105 
106  int64_t starting_frame = *ordered_frame_numbers.begin();
107  int64_t ending_frame = starting_frame;
108 
109  // Loop through all known frames (in sequential order)
110  for (const auto frame_number : ordered_frame_numbers) {
111  if (frame_number - ending_frame > 1) {
112  // End of range detected
113  Json::Value range;
114 
115  // Add JSON object with start/end attributes
116  // Use strings, since int64_ts are supported in JSON
117  std::stringstream start_str;
118  start_str << starting_frame;
119  std::stringstream end_str;
120  end_str << ending_frame;
121  range["start"] = start_str.str();
122  range["end"] = end_str.str();
123  ranges.append(range);
124 
125  // Set new starting range
126  starting_frame = frame_number;
127  }
128 
129  // Set current frame as end of range, and keep looping
130  ending_frame = frame_number;
131  }
132 
133  // APPEND FINAL VALUE
134  Json::Value range;
135 
136  // Add JSON object with start/end attributes
137  // Use strings, since int64_ts are supported in JSON
138  std::stringstream start_str;
139  start_str << starting_frame;
140  std::stringstream end_str;
141  end_str << ending_frame;
142  range["start"] = start_str.str();
143  range["end"] = end_str.str();
144  ranges.append(range);
145 
146  // Cache range JSON as string
147  json_ranges = ranges.toStyledString();
148 
149  // Reset needs_range_processing
150  needs_range_processing = false;
151  }
152 }
153 
154 // Default destructor
156 {
157  frames.clear();
158  frame_numbers.clear();
159  ordered_frame_numbers.clear();
160 
161  // remove critical section
162  delete cacheCriticalSection;
163  cacheCriticalSection = NULL;
164 }
165 
166 // Add a Frame to the cache
167 void CacheDisk::Add(std::shared_ptr<Frame> frame)
168 {
169  // Create a scoped lock, to protect the cache from multiple threads
170  const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
171  int64_t frame_number = frame->number;
172 
173  // Freshen frame if it already exists
174  if (frames.count(frame_number))
175  // Move frame to front of queue
176  MoveToFront(frame_number);
177 
178  else
179  {
180  // Add frame to queue and map
181  frames[frame_number] = frame_number;
182  frame_numbers.push_front(frame_number);
183  ordered_frame_numbers.push_back(frame_number);
184  needs_range_processing = true;
185 
186  // Save image to disk (if needed)
187  QString frame_path(path.path() + "/" + QString("%1.").arg(frame_number) + QString(image_format.c_str()).toLower());
188  frame->Save(frame_path.toStdString(), image_scale, image_format, image_quality);
189  if (frame_size_bytes == 0) {
190  // Get compressed size of frame image (to correctly apply max size against)
191  QFile image_file(frame_path);
192  frame_size_bytes = image_file.size();
193  }
194 
195  // Save audio data (if needed)
196  if (frame->has_audio_data) {
197  QString audio_path(path.path() + "/" + QString("%1").arg(frame_number) + ".audio");
198  QFile audio_file(audio_path);
199 
200  if (audio_file.open(QIODevice::WriteOnly)) {
201  QTextStream audio_stream(&audio_file);
202  audio_stream << frame->SampleRate() << endl;
203  audio_stream << frame->GetAudioChannelsCount() << endl;
204  audio_stream << frame->GetAudioSamplesCount() << endl;
205  audio_stream << frame->ChannelsLayout() << endl;
206 
207  // Loop through all samples
208  for (int channel = 0; channel < frame->GetAudioChannelsCount(); channel++)
209  {
210  // Get audio for this channel
211  float *samples = frame->GetAudioSamples(channel);
212  for (int sample = 0; sample < frame->GetAudioSamplesCount(); sample++)
213  audio_stream << samples[sample] << endl;
214  }
215 
216  }
217 
218  }
219 
220  // Clean up old frames
221  CleanUp();
222  }
223 }
224 
225 // Get a frame from the cache (or NULL shared_ptr if no frame is found)
226 std::shared_ptr<Frame> CacheDisk::GetFrame(int64_t frame_number)
227 {
228  // Create a scoped lock, to protect the cache from multiple threads
229  const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
230 
231  // Does frame exists in cache?
232  if (frames.count(frame_number)) {
233  // Does frame exist on disk
234  QString frame_path(path.path() + "/" + QString("%1.").arg(frame_number) + QString(image_format.c_str()).toLower());
235  if (path.exists(frame_path)) {
236 
237  // Load image file
238  std::shared_ptr<QImage> image = std::shared_ptr<QImage>(new QImage());
239  bool success = image->load(QString::fromStdString(frame_path.toStdString()));
240 
241  // Set pixel formatimage->
242  image = std::shared_ptr<QImage>(new QImage(image->convertToFormat(QImage::Format_RGBA8888)));
243 
244  // Create frame object
245  std::shared_ptr<Frame> frame(new Frame());
246  frame->number = frame_number;
247  frame->AddImage(image);
248 
249  // Get audio data (if found)
250  QString audio_path(path.path() + "/" + QString("%1").arg(frame_number) + ".audio");
251  QFile audio_file(audio_path);
252  if (audio_file.exists()) {
253  // Open audio file
254  QTextStream in(&audio_file);
255  if (audio_file.open(QIODevice::ReadOnly)) {
256  int sample_rate = in.readLine().toInt();
257  int channels = in.readLine().toInt();
258  int sample_count = in.readLine().toInt();
259  int channel_layout = in.readLine().toInt();
260 
261  // Set basic audio properties
262  frame->ResizeAudio(channels, sample_count, sample_rate, (ChannelLayout) channel_layout);
263 
264  // Loop through audio samples and add to frame
265  int current_channel = 0;
266  int current_sample = 0;
267  float *channel_samples = new float[sample_count];
268  while (!in.atEnd()) {
269  // Add sample to channel array
270  channel_samples[current_sample] = in.readLine().toFloat();
271  current_sample++;
272 
273  if (current_sample == sample_count) {
274  // Add audio to frame
275  frame->AddAudio(true, current_channel, 0, channel_samples, sample_count, 1.0);
276 
277  // Increment channel, and reset sample position
278  current_channel++;
279  current_sample = 0;
280  }
281 
282  }
283  }
284  }
285 
286  // return the Frame object
287  return frame;
288  }
289  }
290 
291  // no Frame found
292  return std::shared_ptr<Frame>();
293 }
294 
295 // Get the smallest frame number (or NULL shared_ptr if no frame is found)
296 std::shared_ptr<Frame> CacheDisk::GetSmallestFrame()
297 {
298  // Create a scoped lock, to protect the cache from multiple threads
299  const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
300  std::shared_ptr<openshot::Frame> f;
301 
302  // Loop through frame numbers
303  std::deque<int64_t>::iterator itr;
304  int64_t smallest_frame = -1;
305  for(itr = frame_numbers.begin(); itr != frame_numbers.end(); ++itr)
306  {
307  if (*itr < smallest_frame || smallest_frame == -1)
308  smallest_frame = *itr;
309  }
310 
311  // Return frame
312  f = GetFrame(smallest_frame);
313 
314  return f;
315 }
316 
317 // Gets the maximum bytes value
319 {
320  // Create a scoped lock, to protect the cache from multiple threads
321  const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
322 
323  int64_t total_bytes = 0;
324 
325  // Loop through frames, and calculate total bytes
326  std::deque<int64_t>::reverse_iterator itr;
327  for(itr = frame_numbers.rbegin(); itr != frame_numbers.rend(); ++itr)
328  total_bytes += frame_size_bytes;
329 
330  return total_bytes;
331 }
332 
333 // Remove a specific frame
334 void CacheDisk::Remove(int64_t frame_number)
335 {
336  Remove(frame_number, frame_number);
337 }
338 
339 // Remove range of frames
340 void CacheDisk::Remove(int64_t start_frame_number, int64_t end_frame_number)
341 {
342  // Create a scoped lock, to protect the cache from multiple threads
343  const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
344 
345  // Loop through frame numbers
346  std::deque<int64_t>::iterator itr;
347  for(itr = frame_numbers.begin(); itr != frame_numbers.end();)
348  {
349  //deque<int64_t>::iterator current = itr++;
350  if (*itr >= start_frame_number && *itr <= end_frame_number)
351  {
352  // erase frame number
353  itr = frame_numbers.erase(itr);
354  } else
355  itr++;
356  }
357 
358  // Loop through ordered frame numbers
359  std::vector<int64_t>::iterator itr_ordered;
360  for(itr_ordered = ordered_frame_numbers.begin(); itr_ordered != ordered_frame_numbers.end();)
361  {
362  if (*itr_ordered >= start_frame_number && *itr_ordered <= end_frame_number)
363  {
364  // erase frame number
365  frames.erase(*itr_ordered);
366 
367  // Remove the image file (if it exists)
368  QString frame_path(path.path() + "/" + QString("%1.").arg(*itr_ordered) + QString(image_format.c_str()).toLower());
369  QFile image_file(frame_path);
370  if (image_file.exists())
371  image_file.remove();
372 
373  // Remove audio file (if it exists)
374  QString audio_path(path.path() + "/" + QString("%1").arg(*itr_ordered) + ".audio");
375  QFile audio_file(audio_path);
376  if (audio_file.exists())
377  audio_file.remove();
378 
379  itr_ordered = ordered_frame_numbers.erase(itr_ordered);
380  } else
381  itr_ordered++;
382  }
383 
384  // Needs range processing (since cache has changed)
385  needs_range_processing = true;
386 }
387 
388 // Move frame to front of queue (so it lasts longer)
389 void CacheDisk::MoveToFront(int64_t frame_number)
390 {
391  // Does frame exists in cache?
392  if (frames.count(frame_number))
393  {
394  // Create a scoped lock, to protect the cache from multiple threads
395  const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
396 
397  // Loop through frame numbers
398  std::deque<int64_t>::iterator itr;
399  for(itr = frame_numbers.begin(); itr != frame_numbers.end(); ++itr)
400  {
401  if (*itr == frame_number)
402  {
403  // erase frame number
404  frame_numbers.erase(itr);
405 
406  // add frame number to 'front' of queue
407  frame_numbers.push_front(frame_number);
408  break;
409  }
410  }
411  }
412 }
413 
414 // Clear the cache of all frames
416 {
417  // Create a scoped lock, to protect the cache from multiple threads
418  const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
419 
420  // Clear all containers
421  frames.clear();
422  frame_numbers.clear();
423  ordered_frame_numbers.clear();
424  needs_range_processing = true;
425  frame_size_bytes = 0;
426 
427  // Delete cache directory, and recreate it
428  QString current_path = path.path();
429  path.removeRecursively();
430 
431  // Re-init folder
432  InitPath(current_path.toStdString());
433 }
434 
435 // Count the frames in the queue
437 {
438  // Create a scoped lock, to protect the cache from multiple threads
439  const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
440 
441  // Return the number of frames in the cache
442  return frames.size();
443 }
444 
445 // Clean up cached frames that exceed the number in our max_bytes variable
446 void CacheDisk::CleanUp()
447 {
448  // Do we auto clean up?
449  if (max_bytes > 0)
450  {
451  // Create a scoped lock, to protect the cache from multiple threads
452  const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
453 
454  while (GetBytes() > max_bytes && frame_numbers.size() > 20)
455  {
456  // Get the oldest frame number.
457  int64_t frame_to_remove = frame_numbers.back();
458 
459  // Remove frame_number and frame
460  Remove(frame_to_remove);
461  }
462  }
463 }
464 
465 // Generate JSON string of this object
466 std::string CacheDisk::Json() {
467 
468  // Return formatted string
469  return JsonValue().toStyledString();
470 }
471 
472 // Generate Json::JsonValue for this object
473 Json::Value CacheDisk::JsonValue() {
474 
475  // Process range data (if anything has changed)
476  CalculateRanges();
477 
478  // Create root json object
479  Json::Value root = CacheBase::JsonValue(); // get parent properties
480  root["type"] = cache_type;
481  root["path"] = path.path().toStdString();
482 
483  Json::Value version;
484  std::stringstream range_version_str;
485  range_version_str << range_version;
486  root["version"] = range_version_str.str();
487 
488  // Parse and append range data (if any)
489  Json::Value ranges;
490  Json::CharReaderBuilder rbuilder;
491  Json::CharReader* reader(rbuilder.newCharReader());
492 
493  std::string errors;
494  bool success = reader->parse( json_ranges.c_str(),
495  json_ranges.c_str() + json_ranges.size(), &ranges, &errors );
496  delete reader;
497 
498  if (success)
499  root["ranges"] = ranges;
500 
501  // return JsonValue
502  return root;
503 }
504 
505 // Load JSON string into this object
506 void CacheDisk::SetJson(std::string value) {
507 
508  // Parse JSON string into JSON objects
509  Json::Value root;
510  Json::CharReaderBuilder rbuilder;
511  Json::CharReader* reader(rbuilder.newCharReader());
512 
513  std::string errors;
514  bool success = reader->parse( value.c_str(),
515  value.c_str() + value.size(), &root, &errors );
516  delete reader;
517 
518  if (!success)
519  // Raise exception
520  throw InvalidJSON("JSON could not be parsed (or is invalid)");
521 
522  try
523  {
524  // Set all values that match
525  SetJsonValue(root);
526  }
527  catch (const std::exception& e)
528  {
529  // Error parsing JSON (or missing keys)
530  throw InvalidJSON("JSON is invalid (missing keys or invalid data types)");
531  }
532 }
533 
534 // Load Json::JsonValue into this object
535 void CacheDisk::SetJsonValue(Json::Value root) {
536 
537  // Close timeline before we do anything (this also removes all open and closing clips)
538  Clear();
539 
540  // Set parent data
542 
543  if (!root["type"].isNull())
544  cache_type = root["type"].asString();
545  if (!root["path"].isNull())
546  // Update duration of timeline
547  InitPath(root["path"].asString());
548 }
juce::CriticalSection * cacheCriticalSection
Section lock for multiple threads.
Definition: CacheBase.h:56
void Add(std::shared_ptr< openshot::Frame > frame)
Add a Frame to the cache.
Definition: CacheDisk.cpp:167
int64_t Count()
Count the frames in the queue.
Definition: CacheDisk.cpp:436
This class represents a single frame of video (i.e. image & audio data)
Definition: Frame.h:106
STL namespace.
int64_t GetBytes()
Gets the maximum bytes value.
Definition: CacheDisk.cpp:318
void SetJsonValue(Json::Value root)
Load Json::JsonValue into this object.
Definition: CacheDisk.cpp:535
std::string Json()
Get and Set JSON methods.
Definition: CacheDisk.cpp:466
std::shared_ptr< openshot::Frame > GetFrame(int64_t frame_number)
Get a frame from the cache.
Definition: CacheDisk.cpp:226
void Remove(int64_t frame_number)
Remove a specific frame.
Definition: CacheDisk.cpp:334
Json::Value JsonValue()
Generate Json::JsonValue for this object.
Definition: CacheDisk.cpp:473
void MoveToFront(int64_t frame_number)
Move frame to front of queue (so it lasts longer)
Definition: CacheDisk.cpp:389
All cache managers in libopenshot are based on this CacheBase class.
Definition: CacheBase.h:49
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround...
virtual Json::Value JsonValue()=0
Generate Json::JsonValue for this object.
Definition: CacheBase.cpp:57
void SetJson(std::string value)
Load JSON string into this object.
Definition: CacheDisk.cpp:506
void Clear()
Clear the cache of all frames.
Definition: CacheDisk.cpp:415
std::string cache_type
This is a friendly type name of the derived cache instance.
Definition: CacheBase.h:52
This namespace is the default namespace for all code in the openshot library.
Exception for invalid JSON.
Definition: Exceptions.h:205
std::shared_ptr< openshot::Frame > GetSmallestFrame()
Get the smallest frame number.
Definition: CacheDisk.cpp:296
int64_t max_bytes
This is the max number of bytes to cache (0 = no limit)
Definition: CacheBase.h:53
virtual void SetJsonValue(Json::Value root)=0
Load Json::JsonValue into this object.
Definition: CacheBase.cpp:70
CacheDisk(std::string cache_path, std::string format, float quality, float scale)
Default constructor, no max bytes.
Definition: CacheDisk.cpp:37