Prepare chunks on the fly

cvat-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cvat-core",
-  "version": "3.5.0",
+  "version": "3.6.0",
   "description": "Part of Computer Vision Tool which presents an interface for client-side integration",
   "main": "babel.config.js",
   "scripts": {

cvat-core/src/session.js

@@ -819,6 +819,7 @@
                 data_compressed_chunk_type: undefined,
                 data_original_chunk_type: undefined,
                 use_zip_chunks: undefined,
+                use_cache: undefined,
             };
 
             for (const property in data) {
@@ -1074,6 +1075,24 @@
                         data.use_zip_chunks = useZipChunks;
                     },
                 },
+                /**
+                    * @name useCache
+                    * @type {boolean}
+                    * @memberof module:API.cvat.classes.Task
+                    * @instance
+                    * @throws {module:API.cvat.exceptions.ArgumentError}
+                */
+                useCache: {
+                    get: () => data.use_cache,
+                    set: (useCache) => {
+                        if (typeof (useCache) !== 'boolean') {
+                            throw new ArgumentError(
+                                'Value must be a boolean',
+                            );
+                        }
+                        data.use_cache = useCache;
+                    },
+                },
                 /**
                     * After task has been created value can be appended only.
                     * @name labels
@@ -1645,6 +1664,7 @@
             remote_files: this.remoteFiles,
             image_quality: this.imageQuality,
             use_zip_chunks: this.useZipChunks,
+            use_cache: this.useCache,
         };
 
         if (typeof (this.startFrame) !== 'undefined') {

cvat-ui/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cvat-ui",
-  "version": "1.8.1",
+  "version": "1.9.0",
   "description": "CVAT single-page application",
   "main": "src/index.tsx",
   "scripts": {

cvat-ui/src/actions/tasks-actions.ts

@@ -386,6 +386,7 @@ ThunkAction<Promise<void>, {}, {}, AnyAction> {
             z_order: data.advanced.zOrder,
             image_quality: 70,
             use_zip_chunks: data.advanced.useZipChunks,
+            use_cache: data.advanced.useCache,
         };
 
         if (data.advanced.bugTracker) {

cvat-ui/src/components/create-task-page/advanced-configuration-form.tsx

@@ -26,6 +26,7 @@ export interface AdvancedConfiguration {
     repository?: string;
     useZipChunks: boolean;
     dataChunkSize?: number;
+    useCache: boolean;
 }
 
 type Props = FormComponentProps & {
@@ -380,6 +381,24 @@ class AdvancedConfigurationForm extends React.PureComponent<Props> {
         );
     }
 
+    private renderCreateTaskMethod(): JSX.Element {
+        const { form } = this.props;
+        return (
+            <Form.Item help='Using cache to store data.'>
+                {form.getFieldDecorator('useCache', {
+                    initialValue: false,
+                    valuePropName: 'checked',
+                })(
+                    <Checkbox>
+                        <Text className='cvat-text-color'>
+                            Use cache
+                        </Text>
+                    </Checkbox>,
+                )}
+            </Form.Item>
+        );
+    }
+
     private renderChunkSize(): JSX.Element {
         const { form } = this.props;
 
@@ -434,6 +453,12 @@ class AdvancedConfigurationForm extends React.PureComponent<Props> {
                     </Col>
                 </Row>
 
+                <Row>
+                    <Col>
+                        {this.renderCreateTaskMethod()}
+                    </Col>
+                </Row>
+
                 <Row type='flex' justify='start'>
                     <Col span={7}>
                         {this.renderImageQuality()}

cvat-ui/src/components/create-task-page/create-task-content.tsx

@@ -42,6 +42,7 @@ const defaultState = {
         zOrder: false,
         lfs: false,
         useZipChunks: true,
+        useCache: true,
     },
     labels: [],
     files: {

cvat/apps/engine/cache.py

@@ -0,0 +1,55 @@
+# Copyright (C) 2020 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+from diskcache import Cache
+from django.conf import settings
+from cvat.apps.engine.media_extractors import (Mpeg4ChunkWriter, ZipChunkWriter,
+    Mpeg4CompressedChunkWriter, ZipCompressedChunkWriter)
+from cvat.apps.engine.models import DataChoice
+from .prepare import PrepareInfo
+import os
+
+class CacheInteraction:
+    def __init__(self):
+        self._cache = Cache(settings.CACHE_ROOT)
+
+    def __del__(self):
+        self._cache.close()
+
+    def get_buff_mime(self, chunk_number, quality, db_data):
+        chunk, tag = self._cache.get('{}_{}_{}'.format(db_data.id, chunk_number, quality), tag=True)
+
+        if not chunk:
+            chunk, tag = self.prepare_chunk_buff(db_data, quality, chunk_number)
+            self.save_chunk(db_data.id, chunk_number, quality, chunk, tag)
+        return chunk, tag
+
+    def prepare_chunk_buff(self, db_data, quality, chunk_number):
+        from cvat.apps.engine.frame_provider import FrameProvider
+        extractor_classes = {
+            FrameProvider.Quality.COMPRESSED : Mpeg4CompressedChunkWriter if db_data.compressed_chunk_type == DataChoice.VIDEO else ZipCompressedChunkWriter,
+            FrameProvider.Quality.ORIGINAL : Mpeg4ChunkWriter if db_data.original_chunk_type == DataChoice.VIDEO else ZipChunkWriter,
+        }
+
+        image_quality = 100 if extractor_classes[quality] in [Mpeg4ChunkWriter, ZipChunkWriter] else db_data.image_quality
+        mime_type = 'video/mp4' if extractor_classes[quality] in [Mpeg4ChunkWriter, Mpeg4CompressedChunkWriter] else 'application/zip'
+
+        extractor = extractor_classes[quality](image_quality)
+
+        if os.path.exists(db_data.get_meta_path()):
+            meta = PrepareInfo(source_path=os.path.join(db_data.get_upload_dirname(), db_data.video.path),
+                               meta_path=db_data.get_meta_path())
+            frames = []
+            for frame in meta.decode_needed_frames(chunk_number, db_data):
+                frames.append(frame)
+            buff = extractor.save_as_chunk_to_buff(frames)
+        else:
+            img_paths = None
+            with open(db_data.get_dummy_chunk_path(chunk_number), 'r') as dummy_file:
+                img_paths = [os.path.join(db_data.get_upload_dirname(), line.strip()) for line in dummy_file]
+            buff = extractor.save_as_chunk_to_buff(img_paths)
+        return buff, mime_type
+
+    def save_chunk(self, db_data_id, chunk_number, quality, buff, mime_type):
+        self._cache.set('{}_{}_{}'.format(db_data_id, chunk_number, quality), buff, tag=mime_type)

cvat/apps/engine/frame_provider.py

@@ -11,8 +11,8 @@
 
 from cvat.apps.engine.media_extractors import VideoReader, ZipReader
 from cvat.apps.engine.mime_types import mimetypes
-from cvat.apps.engine.models import DataChoice
-
+from cvat.apps.engine.models import DataChoice, StorageMethodChoice
+from .cache import CacheInteraction
 
 class RandomAccessIterator:
     def __init__(self, iterable):
@@ -65,6 +65,19 @@ def load(self, chunk_id):
                     self.reader_class([self.get_chunk_path(chunk_id)]))
             return self.chunk_reader
 
+    class BuffChunkLoader(ChunkLoader):
+        def __init__(self, reader_class, path_getter, quality, db_data):
+            super().__init__(reader_class, path_getter)
+            self.quality = quality
+            self.db_data = db_data
+
+        def load(self, chunk_id):
+            if self.chunk_id != chunk_id:
+                self.chunk_id = chunk_id
+                self.chunk_reader = RandomAccessIterator(
+                    self.reader_class([self.get_chunk_path(chunk_id, self.quality, self.db_data)[0]]))
+            return self.chunk_reader
+
     def __init__(self, db_data):
         self._db_data = db_data
         self._loaders = {}
@@ -73,12 +86,27 @@ def __init__(self, db_data):
             DataChoice.IMAGESET: ZipReader,
             DataChoice.VIDEO: VideoReader,
         }
-        self._loaders[self.Quality.COMPRESSED] = self.ChunkLoader(
-            reader_class[db_data.compressed_chunk_type],
-            db_data.get_compressed_chunk_path)
-        self._loaders[self.Quality.ORIGINAL] = self.ChunkLoader(
-            reader_class[db_data.original_chunk_type],
-            db_data.get_original_chunk_path)
+
+        if db_data.storage_method == StorageMethodChoice.CACHE:
+            cache = CacheInteraction()
+
+            self._loaders[self.Quality.COMPRESSED] = self.BuffChunkLoader(
+                reader_class[db_data.compressed_chunk_type],
+                cache.get_buff_mime,
+                self.Quality.COMPRESSED,
+                self._db_data)
+            self._loaders[self.Quality.ORIGINAL] = self.BuffChunkLoader(
+                reader_class[db_data.original_chunk_type],
+                cache.get_buff_mime,
+                self.Quality.ORIGINAL,
+                self._db_data)
+        else:
+            self._loaders[self.Quality.COMPRESSED] = self.ChunkLoader(
+                reader_class[db_data.compressed_chunk_type],
+                db_data.get_compressed_chunk_path)
+            self._loaders[self.Quality.ORIGINAL] = self.ChunkLoader(
+                reader_class[db_data.original_chunk_type],
+                db_data.get_original_chunk_path)
 
     def __len__(self):
         return self._db_data.size
@@ -129,6 +157,8 @@ def get_preview(self):
 
     def get_chunk(self, chunk_number, quality=Quality.ORIGINAL):
         chunk_number = self._validate_chunk_number(chunk_number)
+        if self._db_data.storage_method == StorageMethodChoice.CACHE:
+            return self._loaders[quality].get_chunk_path(chunk_number, quality, self._db_data)
         return self._loaders[quality].get_chunk_path(chunk_number)
 
     def get_frame(self, frame_number, quality=Quality.ORIGINAL,