Prepare chunks on the fly

CHANGELOG.md

@@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Ability to configure email verification for new users (<https://github.com/opencv/cvat/pull/1929>)
 - Link to django admin page from UI (<https://github.com/opencv/cvat/pull/2068>)
 - Notification message when users use wrong browser (<https://github.com/opencv/cvat/pull/2070>)
+- Ability to work with data on the fly (https://github.com/opencv/cvat/pull/2007)
 
 ### Changed
 - Shape coordinates are rounded to 2 digits in dumped annotations (<https://github.com/opencv/cvat/pull/1970>)

cvat-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cvat-core",
-  "version": "3.5.0",
+  "version": "3.6.0",
   "description": "Part of Computer Vision Tool which presents an interface for client-side integration",
   "main": "babel.config.js",
   "scripts": {

cvat-core/src/session.js

@@ -819,6 +819,7 @@
                 data_compressed_chunk_type: undefined,
                 data_original_chunk_type: undefined,
                 use_zip_chunks: undefined,
+                use_cache: undefined,
             };
 
             for (const property in data) {
@@ -1074,6 +1075,24 @@
                         data.use_zip_chunks = useZipChunks;
                     },
                 },
+                /**
+                    * @name useCache
+                    * @type {boolean}
+                    * @memberof module:API.cvat.classes.Task
+                    * @instance
+                    * @throws {module:API.cvat.exceptions.ArgumentError}
+                */
+                useCache: {
+                    get: () => data.use_cache,
+                    set: (useCache) => {
+                        if (typeof (useCache) !== 'boolean') {
+                            throw new ArgumentError(
+                                'Value must be a boolean',
+                            );
+                        }
+                        data.use_cache = useCache;
+                    },
+                },
                 /**
                     * After task has been created value can be appended only.
                     * @name labels
@@ -1645,6 +1664,7 @@
             remote_files: this.remoteFiles,
             image_quality: this.imageQuality,
             use_zip_chunks: this.useZipChunks,
+            use_cache: this.useCache,
         };
 
         if (typeof (this.startFrame) !== 'undefined') {

cvat-ui/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cvat-ui",
-  "version": "1.8.4",
+  "version": "1.9.0",
   "description": "CVAT single-page application",
   "main": "src/index.tsx",
   "scripts": {

cvat-ui/src/actions/tasks-actions.ts

@@ -386,6 +386,7 @@ ThunkAction<Promise<void>, {}, {}, AnyAction> {
             z_order: data.advanced.zOrder,
             image_quality: 70,
             use_zip_chunks: data.advanced.useZipChunks,
+            use_cache: data.advanced.useCache,
         };
 
         if (data.advanced.bugTracker) {

cvat-ui/src/components/create-task-page/advanced-configuration-form.tsx

@@ -26,6 +26,7 @@ export interface AdvancedConfiguration {
     repository?: string;
     useZipChunks: boolean;
     dataChunkSize?: number;
+    useCache: boolean;
 }
 
 type Props = FormComponentProps & {
@@ -380,6 +381,24 @@ class AdvancedConfigurationForm extends React.PureComponent<Props> {
         );
     }
 
+    private renderCreateTaskMethod(): JSX.Element {
+        const { form } = this.props;
+        return (
+            <Form.Item help='Using cache to store data.'>
+                {form.getFieldDecorator('useCache', {
+                    initialValue: false,
+                    valuePropName: 'checked',
+                })(
+                    <Checkbox>
+                        <Text className='cvat-text-color'>
+                            Use cache
+                        </Text>
+                    </Checkbox>,
+                )}
+            </Form.Item>
+        );
+    }
+
     private renderChunkSize(): JSX.Element {
         const { form } = this.props;
 
@@ -434,6 +453,12 @@ class AdvancedConfigurationForm extends React.PureComponent<Props> {
                     </Col>
                 </Row>
 
+                <Row>
+                    <Col>
+                        {this.renderCreateTaskMethod()}
+                    </Col>
+                </Row>
+
                 <Row type='flex' justify='start'>
                     <Col span={7}>
                         {this.renderImageQuality()}

cvat-ui/src/components/create-task-page/create-task-content.tsx

@@ -42,6 +42,7 @@ const defaultState = {
         zOrder: false,
         lfs: false,
         useZipChunks: true,
+        useCache: true,
     },
     labels: [],
     files: {

cvat/apps/engine/cache.py

@@ -0,0 +1,57 @@
+# Copyright (C) 2020 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+from diskcache import Cache
+from django.conf import settings
+from cvat.apps.engine.media_extractors import (Mpeg4ChunkWriter, ZipChunkWriter,
+    Mpeg4CompressedChunkWriter, ZipCompressedChunkWriter)
+from cvat.apps.engine.models import DataChoice
+from .prepare import PrepareInfo
+import os
+from io import BytesIO
+
+class CacheInteraction:
+    def __init__(self):
+        self._cache = Cache(settings.CACHE_ROOT)
+
+    def __del__(self):
+        self._cache.close()
+
+    def get_buff_mime(self, chunk_number, quality, db_data):
+        chunk, tag = self._cache.get('{}_{}_{}'.format(db_data.id, chunk_number, quality), tag=True)
+
+        if not chunk:
+            chunk, tag = self.prepare_chunk_buff(db_data, quality, chunk_number)
+            self.save_chunk(db_data.id, chunk_number, quality, chunk, tag)
+        return chunk, tag
+
+    def prepare_chunk_buff(self, db_data, quality, chunk_number):
+        from cvat.apps.engine.frame_provider import FrameProvider
+        extractor_classes = {
+            FrameProvider.Quality.COMPRESSED : Mpeg4CompressedChunkWriter if db_data.compressed_chunk_type == DataChoice.VIDEO else ZipCompressedChunkWriter,
+            FrameProvider.Quality.ORIGINAL : Mpeg4ChunkWriter if db_data.original_chunk_type == DataChoice.VIDEO else ZipChunkWriter,
+        }
+
+        image_quality = 100 if extractor_classes[quality] in [Mpeg4ChunkWriter, ZipChunkWriter] else db_data.image_quality
+        mime_type = 'video/mp4' if extractor_classes[quality] in [Mpeg4ChunkWriter, Mpeg4CompressedChunkWriter] else 'application/zip'
+
+        extractor = extractor_classes[quality](image_quality)
+
+        images = []
+        buff = BytesIO()
+        if os.path.exists(db_data.get_meta_path()):
+            source_path = os.path.join(db_data.get_upload_dirname(), db_data.video.path)
+            meta = PrepareInfo(source_path=source_path, meta_path=db_data.get_meta_path())
+            for frame in meta.decode_needed_frames(chunk_number, db_data):
+                images.append(frame)
+            extractor.save_as_chunk([(image, source_path, None) for image in images], buff)
+        else:
+            with open(db_data.get_dummy_chunk_path(chunk_number), 'r') as dummy_file:
+                images = [os.path.join(db_data.get_upload_dirname(), line.strip()) for line in dummy_file]
+            extractor.save_as_chunk([(image, image, None) for image in images], buff)
+        buff.seek(0)
+        return buff, mime_type
+
+    def save_chunk(self, db_data_id, chunk_number, quality, buff, mime_type):
+        self._cache.set('{}_{}_{}'.format(db_data_id, chunk_number, quality), buff, tag=mime_type)

cvat/apps/engine/frame_provider.py

@@ -11,8 +11,8 @@
 
 from cvat.apps.engine.media_extractors import VideoReader, ZipReader
 from cvat.apps.engine.mime_types import mimetypes
-from cvat.apps.engine.models import DataChoice
-
+from cvat.apps.engine.models import DataChoice, StorageMethodChoice
+from .cache import CacheInteraction
 
 class RandomAccessIterator:
     def __init__(self, iterable):
@@ -65,6 +65,19 @@ def load(self, chunk_id):
                     self.reader_class([self.get_chunk_path(chunk_id)]))
             return self.chunk_reader
 
+    class BuffChunkLoader(ChunkLoader):
+        def __init__(self, reader_class, path_getter, quality, db_data):
+            super().__init__(reader_class, path_getter)
+            self.quality = quality
+            self.db_data = db_data
+
+        def load(self, chunk_id):
+            if self.chunk_id != chunk_id:
+                self.chunk_id = chunk_id
+                self.chunk_reader = RandomAccessIterator(
+                    self.reader_class([self.get_chunk_path(chunk_id, self.quality, self.db_data)[0]]))
+            return self.chunk_reader
+
     def __init__(self, db_data):
         self._db_data = db_data
         self._loaders = {}
@@ -73,12 +86,27 @@ def __init__(self, db_data):
             DataChoice.IMAGESET: ZipReader,
             DataChoice.VIDEO: VideoReader,
         }
-        self._loaders[self.Quality.COMPRESSED] = self.ChunkLoader(
-            reader_class[db_data.compressed_chunk_type],
-            db_data.get_compressed_chunk_path)
-        self._loaders[self.Quality.ORIGINAL] = self.ChunkLoader(
-            reader_class[db_data.original_chunk_type],
-            db_data.get_original_chunk_path)
+
+        if db_data.storage_method == StorageMethodChoice.CACHE:
+            cache = CacheInteraction()
+
+            self._loaders[self.Quality.COMPRESSED] = self.BuffChunkLoader(
+                reader_class[db_data.compressed_chunk_type],
+                cache.get_buff_mime,
+                self.Quality.COMPRESSED,
+                self._db_data)
+            self._loaders[self.Quality.ORIGINAL] = self.BuffChunkLoader(
+                reader_class[db_data.original_chunk_type],
+                cache.get_buff_mime,
+                self.Quality.ORIGINAL,
+                self._db_data)
+        else:
+            self._loaders[self.Quality.COMPRESSED] = self.ChunkLoader(
+                reader_class[db_data.compressed_chunk_type],
+                db_data.get_compressed_chunk_path)
+            self._loaders[self.Quality.ORIGINAL] = self.ChunkLoader(
+                reader_class[db_data.original_chunk_type],
+                db_data.get_original_chunk_path)
 
     def __len__(self):
         return self._db_data.size
@@ -129,6 +157,8 @@ def get_preview(self):
 
     def get_chunk(self, chunk_number, quality=Quality.ORIGINAL):
         chunk_number = self._validate_chunk_number(chunk_number)
+        if self._db_data.storage_method == StorageMethodChoice.CACHE:
+            return self._loaders[quality].get_chunk_path(chunk_number, quality, self._db_data)
         return self._loaders[quality].get_chunk_path(chunk_number)
 
     def get_frame(self, frame_number, quality=Quality.ORIGINAL,

cvat/apps/engine/media_extractors.py

@@ -125,22 +125,17 @@ def __init__(self, source_path, step=1, start=0, stop=None):
 
 class ArchiveReader(DirectoryReader):
     def __init__(self, source_path, step=1, start=0, stop=None):
-        self._tmp_dir = create_tmp_dir()
         self._archive_source = source_path[0]
-        Archive(self._archive_source).extractall(self._tmp_dir)
+        Archive(self._archive_source).extractall(os.path.dirname(source_path[0]))
         super().__init__(
-            source_path=[self._tmp_dir],
+            source_path=[os.path.dirname(source_path[0])],
             step=step,
             start=start,
             stop=stop,
         )
 
     def __del__(self):
-        delete_tmp_dir(self._tmp_dir)
-
-    def get_path(self, i):
-        base_dir = os.path.dirname(self._archive_source)
-        return os.path.join(base_dir, os.path.relpath(self._source_path[i], self._tmp_dir))
+        os.remove(self._archive_source)
 
 class PdfReader(DirectoryReader):
     def __init__(self, source_path, step=1, start=0, stop=None):
@@ -191,7 +186,14 @@ def get_image(self, i):
         return io.BytesIO(self._zip_source.read(self._source_path[i]))
 
     def get_path(self, i):
-        return os.path.join(os.path.dirname(self._zip_source.filename), self._source_path[i])
+        if  self._zip_source.filename:
+            return os.path.join(os.path.dirname(self._zip_source.filename), self._source_path[i])
+        else: # necessary for mime_type definition
+            return self._source_path[i]
+
+    def extract(self):
+        self._zip_source.extractall(os.path.dirname(self._zip_source.filename))
+        os.remove(self._zip_source.filename)
 
 class VideoReader(IMediaReader):
     def __init__(self, source_path, step=1, start=0, stop=None):
@@ -303,14 +305,14 @@ def __init__(self, _):
         self._output_fps = 25
 
     @staticmethod
-    def _create_av_container(path, w, h, rate, options):
+    def _create_av_container(path, w, h, rate, options, f='mp4'):
             # x264 requires width and height must be divisible by 2 for yuv420p
             if h % 2:
                 h += 1
             if w % 2:
                 w += 1
 
-            container = av.open(path, 'w')
+            container = av.open(path, 'w',format=f)
             video_stream = container.add_stream('libx264', rate=rate)
             video_stream.pix_fmt = "yuv420p"
             video_stream.width = w

cvat/apps/engine/migrations/0029_data_storage_method.py

@@ -0,0 +1,36 @@
+# Generated by Django 2.2.13 on 2020-08-13 05:49
+
+from cvat.apps.engine.media_extractors import _is_archive, _is_zip
+import cvat.apps.engine.models
+from django.conf import settings
+from django.db import migrations, models
+import os
+from pyunpack import Archive
+
+def unzip(apps, schema_editor):
+    Data = apps.get_model("engine", "Data")
+    data_q_set = Data.objects.all()
+    archive_paths = []
+
+    for data_instance in data_q_set:
+        for root, _, files in os.walk(os.path.join(settings.MEDIA_DATA_ROOT, '{}/raw/'.format(data_instance.id))):
+            archive_paths.extend([os.path.join(root, file) for file in files if _is_archive(file) or _is_zip(file)])
+
+    for path in archive_paths:
+        Archive(path).extractall(os.path.dirname(path))
+        os.remove(path)
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('engine', '0028_labelcolor'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='data',
+            name='storage_method',
+            field=models.CharField(choices=[('cache', 'CACHE'), ('file_system', 'FILE_SYSTEM')], default=cvat.apps.engine.models.StorageMethodChoice('file_system'), max_length=15),
+        ),
+        migrations.RunPython(unzip),
+    ]