moved engine.settings to engine.media_extractors module

cvat/apps/engine/media_extractors.py

@@ -7,6 +7,20 @@
 from pyunpack import Archive
 from PIL import Image
 
+import mimetypes
+_SCRIPT_DIR = os.path.realpath(os.path.dirname(__file__))
+MEDIA_MIMETYPES_FILES = [
+    os.path.join(_SCRIPT_DIR, "media.mimetypes"),
+]
+mimetypes.init(files=MEDIA_MIMETYPES_FILES)
+
+def get_mime(name):
+    for type_name, type_def in MEDIA_TYPES.items():
+        if type_def['has_mime_type'](name):
+            return type_name
+
+    return 'unknown'
+
 class MediaExtractor:
     def __init__(self, source_path, dest_path, image_quality, step, start, stop):
         self._source_path = source_path
@@ -61,15 +75,14 @@ def save_image(self, k, dest_path):
 #Note step, start, stop have no affect
 class DirectoryExtractor(ImageListExtractor):
     def __init__(self, source_path, dest_path, image_quality, step=1, start=0, stop=0):
-        from cvat.apps.engine.settings import _get_mime
         image_paths = []
         for source in source_path:
             for root, _, files in os.walk(source):
                 paths = [os.path.join(root, f) for f in files]
-                paths = filter(lambda x: _get_mime(x) == 'image', paths)
+                paths = filter(lambda x: get_mime(x) == 'image', paths)
                 image_paths.extend(paths)
         super().__init__(
-            source_path=sorted(source_path),
+            source_path=sorted(image_paths),
             dest_path=dest_path,
             image_quality=image_quality,
             step=1,
@@ -80,16 +93,15 @@ def __init__(self, source_path, dest_path, image_quality, step=1, start=0, stop=
 #Note step, start, stop have no affect
 class ArchiveExtractor(ImageListExtractor):
     def __init__(self, source_path, dest_path, image_quality, step=1, start=0, stop=0):
-        from cvat.apps.engine.settings import _get_mime
         Archive(source_path[0]).extractall(dest_path)
         os.remove(source_path[0])
         image_paths = []
         for root, _, files in os.walk(dest_path):
             paths = [os.path.join(root, f) for f in files]
-            paths = filter(lambda x: _get_mime(x) == 'image', paths)
+            paths = filter(lambda x: get_mime(x) == 'image', paths)
             image_paths.extend(paths)
         super().__init__(
-            source_path=sorted(source_path),
+            source_path=sorted(image_paths),
             dest_path=dest_path,
             image_quality=image_quality,
             step=1,
@@ -153,3 +165,58 @@ def __len__(self):
 
     def save_image(self, k, dest_path):
         shutil.copyfile(self[k], dest_path)
+
+def _is_archive(path):
+    mime = mimetypes.guess_type(path)
+    mime_type = mime[0]
+    encoding = mime[1]
+    supportedArchives = ['application/zip', 'application/x-rar-compressed',
+        'application/x-tar', 'application/x-7z-compressed', 'application/x-cpio',
+        'gzip', 'bzip2']
+    return mime_type in supportedArchives or encoding in supportedArchives
+
+def _is_video(path):
+    mime = mimetypes.guess_type(path)
+    return mime[0] is not None and mime[0].startswith('video')
+
+def _is_image(path):
+    mime = mimetypes.guess_type(path)
+    return mime[0] is not None and mime[0].startswith('image')
+
+def _is_dir(path):
+    return os.path.isdir(path)
+
+# 'has_mime_type': function receives 1 argument - path to file.
+#                  Should return True if file has specified media type.
+# 'extractor': class that extracts images from specified media.
+# 'mode': 'annotation' or 'interpolation' - mode of task that should be created.
+# 'unique': True or False - describes how the type can be combined with other.
+#           True - only one item of this type and no other is allowed
+#           False - this media types can be combined with other which have unique == False
+
+MEDIA_TYPES = {
+    'image': {
+        'has_mime_type': _is_image,
+        'extractor': ImageListExtractor,
+        'mode': 'annotation',
+        'unique': False,
+    },
+    'video': {
+        'has_mime_type': _is_video,
+        'extractor': VideoExtractor,
+        'mode': 'interpolation',
+        'unique': True,
+    },
+    'archive': {
+        'has_mime_type': _is_archive,
+        'extractor': ArchiveExtractor,
+        'mode': 'annotation',
+        'unique': True,
+    },
+    'directory': {
+        'has_mime_type': _is_dir,
+        'extractor': DirectoryExtractor,
+        'mode': 'annotation',
+        'unique': False,
+    },
+}

cvat/apps/engine/migrations/0016_attribute_spec_20190217.py

@@ -5,7 +5,7 @@
 from PIL import Image
 from django.db import migrations
 from django.conf import settings
-from cvat.apps.engine.settings import _get_mime
+from cvat.apps.engine.media_extractors import get_mime
 
 def parse_attribute(value):
     match = re.match(r'^([~@])(\w+)=(\w+):(.+)?$', value)
@@ -81,7 +81,7 @@ def fill_task_meta_data_forward(apps, schema_editor):
             video = ""
             for root, _, files in os.walk(_get_upload_dirname(db_task)):
                 fullnames = map(lambda f: os.path.join(root, f), files)
-                videos = list(filter(lambda x: _get_mime(x) == 'video', fullnames))
+                videos = list(filter(lambda x: get_mime(x) == 'video', fullnames))
                 if len(videos):
                     video = videos[0]
                     break
@@ -100,7 +100,7 @@ def fill_task_meta_data_forward(apps, schema_editor):
             filenames = []
             for root, _, files in os.walk(_get_upload_dirname(db_task)):
                 fullnames = map(lambda f: os.path.join(root, f), files)
-                images = filter(lambda x: _get_mime(x) == 'image', fullnames)
+                images = filter(lambda x: get_mime(x) == 'image', fullnames)
                 filenames.extend(images)
             filenames.sort()
 

cvat/apps/engine/task.py

@@ -14,7 +14,7 @@
 from urllib import parse as urlparse
 from urllib import request as urlrequest
 
-from cvat.apps.engine.settings import _get_mime, MEDIA_TYPES
+from cvat.apps.engine.media_extractors import get_mime, MEDIA_TYPES
 
 import django_rq
 from django.conf import settings
@@ -60,7 +60,7 @@ def make_image_meta_cache(db_task):
             filenames = []
             for root, _, files in os.walk(db_task.get_upload_dirname()):
                 fullnames = map(lambda f: os.path.join(root, f), files)
-                images = filter(lambda x: _get_mime(x) == 'image', fullnames)
+                images = filter(lambda x: get_mime(x) == 'image', fullnames)
                 filenames.extend(images)
             filenames.sort()
 
@@ -152,7 +152,7 @@ def _validate_data(data):
         if '..' in path.split(os.path.sep):
             raise ValueError("Don't use '..' inside file paths")
         full_path = os.path.abspath(os.path.join(share_root, path))
-        if 'directory' == _get_mime(full_path):
+        if 'directory' == get_mime(full_path):
             server_files['dirs'].append(path)
         else:
             server_files['files'].append(path)
@@ -165,7 +165,7 @@ def _validate_data(data):
 
     def count_files(file_mapping, counter):
         for rel_path, full_path in file_mapping.items():
-            mime = _get_mime(full_path)
+            mime = get_mime(full_path)
             counter[mime].append(rel_path)
 
     counter = { media_type: [] for media_type in MEDIA_TYPES.keys() }