From da3fa34ce018df69244d8f6a3f943a092cf52c89 Mon Sep 17 00:00:00 2001 From: zhiltsov-max Date: Thu, 4 Jun 2020 14:12:37 +0300 Subject: [PATCH 1/4] Add tf import checks and tests (#1567) * Add tf import checks and tests * implement disabled formats on server * python 3.5 compatibility * add checks to dm tests * fix tests * Support for disabled formats in UI * add sorting for formats, mark grey disabled items * update changelog * advance package versions --- CHANGELOG.md | 3 +- cvat-core/package.json | 2 +- cvat-core/src/annotation-formats.js | 22 ++++++++++++ cvat-ui/package-lock.json | 2 +- cvat-ui/package.json | 2 +- .../components/actions-menu/actions-menu.tsx | 4 +-- .../components/actions-menu/dump-submenu.tsx | 18 ++++++---- .../actions-menu/export-submenu.tsx | 16 +++++---- .../components/actions-menu/load-submenu.tsx | 23 ++++++++----- .../top-bar/annotation-menu.tsx | 4 +-- .../containers/actions-menu/actions-menu.tsx | 6 ++-- .../top-bar/annotation-menu.tsx | 6 ++-- cvat/apps/dataset_manager/formats/registry.py | 15 +++++--- cvat/apps/dataset_manager/formats/tfrecord.py | 12 +++++-- cvat/apps/dataset_manager/serializers.py | 1 + .../dataset_manager/tests/_test_formats.py | 6 ++++ cvat/apps/engine/tests/_test_rest_api.py | 17 ++++++---- cvat/apps/engine/views.py | 26 +++++++++----- datumaro/datumaro/components/project.py | 12 ++++++- .../datumaro/plugins/openvino_launcher.py | 17 ++-------- datumaro/datumaro/util/os_util.py | 17 ++++++++++ datumaro/datumaro/util/tf_util.py | 34 ++++++++++++++++++- datumaro/tests/test_tfrecord_format.py | 30 ++++++++++++---- 23 files changed, 217 insertions(+), 78 deletions(-) create mode 100644 datumaro/datumaro/util/os_util.py diff --git a/CHANGELOG.md b/CHANGELOG.md index fc541fcea2c..ba199027978 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Throttling policy for unauthenticated users () - Added default label color table for mask export (https://github.com/opencv/cvat/pull/1549) +- Added visual identification for unavailable formats (https://github.com/opencv/cvat/pull/1567) ### Changed - Removed information about e-mail from the basic user information () @@ -19,7 +20,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - ### Fixed -- +- Fixed interpreter crash when trying to import `tensorflow` with no AVX instructions available (https://github.com/opencv/cvat/pull/1567) ### Security - diff --git a/cvat-core/package.json b/cvat-core/package.json index b2f95a0c161..b347fcb15b1 100644 --- a/cvat-core/package.json +++ b/cvat-core/package.json @@ -1,6 +1,6 @@ { "name": "cvat-core", - "version": "2.0.1", + "version": "2.1.1", "description": "Part of Computer Vision Tool which presents an interface for client-side integration", "main": "babel.config.js", "scripts": { diff --git a/cvat-core/src/annotation-formats.js b/cvat-core/src/annotation-formats.js index 641f7bc0616..a248c9d72bd 100644 --- a/cvat-core/src/annotation-formats.js +++ b/cvat-core/src/annotation-formats.js @@ -15,6 +15,7 @@ name: initialData.name, format: initialData.ext, version: initialData.version, + enabled: initialData.enabled, }; Object.defineProperties(this, { @@ -48,6 +49,16 @@ */ get: () => data.version, }, + enabled: { + /** + * @name enabled + * @type {string} + * @memberof module:API.cvat.classes.Loader + * @readonly + * @instance + */ + get: () => data.enabled, + }, }); } } @@ -63,6 +74,7 @@ name: initialData.name, format: initialData.ext, version: initialData.version, + enabled: initialData.enabled, }; Object.defineProperties(this, { @@ -96,6 +108,16 @@ */ get: () => data.version, }, + enabled: { + /** + * @name enabled + * @type {string} + * @memberof module:API.cvat.classes.Loader + * @readonly + * @instance + */ + get: () => data.enabled, + }, }); } } diff --git a/cvat-ui/package-lock.json b/cvat-ui/package-lock.json index 6960db8dcb6..a375e42da9d 100644 --- a/cvat-ui/package-lock.json +++ b/cvat-ui/package-lock.json @@ -1,6 +1,6 @@ { "name": "cvat-ui", - "version": "1.2.0", + "version": "1.2.1", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/cvat-ui/package.json b/cvat-ui/package.json index 00831d57a64..d9c4c6e6d53 100644 --- a/cvat-ui/package.json +++ b/cvat-ui/package.json @@ -1,6 +1,6 @@ { "name": "cvat-ui", - "version": "1.2.0", + "version": "1.2.1", "description": "CVAT single-page application", "main": "src/index.tsx", "scripts": { diff --git a/cvat-ui/src/components/actions-menu/actions-menu.tsx b/cvat-ui/src/components/actions-menu/actions-menu.tsx index 7d4f305bd50..5a1070d05a0 100644 --- a/cvat-ui/src/components/actions-menu/actions-menu.tsx +++ b/cvat-ui/src/components/actions-menu/actions-menu.tsx @@ -16,8 +16,8 @@ interface Props { taskMode: string; bugTracker: string; - loaders: string[]; - dumpers: string[]; + loaders: any[]; + dumpers: any[]; loadActivity: string | null; dumpActivities: string[] | null; exportActivities: string[] | null; diff --git a/cvat-ui/src/components/actions-menu/dump-submenu.tsx b/cvat-ui/src/components/actions-menu/dump-submenu.tsx index 23a6ab6c147..803a5db9892 100644 --- a/cvat-ui/src/components/actions-menu/dump-submenu.tsx +++ b/cvat-ui/src/components/actions-menu/dump-submenu.tsx @@ -15,7 +15,7 @@ function isDefaultFormat(dumperName: string, taskMode: string): boolean { interface Props { taskMode: string; menuKey: string; - dumpers: string[]; + dumpers: any[]; dumpActivities: string[] | null; } @@ -30,17 +30,21 @@ export default function DumpSubmenu(props: Props): JSX.Element { return ( { - dumpers.map((dumper: string): JSX.Element => { - const pending = (dumpActivities || []).includes(dumper); - const isDefault = isDefaultFormat(dumper, taskMode); + dumpers + .sort((a: any, b: any) => a.name.localeCompare(b.name)) + .map((dumper: any): JSX.Element => + { + const pending = (dumpActivities || []).includes(dumper.name); + const disabled = !dumper.enabled || pending; + const isDefault = isDefaultFormat(dumper.name, taskMode); return ( - {dumper} + {dumper.name} {pending && } ); diff --git a/cvat-ui/src/components/actions-menu/export-submenu.tsx b/cvat-ui/src/components/actions-menu/export-submenu.tsx index e5dd53e8388..045d682f85e 100644 --- a/cvat-ui/src/components/actions-menu/export-submenu.tsx +++ b/cvat-ui/src/components/actions-menu/export-submenu.tsx @@ -9,7 +9,7 @@ import Text from 'antd/lib/typography/Text'; interface Props { menuKey: string; - exporters: string[]; + exporters: any[]; exportActivities: string[] | null; } @@ -23,16 +23,20 @@ export default function ExportSubmenu(props: Props): JSX.Element { return ( { - exporters.map((exporter: string): JSX.Element => { - const pending = (exportActivities || []).includes(exporter); + exporters + .sort((a: any, b: any) => a.name.localeCompare(b.name)) + .map((exporter: any): JSX.Element => + { + const pending = (exportActivities || []).includes(exporter.name); + const disabled = !exporter.enabled || pending; return ( - {exporter} + {exporter.name} {pending && } ); diff --git a/cvat-ui/src/components/actions-menu/load-submenu.tsx b/cvat-ui/src/components/actions-menu/load-submenu.tsx index 110761a31ee..2db41d17649 100644 --- a/cvat-ui/src/components/actions-menu/load-submenu.tsx +++ b/cvat-ui/src/components/actions-menu/load-submenu.tsx @@ -11,7 +11,7 @@ import Text from 'antd/lib/typography/Text'; interface Props { menuKey: string; - loaders: string[]; + loaders: any[]; loadActivity: string | null; onFileUpload(file: File): void; } @@ -27,13 +27,20 @@ export default function LoadSubmenu(props: Props): JSX.Element { return ( { - loaders.map((_loader: string): JSX.Element => { - const [loader, accept] = _loader.split('::'); - const pending = loadActivity === loader; + loaders + .sort((a: any, b: any) => a.name.localeCompare(b.name)) + .map((loader: any): JSX.Element => + { + const accept = loader.format + .split(',') + .map((x: string) => '.' + x.trimStart()) + .join(', '); // add '.' to each extension in a list + const pending = loadActivity === loader.name; + const disabled = !loader.enabled || !!loadActivity; return ( - diff --git a/cvat-ui/src/components/annotation-page/top-bar/annotation-menu.tsx b/cvat-ui/src/components/annotation-page/top-bar/annotation-menu.tsx index 51a460ea378..92da2240d25 100644 --- a/cvat-ui/src/components/annotation-page/top-bar/annotation-menu.tsx +++ b/cvat-ui/src/components/annotation-page/top-bar/annotation-menu.tsx @@ -13,8 +13,8 @@ import ReIDPlugin from './reid-plugin'; interface Props { taskMode: string; - loaders: string[]; - dumpers: string[]; + loaders: any[]; + dumpers: any[]; loadActivity: string | null; dumpActivities: string[] | null; exportActivities: string[] | null; diff --git a/cvat-ui/src/containers/actions-menu/actions-menu.tsx b/cvat-ui/src/containers/actions-menu/actions-menu.tsx index c502eece148..a92a42bb1ff 100644 --- a/cvat-ui/src/containers/actions-menu/actions-menu.tsx +++ b/cvat-ui/src/containers/actions-menu/actions-menu.tsx @@ -134,7 +134,7 @@ function ActionsMenuContainer(props: OwnProps & StateToProps & DispatchToProps): dumpAnnotations(taskInstance, dumper); } } else if (action === Actions.LOAD_TASK_ANNO) { - const [format] = additionalKey.split('::'); + const format = additionalKey; const [loader] = loaders .filter((_loader: any): boolean => _loader.name === format); if (loader && file) { @@ -166,8 +166,8 @@ function ActionsMenuContainer(props: OwnProps & StateToProps & DispatchToProps): taskID={taskInstance.id} taskMode={taskInstance.mode} bugTracker={taskInstance.bugTracker} - loaders={loaders.map((loader: any): string => `${loader.name}::${loader.format}`)} - dumpers={dumpers.map((dumper: any): string => dumper.name)} + loaders={loaders} + dumpers={dumpers} loadActivity={loadActivity} dumpActivities={dumpActivities} exportActivities={exportActivities} diff --git a/cvat-ui/src/containers/annotation-page/top-bar/annotation-menu.tsx b/cvat-ui/src/containers/annotation-page/top-bar/annotation-menu.tsx index b6633885c1b..139168a8f29 100644 --- a/cvat-ui/src/containers/annotation-page/top-bar/annotation-menu.tsx +++ b/cvat-ui/src/containers/annotation-page/top-bar/annotation-menu.tsx @@ -123,7 +123,7 @@ function AnnotationMenuContainer(props: Props): JSX.Element { dumpAnnotations(jobInstance.task, dumper); } } else if (action === Actions.LOAD_JOB_ANNO) { - const [format] = additionalKey.split('::'); + const format = additionalKey; const [loader] = loaders .filter((_loader: any): boolean => _loader.name === format); if (loader && file) { @@ -150,8 +150,8 @@ function AnnotationMenuContainer(props: Props): JSX.Element { return ( loader.name)} - dumpers={dumpers.map((dumper: any): string => dumper.name)} + loaders={loaders} + dumpers={dumpers} loadActivity={loadActivity} dumpActivities={dumpActivities} exportActivities={exportActivities} diff --git a/cvat/apps/dataset_manager/formats/registry.py b/cvat/apps/dataset_manager/formats/registry.py index 20377dd6703..ed4defc559d 100644 --- a/cvat/apps/dataset_manager/formats/registry.py +++ b/cvat/apps/dataset_manager/formats/registry.py @@ -13,6 +13,7 @@ class _Format: EXT = '' VERSION = '' DISPLAY_NAME = '{NAME} {VERSION}' + ENABLED = True class Exporter(_Format): def __call__(self, dst_file, task_data, **options): @@ -22,7 +23,7 @@ class Importer(_Format): def __call__(self, src_file, task_data, **options): raise NotImplementedError() -def _wrap_format(f_or_cls, klass, name, version, ext, display_name): +def _wrap_format(f_or_cls, klass, name, version, ext, display_name, enabled): import inspect assert inspect.isclass(f_or_cls) or inspect.isfunction(f_or_cls) if inspect.isclass(f_or_cls): @@ -44,14 +45,17 @@ def __call__(self, *args, **kwargs): target.DISPLAY_NAME = (display_name or klass.DISPLAY_NAME).format( NAME=name, VERSION=version, EXT=ext) assert all([target.NAME, target.VERSION, target.EXT, target.DISPLAY_NAME]) + target.ENABLED = enabled + return target EXPORT_FORMATS = {} -def exporter(name, version, ext, display_name=None): +def exporter(name, version, ext, display_name=None, enabled=True): assert name not in EXPORT_FORMATS, "Export format '%s' already registered" % name def wrap_with_params(f_or_cls): t = _wrap_format(f_or_cls, Exporter, - name=name, ext=ext, version=version, display_name=display_name) + name=name, ext=ext, version=version, display_name=display_name, + enabled=enabled) key = t.DISPLAY_NAME assert key not in EXPORT_FORMATS, "Export format '%s' already registered" % name EXPORT_FORMATS[key] = t @@ -59,10 +63,11 @@ def wrap_with_params(f_or_cls): return wrap_with_params IMPORT_FORMATS = {} -def importer(name, version, ext, display_name=None): +def importer(name, version, ext, display_name=None, enabled=True): def wrap_with_params(f_or_cls): t = _wrap_format(f_or_cls, Importer, - name=name, ext=ext, version=version, display_name=display_name) + name=name, ext=ext, version=version, display_name=display_name, + enabled=enabled) key = t.DISPLAY_NAME assert key not in IMPORT_FORMATS, "Import format '%s' already registered" % name IMPORT_FORMATS[key] = t diff --git a/cvat/apps/dataset_manager/formats/tfrecord.py b/cvat/apps/dataset_manager/formats/tfrecord.py index 0e4962fa6c4..fef95aa710f 100644 --- a/cvat/apps/dataset_manager/formats/tfrecord.py +++ b/cvat/apps/dataset_manager/formats/tfrecord.py @@ -14,7 +14,15 @@ from .registry import dm_env, exporter, importer -@exporter(name='TFRecord', ext='ZIP', version='1.0') +from datumaro.util.tf_util import import_tf +try: + import_tf() + tf_available = True +except ImportError: + tf_available = False + + +@exporter(name='TFRecord', ext='ZIP', version='1.0', enabled=tf_available) def _export(dst_file, task_data, save_images=False): extractor = CvatTaskDataExtractor(task_data, include_images=save_images) extractor = Dataset.from_extractors(extractor) # apply lazy transforms @@ -25,7 +33,7 @@ def _export(dst_file, task_data, save_images=False): make_zip_archive(temp_dir, dst_file) -@importer(name='TFRecord', ext='ZIP', version='1.0') +@importer(name='TFRecord', ext='ZIP', version='1.0', enabled=tf_available) def _import(src_file, task_data): with TemporaryDirectory() as tmp_dir: Archive(src_file.name).extractall(tmp_dir) diff --git a/cvat/apps/dataset_manager/serializers.py b/cvat/apps/dataset_manager/serializers.py index 51cf71ca8da..e64c0cb93bc 100644 --- a/cvat/apps/dataset_manager/serializers.py +++ b/cvat/apps/dataset_manager/serializers.py @@ -9,6 +9,7 @@ class DatasetFormatSerializer(serializers.Serializer): name = serializers.CharField(max_length=64, source='DISPLAY_NAME') ext = serializers.CharField(max_length=64, source='EXT') version = serializers.CharField(max_length=64, source='VERSION') + enabled = serializers.BooleanField(source='ENABLED') class DatasetFormatsSerializer(serializers.Serializer): importers = DatasetFormatSerializer(many=True) diff --git a/cvat/apps/dataset_manager/tests/_test_formats.py b/cvat/apps/dataset_manager/tests/_test_formats.py index bfea13af88e..d5aa950d0f9 100644 --- a/cvat/apps/dataset_manager/tests/_test_formats.py +++ b/cvat/apps/dataset_manager/tests/_test_formats.py @@ -335,6 +335,9 @@ def check(file_path): self.assertTrue(len(f.read()) != 0) for f in dm.views.get_export_formats(): + if not f.ENABLED: + self.skipTest("Format is disabled") + format_name = f.DISPLAY_NAME for save_images in { True, False }: with self.subTest(format=format_name, save_images=save_images): @@ -359,6 +362,9 @@ def test_empty_images_are_exported(self): ('YOLO 1.1', 'yolo'), ]: with self.subTest(format=format_name): + if not dm.formats.registry.EXPORT_FORMATS[format_name].ENABLED: + self.skipTest("Format is disabled") + task = self._generate_task() def check(file_path): diff --git a/cvat/apps/engine/tests/_test_rest_api.py b/cvat/apps/engine/tests/_test_rest_api.py index c8bee86c460..fcd3d59a1d7 100644 --- a/cvat/apps/engine/tests/_test_rest_api.py +++ b/cvat/apps/engine/tests/_test_rest_api.py @@ -3149,8 +3149,8 @@ def _get_initial_annotation(annotation_format): export_formats = data['exporters'] self.assertTrue(isinstance(import_formats, list) and import_formats) self.assertTrue(isinstance(export_formats, list) and export_formats) - import_formats = { v['name'] for v in import_formats } - export_formats = { v['name'] for v in export_formats } + import_formats = { v['name']: v for v in import_formats } + export_formats = { v['name']: v for v in export_formats } formats = { exp: exp if exp in import_formats else None for exp in export_formats } @@ -3159,12 +3159,12 @@ def _get_initial_annotation(annotation_format): formats['CVAT for video 1.1'] = 'CVAT 1.1' if 'CVAT for images 1.1' in export_formats: formats['CVAT for images 1.1'] = 'CVAT 1.1' - if import_formats ^ export_formats: + if set(import_formats) ^ set(export_formats): # NOTE: this may not be an error, so we should not fail print("The following import formats have no pair:", - import_formats - export_formats) + set(import_formats) - set(export_formats)) print("The following export formats have no pair:", - export_formats - import_formats) + set(export_formats) - set(import_formats)) for export_format, import_format in formats.items(): with self.subTest(export_format=export_format, @@ -3183,7 +3183,12 @@ def _get_initial_annotation(annotation_format): # 3. download annotation response = self._dump_api_v1_tasks_id_annotations(task["id"], annotator, "?format={}".format(export_format)) - self.assertEqual(response.status_code, HTTP_202_ACCEPTED) + if annotator and not export_formats[export_format]['enabled']: + self.assertEqual(response.status_code, + status.HTTP_405_METHOD_NOT_ALLOWED) + continue + else: + self.assertEqual(response.status_code, HTTP_202_ACCEPTED) response = self._dump_api_v1_tasks_id_annotations(task["id"], annotator, "?format={}".format(export_format)) diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index 47dfd80036a..735d77c4337 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -482,7 +482,8 @@ def data(self, request, pk): responses={ '202': openapi.Response(description='Dump of annotations has been started'), '201': openapi.Response(description='Annotations file is ready to download'), - '200': openapi.Response(description='Download of file started') + '200': openapi.Response(description='Download of file started'), + '405': openapi.Response(description='Format is not available'), } ) @swagger_auto_schema(method='put', operation_summary='Method allows to upload task annotations', @@ -494,6 +495,7 @@ def data(self, request, pk): responses={ '202': openapi.Response(description='Uploading has been started'), '201': openapi.Response(description='Uploading has finished'), + '405': openapi.Response(description='Format is not available'), } ) @swagger_auto_schema(method='patch', operation_summary='Method performs a partial update of annotations in a specific task', @@ -619,7 +621,8 @@ def data_info(request, pk): ], responses={'202': openapi.Response(description='Exporting has been started'), '201': openapi.Response(description='Output file is ready for downloading'), - '200': openapi.Response(description='Download of file started') + '200': openapi.Response(description='Download of file started'), + '405': openapi.Response(description='Format is not available'), } ) @action(detail=True, methods=['GET'], serializer_class=None, @@ -799,17 +802,20 @@ def rq_handler(job, exc_type, exc_value, tb): # tags=['tasks']) # @api_view(['PUT']) def _import_annotations(request, rq_id, rq_func, pk, format_name): + format_desc = {f.DISPLAY_NAME: f + for f in dm.views.get_import_formats()}.get(format_name) + if format_desc is None: + raise serializers.ValidationError( + "Unknown input format '{}'".format(format_name)) + elif not format_desc.ENABLED: + return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) + queue = django_rq.get_queue("default") rq_job = queue.fetch_job(rq_id) if not rq_job: serializer = AnnotationFileSerializer(data=request.data) if serializer.is_valid(raise_exception=True): - if format_name not in \ - [f.DISPLAY_NAME for f in dm.views.get_import_formats()]: - raise serializers.ValidationError( - "Unknown input format '{}'".format(format_name)) - anno_file = serializer.validated_data['annotation_file'] fd, filename = mkstemp(prefix='cvat_{}'.format(pk)) with open(filename, 'wb+') as f: @@ -843,9 +849,13 @@ def _export_annotations(db_task, rq_id, request, format_name, action, callback, raise serializers.ValidationError( "Unexpected action specified for the request") - if format_name not in [f.DISPLAY_NAME for f in dm.views.get_export_formats()]: + format_desc = {f.DISPLAY_NAME: f + for f in dm.views.get_export_formats()}.get(format_name) + if format_desc is None: raise serializers.ValidationError( "Unknown format specified for the request") + elif not format_desc.ENABLED: + return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) queue = django_rq.get_queue("default") diff --git a/datumaro/datumaro/components/project.py b/datumaro/datumaro/components/project.py index d4468edd9b9..3c590508626 100644 --- a/datumaro/datumaro/components/project.py +++ b/datumaro/datumaro/components/project.py @@ -235,7 +235,17 @@ def _load_plugins(cls, plugins_dir, types): exports = cls._import_module(module_dir, module_name, types, package) except Exception as e: - log.debug("Failed to import module '%s': %s" % (module_name, e)) + module_search_error = ImportError + try: + module_search_error = ModuleNotFoundError # python 3.6+ + except NameError: + pass + + message = ["Failed to import module '%s': %s", module_name, e] + if isinstance(e, module_search_error): + log.debug(*message) + else: + log.warning(*message) continue log.debug("Imported the following symbols from %s: %s" % \ diff --git a/datumaro/datumaro/plugins/openvino_launcher.py b/datumaro/datumaro/plugins/openvino_launcher.py index 10f12feab61..438a4b3da08 100644 --- a/datumaro/datumaro/plugins/openvino_launcher.py +++ b/datumaro/datumaro/plugins/openvino_launcher.py @@ -10,11 +10,11 @@ import os import os.path as osp import platform -import subprocess from openvino.inference_engine import IENetwork, IEPlugin from datumaro.components.launcher import Launcher +from datumaro.util.os_util import check_instruction_set class InterpreterScript: @@ -45,17 +45,6 @@ class OpenVinoLauncher(Launcher): _DEFAULT_IE_PLUGINS_PATH = "/opt/intel/openvino_2019.1.144/deployment_tools/inference_engine/lib/intel64" _IE_PLUGINS_PATH = os.getenv("IE_PLUGINS_PATH", _DEFAULT_IE_PLUGINS_PATH) - @staticmethod - def _check_instruction_set(instruction): - return instruction == str.strip( - # Let's ignore a warning from bandit about using shell=True. - # In this case it isn't a security issue and we use some - # shell features like pipes. - subprocess.check_output( - 'lscpu | grep -o "{}" | head -1'.format(instruction), - shell=True).decode('utf-8') # nosec - ) - @staticmethod def make_plugin(device='cpu', plugins_path=_IE_PLUGINS_PATH): if plugins_path is None or not osp.isdir(plugins_path): @@ -63,10 +52,10 @@ def make_plugin(device='cpu', plugins_path=_IE_PLUGINS_PATH): (plugins_path)) plugin = IEPlugin(device='CPU', plugin_dirs=[plugins_path]) - if (OpenVinoLauncher._check_instruction_set('avx2')): + if (check_instruction_set('avx2')): plugin.add_cpu_extension(os.path.join(plugins_path, 'libcpu_extension_avx2.so')) - elif (OpenVinoLauncher._check_instruction_set('sse4')): + elif (check_instruction_set('sse4')): plugin.add_cpu_extension(os.path.join(plugins_path, 'libcpu_extension_sse4.so')) elif platform.system() == 'Darwin': diff --git a/datumaro/datumaro/util/os_util.py b/datumaro/datumaro/util/os_util.py new file mode 100644 index 00000000000..b4d05e376db --- /dev/null +++ b/datumaro/datumaro/util/os_util.py @@ -0,0 +1,17 @@ + +# Copyright (C) 2020 Intel Corporation +# +# SPDX-License-Identifier: MIT + +import subprocess + + +def check_instruction_set(instruction): + return instruction == str.strip( + # Let's ignore a warning from bandit about using shell=True. + # In this case it isn't a security issue and we use some + # shell features like pipes. + subprocess.check_output( + 'lscpu | grep -o "%s" | head -1' % instruction, + shell=True).decode('utf-8') # nosec + ) \ No newline at end of file diff --git a/datumaro/datumaro/util/tf_util.py b/datumaro/datumaro/util/tf_util.py index 00bf834a0f0..841fc53faf1 100644 --- a/datumaro/datumaro/util/tf_util.py +++ b/datumaro/datumaro/util/tf_util.py @@ -3,7 +3,36 @@ # # SPDX-License-Identifier: MIT -def import_tf(): + +def check_import(): + # Workaround for checking import availability: + # Official TF builds include AVX instructions. Once we try to import, + # the program crashes. We raise an exception instead. + + import subprocess + import sys + + from .os_util import check_instruction_set + + result = subprocess.run([sys.executable, '-c', 'import tensorflow'], + timeout=60, + universal_newlines=True, # use text mode for output stream + stdout=subprocess.PIPE, stderr=subprocess.PIPE) # capture output + + if result.returncode != 0: + message = result.stderr + if not message: + message = "Can't import tensorflow. " \ + "Test process exit code: %s." % result.returncode + if not check_instruction_set('avx'): + # The process has probably crashed for AVX unavalability + message += " This is likely because your CPU does not " \ + "support AVX instructions, " \ + "which are required for tensorflow." + + raise ImportError(message) + +def import_tf(check=True): import sys tf = sys.modules.get('tensorflow', None) @@ -14,6 +43,9 @@ def import_tf(): import os os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' + if check: + check_import() + import tensorflow as tf try: diff --git a/datumaro/tests/test_tfrecord_format.py b/datumaro/tests/test_tfrecord_format.py index 737ea6cf5f7..cc55a9fc9a3 100644 --- a/datumaro/tests/test_tfrecord_format.py +++ b/datumaro/tests/test_tfrecord_format.py @@ -1,17 +1,34 @@ import numpy as np -from unittest import TestCase +from unittest import TestCase, skipIf from datumaro.components.extractor import (Extractor, DatasetItem, AnnotationType, Bbox, Mask, LabelCategories ) -from datumaro.plugins.tf_detection_api_format.importer import TfDetectionApiImporter -from datumaro.plugins.tf_detection_api_format.extractor import TfDetectionApiExtractor -from datumaro.plugins.tf_detection_api_format.converter import TfDetectionApiConverter from datumaro.util.image import Image from datumaro.util.test_utils import TestDir, compare_datasets - - +from datumaro.util.tf_util import check_import + +try: + from datumaro.plugins.tf_detection_api_format.importer import TfDetectionApiImporter + from datumaro.plugins.tf_detection_api_format.extractor import TfDetectionApiExtractor + from datumaro.plugins.tf_detection_api_format.converter import TfDetectionApiConverter + import_failed = False +except ImportError: + import_failed = True + + import importlib + module_found = importlib.util.find_spec('tensorflow') is not None + + @skipIf(not module_found, "Tensorflow package is not found") + class TfImportTest(TestCase): + def test_raises_when_crashes_on_import(self): + # Should fire if import can't be done for any reason except + # module unavailability and import crash + with self.assertRaisesRegex(ImportError, 'Test process exit code'): + check_import() + +@skipIf(import_failed, "Failed to import tensorflow") class TfrecordConverterTest(TestCase): def _test_save_and_load(self, source_dataset, converter, test_dir, target_dataset=None, importer_args=None): @@ -171,6 +188,7 @@ def test_labelmap_parsing(self): self.assertEqual(expected, parsed) +@skipIf(import_failed, "Failed to import tensorflow") class TfrecordImporterTest(TestCase): def test_can_detect(self): class TestExtractor(Extractor): From d1b2960b3d8e63174fe721482dd34aad8412d879 Mon Sep 17 00:00:00 2001 From: zhiltsov-max Date: Thu, 4 Jun 2020 15:25:36 +0300 Subject: [PATCH 2/4] [Datumaro] Add dataset examples (#1640) * add dataset examples --- datumaro/README.md | 13 ++- .../annotations/instances_val.json | 59 +++++++++++ .../coco_dataset/images/val/000000000001.jpg | Bin 0 -> 631 bytes .../cvat_dataset/for_images/images/img0.jpg | Bin 0 -> 631 bytes .../cvat_dataset/for_images/images/img1.jpg | Bin 0 -> 631 bytes .../assets/cvat_dataset/for_images/train.xml | 45 +++++++++ .../cvat_dataset/for_video/annotations.xml | 92 ++++++++++++++++++ .../for_video/images/frame_000010.png | Bin 0 -> 111 bytes .../for_video/images/frame_000013.png | Bin 0 -> 111 bytes datumaro/tests/assets/mot_dataset/gt/gt.txt | 1 + .../tests/assets/mot_dataset/gt/labels.txt | 10 ++ .../tests/assets/mot_dataset/img1/000001.jpg | Bin 0 -> 631 bytes .../tf_detection_api_dataset/label_map.pbtxt | 50 ++++++++++ .../tf_detection_api_dataset/test.tfrecord | Bin 0 -> 803 bytes .../tf_detection_api_dataset/train.tfrecord | Bin 0 -> 1067 bytes .../tf_detection_api_dataset/val.tfrecord | Bin 0 -> 1022 bytes .../voc_dataset/Annotations/2007_000001.xml | 54 ++++++++++ .../voc_dataset/ImageSets/Action/test.txt | 1 + .../voc_dataset/ImageSets/Action/train.txt | 1 + .../voc_dataset/ImageSets/Layout/test.txt | 1 + .../voc_dataset/ImageSets/Layout/train.txt | 1 + .../ImageSets/Main/aeroplane_train.txt | 1 + .../ImageSets/Main/background_train.txt | 1 + .../ImageSets/Main/bicycle_train.txt | 1 + .../voc_dataset/ImageSets/Main/bird_train.txt | 1 + .../voc_dataset/ImageSets/Main/boat_train.txt | 1 + .../ImageSets/Main/bottle_train.txt | 1 + .../voc_dataset/ImageSets/Main/bus_train.txt | 1 + .../voc_dataset/ImageSets/Main/car_train.txt | 1 + .../voc_dataset/ImageSets/Main/cat_train.txt | 1 + .../ImageSets/Main/chair_train.txt | 1 + .../voc_dataset/ImageSets/Main/cow_train.txt | 1 + .../ImageSets/Main/diningtable_train.txt | 1 + .../voc_dataset/ImageSets/Main/dog_train.txt | 1 + .../ImageSets/Main/horse_train.txt | 1 + .../ImageSets/Main/ignored_train.txt | 1 + .../ImageSets/Main/motorbike_train.txt | 1 + .../ImageSets/Main/person_train.txt | 1 + .../ImageSets/Main/pottedplant_train.txt | 1 + .../ImageSets/Main/sheep_train.txt | 1 + .../voc_dataset/ImageSets/Main/sofa_train.txt | 1 + .../voc_dataset/ImageSets/Main/test.txt | 1 + .../voc_dataset/ImageSets/Main/train.txt | 1 + .../ImageSets/Main/train_train.txt | 1 + .../ImageSets/Main/tvmonitor_train.txt | 1 + .../ImageSets/Segmentation/test.txt | 1 + .../ImageSets/Segmentation/train.txt | 1 + .../voc_dataset/JPEGImages/2007_000002.jpg | Bin 0 -> 635 bytes .../SegmentationClass/2007_000001.png | Bin 0 -> 87 bytes .../SegmentationObject/2007_000001.png | Bin 0 -> 82 bytes datumaro/tests/assets/yolo_dataset/obj.data | 4 + datumaro/tests/assets/yolo_dataset/obj.names | 10 ++ .../assets/yolo_dataset/obj_train_data/1.jpg | Bin 0 -> 631 bytes .../assets/yolo_dataset/obj_train_data/1.txt | 2 + datumaro/tests/assets/yolo_dataset/train.txt | 1 + 55 files changed, 370 insertions(+), 1 deletion(-) create mode 100644 datumaro/tests/assets/coco_dataset/annotations/instances_val.json create mode 100644 datumaro/tests/assets/coco_dataset/images/val/000000000001.jpg create mode 100644 datumaro/tests/assets/cvat_dataset/for_images/images/img0.jpg create mode 100644 datumaro/tests/assets/cvat_dataset/for_images/images/img1.jpg create mode 100644 datumaro/tests/assets/cvat_dataset/for_images/train.xml create mode 100644 datumaro/tests/assets/cvat_dataset/for_video/annotations.xml create mode 100644 datumaro/tests/assets/cvat_dataset/for_video/images/frame_000010.png create mode 100644 datumaro/tests/assets/cvat_dataset/for_video/images/frame_000013.png create mode 100644 datumaro/tests/assets/mot_dataset/gt/gt.txt create mode 100644 datumaro/tests/assets/mot_dataset/gt/labels.txt create mode 100644 datumaro/tests/assets/mot_dataset/img1/000001.jpg create mode 100644 datumaro/tests/assets/tf_detection_api_dataset/label_map.pbtxt create mode 100644 datumaro/tests/assets/tf_detection_api_dataset/test.tfrecord create mode 100644 datumaro/tests/assets/tf_detection_api_dataset/train.tfrecord create mode 100644 datumaro/tests/assets/tf_detection_api_dataset/val.tfrecord create mode 100644 datumaro/tests/assets/voc_dataset/Annotations/2007_000001.xml create mode 100644 datumaro/tests/assets/voc_dataset/ImageSets/Action/test.txt create mode 100644 datumaro/tests/assets/voc_dataset/ImageSets/Action/train.txt create mode 100644 datumaro/tests/assets/voc_dataset/ImageSets/Layout/test.txt create mode 100644 datumaro/tests/assets/voc_dataset/ImageSets/Layout/train.txt create mode 100644 datumaro/tests/assets/voc_dataset/ImageSets/Main/aeroplane_train.txt create mode 100644 datumaro/tests/assets/voc_dataset/ImageSets/Main/background_train.txt create mode 100644 datumaro/tests/assets/voc_dataset/ImageSets/Main/bicycle_train.txt create mode 100644 datumaro/tests/assets/voc_dataset/ImageSets/Main/bird_train.txt create mode 100644 datumaro/tests/assets/voc_dataset/ImageSets/Main/boat_train.txt create mode 100644 datumaro/tests/assets/voc_dataset/ImageSets/Main/bottle_train.txt create mode 100644 datumaro/tests/assets/voc_dataset/ImageSets/Main/bus_train.txt create mode 100644 datumaro/tests/assets/voc_dataset/ImageSets/Main/car_train.txt create mode 100644 datumaro/tests/assets/voc_dataset/ImageSets/Main/cat_train.txt create mode 100644 datumaro/tests/assets/voc_dataset/ImageSets/Main/chair_train.txt create mode 100644 datumaro/tests/assets/voc_dataset/ImageSets/Main/cow_train.txt create mode 100644 datumaro/tests/assets/voc_dataset/ImageSets/Main/diningtable_train.txt create mode 100644 datumaro/tests/assets/voc_dataset/ImageSets/Main/dog_train.txt create mode 100644 datumaro/tests/assets/voc_dataset/ImageSets/Main/horse_train.txt create mode 100644 datumaro/tests/assets/voc_dataset/ImageSets/Main/ignored_train.txt create mode 100644 datumaro/tests/assets/voc_dataset/ImageSets/Main/motorbike_train.txt create mode 100644 datumaro/tests/assets/voc_dataset/ImageSets/Main/person_train.txt create mode 100644 datumaro/tests/assets/voc_dataset/ImageSets/Main/pottedplant_train.txt create mode 100644 datumaro/tests/assets/voc_dataset/ImageSets/Main/sheep_train.txt create mode 100644 datumaro/tests/assets/voc_dataset/ImageSets/Main/sofa_train.txt create mode 100644 datumaro/tests/assets/voc_dataset/ImageSets/Main/test.txt create mode 100644 datumaro/tests/assets/voc_dataset/ImageSets/Main/train.txt create mode 100644 datumaro/tests/assets/voc_dataset/ImageSets/Main/train_train.txt create mode 100644 datumaro/tests/assets/voc_dataset/ImageSets/Main/tvmonitor_train.txt create mode 100644 datumaro/tests/assets/voc_dataset/ImageSets/Segmentation/test.txt create mode 100644 datumaro/tests/assets/voc_dataset/ImageSets/Segmentation/train.txt create mode 100644 datumaro/tests/assets/voc_dataset/JPEGImages/2007_000002.jpg create mode 100644 datumaro/tests/assets/voc_dataset/SegmentationClass/2007_000001.png create mode 100644 datumaro/tests/assets/voc_dataset/SegmentationObject/2007_000001.png create mode 100644 datumaro/tests/assets/yolo_dataset/obj.data create mode 100644 datumaro/tests/assets/yolo_dataset/obj.names create mode 100644 datumaro/tests/assets/yolo_dataset/obj_train_data/1.jpg create mode 100644 datumaro/tests/assets/yolo_dataset/obj_train_data/1.txt create mode 100644 datumaro/tests/assets/yolo_dataset/train.txt diff --git a/datumaro/README.md b/datumaro/README.md index f7f1c46ec26..5139976db05 100644 --- a/datumaro/README.md +++ b/datumaro/README.md @@ -32,15 +32,26 @@ VOC-like dataset -- ---> Publication etc. - Dataset format conversions: - COCO (`image_info`, `instances`, `person_keypoints`, `captions`, `labels`*) - [Format specification](http://cocodataset.org/#format-data) + - [Dataset example](tests/assets/coco_dataset) - `labels` are our extension - like `instances` with only `category_id` - PASCAL VOC (`classification`, `detection`, `segmentation` (class, instances), `action_classification`, `person_layout`) - [Format specification](http://host.robots.ox.ac.uk/pascal/VOC/voc2012/htmldoc/index.html) + - [Dataset example](tests/assets/voc_dataset) - YOLO (`bboxes`) - [Format specification](https://github.com/AlexeyAB/darknet#how-to-train-pascal-voc-data) + - [Dataset example](tests/assets/yolo_dataset) - TF Detection API (`bboxes`, `masks`) - Format specifications: [bboxes](https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/using_your_own_dataset.md), [masks](https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/instance_segmentation.md) + - [Dataset example](tests/assets/tf_detection_api_dataset) + - MOT sequences + - [Format specification](https://arxiv.org/pdf/1906.04567.pdf) + - [Dataset example](tests/assets/mot_dataset) - CVAT - [Format specification](https://github.com/opencv/cvat/blob/develop/cvat/apps/documentation/xml_format.md) + - [Dataset example](tests/assets/cvat_dataset) + - LabelMe + - [Format specification](http://labelme.csail.mit.edu/Release3.0) + - [Dataset example](tests/assets/labelme_dataset) - Dataset building operations: - Merging multiple datasets into one - Dataset filtering with custom conditions, for instance: @@ -52,7 +63,7 @@ VOC-like dataset -- ---> Publication etc. - Annotation conversions, for instance - polygons to instance masks and vise-versa - apply a custom colormap for mask annotations - - remap dataset labels + - rename or remove dataset labels - Dataset comparison - Model integration: - Inference (OpenVINO and custom models) diff --git a/datumaro/tests/assets/coco_dataset/annotations/instances_val.json b/datumaro/tests/assets/coco_dataset/annotations/instances_val.json new file mode 100644 index 00000000000..a7f566ccd7e --- /dev/null +++ b/datumaro/tests/assets/coco_dataset/annotations/instances_val.json @@ -0,0 +1,59 @@ +{ + "licenses": [ + { + "name": "", + "id": 0, + "url": "" + } + ], + "info": { + "contributor": "", + "date_created": "", + "description": "", + "url": "", + "version": "", + "year": "" + }, + "categories": [ + { + "id": 1, + "name": "TEST", + "supercategory": "" + } + ], + "images": [ + { + "id": 1, + "width": 5, + "height": 10, + "file_name": "000000000001.jpg", + "license": 0, + "flickr_url": "", + "coco_url": "", + "date_captured": 0 + } + ], + "annotations": [ + { + "id": 1, + "image_id": 1, + "category_id": 1, + "segmentation": [[0, 0, 1, 0, 1, 2, 0, 2]], + "area": 2, + "bbox": [0, 0, 1, 2], + "iscrowd": 0 + }, + { + "id": 2, + "image_id": 1, + "category_id": 1, + "segmentation": { + "counts": [0, 10, 5, 5, 5, 5, 0, 10, 10, 0], + "size": [10, 5] + }, + "area": 30, + "bbox": [0, 0, 10, 4], + "iscrowd": 1 + } + ] + } \ No newline at end of file diff --git a/datumaro/tests/assets/coco_dataset/images/val/000000000001.jpg b/datumaro/tests/assets/coco_dataset/images/val/000000000001.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8bce84d3bf50bd756621338e0da944a42428fb06 GIT binary patch literal 631 zcmex=^(PF6}rMnOeST|r4lSw=>~TvNxu(8R<c1}I=;VrF4wW9Q)H;sz?% zD!{d!pzFb!U9xX3zTPI5o8roG<0MW4oqZMDikqloVbuf*=gfJ(V&YTRE(2~ znmD<{#3dx9RMpfqG__1j&CD$#!v`*nMGf}^(PF6}rMnOeST|r4lSw=>~TvNxu(8R<gTWM0TY@5u?V53ptdXHXalWy7)oGIH{I3zSIJR&kGIVCkMJtH%#xTLhKyrQzI zxuvzOy`!^h(&Q;qr%j(RbJn88OO`HMzGCI7O`ErD-L`$l&RvHNA31vL_=%IJE?vHI z_1g6tH*YuS~;l_iU%Emz-M3agxa*3&!JXHM%@*3D@ k#CfcVET6$WhVa)d1|DWcVB|3iGT1YG;L=#sVE_Ln0Q-o|ng9R* literal 0 HcmV?d00001 diff --git a/datumaro/tests/assets/cvat_dataset/for_images/images/img1.jpg b/datumaro/tests/assets/cvat_dataset/for_images/images/img1.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ee889d22692144aa0004b545eb7c097f101aa3de GIT binary patch literal 631 zcmex=^(PF6}rMnOeST|r4lSw=>~TvNxu(8R<c1}I=;VrF4wW9Q)H;sz?% zD!{d!pzFb!U9xX3zTPI5o8roG<0MW4oqZMDikqloVbuf*=gfJ(V&YTRE(2~ znmD<{#3dx9RMpfqG__1j&CD$#!v`*nMGf} + 1.1 + + + True + annotation + + + + + + + + + true + v3 + + + + + + + + diff --git a/datumaro/tests/assets/cvat_dataset/for_video/annotations.xml b/datumaro/tests/assets/cvat_dataset/for_video/annotations.xml new file mode 100644 index 00000000000..5a68f811a2e --- /dev/null +++ b/datumaro/tests/assets/cvat_dataset/for_video/annotations.xml @@ -0,0 +1,92 @@ + + + 1.1 + + + 5 + v1 + 4 + interpolation + 2 + + 2020-04-23 08:57:24.614217+00:00 + 2020-04-23 09:04:48.168008+00:00 + 10 + 19 + step=3 + True + + + + + + + + 3 + 0 + 3 + http://localhost:7000/?id=3 + + + 4 + 2 + 3 + http://localhost:7000/?id=4 + + + + max + + + + + 25 + 20 + + + 2020-04-23 09:05:02.335612+00:00 + t.mp4 + + + + + + + + + + + + hgkf + + + jk + + + + + + + + + diff --git a/datumaro/tests/assets/cvat_dataset/for_video/images/frame_000010.png b/datumaro/tests/assets/cvat_dataset/for_video/images/frame_000010.png new file mode 100644 index 0000000000000000000000000000000000000000..14996e0c4f1204c754fcc5ab6ffc99f4e9e43f52 GIT binary patch literal 111 zcmeAS@N?(olHy`uVBq!ia0vp^l0YoN!2~3~i!#0fQk(@Ik;M!Q+`=Ht$S`Y;1W-`X z)5S5w;&k$#|Nrfor!ojLe?71wa&cPkEYqxOD^(PF6}rMnOeST|r4lSw=>~TvNxu(8R<c1}I=;VrF4wW9Q)H;sz?% zD!{d!pzFb!U9xX3zTPI5o8roG<0MW4oqZMDikqloVbuf*=gfJ(V&YTRE(2~ znmD<{#3dx9RMpfqG__1j&CD$#!v`*nMGf}>10y2?10xVJ{J+iM z%)r6M#?Hpd!OqUk$;rXRBf`tW&CMezB+M@&D_WTR(nVgxdTk&}~?hnq)&msi3_QAW{-Wbpq0gCGY3D+3ENqY?v?AS1IN z|EI`>>pgaSMAghp~ zp(C4cU?RIxp@>oA#DyHnP8$!323`E1Vw_ae#K|QlE+HwUs-~`?sbyknW^Q3= z=I-I?6&w;879J59m7J2AmY$KBRa{b9R$ftA)!fqB*51+CHEHscsne#9glAUcUPH>GPMb z-@gC&`3vMPMh0exw}2$XXK4Ns1p14Kg@u`g9po=YrgD(S1zA`X4cUYo1KAS`g_VpN zIYgW$F5GyKQ`tD^gJ@FGMJ_QFlZUDwL0$vxYE#}NLy#lXYN2#h>tK?Zw< z4_X?F8tnhyIGdQ@kxWqDZ6Vp@m^OLetlS}lIlJYC`D{?dQgam~6xOgCn!K59R zG#4L4ZCYkdYF=V)st_j^I~S{=URFUmE`t&)U7(t zzqBYhH9j*%2M)G;tLF)@)>x3sk|ve7d(F#;LF$jQmc!_6bX%PV1|D5GdZGWdUhL6Cz%fI)zn zQHg;`kdaxC@&6G9d7wL48NmRSCK#ERSymaka3YSZQ|TeofBv2)j3M&~ka)>xhT)6Qdr?PR-2hpUWi(FzVCJ$9Vg1iRy8F3zKBFkrRk0JbZi-Cuk k5g2*Qf(-TyAGkCYHQ4{Zi7Q3yiD#gsj$ibz?B@PkJPs&P7F40fU zNh~hbFG;N^5#r|JBV$krWc<7m<~cmy?x}kx@|5Q&&*Z zQI?U>FxS*EFf=hSkyp30wJ@^LGd3{-8N$fP$;rdbBf-lnVWcRdXhbsje}F-dg8>Mc z8I>5A1R0qH8UG()kO#Vxl@SaeVGQImF|)9;v2$>8aRU`>6<}auWM*b!VP<7z0fqo$ zEl{3;MUYiU(a@1iI53f2sZhkIapFP_Wv7h?MT0JWP%%y_YU1P)6PJ*bQdLve(9|+9 zH8Z!cv~qTFb#wRd^a>6M4GWKmj7m;PO-s+n%qlJ^Ei136tZHs)ZENr7?3y%r%G7Do zXUv?nXz`Mz%a*TLxoXqqEnBy3-?4Mop~FXx9y@;G#!v`*nMGf} + + VOC2007 + 2007_000001.jpg + + 10 + 20 + 3 + + 1 + + cat + Unspecified + 1 + 0 + + 1 + 2 + 3 + 4 + + + + person + + 4 + 5 + 6 + 7 + + + head + + 5.5 + 6 + 7.5 + 8 + + + + 1 + 0 + 1 + 0 + 1 + 0 + 1 + 0 + 1 + 0 + 1 + + + \ No newline at end of file diff --git a/datumaro/tests/assets/voc_dataset/ImageSets/Action/test.txt b/datumaro/tests/assets/voc_dataset/ImageSets/Action/test.txt new file mode 100644 index 00000000000..c9fdc2510e1 --- /dev/null +++ b/datumaro/tests/assets/voc_dataset/ImageSets/Action/test.txt @@ -0,0 +1 @@ +2007_000002 diff --git a/datumaro/tests/assets/voc_dataset/ImageSets/Action/train.txt b/datumaro/tests/assets/voc_dataset/ImageSets/Action/train.txt new file mode 100644 index 00000000000..640b0d53ff2 --- /dev/null +++ b/datumaro/tests/assets/voc_dataset/ImageSets/Action/train.txt @@ -0,0 +1 @@ +2007_000001 diff --git a/datumaro/tests/assets/voc_dataset/ImageSets/Layout/test.txt b/datumaro/tests/assets/voc_dataset/ImageSets/Layout/test.txt new file mode 100644 index 00000000000..c9fdc2510e1 --- /dev/null +++ b/datumaro/tests/assets/voc_dataset/ImageSets/Layout/test.txt @@ -0,0 +1 @@ +2007_000002 diff --git a/datumaro/tests/assets/voc_dataset/ImageSets/Layout/train.txt b/datumaro/tests/assets/voc_dataset/ImageSets/Layout/train.txt new file mode 100644 index 00000000000..640b0d53ff2 --- /dev/null +++ b/datumaro/tests/assets/voc_dataset/ImageSets/Layout/train.txt @@ -0,0 +1 @@ +2007_000001 diff --git a/datumaro/tests/assets/voc_dataset/ImageSets/Main/aeroplane_train.txt b/datumaro/tests/assets/voc_dataset/ImageSets/Main/aeroplane_train.txt new file mode 100644 index 00000000000..a3decd42ad8 --- /dev/null +++ b/datumaro/tests/assets/voc_dataset/ImageSets/Main/aeroplane_train.txt @@ -0,0 +1 @@ +2007_000001 1 diff --git a/datumaro/tests/assets/voc_dataset/ImageSets/Main/background_train.txt b/datumaro/tests/assets/voc_dataset/ImageSets/Main/background_train.txt new file mode 100644 index 00000000000..d4385b69787 --- /dev/null +++ b/datumaro/tests/assets/voc_dataset/ImageSets/Main/background_train.txt @@ -0,0 +1 @@ +2007_000001 -1 diff --git a/datumaro/tests/assets/voc_dataset/ImageSets/Main/bicycle_train.txt b/datumaro/tests/assets/voc_dataset/ImageSets/Main/bicycle_train.txt new file mode 100644 index 00000000000..d4385b69787 --- /dev/null +++ b/datumaro/tests/assets/voc_dataset/ImageSets/Main/bicycle_train.txt @@ -0,0 +1 @@ +2007_000001 -1 diff --git a/datumaro/tests/assets/voc_dataset/ImageSets/Main/bird_train.txt b/datumaro/tests/assets/voc_dataset/ImageSets/Main/bird_train.txt new file mode 100644 index 00000000000..a3decd42ad8 --- /dev/null +++ b/datumaro/tests/assets/voc_dataset/ImageSets/Main/bird_train.txt @@ -0,0 +1 @@ +2007_000001 1 diff --git a/datumaro/tests/assets/voc_dataset/ImageSets/Main/boat_train.txt b/datumaro/tests/assets/voc_dataset/ImageSets/Main/boat_train.txt new file mode 100644 index 00000000000..d4385b69787 --- /dev/null +++ b/datumaro/tests/assets/voc_dataset/ImageSets/Main/boat_train.txt @@ -0,0 +1 @@ +2007_000001 -1 diff --git a/datumaro/tests/assets/voc_dataset/ImageSets/Main/bottle_train.txt b/datumaro/tests/assets/voc_dataset/ImageSets/Main/bottle_train.txt new file mode 100644 index 00000000000..a3decd42ad8 --- /dev/null +++ b/datumaro/tests/assets/voc_dataset/ImageSets/Main/bottle_train.txt @@ -0,0 +1 @@ +2007_000001 1 diff --git a/datumaro/tests/assets/voc_dataset/ImageSets/Main/bus_train.txt b/datumaro/tests/assets/voc_dataset/ImageSets/Main/bus_train.txt new file mode 100644 index 00000000000..d4385b69787 --- /dev/null +++ b/datumaro/tests/assets/voc_dataset/ImageSets/Main/bus_train.txt @@ -0,0 +1 @@ +2007_000001 -1 diff --git a/datumaro/tests/assets/voc_dataset/ImageSets/Main/car_train.txt b/datumaro/tests/assets/voc_dataset/ImageSets/Main/car_train.txt new file mode 100644 index 00000000000..a3decd42ad8 --- /dev/null +++ b/datumaro/tests/assets/voc_dataset/ImageSets/Main/car_train.txt @@ -0,0 +1 @@ +2007_000001 1 diff --git a/datumaro/tests/assets/voc_dataset/ImageSets/Main/cat_train.txt b/datumaro/tests/assets/voc_dataset/ImageSets/Main/cat_train.txt new file mode 100644 index 00000000000..d4385b69787 --- /dev/null +++ b/datumaro/tests/assets/voc_dataset/ImageSets/Main/cat_train.txt @@ -0,0 +1 @@ +2007_000001 -1 diff --git a/datumaro/tests/assets/voc_dataset/ImageSets/Main/chair_train.txt b/datumaro/tests/assets/voc_dataset/ImageSets/Main/chair_train.txt new file mode 100644 index 00000000000..a3decd42ad8 --- /dev/null +++ b/datumaro/tests/assets/voc_dataset/ImageSets/Main/chair_train.txt @@ -0,0 +1 @@ +2007_000001 1 diff --git a/datumaro/tests/assets/voc_dataset/ImageSets/Main/cow_train.txt b/datumaro/tests/assets/voc_dataset/ImageSets/Main/cow_train.txt new file mode 100644 index 00000000000..d4385b69787 --- /dev/null +++ b/datumaro/tests/assets/voc_dataset/ImageSets/Main/cow_train.txt @@ -0,0 +1 @@ +2007_000001 -1 diff --git a/datumaro/tests/assets/voc_dataset/ImageSets/Main/diningtable_train.txt b/datumaro/tests/assets/voc_dataset/ImageSets/Main/diningtable_train.txt new file mode 100644 index 00000000000..a3decd42ad8 --- /dev/null +++ b/datumaro/tests/assets/voc_dataset/ImageSets/Main/diningtable_train.txt @@ -0,0 +1 @@ +2007_000001 1 diff --git a/datumaro/tests/assets/voc_dataset/ImageSets/Main/dog_train.txt b/datumaro/tests/assets/voc_dataset/ImageSets/Main/dog_train.txt new file mode 100644 index 00000000000..d4385b69787 --- /dev/null +++ b/datumaro/tests/assets/voc_dataset/ImageSets/Main/dog_train.txt @@ -0,0 +1 @@ +2007_000001 -1 diff --git a/datumaro/tests/assets/voc_dataset/ImageSets/Main/horse_train.txt b/datumaro/tests/assets/voc_dataset/ImageSets/Main/horse_train.txt new file mode 100644 index 00000000000..a3decd42ad8 --- /dev/null +++ b/datumaro/tests/assets/voc_dataset/ImageSets/Main/horse_train.txt @@ -0,0 +1 @@ +2007_000001 1 diff --git a/datumaro/tests/assets/voc_dataset/ImageSets/Main/ignored_train.txt b/datumaro/tests/assets/voc_dataset/ImageSets/Main/ignored_train.txt new file mode 100644 index 00000000000..a3decd42ad8 --- /dev/null +++ b/datumaro/tests/assets/voc_dataset/ImageSets/Main/ignored_train.txt @@ -0,0 +1 @@ +2007_000001 1 diff --git a/datumaro/tests/assets/voc_dataset/ImageSets/Main/motorbike_train.txt b/datumaro/tests/assets/voc_dataset/ImageSets/Main/motorbike_train.txt new file mode 100644 index 00000000000..d4385b69787 --- /dev/null +++ b/datumaro/tests/assets/voc_dataset/ImageSets/Main/motorbike_train.txt @@ -0,0 +1 @@ +2007_000001 -1 diff --git a/datumaro/tests/assets/voc_dataset/ImageSets/Main/person_train.txt b/datumaro/tests/assets/voc_dataset/ImageSets/Main/person_train.txt new file mode 100644 index 00000000000..a3decd42ad8 --- /dev/null +++ b/datumaro/tests/assets/voc_dataset/ImageSets/Main/person_train.txt @@ -0,0 +1 @@ +2007_000001 1 diff --git a/datumaro/tests/assets/voc_dataset/ImageSets/Main/pottedplant_train.txt b/datumaro/tests/assets/voc_dataset/ImageSets/Main/pottedplant_train.txt new file mode 100644 index 00000000000..d4385b69787 --- /dev/null +++ b/datumaro/tests/assets/voc_dataset/ImageSets/Main/pottedplant_train.txt @@ -0,0 +1 @@ +2007_000001 -1 diff --git a/datumaro/tests/assets/voc_dataset/ImageSets/Main/sheep_train.txt b/datumaro/tests/assets/voc_dataset/ImageSets/Main/sheep_train.txt new file mode 100644 index 00000000000..a3decd42ad8 --- /dev/null +++ b/datumaro/tests/assets/voc_dataset/ImageSets/Main/sheep_train.txt @@ -0,0 +1 @@ +2007_000001 1 diff --git a/datumaro/tests/assets/voc_dataset/ImageSets/Main/sofa_train.txt b/datumaro/tests/assets/voc_dataset/ImageSets/Main/sofa_train.txt new file mode 100644 index 00000000000..d4385b69787 --- /dev/null +++ b/datumaro/tests/assets/voc_dataset/ImageSets/Main/sofa_train.txt @@ -0,0 +1 @@ +2007_000001 -1 diff --git a/datumaro/tests/assets/voc_dataset/ImageSets/Main/test.txt b/datumaro/tests/assets/voc_dataset/ImageSets/Main/test.txt new file mode 100644 index 00000000000..c9fdc2510e1 --- /dev/null +++ b/datumaro/tests/assets/voc_dataset/ImageSets/Main/test.txt @@ -0,0 +1 @@ +2007_000002 diff --git a/datumaro/tests/assets/voc_dataset/ImageSets/Main/train.txt b/datumaro/tests/assets/voc_dataset/ImageSets/Main/train.txt new file mode 100644 index 00000000000..640b0d53ff2 --- /dev/null +++ b/datumaro/tests/assets/voc_dataset/ImageSets/Main/train.txt @@ -0,0 +1 @@ +2007_000001 diff --git a/datumaro/tests/assets/voc_dataset/ImageSets/Main/train_train.txt b/datumaro/tests/assets/voc_dataset/ImageSets/Main/train_train.txt new file mode 100644 index 00000000000..a3decd42ad8 --- /dev/null +++ b/datumaro/tests/assets/voc_dataset/ImageSets/Main/train_train.txt @@ -0,0 +1 @@ +2007_000001 1 diff --git a/datumaro/tests/assets/voc_dataset/ImageSets/Main/tvmonitor_train.txt b/datumaro/tests/assets/voc_dataset/ImageSets/Main/tvmonitor_train.txt new file mode 100644 index 00000000000..d4385b69787 --- /dev/null +++ b/datumaro/tests/assets/voc_dataset/ImageSets/Main/tvmonitor_train.txt @@ -0,0 +1 @@ +2007_000001 -1 diff --git a/datumaro/tests/assets/voc_dataset/ImageSets/Segmentation/test.txt b/datumaro/tests/assets/voc_dataset/ImageSets/Segmentation/test.txt new file mode 100644 index 00000000000..c9fdc2510e1 --- /dev/null +++ b/datumaro/tests/assets/voc_dataset/ImageSets/Segmentation/test.txt @@ -0,0 +1 @@ +2007_000002 diff --git a/datumaro/tests/assets/voc_dataset/ImageSets/Segmentation/train.txt b/datumaro/tests/assets/voc_dataset/ImageSets/Segmentation/train.txt new file mode 100644 index 00000000000..640b0d53ff2 --- /dev/null +++ b/datumaro/tests/assets/voc_dataset/ImageSets/Segmentation/train.txt @@ -0,0 +1 @@ +2007_000001 diff --git a/datumaro/tests/assets/voc_dataset/JPEGImages/2007_000002.jpg b/datumaro/tests/assets/voc_dataset/JPEGImages/2007_000002.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3c81296b31dcb791dacede93e4be1f2d08b98347 GIT binary patch literal 635 zcmex=^(PF6}rMnOeST|r4lSw=>~TvNxu(8R<W=16jCP7AKLB{__803NOWMu>c1}I=;VrF4wW9Q)H;sz?% zD!{d!pzFb!U9xX3zTPI5o8roG<0MW4oqZMDikqloVbuf*=gfJ(V&YTRE(2~ znmD<{#3dx9RMpfqG__1j&CD$#!v`*nMGYWq|NkZcN!rf* literal 0 HcmV?d00001 diff --git a/datumaro/tests/assets/voc_dataset/SegmentationClass/2007_000001.png b/datumaro/tests/assets/voc_dataset/SegmentationClass/2007_000001.png new file mode 100644 index 0000000000000000000000000000000000000000..0b92051452392a7827f09a27d7b5a179f93b3748 GIT binary patch literal 87 zcmeAS@N?(olHy`uVBq!ia0vp^AT}!p6OjDO_R1be$$7dshH%K%9%N($@|qNC?|qTu iFly>iP(Hz7bcpfk8>Tj~#tAclatxlXelF{r5}E++^A#un literal 0 HcmV?d00001 diff --git a/datumaro/tests/assets/voc_dataset/SegmentationObject/2007_000001.png b/datumaro/tests/assets/voc_dataset/SegmentationObject/2007_000001.png new file mode 100644 index 0000000000000000000000000000000000000000..ebbeee61dd687d6cb27a7bfcef77b503095d10d2 GIT binary patch literal 82 zcmeAS@N?(olHy`uVBq!ia0vp^AT}!pkYI@9SK0!kBt2amLpWqj4;nH8d508U#^1_5 ba0m#B7;6}~8DEFI0x9-%^>bP0l+XkKf*}+L literal 0 HcmV?d00001 diff --git a/datumaro/tests/assets/yolo_dataset/obj.data b/datumaro/tests/assets/yolo_dataset/obj.data new file mode 100644 index 00000000000..16ca4090f42 --- /dev/null +++ b/datumaro/tests/assets/yolo_dataset/obj.data @@ -0,0 +1,4 @@ +classes = 10 +train = data/train.txt +names = data/obj.names +backup = backup/ diff --git a/datumaro/tests/assets/yolo_dataset/obj.names b/datumaro/tests/assets/yolo_dataset/obj.names new file mode 100644 index 00000000000..b24c644df62 --- /dev/null +++ b/datumaro/tests/assets/yolo_dataset/obj.names @@ -0,0 +1,10 @@ +label_0 +label_1 +label_2 +label_3 +label_4 +label_5 +label_6 +label_7 +label_8 +label_9 diff --git a/datumaro/tests/assets/yolo_dataset/obj_train_data/1.jpg b/datumaro/tests/assets/yolo_dataset/obj_train_data/1.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8689b956311969f2efc9e3334f375c0ad65e24f1 GIT binary patch literal 631 zcmex=^(PF6}rMnOeST|r4lSw=>~TvNxu(8R<c1}I=;VrF4wW9Q)H;sz?% zD!{d!pzFb!U9xX3zTPI5o8roG<0MW4oqZMDikqloVbuf*=gfJ(V&YTRE(2~ znmD<{#3dx9RMpfqG__1j&CD$#!v`*nMGf} Date: Thu, 4 Jun 2020 19:24:23 +0300 Subject: [PATCH 3/4] Context menu options for cuboids (#1554) --- cvat-ui/src/assets/reset-perspective.svg | 4 ++ .../objects-side-bar/object-item.tsx | 39 +++++++++++++- .../objects-side-bar/object-item.tsx | 51 +++++++++++++++++++ cvat-ui/src/icons.tsx | 4 ++ cvat-ui/src/utils/math.ts | 8 +++ 5 files changed, 104 insertions(+), 2 deletions(-) create mode 100644 cvat-ui/src/assets/reset-perspective.svg diff --git a/cvat-ui/src/assets/reset-perspective.svg b/cvat-ui/src/assets/reset-perspective.svg new file mode 100644 index 00000000000..2c7c0b83550 --- /dev/null +++ b/cvat-ui/src/assets/reset-perspective.svg @@ -0,0 +1,4 @@ + + + + diff --git a/cvat-ui/src/components/annotation-page/standard-workspace/objects-side-bar/object-item.tsx b/cvat-ui/src/components/annotation-page/standard-workspace/objects-side-bar/object-item.tsx index 1a5b54cbc38..88a4b83f639 100644 --- a/cvat-ui/src/components/annotation-page/standard-workspace/objects-side-bar/object-item.tsx +++ b/cvat-ui/src/components/annotation-page/standard-workspace/objects-side-bar/object-item.tsx @@ -29,6 +29,7 @@ import { NextIcon, BackgroundIcon, ForegroundIcon, + ResetPerspectiveIcon, } from 'icons'; import { ObjectType, ShapeType } from 'reducers/interfaces'; import { clamp } from 'utils/math'; @@ -38,6 +39,7 @@ function ItemMenu( serverID: number | undefined, locked: boolean, objectType: ObjectType, + shapeType: ShapeType, copyShortcut: string, pasteShortcut: string, propagateShortcut: string, @@ -50,6 +52,8 @@ function ItemMenu( createURL: (() => void), toBackground: (() => void), toForeground: (() => void), + switchCuboidOrientation: (() => void), + resetCuboidPerspective: (() => void), ): JSX.Element { return ( @@ -72,7 +76,22 @@ function ItemMenu( - { objectType !== ObjectType.TAG && ( + {shapeType === ShapeType.CUBOID && ( + + + + )} + {shapeType === ShapeType.CUBOID && ( + + + + )} + {objectType !== ObjectType.TAG && (