From 83f2baaab1efeed0b70881bee3aef2d0a004613b Mon Sep 17 00:00:00 2001 From: Kirill Lakhov Date: Fri, 5 Apr 2024 09:01:57 +0300 Subject: [PATCH 1/9] added support for .rar archives --- Dockerfile | 1 + cvat/apps/engine/media.mimetypes | 2 +- cvat/apps/engine/media_extractors.py | 5 ++++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7bed5fac9e81..7c06096dac6d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -110,6 +110,7 @@ ENV DJANGO_SETTINGS_MODULE="cvat.settings.${CVAT_CONFIGURATION}" RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get --no-install-recommends install -yq \ bzip2 \ + unrar \ ca-certificates \ curl \ git \ diff --git a/cvat/apps/engine/media.mimetypes b/cvat/apps/engine/media.mimetypes index 43cd80839b80..0bc8c1040f11 100644 --- a/cvat/apps/engine/media.mimetypes +++ b/cvat/apps/engine/media.mimetypes @@ -200,7 +200,7 @@ image/x-quicktime qif # possible archive mimetypes (limited set) application/gzip gz -application/rar rar +application/x-rar-compressed rar application/x-7z-compressed 7z application/x-bzip bz bz2 application/x-bzip-compressed-tar tar.bz tar.bz2 tb2 tbz tbz2 diff --git a/cvat/apps/engine/media_extractors.py b/cvat/apps/engine/media_extractors.py index 13c515c44c44..99ed6916c8e0 100644 --- a/cvat/apps/engine/media_extractors.py +++ b/cvat/apps/engine/media_extractors.py @@ -1,8 +1,10 @@ # Copyright (C) 2019-2022 Intel Corporation +# Copyright (C) 2024 CVAT.ai Corporation # # SPDX-License-Identifier: MIT import os +import sys import tempfile import shutil import zipfile @@ -266,7 +268,8 @@ def __init__(self, self._archive_source = source_path[0] tmp_dir = extract_dir if extract_dir else os.path.dirname(source_path[0]) - Archive(self._archive_source).extractall(tmp_dir) + patool_path = os.path.join(sys.exec_prefix, 'bin', 'patool') + Archive(self._archive_source).extractall(tmp_dir, False, patool_path) if not extract_dir: os.remove(self._archive_source) super().__init__( From dd531a1f9d8f159f0639385cc93b80e88acefbe9 Mon Sep 17 00:00:00 2001 From: Kirill Lakhov Date: Fri, 5 Apr 2024 09:20:48 +0300 Subject: [PATCH 2/9] added changelog --- changelog.d/20240405_091941_klakhov_rar_support.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changelog.d/20240405_091941_klakhov_rar_support.md diff --git a/changelog.d/20240405_091941_klakhov_rar_support.md b/changelog.d/20240405_091941_klakhov_rar_support.md new file mode 100644 index 000000000000..42a197b3ee1a --- /dev/null +++ b/changelog.d/20240405_091941_klakhov_rar_support.md @@ -0,0 +1,4 @@ +### Added + +- Support for `.rar` archives + () From c77610919cae08af8a7deacea373d0ffa05b1879 Mon Sep 17 00:00:00 2001 From: Kirill Lakhov Date: Tue, 9 Apr 2024 13:28:03 +0300 Subject: [PATCH 3/9] applied comments, added test --- Dockerfile | 2 +- cvat/apps/engine/media_extractors.py | 4 +- cvat/apps/engine/tests/test_rest_api.py | 86 +++++++++++++++++++++++++ 3 files changed, 89 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7c06096dac6d..ab52f3fbefe9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -110,7 +110,6 @@ ENV DJANGO_SETTINGS_MODULE="cvat.settings.${CVAT_CONFIGURATION}" RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get --no-install-recommends install -yq \ bzip2 \ - unrar \ ca-certificates \ curl \ git \ @@ -127,6 +126,7 @@ RUN apt-get update && \ python3-venv \ supervisor \ tzdata \ + unrar \ && ln -fs /usr/share/zoneinfo/${TZ} /etc/localtime && \ dpkg-reconfigure -f noninteractive tzdata && \ rm -rf /var/lib/apt/lists/* && \ diff --git a/cvat/apps/engine/media_extractors.py b/cvat/apps/engine/media_extractors.py index 6a42c9d5b0c0..1e59f172454c 100644 --- a/cvat/apps/engine/media_extractors.py +++ b/cvat/apps/engine/media_extractors.py @@ -4,7 +4,7 @@ # SPDX-License-Identifier: MIT import os -import sys +import sysconfig import tempfile import shutil import zipfile @@ -268,7 +268,7 @@ def __init__(self, self._archive_source = source_path[0] tmp_dir = extract_dir if extract_dir else os.path.dirname(source_path[0]) - patool_path = os.path.join(sys.exec_prefix, 'bin', 'patool') + patool_path = os.path.join(sysconfig.get_path('scripts'), 'patool') Archive(self._archive_source).extractall(tmp_dir, False, patool_path) if not extract_dir: os.remove(self._archive_source) diff --git a/cvat/apps/engine/tests/test_rest_api.py b/cvat/apps/engine/tests/test_rest_api.py index 17113f3851a1..e23c72f06271 100644 --- a/cvat/apps/engine/tests/test_rest_api.py +++ b/cvat/apps/engine/tests/test_rest_api.py @@ -9,7 +9,9 @@ import os import random import shutil +import sysconfig import tempfile +from uuid import uuid4 import xml.etree.ElementTree as ET import zipfile from collections import defaultdict @@ -25,6 +27,7 @@ import av import numpy as np from pdf2image import convert_from_bytes +from pyunpack import Archive from django.conf import settings from django.contrib.auth.models import Group, User from django.http import HttpResponse @@ -3126,6 +3129,7 @@ def setUpClass(cls): cls._share_image_sizes = {} cls._share_files = [] + cls._unpack_dirs = [] for filename in [ "test_1.jpg", "test_2.jpg", "test_3.jpg", "test_10.jpg", "test_qwe.jpg", @@ -3186,6 +3190,18 @@ def setUpClass(cls): image_sizes.append((int(data["WIDTH"]), int(data["HEIGHT"]))) cls._share_image_sizes[filename] = image_sizes + filename = "test_rar.rar" + source_path = os.path.join(os.path.dirname(__file__), 'assets', filename) + path = os.path.join(settings.SHARE_ROOT, filename) + shutil.copyfile(source_path, path) + archive_dir, images = cls._extract_rar_archive(open(source_path, 'rb')) + for [f, image] in images: + width,height = image.size + image_sizes.append((width, height)) + cls._share_image_sizes[filename] = image_sizes + cls._share_files.append(filename) + cls._unpack_dirs.append(archive_dir) + filename = "test_velodyne_points.zip" path = os.path.join(os.path.dirname(__file__), 'assets', filename) image_sizes = [] @@ -3298,6 +3314,9 @@ def tearDownClass(cls): dirs.add(os.path.dirname(filename)) os.remove(os.path.join(settings.SHARE_ROOT, filename)) + for unpack_dir in cls._unpack_dirs: + shutil.rmtree(unpack_dir) + for dirname in sorted(dirs, reverse=True): path = os.path.join(settings.SHARE_ROOT, dirname) if not os.listdir(path): @@ -3364,6 +3383,25 @@ def _extract_zip_archive(archive, dimension=DimensionType.DIM_2D): for f in sorted(chunk.namelist()) ] + @staticmethod + def _extract_rar_archive(archive): + rand_name = uuid4().hex + archive_dir = os.path.join(settings.TMP_FILES_ROOT, rand_name) + os.makedirs(archive_dir) + + arch_file = os.path.join(archive_dir, f"{rand_name}.rar") + with open(arch_file, 'wb') as tmp_file: + tmp_file.write(archive.read()) + + unpack_dir = os.path.join(settings.TMP_FILES_ROOT, rand_name, "unpack") + os.makedirs(unpack_dir) + + patool_path = os.path.join(sysconfig.get_path('scripts'), 'patool') + Archive(arch_file).extractall_patool(unpack_dir, patool_path) + return archive_dir, [(image, Image.open(os.path.join(unpack_dir, image))) + for image in os.listdir(unpack_dir) + ] + @classmethod def _extract_zip_chunk(cls, chunk_buffer, dimension=DimensionType.DIM_2D): return [f[1] for f in cls._extract_zip_archive(chunk_buffer, dimension=dimension)] @@ -3412,6 +3450,8 @@ def _test_api_v2_tasks_id_data_spec(self, user, spec, data, state = response.data['state'] sleep(0.1) max_number_of_attempt -= 1 + # if data["server_files[0]"] == 'test_rar.rar': + # print(response.content) self.assertEqual(state, expected_task_creation_status_state) if expected_task_creation_status_state == 'Failed': self.assertIn(expected_task_creation_status_reason, response.data['message']) @@ -3519,6 +3559,11 @@ def _test_api_v2_tasks_id_data_spec(self, user, spec, data, if zipfile.is_zipfile(f): for frame_name, frame in self._extract_zip_archive(f, dimension=dimension): source_images[frame_name] = frame + elif isinstance(f, str) and f.endswith('.rar'): + archive_dir, images = self._extract_rar_archive(f) + self._unpack_dirs.append(archive_dir) + for frame_name, frame in images: + source_images[frame_name] = frame elif isinstance(f, str) and f.endswith('.pdf'): with open(f, 'rb') as pdf_file: for i, frame in enumerate(convert_from_bytes(pdf_file.read(), fmt='png')): @@ -4562,6 +4607,47 @@ def _send_data_and_fail(*args, **kwargs): image_sizes, StorageMethodChoice.FILE_SYSTEM, StorageChoice.LOCAL, send_data_callback=_send_data_and_fail) + def _test_api_v2_tasks_id_data_create_can_use_server_rar(self, user): + task_spec = { + "name": 'task rar in the shared folder #32', + "overlap": 0, + "segment_size": 0, + "labels": [ + {"name": "car"}, + {"name": "person"}, + ] + } + + task_data = { + "server_files[0]": "test_rar.rar", + "image_quality": 75, + "copy_data": False, + "use_cache": True, + } + image_sizes = self._share_image_sizes[task_data["server_files[0]"]] + + self._test_api_v2_tasks_id_data_spec(user, task_spec, task_data, self.ChunkType.IMAGESET, self.ChunkType.IMAGESET, + image_sizes, StorageMethodChoice.CACHE, StorageChoice.LOCAL) + + def _test_api_v2_tasks_id_data_create_can_use_local_rar(self, user): + task_spec = { + "name": 'task rar in the shared folder #33', + "overlap": 0, + "segment_size": 0, + "labels": [ + {"name": "car"}, + {"name": "person"}, + ] + } + + task_data = { + "client_files[0]": open(os.path.join(os.path.dirname(__file__), 'assets', 'test_rar.rar'), 'rb'), + "image_quality": 75, + } + image_sizes = self._share_image_sizes["test_rar.rar"] + self._test_api_v2_tasks_id_data_spec(user, task_spec, task_data, + self.ChunkType.IMAGESET, self.ChunkType.IMAGESET, image_sizes) + def _test_api_v2_tasks_id_data_create(self, user): method_list = { func: getattr(self, func) for func in dir(self) From af1cf9cb338a806e7a1694931d3497fb937a68c6 Mon Sep 17 00:00:00 2001 From: Kirill Lakhov Date: Tue, 9 Apr 2024 14:24:55 +0300 Subject: [PATCH 4/9] fixed typos, edited mimetypes --- .../20240405_091941_klakhov_rar_support.md | 2 +- cvat/apps/engine/media_extractors.py | 2 +- cvat/apps/engine/tests/assets/test_rar.rar | Bin 0 -> 4511 bytes cvat/apps/engine/tests/test_rest_api.py | 4 +--- 4 files changed, 3 insertions(+), 5 deletions(-) create mode 100644 cvat/apps/engine/tests/assets/test_rar.rar diff --git a/changelog.d/20240405_091941_klakhov_rar_support.md b/changelog.d/20240405_091941_klakhov_rar_support.md index 42a197b3ee1a..3ed735da6662 100644 --- a/changelog.d/20240405_091941_klakhov_rar_support.md +++ b/changelog.d/20240405_091941_klakhov_rar_support.md @@ -1,4 +1,4 @@ ### Added -- Support for `.rar` archives +- Support for `.rar`, `.tar`, `.gz`, `.bz2`, `.cpio`, `.7z`, archives () diff --git a/cvat/apps/engine/media_extractors.py b/cvat/apps/engine/media_extractors.py index 1e59f172454c..d6d306121527 100644 --- a/cvat/apps/engine/media_extractors.py +++ b/cvat/apps/engine/media_extractors.py @@ -848,7 +848,7 @@ def _is_archive(path): encoding = mime[1] supportedArchives = ['application/x-rar-compressed', 'application/x-tar', 'application/x-7z-compressed', 'application/x-cpio', - 'gzip', 'bzip2'] + 'application/gzip', 'application/x-bzip'] return mime_type in supportedArchives or encoding in supportedArchives def _is_video(path): diff --git a/cvat/apps/engine/tests/assets/test_rar.rar b/cvat/apps/engine/tests/assets/test_rar.rar new file mode 100644 index 0000000000000000000000000000000000000000..ef82c0441190d08af75fbb692770f34f596cff44 GIT binary patch literal 4511 zcmeIy`y&&G0|4+nY`W&{63Z|#d?z%oulbf&UgnusIrP;w?dG*FS!sC` z@jW?~(hSXGj*8A$l#&iJCM}PpPO|j<^?teE?}vTJ`|KA% z`(LezlRfeT`g58*%_F;yB+qAN&rh)ALF>w_?~o=7W8(wAPyHcJD)tT`;vai3|_?lMi&WImz}b*VIk$Q(>#ZKIL8k6D=D4W@-b|f zf@61&eL-T<$ZugIehrz_0tY%aB! zTt|TiGIbH|*!P}bg#eY^LYqgM`Se65vW3GOM3_G${~*IFh<(Mu&z!xGZdcWH1$EU|XYX!n89JH^KXDu3Y!!pu zsK`tk>cd$C0VV&sWX-KzZz?OingzI%1}KZbuA&(AeS7lQ9dRNO)VyHQalt zie8#mFo#2I=nk&e0AwdD5Gi8Q`^c1#YXK)dlwP7{XuK_Q$~9vg29oY@BEBYZLo zb&C6z#Ku4;&-|ziFK@+z`2PKyMt2#P|ChY5ZyY5s9Yw8aoqh(PIZ9XPmIrF%{iEc( zlzoJc4>{(INwEy;<$v1CY)1|(XMY~gAQ#`;U(S_9X+&ZLFa_B8QABZ$^w9kxvp%;3 za=1%fO(*Vx?R;QY9lq42cB!JEak;-_rXhiq2&G=qY1wL(a}G?M48d6 zPY%mGTzzo&or`<~BGrZBWtm%*qD+l(`6)-_=MykZABVNCh olfi@Gf1SGHpHj~tg&U|+SEa5>U6r~j^}luMi0qSLV2B#v|2Nr^ng9R* literal 0 HcmV?d00001 diff --git a/cvat/apps/engine/tests/test_rest_api.py b/cvat/apps/engine/tests/test_rest_api.py index e23c72f06271..5da3e1bd9f94 100644 --- a/cvat/apps/engine/tests/test_rest_api.py +++ b/cvat/apps/engine/tests/test_rest_api.py @@ -3450,8 +3450,6 @@ def _test_api_v2_tasks_id_data_spec(self, user, spec, data, state = response.data['state'] sleep(0.1) max_number_of_attempt -= 1 - # if data["server_files[0]"] == 'test_rar.rar': - # print(response.content) self.assertEqual(state, expected_task_creation_status_state) if expected_task_creation_status_state == 'Failed': self.assertIn(expected_task_creation_status_reason, response.data['message']) @@ -4631,7 +4629,7 @@ def _test_api_v2_tasks_id_data_create_can_use_server_rar(self, user): def _test_api_v2_tasks_id_data_create_can_use_local_rar(self, user): task_spec = { - "name": 'task rar in the shared folder #33', + "name": 'task rar client files #33', "overlap": 0, "segment_size": 0, "labels": [ From 52dbe3f76dad2957326009c8d81895b1384f4b21 Mon Sep 17 00:00:00 2001 From: Kirill Lakhov Date: Wed, 10 Apr 2024 11:45:48 +0300 Subject: [PATCH 5/9] fixed tests --- cvat/apps/engine/tests/test_rest_api.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/cvat/apps/engine/tests/test_rest_api.py b/cvat/apps/engine/tests/test_rest_api.py index 5da3e1bd9f94..16ae01a30ddf 100644 --- a/cvat/apps/engine/tests/test_rest_api.py +++ b/cvat/apps/engine/tests/test_rest_api.py @@ -3194,6 +3194,7 @@ def setUpClass(cls): source_path = os.path.join(os.path.dirname(__file__), 'assets', filename) path = os.path.join(settings.SHARE_ROOT, filename) shutil.copyfile(source_path, path) + image_sizes = [] archive_dir, images = cls._extract_rar_archive(open(source_path, 'rb')) for [f, image] in images: width,height = image.size @@ -3390,6 +3391,7 @@ def _extract_rar_archive(archive): os.makedirs(archive_dir) arch_file = os.path.join(archive_dir, f"{rand_name}.rar") + archive.seek(0) with open(arch_file, 'wb') as tmp_file: tmp_file.write(archive.read()) @@ -3399,7 +3401,7 @@ def _extract_rar_archive(archive): patool_path = os.path.join(sysconfig.get_path('scripts'), 'patool') Archive(arch_file).extractall_patool(unpack_dir, patool_path) return archive_dir, [(image, Image.open(os.path.join(unpack_dir, image))) - for image in os.listdir(unpack_dir) + for image in os.listdir(unpack_dir) ] @classmethod @@ -3550,7 +3552,6 @@ def _test_api_v2_tasks_id_data_spec(self, user, spec, data, manifest = next((v for v in source_files if _name_key(v).endswith('.jsonl')), None) source_files = [_add_prefix(f) for f in source_files if not _name_key(f).endswith('jsonl')] - # Load images source_images = {} for f in source_files: @@ -3558,10 +3559,16 @@ def _test_api_v2_tasks_id_data_spec(self, user, spec, data, for frame_name, frame in self._extract_zip_archive(f, dimension=dimension): source_images[frame_name] = frame elif isinstance(f, str) and f.endswith('.rar'): - archive_dir, images = self._extract_rar_archive(f) + with open(f, 'rb') as rar_file: + archive_dir, arhive_frames = self._extract_rar_archive(rar_file) + self._unpack_dirs.append(archive_dir) + for fn, frame in arhive_frames: + source_images[fn] = frame + elif isinstance(f, IOBase) and getattr(f, 'name', '').endswith('.rar'): + archive_dir, arhive_frames = self._extract_rar_archive(f) self._unpack_dirs.append(archive_dir) - for frame_name, frame in images: - source_images[frame_name] = frame + for fn, frame in arhive_frames: + source_images[fn] = frame elif isinstance(f, str) and f.endswith('.pdf'): with open(f, 'rb') as pdf_file: for i, frame in enumerate(convert_from_bytes(pdf_file.read(), fmt='png')): From a6d7ac61cf72c56180e4b58cc08f2cc7a341cae7 Mon Sep 17 00:00:00 2001 From: Kirill Lakhov Date: Thu, 11 Apr 2024 10:35:55 +0300 Subject: [PATCH 6/9] fixed changelog --- changelog.d/20240405_091941_klakhov_rar_support.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/20240405_091941_klakhov_rar_support.md b/changelog.d/20240405_091941_klakhov_rar_support.md index 3ed735da6662..a390854ef672 100644 --- a/changelog.d/20240405_091941_klakhov_rar_support.md +++ b/changelog.d/20240405_091941_klakhov_rar_support.md @@ -1,4 +1,4 @@ ### Added -- Support for `.rar`, `.tar`, `.gz`, `.bz2`, `.cpio`, `.7z`, archives +- Support for `.rar`, `.tar`, `.gz`, `.bz2`, `.cpio`, `.7z` archives () From f890b85c44be58dc0190dae9b26297c35e5f4a5d Mon Sep 17 00:00:00 2001 From: Kirill Lakhov Date: Thu, 11 Apr 2024 11:13:34 +0300 Subject: [PATCH 7/9] applied comments --- cvat/apps/engine/tests/test_rest_api.py | 52 +++++-------------------- 1 file changed, 10 insertions(+), 42 deletions(-) diff --git a/cvat/apps/engine/tests/test_rest_api.py b/cvat/apps/engine/tests/test_rest_api.py index 16ae01a30ddf..81f187ca09ff 100644 --- a/cvat/apps/engine/tests/test_rest_api.py +++ b/cvat/apps/engine/tests/test_rest_api.py @@ -3195,13 +3195,12 @@ def setUpClass(cls): path = os.path.join(settings.SHARE_ROOT, filename) shutil.copyfile(source_path, path) image_sizes = [] - archive_dir, images = cls._extract_rar_archive(open(source_path, 'rb')) + images = cls._extract_rar_archive(source_path) for [f, image] in images: - width,height = image.size + width, height = image.size image_sizes.append((width, height)) cls._share_image_sizes[filename] = image_sizes cls._share_files.append(filename) - cls._unpack_dirs.append(archive_dir) filename = "test_velodyne_points.zip" path = os.path.join(os.path.dirname(__file__), 'assets', filename) @@ -3390,19 +3389,14 @@ def _extract_rar_archive(archive): archive_dir = os.path.join(settings.TMP_FILES_ROOT, rand_name) os.makedirs(archive_dir) - arch_file = os.path.join(archive_dir, f"{rand_name}.rar") - archive.seek(0) - with open(arch_file, 'wb') as tmp_file: - tmp_file.write(archive.read()) - - unpack_dir = os.path.join(settings.TMP_FILES_ROOT, rand_name, "unpack") - os.makedirs(unpack_dir) - patool_path = os.path.join(sysconfig.get_path('scripts'), 'patool') - Archive(arch_file).extractall_patool(unpack_dir, patool_path) - return archive_dir, [(image, Image.open(os.path.join(unpack_dir, image))) - for image in os.listdir(unpack_dir) + Archive(archive).extractall_patool(archive_dir, patool_path) + + images = [(image, Image.open(os.path.join(archive_dir, image))) + for image in os.listdir(archive_dir) ] + shutil.rmtree(archive_dir) + return images @classmethod def _extract_zip_chunk(cls, chunk_buffer, dimension=DimensionType.DIM_2D): @@ -3559,15 +3553,8 @@ def _test_api_v2_tasks_id_data_spec(self, user, spec, data, for frame_name, frame in self._extract_zip_archive(f, dimension=dimension): source_images[frame_name] = frame elif isinstance(f, str) and f.endswith('.rar'): - with open(f, 'rb') as rar_file: - archive_dir, arhive_frames = self._extract_rar_archive(rar_file) - self._unpack_dirs.append(archive_dir) - for fn, frame in arhive_frames: - source_images[fn] = frame - elif isinstance(f, IOBase) and getattr(f, 'name', '').endswith('.rar'): - archive_dir, arhive_frames = self._extract_rar_archive(f) - self._unpack_dirs.append(archive_dir) - for fn, frame in arhive_frames: + archive_frames = self._extract_rar_archive(f) + for fn, frame in archive_frames: source_images[fn] = frame elif isinstance(f, str) and f.endswith('.pdf'): with open(f, 'rb') as pdf_file: @@ -4634,25 +4621,6 @@ def _test_api_v2_tasks_id_data_create_can_use_server_rar(self, user): self._test_api_v2_tasks_id_data_spec(user, task_spec, task_data, self.ChunkType.IMAGESET, self.ChunkType.IMAGESET, image_sizes, StorageMethodChoice.CACHE, StorageChoice.LOCAL) - def _test_api_v2_tasks_id_data_create_can_use_local_rar(self, user): - task_spec = { - "name": 'task rar client files #33', - "overlap": 0, - "segment_size": 0, - "labels": [ - {"name": "car"}, - {"name": "person"}, - ] - } - - task_data = { - "client_files[0]": open(os.path.join(os.path.dirname(__file__), 'assets', 'test_rar.rar'), 'rb'), - "image_quality": 75, - } - image_sizes = self._share_image_sizes["test_rar.rar"] - self._test_api_v2_tasks_id_data_spec(user, task_spec, task_data, - self.ChunkType.IMAGESET, self.ChunkType.IMAGESET, image_sizes) - def _test_api_v2_tasks_id_data_create(self, user): method_list = { func: getattr(self, func) for func in dir(self) From 643e2243c2438c72ca05027338f55931c1ed03ee Mon Sep 17 00:00:00 2001 From: Kirill Lakhov Date: Thu, 11 Apr 2024 13:14:30 +0300 Subject: [PATCH 8/9] fix comments --- cvat/apps/engine/tests/test_rest_api.py | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/cvat/apps/engine/tests/test_rest_api.py b/cvat/apps/engine/tests/test_rest_api.py index 81f187ca09ff..0685e39ceace 100644 --- a/cvat/apps/engine/tests/test_rest_api.py +++ b/cvat/apps/engine/tests/test_rest_api.py @@ -3129,7 +3129,6 @@ def setUpClass(cls): cls._share_image_sizes = {} cls._share_files = [] - cls._unpack_dirs = [] for filename in [ "test_1.jpg", "test_2.jpg", "test_3.jpg", "test_10.jpg", "test_qwe.jpg", @@ -3314,9 +3313,6 @@ def tearDownClass(cls): dirs.add(os.path.dirname(filename)) os.remove(os.path.join(settings.SHARE_ROOT, filename)) - for unpack_dir in cls._unpack_dirs: - shutil.rmtree(unpack_dir) - for dirname in sorted(dirs, reverse=True): path = os.path.join(settings.SHARE_ROOT, dirname) if not os.listdir(path): @@ -3385,18 +3381,14 @@ def _extract_zip_archive(archive, dimension=DimensionType.DIM_2D): @staticmethod def _extract_rar_archive(archive): - rand_name = uuid4().hex - archive_dir = os.path.join(settings.TMP_FILES_ROOT, rand_name) - os.makedirs(archive_dir) - - patool_path = os.path.join(sysconfig.get_path('scripts'), 'patool') - Archive(archive).extractall_patool(archive_dir, patool_path) + with tempfile.TemporaryDirectory(dir=settings.TMP_FILES_ROOT) as archive_dir: + patool_path = os.path.join(sysconfig.get_path('scripts'), 'patool') + Archive(archive).extractall_patool(archive_dir, patool_path) - images = [(image, Image.open(os.path.join(archive_dir, image))) - for image in os.listdir(archive_dir) - ] - shutil.rmtree(archive_dir) - return images + images = [(image, Image.open(os.path.join(archive_dir, image))) + for image in os.listdir(archive_dir) + ] + return images @classmethod def _extract_zip_chunk(cls, chunk_buffer, dimension=DimensionType.DIM_2D): @@ -3546,6 +3538,7 @@ def _test_api_v2_tasks_id_data_spec(self, user, spec, data, manifest = next((v for v in source_files if _name_key(v).endswith('.jsonl')), None) source_files = [_add_prefix(f) for f in source_files if not _name_key(f).endswith('jsonl')] + # Load images source_images = {} for f in source_files: From aee6baf02e321bdbd240671def32eed54418bbf5 Mon Sep 17 00:00:00 2001 From: Kirill Lakhov Date: Thu, 11 Apr 2024 13:20:20 +0300 Subject: [PATCH 9/9] remove unused import --- cvat/apps/engine/tests/test_rest_api.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cvat/apps/engine/tests/test_rest_api.py b/cvat/apps/engine/tests/test_rest_api.py index 0685e39ceace..37f4fbd44c4e 100644 --- a/cvat/apps/engine/tests/test_rest_api.py +++ b/cvat/apps/engine/tests/test_rest_api.py @@ -11,7 +11,6 @@ import shutil import sysconfig import tempfile -from uuid import uuid4 import xml.etree.ElementTree as ET import zipfile from collections import defaultdict