Extension of cloud storage server part (#3386)

* Add preview && some fixes * Fix case with sub dirs on cloud storage * Move server part from ui_support_cloud_storage && fix missing id field * Add support_key_secret_key_pair * Fix several moments * Add index resetting * Fix pylint errors * Remove excess migration * tmp * Some fixes * Fixes * fix * [server] Add cloud storage status && fixes * Remove unused import * Add manifest set_index method * Implement status support for Azure blob container * Move specific attributes parsing into utils * Fix missing in migration * Fix error display * some fix * Update migration dependency * Update google cloud storage status * Update migrtaions * Update CHANGELOG
cvat-ai · Aug 31, 2021 · fa15f78 · fa15f78
1 parent 7bad531
commit fa15f78
Show file tree

Hide file tree

Showing 10 changed files with 573 additions and 164 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -15,19 +15,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   and project with 3D tasks (<https://github.com/openvinotoolkit/cvat/pull/3502>)
 - Additional inline tips in interactors with demo gifs (<https://github.com/openvinotoolkit/cvat/pull/3473>)
 - Added intelligent scissors blocking feature (<https://github.com/openvinotoolkit/cvat/pull/3510>)
+- Support cloud storage status (<https://github.com/openvinotoolkit/cvat/pull/3386>)
+- Support cloud storage preview (<https://github.com/openvinotoolkit/cvat/pull/3386>)
 
 ### Changed
 
 - Non-blocking UI when using interactors (<https://github.com/openvinotoolkit/cvat/pull/3473>)
 - "Selected opacity" slider now defines opacity level for shapes being drawnSelected opacity (<https://github.com/openvinotoolkit/cvat/pull/3473>)
+- Cloud storage creating and updating (<https://github.com/openvinotoolkit/cvat/pull/3386>)
+- Way of working with cloud storage content (<https://github.com/openvinotoolkit/cvat/pull/3386>)
 
 ### Deprecated
 
 - TDB
 
 ### Removed
 
-- TDB
+- Support TEMP_KEY_SECRET_KEY_TOKEN_SET for AWS S3 cloud storage (<https://github.com/openvinotoolkit/cvat/pull/3386>)
 
 ### Fixed
 

diff --git a/cvat/apps/engine/cache.py b/cvat/apps/engine/cache.py
@@ -15,7 +15,7 @@
     ImageDatasetManifestReader, VideoDatasetManifestReader)
 from cvat.apps.engine.models import DataChoice, StorageChoice
 from cvat.apps.engine.models import DimensionType
-from cvat.apps.engine.cloud_provider import get_cloud_storage_instance, Credentials
+from cvat.apps.engine.cloud_provider import get_cloud_storage_instance, Credentials, Status
 from cvat.apps.engine.utils import md5_hash
 class CacheInteraction:
     def __init__(self, dimension=DimensionType.DIM_2D):
@@ -71,6 +71,7 @@ def prepare_chunk_buff(self, db_data, quality, chunk_number):
                 step=db_data.get_frame_step())
             if db_data.storage == StorageChoice.CLOUD_STORAGE:
                 db_cloud_storage = db_data.cloud_storage
+                assert db_cloud_storage, 'Cloud storage instance was deleted'
                 credentials = Credentials()
                 credentials.convert_from_db({
                     'type': db_cloud_storage.credentials_type,
@@ -81,22 +82,38 @@ def prepare_chunk_buff(self, db_data, quality, chunk_number):
                     'credentials': credentials,
                     'specific_attributes': db_cloud_storage.get_specific_attributes()
                 }
-                cloud_storage_instance = get_cloud_storage_instance(cloud_provider=db_cloud_storage.provider_type, **details)
-                cloud_storage_instance.initialize_content()
-                for item in reader:
-                    name = f"{item['name']}{item['extension']}"
-                    if name not in cloud_storage_instance:
-                        raise Exception('{} file was not found on a {} storage'.format(name, cloud_storage_instance.name))
-                    with NamedTemporaryFile(mode='w+b', prefix='cvat', suffix=name.replace(os.path.sep, '#'), delete=False) as temp_file:
-                        source_path = temp_file.name
-                        buf = cloud_storage_instance.download_fileobj(name)
-                        temp_file.write(buf.getvalue())
-                        checksum = item.get('checksum', None)
-                        if not checksum:
-                            slogger.glob.warning('A manifest file does not contain checksum for image {}'.format(item.get('name')))
-                        if checksum and not md5_hash(source_path) == checksum:
-                            slogger.glob.warning('Hash sums of files {} do not match'.format(name))
-                        images.append((source_path, source_path, None))
+                try:
+                    cloud_storage_instance = get_cloud_storage_instance(cloud_provider=db_cloud_storage.provider_type, **details)
+                    cloud_storage_instance.initialize_content()
+                    for item in reader:
+                        file_name = f"{item['name']}{item['extension']}"
+                        if file_name not in cloud_storage_instance:
+                            raise Exception('{} file was not found on a {} storage'.format(file_name, cloud_storage_instance.name))
+                        with NamedTemporaryFile(mode='w+b', prefix='cvat', suffix=file_name.replace(os.path.sep, '#'), delete=False) as temp_file:
+                            source_path = temp_file.name
+                            buf = cloud_storage_instance.download_fileobj(file_name)
+                            temp_file.write(buf.getvalue())
+                            checksum = item.get('checksum', None)
+                            if not checksum:
+                                slogger.cloud_storage[db_cloud_storage.id].warning('A manifest file does not contain checksum for image {}'.format(item.get('name')))
+                            if checksum and not md5_hash(source_path) == checksum:
+                                slogger.cloud_storage[db_cloud_storage.id].warning('Hash sums of files {} do not match'.format(file_name))
+                            images.append((source_path, source_path, None))
+                except Exception as ex:
+                    storage_status = cloud_storage_instance.get_status()
+                    if storage_status == Status.FORBIDDEN:
+                        msg = 'The resource {} is no longer available. Access forbidden.'.format(cloud_storage_instance.name)
+                    elif storage_status == Status.NOT_FOUND:
+                        msg = 'The resource {} not found. It may have been deleted.'.format(cloud_storage_instance.name)
+                    else:
+                        # check status of last file
+                        file_status = cloud_storage_instance.get_file_status(file_name)
+                        if file_status == Status.NOT_FOUND:
+                            raise Exception("'{}' not found on the cloud storage '{}'".format(file_name, cloud_storage_instance.name))
+                        elif file_status == Status.FORBIDDEN:
+                            raise Exception("Access to the file '{}' on the '{}' cloud storage is denied".format(file_name, cloud_storage_instance.name))
+                        msg = str(ex)
+                    raise Exception(msg)
             else:
                 for item in reader:
                     source_path = os.path.join(upload_dir, f"{item['name']}{item['extension']}")