Add REST API GET, POST, PATCH tests for cloud storage (cvat-ai#4353)

Co-authored-by: kirill.sizov <[email protected]>

.github/workflows/main.yml

@@ -74,12 +74,15 @@ jobs:
       - name: Running REST API tests
         env:
           API_ABOUT_PAGE: "localhost:8080/api/server/about"
+          # Access key length should be at least 3, and secret key length at least 8 characters
+          MINIO_ACCESS_KEY: "minio_access_key"
+          MINIO_SECRET_KEY: "minio_secret_key"
         run: |
-          docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f components/serverless/docker-compose.serverless.yml -f components/analytics/docker-compose.analytics.yml up -d
+          docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f components/serverless/docker-compose.serverless.yml -f components/analytics/docker-compose.analytics.yml -f tests/rest_api/docker-compose.minio.yml up -d
           /bin/bash -c 'while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' ${API_ABOUT_PAGE})" != "401" ]]; do sleep 5; done'
           pip3 install --user -r tests/rest_api/requirements.txt
           pytest tests/rest_api/
-          docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f components/serverless/docker-compose.serverless.yml -f components/analytics/docker-compose.analytics.yml down -v
+          docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f components/serverless/docker-compose.serverless.yml -f components/analytics/docker-compose.analytics.yml -f tests/rest_api/docker-compose.minio.yml down -v
       - name: Running unit tests
         env:
           HOST_COVERAGE_DATA_DIR: ${{ github.workspace }}

cvat-ui/src/components/create-cloud-storage-page/cloud-storage-form.tsx

@@ -48,6 +48,7 @@ interface CloudStorageForm {
     prefix?: string;
     project_id?: string;
     manifests: string[];
+    endpoint_url?: string;
 }
 
 const { Dragger } = Upload;
@@ -117,16 +118,20 @@ export default function CreateCloudStorageForm(props: Props): JSX.Element {
             const location = parsedOptions.get('region') || parsedOptions.get('location');
             const prefix = parsedOptions.get('prefix');
             const projectId = parsedOptions.get('project_id');
+            const endpointUrl = parsedOptions.get('endpoint_url');
+
             if (location) {
                 setSelectedRegion(location);
             }
             if (prefix) {
                 fieldsValue.prefix = prefix;
             }
-
             if (projectId) {
                 fieldsValue.project_id = projectId;
             }
+            if (endpointUrl) {
+                fieldsValue.endpoint_url = endpointUrl;
+            }
         }
 
         form.setFieldsValue(fieldsValue);
@@ -222,6 +227,10 @@ export default function CreateCloudStorageForm(props: Props): JSX.Element {
             delete cloudStorageData.project_id;
             specificAttributes.append('project_id', formValues.project_id);
         }
+        if (formValues.endpoint_url) {
+            delete cloudStorageData.endpoint_url;
+            specificAttributes.append('endpoint_url', formValues.endpoint_url);
+        }
 
         cloudStorageData.specific_attributes = specificAttributes.toString();
 
@@ -489,6 +498,14 @@ export default function CreateCloudStorageForm(props: Props): JSX.Element {
                     </Select>
                 </Form.Item>
                 {credentialsBlok()}
+                <Form.Item
+                    label='Endpoint URL'
+                    help='You can specify an endpoint for your storage when using the AWS S3 cloud storage compatible API'
+                    name='endpoint_url'
+                    {...internalCommonProps}
+                >
+                    <Input />
+                </Form.Item>
                 <S3Region
                     selectedRegion={selectedRegion}
                     onSelectRegion={onSelectRegion}

cvat/apps/engine/cloud_provider.py

@@ -99,15 +99,16 @@ def __len__(self):
     def content(self):
         return list(map(lambda x: x['name'] , self._files))
 
-def get_cloud_storage_instance(cloud_provider, resource, credentials, specific_attributes=None):
+def get_cloud_storage_instance(cloud_provider, resource, credentials, specific_attributes=None, endpoint=None):
     instance = None
     if cloud_provider == CloudProviderChoice.AWS_S3:
         instance = AWS_S3(
             bucket=resource,
             access_key_id=credentials.key,
             secret_key=credentials.secret_key,
             session_token=credentials.session_token,
-            region=specific_attributes.get('region', 'us-east-2')
+            region=specific_attributes.get('region'),
+            endpoint_url=specific_attributes.get('endpoint_url'),
         )
     elif cloud_provider == CloudProviderChoice.AZURE_CONTAINER:
         instance = AzureBlobContainer(
@@ -137,28 +138,31 @@ def __init__(self,
                 region,
                 access_key_id=None,
                 secret_key=None,
-                session_token=None):
+                session_token=None,
+                endpoint_url=None):
         super().__init__()
         if all([access_key_id, secret_key, session_token]):
             self._s3 = boto3.resource(
                 's3',
                 aws_access_key_id=access_key_id,
                 aws_secret_access_key=secret_key,
                 aws_session_token=session_token,
-                region_name=region
+                region_name=region,
+                endpoint_url=endpoint_url
             )
         elif access_key_id and secret_key:
             self._s3 = boto3.resource(
                 's3',
                 aws_access_key_id=access_key_id,
                 aws_secret_access_key=secret_key,
-                region_name=region
+                region_name=region,
+                endpoint_url=endpoint_url
             )
         elif any([access_key_id, secret_key, session_token]):
             raise Exception('Insufficient data for authorization')
         # anonymous access
         if not any([access_key_id, secret_key, session_token]):
-            self._s3 = boto3.resource('s3', region_name=region)
+            self._s3 = boto3.resource('s3', region_name=region, endpoint_url=endpoint_url)
             self._s3.meta.client.meta.events.register('choose-signer.s3.*', disable_signing)
         self._client_s3 = self._s3.meta.client
         self._bucket = self._s3.Bucket(bucket)

cvat/apps/engine/migrations/0052_alter_cloudstorage_specific_attributes.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.12 on 2022-03-14 10:51
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('engine', '0051_auto_20220220_1824'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='cloudstorage',
+            name='specific_attributes',
+            field=models.CharField(blank=True, max_length=1024),
+        ),
+    ]

cvat/apps/engine/models.py

@@ -673,7 +673,7 @@ class CloudStorage(models.Model):
     updated_date = models.DateTimeField(auto_now=True)
     credentials = models.CharField(max_length=500)
     credentials_type = models.CharField(max_length=29, choices=CredentialsTypeChoice.choices())#auth_type
-    specific_attributes = models.CharField(max_length=128, blank=True)
+    specific_attributes = models.CharField(max_length=1024, blank=True)
     description = models.TextField(blank=True)
     organization = models.ForeignKey(Organization, null=True, default=None,
         blank=True, on_delete=models.SET_NULL, related_name="cloudstorages")

cvat/apps/engine/serializers.py

@@ -972,6 +972,22 @@ def validate(self, attrs):
                 raise serializers.ValidationError('Account name for Azure container was not specified')
         return attrs
 
+    @staticmethod
+    def _manifests_validation(storage, manifests):
+            # check manifest files availability
+            for manifest in manifests:
+                file_status = storage.get_file_status(manifest)
+                if file_status == Status.NOT_FOUND:
+                    raise serializers.ValidationError({
+                        'manifests': "The '{}' file does not exist on '{}' cloud storage" \
+                            .format(manifest, storage.name)
+                    })
+                elif file_status == Status.FORBIDDEN:
+                    raise serializers.ValidationError({
+                        'manifests': "The '{}' file does not available on '{}' cloud storage. Access denied" \
+                            .format(manifest, storage.name)
+                    })
+
     def create(self, validated_data):
         provider_type = validated_data.get('provider_type')
         should_be_created = validated_data.pop('should_be_created', None)
@@ -1008,28 +1024,16 @@ def create(self, validated_data):
 
         storage_status = storage.get_status()
         if storage_status == Status.AVAILABLE:
-            manifests = validated_data.pop('manifests')
-            # check manifest files availability
-            for manifest in manifests:
-                file_status = storage.get_file_status(manifest.get('filename'))
-                if file_status == Status.NOT_FOUND:
-                    raise serializers.ValidationError({
-                        'manifests': "The '{}' file does not exist on '{}' cloud storage" \
-                            .format(manifest.get('filename'), storage.name)
-                    })
-                elif file_status == Status.FORBIDDEN:
-                    raise serializers.ValidationError({
-                        'manifests': "The '{}' file does not available on '{}' cloud storage. Access denied" \
-                            .format(manifest.get('filename'), storage.name)
-                    })
+            manifests = [m.get('filename') for m in validated_data.pop('manifests')]
+            self._manifests_validation(storage, manifests)
 
             db_storage = models.CloudStorage.objects.create(
                 credentials=credentials.convert_to_db(),
                 **validated_data
             )
             db_storage.save()
 
-            manifest_file_instances = [models.Manifest(**manifest, cloud_storage=db_storage) for manifest in manifests]
+            manifest_file_instances = [models.Manifest(filename=manifest, cloud_storage=db_storage) for manifest in manifests]
             models.Manifest.objects.bulk_create(manifest_file_instances)
 
             cloud_storage_path = db_storage.get_storage_dirname()
@@ -1105,18 +1109,7 @@ def update(self, instance, validated_data):
                 instance.manifests.filter(filename__in=delta_to_delete).delete()
             if delta_to_create:
                 # check manifest files existing
-                for manifest in delta_to_create:
-                    file_status = storage.get_file_status(manifest)
-                    if file_status == Status.NOT_FOUND:
-                        raise serializers.ValidationError({
-                            'manifests': "The '{}' file does not exist on '{}' cloud storage"
-                                .format(manifest, storage.name)
-                        })
-                    elif file_status == Status.FORBIDDEN:
-                        raise serializers.ValidationError({
-                            'manifests': "The '{}' file does not available on '{}' cloud storage. Access denied" \
-                                .format(manifest.get('filename'), storage.name)
-                        })
+                self._manifests_validation(storage, delta_to_create)
                 manifest_instances = [models.Manifest(filename=f, cloud_storage=instance) for f in delta_to_create]
                 models.Manifest.objects.bulk_create(manifest_instances)
             if temporary_file:

cvat/apps/engine/utils.py

@@ -11,6 +11,8 @@
 import traceback
 import subprocess
 import os
+import urllib.parse
+
 from av import VideoFrame
 from PIL import Image
 
@@ -102,7 +104,7 @@ def md5_hash(frame):
 
 def parse_specific_attributes(specific_attributes):
     assert isinstance(specific_attributes, str), 'Specific attributes must be a string'
+    parsed_specific_attributes = urllib.parse.parse_qsl(specific_attributes)
     return {
-        item.split('=')[0].strip(): item.split('=')[1].strip()
-            for item in specific_attributes.split('&')
-    } if specific_attributes else dict()
+        key: value for (key, value) in parsed_specific_attributes
+    } if parsed_specific_attributes else dict()

tests/rest_api/README.md

@@ -19,14 +19,20 @@ the server calling REST API directly (as it done by users).
 
 ## How to run?
 
-Please look at documentation for [pytest](https://docs.pytest.org/en/6.2.x/).
-Generally you have to install requirements and run the following command from
-the root directory of the cloned CVAT repository:
+1. Execute commands below to run docker containers:
+   ```console
+   export MINIO_ACCESS_KEY="minio_access_key"
+   export MINIO_SECRET_KEY="minio_secret_key"
+   docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f components/analytics/docker-compose.analytics.yml -f tests/rest_api/docker-compose.minio.yml up -d --build
+   ```
+1. After that please look at documentation for [pytest](https://docs.pytest.org/en/6.2.x/).
+   Generally, you have to install requirements and run the following command from
+   the root directory of the cloned CVAT repository:
 
-```console
-pip3 install --user -r tests/rest_api/requirements.txt
-pytest tests/rest_api/
-```
+   ```console
+   pip3 install --user -r tests/rest_api/requirements.txt
+   pytest tests/rest_api/
+   ```
 
 ## How to upgrade testing assets?
 
@@ -151,7 +157,7 @@ Assets directory has two parts:
    ```
 
 1. If your tests was failed due to date field incompatibility and you have
-error message like this:
+   error message like this:
    ```
    assert {'values_chan...34.908528Z'}}} == {}
    E                 Left contains 1 more item:
@@ -182,7 +188,7 @@ error message like this:
    ```
 
 1. If for some reason you need to recreate cvat database, but using `dropdb`
-you have error message:
+   you have error message:
    ```
    ERROR:  database "cvat" is being accessed by other users
    DETAIL:  There are 1 other session(s) using the database.

tests/rest_api/assets/cloudstorages.json

@@ -0,0 +1,51 @@
+{
+  "count": 2,
+  "next": null,
+  "previous": null,
+  "results": [
+    {
+      "created_date": "2022-03-17T07:23:59.305000Z",
+      "credentials_type": "KEY_SECRET_KEY_PAIR",
+      "description": "",
+      "display_name": "Bucket 2",
+      "id": 2,
+      "manifests": [
+        "manifest.jsonl"
+      ],
+      "organization": 2,
+      "owner": {
+        "first_name": "Business",
+        "id": 11,
+        "last_name": "Second",
+        "url": "http://localhost:8080/api/users/11",
+        "username": "business2"
+      },
+      "provider_type": "AWS_S3_BUCKET",
+      "resource": "private",
+      "specific_attributes": "endpoint_url=http%3A%2F%2Fminio%3A9000",
+      "updated_date": "2022-03-17T07:23:59.309000Z"
+    },
+    {
+      "created_date": "2022-03-17T07:22:49.519000Z",
+      "credentials_type": "ANONYMOUS_ACCESS",
+      "description": "",
+      "display_name": "Bucket 1",
+      "id": 1,
+      "manifests": [
+        "manifest.jsonl"
+      ],
+      "organization": null,
+      "owner": {
+        "first_name": "User",
+        "id": 2,
+        "last_name": "First",
+        "url": "http://localhost:8080/api/users/2",
+        "username": "user1"
+      },
+      "provider_type": "AWS_S3_BUCKET",
+      "resource": "public",
+      "specific_attributes": "endpoint_url=http%3A%2F%2Fminio%3A9000",
+      "updated_date": "2022-03-17T07:22:49.529000Z"
+    }
+  ]
+}