Merge CVAT milestone 2 changes

.github/workflows/cd-cvat-exchange-oracle.yaml

@@ -2,11 +2,11 @@ name: Deploy CVAT Exchange Oracle
 
 on:
   push:
-    branches: [ develop, main ]
+    branches: [ develop, main, cvat-milestone-2 ]
     paths:
       - 'packages/examples/cvat/exchange-oracle/**'
   pull_request:
-    branches: [ develop, main ]
+    branches: [ develop, main, cvat-milestone-2 ]
     paths:
       - 'packages/examples/cvat/exchange-oracle/**'
   workflow_dispatch:

.github/workflows/cd-cvat-recording-oracle.yaml

@@ -2,11 +2,11 @@ name: Deploy CVAT Recording Oracle
 
 on:
   push:
-    branches: [ develop, main ]
+    branches: [ develop, main, cvat-milestone-2 ]
     paths:
       - 'packages/examples/cvat/recording-oracle/**'
   pull_request:
-    branches: [ develop, main ]
+    branches: [ develop, main, cvat-milestone-2 ]
     paths:
       - 'packages/examples/cvat/recording-oracle/**'
   workflow_dispatch:

packages/examples/cvat/exchange-oracle/alembic/versions/c1e74c227cfe_non_unique_escrows.py

@@ -0,0 +1,29 @@
+"""non-unique-escrows
+
+Revision ID: c1e74c227cfe
+Revises: 16ecc586d685
+Create Date: 2024-02-05 22:54:42.478270
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import sqlalchemy_utils
+
+
+# revision identifiers, used by Alembic.
+revision = 'c1e74c227cfe'
+down_revision = '16ecc586d685'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint('projects_escrow_address_key', 'projects', type_='unique')
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_unique_constraint('projects_escrow_address_key', 'projects', ['escrow_address'])
+    # ### end Alembic commands ###

packages/examples/cvat/exchange-oracle/docker-compose.test.yml

@@ -70,6 +70,7 @@ services:
       STORAGE_SECRET_KEY: 'devdevdev'
       STORAGE_RESULTS_BUCKET_NAME: 'results'
       STORAGE_USE_SSL: False
+      STORAGE_PROVIDER: 'aws'
       ENABLE_CUSTOM_CLOUD_HOST: Yes
     depends_on:
       postgres:

packages/examples/cvat/exchange-oracle/pyproject.toml

@@ -23,6 +23,7 @@ human-protocol-sdk = "^1.1.19"
 xmltodict = "^0.13.0"
 datumaro = {git = "https://github.com/cvat-ai/datumaro.git", rev = "ff83c00c2c1bc4b8fdfcc55067fcab0a9b5b6b11"}
 boto3 = "^1.28.33"
+google-cloud-storage = "^2.14.0"
 
 
 [tool.poetry.group.dev.dependencies]

packages/examples/cvat/exchange-oracle/src/.env.template

@@ -35,10 +35,12 @@ PROCESS_RECORDING_ORACLE_WEBHOOKS_CHUNK_SIZE=
 TRACK_COMPLETED_PROJECTS_INT=
 TRACK_COMPLETED_PROJECTS_CHUNK_SIZE=
 TRACK_COMPLETED_TASKS_INT=
-RETRIEVE_ANNOTATIONS_INT=
-RETRIEVE_ANNOTATIONS_CHUNK_SIZE=
+TRACK_COMPLETED_ESCROWS_INT=
+TRACK_COMPLETED_ESCROWS_CHUNK_SIZE=
 PROCESS_JOB_LAUNCHER_WEBHOOKS_INT=
 TRACK_CREATING_TASKS_INT=
+REJECTED_PROJECTS_CHUNK_SIZE=
+ACCEPTED_PROJECTS_CHUNK_SIZE=
 
 # CVAT Config
 
@@ -50,12 +52,14 @@ CVAT_INCOMING_WEBHOOKS_URL=
 CVAT_WEBHOOK_SECRET=
 CVAT_ORG_SLUG=
 
-# S3 Storage Config
+# Storage Config (S3/GCS)
 
+STORAGE_PROVIDER=
 STORAGE_ENDPOINT_URL=
 STORAGE_REGION=
 STORAGE_ACCESS_KEY=
 STORAGE_SECRET_KEY=
+STORAGE_KEY_FILE_PATH=
 STORAGE_RESULTS_BUCKET_NAME=
 STORAGE_USE_SSL=
 
@@ -73,5 +77,6 @@ HUMAN_APP_SIGNATURE=
 
 # Localhost
 
+LOCALHOST_RECORDING_ORACLE_ADDRESS=
 LOCALHOST_RECORDING_ORACLE_URL=
 LOCALHOST_JOB_LAUNCHER_URL=

packages/examples/cvat/exchange-oracle/src/chain/escrow.py

@@ -5,6 +5,8 @@
 from human_protocol_sdk.escrow import EscrowData, EscrowUtils
 from human_protocol_sdk.storage import StorageUtils
 
+from src.core.config import Config
+
 
 def get_escrow(chain_id: int, escrow_address: str) -> EscrowData:
     escrow = EscrowUtils.get_escrow(ChainId(chain_id), escrow_address)
@@ -51,4 +53,7 @@ def get_job_launcher_address(chain_id: int, escrow_address: str) -> str:
 
 
 def get_recording_oracle_address(chain_id: int, escrow_address: str) -> str:
+    if address := Config.localhost.recording_oracle_address:
+        return address
+
     return get_escrow(chain_id, escrow_address).recording_oracle

packages/examples/cvat/exchange-oracle/src/core/annotation_meta.py

@@ -3,7 +3,7 @@
 
 from pydantic import BaseModel
 
-ANNOTATION_METAFILE_NAME = "annotation_meta.json"
+ANNOTATION_RESULTS_METAFILE_NAME = "annotation_meta.json"
 RESULTING_ANNOTATIONS_FILE = "resulting_annotations.zip"
 
 

packages/examples/cvat/exchange-oracle/src/core/config.py

@@ -1,7 +1,9 @@
 # pylint: disable=too-few-public-methods,missing-class-docstring
 """ Project configuration from env vars """
 import os
+from typing import ClassVar, Optional
 
+from attrs.converters import to_bool
 from dotenv import load_dotenv
 
 from src.utils.logging import parse_log_level
@@ -10,12 +12,6 @@
 load_dotenv()
 
 
-def str_to_bool(val: str) -> bool:
-    from distutils.util import strtobool
-
-    return val is True or strtobool(val)
-
-
 class PostgresConfig:
     port = os.environ.get("PG_PORT", "5432")
     host = os.environ.get("PG_HOST", "0.0.0.0")
@@ -53,6 +49,8 @@ class LocalhostConfig:
     addr = os.environ.get("LOCALHOST_MUMBAI_ADDR", "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266")
 
     job_launcher_url = os.environ.get("LOCALHOST_JOB_LAUNCHER_URL")
+
+    recording_oracle_address = os.environ.get("LOCALHOST_RECORDING_ORACLE_ADDRESS")
     recording_oracle_url = os.environ.get("LOCALHOST_RECORDING_ORACLE_URL")
 
 
@@ -75,8 +73,20 @@ class CronConfig:
     track_assignments_int = int(os.environ.get("TRACK_ASSIGNMENTS_INT", 5))
     track_assignments_chunk_size = os.environ.get("TRACK_ASSIGNMENTS_CHUNK_SIZE", 10)
 
-    retrieve_annotations_int = int(os.environ.get("RETRIEVE_ANNOTATIONS_INT", 60))
-    retrieve_annotations_chunk_size = os.environ.get("RETRIEVE_ANNOTATIONS_CHUNK_SIZE", 5)
+    track_completed_escrows_int = int(
+        # backward compatibility
+        os.environ.get(
+            "TRACK_COMPLETED_ESCROWS_INT", os.environ.get("RETRIEVE_ANNOTATIONS_INT", 60)
+        )
+    )
+    track_completed_escrows_chunk_size = os.environ.get(
+        # backward compatibility
+        "TRACK_COMPLETED_ESCROWS_CHUNK_SIZE",
+        os.environ.get("RETRIEVE_ANNOTATIONS_CHUNK_SIZE", 5),
+    )
+
+    rejected_projects_chunk_size = os.environ.get("REJECTED_PROJECTS_CHUNK_SIZE", 20)
+    accepted_projects_chunk_size = os.environ.get("ACCEPTED_PROJECTS_CHUNK_SIZE", 20)
 
 
 class CvatConfig:
@@ -94,39 +104,51 @@ class CvatConfig:
 
 
 class StorageConfig:
-    endpoint_url = os.environ.get("STORAGE_ENDPOINT_URL", "storage.googleapis.com")
-    region = os.environ.get("STORAGE_REGION", "")
-    access_key = os.environ.get("STORAGE_ACCESS_KEY", "")
-    secret_key = os.environ.get("STORAGE_SECRET_KEY", "")
-    results_bucket_name = os.environ.get("STORAGE_RESULTS_BUCKET_NAME", "")
-    secure = str_to_bool(os.environ.get("STORAGE_USE_SSL", "true"))
+    provider: ClassVar[str] = os.environ["STORAGE_PROVIDER"].lower()
+    data_bucket_name: ClassVar[str] = (
+        os.environ.get("STORAGE_RESULTS_BUCKET_NAME")  # backward compatibility
+        or os.environ["STORAGE_BUCKET_NAME"]
+    )
+    endpoint_url: ClassVar[str] = os.environ[
+        "STORAGE_ENDPOINT_URL"
+    ]  # TODO: probably should be optional
+    region: ClassVar[Optional[str]] = os.environ.get("STORAGE_REGION")
+    results_dir_suffix: ClassVar[str] = os.environ.get("STORAGE_RESULTS_DIR_SUFFIX", "-results")
+    secure: ClassVar[bool] = to_bool(os.environ.get("STORAGE_USE_SSL", "true"))
+
+    # S3 specific attributes
+    access_key: ClassVar[Optional[str]] = os.environ.get("STORAGE_ACCESS_KEY")
+    secret_key: ClassVar[Optional[str]] = os.environ.get("STORAGE_SECRET_KEY")
+
+    # GCS specific attributes
+    key_file_path: ClassVar[Optional[str]] = os.environ.get("STORAGE_KEY_FILE_PATH")
 
     @classmethod
-    def provider_endpoint_url(cls):
-        scheme = "https://" if cls.secure else "http://"
+    def get_scheme(cls) -> str:
+        return "https://" if cls.secure else "http://"
 
-        return f"{scheme}{cls.endpoint_url}"
+    @classmethod
+    def provider_endpoint_url(cls):
+        return f"{cls.get_scheme()}{cls.endpoint_url}"
 
     @classmethod
     def bucket_url(cls):
-        scheme = "https://" if cls.secure else "http://"
-
         if is_ipv4(cls.endpoint_url):
-            return f"{scheme}{cls.endpoint_url}/{cls.results_bucket_name}/"
+            return f"{cls.get_scheme()}{cls.endpoint_url}/{cls.data_bucket_name}/"
         else:
-            return f"{scheme}{cls.results_bucket_name}.{cls.endpoint_url}/"
+            return f"{cls.get_scheme()}{cls.data_bucket_name}.{cls.endpoint_url}/"
 
 
 class FeaturesConfig:
-    enable_custom_cloud_host = str_to_bool(os.environ.get("ENABLE_CUSTOM_CLOUD_HOST", "no"))
+    enable_custom_cloud_host = to_bool(os.environ.get("ENABLE_CUSTOM_CLOUD_HOST", "no"))
     "Allows using a custom host in manifest bucket urls"
 
     default_export_timeout = int(os.environ.get("DEFAULT_EXPORT_TIMEOUT", 60))
     "Timeout, in seconds, for annotations or dataset export waiting"
 
 
 class CoreConfig:
-    default_assignment_time = int(os.environ.get("DEFAULT_ASSIGNMENT_TIME", 300))
+    default_assignment_time = int(os.environ.get("DEFAULT_ASSIGNMENT_TIME", 1800))
 
 
 class HumanAppConfig: