chore: BI-0 add s3-storage with minio to common docker-compose, use it everywhere

lib/dl_connector_bundle_chs3/docker-compose.yml

@@ -17,13 +17,9 @@ services:
       - 52611:9000
 
   s3-storage:
-    build:
-      context: ../testenv-common/images
-      dockerfile: Dockerfile.s3-storage
-    command: bash /data/entrypoint.sh
-    environment:
-      S3BACKEND: "mem"
-      REMOTE_MANAGEMENT_DISABLE: 1
+    extends:
+      file: ../testenv-common/docker-compose.common.yml
+      service: s3-storage
     ports:
       - 52620:8000
 

lib/dl_file_uploader_api_lib/dl_file_uploader_api_lib_tests/conftest.py

@@ -77,7 +77,6 @@
 from dl_testing.utils import (
     get_default_aiohttp_session,
     get_root_certificates,
-    wait_for_initdb,
 )
 
 from dl_connector_bundle_chs3.chs3_base.core.settings import FileS3ConnectorSettings
@@ -116,11 +115,6 @@ def loop(event_loop):
     return event_loop
 
 
-@pytest.fixture(scope="session")
-def initdb_ready():
-    return wait_for_initdb(initdb_port=51408)
-
-
 @pytest.fixture(scope="session")
 def us_config():
     return TestingUSConfig(
@@ -250,14 +244,14 @@ async def s3_client(s3_settings) -> AsyncS3Client:
 
 
 @pytest.fixture(scope="function")
-async def s3_tmp_bucket(initdb_ready, s3_client, app_settings) -> str:
+async def s3_tmp_bucket(s3_client, app_settings) -> str:
     bucket_name = app_settings.S3_TMP_BUCKET_NAME
     await create_s3_bucket(s3_client, bucket_name, max_attempts=1)
     return bucket_name
 
 
 @pytest.fixture(scope="function")
-async def s3_persistent_bucket(initdb_ready, s3_client, app_settings) -> str:
+async def s3_persistent_bucket(s3_client, app_settings) -> str:
     bucket_name = app_settings.S3_PERSISTENT_BUCKET_NAME
     await create_s3_bucket(s3_client, bucket_name, max_attempts=1)
     return bucket_name

lib/dl_file_uploader_api_lib/docker-compose.yml

@@ -11,25 +11,12 @@ services:
       - 51404:6379
 
   s3-storage:
-    image: minio/minio:RELEASE.2024-12-18T13-15-44Z@sha256:1dce27c494a16bae114774f1cec295493f3613142713130c2d22dd5696be6ad3
-    environment:
-      MINIO_ROOT_USER: accessKey1
-      MINIO_ROOT_PASSWORD: verySecretKey1
-      MINIO_DOMAIN: local
-    command: server --address ":8000" /export
+    extends:
+      file: ../testenv-common/docker-compose.common.yml
+      service: s3-storage
     ports:
       - 51420:8000
 
-  init-db:
-    depends_on:
-      - redis
-      - s3-storage
-    build:
-      context: docker-compose
-      dockerfile: Dockerfile.init-db
-    ports:
-     - "51408:8000"
-
   pg-us:
     extends:
       file: ../testenv-common/docker-compose.common.yml
@@ -44,16 +31,7 @@ services:
     ports:
       - 51400:8083
 
-  zookeeper-1:
-    # image: "zookeeper:3.4"
-    image: "zookeeper:3.4@sha256:d2b1ea8db9241d31daed9b1b90e22b62d9ffb14e29d355d0d0c8d0a87819d929"
-    environment:
-      ZOO_MY_ID: 1
-      ZOO_SERVERS: server.1=0.0.0.0:2888:3888
-
   db-clickhouse:
-    depends_on:
-      - zookeeper-1
     build:
       context: docker-compose
       dockerfile: Dockerfile.db-clickhouse

lib/dl_file_uploader_worker_lib/dl_file_uploader_worker_lib_tests/conftest.py

@@ -68,10 +68,7 @@
     create_s3_client,
     create_sync_s3_client,
 )
-from dl_testing.utils import (
-    get_root_certificates,
-    wait_for_initdb,
-)
+from dl_testing.utils import get_root_certificates
 
 from dl_connector_bundle_chs3.chs3_base.core.settings import FileS3ConnectorSettings
 
@@ -103,11 +100,6 @@ def loop(event_loop):
     return event_loop
 
 
-@pytest.fixture(scope="session")
-def initdb_ready():
-    return wait_for_initdb(initdb_port=get_test_container_hostport("init-db", fallback_port=51508).port)
-
-
 @pytest.fixture(scope="function")
 def rci() -> RequestContextInfo:
     return RequestContextInfo(user_id="_the_tests_asyncapp_user_id_")
@@ -321,14 +313,14 @@ def redis_model_manager(redis_cli, rci) -> RedisModelManager:
 
 
 @pytest.fixture(scope="function")
-async def s3_tmp_bucket(initdb_ready, s3_client, file_uploader_worker_settings) -> str:
+async def s3_tmp_bucket(s3_client, file_uploader_worker_settings) -> str:
     bucket_name = file_uploader_worker_settings.S3_TMP_BUCKET_NAME
     await create_s3_bucket(s3_client, bucket_name, max_attempts=1)
     return bucket_name
 
 
 @pytest.fixture(scope="function")
-async def s3_persistent_bucket(initdb_ready, s3_client, file_uploader_worker_settings) -> str:
+async def s3_persistent_bucket(s3_client, file_uploader_worker_settings) -> str:
     bucket_name = file_uploader_worker_settings.S3_PERSISTENT_BUCKET_NAME
     await create_s3_bucket(s3_client, bucket_name, max_attempts=1)
     return bucket_name

lib/dl_file_uploader_worker_lib/docker-compose.yml

@@ -11,26 +11,12 @@ services:
       - 51504:6379
 
   s3-storage:
-    build:
-      context: ../testenv-common/images
-      dockerfile: Dockerfile.s3-storage
-    command: bash /data/entrypoint.sh
-    environment:
-      S3BACKEND: "mem"
-      REMOTE_MANAGEMENT_DISABLE: 1
+    extends:
+      file: ../testenv-common/docker-compose.common.yml
+      service: s3-storage
     ports:
       - 51520:8000
 
-  init-db:
-    depends_on:
-      - redis
-      - s3-storage
-    build:
-      context: docker-compose
-      dockerfile: Dockerfile.init-db
-    ports:
-     - "51508:8000"
-
   pg-us:
     extends:
       file: ../testenv-common/docker-compose.common.yml
@@ -45,16 +31,7 @@ services:
     ports:
       - 51500:8083
 
-  zookeeper-1:
-    # image: "zookeeper:3.4"
-    image: "zookeeper:3.4@sha256:d2b1ea8db9241d31daed9b1b90e22b62d9ffb14e29d355d0d0c8d0a87819d929"
-    environment:
-      ZOO_MY_ID: 1
-      ZOO_SERVERS: server.1=0.0.0.0:2888:3888
-
   db-clickhouse:
-    depends_on:
-      - zookeeper-1
     build:
       context: docker-compose
       dockerfile: Dockerfile.db-clickhouse

lib/testenv-common/docker-compose.common.yml

@@ -32,3 +32,11 @@ services:
       AUTH_POLICY: "required"
       MASTER_TOKEN: *c-us-master-token
     restart: on-failure
+
+  s3-storage:
+    image: minio/minio:RELEASE.2024-12-18T13-15-44Z@sha256:1dce27c494a16bae114774f1cec295493f3613142713130c2d22dd5696be6ad3
+    environment:
+      MINIO_ROOT_USER: accessKey1
+      MINIO_ROOT_PASSWORD: verySecretKey1
+      MINIO_DOMAIN: local
+    command: server --address ":8000" /export

terrarium/bi_ci/bi_ci_tests/unit/resources/docker-compose.expected.yml

@@ -51,10 +51,9 @@ services:
       ALLOW_EMPTY_PASSWORD: 'yes'
     image: some.cr.example/datalens/redis@sha256:3127620da977815556439a9dc347fff89432a79b6bb6e93a16f20ac4a34ce337
   s3-storage:
-    build:
-      context: ../../lib/testenv-common/images
-      dockerfile: Dockerfile.s3-storage
-    command: bash /data/entrypoint.sh
+    extends:
+      file: ../../lib/testenv-common/docker-compose.common.yml
+      service: s3-storage
   us:
     build:
       context: ../../lib/testenv-common/images

terrarium/bi_ci/bi_ci_tests/unit/resources/docker-compose.src.yml

@@ -117,9 +117,8 @@ services:
      - "50305:6379"
 
   s3-storage:
-    build:
-      context: ../../lib/testenv-common/images
-      dockerfile: Dockerfile.s3-storage
-    command: bash /data/entrypoint.sh
+    extends:
+      file: ../../lib/testenv-common/docker-compose.common.yml
+      service: s3-storage
     ports:
       - "51620:8000"