diff --git a/lib/dl_connector_bundle_chs3/docker-compose.yml b/lib/dl_connector_bundle_chs3/docker-compose.yml index b700d4ca0..bf07697ee 100644 --- a/lib/dl_connector_bundle_chs3/docker-compose.yml +++ b/lib/dl_connector_bundle_chs3/docker-compose.yml @@ -17,13 +17,9 @@ services: - 52611:9000 s3-storage: - build: - context: ../testenv-common/images - dockerfile: Dockerfile.s3-storage - command: bash /data/entrypoint.sh - environment: - S3BACKEND: "mem" - REMOTE_MANAGEMENT_DISABLE: 1 + extends: + file: ../testenv-common/docker-compose.common.yml + service: s3-storage ports: - 52620:8000 diff --git a/lib/dl_file_uploader_api_lib/dl_file_uploader_api_lib_tests/conftest.py b/lib/dl_file_uploader_api_lib/dl_file_uploader_api_lib_tests/conftest.py index 546f74c9d..82542d919 100644 --- a/lib/dl_file_uploader_api_lib/dl_file_uploader_api_lib_tests/conftest.py +++ b/lib/dl_file_uploader_api_lib/dl_file_uploader_api_lib_tests/conftest.py @@ -77,7 +77,6 @@ from dl_testing.utils import ( get_default_aiohttp_session, get_root_certificates, - wait_for_initdb, ) from dl_connector_bundle_chs3.chs3_base.core.settings import FileS3ConnectorSettings @@ -116,11 +115,6 @@ def loop(event_loop): return event_loop -@pytest.fixture(scope="session") -def initdb_ready(): - return wait_for_initdb(initdb_port=51408) - - @pytest.fixture(scope="session") def us_config(): return TestingUSConfig( @@ -250,14 +244,14 @@ async def s3_client(s3_settings) -> AsyncS3Client: @pytest.fixture(scope="function") -async def s3_tmp_bucket(initdb_ready, s3_client, app_settings) -> str: +async def s3_tmp_bucket(s3_client, app_settings) -> str: bucket_name = app_settings.S3_TMP_BUCKET_NAME await create_s3_bucket(s3_client, bucket_name, max_attempts=1) return bucket_name @pytest.fixture(scope="function") -async def s3_persistent_bucket(initdb_ready, s3_client, app_settings) -> str: +async def s3_persistent_bucket(s3_client, app_settings) -> str: bucket_name = app_settings.S3_PERSISTENT_BUCKET_NAME await create_s3_bucket(s3_client, bucket_name, max_attempts=1) return bucket_name diff --git a/lib/dl_file_uploader_api_lib/docker-compose.yml b/lib/dl_file_uploader_api_lib/docker-compose.yml index fcbf6bd12..0674e8803 100644 --- a/lib/dl_file_uploader_api_lib/docker-compose.yml +++ b/lib/dl_file_uploader_api_lib/docker-compose.yml @@ -11,25 +11,12 @@ services: - 51404:6379 s3-storage: - image: minio/minio:RELEASE.2024-12-18T13-15-44Z@sha256:1dce27c494a16bae114774f1cec295493f3613142713130c2d22dd5696be6ad3 - environment: - MINIO_ROOT_USER: accessKey1 - MINIO_ROOT_PASSWORD: verySecretKey1 - MINIO_DOMAIN: local - command: server --address ":8000" /export + extends: + file: ../testenv-common/docker-compose.common.yml + service: s3-storage ports: - 51420:8000 - init-db: - depends_on: - - redis - - s3-storage - build: - context: docker-compose - dockerfile: Dockerfile.init-db - ports: - - "51408:8000" - pg-us: extends: file: ../testenv-common/docker-compose.common.yml @@ -44,16 +31,7 @@ services: ports: - 51400:8083 - zookeeper-1: - # image: "zookeeper:3.4" - image: "zookeeper:3.4@sha256:d2b1ea8db9241d31daed9b1b90e22b62d9ffb14e29d355d0d0c8d0a87819d929" - environment: - ZOO_MY_ID: 1 - ZOO_SERVERS: server.1=0.0.0.0:2888:3888 - db-clickhouse: - depends_on: - - zookeeper-1 build: context: docker-compose dockerfile: Dockerfile.db-clickhouse diff --git a/lib/dl_file_uploader_api_lib/docker-compose/Dockerfile.init-db b/lib/dl_file_uploader_api_lib/docker-compose/Dockerfile.init-db deleted file mode 100644 index 166b6a2c2..000000000 --- a/lib/dl_file_uploader_api_lib/docker-compose/Dockerfile.init-db +++ /dev/null @@ -1,13 +0,0 @@ -FROM python@sha256:aa053a66b9defa644886d62c78eaea6a13a2ceb49a979a9ad904561d8d4ba8da -# ^ python:3.10-slim-bullseye - -ENV DEBIAN_FRONTEND=noninteractive -RUN apt update &&\ - apt install -y curl &&\ - apt clean &&\ - rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - -COPY init-db/s3 /s3 -COPY init-db/entrypoint.sh /entrypoint.sh - -ENTRYPOINT ["sh", "/entrypoint.sh"] diff --git a/lib/dl_file_uploader_api_lib/docker-compose/init-db/entrypoint.sh b/lib/dl_file_uploader_api_lib/docker-compose/init-db/entrypoint.sh deleted file mode 100755 index 2efbd242e..000000000 --- a/lib/dl_file_uploader_api_lib/docker-compose/init-db/entrypoint.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -set -eu - -BEACON_WS_ROOT=/beacon_ws_root/ - -chmod +x /s3/docker-entrypoint-initdb.d/* -run-parts -v --regex ".\\.sh$" /s3/docker-entrypoint-initdb.d - -echo "Starting beacon HTTP server" - -mkdir -p ${BEACON_WS_ROOT} -cd ${BEACON_WS_ROOT} - -# Tests will wait for this port before launch -python3 -m http.server 8000 diff --git a/lib/dl_file_uploader_api_lib/docker-compose/init-db/s3/docker-entrypoint-initdb.d/initialize.sh b/lib/dl_file_uploader_api_lib/docker-compose/init-db/s3/docker-entrypoint-initdb.d/initialize.sh deleted file mode 100644 index f2f19ee70..000000000 --- a/lib/dl_file_uploader_api_lib/docker-compose/init-db/s3/docker-entrypoint-initdb.d/initialize.sh +++ /dev/null @@ -1,9 +0,0 @@ -#! /bin/bash - -echo 'Waiting for S3 to initialize...' -until curl -s s3-storage:8000 > /dev/null -do - sleep 5 - echo 'Waiting for S3 to initialize...' -done -echo Connected to S3 diff --git a/lib/dl_file_uploader_worker_lib/dl_file_uploader_worker_lib_tests/conftest.py b/lib/dl_file_uploader_worker_lib/dl_file_uploader_worker_lib_tests/conftest.py index 6f86b7e41..c8cec1b90 100644 --- a/lib/dl_file_uploader_worker_lib/dl_file_uploader_worker_lib_tests/conftest.py +++ b/lib/dl_file_uploader_worker_lib/dl_file_uploader_worker_lib_tests/conftest.py @@ -68,10 +68,7 @@ create_s3_client, create_sync_s3_client, ) -from dl_testing.utils import ( - get_root_certificates, - wait_for_initdb, -) +from dl_testing.utils import get_root_certificates from dl_connector_bundle_chs3.chs3_base.core.settings import FileS3ConnectorSettings @@ -103,11 +100,6 @@ def loop(event_loop): return event_loop -@pytest.fixture(scope="session") -def initdb_ready(): - return wait_for_initdb(initdb_port=get_test_container_hostport("init-db", fallback_port=51508).port) - - @pytest.fixture(scope="function") def rci() -> RequestContextInfo: return RequestContextInfo(user_id="_the_tests_asyncapp_user_id_") @@ -321,14 +313,14 @@ def redis_model_manager(redis_cli, rci) -> RedisModelManager: @pytest.fixture(scope="function") -async def s3_tmp_bucket(initdb_ready, s3_client, file_uploader_worker_settings) -> str: +async def s3_tmp_bucket(s3_client, file_uploader_worker_settings) -> str: bucket_name = file_uploader_worker_settings.S3_TMP_BUCKET_NAME await create_s3_bucket(s3_client, bucket_name, max_attempts=1) return bucket_name @pytest.fixture(scope="function") -async def s3_persistent_bucket(initdb_ready, s3_client, file_uploader_worker_settings) -> str: +async def s3_persistent_bucket(s3_client, file_uploader_worker_settings) -> str: bucket_name = file_uploader_worker_settings.S3_PERSISTENT_BUCKET_NAME await create_s3_bucket(s3_client, bucket_name, max_attempts=1) return bucket_name diff --git a/lib/dl_file_uploader_worker_lib/docker-compose.yml b/lib/dl_file_uploader_worker_lib/docker-compose.yml index 0ca78e692..412d8868c 100644 --- a/lib/dl_file_uploader_worker_lib/docker-compose.yml +++ b/lib/dl_file_uploader_worker_lib/docker-compose.yml @@ -11,26 +11,12 @@ services: - 51504:6379 s3-storage: - build: - context: ../testenv-common/images - dockerfile: Dockerfile.s3-storage - command: bash /data/entrypoint.sh - environment: - S3BACKEND: "mem" - REMOTE_MANAGEMENT_DISABLE: 1 + extends: + file: ../testenv-common/docker-compose.common.yml + service: s3-storage ports: - 51520:8000 - init-db: - depends_on: - - redis - - s3-storage - build: - context: docker-compose - dockerfile: Dockerfile.init-db - ports: - - "51508:8000" - pg-us: extends: file: ../testenv-common/docker-compose.common.yml @@ -45,13 +31,6 @@ services: ports: - 51500:8083 - zookeeper-1: - # image: "zookeeper:3.4" - image: "zookeeper:3.4@sha256:d2b1ea8db9241d31daed9b1b90e22b62d9ffb14e29d355d0d0c8d0a87819d929" - environment: - ZOO_MY_ID: 1 - ZOO_SERVERS: server.1=0.0.0.0:2888:3888 - db-clickhouse: depends_on: - zookeeper-1 diff --git a/lib/dl_file_uploader_worker_lib/docker-compose/Dockerfile.init-db b/lib/dl_file_uploader_worker_lib/docker-compose/Dockerfile.init-db deleted file mode 100644 index 166b6a2c2..000000000 --- a/lib/dl_file_uploader_worker_lib/docker-compose/Dockerfile.init-db +++ /dev/null @@ -1,13 +0,0 @@ -FROM python@sha256:aa053a66b9defa644886d62c78eaea6a13a2ceb49a979a9ad904561d8d4ba8da -# ^ python:3.10-slim-bullseye - -ENV DEBIAN_FRONTEND=noninteractive -RUN apt update &&\ - apt install -y curl &&\ - apt clean &&\ - rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - -COPY init-db/s3 /s3 -COPY init-db/entrypoint.sh /entrypoint.sh - -ENTRYPOINT ["sh", "/entrypoint.sh"] diff --git a/lib/dl_file_uploader_worker_lib/docker-compose/init-db/entrypoint.sh b/lib/dl_file_uploader_worker_lib/docker-compose/init-db/entrypoint.sh deleted file mode 100755 index 2efbd242e..000000000 --- a/lib/dl_file_uploader_worker_lib/docker-compose/init-db/entrypoint.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -set -eu - -BEACON_WS_ROOT=/beacon_ws_root/ - -chmod +x /s3/docker-entrypoint-initdb.d/* -run-parts -v --regex ".\\.sh$" /s3/docker-entrypoint-initdb.d - -echo "Starting beacon HTTP server" - -mkdir -p ${BEACON_WS_ROOT} -cd ${BEACON_WS_ROOT} - -# Tests will wait for this port before launch -python3 -m http.server 8000 diff --git a/lib/dl_file_uploader_worker_lib/docker-compose/init-db/s3/docker-entrypoint-initdb.d/initialize.sh b/lib/dl_file_uploader_worker_lib/docker-compose/init-db/s3/docker-entrypoint-initdb.d/initialize.sh deleted file mode 100644 index f2f19ee70..000000000 --- a/lib/dl_file_uploader_worker_lib/docker-compose/init-db/s3/docker-entrypoint-initdb.d/initialize.sh +++ /dev/null @@ -1,9 +0,0 @@ -#! /bin/bash - -echo 'Waiting for S3 to initialize...' -until curl -s s3-storage:8000 > /dev/null -do - sleep 5 - echo 'Waiting for S3 to initialize...' -done -echo Connected to S3 diff --git a/lib/testenv-common/docker-compose.common.yml b/lib/testenv-common/docker-compose.common.yml index b63ffb473..7f93bc079 100644 --- a/lib/testenv-common/docker-compose.common.yml +++ b/lib/testenv-common/docker-compose.common.yml @@ -32,3 +32,11 @@ services: AUTH_POLICY: "required" MASTER_TOKEN: *c-us-master-token restart: on-failure + + s3-storage: + image: minio/minio:RELEASE.2024-12-18T13-15-44Z@sha256:1dce27c494a16bae114774f1cec295493f3613142713130c2d22dd5696be6ad3 + environment: + MINIO_ROOT_USER: accessKey1 + MINIO_ROOT_PASSWORD: verySecretKey1 + MINIO_DOMAIN: local + command: server --address ":8000" /export diff --git a/terrarium/bi_ci/bi_ci_tests/unit/resources/docker-compose.expected.yml b/terrarium/bi_ci/bi_ci_tests/unit/resources/docker-compose.expected.yml index 0a1d107c5..0cc97566a 100644 --- a/terrarium/bi_ci/bi_ci_tests/unit/resources/docker-compose.expected.yml +++ b/terrarium/bi_ci/bi_ci_tests/unit/resources/docker-compose.expected.yml @@ -51,10 +51,9 @@ services: ALLOW_EMPTY_PASSWORD: 'yes' image: some.cr.example/datalens/redis@sha256:3127620da977815556439a9dc347fff89432a79b6bb6e93a16f20ac4a34ce337 s3-storage: - build: - context: ../../lib/testenv-common/images - dockerfile: Dockerfile.s3-storage - command: bash /data/entrypoint.sh + extends: + file: ../../lib/testenv-common/docker-compose.common.yml + service: s3-storage us: build: context: ../../lib/testenv-common/images diff --git a/terrarium/bi_ci/bi_ci_tests/unit/resources/docker-compose.src.yml b/terrarium/bi_ci/bi_ci_tests/unit/resources/docker-compose.src.yml index 221518054..9b6773acb 100644 --- a/terrarium/bi_ci/bi_ci_tests/unit/resources/docker-compose.src.yml +++ b/terrarium/bi_ci/bi_ci_tests/unit/resources/docker-compose.src.yml @@ -117,9 +117,8 @@ services: - "50305:6379" s3-storage: - build: - context: ../../lib/testenv-common/images - dockerfile: Dockerfile.s3-storage - command: bash /data/entrypoint.sh + extends: + file: ../../lib/testenv-common/docker-compose.common.yml + service: s3-storage ports: - "51620:8000"