From a8164d24d888a6628ec256fc669df91f81bdc1f2 Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Sat, 27 Jan 2024 09:44:05 -0500 Subject: [PATCH] build(docker): clean up minio setup --- ci/schema/trino.sql | 10 +++++----- compose.yaml | 6 ++---- docker/minio/config.json | 2 +- ibis/backends/trino/tests/conftest.py | 20 +++----------------- 4 files changed, 11 insertions(+), 27 deletions(-) diff --git a/ci/schema/trino.sql b/ci/schema/trino.sql index d6281f42ea0d..18477e441595 100644 --- a/ci/schema/trino.sql +++ b/ci/schema/trino.sql @@ -11,7 +11,7 @@ CREATE TABLE hive.default.diamonds ( "y" DOUBLE, "z" DOUBLE ) WITH ( - external_location = 's3a://warehouse/diamonds', + external_location = 's3a://trino/diamonds', format = 'PARQUET' ); @@ -45,7 +45,7 @@ CREATE TABLE hive.default.astronauts ( "eva_hrs_mission" REAL, "total_eva_hrs" REAL ) WITH ( - external_location = 's3a://warehouse/astronauts', + external_location = 's3a://trino/astronauts', format = 'PARQUET' ); @@ -77,7 +77,7 @@ CREATE TABLE hive.default.batting ( "SF" BIGINT, "GIDP" BIGINT ) WITH ( - external_location = 's3a://warehouse/batting', + external_location = 's3a://trino/batting', format = 'PARQUET' ); @@ -93,7 +93,7 @@ CREATE TABLE hive.default.awards_players ( "tie" VARCHAR, "notes" VARCHAR ) WITH ( - external_location = 's3a://warehouse/awards-players', + external_location = 's3a://trino/awards-players', format = 'PARQUET' ); @@ -116,7 +116,7 @@ CREATE TABLE hive.default.functional_alltypes ( "year" INTEGER, "month" INTEGER ) WITH ( - external_location = 's3a://warehouse/functional-alltypes', + external_location = 's3a://trino/functional-alltypes', format = 'PARQUET' ); CREATE OR REPLACE VIEW memory.default.functional_alltypes AS diff --git a/compose.yaml b/compose.yaml index 2f588ac4d85e..369b713e828e 100644 --- a/compose.yaml +++ b/compose.yaml @@ -104,11 +104,10 @@ services: retries: 20 test: - CMD-SHELL - - mc ping --count 1 trino + - mc ready data && mc mb --ignore-existing data/trino networks: - trino volumes: - - minio:/data - $PWD/docker/minio/config.json:/.mc/config.json:ro hive-metastore: @@ -119,7 +118,7 @@ services: HIVE_METASTORE_JDBC_URL: jdbc:postgresql://hive-metastore-db:23456/metastore HIVE_METASTORE_USER: admin HIVE_METASTORE_PASSWORD: admin - HIVE_METASTORE_WAREHOUSE_DIR: s3://warehouse/ + HIVE_METASTORE_WAREHOUSE_DIR: s3://trino/ HIVE_METASTORE_USERS_IN_ADMIN_ROLE: "admin" S3_ENDPOINT: http://minio:9000 S3_ACCESS_KEY: accesskey @@ -569,6 +568,5 @@ volumes: mysql: oracle: postgres: - minio: exasol: impala: diff --git a/docker/minio/config.json b/docker/minio/config.json index df7c82ec0be8..f0064dd72aef 100644 --- a/docker/minio/config.json +++ b/docker/minio/config.json @@ -1,7 +1,7 @@ { "version": "10", "aliases": { - "trino": { + "data": { "url": "http://minio:9000", "accessKey": "accesskey", "secretKey": "secretkey", diff --git a/ibis/backends/trino/tests/conftest.py b/ibis/backends/trino/tests/conftest.py index 1f27b798761f..da6af3583daf 100644 --- a/ibis/backends/trino/tests/conftest.py +++ b/ibis/backends/trino/tests/conftest.py @@ -37,6 +37,7 @@ class TestConf(ServiceBackendTest): # for numeric and decimal service_name = "minio" + data_volume = "/bitnami/minio/data" returned_timestamp_unit = "s" supports_structs = True supports_map = True @@ -47,21 +48,6 @@ class TestConf(ServiceBackendTest): _tpch_query_schema = "hive.ibis_sf1" def preload(self): - # create buckets - subprocess.run( - [ - "docker", - "compose", - "exec", - "minio", - "mc", - "mb", - "--ignore-existing", - "trino/warehouse", - ], - check=True, - ) - # copy files to the minio host super().preload() @@ -74,11 +60,11 @@ def preload(self): "docker", "compose", "exec", - "minio", + self.service_name, "mc", "cp", f"{self.data_volume}/{path.name}", - f"trino/warehouse/{dirname}/{path.name}", + f"data/trino/{dirname}/{path.name}", ], check=True, )