From 0711471ccadaa7e7975f4c9ffbae41aaa51bf707 Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Tue, 13 Jul 2021 22:03:13 -0400 Subject: [PATCH 1/3] Adding internal and external url to minio client for connections This works around a longstanding issue with minio and how it checks that the presigned url hostname matches. --- .../conda_store_server/storage.py | 23 +++++++++++++------ docker-compose.yaml | 11 ++++----- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/conda-store-server/conda_store_server/storage.py b/conda-store-server/conda_store_server/storage.py index f6c940fdb..574908c97 100644 --- a/conda-store-server/conda_store_server/storage.py +++ b/conda-store-server/conda_store_server/storage.py @@ -19,27 +19,36 @@ def get_url(self, key): class S3Storage(Storage): def __init__(self): - self.endpoint = os.environ["CONDA_STORE_S3_ENDPOINT"] + self.internal_endpoint = os.environ["CONDA_STORE_S3_INTERNAL_ENDPOINT"] + self.external_endpoint = os.environ["CONDA_STORE_S3_EXTERNAL_ENDPOINT"] self.bucket_name = os.environ.get("CONDA_STORE_S3_BUCKET_NAME", "conda-store") - self.client = minio.Minio( - self.endpoint, + self.internal_client = minio.Minio( + self.internal_endpoint, os.environ["CONDA_STORE_S3_ACCESS_KEY"], os.environ["CONDA_STORE_S3_SECRET_KEY"], + region='us-east-1', + secure=False, + ) + self.external_client = minio.Minio( + self.external_endpoint, + os.environ["CONDA_STORE_S3_ACCESS_KEY"], + os.environ["CONDA_STORE_S3_SECRET_KEY"], + region='us-east-1', secure=False, ) self._check_bucket_exists() def _check_bucket_exists(self): - if not self.client.bucket_exists(self.bucket_name): + if not self.internal_client.bucket_exists(self.bucket_name): raise ValueError(f"S3 bucket={self.bucket_name} does not exist") def fset(self, key, filename, content_type="application/octet-stream"): - self.client.fput_object( + self.internal_client.fput_object( self.bucket_name, key, filename, content_type=content_type ) def set(self, key, value, content_type="application/octet-stream"): - self.client.put_object( + self.internal_client.put_object( self.bucket_name, key, io.BytesIO(value), @@ -48,7 +57,7 @@ def set(self, key, value, content_type="application/octet-stream"): ) def get_url(self, key): - return self.client.presigned_get_object(self.bucket_name, key) + return self.external_client.presigned_get_object(self.bucket_name, key) class LocalStorage(Storage): diff --git a/docker-compose.yaml b/docker-compose.yaml index 8479ea3b2..b0c4f7f80 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -8,12 +8,12 @@ services: - "minio" volumes: - ./tests/assets/environments:/opt/environments:ro - - ./data/conda-store:/data platform: linux/amd64 command: ["wait-for-it", "postgres:5432", '--', 'conda-store-server', 'build', '-p', '/opt/environments', '-e', '/data/envs', '-s', '/data/store', '--uid', '1000', '--gid', '100', '--permissions', '775', '--storage-backend', 's3'] environment: CONDA_STORE_DB_URL: "postgresql+psycopg2://admin:password@postgres/conda-store" - CONDA_STORE_S3_ENDPOINT: minio:9000 + CONDA_STORE_S3_INTERNAL_ENDPOINT: minio:9000 + CONDA_STORE_S3_EXTERNAL_ENDPOINT: localhost:9000 CONDA_STORE_S3_ACCESS_KEY: admin CONDA_STORE_S3_SECRET_KEY: password @@ -28,7 +28,8 @@ services: - "5000:5000" environment: CONDA_STORE_DB_URL: "postgresql+psycopg2://admin:password@postgres/conda-store" - CONDA_STORE_S3_ENDPOINT: minio:9000 + CONDA_STORE_S3_INTERNAL_ENDPOINT: minio:9000 + CONDA_STORE_S3_EXTERNAL_ENDPOINT: localhost:9000 CONDA_STORE_S3_ACCESS_KEY: admin CONDA_STORE_S3_SECRET_KEY: password @@ -44,8 +45,6 @@ services: - "9000:9000" entrypoint: sh command: -c 'mkdir -p /data/conda-store && /usr/bin/minio server /data' - volumes: - - ./data/minio:/data environment: MINIO_ACCESS_KEY: admin MINIO_SECRET_KEY: password @@ -55,8 +54,6 @@ services: # TODO: need to properly fix this without this hack # reuse sqlalchemy connections command: postgres -c 'max_connections=200' - volumes: - - ./data/postgresql:/var/lib/postgresql/data ports: - 5432:5432 environment: From 7d306c639ce616980d6d8014bd5281449952872c Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Tue, 13 Jul 2021 22:44:12 -0400 Subject: [PATCH 2/3] Additionally store docker manifest via sha256:<....>. Looks like newer docker checks that a sha256:... exists for a given manifest. --- conda-store-server/conda_store_server/build.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/conda-store-server/conda_store_server/build.py b/conda-store-server/conda_store_server/build.py index d57c67bd7..34bc891fc 100644 --- a/conda-store-server/conda_store_server/build.py +++ b/conda-store-server/conda_store_server/build.py @@ -309,6 +309,9 @@ def build_docker_image(conda_store, conda_prefix, build): docker_manifest.config = schema.DockerManifestConfig( size=len(docker_config_content), digest=f"sha256:{docker_config_hash}" ) + docker_manifest_content = docker_manifest.json().encode("utf-8") + docker_manifest_hash = hashlib.sha256(docker_manifest_content).hexdigest() + conda_store.storage.set( build.docker_blob_key(docker_config_hash), docker_config_content, @@ -317,9 +320,19 @@ def build_docker_image(conda_store, conda_prefix, build): conda_store.storage.set( build.docker_manifest_key, - docker_manifest.json().encode("utf-8"), + docker_manifest_content, + content_type="application/vnd.docker.distribution.manifest.v2+json", + ) + + # docker likes to have a sha256 key version of the manifest this + # is sort of hack to avoid having to figure out which sha256 + # refers to which manifest. + conda_store.storage.set( + f"docker/manifest/{build.specification.name}/sha256:{docker_manifest_hash}", + docker_manifest_content, content_type="application/vnd.docker.distribution.manifest.v2+json", ) + logger.info( f"built docker image: {image.name}:{image.tag} layers={len(image.layers)}" ) From 4fb78ae79eb829e3b4edc198a202c8a2fefbc462 Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Tue, 13 Jul 2021 22:54:15 -0400 Subject: [PATCH 3/3] Black formatting --- conda-store-server/conda_store_server/storage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda-store-server/conda_store_server/storage.py b/conda-store-server/conda_store_server/storage.py index 574908c97..42c2e2c12 100644 --- a/conda-store-server/conda_store_server/storage.py +++ b/conda-store-server/conda_store_server/storage.py @@ -26,14 +26,14 @@ def __init__(self): self.internal_endpoint, os.environ["CONDA_STORE_S3_ACCESS_KEY"], os.environ["CONDA_STORE_S3_SECRET_KEY"], - region='us-east-1', + region="us-east-1", secure=False, ) self.external_client = minio.Minio( self.external_endpoint, os.environ["CONDA_STORE_S3_ACCESS_KEY"], os.environ["CONDA_STORE_S3_SECRET_KEY"], - region='us-east-1', + region="us-east-1", secure=False, ) self._check_bucket_exists()