Add support for external url for s3 using minio (#81)

* Adding internal and external url to minio client for connections

This works around a longstanding issue with minio and how it checks
that the presigned url hostname matches.

* Additionally store docker manifest via sha256:<....>.

Looks like newer docker checks that a sha256:... exists for a given
manifest.

* Black formatting

conda-store-server/conda_store_server/build.py

@@ -309,6 +309,9 @@ def build_docker_image(conda_store, conda_prefix, build):
     docker_manifest.config = schema.DockerManifestConfig(
         size=len(docker_config_content), digest=f"sha256:{docker_config_hash}"
     )
+    docker_manifest_content = docker_manifest.json().encode("utf-8")
+    docker_manifest_hash = hashlib.sha256(docker_manifest_content).hexdigest()
+
     conda_store.storage.set(
         build.docker_blob_key(docker_config_hash),
         docker_config_content,
@@ -317,9 +320,19 @@ def build_docker_image(conda_store, conda_prefix, build):
 
     conda_store.storage.set(
         build.docker_manifest_key,
-        docker_manifest.json().encode("utf-8"),
+        docker_manifest_content,
+        content_type="application/vnd.docker.distribution.manifest.v2+json",
+    )
+
+    # docker likes to have a sha256 key version of the manifest this
+    # is sort of hack to avoid having to figure out which sha256
+    # refers to which manifest.
+    conda_store.storage.set(
+        f"docker/manifest/{build.specification.name}/sha256:{docker_manifest_hash}",
+        docker_manifest_content,
         content_type="application/vnd.docker.distribution.manifest.v2+json",
     )
+
     logger.info(
         f"built docker image: {image.name}:{image.tag} layers={len(image.layers)}"
     )

conda-store-server/conda_store_server/storage.py

@@ -19,27 +19,36 @@ def get_url(self, key):
 
 class S3Storage(Storage):
     def __init__(self):
-        self.endpoint = os.environ["CONDA_STORE_S3_ENDPOINT"]
+        self.internal_endpoint = os.environ["CONDA_STORE_S3_INTERNAL_ENDPOINT"]
+        self.external_endpoint = os.environ["CONDA_STORE_S3_EXTERNAL_ENDPOINT"]
         self.bucket_name = os.environ.get("CONDA_STORE_S3_BUCKET_NAME", "conda-store")
-        self.client = minio.Minio(
-            self.endpoint,
+        self.internal_client = minio.Minio(
+            self.internal_endpoint,
             os.environ["CONDA_STORE_S3_ACCESS_KEY"],
             os.environ["CONDA_STORE_S3_SECRET_KEY"],
+            region="us-east-1",
+            secure=False,
+        )
+        self.external_client = minio.Minio(
+            self.external_endpoint,
+            os.environ["CONDA_STORE_S3_ACCESS_KEY"],
+            os.environ["CONDA_STORE_S3_SECRET_KEY"],
+            region="us-east-1",
             secure=False,
         )
         self._check_bucket_exists()
 
     def _check_bucket_exists(self):
-        if not self.client.bucket_exists(self.bucket_name):
+        if not self.internal_client.bucket_exists(self.bucket_name):
             raise ValueError(f"S3 bucket={self.bucket_name} does not exist")
 
     def fset(self, key, filename, content_type="application/octet-stream"):
-        self.client.fput_object(
+        self.internal_client.fput_object(
             self.bucket_name, key, filename, content_type=content_type
         )
 
     def set(self, key, value, content_type="application/octet-stream"):
-        self.client.put_object(
+        self.internal_client.put_object(
             self.bucket_name,
             key,
             io.BytesIO(value),
@@ -48,7 +57,7 @@ def set(self, key, value, content_type="application/octet-stream"):
         )
 
     def get_url(self, key):
-        return self.client.presigned_get_object(self.bucket_name, key)
+        return self.external_client.presigned_get_object(self.bucket_name, key)
 
 
 class LocalStorage(Storage):

docker-compose.yaml

@@ -8,12 +8,12 @@ services:
       - "minio"
     volumes:
       - ./tests/assets/environments:/opt/environments:ro
-      - ./data/conda-store:/data
     platform: linux/amd64
     command: ["wait-for-it", "postgres:5432", '--', 'conda-store-server', 'build', '-p', '/opt/environments', '-e', '/data/envs', '-s', '/data/store', '--uid', '1000', '--gid', '100', '--permissions', '775', '--storage-backend', 's3']
     environment:
       CONDA_STORE_DB_URL: "postgresql+psycopg2://admin:password@postgres/conda-store"
-      CONDA_STORE_S3_ENDPOINT: minio:9000
+      CONDA_STORE_S3_INTERNAL_ENDPOINT: minio:9000
+      CONDA_STORE_S3_EXTERNAL_ENDPOINT: localhost:9000
       CONDA_STORE_S3_ACCESS_KEY: admin
       CONDA_STORE_S3_SECRET_KEY: password
 
@@ -28,7 +28,8 @@ services:
       - "5000:5000"
     environment:
       CONDA_STORE_DB_URL: "postgresql+psycopg2://admin:password@postgres/conda-store"
-      CONDA_STORE_S3_ENDPOINT: minio:9000
+      CONDA_STORE_S3_INTERNAL_ENDPOINT: minio:9000
+      CONDA_STORE_S3_EXTERNAL_ENDPOINT: localhost:9000
       CONDA_STORE_S3_ACCESS_KEY: admin
       CONDA_STORE_S3_SECRET_KEY: password
 
@@ -44,8 +45,6 @@ services:
       - "9000:9000"
     entrypoint: sh
     command: -c 'mkdir -p /data/conda-store && /usr/bin/minio server /data'
-    volumes:
-      - ./data/minio:/data
     environment:
       MINIO_ACCESS_KEY: admin
       MINIO_SECRET_KEY: password
@@ -55,8 +54,6 @@ services:
     # TODO: need to properly fix this without this hack
     # reuse sqlalchemy connections
     command: postgres -c 'max_connections=200'
-    volumes:
-      - ./data/postgresql:/var/lib/postgresql/data
     ports:
       - 5432:5432
     environment: