Fixing docker registry protocol bits (#112)

* Fixing docker registry protocol

* Black formatting

conda-store-server/conda_store_server/api.py

@@ -104,6 +104,14 @@ def get_build_lockfile(db, build_id):
     )
 
 
+def get_build_artifact_types(db, build_id):
+    return (
+        db.query(orm.BuildArtifact.artifact_type)
+        .filter(orm.BuildArtifact.build_id == build_id)
+        .distinct()
+    )
+
+
 def list_build_artifacts(
     db,
     limit: int = 25,

conda-store-server/conda_store_server/app.py

@@ -58,7 +58,11 @@ class CondaStore(LoggingConfigurable):
     )
 
     build_artifacts_kept_on_deletion = List(
-        [orm.BuildArtifactType.LOGS, orm.BuildArtifactType.YAML],
+        [
+            orm.BuildArtifactType.LOGS,
+            orm.BuildArtifactType.LOCKFILE,
+            orm.BuildArtifactType.YAML,
+        ],
         help="artifacts to keep on build deletion",
         config=True,
     )

conda-store-server/conda_store_server/build.py

@@ -87,6 +87,18 @@ def package_query(package):
     build.status = orm.BuildStatus.COMPLETED
     build.ended_on = datetime.datetime.utcnow()
 
+    # add records for lockfile and directory build artifacts
+    lockfile_build_artifact = orm.BuildArtifact(
+        build_id=build.id, artifact_type=orm.BuildArtifactType.LOCKFILE, key=""
+    )
+    directory_build_artifact = orm.BuildArtifact(
+        build_id=build.id,
+        artifact_type=orm.BuildArtifactType.DIRECTORY,
+        key=build.build_path(conda_store.store_directory),
+    )
+    conda_store.db.add(lockfile_build_artifact)
+    conda_store.db.add(directory_build_artifact)
+
     environment = (
         conda_store.db.query(orm.Environment)
         .filter(orm.Environment.name == build.specification.name)
@@ -259,7 +271,7 @@ def build_conda_docker(conda_store, build):
             build.docker_blob_key(content_compressed_hash),
             content_compressed,
             content_type="application/gzip",
-            artifact_type=orm.BuildArtifactType.DOCKER,
+            artifact_type=orm.BuildArtifactType.DOCKER_BLOB,
         )
 
         docker_layer = schema.DockerManifestLayer(
@@ -286,28 +298,28 @@ def build_conda_docker(conda_store, build):
         build.docker_blob_key(docker_config_hash),
         docker_config_content,
         content_type="application/vnd.docker.container.image.v1+json",
-        artifact_type=orm.BuildArtifactType.DOCKER,
+        artifact_type=orm.BuildArtifactType.DOCKER_BLOB,
     )
 
+    # docker likes to have a sha256 key version of the manifest this
+    # is sort of hack to avoid having to figure out which sha256
+    # refers to which manifest.
     conda_store.storage.set(
         conda_store.db,
         build.id,
-        build.docker_manifest_key,
+        f"docker/manifest/sha256:{docker_manifest_hash}",
         docker_manifest_content,
         content_type="application/vnd.docker.distribution.manifest.v2+json",
-        artifact_type=orm.BuildArtifactType.DOCKER,
+        artifact_type=orm.BuildArtifactType.DOCKER_BLOB,
     )
 
-    # docker likes to have a sha256 key version of the manifest this
-    # is sort of hack to avoid having to figure out which sha256
-    # refers to which manifest.
     conda_store.storage.set(
         conda_store.db,
         build.id,
-        f"docker/manifest/{build.specification.name}/sha256:{docker_manifest_hash}",
+        build.docker_manifest_key,
         docker_manifest_content,
         content_type="application/vnd.docker.distribution.manifest.v2+json",
-        artifact_type=orm.BuildArtifactType.DOCKER,
+        artifact_type=orm.BuildArtifactType.DOCKER_MANIFEST,
     )
 
     conda_store.log.info(

conda-store-server/conda_store_server/orm.py

@@ -28,10 +28,13 @@
 
 
 class BuildArtifactType(enum.Enum):
+    DIRECTORY = "DIRECTORY"
+    LOCKFILE = "LOCKFILE"
     LOGS = "LOGS"
     YAML = "YAML"
     CONDA_PACK = "CONDA_PACK"
-    DOCKER = "DOCKER"
+    DOCKER_BLOB = "DOCKER_BLOB"
+    DOCKER_MANIFEST = "DOCKER_MANIFEST"
 
 
 class BuildStatus(enum.Enum):

conda-store-server/conda_store_server/server/app.py

@@ -45,6 +45,12 @@ class CondaStoreServer(Application):
 
     port = Integer(5000, help="port for conda-store server", config=True)
 
+    registry_external_url = Unicode(
+        "localhost:5000",
+        help='external hostname and port to access docker registry cannot contain "http://" or "https://"',
+        config=True,
+    )
+
     url_prefix = Unicode(
         "/",
         help="the prefix URL (subdirectory) for the entire application; "
@@ -87,7 +93,8 @@ def start(self):
             app.register_blueprint(views.app_api, url_prefix=self.url_prefix)
 
         if self.enable_registry:
-            app.register_blueprint(views.app_registry, url_prefix=self.url_prefix)
+            # docker registry api specification does not support a url_prefix
+            app.register_blueprint(views.app_registry)
 
         if self.enable_ui:
             app.register_blueprint(views.app_ui, url_prefix=self.url_prefix)
@@ -96,6 +103,7 @@ def start(self):
             app.register_blueprint(views.app_metrics, url_prefix=self.url_prefix)
 
         app.conda_store = CondaStore(parent=self, log=self.log)
+        app.server = self
         app.authentication = self.authentication_class(parent=self, log=self.log)
 
         @app.after_request

conda-store-server/conda_store_server/server/templates/build.html

@@ -66,9 +66,18 @@ <h3 class="card-title">Conda Packages
         <h3 class="card-title">Conda Environment Artifacts</h3>
     </div>
     <ul class="list-group list-group-flush">
+        {% if 'YAML' in build_artifact_types %}
         <li class="list-group-item">YAML: <a href="{{ url_for('ui.api_get_build_yaml', build_id=build.id) }}">environment.yaml</a></li>
+        {% endif %}
+        {% if 'LOCKFILE' in build_artifact_types %}
         <li class="list-group-item">Lockfile: <a href="{{ url_for('ui.api_get_build_lockfile', build_id=build.id) }}">conda-{{ platform }}.lock</a></li>
+        {% endif %}
+        {% if 'CONDA_PACK' in build_artifact_types %}
         <li class="list-group-item">Archive: <a href="{{ url_for('ui.api_get_build_archive', build_id=build.id) }}">environment.tar.gz</a></li>
+        {% endif %}
+        {% if 'DOCKER_MANIFEST' in build_artifact_types %}
+        <li class="list-group-item">Docker: {{ registry_external_url }}/{{ build.namespace.name }}/{{ build.specification.name }}:{{ build.build_key }}</a></li>
+        {% endif %}
     </ul>
 </div>
 {% endif %}

conda-store-server/conda_store_server/server/templates/home.html

@@ -15,7 +15,7 @@ <h5 class="card-title">
         <a class="card-link" href="{{ url_for('ui.api_get_build_yaml', build_id=environment.build_id) }}"><ion-icon name="code-download"></ion-icon> YAML</a>
         <a class="card-link" href="{{ url_for('ui.api_get_build_lockfile', build_id=environment.build_id) }}"><ion-icon name="lock-closed-outline"></ion-icon> Lockfile</a>
         <a class="card-link" href="{{ url_for('ui.api_get_build_archive', build_id=environment.build_id) }}"><ion-icon name="archive-outline"></ion-icon> Archive</a>
-        <a class="card-link" onclick="setClipboard('{{ url_for('ui.ui_list_environments', _external=True) }}{{ environment.namespace.name }}/{{ environment.name }}:{{ environment.build.specification.sha256 }}')" role="button" data-toggle="popover" data-trigger="hover focus" data-content="Click to copy!"><ion-icon name="logo-docker"></ion-icon> Docker</a>
+        <a class="card-link" onclick="setClipboard('{{ registry_external_url }}/{{ environment.namespace.name }}/{{ environment.name }}:{{ environment.build.build_key }}')" role="button" data-toggle="popover" data-trigger="hover focus" data-content="Click to copy!"><ion-icon name="logo-docker"></ion-icon> Docker</a>
         {% endif %}
     </div>
 </div>

conda-store-server/conda_store_server/server/utils.py

@@ -5,5 +5,9 @@ def get_conda_store():
     return current_app.conda_store
 
 
+def get_server():
+    return current_app.server
+
+
 def get_auth():
     return current_app.authentication

conda-store-server/conda_store_server/server/views/registry.py

@@ -4,7 +4,7 @@
 from flask import Blueprint, redirect, Response
 
 from conda_store_server.server.utils import get_conda_store
-from conda_store_server import schema, api
+from conda_store_server import schema, api, orm
 
 
 app_registry = Blueprint("registry", __name__)
@@ -81,25 +81,30 @@ def get_docker_image_manifest(conda_store, image, tag, timeout=10 * 60):
 
     # check that namespace/environment_name exist
     environment = api.get_environment(
-        conda_store.db, environment_name, namespace=namespace
+        conda_store.db, namespace=namespace, name=environment_name
     )
     if environment is None:
         return docker_error_message(schema.DockerRegistryError.NAME_UNKNOWN)
 
     if tag == "latest":
-        # waiting for image to be built by conda-store
-        start_time = time.time()
-        while environment.specification is None:
-            conda_store.db.refresh(environment)
-            time.sleep(10)
-            if time.time() - start_time > timeout:
-                return docker_error_message(schema.DockerRegistryError.MANIFEST_UNKNOWN)
-
-        specification_sha256 = environment.specification.sha256
+        build_key = environment.build.build_key
     else:
-        specification_sha256 = tag
-
-    manifests_key = f"docker/manifest/{environment_name}/{specification_sha256}"
+        build_key = tag
+
+    # waiting for image to be built by conda-store
+    start_time = time.time()
+    while orm.BuildArtifactType.DOCKER_MANIFEST not in {
+        _.artifact_type
+        for _ in api.get_build_artifact_types(
+            conda_store.db, environment.build.id
+        ).all()
+    }:
+        conda_store.db.refresh(environment)
+        time.sleep(10)
+        if time.time() - start_time > timeout:
+            return docker_error_message(schema.DockerRegistryError.MANIFEST_UNKNOWN)
+
+    manifests_key = f"docker/manifest/{build_key}"
     return redirect(conda_store.storage.get_url(manifests_key))
 
 

conda-store-server/conda_store_server/server/views/ui.py

@@ -10,7 +10,7 @@
 import yaml
 
 from conda_store_server import api, schema
-from conda_store_server.server.utils import get_conda_store, get_auth
+from conda_store_server.server.utils import get_conda_store, get_auth, get_server
 from conda_store_server.server.auth import Permissions
 from conda_store_server.conda import conda_platform
 
@@ -60,12 +60,14 @@ def ui_create_get_environment():
 @app_ui.route("/", methods=["GET"])
 def ui_list_environments():
     conda_store = get_conda_store()
+    server = get_server()
     auth = get_auth()
 
     orm_environments = auth.filter_environments(api.list_environments(conda_store.db))
 
     context = {
         "environments": orm_environments.all(),
+        "registry_external_url": server.registry_external_url,
         "entity": auth.authenticate_request(),
     }
 
@@ -135,6 +137,7 @@ def ui_edit_environment(namespace, name):
 @app_ui.route("/build/<build_id>/", methods=["GET"])
 def ui_get_build(build_id):
     conda_store = get_conda_store()
+    server = get_server()
     auth = get_auth()
 
     build = api.get_build(conda_store.db, build_id)
@@ -150,8 +153,14 @@ def ui_get_build(build_id):
         require=True,
     )
 
+    build_artifact_types = api.get_build_artifact_types(conda_store.db, build.id)
+
     context = {
         "build": build,
+        "build_artifact_types": [
+            _.artifact_type.value for _ in build_artifact_types.all()
+        ],
+        "registry_external_url": server.registry_external_url,
         "entity": auth.authenticate_request(),
         "platform": conda_platform(),
         "spec": yaml.dump(build.specification.spec),

conda-store-server/conda_store_server/worker/tasks.py

@@ -5,7 +5,7 @@
 import yaml
 
 from conda_store_server.worker.utils import create_worker
-from conda_store_server import api, environment, utils
+from conda_store_server import api, environment, utils, orm
 from conda_store_server.build import (
     build_conda_environment,
     build_conda_env_export,
@@ -101,23 +101,27 @@ def task_delete_build(build_id):
     conda_store = create_worker().conda_store
     build = api.get_build(conda_store.db, build_id)
 
-    conda_prefix = build.build_path(conda_store.store_directory)
-
-    # be REALLY sure this is a directory within store directory
-    if conda_prefix.startswith(conda_store.store_directory) and os.path.isdir(
-        conda_prefix
-    ):
-        shutil.rmtree(conda_prefix)
-
     conda_store.log.error("deleting artifacts")
     for build_artifact in api.list_build_artifacts(
         conda_store.db,
         limit=None,
         build_id=build_id,
         excluded_artifact_types=conda_store.build_artifacts_kept_on_deletion,
     ):
-        conda_store.log.error(f"deleting {build_artifact.key}")
-        conda_store.storage.delete(conda_store.db, build_id, build_artifact.key)
+        if build_artifact.artifact_type == orm.BuildArtifactType.DIRECTORY:
+            # ignore key
+            conda_prefix = build.build_path(conda_store.store_directory)
+            # be REALLY sure this is a directory within store directory
+            if conda_prefix.startswith(conda_store.store_directory) and os.path.isdir(
+                conda_prefix
+            ):
+                shutil.rmtree(conda_prefix)
+                conda_store.db.delete(build_artifact)
+        elif build_artifact.artifact_type == orm.BuildArtifactType.LOCKFILE:
+            pass
+        else:
+            conda_store.log.error(f"deleting {build_artifact.key}")
+            conda_store.storage.delete(conda_store.db, build_id, build_artifact.key)
 
     conda_store.db.commit()
     conda_store.session_factory.remove()