From 2f98e25e13a4ef91b4b7ee06505fe3c55bd3e111 Mon Sep 17 00:00:00 2001 From: Christopher Ostrouchov Date: Tue, 24 Aug 2021 00:12:35 -0400 Subject: [PATCH] Fixing docker registry protocol bits (#112) * Fixing docker registry protocol * Black formatting --- conda-store-server/conda_store_server/api.py | 8 ++++ conda-store-server/conda_store_server/app.py | 6 ++- .../conda_store_server/build.py | 30 +++++++++---- conda-store-server/conda_store_server/orm.py | 5 ++- .../conda_store_server/server/app.py | 10 ++++- .../server/templates/build.html | 9 ++++ .../server/templates/home.html | 2 +- .../conda_store_server/server/utils.py | 4 ++ .../server/views/registry.py | 33 ++++++++------ .../conda_store_server/server/views/ui.py | 11 ++++- .../conda_store_server/worker/tasks.py | 26 ++++++----- examples/docker/docker-compose.yaml | 44 ++++++++----------- 12 files changed, 124 insertions(+), 64 deletions(-) diff --git a/conda-store-server/conda_store_server/api.py b/conda-store-server/conda_store_server/api.py index c322f5184..fc613f202 100644 --- a/conda-store-server/conda_store_server/api.py +++ b/conda-store-server/conda_store_server/api.py @@ -104,6 +104,14 @@ def get_build_lockfile(db, build_id): ) +def get_build_artifact_types(db, build_id): + return ( + db.query(orm.BuildArtifact.artifact_type) + .filter(orm.BuildArtifact.build_id == build_id) + .distinct() + ) + + def list_build_artifacts( db, limit: int = 25, diff --git a/conda-store-server/conda_store_server/app.py b/conda-store-server/conda_store_server/app.py index 124f80daf..484e34253 100644 --- a/conda-store-server/conda_store_server/app.py +++ b/conda-store-server/conda_store_server/app.py @@ -58,7 +58,11 @@ class CondaStore(LoggingConfigurable): ) build_artifacts_kept_on_deletion = List( - [orm.BuildArtifactType.LOGS, orm.BuildArtifactType.YAML], + [ + orm.BuildArtifactType.LOGS, + orm.BuildArtifactType.LOCKFILE, + orm.BuildArtifactType.YAML, + ], help="artifacts to keep on build deletion", config=True, ) diff --git a/conda-store-server/conda_store_server/build.py b/conda-store-server/conda_store_server/build.py index 19efecb61..96fda8986 100644 --- a/conda-store-server/conda_store_server/build.py +++ b/conda-store-server/conda_store_server/build.py @@ -87,6 +87,18 @@ def package_query(package): build.status = orm.BuildStatus.COMPLETED build.ended_on = datetime.datetime.utcnow() + # add records for lockfile and directory build artifacts + lockfile_build_artifact = orm.BuildArtifact( + build_id=build.id, artifact_type=orm.BuildArtifactType.LOCKFILE, key="" + ) + directory_build_artifact = orm.BuildArtifact( + build_id=build.id, + artifact_type=orm.BuildArtifactType.DIRECTORY, + key=build.build_path(conda_store.store_directory), + ) + conda_store.db.add(lockfile_build_artifact) + conda_store.db.add(directory_build_artifact) + environment = ( conda_store.db.query(orm.Environment) .filter(orm.Environment.name == build.specification.name) @@ -259,7 +271,7 @@ def build_conda_docker(conda_store, build): build.docker_blob_key(content_compressed_hash), content_compressed, content_type="application/gzip", - artifact_type=orm.BuildArtifactType.DOCKER, + artifact_type=orm.BuildArtifactType.DOCKER_BLOB, ) docker_layer = schema.DockerManifestLayer( @@ -286,28 +298,28 @@ def build_conda_docker(conda_store, build): build.docker_blob_key(docker_config_hash), docker_config_content, content_type="application/vnd.docker.container.image.v1+json", - artifact_type=orm.BuildArtifactType.DOCKER, + artifact_type=orm.BuildArtifactType.DOCKER_BLOB, ) + # docker likes to have a sha256 key version of the manifest this + # is sort of hack to avoid having to figure out which sha256 + # refers to which manifest. conda_store.storage.set( conda_store.db, build.id, - build.docker_manifest_key, + f"docker/manifest/sha256:{docker_manifest_hash}", docker_manifest_content, content_type="application/vnd.docker.distribution.manifest.v2+json", - artifact_type=orm.BuildArtifactType.DOCKER, + artifact_type=orm.BuildArtifactType.DOCKER_BLOB, ) - # docker likes to have a sha256 key version of the manifest this - # is sort of hack to avoid having to figure out which sha256 - # refers to which manifest. conda_store.storage.set( conda_store.db, build.id, - f"docker/manifest/{build.specification.name}/sha256:{docker_manifest_hash}", + build.docker_manifest_key, docker_manifest_content, content_type="application/vnd.docker.distribution.manifest.v2+json", - artifact_type=orm.BuildArtifactType.DOCKER, + artifact_type=orm.BuildArtifactType.DOCKER_MANIFEST, ) conda_store.log.info( diff --git a/conda-store-server/conda_store_server/orm.py b/conda-store-server/conda_store_server/orm.py index 9ff888194..446856262 100644 --- a/conda-store-server/conda_store_server/orm.py +++ b/conda-store-server/conda_store_server/orm.py @@ -28,10 +28,13 @@ class BuildArtifactType(enum.Enum): + DIRECTORY = "DIRECTORY" + LOCKFILE = "LOCKFILE" LOGS = "LOGS" YAML = "YAML" CONDA_PACK = "CONDA_PACK" - DOCKER = "DOCKER" + DOCKER_BLOB = "DOCKER_BLOB" + DOCKER_MANIFEST = "DOCKER_MANIFEST" class BuildStatus(enum.Enum): diff --git a/conda-store-server/conda_store_server/server/app.py b/conda-store-server/conda_store_server/server/app.py index 0f501f230..621a07069 100644 --- a/conda-store-server/conda_store_server/server/app.py +++ b/conda-store-server/conda_store_server/server/app.py @@ -45,6 +45,12 @@ class CondaStoreServer(Application): port = Integer(5000, help="port for conda-store server", config=True) + registry_external_url = Unicode( + "localhost:5000", + help='external hostname and port to access docker registry cannot contain "http://" or "https://"', + config=True, + ) + url_prefix = Unicode( "/", help="the prefix URL (subdirectory) for the entire application; " @@ -87,7 +93,8 @@ def start(self): app.register_blueprint(views.app_api, url_prefix=self.url_prefix) if self.enable_registry: - app.register_blueprint(views.app_registry, url_prefix=self.url_prefix) + # docker registry api specification does not support a url_prefix + app.register_blueprint(views.app_registry) if self.enable_ui: app.register_blueprint(views.app_ui, url_prefix=self.url_prefix) @@ -96,6 +103,7 @@ def start(self): app.register_blueprint(views.app_metrics, url_prefix=self.url_prefix) app.conda_store = CondaStore(parent=self, log=self.log) + app.server = self app.authentication = self.authentication_class(parent=self, log=self.log) @app.after_request diff --git a/conda-store-server/conda_store_server/server/templates/build.html b/conda-store-server/conda_store_server/server/templates/build.html index cec6f742a..3e9bfcc80 100644 --- a/conda-store-server/conda_store_server/server/templates/build.html +++ b/conda-store-server/conda_store_server/server/templates/build.html @@ -66,9 +66,18 @@

Conda Packages

Conda Environment Artifacts

{% endif %} diff --git a/conda-store-server/conda_store_server/server/templates/home.html b/conda-store-server/conda_store_server/server/templates/home.html index ab2138c86..71b601714 100644 --- a/conda-store-server/conda_store_server/server/templates/home.html +++ b/conda-store-server/conda_store_server/server/templates/home.html @@ -15,7 +15,7 @@
YAML Lockfile Archive - Docker + Docker {% endif %} diff --git a/conda-store-server/conda_store_server/server/utils.py b/conda-store-server/conda_store_server/server/utils.py index c9bfbaf8e..3026011a0 100644 --- a/conda-store-server/conda_store_server/server/utils.py +++ b/conda-store-server/conda_store_server/server/utils.py @@ -5,5 +5,9 @@ def get_conda_store(): return current_app.conda_store +def get_server(): + return current_app.server + + def get_auth(): return current_app.authentication diff --git a/conda-store-server/conda_store_server/server/views/registry.py b/conda-store-server/conda_store_server/server/views/registry.py index 2c53e6461..d5cad14a4 100644 --- a/conda-store-server/conda_store_server/server/views/registry.py +++ b/conda-store-server/conda_store_server/server/views/registry.py @@ -4,7 +4,7 @@ from flask import Blueprint, redirect, Response from conda_store_server.server.utils import get_conda_store -from conda_store_server import schema, api +from conda_store_server import schema, api, orm app_registry = Blueprint("registry", __name__) @@ -81,25 +81,30 @@ def get_docker_image_manifest(conda_store, image, tag, timeout=10 * 60): # check that namespace/environment_name exist environment = api.get_environment( - conda_store.db, environment_name, namespace=namespace + conda_store.db, namespace=namespace, name=environment_name ) if environment is None: return docker_error_message(schema.DockerRegistryError.NAME_UNKNOWN) if tag == "latest": - # waiting for image to be built by conda-store - start_time = time.time() - while environment.specification is None: - conda_store.db.refresh(environment) - time.sleep(10) - if time.time() - start_time > timeout: - return docker_error_message(schema.DockerRegistryError.MANIFEST_UNKNOWN) - - specification_sha256 = environment.specification.sha256 + build_key = environment.build.build_key else: - specification_sha256 = tag - - manifests_key = f"docker/manifest/{environment_name}/{specification_sha256}" + build_key = tag + + # waiting for image to be built by conda-store + start_time = time.time() + while orm.BuildArtifactType.DOCKER_MANIFEST not in { + _.artifact_type + for _ in api.get_build_artifact_types( + conda_store.db, environment.build.id + ).all() + }: + conda_store.db.refresh(environment) + time.sleep(10) + if time.time() - start_time > timeout: + return docker_error_message(schema.DockerRegistryError.MANIFEST_UNKNOWN) + + manifests_key = f"docker/manifest/{build_key}" return redirect(conda_store.storage.get_url(manifests_key)) diff --git a/conda-store-server/conda_store_server/server/views/ui.py b/conda-store-server/conda_store_server/server/views/ui.py index f7a8a6e51..65395bf72 100644 --- a/conda-store-server/conda_store_server/server/views/ui.py +++ b/conda-store-server/conda_store_server/server/views/ui.py @@ -10,7 +10,7 @@ import yaml from conda_store_server import api, schema -from conda_store_server.server.utils import get_conda_store, get_auth +from conda_store_server.server.utils import get_conda_store, get_auth, get_server from conda_store_server.server.auth import Permissions from conda_store_server.conda import conda_platform @@ -60,12 +60,14 @@ def ui_create_get_environment(): @app_ui.route("/", methods=["GET"]) def ui_list_environments(): conda_store = get_conda_store() + server = get_server() auth = get_auth() orm_environments = auth.filter_environments(api.list_environments(conda_store.db)) context = { "environments": orm_environments.all(), + "registry_external_url": server.registry_external_url, "entity": auth.authenticate_request(), } @@ -135,6 +137,7 @@ def ui_edit_environment(namespace, name): @app_ui.route("/build//", methods=["GET"]) def ui_get_build(build_id): conda_store = get_conda_store() + server = get_server() auth = get_auth() build = api.get_build(conda_store.db, build_id) @@ -150,8 +153,14 @@ def ui_get_build(build_id): require=True, ) + build_artifact_types = api.get_build_artifact_types(conda_store.db, build.id) + context = { "build": build, + "build_artifact_types": [ + _.artifact_type.value for _ in build_artifact_types.all() + ], + "registry_external_url": server.registry_external_url, "entity": auth.authenticate_request(), "platform": conda_platform(), "spec": yaml.dump(build.specification.spec), diff --git a/conda-store-server/conda_store_server/worker/tasks.py b/conda-store-server/conda_store_server/worker/tasks.py index 66bfa4b7e..38200e3de 100644 --- a/conda-store-server/conda_store_server/worker/tasks.py +++ b/conda-store-server/conda_store_server/worker/tasks.py @@ -5,7 +5,7 @@ import yaml from conda_store_server.worker.utils import create_worker -from conda_store_server import api, environment, utils +from conda_store_server import api, environment, utils, orm from conda_store_server.build import ( build_conda_environment, build_conda_env_export, @@ -101,14 +101,6 @@ def task_delete_build(build_id): conda_store = create_worker().conda_store build = api.get_build(conda_store.db, build_id) - conda_prefix = build.build_path(conda_store.store_directory) - - # be REALLY sure this is a directory within store directory - if conda_prefix.startswith(conda_store.store_directory) and os.path.isdir( - conda_prefix - ): - shutil.rmtree(conda_prefix) - conda_store.log.error("deleting artifacts") for build_artifact in api.list_build_artifacts( conda_store.db, @@ -116,8 +108,20 @@ def task_delete_build(build_id): build_id=build_id, excluded_artifact_types=conda_store.build_artifacts_kept_on_deletion, ): - conda_store.log.error(f"deleting {build_artifact.key}") - conda_store.storage.delete(conda_store.db, build_id, build_artifact.key) + if build_artifact.artifact_type == orm.BuildArtifactType.DIRECTORY: + # ignore key + conda_prefix = build.build_path(conda_store.store_directory) + # be REALLY sure this is a directory within store directory + if conda_prefix.startswith(conda_store.store_directory) and os.path.isdir( + conda_prefix + ): + shutil.rmtree(conda_prefix) + conda_store.db.delete(build_artifact) + elif build_artifact.artifact_type == orm.BuildArtifactType.LOCKFILE: + pass + else: + conda_store.log.error(f"deleting {build_artifact.key}") + conda_store.storage.delete(conda_store.db, build_id, build_artifact.key) conda_store.db.commit() conda_store.session_factory.remove() diff --git a/examples/docker/docker-compose.yaml b/examples/docker/docker-compose.yaml index bc45310d0..dfc8f43a3 100644 --- a/examples/docker/docker-compose.yaml +++ b/examples/docker/docker-compose.yaml @@ -1,40 +1,38 @@ version: "2" services: - conda-store-build: - image: quansight/conda-store-server:v0.2.5 + conda-store-worker: + build: conda-store-server depends_on: - "postgres" - "minio" volumes: - - ../environments:/opt/environments:ro - - ./data/conda-store:/data - command: ["wait-for-it", "postgres:5432", '--', 'conda-store-server', 'build', '-p', '/opt/environments', '-e', '/data/envs', '-s', '/data/store', '--uid', '1000', '--gid', '100', '--permissions', '775', '--storage-backend', 's3'] - environment: - CONDA_STORE_DB_URL: "postgresql+psycopg2://admin:password@postgres/conda-store" - CONDA_STORE_S3_ENDPOINT: minio:9000 - CONDA_STORE_S3_ACCESS_KEY: admin - CONDA_STORE_S3_SECRET_KEY: password + - ./tests/assets/environments:/opt/environments:ro + - ./tests/assets/conda_store_config.py:/opt/conda_store/conda_store_config.py:ro + platform: linux/amd64 + command: ["wait-for-it", "conda-store-server:5000", "-t", "60", '--', 'conda-store-worker', '--config', '/opt/conda_store/conda_store_config.py'] conda-store-server: - image: quansight/conda-store-server:v0.2.5 + build: conda-store-server depends_on: - "postgres" - "minio" - command: ["wait-for-it", "postgres:5432", '--', 'conda-store-server', 'server', '-s', '/data/store', '--port', '5000'] + volumes: + - ./tests/assets/conda_store_config.py:/opt/conda_store/conda_store_config.py:ro + platform: linux/amd64 + command: ["wait-for-it", "postgres:5432", '--', 'conda-store-server', '--config', '/opt/conda_store/conda_store_config.py'] ports: - "5000:5000" - environment: - CONDA_STORE_DB_URL: "postgresql+psycopg2://admin:password@postgres/conda-store" - CONDA_STORE_S3_ENDPOINT: minio:9000 - CONDA_STORE_S3_ACCESS_KEY: admin - CONDA_STORE_S3_SECRET_KEY: password - jupyterlab: - image: quansight/conda-store:v0.2.5 - command: /opt/conda/envs/conda-store/bin/jupyter lab --allow-root --ip=0.0.0.0 --NotebookApp.token='' + jupyterhub: + build: conda-store + user: "1000:1000" + volumes: + - ./tests/assets/jupyter_notebook_config.py:/etc/jupyter/jupyter_notebook_config.py + - ./tests/assets/jupyterhub_config.py:/opt/jupyterhub/jupyterhub_config.py:ro + command: ['/opt/conda/envs/conda-store/bin/jupyterhub', '--config', '/opt/jupyterhub/jupyterhub_config.py'] ports: - - "8888:8888" + - "8000:8000" minio: image: minio/minio:RELEASE.2020-11-10T21-02-24Z @@ -42,8 +40,6 @@ services: - "9000:9000" entrypoint: sh command: -c 'mkdir -p /data/conda-store && /usr/bin/minio server /data' - volumes: - - ./data/minio:/data environment: MINIO_ACCESS_KEY: admin MINIO_SECRET_KEY: password @@ -53,8 +49,6 @@ services: # TODO: need to properly fix this without this hack # reuse sqlalchemy connections command: postgres -c 'max_connections=200' - volumes: - - ./data/postgresql:/var/lib/postgresql/data ports: - 5432:5432 environment: