From 690e8f8aeb8453efc062500ccbb7eaea16ca0e85 Mon Sep 17 00:00:00 2001 From: Christopher Ostrouchov Date: Tue, 24 Aug 2021 16:12:58 -0400 Subject: [PATCH] Only show artifacts when they are complete building (#113) * Fixing docker registry protocol * Black formatting * Trying to use hybrid atributes * Fixing regex for environment names allowing numbers * Black and flake8 formatting --- conda-store-server/conda_store_server/orm.py | 40 ++++++++++++++++++- .../conda_store_server/server/auth.py | 6 ++- .../server/templates/build.html | 8 ++-- .../server/templates/home.html | 8 +++- .../server/views/registry.py | 21 ++++++---- .../conda_store_server/server/views/ui.py | 5 --- 6 files changed, 68 insertions(+), 20 deletions(-) diff --git a/conda-store-server/conda_store_server/orm.py b/conda-store-server/conda_store_server/orm.py index 446856262..1eb2ce9f7 100644 --- a/conda-store-server/conda_store_server/orm.py +++ b/conda-store-server/conda_store_server/orm.py @@ -17,6 +17,7 @@ ) from sqlalchemy.orm import sessionmaker, relationship, scoped_session from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy import create_engine from conda_store_server import utils @@ -109,6 +110,8 @@ class Build(Base): ended_on = Column(DateTime, default=None) deleted_on = Column(DateTime, default=None) + build_artifacts = relationship("BuildArtifact", back_populates="build") + def build_path(self, store_directory): store_path = os.path.abspath(store_directory) return os.path.join(store_path, self.build_key) @@ -131,6 +134,13 @@ def build_key(self): datetime_format = "%Y%m%d-%H%M%S-%f" return f"{self.specification.sha256}-{self.scheduled_on.strftime(datetime_format)}-{self.id}-{self.specification.name}" + @staticmethod + def parse_build_key(key): + parts = key.split("-") + if len(parts) < 5: + return None + return int(parts[4]) # build_id + @property def log_key(self): return f"logs/{self.build_key}.log" @@ -150,6 +160,34 @@ def docker_manifest_key(self): def docker_blob_key(self, blob_hash): return f"docker/blobs/{blob_hash}" + @hybrid_property + def has_lockfile(self): + return any( + artifact.artifact_type == BuildArtifactType.LOCKFILE + for artifact in self.build_artifacts + ) + + @hybrid_property + def has_yaml(self): + return any( + artifact.artifact_type == BuildArtifactType.YAML + for artifact in self.build_artifacts + ) + + @hybrid_property + def has_conda_pack(self): + return any( + artifact.artifact_type == BuildArtifactType.CONDA_PACK + for artifact in self.build_artifacts + ) + + @hybrid_property + def has_docker_manifest(self): + return any( + artifact.artifact_type == BuildArtifactType.DOCKER_MANIFEST + for artifact in self.build_artifacts + ) + class BuildArtifact(Base): """Artifacts of a given build""" @@ -159,7 +197,7 @@ class BuildArtifact(Base): id = Column(Integer, primary_key=True) build_id = Column(Integer, ForeignKey("build.id")) - build = relationship(Build) + build = relationship(Build, back_populates="build_artifacts") artifact_type = Column(Enum(BuildArtifactType), nullable=False) diff --git a/conda-store-server/conda_store_server/server/auth.py b/conda-store-server/conda_store_server/server/auth.py index 7fe476c66..2da490500 100644 --- a/conda-store-server/conda_store_server/server/auth.py +++ b/conda-store-server/conda_store_server/server/auth.py @@ -22,7 +22,9 @@ from conda_store_server import schema, orm -ARN_ALLOWED_REGEX = re.compile(r"^([A-Za-z\_\-\*]+)/([A-Za-z\_\-\*]+)$") +ARN_ALLOWED_REGEX = re.compile( + r"^([A-Za-z0-9|<>=\.\_\-\*]+)/([A-Za-z0-9|<>=\.\_\-\*]+)$" +) class Permissions(enum.Enum): @@ -104,7 +106,7 @@ def compile_arn_regex(arn): if not ARN_ALLOWED_REGEX.match(arn): raise ValueError(f"invalid arn={arn}") - regex_arn = "^" + re.sub(r"\*", r"[A-Za-z_\-]*", arn) + "$" + regex_arn = "^" + re.sub(r"\*", r"[A-Za-z0-9_\-\.|<>=]*", arn) + "$" return re.compile(regex_arn) @staticmethod diff --git a/conda-store-server/conda_store_server/server/templates/build.html b/conda-store-server/conda_store_server/server/templates/build.html index 3e9bfcc80..626faa0e1 100644 --- a/conda-store-server/conda_store_server/server/templates/build.html +++ b/conda-store-server/conda_store_server/server/templates/build.html @@ -66,16 +66,16 @@

Conda Packages

Conda Environment Artifacts

diff --git a/conda-store-server/conda_store_server/server/templates/home.html b/conda-store-server/conda_store_server/server/templates/home.html index 71b601714..9c68bb523 100644 --- a/conda-store-server/conda_store_server/server/templates/home.html +++ b/conda-store-server/conda_store_server/server/templates/home.html @@ -11,10 +11,16 @@
{{ environment.namespace.name }}/{{ environment.name }} {{ (environment.build.size or 0) | filesizeformat(true) }}
- {% if environment.build.status.value == 'COMPLETED' %} + {% if environment.build.has_yaml %} YAML + {% endif %} + {% if environment.build.has_lockfile %} Lockfile + {% endif %} + {% if environment.build.has_conda_pack %} Archive + {% endif %} + {% if environment.build.has_docker_manifest %} Docker {% endif %} diff --git a/conda-store-server/conda_store_server/server/views/registry.py b/conda-store-server/conda_store_server/server/views/registry.py index d5cad14a4..636542c02 100644 --- a/conda-store-server/conda_store_server/server/views/registry.py +++ b/conda-store-server/conda_store_server/server/views/registry.py @@ -88,18 +88,25 @@ def get_docker_image_manifest(conda_store, image, tag, timeout=10 * 60): if tag == "latest": build_key = environment.build.build_key + elif tag.startswith("sha256:"): + # looking for sha256 of docker manifest + manifests_key = f"docker/manifest/{tag}" + return redirect(conda_store.storage.get_url(manifests_key)) else: build_key = tag + build_id = orm.Build.parse_build_key(build_key) + if build_id is None: + return docker_error_message(schema.DockerRegistryError.MANIFEST_UNKNOWN) + + build = api.get_build(conda_store.db, build_id) + if build is None: + return docker_error_message(schema.DockerRegistryError.MANIFEST_UNKNOWN) + # waiting for image to be built by conda-store start_time = time.time() - while orm.BuildArtifactType.DOCKER_MANIFEST not in { - _.artifact_type - for _ in api.get_build_artifact_types( - conda_store.db, environment.build.id - ).all() - }: - conda_store.db.refresh(environment) + while not build.has_docker_manifest: + conda_store.db.refresh(build) time.sleep(10) if time.time() - start_time > timeout: return docker_error_message(schema.DockerRegistryError.MANIFEST_UNKNOWN) diff --git a/conda-store-server/conda_store_server/server/views/ui.py b/conda-store-server/conda_store_server/server/views/ui.py index 65395bf72..0c634aeb7 100644 --- a/conda-store-server/conda_store_server/server/views/ui.py +++ b/conda-store-server/conda_store_server/server/views/ui.py @@ -153,13 +153,8 @@ def ui_get_build(build_id): require=True, ) - build_artifact_types = api.get_build_artifact_types(conda_store.db, build.id) - context = { "build": build, - "build_artifact_types": [ - _.artifact_type.value for _ in build_artifact_types.all() - ], "registry_external_url": server.registry_external_url, "entity": auth.authenticate_request(), "platform": conda_platform(),