diff --git a/conda-store-server/conda_store_server/api.py b/conda-store-server/conda_store_server/api.py index 16d004a39..c322f5184 100644 --- a/conda-store-server/conda_store_server/api.py +++ b/conda-store-server/conda_store_server/api.py @@ -1,3 +1,5 @@ +from typing import List + from sqlalchemy import func from conda_store_server import orm @@ -102,12 +104,22 @@ def get_build_lockfile(db, build_id): ) -def list_build_artifacts(db, limit: int = 25, build_id: int = None, key: str = None): +def list_build_artifacts( + db, + limit: int = 25, + build_id: int = None, + key: str = None, + excluded_artifact_types: List[orm.BuildArtifactType] = None, +): filters = [] if build_id: filters.append(orm.BuildArtifact.build_id == build_id) if key: filters.append(orm.BuildArtifact.key == key) + if excluded_artifact_types: + filters.append( + func.not_(orm.BuildArtifact.artifact_type.in_(excluded_artifact_types)) + ) return db.query(orm.BuildArtifact).filter(*filters).limit(limit).all() diff --git a/conda-store-server/conda_store_server/app.py b/conda-store-server/conda_store_server/app.py index 650a80b20..037c6cc2d 100644 --- a/conda-store-server/conda_store_server/app.py +++ b/conda-store-server/conda_store_server/app.py @@ -1,4 +1,5 @@ import os +import datetime from celery import Celery from traitlets import Type, Unicode, Integer, List, default @@ -56,6 +57,12 @@ class CondaStore(LoggingConfigurable): config=True, ) + build_artifacts_kept_on_deletion = List( + [orm.BuildArtifactType.LOGS, orm.BuildArtifactType.YAML], + help="artifacts to keep on build deletion", + config=True, + ) + @default("celery_broker_url") def _default_celery_broker_url(self): return f"sqla+{self.database_url}" @@ -307,6 +314,9 @@ def delete_build(self, build_id): if build.status not in [orm.BuildStatus.FAILED, orm.BuildStatus.COMPLETED]: raise ValueError("cannot delete build since not finished building") + build.deleted_on = datetime.datetime.utcnow() + self.db.commit() + self.celery_app # must import tasks after a celery app has been initialized diff --git a/conda-store-server/conda_store_server/build.py b/conda-store-server/conda_store_server/build.py index 4cb1c118c..19efecb61 100644 --- a/conda-store-server/conda_store_server/build.py +++ b/conda-store-server/conda_store_server/build.py @@ -21,7 +21,12 @@ def set_build_started(conda_store, build): def set_build_failed(conda_store, build, logs): conda_store.storage.set( - conda_store.db, build.id, build.log_key, logs, content_type="text/plain" + conda_store.db, + build.id, + build.log_key, + logs, + content_type="text/plain", + artifact_type=orm.BuildArtifactType.LOGS, ) build.status = orm.BuildStatus.FAILED build.ended_on = datetime.datetime.utcnow() @@ -72,7 +77,12 @@ def package_query(package): build.packages.append(_package) conda_store.storage.set( - conda_store.db, build.id, build.log_key, logs, content_type="text/plain" + conda_store.db, + build.id, + build.log_key, + logs, + content_type="text/plain", + artifact_type=orm.BuildArtifactType.LOGS, ) build.status = orm.BuildStatus.COMPLETED build.ended_on = datetime.datetime.utcnow() @@ -178,6 +188,7 @@ def build_conda_env_export(conda_store, build): build.conda_env_export_key, output, content_type="text/yaml", + artifact_type=orm.BuildArtifactType.YAML, ) @@ -194,6 +205,7 @@ def build_conda_pack(conda_store, build): build.conda_pack_key, output_filename, content_type="application/gzip", + artifact_type=orm.BuildArtifactType.CONDA_PACK, ) @@ -247,6 +259,7 @@ def build_conda_docker(conda_store, build): build.docker_blob_key(content_compressed_hash), content_compressed, content_type="application/gzip", + artifact_type=orm.BuildArtifactType.DOCKER, ) docker_layer = schema.DockerManifestLayer( @@ -273,6 +286,7 @@ def build_conda_docker(conda_store, build): build.docker_blob_key(docker_config_hash), docker_config_content, content_type="application/vnd.docker.container.image.v1+json", + artifact_type=orm.BuildArtifactType.DOCKER, ) conda_store.storage.set( @@ -281,6 +295,7 @@ def build_conda_docker(conda_store, build): build.docker_manifest_key, docker_manifest_content, content_type="application/vnd.docker.distribution.manifest.v2+json", + artifact_type=orm.BuildArtifactType.DOCKER, ) # docker likes to have a sha256 key version of the manifest this @@ -292,6 +307,7 @@ def build_conda_docker(conda_store, build): f"docker/manifest/{build.specification.name}/sha256:{docker_manifest_hash}", docker_manifest_content, content_type="application/vnd.docker.distribution.manifest.v2+json", + artifact_type=orm.BuildArtifactType.DOCKER, ) conda_store.log.info( diff --git a/conda-store-server/conda_store_server/orm.py b/conda-store-server/conda_store_server/orm.py index 09c6fa8d6..ff91df9a0 100644 --- a/conda-store-server/conda_store_server/orm.py +++ b/conda-store-server/conda_store_server/orm.py @@ -27,6 +27,13 @@ Base = declarative_base() +class BuildArtifactType(enum.Enum): + LOGS = "LOGS" + YAML = "YAML" + CONDA_PACK = "CONDA_PACK" + DOCKER = "DOCKER" + + class BuildStatus(enum.Enum): QUEUED = "QUEUED" BUILDING = "BUILDING" @@ -97,6 +104,7 @@ class Build(Base): scheduled_on = Column(DateTime, default=datetime.datetime.utcnow) started_on = Column(DateTime, default=None) ended_on = Column(DateTime, default=None) + deleted_on = Column(DateTime, default=None) def build_path(self, store_directory): store_path = os.path.abspath(store_directory) @@ -150,6 +158,8 @@ class BuildArtifact(Base): build_id = Column(Integer, ForeignKey("build.id")) build = relationship(Build) + artifact_type = Column(Enum(BuildArtifactType), nullable=False) + key = Column(String) diff --git a/conda-store-server/conda_store_server/server/templates/environment.html b/conda-store-server/conda_store_server/server/templates/environment.html index c6dec1977..633d37dd4 100644 --- a/conda-store-server/conda_store_server/server/templates/environment.html +++ b/conda-store-server/conda_store_server/server/templates/environment.html @@ -25,11 +25,11 @@