Adding syncronous delete (#107)

conda-store-server/conda_store_server/api.py

@@ -1,3 +1,5 @@
+from typing import List
+
 from sqlalchemy import func
 
 from conda_store_server import orm
@@ -102,12 +104,22 @@ def get_build_lockfile(db, build_id):
     )
 
 
-def list_build_artifacts(db, limit: int = 25, build_id: int = None, key: str = None):
+def list_build_artifacts(
+    db,
+    limit: int = 25,
+    build_id: int = None,
+    key: str = None,
+    excluded_artifact_types: List[orm.BuildArtifactType] = None,
+):
     filters = []
     if build_id:
         filters.append(orm.BuildArtifact.build_id == build_id)
     if key:
         filters.append(orm.BuildArtifact.key == key)
+    if excluded_artifact_types:
+        filters.append(
+            func.not_(orm.BuildArtifact.artifact_type.in_(excluded_artifact_types))
+        )
 
     return db.query(orm.BuildArtifact).filter(*filters).limit(limit).all()
 

conda-store-server/conda_store_server/app.py

@@ -1,4 +1,5 @@
 import os
+import datetime
 
 from celery import Celery
 from traitlets import Type, Unicode, Integer, List, default
@@ -56,6 +57,12 @@ class CondaStore(LoggingConfigurable):
         config=True,
     )
 
+    build_artifacts_kept_on_deletion = List(
+        [orm.BuildArtifactType.LOGS, orm.BuildArtifactType.YAML],
+        help="artifacts to keep on build deletion",
+        config=True,
+    )
+
     @default("celery_broker_url")
     def _default_celery_broker_url(self):
         return f"sqla+{self.database_url}"
@@ -307,6 +314,9 @@ def delete_build(self, build_id):
         if build.status not in [orm.BuildStatus.FAILED, orm.BuildStatus.COMPLETED]:
             raise ValueError("cannot delete build since not finished building")
 
+        build.deleted_on = datetime.datetime.utcnow()
+        self.db.commit()
+
         self.celery_app
 
         # must import tasks after a celery app has been initialized

conda-store-server/conda_store_server/build.py

@@ -21,7 +21,12 @@ def set_build_started(conda_store, build):
 
 def set_build_failed(conda_store, build, logs):
     conda_store.storage.set(
-        conda_store.db, build.id, build.log_key, logs, content_type="text/plain"
+        conda_store.db,
+        build.id,
+        build.log_key,
+        logs,
+        content_type="text/plain",
+        artifact_type=orm.BuildArtifactType.LOGS,
     )
     build.status = orm.BuildStatus.FAILED
     build.ended_on = datetime.datetime.utcnow()
@@ -72,7 +77,12 @@ def package_query(package):
             build.packages.append(_package)
 
     conda_store.storage.set(
-        conda_store.db, build.id, build.log_key, logs, content_type="text/plain"
+        conda_store.db,
+        build.id,
+        build.log_key,
+        logs,
+        content_type="text/plain",
+        artifact_type=orm.BuildArtifactType.LOGS,
     )
     build.status = orm.BuildStatus.COMPLETED
     build.ended_on = datetime.datetime.utcnow()
@@ -178,6 +188,7 @@ def build_conda_env_export(conda_store, build):
         build.conda_env_export_key,
         output,
         content_type="text/yaml",
+        artifact_type=orm.BuildArtifactType.YAML,
     )
 
 
@@ -194,6 +205,7 @@ def build_conda_pack(conda_store, build):
             build.conda_pack_key,
             output_filename,
             content_type="application/gzip",
+            artifact_type=orm.BuildArtifactType.CONDA_PACK,
         )
 
 
@@ -247,6 +259,7 @@ def build_conda_docker(conda_store, build):
             build.docker_blob_key(content_compressed_hash),
             content_compressed,
             content_type="application/gzip",
+            artifact_type=orm.BuildArtifactType.DOCKER,
         )
 
         docker_layer = schema.DockerManifestLayer(
@@ -273,6 +286,7 @@ def build_conda_docker(conda_store, build):
         build.docker_blob_key(docker_config_hash),
         docker_config_content,
         content_type="application/vnd.docker.container.image.v1+json",
+        artifact_type=orm.BuildArtifactType.DOCKER,
     )
 
     conda_store.storage.set(
@@ -281,6 +295,7 @@ def build_conda_docker(conda_store, build):
         build.docker_manifest_key,
         docker_manifest_content,
         content_type="application/vnd.docker.distribution.manifest.v2+json",
+        artifact_type=orm.BuildArtifactType.DOCKER,
     )
 
     # docker likes to have a sha256 key version of the manifest this
@@ -292,6 +307,7 @@ def build_conda_docker(conda_store, build):
         f"docker/manifest/{build.specification.name}/sha256:{docker_manifest_hash}",
         docker_manifest_content,
         content_type="application/vnd.docker.distribution.manifest.v2+json",
+        artifact_type=orm.BuildArtifactType.DOCKER,
     )
 
     conda_store.log.info(

conda-store-server/conda_store_server/orm.py

@@ -27,6 +27,13 @@
 Base = declarative_base()
 
 
+class BuildArtifactType(enum.Enum):
+    LOGS = "LOGS"
+    YAML = "YAML"
+    CONDA_PACK = "CONDA_PACK"
+    DOCKER = "DOCKER"
+
+
 class BuildStatus(enum.Enum):
     QUEUED = "QUEUED"
     BUILDING = "BUILDING"
@@ -97,6 +104,7 @@ class Build(Base):
     scheduled_on = Column(DateTime, default=datetime.datetime.utcnow)
     started_on = Column(DateTime, default=None)
     ended_on = Column(DateTime, default=None)
+    deleted_on = Column(DateTime, default=None)
 
     def build_path(self, store_directory):
         store_path = os.path.abspath(store_directory)
@@ -150,6 +158,8 @@ class BuildArtifact(Base):
     build_id = Column(Integer, ForeignKey("build.id"))
     build = relationship(Build)
 
+    artifact_type = Column(Enum(BuildArtifactType), nullable=False)
+
     key = Column(String)
 
 

conda-store-server/conda_store_server/server/templates/environment.html

@@ -25,21 +25,23 @@ <h5 class="card-title">{{ environment.namespace.name }}/{{ environment.name }}
 <h3>Builds</h3>
 <ul class="list-group">
     {% for build in environment_builds %}
-    <li class="list-group-item d-flex justify-content-between align-items-center {{ 'list-group-item-secondary' if build.id == environment.build_id else ''}}">
+    <li class="list-group-item d-flex justify-content-between align-items-center {% if build.id == environment.build_id %}list-group-item-success{% elif build.deleted_on is not none %}list-group-item-secondary{% endif %}">
         <a href="/build/{{ build.id }}/">Build {{ build.id }}</a>
         <span>{{ build.status.value }}</span>
         <div class="btn-group" role="group" aria-label="Build actions">
-            {% if build.id != environment.build_id and build.status.value == 'COMPLETED' %}
+            {% if build.id != environment.build_id and build.status.value == 'COMPLETED' and build.deleted_on is none %}
             <button type="button" onclick="updateEnvironmentBuild('{{ build.id }}')" class="btn btn-primary mb-2">
                 <ion-icon name="checkmark"></ion-icon>
             </button>
             {% endif %}
             <button type="button" onclick="buildAction('PUT', '{{ build.id }}')" class="btn btn-primary mb-2">
                 <ion-icon name="refresh"></ion-icon>
             </button>
+            {% if build.status.value in ["COMPLETED", "FAILED"] and  build.deleted_on is none %}
             <button type="button" onclick="buildAction('DELETE', '{{ build.id }}')" class="btn btn-primary mb-2">
                 <ion-icon name="trash"></ion-icon>
             </button>
+            {% endif %}
         </div>
     </li>
     {% endfor %}

conda-store-server/conda_store_server/storage.py

@@ -10,12 +10,30 @@
 
 
 class Storage(LoggingConfigurable):
-    def fset(self, db, build_id: int, key: str, filename: str):
-        db.add(orm.BuildArtifact(build_id=build_id, key=key))
+    def fset(
+        self,
+        db,
+        build_id: int,
+        key: str,
+        filename: str,
+        artifact_type: orm.BuildArtifactType,
+    ):
+        db.add(
+            orm.BuildArtifact(build_id=build_id, key=key, artifact_type=artifact_type)
+        )
         db.commit()
 
-    def set(self, db, build_id: int, key: str, value):
-        db.add(orm.BuildArtifact(build_id=build_id, key=key))
+    def set(
+        self,
+        db,
+        build_id: int,
+        key: str,
+        value: bytes,
+        artifact_type: orm.BuildArtifactType,
+    ):
+        db.add(
+            orm.BuildArtifact(build_id=build_id, key=key, artifact_type=artifact_type)
+        )
         db.commit()
 
     def get_url(self, key: str):
@@ -105,23 +123,21 @@ def _check_bucket_exists(self):
         if not self._internal_client.bucket_exists(self.bucket_name):
             raise ValueError(f"S3 bucket={self.bucket_name} does not exist")
 
-    def fset(
-        self, db, build_id, key, filename, content_type="application/octet-stream"
-    ):
+    def fset(self, db, build_id, key, filename, content_type, artifact_type):
         self.internal_client.fput_object(
             self.bucket_name, key, filename, content_type=content_type
         )
-        super().fset(db, build_id, key, filename)
+        super().fset(db, build_id, key, filename, artifact_type)
 
-    def set(self, db, build_id, key, value, content_type="application/octet-stream"):
+    def set(self, db, build_id, key, value, content_type, artifact_type):
         self.internal_client.put_object(
             self.bucket_name,
             key,
             io.BytesIO(value),
             length=len(value),
             content_type=content_type,
         )
-        super().fset(db, build_id, key, value)
+        super().fset(db, build_id, key, value, artifact_type)
 
     def get_url(self, key):
         return self.external_client.presigned_get_object(self.bucket_name, key)
@@ -143,20 +159,20 @@ class LocalStorage(Storage):
         config=True,
     )
 
-    def fset(self, db, build_id, key, filename, content_type=None):
+    def fset(self, db, build_id, key, filename, content_type=None, artifact_type=None):
         filename = os.path.join(self.storage_path, key)
         os.makedirs(os.path.dirname(filename), exist_ok=True)
 
         shutil.copyfile(filename, os.path.join(self.storage_path, key))
-        super().fset(db, build_id, key, filename)
+        super().fset(db, build_id, key, filename, artifact_type)
 
-    def set(self, db, build_id, key, value, content_type=None):
+    def set(self, db, build_id, key, value, content_type=None, artifact_type=None):
         filename = os.path.join(self.storage_path, key)
         os.makedirs(os.path.dirname(filename), exist_ok=True)
 
         with open(filename, "wb") as f:
             f.write(value)
-        super().set(db, build_id, key, value)
+        super().set(db, build_id, key, value, artifact_type)
 
     def get_url(self, key):
         return os.path.join(self.storage_url, key)

conda-store-server/conda_store_server/worker/tasks.py

@@ -98,10 +98,14 @@ def task_delete_build(build_id):
     ):
         shutil.rmtree(conda_prefix)
 
+    conda_store.log.error("deleting artifacts")
     for build_artifact in api.list_build_artifacts(
-        conda_store.db, limit=None, build_id=build_id
+        conda_store.db,
+        limit=None,
+        build_id=build_id,
+        excluded_artifact_types=conda_store.build_artifacts_kept_on_deletion,
     ):
+        conda_store.log.error(f"deleting {build_artifact.key}")
         conda_store.storage.delete(conda_store.db, build_id, build_artifact.key)
 
-    conda_store.db.delete(build)
     conda_store.db.commit()