Try to update conda channels within task

conda-store-server/conda_store_server/app.py

@@ -199,9 +199,6 @@ def ensure_conda_channels(self):
                 self.db.add(conda_channel)
                 self.db.commit()
 
-        for channel in api.list_conda_channels(self.db):
-            channel.update_packages(self.db)
-
     def register_environment(
         self, specification: dict, namespace: str = None, force_build=False
     ):
@@ -270,6 +267,7 @@ def create_build(self, namespace_id: int, specification_sha256: str):
 
         (
             tasks.task_update_storage_metrics.si()
+            | tasks.task_update_conda_channels.si()
             | tasks.task_build_conda_environment.si(build.id)
             | tasks.task_build_conda_env_export.si(build.id)
             | tasks.task_build_conda_pack.si(build.id)

conda-store-server/conda_store_server/orm.py

@@ -194,22 +194,28 @@ class CondaChannel(Base):
     last_update = Column(DateTime)
 
     def update_packages(self, db):
-        repodata = download_repodata(self.name, self.last_update)
+        # # TODO: needs to be done within transaction
+        previous_last_update = self.last_update
+        self.last_update = datetime.datetime.utcnow()
+        db.commit()
+
+        repodata = download_repodata(self.name, previous_last_update)
         if not repodata:
             # nothing to update
             return
 
-        existing_sha256 = {
-            _[0]
-            for _ in db.query(CondaPackage.sha256)
-            .filter(CondaPackage.channel_id == self.id)
-            .all()
-        }
-
         for architecture in repodata:
             packages = list(repodata[architecture]["packages"].values())
+
+            existing_architecture_sha256 = {
+                _[0]
+                for _ in db.query(CondaPackage.sha256)
+                .filter(CondaPackage.channel_id == self.id)
+                .filter(CondaPackage.subdir == architecture)
+                .all()
+            }
             for package in packages:
-                if package["sha256"] not in existing_sha256:
+                if package["sha256"] not in existing_architecture_sha256:
                     db.add(
                         CondaPackage(
                             build=package["build"],
@@ -228,8 +234,8 @@ def update_packages(self, db):
                             channel_id=self.id,
                         )
                     )
-        self.last_update = datetime.datetime.utcnow()
-        db.commit()
+                    existing_architecture_sha256.add(package["sha256"])
+            db.commit()
 
 
 class CondaPackage(Base):
@@ -243,6 +249,7 @@ class CondaPackage(Base):
             "version",
             "build",
             "build_number",
+            "sha256",
             name="_conda_package_uc",
         ),
     )

conda-store-server/conda_store_server/worker/tasks.py

@@ -25,6 +25,8 @@ def task_watch_paths():
                 specification=yaml.safe_load(f), namespace="filesystem"
             )
 
+    worker.conda_store.session_factory.remove()
+
 
 @task(name="task_update_storage_metrics")
 def task_update_storage_metrics():
@@ -33,6 +35,8 @@ def task_update_storage_metrics():
         conda_store.db, conda_store.store_directory
     )
 
+    conda_store.session_factory.remove()
+
 
 @task(name="task_update_conda_channels")
 def task_update_conda_channels():
@@ -43,34 +47,38 @@ def task_update_conda_channels():
     for channel in api.list_conda_channels(conda_store.db):
         channel.update_packages(conda_store.db)
 
+    conda_store.session_factory.remove()
+
 
 @task(name="task_build_conda_environment")
 def task_build_conda_environment(build_id):
     conda_store = create_worker().conda_store
     build = api.get_build(conda_store.db, build_id)
     build_conda_environment(conda_store, build)
+    conda_store.session_factory.remove()
 
 
 @task(name="task_build_conda_env_export")
 def task_build_conda_env_export(build_id):
     conda_store = create_worker().conda_store
     build = api.get_build(conda_store.db, build_id)
     build_conda_env_export(conda_store, build)
+    conda_store.session_factory.remove()
 
 
 @task(name="task_build_conda_pack")
 def task_build_conda_pack(build_id):
     conda_store = create_worker().conda_store
     build = api.get_build(conda_store.db, build_id)
     build_conda_pack(conda_store, build)
-
+    conda_store.session_factory.remove()
 
 @task(name="task_build_conda_docker")
 def task_build_conda_docker(build_id):
     conda_store = create_worker().conda_store
     build = api.get_build(conda_store.db, build_id)
     build_conda_docker(conda_store, build)
-
+    conda_store.session_factory.remove()
 
 @task(name="task_update_environment_build")
 def task_update_environment_build(environment_id):
@@ -83,6 +91,7 @@ def task_update_environment_build(environment_id):
     )
 
     utils.symlink(conda_prefix, environment_prefix)
+    conda_store.session_factory.remove()
 
 
 @task(name="task_delete_build")
@@ -109,3 +118,4 @@ def task_delete_build(build_id):
         conda_store.storage.delete(conda_store.db, build_id, build_artifact.key)
 
     conda_store.db.commit()
+    conda_store.session_factory.remove()