diff --git a/conda-store-server/conda_store_server/app.py b/conda-store-server/conda_store_server/app.py index 037c6cc2d..124f80daf 100644 --- a/conda-store-server/conda_store_server/app.py +++ b/conda-store-server/conda_store_server/app.py @@ -199,9 +199,6 @@ def ensure_conda_channels(self): self.db.add(conda_channel) self.db.commit() - for channel in api.list_conda_channels(self.db): - channel.update_packages(self.db) - def register_environment( self, specification: dict, namespace: str = None, force_build=False ): @@ -270,6 +267,7 @@ def create_build(self, namespace_id: int, specification_sha256: str): ( tasks.task_update_storage_metrics.si() + | tasks.task_update_conda_channels.si() | tasks.task_build_conda_environment.si(build.id) | tasks.task_build_conda_env_export.si(build.id) | tasks.task_build_conda_pack.si(build.id) diff --git a/conda-store-server/conda_store_server/orm.py b/conda-store-server/conda_store_server/orm.py index ff91df9a0..6c8a45199 100644 --- a/conda-store-server/conda_store_server/orm.py +++ b/conda-store-server/conda_store_server/orm.py @@ -194,22 +194,28 @@ class CondaChannel(Base): last_update = Column(DateTime) def update_packages(self, db): - repodata = download_repodata(self.name, self.last_update) + # # TODO: needs to be done within transaction + previous_last_update = self.last_update + self.last_update = datetime.datetime.utcnow() + db.commit() + + repodata = download_repodata(self.name, previous_last_update) if not repodata: # nothing to update return - existing_sha256 = { - _[0] - for _ in db.query(CondaPackage.sha256) - .filter(CondaPackage.channel_id == self.id) - .all() - } - for architecture in repodata: packages = list(repodata[architecture]["packages"].values()) + + existing_architecture_sha256 = { + _[0] + for _ in db.query(CondaPackage.sha256) + .filter(CondaPackage.channel_id == self.id) + .filter(CondaPackage.subdir == architecture) + .all() + } for package in packages: - if package["sha256"] not in existing_sha256: + if package["sha256"] not in existing_architecture_sha256: db.add( CondaPackage( build=package["build"], @@ -228,8 +234,8 @@ def update_packages(self, db): channel_id=self.id, ) ) - self.last_update = datetime.datetime.utcnow() - db.commit() + existing_architecture_sha256.add(package["sha256"]) + db.commit() class CondaPackage(Base): @@ -243,6 +249,7 @@ class CondaPackage(Base): "version", "build", "build_number", + "sha256", name="_conda_package_uc", ), ) diff --git a/conda-store-server/conda_store_server/worker/tasks.py b/conda-store-server/conda_store_server/worker/tasks.py index 1b488dc73..0c4e6852c 100644 --- a/conda-store-server/conda_store_server/worker/tasks.py +++ b/conda-store-server/conda_store_server/worker/tasks.py @@ -25,6 +25,8 @@ def task_watch_paths(): specification=yaml.safe_load(f), namespace="filesystem" ) + worker.conda_store.session_factory.remove() + @task(name="task_update_storage_metrics") def task_update_storage_metrics(): @@ -33,6 +35,8 @@ def task_update_storage_metrics(): conda_store.db, conda_store.store_directory ) + conda_store.session_factory.remove() + @task(name="task_update_conda_channels") def task_update_conda_channels(): @@ -43,12 +47,15 @@ def task_update_conda_channels(): for channel in api.list_conda_channels(conda_store.db): channel.update_packages(conda_store.db) + conda_store.session_factory.remove() + @task(name="task_build_conda_environment") def task_build_conda_environment(build_id): conda_store = create_worker().conda_store build = api.get_build(conda_store.db, build_id) build_conda_environment(conda_store, build) + conda_store.session_factory.remove() @task(name="task_build_conda_env_export") @@ -56,6 +63,7 @@ def task_build_conda_env_export(build_id): conda_store = create_worker().conda_store build = api.get_build(conda_store.db, build_id) build_conda_env_export(conda_store, build) + conda_store.session_factory.remove() @task(name="task_build_conda_pack") @@ -63,14 +71,14 @@ def task_build_conda_pack(build_id): conda_store = create_worker().conda_store build = api.get_build(conda_store.db, build_id) build_conda_pack(conda_store, build) - + conda_store.session_factory.remove() @task(name="task_build_conda_docker") def task_build_conda_docker(build_id): conda_store = create_worker().conda_store build = api.get_build(conda_store.db, build_id) build_conda_docker(conda_store, build) - + conda_store.session_factory.remove() @task(name="task_update_environment_build") def task_update_environment_build(environment_id): @@ -83,6 +91,7 @@ def task_update_environment_build(environment_id): ) utils.symlink(conda_prefix, environment_prefix) + conda_store.session_factory.remove() @task(name="task_delete_build") @@ -109,3 +118,4 @@ def task_delete_build(build_id): conda_store.storage.delete(conda_store.db, build_id, build_artifact.key) conda_store.db.commit() + conda_store.session_factory.remove()