From 5a16a49cf91a4bca3f9f435aa46b8b616fe2f81f Mon Sep 17 00:00:00 2001 From: Adam Lewis <23342526+Adam-D-Lewis@users.noreply.github.com> Date: Tue, 2 Jul 2024 12:00:11 -0500 Subject: [PATCH 1/4] restart workers after executing 10 tasks to mitigate memory leaks --- conda-store-server/conda_store_server/_internal/worker/app.py | 1 + 1 file changed, 1 insertion(+) diff --git a/conda-store-server/conda_store_server/_internal/worker/app.py b/conda-store-server/conda_store_server/_internal/worker/app.py index e8400c7f3..48c996348 100644 --- a/conda-store-server/conda_store_server/_internal/worker/app.py +++ b/conda-store-server/conda_store_server/_internal/worker/app.py @@ -72,6 +72,7 @@ def start(self): argv = [ "worker", "--loglevel=INFO", + "--max-tasks-per-child=10", # mitigate memory leaks ] # The default Celery pool requires this on Windows. See From d097c90985a6ede7597e31af1385c8ea975cf2f1 Mon Sep 17 00:00:00 2001 From: Adam Lewis <23342526+Adam-D-Lewis@users.noreply.github.com> Date: Fri, 5 Jul 2024 16:58:41 -0500 Subject: [PATCH 2/4] add memory leak test --- conda-store-server/tests/test_memory_leak.py | 53 ++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 conda-store-server/tests/test_memory_leak.py diff --git a/conda-store-server/tests/test_memory_leak.py b/conda-store-server/tests/test_memory_leak.py new file mode 100644 index 000000000..f03395320 --- /dev/null +++ b/conda-store-server/tests/test_memory_leak.py @@ -0,0 +1,53 @@ +from uuid import uuid4 +from conda_store_server import api +from conda_store_server._internal import schema +from conda_store_server._internal.worker import tasks + + +# def test_memory_leak(testclient): +def test_memory_leak(db, conda_store, celery_worker): + # DIDNT WORK + # conda_store.celery_app + + # @conda_store.celery_app.task(name='return_memory') + # def return_memory(): + # return "4GiB" + + # breakpoint() + # # conda_store.celery_app.register_task(return_memory) + # result = return_memory.apply_async() + # real_result = result.get() + # print(f'REAL RESULT = {real_result}') + conda_specification = schema.CondaSpecification( + name=f"pytest-{uuid4()}", + channels=["conda-forge"], + dependencies=["python"], + ) + namespace_name = "default" + + build_id = conda_store.register_environment( + db, specification=conda_specification.dict(), namespace=namespace_name + ) + + build = api.get_build(db, build_id=build_id) + assert build is not None + assert build.status == schema.BuildStatus.QUEUED + assert build.environment.name == conda_specification.name + assert build.environment.namespace.name == namespace_name + assert build.specification.spec["name"] == conda_specification.name + assert build.specification.spec["channels"] == conda_specification.channels + assert build.specification.spec["dependencies"] == conda_specification.dependencies + # when new environment is created current_build should be the default build + assert build.environment.current_build == build + + from celery.result import AsyncResult + + # wait for task to complete + # build: environment, export, archive + # docker is expected to fail will be fixed soon + breakpoint() + task = AsyncResult(f"build-{build.id}-environment") + task.wait(timeout=1) + + + print(f'build_id = {build_id}') From 5befd8466e9e25a9e0458032bcfbc23659103bf7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 5 Jul 2024 21:58:55 +0000 Subject: [PATCH 3/4] [pre-commit.ci] Apply automatic pre-commit fixes --- conda-store-server/tests/test_memory_leak.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/conda-store-server/tests/test_memory_leak.py b/conda-store-server/tests/test_memory_leak.py index f03395320..fc8421406 100644 --- a/conda-store-server/tests/test_memory_leak.py +++ b/conda-store-server/tests/test_memory_leak.py @@ -1,18 +1,18 @@ from uuid import uuid4 + from conda_store_server import api from conda_store_server._internal import schema -from conda_store_server._internal.worker import tasks # def test_memory_leak(testclient): def test_memory_leak(db, conda_store, celery_worker): # DIDNT WORK # conda_store.celery_app - + # @conda_store.celery_app.task(name='return_memory') # def return_memory(): # return "4GiB" - + # breakpoint() # # conda_store.celery_app.register_task(return_memory) # result = return_memory.apply_async() @@ -28,7 +28,7 @@ def test_memory_leak(db, conda_store, celery_worker): build_id = conda_store.register_environment( db, specification=conda_specification.dict(), namespace=namespace_name ) - + build = api.get_build(db, build_id=build_id) assert build is not None assert build.status == schema.BuildStatus.QUEUED @@ -49,5 +49,4 @@ def test_memory_leak(db, conda_store, celery_worker): task = AsyncResult(f"build-{build.id}-environment") task.wait(timeout=1) - - print(f'build_id = {build_id}') + print(f"build_id = {build_id}") From c8cdb5f6543502085031a9a5a5f1959adb63421d Mon Sep 17 00:00:00 2001 From: Adam Lewis <23342526+Adam-D-Lewis@users.noreply.github.com> Date: Tue, 23 Jul 2024 15:17:20 -0500 Subject: [PATCH 4/4] remove unfinished test --- conda-store-server/tests/test_memory_leak.py | 52 -------------------- 1 file changed, 52 deletions(-) delete mode 100644 conda-store-server/tests/test_memory_leak.py diff --git a/conda-store-server/tests/test_memory_leak.py b/conda-store-server/tests/test_memory_leak.py deleted file mode 100644 index fc8421406..000000000 --- a/conda-store-server/tests/test_memory_leak.py +++ /dev/null @@ -1,52 +0,0 @@ -from uuid import uuid4 - -from conda_store_server import api -from conda_store_server._internal import schema - - -# def test_memory_leak(testclient): -def test_memory_leak(db, conda_store, celery_worker): - # DIDNT WORK - # conda_store.celery_app - - # @conda_store.celery_app.task(name='return_memory') - # def return_memory(): - # return "4GiB" - - # breakpoint() - # # conda_store.celery_app.register_task(return_memory) - # result = return_memory.apply_async() - # real_result = result.get() - # print(f'REAL RESULT = {real_result}') - conda_specification = schema.CondaSpecification( - name=f"pytest-{uuid4()}", - channels=["conda-forge"], - dependencies=["python"], - ) - namespace_name = "default" - - build_id = conda_store.register_environment( - db, specification=conda_specification.dict(), namespace=namespace_name - ) - - build = api.get_build(db, build_id=build_id) - assert build is not None - assert build.status == schema.BuildStatus.QUEUED - assert build.environment.name == conda_specification.name - assert build.environment.namespace.name == namespace_name - assert build.specification.spec["name"] == conda_specification.name - assert build.specification.spec["channels"] == conda_specification.channels - assert build.specification.spec["dependencies"] == conda_specification.dependencies - # when new environment is created current_build should be the default build - assert build.environment.current_build == build - - from celery.result import AsyncResult - - # wait for task to complete - # build: environment, export, archive - # docker is expected to fail will be fixed soon - breakpoint() - task = AsyncResult(f"build-{build.id}-environment") - task.wait(timeout=1) - - print(f"build_id = {build_id}")