From e0d4819dd38ca473a5a7158a409495b66ac187f5 Mon Sep 17 00:00:00 2001 From: Sean Preston Date: Wed, 1 Jun 2022 10:54:47 -0400 Subject: [PATCH 01/11] adds celery worker to project and code to dispatch tasks --- docker-compose.yml | 21 ++++++++++++++++++++- fidesops.toml | 2 ++ requirements.txt | 1 + src/fidesops/core/config.py | 2 ++ src/fidesops/tasks/__init__.py | 10 ++++++++++ tests/conftest.py | 6 ++++++ 6 files changed, 41 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 47d6bf610..590d26718 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,7 +10,7 @@ services: expose: - 8080 healthcheck: - test: ["CMD", "curl", "-f", "http://0.0.0.0:8080/health"] + test: [ "CMD", "curl", "-f", "http://0.0.0.0:8080/health" ] interval: 30s timeout: 10s retries: 3 @@ -57,6 +57,25 @@ services: ports: - "0.0.0.0:6379:6379" + celery: + build: + context: . + dockerfile: Dockerfile + command: celery -A fidesops.tasks beat -l info --logfile=/var/log/celery.log + depends_on: + fidesops: + condition: service_healthy + db: + condition: service_started + redis: + condition: service_started + restart: always + volumes: + - type: bind + source: ./ + target: /fidesops + read_only: False + docs: build: context: docs/fidesops/ diff --git a/fidesops.toml b/fidesops.toml index 78801dda5..f4e38ec08 100644 --- a/fidesops.toml +++ b/fidesops.toml @@ -29,3 +29,5 @@ TASK_RETRY_DELAY=1 TASK_RETRY_BACKOFF=1 REQUIRE_MANUAL_REQUEST_APPROVAL=false MASKING_STRICT=true +CELERY_BROKER_URL=redis://:testpassword@redis:6379 +CELERY_RESULT_BACKEND=redis://:testpassword@redis:6379 diff --git a/requirements.txt b/requirements.txt index cb272aa1e..d0fbf4c46 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,7 @@ alembic==1.6.5 APScheduler==3.8.0 bcrypt~=3.2.0 boto3~=1.18.14 +celery==5.2.7 click==7.1.2 cryptography~=3.4.8 dask==2021.10.0 diff --git a/src/fidesops/core/config.py b/src/fidesops/core/config.py index d32e602b3..ff23e8ee8 100644 --- a/src/fidesops/core/config.py +++ b/src/fidesops/core/config.py @@ -89,6 +89,8 @@ class ExecutionSettings(FidesSettings): TASK_RETRY_BACKOFF: int REQUIRE_MANUAL_REQUEST_APPROVAL: bool = False MASKING_STRICT: bool = True + CELERY_BROKER_URL: str + CELERY_RESULT_BACKEND: str class Config: env_prefix = "FIDESOPS__EXECUTION__" diff --git a/src/fidesops/tasks/__init__.py b/src/fidesops/tasks/__init__.py index e69de29bb..e2e9c8060 100644 --- a/src/fidesops/tasks/__init__.py +++ b/src/fidesops/tasks/__init__.py @@ -0,0 +1,10 @@ +from celery import Celery + + +app = Celery("tasks") +app.config_from_object("fidesops.core.config.settings", namespace="EXECUTION") +app.autodiscover_tasks(["fidesops.tasks", "fidesops.tasks.scheduled"]) + + +if __name__ == "__main__": + app.worker_main() diff --git a/tests/conftest.py b/tests/conftest.py index 26764acc0..7652f427d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -154,3 +154,9 @@ def _build_jwt(webhook: PolicyPreWebhook) -> Dict[str, str]: @pytest.fixture(scope="session") def integration_config() -> MutableMapping[str, Any]: yield load_toml("fidesops-integration.toml") + + +@pytest.fixture(autouse=True, scope="session") +def celery_enable_logging(): + """Turns on celery output logs.""" + return True From d80b41510e39654b68025a342dac40243e935560 Mon Sep 17 00:00:00 2001 From: Sean Preston Date: Tue, 7 Jun 2022 17:52:29 -0400 Subject: [PATCH 02/11] use celery 5.1.2 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 341a0d451..19d48d771 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ alembic==1.6.5 APScheduler==3.8.0 bcrypt~=3.2.0 boto3~=1.18.14 -celery==5.2.7 +celery==5.1.2 click==7.1.2 cryptography~=3.4.8 dask==2021.10.0 From 8ef2141372e3ed9bc883f7fe8987307bdf309bd4 Mon Sep 17 00:00:00 2001 From: Sean Preston Date: Tue, 7 Jun 2022 18:14:13 -0400 Subject: [PATCH 03/11] fix import order --- src/fidesops/tasks/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/fidesops/tasks/__init__.py b/src/fidesops/tasks/__init__.py index e2e9c8060..0fb9311b2 100644 --- a/src/fidesops/tasks/__init__.py +++ b/src/fidesops/tasks/__init__.py @@ -1,6 +1,5 @@ from celery import Celery - app = Celery("tasks") app.config_from_object("fidesops.core.config.settings", namespace="EXECUTION") app.autodiscover_tasks(["fidesops.tasks", "fidesops.tasks.scheduled"]) From e623f44667d3113849ae93b855c0638d3f5939ef Mon Sep 17 00:00:00 2001 From: Sean Preston Date: Tue, 7 Jun 2022 19:15:32 -0400 Subject: [PATCH 04/11] make celery details strings --- fidesops.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fidesops.toml b/fidesops.toml index 1948c7c67..18b53763b 100644 --- a/fidesops.toml +++ b/fidesops.toml @@ -27,8 +27,8 @@ DRP_JWT_SECRET = "secret" LOG_LEVEL = "INFO" [execution] -CELERY_BROKER_URL=redis://:testpassword@redis:6379 -CELERY_RESULT_BACKEND=redis://:testpassword@redis:6379 +CELERY_BROKER_URL = "redis://:testpassword@redis:6379" +CELERY_RESULT_BACKEND = "redis://:testpassword@redis:6379" MASKING_STRICT=true REQUIRE_MANUAL_REQUEST_APPROVAL=false TASK_RETRY_COUNT=0 From 592962d457fdac852128400b981c15d0bc81cf2d Mon Sep 17 00:00:00 2001 From: Sean Preston Date: Tue, 7 Jun 2022 22:53:26 -0400 Subject: [PATCH 05/11] update .toml files --- data/config/fidesops.toml | 2 ++ fidesops.toml | 13 +++++++------ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/data/config/fidesops.toml b/data/config/fidesops.toml index 3c143a882..25142ea5f 100644 --- a/data/config/fidesops.toml +++ b/data/config/fidesops.toml @@ -26,6 +26,8 @@ DRP_JWT_SECRET = "testdrpsecret" LOG_LEVEL = "DEBUG" [execution] +CELERY_BROKER_URL = "redis://:testpassword@redis:6379" +CELERY_RESULT_BACKEND = "redis://:testpassword@redis:6379" TASK_RETRY_COUNT = 0 TASK_RETRY_DELAY = 1 TASK_RETRY_BACKOFF = 1 diff --git a/fidesops.toml b/fidesops.toml index 18b53763b..73dc0cba6 100644 --- a/fidesops.toml +++ b/fidesops.toml @@ -1,4 +1,4 @@ -PORT=8080 +PORT = 8080 [database] SERVER = "db" @@ -29,11 +29,12 @@ LOG_LEVEL = "INFO" [execution] CELERY_BROKER_URL = "redis://:testpassword@redis:6379" CELERY_RESULT_BACKEND = "redis://:testpassword@redis:6379" -MASKING_STRICT=true -REQUIRE_MANUAL_REQUEST_APPROVAL=false -TASK_RETRY_COUNT=0 -TASK_RETRY_DELAY=1 -TASK_RETRY_BACKOFF=1 +MASKING_STRICT = true +REQUIRE_MANUAL_REQUEST_APPROVAL = false +TASK_RETRY_COUNT = 0 +TASK_RETRY_DELAY = 1 +TASK_RETRY_BACKOFF = 1 [root_user] ANALYTICS_OPT_OUT = false +ANALYTICS_ID = "89187e1f107ebe56c35e4f864af83a90" From b703cf5ad8755b40e7da192f46d8e5c5e5fbd334 Mon Sep 17 00:00:00 2001 From: Sean Preston Date: Wed, 8 Jun 2022 12:26:03 -0400 Subject: [PATCH 06/11] adds file to test celery --- tests/tasks/test_celery.py | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 tests/tasks/test_celery.py diff --git a/tests/tasks/test_celery.py b/tests/tasks/test_celery.py new file mode 100644 index 000000000..5980ca6e7 --- /dev/null +++ b/tests/tasks/test_celery.py @@ -0,0 +1,8 @@ +def test_create_task(celery_app, celery_worker): + @celery_app.task + def multiply(x, y): + return x * y + + celery_worker.reload() # Force `celery_app` to register our new task + assert multiply.run(4, 4) == 16 + assert multiply.delay(4, 4).get(timeout=10) == 16 From 9692e829d6babd1a6dcc2c54e6a3aa801517009e Mon Sep 17 00:00:00 2001 From: Sean Preston Date: Wed, 8 Jun 2022 12:27:22 -0400 Subject: [PATCH 07/11] elaborate on comment --- tests/tasks/test_celery.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/tasks/test_celery.py b/tests/tasks/test_celery.py index 5980ca6e7..fe5d80286 100644 --- a/tests/tasks/test_celery.py +++ b/tests/tasks/test_celery.py @@ -3,6 +3,8 @@ def test_create_task(celery_app, celery_worker): def multiply(x, y): return x * y - celery_worker.reload() # Force `celery_app` to register our new task + # Force `celery_app` to register our new task + # See: https://github.com/celery/celery/issues/3642#issuecomment-369057682 + celery_worker.reload() assert multiply.run(4, 4) == 16 assert multiply.delay(4, 4).get(timeout=10) == 16 From 96d79499d13c30224c5d7b8ddabe411348bee53f Mon Sep 17 00:00:00 2001 From: Sean Preston Date: Wed, 8 Jun 2022 12:33:03 -0400 Subject: [PATCH 08/11] check new version of celery --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0b4aae7be..ed79c4fea 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ alembic==1.6.5 APScheduler==3.8.0 boto3~=1.18.14 -celery==5.1.2 +celery[pytest]==5.2.7 click==8.1.3 cryptography~=3.4.8 dask==2021.10.0 From 1067cf1847a4e183e797795e8c47a3fc0fc1378c Mon Sep 17 00:00:00 2001 From: Sean Preston Date: Wed, 8 Jun 2022 17:37:44 -0400 Subject: [PATCH 09/11] fix task config path --- src/fidesops/tasks/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fidesops/tasks/__init__.py b/src/fidesops/tasks/__init__.py index 0fb9311b2..08d60f0b9 100644 --- a/src/fidesops/tasks/__init__.py +++ b/src/fidesops/tasks/__init__.py @@ -1,7 +1,7 @@ from celery import Celery app = Celery("tasks") -app.config_from_object("fidesops.core.config.settings", namespace="EXECUTION") +app.config_from_object("fidesops.core.config", namespace="EXECUTION") app.autodiscover_tasks(["fidesops.tasks", "fidesops.tasks.scheduled"]) From e1eb995a17d2beca01c52b35d4a35d6fb75b481a Mon Sep 17 00:00:00 2001 From: Sean Preston Date: Wed, 8 Jun 2022 19:16:15 -0400 Subject: [PATCH 10/11] add fixture for pending request --- tests/fixtures/application_fixtures.py | 73 ++++++++++++++++---------- 1 file changed, 45 insertions(+), 28 deletions(-) diff --git a/tests/fixtures/application_fixtures.py b/tests/fixtures/application_fixtures.py index c905894c3..ab60be60a 100644 --- a/tests/fixtures/application_fixtures.py +++ b/tests/fixtures/application_fixtures.py @@ -744,42 +744,59 @@ def privacy_requests(db: Session, policy: Policy) -> Generator: def _create_privacy_request_for_policy( db: Session, policy: Policy, + status: PrivacyRequestStatus = PrivacyRequestStatus.in_processing, ) -> PrivacyRequest: + data = { + "external_id": f"ext-{str(uuid4())}", + "requested_at": datetime( + 2018, + 12, + 31, + hour=2, + minute=30, + second=23, + microsecond=916482, + tzinfo=timezone.utc, + ), + "status": status, + "origin": f"https://example.com/", + "policy_id": policy.id, + "client_id": policy.client_id, + } + if status != PrivacyRequestStatus.pending: + data["started_processing_at"] = datetime( + 2019, + 1, + 1, + hour=1, + minute=45, + second=55, + microsecond=393185, + tzinfo=timezone.utc, + ) return PrivacyRequest.create( db=db, - data={ - "external_id": f"ext-{str(uuid4())}", - "started_processing_at": datetime( - 2019, - 1, - 1, - hour=1, - minute=45, - second=55, - microsecond=393185, - tzinfo=timezone.utc, - ), - "requested_at": datetime( - 2018, - 12, - 31, - hour=2, - minute=30, - second=23, - microsecond=916482, - tzinfo=timezone.utc, - ), - "status": PrivacyRequestStatus.in_processing, - "origin": f"https://example.com/", - "policy_id": policy.id, - "client_id": policy.client_id, - }, + data=data, ) @pytest.fixture(scope="function") def privacy_request(db: Session, policy: Policy) -> PrivacyRequest: - privacy_request = _create_privacy_request_for_policy(db, policy) + privacy_request = _create_privacy_request_for_policy( + db, + policy, + ) + yield privacy_request + privacy_request.delete(db) + + +@pytest.fixture(scope="function") +def privacy_request_status_pending(db: Session, policy: Policy) -> PrivacyRequest: + privacy_request = _create_privacy_request_for_policy( + db, + policy, + PrivacyRequestStatus.pending, + ) yield privacy_request privacy_request.delete(db) From 6ceed350492973a5b2a57ceedf5fe734cab94c44 Mon Sep 17 00:00:00 2001 From: Sean Preston Date: Wed, 8 Jun 2022 21:18:02 -0400 Subject: [PATCH 11/11] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 089bbc366..25838d736 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ The types of changes are: * Subject Request Details page [#563](https://github.com/ethyca/fidesops/pull/563) * Restart Graph from Failure [#578](https://github.com/ethyca/fidesops/pull/578) * Redis SSL Support [#611](https://github.com/ethyca/fidesops/pull/611) +* Celery as a dependency for use in the execution layer [#610](https://github.com/ethyca/fidesops/pull/610) ## [1.5.2](https://github.com/ethyca/fidesops/compare/1.5.1...1.5.2)