From a3e3d077cb4ae0e17601e7f115ab1f9926f6730a Mon Sep 17 00:00:00 2001 From: Adam Sachs Date: Mon, 23 May 2022 15:45:10 -0400 Subject: [PATCH 1/3] environment variables to disable app db and cache. some related cleanup --- Makefile | 3 ++ docker-compose.no-db.yml | 38 +++++++++++++++++++ .../docs/guides/configuration_reference.md | 14 ++++--- fidesops.toml | 2 + src/fidesops/api/deps.py | 10 +++++ src/fidesops/api/v1/exception_handlers.py | 20 ++++++++++ src/fidesops/common_exceptions.py | 4 ++ src/fidesops/core/config.py | 2 + src/fidesops/main.py | 20 +++++++--- tests/conftest.py | 3 +- 10 files changed, 105 insertions(+), 11 deletions(-) create mode 100644 docker-compose.no-db.yml create mode 100644 src/fidesops/api/v1/exception_handlers.py diff --git a/Makefile b/Makefile index 3c67e2a83..cdd8dd89a 100644 --- a/Makefile +++ b/Makefile @@ -43,6 +43,9 @@ reset-db: server: compose-build @docker-compose up +server-no-db: compose-build + @docker-compose -f docker-compose.no-db.yml up + server-shell: compose-build @docker-compose run $(IMAGE_NAME) /bin/bash diff --git a/docker-compose.no-db.yml b/docker-compose.no-db.yml new file mode 100644 index 000000000..ba8aea237 --- /dev/null +++ b/docker-compose.no-db.yml @@ -0,0 +1,38 @@ +services: + fidesops: + container_name: fidesops + build: + context: . + dockerfile: Dockerfile + expose: + - 8080 + healthcheck: + test: [ "CMD", "curl", "-f", "http://0.0.0.0:8080/health" ] + interval: 30s + timeout: 10s + retries: 3 + start_period: 1s + ports: + - "8080:8080" + volumes: + - type: bind + source: ./ + target: /fidesops + read_only: False + - /fidesops/src/fidesops.egg-info + environment: + - FIDESOPS__DEV_MODE=${FIDESOPS__DEV_MODE} + - FIDESOPS__LOG_PII=${FIDESOPS__LOG_PII} + - FIDESOPS__HOT_RELOAD=${FIDESOPS__HOT_RELOAD} + - FIDESOPS__DATABASE__ENABLED=false + - FIDESOPS__REDIS__ENABLED=false + docs: + build: + context: docs/fidesops/ + dockerfile: Dockerfile + volumes: + - ./docs/fidesops:/docs + expose: + - 8000 + ports: + - "8000:8000" diff --git a/docs/fidesops/docs/guides/configuration_reference.md b/docs/fidesops/docs/guides/configuration_reference.md index cab1b9920..d76b6b086 100644 --- a/docs/fidesops/docs/guides/configuration_reference.md +++ b/docs/fidesops/docs/guides/configuration_reference.md @@ -30,16 +30,18 @@ The `fidesops.toml` file should specify the following variables: | TOML Variable | ENV Variable | Type | Example | Default | Description | |---|---|---|---|---|---| | `SERVER` | `FIDESOPS__DATABASE__SERVER` | string | postgres.internal | N/A | The networking address for the Fideops Postgres database server | -| `USER` | `FIDESOPS__DATABASE_USER` | string | postgres | N/A | The database user with which to login to the Fidesops application database | -| `PASSWORD` | `FIDESOPS__DATABASE_PASSWORD` | string | apassword | N/A | The password with which to login to the Fidesops application database | +| `USER` | `FIDESOPS__DATABASE__USER` | string | postgres | N/A | The database user with which to login to the Fidesops application database | +| `PASSWORD` | `FIDESOPS__DATABASE__PASSWORD` | string | apassword | N/A | The password with which to login to the Fidesops application database | | `PORT` | `FIDESOPS__DATABASE__PORT` | int | 5432 | 5432 | The port at which the Fidesops application database will be accessible | -| `DB` | `FIDESOPS__DATABASE_DB` | string | db | N/A | The name of the database to use in the Fidesops application database | +| `DB` | `FIDESOPS__DATABASE__DB` | string | db | N/A | The name of the database to use in the Fidesops application database | +| `ENABLED` | `FIDESOPS__DATABASE__ENABLED` | bool | True | True | Whether the application database should be enabled. Only set to false for certain narrow uses of the application that do not require a backing application database. | |---|---|---|---|---|---| | `HOST` | `FIDESOPS__REDIS__HOST` | string | redis.internal | N/A | The networking address for the Fidesops application Redis cache | | `PORT` | `FIDESOPS__REDIS__PORT` | int | 6379 | 6379 | The port at which the Fidesops application cache will be accessible | | `PASSWORD` | `FIDESOPS__REDIS__PASSWORD` | string | anotherpassword | N/A | The password with which to login to the Fidesops application cache | | `DB_INDEX` | `FIDESOPS__REDIS__DB_INDEX` | int | 0 | 0 | The Fidesops application will use this index in the Redis cache to cache data | | `DEFAULT_TTL_SECONDS` | `FIDESOPS__REDIS__DEFAULT_TTL_SECONDS` | int | 3600 | 604800 | The number of seconds for which data will live in Redis before automatically expiring | +| `ENABLED` | `FIDESOPS__REDIS__ENABLED` | bool | True | True | Whether the application's redis cache should be enabled. Only set to false for certain narrow uses of the application that do not require a backing redis cache. | |---|---|---|---|---|---| | `APP_ENCRYPTION_KEY` | `FIDESOPS__SECURITY__APP_ENCRYPTION_KEY` | string | OLMkv91j8DHiDAULnK5Lxx3kSCov30b3 | N/A | The key used to sign Fidesops API access tokens | | `CORS_ORIGINS` | `FIDESOPS__SECURITY__CORS_ORIGINS` | List[AnyHttpUrl] | ["https://a-client.com/", "https://another-client.com"/] | N/A | A list of pre-approved addresses of clients allowed to communicate with the Fidesops application server | @@ -64,6 +66,7 @@ USER="postgres" PASSWORD="a-password" DB="app" TEST_DB="test" +ENABLED=true [redis] HOST="redis" @@ -72,6 +75,7 @@ PORT=6379 CHARSET="utf8" DEFAULT_TTL_SECONDS=3600 DB_INDEX=0 +ENABLED=true [security] APP_ENCRYPTION_KEY="OLMkv91j8DHiDAULnK5Lxx3kSCov30b3" @@ -84,8 +88,8 @@ OAUTH_ROOT_CLIENT_SECRET="fidesopsadminsecret" TASK_RETRY_COUNT=3 TASK_RETRY_DELAY=20 TASK_RETRY_BACKOFF=2 -REQUIRE_MANUAL_REQUEST_APPROVAL=True -MASKING_STRICT=True +REQUIRE_MANUAL_REQUEST_APPROVAL=true +MASKING_STRICT=true ``` Please note: The configuration is case-sensitive, so the variables must be specified in UPPERCASE. diff --git a/fidesops.toml b/fidesops.toml index 5fe6a6bc3..78801dda5 100644 --- a/fidesops.toml +++ b/fidesops.toml @@ -4,6 +4,7 @@ USER="postgres" PASSWORD="216f4b49bea5da4f84f05288258471852c3e325cd336821097e1e65ff92b528a" DB="app" TEST_DB="test" +ENABLED=true [redis] HOST="redis" @@ -12,6 +13,7 @@ PORT=6379 CHARSET="utf8" DEFAULT_TTL_SECONDS=604800 DB_INDEX=0 +ENABLED=true [security] APP_ENCRYPTION_KEY="OLMkv91j8DHiDAULnK5Lxx3kSCov30b3" diff --git a/src/fidesops/api/deps.py b/src/fidesops/api/deps.py index 46622e295..e23cd7f3e 100644 --- a/src/fidesops/api/deps.py +++ b/src/fidesops/api/deps.py @@ -1,11 +1,17 @@ from typing import Generator +from fidesops.common_exceptions import FunctionalityNotConfigured +from fidesops.core.config import config from fidesops.db.session import get_db_session from fidesops.util.cache import get_cache as get_redis_connection def get_db() -> Generator: """Return our database session""" + if not config.database.ENABLED: + raise FunctionalityNotConfigured( + "Application database required, but it is currently disabled! Please update your application configuration to enable integration with an application database." + ) try: SessionLocal = get_db_session() db = SessionLocal() @@ -16,4 +22,8 @@ def get_db() -> Generator: def get_cache() -> Generator: """Return a connection to our redis cache""" + if not config.redis.ENABLED: + raise FunctionalityNotConfigured( + "Application redis cache required, but it is currently disabled! Please update your application configuration to enable integration with a redis cache." + ) yield get_redis_connection() diff --git a/src/fidesops/api/v1/exception_handlers.py b/src/fidesops/api/v1/exception_handlers.py new file mode 100644 index 000000000..d3206a9bf --- /dev/null +++ b/src/fidesops/api/v1/exception_handlers.py @@ -0,0 +1,20 @@ +from typing import Callable, List + +from fastapi import Request +from fastapi.responses import JSONResponse, Response +from starlette.status import HTTP_500_INTERNAL_SERVER_ERROR + +from fidesops.common_exceptions import FunctionalityNotConfigured + + +class ExceptionHandlers: + def functionality_not_configured_handler( + request: Request, exc: FunctionalityNotConfigured + ) -> JSONResponse: + return JSONResponse( + status_code=HTTP_500_INTERNAL_SERVER_ERROR, content={"message": str(exc)} + ) + + @classmethod + def get_handlers(cls) -> List[Callable[[Request, Exception], Response]]: + return [ExceptionHandlers.functionality_not_configured_handler] diff --git a/src/fidesops/common_exceptions.py b/src/fidesops/common_exceptions.py index 26be8156e..7c0c1e53b 100644 --- a/src/fidesops/common_exceptions.py +++ b/src/fidesops/common_exceptions.py @@ -156,3 +156,7 @@ class NoSuchStrategyException(ValueError): class MissingConfig(Exception): """Custom exception for when no valid configuration file is provided.""" + + +class FunctionalityNotConfigured(Exception): + """Custom exception for when invoked functionality is unavailable due to configuration.""" diff --git a/src/fidesops/core/config.py b/src/fidesops/core/config.py index f03fe72b9..6459d62d2 100644 --- a/src/fidesops/core/config.py +++ b/src/fidesops/core/config.py @@ -41,6 +41,7 @@ class DatabaseSettings(FidesSettings): DB: str PORT: str = "5432" TEST_DB: str = "test" + ENABLED: bool = True SQLALCHEMY_DATABASE_URI: Optional[PostgresDsn] = None SQLALCHEMY_TEST_DATABASE_URI: Optional[PostgresDsn] = None @@ -103,6 +104,7 @@ class RedisSettings(FidesSettings): DECODE_RESPONSES: bool = True DEFAULT_TTL_SECONDS: int = 604800 DB_INDEX: int + ENABLED: bool = True class Config: env_prefix = "FIDESOPS__REDIS__" diff --git a/src/fidesops/main.py b/src/fidesops/main.py index 2208e6501..4a987ad0a 100644 --- a/src/fidesops/main.py +++ b/src/fidesops/main.py @@ -5,7 +5,9 @@ from starlette.middleware.cors import CORSMiddleware from fidesops.api.v1.api import api_router +from fidesops.api.v1.exception_handlers import ExceptionHandlers from fidesops.api.v1.urn_registry import V1_URL_PREFIX +from fidesops.common_exceptions import FunctionalityNotConfigured from fidesops.core.config import config from fidesops.db.database import init_db from fidesops.tasks.scheduled.scheduler import scheduler @@ -29,21 +31,29 @@ ) app.include_router(api_router) +for handler in ExceptionHandlers.get_handlers(): + app.add_exception_handler(FunctionalityNotConfigured, handler) def start_webserver() -> None: """Run any pending DB migrations and start the webserver.""" logger.info("****************fidesops****************") - logger.info("Running any pending DB migrations...") - init_db(config.database.SQLALCHEMY_DATABASE_URI) + if config.database.ENABLED: + # don't run db migrations if database is disabled + logger.info("Running any pending DB migrations...") + init_db(config.database.SQLALCHEMY_DATABASE_URI) + scheduler.start() - logger.info("Starting scheduled request intake...") - initiate_scheduled_request_intake() + if config.database.ENABLED: + # don't schedule request intake if database is disabled + logger.info("Starting scheduled request intake...") + initiate_scheduled_request_intake() logger.info("Starting web server...") + uvicorn.run( - "src.fidesops.main:app", + "fidesops.main:app", host="0.0.0.0", port=8080, log_config=None, diff --git a/tests/conftest.py b/tests/conftest.py index 38d71e0e3..7de532efb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -45,7 +45,8 @@ def migrate_test_db() -> None: """Apply migrations at beginning and end of testing session""" logger.debug("Applying migrations...") assert config.is_test_mode - init_db(config.database.SQLALCHEMY_TEST_DATABASE_URI) + if config.database.ENABLED: + init_db(config.database.SQLALCHEMY_TEST_DATABASE_URI) logger.debug("Migrations successfully applied") From d39068733b740c7b019de5bfc9885224709edf9a Mon Sep 17 00:00:00 2001 From: Adam Sachs Date: Mon, 23 May 2022 15:53:40 -0400 Subject: [PATCH 2/3] update changelog.md --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ced374f74..edb5be4c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,9 @@ The types of changes are: ## [Unreleased](https://github.com/ethyca/fidesops/compare/1.5.0...main) +### Added +* Added `FIDESOPS__DATABASE__ENABLED` and `FIDESOPS__REDIS__ENABLED` configuration variables to allow `fidesops` to run cleanly in a "stateless" mode without any database or redis cache integration + ### Developer Experience * Import ordering is now enforced using [isort](https://pycqa.github.io/isort/) in CI [#533](https://github.com/ethyca/fidesops/pull/533) From 4810006797dfa07b9464348ab957b3969f10a084 Mon Sep 17 00:00:00 2001 From: Adam Sachs Date: Tue, 24 May 2022 09:38:40 -0400 Subject: [PATCH 3/3] add tests db and redis disabled configs. add tests for endpoint exception handlers. make exception handler a static method. --- src/fidesops/api/v1/exception_handlers.py | 1 + src/fidesops/main.py | 1 - tests/api/test_deps.py | 28 ++++++++++ tests/api/v1/test_exception_handlers.py | 66 +++++++++++++++++++++++ 4 files changed, 95 insertions(+), 1 deletion(-) create mode 100644 tests/api/test_deps.py create mode 100644 tests/api/v1/test_exception_handlers.py diff --git a/src/fidesops/api/v1/exception_handlers.py b/src/fidesops/api/v1/exception_handlers.py index d3206a9bf..9065e4bb1 100644 --- a/src/fidesops/api/v1/exception_handlers.py +++ b/src/fidesops/api/v1/exception_handlers.py @@ -8,6 +8,7 @@ class ExceptionHandlers: + @staticmethod def functionality_not_configured_handler( request: Request, exc: FunctionalityNotConfigured ) -> JSONResponse: diff --git a/src/fidesops/main.py b/src/fidesops/main.py index 4a987ad0a..da2843e84 100644 --- a/src/fidesops/main.py +++ b/src/fidesops/main.py @@ -51,7 +51,6 @@ def start_webserver() -> None: initiate_scheduled_request_intake() logger.info("Starting web server...") - uvicorn.run( "fidesops.main:app", host="0.0.0.0", diff --git a/tests/api/test_deps.py b/tests/api/test_deps.py new file mode 100644 index 000000000..61cd16700 --- /dev/null +++ b/tests/api/test_deps.py @@ -0,0 +1,28 @@ +import pytest + +from fidesops.api.deps import get_cache, get_db +from fidesops.common_exceptions import FunctionalityNotConfigured +from fidesops.core import config + + +@pytest.fixture +def mock_config(): + db_enabled = config.config.database.ENABLED + redis_enabled = config.config.redis.ENABLED + config.config.database.ENABLED = False + config.config.redis.ENABLED = False + yield + config.config.database.ENABLED = db_enabled + config.config.redis.ENABLED = redis_enabled + + +@pytest.mark.usefixtures("mock_config") +def test_get_cache_not_enabled(): + with pytest.raises(FunctionalityNotConfigured): + next(get_cache()) + + +@pytest.mark.usefixtures("mock_config") +def test_get_db_not_enabled(): + with pytest.raises(FunctionalityNotConfigured): + next(get_db()) diff --git a/tests/api/v1/test_exception_handlers.py b/tests/api/v1/test_exception_handlers.py new file mode 100644 index 000000000..47fbc401a --- /dev/null +++ b/tests/api/v1/test_exception_handlers.py @@ -0,0 +1,66 @@ +import json + +import pytest +from starlette.testclient import TestClient + +from fidesops.api.v1.scope_registry import CLIENT_CREATE +from fidesops.api.v1.urn_registry import CLIENT, HEALTH, PRIVACY_REQUESTS, V1_URL_PREFIX +from fidesops.core import config + + +@pytest.fixture +def mock_config_db_disabled(): + db_enabled = config.config.database.ENABLED + config.config.database.ENABLED = False + yield + config.config.database.ENABLED = db_enabled + +@pytest.fixture +def mock_config_redis_disabled(): + redis_enabled = config.config.redis.ENABLED + config.config.redis.ENABLED = False + yield + config.config.redis.ENABLED = redis_enabled + + +class TestExceptionHandlers: + @pytest.mark.usefixtures("mock_config_db_disabled") + def test_db_disabled(self, api_client: TestClient, generate_auth_header): + auth_header = generate_auth_header([CLIENT_CREATE]) + # oauth endpoint should not work + expected_response = {"message": "Application database required, but it is currently disabled! Please update your application configuration to enable integration with an application database."} + response = api_client.post(V1_URL_PREFIX + CLIENT, headers=auth_header) + response_body = json.loads(response.text) + assert 500 == response.status_code + assert expected_response == response_body + + # health endpoint should still work + expected_response = {"healthy": True} + response = api_client.get(HEALTH) + response_body = json.loads(response.text) + assert 200 == response.status_code + assert expected_response == response_body + + @pytest.mark.usefixtures("mock_config_redis_disabled") + def test_redis_disabled(self, api_client: TestClient, generate_auth_header): + auth_header = generate_auth_header([CLIENT_CREATE]) + # Privacy requests endpoint should not work + request_body = [ + { + "requested_at": "2021-08-30T16:09:37.359Z", + "identity": { "email": "customer-1@example.com" }, + "policy_key": "my_separate_policy" + } + ] + expected_response = {"message": "Application redis cache required, but it is currently disabled! Please update your application configuration to enable integration with a redis cache."} + response = api_client.post(V1_URL_PREFIX + PRIVACY_REQUESTS, headers=auth_header, json=request_body) + response_body = json.loads(response.text) + assert 500 == response.status_code + assert expected_response == response_body + + # health endpoint should still work + expected_response = {"healthy": True} + response = api_client.get(HEALTH) + response_body = json.loads(response.text) + assert 200 == response.status_code + assert expected_response == response_body