From 408b941edbee758a770a800f3d64846d0b94c5a6 Mon Sep 17 00:00:00 2001 From: Paul Sanders Date: Thu, 9 Jun 2022 16:55:39 -0400 Subject: [PATCH 1/3] Add fideslib config --- Makefile | 3 +- fidesops.toml | 1 + requirements.txt | 1 + src/fidesops/core/config.py | 205 ++---------------- tests/conftest.py | 3 +- tests/core/test_config.py | 75 ++++--- tests/fixtures/application_fixtures.py | 8 +- tests/fixtures/saas/hubspot_fixtures.py | 4 +- tests/fixtures/saas/mailchimp_fixtures.py | 4 +- tests/fixtures/saas/outreach_fixtures.py | 4 +- tests/fixtures/saas/segment_fixtures.py | 4 +- tests/fixtures/saas/sentry_fixtures.py | 4 +- tests/fixtures/saas/stripe_fixtures.py | 4 +- tests/fixtures/saas_example_fixtures.py | 6 +- .../setup_scripts/mariadb_setup.py | 4 +- .../setup_scripts/postgres_setup.py | 4 +- .../test_external_database_connections.py | 4 +- 17 files changed, 92 insertions(+), 246 deletions(-) diff --git a/Makefile b/Makefile index 3fd5c0c79..759781cc0 100644 --- a/Makefile +++ b/Makefile @@ -126,7 +126,8 @@ pytest: compose-build @docker-compose run \ -e ANALYTICS_OPT_OUT \ $(IMAGE_NAME) \ - pytest $(pytestpath) -m "not integration and not integration_external and not integration_saas" + pytest $(pytestpath) -m "not integration and not integration_external and not integration_saas" --lf -x + @make teardown pytest-integration: diff --git a/fidesops.toml b/fidesops.toml index a098b6789..cda19c198 100644 --- a/fidesops.toml +++ b/fidesops.toml @@ -37,3 +37,4 @@ MASKING_STRICT = true [root_user] ANALYTICS_OPT_OUT = false +ANALYTICS_ID = "73e110fecf8ce7b018b94abc08240cd6" diff --git a/requirements.txt b/requirements.txt index 170721219..158a523d3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,6 +9,7 @@ fastapi-caching[redis] fastapi-pagination[sqlalchemy]~= 0.8.3 fastapi[all]==0.78.0 fideslang==1.0.0 +fideslib==2.0.2 fideslog==1.1.5 multidimensional_urlencode==0.0.4 pandas==1.3.3 diff --git a/src/fidesops/core/config.py b/src/fidesops/core/config.py index 4e44d8de6..59df5929c 100644 --- a/src/fidesops/core/config.py +++ b/src/fidesops/core/config.py @@ -1,82 +1,31 @@ # pylint: disable=C0115,C0116, E0213 -import hashlib import logging import os -from typing import Any, Dict, List, MutableMapping, Optional, Tuple, Union +from typing import Any, Dict, MutableMapping, Optional -import bcrypt import toml +from fideslib.core.config import ( + DatabaseSettings, + FidesSettings, + SecuritySettings, + get_config, + load_file, + load_toml, +) from fideslog.sdk.python.utils import FIDESOPS, generate_client_id -from pydantic import AnyHttpUrl, BaseSettings, PostgresDsn, ValidationError, validator -from pydantic.env_settings import SettingsSourceCallable +from pydantic import validator -from fidesops.common_exceptions import MissingConfig from fidesops.util.logger import NotPii logger = logging.getLogger(__name__) -class FidesSettings(BaseSettings): - """Class used as a base model for configuration subsections.""" - - class Config: - - # Set environment variables to take precedence over init values - @classmethod - def customise_sources( - cls, - init_settings: SettingsSourceCallable, - env_settings: SettingsSourceCallable, - file_secret_settings: SettingsSourceCallable, - ) -> Tuple[SettingsSourceCallable, ...]: - return env_settings, init_settings - - -class DatabaseSettings(FidesSettings): +class FidesopsDatabaseSettings(DatabaseSettings): """Configuration settings for Postgres.""" - SERVER: str - USER: str - PASSWORD: str - DB: str - PORT: str = "5432" - TEST_DB: str = "test" ENABLED: bool = True - SQLALCHEMY_DATABASE_URI: Optional[PostgresDsn] = None - SQLALCHEMY_TEST_DATABASE_URI: Optional[PostgresDsn] = None - - @validator("SQLALCHEMY_DATABASE_URI", pre=True) - def assemble_db_connection(cls, v: Optional[str], values: Dict[str, str]) -> str: - """Join DB connection credentials into a connection string""" - if isinstance(v, str): - return v - return PostgresDsn.build( - scheme="postgresql", - user=values["USER"], - password=values["PASSWORD"], - host=values["SERVER"], - port=values.get("PORT"), - path=f"/{values.get('DB') or ''}", - ) - - @validator("SQLALCHEMY_TEST_DATABASE_URI", pre=True) - def assemble_test_db_connection( - cls, v: Optional[str], values: Dict[str, str] - ) -> str: - """Join DB connection credentials into a connection string""" - if isinstance(v, str): - return v - return PostgresDsn.build( - scheme="postgresql", - user=values["USER"], - password=values["PASSWORD"], - host=values["SERVER"], - port=values["PORT"], - path=f"/{values.get('TEST_DB') or ''}", - ) - class Config: env_prefix = "FIDESOPS__DATABASE__" @@ -138,64 +87,9 @@ class Config: env_prefix = "FIDESOPS__REDIS__" -class SecuritySettings(FidesSettings): +class FidesopsSecuritySettings(SecuritySettings): """Configuration settings for Security variables.""" - AES_ENCRYPTION_KEY_LENGTH: int = 16 - AES_GCM_NONCE_LENGTH: int = 12 - APP_ENCRYPTION_KEY: str - DRP_JWT_SECRET: str - - @validator("APP_ENCRYPTION_KEY") - def validate_encryption_key_length( - cls, v: Optional[str], values: Dict[str, str] - ) -> Optional[str]: - """Validate the encryption key is exactly 32 characters""" - if v is None: - raise ValueError("APP_ENCRYPTION_KEY value not provided!") - encryption_key = v.encode(values.get("ENCODING", "UTF-8")) - if len(encryption_key) != 32: - raise ValueError( - f"APP_ENCRYPTION_KEY value must be exactly 32 characters, " - f"received {len(encryption_key)} characters!" - ) - return v - - CORS_ORIGINS: List[AnyHttpUrl] = [] - - @validator("CORS_ORIGINS", pre=True) - def assemble_cors_origins(cls, v: Union[str, List[str]]) -> Union[List[str], str]: - """Return a list of valid origins for CORS requests""" - if isinstance(v, str) and not v.startswith("["): - return [i.strip() for i in v.split(",")] - if isinstance(v, (list, str)): - return v - raise ValueError(v) - - ENCODING: str = "UTF-8" - - # OAuth - OAUTH_ROOT_CLIENT_ID: str - OAUTH_ROOT_CLIENT_SECRET: str - OAUTH_ROOT_CLIENT_SECRET_HASH: Optional[Tuple] - OAUTH_ACCESS_TOKEN_EXPIRE_MINUTES: int = 60 * 24 * 8 - OAUTH_CLIENT_ID_LENGTH_BYTES = 16 - OAUTH_CLIENT_SECRET_LENGTH_BYTES = 16 - - @validator("OAUTH_ROOT_CLIENT_SECRET_HASH", pre=True) - def assemble_root_access_token( - cls, v: Optional[str], values: Dict[str, str] - ) -> Tuple: - """Returns a hashed value of the root access key. This is hashed as it is not wise to - return a plaintext for of the root credential anywhere in the system""" - value = values["OAUTH_ROOT_CLIENT_SECRET"] - encoding = values["ENCODING"] - assert value is not None - assert encoding is not None - salt = bcrypt.gensalt() - hashed_client_id = hashlib.sha512(value.encode(encoding) + salt).hexdigest() - return hashed_client_id, salt - LOG_LEVEL: str = "INFO" @validator("LOG_LEVEL", pre=True) @@ -252,10 +146,10 @@ class Config: class FidesopsConfig(FidesSettings): """Configuration variables for the FastAPI project""" - database: DatabaseSettings + database: FidesopsDatabaseSettings package = PackageSettings() redis: RedisSettings - security: SecuritySettings + security: FidesopsSecuritySettings execution: ExecutionSettings root_user: RootUserSettings @@ -286,71 +180,6 @@ def log_all_config_values(self) -> None: ) -def load_file(file_name: str) -> str: - """Load a file and from the first matching location. - - In order, will check: - - A path set at ENV variable FIDESOPS__CONFIG_PATH - - The current directory - - The parent directory - - users home (~) directory - - raises FileNotFound if none is found - """ - possible_directories = [ - os.getenv("FIDESOPS__CONFIG_PATH"), - os.curdir, - os.pardir, - os.path.expanduser("~"), - ] - - directories: List[str] = [d for d in possible_directories if d] - - for dir_str in directories: - possible_location = os.path.join(dir_str, file_name) - if possible_location and os.path.isfile(possible_location): - logger.info("Loading file %s from %s", NotPii(file_name), NotPii(dir_str)) - return possible_location - logger.debug("%s not found at %s", NotPii(file_name), NotPii(dir_str)) - raise FileNotFoundError - - -def load_toml(file_name: str) -> MutableMapping[str, Any]: - """ - Load toml file from possible locations specified in load_file. - - Will raise FileNotFoundError or ValidationError on missing or - bad file - """ - return toml.load(load_file(file_name)) - - -def get_config() -> FidesopsConfig: - """ - Attempt to read config file from: - a) env var FIDESOPS__CONFIG_PATH - b) local directory - c) parent directory - d) home directory - This will fail on the first encountered bad conf file. - """ - try: - return FidesopsConfig.parse_obj(load_toml("fidesops.toml")) - except (FileNotFoundError) as e: - logger.warning("fidesops.toml could not be loaded: %s", NotPii(e)) - # If no path is specified Pydantic will attempt to read settings from - # the environment. Default values will still be used if the matching - # environment variable is not set. - try: - return FidesopsConfig() - except ValidationError as exc: - logger.error("Fidesops config could not be loaded: %s", NotPii(exc)) - # If FidesopsConfig is missing any required values Pydantic will throw - # an ImportError. This means the config has not been correctly specified - # so we can throw the missing config error. - raise MissingConfig(exc.args[0]) - - CONFIG_KEY_ALLOWLIST = { "database": [ "SERVER", @@ -403,8 +232,8 @@ def update_config_file(updates: Dict[str, Dict[str, Any]]) -> None: :param updates: A nested `dict`, where top-level keys correspond to configuration sections and top-level values contain `dict`s whose key/value pairs correspond to the desired option/value updates. """ try: - config_path: str = load_file("fidesops.toml") - current_config: MutableMapping[str, Any] = load_toml("fidesops.toml") + config_path: str = load_file(["fidesops.toml"]) + current_config: MutableMapping[str, Any] = load_toml(["fidesops.toml"]) except FileNotFoundError as e: logger.warning("fidesops.toml could not be loaded: %s", NotPii(e)) @@ -424,7 +253,7 @@ def update_config_file(updates: Dict[str, Dict[str, Any]]) -> None: logger.info("\tSet %s.%s = %s", NotPii(key), NotPii(subkey), NotPii(val)) -config = get_config() +config = get_config(FidesopsConfig) # `censored_config` is included below because it's important we keep the censored # config at parity with `config`. This means if we change the path at which fidesops # loads `config`, we should also change `censored_config`. diff --git a/tests/conftest.py b/tests/conftest.py index 1140e8b2b..6c0fefce2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,6 +6,7 @@ import pytest from fastapi.testclient import TestClient +from fideslib.core.config import load_toml from sqlalchemy_utils.functions import create_database, database_exists, drop_database from fidesops.api.v1.scope_registry import SCOPE_REGISTRY @@ -154,4 +155,4 @@ def _build_jwt(webhook: PolicyPreWebhook) -> Dict[str, str]: @pytest.fixture(scope="session") def integration_config() -> MutableMapping[str, Any]: - yield load_toml("fidesops-integration.toml") + yield load_toml(["fidesops-integration.toml"]) diff --git a/tests/core/test_config.py b/tests/core/test_config.py index 626d8aca3..a0e353c0d 100644 --- a/tests/core/test_config.py +++ b/tests/core/test_config.py @@ -3,14 +3,15 @@ from unittest.mock import patch import pytest +from fideslib.core.config import get_config from pydantic import ValidationError -from fidesops.core.config import get_config +from fidesops.core.config import FidesopsConfig def test_config_from_default() -> None: "Test building a config from default local TOML" - config = get_config() + config = get_config(FidesopsConfig) assert config.database.SERVER == "db" assert config.redis.HOST == "redis" @@ -24,9 +25,10 @@ def test_config_from_default() -> None: }, clear=True, ) +@pytest.mark.xfail(strict=True, reason="Fix available in next release of fideslib") def test_config_from_path() -> None: """Test reading config using the FIDESOPS__CONFIG_PATH option.""" - config = get_config() + config = get_config(FidesopsConfig) assert config.database.SERVER == "testserver" assert config.redis.HOST == "testredis" assert config.security.APP_ENCRYPTION_KEY == "atestencryptionkeythatisvalidlen" @@ -42,26 +44,33 @@ def test_config_from_path() -> None: ) def test_config_from_env_vars() -> None: """Test overriding config using ENV vars.""" - config = get_config() + config = get_config(FidesopsConfig) assert config.database.SERVER == "envserver" assert config.redis.HOST == "envhost" assert config.security.APP_ENCRYPTION_KEY == "OLMkv91j8DHiDAULnK5Lxx3kSCov30b3" +def test_config_app_encryption_key_validation() -> None: + """Test APP_ENCRYPTION_KEY is validated to be exactly 32 characters.""" + app_encryption_key = "atestencryptionkeythatisvalidlen" + + with patch.dict( + os.environ, + { + "FIDESOPS__SECURITY__APP_ENCRYPTION_KEY": app_encryption_key, + }, + clear=True, + ): + config = get_config(FidesopsConfig) + assert config.security.APP_ENCRYPTION_KEY == app_encryption_key + + @pytest.mark.parametrize( - "app_encryption_key,expected_error", - [ - ("tooshortkey", "must be exactly 32 characters, received 11"), - ( - "muchmuchmuchmuchmuchmuchmuchmuchtoolongkey", - "must be exactly 32 characters, received 42", - ), - ("atestencryptionkeythatisvalidlen", None), - ], + "app_encryption_key", + ["tooshortkey", "muchmuchmuchmuchmuchmuchmuchmuchtoolongkey"], ) -def test_config_app_encryption_key_validation( - app_encryption_key, expected_error -) -> None: +@pytest.mark.xfail(strict=True, reason="Fix available in next release of fideslib") +def test_config_app_encryption_key_validation_length_error(app_encryption_key) -> None: """Test APP_ENCRYPTION_KEY is validated to be exactly 32 characters.""" with patch.dict( os.environ, @@ -70,13 +79,9 @@ def test_config_app_encryption_key_validation( }, clear=True, ): - if expected_error is not None: - with pytest.raises(ValidationError) as err: - config = get_config() - assert expected_error in str(err.value) - else: - config = get_config() - assert config.security.APP_ENCRYPTION_KEY == app_encryption_key + with pytest.raises(ValidationError) as err: + get_config(FidesopsConfig) + assert "must be exactly 32 characters" in str(err.value) @pytest.mark.parametrize( @@ -88,7 +93,6 @@ def test_config_app_encryption_key_validation( ("WARNING", "WARNING"), ("ERROR", "ERROR"), ("CRITICAL", "CRITICAL"), - ("INVALID", None), ], ) def test_config_log_level(log_level, expected_log_level): @@ -100,10 +104,19 @@ def test_config_log_level(log_level, expected_log_level): }, clear=True, ): - if expected_log_level is not None: - config = get_config() - assert config.security.LOG_LEVEL == expected_log_level - else: - with pytest.raises(ValidationError) as err: - config = get_config() - assert "Invalid LOG_LEVEL" in str(err.value) + config = get_config(FidesopsConfig) + assert config.security.LOG_LEVEL == expected_log_level + + +@pytest.mark.xfail(strict=True, reason="Fix available in next release of fideslib") +def test_config_log_level_invalid(): + with patch.dict( + os.environ, + { + "FIDESOPS__SECURITY__LOG_LEVEL": "INVALID", + }, + clear=True, + ): + with pytest.raises(ValidationError) as err: + get_config(FidesopsConfig) + assert "Invalid LOG_LEVEL" in str(err.value) diff --git a/tests/fixtures/application_fixtures.py b/tests/fixtures/application_fixtures.py index c905894c3..0c8b5c83a 100644 --- a/tests/fixtures/application_fixtures.py +++ b/tests/fixtures/application_fixtures.py @@ -8,11 +8,11 @@ import pytest import yaml from faker import Faker +from fideslib.core.config import load_file, load_toml from sqlalchemy.orm import Session from sqlalchemy.orm.exc import ObjectDeletedError from fidesops.api.v1.scope_registry import PRIVACY_REQUEST_READ, SCOPE_REGISTRY -from fidesops.core.config import load_file, load_toml from fidesops.models.client import ClientDetail from fidesops.models.connectionconfig import ( AccessLevel, @@ -52,7 +52,7 @@ logging.getLogger("faker").setLevel(logging.ERROR) # disable verbose faker logging faker = Faker() -integration_config = load_toml("fidesops-integration.toml") +integration_config = load_toml(["fidesops-integration.toml"]) logger = logging.getLogger(__name__) @@ -952,13 +952,13 @@ def dataset_config_preview( def load_dataset(filename: str) -> Dict: - yaml_file = load_file(filename) + yaml_file = load_file([filename]) with open(yaml_file, "r") as file: return yaml.safe_load(file).get("dataset", []) def load_dataset_as_string(filename: str) -> str: - yaml_file = load_file(filename) + yaml_file = load_file([filename]) with open(yaml_file, "r") as file: return file.read() diff --git a/tests/fixtures/saas/hubspot_fixtures.py b/tests/fixtures/saas/hubspot_fixtures.py index 144006651..cbc4ba9c2 100644 --- a/tests/fixtures/saas/hubspot_fixtures.py +++ b/tests/fixtures/saas/hubspot_fixtures.py @@ -5,9 +5,9 @@ import pydash import pytest +from fideslib.core.config import load_toml from sqlalchemy.orm import Session -from fidesops.core.config import load_toml from fidesops.models.connectionconfig import ( AccessLevel, ConnectionConfig, @@ -21,7 +21,7 @@ from tests.fixtures.application_fixtures import load_dataset from tests.fixtures.saas_example_fixtures import load_config -saas_config = load_toml("saas_config.toml") +saas_config = load_toml(["saas_config.toml"]) HUBSPOT_FIRSTNAME = "SomeoneFirstname" diff --git a/tests/fixtures/saas/mailchimp_fixtures.py b/tests/fixtures/saas/mailchimp_fixtures.py index b25311894..37466ce9d 100644 --- a/tests/fixtures/saas/mailchimp_fixtures.py +++ b/tests/fixtures/saas/mailchimp_fixtures.py @@ -4,9 +4,9 @@ import pydash import pytest +from fideslib.core.config import load_toml from sqlalchemy.orm import Session -from fidesops.core.config import load_toml from fidesops.db import session from fidesops.models.connectionconfig import ( AccessLevel, @@ -19,7 +19,7 @@ from tests.fixtures.application_fixtures import load_dataset from tests.fixtures.saas_example_fixtures import load_config -saas_config = load_toml("saas_config.toml") +saas_config = load_toml(["saas_config.toml"]) @pytest.fixture(scope="function") diff --git a/tests/fixtures/saas/outreach_fixtures.py b/tests/fixtures/saas/outreach_fixtures.py index 875a8e045..4abc0a490 100644 --- a/tests/fixtures/saas/outreach_fixtures.py +++ b/tests/fixtures/saas/outreach_fixtures.py @@ -3,9 +3,9 @@ import pydash import pytest +from fideslib.core.config import load_toml from sqlalchemy.orm import Session -from fidesops.core.config import load_toml from fidesops.db import session from fidesops.models.connectionconfig import ( AccessLevel, @@ -16,7 +16,7 @@ from tests.fixtures.application_fixtures import load_dataset from tests.fixtures.saas_example_fixtures import load_config -saas_config = load_toml("saas_config.toml") +saas_config = load_toml(["saas_config.toml"]) @pytest.fixture(scope="function") diff --git a/tests/fixtures/saas/segment_fixtures.py b/tests/fixtures/saas/segment_fixtures.py index ee23bf825..8d2c07bcc 100644 --- a/tests/fixtures/saas/segment_fixtures.py +++ b/tests/fixtures/saas/segment_fixtures.py @@ -7,9 +7,9 @@ import pytest import requests from faker import Faker +from fideslib.core.config import load_toml from sqlalchemy.orm import Session -from fidesops.core.config import load_toml from fidesops.db import session from fidesops.models.connectionconfig import ( AccessLevel, @@ -20,7 +20,7 @@ from tests.fixtures.application_fixtures import load_dataset from tests.fixtures.saas_example_fixtures import load_config -saas_config = load_toml("saas_config.toml") +saas_config = load_toml(["saas_config.toml"]) @pytest.fixture(scope="function") diff --git a/tests/fixtures/saas/sentry_fixtures.py b/tests/fixtures/saas/sentry_fixtures.py index 923953b24..ac16c3bf9 100644 --- a/tests/fixtures/saas/sentry_fixtures.py +++ b/tests/fixtures/saas/sentry_fixtures.py @@ -3,9 +3,9 @@ import pydash import pytest +from fideslib.core.config import load_toml from sqlalchemy.orm import Session -from fidesops.core.config import load_toml from fidesops.db import session from fidesops.models.connectionconfig import ( AccessLevel, @@ -16,7 +16,7 @@ from tests.fixtures.application_fixtures import load_dataset from tests.fixtures.saas_example_fixtures import load_config -saas_config = load_toml("saas_config.toml") +saas_config = load_toml(["saas_config.toml"]) @pytest.fixture(scope="function") diff --git a/tests/fixtures/saas/stripe_fixtures.py b/tests/fixtures/saas/stripe_fixtures.py index 7c476a991..900a22ca5 100644 --- a/tests/fixtures/saas/stripe_fixtures.py +++ b/tests/fixtures/saas/stripe_fixtures.py @@ -4,10 +4,10 @@ import pydash import pytest import requests +from fideslib.core.config import load_toml from multidimensional_urlencode import urlencode as multidimensional_urlencode from sqlalchemy.orm import Session -from fidesops.core.config import load_toml from fidesops.db import session from fidesops.models.connectionconfig import ( AccessLevel, @@ -18,7 +18,7 @@ from tests.fixtures.application_fixtures import load_dataset from tests.fixtures.saas_example_fixtures import load_config -saas_config = load_toml("saas_config.toml") +saas_config = load_toml(["saas_config.toml"]) @pytest.fixture(scope="function") diff --git a/tests/fixtures/saas_example_fixtures.py b/tests/fixtures/saas_example_fixtures.py index 4dbe81b97..4cfd42387 100644 --- a/tests/fixtures/saas_example_fixtures.py +++ b/tests/fixtures/saas_example_fixtures.py @@ -3,9 +3,9 @@ import pydash import pytest import yaml +from fideslib.core.config import load_file, load_toml from sqlalchemy.orm import Session -from fidesops.core.config import load_file, load_toml from fidesops.models.connectionconfig import ( AccessLevel, ConnectionConfig, @@ -19,12 +19,12 @@ def load_config(filename: str) -> Dict: - yaml_file = load_file(filename) + yaml_file = load_file([filename]) with open(yaml_file, "r") as file: return yaml.safe_load(file).get("saas_config", []) -saas_config = load_toml("saas_config.toml") +saas_config = load_toml(["saas_config.toml"]) @pytest.fixture(scope="function") diff --git a/tests/integration_tests/setup_scripts/mariadb_setup.py b/tests/integration_tests/setup_scripts/mariadb_setup.py index 41ce6a254..b8a46d898 100644 --- a/tests/integration_tests/setup_scripts/mariadb_setup.py +++ b/tests/integration_tests/setup_scripts/mariadb_setup.py @@ -3,8 +3,8 @@ import pydash import sqlalchemy +from fideslib.core.config import load_toml -from fidesops.core.config import load_toml from fidesops.db.session import get_db_engine, get_db_session from fidesops.models.connectionconfig import ( AccessLevel, @@ -13,7 +13,7 @@ ) from fidesops.service.connectors.sql_connector import MariaDBConnector -integration_config = load_toml("fidesops-integration.toml") +integration_config = load_toml(["fidesops-integration.toml"]) def truncate_tables(db_session): diff --git a/tests/integration_tests/setup_scripts/postgres_setup.py b/tests/integration_tests/setup_scripts/postgres_setup.py index 935e08820..fbfaccf7e 100644 --- a/tests/integration_tests/setup_scripts/postgres_setup.py +++ b/tests/integration_tests/setup_scripts/postgres_setup.py @@ -2,9 +2,9 @@ import pydash import sqlalchemy +from fideslib.core.config import load_toml from sqlalchemy_utils.functions import create_database, database_exists, drop_database -from fidesops.core.config import load_toml from fidesops.db.session import get_db_engine, get_db_session from fidesops.models.connectionconfig import ( AccessLevel, @@ -13,7 +13,7 @@ ) from fidesops.service.connectors.sql_connector import PostgreSQLConnector -integration_config = load_toml("fidesops-integration.toml") +integration_config = load_toml(["fidesops-integration.toml"]) def setup(): diff --git a/tests/integration_tests/test_external_database_connections.py b/tests/integration_tests/test_external_database_connections.py index 5abe00181..26ba0e6b5 100644 --- a/tests/integration_tests/test_external_database_connections.py +++ b/tests/integration_tests/test_external_database_connections.py @@ -3,9 +3,9 @@ from typing import Generator import pytest +from fideslib.core.config import load_toml from sqlalchemy import inspect -from fidesops.core.config import load_toml from fidesops.models.connectionconfig import ConnectionConfig, ConnectionType from fidesops.schemas.connection_configuration import RedshiftSchema, SnowflakeSchema from fidesops.service.connectors import ( @@ -16,7 +16,7 @@ logger = logging.getLogger(__name__) -integration_config = load_toml("fidesops-integration.toml") +integration_config = load_toml(["fidesops-integration.toml"]) @pytest.fixture(scope="session") From ee244a3ad2d8c952075ac7dc6abbc6558aa484b5 Mon Sep 17 00:00:00 2001 From: Paul Sanders Date: Fri, 10 Jun 2022 19:25:58 -0400 Subject: [PATCH 2/3] Bump fideslib from 2.0.2 to 2.0.3 --- requirements.txt | 2 +- tests/core/test_config.py | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index 158a523d3..2e922ac17 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,7 @@ fastapi-caching[redis] fastapi-pagination[sqlalchemy]~= 0.8.3 fastapi[all]==0.78.0 fideslang==1.0.0 -fideslib==2.0.2 +fideslib==2.0.3 fideslog==1.1.5 multidimensional_urlencode==0.0.4 pandas==1.3.3 diff --git a/tests/core/test_config.py b/tests/core/test_config.py index a0e353c0d..25eba90e3 100644 --- a/tests/core/test_config.py +++ b/tests/core/test_config.py @@ -25,7 +25,6 @@ def test_config_from_default() -> None: }, clear=True, ) -@pytest.mark.xfail(strict=True, reason="Fix available in next release of fideslib") def test_config_from_path() -> None: """Test reading config using the FIDESOPS__CONFIG_PATH option.""" config = get_config(FidesopsConfig) @@ -69,7 +68,6 @@ def test_config_app_encryption_key_validation() -> None: "app_encryption_key", ["tooshortkey", "muchmuchmuchmuchmuchmuchmuchmuchtoolongkey"], ) -@pytest.mark.xfail(strict=True, reason="Fix available in next release of fideslib") def test_config_app_encryption_key_validation_length_error(app_encryption_key) -> None: """Test APP_ENCRYPTION_KEY is validated to be exactly 32 characters.""" with patch.dict( @@ -108,7 +106,6 @@ def test_config_log_level(log_level, expected_log_level): assert config.security.LOG_LEVEL == expected_log_level -@pytest.mark.xfail(strict=True, reason="Fix available in next release of fideslib") def test_config_log_level_invalid(): with patch.dict( os.environ, From 7888f2900b78c7943dc927d008cdb9387ebacf21 Mon Sep 17 00:00:00 2001 From: Paul Sanders Date: Fri, 10 Jun 2022 21:18:59 -0400 Subject: [PATCH 3/3] Fix mypy error --- src/fidesops/core/config.py | 2 +- tests/core/test_config.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fidesops/core/config.py b/src/fidesops/core/config.py index 78e7ad74f..45ebffdaa 100644 --- a/src/fidesops/core/config.py +++ b/src/fidesops/core/config.py @@ -147,7 +147,7 @@ class Config: # pylint: disable=C0115 def log_all_config_values(self) -> None: """Output DEBUG logs of all the config values.""" for settings in [self.database, self.redis, self.security, self.execution]: - for key, value in settings.dict().items(): + for key, value in settings.dict().items(): # type: ignore logger.debug( "Using config: %s%s = %s", NotPii(settings.Config.env_prefix), # type: ignore diff --git a/tests/core/test_config.py b/tests/core/test_config.py index 25eba90e3..d276e66fa 100644 --- a/tests/core/test_config.py +++ b/tests/core/test_config.py @@ -21,7 +21,7 @@ def test_config_from_default() -> None: @patch.dict( os.environ, { - "FIDESOPS__CONFIG_PATH": "data/config/", + "FIDES__CONFIG_PATH": "data/config/", }, clear=True, )