diff --git a/docs/fidesops/docs/guides/saas_config.md b/docs/fidesops/docs/guides/saas_config.md index 9b0a15ab6..6cc06fe7d 100644 --- a/docs/fidesops/docs/guides/saas_config.md +++ b/docs/fidesops/docs/guides/saas_config.md @@ -191,7 +191,7 @@ This is where we define how we are going to access and update each collection in - `pagination` An optional strategy used to get the next set of results from APIs with resources spanning multiple pages. Details can be found under [SaaS Pagination](saas_pagination.md). - `grouped_inputs` An optional list of reference fields whose inputs are dependent upon one another. For example, an endpoint may need both an `organization_id` and a `project_id` from another endpoint. These aren't independent values, as a `project_id` belongs to an `organization_id`. You would specify this as ["organization_id", "project_id"]. -## Request params in more detail +## Param values in more detail The `param_values` list is what provides the values to our various placeholders in the path, headers, query params and body. Values can be `identities` such as email or phone number, `references` to fields in other collections, or `connector_params` which are defined as part of configuring a SaaS connector. Whenever a placeholder is encountered, the placeholder name is looked up in the list of `param_values` and corresponding value is used instead. Here is an example of placeholders being used in various locations: ```yaml diff --git a/saas_config.toml b/saas_config.toml index 1385cfb34..f0e774f61 100644 --- a/saas_config.toml +++ b/saas_config.toml @@ -9,4 +9,11 @@ page_limit = "10" domain = "" username = "" api_key = "" -identity_email = "" \ No newline at end of file +identity_email = "" + +[stripe] +host = "" +api_key = "" +payment_types = "" +page_limit = "" +identity_email = "" diff --git a/tests/api/v1/endpoints/test_connection_config_endpoints.py b/tests/api/v1/endpoints/test_connection_config_endpoints.py index 7a691a7cb..d94e56f84 100644 --- a/tests/api/v1/endpoints/test_connection_config_endpoints.py +++ b/tests/api/v1/endpoints/test_connection_config_endpoints.py @@ -815,17 +815,17 @@ def test_put_http_connection_config_secrets( assert https_connection_config.last_test_succeeded is None @pytest.mark.unit_saas - def test_put_connection_config_saas_example_secrets( + def test_put_saas_example_connection_config_secrets( self, api_client: TestClient, db: Session, generate_auth_header, - connection_config_saas_example, - saas_secrets, + saas_example_connection_config, + saas_example_secrets, ): auth_header = generate_auth_header(scopes=[CONNECTION_CREATE_OR_UPDATE]) - url = f"{V1_URL_PREFIX}{CONNECTIONS}/{connection_config_saas_example.key}/secret" - payload = saas_secrets["saas_example"] + url = f"{V1_URL_PREFIX}{CONNECTIONS}/{saas_example_connection_config.key}/secret" + payload = saas_example_secrets resp = api_client.put( url + "?verify=False", @@ -837,25 +837,25 @@ def test_put_connection_config_saas_example_secrets( body = json.loads(resp.text) assert ( body["msg"] - == f"Secrets updated for ConnectionConfig with key: {connection_config_saas_example.key}." + == f"Secrets updated for ConnectionConfig with key: {saas_example_connection_config.key}." ) - db.refresh(connection_config_saas_example) - assert connection_config_saas_example.secrets == saas_secrets["saas_example"] - assert connection_config_saas_example.last_test_timestamp is None - assert connection_config_saas_example.last_test_succeeded is None + db.refresh(saas_example_connection_config) + assert saas_example_connection_config.secrets == saas_example_secrets + assert saas_example_connection_config.last_test_timestamp is None + assert saas_example_connection_config.last_test_succeeded is None @pytest.mark.unit_saas - def test_put_connection_config_saas_example_secrets_missing_saas_config( + def test_put_saas_example_connection_config_secrets_missing_saas_config( self, api_client: TestClient, generate_auth_header, - connection_config_saas_example_without_saas_config, - saas_secrets, + saas_example_connection_config_without_saas_config, + saas_example_secrets, ): auth_header = generate_auth_header(scopes=[CONNECTION_CREATE_OR_UPDATE]) - url = f"{V1_URL_PREFIX}{CONNECTIONS}/{connection_config_saas_example_without_saas_config.key}/secret" - payload = saas_secrets["saas_example"] + url = f"{V1_URL_PREFIX}{CONNECTIONS}/{saas_example_connection_config_without_saas_config.key}/secret" + payload = saas_example_secrets resp = api_client.put( url + "?verify=False", diff --git a/tests/api/v1/endpoints/test_dataset_endpoints.py b/tests/api/v1/endpoints/test_dataset_endpoints.py index 23672ae35..a28489a7d 100644 --- a/tests/api/v1/endpoints/test_dataset_endpoints.py +++ b/tests/api/v1/endpoints/test_dataset_endpoints.py @@ -359,14 +359,14 @@ def test_validate_dataset_that_references_another_dataset( def test_validate_saas_dataset_invalid_traversal( self, db, - connection_config_saas_example_with_invalid_saas_config, - saas_datasets, + saas_example_connection_config_with_invalid_saas_config, + saas_example_dataset, api_client: TestClient, generate_auth_header, ): path = V1_URL_PREFIX + DATASET_VALIDATE path_params = { - "connection_key": connection_config_saas_example_with_invalid_saas_config.key + "connection_key": saas_example_connection_config_with_invalid_saas_config.key } validate_dataset_url = path.format(**path_params) @@ -374,7 +374,7 @@ def test_validate_saas_dataset_invalid_traversal( response = api_client.put( validate_dataset_url, headers=auth_header, - json=saas_datasets["saas_example"], + json=saas_example_dataset, ) assert response.status_code == 200 @@ -669,19 +669,19 @@ def test_patch_datasets_bulk_update( @pytest.mark.unit_saas def test_patch_datasets_missing_saas_config( self, - connection_config_saas_example_without_saas_config, - saas_datasets, + saas_example_connection_config_without_saas_config, + saas_example_dataset, api_client: TestClient, db: Session, generate_auth_header, ): path = V1_URL_PREFIX + DATASETS - path_params = {"connection_key": connection_config_saas_example_without_saas_config.key} + path_params = {"connection_key": saas_example_connection_config_without_saas_config.key} datasets_url = path.format(**path_params) auth_header = generate_auth_header(scopes=[DATASET_CREATE_OR_UPDATE]) response = api_client.patch( - datasets_url, headers=auth_header, json=[saas_datasets["saas_example"]] + datasets_url, headers=auth_header, json=[saas_example_dataset] ) assert response.status_code == 200 @@ -690,24 +690,24 @@ def test_patch_datasets_missing_saas_config( assert len(response_body["failed"]) == 1 assert ( response_body["failed"][0]["message"] - == f"Connection config '{connection_config_saas_example_without_saas_config.key}' " + == f"Connection config '{saas_example_connection_config_without_saas_config.key}' " "must have a SaaS config before validating or adding a dataset" ) @pytest.mark.unit_saas def test_patch_datasets_extra_reference( self, - connection_config_saas_example, - saas_datasets, + saas_example_connection_config, + saas_example_dataset, api_client: TestClient, db: Session, generate_auth_header, ): path = V1_URL_PREFIX + DATASETS - path_params = {"connection_key": connection_config_saas_example.key} + path_params = {"connection_key": saas_example_connection_config.key} datasets_url = path.format(**path_params) - invalid_dataset = saas_datasets["saas_example"] + invalid_dataset = saas_example_dataset invalid_dataset["collections"][0]["fields"][0]["fidesops_meta"] = { "references": [ { @@ -736,17 +736,17 @@ def test_patch_datasets_extra_reference( @pytest.mark.unit_saas def test_patch_datasets_extra_identity( self, - connection_config_saas_example, - saas_datasets, + saas_example_connection_config, + saas_example_dataset, api_client: TestClient, db: Session, generate_auth_header, ): path = V1_URL_PREFIX + DATASETS - path_params = {"connection_key": connection_config_saas_example.key} + path_params = {"connection_key": saas_example_connection_config.key} datasets_url = path.format(**path_params) - invalid_dataset = saas_datasets["saas_example"] + invalid_dataset = saas_example_dataset invalid_dataset["collections"][0]["fields"][0]["fidesops_meta"] = { "identity": "email" } @@ -769,17 +769,17 @@ def test_patch_datasets_extra_identity( @pytest.mark.unit_saas def test_patch_datasets_fides_key_mismatch( self, - connection_config_saas_example, - saas_datasets, + saas_example_connection_config, + saas_example_dataset, api_client: TestClient, db: Session, generate_auth_header, ): path = V1_URL_PREFIX + DATASETS - path_params = {"connection_key": connection_config_saas_example.key} + path_params = {"connection_key": saas_example_connection_config.key} datasets_url = path.format(**path_params) - invalid_dataset = saas_datasets["saas_example"] + invalid_dataset = saas_example_dataset invalid_dataset["fides_key"] = "different_key" auth_header = generate_auth_header(scopes=[DATASET_CREATE_OR_UPDATE]) diff --git a/tests/api/v1/endpoints/test_saas_config_endpoints.py b/tests/api/v1/endpoints/test_saas_config_endpoints.py index 904c47bc1..dbfadfd47 100644 --- a/tests/api/v1/endpoints/test_saas_config_endpoints.py +++ b/tests/api/v1/endpoints/test_saas_config_endpoints.py @@ -24,22 +24,22 @@ @pytest.mark.unit_saas class TestValidateSaaSConfig: @pytest.fixture - def validate_saas_config_url(self, connection_config_saas_example) -> str: + def validate_saas_config_url(self, saas_example_connection_config) -> str: path = V1_URL_PREFIX + SAAS_CONFIG_VALIDATE - path_params = {"connection_key": connection_config_saas_example.key} + path_params = {"connection_key": saas_example_connection_config.key} return path.format(**path_params) def test_put_validate_saas_config_not_authenticated( - self, saas_configs, validate_saas_config_url: str, api_client + self, saas_example_config, validate_saas_config_url: str, api_client ) -> None: response = api_client.put( - validate_saas_config_url, headers={}, json=saas_configs["saas_example"] + validate_saas_config_url, headers={}, json=saas_example_config ) assert response.status_code == 401 def test_put_validate_dataset_wrong_scope( self, - saas_configs, + saas_example_config, validate_saas_config_url, api_client: TestClient, generate_auth_header, @@ -48,19 +48,19 @@ def test_put_validate_dataset_wrong_scope( response = api_client.put( validate_saas_config_url, headers=auth_header, - json=saas_configs["saas_example"], + json=saas_example_config, ) assert response.status_code == 403 def test_put_validate_saas_config_missing_key( self, - saas_configs, + saas_example_config, validate_saas_config_url, api_client: TestClient, generate_auth_header, ) -> None: auth_header = generate_auth_header(scopes=[SAAS_CONFIG_READ]) - invalid_config = _reject_key(saas_configs["saas_example"], "fides_key") + invalid_config = _reject_key(saas_example_config, "fides_key") response = api_client.put( validate_saas_config_url, headers=auth_header, json=invalid_config ) @@ -71,13 +71,13 @@ def test_put_validate_saas_config_missing_key( def test_put_validate_saas_config_missing_endpoints( self, - saas_configs, + saas_example_config, validate_saas_config_url, api_client: TestClient, generate_auth_header, ) -> None: auth_header = generate_auth_header(scopes=[SAAS_CONFIG_READ]) - invalid_config = _reject_key(saas_configs["saas_example"], "endpoints") + invalid_config = _reject_key(saas_example_config, "endpoints") response = api_client.put( validate_saas_config_url, headers=auth_header, json=invalid_config ) @@ -88,13 +88,13 @@ def test_put_validate_saas_config_missing_endpoints( def test_put_validate_saas_config_reference_and_identity( self, - saas_configs, + saas_example_config, validate_saas_config_url, api_client: TestClient, generate_auth_header, ) -> None: auth_header = generate_auth_header(scopes=[SAAS_CONFIG_READ]) - saas_config = saas_configs["saas_example"] + saas_config = saas_example_config param_values = saas_config["endpoints"][0]["requests"]["read"][ "param_values" ][0] @@ -115,13 +115,13 @@ def test_put_validate_saas_config_reference_and_identity( def test_put_validate_saas_config_wrong_reference_direction( self, - saas_configs, + saas_example_config, validate_saas_config_url, api_client: TestClient, generate_auth_header, ) -> None: auth_header = generate_auth_header(scopes=[SAAS_CONFIG_READ]) - saas_config = saas_configs["saas_example"] + saas_config = saas_example_config param_values = saas_config["endpoints"][0]["requests"]["read"][ "param_values" ][0] @@ -142,34 +142,34 @@ def test_put_validate_saas_config_wrong_reference_direction( @pytest.mark.unit_saas class TestPutSaaSConfig: @pytest.fixture - def saas_config_url(self, connection_config_saas_example) -> str: + def saas_config_url(self, saas_example_connection_config) -> str: path = V1_URL_PREFIX + SAAS_CONFIG - path_params = {"connection_key": connection_config_saas_example.key} + path_params = {"connection_key": saas_example_connection_config.key} return path.format(**path_params) def test_patch_saas_config_not_authenticated( - self, saas_configs, saas_config_url, api_client + self, saas_example_config, saas_config_url, api_client ) -> None: response = api_client.patch( - saas_config_url, headers={}, json=saas_configs["saas_example"] + saas_config_url, headers={}, json=saas_example_config ) assert response.status_code == 401 def test_patch_saas_config_wrong_scope( self, - saas_configs, + saas_example_config, saas_config_url, api_client: TestClient, generate_auth_header, ) -> None: auth_header = generate_auth_header(scopes=[SAAS_CONFIG_READ]) response = api_client.patch( - saas_config_url, headers=auth_header, json=saas_configs["saas_example"] + saas_config_url, headers=auth_header, json=saas_example_config ) assert response.status_code == 403 def test_patch_saas_config_invalid_connection_key( - self, saas_configs, api_client: TestClient, generate_auth_header + self, saas_example_config, api_client: TestClient, generate_auth_header ) -> None: path = V1_URL_PREFIX + SAAS_CONFIG path_params = {"connection_key": "nonexistent_key"} @@ -177,30 +177,30 @@ def test_patch_saas_config_invalid_connection_key( auth_header = generate_auth_header(scopes=[SAAS_CONFIG_CREATE_OR_UPDATE]) response = api_client.patch( - saas_config_url, headers=auth_header, json=saas_configs["saas_example"] + saas_config_url, headers=auth_header, json=saas_example_config ) assert response.status_code == 404 def test_patch_saas_config_create( self, - connection_config_saas_example_without_saas_config, - saas_configs, + saas_example_connection_config_without_saas_config, + saas_example_config, api_client: TestClient, db: Session, generate_auth_header, ) -> None: path = V1_URL_PREFIX + SAAS_CONFIG - path_params = {"connection_key": connection_config_saas_example_without_saas_config.key} + path_params = {"connection_key": saas_example_connection_config_without_saas_config.key} saas_config_url = path.format(**path_params) auth_header = generate_auth_header(scopes=[SAAS_CONFIG_CREATE_OR_UPDATE]) response = api_client.patch( - saas_config_url, headers=auth_header, json=saas_configs["saas_example"] + saas_config_url, headers=auth_header, json=saas_example_config ) assert response.status_code == 200 updated_config = ConnectionConfig.get_by( - db=db, field="key", value=connection_config_saas_example_without_saas_config.key + db=db, field="key", value=saas_example_connection_config_without_saas_config.key ) db.expire(updated_config) saas_config = updated_config.saas_config @@ -208,21 +208,21 @@ def test_patch_saas_config_create( def test_patch_saas_config_update( self, - saas_configs, + saas_example_config, saas_config_url, api_client: TestClient, db: Session, generate_auth_header, ) -> None: auth_header = generate_auth_header(scopes=[SAAS_CONFIG_CREATE_OR_UPDATE]) - saas_configs["saas_example"]["endpoints"].pop() + saas_example_config["endpoints"].pop() response = api_client.patch( - saas_config_url, headers=auth_header, json=saas_configs["saas_example"] + saas_config_url, headers=auth_header, json=saas_example_config ) assert response.status_code == 200 connection_config = ConnectionConfig.get_by( - db=db, field="key", value=saas_configs["saas_example"]["fides_key"] + db=db, field="key", value=saas_example_config["fides_key"] ) saas_config = connection_config.saas_config assert saas_config is not None @@ -243,32 +243,32 @@ def get_saas_config_url(connection_config: Optional[ConnectionConfig] = None) -> class TestGetSaaSConfig: def test_get_saas_config_not_authenticated( self, - connection_config_saas_example, + saas_example_connection_config, api_client: TestClient, ) -> None: - saas_config_url = get_saas_config_url(connection_config_saas_example) + saas_config_url = get_saas_config_url(saas_example_connection_config) response = api_client.get(saas_config_url, headers={}) assert response.status_code == 401 def test_get_saas_config_wrong_scope( self, - connection_config_saas_example, + saas_example_connection_config, api_client: TestClient, generate_auth_header, ) -> None: - saas_config_url = get_saas_config_url(connection_config_saas_example) + saas_config_url = get_saas_config_url(saas_example_connection_config) auth_header = generate_auth_header(scopes=[SAAS_CONFIG_CREATE_OR_UPDATE]) response = api_client.get(saas_config_url, headers=auth_header) assert response.status_code == 403 def test_get_saas_config_does_not_exist( self, - connection_config_saas_example_without_saas_config, + saas_example_connection_config_without_saas_config, api_client: TestClient, generate_auth_header, ) -> None: saas_config_url = get_saas_config_url( - connection_config_saas_example_without_saas_config + saas_example_connection_config_without_saas_config ) auth_header = generate_auth_header(scopes=[SAAS_CONFIG_READ]) response = api_client.get(saas_config_url, headers=auth_header) @@ -286,11 +286,11 @@ def test_get_saas_config_invalid_connection_key( def test_get_saas_config( self, - connection_config_saas_example, + saas_example_connection_config, api_client: TestClient, generate_auth_header, ) -> None: - saas_config_url = get_saas_config_url(connection_config_saas_example) + saas_config_url = get_saas_config_url(saas_example_connection_config) auth_header = generate_auth_header(scopes=[SAAS_CONFIG_READ]) response = api_client.get(saas_config_url, headers=auth_header) assert response.status_code == 200 @@ -298,7 +298,7 @@ def test_get_saas_config( response_body = json.loads(response.text) assert ( response_body["fides_key"] - == connection_config_saas_example.get_saas_config().fides_key + == saas_example_connection_config.get_saas_config().fides_key ) assert len(response_body["endpoints"]) == 6 @@ -306,31 +306,31 @@ def test_get_saas_config( @pytest.mark.unit_saas class TestDeleteSaaSConfig: def test_delete_saas_config_not_authenticated( - self, connection_config_saas_example, api_client + self, saas_example_connection_config, api_client ) -> None: - saas_config_url = get_saas_config_url(connection_config_saas_example) + saas_config_url = get_saas_config_url(saas_example_connection_config) response = api_client.delete(saas_config_url, headers={}) assert response.status_code == 401 def test_delete_saas_config_wrong_scope( self, - connection_config_saas_example, + saas_example_connection_config, api_client: TestClient, generate_auth_header, ) -> None: - saas_config_url = get_saas_config_url(connection_config_saas_example) + saas_config_url = get_saas_config_url(saas_example_connection_config) auth_header = generate_auth_header(scopes=[SAAS_CONFIG_READ]) response = api_client.delete(saas_config_url, headers=auth_header) assert response.status_code == 403 def test_delete_saas_config_does_not_exist( self, - connection_config_saas_example_without_saas_config, + saas_example_connection_config_without_saas_config, api_client: TestClient, generate_auth_header, ) -> None: saas_config_url = get_saas_config_url( - connection_config_saas_example_without_saas_config + saas_example_connection_config_without_saas_config ) auth_header = generate_auth_header(scopes=[SAAS_CONFIG_DELETE]) response = api_client.delete(saas_config_url, headers=auth_header) @@ -349,14 +349,14 @@ def test_delete_saas_config_invalid_connection_key( def test_delete_saas_config( self, db: Session, - saas_configs, + saas_example_config, api_client: TestClient, generate_auth_header, ) -> None: # Create a new connection config so we don't run into issues trying to clean up an # already deleted fixture fides_key = "saas_config_for_deletion_test" - saas_configs["saas_example"]["fides_key"] = fides_key + saas_example_config["fides_key"] = fides_key config_to_delete = ConnectionConfig.create( db=db, data={ @@ -364,7 +364,7 @@ def test_delete_saas_config( "name": fides_key, "connection_type": ConnectionType.saas, "access": AccessLevel.read, - "saas_config": saas_configs["saas_example"], + "saas_config": saas_example_config, }, ) saas_config_url = get_saas_config_url(config_to_delete) @@ -378,12 +378,12 @@ def test_delete_saas_config( def test_delete_saas_config_with_dataset_and_secrets( self, - connection_config_saas_example, - dataset_config_saas_example, + saas_example_connection_config, + saas_example_dataset_config, api_client: TestClient, generate_auth_header, ) -> None: - saas_config_url = get_saas_config_url(connection_config_saas_example) + saas_config_url = get_saas_config_url(saas_example_connection_config) auth_header = generate_auth_header(scopes=[SAAS_CONFIG_DELETE]) response = api_client.delete(saas_config_url, headers=auth_header) assert response.status_code == 400 @@ -391,7 +391,7 @@ def test_delete_saas_config_with_dataset_and_secrets( response_body = json.loads(response.text) assert ( response_body["detail"] - == f"Must delete the dataset with fides_key '{dataset_config_saas_example.fides_key}' " + == f"Must delete the dataset with fides_key '{saas_example_dataset_config.fides_key}' " "before deleting this SaaS config. Must clear the secrets from this connection " "config before deleting the SaaS config." ) diff --git a/tests/conftest.py b/tests/conftest.py index 4dfa8f844..0eb23056a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -38,7 +38,8 @@ from .fixtures.redshift_fixtures import * from .fixtures.snowflake_fixtures import * from .fixtures.bigquery_fixtures import * -from .fixtures.saas_fixtures import * +from .fixtures.saas_example_fixtures import * +from .fixtures.saas.mailchimp_fixtures import * logger = logging.getLogger(__name__) diff --git a/tests/fixtures/saas/mailchimp_fixtures.py b/tests/fixtures/saas/mailchimp_fixtures.py new file mode 100644 index 000000000..a611f3d52 --- /dev/null +++ b/tests/fixtures/saas/mailchimp_fixtures.py @@ -0,0 +1,116 @@ +import json +from fidesops.core.config import load_toml +from fidesops.db import session +from fidesops.models.connectionconfig import ( + AccessLevel, + ConnectionConfig, + ConnectionType, +) +from fidesops.models.datasetconfig import DatasetConfig +import pytest +import pydash +import os +from typing import Any, Dict, Generator +from fidesops.schemas.saas.shared_schemas import HTTPMethod, SaaSRequestParams +from fidesops.service.connectors.saas_connector import SaaSConnector +from tests.fixtures.application_fixtures import load_dataset +from tests.fixtures.saas_example_fixtures import load_config +from sqlalchemy.orm import Session + +saas_config = load_toml("saas_config.toml") + + +@pytest.fixture(scope="function") +def mailchimp_secrets(): + return { + "domain": pydash.get(saas_config, "mailchimp.domain") + or os.environ.get("MAILCHIMP_DOMAIN"), + "username": pydash.get(saas_config, "mailchimp.username") + or os.environ.get("MAILCHIMP_USERNAME"), + "api_key": pydash.get(saas_config, "mailchimp.api_key") + or os.environ.get("MAILCHIMP_API_KEY"), + } + + +@pytest.fixture(scope="function") +def mailchimp_identity_email(): + return pydash.get(saas_config, "mailchimp.identity_email") or os.environ.get( + "MAILCHIMP_IDENTITY_EMAIL" + ) + + +@pytest.fixture +def mailchimp_config() -> Dict[str, Any]: + return load_config("data/saas/config/mailchimp_config.yml") + + +@pytest.fixture +def mailchimp_dataset() -> Dict[str, Any]: + return load_dataset("data/saas/dataset/mailchimp_dataset.yml")[0] + + +@pytest.fixture(scope="function") +def mailchimp_connection_config( + db: session, mailchimp_config, mailchimp_secrets +) -> Generator: + fides_key = mailchimp_config["fides_key"] + connection_config = ConnectionConfig.create( + db=db, + data={ + "key": fides_key, + "name": fides_key, + "connection_type": ConnectionType.saas, + "access": AccessLevel.write, + "secrets": mailchimp_secrets, + "saas_config": mailchimp_config, + }, + ) + yield connection_config + connection_config.delete(db) + + +@pytest.fixture +def mailchimp_dataset_config( + db: Session, + mailchimp_connection_config: ConnectionConfig, + mailchimp_dataset: Dict[str, Any], +) -> Generator: + fides_key = mailchimp_dataset["fides_key"] + mailchimp_connection_config.name = fides_key + mailchimp_connection_config.key = fides_key + mailchimp_connection_config.save(db=db) + dataset = DatasetConfig.create( + db=db, + data={ + "connection_config_id": mailchimp_connection_config.id, + "fides_key": fides_key, + "dataset": mailchimp_dataset, + }, + ) + yield dataset + dataset.delete(db=db) + + +@pytest.fixture(scope="function") +def reset_mailchimp_data(mailchimp_connection_config, mailchimp_identity_email) -> Generator: + """ + Gets the current value of the resource and restores it after the test is complete. + Used for erasure tests. + """ + connector = SaaSConnector(mailchimp_connection_config) + request: SaaSRequestParams = SaaSRequestParams( + method=HTTPMethod.GET, + path="/3.0/search-members", + query_params={"query": mailchimp_identity_email}, + body=None, + ) + response = connector.create_client().send(request) + body = response.json() + member = body["exact_matches"]["members"][0] + yield member + request: SaaSRequestParams = SaaSRequestParams( + method=HTTPMethod.PUT, + path=f'/3.0/lists/{member["list_id"]}/members/{member["id"]}', + body=json.dumps(member), + ) + connector.create_client().send(request) diff --git a/tests/fixtures/saas_example_fixtures.py b/tests/fixtures/saas_example_fixtures.py new file mode 100644 index 000000000..ab9c20469 --- /dev/null +++ b/tests/fixtures/saas_example_fixtures.py @@ -0,0 +1,126 @@ +import pytest +import pydash +import yaml + +from sqlalchemy.orm import Session +from typing import Any, Dict, Generator + +from fidesops.core.config import load_file, load_toml +from fidesops.models.connectionconfig import ( + AccessLevel, + ConnectionConfig, + ConnectionType, +) +from fidesops.models.datasetconfig import DatasetConfig +from tests.fixtures.application_fixtures import load_dataset + +def load_config(filename: str) -> Dict: + yaml_file = load_file(filename) + with open(yaml_file, "r") as file: + return yaml.safe_load(file).get("saas_config", []) + +saas_config = load_toml("saas_config.toml") + +@pytest.fixture(scope="function") +def saas_example_secrets(): + return { + "domain": pydash.get(saas_config, "saas_example.domain"), + "username": pydash.get(saas_config, "saas_example.username"), + "api_key": pydash.get(saas_config, "saas_example.api_key"), + "api_version": pydash.get(saas_config, "saas_example.api_version"), + "page_limit": pydash.get(saas_config, "saas_example.page_limit"), + } + +@pytest.fixture +def saas_example_config() -> Dict: + return load_config("data/saas/config/saas_example_config.yml") + + +@pytest.fixture +def saas_example_dataset() -> Dict: + return load_dataset("data/saas/dataset/saas_example_dataset.yml")[0] + + +@pytest.fixture(scope="function") +def saas_example_connection_config( + db: Session, + saas_example_config: Dict[str, Any], + saas_example_secrets: Dict[str, Any], +) -> Generator: + fides_key = saas_example_config["fides_key"] + connection_config = ConnectionConfig.create( + db=db, + data={ + "key": fides_key, + "name": fides_key, + "connection_type": ConnectionType.saas, + "access": AccessLevel.write, + "secrets": saas_example_secrets, + "saas_config": saas_example_config, + }, + ) + yield connection_config + connection_config.delete(db) + + +@pytest.fixture +def saas_example_dataset_config( + db: Session, + saas_example_connection_config: ConnectionConfig, + saas_example_dataset: Dict, +) -> Generator: + fides_key = saas_example_dataset["fides_key"] + saas_example_connection_config.name = fides_key + saas_example_connection_config.key = fides_key + saas_example_connection_config.save(db=db) + dataset = DatasetConfig.create( + db=db, + data={ + "connection_config_id": saas_example_connection_config.id, + "fides_key": fides_key, + "dataset": saas_example_dataset, + }, + ) + yield dataset + dataset.delete(db=db) + + +@pytest.fixture(scope="function") +def saas_example_connection_config_without_saas_config( + db: Session, saas_example_secrets +) -> Generator: + connection_config = ConnectionConfig.create( + db=db, + data={ + "key": "connection_config_without_saas_config", + "name": "connection_config_without_saas_config", + "connection_type": ConnectionType.saas, + "access": AccessLevel.read, + "secrets": saas_example_secrets, + }, + ) + yield connection_config + connection_config.delete(db) + + +@pytest.fixture(scope="function") +def saas_example_connection_config_with_invalid_saas_config( + db: Session, + saas_example_config: Dict[str, Any], + saas_example_secrets: Dict[str, Any], +) -> Generator: + invalid_saas_config = saas_example_config.copy() + invalid_saas_config["endpoints"][0]["requests"]["read"]["param_values"].pop() + connection_config = ConnectionConfig.create( + db=db, + data={ + "key": "connection_config_with_invalid_saas_config", + "name": "connection_config_with_invalid_saas_config", + "connection_type": ConnectionType.saas, + "access": AccessLevel.read, + "secrets": saas_example_secrets, + "saas_config": invalid_saas_config, + }, + ) + yield connection_config + connection_config.delete(db) \ No newline at end of file diff --git a/tests/fixtures/saas_fixtures.py b/tests/fixtures/saas_fixtures.py deleted file mode 100644 index 026f7521b..000000000 --- a/tests/fixtures/saas_fixtures.py +++ /dev/null @@ -1,232 +0,0 @@ -import json -import pytest -import os -import pydash -import yaml - -from sqlalchemy.orm import Session -from typing import Dict, Generator - -from fidesops.core.config import load_file, load_toml -from fidesops.models.connectionconfig import ( - AccessLevel, - ConnectionConfig, - ConnectionType, -) -from fidesops.models.datasetconfig import DatasetConfig -from fidesops.schemas.saas.saas_config import SaaSConfig -from fidesops.schemas.saas.shared_schemas import HTTPMethod, SaaSRequestParams -from fidesops.service.connectors.saas_connector import SaaSConnector -from tests.fixtures.application_fixtures import load_dataset - - -saas_config = load_toml("saas_config.toml") - -saas_secrets_dict = { - "saas_example": { - "domain": pydash.get(saas_config, "saas_example.domain"), - "username": pydash.get(saas_config, "saas_example.username"), - "api_key": pydash.get(saas_config, "saas_example.api_key"), - "api_version": pydash.get(saas_config, "saas_example.api_version"), - "page_limit": pydash.get(saas_config, "saas_example.page_limit") - }, - "mailchimp": { - "domain": pydash.get(saas_config, "mailchimp.domain") - or os.environ.get("MAILCHIMP_DOMAIN"), - "username": pydash.get(saas_config, "mailchimp.username") - or os.environ.get("MAILCHIMP_USERNAME"), - "api_key": pydash.get(saas_config, "mailchimp.api_key") - or os.environ.get("MAILCHIMP_API_KEY"), - }, -} - - -def load_config(filename: str) -> Dict: - yaml_file = load_file(filename) - with open(yaml_file, "r") as file: - return yaml.safe_load(file).get("saas_config", []) - - -@pytest.fixture -def saas_configs() -> Dict[str, Dict]: - saas_configs = {} - saas_configs["saas_example"] = load_config( - "data/saas/config/saas_example_config.yml" - ) - saas_configs["mailchimp"] = load_config("data/saas/config/mailchimp_config.yml") - return saas_configs - - -@pytest.fixture -def saas_datasets() -> Dict[str, Dict]: - saas_datasets = {} - saas_datasets["saas_example"] = load_dataset( - "data/saas/dataset/saas_example_dataset.yml" - )[0] - saas_datasets["mailchimp"] = load_dataset( - "data/saas/dataset/mailchimp_dataset.yml" - )[0] - return saas_datasets - - -@pytest.fixture(scope="function") -def connection_config_saas_example( - db: Session, - saas_configs: Dict[str, Dict], -) -> Generator: - saas_config = SaaSConfig(**saas_configs["saas_example"]) - connection_config = ConnectionConfig.create( - db=db, - data={ - "key": saas_config.fides_key, - "name": saas_config.fides_key, - "connection_type": ConnectionType.saas, - "access": AccessLevel.write, - "secrets": saas_secrets_dict["saas_example"], - "saas_config": saas_configs["saas_example"], - }, - ) - yield connection_config - connection_config.delete(db) - - -@pytest.fixture -def dataset_config_saas_example( - db: Session, - connection_config_saas_example: ConnectionConfig, - saas_datasets: Dict[str, Dict], -) -> Generator: - saas_dataset = saas_datasets["saas_example"] - fides_key = saas_dataset["fides_key"] - connection_config_saas_example.name = fides_key - connection_config_saas_example.key = fides_key - connection_config_saas_example.save(db=db) - dataset = DatasetConfig.create( - db=db, - data={ - "connection_config_id": connection_config_saas_example.id, - "fides_key": fides_key, - "dataset": saas_dataset, - }, - ) - yield dataset - dataset.delete(db=db) - - -@pytest.fixture(scope="function") -def connection_config_mailchimp( - db: Session, saas_configs: Dict[str, Dict] -) -> Generator: - saas_config = SaaSConfig(**saas_configs["mailchimp"]) - connection_config = ConnectionConfig.create( - db=db, - data={ - "key": saas_config.fides_key, - "name": saas_config.fides_key, - "connection_type": ConnectionType.saas, - "access": AccessLevel.write, - "secrets": saas_secrets_dict["mailchimp"], - "saas_config": saas_configs["mailchimp"], - }, - ) - yield connection_config - connection_config.delete(db) - - -@pytest.fixture -def dataset_config_mailchimp( - db: Session, - connection_config_mailchimp: ConnectionConfig, - saas_datasets: Dict[str, Dict] -) -> Generator: - saas_dataset = saas_datasets["mailchimp"] - fides_key = saas_dataset["fides_key"] - connection_config_mailchimp.name = fides_key - connection_config_mailchimp.key = fides_key - connection_config_mailchimp.save(db=db) - dataset = DatasetConfig.create( - db=db, - data={ - "connection_config_id": connection_config_mailchimp.id, - "fides_key": fides_key, - "dataset": saas_dataset, - }, - ) - yield dataset - dataset.delete(db=db) - - -@pytest.fixture(scope="function") -def connection_config_saas_example_without_saas_config( - db: Session, -) -> Generator: - connection_config = ConnectionConfig.create( - db=db, - data={ - "key": "connection_config_without_saas_config", - "name": "connection_config_without_saas_config", - "connection_type": ConnectionType.saas, - "access": AccessLevel.read, - "secrets": saas_secrets_dict["saas_example"], - }, - ) - yield connection_config - connection_config.delete(db) - - -@pytest.fixture(scope="function") -def connection_config_saas_example_with_invalid_saas_config( - db: Session, saas_configs: Dict[str, Dict] -) -> Generator: - invalid_saas_config = saas_configs["saas_example"].copy() - invalid_saas_config["endpoints"][0]["requests"]["read"]["param_values"].pop() - connection_config = ConnectionConfig.create( - db=db, - data={ - "key": "connection_config_without_saas_config", - "name": "connection_config_without_saas_config", - "connection_type": ConnectionType.saas, - "access": AccessLevel.read, - "secrets": saas_secrets_dict["saas_example"], - "saas_config": invalid_saas_config, - }, - ) - yield connection_config - connection_config.delete(db) - - -@pytest.fixture(scope="function") -def saas_secrets(): - return saas_secrets_dict - - -@pytest.fixture(scope="function") -def mailchimp_identity_email(): - return pydash.get(saas_config, "mailchimp.identity_email") or os.environ.get( - "MAILCHIMP_IDENTITY_EMAIL" - ) - - -@pytest.fixture(scope="function") -def reset_mailchimp_data( - connection_config_mailchimp, mailchimp_identity_email -) -> Generator: - """ - Gets the current value of the resource and restores it after the test is complete. - Used for erasure tests. - """ - connector = SaaSConnector(connection_config_mailchimp) - request: SaaSRequestParams = SaaSRequestParams( - method=HTTPMethod.GET, path="/3.0/search-members", query_params={"query": mailchimp_identity_email}, body=None - ) - response = connector.create_client().send(request) - body = response.json() - member = body["exact_matches"]["members"][0] - yield member - request: SaaSRequestParams = SaaSRequestParams( - method=HTTPMethod.PUT, - path=f'/3.0/lists/{member["list_id"]}/members/{member["id"]}', - query_params={}, - body=json.dumps(member) - ) - connector.create_client().send(request) diff --git a/tests/integration_tests/saas/test_mailchimp_task.py b/tests/integration_tests/saas/test_mailchimp_task.py new file mode 100644 index 000000000..c48a54183 --- /dev/null +++ b/tests/integration_tests/saas/test_mailchimp_task.py @@ -0,0 +1,200 @@ +import pytest +import random + +from fidesops.graph.graph import DatasetGraph +from fidesops.models.privacy_request import ExecutionLog, PrivacyRequest +from fidesops.schemas.redis_cache import PrivacyRequestIdentity + +from fidesops.task import graph_task +from fidesops.task.graph_task import get_cached_data_for_erasures +from tests.graph.graph_test_util import assert_rows_match, records_matching_fields + + +@pytest.mark.integration_saas +@pytest.mark.integration_mailchimp +def test_saas_access_request_task( + db, + policy, + mailchimp_connection_config, + mailchimp_dataset_config, + mailchimp_identity_email, +) -> None: + """Full access request based on the Mailchimp SaaS config""" + + privacy_request = PrivacyRequest( + id=f"test_saas_access_request_task_{random.randint(0, 1000)}" + ) + identity_attribute = "email" + identity_value = mailchimp_identity_email + identity_kwargs = {identity_attribute: identity_value} + identity = PrivacyRequestIdentity(**identity_kwargs) + privacy_request.cache_identity(identity) + + dataset_name = mailchimp_connection_config.get_saas_config().fides_key + merged_graph = mailchimp_dataset_config.get_graph() + graph = DatasetGraph(merged_graph) + + v = graph_task.run_access_request( + privacy_request, + policy, + graph, + [mailchimp_connection_config], + {"email": mailchimp_identity_email}, + ) + + assert_rows_match( + v[f"{dataset_name}:member"], + min_size=1, + keys=[ + "id", + "list_id", + "email_address", + "unique_email_id", + "web_id", + "email_type", + "status", + "merge_fields", + "ip_signup", + "timestamp_signup", + "ip_opt", + "timestamp_opt", + "language", + "email_client", + "location", + "source", + "tags", + ], + ) + assert_rows_match( + v[f"{dataset_name}:conversations"], + min_size=2, + keys=["id", "campaign_id", "list_id", "from_email", "from_label", "subject"], + ) + assert_rows_match( + v[f"{dataset_name}:messages"], + min_size=3, + keys=[ + "id", + "conversation_id", + "from_label", + "from_email", + "subject", + "message", + "read", + "timestamp", + ], + ) + + # links + assert v[f"{dataset_name}:member"][0]["email_address"] == mailchimp_identity_email + + logs = ( + ExecutionLog.query(db=db) + .filter(ExecutionLog.privacy_request_id == privacy_request.id) + .all() + ) + + logs = [log.__dict__ for log in logs] + assert ( + len( + records_matching_fields( + logs, dataset_name=dataset_name, collection_name="member" + ) + ) + > 0 + ) + assert ( + len( + records_matching_fields( + logs, + dataset_name=dataset_name, + collection_name="conversations", + ) + ) + > 0 + ) + assert ( + len( + records_matching_fields( + logs, dataset_name=dataset_name, collection_name="messages" + ) + ) + > 0 + ) + + +@pytest.mark.integration_saas +@pytest.mark.integration_mailchimp +def test_saas_erasure_request_task( + db, + policy, + mailchimp_connection_config, + mailchimp_dataset_config, + mailchimp_identity_email, + reset_mailchimp_data, +) -> None: + """Full erasure request based on the Mailchimp SaaS config""" + + privacy_request = PrivacyRequest( + id=f"test_saas_erasure_request_task_{random.randint(0, 1000)}" + ) + identity_attribute = "email" + identity_value = mailchimp_identity_email + identity_kwargs = {identity_attribute: identity_value} + identity = PrivacyRequestIdentity(**identity_kwargs) + privacy_request.cache_identity(identity) + + dataset_name = mailchimp_connection_config.get_saas_config().fides_key + merged_graph = mailchimp_dataset_config.get_graph() + graph = DatasetGraph(merged_graph) + + graph_task.run_access_request( + privacy_request, + policy, + graph, + [mailchimp_connection_config], + {"email": mailchimp_identity_email}, + ) + + v = graph_task.run_erasure( + privacy_request, + policy, + graph, + [mailchimp_connection_config], + {"email": mailchimp_identity_email}, + get_cached_data_for_erasures(privacy_request.id), + ) + + logs = ( + ExecutionLog.query(db=db) + .filter(ExecutionLog.privacy_request_id == privacy_request.id) + .all() + ) + logs = [log.__dict__ for log in logs] + assert ( + len( + records_matching_fields( + logs, + dataset_name=dataset_name, + collection_name="conversations", + message="No values were erased since no primary key was defined for this collection", + ) + ) + == 1 + ) + assert ( + len( + records_matching_fields( + logs, + dataset_name=dataset_name, + collection_name="messages", + message="No values were erased since no primary key was defined for this collection", + ) + ) + == 1 + ) + assert v == { + f"{dataset_name}:member": 1, + f"{dataset_name}:conversations": 0, + f"{dataset_name}:messages": 0, + } diff --git a/tests/integration_tests/test_connection_configuration_integration.py b/tests/integration_tests/test_connection_configuration_integration.py index 94bff288a..568d7e160 100644 --- a/tests/integration_tests/test_connection_configuration_integration.py +++ b/tests/integration_tests/test_connection_configuration_integration.py @@ -1192,16 +1192,16 @@ def test_mongo_db_connection_connect_with_url( class TestSaaSConnectionPutSecretsAPI: @pytest.fixture(scope="function") def url( - self, oauth_client: ClientDetail, policy, connection_config_mailchimp + self, oauth_client: ClientDetail, policy, mailchimp_connection_config ) -> str: - return f"{V1_URL_PREFIX}{CONNECTIONS}/{connection_config_mailchimp.key}/secret" + return f"{V1_URL_PREFIX}{CONNECTIONS}/{mailchimp_connection_config.key}/secret" def test_saas_connection_incorrect_secrets( self, api_client: TestClient, db: Session, generate_auth_header, - connection_config_mailchimp, + mailchimp_connection_config, url, ): auth_header = generate_auth_header(scopes=[CONNECTION_CREATE_OR_UPDATE]) @@ -1216,22 +1216,22 @@ def test_saas_connection_incorrect_secrets( body = json.loads(resp.text) assert ( body["msg"] - == f"Secrets updated for ConnectionConfig with key: {connection_config_mailchimp.key}." + == f"Secrets updated for ConnectionConfig with key: {mailchimp_connection_config.key}." ) assert body["test_status"] == "failed" assert ( - f"Operational Error connecting to '{connection_config_mailchimp.key}'." + f"Operational Error connecting to '{mailchimp_connection_config.key}'." == body["failure_reason"] ) - db.refresh(connection_config_mailchimp) - assert connection_config_mailchimp.secrets == { + db.refresh(mailchimp_connection_config) + assert mailchimp_connection_config.secrets == { "domain": "can", "username": "someone", "api_key": "letmein", } - assert connection_config_mailchimp.last_test_timestamp is not None - assert connection_config_mailchimp.last_test_succeeded is False + assert mailchimp_connection_config.last_test_timestamp is not None + assert mailchimp_connection_config.last_test_succeeded is False def test_saas_connection_connect_with_components( self, @@ -1239,11 +1239,11 @@ def test_saas_connection_connect_with_components( api_client: TestClient, db: Session, generate_auth_header, - connection_config_mailchimp, - saas_secrets, + mailchimp_connection_config, + mailchimp_secrets, ): auth_header = generate_auth_header(scopes=[CONNECTION_CREATE_OR_UPDATE]) - payload = saas_secrets["mailchimp"] + payload = mailchimp_secrets resp = api_client.put( url, headers=auth_header, @@ -1254,27 +1254,27 @@ def test_saas_connection_connect_with_components( body = json.loads(resp.text) assert ( body["msg"] - == f"Secrets updated for ConnectionConfig with key: {connection_config_mailchimp.key}." + == f"Secrets updated for ConnectionConfig with key: {mailchimp_connection_config.key}." ) assert body["test_status"] == "succeeded" assert body["failure_reason"] is None - db.refresh(connection_config_mailchimp) - assert connection_config_mailchimp.secrets == saas_secrets["mailchimp"] - assert connection_config_mailchimp.last_test_timestamp is not None - assert connection_config_mailchimp.last_test_succeeded is True + db.refresh(mailchimp_connection_config) + assert mailchimp_connection_config.secrets == mailchimp_secrets + assert mailchimp_connection_config.last_test_timestamp is not None + assert mailchimp_connection_config.last_test_succeeded is True def test_saas_connection_connect_missing_secrets( self, url, api_client: TestClient, generate_auth_header, - saas_secrets, + saas_example_secrets, ): auth_header = generate_auth_header(scopes=[CONNECTION_CREATE_OR_UPDATE]) payload = { - "domain": saas_secrets["saas_example"]["domain"], - "username": saas_secrets["saas_example"]["username"], + "domain": saas_example_secrets["domain"], + "username": saas_example_secrets["username"], } resp = api_client.put( url, @@ -1291,10 +1291,10 @@ def test_saas_connection_connect_with_extra_secrets( url, api_client: TestClient, generate_auth_header, - saas_secrets, + mailchimp_secrets, ): auth_header = generate_auth_header(scopes=[CONNECTION_CREATE_OR_UPDATE]) - payload = {**saas_secrets["mailchimp"], "extra": "junk"} + payload = {**mailchimp_secrets, "extra": "junk"} resp = api_client.put( url, headers=auth_header, @@ -1314,10 +1314,10 @@ def url( self, oauth_client: ClientDetail, policy, - connection_config_mailchimp, - dataset_config_mailchimp, + mailchimp_connection_config, + mailchimp_dataset_config, ) -> str: - return f"{V1_URL_PREFIX}{CONNECTIONS}/{connection_config_mailchimp.key}/test" + return f"{V1_URL_PREFIX}{CONNECTIONS}/{mailchimp_connection_config.key}/test" def test_connection_configuration_test_not_authenticated( self, @@ -1325,16 +1325,16 @@ def test_connection_configuration_test_not_authenticated( api_client: TestClient, db: Session, generate_auth_header, - connection_config_mailchimp, + mailchimp_connection_config, ): - assert connection_config_mailchimp.last_test_timestamp is None + assert mailchimp_connection_config.last_test_timestamp is None resp = api_client.get(url) assert resp.status_code == 401 - db.refresh(connection_config_mailchimp) - assert connection_config_mailchimp.last_test_timestamp is None - assert connection_config_mailchimp.last_test_succeeded is None + db.refresh(mailchimp_connection_config) + assert mailchimp_connection_config.last_test_timestamp is None + assert mailchimp_connection_config.last_test_succeeded is None def test_connection_configuration_test_incorrect_scopes( self, @@ -1342,9 +1342,9 @@ def test_connection_configuration_test_incorrect_scopes( api_client: TestClient, db: Session, generate_auth_header, - connection_config_mailchimp, + mailchimp_connection_config, ): - assert connection_config_mailchimp.last_test_timestamp is None + assert mailchimp_connection_config.last_test_timestamp is None auth_header = generate_auth_header(scopes=[STORAGE_READ]) resp = api_client.get( @@ -1353,9 +1353,9 @@ def test_connection_configuration_test_incorrect_scopes( ) assert resp.status_code == 403 - db.refresh(connection_config_mailchimp) - assert connection_config_mailchimp.last_test_timestamp is None - assert connection_config_mailchimp.last_test_succeeded is None + db.refresh(mailchimp_connection_config) + assert mailchimp_connection_config.last_test_timestamp is None + assert mailchimp_connection_config.last_test_succeeded is None def test_connection_configuration_test_failed_response( self, @@ -1363,12 +1363,12 @@ def test_connection_configuration_test_failed_response( api_client: TestClient, db: Session, generate_auth_header, - connection_config_mailchimp, + mailchimp_connection_config, ): - assert connection_config_mailchimp.last_test_timestamp is None + assert mailchimp_connection_config.last_test_timestamp is None - connection_config_mailchimp.secrets = {"domain": "invalid_domain"} - connection_config_mailchimp.save(db) + mailchimp_connection_config.secrets = {"domain": "invalid_domain"} + mailchimp_connection_config.save(db) auth_header = generate_auth_header(scopes=[CONNECTION_READ]) resp = api_client.get( url, @@ -1379,17 +1379,17 @@ def test_connection_configuration_test_failed_response( body = json.loads(resp.text) assert body["test_status"] == "failed" assert ( - f"Operational Error connecting to '{connection_config_mailchimp.key}'." + f"Operational Error connecting to '{mailchimp_connection_config.key}'." == body["failure_reason"] ) assert ( body["msg"] - == f"Test completed for ConnectionConfig with key: {connection_config_mailchimp.key}." + == f"Test completed for ConnectionConfig with key: {mailchimp_connection_config.key}." ) - db.refresh(connection_config_mailchimp) - assert connection_config_mailchimp.last_test_timestamp is not None - assert connection_config_mailchimp.last_test_succeeded is False + db.refresh(mailchimp_connection_config) + assert mailchimp_connection_config.last_test_timestamp is not None + assert mailchimp_connection_config.last_test_succeeded is False def test_connection_configuration_test( self, @@ -1397,9 +1397,9 @@ def test_connection_configuration_test( api_client: TestClient, db: Session, generate_auth_header, - connection_config_mailchimp, + mailchimp_connection_config, ): - assert connection_config_mailchimp.last_test_timestamp is None + assert mailchimp_connection_config.last_test_timestamp is None auth_header = generate_auth_header(scopes=[CONNECTION_READ]) resp = api_client.get( @@ -1411,31 +1411,31 @@ def test_connection_configuration_test( body = json.loads(resp.text) assert ( body["msg"] - == f"Test completed for ConnectionConfig with key: {connection_config_mailchimp.key}." + == f"Test completed for ConnectionConfig with key: {mailchimp_connection_config.key}." ) assert body["failure_reason"] is None assert body["test_status"] == "succeeded" - db.refresh(connection_config_mailchimp) - assert connection_config_mailchimp.last_test_timestamp is not None - assert connection_config_mailchimp.last_test_succeeded is True + db.refresh(mailchimp_connection_config) + assert mailchimp_connection_config.last_test_timestamp is not None + assert mailchimp_connection_config.last_test_succeeded is True @pytest.mark.integration_saas @pytest.mark.integration_mailchimp class TestSaasConnector: def test_saas_connector( - self, db: Session, connection_config_mailchimp, dataset_config_mailchimp + self, db: Session, mailchimp_connection_config, mailchimp_dataset_config ): - connector = get_connector(connection_config_mailchimp) + connector = get_connector(mailchimp_connection_config) assert connector.__class__ == SaaSConnector client = connector.client() assert client.__class__ == AuthenticatedClient assert connector.test_connection() == ConnectionTestStatus.succeeded - connection_config_mailchimp.secrets = {"domain": "bad_host"} - connection_config_mailchimp.save(db) - connector = get_connector(connection_config_mailchimp) + mailchimp_connection_config.secrets = {"domain": "bad_host"} + mailchimp_connection_config.save(db) + connector = get_connector(mailchimp_connection_config) with pytest.raises(ConnectionException): connector.test_connection() diff --git a/tests/integration_tests/test_saas_task.py b/tests/integration_tests/test_saas_task.py index a510fa029..c48a54183 100644 --- a/tests/integration_tests/test_saas_task.py +++ b/tests/integration_tests/test_saas_task.py @@ -15,8 +15,8 @@ def test_saas_access_request_task( db, policy, - connection_config_mailchimp, - dataset_config_mailchimp, + mailchimp_connection_config, + mailchimp_dataset_config, mailchimp_identity_email, ) -> None: """Full access request based on the Mailchimp SaaS config""" @@ -30,15 +30,15 @@ def test_saas_access_request_task( identity = PrivacyRequestIdentity(**identity_kwargs) privacy_request.cache_identity(identity) - dataset_name = connection_config_mailchimp.get_saas_config().fides_key - merged_graph = dataset_config_mailchimp.get_graph() + dataset_name = mailchimp_connection_config.get_saas_config().fides_key + merged_graph = mailchimp_dataset_config.get_graph() graph = DatasetGraph(merged_graph) v = graph_task.run_access_request( privacy_request, policy, graph, - [connection_config_mailchimp], + [mailchimp_connection_config], {"email": mailchimp_identity_email}, ) @@ -128,8 +128,8 @@ def test_saas_access_request_task( def test_saas_erasure_request_task( db, policy, - connection_config_mailchimp, - dataset_config_mailchimp, + mailchimp_connection_config, + mailchimp_dataset_config, mailchimp_identity_email, reset_mailchimp_data, ) -> None: @@ -144,15 +144,15 @@ def test_saas_erasure_request_task( identity = PrivacyRequestIdentity(**identity_kwargs) privacy_request.cache_identity(identity) - dataset_name = connection_config_mailchimp.get_saas_config().fides_key - merged_graph = dataset_config_mailchimp.get_graph() + dataset_name = mailchimp_connection_config.get_saas_config().fides_key + merged_graph = mailchimp_dataset_config.get_graph() graph = DatasetGraph(merged_graph) graph_task.run_access_request( privacy_request, policy, graph, - [connection_config_mailchimp], + [mailchimp_connection_config], {"email": mailchimp_identity_email}, ) @@ -160,7 +160,7 @@ def test_saas_erasure_request_task( privacy_request, policy, graph, - [connection_config_mailchimp], + [mailchimp_connection_config], {"email": mailchimp_identity_email}, get_cached_data_for_erasures(privacy_request.id), ) diff --git a/tests/models/test_saasconfig.py b/tests/models/test_saasconfig.py index 6bfd6a689..b9b569176 100644 --- a/tests/models/test_saasconfig.py +++ b/tests/models/test_saasconfig.py @@ -7,11 +7,15 @@ @pytest.mark.unit_saas -def test_saas_configs(saas_configs): - """Simple test to verify that the available configs can be deserialized into SaaSConfigs""" - for saas_config in saas_configs.values(): - SaaSConfig(**saas_config) +def test_saas_configs(saas_example_config) -> None: + """Simple test to verify that the example config can be deserialized into SaaSConfigs""" + SaaSConfig(**saas_example_config) +@pytest.mark.unit_saas +def test_saas_request_without_method(): + with pytest.raises(ValidationError) as exc: + SaaSRequest(path="/test") + assert "field required" in str(exc.value) @pytest.mark.unit_saas def test_saas_request_without_method(): @@ -21,10 +25,10 @@ def test_saas_request_without_method(): @pytest.mark.unit_saas -def test_saas_config_to_dataset(saas_configs: Dict[str, Dict]): +def test_saas_config_to_dataset(saas_example_config: Dict[str, Dict]): """Verify dataset generated by SaaS config""" # convert endpoint references to dataset references to be able to hook SaaS connectors into the graph traversal - saas_config = SaaSConfig(**saas_configs["saas_example"]) + saas_config = SaaSConfig(**saas_example_config) saas_dataset = saas_config.get_graph() messages_collection = saas_dataset.collections[0] @@ -66,10 +70,10 @@ def test_saas_config_to_dataset(saas_configs: Dict[str, Dict]): @pytest.mark.unit_saas -def test_saas_config_ignore_errors_param(saas_configs: Dict[str, Dict]): +def test_saas_config_ignore_errors_param(saas_example_config: Dict[str, Dict]): """Verify saas config ignore errors""" # convert endpoint references to dataset references to be able to hook SaaS connectors into the graph traversal - saas_config = SaaSConfig(**saas_configs["saas_example"]) + saas_config = SaaSConfig(**saas_example_config) collections_endpoint = next( end for end in saas_config.endpoints if end.name == "conversations" diff --git a/tests/schemas/connection_configuration/test_connection_secrets_saas.py b/tests/schemas/connection_configuration/test_connection_secrets_saas.py index 51fb7de5a..16e2a07c1 100644 --- a/tests/schemas/connection_configuration/test_connection_secrets_saas.py +++ b/tests/schemas/connection_configuration/test_connection_secrets_saas.py @@ -10,8 +10,8 @@ @pytest.mark.unit_saas class TestSaaSConnectionSecrets: @pytest.fixture(scope="function") - def saas_config(self, saas_configs) -> SaaSConfig: - return SaaSConfig(**saas_configs["saas_example"]) + def saas_config(self, saas_example_config) -> SaaSConfig: + return SaaSConfig(**saas_example_config) def test_get_saas_schema(self, saas_config): """ @@ -22,9 +22,9 @@ def test_get_saas_schema(self, saas_config): assert schema.__name__ == f"{saas_config.fides_key}_schema" assert issubclass(schema.__base__, SaaSSchema) - def test_validation(self, saas_config, saas_secrets): + def test_validation(self, saas_config, saas_example_secrets): schema = SaaSSchemaFactory(saas_config).get_saas_schema() - config = saas_secrets["saas_example"] + config = saas_example_secrets schema.parse_obj(config) def test_missing_fields(self, saas_config): @@ -38,10 +38,10 @@ def test_missing_fields(self, saas_config): in str(exc.value) ) - def test_extra_fields(self, saas_config, saas_secrets): + def test_extra_fields(self, saas_config, saas_example_secrets): schema = SaaSSchemaFactory(saas_config).get_saas_schema() config = { - **saas_secrets["saas_example"], + **saas_example_secrets, "extra": "extra", } with pytest.raises(ValidationError) as exc: diff --git a/tests/service/connectors/test_queryconfig.py b/tests/service/connectors/test_queryconfig.py index 3d1b93e55..7bc13ae54 100644 --- a/tests/service/connectors/test_queryconfig.py +++ b/tests/service/connectors/test_queryconfig.py @@ -604,16 +604,16 @@ def test_generate_update_stmt_multiple_rules( class TestSaaSQueryConfig: @pytest.fixture(scope="function") def combined_traversal( - self, connection_config_saas_example, dataset_config_saas_example + self, saas_example_connection_config, saas_example_dataset_config ): - merged_graph = dataset_config_saas_example.get_graph() + merged_graph = saas_example_dataset_config.get_graph() graph = DatasetGraph(merged_graph) return Traversal(graph, {"email": "customer-1@example.com"}) def test_generate_query( - self, policy, combined_traversal, connection_config_saas_example + self, policy, combined_traversal, saas_example_connection_config ): - saas_config = connection_config_saas_example.get_saas_config() + saas_config = saas_example_connection_config.get_saas_config() endpoints = saas_config.top_level_endpoint_dict member = combined_traversal.traversal_node_dict[ @@ -698,9 +698,9 @@ def test_generate_update_stmt( self, erasure_policy_string_rewrite, combined_traversal, - connection_config_saas_example, + saas_example_connection_config, ): - saas_config = connection_config_saas_example.get_saas_config() + saas_config = saas_example_connection_config.get_saas_config() endpoints = saas_config.top_level_endpoint_dict member = combined_traversal.traversal_node_dict[ @@ -732,11 +732,11 @@ def test_generate_update_stmt_custom_http_method( self, erasure_policy_string_rewrite, combined_traversal, - connection_config_saas_example, + saas_example_connection_config, ): saas_config: Optional[ SaaSConfig - ] = connection_config_saas_example.get_saas_config() + ] = saas_example_connection_config.get_saas_config() saas_config.endpoints[2].requests.get("update").method = HTTPMethod.POST endpoints = saas_config.top_level_endpoint_dict @@ -769,11 +769,11 @@ def test_generate_update_stmt_with_request_body( self, erasure_policy_string_rewrite, combined_traversal, - connection_config_saas_example, + saas_example_connection_config, ): saas_config: Optional[ SaaSConfig - ] = connection_config_saas_example.get_saas_config() + ] = saas_example_connection_config.get_saas_config() saas_config.endpoints[2].requests.get( "update" ).body = '{"properties": {, "list_id": ""}}' diff --git a/tests/service/privacy_request/request_runner_service_test.py b/tests/service/privacy_request/request_runner_service_test.py index 26e94a0cf..acdf2467a 100644 --- a/tests/service/privacy_request/request_runner_service_test.py +++ b/tests/service/privacy_request/request_runner_service_test.py @@ -308,8 +308,8 @@ def test_create_and_process_access_request_mariadb( @mock.patch("fidesops.models.privacy_request.PrivacyRequest.trigger_policy_webhook") def test_create_and_process_access_request_saas( trigger_webhook_mock, - connection_config_mailchimp, - dataset_config_mailchimp, + mailchimp_connection_config, + mailchimp_dataset_config, db, cache, policy, @@ -347,8 +347,8 @@ def test_create_and_process_access_request_saas( @mock.patch("fidesops.models.privacy_request.PrivacyRequest.trigger_policy_webhook") def test_create_and_process_erasure_request_saas( trigger_webhook_mock, - connection_config_mailchimp, - dataset_config_mailchimp, + mailchimp_connection_config, + mailchimp_dataset_config, db, cache, erasure_policy_hmac, @@ -365,7 +365,7 @@ def test_create_and_process_erasure_request_saas( pr = get_privacy_request_results(db, erasure_policy_hmac, cache, data) - connector = SaaSConnector(connection_config_mailchimp) + connector = SaaSConnector(mailchimp_connection_config) request: SaaSRequestParams = SaaSRequestParams( method=HTTPMethod.GET, path="/3.0/search-members", query_params={"query": mailchimp_identity_email} )