From 2e7e85f52f5d969315440ecc830d2d6b3a935aa9 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Thu, 17 Mar 2022 08:17:52 -0700 Subject: [PATCH 01/24] Adding connector_param references to generate_query and generate_update_stmt --- data/saas/config/saas_example_config.yml | 17 ++++++++ data/saas/dataset/saas_example_dataset.yml | 4 ++ src/fidesops/schemas/saas/saas_config.py | 1 + .../service/connectors/query_config.py | 42 ++++++++++++++++--- .../service/connectors/saas_connector.py | 2 +- .../endpoints/test_saas_config_endpoints.py | 4 +- tests/fixtures/saas_fixtures.py | 2 + .../test_connection_secrets_saas.py | 22 ++++------ tests/service/connectors/test_queryconfig.py | 39 +++++++++++++---- 9 files changed, 104 insertions(+), 29 deletions(-) diff --git a/data/saas/config/saas_example_config.yml b/data/saas/config/saas_example_config.yml index ea4395894..8214f5e5d 100644 --- a/data/saas/config/saas_example_config.yml +++ b/data/saas/config/saas_example_config.yml @@ -8,6 +8,8 @@ saas_config: - name: domain - name: username - name: api_key + - name: api_version + - name: page_limit client_config: protocol: https @@ -91,3 +93,18 @@ saas_config: - dataset: saas_connector_example field: member.id direction: from + - name: payment_methods + requests: + read: + path: //payment_methods + request_params: + - name: limit + type: query + connector_param: page_limit + - name: version + type: path + connector_param: api_version + - name: query + type: query + identity: email + diff --git a/data/saas/dataset/saas_example_dataset.yml b/data/saas/dataset/saas_example_dataset.yml index 3aef51a54..aa8f56cdb 100644 --- a/data/saas/dataset/saas_example_dataset.yml +++ b/data/saas/dataset/saas_example_dataset.yml @@ -143,3 +143,7 @@ dataset: data_categories: [system.operations] - name: name data_categories: [system.operations] + - name: payment_methods + fields: + - name: type + data_categories: [system.operations] \ No newline at end of file diff --git a/src/fidesops/schemas/saas/saas_config.py b/src/fidesops/schemas/saas/saas_config.py index 3fe5ad03e..866c7d9a0 100644 --- a/src/fidesops/schemas/saas/saas_config.py +++ b/src/fidesops/schemas/saas/saas_config.py @@ -29,6 +29,7 @@ class RequestParam(BaseModel): identity: Optional[str] data_type: Optional[str] references: Optional[List[FidesopsDatasetReference]] + connector_param: Optional[str] @validator("references") def check_references_or_identity( diff --git a/src/fidesops/service/connectors/query_config.py b/src/fidesops/service/connectors/query_config.py index a91cbd370..24609b1a7 100644 --- a/src/fidesops/service/connectors/query_config.py +++ b/src/fidesops/service/connectors/query_config.py @@ -674,9 +674,15 @@ def dry_run_query(self) -> Optional[str]: class SaaSQueryConfig(QueryConfig[SaaSRequestParams]): """Query config that generates populated SaaS requests for a given collection""" - def __init__(self, node: TraversalNode, endpoints: Dict[str, Endpoint]): + def __init__( + self, + node: TraversalNode, + endpoints: Dict[str, Endpoint], + connector_params: Dict[str, Any] + ): super().__init__(node) self.endpoints = endpoints + self.connector_params = connector_params def get_request_by_action(self, action: str) -> SaaSRequest: """ @@ -731,8 +737,18 @@ def generate_query( params[param.name] = param.default_value elif param.references or param.identity: params[param.name] = input_data[param.name][0] + elif param.connector_param: + params[param.name] = pydash.get( + self.connector_params, param.connector_param + ) elif param.type == "path": - path = path.replace(f"<{param.name}>", input_data[param.name][0]) + if param.references or param.identity: + path = path.replace(f"<{param.name}>", input_data[param.name][0]) + elif param.connector_param: + path = path.replace( + f"<{param.name}>", + pydash.get(self.connector_params, param.connector_param), + ) logger.info(f"Populated request params for {current_request.path}") return "GET", path, params, None @@ -763,11 +779,25 @@ def generate_update_stmt( ) elif param.identity: params[param.name] = pydash.get(param_values, param.identity) + elif param.connector_param: + params[param.name] = pydash.get( + self.connector_params, param.connector_param + ) elif param.type == "path": - path = path.replace( - f"<{param.name}>", - pydash.get(param_values, param.references[0].field), - ) + if param.references: + path = path.replace( + f"<{param.name}>", + pydash.get(param_values, param.references[0].field), + ) + elif param.identity: + path = path.replace( + f"<{param.name}>", pydash.get(param_values, param.identity) + ) + elif param.connector_param: + path = path.replace( + f"<{param.name}>", + pydash.get(self.connector_params, param.connector_param), + ) logger.info(f"Populated request params for {current_request.path}") diff --git a/src/fidesops/service/connectors/saas_connector.py b/src/fidesops/service/connectors/saas_connector.py index 363e52b95..11d718ea0 100644 --- a/src/fidesops/service/connectors/saas_connector.py +++ b/src/fidesops/service/connectors/saas_connector.py @@ -96,7 +96,7 @@ def __init__(self, configuration: ConnectionConfig): def query_config(self, node: TraversalNode) -> SaaSQueryConfig: """Returns the query config for a SaaS connector""" - return SaaSQueryConfig(node, self.endpoints) + return SaaSQueryConfig(node, self.endpoints, self.secrets) def test_connection(self) -> Optional[ConnectionTestStatus]: """Generates and executes a test connection based on the SaaS config""" diff --git a/tests/api/v1/endpoints/test_saas_config_endpoints.py b/tests/api/v1/endpoints/test_saas_config_endpoints.py index 8b0fdb429..d5159b1b7 100644 --- a/tests/api/v1/endpoints/test_saas_config_endpoints.py +++ b/tests/api/v1/endpoints/test_saas_config_endpoints.py @@ -226,7 +226,7 @@ def test_patch_saas_config_update( ) saas_config = connection_config.saas_config assert saas_config is not None - assert len(saas_config["endpoints"]) == 2 + assert len(saas_config["endpoints"]) == 3 def get_saas_config_url(connection_config: Optional[ConnectionConfig] = None) -> str: @@ -300,7 +300,7 @@ def test_get_saas_config( response_body["fides_key"] == connection_config_saas_example.get_saas_config().fides_key ) - assert len(response_body["endpoints"]) == 3 + assert len(response_body["endpoints"]) == 4 @pytest.mark.unit_saas diff --git a/tests/fixtures/saas_fixtures.py b/tests/fixtures/saas_fixtures.py index 7d335cd3a..a26442c5e 100644 --- a/tests/fixtures/saas_fixtures.py +++ b/tests/fixtures/saas_fixtures.py @@ -26,6 +26,8 @@ "domain": pydash.get(saas_config, "saas_example.domain"), "username": pydash.get(saas_config, "saas_example.username"), "api_key": pydash.get(saas_config, "saas_example.api_key"), + "api_version": pydash.get(saas_config, "saas_example.api_version"), + "page_limit": pydash.get(saas_config, "saas_example.page_limit") }, "mailchimp": { "domain": pydash.get(saas_config, "mailchimp.domain") diff --git a/tests/schemas/connection_configuration/test_connection_secrets_saas.py b/tests/schemas/connection_configuration/test_connection_secrets_saas.py index 0b8dce503..51fb7de5a 100644 --- a/tests/schemas/connection_configuration/test_connection_secrets_saas.py +++ b/tests/schemas/connection_configuration/test_connection_secrets_saas.py @@ -22,32 +22,28 @@ def test_get_saas_schema(self, saas_config): assert schema.__name__ == f"{saas_config.fides_key}_schema" assert issubclass(schema.__base__, SaaSSchema) - def test_validation(self, saas_config): + def test_validation(self, saas_config, saas_secrets): schema = SaaSSchemaFactory(saas_config).get_saas_schema() - config = {"domain": "domain", "username": "username", "api_key": "api_key"} + config = saas_secrets["saas_example"] schema.parse_obj(config) def test_missing_fields(self, saas_config): schema = SaaSSchemaFactory(saas_config).get_saas_schema() config = {"domain": "domain", "username": "username"} - with pytest.raises(ValidationError) as exception: + with pytest.raises(ValidationError) as exc: schema.parse_obj(config) - errors = exception.value.errors() - assert errors[0]["msg"] == "field required" assert ( - errors[1]["msg"] - == f"{saas_config.fides_key}_schema must be supplied all of: " + f"{saas_config.fides_key}_schema must be supplied all of: " f"[{', '.join([connector_param.name for connector_param in saas_config.connector_params])}]." + in str(exc.value) ) - def test_extra_fields(self, saas_config): + def test_extra_fields(self, saas_config, saas_secrets): schema = SaaSSchemaFactory(saas_config).get_saas_schema() config = { - "domain": "domain", - "username": "username", - "api_key": "api_key", + **saas_secrets["saas_example"], "extra": "extra", } - with pytest.raises(ValidationError) as exception: + with pytest.raises(ValidationError) as exc: schema.parse_obj(config) - assert exception.value.errors()[0]["msg"] == "extra fields not permitted" + assert exc.value.errors()[0]["msg"] == "extra fields not permitted" diff --git a/tests/service/connectors/test_queryconfig.py b/tests/service/connectors/test_queryconfig.py index 7ef1543d7..8b4bbe14a 100644 --- a/tests/service/connectors/test_queryconfig.py +++ b/tests/service/connectors/test_queryconfig.py @@ -604,12 +604,16 @@ def test_generate_update_stmt_multiple_rules( @pytest.mark.unit_saas class TestSaaSQueryConfig: @pytest.fixture(scope="function") - def combined_traversal(self, connection_config_saas_example, dataset_config_saas_example): + def combined_traversal( + self, connection_config_saas_example, dataset_config_saas_example + ): merged_graph = dataset_config_saas_example.get_graph() graph = DatasetGraph(merged_graph) return Traversal(graph, {"email": "customer-1@example.com"}) - def test_generate_query(self, policy, combined_traversal, connection_config_saas_example): + def test_generate_query( + self, policy, combined_traversal, connection_config_saas_example + ): saas_config = connection_config_saas_example.get_saas_config() endpoints = saas_config.top_level_endpoint_dict @@ -623,8 +627,12 @@ def test_generate_query(self, policy, combined_traversal, connection_config_saas CollectionAddress(saas_config.fides_key, "messages") ] + payment_methods: Traversal = combined_traversal.traversal_node_dict[ + CollectionAddress(saas_config.fides_key, "payment_methods") + ] + # static path with single query param - config = SaaSQueryConfig(member, endpoints) + config = SaaSQueryConfig(member, endpoints, {}) prepared_request = config.generate_query( {"query": ["customer-1@example.com"]}, policy ) @@ -636,7 +644,7 @@ def test_generate_query(self, policy, combined_traversal, connection_config_saas ) # static path with multiple query params with default values - config = SaaSQueryConfig(conversations, endpoints) + config = SaaSQueryConfig(conversations, endpoints, {}) prepared_request = config.generate_query( {"placeholder": ["customer-1@example.com"]}, policy ) @@ -648,7 +656,7 @@ def test_generate_query(self, policy, combined_traversal, connection_config_saas ) # dynamic path with no query params - config = SaaSQueryConfig(messages, endpoints) + config = SaaSQueryConfig(messages, endpoints, {}) prepared_request = config.generate_query({"conversation_id": ["abc"]}, policy) assert prepared_request == ( "GET", @@ -657,8 +665,25 @@ def test_generate_query(self, policy, combined_traversal, connection_config_saas None, ) + # query and path params with a connector param reference + config = SaaSQueryConfig( + payment_methods, endpoints, {"api_version": "2.0", "page_limit": 10}, {} + ) + prepared_request = config.generate_query( + {"query": ["customer-1@example.com"]}, policy + ) + assert prepared_request == ( + "GET", + "/2.0/payment_methods", + {"limit": 10, "query": "customer-1@example.com"}, + None, + ) + def test_generate_update_stmt( - self, erasure_policy_string_rewrite, combined_traversal, connection_config_saas_example + self, + erasure_policy_string_rewrite, + combined_traversal, + connection_config_saas_example, ): saas_config = connection_config_saas_example.get_saas_config() endpoints = saas_config.top_level_endpoint_dict @@ -667,7 +692,7 @@ def test_generate_update_stmt( CollectionAddress(saas_config.fides_key, "member") ] - config = SaaSQueryConfig(member, endpoints) + config = SaaSQueryConfig(member, endpoints, {}) row = { "id": "123", "merge_fields": {"FNAME": "First", "LNAME": "Last"}, From 29167d8c67950598fcece99e3f2a68c0994c6d73 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Thu, 17 Mar 2022 16:24:00 -0700 Subject: [PATCH 02/24] Cleaning up after merging in main --- data/saas/config/saas_example_config.yml | 7 ++- data/saas/dataset/saas_example_dataset.yml | 6 ++- docs/fidesops/docs/guides/saas_config.md | 5 ++- saas_config.toml | 2 + src/fidesops/schemas/saas/saas_config.py | 18 +------- .../service/connectors/query_config.py | 1 - .../service/connectors/saas_query_config.py | 45 ++++++++++++++++--- .../endpoints/test_saas_config_endpoints.py | 2 +- tests/service/connectors/test_queryconfig.py | 23 ++++++++-- 9 files changed, 76 insertions(+), 33 deletions(-) diff --git a/data/saas/config/saas_example_config.yml b/data/saas/config/saas_example_config.yml index 8214f5e5d..44b380b2d 100644 --- a/data/saas/config/saas_example_config.yml +++ b/data/saas/config/saas_example_config.yml @@ -107,4 +107,9 @@ saas_config: - name: query type: query identity: email - + update: + path: //payment_methods + request_params: + - name: version + type: path + connector_param: api_version diff --git a/data/saas/dataset/saas_example_dataset.yml b/data/saas/dataset/saas_example_dataset.yml index aa8f56cdb..ed8ae0400 100644 --- a/data/saas/dataset/saas_example_dataset.yml +++ b/data/saas/dataset/saas_example_dataset.yml @@ -146,4 +146,8 @@ dataset: - name: payment_methods fields: - name: type - data_categories: [system.operations] \ No newline at end of file + data_categories: [system.operations] + - name: customer_name + data_categories: [user.provided.identifiable.name] + fidesops_meta: + data_type: string \ No newline at end of file diff --git a/docs/fidesops/docs/guides/saas_config.md b/docs/fidesops/docs/guides/saas_config.md index 5fe646771..d60cc0e4f 100644 --- a/docs/fidesops/docs/guides/saas_config.md +++ b/docs/fidesops/docs/guides/saas_config.md @@ -170,8 +170,9 @@ This is where we define how we are going to access and update each collection in - `name` Used as the key for query param values, or to map this param to a value placeholder in the path. - `type` Either "query" or "path". - `references` These are the same as `references` in the Dataset schema. It is used to define the source of the value for the given request_param. - - `identity` This denotes the identity value that this request_param should take. - - `default_value` Hard-coded default value for a `request_param`. This is most often used for query params since a static path param can just be included in the `path`. + - `identity` Used to access the identity values passed into the privacy request such as email or phone number. + - `default_value` Hard-coded default value for a `request_param`. This is most often used for query params since a static path param can just be included in the `path`. + - `connector_param` Used to access the user-configured secrets for the connection. - `data_path`: The expression used to access the collection information from the raw JSON response. - `postprocessors` An optional list of response post-processing strategies. We will ignore this for the example scenarios below but an in depth-explanation can be found under [SaaS Post-Processors](saas_postprocessors.md) - `pagination` An optional strategy used to get the next set of results from APIs with resources spanning multiple pages. Details can be found under [SaaS Pagination](saas_pagination.md) diff --git a/saas_config.toml b/saas_config.toml index d0bd16e60..1385cfb34 100644 --- a/saas_config.toml +++ b/saas_config.toml @@ -2,6 +2,8 @@ domain = "domain" username = "username" api_key = "api_key" +api_version = "2.0" +page_limit = "10" [mailchimp] domain = "" diff --git a/src/fidesops/schemas/saas/saas_config.py b/src/fidesops/schemas/saas/saas_config.py index d1afba1af..1336aaa62 100644 --- a/src/fidesops/schemas/saas/saas_config.py +++ b/src/fidesops/schemas/saas/saas_config.py @@ -30,21 +30,6 @@ class RequestParam(BaseModel): identity: Optional[str] references: Optional[List[FidesopsDatasetReference]] connector_param: Optional[str] - data_type: Optional[str] - - @validator("references") - def check_references_or_identity( - cls, - references: Optional[List[FidesopsDatasetReference]], - values: Dict[str, str], - ) -> Optional[List[FidesopsDatasetReference]]: - """Validates that each request_param only has an identity or references, not both""" - if values["identity"] is not None and references is not None: - raise ValueError( - "Can only have one of 'reference' or 'identity' per request_param, not both" - ) - return references - @validator("references") def check_reference_direction( @@ -66,10 +51,11 @@ def check_exactly_one_value_field(cls, values: Dict[str, Any]) -> Dict[str, Any] bool( values.get("default_value") is not None ), # to prevent a value of 0 from returning False + bool(values.get("connector_param")), ] if sum(value_fields) != 1: raise ValueError( - "Must have exactly one of 'identity', 'references', or 'default_value'" + "Must have exactly one of 'identity', 'references', 'default_value', or 'connector_param'" ) return values diff --git a/src/fidesops/service/connectors/query_config.py b/src/fidesops/service/connectors/query_config.py index df29d3dd8..570202197 100644 --- a/src/fidesops/service/connectors/query_config.py +++ b/src/fidesops/service/connectors/query_config.py @@ -662,4 +662,3 @@ def dry_run_query(self) -> Optional[str]: if mongo_query is not None: return self.query_to_str(mongo_query, data) return None - diff --git a/src/fidesops/service/connectors/saas_query_config.py b/src/fidesops/service/connectors/saas_query_config.py index 7b5903552..a6fc8964e 100644 --- a/src/fidesops/service/connectors/saas_query_config.py +++ b/src/fidesops/service/connectors/saas_query_config.py @@ -19,9 +19,15 @@ class SaaSQueryConfig(QueryConfig[SaaSRequestParams]): """Query config that generates populated SaaS requests for a given collection""" - def __init__(self, node: TraversalNode, endpoints: Dict[str, Endpoint]): + def __init__( + self, + node: TraversalNode, + endpoints: Dict[str, Endpoint], + connector_params: Dict[str, Any], + ): super().__init__(node) self.endpoints = endpoints + self.connector_params = connector_params def get_request_by_action(self, action: str) -> SaaSRequest: """ @@ -76,8 +82,18 @@ def generate_query( params[param.name] = param.default_value elif param.references or param.identity: params[param.name] = input_data[param.name][0] + elif param.connector_param: + params[param.name] = pydash.get( + self.connector_params, param.connector_param + ) elif param.type == "path": - path = path.replace(f"<{param.name}>", input_data[param.name][0]) + if param.references or param.identity: + path = path.replace(f"<{param.name}>", input_data[param.name][0]) + elif param.connector_param: + path = path.replace( + f"<{param.name}>", + pydash.get(self.connector_params, param.connector_param), + ) logger.info(f"Populated request params for {current_request.path}") return "GET", path, params, None @@ -93,6 +109,7 @@ def generate_update_stmt( current_request: SaaSRequest = self.get_request_by_action("update") collection_name: str = self.node.address.collection param_values: Dict[str, Row] = {collection_name: row} + identity_data: Dict[str, Any] = request.get_cached_identity_data() path: str = current_request.path params: Dict[str, Any] = {} @@ -108,12 +125,26 @@ def generate_update_stmt( ) elif param.identity: params[param.name] = pydash.get(param_values, param.identity) + elif param.connector_param: + params[param.name] = pydash.get( + self.connector_params, param.connector_param + ) elif param.type == "path": - path = path.replace( - f"<{param.name}>", - pydash.get(param_values, param.references[0].field), - ) - + if param.references: + path = path.replace( + f"<{param.name}>", + pydash.get(param_values, param.references[0].field), + ) + elif param.identity: + path = path.replace( + f"<{param.name}>", + pydash.get(identity_data, param.identity), + ) + elif param.connector_param: + path = path.replace( + f"<{param.name}>", + pydash.get(self.connector_params, param.connector_param), + ) logger.info(f"Populated request params for {current_request.path}") update_value_map: Dict[str, Any] = self.update_value_map(row, policy, request) diff --git a/tests/api/v1/endpoints/test_saas_config_endpoints.py b/tests/api/v1/endpoints/test_saas_config_endpoints.py index 50416a8c6..d6b16b35f 100644 --- a/tests/api/v1/endpoints/test_saas_config_endpoints.py +++ b/tests/api/v1/endpoints/test_saas_config_endpoints.py @@ -111,7 +111,7 @@ def test_put_validate_saas_config_reference_and_identity( ) assert response.status_code == 422 details = json.loads(response.text)["detail"] - assert details[0]["msg"] == "Must have exactly one of 'identity', 'references', or 'default_value'" + assert details[0]["msg"] == "Must have exactly one of 'identity', 'references', 'default_value', or 'connector_param'" def test_put_validate_saas_config_wrong_reference_direction( self, diff --git a/tests/service/connectors/test_queryconfig.py b/tests/service/connectors/test_queryconfig.py index fefa5c629..edde74bfa 100644 --- a/tests/service/connectors/test_queryconfig.py +++ b/tests/service/connectors/test_queryconfig.py @@ -623,8 +623,7 @@ def test_generate_query( messages = combined_traversal.traversal_node_dict[ CollectionAddress(saas_config.fides_key, "messages") ] - - payment_methods: Traversal = combined_traversal.traversal_node_dict[ + payment_methods = combined_traversal.traversal_node_dict[ CollectionAddress(saas_config.fides_key, "payment_methods") ] @@ -662,9 +661,9 @@ def test_generate_query( None, ) - # query and path params with a connector param reference + # query and path params with connector param references config = SaaSQueryConfig( - payment_methods, endpoints, {"api_version": "2.0", "page_limit": 10}, {} + payment_methods, endpoints, {"api_version": "2.0", "page_limit": 10} ) prepared_request = config.generate_query( {"query": ["customer-1@example.com"]}, policy @@ -688,6 +687,9 @@ def test_generate_update_stmt( member = combined_traversal.traversal_node_dict[ CollectionAddress(saas_config.fides_key, "member") ] + payment_methods = combined_traversal.traversal_node_dict[ + CollectionAddress(saas_config.fides_key, "payment_methods") + ] config = SaaSQueryConfig(member, endpoints, {}) row = { @@ -711,3 +713,16 @@ def test_generate_update_stmt( } ), ) + + # update with connector_param reference + config = SaaSQueryConfig(payment_methods, endpoints, {"api_version": "2.0"}) + row = {"type": "card", "customer_name": "First Last"} + prepared_request = config.generate_update_stmt( + row, erasure_policy_string_rewrite, privacy_request + ) + assert prepared_request == ( + "PUT", + "/2.0/payment_methods", + {}, + json.dumps({"customer_name": "MASKED"}), + ) From 3ecdca3963becb3bd0bef37388747e96184ac20d Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Tue, 22 Mar 2022 14:44:06 -0700 Subject: [PATCH 03/24] Updating variable name from connector_params to secrets to better reflect the confidential nature of the values --- .../service/connectors/saas_query_config.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/src/fidesops/service/connectors/saas_query_config.py b/src/fidesops/service/connectors/saas_query_config.py index a6fc8964e..4571d1c7c 100644 --- a/src/fidesops/service/connectors/saas_query_config.py +++ b/src/fidesops/service/connectors/saas_query_config.py @@ -23,11 +23,11 @@ def __init__( self, node: TraversalNode, endpoints: Dict[str, Endpoint], - connector_params: Dict[str, Any], + secrets: Dict[str, Any], ): super().__init__(node) self.endpoints = endpoints - self.connector_params = connector_params + self.secrets = secrets def get_request_by_action(self, action: str) -> SaaSRequest: """ @@ -83,16 +83,14 @@ def generate_query( elif param.references or param.identity: params[param.name] = input_data[param.name][0] elif param.connector_param: - params[param.name] = pydash.get( - self.connector_params, param.connector_param - ) + params[param.name] = pydash.get(self.secrets, param.connector_param) elif param.type == "path": if param.references or param.identity: path = path.replace(f"<{param.name}>", input_data[param.name][0]) elif param.connector_param: path = path.replace( f"<{param.name}>", - pydash.get(self.connector_params, param.connector_param), + pydash.get(self.secrets, param.connector_param), ) logger.info(f"Populated request params for {current_request.path}") @@ -126,9 +124,7 @@ def generate_update_stmt( elif param.identity: params[param.name] = pydash.get(param_values, param.identity) elif param.connector_param: - params[param.name] = pydash.get( - self.connector_params, param.connector_param - ) + params[param.name] = pydash.get(self.secrets, param.connector_param) elif param.type == "path": if param.references: path = path.replace( @@ -143,7 +139,7 @@ def generate_update_stmt( elif param.connector_param: path = path.replace( f"<{param.name}>", - pydash.get(self.connector_params, param.connector_param), + pydash.get(self.secrets, param.connector_param), ) logger.info(f"Populated request params for {current_request.path}") From b7528e67d7dd8cf607e1653421996bb8f4cfd142 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Wed, 23 Mar 2022 15:02:03 -0700 Subject: [PATCH 04/24] Updating tests --- tests/service/connectors/test_queryconfig.py | 28 +++++++++----------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/tests/service/connectors/test_queryconfig.py b/tests/service/connectors/test_queryconfig.py index 2ebe9f7b9..959bd2869 100644 --- a/tests/service/connectors/test_queryconfig.py +++ b/tests/service/connectors/test_queryconfig.py @@ -664,12 +664,10 @@ def test_generate_query( prepared_request = config.generate_query( {"query": ["customer-1@example.com"]}, policy ) - assert prepared_request == ( - "GET", - "/2.0/payment_methods", - {"limit": 10, "query": "customer-1@example.com"}, - None, - ) + assert prepared_request.method == HTTPMethod.GET.value + assert prepared_request.path == "/2.0/payment_methods" + assert prepared_request.params == {"limit": 10, "query": "customer-1@example.com"} + assert prepared_request.body is None def test_generate_update_stmt( self, @@ -683,10 +681,7 @@ def test_generate_update_stmt( member = combined_traversal.traversal_node_dict[ CollectionAddress(saas_config.fides_key, "member") ] - payment_methods = combined_traversal.traversal_node_dict[ - CollectionAddress(saas_config.fides_key, "payment_methods") - ] - + config = SaaSQueryConfig(member, endpoints, {}) row = { "id": "123", @@ -755,6 +750,9 @@ def test_generate_update_stmt_with_request_body( member = combined_traversal.traversal_node_dict[ CollectionAddress(saas_config.fides_key, "member") ] + payment_methods = combined_traversal.traversal_node_dict[ + CollectionAddress(saas_config.fides_key, "payment_methods") + ] config = SaaSQueryConfig(member, endpoints) row = { "id": "123", @@ -779,9 +777,7 @@ def test_generate_update_stmt_with_request_body( prepared_request = config.generate_update_stmt( row, erasure_policy_string_rewrite, privacy_request ) - assert prepared_request == ( - "PUT", - "/2.0/payment_methods", - {}, - json.dumps({"customer_name": "MASKED"}), - ) + assert prepared_request.method == HTTPMethod.PUT.value + assert prepared_request.path == "/2.0/payment_methods" + assert prepared_request.params == {} + assert prepared_request.body == json.dumps({"customer_name": "MASKED"}) \ No newline at end of file From 23bed420875b15bf9048a9351249f14ecb56691b Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Wed, 23 Mar 2022 15:36:10 -0700 Subject: [PATCH 05/24] Fixing tests --- tests/service/connectors/test_queryconfig.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/service/connectors/test_queryconfig.py b/tests/service/connectors/test_queryconfig.py index 959bd2869..95e980e69 100644 --- a/tests/service/connectors/test_queryconfig.py +++ b/tests/service/connectors/test_queryconfig.py @@ -681,7 +681,7 @@ def test_generate_update_stmt( member = combined_traversal.traversal_node_dict[ CollectionAddress(saas_config.fides_key, "member") ] - + config = SaaSQueryConfig(member, endpoints, {}) row = { "id": "123", @@ -714,7 +714,7 @@ def test_generate_update_stmt_custom_http_method( CollectionAddress(saas_config.fides_key, "member") ] - config = SaaSQueryConfig(member, endpoints) + config = SaaSQueryConfig(member, endpoints, {}) row = { "id": "123", "merge_fields": {"FNAME": "First", "LNAME": "Last"}, @@ -753,7 +753,7 @@ def test_generate_update_stmt_with_request_body( payment_methods = combined_traversal.traversal_node_dict[ CollectionAddress(saas_config.fides_key, "payment_methods") ] - config = SaaSQueryConfig(member, endpoints) + config = SaaSQueryConfig(member, endpoints, {}) row = { "id": "123", "merge_fields": {"FNAME": "First", "LNAME": "Last"}, From c7394f6152c97059bed2416fc93c3442a91ce3f6 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Thu, 24 Mar 2022 10:26:18 -0700 Subject: [PATCH 06/24] Adding support for HTTP headers --- data/saas/config/mailchimp_config.yml | 24 +- data/saas/config/saas_example_config.yml | 46 ++-- src/fidesops/schemas/saas/saas_config.py | 15 +- src/fidesops/schemas/saas/shared_schemas.py | 3 +- .../service/connectors/saas_connector.py | 5 +- .../service/connectors/saas_query_config.py | 233 ++++++++---------- .../pagination/pagination_strategy_offset.py | 13 +- tests/models/test_saasconfig.py | 2 +- tests/service/connectors/test_queryconfig.py | 81 ++++-- 9 files changed, 224 insertions(+), 198 deletions(-) diff --git a/data/saas/config/mailchimp_config.yml b/data/saas/config/mailchimp_config.yml index eeb1d34c2..cfbdf142a 100644 --- a/data/saas/config/mailchimp_config.yml +++ b/data/saas/config/mailchimp_config.yml @@ -22,16 +22,17 @@ saas_config: connector_param: api_key test_request: + method: GET path: /3.0/lists endpoints: - name: messages requests: read: + method: GET path: /3.0/conversations//messages request_params: - name: conversation_id - type: path references: - dataset: mailchimp_connector_example field: conversations.id @@ -46,16 +47,15 @@ saas_config: - name: conversations requests: read: + method: GET path: /3.0/conversations - request_params: + query_params: - name: count - type: query - default_value: 1000 + value: 1000 - name: offset - type: query - default_value: 0 + value: 0 + request_params: - name: placeholder - type: query identity: email data_path: conversations pagination: @@ -67,23 +67,25 @@ saas_config: - name: member requests: read: + method: GET path: /3.0/search-members - request_params: + query_params: - name: query - type: query + value: + request_params: + - name: email identity: email data_path: exact_matches.members update: + method: PUT path: /3.0/lists//members/ request_params: - name: list_id - type: path references: - dataset: mailchimp_connector_example field: member.list_id direction: from - name: subscriber_hash - type: path references: - dataset: mailchimp_connector_example field: member.id diff --git a/data/saas/config/saas_example_config.yml b/data/saas/config/saas_example_config.yml index 44b380b2d..96bd2926c 100644 --- a/data/saas/config/saas_example_config.yml +++ b/data/saas/config/saas_example_config.yml @@ -30,10 +30,10 @@ saas_config: - name: messages requests: read: + method: GET path: /3.0/conversations//messages request_params: - name: conversation_id - type: path references: - dataset: saas_connector_example field: conversations.id @@ -50,16 +50,15 @@ saas_config: - name: conversations requests: read: + method: GET path: /3.0/conversations - request_params: + query_params: - name: count - type: query - default_value: 1000 + value: 1000 - name: offset - type: query - default_value: 0 + value: 0 + request_params: - name: placeholder - type: query identity: email postprocessors: - strategy: unwrap @@ -68,27 +67,28 @@ saas_config: - name: member requests: read: + method: GET path: /3.0/search-members - request_params: + query_params: - name: query - type: query + value: + request_params: + - name: email identity: email - data_type: string postprocessors: - strategy: unwrap configuration: data_path: exact_matches.members update: + method: PUT path: /3.0/lists//members/ request_params: - name: list_id - type: path references: - dataset: saas_connector_example field: member.list_id direction: from - name: subscriber_hash - type: path references: - dataset: saas_connector_example field: member.id @@ -96,20 +96,32 @@ saas_config: - name: payment_methods requests: read: + method: GET path: //payment_methods + headers: + - name: Content-Type + value: application/json + - name: On-Behalf-Of + value: + - name: Token + value: Custom + query_params: + - name: limit + value: + - name: query + value: request_params: - name: limit - type: query connector_param: page_limit - name: version - type: path connector_param: api_version - - name: query - type: query + - name: email identity: email + - name: api_key + connector_param: api_key update: + method: PUT path: //payment_methods request_params: - name: version - type: path connector_param: api_version diff --git a/src/fidesops/schemas/saas/saas_config.py b/src/fidesops/schemas/saas/saas_config.py index 59a40d2b8..15b1d950d 100644 --- a/src/fidesops/schemas/saas/saas_config.py +++ b/src/fidesops/schemas/saas/saas_config.py @@ -26,9 +26,6 @@ class RequestParam(BaseModel): """ name: str - type: Literal[ - "query", "path", "body" - ] # used to determine location in the generated request default_value: Optional[Any] identity: Optional[str] references: Optional[List[FidesopsDatasetReference]] @@ -70,6 +67,16 @@ class Strategy(BaseModel): configuration: Dict[str, Any] +class Header(BaseModel): + name: str + value: str + + +class QueryParam(BaseModel): + name: str + value: str + + class SaaSRequest(BaseModel): """ A single request with a static or dynamic path, and the request params needed to build the request. @@ -78,6 +85,8 @@ class SaaSRequest(BaseModel): path: str method: Optional[HTTPMethod] + headers: Optional[List[Header]] + query_params: Optional[List[QueryParam]] body: Optional[str] request_params: Optional[List[RequestParam]] data_path: Optional[str] diff --git a/src/fidesops/schemas/saas/shared_schemas.py b/src/fidesops/schemas/saas/shared_schemas.py index 8f702ba89..0c239db15 100644 --- a/src/fidesops/schemas/saas/shared_schemas.py +++ b/src/fidesops/schemas/saas/shared_schemas.py @@ -20,7 +20,8 @@ class SaaSRequestParams(BaseModel): method: HTTPMethod path: str - params: Dict[str, Any] + headers: Optional[Dict[str, Any]] + query_params: Optional[Dict[str, Any]] body: Optional[str] class Config: diff --git a/src/fidesops/service/connectors/saas_connector.py b/src/fidesops/service/connectors/saas_connector.py index e50bd6b1e..724e5f01a 100644 --- a/src/fidesops/service/connectors/saas_connector.py +++ b/src/fidesops/service/connectors/saas_connector.py @@ -67,12 +67,13 @@ def get_authenticated_request( ) -> PreparedRequest: """ Returns an authenticated request based on the client config and - incoming path, query, and body params. + incoming path, headers, query, and body params. """ req = Request( method=request_params.method, url=f"{self.uri}{request_params.path}", - params=request_params.params, + headers=request_params.headers, + params=request_params.query_params, data=request_params.body, ).prepare() return self.add_authentication(req, self.client_config.authentication) diff --git a/src/fidesops/service/connectors/saas_query_config.py b/src/fidesops/service/connectors/saas_query_config.py index 90cc39ac3..2774e7f3c 100644 --- a/src/fidesops/service/connectors/saas_query_config.py +++ b/src/fidesops/service/connectors/saas_query_config.py @@ -1,8 +1,9 @@ import logging import json +import re from typing import Any, Dict, List, Optional, TypeVar import pydash -from fidesops.schemas.saas.shared_schemas import SaaSRequestParams, HTTPMethod +from fidesops.schemas.saas.shared_schemas import SaaSRequestParams from fidesops.graph.traversal import TraversalNode from fidesops.models.policy import Policy from fidesops.models.privacy_request import PrivacyRequest @@ -29,39 +30,6 @@ def __init__( self.endpoints = endpoints self.secrets = secrets - @staticmethod - def _build_request_body( # pylint: disable=R0913 - path: str, - param_name: str, - custom_body: Optional[str] = None, - default_value: Optional[str] = None, - field_reference: Optional[str] = None, - identity: Optional[str] = None, - ) -> Optional[str]: - """ - Method to build request body based on config vals. Common to both read and update requests. - Attempts to - """ - if not custom_body: - logger.info(f"Missing custom body {path}") - return None - if default_value: - custom_body = custom_body.replace(f"<{param_name}>", f'"{default_value}"') - elif field_reference: - custom_body = custom_body.replace( - f"<{param_name}>", - f'"{field_reference}"', - ) - elif identity: - custom_body = custom_body.replace( - f"<{param_name}>", - f'"{identity}"', - ) - else: - logger.info(f"Missing body param value(s) for {path}") - return None - return custom_body - def get_request_by_action(self, action: str) -> SaaSRequest: """ Returns the appropriate request config based on the @@ -95,59 +63,81 @@ def generate_requests( ) return request_params + @staticmethod + def assign_placeholders(value: str, param_values: Dict[str, Any]) -> str: + """ + Finds all the placeholders (indicated by <>) in the passed in value + and replaces them with the actual param values + """ + if value: + placeholders = re.findall("<(.+?)>", value) + for placeholder in placeholders: + value = value.replace( + f"<{placeholder}>", str(param_values[placeholder]) + ) + return value + + def map_param_values( + self, current_request: SaaSRequest, param_values: Dict[str, Any] + ) -> SaaSRequestParams: + """ + Visits path, headers, and query_params in the current request and replaces + the placeholders with the request param values + """ + + path: str = self.assign_placeholders(current_request.path, param_values) + + headers: Dict[str, Any] = {} + for header in current_request.headers or []: + headers[header.name] = self.assign_placeholders(header.value, param_values) + + query_params: Dict[str, Any] = {} + for query_param in current_request.query_params or []: + query_params[query_param.name] = self.assign_placeholders( + query_param.value, param_values + ) + + return SaaSRequestParams( + method=current_request.method, + path=path, + headers=headers, + query_params=query_params, + ) + def generate_query( self, input_data: Dict[str, List[Any]], policy: Optional[Policy] ) -> SaaSRequestParams: """ - This returns the query/path params needed to make an API call. + This returns the header, query, and path params needed to make an API call. This is the API equivalent of building the components of a database query statement (select statement, where clause, limit, offset, etc.) """ - current_request = self.get_request_by_action("read") - path: str = current_request.path - query_params: Dict[str, Any] = {} - body: Optional[str] = current_request.body - - # uses the param names to read from the input data - for param in current_request.request_params: - if param.type == "query": - if param.default_value is not None: - query_params[param.name] = param.default_value - elif param.references or param.identity: - query_params[param.name] = input_data[param.name][0] - elif param.connector_param: - query_params[param.name] = pydash.get( - self.secrets, param.connector_param - ) - elif param.type == "path": - if param.references or param.identity: - path = path.replace(f"<{param.name}>", input_data[param.name][0]) - elif param.connector_param: - path = path.replace( - f"<{param.name}>", - pydash.get(self.secrets, param.connector_param), - ) - elif param.type == "body": - body = SaaSQueryConfig._build_request_body( - path, - param.name, - body, - param.default_value, - input_data[param.name][0] if param.references else None, - input_data[param.name][0] if param.identity else None, + current_request: SaaSRequest = self.get_request_by_action("read") + + # create the source of param values to populate the various placeholders + # in the path, headers, query_params, and body + param_values: Dict[str, Any] = {} + for request_param in current_request.request_params: + if request_param.references or request_param.identity: + param_values[request_param.name] = input_data[request_param.name][0] + elif request_param.connector_param: + param_values[request_param.name] = pydash.get( + self.secrets, request_param.connector_param ) - logger.info(f"Populated request params for {current_request.path}") - method: HTTPMethod = ( - current_request.method if current_request.method else HTTPMethod.GET - ) - return SaaSRequestParams( - method=method, - path=path, - params=query_params, - body=json.loads(body) if body else None, + + # map param values to placeholders in path, headers, and query params + saas_request_params: SaaSRequestParams = self.map_param_values( + current_request, param_values ) + # map body + body = self.assign_placeholders(current_request.body, param_values) + saas_request_params.body = json.loads(body) if body else None + logger.info(f"Populated request params for {current_request.path}") + + return saas_request_params + def generate_update_stmt( self, row: Row, policy: Policy, request: PrivacyRequest ) -> SaaSRequestParams: @@ -158,73 +148,46 @@ def generate_update_stmt( current_request: SaaSRequest = self.get_request_by_action("update") collection_name: str = self.node.address.collection - param_values: Dict[str, Row] = {collection_name: row} + collection_values: Dict[str, Row] = {collection_name: row} identity_data: Dict[str, Any] = request.get_cached_identity_data() - path: str = current_request.path - params: Dict[str, Any] = {} - body: Optional[str] = current_request.body or None - - # uses the reference fields to read from the param_values - for param in current_request.request_params: - if param.type == "query": - if param.default_value is not None: - params[param.name] = param.default_value - elif param.references: - params[param.name] = pydash.get( - param_values, param.references[0].field - ) - elif param.identity: - params[param.name] = pydash.get(param_values, param.identity) - elif param.connector_param: - params[param.name] = pydash.get(self.secrets, param.connector_param) - elif param.type == "path": - if param.references: - path = path.replace( - f"<{param.name}>", - pydash.get(param_values, param.references[0].field), - ) - elif param.identity: - path = path.replace( - f"<{param.name}>", - pydash.get(identity_data, param.identity), - ) - elif param.connector_param: - path = path.replace( - f"<{param.name}>", - pydash.get(self.secrets, param.connector_param), - ) - elif param.type == "body": - body = SaaSQueryConfig._build_request_body( - path, - param.name, - body, - param.default_value, - pydash.get(param_values, param.references[0].field) - if param.references - else None, - pydash.get(param_values, param.identity) - if param.identity - else None, + # create the source of param values to populate the various placeholders + # in the path, headers, query_params, and body + param_values: Dict[str, Any] = {} + for request_param in current_request.request_params: + if request_param.references: + param_values[request_param.name] = pydash.get( + collection_values, request_param.references[0].field + ) + elif request_param.identity: + param_values[request_param.name] = pydash.get( + identity_data, request_param.identity + ) + elif request_param.connector_param: + param_values[request_param.name] = pydash.get( + self.secrets, request_param.connector_param ) - logger.info(f"Populated request params for {current_request.path}") + # mask row values update_value_map: Dict[str, Any] = self.update_value_map(row, policy, request) update_values: Dict[str, Any] = unflatten_dict(update_value_map) - method: HTTPMethod = ( - current_request.method if current_request.method else HTTPMethod.PUT + + # removes outer {} wrapper from body for greater flexibility in custom body config + param_values["masked_object_fields"] = json.dumps(update_values)[1:-1] + + # map param values to placeholders in path, headers, and query params + saas_request_params: SaaSRequestParams = self.map_param_values( + current_request, param_values ) - if body: - # removes outer {} wrapper from body for greater flexibility in custom body config - body = body.replace( - "", json.dumps(update_values)[1:-1] - ) - return SaaSRequestParams( - method=method, - path=path, - params=params, - body=json.dumps(json.loads(body) if body else update_values), + + # map body + body = self.assign_placeholders(current_request.body, param_values) + saas_request_params.body = json.dumps( + json.loads(body) if body else update_values ) + logger.info(f"Populated request params for {current_request.path}") + + return saas_request_params def query_to_str(self, t: T, input_data: Dict[str, List[Any]]) -> str: """Convert query to string""" diff --git a/src/fidesops/service/pagination/pagination_strategy_offset.py b/src/fidesops/service/pagination/pagination_strategy_offset.py index 6cf5dfa60..e44529039 100644 --- a/src/fidesops/service/pagination/pagination_strategy_offset.py +++ b/src/fidesops/service/pagination/pagination_strategy_offset.py @@ -73,12 +73,11 @@ def get_configuration_model() -> StrategyConfiguration: def validate_request(self, request: Dict[str, Any]) -> None: """Ensures that the query param specified by 'incremental_param' exists in the request""" - request_params = ( - request_param - for request_param in request.get("request_params", []) - if request_param.get("name") == self.incremental_param - and request_param.get("type") == "query" + query_params = ( + query_params + for query_params in request.get("query_params", []) + if query_params.get("name") == self.incremental_param ) - request_param = next(request_params, None) - if request_param is None: + query_param = next(query_params, None) + if query_param is None: raise ValueError(f"Query param '{self.incremental_param}' not found.") diff --git a/tests/models/test_saasconfig.py b/tests/models/test_saasconfig.py index 2a82f8a5f..44673d611 100644 --- a/tests/models/test_saasconfig.py +++ b/tests/models/test_saasconfig.py @@ -30,5 +30,5 @@ def test_saas_config_to_dataset(saas_configs: Dict[str, Dict]): assert field_address == FieldAddress(saas_config.fides_key, "conversations", "id") assert direction == "from" - assert query_field.name == "query" + assert query_field.name == "email" assert query_field.identity == "email" diff --git a/tests/service/connectors/test_queryconfig.py b/tests/service/connectors/test_queryconfig.py index 95e980e69..57bc5e523 100644 --- a/tests/service/connectors/test_queryconfig.py +++ b/tests/service/connectors/test_queryconfig.py @@ -632,11 +632,11 @@ def test_generate_query( # static path with single query param config = SaaSQueryConfig(member, endpoints, {}) prepared_request: SaaSRequestParams = config.generate_query( - {"query": ["customer-1@example.com"]}, policy + {"email": ["customer-1@example.com"]}, policy ) assert prepared_request.method == HTTPMethod.GET.value assert prepared_request.path == "/3.0/search-members" - assert prepared_request.params == {"query": "customer-1@example.com"} + assert prepared_request.query_params == {"query": "customer-1@example.com"} assert prepared_request.body is None # static path with multiple query params with default values @@ -646,7 +646,7 @@ def test_generate_query( ) assert prepared_request.method == HTTPMethod.GET.value assert prepared_request.path == "/3.0/conversations" - assert prepared_request.params == {"count": 1000, "offset": 0, "placeholder": "customer-1@example.com"} + assert prepared_request.query_params == {"count": "1000", "offset": "0"} assert prepared_request.body is None # dynamic path with no query params @@ -654,19 +654,30 @@ def test_generate_query( prepared_request = config.generate_query({"conversation_id": ["abc"]}, policy) assert prepared_request.method == HTTPMethod.GET.value assert prepared_request.path == "/3.0/conversations/abc/messages" - assert prepared_request.params == {} + assert prepared_request.query_params == {} assert prepared_request.body is None - # query and path params with connector param references + # header, query, and path params with connector param references config = SaaSQueryConfig( - payment_methods, endpoints, {"api_version": "2.0", "page_limit": 10} + payment_methods, + endpoints, + {"api_version": "2.0", "page_limit": "10", "api_key": "letmein"}, ) prepared_request = config.generate_query( - {"query": ["customer-1@example.com"]}, policy + {"email": ["customer-1@example.com"]}, policy ) + import pdb; pdb.set_trace() assert prepared_request.method == HTTPMethod.GET.value assert prepared_request.path == "/2.0/payment_methods" - assert prepared_request.params == {"limit": 10, "query": "customer-1@example.com"} + assert prepared_request.headers == { + "Content-Type": "application/json", + "On-Behalf-Of": "customer-1@example.com", + "Token": "Custom letmein", + } + assert prepared_request.query_params == { + "limit": "10", + "query": "customer-1@example.com", + } assert prepared_request.body is None def test_generate_update_stmt( @@ -696,7 +707,7 @@ def test_generate_update_stmt( ) assert prepared_request.method == HTTPMethod.PUT.value assert prepared_request.path == "/3.0/lists/abc/members/123" - assert prepared_request.params == {} + assert prepared_request.query_params == {} assert prepared_request.body == json.dumps( { "merge_fields": {"FNAME": "MASKED", "LNAME": "MASKED"}, @@ -704,9 +715,14 @@ def test_generate_update_stmt( ) def test_generate_update_stmt_custom_http_method( - self, erasure_policy_string_rewrite, combined_traversal, connection_config_saas_example + self, + erasure_policy_string_rewrite, + combined_traversal, + connection_config_saas_example, ): - saas_config: Optional[SaaSConfig] = connection_config_saas_example.get_saas_config() + saas_config: Optional[ + SaaSConfig + ] = connection_config_saas_example.get_saas_config() saas_config.endpoints[2].requests.get("update").method = HTTPMethod.POST endpoints = saas_config.top_level_endpoint_dict @@ -728,7 +744,7 @@ def test_generate_update_stmt_custom_http_method( ) assert prepared_request.method == HTTPMethod.POST.value assert prepared_request.path == "/3.0/lists/abc/members/123" - assert prepared_request.params == {} + assert prepared_request.query_params == {} assert prepared_request.body == json.dumps( { "merge_fields": {"FNAME": "MASKED", "LNAME": "MASKED"}, @@ -736,16 +752,31 @@ def test_generate_update_stmt_custom_http_method( ) def test_generate_update_stmt_with_request_body( - self, erasure_policy_string_rewrite, combined_traversal, connection_config_saas_example + self, + erasure_policy_string_rewrite, + combined_traversal, + connection_config_saas_example, ): - saas_config: Optional[SaaSConfig] = connection_config_saas_example.get_saas_config() - saas_config.endpoints[2].requests.get("update").body = '{"properties": {, "list_id": }}' + saas_config: Optional[ + SaaSConfig + ] = connection_config_saas_example.get_saas_config() + saas_config.endpoints[2].requests.get( + "update" + ).body = '{"properties": {, "list_id": ""}}' body_request_params = RequestParam( name="list_id", type="body", - references=[{"dataset": "saas_connector_example", "field": "member.list_id", "direction": "from"}] + references=[ + { + "dataset": "saas_connector_example", + "field": "member.list_id", + "direction": "from", + } + ], + ) + saas_config.endpoints[2].requests.get("update").request_params.append( + body_request_params ) - saas_config.endpoints[2].requests.get("update").request_params.append(body_request_params) endpoints = saas_config.top_level_endpoint_dict member = combined_traversal.traversal_node_dict[ CollectionAddress(saas_config.fides_key, "member") @@ -767,8 +798,16 @@ def test_generate_update_stmt_with_request_body( assert prepared_request == SaaSRequestParams( method=HTTPMethod.PUT, path="/3.0/lists/abc/members/123", - params={}, - body=json.dumps({'properties': {"merge_fields": {"FNAME": "MASKED", "LNAME": "MASKED"}, 'list_id': 'abc'}}), + headers={}, + query_params={}, + body=json.dumps( + { + "properties": { + "merge_fields": {"FNAME": "MASKED", "LNAME": "MASKED"}, + "list_id": "abc", + } + } + ), ) # update with connector_param reference @@ -779,5 +818,5 @@ def test_generate_update_stmt_with_request_body( ) assert prepared_request.method == HTTPMethod.PUT.value assert prepared_request.path == "/2.0/payment_methods" - assert prepared_request.params == {} - assert prepared_request.body == json.dumps({"customer_name": "MASKED"}) \ No newline at end of file + assert prepared_request.query_params == {} + assert prepared_request.body == json.dumps({"customer_name": "MASKED"}) From f6190b3c2add45cb4366f6d559c6134db7a8a2c2 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Thu, 24 Mar 2022 10:58:56 -0700 Subject: [PATCH 07/24] Updating documentation --- docs/fidesops/docs/guides/saas_config.md | 85 +++++++++++-------- .../docs/guides/saas_postprocessors.md | 28 +++--- src/fidesops/schemas/saas/saas_config.py | 6 +- .../endpoints/test_saas_config_endpoints.py | 2 +- 4 files changed, 63 insertions(+), 58 deletions(-) diff --git a/docs/fidesops/docs/guides/saas_config.md b/docs/fidesops/docs/guides/saas_config.md index 17e1a108f..3b3205410 100644 --- a/docs/fidesops/docs/guides/saas_config.md +++ b/docs/fidesops/docs/guides/saas_config.md @@ -41,16 +41,17 @@ saas_config: connector_param: api_key test_request: + method: GET path: /3.0/lists - + endpoints: - name: messages requests: read: + method: GET path: /3.0/conversations//messages request_params: - name: conversation_id - type: path references: - dataset: mailchimp_connector_example field: conversations.id @@ -65,39 +66,45 @@ saas_config: - name: conversations requests: read: + method: GET path: /3.0/conversations - request_params: + query_params: - name: count - type: query - default_value: 1000 + value: 1000 - name: offset - type: query - default_value: 0 + value: 0 + request_params: - name: placeholder - type: query identity: email data_path: conversations + pagination: + strategy: offset + configuration: + incremental_param: offset + increment_by: 1000 + limit: 10000 - name: member requests: read: + method: GET path: /3.0/search-members - request_params: + query_params: - name: query - type: query + value: + request_params: + - name: email identity: email - data_type: string data_path: exact_matches.members update: + method: PUT path: /3.0/lists//members/ request_params: - name: list_id - type: path references: - dataset: mailchimp_connector_example field: member.list_id direction: from - name: subscriber_hash - type: path references: - dataset: mailchimp_connector_example field: member.id @@ -158,26 +165,28 @@ authentication: Once the base client is defined we can use a `test_request` to verify our hostname and credentials. This is in the form of an idempotent request (usually a read). The testing approach is the same for any [ConnectionConfig test](database_connectors.md#testing-your-connection). ```yaml test_request: + method: GET path: /3.0/lists ``` #### Endpoints This is where we define how we are going to access and update each collection in the corresponding Dataset. The endpoint section contains the following members: - `name` This name corresponds to a Collection in the corresponding Dataset. -- `requests` A map of `read` and `update` requests for this collection. Each collection can define a way to read and a way to update the data. Each request is made up of: +- `requests` A map of `read`, `update`, and `delete` requests for this collection. Each collection can define a way to read and a way to update the data. Each request is made up of: + - `method` The HTTP method used for the endpoint. - `path` A static or dynamic resource path. The dynamic portions of the path are enclosed within angle brackets `` and are replaced with values from `request_params`. - - `method` (optional) HTTP method. Defaults to `GET` for read requests, `PUT` for update requests. Other options are `POST`, `PATCH`, or `DELETE`. - - `body` (optional) static or dynamic request body, with dynamic portions enclosed in brackets, just like `path`. These dynamic values will be replaced with values from `request_params`. For update requests, you'll need to additionally annotated `` as a placeholder for the fidesops generated update values. + - `headers` and `query_params` The HTTP headers and query parameters to include in the request. + - `name` the value to use for the header or query param name. + - `value` can be a static value or use a `` which will be replaced with the value sourced from the `request_param` with a matching name. + - `body` (optional) static or dynamic request body, with dynamic portions enclosed in brackets, just like `path`. These dynamic values will be replaced with values from `request_params`. For update requests, you'll need to additionally annotated `` as a placeholder for the Fidesops generated update values. - `request_params` - - `name` Used as the key for query param values, or to map this param to a value placeholder in the path. - - `type` Can be "query", "path", or "body". + - `name` Used as the key to reference this value from dynamic values in the path, headers, query, or body params. - `references` These are the same as `references` in the Dataset schema. It is used to define the source of the value for the given request_param. - `identity` Used to access the identity values passed into the privacy request such as email or phone number. - - `default_value` Hard-coded default value for a `request_param`. This is most often used for query params since a static path param can just be included in the `path`. - `connector_param` Used to access the user-configured secrets for the connection. - `data_path`: The expression used to access the collection information from the raw JSON response. - `postprocessors` An optional list of response post-processing strategies. We will ignore this for the example scenarios below but an in depth-explanation can be found under [SaaS Post-Processors](saas_postprocessors.md) - - `pagination` An optional strategy used to get the next set of results from APIs with resources spanning multiple pages. Details can be found under [SaaS Pagination](saas_pagination.md) + - `pagination` An optional strategy used to get the next set of results from APIs with resources spanning multiple pages. Details can be found under [SaaS Pagination](saas_pagination.md). ## Example scenarios #### Dynamic path with dataset references @@ -186,10 +195,10 @@ endpoints: - name: messages requests: read: + method: GET path: /3.0/conversations//messages request_params: - name: conversation_id - type: path references: - dataset: mailchimp_connector_example field: conversations.id @@ -210,13 +219,16 @@ endpoints: - name: member requests: read: + method: GET path: /3.0/search-members - request_params: + query_params: - name: query - type: query + value: + request_params: + - name: email identity: email ``` -In this example, the `email` identity value is used as a param named "query" and would look like this: +In this example, the placeholder in the `query` query param would be replaced with the value of the `request_param` with a name of `email`, which is the `email` identity. The result would look like this: ``` GET /3.0/search-members?query=name@email.com ``` @@ -227,16 +239,15 @@ endpoints: - name: member requests: update: + method: PUT path: /3.0/lists//members/ request_params: - name: list_id - type: path references: - dataset: mailchimp_connector_example field: member.list_id direction: from - name: subscriber_hash - type: path references: - dataset: mailchimp_connector_example field: member.id @@ -272,8 +283,9 @@ and the contents of the body would be masked according to the configured [policy #### Data update with a dynamic HTTP body Sometimes, the update request needs a different body structure than what we obtain from the read request. In this example, we use a custom HTTP body that contains our masked object fields. -```yml +```yaml update: + method: PUT path: /crm/v3/objects/contacts body: { "properties": { @@ -283,7 +295,6 @@ update: } request_params: - name: user_ref_id - type: body references: - dataset: dataset_test field: contacts.user_ref_id @@ -295,6 +306,7 @@ Fidesops will replace the `` placeholder with the result o This results in the following update request: ```yaml PUT /crm/v3/objects/contacts + { "properties": { "company": "None", @@ -336,10 +348,10 @@ endpoints: - name: messages requests: read: + method: GET path: /3.0/conversations//messages request_params: - name: conversation_id - type: path references: - dataset: mailchimp_connector_example field: conversations.id @@ -385,22 +397,21 @@ collections: Notice how the `conversation_id` field is updated with a reference from `mailchimp_connector_example.conversations.id`. This means that the `conversations` collection must be retrieved first to forward the conversation IDs to the messages collection for further processing. ## What if a collection has no dependencies? -In the Mailchimp example, you might have noticed the `placeholder` query param. +In the Mailchimp example, you might have noticed the `placeholder` request param. ```yaml endpoints: - name: conversations requests: read: + method: GET path: /3.0/conversations - request_params: + query_params: - name: count - type: query - default_value: 1000 + value: 1000 - name: offset - type: query - default_value: 0 + value: 0 + request_params: - name: placeholder - type: query identity: email ``` -Some endpoints might not have any external dependencies on `identity` or Dataset `reference` values. The way the Fidesops [graph traversal](query_execution.md) interprets this is as an unreachable collection. At this time, the way to mark this as reachable is to include `request_param` with a "synthetic link" (an unused query param) to an identity or a reference. In the future we plan on having collections like these still be considered reachable even without this synthetic placeholder (the request_param name is not relevant, we just chose placeholder for this example). \ No newline at end of file +Some endpoints might not have any external dependencies on `identity` or Dataset `reference` values. The way the Fidesops [graph traversal](query_execution.md) interprets this is as an unreachable collection. At this time, the way to mark this as reachable is to include a `request_param` with an identity or a reference. In the future we plan on having collections like these still be considered reachable even without this placeholder (the request_param name is not relevant, we just chose placeholder for this example). \ No newline at end of file diff --git a/docs/fidesops/docs/guides/saas_postprocessors.md b/docs/fidesops/docs/guides/saas_postprocessors.md index 9967b5bd5..6ff967416 100644 --- a/docs/fidesops/docs/guides/saas_postprocessors.md +++ b/docs/fidesops/docs/guides/saas_postprocessors.md @@ -11,6 +11,7 @@ endpoints: - name: messages requests: read: + method: GET path: /conversations//messages request_params: ... @@ -61,14 +62,14 @@ Post-Processor Config: Identity data passed in through request: -``` +```json { "email": "somebody@email.com" } ``` Data to be processed: -``` +```json data = [ { "id": 1397429347 @@ -84,14 +85,14 @@ data = [ ``` Result: -``` +```json result = [ - { - "id": 1397429347 - "email_contact": "somebody@email.com" - "name": "Somebody Awesome" - } - ] + { + "id": 1397429347 + "email_contact": "somebody@email.com" + "name": "Somebody Awesome" + } +] ``` Note: Type casting is not supported at this time. We currently only support filtering by string values. e.g. `bob@mail.com` and not `12344245`. @@ -120,7 +121,7 @@ Post-Processor Config: ``` Data to be processed: -``` +```json data = { "exact_matches": { "members": [ @@ -128,13 +129,10 @@ data = { { "meow": 841 } ] } -} - -data_path = exact_matches.members - +} ``` Result: -``` +```json result = [ { "howdy": 123 }, { "meow": 841 } diff --git a/src/fidesops/schemas/saas/saas_config.py b/src/fidesops/schemas/saas/saas_config.py index 15b1d950d..bdccf4e1c 100644 --- a/src/fidesops/schemas/saas/saas_config.py +++ b/src/fidesops/schemas/saas/saas_config.py @@ -26,7 +26,6 @@ class RequestParam(BaseModel): """ name: str - default_value: Optional[Any] identity: Optional[str] references: Optional[List[FidesopsDatasetReference]] connector_param: Optional[str] @@ -48,14 +47,11 @@ def check_exactly_one_value_field(cls, values: Dict[str, Any]) -> Dict[str, Any] value_fields = [ bool(values.get("identity")), bool(values.get("references")), - bool( - values.get("default_value") is not None - ), # to prevent a value of 0 from returning False bool(values.get("connector_param")), ] if sum(value_fields) != 1: raise ValueError( - "Must have exactly one of 'identity', 'references', 'default_value', or 'connector_param'" + "Must have exactly one of 'identity', 'references', or 'connector_param'" ) return values diff --git a/tests/api/v1/endpoints/test_saas_config_endpoints.py b/tests/api/v1/endpoints/test_saas_config_endpoints.py index d6b16b35f..1b973ee3a 100644 --- a/tests/api/v1/endpoints/test_saas_config_endpoints.py +++ b/tests/api/v1/endpoints/test_saas_config_endpoints.py @@ -111,7 +111,7 @@ def test_put_validate_saas_config_reference_and_identity( ) assert response.status_code == 422 details = json.loads(response.text)["detail"] - assert details[0]["msg"] == "Must have exactly one of 'identity', 'references', 'default_value', or 'connector_param'" + assert details[0]["msg"] == "Must have exactly one of 'identity', 'references', or 'connector_param'" def test_put_validate_saas_config_wrong_reference_direction( self, From 6eb40774aeb2c403f716a6cc901748c790ebcc75 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Thu, 24 Mar 2022 11:06:55 -0700 Subject: [PATCH 08/24] Updating Postman collection --- .../postman/Fidesops.postman_collection.json | 20 +++++-------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/docs/fidesops/docs/postman/Fidesops.postman_collection.json b/docs/fidesops/docs/postman/Fidesops.postman_collection.json index 9bd5f476f..8231fcfb3 100644 --- a/docs/fidesops/docs/postman/Fidesops.postman_collection.json +++ b/docs/fidesops/docs/postman/Fidesops.postman_collection.json @@ -1,6 +1,6 @@ { "info": { - "_postman_id": "7a31c401-46b8-4d1f-a55c-22c7b05d5b90", + "_postman_id": "ddf34df5-936a-4a8f-9dac-9d853ee53a95", "name": "Fidesops", "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" }, @@ -29,12 +29,7 @@ "value": "{{OAUTH_ROOT_CLIENT_SECRET}}", "type": "text" } - ], - "options": { - "raw": { - "language": "json" - } - } + ] }, "url": { "raw": "{{host}}/oauth/token", @@ -107,12 +102,7 @@ "value": "{{client_secret}}", "type": "text" } - ], - "options": { - "raw": { - "language": "json" - } - } + ] }, "url": { "raw": "{{host}}/oauth/token", @@ -2376,7 +2366,7 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"fides_key\": \"mailchimp_connector_example\",\n \"name\": \"Mailchimp SaaS Config\",\n \"description\": \"A sample schema representing the Mailchimp connector for Fidesops\",\n \"version\": \"0.0.2\",\n \"connector_params\": [\n {\n \"name\": \"domain\"\n },\n {\n \"name\": \"username\"\n },\n {\n \"name\": \"api_key\"\n }\n ],\n \"client_config\": {\n \"protocol\": \"https\",\n \"host\": {\n \"connector_param\": \"domain\"\n },\n \"authentication\": {\n \"strategy\": \"basic_authentication\",\n \"configuration\": {\n \"username\": {\n \"connector_param\": \"username\"\n },\n \"password\": {\n \"connector_param\": \"api_key\"\n }\n }\n }\n },\n \"test_request\": {\n \"path\": \"/3.0/lists\"\n },\n \"endpoints\": [\n {\n \"name\": \"messages\",\n \"requests\": {\n \"read\": {\n \"path\": \"/3.0/conversations//messages\",\n \"request_params\": [\n {\n \"name\": \"conversation_id\",\n \"type\": \"path\",\n \"references\": [\n {\n \"dataset\": \"mailchimp_connector_example\",\n \"field\": \"conversations.id\",\n \"direction\": \"from\"\n }\n ]\n }\n ],\n \"data_path\": \"conversation_messages\",\n \"postprocessors\": [\n {\n \"strategy\": \"filter\",\n \"configuration\": {\n \"field\": \"from_email\",\n \"value\": {\n \"identity\": \"email\"\n }\n }\n }\n ]\n }\n }\n },\n {\n \"name\": \"conversations\",\n \"requests\": {\n \"read\": {\n \"path\": \"/3.0/conversations\",\n \"request_params\": [\n {\n \"name\": \"count\",\n \"type\": \"query\",\n \"default_value\": 1000\n },\n {\n \"name\": \"offset\",\n \"type\": \"query\",\n \"default_value\": 0\n },\n {\n \"name\": \"placeholder\",\n \"type\": \"query\",\n \"identity\": \"email\"\n }\n ],\n \"data_path\": \"conversations\",\n \"pagination\": {\n \"strategy\": \"offset\",\n \"configuration\": {\n \"incremental_param\": \"offset\",\n \"increment_by\": 1000,\n \"limit\": 10000\n }\n }\n }\n }\n },\n {\n \"name\": \"member\",\n \"requests\": {\n \"read\": {\n \"path\": \"/3.0/search-members\",\n \"request_params\": [\n {\n \"name\": \"query\",\n \"type\": \"query\",\n \"identity\": \"email\"\n }\n ],\n \"data_path\": \"exact_matches.members\"\n },\n \"update\": {\n \"path\": \"/3.0/lists//members/\",\n \"request_params\": [\n {\n \"name\": \"list_id\",\n \"type\": \"path\",\n \"references\": [\n {\n \"dataset\": \"mailchimp_connector_example\",\n \"field\": \"member.list_id\",\n \"direction\": \"from\"\n }\n ]\n },\n {\n \"name\": \"subscriber_hash\",\n \"type\": \"path\",\n \"references\": [\n {\n \"dataset\": \"mailchimp_connector_example\",\n \"field\": \"member.id\",\n \"direction\": \"from\"\n }\n ]\n }\n ]\n }\n }\n }\n ]\n}", + "raw": "{\n \"fides_key\": \"mailchimp_connector_example\",\n \"name\": \"Mailchimp SaaS Config\",\n \"description\": \"A sample schema representing the Mailchimp connector for Fidesops\",\n \"version\": \"0.0.1\",\n \"connector_params\": [\n {\n \"name\": \"domain\"\n },\n {\n \"name\": \"username\"\n },\n {\n \"name\": \"api_key\"\n }\n ],\n \"client_config\": {\n \"protocol\": \"https\",\n \"host\": {\n \"connector_param\": \"domain\"\n },\n \"authentication\": {\n \"strategy\": \"basic_authentication\",\n \"configuration\": {\n \"username\": {\n \"connector_param\": \"username\"\n },\n \"password\": {\n \"connector_param\": \"api_key\"\n }\n }\n }\n },\n \"test_request\": {\n \"method\": \"GET\",\n \"path\": \"/3.0/lists\"\n },\n \"endpoints\": [\n {\n \"name\": \"messages\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/3.0/conversations//messages\",\n \"request_params\": [\n {\n \"name\": \"conversation_id\",\n \"references\": [\n {\n \"dataset\": \"mailchimp_connector_example\",\n \"field\": \"conversations.id\",\n \"direction\": \"from\"\n }\n ]\n }\n ],\n \"data_path\": \"conversation_messages\",\n \"postprocessors\": [\n {\n \"strategy\": \"filter\",\n \"configuration\": {\n \"field\": \"from_email\",\n \"value\": {\n \"identity\": \"email\"\n }\n }\n }\n ]\n }\n }\n },\n {\n \"name\": \"conversations\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/3.0/conversations\",\n \"query_params\": [\n {\n \"name\": \"count\",\n \"value\": 1000\n },\n {\n \"name\": \"offset\",\n \"value\": 0\n }\n ],\n \"request_params\": [\n {\n \"name\": \"placeholder\",\n \"identity\": \"email\"\n }\n ],\n \"data_path\": \"conversations\",\n \"pagination\": {\n \"strategy\": \"offset\",\n \"configuration\": {\n \"incremental_param\": \"offset\",\n \"increment_by\": 1000,\n \"limit\": 10000\n }\n }\n }\n }\n },\n {\n \"name\": \"member\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/3.0/search-members\",\n \"query_params\": [\n {\n \"name\": \"query\",\n \"value\": \"\"\n }\n ],\n \"request_params\": [\n {\n \"name\": \"email\",\n \"identity\": \"email\"\n }\n ],\n \"data_path\": \"exact_matches.members\"\n },\n \"update\": {\n \"method\": \"PUT\",\n \"path\": \"/3.0/lists//members/\",\n \"request_params\": [\n {\n \"name\": \"list_id\",\n \"references\": [\n {\n \"dataset\": \"mailchimp_connector_example\",\n \"field\": \"member.list_id\",\n \"direction\": \"from\"\n }\n ]\n },\n {\n \"name\": \"subscriber_hash\",\n \"references\": [\n {\n \"dataset\": \"mailchimp_connector_example\",\n \"field\": \"member.id\",\n \"direction\": \"from\"\n }\n ]\n }\n ]\n }\n }\n }\n ]\n}", "options": { "raw": { "language": "json" @@ -2414,7 +2404,7 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"fides_key\": \"mailchimp_connector_example\",\n \"name\": \"Mailchimp SaaS Config\",\n \"description\": \"A sample schema representing the Mailchimp connector for Fidesops\",\n \"version\": \"0.0.2\",\n \"connector_params\": [\n {\n \"name\": \"domain\"\n },\n {\n \"name\": \"username\"\n },\n {\n \"name\": \"api_key\"\n }\n ],\n \"client_config\": {\n \"protocol\": \"https\",\n \"host\": {\n \"connector_param\": \"domain\"\n },\n \"authentication\": {\n \"strategy\": \"basic_authentication\",\n \"configuration\": {\n \"username\": {\n \"connector_param\": \"username\"\n },\n \"password\": {\n \"connector_param\": \"api_key\"\n }\n }\n }\n },\n \"test_request\": {\n \"path\": \"/3.0/lists\"\n },\n \"endpoints\": [\n {\n \"name\": \"messages\",\n \"requests\": {\n \"read\": {\n \"path\": \"/3.0/conversations//messages\",\n \"request_params\": [\n {\n \"name\": \"conversation_id\",\n \"type\": \"path\",\n \"references\": [\n {\n \"dataset\": \"mailchimp_connector_example\",\n \"field\": \"conversations.id\",\n \"direction\": \"from\"\n }\n ]\n }\n ],\n \"data_path\": \"conversation_messages\",\n \"postprocessors\": [\n {\n \"strategy\": \"filter\",\n \"configuration\": {\n \"field\": \"from_email\",\n \"value\": {\n \"identity\": \"email\"\n }\n }\n }\n ]\n }\n }\n },\n {\n \"name\": \"conversations\",\n \"requests\": {\n \"read\": {\n \"path\": \"/3.0/conversations\",\n \"request_params\": [\n {\n \"name\": \"count\",\n \"type\": \"query\",\n \"default_value\": 1000\n },\n {\n \"name\": \"offset\",\n \"type\": \"query\",\n \"default_value\": 0\n },\n {\n \"name\": \"placeholder\",\n \"type\": \"query\",\n \"identity\": \"email\"\n }\n ],\n \"data_path\": \"conversations\",\n \"pagination\": {\n \"strategy\": \"offset\",\n \"configuration\": {\n \"incremental_param\": \"offset\",\n \"increment_by\": 1000,\n \"limit\": 10000\n }\n }\n }\n }\n },\n {\n \"name\": \"member\",\n \"requests\": {\n \"read\": {\n \"path\": \"/3.0/search-members\",\n \"request_params\": [\n {\n \"name\": \"query\",\n \"type\": \"query\",\n \"identity\": \"email\"\n }\n ],\n \"data_path\": \"exact_matches.members\"\n },\n \"update\": {\n \"path\": \"/3.0/lists//members/\",\n \"request_params\": [\n {\n \"name\": \"list_id\",\n \"type\": \"path\",\n \"references\": [\n {\n \"dataset\": \"mailchimp_connector_example\",\n \"field\": \"member.list_id\",\n \"direction\": \"from\"\n }\n ]\n },\n {\n \"name\": \"subscriber_hash\",\n \"type\": \"path\",\n \"references\": [\n {\n \"dataset\": \"mailchimp_connector_example\",\n \"field\": \"member.id\",\n \"direction\": \"from\"\n }\n ]\n }\n ]\n }\n }\n }\n ]\n}", + "raw": "{\n \"fides_key\": \"mailchimp_connector_example\",\n \"name\": \"Mailchimp SaaS Config\",\n \"description\": \"A sample schema representing the Mailchimp connector for Fidesops\",\n \"version\": \"0.0.1\",\n \"connector_params\": [\n {\n \"name\": \"domain\"\n },\n {\n \"name\": \"username\"\n },\n {\n \"name\": \"api_key\"\n }\n ],\n \"client_config\": {\n \"protocol\": \"https\",\n \"host\": {\n \"connector_param\": \"domain\"\n },\n \"authentication\": {\n \"strategy\": \"basic_authentication\",\n \"configuration\": {\n \"username\": {\n \"connector_param\": \"username\"\n },\n \"password\": {\n \"connector_param\": \"api_key\"\n }\n }\n }\n },\n \"test_request\": {\n \"method\": \"GET\",\n \"path\": \"/3.0/lists\"\n },\n \"endpoints\": [\n {\n \"name\": \"messages\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/3.0/conversations//messages\",\n \"request_params\": [\n {\n \"name\": \"conversation_id\",\n \"references\": [\n {\n \"dataset\": \"mailchimp_connector_example\",\n \"field\": \"conversations.id\",\n \"direction\": \"from\"\n }\n ]\n }\n ],\n \"data_path\": \"conversation_messages\",\n \"postprocessors\": [\n {\n \"strategy\": \"filter\",\n \"configuration\": {\n \"field\": \"from_email\",\n \"value\": {\n \"identity\": \"email\"\n }\n }\n }\n ]\n }\n }\n },\n {\n \"name\": \"conversations\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/3.0/conversations\",\n \"query_params\": [\n {\n \"name\": \"count\",\n \"value\": 1000\n },\n {\n \"name\": \"offset\",\n \"value\": 0\n }\n ],\n \"request_params\": [\n {\n \"name\": \"placeholder\",\n \"identity\": \"email\"\n }\n ],\n \"data_path\": \"conversations\",\n \"pagination\": {\n \"strategy\": \"offset\",\n \"configuration\": {\n \"incremental_param\": \"offset\",\n \"increment_by\": 1000,\n \"limit\": 10000\n }\n }\n }\n }\n },\n {\n \"name\": \"member\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/3.0/search-members\",\n \"query_params\": [\n {\n \"name\": \"query\",\n \"value\": \"\"\n }\n ],\n \"request_params\": [\n {\n \"name\": \"email\",\n \"identity\": \"email\"\n }\n ],\n \"data_path\": \"exact_matches.members\"\n },\n \"update\": {\n \"method\": \"PUT\",\n \"path\": \"/3.0/lists//members/\",\n \"request_params\": [\n {\n \"name\": \"list_id\",\n \"references\": [\n {\n \"dataset\": \"mailchimp_connector_example\",\n \"field\": \"member.list_id\",\n \"direction\": \"from\"\n }\n ]\n },\n {\n \"name\": \"subscriber_hash\",\n \"references\": [\n {\n \"dataset\": \"mailchimp_connector_example\",\n \"field\": \"member.id\",\n \"direction\": \"from\"\n }\n ]\n }\n ]\n }\n }\n }\n ]\n}", "options": { "raw": { "language": "json" From 810f94a25da01a0fbd53760e2a8abcc90951c166 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Thu, 24 Mar 2022 11:22:22 -0700 Subject: [PATCH 09/24] Minor documentation change --- docs/fidesops/docs/guides/saas_pagination.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/fidesops/docs/guides/saas_pagination.md b/docs/fidesops/docs/guides/saas_pagination.md index c4d1dc7b1..ae8e756f6 100644 --- a/docs/fidesops/docs/guides/saas_pagination.md +++ b/docs/fidesops/docs/guides/saas_pagination.md @@ -76,7 +76,7 @@ This strategy is used when a specific value from a response object is used as a #### Examples If an API request returns the following: -``` +```json { "messages": [ {"id": 1, msg: "this is"}, From be629027275619cc2204660e18445345d914e903 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Thu, 24 Mar 2022 11:58:40 -0700 Subject: [PATCH 10/24] Cleaning up usages of SaaSRequestParams --- .../service/connectors/saas_connector.py | 2 +- .../pagination/pagination_strategy_cursor.py | 2 +- .../pagination/pagination_strategy_link.py | 6 +++--- .../pagination/pagination_strategy_offset.py | 2 +- tests/fixtures/saas_fixtures.py | 4 ++-- tests/service/connectors/test_queryconfig.py | 1 - .../test_pagination_strategy_cursor.py | 11 ++-------- .../test_pagination_strategy_link.py | 18 ++++++---------- .../test_pagination_strategy_offset.py | 21 +++++++------------ .../request_runner_service_test.py | 2 +- 10 files changed, 24 insertions(+), 45 deletions(-) diff --git a/src/fidesops/service/connectors/saas_connector.py b/src/fidesops/service/connectors/saas_connector.py index 724e5f01a..3f5b71a52 100644 --- a/src/fidesops/service/connectors/saas_connector.py +++ b/src/fidesops/service/connectors/saas_connector.py @@ -114,7 +114,7 @@ def test_connection(self) -> Optional[ConnectionTestStatus]: """Generates and executes a test connection based on the SaaS config""" test_request_path = self.saas_config.test_request.path prepared_request: SaaSRequestParams = SaaSRequestParams( - method=HTTPMethod.GET, path=test_request_path, params={}, body=None + method=HTTPMethod.GET, path=test_request_path ) self.client().send(prepared_request) return ConnectionTestStatus.succeeded diff --git a/src/fidesops/service/pagination/pagination_strategy_cursor.py b/src/fidesops/service/pagination/pagination_strategy_cursor.py index 990b57bfe..c15be1f71 100644 --- a/src/fidesops/service/pagination/pagination_strategy_cursor.py +++ b/src/fidesops/service/pagination/pagination_strategy_cursor.py @@ -46,7 +46,7 @@ def get_next_request( return SaaSRequestParams( method=request_params.method, path=request_params.path, - params=request_params.params, + query_params=request_params.query_params, body=request_params.body, ) diff --git a/src/fidesops/service/pagination/pagination_strategy_link.py b/src/fidesops/service/pagination/pagination_strategy_link.py index 483bfc72e..12fbcc896 100644 --- a/src/fidesops/service/pagination/pagination_strategy_link.py +++ b/src/fidesops/service/pagination/pagination_strategy_link.py @@ -56,14 +56,14 @@ def get_next_request( # replace existing path and params with updated path and query params updated_path = urlsplit(next_link).path - updated_params = dict(parse.parse_qsl(urlsplit(next_link).query)) + updated_query_params = dict(parse.parse_qsl(urlsplit(next_link).query)) logger.debug( - f"Replacing path with {updated_path} and params with {updated_params}" + f"Replacing path with {updated_path} and query params with {updated_query_params}" ) return SaaSRequestParams( method=request_params.method, path=updated_path, - params=updated_params, + query_params=updated_query_params, body=request_params.body, ) diff --git a/src/fidesops/service/pagination/pagination_strategy_offset.py b/src/fidesops/service/pagination/pagination_strategy_offset.py index e44529039..b9fb8ef46 100644 --- a/src/fidesops/service/pagination/pagination_strategy_offset.py +++ b/src/fidesops/service/pagination/pagination_strategy_offset.py @@ -63,7 +63,7 @@ def get_next_request( return SaaSRequestParams( method=request_params.method, path=request_params.path, - params=request_params.params, + query_params=request_params.query_params, body=request_params.body, ) diff --git a/tests/fixtures/saas_fixtures.py b/tests/fixtures/saas_fixtures.py index 95e48a8a6..09cc61029 100644 --- a/tests/fixtures/saas_fixtures.py +++ b/tests/fixtures/saas_fixtures.py @@ -217,7 +217,7 @@ def reset_mailchimp_data( """ connector = SaaSConnector(connection_config_mailchimp) request: SaaSRequestParams = SaaSRequestParams( - method=HTTPMethod.GET, path="/3.0/search-members", params={"query": mailchimp_identity_email}, body=None + method=HTTPMethod.GET, path="/3.0/search-members", query_params={"query": mailchimp_identity_email}, body=None ) response = connector.create_client().send(request) body = response.json() @@ -226,7 +226,7 @@ def reset_mailchimp_data( request: SaaSRequestParams = SaaSRequestParams( method=HTTPMethod.PUT, path=f'/3.0/lists/{member["list_id"]}/members/{member["id"]}', - params={}, + query_params={}, body=json.dumps(member) ) connector.create_client().send(request) diff --git a/tests/service/connectors/test_queryconfig.py b/tests/service/connectors/test_queryconfig.py index 57bc5e523..330b27fb8 100644 --- a/tests/service/connectors/test_queryconfig.py +++ b/tests/service/connectors/test_queryconfig.py @@ -666,7 +666,6 @@ def test_generate_query( prepared_request = config.generate_query( {"email": ["customer-1@example.com"]}, policy ) - import pdb; pdb.set_trace() assert prepared_request.method == HTTPMethod.GET.value assert prepared_request.path == "/2.0/payment_methods" assert prepared_request.headers == { diff --git a/tests/service/pagination/test_pagination_strategy_cursor.py b/tests/service/pagination/test_pagination_strategy_cursor.py index 8554d1be0..baa39fbef 100644 --- a/tests/service/pagination/test_pagination_strategy_cursor.py +++ b/tests/service/pagination/test_pagination_strategy_cursor.py @@ -36,9 +36,7 @@ def test_cursor(response_with_body): ) request_params: SaaSRequestParams = SaaSRequestParams( method=HTTPMethod.GET, - path="/conversations", - params={}, - body=None + path="/conversations" ) paginator = CursorPaginationStrategy(config) @@ -48,8 +46,7 @@ def test_cursor(response_with_body): assert next_request == SaaSRequestParams( method=HTTPMethod.GET, path="/conversations", - params={"after": 3}, - body=None + query_params={"after": 3}, ) @@ -60,8 +57,6 @@ def test_missing_cursor_value(response_with_body): request_params: SaaSRequestParams = SaaSRequestParams( method=HTTPMethod.GET, path="/conversations", - params={}, - body=None ) paginator = CursorPaginationStrategy(config) @@ -78,8 +73,6 @@ def test_cursor_with_empty_list(response_with_empty_list): request_params: SaaSRequestParams = SaaSRequestParams( method=HTTPMethod.GET, path="/conversations", - params={}, - body=None ) paginator = CursorPaginationStrategy(config) diff --git a/tests/service/pagination/test_pagination_strategy_link.py b/tests/service/pagination/test_pagination_strategy_link.py index 1b988ca9f..78216d6bf 100644 --- a/tests/service/pagination/test_pagination_strategy_link.py +++ b/tests/service/pagination/test_pagination_strategy_link.py @@ -38,8 +38,7 @@ def test_link_in_headers(response_with_header_link): request_params: SaaSRequestParams = SaaSRequestParams( method=HTTPMethod.GET, path="/customers", - params={"page": "abc"}, - body=None + query_params={"page": "abc"}, ) paginator = LinkPaginationStrategy(config) @@ -49,8 +48,7 @@ def test_link_in_headers(response_with_header_link): assert next_request == SaaSRequestParams( method=HTTPMethod.GET, path="/customers", - params={"page": "def"}, - body=None + query_params={"page": "def"}, ) @@ -59,8 +57,7 @@ def test_link_in_headers_missing(response_with_body_link): request_params: SaaSRequestParams = SaaSRequestParams( method=HTTPMethod.GET, path="/customers", - params={"page": "abc"}, - body=None + query_params={"page": "abc"}, ) paginator = LinkPaginationStrategy(config) @@ -75,8 +72,7 @@ def test_link_in_body(response_with_body_link): request_params: SaaSRequestParams = SaaSRequestParams( method=HTTPMethod.GET, path="/customers", - params={"page": "abc"}, - body=None + query_params={"page": "abc"}, ) paginator = LinkPaginationStrategy(config) @@ -86,8 +82,7 @@ def test_link_in_body(response_with_body_link): assert next_request == SaaSRequestParams( method=HTTPMethod.GET, path="/customers", - params={"page": "def"}, - body=None + query_params={"page": "def"}, ) @@ -96,8 +91,7 @@ def test_link_in_body_missing(response_with_header_link): request_params: SaaSRequestParams = SaaSRequestParams( method=HTTPMethod.GET, path="/customers", - params={"page": "abc"}, - body=None + query_params={"page": "abc"}, ) paginator = LinkPaginationStrategy(config) diff --git a/tests/service/pagination/test_pagination_strategy_offset.py b/tests/service/pagination/test_pagination_strategy_offset.py index e4cc4471d..bcafc29bf 100644 --- a/tests/service/pagination/test_pagination_strategy_offset.py +++ b/tests/service/pagination/test_pagination_strategy_offset.py @@ -31,8 +31,7 @@ def test_offset(response_with_body): request_params: SaaSRequestParams = SaaSRequestParams( method=HTTPMethod.GET, path="/conversations", - params={"page": 1}, - body=None + query_params={"page": 1}, ) paginator = OffsetPaginationStrategy(config) next_request: Optional[SaaSRequestParams] = paginator.get_next_request( @@ -41,8 +40,7 @@ def test_offset(response_with_body): assert next_request == SaaSRequestParams( method=HTTPMethod.GET, path="/conversations", - params={"page": 2}, - body=None + query_params={"page": 2}, ) @@ -56,8 +54,7 @@ def test_offset_with_connector_param_reference(response_with_body): request_params: SaaSRequestParams = SaaSRequestParams( method=HTTPMethod.GET, path="/conversations", - params={"page": 1}, - body=None + query_params={"page": 1}, ) paginator = OffsetPaginationStrategy(config) @@ -67,8 +64,7 @@ def test_offset_with_connector_param_reference(response_with_body): assert next_request == SaaSRequestParams( method=HTTPMethod.GET, path="/conversations", - params={"page": 2}, - body=None + query_params={"page": 2}, ) @@ -81,8 +77,7 @@ def test_offset_with_connector_param_reference_not_found(response_with_body): request_params: SaaSRequestParams = SaaSRequestParams( method=HTTPMethod.GET, path="/conversations", - params={"page": 1}, - body=None + query_params={"page": 1}, ) paginator = OffsetPaginationStrategy(config) @@ -101,8 +96,7 @@ def test_offset_limit(response_with_body): request_params: SaaSRequestParams = SaaSRequestParams( method=HTTPMethod.GET, path="/conversations", - params={"page": 10}, - body=None + query_params={"page": 10}, ) paginator = OffsetPaginationStrategy(config) @@ -135,8 +129,7 @@ def test_offset_missing_start_value(response_with_body): request_params: SaaSRequestParams = SaaSRequestParams( method=HTTPMethod.GET, path="/conversations", - params={"row": 1}, - body=None + query_params={"row": 1}, ) paginator = OffsetPaginationStrategy(config) diff --git a/tests/service/privacy_request/request_runner_service_test.py b/tests/service/privacy_request/request_runner_service_test.py index e0a7b36ec..26e94a0cf 100644 --- a/tests/service/privacy_request/request_runner_service_test.py +++ b/tests/service/privacy_request/request_runner_service_test.py @@ -367,7 +367,7 @@ def test_create_and_process_erasure_request_saas( connector = SaaSConnector(connection_config_mailchimp) request: SaaSRequestParams = SaaSRequestParams( - method=HTTPMethod.GET, path="/3.0/search-members", params={"query": mailchimp_identity_email}, body=None + method=HTTPMethod.GET, path="/3.0/search-members", query_params={"query": mailchimp_identity_email} ) resp = connector.create_client().send(request) body = resp.json() From 9301a6f196f08650639f3a70a40eae8816ad8cd6 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Thu, 24 Mar 2022 12:14:39 -0700 Subject: [PATCH 11/24] Fixing JSON in documentation --- docs/fidesops/docs/guides/saas_pagination.md | 6 +++--- docs/fidesops/docs/guides/saas_postprocessors.md | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/fidesops/docs/guides/saas_pagination.md b/docs/fidesops/docs/guides/saas_pagination.md index ae8e756f6..35885aaf7 100644 --- a/docs/fidesops/docs/guides/saas_pagination.md +++ b/docs/fidesops/docs/guides/saas_pagination.md @@ -79,9 +79,9 @@ If an API request returns the following: ```json { "messages": [ - {"id": 1, msg: "this is"}, - {"id": 2, msg: "a"} - {"id": 3, msg: "test"} + {"id": 1, "msg": "this is"}, + {"id": 2, "msg": "a"} + {"id": 3, "msg": "test"} ] } ``` diff --git a/docs/fidesops/docs/guides/saas_postprocessors.md b/docs/fidesops/docs/guides/saas_postprocessors.md index 6ff967416..b672d916d 100644 --- a/docs/fidesops/docs/guides/saas_postprocessors.md +++ b/docs/fidesops/docs/guides/saas_postprocessors.md @@ -70,7 +70,7 @@ Identity data passed in through request: Data to be processed: ```json -data = [ +"data" = [ { "id": 1397429347 "email_contact": "somebody@email.com" @@ -86,7 +86,7 @@ data = [ Result: ```json -result = [ +"result" = [ { "id": 1397429347 "email_contact": "somebody@email.com" @@ -122,7 +122,7 @@ Post-Processor Config: Data to be processed: ```json -data = { +"data" = { "exact_matches": { "members": [ { "howdy": 123 }, @@ -133,7 +133,7 @@ data = { ``` Result: ```json -result = [ +"result" = [ { "howdy": 123 }, { "meow": 841 } ] From 459860e5ad6d6eb3f9058713176ae25746be6170 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Thu, 24 Mar 2022 13:42:55 -0700 Subject: [PATCH 12/24] Adding more info about request_params in the documentation --- docs/fidesops/docs/guides/saas_config.md | 34 ++++++++++++++++++++ docs/fidesops/docs/guides/saas_pagination.md | 2 +- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/docs/fidesops/docs/guides/saas_config.md b/docs/fidesops/docs/guides/saas_config.md index 3b3205410..eef742cc7 100644 --- a/docs/fidesops/docs/guides/saas_config.md +++ b/docs/fidesops/docs/guides/saas_config.md @@ -188,6 +188,40 @@ This is where we define how we are going to access and update each collection in - `postprocessors` An optional list of response post-processing strategies. We will ignore this for the example scenarios below but an in depth-explanation can be found under [SaaS Post-Processors](saas_postprocessors.md) - `pagination` An optional strategy used to get the next set of results from APIs with resources spanning multiple pages. Details can be found under [SaaS Pagination](saas_pagination.md). +## Request params in more detail +The `request_params` list is what provides the values to our various placeholders in the path, headers, query params and body. Values can be `identities` such as email or phone number, `references` to fields in other collections, or `connector_params` which are defined as part of configuring a SaaS connector. Whenever a placeholder is encountered, the placeholder name is looked up in the list of `request_params` and corresponding value is used instead. Here is an example of placeholders being used in various locations: + +```yaml +messages: + requests: + read: + method: GET + path: //messages + headers: + - name: Content-Type + value: application/json + - name: On-Behalf-Of + value: + - name: Token + value: Custom + query_params: + - name: count + value: 100 + - name: organization: + value: + - name: where: + value: properties["$email"]=="" + request_params: + - name: email + identity: email + - name: api_key + connector_param: api_key + - name: org_id + connector_param: org_id + - name: version + connector_param: version +``` + ## Example scenarios #### Dynamic path with dataset references ```yaml diff --git a/docs/fidesops/docs/guides/saas_pagination.md b/docs/fidesops/docs/guides/saas_pagination.md index 35885aaf7..bce62acbd 100644 --- a/docs/fidesops/docs/guides/saas_pagination.md +++ b/docs/fidesops/docs/guides/saas_pagination.md @@ -49,7 +49,7 @@ pagination: ``` We can also access links returned in the body. If we receive this value in the body: -``` +```json { ... "next_page": { From ebdfa296b9ffe9ed77a1603cf7f14e33497fe502 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Fri, 25 Mar 2022 13:54:00 -0700 Subject: [PATCH 13/24] Fixing pagination tests --- src/fidesops/schemas/saas/shared_schemas.py | 4 ++-- .../pagination/pagination_strategy_cursor.py | 2 +- .../pagination/pagination_strategy_offset.py | 4 ++-- .../test_pagination_strategy_offset.py | 16 ++++++++++------ 4 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/fidesops/schemas/saas/shared_schemas.py b/src/fidesops/schemas/saas/shared_schemas.py index 0c239db15..ec3c83d50 100644 --- a/src/fidesops/schemas/saas/shared_schemas.py +++ b/src/fidesops/schemas/saas/shared_schemas.py @@ -20,8 +20,8 @@ class SaaSRequestParams(BaseModel): method: HTTPMethod path: str - headers: Optional[Dict[str, Any]] - query_params: Optional[Dict[str, Any]] + headers: Dict[str, Any] = {} + query_params: Dict[str, Any] = {} body: Optional[str] class Config: diff --git a/src/fidesops/service/pagination/pagination_strategy_cursor.py b/src/fidesops/service/pagination/pagination_strategy_cursor.py index c15be1f71..f15fb2910 100644 --- a/src/fidesops/service/pagination/pagination_strategy_cursor.py +++ b/src/fidesops/service/pagination/pagination_strategy_cursor.py @@ -41,7 +41,7 @@ def get_next_request( return None # add or replace cursor_param with new cursor value - request_params.params[self.cursor_param] = cursor + request_params.query_params[self.cursor_param] = cursor return SaaSRequestParams( method=request_params.method, diff --git a/src/fidesops/service/pagination/pagination_strategy_offset.py b/src/fidesops/service/pagination/pagination_strategy_offset.py index b9fb8ef46..a4a87a3af 100644 --- a/src/fidesops/service/pagination/pagination_strategy_offset.py +++ b/src/fidesops/service/pagination/pagination_strategy_offset.py @@ -40,7 +40,7 @@ def get_next_request( return None # find query param value from deconstructed request_params, throw exception if query param not found - param_value = request_params.params.get(self.incremental_param) + param_value = request_params.query_params.get(self.incremental_param) if param_value is None: raise FidesopsException( f"Unable to find query param named '{self.incremental_param}' in request" @@ -59,7 +59,7 @@ def get_next_request( return None # update query param and return updated request_param tuple - request_params.params[self.incremental_param] = param_value + request_params.query_params[self.incremental_param] = param_value return SaaSRequestParams( method=request_params.method, path=request_params.path, diff --git a/tests/service/pagination/test_pagination_strategy_offset.py b/tests/service/pagination/test_pagination_strategy_offset.py index bcafc29bf..38bf970c0 100644 --- a/tests/service/pagination/test_pagination_strategy_offset.py +++ b/tests/service/pagination/test_pagination_strategy_offset.py @@ -82,7 +82,9 @@ def test_offset_with_connector_param_reference_not_found(response_with_body): paginator = OffsetPaginationStrategy(config) with pytest.raises(FidesopsException) as exc: - paginator.get_next_request(request_params, {}, response_with_body, "conversations") + paginator.get_next_request( + request_params, {}, response_with_body, "conversations" + ) assert ( f"Unable to find value for 'limit' with the connector_param reference '{config.limit.connector_param}'" == str(exc.value) @@ -134,7 +136,9 @@ def test_offset_missing_start_value(response_with_body): paginator = OffsetPaginationStrategy(config) with pytest.raises(FidesopsException) as exc: - paginator.get_next_request(request_params, {}, response_with_body, "conversations") + paginator.get_next_request( + request_params, {}, response_with_body, "conversations" + ) assert ( f"Unable to find query param named '{config.incremental_param}' in request" == str(exc.value) @@ -142,7 +146,7 @@ def test_offset_missing_start_value(response_with_body): def test_validate_request(): - request_params = [{"name": "page", "type": "query", "default_value": 1}] + query_params = [{"name": "page", "value": 1}] pagination = { "strategy": "offset", "configuration": { @@ -151,11 +155,11 @@ def test_validate_request(): "limit": 10, }, } - SaaSRequest(path="/test", request_params=request_params, pagination=pagination) + SaaSRequest(path="/test", query_params=query_params, pagination=pagination) def test_validate_request_missing_param(): - request_params = [{"name": "row", "type": "query", "default_value": 1}] + query_params = [{"name": "row", "value": 1}] pagination = { "strategy": "offset", "configuration": { @@ -165,5 +169,5 @@ def test_validate_request_missing_param(): }, } with pytest.raises(ValueError) as exc: - SaaSRequest(path="/test", request_params=request_params, pagination=pagination) + SaaSRequest(path="/test", query_params=query_params, pagination=pagination) assert "Query param 'page' not found." in str(exc.value) From 87cd12040b4d7766dc3d8dfe82a260a88549b747 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Sat, 26 Mar 2022 11:38:06 -0700 Subject: [PATCH 14/24] Fixing tests and cleaning up based on PR review feedback --- docs/fidesops/docs/guides/saas_config.md | 4 ++-- .../docs/guides/saas_postprocessors.md | 20 ++++++++-------- src/fidesops/schemas/saas/saas_config.py | 14 +++-------- .../service/connectors/saas_connector.py | 4 ++-- .../service/connectors/saas_query_config.py | 24 +++++++++---------- tests/models/test_saasconfig.py | 10 ++++++-- tests/service/connectors/test_queryconfig.py | 2 +- 7 files changed, 37 insertions(+), 41 deletions(-) diff --git a/docs/fidesops/docs/guides/saas_config.md b/docs/fidesops/docs/guides/saas_config.md index 34fa2fbd4..12e3614d7 100644 --- a/docs/fidesops/docs/guides/saas_config.md +++ b/docs/fidesops/docs/guides/saas_config.md @@ -177,8 +177,8 @@ This is where we define how we are going to access and update each collection in - `path` A static or dynamic resource path. The dynamic portions of the path are enclosed within angle brackets `` and are replaced with values from `request_params`. - `headers` and `query_params` The HTTP headers and query parameters to include in the request. - `name` the value to use for the header or query param name. - - `value` can be a static value or use a `` which will be replaced with the value sourced from the `request_param` with a matching name. - - `body` (optional) static or dynamic request body, with dynamic portions enclosed in brackets, just like `path`. These dynamic values will be replaced with values from `request_params`. For update requests, you'll need to additionally annotated `` as a placeholder for the Fidesops generated update values. + - `value` can be a static value, one or more of ``, or a mix of static and dynamic values (prefix ``) which will be replaced with the value sourced from the `request_param` with a matching name. + - `body` (optional) static or dynamic request body, with dynamic portions enclosed in brackets, just like `path`. These dynamic values will be replaced with values from `request_params`. For update requests, you'll need to additionally annotate `` as a placeholder for the Fidesops generated update values. - `request_params` - `name` Used as the key to reference this value from dynamic values in the path, headers, query, or body params. - `references` These are the same as `references` in the Dataset schema. It is used to define the source of the value for the given request_param. diff --git a/docs/fidesops/docs/guides/saas_postprocessors.md b/docs/fidesops/docs/guides/saas_postprocessors.md index b672d916d..be0c96eb7 100644 --- a/docs/fidesops/docs/guides/saas_postprocessors.md +++ b/docs/fidesops/docs/guides/saas_postprocessors.md @@ -70,15 +70,15 @@ Identity data passed in through request: Data to be processed: ```json -"data" = [ +[ { - "id": 1397429347 - "email_contact": "somebody@email.com" + "id": 1397429347, + "email_contact": "somebody@email.com", "name": "Somebody Awesome" }, { - "id": 238475234 - "email_contact": "somebody-else@email.com" + "id": 238475234, + "email_contact": "somebody-else@email.com", "name": "Somebody Cool" } ] @@ -86,10 +86,10 @@ Data to be processed: Result: ```json -"result" = [ +[ { - "id": 1397429347 - "email_contact": "somebody@email.com" + "id": 1397429347, + "email_contact": "somebody@email.com", "name": "Somebody Awesome" } ] @@ -122,7 +122,7 @@ Post-Processor Config: Data to be processed: ```json -"data" = { +{ "exact_matches": { "members": [ { "howdy": 123 }, @@ -133,7 +133,7 @@ Data to be processed: ``` Result: ```json -"result" = [ +[ { "howdy": 123 }, { "meow": 841 } ] diff --git a/src/fidesops/schemas/saas/saas_config.py b/src/fidesops/schemas/saas/saas_config.py index dc11a5f0b..cdb335c14 100644 --- a/src/fidesops/schemas/saas/saas_config.py +++ b/src/fidesops/schemas/saas/saas_config.py @@ -2,7 +2,7 @@ from fidesops.schemas.saas.shared_schemas import HTTPMethod from fidesops.service.pagination.pagination_strategy_factory import get_strategy -from pydantic import BaseModel, validator, root_validator +from pydantic import BaseModel, validator, root_validator, Extra from fidesops.schemas.base_class import BaseSchema from fidesops.schemas.dataset import FidesopsDatasetReference from fidesops.graph.config import Collection, Dataset, FieldAddress, ScalarField @@ -10,14 +10,6 @@ from fidesops.schemas.shared_schemas import FidesOpsKey -class ConnectorParams(BaseModel): - """ - Required information for the given SaaS connector. - """ - - name: str - - class RequestParam(BaseModel): """ A request parameter which includes the type (query, path, or body) along with a default value or @@ -79,7 +71,7 @@ class SaaSRequest(BaseModel): """ path: str - method: Optional[HTTPMethod] + method: HTTPMethod headers: Optional[List[Header]] query_params: Optional[List[QueryParam]] body: Optional[str] @@ -94,6 +86,7 @@ class Config: orm_mode = True use_enum_values = True + extra = Extra.forbid @root_validator(pre=True) def validate_request_for_pagination(cls, values: Dict[str, Any]) -> Dict[str, Any]: @@ -154,7 +147,6 @@ class ConnectorParam(BaseModel): """Used to define the required parameters for the connector (user-provided and constants)""" name: str - default_value: Optional[Any] class ClientConfig(BaseModel): diff --git a/src/fidesops/service/connectors/saas_connector.py b/src/fidesops/service/connectors/saas_connector.py index 3f5b71a52..d8f895827 100644 --- a/src/fidesops/service/connectors/saas_connector.py +++ b/src/fidesops/service/connectors/saas_connector.py @@ -112,9 +112,9 @@ def query_config(self, node: TraversalNode) -> SaaSQueryConfig: def test_connection(self) -> Optional[ConnectionTestStatus]: """Generates and executes a test connection based on the SaaS config""" - test_request_path = self.saas_config.test_request.path + test_request = self.saas_config.test_request prepared_request: SaaSRequestParams = SaaSRequestParams( - method=HTTPMethod.GET, path=test_request_path + method=test_request.method, path=test_request.path ) self.client().send(prepared_request) return ConnectionTestStatus.succeeded diff --git a/src/fidesops/service/connectors/saas_query_config.py b/src/fidesops/service/connectors/saas_query_config.py index 78d143f7c..1073d939f 100644 --- a/src/fidesops/service/connectors/saas_query_config.py +++ b/src/fidesops/service/connectors/saas_query_config.py @@ -86,11 +86,14 @@ def assign_placeholders(value: str, param_values: Dict[str, Any]) -> str: return value def map_param_values( - self, current_request: SaaSRequest, param_values: Dict[str, Any] + self, current_request: SaaSRequest, param_values: Dict[str, Any], update_values: Optional[Dict[str, Any]] ) -> SaaSRequestParams: """ - Visits path, headers, and query_params in the current request and replaces - the placeholders with the request param values + Visits path, headers, query, and body params in the current request and replaces + the placeholders with the request param values. + + The update_values are added to the body, if available, and the current_request + does not specify a body. """ path: str = self.assign_placeholders(current_request.path, param_values) @@ -105,11 +108,14 @@ def map_param_values( query_param.value, param_values ) + body = self.assign_placeholders(current_request.body, param_values) + return SaaSRequestParams( method=current_request.method, path=path, headers=headers, query_params=query_params, + body=json.loads(body) if body else update_values ) def generate_query( @@ -136,12 +142,9 @@ def generate_query( # map param values to placeholders in path, headers, and query params saas_request_params: SaaSRequestParams = self.map_param_values( - current_request, param_values + current_request, param_values, None ) - # map body - body = self.assign_placeholders(current_request.body, param_values) - saas_request_params.body = json.loads(body) if body else None logger.info(f"Populated request params for {current_request.path}") return saas_request_params @@ -185,14 +188,9 @@ def generate_update_stmt( # map param values to placeholders in path, headers, and query params saas_request_params: SaaSRequestParams = self.map_param_values( - current_request, param_values + current_request, param_values, update_values ) - # map body - body = self.assign_placeholders(current_request.body, param_values) - saas_request_params.body = json.dumps( - json.loads(body) if body else update_values - ) logger.info(f"Populated request params for {current_request.path}") return saas_request_params diff --git a/tests/models/test_saasconfig.py b/tests/models/test_saasconfig.py index 814cac3f1..81302482b 100644 --- a/tests/models/test_saasconfig.py +++ b/tests/models/test_saasconfig.py @@ -1,16 +1,22 @@ from typing import Dict import pytest +from pydantic import ValidationError from fidesops.graph.config import FieldAddress -from fidesops.schemas.saas.saas_config import SaaSConfig +from fidesops.schemas.saas.saas_config import SaaSConfig, SaaSRequest @pytest.mark.unit_saas -def test_saas_configs(saas_configs) -> None: +def test_saas_configs(saas_configs): """Simple test to verify that the available configs can be deserialized into SaaSConfigs""" for saas_config in saas_configs.values(): SaaSConfig(**saas_config) +@pytest.mark.unit_saas +def test_saas_request_without_method(): + with pytest.raises(ValidationError) as exc: + SaaSRequest(path="/test") + assert "field required" in str(exc.value) @pytest.mark.unit_saas def test_saas_config_to_dataset(saas_configs: Dict[str, Dict]): diff --git a/tests/service/connectors/test_queryconfig.py b/tests/service/connectors/test_queryconfig.py index 5b10a0e11..b65dc1cfe 100644 --- a/tests/service/connectors/test_queryconfig.py +++ b/tests/service/connectors/test_queryconfig.py @@ -684,7 +684,7 @@ def test_generate_query( payment_methods, endpoints, {"api_version": "2.0", "page_limit": 10} ) prepared_request = config.generate_query( - {"query": ["customer-1@example.com"]}, policy + {"email": ["customer-1@example.com"]}, policy ) assert prepared_request.method == HTTPMethod.GET.value assert prepared_request.path == "/2.0/payment_methods" From 75c8075f4ef82a66822bbac92a50b633d023935f Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Sat, 26 Mar 2022 12:26:13 -0700 Subject: [PATCH 15/24] Changing name of request_param to param_values --- data/saas/config/mailchimp_config.yml | 8 ++-- data/saas/config/saas_example_config.yml | 16 ++++---- docs/fidesops/docs/guides/saas_config.md | 32 +++++++-------- .../docs/guides/saas_postprocessors.md | 2 +- .../postman/Fidesops.postman_collection.json | 4 +- src/fidesops/schemas/saas/saas_config.py | 26 +++++++------ src/fidesops/schemas/saas/shared_schemas.py | 4 +- .../service/connectors/saas_query_config.py | 39 ++++++++++--------- .../endpoints/test_saas_config_endpoints.py | 14 +++---- tests/service/connectors/test_queryconfig.py | 8 ++-- 10 files changed, 80 insertions(+), 73 deletions(-) diff --git a/data/saas/config/mailchimp_config.yml b/data/saas/config/mailchimp_config.yml index cfbdf142a..2c9bf594d 100644 --- a/data/saas/config/mailchimp_config.yml +++ b/data/saas/config/mailchimp_config.yml @@ -31,7 +31,7 @@ saas_config: read: method: GET path: /3.0/conversations//messages - request_params: + param_values: - name: conversation_id references: - dataset: mailchimp_connector_example @@ -54,7 +54,7 @@ saas_config: value: 1000 - name: offset value: 0 - request_params: + param_values: - name: placeholder identity: email data_path: conversations @@ -72,14 +72,14 @@ saas_config: query_params: - name: query value: - request_params: + param_values: - name: email identity: email data_path: exact_matches.members update: method: PUT path: /3.0/lists//members/ - request_params: + param_values: - name: list_id references: - dataset: mailchimp_connector_example diff --git a/data/saas/config/saas_example_config.yml b/data/saas/config/saas_example_config.yml index c25318419..8b0a57af2 100644 --- a/data/saas/config/saas_example_config.yml +++ b/data/saas/config/saas_example_config.yml @@ -32,7 +32,7 @@ saas_config: read: method: GET path: /3.0/conversations//messages - request_params: + param_values: - name: conversation_id references: - dataset: saas_connector_example @@ -57,7 +57,7 @@ saas_config: value: 1000 - name: offset value: 0 - request_params: + param_values: - name: placeholder identity: email postprocessors: @@ -72,7 +72,7 @@ saas_config: query_params: - name: query value: - request_params: + param_values: - name: email identity: email postprocessors: @@ -82,7 +82,7 @@ saas_config: update: method: PUT path: /3.0/lists//members/ - request_params: + param_values: - name: list_id references: - dataset: saas_connector_example @@ -110,7 +110,7 @@ saas_config: value: - name: query value: - request_params: + param_values: - name: limit connector_param: page_limit - name: version @@ -122,7 +122,7 @@ saas_config: update: method: PUT path: //payment_methods - request_params: + param_values: - name: version connector_param: api_version - name: projects @@ -130,7 +130,7 @@ saas_config: read: method: GET path: /api/0/projects/ - request_params: + param_values: - name: placeholder identity: email - name: user_feedback @@ -139,7 +139,7 @@ saas_config: method: GET path: /api/0/projects///user-feedback/ grouped_inputs: [ organization_slug, project_slug ] - request_params: + param_values: - name: organization_slug references: - dataset: saas_connector_example diff --git a/docs/fidesops/docs/guides/saas_config.md b/docs/fidesops/docs/guides/saas_config.md index 12e3614d7..cadfa6a65 100644 --- a/docs/fidesops/docs/guides/saas_config.md +++ b/docs/fidesops/docs/guides/saas_config.md @@ -50,7 +50,7 @@ saas_config: read: method: GET path: /3.0/conversations//messages - request_params: + param_values: - name: conversation_id references: - dataset: mailchimp_connector_example @@ -73,7 +73,7 @@ saas_config: value: 1000 - name: offset value: 0 - request_params: + param_values: - name: placeholder identity: email data_path: conversations @@ -91,14 +91,14 @@ saas_config: query_params: - name: query value: - request_params: + param_values: - name: email identity: email data_path: exact_matches.members update: method: PUT path: /3.0/lists//members/ - request_params: + param_values: - name: list_id references: - dataset: mailchimp_connector_example @@ -174,12 +174,12 @@ This is where we define how we are going to access and update each collection in - `name` This name corresponds to a Collection in the corresponding Dataset. - `requests` A map of `read`, `update`, and `delete` requests for this collection. Each collection can define a way to read and a way to update the data. Each request is made up of: - `method` The HTTP method used for the endpoint. - - `path` A static or dynamic resource path. The dynamic portions of the path are enclosed within angle brackets `` and are replaced with values from `request_params`. + - `path` A static or dynamic resource path. The dynamic portions of the path are enclosed within angle brackets `` and are replaced with values from `param_values`. - `headers` and `query_params` The HTTP headers and query parameters to include in the request. - `name` the value to use for the header or query param name. - `value` can be a static value, one or more of ``, or a mix of static and dynamic values (prefix ``) which will be replaced with the value sourced from the `request_param` with a matching name. - - `body` (optional) static or dynamic request body, with dynamic portions enclosed in brackets, just like `path`. These dynamic values will be replaced with values from `request_params`. For update requests, you'll need to additionally annotate `` as a placeholder for the Fidesops generated update values. - - `request_params` + - `body` (optional) static or dynamic request body, with dynamic portions enclosed in brackets, just like `path`. These dynamic values will be replaced with values from `param_values`. For update requests, you'll need to additionally annotate `` as a placeholder for the Fidesops generated update values. + - `param_values` - `name` Used as the key to reference this value from dynamic values in the path, headers, query, or body params. - `references` These are the same as `references` in the Dataset schema. It is used to define the source of the value for the given request_param. - `identity` Used to access the identity values passed into the privacy request such as email or phone number. @@ -190,7 +190,7 @@ This is where we define how we are going to access and update each collection in - `grouped_inputs` An optional list of reference fields whose inputs are dependent upon one another. For example, an endpoint may need both an `organization_id` and a `project_id` from another endpoint. These aren't independent values, as a `project_id` belongs to an `organization_id`. You would specify this as ["organization_id", "project_id"]. ## Request params in more detail -The `request_params` list is what provides the values to our various placeholders in the path, headers, query params and body. Values can be `identities` such as email or phone number, `references` to fields in other collections, or `connector_params` which are defined as part of configuring a SaaS connector. Whenever a placeholder is encountered, the placeholder name is looked up in the list of `request_params` and corresponding value is used instead. Here is an example of placeholders being used in various locations: +The `param_values` list is what provides the values to our various placeholders in the path, headers, query params and body. Values can be `identities` such as email or phone number, `references` to fields in other collections, or `connector_params` which are defined as part of configuring a SaaS connector. Whenever a placeholder is encountered, the placeholder name is looked up in the list of `param_values` and corresponding value is used instead. Here is an example of placeholders being used in various locations: ```yaml messages: @@ -212,7 +212,7 @@ messages: value: - name: where: value: properties["$email"]=="" - request_params: + param_values: - name: email identity: email - name: api_key @@ -232,7 +232,7 @@ endpoints: read: method: GET path: /3.0/conversations//messages - request_params: + param_values: - name: conversation_id references: - dataset: mailchimp_connector_example @@ -259,7 +259,7 @@ endpoints: query_params: - name: query value: - request_params: + param_values: - name: email identity: email ``` @@ -276,7 +276,7 @@ endpoints: update: method: PUT path: /3.0/lists//members/ - request_params: + param_values: - name: list_id references: - dataset: mailchimp_connector_example @@ -328,7 +328,7 @@ update: "user_ref_id": } } - request_params: + param_values: - name: user_ref_id references: - dataset: dataset_test @@ -354,7 +354,7 @@ PUT /crm/v3/objects/contacts ## How does this relate to graph traversal? -Fidesops uses the available Datasets to [generate a graph](query_execution.md) of all reachable data and the dependencies between Datasets. For SaaS connectors, all the references and identities are stored in the `request_params`, therefore we must merge both the SaaS config and Dataset to provide a complete picture for the graph traversal. Using Mailchimp as an example the Dataset collection and SaaS config endpoints for `messages` looks like this: +Fidesops uses the available Datasets to [generate a graph](query_execution.md) of all reachable data and the dependencies between Datasets. For SaaS connectors, all the references and identities are stored in the `param_values`, therefore we must merge both the SaaS config and Dataset to provide a complete picture for the graph traversal. Using Mailchimp as an example the Dataset collection and SaaS config endpoints for `messages` looks like this: ```yaml collections: @@ -385,7 +385,7 @@ endpoints: read: method: GET path: /3.0/conversations//messages - request_params: + param_values: - name: conversation_id references: - dataset: mailchimp_connector_example @@ -445,7 +445,7 @@ endpoints: value: 1000 - name: offset value: 0 - request_params: + param_values: - name: placeholder identity: email ``` diff --git a/docs/fidesops/docs/guides/saas_postprocessors.md b/docs/fidesops/docs/guides/saas_postprocessors.md index be0c96eb7..e3d6663e1 100644 --- a/docs/fidesops/docs/guides/saas_postprocessors.md +++ b/docs/fidesops/docs/guides/saas_postprocessors.md @@ -13,7 +13,7 @@ endpoints: read: method: GET path: /conversations//messages - request_params: + param_values: ... postprocessors: - strategy: unwrap diff --git a/docs/fidesops/docs/postman/Fidesops.postman_collection.json b/docs/fidesops/docs/postman/Fidesops.postman_collection.json index 8231fcfb3..19720f1f5 100644 --- a/docs/fidesops/docs/postman/Fidesops.postman_collection.json +++ b/docs/fidesops/docs/postman/Fidesops.postman_collection.json @@ -2366,7 +2366,7 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"fides_key\": \"mailchimp_connector_example\",\n \"name\": \"Mailchimp SaaS Config\",\n \"description\": \"A sample schema representing the Mailchimp connector for Fidesops\",\n \"version\": \"0.0.1\",\n \"connector_params\": [\n {\n \"name\": \"domain\"\n },\n {\n \"name\": \"username\"\n },\n {\n \"name\": \"api_key\"\n }\n ],\n \"client_config\": {\n \"protocol\": \"https\",\n \"host\": {\n \"connector_param\": \"domain\"\n },\n \"authentication\": {\n \"strategy\": \"basic_authentication\",\n \"configuration\": {\n \"username\": {\n \"connector_param\": \"username\"\n },\n \"password\": {\n \"connector_param\": \"api_key\"\n }\n }\n }\n },\n \"test_request\": {\n \"method\": \"GET\",\n \"path\": \"/3.0/lists\"\n },\n \"endpoints\": [\n {\n \"name\": \"messages\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/3.0/conversations//messages\",\n \"request_params\": [\n {\n \"name\": \"conversation_id\",\n \"references\": [\n {\n \"dataset\": \"mailchimp_connector_example\",\n \"field\": \"conversations.id\",\n \"direction\": \"from\"\n }\n ]\n }\n ],\n \"data_path\": \"conversation_messages\",\n \"postprocessors\": [\n {\n \"strategy\": \"filter\",\n \"configuration\": {\n \"field\": \"from_email\",\n \"value\": {\n \"identity\": \"email\"\n }\n }\n }\n ]\n }\n }\n },\n {\n \"name\": \"conversations\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/3.0/conversations\",\n \"query_params\": [\n {\n \"name\": \"count\",\n \"value\": 1000\n },\n {\n \"name\": \"offset\",\n \"value\": 0\n }\n ],\n \"request_params\": [\n {\n \"name\": \"placeholder\",\n \"identity\": \"email\"\n }\n ],\n \"data_path\": \"conversations\",\n \"pagination\": {\n \"strategy\": \"offset\",\n \"configuration\": {\n \"incremental_param\": \"offset\",\n \"increment_by\": 1000,\n \"limit\": 10000\n }\n }\n }\n }\n },\n {\n \"name\": \"member\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/3.0/search-members\",\n \"query_params\": [\n {\n \"name\": \"query\",\n \"value\": \"\"\n }\n ],\n \"request_params\": [\n {\n \"name\": \"email\",\n \"identity\": \"email\"\n }\n ],\n \"data_path\": \"exact_matches.members\"\n },\n \"update\": {\n \"method\": \"PUT\",\n \"path\": \"/3.0/lists//members/\",\n \"request_params\": [\n {\n \"name\": \"list_id\",\n \"references\": [\n {\n \"dataset\": \"mailchimp_connector_example\",\n \"field\": \"member.list_id\",\n \"direction\": \"from\"\n }\n ]\n },\n {\n \"name\": \"subscriber_hash\",\n \"references\": [\n {\n \"dataset\": \"mailchimp_connector_example\",\n \"field\": \"member.id\",\n \"direction\": \"from\"\n }\n ]\n }\n ]\n }\n }\n }\n ]\n}", + "raw": "{\n \"fides_key\": \"mailchimp_connector_example\",\n \"name\": \"Mailchimp SaaS Config\",\n \"description\": \"A sample schema representing the Mailchimp connector for Fidesops\",\n \"version\": \"0.0.1\",\n \"connector_params\": [\n {\n \"name\": \"domain\"\n },\n {\n \"name\": \"username\"\n },\n {\n \"name\": \"api_key\"\n }\n ],\n \"client_config\": {\n \"protocol\": \"https\",\n \"host\": {\n \"connector_param\": \"domain\"\n },\n \"authentication\": {\n \"strategy\": \"basic_authentication\",\n \"configuration\": {\n \"username\": {\n \"connector_param\": \"username\"\n },\n \"password\": {\n \"connector_param\": \"api_key\"\n }\n }\n }\n },\n \"test_request\": {\n \"method\": \"GET\",\n \"path\": \"/3.0/lists\"\n },\n \"endpoints\": [\n {\n \"name\": \"messages\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/3.0/conversations//messages\",\n \"param_values\": [\n {\n \"name\": \"conversation_id\",\n \"references\": [\n {\n \"dataset\": \"mailchimp_connector_example\",\n \"field\": \"conversations.id\",\n \"direction\": \"from\"\n }\n ]\n }\n ],\n \"data_path\": \"conversation_messages\",\n \"postprocessors\": [\n {\n \"strategy\": \"filter\",\n \"configuration\": {\n \"field\": \"from_email\",\n \"value\": {\n \"identity\": \"email\"\n }\n }\n }\n ]\n }\n }\n },\n {\n \"name\": \"conversations\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/3.0/conversations\",\n \"query_params\": [\n {\n \"name\": \"count\",\n \"value\": 1000\n },\n {\n \"name\": \"offset\",\n \"value\": 0\n }\n ],\n \"param_values\": [\n {\n \"name\": \"placeholder\",\n \"identity\": \"email\"\n }\n ],\n \"data_path\": \"conversations\",\n \"pagination\": {\n \"strategy\": \"offset\",\n \"configuration\": {\n \"incremental_param\": \"offset\",\n \"increment_by\": 1000,\n \"limit\": 10000\n }\n }\n }\n }\n },\n {\n \"name\": \"member\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/3.0/search-members\",\n \"query_params\": [\n {\n \"name\": \"query\",\n \"value\": \"\"\n }\n ],\n \"param_values\": [\n {\n \"name\": \"email\",\n \"identity\": \"email\"\n }\n ],\n \"data_path\": \"exact_matches.members\"\n },\n \"update\": {\n \"method\": \"PUT\",\n \"path\": \"/3.0/lists//members/\",\n \"param_values\": [\n {\n \"name\": \"list_id\",\n \"references\": [\n {\n \"dataset\": \"mailchimp_connector_example\",\n \"field\": \"member.list_id\",\n \"direction\": \"from\"\n }\n ]\n },\n {\n \"name\": \"subscriber_hash\",\n \"references\": [\n {\n \"dataset\": \"mailchimp_connector_example\",\n \"field\": \"member.id\",\n \"direction\": \"from\"\n }\n ]\n }\n ]\n }\n }\n }\n ]\n}", "options": { "raw": { "language": "json" @@ -2404,7 +2404,7 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"fides_key\": \"mailchimp_connector_example\",\n \"name\": \"Mailchimp SaaS Config\",\n \"description\": \"A sample schema representing the Mailchimp connector for Fidesops\",\n \"version\": \"0.0.1\",\n \"connector_params\": [\n {\n \"name\": \"domain\"\n },\n {\n \"name\": \"username\"\n },\n {\n \"name\": \"api_key\"\n }\n ],\n \"client_config\": {\n \"protocol\": \"https\",\n \"host\": {\n \"connector_param\": \"domain\"\n },\n \"authentication\": {\n \"strategy\": \"basic_authentication\",\n \"configuration\": {\n \"username\": {\n \"connector_param\": \"username\"\n },\n \"password\": {\n \"connector_param\": \"api_key\"\n }\n }\n }\n },\n \"test_request\": {\n \"method\": \"GET\",\n \"path\": \"/3.0/lists\"\n },\n \"endpoints\": [\n {\n \"name\": \"messages\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/3.0/conversations//messages\",\n \"request_params\": [\n {\n \"name\": \"conversation_id\",\n \"references\": [\n {\n \"dataset\": \"mailchimp_connector_example\",\n \"field\": \"conversations.id\",\n \"direction\": \"from\"\n }\n ]\n }\n ],\n \"data_path\": \"conversation_messages\",\n \"postprocessors\": [\n {\n \"strategy\": \"filter\",\n \"configuration\": {\n \"field\": \"from_email\",\n \"value\": {\n \"identity\": \"email\"\n }\n }\n }\n ]\n }\n }\n },\n {\n \"name\": \"conversations\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/3.0/conversations\",\n \"query_params\": [\n {\n \"name\": \"count\",\n \"value\": 1000\n },\n {\n \"name\": \"offset\",\n \"value\": 0\n }\n ],\n \"request_params\": [\n {\n \"name\": \"placeholder\",\n \"identity\": \"email\"\n }\n ],\n \"data_path\": \"conversations\",\n \"pagination\": {\n \"strategy\": \"offset\",\n \"configuration\": {\n \"incremental_param\": \"offset\",\n \"increment_by\": 1000,\n \"limit\": 10000\n }\n }\n }\n }\n },\n {\n \"name\": \"member\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/3.0/search-members\",\n \"query_params\": [\n {\n \"name\": \"query\",\n \"value\": \"\"\n }\n ],\n \"request_params\": [\n {\n \"name\": \"email\",\n \"identity\": \"email\"\n }\n ],\n \"data_path\": \"exact_matches.members\"\n },\n \"update\": {\n \"method\": \"PUT\",\n \"path\": \"/3.0/lists//members/\",\n \"request_params\": [\n {\n \"name\": \"list_id\",\n \"references\": [\n {\n \"dataset\": \"mailchimp_connector_example\",\n \"field\": \"member.list_id\",\n \"direction\": \"from\"\n }\n ]\n },\n {\n \"name\": \"subscriber_hash\",\n \"references\": [\n {\n \"dataset\": \"mailchimp_connector_example\",\n \"field\": \"member.id\",\n \"direction\": \"from\"\n }\n ]\n }\n ]\n }\n }\n }\n ]\n}", + "raw": "{\n \"fides_key\": \"mailchimp_connector_example\",\n \"name\": \"Mailchimp SaaS Config\",\n \"description\": \"A sample schema representing the Mailchimp connector for Fidesops\",\n \"version\": \"0.0.1\",\n \"connector_params\": [\n {\n \"name\": \"domain\"\n },\n {\n \"name\": \"username\"\n },\n {\n \"name\": \"api_key\"\n }\n ],\n \"client_config\": {\n \"protocol\": \"https\",\n \"host\": {\n \"connector_param\": \"domain\"\n },\n \"authentication\": {\n \"strategy\": \"basic_authentication\",\n \"configuration\": {\n \"username\": {\n \"connector_param\": \"username\"\n },\n \"password\": {\n \"connector_param\": \"api_key\"\n }\n }\n }\n },\n \"test_request\": {\n \"method\": \"GET\",\n \"path\": \"/3.0/lists\"\n },\n \"endpoints\": [\n {\n \"name\": \"messages\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/3.0/conversations//messages\",\n \"param_values\": [\n {\n \"name\": \"conversation_id\",\n \"references\": [\n {\n \"dataset\": \"mailchimp_connector_example\",\n \"field\": \"conversations.id\",\n \"direction\": \"from\"\n }\n ]\n }\n ],\n \"data_path\": \"conversation_messages\",\n \"postprocessors\": [\n {\n \"strategy\": \"filter\",\n \"configuration\": {\n \"field\": \"from_email\",\n \"value\": {\n \"identity\": \"email\"\n }\n }\n }\n ]\n }\n }\n },\n {\n \"name\": \"conversations\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/3.0/conversations\",\n \"query_params\": [\n {\n \"name\": \"count\",\n \"value\": 1000\n },\n {\n \"name\": \"offset\",\n \"value\": 0\n }\n ],\n \"param_values\": [\n {\n \"name\": \"placeholder\",\n \"identity\": \"email\"\n }\n ],\n \"data_path\": \"conversations\",\n \"pagination\": {\n \"strategy\": \"offset\",\n \"configuration\": {\n \"incremental_param\": \"offset\",\n \"increment_by\": 1000,\n \"limit\": 10000\n }\n }\n }\n }\n },\n {\n \"name\": \"member\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/3.0/search-members\",\n \"query_params\": [\n {\n \"name\": \"query\",\n \"value\": \"\"\n }\n ],\n \"param_values\": [\n {\n \"name\": \"email\",\n \"identity\": \"email\"\n }\n ],\n \"data_path\": \"exact_matches.members\"\n },\n \"update\": {\n \"method\": \"PUT\",\n \"path\": \"/3.0/lists//members/\",\n \"param_values\": [\n {\n \"name\": \"list_id\",\n \"references\": [\n {\n \"dataset\": \"mailchimp_connector_example\",\n \"field\": \"member.list_id\",\n \"direction\": \"from\"\n }\n ]\n },\n {\n \"name\": \"subscriber_hash\",\n \"references\": [\n {\n \"dataset\": \"mailchimp_connector_example\",\n \"field\": \"member.id\",\n \"direction\": \"from\"\n }\n ]\n }\n ]\n }\n }\n }\n ]\n}", "options": { "raw": { "language": "json" diff --git a/src/fidesops/schemas/saas/saas_config.py b/src/fidesops/schemas/saas/saas_config.py index cdb335c14..6976526d3 100644 --- a/src/fidesops/schemas/saas/saas_config.py +++ b/src/fidesops/schemas/saas/saas_config.py @@ -10,10 +10,10 @@ from fidesops.schemas.shared_schemas import FidesOpsKey -class RequestParam(BaseModel): +class ParamValue(BaseModel): """ - A request parameter which includes the type (query, path, or body) along with a default value or - a reference to an identity value or a value in another dataset. + A named variable which can be sourced from identities, dataset references, or connector params. These values + are used to replace the placeholders in the path, header, query, and body param values. """ name: str @@ -66,8 +66,10 @@ class QueryParam(BaseModel): class SaaSRequest(BaseModel): """ - A single request with a static or dynamic path, and the request params needed to build the request. - Also includes optional strategies for postprocessing and pagination. + A single request with static or dynamic path, headers, query, and body params. + Also specifies the names and sources for the param values needed to build the request. + + Includes optional strategies for postprocessing and pagination. """ path: str @@ -75,7 +77,7 @@ class SaaSRequest(BaseModel): headers: Optional[List[Header]] query_params: Optional[List[QueryParam]] body: Optional[str] - request_params: Optional[List[RequestParam]] + param_values: Optional[List[ParamValue]] data_path: Optional[str] postprocessors: Optional[List[Strategy]] pagination: Optional[Strategy] @@ -109,16 +111,16 @@ def validate_grouped_inputs(cls, values: Dict[str, Any]) -> Dict[str, Any]: grouped_inputs = set(values.get("grouped_inputs", [])) if grouped_inputs: - request_params = values.get("request_params", []) - names = {param.name for param in request_params} + param_values = values.get("param_values", []) + names = {param.name for param in param_values} if not grouped_inputs.issubset(names): raise ValueError( - "Grouped input fields must also be declared as request_params." + "Grouped input fields must also be declared as param_values." ) referenced_collections: List[str] = [] - for param in request_params: + for param in param_values: if param.name in grouped_inputs: if not param.references and not param.identity: raise ValueError( @@ -167,7 +169,7 @@ class SaaSConfig(BaseModel): The required fields for the config are converted into a Dataset which is merged with the standard Fidesops Dataset to provide a complete set of dependencies - for the graph traversal + for the graph traversal. """ fides_key: FidesOpsKey @@ -189,7 +191,7 @@ def get_graph(self) -> Dataset: collections = [] for endpoint in self.endpoints: fields = [] - for param in endpoint.requests["read"].request_params or []: + for param in endpoint.requests["read"].param_values or []: if param.references: references = [] for reference in param.references: diff --git a/src/fidesops/schemas/saas/shared_schemas.py b/src/fidesops/schemas/saas/shared_schemas.py index ec3c83d50..1c89e15ed 100644 --- a/src/fidesops/schemas/saas/shared_schemas.py +++ b/src/fidesops/schemas/saas/shared_schemas.py @@ -16,7 +16,9 @@ class HTTPMethod(Enum): class SaaSRequestParams(BaseModel): - """Custom type to represent a SaaS request param""" + """ + Holds the method, path, headers, query, and body params to build a SaaS HTTP request. + """ method: HTTPMethod path: str diff --git a/src/fidesops/service/connectors/saas_query_config.py b/src/fidesops/service/connectors/saas_query_config.py index 1073d939f..f8b57e1c7 100644 --- a/src/fidesops/service/connectors/saas_query_config.py +++ b/src/fidesops/service/connectors/saas_query_config.py @@ -86,7 +86,10 @@ def assign_placeholders(value: str, param_values: Dict[str, Any]) -> str: return value def map_param_values( - self, current_request: SaaSRequest, param_values: Dict[str, Any], update_values: Optional[Dict[str, Any]] + self, + current_request: SaaSRequest, + param_values: Dict[str, Any], + update_values: Optional[Dict[str, Any]], ) -> SaaSRequestParams: """ Visits path, headers, query, and body params in the current request and replaces @@ -115,7 +118,7 @@ def map_param_values( path=path, headers=headers, query_params=query_params, - body=json.loads(body) if body else update_values + body=json.loads(body) if body else update_values, ) def generate_query( @@ -132,12 +135,12 @@ def generate_query( # create the source of param values to populate the various placeholders # in the path, headers, query_params, and body param_values: Dict[str, Any] = {} - for request_param in current_request.request_params: - if request_param.references or request_param.identity: - param_values[request_param.name] = input_data[request_param.name][0] - elif request_param.connector_param: - param_values[request_param.name] = pydash.get( - self.secrets, request_param.connector_param + for param_value in current_request.param_values: + if param_value.references or param_value.identity: + param_values[param_value.name] = input_data[param_value.name][0] + elif param_value.connector_param: + param_values[param_value.name] = pydash.get( + self.secrets, param_value.connector_param ) # map param values to placeholders in path, headers, and query params @@ -165,18 +168,18 @@ def generate_update_stmt( # create the source of param values to populate the various placeholders # in the path, headers, query_params, and body param_values: Dict[str, Any] = {} - for request_param in current_request.request_params: - if request_param.references: - param_values[request_param.name] = pydash.get( - collection_values, request_param.references[0].field + for param_value in current_request.param_values: + if param_value.references: + param_values[param_value.name] = pydash.get( + collection_values, param_value.references[0].field ) - elif request_param.identity: - param_values[request_param.name] = pydash.get( - identity_data, request_param.identity + elif param_value.identity: + param_values[param_value.name] = pydash.get( + identity_data, param_value.identity ) - elif request_param.connector_param: - param_values[request_param.name] = pydash.get( - self.secrets, request_param.connector_param + elif param_value.connector_param: + param_values[param_value.name] = pydash.get( + self.secrets, param_value.connector_param ) # mask row values diff --git a/tests/api/v1/endpoints/test_saas_config_endpoints.py b/tests/api/v1/endpoints/test_saas_config_endpoints.py index a14b12b5b..904c47bc1 100644 --- a/tests/api/v1/endpoints/test_saas_config_endpoints.py +++ b/tests/api/v1/endpoints/test_saas_config_endpoints.py @@ -95,11 +95,11 @@ def test_put_validate_saas_config_reference_and_identity( ) -> None: auth_header = generate_auth_header(scopes=[SAAS_CONFIG_READ]) saas_config = saas_configs["saas_example"] - request_params = saas_config["endpoints"][0]["requests"]["read"][ - "request_params" + param_values = saas_config["endpoints"][0]["requests"]["read"][ + "param_values" ][0] - request_params["identity"] = "email" - request_params["references"] = [ + param_values["identity"] = "email" + param_values["references"] = [ { "dataset": "postgres_example_test_dataset", "field": "another.field", @@ -122,10 +122,10 @@ def test_put_validate_saas_config_wrong_reference_direction( ) -> None: auth_header = generate_auth_header(scopes=[SAAS_CONFIG_READ]) saas_config = saas_configs["saas_example"] - request_params = saas_config["endpoints"][0]["requests"]["read"][ - "request_params" + param_values = saas_config["endpoints"][0]["requests"]["read"][ + "param_values" ][0] - request_params["references"] = [ + param_values["references"] = [ { "dataset": "postgres_example_test_dataset", "field": "another.field", diff --git a/tests/service/connectors/test_queryconfig.py b/tests/service/connectors/test_queryconfig.py index b65dc1cfe..9f7943896 100644 --- a/tests/service/connectors/test_queryconfig.py +++ b/tests/service/connectors/test_queryconfig.py @@ -17,7 +17,7 @@ from fidesops.schemas.dataset import FidesopsDataset from fidesops.schemas.masking.masking_configuration import HashMaskingConfiguration from fidesops.schemas.masking.masking_secrets import MaskingSecretCache, SecretType -from fidesops.schemas.saas.saas_config import SaaSConfig, RequestParam +from fidesops.schemas.saas.saas_config import SaaSConfig, ParamValue from fidesops.schemas.saas.shared_schemas import SaaSRequestParams, HTTPMethod from fidesops.service.connectors.saas_query_config import SaaSQueryConfig from fidesops.service.connectors.query_config import SQLQueryConfig, MongoQueryConfig @@ -774,7 +774,7 @@ def test_generate_update_stmt_with_request_body( saas_config.endpoints[2].requests.get( "update" ).body = '{"properties": {, "list_id": ""}}' - body_request_params = RequestParam( + body_param_value = ParamValue( name="list_id", type="body", references=[ @@ -785,8 +785,8 @@ def test_generate_update_stmt_with_request_body( } ], ) - saas_config.endpoints[2].requests.get("update").request_params.append( - body_request_params + saas_config.endpoints[2].requests.get("update").param_values.append( + body_param_value ) endpoints = saas_config.top_level_endpoint_dict member = combined_traversal.traversal_node_dict[ From 84fad3166c4b1a429e9844b0cb083e6ddec36b49 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Sat, 26 Mar 2022 13:05:11 -0700 Subject: [PATCH 16/24] Removing unused import --- src/fidesops/service/connectors/saas_connector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fidesops/service/connectors/saas_connector.py b/src/fidesops/service/connectors/saas_connector.py index d8f895827..0b68c8e17 100644 --- a/src/fidesops/service/connectors/saas_connector.py +++ b/src/fidesops/service/connectors/saas_connector.py @@ -5,7 +5,7 @@ from requests import Session, Request, PreparedRequest, Response from fidesops.common_exceptions import FidesopsException from fidesops.service.pagination.pagination_strategy import PaginationStrategy -from fidesops.schemas.saas.shared_schemas import SaaSRequestParams, HTTPMethod +from fidesops.schemas.saas.shared_schemas import SaaSRequestParams from fidesops.service.connectors.saas_query_config import SaaSQueryConfig from fidesops.service.connectors.base_connector import BaseConnector from fidesops.graph.traversal import Row, TraversalNode From a65d5f83395712c40e36fc1dc67a5899a1d1b864 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Sat, 26 Mar 2022 15:59:34 -0700 Subject: [PATCH 17/24] Fixing tests --- data/saas/config/saas_example_config.yml | 1 + src/fidesops/schemas/saas/saas_config.py | 2 +- src/fidesops/service/connectors/saas_query_config.py | 10 +++++----- tests/fixtures/saas_fixtures.py | 2 +- tests/service/connectors/test_queryconfig.py | 9 ++++++--- .../pagination/test_pagination_strategy_offset.py | 4 ++-- 6 files changed, 16 insertions(+), 12 deletions(-) diff --git a/data/saas/config/saas_example_config.yml b/data/saas/config/saas_example_config.yml index 8b0a57af2..11fe5e83b 100644 --- a/data/saas/config/saas_example_config.yml +++ b/data/saas/config/saas_example_config.yml @@ -24,6 +24,7 @@ saas_config: connector_param: api_key test_request: + method: GET path: /3.0/lists endpoints: diff --git a/src/fidesops/schemas/saas/saas_config.py b/src/fidesops/schemas/saas/saas_config.py index 6976526d3..c16b9c1ae 100644 --- a/src/fidesops/schemas/saas/saas_config.py +++ b/src/fidesops/schemas/saas/saas_config.py @@ -61,7 +61,7 @@ class Header(BaseModel): class QueryParam(BaseModel): name: str - value: str + value: Union[int, str] class SaaSRequest(BaseModel): diff --git a/src/fidesops/service/connectors/saas_query_config.py b/src/fidesops/service/connectors/saas_query_config.py index f8b57e1c7..d1ef8e5d7 100644 --- a/src/fidesops/service/connectors/saas_query_config.py +++ b/src/fidesops/service/connectors/saas_query_config.py @@ -77,7 +77,7 @@ def assign_placeholders(value: str, param_values: Dict[str, Any]) -> str: Finds all the placeholders (indicated by <>) in the passed in value and replaces them with the actual param values """ - if value: + if value and isinstance(value, str): placeholders = re.findall("<(.+?)>", value) for placeholder in placeholders: value = value.replace( @@ -89,7 +89,7 @@ def map_param_values( self, current_request: SaaSRequest, param_values: Dict[str, Any], - update_values: Optional[Dict[str, Any]], + update_values: Optional[str], ) -> SaaSRequestParams: """ Visits path, headers, query, and body params in the current request and replaces @@ -118,7 +118,7 @@ def map_param_values( path=path, headers=headers, query_params=query_params, - body=json.loads(body) if body else update_values, + body=body if body else update_values, ) def generate_query( @@ -184,10 +184,10 @@ def generate_update_stmt( # mask row values update_value_map: Dict[str, Any] = self.update_value_map(row, policy, request) - update_values: Dict[str, Any] = unflatten_dict(update_value_map) + update_values: str = json.dumps(unflatten_dict(update_value_map)) # removes outer {} wrapper from body for greater flexibility in custom body config - param_values["masked_object_fields"] = json.dumps(update_values)[1:-1] + param_values["masked_object_fields"] = update_values[1:-1] # map param values to placeholders in path, headers, and query params saas_request_params: SaaSRequestParams = self.map_param_values( diff --git a/tests/fixtures/saas_fixtures.py b/tests/fixtures/saas_fixtures.py index 09cc61029..026f7521b 100644 --- a/tests/fixtures/saas_fixtures.py +++ b/tests/fixtures/saas_fixtures.py @@ -179,7 +179,7 @@ def connection_config_saas_example_with_invalid_saas_config( db: Session, saas_configs: Dict[str, Dict] ) -> Generator: invalid_saas_config = saas_configs["saas_example"].copy() - invalid_saas_config["endpoints"][0]["requests"]["read"]["request_params"].pop() + invalid_saas_config["endpoints"][0]["requests"]["read"]["param_values"].pop() connection_config = ConnectionConfig.create( db=db, data={ diff --git a/tests/service/connectors/test_queryconfig.py b/tests/service/connectors/test_queryconfig.py index 9f7943896..3d1b93e55 100644 --- a/tests/service/connectors/test_queryconfig.py +++ b/tests/service/connectors/test_queryconfig.py @@ -646,7 +646,7 @@ def test_generate_query( ) assert prepared_request.method == HTTPMethod.GET.value assert prepared_request.path == "/3.0/conversations" - assert prepared_request.query_params == {"count": "1000", "offset": "0"} + assert prepared_request.query_params == {"count": 1000, "offset": 0} assert prepared_request.body is None # dynamic path with no query params @@ -661,7 +661,7 @@ def test_generate_query( config = SaaSQueryConfig( payment_methods, endpoints, - {"api_version": "2.0", "page_limit": "10", "api_key": "letmein"}, + {"api_version": "2.0", "page_limit": 10, "api_key": "letmein"}, ) prepared_request = config.generate_query( {"email": ["customer-1@example.com"]}, policy @@ -688,7 +688,10 @@ def test_generate_query( ) assert prepared_request.method == HTTPMethod.GET.value assert prepared_request.path == "/2.0/payment_methods" - assert prepared_request.params == {"limit": 10, "query": "customer-1@example.com"} + assert prepared_request.query_params == { + "limit": "10", + "query": "customer-1@example.com", + } assert prepared_request.body is None def test_generate_update_stmt( diff --git a/tests/service/pagination/test_pagination_strategy_offset.py b/tests/service/pagination/test_pagination_strategy_offset.py index 38bf970c0..a0581028e 100644 --- a/tests/service/pagination/test_pagination_strategy_offset.py +++ b/tests/service/pagination/test_pagination_strategy_offset.py @@ -155,7 +155,7 @@ def test_validate_request(): "limit": 10, }, } - SaaSRequest(path="/test", query_params=query_params, pagination=pagination) + SaaSRequest(method="GET", path="/test", query_params=query_params, pagination=pagination) def test_validate_request_missing_param(): @@ -169,5 +169,5 @@ def test_validate_request_missing_param(): }, } with pytest.raises(ValueError) as exc: - SaaSRequest(path="/test", query_params=query_params, pagination=pagination) + SaaSRequest(method="GET", path="/test", query_params=query_params, pagination=pagination) assert "Query param 'page' not found." in str(exc.value) From 4acc7f8618ba042278d926766e121bb07ebe0e7c Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Sat, 26 Mar 2022 19:38:33 -0700 Subject: [PATCH 18/24] Updating branch --- data/saas/config/stripe_config.yml | 99 ++++++++ data/saas/dataset/stripe_dataset.yml | 149 +++++++++++ saas_config.toml | 9 +- .../test_connection_config_endpoints.py | 30 +-- .../v1/endpoints/test_dataset_endpoints.py | 42 ++-- .../endpoints/test_saas_config_endpoints.py | 106 ++++---- tests/conftest.py | 4 +- tests/fixtures/saas/mailchimp_fixtures.py | 117 +++++++++ tests/fixtures/saas/stripe_fixtures.py | 89 +++++++ tests/fixtures/saas_example_fixtures.py | 126 ++++++++++ tests/fixtures/saas_fixtures.py | 232 ------------------ .../saas/test_mailchimp_task.py | 200 +++++++++++++++ .../saas/test_stripe_task.py | 85 +++++++ ...st_connection_configuration_integration.py | 110 ++++----- tests/integration_tests/test_saas_task.py | 22 +- tests/models/test_saasconfig.py | 11 +- .../test_connection_secrets_saas.py | 12 +- tests/service/connectors/test_queryconfig.py | 20 +- .../request_runner_service_test.py | 10 +- 19 files changed, 1057 insertions(+), 416 deletions(-) create mode 100644 data/saas/config/stripe_config.yml create mode 100644 data/saas/dataset/stripe_dataset.yml create mode 100644 tests/fixtures/saas/mailchimp_fixtures.py create mode 100644 tests/fixtures/saas/stripe_fixtures.py create mode 100644 tests/fixtures/saas_example_fixtures.py delete mode 100644 tests/fixtures/saas_fixtures.py create mode 100644 tests/integration_tests/saas/test_mailchimp_task.py create mode 100644 tests/integration_tests/saas/test_stripe_task.py diff --git a/data/saas/config/stripe_config.yml b/data/saas/config/stripe_config.yml new file mode 100644 index 000000000..b56423b68 --- /dev/null +++ b/data/saas/config/stripe_config.yml @@ -0,0 +1,99 @@ +saas_config: + fides_key: stripe_connector_example + name: Stripe SaaS Config + description: A sample schema representing the Stripe connector for Fidesops + version: 0.0.1 + + connector_params: + - name: host + - name: api_key + - name: payment_types + - name: page_limit + + client_config: + protocol: https + host: + connector_param: host + authentication: + strategy: bearer_authentication + configuration: + token: + connector_param: api_key + + test_request: + path: /v1/customers + + endpoints: + - name: customer + requests: + read: + path: /v1/customers + request_params: + - name: email + type: query + identity: email + postprocessors: + - strategy: unwrap + configuration: + data_path: data + update: + path: /v1/customers/ + request_params: + - name: id + type: path + references: + - dataset: stripe_connector_example + field: customer.id + direction: from + - name: payment_methods + requests: + read: + path: /v1/payment_methods + request_params: + - name: customer + type: query + references: + - dataset: stripe_connector_example + field: customer.id + direction: from + - name: type + type: query + connector_param: payment_types + - name: limit + type: query + connector_param: page_limit + postprocessors: + - strategy: unwrap + configuration: + data_path: data + pagination: + strategy: cursor + configuration: + cursor_param: starting_after + field: id + - name: bank_accounts + requests: + read: + path: /v1/customers//sources + request_params: + - name: customer_id + type: path + references: + - dataset: stripe_connector_example + field: customer.id + direction: from + - name: object + type: query + default_value: bank_account + - name: limit + type: query + connector_param: page_limit + postprocessors: + - strategy: unwrap + configuration: + data_path: data + pagination: + strategy: cursor + configuration: + cursor_param: starting_after + field: id \ No newline at end of file diff --git a/data/saas/dataset/stripe_dataset.yml b/data/saas/dataset/stripe_dataset.yml new file mode 100644 index 000000000..682ad1303 --- /dev/null +++ b/data/saas/dataset/stripe_dataset.yml @@ -0,0 +1,149 @@ +dataset: + - fides_key: stripe_connector_example + name: Stripe Dataset + description: A sample dataset representing the Stripe connector for Fidesops + collections: + - name: customer + fields: + - name: id + data_categories: [system.operations] + - name: object + data_categories: [system.operations] + - name: address + fields: + - name: city + data_categories: [system.operations] + - name: country + data_categories: [system.operations] + - name: line1 + data_categories: [system.operations] + - name: line2 + data_categories: [system.operations] + - name: postal_code + data_categories: [system.operations] + - name: state + data_categories: [system.operations] + - name: currency + data_categories: [system.operations] + - name: default_source + data_categories: [system.operations] + - name: description + data_categories: [system.operations] + - name: email + data_categories: [system.operations] + - name: invoice_settings + fields: + - name: custom_fields + data_categories: [system.operations] + - name: default_payment_method + data_categories: [system.operations] + - name: livemode + data_categories: [system.operations] + - name: name + data_categories: [system.operations] + - name: phone + data_categories: [system.operations] + - name: preferred_locales + data_categories: [system.operations] + - name: shipping + fields: + - name: address + fields: + - name: city + data_categories: [system.operations] + - name: country + data_categories: [system.operations] + - name: line1 + data_categories: [system.operations] + - name: line2 + data_categories: [system.operations] + - name: postal_code + data_categories: [system.operations] + - name: state + data_categories: [system.operations] + - name: sources + fields: + - name: object + data_categories: [system.operations] + - name: data + fields: + - name: id + data_categories: [system.operations] + - name: object + data_categories: [system.operations] + - name: address_city + data_categories: [system.operations] + - name: address_country + data_categories: [system.operations] + - name: address_line1 + data_categories: [system.operations] + - name: address_line1_check + data_categories: [system.operations] + - name: address_line2 + data_categories: [system.operations] + - name: address_state + data_categories: [system.operations] + - name: address_zip + data_categories: [system.operations] + - name: address_zip_check + data_categories: [system.operations] + - name: brand + data_categories: [system.operations] + - name: country + data_categories: [system.operations] + - name: customer + data_categories: [system.operations] + - name: cvc_check + data_categories: [system.operations] + - name: dynamic_last4 + data_categories: [system.operations] + - name: exp_month + data_categories: [system.operations] + - name: exp_year + data_categories: [system.operations] + - name: fingerprint + data_categories: [system.operations] + - name: funding + data_categories: [system.operations] + - name: last4 + data_categories: [system.operations] + - name: name + data_categories: [system.operations] + - name: tokenization_method + data_categories: [system.operations] + - name: url + data_categories: [system.operations] + - name: subscriptions + fields: + - name: object + data_categories: [system.operations] + - name: data + fields: + - name: id + data_categories: [system.operations] + - name: object + data_categories: [system.operations] + - name: application_fee_percent + data_categories: [system.operations] + - name: customer + data_categories: [system.operations] + - name: default_payment_method + data_categories: [system.operations] + - name: default_source + data_categories: [system.operations] + - name: default_tax_rates + data_categories: [system.operations] + - name: latest_invoice + data_categories: [system.operations] + - name: livemode + data_categories: [system.operations] + - name: url + data_categories: [system.operations] + - name: tax_exempt + data_categories: [system.operations] + - name: tax_ids + fields: + - name: object + data_categories: [system.operations] + - name: data + data_categories: [system.operations] \ No newline at end of file diff --git a/saas_config.toml b/saas_config.toml index 1385cfb34..f0e774f61 100644 --- a/saas_config.toml +++ b/saas_config.toml @@ -9,4 +9,11 @@ page_limit = "10" domain = "" username = "" api_key = "" -identity_email = "" \ No newline at end of file +identity_email = "" + +[stripe] +host = "" +api_key = "" +payment_types = "" +page_limit = "" +identity_email = "" diff --git a/tests/api/v1/endpoints/test_connection_config_endpoints.py b/tests/api/v1/endpoints/test_connection_config_endpoints.py index 7a691a7cb..d94e56f84 100644 --- a/tests/api/v1/endpoints/test_connection_config_endpoints.py +++ b/tests/api/v1/endpoints/test_connection_config_endpoints.py @@ -815,17 +815,17 @@ def test_put_http_connection_config_secrets( assert https_connection_config.last_test_succeeded is None @pytest.mark.unit_saas - def test_put_connection_config_saas_example_secrets( + def test_put_saas_example_connection_config_secrets( self, api_client: TestClient, db: Session, generate_auth_header, - connection_config_saas_example, - saas_secrets, + saas_example_connection_config, + saas_example_secrets, ): auth_header = generate_auth_header(scopes=[CONNECTION_CREATE_OR_UPDATE]) - url = f"{V1_URL_PREFIX}{CONNECTIONS}/{connection_config_saas_example.key}/secret" - payload = saas_secrets["saas_example"] + url = f"{V1_URL_PREFIX}{CONNECTIONS}/{saas_example_connection_config.key}/secret" + payload = saas_example_secrets resp = api_client.put( url + "?verify=False", @@ -837,25 +837,25 @@ def test_put_connection_config_saas_example_secrets( body = json.loads(resp.text) assert ( body["msg"] - == f"Secrets updated for ConnectionConfig with key: {connection_config_saas_example.key}." + == f"Secrets updated for ConnectionConfig with key: {saas_example_connection_config.key}." ) - db.refresh(connection_config_saas_example) - assert connection_config_saas_example.secrets == saas_secrets["saas_example"] - assert connection_config_saas_example.last_test_timestamp is None - assert connection_config_saas_example.last_test_succeeded is None + db.refresh(saas_example_connection_config) + assert saas_example_connection_config.secrets == saas_example_secrets + assert saas_example_connection_config.last_test_timestamp is None + assert saas_example_connection_config.last_test_succeeded is None @pytest.mark.unit_saas - def test_put_connection_config_saas_example_secrets_missing_saas_config( + def test_put_saas_example_connection_config_secrets_missing_saas_config( self, api_client: TestClient, generate_auth_header, - connection_config_saas_example_without_saas_config, - saas_secrets, + saas_example_connection_config_without_saas_config, + saas_example_secrets, ): auth_header = generate_auth_header(scopes=[CONNECTION_CREATE_OR_UPDATE]) - url = f"{V1_URL_PREFIX}{CONNECTIONS}/{connection_config_saas_example_without_saas_config.key}/secret" - payload = saas_secrets["saas_example"] + url = f"{V1_URL_PREFIX}{CONNECTIONS}/{saas_example_connection_config_without_saas_config.key}/secret" + payload = saas_example_secrets resp = api_client.put( url + "?verify=False", diff --git a/tests/api/v1/endpoints/test_dataset_endpoints.py b/tests/api/v1/endpoints/test_dataset_endpoints.py index 23672ae35..a28489a7d 100644 --- a/tests/api/v1/endpoints/test_dataset_endpoints.py +++ b/tests/api/v1/endpoints/test_dataset_endpoints.py @@ -359,14 +359,14 @@ def test_validate_dataset_that_references_another_dataset( def test_validate_saas_dataset_invalid_traversal( self, db, - connection_config_saas_example_with_invalid_saas_config, - saas_datasets, + saas_example_connection_config_with_invalid_saas_config, + saas_example_dataset, api_client: TestClient, generate_auth_header, ): path = V1_URL_PREFIX + DATASET_VALIDATE path_params = { - "connection_key": connection_config_saas_example_with_invalid_saas_config.key + "connection_key": saas_example_connection_config_with_invalid_saas_config.key } validate_dataset_url = path.format(**path_params) @@ -374,7 +374,7 @@ def test_validate_saas_dataset_invalid_traversal( response = api_client.put( validate_dataset_url, headers=auth_header, - json=saas_datasets["saas_example"], + json=saas_example_dataset, ) assert response.status_code == 200 @@ -669,19 +669,19 @@ def test_patch_datasets_bulk_update( @pytest.mark.unit_saas def test_patch_datasets_missing_saas_config( self, - connection_config_saas_example_without_saas_config, - saas_datasets, + saas_example_connection_config_without_saas_config, + saas_example_dataset, api_client: TestClient, db: Session, generate_auth_header, ): path = V1_URL_PREFIX + DATASETS - path_params = {"connection_key": connection_config_saas_example_without_saas_config.key} + path_params = {"connection_key": saas_example_connection_config_without_saas_config.key} datasets_url = path.format(**path_params) auth_header = generate_auth_header(scopes=[DATASET_CREATE_OR_UPDATE]) response = api_client.patch( - datasets_url, headers=auth_header, json=[saas_datasets["saas_example"]] + datasets_url, headers=auth_header, json=[saas_example_dataset] ) assert response.status_code == 200 @@ -690,24 +690,24 @@ def test_patch_datasets_missing_saas_config( assert len(response_body["failed"]) == 1 assert ( response_body["failed"][0]["message"] - == f"Connection config '{connection_config_saas_example_without_saas_config.key}' " + == f"Connection config '{saas_example_connection_config_without_saas_config.key}' " "must have a SaaS config before validating or adding a dataset" ) @pytest.mark.unit_saas def test_patch_datasets_extra_reference( self, - connection_config_saas_example, - saas_datasets, + saas_example_connection_config, + saas_example_dataset, api_client: TestClient, db: Session, generate_auth_header, ): path = V1_URL_PREFIX + DATASETS - path_params = {"connection_key": connection_config_saas_example.key} + path_params = {"connection_key": saas_example_connection_config.key} datasets_url = path.format(**path_params) - invalid_dataset = saas_datasets["saas_example"] + invalid_dataset = saas_example_dataset invalid_dataset["collections"][0]["fields"][0]["fidesops_meta"] = { "references": [ { @@ -736,17 +736,17 @@ def test_patch_datasets_extra_reference( @pytest.mark.unit_saas def test_patch_datasets_extra_identity( self, - connection_config_saas_example, - saas_datasets, + saas_example_connection_config, + saas_example_dataset, api_client: TestClient, db: Session, generate_auth_header, ): path = V1_URL_PREFIX + DATASETS - path_params = {"connection_key": connection_config_saas_example.key} + path_params = {"connection_key": saas_example_connection_config.key} datasets_url = path.format(**path_params) - invalid_dataset = saas_datasets["saas_example"] + invalid_dataset = saas_example_dataset invalid_dataset["collections"][0]["fields"][0]["fidesops_meta"] = { "identity": "email" } @@ -769,17 +769,17 @@ def test_patch_datasets_extra_identity( @pytest.mark.unit_saas def test_patch_datasets_fides_key_mismatch( self, - connection_config_saas_example, - saas_datasets, + saas_example_connection_config, + saas_example_dataset, api_client: TestClient, db: Session, generate_auth_header, ): path = V1_URL_PREFIX + DATASETS - path_params = {"connection_key": connection_config_saas_example.key} + path_params = {"connection_key": saas_example_connection_config.key} datasets_url = path.format(**path_params) - invalid_dataset = saas_datasets["saas_example"] + invalid_dataset = saas_example_dataset invalid_dataset["fides_key"] = "different_key" auth_header = generate_auth_header(scopes=[DATASET_CREATE_OR_UPDATE]) diff --git a/tests/api/v1/endpoints/test_saas_config_endpoints.py b/tests/api/v1/endpoints/test_saas_config_endpoints.py index 0d3e6551e..c743db268 100644 --- a/tests/api/v1/endpoints/test_saas_config_endpoints.py +++ b/tests/api/v1/endpoints/test_saas_config_endpoints.py @@ -24,22 +24,22 @@ @pytest.mark.unit_saas class TestValidateSaaSConfig: @pytest.fixture - def validate_saas_config_url(self, connection_config_saas_example) -> str: + def validate_saas_config_url(self, saas_example_connection_config) -> str: path = V1_URL_PREFIX + SAAS_CONFIG_VALIDATE - path_params = {"connection_key": connection_config_saas_example.key} + path_params = {"connection_key": saas_example_connection_config.key} return path.format(**path_params) def test_put_validate_saas_config_not_authenticated( - self, saas_configs, validate_saas_config_url: str, api_client + self, saas_example_config, validate_saas_config_url: str, api_client ) -> None: response = api_client.put( - validate_saas_config_url, headers={}, json=saas_configs["saas_example"] + validate_saas_config_url, headers={}, json=saas_example_config ) assert response.status_code == 401 def test_put_validate_dataset_wrong_scope( self, - saas_configs, + saas_example_config, validate_saas_config_url, api_client: TestClient, generate_auth_header, @@ -48,19 +48,19 @@ def test_put_validate_dataset_wrong_scope( response = api_client.put( validate_saas_config_url, headers=auth_header, - json=saas_configs["saas_example"], + json=saas_example_config, ) assert response.status_code == 403 def test_put_validate_saas_config_missing_key( self, - saas_configs, + saas_example_config, validate_saas_config_url, api_client: TestClient, generate_auth_header, ) -> None: auth_header = generate_auth_header(scopes=[SAAS_CONFIG_READ]) - invalid_config = _reject_key(saas_configs["saas_example"], "fides_key") + invalid_config = _reject_key(saas_example_config, "fides_key") response = api_client.put( validate_saas_config_url, headers=auth_header, json=invalid_config ) @@ -71,13 +71,13 @@ def test_put_validate_saas_config_missing_key( def test_put_validate_saas_config_missing_endpoints( self, - saas_configs, + saas_example_config, validate_saas_config_url, api_client: TestClient, generate_auth_header, ) -> None: auth_header = generate_auth_header(scopes=[SAAS_CONFIG_READ]) - invalid_config = _reject_key(saas_configs["saas_example"], "endpoints") + invalid_config = _reject_key(saas_example_config, "endpoints") response = api_client.put( validate_saas_config_url, headers=auth_header, json=invalid_config ) @@ -88,13 +88,13 @@ def test_put_validate_saas_config_missing_endpoints( def test_put_validate_saas_config_reference_and_identity( self, - saas_configs, + saas_example_config, validate_saas_config_url, api_client: TestClient, generate_auth_header, ) -> None: auth_header = generate_auth_header(scopes=[SAAS_CONFIG_READ]) - saas_config = saas_configs["saas_example"] + saas_config = saas_example_config request_params = saas_config["endpoints"][0]["requests"]["read"][ "request_params" ][0] @@ -115,13 +115,13 @@ def test_put_validate_saas_config_reference_and_identity( def test_put_validate_saas_config_wrong_reference_direction( self, - saas_configs, + saas_example_config, validate_saas_config_url, api_client: TestClient, generate_auth_header, ) -> None: auth_header = generate_auth_header(scopes=[SAAS_CONFIG_READ]) - saas_config = saas_configs["saas_example"] + saas_config = saas_example_config request_params = saas_config["endpoints"][0]["requests"]["read"][ "request_params" ][0] @@ -142,34 +142,34 @@ def test_put_validate_saas_config_wrong_reference_direction( @pytest.mark.unit_saas class TestPutSaaSConfig: @pytest.fixture - def saas_config_url(self, connection_config_saas_example) -> str: + def saas_config_url(self, saas_example_connection_config) -> str: path = V1_URL_PREFIX + SAAS_CONFIG - path_params = {"connection_key": connection_config_saas_example.key} + path_params = {"connection_key": saas_example_connection_config.key} return path.format(**path_params) def test_patch_saas_config_not_authenticated( - self, saas_configs, saas_config_url, api_client + self, saas_example_config, saas_config_url, api_client ) -> None: response = api_client.patch( - saas_config_url, headers={}, json=saas_configs["saas_example"] + saas_config_url, headers={}, json=saas_example_config ) assert response.status_code == 401 def test_patch_saas_config_wrong_scope( self, - saas_configs, + saas_example_config, saas_config_url, api_client: TestClient, generate_auth_header, ) -> None: auth_header = generate_auth_header(scopes=[SAAS_CONFIG_READ]) response = api_client.patch( - saas_config_url, headers=auth_header, json=saas_configs["saas_example"] + saas_config_url, headers=auth_header, json=saas_example_config ) assert response.status_code == 403 def test_patch_saas_config_invalid_connection_key( - self, saas_configs, api_client: TestClient, generate_auth_header + self, saas_example_config, api_client: TestClient, generate_auth_header ) -> None: path = V1_URL_PREFIX + SAAS_CONFIG path_params = {"connection_key": "nonexistent_key"} @@ -177,30 +177,30 @@ def test_patch_saas_config_invalid_connection_key( auth_header = generate_auth_header(scopes=[SAAS_CONFIG_CREATE_OR_UPDATE]) response = api_client.patch( - saas_config_url, headers=auth_header, json=saas_configs["saas_example"] + saas_config_url, headers=auth_header, json=saas_example_config ) assert response.status_code == 404 def test_patch_saas_config_create( self, - connection_config_saas_example_without_saas_config, - saas_configs, + saas_example_connection_config_without_saas_config, + saas_example_config, api_client: TestClient, db: Session, generate_auth_header, ) -> None: path = V1_URL_PREFIX + SAAS_CONFIG - path_params = {"connection_key": connection_config_saas_example_without_saas_config.key} + path_params = {"connection_key": saas_example_connection_config_without_saas_config.key} saas_config_url = path.format(**path_params) auth_header = generate_auth_header(scopes=[SAAS_CONFIG_CREATE_OR_UPDATE]) response = api_client.patch( - saas_config_url, headers=auth_header, json=saas_configs["saas_example"] + saas_config_url, headers=auth_header, json=saas_example_config ) assert response.status_code == 200 updated_config = ConnectionConfig.get_by( - db=db, field="key", value=connection_config_saas_example_without_saas_config.key + db=db, field="key", value=saas_example_connection_config_without_saas_config.key ) db.expire(updated_config) saas_config = updated_config.saas_config @@ -208,21 +208,21 @@ def test_patch_saas_config_create( def test_patch_saas_config_update( self, - saas_configs, + saas_example_config, saas_config_url, api_client: TestClient, db: Session, generate_auth_header, ) -> None: auth_header = generate_auth_header(scopes=[SAAS_CONFIG_CREATE_OR_UPDATE]) - saas_configs["saas_example"]["endpoints"].pop() + saas_example_config["endpoints"].pop() response = api_client.patch( - saas_config_url, headers=auth_header, json=saas_configs["saas_example"] + saas_config_url, headers=auth_header, json=saas_example_config ) assert response.status_code == 200 connection_config = ConnectionConfig.get_by( - db=db, field="key", value=saas_configs["saas_example"]["fides_key"] + db=db, field="key", value=saas_example_config["fides_key"] ) saas_config = connection_config.saas_config assert saas_config is not None @@ -243,32 +243,32 @@ def get_saas_config_url(connection_config: Optional[ConnectionConfig] = None) -> class TestGetSaaSConfig: def test_get_saas_config_not_authenticated( self, - connection_config_saas_example, + saas_example_connection_config, api_client: TestClient, ) -> None: - saas_config_url = get_saas_config_url(connection_config_saas_example) + saas_config_url = get_saas_config_url(saas_example_connection_config) response = api_client.get(saas_config_url, headers={}) assert response.status_code == 401 def test_get_saas_config_wrong_scope( self, - connection_config_saas_example, + saas_example_connection_config, api_client: TestClient, generate_auth_header, ) -> None: - saas_config_url = get_saas_config_url(connection_config_saas_example) + saas_config_url = get_saas_config_url(saas_example_connection_config) auth_header = generate_auth_header(scopes=[SAAS_CONFIG_CREATE_OR_UPDATE]) response = api_client.get(saas_config_url, headers=auth_header) assert response.status_code == 403 def test_get_saas_config_does_not_exist( self, - connection_config_saas_example_without_saas_config, + saas_example_connection_config_without_saas_config, api_client: TestClient, generate_auth_header, ) -> None: saas_config_url = get_saas_config_url( - connection_config_saas_example_without_saas_config + saas_example_connection_config_without_saas_config ) auth_header = generate_auth_header(scopes=[SAAS_CONFIG_READ]) response = api_client.get(saas_config_url, headers=auth_header) @@ -286,11 +286,11 @@ def test_get_saas_config_invalid_connection_key( def test_get_saas_config( self, - connection_config_saas_example, + saas_example_connection_config, api_client: TestClient, generate_auth_header, ) -> None: - saas_config_url = get_saas_config_url(connection_config_saas_example) + saas_config_url = get_saas_config_url(saas_example_connection_config) auth_header = generate_auth_header(scopes=[SAAS_CONFIG_READ]) response = api_client.get(saas_config_url, headers=auth_header) assert response.status_code == 200 @@ -298,7 +298,7 @@ def test_get_saas_config( response_body = json.loads(response.text) assert ( response_body["fides_key"] - == connection_config_saas_example.get_saas_config().fides_key + == saas_example_connection_config.get_saas_config().fides_key ) assert len(response_body["endpoints"]) == 6 @@ -306,31 +306,31 @@ def test_get_saas_config( @pytest.mark.unit_saas class TestDeleteSaaSConfig: def test_delete_saas_config_not_authenticated( - self, connection_config_saas_example, api_client + self, saas_example_connection_config, api_client ) -> None: - saas_config_url = get_saas_config_url(connection_config_saas_example) + saas_config_url = get_saas_config_url(saas_example_connection_config) response = api_client.delete(saas_config_url, headers={}) assert response.status_code == 401 def test_delete_saas_config_wrong_scope( self, - connection_config_saas_example, + saas_example_connection_config, api_client: TestClient, generate_auth_header, ) -> None: - saas_config_url = get_saas_config_url(connection_config_saas_example) + saas_config_url = get_saas_config_url(saas_example_connection_config) auth_header = generate_auth_header(scopes=[SAAS_CONFIG_READ]) response = api_client.delete(saas_config_url, headers=auth_header) assert response.status_code == 403 def test_delete_saas_config_does_not_exist( self, - connection_config_saas_example_without_saas_config, + saas_example_connection_config_without_saas_config, api_client: TestClient, generate_auth_header, ) -> None: saas_config_url = get_saas_config_url( - connection_config_saas_example_without_saas_config + saas_example_connection_config_without_saas_config ) auth_header = generate_auth_header(scopes=[SAAS_CONFIG_DELETE]) response = api_client.delete(saas_config_url, headers=auth_header) @@ -349,14 +349,14 @@ def test_delete_saas_config_invalid_connection_key( def test_delete_saas_config( self, db: Session, - saas_configs, + saas_example_config, api_client: TestClient, generate_auth_header, ) -> None: # Create a new connection config so we don't run into issues trying to clean up an # already deleted fixture fides_key = "saas_config_for_deletion_test" - saas_configs["saas_example"]["fides_key"] = fides_key + saas_example_config["fides_key"] = fides_key config_to_delete = ConnectionConfig.create( db=db, data={ @@ -364,7 +364,7 @@ def test_delete_saas_config( "name": fides_key, "connection_type": ConnectionType.saas, "access": AccessLevel.read, - "saas_config": saas_configs["saas_example"], + "saas_config": saas_example_config, }, ) saas_config_url = get_saas_config_url(config_to_delete) @@ -378,12 +378,12 @@ def test_delete_saas_config( def test_delete_saas_config_with_dataset_and_secrets( self, - connection_config_saas_example, - dataset_config_saas_example, + saas_example_connection_config, + saas_example_dataset_config, api_client: TestClient, generate_auth_header, ) -> None: - saas_config_url = get_saas_config_url(connection_config_saas_example) + saas_config_url = get_saas_config_url(saas_example_connection_config) auth_header = generate_auth_header(scopes=[SAAS_CONFIG_DELETE]) response = api_client.delete(saas_config_url, headers=auth_header) assert response.status_code == 400 @@ -391,7 +391,7 @@ def test_delete_saas_config_with_dataset_and_secrets( response_body = json.loads(response.text) assert ( response_body["detail"] - == f"Must delete the dataset with fides_key '{dataset_config_saas_example.fides_key}' " + == f"Must delete the dataset with fides_key '{saas_example_dataset_config.fides_key}' " "before deleting this SaaS config. Must clear the secrets from this connection " "config before deleting the SaaS config." ) diff --git a/tests/conftest.py b/tests/conftest.py index 4dfa8f844..cbfb8d75e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -38,7 +38,9 @@ from .fixtures.redshift_fixtures import * from .fixtures.snowflake_fixtures import * from .fixtures.bigquery_fixtures import * -from .fixtures.saas_fixtures import * +from .fixtures.saas_example_fixtures import * +from .fixtures.saas.mailchimp_fixtures import * +from .fixtures.saas.stripe_fixtures import * logger = logging.getLogger(__name__) diff --git a/tests/fixtures/saas/mailchimp_fixtures.py b/tests/fixtures/saas/mailchimp_fixtures.py new file mode 100644 index 000000000..752bec6dd --- /dev/null +++ b/tests/fixtures/saas/mailchimp_fixtures.py @@ -0,0 +1,117 @@ +import json +from fidesops.core.config import load_toml +from fidesops.db import session +from fidesops.models.connectionconfig import ( + AccessLevel, + ConnectionConfig, + ConnectionType, +) +from fidesops.models.datasetconfig import DatasetConfig +import pytest +import pydash +import os +from typing import Any, Dict, Generator +from fidesops.schemas.saas.shared_schemas import HTTPMethod, SaaSRequestParams +from fidesops.service.connectors.saas_connector import SaaSConnector +from tests.fixtures.application_fixtures import load_dataset +from tests.fixtures.saas_example_fixtures import load_config +from sqlalchemy.orm import Session + +saas_config = load_toml("saas_config.toml") + + +@pytest.fixture(scope="function") +def mailchimp_secrets(): + return { + "domain": pydash.get(saas_config, "mailchimp.domain") + or os.environ.get("MAILCHIMP_DOMAIN"), + "username": pydash.get(saas_config, "mailchimp.username") + or os.environ.get("MAILCHIMP_USERNAME"), + "api_key": pydash.get(saas_config, "mailchimp.api_key") + or os.environ.get("MAILCHIMP_API_KEY"), + } + + +@pytest.fixture(scope="function") +def mailchimp_identity_email(): + return pydash.get(saas_config, "mailchimp.identity_email") or os.environ.get( + "MAILCHIMP_IDENTITY_EMAIL" + ) + + +@pytest.fixture +def mailchimp_config() -> Dict[str, Any]: + return load_config("data/saas/config/mailchimp_config.yml") + + +@pytest.fixture +def mailchimp_dataset() -> Dict[str, Any]: + return load_dataset("data/saas/dataset/mailchimp_dataset.yml")[0] + + +@pytest.fixture(scope="function") +def mailchimp_connection_config( + db: session, mailchimp_config, mailchimp_secrets +) -> Generator: + fides_key = mailchimp_config["fides_key"] + connection_config = ConnectionConfig.create( + db=db, + data={ + "key": fides_key, + "name": fides_key, + "connection_type": ConnectionType.saas, + "access": AccessLevel.write, + "secrets": mailchimp_secrets, + "saas_config": mailchimp_config, + }, + ) + yield connection_config + connection_config.delete(db) + + +@pytest.fixture +def mailchimp_dataset_config( + db: Session, + mailchimp_connection_config: ConnectionConfig, + mailchimp_dataset: Dict[str, Any], +) -> Generator: + fides_key = mailchimp_dataset["fides_key"] + mailchimp_connection_config.name = fides_key + mailchimp_connection_config.key = fides_key + mailchimp_connection_config.save(db=db) + dataset = DatasetConfig.create( + db=db, + data={ + "connection_config_id": mailchimp_connection_config.id, + "fides_key": fides_key, + "dataset": mailchimp_dataset, + }, + ) + yield dataset + dataset.delete(db=db) + + +@pytest.fixture(scope="function") +def reset_mailchimp_data(mailchimp_connection_config, mailchimp_identity_email) -> Generator: + """ + Gets the current value of the resource and restores it after the test is complete. + Used for erasure tests. + """ + connector = SaaSConnector(mailchimp_connection_config) + request: SaaSRequestParams = SaaSRequestParams( + method=HTTPMethod.GET, + path="/3.0/search-members", + params={"query": mailchimp_identity_email}, + body=None, + ) + response = connector.create_client().send(request) + body = response.json() + member = body["exact_matches"]["members"][0] + yield member + request: SaaSRequestParams = SaaSRequestParams( + method=HTTPMethod.PUT, + path=f'/3.0/lists/{member["list_id"]}/members/{member["id"]}', + params={}, + body=json.dumps(member), + ) + connector.create_client().send(request) diff --git a/tests/fixtures/saas/stripe_fixtures.py b/tests/fixtures/saas/stripe_fixtures.py new file mode 100644 index 000000000..3300a1550 --- /dev/null +++ b/tests/fixtures/saas/stripe_fixtures.py @@ -0,0 +1,89 @@ +import os +from typing import Any, Dict, Generator +from fidesops.core.config import load_toml +from fidesops.db import session +from fidesops.models.connectionconfig import ( + AccessLevel, + ConnectionConfig, + ConnectionType, +) +from fidesops.models.datasetconfig import DatasetConfig +import pytest +import pydash +from tests.fixtures.application_fixtures import load_dataset +from tests.fixtures.saas_example_fixtures import load_config +from sqlalchemy.orm import Session + +saas_config = load_toml("saas_config.toml") + + +@pytest.fixture(scope="function") +def stripe_secrets(): + return { + "host": pydash.get(saas_config, "stripe.host") or os.environ.get("STRIPE_HOST"), + "api_key": pydash.get(saas_config, "stripe.api_key") + or os.environ.get("STRIPE_API_KEY"), + "payment_types": pydash.get(saas_config, "stripe.payment_types") + or os.environ.get("STRIPE_PAYMENT_TYPES"), + "page_limit": pydash.get(saas_config, "stripe.page_limit") + or os.environ.get("STRIPE_PAGE_LIMIT"), + } + + +@pytest.fixture(scope="function") +def stripe_identity_email(): + return pydash.get(saas_config, "stripe.identity_email") or os.environ.get( + "STRIPE_IDENTITY_EMAIL" + ) + + +@pytest.fixture +def stripe_config() -> Dict[str, Any]: + return load_config("data/saas/config/stripe_config.yml") + + +@pytest.fixture +def stripe_dataset() -> Dict[str, Any]: + return load_dataset("data/saas/dataset/stripe_dataset.yml")[0] + + +@pytest.fixture(scope="function") +def stripe_connection_config( + db: Session, stripe_config: Dict[str, Dict], stripe_secrets: Dict[str, Any] +) -> Generator: + fides_key = stripe_config["fides_key"] + connection_config = ConnectionConfig.create( + db=db, + data={ + "key": fides_key, + "name": fides_key, + "connection_type": ConnectionType.saas, + "access": AccessLevel.write, + "secrets": stripe_secrets, + "saas_config": stripe_config, + }, + ) + yield connection_config + connection_config.delete(db) + + +@pytest.fixture +def stripe_dataset_config( + db: session, + stripe_connection_config: ConnectionConfig, + stripe_dataset: Dict[str, Dict], +) -> Generator: + fides_key = stripe_dataset["fides_key"] + stripe_connection_config.name = fides_key + stripe_connection_config.key = fides_key + stripe_connection_config.save(db=db) + dataset = DatasetConfig.create( + db=db, + data={ + "connection_config_id": stripe_connection_config.id, + "fides_key": fides_key, + "dataset": stripe_dataset, + }, + ) + yield dataset + dataset.delete(db=db) diff --git a/tests/fixtures/saas_example_fixtures.py b/tests/fixtures/saas_example_fixtures.py new file mode 100644 index 000000000..96af7ea90 --- /dev/null +++ b/tests/fixtures/saas_example_fixtures.py @@ -0,0 +1,126 @@ +import pytest +import pydash +import yaml + +from sqlalchemy.orm import Session +from typing import Any, Dict, Generator + +from fidesops.core.config import load_file, load_toml +from fidesops.models.connectionconfig import ( + AccessLevel, + ConnectionConfig, + ConnectionType, +) +from fidesops.models.datasetconfig import DatasetConfig +from tests.fixtures.application_fixtures import load_dataset + +def load_config(filename: str) -> Dict: + yaml_file = load_file(filename) + with open(yaml_file, "r") as file: + return yaml.safe_load(file).get("saas_config", []) + +saas_config = load_toml("saas_config.toml") + +@pytest.fixture(scope="function") +def saas_example_secrets(): + return { + "domain": pydash.get(saas_config, "saas_example.domain"), + "username": pydash.get(saas_config, "saas_example.username"), + "api_key": pydash.get(saas_config, "saas_example.api_key"), + "api_version": pydash.get(saas_config, "saas_example.api_version"), + "page_limit": pydash.get(saas_config, "saas_example.page_limit"), + } + +@pytest.fixture +def saas_example_config() -> Dict: + return load_config("data/saas/config/saas_example_config.yml") + + +@pytest.fixture +def saas_example_dataset() -> Dict: + return load_dataset("data/saas/dataset/saas_example_dataset.yml")[0] + + +@pytest.fixture(scope="function") +def saas_example_connection_config( + db: Session, + saas_example_config: Dict[str, Any], + saas_example_secrets: Dict[str, Any], +) -> Generator: + fides_key = saas_example_config["fides_key"] + connection_config = ConnectionConfig.create( + db=db, + data={ + "key": fides_key, + "name": fides_key, + "connection_type": ConnectionType.saas, + "access": AccessLevel.write, + "secrets": saas_example_secrets, + "saas_config": saas_example_config, + }, + ) + yield connection_config + connection_config.delete(db) + + +@pytest.fixture +def saas_example_dataset_config( + db: Session, + saas_example_connection_config: ConnectionConfig, + saas_example_dataset: Dict, +) -> Generator: + fides_key = saas_example_dataset["fides_key"] + saas_example_connection_config.name = fides_key + saas_example_connection_config.key = fides_key + saas_example_connection_config.save(db=db) + dataset = DatasetConfig.create( + db=db, + data={ + "connection_config_id": saas_example_connection_config.id, + "fides_key": fides_key, + "dataset": saas_example_dataset, + }, + ) + yield dataset + dataset.delete(db=db) + + +@pytest.fixture(scope="function") +def saas_example_connection_config_without_saas_config( + db: Session, saas_example_secrets +) -> Generator: + connection_config = ConnectionConfig.create( + db=db, + data={ + "key": "connection_config_without_saas_config", + "name": "connection_config_without_saas_config", + "connection_type": ConnectionType.saas, + "access": AccessLevel.read, + "secrets": saas_example_secrets, + }, + ) + yield connection_config + connection_config.delete(db) + + +@pytest.fixture(scope="function") +def saas_example_connection_config_with_invalid_saas_config( + db: Session, + saas_example_config: Dict[str, Any], + saas_example_secrets: Dict[str, Any], +) -> Generator: + invalid_saas_config = saas_example_config.copy() + invalid_saas_config["endpoints"][0]["requests"]["read"]["request_params"].pop() + connection_config = ConnectionConfig.create( + db=db, + data={ + "key": "connection_config_with_invalid_saas_config", + "name": "connection_config_with_invalid_saas_config", + "connection_type": ConnectionType.saas, + "access": AccessLevel.read, + "secrets": saas_example_secrets, + "saas_config": invalid_saas_config, + }, + ) + yield connection_config + connection_config.delete(db) \ No newline at end of file diff --git a/tests/fixtures/saas_fixtures.py b/tests/fixtures/saas_fixtures.py deleted file mode 100644 index 95e48a8a6..000000000 --- a/tests/fixtures/saas_fixtures.py +++ /dev/null @@ -1,232 +0,0 @@ -import json -import pytest -import os -import pydash -import yaml - -from sqlalchemy.orm import Session -from typing import Dict, Generator - -from fidesops.core.config import load_file, load_toml -from fidesops.models.connectionconfig import ( - AccessLevel, - ConnectionConfig, - ConnectionType, -) -from fidesops.models.datasetconfig import DatasetConfig -from fidesops.schemas.saas.saas_config import SaaSConfig -from fidesops.schemas.saas.shared_schemas import HTTPMethod, SaaSRequestParams -from fidesops.service.connectors.saas_connector import SaaSConnector -from tests.fixtures.application_fixtures import load_dataset - - -saas_config = load_toml("saas_config.toml") - -saas_secrets_dict = { - "saas_example": { - "domain": pydash.get(saas_config, "saas_example.domain"), - "username": pydash.get(saas_config, "saas_example.username"), - "api_key": pydash.get(saas_config, "saas_example.api_key"), - "api_version": pydash.get(saas_config, "saas_example.api_version"), - "page_limit": pydash.get(saas_config, "saas_example.page_limit") - }, - "mailchimp": { - "domain": pydash.get(saas_config, "mailchimp.domain") - or os.environ.get("MAILCHIMP_DOMAIN"), - "username": pydash.get(saas_config, "mailchimp.username") - or os.environ.get("MAILCHIMP_USERNAME"), - "api_key": pydash.get(saas_config, "mailchimp.api_key") - or os.environ.get("MAILCHIMP_API_KEY"), - }, -} - - -def load_config(filename: str) -> Dict: - yaml_file = load_file(filename) - with open(yaml_file, "r") as file: - return yaml.safe_load(file).get("saas_config", []) - - -@pytest.fixture -def saas_configs() -> Dict[str, Dict]: - saas_configs = {} - saas_configs["saas_example"] = load_config( - "data/saas/config/saas_example_config.yml" - ) - saas_configs["mailchimp"] = load_config("data/saas/config/mailchimp_config.yml") - return saas_configs - - -@pytest.fixture -def saas_datasets() -> Dict[str, Dict]: - saas_datasets = {} - saas_datasets["saas_example"] = load_dataset( - "data/saas/dataset/saas_example_dataset.yml" - )[0] - saas_datasets["mailchimp"] = load_dataset( - "data/saas/dataset/mailchimp_dataset.yml" - )[0] - return saas_datasets - - -@pytest.fixture(scope="function") -def connection_config_saas_example( - db: Session, - saas_configs: Dict[str, Dict], -) -> Generator: - saas_config = SaaSConfig(**saas_configs["saas_example"]) - connection_config = ConnectionConfig.create( - db=db, - data={ - "key": saas_config.fides_key, - "name": saas_config.fides_key, - "connection_type": ConnectionType.saas, - "access": AccessLevel.write, - "secrets": saas_secrets_dict["saas_example"], - "saas_config": saas_configs["saas_example"], - }, - ) - yield connection_config - connection_config.delete(db) - - -@pytest.fixture -def dataset_config_saas_example( - db: Session, - connection_config_saas_example: ConnectionConfig, - saas_datasets: Dict[str, Dict], -) -> Generator: - saas_dataset = saas_datasets["saas_example"] - fides_key = saas_dataset["fides_key"] - connection_config_saas_example.name = fides_key - connection_config_saas_example.key = fides_key - connection_config_saas_example.save(db=db) - dataset = DatasetConfig.create( - db=db, - data={ - "connection_config_id": connection_config_saas_example.id, - "fides_key": fides_key, - "dataset": saas_dataset, - }, - ) - yield dataset - dataset.delete(db=db) - - -@pytest.fixture(scope="function") -def connection_config_mailchimp( - db: Session, saas_configs: Dict[str, Dict] -) -> Generator: - saas_config = SaaSConfig(**saas_configs["mailchimp"]) - connection_config = ConnectionConfig.create( - db=db, - data={ - "key": saas_config.fides_key, - "name": saas_config.fides_key, - "connection_type": ConnectionType.saas, - "access": AccessLevel.write, - "secrets": saas_secrets_dict["mailchimp"], - "saas_config": saas_configs["mailchimp"], - }, - ) - yield connection_config - connection_config.delete(db) - - -@pytest.fixture -def dataset_config_mailchimp( - db: Session, - connection_config_mailchimp: ConnectionConfig, - saas_datasets: Dict[str, Dict] -) -> Generator: - saas_dataset = saas_datasets["mailchimp"] - fides_key = saas_dataset["fides_key"] - connection_config_mailchimp.name = fides_key - connection_config_mailchimp.key = fides_key - connection_config_mailchimp.save(db=db) - dataset = DatasetConfig.create( - db=db, - data={ - "connection_config_id": connection_config_mailchimp.id, - "fides_key": fides_key, - "dataset": saas_dataset, - }, - ) - yield dataset - dataset.delete(db=db) - - -@pytest.fixture(scope="function") -def connection_config_saas_example_without_saas_config( - db: Session, -) -> Generator: - connection_config = ConnectionConfig.create( - db=db, - data={ - "key": "connection_config_without_saas_config", - "name": "connection_config_without_saas_config", - "connection_type": ConnectionType.saas, - "access": AccessLevel.read, - "secrets": saas_secrets_dict["saas_example"], - }, - ) - yield connection_config - connection_config.delete(db) - - -@pytest.fixture(scope="function") -def connection_config_saas_example_with_invalid_saas_config( - db: Session, saas_configs: Dict[str, Dict] -) -> Generator: - invalid_saas_config = saas_configs["saas_example"].copy() - invalid_saas_config["endpoints"][0]["requests"]["read"]["request_params"].pop() - connection_config = ConnectionConfig.create( - db=db, - data={ - "key": "connection_config_without_saas_config", - "name": "connection_config_without_saas_config", - "connection_type": ConnectionType.saas, - "access": AccessLevel.read, - "secrets": saas_secrets_dict["saas_example"], - "saas_config": invalid_saas_config, - }, - ) - yield connection_config - connection_config.delete(db) - - -@pytest.fixture(scope="function") -def saas_secrets(): - return saas_secrets_dict - - -@pytest.fixture(scope="function") -def mailchimp_identity_email(): - return pydash.get(saas_config, "mailchimp.identity_email") or os.environ.get( - "MAILCHIMP_IDENTITY_EMAIL" - ) - - -@pytest.fixture(scope="function") -def reset_mailchimp_data( - connection_config_mailchimp, mailchimp_identity_email -) -> Generator: - """ - Gets the current value of the resource and restores it after the test is complete. - Used for erasure tests. - """ - connector = SaaSConnector(connection_config_mailchimp) - request: SaaSRequestParams = SaaSRequestParams( - method=HTTPMethod.GET, path="/3.0/search-members", params={"query": mailchimp_identity_email}, body=None - ) - response = connector.create_client().send(request) - body = response.json() - member = body["exact_matches"]["members"][0] - yield member - request: SaaSRequestParams = SaaSRequestParams( - method=HTTPMethod.PUT, - path=f'/3.0/lists/{member["list_id"]}/members/{member["id"]}', - params={}, - body=json.dumps(member) - ) - connector.create_client().send(request) diff --git a/tests/integration_tests/saas/test_mailchimp_task.py b/tests/integration_tests/saas/test_mailchimp_task.py new file mode 100644 index 000000000..c48a54183 --- /dev/null +++ b/tests/integration_tests/saas/test_mailchimp_task.py @@ -0,0 +1,200 @@ +import pytest +import random + +from fidesops.graph.graph import DatasetGraph +from fidesops.models.privacy_request import ExecutionLog, PrivacyRequest +from fidesops.schemas.redis_cache import PrivacyRequestIdentity + +from fidesops.task import graph_task +from fidesops.task.graph_task import get_cached_data_for_erasures +from tests.graph.graph_test_util import assert_rows_match, records_matching_fields + + +@pytest.mark.integration_saas +@pytest.mark.integration_mailchimp +def test_saas_access_request_task( + db, + policy, + mailchimp_connection_config, + mailchimp_dataset_config, + mailchimp_identity_email, +) -> None: + """Full access request based on the Mailchimp SaaS config""" + + privacy_request = PrivacyRequest( + id=f"test_saas_access_request_task_{random.randint(0, 1000)}" + ) + identity_attribute = "email" + identity_value = mailchimp_identity_email + identity_kwargs = {identity_attribute: identity_value} + identity = PrivacyRequestIdentity(**identity_kwargs) + privacy_request.cache_identity(identity) + + dataset_name = mailchimp_connection_config.get_saas_config().fides_key + merged_graph = mailchimp_dataset_config.get_graph() + graph = DatasetGraph(merged_graph) + + v = graph_task.run_access_request( + privacy_request, + policy, + graph, + [mailchimp_connection_config], + {"email": mailchimp_identity_email}, + ) + + assert_rows_match( + v[f"{dataset_name}:member"], + min_size=1, + keys=[ + "id", + "list_id", + "email_address", + "unique_email_id", + "web_id", + "email_type", + "status", + "merge_fields", + "ip_signup", + "timestamp_signup", + "ip_opt", + "timestamp_opt", + "language", + "email_client", + "location", + "source", + "tags", + ], + ) + assert_rows_match( + v[f"{dataset_name}:conversations"], + min_size=2, + keys=["id", "campaign_id", "list_id", "from_email", "from_label", "subject"], + ) + assert_rows_match( + v[f"{dataset_name}:messages"], + min_size=3, + keys=[ + "id", + "conversation_id", + "from_label", + "from_email", + "subject", + "message", + "read", + "timestamp", + ], + ) + + # links + assert v[f"{dataset_name}:member"][0]["email_address"] == mailchimp_identity_email + + logs = ( + ExecutionLog.query(db=db) + .filter(ExecutionLog.privacy_request_id == privacy_request.id) + .all() + ) + + logs = [log.__dict__ for log in logs] + assert ( + len( + records_matching_fields( + logs, dataset_name=dataset_name, collection_name="member" + ) + ) + > 0 + ) + assert ( + len( + records_matching_fields( + logs, + dataset_name=dataset_name, + collection_name="conversations", + ) + ) + > 0 + ) + assert ( + len( + records_matching_fields( + logs, dataset_name=dataset_name, collection_name="messages" + ) + ) + > 0 + ) + + +@pytest.mark.integration_saas +@pytest.mark.integration_mailchimp +def test_saas_erasure_request_task( + db, + policy, + mailchimp_connection_config, + mailchimp_dataset_config, + mailchimp_identity_email, + reset_mailchimp_data, +) -> None: + """Full erasure request based on the Mailchimp SaaS config""" + + privacy_request = PrivacyRequest( + id=f"test_saas_erasure_request_task_{random.randint(0, 1000)}" + ) + identity_attribute = "email" + identity_value = mailchimp_identity_email + identity_kwargs = {identity_attribute: identity_value} + identity = PrivacyRequestIdentity(**identity_kwargs) + privacy_request.cache_identity(identity) + + dataset_name = mailchimp_connection_config.get_saas_config().fides_key + merged_graph = mailchimp_dataset_config.get_graph() + graph = DatasetGraph(merged_graph) + + graph_task.run_access_request( + privacy_request, + policy, + graph, + [mailchimp_connection_config], + {"email": mailchimp_identity_email}, + ) + + v = graph_task.run_erasure( + privacy_request, + policy, + graph, + [mailchimp_connection_config], + {"email": mailchimp_identity_email}, + get_cached_data_for_erasures(privacy_request.id), + ) + + logs = ( + ExecutionLog.query(db=db) + .filter(ExecutionLog.privacy_request_id == privacy_request.id) + .all() + ) + logs = [log.__dict__ for log in logs] + assert ( + len( + records_matching_fields( + logs, + dataset_name=dataset_name, + collection_name="conversations", + message="No values were erased since no primary key was defined for this collection", + ) + ) + == 1 + ) + assert ( + len( + records_matching_fields( + logs, + dataset_name=dataset_name, + collection_name="messages", + message="No values were erased since no primary key was defined for this collection", + ) + ) + == 1 + ) + assert v == { + f"{dataset_name}:member": 1, + f"{dataset_name}:conversations": 0, + f"{dataset_name}:messages": 0, + } diff --git a/tests/integration_tests/saas/test_stripe_task.py b/tests/integration_tests/saas/test_stripe_task.py new file mode 100644 index 000000000..b75b49d94 --- /dev/null +++ b/tests/integration_tests/saas/test_stripe_task.py @@ -0,0 +1,85 @@ +import pytest +import random + +from fidesops.graph.graph import DatasetGraph +from fidesops.models.privacy_request import ExecutionLog, PrivacyRequest +from fidesops.schemas.redis_cache import PrivacyRequestIdentity + +from fidesops.task import graph_task +from tests.graph.graph_test_util import assert_rows_match, records_matching_fields + + +@pytest.mark.integration +@pytest.mark.integration_saas +def test_stripe_access_request_task( + db, + policy, + stripe_connection_config, + stripe_dataset_config, + stripe_identity_email, +) -> None: + """Full access request based on the Stripe SaaS config""" + + privacy_request = PrivacyRequest( + id=f"test_stripe_access_request_task_{random.randint(0, 1000)}" + ) + identity_attribute = "email" + identity_value = stripe_identity_email + identity_kwargs = {identity_attribute: identity_value} + identity = PrivacyRequestIdentity(**identity_kwargs) + privacy_request.cache_identity(identity) + + dataset_name = stripe_connection_config.get_saas_config().fides_key + merged_graph = stripe_dataset_config.get_graph() + graph = DatasetGraph(merged_graph) + + v = graph_task.run_access_request( + privacy_request, + policy, + graph, + [stripe_connection_config], + {"email": stripe_identity_email}, + ) + + assert_rows_match( + v[f"{dataset_name}:customer"], + min_size=1, + keys=[ + "id", + "object", + "address", + "currency", + "default_source", + "description", + "email", + "invoice_settings", + "livemode", + "name", + "phone", + "preferred_locales", + "shipping", + "sources", + "subscriptions", + "tax_exempt", + "tax_ids", + ], + ) + + # links + assert v[f"{dataset_name}:customer"][0]["email"] == stripe_identity_email + + logs = ( + ExecutionLog.query(db=db) + .filter(ExecutionLog.privacy_request_id == privacy_request.id) + .all() + ) + + logs = [log.__dict__ for log in logs] + assert ( + len( + records_matching_fields( + logs, dataset_name=dataset_name, collection_name="customer" + ) + ) + > 0 + ) \ No newline at end of file diff --git a/tests/integration_tests/test_connection_configuration_integration.py b/tests/integration_tests/test_connection_configuration_integration.py index 94bff288a..568d7e160 100644 --- a/tests/integration_tests/test_connection_configuration_integration.py +++ b/tests/integration_tests/test_connection_configuration_integration.py @@ -1192,16 +1192,16 @@ def test_mongo_db_connection_connect_with_url( class TestSaaSConnectionPutSecretsAPI: @pytest.fixture(scope="function") def url( - self, oauth_client: ClientDetail, policy, connection_config_mailchimp + self, oauth_client: ClientDetail, policy, mailchimp_connection_config ) -> str: - return f"{V1_URL_PREFIX}{CONNECTIONS}/{connection_config_mailchimp.key}/secret" + return f"{V1_URL_PREFIX}{CONNECTIONS}/{mailchimp_connection_config.key}/secret" def test_saas_connection_incorrect_secrets( self, api_client: TestClient, db: Session, generate_auth_header, - connection_config_mailchimp, + mailchimp_connection_config, url, ): auth_header = generate_auth_header(scopes=[CONNECTION_CREATE_OR_UPDATE]) @@ -1216,22 +1216,22 @@ def test_saas_connection_incorrect_secrets( body = json.loads(resp.text) assert ( body["msg"] - == f"Secrets updated for ConnectionConfig with key: {connection_config_mailchimp.key}." + == f"Secrets updated for ConnectionConfig with key: {mailchimp_connection_config.key}." ) assert body["test_status"] == "failed" assert ( - f"Operational Error connecting to '{connection_config_mailchimp.key}'." + f"Operational Error connecting to '{mailchimp_connection_config.key}'." == body["failure_reason"] ) - db.refresh(connection_config_mailchimp) - assert connection_config_mailchimp.secrets == { + db.refresh(mailchimp_connection_config) + assert mailchimp_connection_config.secrets == { "domain": "can", "username": "someone", "api_key": "letmein", } - assert connection_config_mailchimp.last_test_timestamp is not None - assert connection_config_mailchimp.last_test_succeeded is False + assert mailchimp_connection_config.last_test_timestamp is not None + assert mailchimp_connection_config.last_test_succeeded is False def test_saas_connection_connect_with_components( self, @@ -1239,11 +1239,11 @@ def test_saas_connection_connect_with_components( api_client: TestClient, db: Session, generate_auth_header, - connection_config_mailchimp, - saas_secrets, + mailchimp_connection_config, + mailchimp_secrets, ): auth_header = generate_auth_header(scopes=[CONNECTION_CREATE_OR_UPDATE]) - payload = saas_secrets["mailchimp"] + payload = mailchimp_secrets resp = api_client.put( url, headers=auth_header, @@ -1254,27 +1254,27 @@ def test_saas_connection_connect_with_components( body = json.loads(resp.text) assert ( body["msg"] - == f"Secrets updated for ConnectionConfig with key: {connection_config_mailchimp.key}." + == f"Secrets updated for ConnectionConfig with key: {mailchimp_connection_config.key}." ) assert body["test_status"] == "succeeded" assert body["failure_reason"] is None - db.refresh(connection_config_mailchimp) - assert connection_config_mailchimp.secrets == saas_secrets["mailchimp"] - assert connection_config_mailchimp.last_test_timestamp is not None - assert connection_config_mailchimp.last_test_succeeded is True + db.refresh(mailchimp_connection_config) + assert mailchimp_connection_config.secrets == mailchimp_secrets + assert mailchimp_connection_config.last_test_timestamp is not None + assert mailchimp_connection_config.last_test_succeeded is True def test_saas_connection_connect_missing_secrets( self, url, api_client: TestClient, generate_auth_header, - saas_secrets, + saas_example_secrets, ): auth_header = generate_auth_header(scopes=[CONNECTION_CREATE_OR_UPDATE]) payload = { - "domain": saas_secrets["saas_example"]["domain"], - "username": saas_secrets["saas_example"]["username"], + "domain": saas_example_secrets["domain"], + "username": saas_example_secrets["username"], } resp = api_client.put( url, @@ -1291,10 +1291,10 @@ def test_saas_connection_connect_with_extra_secrets( url, api_client: TestClient, generate_auth_header, - saas_secrets, + mailchimp_secrets, ): auth_header = generate_auth_header(scopes=[CONNECTION_CREATE_OR_UPDATE]) - payload = {**saas_secrets["mailchimp"], "extra": "junk"} + payload = {**mailchimp_secrets, "extra": "junk"} resp = api_client.put( url, headers=auth_header, @@ -1314,10 +1314,10 @@ def url( self, oauth_client: ClientDetail, policy, - connection_config_mailchimp, - dataset_config_mailchimp, + mailchimp_connection_config, + mailchimp_dataset_config, ) -> str: - return f"{V1_URL_PREFIX}{CONNECTIONS}/{connection_config_mailchimp.key}/test" + return f"{V1_URL_PREFIX}{CONNECTIONS}/{mailchimp_connection_config.key}/test" def test_connection_configuration_test_not_authenticated( self, @@ -1325,16 +1325,16 @@ def test_connection_configuration_test_not_authenticated( api_client: TestClient, db: Session, generate_auth_header, - connection_config_mailchimp, + mailchimp_connection_config, ): - assert connection_config_mailchimp.last_test_timestamp is None + assert mailchimp_connection_config.last_test_timestamp is None resp = api_client.get(url) assert resp.status_code == 401 - db.refresh(connection_config_mailchimp) - assert connection_config_mailchimp.last_test_timestamp is None - assert connection_config_mailchimp.last_test_succeeded is None + db.refresh(mailchimp_connection_config) + assert mailchimp_connection_config.last_test_timestamp is None + assert mailchimp_connection_config.last_test_succeeded is None def test_connection_configuration_test_incorrect_scopes( self, @@ -1342,9 +1342,9 @@ def test_connection_configuration_test_incorrect_scopes( api_client: TestClient, db: Session, generate_auth_header, - connection_config_mailchimp, + mailchimp_connection_config, ): - assert connection_config_mailchimp.last_test_timestamp is None + assert mailchimp_connection_config.last_test_timestamp is None auth_header = generate_auth_header(scopes=[STORAGE_READ]) resp = api_client.get( @@ -1353,9 +1353,9 @@ def test_connection_configuration_test_incorrect_scopes( ) assert resp.status_code == 403 - db.refresh(connection_config_mailchimp) - assert connection_config_mailchimp.last_test_timestamp is None - assert connection_config_mailchimp.last_test_succeeded is None + db.refresh(mailchimp_connection_config) + assert mailchimp_connection_config.last_test_timestamp is None + assert mailchimp_connection_config.last_test_succeeded is None def test_connection_configuration_test_failed_response( self, @@ -1363,12 +1363,12 @@ def test_connection_configuration_test_failed_response( api_client: TestClient, db: Session, generate_auth_header, - connection_config_mailchimp, + mailchimp_connection_config, ): - assert connection_config_mailchimp.last_test_timestamp is None + assert mailchimp_connection_config.last_test_timestamp is None - connection_config_mailchimp.secrets = {"domain": "invalid_domain"} - connection_config_mailchimp.save(db) + mailchimp_connection_config.secrets = {"domain": "invalid_domain"} + mailchimp_connection_config.save(db) auth_header = generate_auth_header(scopes=[CONNECTION_READ]) resp = api_client.get( url, @@ -1379,17 +1379,17 @@ def test_connection_configuration_test_failed_response( body = json.loads(resp.text) assert body["test_status"] == "failed" assert ( - f"Operational Error connecting to '{connection_config_mailchimp.key}'." + f"Operational Error connecting to '{mailchimp_connection_config.key}'." == body["failure_reason"] ) assert ( body["msg"] - == f"Test completed for ConnectionConfig with key: {connection_config_mailchimp.key}." + == f"Test completed for ConnectionConfig with key: {mailchimp_connection_config.key}." ) - db.refresh(connection_config_mailchimp) - assert connection_config_mailchimp.last_test_timestamp is not None - assert connection_config_mailchimp.last_test_succeeded is False + db.refresh(mailchimp_connection_config) + assert mailchimp_connection_config.last_test_timestamp is not None + assert mailchimp_connection_config.last_test_succeeded is False def test_connection_configuration_test( self, @@ -1397,9 +1397,9 @@ def test_connection_configuration_test( api_client: TestClient, db: Session, generate_auth_header, - connection_config_mailchimp, + mailchimp_connection_config, ): - assert connection_config_mailchimp.last_test_timestamp is None + assert mailchimp_connection_config.last_test_timestamp is None auth_header = generate_auth_header(scopes=[CONNECTION_READ]) resp = api_client.get( @@ -1411,31 +1411,31 @@ def test_connection_configuration_test( body = json.loads(resp.text) assert ( body["msg"] - == f"Test completed for ConnectionConfig with key: {connection_config_mailchimp.key}." + == f"Test completed for ConnectionConfig with key: {mailchimp_connection_config.key}." ) assert body["failure_reason"] is None assert body["test_status"] == "succeeded" - db.refresh(connection_config_mailchimp) - assert connection_config_mailchimp.last_test_timestamp is not None - assert connection_config_mailchimp.last_test_succeeded is True + db.refresh(mailchimp_connection_config) + assert mailchimp_connection_config.last_test_timestamp is not None + assert mailchimp_connection_config.last_test_succeeded is True @pytest.mark.integration_saas @pytest.mark.integration_mailchimp class TestSaasConnector: def test_saas_connector( - self, db: Session, connection_config_mailchimp, dataset_config_mailchimp + self, db: Session, mailchimp_connection_config, mailchimp_dataset_config ): - connector = get_connector(connection_config_mailchimp) + connector = get_connector(mailchimp_connection_config) assert connector.__class__ == SaaSConnector client = connector.client() assert client.__class__ == AuthenticatedClient assert connector.test_connection() == ConnectionTestStatus.succeeded - connection_config_mailchimp.secrets = {"domain": "bad_host"} - connection_config_mailchimp.save(db) - connector = get_connector(connection_config_mailchimp) + mailchimp_connection_config.secrets = {"domain": "bad_host"} + mailchimp_connection_config.save(db) + connector = get_connector(mailchimp_connection_config) with pytest.raises(ConnectionException): connector.test_connection() diff --git a/tests/integration_tests/test_saas_task.py b/tests/integration_tests/test_saas_task.py index a510fa029..c48a54183 100644 --- a/tests/integration_tests/test_saas_task.py +++ b/tests/integration_tests/test_saas_task.py @@ -15,8 +15,8 @@ def test_saas_access_request_task( db, policy, - connection_config_mailchimp, - dataset_config_mailchimp, + mailchimp_connection_config, + mailchimp_dataset_config, mailchimp_identity_email, ) -> None: """Full access request based on the Mailchimp SaaS config""" @@ -30,15 +30,15 @@ def test_saas_access_request_task( identity = PrivacyRequestIdentity(**identity_kwargs) privacy_request.cache_identity(identity) - dataset_name = connection_config_mailchimp.get_saas_config().fides_key - merged_graph = dataset_config_mailchimp.get_graph() + dataset_name = mailchimp_connection_config.get_saas_config().fides_key + merged_graph = mailchimp_dataset_config.get_graph() graph = DatasetGraph(merged_graph) v = graph_task.run_access_request( privacy_request, policy, graph, - [connection_config_mailchimp], + [mailchimp_connection_config], {"email": mailchimp_identity_email}, ) @@ -128,8 +128,8 @@ def test_saas_access_request_task( def test_saas_erasure_request_task( db, policy, - connection_config_mailchimp, - dataset_config_mailchimp, + mailchimp_connection_config, + mailchimp_dataset_config, mailchimp_identity_email, reset_mailchimp_data, ) -> None: @@ -144,15 +144,15 @@ def test_saas_erasure_request_task( identity = PrivacyRequestIdentity(**identity_kwargs) privacy_request.cache_identity(identity) - dataset_name = connection_config_mailchimp.get_saas_config().fides_key - merged_graph = dataset_config_mailchimp.get_graph() + dataset_name = mailchimp_connection_config.get_saas_config().fides_key + merged_graph = mailchimp_dataset_config.get_graph() graph = DatasetGraph(merged_graph) graph_task.run_access_request( privacy_request, policy, graph, - [connection_config_mailchimp], + [mailchimp_connection_config], {"email": mailchimp_identity_email}, ) @@ -160,7 +160,7 @@ def test_saas_erasure_request_task( privacy_request, policy, graph, - [connection_config_mailchimp], + [mailchimp_connection_config], {"email": mailchimp_identity_email}, get_cached_data_for_erasures(privacy_request.id), ) diff --git a/tests/models/test_saasconfig.py b/tests/models/test_saasconfig.py index 52a1857b5..30623ebd7 100644 --- a/tests/models/test_saasconfig.py +++ b/tests/models/test_saasconfig.py @@ -6,17 +6,16 @@ @pytest.mark.unit_saas -def test_saas_configs(saas_configs) -> None: - """Simple test to verify that the available configs can be deserialized into SaaSConfigs""" - for saas_config in saas_configs.values(): - SaaSConfig(**saas_config) +def test_saas_configs(saas_example_config) -> None: + """Simple test to verify that the example config can be deserialized into SaaSConfigs""" + SaaSConfig(**saas_example_config) @pytest.mark.unit_saas -def test_saas_config_to_dataset(saas_configs: Dict[str, Dict]): +def test_saas_config_to_dataset(saas_example_config: Dict[str, Dict]): """Verify dataset generated by SaaS config""" # convert endpoint references to dataset references to be able to hook SaaS connectors into the graph traversal - saas_config = SaaSConfig(**saas_configs["saas_example"]) + saas_config = SaaSConfig(**saas_example_config) saas_dataset = saas_config.get_graph() messages_collection = saas_dataset.collections[0] diff --git a/tests/schemas/connection_configuration/test_connection_secrets_saas.py b/tests/schemas/connection_configuration/test_connection_secrets_saas.py index 51fb7de5a..16e2a07c1 100644 --- a/tests/schemas/connection_configuration/test_connection_secrets_saas.py +++ b/tests/schemas/connection_configuration/test_connection_secrets_saas.py @@ -10,8 +10,8 @@ @pytest.mark.unit_saas class TestSaaSConnectionSecrets: @pytest.fixture(scope="function") - def saas_config(self, saas_configs) -> SaaSConfig: - return SaaSConfig(**saas_configs["saas_example"]) + def saas_config(self, saas_example_config) -> SaaSConfig: + return SaaSConfig(**saas_example_config) def test_get_saas_schema(self, saas_config): """ @@ -22,9 +22,9 @@ def test_get_saas_schema(self, saas_config): assert schema.__name__ == f"{saas_config.fides_key}_schema" assert issubclass(schema.__base__, SaaSSchema) - def test_validation(self, saas_config, saas_secrets): + def test_validation(self, saas_config, saas_example_secrets): schema = SaaSSchemaFactory(saas_config).get_saas_schema() - config = saas_secrets["saas_example"] + config = saas_example_secrets schema.parse_obj(config) def test_missing_fields(self, saas_config): @@ -38,10 +38,10 @@ def test_missing_fields(self, saas_config): in str(exc.value) ) - def test_extra_fields(self, saas_config, saas_secrets): + def test_extra_fields(self, saas_config, saas_example_secrets): schema = SaaSSchemaFactory(saas_config).get_saas_schema() config = { - **saas_secrets["saas_example"], + **saas_example_secrets, "extra": "extra", } with pytest.raises(ValidationError) as exc: diff --git a/tests/service/connectors/test_queryconfig.py b/tests/service/connectors/test_queryconfig.py index 95e980e69..32bca671e 100644 --- a/tests/service/connectors/test_queryconfig.py +++ b/tests/service/connectors/test_queryconfig.py @@ -604,16 +604,16 @@ def test_generate_update_stmt_multiple_rules( class TestSaaSQueryConfig: @pytest.fixture(scope="function") def combined_traversal( - self, connection_config_saas_example, dataset_config_saas_example + self, saas_example_connection_config, saas_example_dataset_config ): - merged_graph = dataset_config_saas_example.get_graph() + merged_graph = saas_example_dataset_config.get_graph() graph = DatasetGraph(merged_graph) return Traversal(graph, {"email": "customer-1@example.com"}) def test_generate_query( - self, policy, combined_traversal, connection_config_saas_example + self, policy, combined_traversal, saas_example_connection_config ): - saas_config = connection_config_saas_example.get_saas_config() + saas_config = saas_example_connection_config.get_saas_config() endpoints = saas_config.top_level_endpoint_dict member = combined_traversal.traversal_node_dict[ @@ -673,9 +673,9 @@ def test_generate_update_stmt( self, erasure_policy_string_rewrite, combined_traversal, - connection_config_saas_example, + saas_example_connection_config, ): - saas_config = connection_config_saas_example.get_saas_config() + saas_config = saas_example_connection_config.get_saas_config() endpoints = saas_config.top_level_endpoint_dict member = combined_traversal.traversal_node_dict[ @@ -704,9 +704,9 @@ def test_generate_update_stmt( ) def test_generate_update_stmt_custom_http_method( - self, erasure_policy_string_rewrite, combined_traversal, connection_config_saas_example + self, erasure_policy_string_rewrite, combined_traversal, saas_example_connection_config ): - saas_config: Optional[SaaSConfig] = connection_config_saas_example.get_saas_config() + saas_config: Optional[SaaSConfig] = saas_example_connection_config.get_saas_config() saas_config.endpoints[2].requests.get("update").method = HTTPMethod.POST endpoints = saas_config.top_level_endpoint_dict @@ -736,9 +736,9 @@ def test_generate_update_stmt_custom_http_method( ) def test_generate_update_stmt_with_request_body( - self, erasure_policy_string_rewrite, combined_traversal, connection_config_saas_example + self, erasure_policy_string_rewrite, combined_traversal, saas_example_connection_config ): - saas_config: Optional[SaaSConfig] = connection_config_saas_example.get_saas_config() + saas_config: Optional[SaaSConfig] = saas_example_connection_config.get_saas_config() saas_config.endpoints[2].requests.get("update").body = '{"properties": {, "list_id": }}' body_request_params = RequestParam( name="list_id", diff --git a/tests/service/privacy_request/request_runner_service_test.py b/tests/service/privacy_request/request_runner_service_test.py index e0a7b36ec..b6cc3dd11 100644 --- a/tests/service/privacy_request/request_runner_service_test.py +++ b/tests/service/privacy_request/request_runner_service_test.py @@ -308,8 +308,8 @@ def test_create_and_process_access_request_mariadb( @mock.patch("fidesops.models.privacy_request.PrivacyRequest.trigger_policy_webhook") def test_create_and_process_access_request_saas( trigger_webhook_mock, - connection_config_mailchimp, - dataset_config_mailchimp, + mailchimp_connection_config, + mailchimp_dataset_config, db, cache, policy, @@ -347,8 +347,8 @@ def test_create_and_process_access_request_saas( @mock.patch("fidesops.models.privacy_request.PrivacyRequest.trigger_policy_webhook") def test_create_and_process_erasure_request_saas( trigger_webhook_mock, - connection_config_mailchimp, - dataset_config_mailchimp, + mailchimp_connection_config, + mailchimp_dataset_config, db, cache, erasure_policy_hmac, @@ -365,7 +365,7 @@ def test_create_and_process_erasure_request_saas( pr = get_privacy_request_results(db, erasure_policy_hmac, cache, data) - connector = SaaSConnector(connection_config_mailchimp) + connector = SaaSConnector(mailchimp_connection_config) request: SaaSRequestParams = SaaSRequestParams( method=HTTPMethod.GET, path="/3.0/search-members", params={"query": mailchimp_identity_email}, body=None ) From 62a6807ed2dc1403e76c7cddbbfbf3a1497fda2b Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Sat, 26 Mar 2022 19:42:31 -0700 Subject: [PATCH 19/24] Removing in-progress Stripe files --- data/saas/config/stripe_config.yml | 99 ------------ data/saas/dataset/stripe_dataset.yml | 149 ------------------ tests/conftest.py | 1 - tests/fixtures/saas/stripe_fixtures.py | 89 ----------- .../saas/test_stripe_task.py | 85 ---------- 5 files changed, 423 deletions(-) delete mode 100644 data/saas/config/stripe_config.yml delete mode 100644 data/saas/dataset/stripe_dataset.yml delete mode 100644 tests/fixtures/saas/stripe_fixtures.py delete mode 100644 tests/integration_tests/saas/test_stripe_task.py diff --git a/data/saas/config/stripe_config.yml b/data/saas/config/stripe_config.yml deleted file mode 100644 index b56423b68..000000000 --- a/data/saas/config/stripe_config.yml +++ /dev/null @@ -1,99 +0,0 @@ -saas_config: - fides_key: stripe_connector_example - name: Stripe SaaS Config - description: A sample schema representing the Stripe connector for Fidesops - version: 0.0.1 - - connector_params: - - name: host - - name: api_key - - name: payment_types - - name: page_limit - - client_config: - protocol: https - host: - connector_param: host - authentication: - strategy: bearer_authentication - configuration: - token: - connector_param: api_key - - test_request: - path: /v1/customers - - endpoints: - - name: customer - requests: - read: - path: /v1/customers - request_params: - - name: email - type: query - identity: email - postprocessors: - - strategy: unwrap - configuration: - data_path: data - update: - path: /v1/customers/ - request_params: - - name: id - type: path - references: - - dataset: stripe_connector_example - field: customer.id - direction: from - - name: payment_methods - requests: - read: - path: /v1/payment_methods - request_params: - - name: customer - type: query - references: - - dataset: stripe_connector_example - field: customer.id - direction: from - - name: type - type: query - connector_param: payment_types - - name: limit - type: query - connector_param: page_limit - postprocessors: - - strategy: unwrap - configuration: - data_path: data - pagination: - strategy: cursor - configuration: - cursor_param: starting_after - field: id - - name: bank_accounts - requests: - read: - path: /v1/customers//sources - request_params: - - name: customer_id - type: path - references: - - dataset: stripe_connector_example - field: customer.id - direction: from - - name: object - type: query - default_value: bank_account - - name: limit - type: query - connector_param: page_limit - postprocessors: - - strategy: unwrap - configuration: - data_path: data - pagination: - strategy: cursor - configuration: - cursor_param: starting_after - field: id \ No newline at end of file diff --git a/data/saas/dataset/stripe_dataset.yml b/data/saas/dataset/stripe_dataset.yml deleted file mode 100644 index 682ad1303..000000000 --- a/data/saas/dataset/stripe_dataset.yml +++ /dev/null @@ -1,149 +0,0 @@ -dataset: - - fides_key: stripe_connector_example - name: Stripe Dataset - description: A sample dataset representing the Stripe connector for Fidesops - collections: - - name: customer - fields: - - name: id - data_categories: [system.operations] - - name: object - data_categories: [system.operations] - - name: address - fields: - - name: city - data_categories: [system.operations] - - name: country - data_categories: [system.operations] - - name: line1 - data_categories: [system.operations] - - name: line2 - data_categories: [system.operations] - - name: postal_code - data_categories: [system.operations] - - name: state - data_categories: [system.operations] - - name: currency - data_categories: [system.operations] - - name: default_source - data_categories: [system.operations] - - name: description - data_categories: [system.operations] - - name: email - data_categories: [system.operations] - - name: invoice_settings - fields: - - name: custom_fields - data_categories: [system.operations] - - name: default_payment_method - data_categories: [system.operations] - - name: livemode - data_categories: [system.operations] - - name: name - data_categories: [system.operations] - - name: phone - data_categories: [system.operations] - - name: preferred_locales - data_categories: [system.operations] - - name: shipping - fields: - - name: address - fields: - - name: city - data_categories: [system.operations] - - name: country - data_categories: [system.operations] - - name: line1 - data_categories: [system.operations] - - name: line2 - data_categories: [system.operations] - - name: postal_code - data_categories: [system.operations] - - name: state - data_categories: [system.operations] - - name: sources - fields: - - name: object - data_categories: [system.operations] - - name: data - fields: - - name: id - data_categories: [system.operations] - - name: object - data_categories: [system.operations] - - name: address_city - data_categories: [system.operations] - - name: address_country - data_categories: [system.operations] - - name: address_line1 - data_categories: [system.operations] - - name: address_line1_check - data_categories: [system.operations] - - name: address_line2 - data_categories: [system.operations] - - name: address_state - data_categories: [system.operations] - - name: address_zip - data_categories: [system.operations] - - name: address_zip_check - data_categories: [system.operations] - - name: brand - data_categories: [system.operations] - - name: country - data_categories: [system.operations] - - name: customer - data_categories: [system.operations] - - name: cvc_check - data_categories: [system.operations] - - name: dynamic_last4 - data_categories: [system.operations] - - name: exp_month - data_categories: [system.operations] - - name: exp_year - data_categories: [system.operations] - - name: fingerprint - data_categories: [system.operations] - - name: funding - data_categories: [system.operations] - - name: last4 - data_categories: [system.operations] - - name: name - data_categories: [system.operations] - - name: tokenization_method - data_categories: [system.operations] - - name: url - data_categories: [system.operations] - - name: subscriptions - fields: - - name: object - data_categories: [system.operations] - - name: data - fields: - - name: id - data_categories: [system.operations] - - name: object - data_categories: [system.operations] - - name: application_fee_percent - data_categories: [system.operations] - - name: customer - data_categories: [system.operations] - - name: default_payment_method - data_categories: [system.operations] - - name: default_source - data_categories: [system.operations] - - name: default_tax_rates - data_categories: [system.operations] - - name: latest_invoice - data_categories: [system.operations] - - name: livemode - data_categories: [system.operations] - - name: url - data_categories: [system.operations] - - name: tax_exempt - data_categories: [system.operations] - - name: tax_ids - fields: - - name: object - data_categories: [system.operations] - - name: data - data_categories: [system.operations] \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index cbfb8d75e..0eb23056a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -40,7 +40,6 @@ from .fixtures.bigquery_fixtures import * from .fixtures.saas_example_fixtures import * from .fixtures.saas.mailchimp_fixtures import * -from .fixtures.saas.stripe_fixtures import * logger = logging.getLogger(__name__) diff --git a/tests/fixtures/saas/stripe_fixtures.py b/tests/fixtures/saas/stripe_fixtures.py deleted file mode 100644 index 3300a1550..000000000 --- a/tests/fixtures/saas/stripe_fixtures.py +++ /dev/null @@ -1,89 +0,0 @@ -import os -from typing import Any, Dict, Generator -from fidesops.core.config import load_toml -from fidesops.db import session -from fidesops.models.connectionconfig import ( - AccessLevel, - ConnectionConfig, - ConnectionType, -) -from fidesops.models.datasetconfig import DatasetConfig -import pytest -import pydash -from tests.fixtures.application_fixtures import load_dataset -from tests.fixtures.saas_example_fixtures import load_config -from sqlalchemy.orm import Session - -saas_config = load_toml("saas_config.toml") - - -@pytest.fixture(scope="function") -def stripe_secrets(): - return { - "host": pydash.get(saas_config, "stripe.host") or os.environ.get("STRIPE_HOST"), - "api_key": pydash.get(saas_config, "stripe.api_key") - or os.environ.get("STRIPE_API_KEY"), - "payment_types": pydash.get(saas_config, "stripe.payment_types") - or os.environ.get("STRIPE_PAYMENT_TYPES"), - "page_limit": pydash.get(saas_config, "stripe.page_limit") - or os.environ.get("STRIPE_PAGE_LIMIT"), - } - - -@pytest.fixture(scope="function") -def stripe_identity_email(): - return pydash.get(saas_config, "stripe.identity_email") or os.environ.get( - "STRIPE_IDENTITY_EMAIL" - ) - - -@pytest.fixture -def stripe_config() -> Dict[str, Any]: - return load_config("data/saas/config/stripe_config.yml") - - -@pytest.fixture -def stripe_dataset() -> Dict[str, Any]: - return load_dataset("data/saas/dataset/stripe_dataset.yml")[0] - - -@pytest.fixture(scope="function") -def stripe_connection_config( - db: Session, stripe_config: Dict[str, Dict], stripe_secrets: Dict[str, Any] -) -> Generator: - fides_key = stripe_config["fides_key"] - connection_config = ConnectionConfig.create( - db=db, - data={ - "key": fides_key, - "name": fides_key, - "connection_type": ConnectionType.saas, - "access": AccessLevel.write, - "secrets": stripe_secrets, - "saas_config": stripe_config, - }, - ) - yield connection_config - connection_config.delete(db) - - -@pytest.fixture -def stripe_dataset_config( - db: session, - stripe_connection_config: ConnectionConfig, - stripe_dataset: Dict[str, Dict], -) -> Generator: - fides_key = stripe_dataset["fides_key"] - stripe_connection_config.name = fides_key - stripe_connection_config.key = fides_key - stripe_connection_config.save(db=db) - dataset = DatasetConfig.create( - db=db, - data={ - "connection_config_id": stripe_connection_config.id, - "fides_key": fides_key, - "dataset": stripe_dataset, - }, - ) - yield dataset - dataset.delete(db=db) diff --git a/tests/integration_tests/saas/test_stripe_task.py b/tests/integration_tests/saas/test_stripe_task.py deleted file mode 100644 index b75b49d94..000000000 --- a/tests/integration_tests/saas/test_stripe_task.py +++ /dev/null @@ -1,85 +0,0 @@ -import pytest -import random - -from fidesops.graph.graph import DatasetGraph -from fidesops.models.privacy_request import ExecutionLog, PrivacyRequest -from fidesops.schemas.redis_cache import PrivacyRequestIdentity - -from fidesops.task import graph_task -from tests.graph.graph_test_util import assert_rows_match, records_matching_fields - - -@pytest.mark.integration -@pytest.mark.integration_saas -def test_stripe_access_request_task( - db, - policy, - stripe_connection_config, - stripe_dataset_config, - stripe_identity_email, -) -> None: - """Full access request based on the Stripe SaaS config""" - - privacy_request = PrivacyRequest( - id=f"test_stripe_access_request_task_{random.randint(0, 1000)}" - ) - identity_attribute = "email" - identity_value = stripe_identity_email - identity_kwargs = {identity_attribute: identity_value} - identity = PrivacyRequestIdentity(**identity_kwargs) - privacy_request.cache_identity(identity) - - dataset_name = stripe_connection_config.get_saas_config().fides_key - merged_graph = stripe_dataset_config.get_graph() - graph = DatasetGraph(merged_graph) - - v = graph_task.run_access_request( - privacy_request, - policy, - graph, - [stripe_connection_config], - {"email": stripe_identity_email}, - ) - - assert_rows_match( - v[f"{dataset_name}:customer"], - min_size=1, - keys=[ - "id", - "object", - "address", - "currency", - "default_source", - "description", - "email", - "invoice_settings", - "livemode", - "name", - "phone", - "preferred_locales", - "shipping", - "sources", - "subscriptions", - "tax_exempt", - "tax_ids", - ], - ) - - # links - assert v[f"{dataset_name}:customer"][0]["email"] == stripe_identity_email - - logs = ( - ExecutionLog.query(db=db) - .filter(ExecutionLog.privacy_request_id == privacy_request.id) - .all() - ) - - logs = [log.__dict__ for log in logs] - assert ( - len( - records_matching_fields( - logs, dataset_name=dataset_name, collection_name="customer" - ) - ) - > 0 - ) \ No newline at end of file From 2cf40b0d37ffb7ef35faebcc17c2c1982a3bdc28 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Sat, 26 Mar 2022 20:18:11 -0700 Subject: [PATCH 20/24] Fixing test --- tests/fixtures/saas_example_fixtures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fixtures/saas_example_fixtures.py b/tests/fixtures/saas_example_fixtures.py index 96af7ea90..ab9c20469 100644 --- a/tests/fixtures/saas_example_fixtures.py +++ b/tests/fixtures/saas_example_fixtures.py @@ -110,7 +110,7 @@ def saas_example_connection_config_with_invalid_saas_config( saas_example_secrets: Dict[str, Any], ) -> Generator: invalid_saas_config = saas_example_config.copy() - invalid_saas_config["endpoints"][0]["requests"]["read"]["request_params"].pop() + invalid_saas_config["endpoints"][0]["requests"]["read"]["param_values"].pop() connection_config = ConnectionConfig.create( db=db, data={ From 9159c2db5cc66f3c624f9a2ec0c467432d08b478 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Fri, 1 Apr 2022 10:04:41 -0700 Subject: [PATCH 21/24] Minor formatting fix --- tests/service/connectors/test_queryconfig.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/service/connectors/test_queryconfig.py b/tests/service/connectors/test_queryconfig.py index e45947e2e..7bc13ae54 100644 --- a/tests/service/connectors/test_queryconfig.py +++ b/tests/service/connectors/test_queryconfig.py @@ -729,9 +729,14 @@ def test_generate_update_stmt( ) def test_generate_update_stmt_custom_http_method( - self, erasure_policy_string_rewrite, combined_traversal, saas_example_connection_config + self, + erasure_policy_string_rewrite, + combined_traversal, + saas_example_connection_config, ): - saas_config: Optional[SaaSConfig] = saas_example_connection_config.get_saas_config() + saas_config: Optional[ + SaaSConfig + ] = saas_example_connection_config.get_saas_config() saas_config.endpoints[2].requests.get("update").method = HTTPMethod.POST endpoints = saas_config.top_level_endpoint_dict From 3bad5db8abf65e95cb6a55ab2cdf6e02a8c7c7c4 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Fri, 1 Apr 2022 10:24:36 -0700 Subject: [PATCH 22/24] Fixing test --- tests/models/test_saasconfig.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/models/test_saasconfig.py b/tests/models/test_saasconfig.py index a9fb4be41..b9b569176 100644 --- a/tests/models/test_saasconfig.py +++ b/tests/models/test_saasconfig.py @@ -70,10 +70,10 @@ def test_saas_config_to_dataset(saas_example_config: Dict[str, Dict]): @pytest.mark.unit_saas -def test_saas_config_ignore_errors_param(saas_configs: Dict[str, Dict]): +def test_saas_config_ignore_errors_param(saas_example_config: Dict[str, Dict]): """Verify saas config ignore errors""" # convert endpoint references to dataset references to be able to hook SaaS connectors into the graph traversal - saas_config = SaaSConfig(**saas_configs["saas_example"]) + saas_config = SaaSConfig(**saas_example_config) collections_endpoint = next( end for end in saas_config.endpoints if end.name == "conversations" From 1cf4bcf80ab389bf1d672b602d9fdd466f37da56 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Fri, 1 Apr 2022 11:42:25 -0700 Subject: [PATCH 23/24] Fixing integration test --- tests/fixtures/saas/mailchimp_fixtures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fixtures/saas/mailchimp_fixtures.py b/tests/fixtures/saas/mailchimp_fixtures.py index 752bec6dd..f43f6632d 100644 --- a/tests/fixtures/saas/mailchimp_fixtures.py +++ b/tests/fixtures/saas/mailchimp_fixtures.py @@ -101,7 +101,7 @@ def reset_mailchimp_data(mailchimp_connection_config, mailchimp_identity_email) request: SaaSRequestParams = SaaSRequestParams( method=HTTPMethod.GET, path="/3.0/search-members", - params={"query": mailchimp_identity_email}, + query_params={"query": mailchimp_identity_email}, body=None, ) response = connector.create_client().send(request) From a06a42c92f3d3d770ca557c47c9360d99c495161 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Fri, 1 Apr 2022 12:19:01 -0700 Subject: [PATCH 24/24] Removing unnecessary param --- tests/fixtures/saas/mailchimp_fixtures.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/fixtures/saas/mailchimp_fixtures.py b/tests/fixtures/saas/mailchimp_fixtures.py index f43f6632d..a611f3d52 100644 --- a/tests/fixtures/saas/mailchimp_fixtures.py +++ b/tests/fixtures/saas/mailchimp_fixtures.py @@ -111,7 +111,6 @@ def reset_mailchimp_data(mailchimp_connection_config, mailchimp_identity_email) request: SaaSRequestParams = SaaSRequestParams( method=HTTPMethod.PUT, path=f'/3.0/lists/{member["list_id"]}/members/{member["id"]}', - params={}, body=json.dumps(member), ) connector.create_client().send(request)