diff --git a/docs/fidesops/docs/postman/Fidesops.postman_collection.json b/docs/fidesops/docs/postman/Fidesops.postman_collection.json index 28f5c9edf..c473db80c 100644 --- a/docs/fidesops/docs/postman/Fidesops.postman_collection.json +++ b/docs/fidesops/docs/postman/Fidesops.postman_collection.json @@ -1,6 +1,6 @@ { "info": { - "_postman_id": "695c6d80-2cca-4b75-ac2e-933d3f68a03a", + "_postman_id": "f8e7f492-c328-4177-9872-4b7e7b2b6ec5", "name": "Fidesops", "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" }, @@ -1883,6 +1883,124 @@ "response": [] } ] + }, + { + "name": "MySQL", + "item": [ + { + "name": "Create/Update Connection Configs: MySQL", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "[\n {\"name\": \"Application MySQL DB\",\n \"key\": \"{{mysql_key}}\",\n \"connection_type\": \"mysql\",\n \"access\": \"read\"\n}]", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/connection/", + "host": [ + "{{host}}" + ], + "path": [ + "connection", + "" + ] + } + }, + "response": [] + }, + { + "name": "Update Connection Secrets: MySQL", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PUT", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"host\": \"mysql_example\",\n \"port\": 3306,\n \"dbname\": \"mysql_example\",\n \"username\": \"mysql_user\",\n \"password\": \"mysql_pw\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/connection/{{mysql_key}}/secret", + "host": [ + "{{host}}" + ], + "path": [ + "connection", + "{{mysql_key}}", + "secret" + ] + } + }, + "response": [] + }, + { + "name": "Create/Update MySQL Dataset", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "[{\n \"fides_key\": \"mysql_example_test_dataset\",\n \"name\": \"MySQL Example Test Dataset\",\n \"description\": \"Example of a MySQL dataset containing a variety of related tables like customers, products, addresses, etc.\",\n \"collections\": [\n {\n \"name\": \"address\",\n \"fields\": [\n {\n \"name\": \"city\",\n \"data_categories\": [\n \"user.provided.identifiable.contact.city\"\n ]\n },\n {\n \"name\": \"house\",\n \"data_categories\": [\n \"user.provided.identifiable.contact.street\"\n ]\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"state\",\n \"data_categories\": [\n \"user.provided.identifiable.contact.state\"\n ]\n },\n {\n \"name\": \"street\",\n \"data_categories\": [\n \"user.provided.identifiable.contact.street\"\n ]\n },\n {\n \"name\": \"zip\",\n \"data_categories\": [\n \"user.provided.identifiable.contact.postal_code\"\n ]\n }\n ]\n },\n {\n \"name\": \"customer\",\n \"fields\": [\n {\n \"name\": \"address_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mysql_example_test_dataset\",\n \"field\": \"address.id\",\n \"direction\": \"to\"\n }\n ]\n }\n },\n {\n \"name\": \"created\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.provided.identifiable.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\",\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"user.derived.identifiable.unique_id\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"user.provided.identifiable.name\"\n ]\n }\n ]\n },\n {\n \"name\": \"employee\",\n \"fields\": [\n {\n \"name\": \"address_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mysql_example_test_dataset\",\n \"field\": \"address.id\",\n \"direction\": \"to\"\n }\n ]\n }\n },\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.provided.identifiable.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\",\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"user.derived.identifiable.unique_id\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"user.provided.identifiable.name\"\n ]\n }\n ]\n },\n {\n \"name\": \"login\",\n \"fields\": [\n {\n \"name\": \"customer_id\",\n \"data_categories\": [\n \"user.derived.identifiable.unique_id\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mysql_example_test_dataset\",\n \"field\": \"customer.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"time\",\n \"data_categories\": [\n \"user.derived.nonidentifiable.sensor\"\n ]\n }\n ]\n },\n {\n \"name\": \"orders\",\n \"fields\": [\n {\n \"name\": \"customer_id\",\n \"data_categories\": [\n \"user.derived.identifiable.unique_id\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mysql_example_test_dataset\",\n \"field\": \"customer.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"shipping_address_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mysql_example_test_dataset\",\n \"field\": \"address.id\",\n \"direction\": \"to\"\n }\n ]\n }\n }\n ]\n },\n {\n \"name\": \"order_item\",\n \"fields\": [\n {\n \"name\": \"order_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mysql_example_test_dataset\",\n \"field\": \"orders.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"product_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mysql_example_test_dataset\",\n \"field\": \"product.id\",\n \"direction\": \"to\"\n }\n ]\n }\n },\n {\n \"name\": \"quantity\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"payment_card\",\n \"fields\": [\n {\n \"name\": \"billing_address_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mysql_example_test_dataset\",\n \"field\": \"address.id\",\n \"direction\": \"to\"\n }\n ]\n }\n },\n {\n \"name\": \"ccn\",\n \"data_categories\": [\n \"user.provided.identifiable.financial.account_number\"\n ]\n },\n {\n \"name\": \"code\",\n \"data_categories\": [\n \"user.provided.identifiable.financial\"\n ]\n },\n {\n \"name\": \"customer_id\",\n \"data_categories\": [\n \"user.derived.identifiable.unique_id\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mysql_example_test_dataset\",\n \"field\": \"customer.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"user.provided.identifiable.financial\"\n ]\n },\n {\n \"name\": \"preferred\",\n \"data_categories\": [\n \"user.provided.nonidentifiable\"\n ]\n }\n ]\n },\n {\n \"name\": \"product\",\n \"fields\": [\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"price\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"report\",\n \"fields\": [\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.provided.identifiable.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\",\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"month\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"total_visits\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"year\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"service_request\",\n \"fields\": [\n {\n \"name\": \"alt_email\",\n \"data_categories\": [\n \"user.provided.identifiable.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\",\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"closed\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\",\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"employee_id\",\n \"data_categories\": [\n \"user.derived.identifiable.unique_id\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mysql_example_test_dataset\",\n \"field\": \"employee.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"opened\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"visit\",\n \"fields\": [\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.provided.identifiable.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\",\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"last_visit\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n }\n ]\n}]", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/connection/{{mysql_key}}/dataset", + "host": [ + "{{host}}" + ], + "path": [ + "connection", + "{{mysql_key}}", + "dataset" + ] + } + }, + "response": [] + } + ] } ], "event": [ @@ -1954,6 +2072,11 @@ "key": "mssql_key", "value": "app_mssql_db" }, + { + "key": "mysql_key", + "value": "app_mysql_db", + "type": "string" + }, { "key": "mongo_key", "value": "app_mongo_db" diff --git a/docs/fidesops/docs/postman/using_postman.md b/docs/fidesops/docs/postman/using_postman.md index d535313b7..62b309181 100644 --- a/docs/fidesops/docs/postman/using_postman.md +++ b/docs/fidesops/docs/postman/using_postman.md @@ -70,6 +70,8 @@ Inspect the `Body` of each request to see what we're sending to fidesops: 1. SEND `Create/Update Postgres Dataset` 2. SEND `Create/Update Dataset Mongo` +Note: API calls to additional supported datastores (MsSQL, MySQL) are in separate folders within the collection. + ## Run a privacy request Now, we should have all the basic pieces needed to create an Access request. diff --git a/src/fidesops/service/connectors/query_config.py b/src/fidesops/service/connectors/query_config.py index 89aeb7b5d..3e555e9ef 100644 --- a/src/fidesops/service/connectors/query_config.py +++ b/src/fidesops/service/connectors/query_config.py @@ -320,9 +320,6 @@ def generate_query( self.format_clause_for_query(string_path, "IN", string_path) ) query_data[string_path] = tuple(data) - else: - # if there's no data, create no clause - pass if len(clauses) > 0: query_str = self.get_formatted_query_string(field_list, clauses) return text(query_str).params(query_data) @@ -452,9 +449,6 @@ def generate_query( # pylint: disable=R0914 clauses.append( self.format_clause_for_query(string_path, "IN", operand) ) - else: - # if there's no data, create no clause - pass if len(clauses) > 0: query_str = self.get_formatted_query_string(field_list, clauses) return text(query_str).params(query_data) @@ -566,9 +560,6 @@ def transform_query_pairs(pairs: Dict[str, Any]) -> Dict[str, Any]: elif len(data) > 1: query_pairs[string_field_path] = {"$in": data} - else: - # if there's no data, create no clause - pass query_fields, return_fields = ( transform_query_pairs(query_pairs), field_list, diff --git a/src/fidesops/service/connectors/sql_connector.py b/src/fidesops/service/connectors/sql_connector.py index e5c96add9..ee7e91498 100644 --- a/src/fidesops/service/connectors/sql_connector.py +++ b/src/fidesops/service/connectors/sql_connector.py @@ -11,8 +11,8 @@ LegacyCursorResult, Connection, ) -from sqlalchemy.sql.elements import TextClause from sqlalchemy.exc import OperationalError, InternalError +from sqlalchemy.sql.elements import TextClause from snowflake.sqlalchemy import URL as Snowflake_URL from fidesops.common_exceptions import ConnectionException @@ -48,10 +48,11 @@ class SQLConnector(BaseConnector[Engine]): def cursor_result_to_rows(results: CursorResult) -> List[Row]: """Convert SQLAlchemy results to a list of dictionaries""" columns: List[Column] = results.cursor.description - l = len(columns) rows = [] for row_tuple in results: - rows.append({columns[i].name: row_tuple[i] for i in range(l)}) + rows.append( + {col.name: row_tuple[count] for count, col in enumerate(columns)} + ) return rows @abstractmethod @@ -95,7 +96,7 @@ def retrieve_data( logger.info(f"Starting data retrieval for {node.address}") with client.connect() as connection: results = connection.execute(stmt) - return SQLConnector.cursor_result_to_rows(results) + return self.cursor_result_to_rows(results) def mask_data( self, @@ -183,6 +184,19 @@ def create_client(self) -> Engine: echo=not self.hide_parameters, ) + # Overrides BaseConnector.cursor_result_to_rows + @staticmethod + def cursor_result_to_rows(results: LegacyCursorResult) -> List[Row]: + """ + Convert SQLAlchemy results to a list of dictionaries + Overrides BaseConnector.cursor_result_to_rows since SQLAlchemy execute returns LegacyCursorResult for MySQL + """ + columns: List[Column] = results.cursor.description + rows = [] + for row_tuple in results: + rows.append({col[0]: row_tuple[count] for count, col in enumerate(columns)}) + return rows + class RedshiftConnector(SQLConnector): """Connector specific to Amazon Redshift""" @@ -355,25 +369,13 @@ def query_config(self, node: TraversalNode) -> SQLQueryConfig: # Overrides BaseConnector.cursor_result_to_rows @staticmethod - def cursor_result_to_rows(results: CursorResult) -> List[Row]: - """Convert SQLAlchemy results to a list of dictionaries""" + def cursor_result_to_rows(results: LegacyCursorResult) -> List[Row]: + """ + Convert SQLAlchemy results to a list of dictionaries + Overrides BaseConnector.cursor_result_to_rows since SQLAlchemy execute returns LegacyCursorResult for MsSQL + """ columns: List[Column] = results.cursor.description - l = len(columns) rows = [] for row_tuple in results: - rows.append({columns[i][0]: row_tuple[i] for i in range(l)}) + rows.append({col[0]: row_tuple[count] for count, col in enumerate(columns)}) return rows - - def retrieve_data( - self, node: TraversalNode, policy: Policy, input_data: Dict[str, List[Any]] - ) -> List[Row]: - """Retrieve sql data""" - query_config = self.query_config(node) - client = self.client() - stmt: Optional[TextClause] = query_config.generate_query(input_data, policy) - if stmt is None: - return [] - logger.info(f"Starting data retrieval for {node.address}") - with client.connect() as connection: - results: CursorResult = connection.execute(stmt) - return MicrosoftSQLServerConnector.cursor_result_to_rows(results) diff --git a/tests/api/v1/endpoints/test_dataset_endpoints.py b/tests/api/v1/endpoints/test_dataset_endpoints.py index aa728da50..fa6816a1e 100644 --- a/tests/api/v1/endpoints/test_dataset_endpoints.py +++ b/tests/api/v1/endpoints/test_dataset_endpoints.py @@ -46,6 +46,8 @@ def test_example_datasets(example_datasets): assert len(example_datasets[3]["collections"]) == 11 assert example_datasets[4]["fides_key"] == "mssql_example_test_dataset" assert len(example_datasets[4]["collections"]) == 11 + assert example_datasets[5]["fides_key"] == "mysql_example_test_dataset" + assert len(example_datasets[5]["collections"]) == 11 class TestValidateDataset: @@ -440,7 +442,7 @@ def test_patch_datasets_bulk_create( assert response.status_code == 200 response_body = json.loads(response.text) - assert len(response_body["succeeded"]) == 5 + assert len(response_body["succeeded"]) == 6 assert len(response_body["failed"]) == 0 # Confirm that postgres dataset matches the values we provided @@ -476,6 +478,17 @@ def test_patch_datasets_bulk_create( assert "Example of a Microsoft SQLServer dataset" in mssql_dataset["description"] assert len(mssql_dataset["collections"]) == 11 + # check the mysql dataset + mysql_dataset = response_body["succeeded"][5] + mysql_config = DatasetConfig.get_by( + db=db, field="fides_key", value="mysql_example_test_dataset" + ) + assert mysql_config is not None + assert mysql_dataset["fides_key"] == "mysql_example_test_dataset" + assert mysql_dataset["name"] == "MySQL Example Test Dataset" + assert "Example of a MySQL dataset" in mysql_dataset["description"] + assert len(mysql_dataset["collections"]) == 11 + postgres_config.delete(db) mongo_config.delete(db) mssql_config.delete(db) @@ -527,7 +540,7 @@ def test_patch_datasets_bulk_update( assert response.status_code == 200 response_body = json.loads(response.text) - assert len(response_body["succeeded"]) == 5 + assert len(response_body["succeeded"]) == 6 assert len(response_body["failed"]) == 0 # test postgres @@ -600,7 +613,7 @@ def test_patch_datasets_failed_response( assert response.status_code == 200 # Returns 200 regardless response_body = json.loads(response.text) assert len(response_body["succeeded"]) == 0 - assert len(response_body["failed"]) == 5 + assert len(response_body["failed"]) == 6 for failed_response in response_body["failed"]: assert "Dataset create/update failed" in failed_response["message"] diff --git a/tests/fixtures.py b/tests/fixtures.py index 2d8c49a54..5e88c3e05 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -981,7 +981,8 @@ def example_datasets() -> List[Dict]: "data/dataset/mongo_example_test_dataset.yml", "data/dataset/snowflake_example_test_dataset.yml", "data/dataset/redshift_example_test_dataset.yml", - "data/dataset/mssql_example_test_dataset.yml" + "data/dataset/mssql_example_test_dataset.yml", + "data/dataset/mysql_example_test_dataset.yml" ] for filename in example_filenames: example_datasets += load_dataset(filename) @@ -1100,8 +1101,7 @@ def mysql_example_test_dataset_config( db: Session, example_datasets: List[Dict], ) -> Generator: - # todo: this references the incorrect dataset - mysql_dataset = example_datasets[0] + mysql_dataset = example_datasets[5] fides_key = mysql_dataset["fides_key"] connection_config_mysql.name = fides_key connection_config_mysql.key = fides_key diff --git a/tests/integration_tests/test_sql_task.py b/tests/integration_tests/test_sql_task.py index 8a39db4d6..2144254ac 100644 --- a/tests/integration_tests/test_sql_task.py +++ b/tests/integration_tests/test_sql_task.py @@ -253,10 +253,10 @@ def test_sql_erasure_task(db, postgres_inserts, integration_postgres_config): @pytest.mark.integration -def test_sql_access_request_task(db, policy, integration_postgres_config) -> None: +def test_postgres_access_request_task(db, policy, integration_postgres_config) -> None: privacy_request = PrivacyRequest( - id=f"test_sql_access_request_task_{random.randint(0, 1000)}" + id=f"test_postgres_access_request_task_{random.randint(0, 1000)}" ) v = graph_task.run_access_request( @@ -334,6 +334,170 @@ def test_sql_access_request_task(db, policy, integration_postgres_config) -> Non ) +@pytest.mark.integration +def test_mssql_access_request_task(db, policy, connection_config_mssql) -> None: + + privacy_request = PrivacyRequest( + id=f"test_mssql_access_request_task_{random.randint(0, 1000)}" + ) + + v = graph_task.run_access_request( + privacy_request, + policy, + integration_db_graph("my_mssql_db_1"), + [connection_config_mssql], + {"email": "customer-1@example.com"}, + ) + + assert_rows_match( + v["my_mssql_db_1:address"], + min_size=2, + keys=["id", "street", "city", "state", "zip"], + ) + assert_rows_match( + v["my_mssql_db_1:orders"], + min_size=3, + keys=["id", "customer_id", "shipping_address_id", "payment_card_id"], + ) + assert_rows_match( + v["my_mssql_db_1:payment_card"], + min_size=2, + keys=["id", "name", "ccn", "customer_id", "billing_address_id"], + ) + assert_rows_match( + v["my_mssql_db_1:customer"], + min_size=1, + keys=["id", "name", "email", "address_id"], + ) + + # links + assert v["my_mssql_db_1:customer"][0]["email"] == "customer-1@example.com" + + logs = ( + ExecutionLog.query(db=db) + .filter(ExecutionLog.privacy_request_id == privacy_request.id) + .all() + ) + + logs = [log.__dict__ for log in logs] + assert ( + len( + records_matching_fields( + logs, dataset_name="my_mssql_db_1", collection_name="customer" + ) + ) + > 0 + ) + assert ( + len( + records_matching_fields( + logs, dataset_name="my_mssql_db_1", collection_name="address" + ) + ) + > 0 + ) + assert ( + len( + records_matching_fields( + logs, dataset_name="my_mssql_db_1", collection_name="orders" + ) + ) + > 0 + ) + assert ( + len( + records_matching_fields( + logs, + dataset_name="my_mssql_db_1", + collection_name="payment_card", + ) + ) + > 0 + ) + + +@pytest.mark.integration +def test_mysql_access_request_task(db, policy, connection_config_mysql) -> None: + + privacy_request = PrivacyRequest( + id=f"test_mysql_access_request_task_{random.randint(0, 1000)}" + ) + + v = graph_task.run_access_request( + privacy_request, + policy, + integration_db_graph("my_mysql_db_1"), + [connection_config_mysql], + {"email": "customer-1@example.com"}, + ) + + assert_rows_match( + v["my_mysql_db_1:address"], + min_size=2, + keys=["id", "street", "city", "state", "zip"], + ) + assert_rows_match( + v["my_mysql_db_1:orders"], + min_size=3, + keys=["id", "customer_id", "shipping_address_id", "payment_card_id"], + ) + assert_rows_match( + v["my_mysql_db_1:payment_card"], + min_size=2, + keys=["id", "name", "ccn", "customer_id", "billing_address_id"], + ) + assert_rows_match( + v["my_mysql_db_1:customer"], + min_size=1, + keys=["id", "name", "email", "address_id"], + ) + + # links + assert v["my_mysql_db_1:customer"][0]["email"] == "customer-1@example.com" + + logs = ( + ExecutionLog.query(db=db) + .filter(ExecutionLog.privacy_request_id == privacy_request.id) + .all() + ) + + logs = [log.__dict__ for log in logs] + assert ( + len( + records_matching_fields( + logs, dataset_name="my_mysql_db_1", collection_name="customer" + ) + ) + > 0 + ) + assert ( + len( + records_matching_fields( + logs, dataset_name="my_mysql_db_1", collection_name="address" + ) + ) + > 0 + ) + assert ( + len( + records_matching_fields( + logs, dataset_name="my_mysql_db_1", collection_name="orders" + ) + ) + > 0 + ) + assert ( + len( + records_matching_fields( + logs, + dataset_name="my_mysql_db_1", + collection_name="payment_card", + ) + ) + > 0 + ) + + @pytest.mark.integration def test_filter_on_data_categories( db, diff --git a/tests/service/privacy_request/request_runner_service_test.py b/tests/service/privacy_request/request_runner_service_test.py index ff6c7f4ec..ea407c0ee 100644 --- a/tests/service/privacy_request/request_runner_service_test.py +++ b/tests/service/privacy_request/request_runner_service_test.py @@ -29,7 +29,9 @@ from fidesops.service.connectors import PostgreSQLConnector from fidesops.service.connectors.sql_connector import ( SnowflakeConnector, - RedshiftConnector, MicrosoftSQLServerConnector, + RedshiftConnector, + MicrosoftSQLServerConnector, + MySQLConnector, ) from fidesops.service.masking.strategy.masking_strategy_factory import get_strategy from fidesops.service.privacy_request.request_runner_service import PrivacyRequestRunner @@ -212,6 +214,45 @@ def test_create_and_process_access_request_mssql( pr.delete(db=db) +@pytest.mark.integration +@mock.patch("fidesops.models.privacy_request.PrivacyRequest.trigger_policy_webhook") +def test_create_and_process_access_request_mysql( + trigger_webhook_mock, + mysql_example_test_dataset_config, + db, + cache, + policy, + policy_pre_execution_webhooks, + policy_post_execution_webhooks, +): + + customer_email = "customer-1@example.com" + data = { + "requested_at": "2021-08-30T16:09:37.359Z", + "policy_key": policy.key, + "identity": {"email": customer_email}, + } + + pr = get_privacy_request_results(db, policy, cache, data) + + results = pr.get_results() + assert len(results.keys()) == 11 + + for key in results.keys(): + assert results[key] is not None + assert results[key] != {} + + result_key_prefix = f"EN_{pr.id}__access_request__mysql_example_test_dataset:" + customer_key = result_key_prefix + "customer" + assert results[customer_key][0]["email"] == customer_email + + visit_key = result_key_prefix + "visit" + assert results[visit_key][0]["email"] == customer_email + # Both pre-execution webhooks and both post-execution webhooks were called + assert trigger_webhook_mock.call_count == 4 + pr.delete(db=db) + + @pytest.mark.integration_erasure def test_create_and_process_erasure_request_specific_category( postgres_example_test_dataset_config, @@ -290,6 +331,45 @@ def test_create_and_process_erasure_request_specific_category_mssql( assert customer_found +@pytest.mark.integration_erasure +def test_create_and_process_erasure_request_specific_category_mysql( + mysql_example_test_dataset_config, + cache, + db, + generate_auth_header, + erasure_policy, + connection_config_mysql, +): + customer_email = "customer-1@example.com" + customer_id = 1 + data = { + "requested_at": "2021-08-30T16:09:37.359Z", + "policy_key": erasure_policy.key, + "identity": {"email": customer_email}, + } + + pr = get_privacy_request_results(db, erasure_policy, cache, data) + pr.delete(db=db) + + example_mysql_uri = MySQLConnector(connection_config_mysql).build_uri() + engine = get_db_engine(database_uri=example_mysql_uri) + SessionLocal = get_db_session(engine=engine) + integration_db = SessionLocal() + stmt = select( + column("id"), + column("name"), + ).select_from(table("customer")) + res = integration_db.execute(stmt).all() + + customer_found = False + for row in res: + if customer_id in row: + customer_found = True + # Check that the `name` field is `None` + assert row.name is None + assert customer_found + + @pytest.mark.integration_erasure def test_create_and_process_erasure_request_generic_category( postgres_example_test_dataset_config,