Skip to content
This repository has been archived by the owner on Nov 30, 2022. It is now read-only.

136 mysql query execution #168

Merged
merged 6 commits into from
Jan 25, 2022
+416 −41
Merged

136 mysql query execution #168

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
3407cd1
getting started, add integration test for mysql
eastandwestwind Jan 20, 2022
e731859
adds all access/integration tests for mysql
eastandwestwind Jan 24, 2022
1c89004
merge in latest main
eastandwestwind Jan 24, 2022
754d187
fix tests
eastandwestwind Jan 24, 2022
7369350
cr changes
eastandwestwind Jan 25, 2022
ea8420a
formatting
eastandwestwind Jan 25, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
125 changes: 124 additions & 1 deletion docs/fidesops/docs/postman/Fidesops.postman_collection.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"info": {
"_postman_id": "695c6d80-2cca-4b75-ac2e-933d3f68a03a",
"_postman_id": "f8e7f492-c328-4177-9872-4b7e7b2b6ec5",
"name": "Fidesops",
"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json"
},
Expand Down Expand Up @@ -1883,6 +1883,124 @@
"response": []
}
]
},
{
"name": "MySQL",
"item": [
{
"name": "Create/Update Connection Configs: MySQL",
"request": {
"auth": {
"type": "bearer",
"bearer": [
{
"key": "token",
"value": "{{client_token}}",
"type": "string"
}
]
},
"method": "PATCH",
"header": [],
"body": {
"mode": "raw",
"raw": "[\n {\"name\": \"Application MySQL DB\",\n \"key\": \"{{mysql_key}}\",\n \"connection_type\": \"mysql\",\n \"access\": \"read\"\n}]",
"options": {
"raw": {
"language": "json"
}
}
},
"url": {
"raw": "{{host}}/connection/",
"host": [
"{{host}}"
],
"path": [
"connection",
""
]
}
},
"response": []
},
{
"name": "Update Connection Secrets: MySQL",
"request": {
"auth": {
"type": "bearer",
"bearer": [
{
"key": "token",
"value": "{{client_token}}",
"type": "string"
}
]
},
"method": "PUT",
"header": [],
"body": {
"mode": "raw",
"raw": "{\n \"host\": \"mysql_example\",\n \"port\": 3306,\n \"dbname\": \"mysql_example\",\n \"username\": \"mysql_user\",\n \"password\": \"mysql_pw\"\n}",
"options": {
"raw": {
"language": "json"
}
}
},
"url": {
"raw": "{{host}}/connection/{{mysql_key}}/secret",
"host": [
"{{host}}"
],
"path": [
"connection",
"{{mysql_key}}",
"secret"
]
}
},
"response": []
},
{
"name": "Create/Update MySQL Dataset",
"request": {
"auth": {
"type": "bearer",
"bearer": [
{
"key": "token",
"value": "{{client_token}}",
"type": "string"
}
]
},
"method": "PATCH",
"header": [],
"body": {
"mode": "raw",
"raw": "[{\n \"fides_key\": \"mysql_example_test_dataset\",\n \"name\": \"MySQL Example Test Dataset\",\n \"description\": \"Example of a MySQL dataset containing a variety of related tables like customers, products, addresses, etc.\",\n \"collections\": [\n {\n \"name\": \"address\",\n \"fields\": [\n {\n \"name\": \"city\",\n \"data_categories\": [\n \"user.provided.identifiable.contact.city\"\n ]\n },\n {\n \"name\": \"house\",\n \"data_categories\": [\n \"user.provided.identifiable.contact.street\"\n ]\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"state\",\n \"data_categories\": [\n \"user.provided.identifiable.contact.state\"\n ]\n },\n {\n \"name\": \"street\",\n \"data_categories\": [\n \"user.provided.identifiable.contact.street\"\n ]\n },\n {\n \"name\": \"zip\",\n \"data_categories\": [\n \"user.provided.identifiable.contact.postal_code\"\n ]\n }\n ]\n },\n {\n \"name\": \"customer\",\n \"fields\": [\n {\n \"name\": \"address_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mysql_example_test_dataset\",\n \"field\": \"address.id\",\n \"direction\": \"to\"\n }\n ]\n }\n },\n {\n \"name\": \"created\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.provided.identifiable.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\",\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"user.derived.identifiable.unique_id\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"user.provided.identifiable.name\"\n ]\n }\n ]\n },\n {\n \"name\": \"employee\",\n \"fields\": [\n {\n \"name\": \"address_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mysql_example_test_dataset\",\n \"field\": \"address.id\",\n \"direction\": \"to\"\n }\n ]\n }\n },\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.provided.identifiable.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\",\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"user.derived.identifiable.unique_id\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"user.provided.identifiable.name\"\n ]\n }\n ]\n },\n {\n \"name\": \"login\",\n \"fields\": [\n {\n \"name\": \"customer_id\",\n \"data_categories\": [\n \"user.derived.identifiable.unique_id\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mysql_example_test_dataset\",\n \"field\": \"customer.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"time\",\n \"data_categories\": [\n \"user.derived.nonidentifiable.sensor\"\n ]\n }\n ]\n },\n {\n \"name\": \"orders\",\n \"fields\": [\n {\n \"name\": \"customer_id\",\n \"data_categories\": [\n \"user.derived.identifiable.unique_id\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mysql_example_test_dataset\",\n \"field\": \"customer.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"shipping_address_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mysql_example_test_dataset\",\n \"field\": \"address.id\",\n \"direction\": \"to\"\n }\n ]\n }\n }\n ]\n },\n {\n \"name\": \"order_item\",\n \"fields\": [\n {\n \"name\": \"order_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mysql_example_test_dataset\",\n \"field\": \"orders.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"product_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mysql_example_test_dataset\",\n \"field\": \"product.id\",\n \"direction\": \"to\"\n }\n ]\n }\n },\n {\n \"name\": \"quantity\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"payment_card\",\n \"fields\": [\n {\n \"name\": \"billing_address_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mysql_example_test_dataset\",\n \"field\": \"address.id\",\n \"direction\": \"to\"\n }\n ]\n }\n },\n {\n \"name\": \"ccn\",\n \"data_categories\": [\n \"user.provided.identifiable.financial.account_number\"\n ]\n },\n {\n \"name\": \"code\",\n \"data_categories\": [\n \"user.provided.identifiable.financial\"\n ]\n },\n {\n \"name\": \"customer_id\",\n \"data_categories\": [\n \"user.derived.identifiable.unique_id\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mysql_example_test_dataset\",\n \"field\": \"customer.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"user.provided.identifiable.financial\"\n ]\n },\n {\n \"name\": \"preferred\",\n \"data_categories\": [\n \"user.provided.nonidentifiable\"\n ]\n }\n ]\n },\n {\n \"name\": \"product\",\n \"fields\": [\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"price\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"report\",\n \"fields\": [\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.provided.identifiable.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\",\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"month\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"total_visits\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"year\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"service_request\",\n \"fields\": [\n {\n \"name\": \"alt_email\",\n \"data_categories\": [\n \"user.provided.identifiable.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\",\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"closed\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\",\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"employee_id\",\n \"data_categories\": [\n \"user.derived.identifiable.unique_id\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mysql_example_test_dataset\",\n \"field\": \"employee.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"opened\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"visit\",\n \"fields\": [\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.provided.identifiable.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\",\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"last_visit\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n }\n ]\n}]",
"options": {
"raw": {
"language": "json"
}
}
},
"url": {
"raw": "{{host}}/connection/{{mysql_key}}/dataset",
"host": [
"{{host}}"
],
"path": [
"connection",
"{{mysql_key}}",
"dataset"
]
}
},
"response": []
}
]
}
],
"event": [
Expand Down Expand Up @@ -1954,6 +2072,11 @@
"key": "mssql_key",
"value": "app_mssql_db"
},
{
"key": "mysql_key",
"value": "app_mysql_db",
"type": "string"
},
{
"key": "mongo_key",
"value": "app_mongo_db"
Expand Down
2 changes: 2 additions & 0 deletions docs/fidesops/docs/postman/using_postman.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,8 @@ Inspect the `Body` of each request to see what we're sending to fidesops:
1. SEND `Create/Update Postgres Dataset`
2. SEND `Create/Update Dataset Mongo`

Note: API calls to additional supported datastores (MsSQL, MySQL) are in separate folders within the collection.


## Run a privacy request
Now, we should have all the basic pieces needed to create an Access request.
Expand Down
9 changes: 0 additions & 9 deletions src/fidesops/service/connectors/query_config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -320,9 +320,6 @@ def generate_query(
self.format_clause_for_query(string_path, "IN", string_path)
)
query_data[string_path] = tuple(data)
else:
# if there's no data, create no clause
pass
if len(clauses) > 0:
query_str = self.get_formatted_query_string(field_list, clauses)
return text(query_str).params(query_data)
Expand Down Expand Up @@ -452,9 +449,6 @@ def generate_query( # pylint: disable=R0914
clauses.append(
self.format_clause_for_query(string_path, "IN", operand)
)
else:
# if there's no data, create no clause
pass
if len(clauses) > 0:
query_str = self.get_formatted_query_string(field_list, clauses)
return text(query_str).params(query_data)
Expand Down Expand Up @@ -566,9 +560,6 @@ def transform_query_pairs(pairs: Dict[str, Any]) -> Dict[str, Any]:
elif len(data) > 1:
query_pairs[string_field_path] = {"$in": data}

else:
# if there's no data, create no clause
pass
query_fields, return_fields = (
transform_query_pairs(query_pairs),
field_list,
Expand Down
Loading