diff --git a/CHANGELOG.md b/CHANGELOG.md index 40caaf0c8..f5941b3c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,8 @@ The types of changes are: ### Added * Adds users and owners configuration for Hubspot connector [#1091](https://github.com/ethyca/fidesops/pull/1091) +* Foundations for a new email connector type [#1142](https://github.com/ethyca/fidesops/pull/1142) + ## [1.7.1](https://github.com/ethyca/fidesops/compare/1.7.0...1.7.1) diff --git a/data/dataset/email_dataset.yml b/data/dataset/email_dataset.yml new file mode 100644 index 000000000..56a87a4b7 --- /dev/null +++ b/data/dataset/email_dataset.yml @@ -0,0 +1,39 @@ +dataset: + - fides_key: email_dataset + name: Dataset not accessible automatically + description: Example of a email dataset with a collection waiting on postgres input + collections: + - name: daycare_customer + fields: + - name: id + data_categories: [system.operations] + fidesops_meta: + primary_key: true + - name: customer_id + data_categories: [user] + fidesops_meta: + references: + - dataset: postgres_example_test_dataset + field: customer.id + direction: from + - name: children + fields: + - name: id + data_categories: [system.operations] + fidesops_meta: + primary_key: true + - name: first_name + data_categories: [user.childrens] + - name: last_name + data_categories: [user.childrens] + - name: birthday + data_categories: [user.childrens] + fidesops_meta: + data_type: string + - name: parent_id + data_categories: [user] + fidesops_meta: + references: + - dataset: email_dataset + field: daycare_customer.id + direction: from \ No newline at end of file diff --git a/docs/fidesops/docs/postman/Fidesops.postman_collection.json b/docs/fidesops/docs/postman/Fidesops.postman_collection.json index b5f8c5cfd..47425d4cf 100644 --- a/docs/fidesops/docs/postman/Fidesops.postman_collection.json +++ b/docs/fidesops/docs/postman/Fidesops.postman_collection.json @@ -1,6 +1,6 @@ { "info": { - "_postman_id": "645bae7d-d9af-4b08-84cf-b98d9ae26014", + "_postman_id": "8d7f46be-8de5-45a5-88e3-a4860feb1b42", "name": "Fidesops", "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json", "_exporter_id": "1984786" @@ -2387,6 +2387,124 @@ } ] }, + { + "name": "Email ConnectionConfig", + "item": [ + { + "name": "Create/Update Connection Configs: BigQuery Copy", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "[\n {\"name\": \"Email ConnectionConfig\",\n \"key\": \"{{email_connection_config_key}}\",\n \"connection_type\": \"email\",\n \"access\": \"read\",\n \"email_service_type\": \"mailgun\"\n}]", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/connection/", + "host": [ + "{{host}}" + ], + "path": [ + "connection", + "" + ] + } + }, + "response": [] + }, + { + "name": "Update Connection Secrets: BigQuery Copy", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PUT", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"to_email\": \"customer-1@example.com\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/connection/{{email_connection_config_key}}/secret", + "host": [ + "{{host}}" + ], + "path": [ + "connection", + "{{email_connection_config_key}}", + "secret" + ] + } + }, + "response": [] + }, + { + "name": "Create/Update Email Dataset", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "[\n {\n \"fides_key\":\"email_dataset\",\n \"name\":\"An example of a dataset not automatically accessible\",\n \"description\":\"Example of a email dataset with a collection waiting on postgres input\",\n \"collections\":[\n {\n \"name\":\"daycare_customer\",\n \"fields\":[\n {\n \"name\":\"id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"primary_key\":true\n }\n },\n {\n \"name\":\"customer_id\",\n \"data_categories\":[\n \"user\"\n ],\n \"fidesops_meta\":{\n \"references\":[\n {\n \"dataset\":\"postgres_example\",\n \"field\":\"customer.id\",\n \"direction\":\"from\"\n }\n ]\n }\n }\n ]\n },\n {\n \"name\":\"children\",\n \"fields\":[\n {\n \"name\":\"id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"primary_key\":true\n }\n },\n {\n \"name\":\"first_name\",\n \"data_categories\":[\n \"user.childrens\"\n ]\n },\n {\n \"name\":\"last_name\",\n \"data_categories\":[\n \"user.childrens\"\n ]\n },\n {\n \"name\":\"birthday\",\n \"data_categories\":[\n \"user.childrens\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"parent_id\",\n \"data_categories\":[\n \"user\"\n ],\n \"fidesops_meta\":{\n \"references\":[\n {\n \"dataset\":\"email_dataset\",\n \"field\":\"daycare_customer.id\",\n \"direction\":\"from\"\n }\n ]\n }\n }\n ]\n }\n ]\n }\n]", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/connection/{{email_connection_config_key}}/dataset", + "host": [ + "{{host}}" + ], + "path": [ + "connection", + "{{email_connection_config_key}}", + "dataset" + ] + } + }, + "response": [] + } + ] + }, { "name": "SaaS", "item": [ @@ -3971,7 +4089,7 @@ ] }, { - "name": "Email Config", + "name": "Primary Email Config", "item": [ { "name": "Post Email Config", @@ -4431,6 +4549,21 @@ "key": "saas_connector_type", "value": "mailchimp", "type": "string" + }, + { + "key": "email_connection_config_key", + "value": "email_connection_config_key", + "type": "string" + }, + { + "key": "mailgun_domain", + "value": "", + "type": "string" + }, + { + "key": "mailgun_api_key", + "value": "", + "type": "string" } ] } \ No newline at end of file diff --git a/src/fidesops/ops/api/v1/endpoints/connection_type_endpoints.py b/src/fidesops/ops/api/v1/endpoints/connection_type_endpoints.py index 9e88504a2..3c333ae68 100644 --- a/src/fidesops/ops/api/v1/endpoints/connection_type_endpoints.py +++ b/src/fidesops/ops/api/v1/endpoints/connection_type_endpoints.py @@ -49,6 +49,7 @@ def is_match(elem: str) -> bool: ConnectionType.saas, ConnectionType.https, ConnectionType.manual, + ConnectionType.email, ] and is_match(conn_type.value) ] diff --git a/src/fidesops/ops/migrations/versions/c2f7a29c4780_email_connection_config.py b/src/fidesops/ops/migrations/versions/c2f7a29c4780_email_connection_config.py new file mode 100644 index 000000000..e7bc39994 --- /dev/null +++ b/src/fidesops/ops/migrations/versions/c2f7a29c4780_email_connection_config.py @@ -0,0 +1,41 @@ +"""email_connection_config +Revision ID: c2f7a29c4780 +Revises: 97801300fedd +Create Date: 2022-08-24 14:02:25.096312 +""" +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "c2f7a29c4780" +down_revision = "97801300fedd" +branch_labels = None +depends_on = None + + +def upgrade(): + op.execute("alter type connectiontype rename to connectiontype_old") + op.execute( + "create type connectiontype as enum('postgres', 'mongodb', 'mysql', 'https', 'snowflake', 'redshift', 'mssql', 'mariadb', 'bigquery', 'saas', 'manual', 'email')" + ) + op.execute( + ( + "alter table connectionconfig alter column connection_type type connectiontype using " + "connection_type::text::connectiontype" + ) + ) + op.execute("drop type connectiontype_old") + + +def downgrade(): + op.execute("alter type connectiontype rename to connectiontype_old") + op.execute( + "create type connectiontype as enum('postgres', 'mongodb', 'mysql', 'https', 'snowflake', 'redshift', 'mssql', 'mariadb', 'bigquery', 'saas', 'manual')" + ) + op.execute( + ( + "alter table connectionconfig alter column connection_type type connectiontype using " + "connection_type::text::connectiontype" + ) + ) + op.execute("drop type connectiontype_old") diff --git a/src/fidesops/ops/models/connectionconfig.py b/src/fidesops/ops/models/connectionconfig.py index 094de5e92..e9576f7e7 100644 --- a/src/fidesops/ops/models/connectionconfig.py +++ b/src/fidesops/ops/models/connectionconfig.py @@ -45,6 +45,7 @@ class ConnectionType(enum.Enum): mariadb = "mariadb" bigquery = "bigquery" manual = "manual" + email = "email" class AccessLevel(enum.Enum): diff --git a/src/fidesops/ops/schemas/connection_configuration/__init__.py b/src/fidesops/ops/schemas/connection_configuration/__init__.py index aef9c381b..bbac017e1 100644 --- a/src/fidesops/ops/schemas/connection_configuration/__init__.py +++ b/src/fidesops/ops/schemas/connection_configuration/__init__.py @@ -8,6 +8,10 @@ BigQueryDocsSchema, BigQuerySchema, ) +from fidesops.ops.schemas.connection_configuration.connection_secrets_email import ( + EmailDocsSchema, + EmailSchema, +) from fidesops.ops.schemas.connection_configuration.connection_secrets_mariadb import ( MariaDBDocsSchema, MariaDBSchema, @@ -56,6 +60,7 @@ ConnectionType.mariadb.value: MariaDBSchema, ConnectionType.bigquery.value: BigQuerySchema, ConnectionType.saas.value: SaaSSchema, + ConnectionType.email.value: EmailSchema, } @@ -95,4 +100,5 @@ def get_connection_secrets_validator( MariaDBDocsSchema, BigQueryDocsSchema, SaaSSchema, + EmailDocsSchema, ] diff --git a/src/fidesops/ops/schemas/connection_configuration/connection_secrets_email.py b/src/fidesops/ops/schemas/connection_configuration/connection_secrets_email.py new file mode 100644 index 000000000..6cf065a74 --- /dev/null +++ b/src/fidesops/ops/schemas/connection_configuration/connection_secrets_email.py @@ -0,0 +1,19 @@ +from typing import List, Optional + +from fidesops.ops.schemas.base_class import NoValidationSchema +from fidesops.ops.schemas.connection_configuration.connection_secrets import ( + ConnectionConfigSecretsSchema, +) + + +class EmailSchema(ConnectionConfigSecretsSchema): + """Schema to validate the secrets needed for the EmailConnector""" + + to_email: str + test_email: Optional[str] # Email to send a connection test email + + _required_components: List[str] = ["to_email"] + + +class EmailDocsSchema(EmailSchema, NoValidationSchema): + """EmailDocsSchema Secrets Schema for API Docs""" diff --git a/src/fidesops/ops/service/connectors/__init__.py b/src/fidesops/ops/service/connectors/__init__.py index 155e0fb1b..cca99c45b 100644 --- a/src/fidesops/ops/service/connectors/__init__.py +++ b/src/fidesops/ops/service/connectors/__init__.py @@ -2,6 +2,7 @@ from fidesops.ops.models.connectionconfig import ConnectionConfig, ConnectionType from fidesops.ops.service.connectors.base_connector import BaseConnector +from fidesops.ops.service.connectors.email_connector import EmailConnector from fidesops.ops.service.connectors.http_connector import HTTPSConnector from fidesops.ops.service.connectors.manual_connector import ManualConnector from fidesops.ops.service.connectors.mongodb_connector import MongoDBConnector @@ -28,6 +29,7 @@ ConnectionType.mariadb.value: MariaDBConnector, ConnectionType.bigquery.value: BigQueryConnector, ConnectionType.manual.value: ManualConnector, + ConnectionType.email.value: EmailConnector, } diff --git a/src/fidesops/ops/service/connectors/email_connector.py b/src/fidesops/ops/service/connectors/email_connector.py new file mode 100644 index 000000000..81675d867 --- /dev/null +++ b/src/fidesops/ops/service/connectors/email_connector.py @@ -0,0 +1,50 @@ +import logging +from typing import Any, Dict, List, Optional + +from fidesops.ops.graph.traversal import TraversalNode +from fidesops.ops.models.connectionconfig import ConnectionTestStatus +from fidesops.ops.models.policy import Policy +from fidesops.ops.models.privacy_request import PrivacyRequest +from fidesops.ops.service.connectors.base_connector import BaseConnector +from fidesops.ops.service.connectors.query_config import ManualQueryConfig +from fidesops.ops.util.collection_util import Row + +logger = logging.getLogger(__name__) + + +class EmailConnector(BaseConnector[None]): + def query_config(self, node: TraversalNode) -> ManualQueryConfig: + """ + Stub + """ + + def create_client(self) -> None: + """Stub""" + + def close(self) -> None: + """Stub""" + + def test_connection(self) -> Optional[ConnectionTestStatus]: + """ + Override to skip connection test for now + """ + return ConnectionTestStatus.skipped + + def retrieve_data( # type: ignore + self, + node: TraversalNode, + policy: Policy, + privacy_request: PrivacyRequest, + input_data: Dict[str, List[Any]], + ) -> Optional[List[Row]]: + """Access requests are not supported at this time.""" + return [] + + def mask_data( # type: ignore + self, + node: TraversalNode, + policy: Policy, + privacy_request: PrivacyRequest, + rows: List[Row], + ) -> Optional[int]: + """Stub""" diff --git a/src/fidesops/ops/task/task_resources.py b/src/fidesops/ops/task/task_resources.py index f0144df48..cf44a8f3c 100644 --- a/src/fidesops/ops/task/task_resources.py +++ b/src/fidesops/ops/task/task_resources.py @@ -16,6 +16,7 @@ from fidesops.ops.service.connectors import ( BaseConnector, BigQueryConnector, + EmailConnector, ManualConnector, MariaDBConnector, MicrosoftSQLServerConnector, @@ -72,6 +73,8 @@ def build_connector( # pylint: disable=R0911 return SaaSConnector(connection_config) if connection_config.connection_type == ConnectionType.manual: return ManualConnector(connection_config) + if connection_config.connection_type == ConnectionType.email: + return EmailConnector(connection_config) raise NotImplementedError( f"No connector available for {connection_config.connection_type}" ) diff --git a/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py b/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py index 8b7cedd15..28c300575 100644 --- a/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py @@ -241,6 +241,12 @@ def test_patch_connections_bulk_update( "access": "write", "description": "Backup snowflake db", }, + { + "key": "email_connector", + "name": "Third Party Email Connector", + "connection_type": "email", + "access": "write", + }, ] response = api_client.patch( @@ -250,7 +256,7 @@ def test_patch_connections_bulk_update( assert 200 == response.status_code response_body = json.loads(response.text) assert len(response_body) == 2 - assert len(response_body["succeeded"]) == 8 + assert len(response_body["succeeded"]) == 9 assert len(response_body["failed"]) == 0 postgres_connection = response_body["succeeded"][0] @@ -325,6 +331,15 @@ def test_patch_connections_bulk_update( assert snowflake_resource.description == "Backup snowflake db" assert "secrets" not in snowflake_connection + email_connection = response_body["succeeded"][8] + assert email_connection["access"] == "write" + assert email_connection["updated_at"] is not None + email_resource = ( + db.query(ConnectionConfig).filter_by(key="email_connector").first() + ) + assert email_resource.access.value == "write" + assert "secrets" not in email_connection + postgres_resource.delete(db) mongo_resource.delete(db) redshift_resource.delete(db) @@ -333,6 +348,7 @@ def test_patch_connections_bulk_update( mysql_resource.delete(db) mssql_resource.delete(db) bigquery_resource.delete(db) + email_resource.delete(db) @mock.patch("fideslib.db.base_class.OrmWrappedFidesBase.create_or_update") def test_patch_connections_failed_response( @@ -1131,3 +1147,37 @@ def test_put_saas_example_connection_config_secrets_missing_saas_config( body["detail"] == f"A SaaS config to validate the secrets is unavailable for this connection config, please add one via {SAAS_CONFIG}" ) + + def test_put_email_connection_config_secrets( + self, + api_client: TestClient, + db: Session, + generate_auth_header, + email_connection_config, + url, + ) -> None: + auth_header = generate_auth_header(scopes=[CONNECTION_CREATE_OR_UPDATE]) + payload = {"url": None, "to_email": "test@example.com"} + url = f"{V1_URL_PREFIX}{CONNECTIONS}/{email_connection_config.key}/secret" + + resp = api_client.put( + url, + headers=auth_header, + json=payload, + ) + + assert resp.status_code == 200 + body = json.loads(resp.text) + assert ( + body["msg"] + == f"Secrets updated for ConnectionConfig with key: {email_connection_config.key}." + ) + assert body["test_status"] == "skipped" "" + db.refresh(email_connection_config) + assert email_connection_config.secrets == { + "to_email": "test@example.com", + "url": None, + "test_email": None, + } + assert email_connection_config.last_test_timestamp is None + assert email_connection_config.last_test_succeeded is None diff --git a/tests/ops/api/v1/endpoints/test_dataset_endpoints.py b/tests/ops/api/v1/endpoints/test_dataset_endpoints.py index 5aa1ec049..d785d0287 100644 --- a/tests/ops/api/v1/endpoints/test_dataset_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_dataset_endpoints.py @@ -53,6 +53,8 @@ def test_example_datasets(example_datasets): assert len(example_datasets[6]["collections"]) == 11 assert example_datasets[7]["fides_key"] == "bigquery_example_test_dataset" assert len(example_datasets[7]["collections"]) == 11 + assert example_datasets[9]["fides_key"] == "email_dataset" + assert len(example_datasets[9]["collections"]) == 2 class TestValidateDataset: @@ -481,7 +483,7 @@ def test_patch_datasets_bulk_create( assert response.status_code == 200 response_body = json.loads(response.text) - assert len(response_body["succeeded"]) == 9 + assert len(response_body["succeeded"]) == 10 assert len(response_body["failed"]) == 0 # Confirm that postgres dataset matches the values we provided @@ -600,7 +602,7 @@ def test_patch_datasets_bulk_update( assert response.status_code == 200 response_body = json.loads(response.text) - assert len(response_body["succeeded"]) == 9 + assert len(response_body["succeeded"]) == 10 assert len(response_body["failed"]) == 0 # test postgres @@ -819,7 +821,7 @@ def test_patch_datasets_failed_response( assert response.status_code == 200 # Returns 200 regardless response_body = json.loads(response.text) assert len(response_body["succeeded"]) == 0 - assert len(response_body["failed"]) == 9 + assert len(response_body["failed"]) == 10 for failed_response in response_body["failed"]: assert "Dataset create/update failed" in failed_response["message"] diff --git a/tests/ops/conftest.py b/tests/ops/conftest.py index 9196e1ee0..9deae1600 100644 --- a/tests/ops/conftest.py +++ b/tests/ops/conftest.py @@ -29,6 +29,7 @@ from .fixtures.application_fixtures import * from .fixtures.bigquery_fixtures import * +from .fixtures.email_fixtures import * from .fixtures.integration_fixtures import * from .fixtures.manual_fixtures import * from .fixtures.mariadb_fixtures import * diff --git a/tests/ops/fixtures/application_fixtures.py b/tests/ops/fixtures/application_fixtures.py index b31e82793..5f57e7b47 100644 --- a/tests/ops/fixtures/application_fixtures.py +++ b/tests/ops/fixtures/application_fixtures.py @@ -1060,6 +1060,7 @@ def example_datasets() -> List[Dict]: "data/dataset/mariadb_example_test_dataset.yml", "data/dataset/bigquery_example_test_dataset.yml", "data/dataset/manual_dataset.yml", + "data/dataset/email_dataset.yml", ] for filename in example_filenames: example_datasets += load_dataset(filename) diff --git a/tests/ops/fixtures/email_fixtures.py b/tests/ops/fixtures/email_fixtures.py new file mode 100644 index 000000000..49502b3ad --- /dev/null +++ b/tests/ops/fixtures/email_fixtures.py @@ -0,0 +1,51 @@ +from typing import Dict, Generator, List +from uuid import uuid4 + +import pytest +from sqlalchemy.orm import Session + +from fidesops.ops.models.connectionconfig import ( + AccessLevel, + ConnectionConfig, + ConnectionType, +) +from fidesops.ops.models.datasetconfig import DatasetConfig + + +@pytest.fixture(scope="function") +def email_connection_config(db: Session) -> Generator: + name = str(uuid4()) + connection_config = ConnectionConfig.create( + db=db, + data={ + "name": name, + "key": "my_email_connection_config", + "connection_type": ConnectionType.email, + "access": AccessLevel.read, + }, + ) + yield connection_config + connection_config.delete(db) + + +@pytest.fixture(scope="function") +def email_dataset_config( + email_connection_config: ConnectionConfig, + db: Session, + example_datasets: List[Dict], +) -> Generator: + email_dataset = example_datasets[9] + fides_key = email_dataset["fides_key"] + email_connection_config.name = fides_key + email_connection_config.key = fides_key + email_connection_config.save(db=db) + dataset = DatasetConfig.create( + db=db, + data={ + "connection_config_id": email_connection_config.id, + "fides_key": fides_key, + "dataset": email_dataset, + }, + ) + yield dataset + dataset.delete(db=db)