Start a new "email" ConnectionConfig type [#1134]

CHANGELOG.md

@@ -19,6 +19,10 @@ The types of changes are:
 
 ## [Unreleased](https://github.com/ethyca/fidesops/compare/1.7.1...main)
 
+### Added
+
+* Foundations for a new email connector type [#1142] https://github.com/ethyca/fidesops/pull/1142
+
 ## [1.7.1](https://github.com/ethyca/fidesops/compare/1.7.0...1.7.1)
 
 ### Added

data/dataset/email_dataset.yml

@@ -0,0 +1,39 @@
+dataset:
+  - fides_key: email_dataset
+    name: Dataset not accessible automatically
+    description: Example of a email dataset with a collection waiting on postgres input
+    collections:
+      - name: daycare_customer
+        fields:
+          - name: id
+            data_categories: [system.operations]
+            fidesops_meta:
+              primary_key: true
+          - name: customer_id
+            data_categories: [user]
+            fidesops_meta:
+              references:
+                - dataset: postgres_example_test_dataset
+                  field: customer.id
+                  direction: from
+      - name: children
+        fields:
+          - name: id
+            data_categories: [system.operations]
+            fidesops_meta:
+              primary_key: true
+          - name: first_name
+            data_categories: [user.childrens]
+          - name: last_name
+            data_categories: [user.childrens]
+          - name: birthday
+            data_categories: [user.childrens]
+            fidesops_meta:
+              data_type: string
+          - name: parent_id
+            data_categories: [user]
+            fidesops_meta:
+              references:
+                - dataset: email_dataset
+                  field: daycare_customer.id
+                  direction: from

docs/fidesops/docs/postman/Fidesops.postman_collection.json

@@ -1,6 +1,6 @@
 {
 	"info": {
-		"_postman_id": "645bae7d-d9af-4b08-84cf-b98d9ae26014",
+		"_postman_id": "8d7f46be-8de5-45a5-88e3-a4860feb1b42",
 		"name": "Fidesops",
 		"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json",
 		"_exporter_id": "1984786"
@@ -2387,6 +2387,124 @@
 				}
 			]
 		},
+		{
+			"name": "Email ConnectionConfig",
+			"item": [
+				{
+					"name": "Create/Update Connection Configs: BigQuery Copy",
+					"request": {
+						"auth": {
+							"type": "bearer",
+							"bearer": [
+								{
+									"key": "token",
+									"value": "{{client_token}}",
+									"type": "string"
+								}
+							]
+						},
+						"method": "PATCH",
+						"header": [],
+						"body": {
+							"mode": "raw",
+							"raw": "[\n    {\"name\": \"Email ConnectionConfig\",\n    \"key\": \"{{email_connection_config_key}}\",\n    \"connection_type\": \"email\",\n    \"access\": \"read\",\n    \"email_service_type\": \"mailgun\"\n}]",
+							"options": {
+								"raw": {
+									"language": "json"
+								}
+							}
+						},
+						"url": {
+							"raw": "{{host}}/connection/",
+							"host": [
+								"{{host}}"
+							],
+							"path": [
+								"connection",
+								""
+							]
+						}
+					},
+					"response": []
+				},
+				{
+					"name": "Update Connection Secrets: BigQuery Copy",
+					"request": {
+						"auth": {
+							"type": "bearer",
+							"bearer": [
+								{
+									"key": "token",
+									"value": "{{client_token}}",
+									"type": "string"
+								}
+							]
+						},
+						"method": "PUT",
+						"header": [],
+						"body": {
+							"mode": "raw",
+							"raw": "{\n    \"to_email\": \"[email protected]\"\n}",
+							"options": {
+								"raw": {
+									"language": "json"
+								}
+							}
+						},
+						"url": {
+							"raw": "{{host}}/connection/{{email_connection_config_key}}/secret",
+							"host": [
+								"{{host}}"
+							],
+							"path": [
+								"connection",
+								"{{email_connection_config_key}}",
+								"secret"
+							]
+						}
+					},
+					"response": []
+				},
+				{
+					"name": "Create/Update Email Dataset",
+					"request": {
+						"auth": {
+							"type": "bearer",
+							"bearer": [
+								{
+									"key": "token",
+									"value": "{{client_token}}",
+									"type": "string"
+								}
+							]
+						},
+						"method": "PATCH",
+						"header": [],
+						"body": {
+							"mode": "raw",
+							"raw": "[\n   {\n      \"fides_key\":\"email_dataset\",\n      \"name\":\"An example of a dataset not automatically accessible\",\n      \"description\":\"Example of a email dataset with a collection waiting on postgres input\",\n      \"collections\":[\n         {\n            \"name\":\"daycare_customer\",\n            \"fields\":[\n               {\n                  \"name\":\"id\",\n                  \"data_categories\":[\n                     \"system.operations\"\n                  ],\n                  \"fidesops_meta\":{\n                     \"primary_key\":true\n                  }\n               },\n               {\n                  \"name\":\"customer_id\",\n                  \"data_categories\":[\n                     \"user\"\n                  ],\n                  \"fidesops_meta\":{\n                     \"references\":[\n                        {\n                           \"dataset\":\"postgres_example\",\n                           \"field\":\"customer.id\",\n                           \"direction\":\"from\"\n                        }\n                     ]\n                  }\n               }\n            ]\n         },\n         {\n            \"name\":\"children\",\n            \"fields\":[\n               {\n                  \"name\":\"id\",\n                  \"data_categories\":[\n                     \"system.operations\"\n                  ],\n                  \"fidesops_meta\":{\n                     \"primary_key\":true\n                  }\n               },\n               {\n                  \"name\":\"first_name\",\n                  \"data_categories\":[\n                     \"user.childrens\"\n                  ]\n               },\n               {\n                  \"name\":\"last_name\",\n                  \"data_categories\":[\n                     \"user.childrens\"\n                  ]\n               },\n               {\n                  \"name\":\"birthday\",\n                  \"data_categories\":[\n                     \"user.childrens\"\n                  ],\n                  \"fidesops_meta\":{\n                     \"data_type\":\"string\"\n                  }\n               },\n               {\n                  \"name\":\"parent_id\",\n                  \"data_categories\":[\n                     \"user\"\n                  ],\n                  \"fidesops_meta\":{\n                     \"references\":[\n                        {\n                           \"dataset\":\"email_dataset\",\n                           \"field\":\"daycare_customer.id\",\n                           \"direction\":\"from\"\n                        }\n                     ]\n                  }\n               }\n            ]\n         }\n      ]\n   }\n]",
+							"options": {
+								"raw": {
+									"language": "json"
+								}
+							}
+						},
+						"url": {
+							"raw": "{{host}}/connection/{{email_connection_config_key}}/dataset",
+							"host": [
+								"{{host}}"
+							],
+							"path": [
+								"connection",
+								"{{email_connection_config_key}}",
+								"dataset"
+							]
+						}
+					},
+					"response": []
+				}
+			]
+		},
 		{
 			"name": "SaaS",
 			"item": [
@@ -3971,7 +4089,7 @@
 			]
 		},
 		{
-			"name": "Email Config",
+			"name": "Primary Email Config",
 			"item": [
 				{
 					"name": "Post Email Config",
@@ -4431,6 +4549,21 @@
 			"key": "saas_connector_type",
 			"value": "mailchimp",
 			"type": "string"
+		},
+		{
+			"key": "email_connection_config_key",
+			"value": "email_connection_config_key",
+			"type": "string"
+		},
+		{
+			"key": "mailgun_domain",
+			"value": "",
+			"type": "string"
+		},
+		{
+			"key": "mailgun_api_key",
+			"value": "",
+			"type": "string"
 		}
 	]
 }

src/fidesops/ops/api/v1/endpoints/connection_type_endpoints.py

@@ -49,6 +49,7 @@ def is_match(elem: str) -> bool:
                     ConnectionType.saas,
                     ConnectionType.https,
                     ConnectionType.manual,
+                    ConnectionType.email,
                 ]
                 and is_match(conn_type.value)
             ]

src/fidesops/ops/migrations/versions/c2f7a29c4780_email_connection_config.py

@@ -0,0 +1,41 @@
+"""email_connection_config
+Revision ID: c2f7a29c4780
+Revises: 97801300fedd
+Create Date: 2022-08-24 14:02:25.096312
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "c2f7a29c4780"
+down_revision = "97801300fedd"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.execute("alter type connectiontype rename to connectiontype_old")
+    op.execute(
+        "create type connectiontype as enum('postgres', 'mongodb', 'mysql', 'https', 'snowflake', 'redshift', 'mssql', 'mariadb', 'bigquery', 'saas', 'manual', 'email')"
+    )
+    op.execute(
+        (
+            "alter table connectionconfig alter column connection_type type connectiontype using "
+            "connection_type::text::connectiontype"
+        )
+    )
+    op.execute("drop type connectiontype_old")
+
+
+def downgrade():
+    op.execute("alter type connectiontype rename to connectiontype_old")
+    op.execute(
+        "create type connectiontype as enum('postgres', 'mongodb', 'mysql', 'https', 'snowflake', 'redshift', 'mssql', 'mariadb', 'bigquery', 'saas', 'manual')"
+    )
+    op.execute(
+        (
+            "alter table connectionconfig alter column connection_type type connectiontype using "
+            "connection_type::text::connectiontype"
+        )
+    )
+    op.execute("drop type connectiontype_old")

src/fidesops/ops/models/connectionconfig.py

@@ -45,6 +45,7 @@ class ConnectionType(enum.Enum):
     mariadb = "mariadb"
     bigquery = "bigquery"
     manual = "manual"
+    email = "email"
 
 
 class AccessLevel(enum.Enum):

src/fidesops/ops/schemas/connection_configuration/__init__.py

@@ -8,6 +8,10 @@
     BigQueryDocsSchema,
     BigQuerySchema,
 )
+from fidesops.ops.schemas.connection_configuration.connection_secrets_email import (
+    EmailDocsSchema,
+    EmailSchema,
+)
 from fidesops.ops.schemas.connection_configuration.connection_secrets_mariadb import (
     MariaDBDocsSchema,
     MariaDBSchema,
@@ -56,6 +60,7 @@
     ConnectionType.mariadb.value: MariaDBSchema,
     ConnectionType.bigquery.value: BigQuerySchema,
     ConnectionType.saas.value: SaaSSchema,
+    ConnectionType.email.value: EmailSchema,
 }
 
 
@@ -95,4 +100,5 @@ def get_connection_secrets_validator(
     MariaDBDocsSchema,
     BigQueryDocsSchema,
     SaaSSchema,
+    EmailDocsSchema,
 ]

src/fidesops/ops/schemas/connection_configuration/connection_secrets_email.py

@@ -0,0 +1,19 @@
+from typing import List, Optional
+
+from fidesops.ops.schemas.base_class import NoValidationSchema
+from fidesops.ops.schemas.connection_configuration.connection_secrets import (
+    ConnectionConfigSecretsSchema,
+)
+
+
+class EmailSchema(ConnectionConfigSecretsSchema):
+    """Schema to validate the secrets needed for the EmailConnector"""
+
+    to_email: str
+    test_email: Optional[str]  # Email to send a connection test email
+
+    _required_components: List[str] = ["to_email"]
+
+
+class EmailDocsSchema(EmailSchema, NoValidationSchema):
+    """EmailDocsSchema Secrets Schema for API Docs"""

src/fidesops/ops/service/connectors/__init__.py

@@ -2,6 +2,7 @@
 
 from fidesops.ops.models.connectionconfig import ConnectionConfig, ConnectionType
 from fidesops.ops.service.connectors.base_connector import BaseConnector
+from fidesops.ops.service.connectors.email_connector import EmailConnector
 from fidesops.ops.service.connectors.http_connector import HTTPSConnector
 from fidesops.ops.service.connectors.manual_connector import ManualConnector
 from fidesops.ops.service.connectors.mongodb_connector import MongoDBConnector
@@ -28,6 +29,7 @@
     ConnectionType.mariadb.value: MariaDBConnector,
     ConnectionType.bigquery.value: BigQueryConnector,
     ConnectionType.manual.value: ManualConnector,
+    ConnectionType.email.value: EmailConnector,
 }
 
 

src/fidesops/ops/service/connectors/email_connector.py

@@ -0,0 +1,50 @@
+import logging
+from typing import Any, Dict, List, Optional
+
+from fidesops.ops.graph.traversal import TraversalNode
+from fidesops.ops.models.connectionconfig import ConnectionTestStatus
+from fidesops.ops.models.policy import Policy
+from fidesops.ops.models.privacy_request import PrivacyRequest
+from fidesops.ops.service.connectors.base_connector import BaseConnector
+from fidesops.ops.service.connectors.query_config import ManualQueryConfig
+from fidesops.ops.util.collection_util import Row
+
+logger = logging.getLogger(__name__)
+
+
+class EmailConnector(BaseConnector[None]):
+    def query_config(self, node: TraversalNode) -> ManualQueryConfig:
+        """
+        Stub
+        """
+
+    def create_client(self) -> None:
+        """Stub"""
+
+    def close(self) -> None:
+        """Stub"""
+
+    def test_connection(self) -> Optional[ConnectionTestStatus]:
+        """
+        Override to skip connection test for now
+        """
+        return ConnectionTestStatus.skipped
+
+    def retrieve_data(  # type: ignore
+        self,
+        node: TraversalNode,
+        policy: Policy,
+        privacy_request: PrivacyRequest,
+        input_data: Dict[str, List[Any]],
+    ) -> Optional[List[Row]]:
+        """Access requests are not supported at this time."""
+        return []
+
+    def mask_data(  # type: ignore
+        self,
+        node: TraversalNode,
+        policy: Policy,
+        privacy_request: PrivacyRequest,
+        rows: List[Row],
+    ) -> Optional[int]:
+        """Stub"""