[SaaS Connector] Outreach (Erasures) (#619)

CHANGELOG.md

@@ -26,6 +26,7 @@ The types of changes are:
 * Celery as a dependency for use in the execution layer [#610](https://github.com/ethyca/fidesops/pull/610)
 * Cache and Surface Resume/Restart Instructions [#591](https://github.com/ethyca/fidesops/pull/591)
 * Allow disabling a ConnectionConfig [#637](https://github.com/ethyca/fidesops/pull/637)
+* Erasure support for Outreach connector [#619](https://github.com/ethyca/fidesops/pull/619)
 
 ### Changed
 

data/saas/config/outreach_config.yml

@@ -6,10 +6,13 @@ saas_config:
 
   connector_params:
     - name: domain
+    - name: requester_email
+      description: The email of the Outreach user to associate with each automated compliance request (data_protection_request)
     - name: client_id
     - name: client_secret
     - name: redirect_uri
     - name: page_size
+      description: The number of entries to return per page
 
   client_config:
     protocol: https
@@ -83,6 +86,26 @@ saas_config:
             - name: email
               identity: email
           data_path: data
+        delete:
+          method: POST
+          path: /api/v2/complianceRequests
+          param_values:
+            - name: requester_email
+              connector_param: requester_email
+            - name: email
+              identity: email
+          body: |
+            {
+              "data": {
+                "type": "complianceRequest",
+                "attributes": {
+                  "requester_email": "<requester_email>",
+                  "request_type": "Delete",
+                  "object_type": "Prospect",
+                  "request_object_email": "<email>"
+                }
+              }
+            }
     - name: recipients
       requests:
         read:
@@ -110,3 +133,23 @@ saas_config:
                   identity: email
                 exact: False
                 case_sensitive: False
+        delete:
+          method: POST
+          path: /api/v2/complianceRequests
+          param_values:
+            - name: requester_email
+              connector_param: requester_email
+            - name: email
+              identity: email
+          body: |
+            {
+              "data": {
+                "type": "complianceRequest",
+                "attributes": {
+                  "requester_email": "<requester_email>",
+                  "request_type": "Delete",
+                  "object_type": "Recipient",
+                  "request_object_email": "<email>"
+                }
+              }
+            }

data/saas/dataset/outreach_dataset.yml

@@ -7,6 +7,8 @@ dataset:
         fields:
           - name: id
             data_categories: [system.operations]
+            fidesops_meta:
+              primary_key: True
           - name: attributes
             fields:
               - name: createdAt
@@ -23,6 +25,8 @@ dataset:
         fields:
           - name: id
             data_categories: [system.operations]
+            fidesops_meta:
+              primary_key: True
           - name: attributes
             fields:
               - name: addressCity

saas_config.toml

@@ -48,4 +48,6 @@ domain = ""
 client_id = ""
 client_secret = ""
 redirect_uri = ""
-page_size = ""
+page_size = ""
+requester_email = ""
+identity_email = ""

src/fidesops/schemas/saas/saas_config.py

@@ -169,6 +169,7 @@ class ConnectorParam(BaseModel):
     """Used to define the required parameters for the connector (user-provided and constants)"""
 
     name: str
+    description: Optional[str]
 
 
 class SaaSConfig(BaseModel):

tests/fixtures/saas/outreach_fixtures.py

@@ -3,6 +3,7 @@
 
 import pydash
 import pytest
+import requests
 from fideslib.core.config import load_toml
 from sqlalchemy.orm import Session
 
@@ -13,6 +14,7 @@
     ConnectionType,
 )
 from fidesops.models.datasetconfig import DatasetConfig
+from fidesops.util import cryptographic_util
 from tests.fixtures.application_fixtures import load_dataset
 from tests.fixtures.saas_example_fixtures import load_config
 
@@ -24,6 +26,8 @@ def outreach_secrets():
     return {
         "domain": pydash.get(saas_config, "outreach.domain")
         or os.environ.get("OUTREACH_DOMAIN"),
+        "requester_email": pydash.get(saas_config, "outreach.requester_email")
+        or os.environ.get("OUTREACH_REQUESTER_EMAIL"),
         "client_id": pydash.get(saas_config, "outreach.client_id")
         or os.environ.get("OUTREACH_CLIENT_ID"),
         "client_secret": pydash.get(saas_config, "outreach.client_secret")
@@ -42,6 +46,11 @@ def outreach_identity_email():
     )
 
 
+@pytest.fixture(scope="function")
+def outreach_erasure_identity_email() -> str:
+    return f"{cryptographic_util.generate_secure_random_string(13)}@email.com"
+
+
 @pytest.fixture
 def outreach_config() -> Dict[str, Any]:
     return load_config("data/saas/config/outreach_config.yml")
@@ -92,3 +101,52 @@ def outreach_dataset_config(
     )
     yield dataset
     dataset.delete(db=db)
+
+
+@pytest.fixture(scope="function")
+def outreach_create_erasure_data(
+    outreach_connection_config: ConnectionConfig, outreach_erasure_identity_email: str
+) -> None:
+
+    outreach_secrets = outreach_connection_config.secrets
+    base_url = f"https://{outreach_secrets['domain']}"
+    headers = {
+        "Authorization": f"Bearer {outreach_secrets['access_token']}",
+    }
+
+    # prospect
+    prospect_data = {
+        "data": {
+            "type": "prospect",
+            "attributes": {
+                "emails": [outreach_erasure_identity_email],
+                "firstName": "Ethyca",
+                "lastName": "RTF",
+            },
+        }
+    }
+
+    response = requests.post(
+        url=f"{base_url}/api/v2/prospects",
+        headers=headers,
+        json=prospect_data,
+    )
+    assert response.ok
+
+    # recipient
+    recipient_data = {
+        "data": {
+            "type": "recipient",
+            "attributes": {
+                "recipientType": "to",
+                "value": outreach_erasure_identity_email,
+            },
+        }
+    }
+
+    response = requests.post(
+        url=f"{base_url}/api/v2/recipients",
+        headers=headers,
+        json=recipient_data,
+    )
+    assert response.ok

tests/integration_tests/saas/test_outreach_task.py

@@ -2,11 +2,13 @@
 
 import pytest
 
+from fidesops.core.config import config
 from fidesops.graph.graph import DatasetGraph
 from fidesops.models.privacy_request import PrivacyRequest
 from fidesops.schemas.redis_cache import PrivacyRequestIdentity
 from fidesops.task import graph_task
 from fidesops.task.filter_results import filter_data_categories
+from fidesops.task.graph_task import get_cached_data_for_erasures
 from tests.graph.graph_test_util import assert_rows_match
 
 
@@ -80,3 +82,64 @@ def test_outreach_access_request_task(
     assert set(filtered_results[f"{dataset_name}:recipients"][0].keys()) == {
         "attributes",
     }
+
+
+@pytest.mark.skip(reason="Currently unable to test OAuth2 connectors")
+@pytest.mark.integration_saas
+@pytest.mark.integration_outreach
+def test_outreach_erasure_request_task(
+    db,
+    policy,
+    erasure_policy_string_rewrite,
+    outreach_connection_config,
+    outreach_dataset_config,
+    outreach_erasure_identity_email,
+    outreach_create_erasure_data,
+) -> None:
+    """Full erasure request based on the Outreach SaaS config"""
+    config.execution.MASKING_STRICT = False  # Allow Delete
+
+    privacy_request = PrivacyRequest(
+        id=f"test_outreach_erasure_request_task_{random.randint(0, 1000)}"
+    )
+    identity = PrivacyRequestIdentity(**{"email": outreach_erasure_identity_email})
+    privacy_request.cache_identity(identity)
+
+    dataset_name = outreach_connection_config.get_saas_config().fides_key
+    merged_graph = outreach_dataset_config.get_graph()
+    graph = DatasetGraph(merged_graph)
+
+    v = graph_task.run_access_request(
+        privacy_request,
+        policy,
+        graph,
+        [outreach_connection_config],
+        {"email": outreach_erasure_identity_email},
+    )
+
+    # verify staged data is available for erasure
+    assert_rows_match(
+        v[f"{dataset_name}:prospects"],
+        min_size=1,
+        keys=["type", "id", "attributes", "relationships", "links"],
+    )
+    assert_rows_match(
+        v[f"{dataset_name}:recipients"],
+        min_size=1,
+        keys=["type", "id", "attributes", "links"],
+    )
+
+    x = graph_task.run_erasure(
+        privacy_request,
+        erasure_policy_string_rewrite,
+        graph,
+        [outreach_connection_config],
+        {"email": outreach_erasure_identity_email},
+        get_cached_data_for_erasures(privacy_request.id),
+    )
+
+    # Assert erasure request made to prospects and recipients
+    # cannot verify success immediately as this can take days, weeks to process
+    assert x == {f"{dataset_name}:prospects": 1, f"{dataset_name}:recipients": 1}
+
+    config.execution.MASKING_STRICT = True