Manual Webhook Followup: UI Sync [#1326]

CHANGELOG.md

@@ -51,7 +51,7 @@ The types of changes are:
 * Adds ability to send email notification upon privacy request receipt [#1303](https://github.com/ethyca/fidesops/pull/1303)
 * Utility to update SaaS config instances based on template updates [#1307](https://github.com/ethyca/fidesops/pull/1307)
 * Added generic request sorting button [#1320](https://github.com/ethyca/fidesops/pull/1320)
-
+* Manual webhook test functionality (#1323)[https://github.com/ethyca/fidesops/pull/1323/] 
 
 ### Docs
 

src/fidesops/ops/models/manual_webhook.py

@@ -3,7 +3,7 @@
 from fideslib.db.base_class import Base
 from fideslib.schemas.base_class import BaseSchema
 from pydantic import create_model
-from sqlalchemy import Column, ForeignKey, String
+from sqlalchemy import Column, ForeignKey, String, text
 from sqlalchemy.dialects.postgresql import JSONB
 from sqlalchemy.ext.mutable import MutableList
 from sqlalchemy.orm import Session, relationship
@@ -63,12 +63,14 @@ def empty_fields_dict(self) -> Dict[str, None]:
 
     @classmethod
     def get_enabled(cls, db: Session) -> List["AccessManualWebhook"]:
-        """Get all enabled access manual webhooks"""
+        """Get all enabled access manual webhooks with fields"""
         return (
             db.query(cls)
             .filter(
                 AccessManualWebhook.connection_config_id == ConnectionConfig.id,
                 ConnectionConfig.disabled.is_(False),
+                AccessManualWebhook.fields != text("'null'"),
+                AccessManualWebhook.fields != "[]",
             )
             .all()
         )

src/fidesops/ops/schemas/manual_webhook_schemas.py

@@ -9,18 +9,24 @@
 )
 
 
-class ManualWebhookFieldType(ConstrainedStr):
+class PIIFieldType(ConstrainedStr):
     """Using ConstrainedStr instead of constr to keep mypy happy"""
 
     min_length = 1
     max_length = 200
 
 
+class DSRLabelFieldType(ConstrainedStr):
+    """Using ConstrainedStr instead of constr to keep mypy happy"""
+
+    max_length = 200
+
+
 class ManualWebhookField(BaseSchema):
     """Schema to describe the attributes on a manual webhook field"""
 
-    pii_field: ManualWebhookFieldType
-    dsr_package_label: Optional[ManualWebhookFieldType] = None
+    pii_field: PIIFieldType
+    dsr_package_label: Optional[DSRLabelFieldType] = None
 
     class Config:
         orm_mode = True

src/fidesops/ops/service/connectors/__init__.py

@@ -5,6 +5,9 @@
 from fidesops.ops.service.connectors.email_connector import EmailConnector
 from fidesops.ops.service.connectors.http_connector import HTTPSConnector
 from fidesops.ops.service.connectors.manual_connector import ManualConnector
+from fidesops.ops.service.connectors.manual_webhook_connector import (
+    ManualWebhookConnector,
+)
 from fidesops.ops.service.connectors.mongodb_connector import MongoDBConnector
 from fidesops.ops.service.connectors.saas_connector import SaaSConnector
 from fidesops.ops.service.connectors.sql_connector import (
@@ -30,6 +33,7 @@
     ConnectionType.bigquery.value: BigQueryConnector,
     ConnectionType.manual.value: ManualConnector,
     ConnectionType.email.value: EmailConnector,
+    ConnectionType.manual_webhook.value: ManualWebhookConnector,
 }
 
 

src/fidesops/ops/service/connectors/manual_webhook_connector.py

@@ -0,0 +1,60 @@
+import logging
+from typing import Any, Dict, List, Optional
+
+from fidesops.ops.graph.traversal import TraversalNode
+from fidesops.ops.models.connectionconfig import ConnectionConfig, ConnectionTestStatus
+from fidesops.ops.models.policy import Policy
+from fidesops.ops.models.privacy_request import PrivacyRequest
+from fidesops.ops.service.connectors.base_connector import BaseConnector
+from fidesops.ops.util.collection_util import Row
+
+logger = logging.getLogger(__name__)
+
+
+class ManualWebhookConnector(BaseConnector[None]):
+    def query_config(self, node: TraversalNode) -> None:  # type: ignore
+        """
+        Not applicable for this connector type. Manual Webhooks are not run as part of the traversal.
+        There will not be a node associated with the ManualWebhook.
+        """
+        return None
+
+    def create_client(self) -> None:
+        """Not needed because this connector involves a human performing some lookup step"""
+        return None
+
+    def close(self) -> None:
+        """Not applicable for this connector type."""
+        return None
+
+    def test_connection(self) -> ConnectionTestStatus:
+        """Very simple checks to verify that a ManualWebhook configuration exists"""
+        connection_config: ConnectionConfig = self.configuration
+        if not connection_config.access_manual_webhook:
+            return ConnectionTestStatus.failed
+        if not connection_config.access_manual_webhook.fields:
+            return ConnectionTestStatus.failed
+        return ConnectionTestStatus.succeeded
+
+    def retrieve_data(  # type: ignore
+        self,
+        node: TraversalNode,
+        policy: Policy,
+        privacy_request: PrivacyRequest,
+        input_data: Dict[str, List[Any]],
+    ) -> Optional[List[Row]]:
+        """
+        Not applicable for a manual webhook.  Manual webhooks are not called as part of the traversal.
+        """
+
+    def mask_data(  # type: ignore
+        self,
+        node: TraversalNode,
+        policy: Policy,
+        privacy_request: PrivacyRequest,
+        rows: List[Row],
+        input_data: Dict[str, List[Any]],
+    ) -> Optional[int]:
+        """
+        Not applicable for a manual webhook.  Manual webhooks are not called as part of the traversal.
+        """

tests/ops/api/v1/endpoints/test_manual_webhooks.py

@@ -3,6 +3,7 @@
 from starlette.testclient import TestClient
 
 from fidesops.ops.api.v1.scope_registry import (
+    CONNECTION_READ,
     STORAGE_READ,
     WEBHOOK_CREATE_OR_UPDATE,
     WEBHOOK_DELETE,
@@ -11,6 +12,7 @@
 from fidesops.ops.api.v1.urn_registry import (
     ACCESS_MANUAL_WEBHOOK,
     ACCESS_MANUAL_WEBHOOKS,
+    CONNECTION_TEST,
     V1_URL_PREFIX,
 )
 from fidesops.ops.models.manual_webhook import AccessManualWebhook
@@ -181,6 +183,23 @@ def test_post_access_manual_webhook_fields_empty_string(
             == "ensure this value has at least 1 characters"
         )
 
+    def test_post_access_manual_webhook_dsr_package_labels_empty_string(
+        self, db, api_client, url, generate_auth_header
+    ):
+        payload = {
+            "fields": [
+                {"pii_field": "first_name", "dsr_package_label": "First Name"},
+                {"pii_field": "last_name", "dsr_package_label": ""},
+            ]
+        }
+        auth_header = generate_auth_header([WEBHOOK_CREATE_OR_UPDATE])
+        response = api_client.post(url, headers=auth_header, json=payload)
+        assert response.status_code == 201
+        assert response.json()["fields"] == [
+            {"pii_field": "first_name", "dsr_package_label": "First Name"},
+            {"pii_field": "last_name", "dsr_package_label": "last_name"},
+        ]
+
     def test_post_access_manual_webhook_wrong_connection_config_type(
         self, connection_config, payload, generate_auth_header, api_client
     ):
@@ -420,3 +439,74 @@ def test_get_manual_webhooks(
         assert connection_config_details["created_at"] is not None
         assert connection_config_details["updated_at"] is not None
         assert "secrets" not in connection_config_details
+
+
+class TestManualWebhookTest:
+    @pytest.fixture(scope="function")
+    def url(self, integration_manual_webhook_config) -> str:
+        return V1_URL_PREFIX + CONNECTION_TEST.format(
+            connection_key=integration_manual_webhook_config.key
+        )
+
+    def test_connection_test_manual_webhook_not_authenticated(
+        self, api_client: TestClient, url
+    ):
+        response = api_client.get(url, headers={})
+        assert 401 == response.status_code
+
+    def test_connection_test_manual_webhook_wrong_scopes(
+        self, api_client: TestClient, url, generate_auth_header
+    ):
+        auth_header = generate_auth_header([STORAGE_READ])
+
+        response = api_client.get(url, headers=auth_header)
+        assert 403 == response.status_code
+
+    def test_connection_test_manual_webhook_no_webhook_resource(
+        self,
+        api_client: TestClient,
+        db,
+        url,
+        generate_auth_header,
+        integration_manual_webhook_config,
+    ):
+        auth_header = generate_auth_header([CONNECTION_READ])
+
+        response = api_client.get(url, headers=auth_header)
+        assert 200 == response.status_code
+        assert response.json()["test_status"] == "failed"
+        assert response.status_code == 200
+
+    def test_connection_test_manual_webhook_no_webhook_fields(
+        self,
+        api_client: TestClient,
+        db,
+        url,
+        generate_auth_header,
+        integration_manual_webhook_config,
+        access_manual_webhook,
+    ):
+        auth_header = generate_auth_header([CONNECTION_READ])
+        access_manual_webhook.fields = None
+        access_manual_webhook.save(db)
+
+        response = api_client.get(url, headers=auth_header)
+        assert 200 == response.status_code
+        assert response.json()["test_status"] == "failed"
+        assert response.status_code == 200
+
+    def test_connection_test_manual_webhook(
+        self,
+        api_client: TestClient,
+        db,
+        url,
+        generate_auth_header,
+        access_manual_webhook,
+        integration_manual_webhook_config,
+    ):
+        auth_header = generate_auth_header([CONNECTION_READ])
+
+        response = api_client.get(url, headers=auth_header)
+        assert 200 == response.status_code
+        assert response.json()["test_status"] == "succeeded"
+        assert response.status_code == 200

tests/ops/models/test_connectionconfig.py

@@ -154,14 +154,3 @@ def test_connection_type_human_readable(self):
     def test_connection_type_human_readable_invalid(self):
         with pytest.raises(ValueError):
             ConnectionType("nonmapped_type").human_readable()
-
-    def test_manual_webhook(
-        self, integration_manual_webhook_config, access_manual_webhook
-    ):
-        assert (
-            integration_manual_webhook_config.access_manual_webhook
-            == access_manual_webhook
-        )
-        assert (
-            access_manual_webhook.connection_config == integration_manual_webhook_config
-        )

tests/ops/models/test_manual_webhook.py

@@ -0,0 +1,41 @@
+from fidesops.ops.models.manual_webhook import AccessManualWebhook
+
+
+class TestManualWebhook:
+    def test_manual_webhook(
+        self, integration_manual_webhook_config, access_manual_webhook
+    ):
+        assert (
+            integration_manual_webhook_config.access_manual_webhook
+            == access_manual_webhook
+        )
+        assert (
+            access_manual_webhook.connection_config == integration_manual_webhook_config
+        )
+
+    def test_get_enabled_no_enabled_webhooks(self, db):
+        assert AccessManualWebhook.get_enabled(db) == []
+
+    def test_get_enabled_webhooks(self, db, access_manual_webhook):
+        assert AccessManualWebhook.get_enabled(db) == [access_manual_webhook]
+
+    def test_get_enabled_webhooks_connection_config_disabled(
+        self, db, access_manual_webhook, integration_manual_webhook_config
+    ):
+        integration_manual_webhook_config.disabled = True
+        integration_manual_webhook_config.save(db)
+        assert AccessManualWebhook.get_enabled(db) == []
+
+    def test_get_enabled_webhooks_connection_config_fields_is_none(
+        self, db, access_manual_webhook
+    ):
+        access_manual_webhook.fields = None
+        access_manual_webhook.save(db)
+        assert AccessManualWebhook.get_enabled(db) == []
+
+    def test_get_enabled_webhooks_connection_config_fields_is_empty(
+        self, db, access_manual_webhook
+    ):
+        access_manual_webhook.fields = []
+        access_manual_webhook.save(db)
+        assert AccessManualWebhook.get_enabled(db) == []