Add consent table (#1301)

Co-authored-by: Paul Sanders <[email protected]>

CHANGELOG.md

@@ -46,8 +46,8 @@ The types of changes are:
 * Added erasure endpoints for Shopify connector [#1289](https://github.com/ethyca/fidesops/pull/1289)
 * Adds ability to send email notification upon privacy request completion [#1282](https://github.com/ethyca/fidesops/pull/1282)
 * Enable new manual webhooks in privacy request execution [#1285](https://github.com/ethyca/fidesops/pull/1285)
-
 * Added human readable label to ConnectionType endpoint [#1297](https://github.com/ethyca/fidesops/pull/1297)
+* Add table for consent (#1301)[https://github.com/ethyca/fidesops/pull/1301]
 
 ### Docs
 

src/fidesops/ops/api/v1/endpoints/privacy_request_endpoints.py

@@ -465,11 +465,13 @@ def _filter_privacy_request_queryset(
             identity[0]
             for identity in ProvidedIdentity.filter(
                 db=db,
-                conditions=(ProvidedIdentity.hashed_value == hashed_identity),
+                conditions=(
+                    (ProvidedIdentity.hashed_value == hashed_identity)
+                    & (ProvidedIdentity.privacy_request_id.isnot(None))
+                ),
             ).values(column("privacy_request_id"))
         }
         query = query.filter(PrivacyRequest.id.in_(identities))
-
     # Further restrict all PrivacyRequests by query params
     if request_id:
         query = query.filter(PrivacyRequest.id.ilike(f"{request_id}%"))
@@ -608,7 +610,6 @@ def get_request_status(
     To fetch a single privacy request, use the request_id query param `?request_id=`.
     To see individual execution logs, use the verbose query param `?verbose=True`.
     """
-
     logger.info("Finding all request statuses with pagination params %s", params)
 
     query = db.query(PrivacyRequest)

src/fidesops/ops/migrations/versions/a0e6feb5bdc8_add_consent.py

@@ -0,0 +1,66 @@
+"""Add consent
+
+Revision ID: a0e6feb5bdc8
+Revises: 7a4f4042091e
+Create Date: 2022-09-12 14:46:05.443579
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "a0e6feb5bdc8"
+down_revision = "7a4f4042091e"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "consent",
+        sa.Column("id", sa.String(length=255), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column("provided_identity_id", sa.String(), nullable=True),
+        sa.Column("data_use", sa.String(), nullable=False),
+        sa.Column("data_use_description", sa.String(), nullable=True),
+        sa.Column("opt_in", sa.Boolean(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["provided_identity_id"],
+            ["providedidentity.id"],
+        ),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint("data_use"),
+    )
+    op.create_index(op.f("ix_consent_id"), "consent", ["id"], unique=False)
+    op.alter_column(
+        "providedidentity",
+        "privacy_request_id",
+        existing_type=sa.VARCHAR(),
+        nullable=True,
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column(
+        "providedidentity",
+        "privacy_request_id",
+        existing_type=sa.VARCHAR(),
+        nullable=False,
+    )
+    op.drop_index(op.f("ix_consent_id"), table_name="consent")
+    op.drop_table("consent")
+    # ### end Alembic commands ###

src/fidesops/ops/models/privacy_request.py

@@ -16,7 +16,7 @@
 from fideslib.models.client import ClientDetail
 from fideslib.models.fides_user import FidesUser
 from fideslib.oauth.jwt import generate_jwe
-from sqlalchemy import Column, DateTime
+from sqlalchemy import Boolean, Column, DateTime
 from sqlalchemy import Enum as EnumColumn
 from sqlalchemy import ForeignKey, String
 from sqlalchemy.dialects.postgresql import JSONB
@@ -280,6 +280,7 @@ def persist_identity(self, db: Session, identity: PrivacyRequestIdentity) -> Non
         for key, value in identity_dict.items():
             if value is not None:
                 hashed_value = ProvidedIdentity.hash_value(value)
+
                 ProvidedIdentity.create(
                     db=db,
                     data={
@@ -728,7 +729,6 @@ class ProvidedIdentity(Base):  # pylint: disable=R0904
     privacy_request_id = Column(
         String,
         ForeignKey(PrivacyRequest.id_field_path),
-        nullable=False,
     )
     privacy_request = relationship(
         PrivacyRequest,
@@ -757,6 +757,9 @@ class ProvidedIdentity(Base):  # pylint: disable=R0904
         ),
         nullable=True,
     )  # Type bytea in the db
+    consent = relationship(
+        "Consent", back_populates="provided_identity", cascade="delete, delete-orphan"
+    )
 
     @classmethod
     def hash_value(
@@ -773,6 +776,17 @@ def hash_value(
         return hashed_value
 
 
+class Consent(Base):
+    """The DB ORM model for Consent."""
+
+    provided_identity_id = Column(String, ForeignKey(ProvidedIdentity.id))
+    data_use = Column(String, nullable=False, unique=True)
+    data_use_description = Column(String)
+    opt_in = Column(Boolean, nullable=False)
+
+    provided_identity = relationship(ProvidedIdentity, back_populates="consent")
+
+
 # Unique text to separate a step from a collection address, so we can store two values in one.
 PAUSED_SEPARATOR = "__fidesops_paused_sep__"
 

tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py

@@ -50,7 +50,6 @@
     V1_URL_PREFIX,
 )
 from fidesops.ops.core.config import config
-from fidesops.ops.email_templates import get_email_template
 from fidesops.ops.graph.config import CollectionAddress
 from fidesops.ops.graph.graph import DatasetGraph
 from fidesops.ops.models.datasetconfig import DatasetConfig
@@ -790,7 +789,7 @@ def test_filter_privacy_requests_by_internal_id(
         assert len(resp["items"]) == 1
         assert resp["items"][0]["id"] == privacy_request.id
 
-    def test_filter_privacy_requests_by_identity_exact(
+    def test_filter_privacy_requests_by_identity_no_request_id(
         self,
         db,
         api_client,

tests/ops/models/test_privacy_request.py

@@ -16,8 +16,10 @@
 from fidesops.ops.models.policy import CurrentStep, Policy
 from fidesops.ops.models.privacy_request import (
     CheckpointActionRequired,
+    Consent,
     PrivacyRequest,
     PrivacyRequestStatus,
+    ProvidedIdentity,
     can_run_checkpoint,
 )
 from fidesops.ops.schemas.redis_cache import PrivacyRequestIdentity
@@ -667,3 +669,35 @@ def test_can_run_if_no_saved_checkpoint(self):
             )
             is True
         )
+
+
+def test_consent(db):
+    provided_identity_data = {
+        "privacy_request_id": None,
+        "field_name": "email",
+        "encrypted_value": {"value": "[email protected]"},
+    }
+    provided_identity = ProvidedIdentity.create(db, data=provided_identity_data)
+
+    consent_data_1 = {
+        "provided_identity_id": provided_identity.id,
+        "data_use": "user.biometric_health",
+        "opt_in": True,
+    }
+    consent_1 = Consent.create(db, data=consent_data_1)
+
+    consent_data_2 = {
+        "provided_identity_id": provided_identity.id,
+        "data_use": "user.browsing_history",
+        "opt_in": False,
+    }
+    consent_2 = Consent.create(db, data=consent_data_2)
+    data_uses = [x.data_use for x in provided_identity.consent]
+
+    assert consent_data_1["data_use"] in data_uses
+    assert consent_data_2["data_use"] in data_uses
+
+    provided_identity.delete(db)
+
+    assert Consent.get(db, object_id=consent_1.id) is None
+    assert Consent.get(db, object_id=consent_2.id) is None