From 188c57478c2e7fd4aefb4f08cbdc1e233523aa01 Mon Sep 17 00:00:00 2001 From: Sean Preston Date: Fri, 8 Jul 2022 11:10:21 +0200 Subject: [PATCH 01/27] adds identity fields to PrivacyRequest model --- ...68_add_identity_data_to_privacy_request.py | 34 +++++++++++++++++++ src/fidesops/models/privacy_request.py | 16 +++++++++ 2 files changed, 50 insertions(+) create mode 100644 src/fidesops/migrations/versions/cdaac83e0a68_add_identity_data_to_privacy_request.py diff --git a/src/fidesops/migrations/versions/cdaac83e0a68_add_identity_data_to_privacy_request.py b/src/fidesops/migrations/versions/cdaac83e0a68_add_identity_data_to_privacy_request.py new file mode 100644 index 000000000..1ffbe5fc8 --- /dev/null +++ b/src/fidesops/migrations/versions/cdaac83e0a68_add_identity_data_to_privacy_request.py @@ -0,0 +1,34 @@ +"""add identity data to privacy_request + +Revision ID: cdaac83e0a68 +Revises: fc90277bbcde +Create Date: 2022-07-08 09:02:11.717954 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'cdaac83e0a68' +down_revision = 'fc90277bbcde' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('privacyrequest', sa.Column('identity_email', sa.String(), nullable=True)) + op.add_column('privacyrequest', sa.Column('identity_phone_number', sa.String(), nullable=True)) + op.create_index(op.f('ix_privacyrequest_identity_email'), 'privacyrequest', ['identity_email'], unique=False) + op.create_index(op.f('ix_privacyrequest_identity_phone_number'), 'privacyrequest', ['identity_phone_number'], unique=False) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f('ix_privacyrequest_identity_phone_number'), table_name='privacyrequest') + op.drop_index(op.f('ix_privacyrequest_identity_email'), table_name='privacyrequest') + op.drop_column('privacyrequest', 'identity_phone_number') + op.drop_column('privacyrequest', 'identity_email') + # ### end Alembic commands ### diff --git a/src/fidesops/models/privacy_request.py b/src/fidesops/models/privacy_request.py index 6d94cd9cb..0433cea91 100644 --- a/src/fidesops/models/privacy_request.py +++ b/src/fidesops/models/privacy_request.py @@ -190,6 +190,22 @@ class PrivacyRequest(Base): # pylint: disable=R0904 ) paused_at = Column(DateTime(timezone=True), nullable=True) + # Provided PII fields at request creation, stored to facilitate searching by identity in the UIs + # In the future these can be moved into a JSON field, but for now since there are only two fields + # supported as identities it's simpler for indexing and searching to leave them on the table + identity_email = Column( + String, + index=True, + unique=False, + nullable=True, + ) + identity_phone_number = Column( + String, + index=True, + unique=False, + nullable=True, + ) + @classmethod def create(cls, db: Session, *, data: Dict[str, Any]) -> FidesBase: """ From 18efc9f0e52c43a7a0ca402767d284c3ff5d588d Mon Sep 17 00:00:00 2001 From: Sean Preston Date: Fri, 8 Jul 2022 11:12:13 +0200 Subject: [PATCH 02/27] store identity data inside database --- .../v1/endpoints/privacy_request_endpoints.py | 6 ++++ .../test_privacy_request_endpoints.py | 33 +++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/src/fidesops/api/v1/endpoints/privacy_request_endpoints.py b/src/fidesops/api/v1/endpoints/privacy_request_endpoints.py index fd2007baf..4b36d7fc7 100644 --- a/src/fidesops/api/v1/endpoints/privacy_request_endpoints.py +++ b/src/fidesops/api/v1/endpoints/privacy_request_endpoints.py @@ -176,6 +176,12 @@ def create_privacy_request( if attr is not None: kwargs[field] = attr + if privacy_request_data.identity.email: + kwargs["identity_email"] = privacy_request_data.identity.email + + if privacy_request_data.identity.phone_number: + kwargs["identity_phone_number"] = privacy_request_data.identity.phone_number + try: privacy_request: PrivacyRequest = PrivacyRequest.create(db=db, data=kwargs) diff --git a/tests/api/v1/endpoints/test_privacy_request_endpoints.py b/tests/api/v1/endpoints/test_privacy_request_endpoints.py index 8da31bc52..7438c41aa 100644 --- a/tests/api/v1/endpoints/test_privacy_request_endpoints.py +++ b/tests/api/v1/endpoints/test_privacy_request_endpoints.py @@ -102,6 +102,39 @@ def test_create_privacy_request( pr.delete(db=db) assert run_access_request_mock.called + @mock.patch( + "fidesops.service.privacy_request.request_runner_service.run_privacy_request.delay" + ) + def test_create_privacy_request_stores_identities( + self, + run_access_request_mock, + url, + db, + api_client: TestClient, + policy, + ): + TEST_EMAIL = "test@example.com" + TEST_PHONE_NUMBER = "+1 234 567 8910" + data = [ + { + "requested_at": "2021-08-30T16:09:37.359Z", + "policy_key": policy.key, + "identity": { + "email": TEST_EMAIL, + "phone_number": TEST_PHONE_NUMBER, + }, + } + ] + resp = api_client.post(url, json=data) + assert resp.status_code == 200 + response_data = resp.json()["succeeded"] + assert len(response_data) == 1 + pr = PrivacyRequest.get(db=db, object_id=response_data[0]["id"]) + assert pr.identity_email == TEST_EMAIL + assert pr.identity_phone_number == TEST_PHONE_NUMBER + pr.delete(db=db) + assert run_access_request_mock.called + @mock.patch( "fidesops.service.privacy_request.request_runner_service.run_privacy_request.delay" ) From 13c0de9cd9b5807407dbc360cdc20efeed7eae57 Mon Sep 17 00:00:00 2001 From: Sean Preston Date: Fri, 8 Jul 2022 11:19:52 +0200 Subject: [PATCH 03/27] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b7d34a9f8..265fbf3c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ The types of changes are: * Adds SaaS connection type to SaaS yaml config [748](https://github.com/ethyca/fidesops/pull/748) * Adds endpoint to get available connectors (database and saas) [#768](https://github.com/ethyca/fidesops/pull/768) * Adds endpoint to get the secrets required for different connectors [#795](https://github.com/ethyca/fidesops/pull/795) +* Store provided identity data in the privacy request table [#743](https://github.com/ethyca/fidesops/pull/834) ### Developer Experience * Replace user authentication routes with fideslib routes [#811](https://github.com/ethyca/fidesops/pull/811) From 37cc67ee4e4c17f2c90fb1e174c062af7dc76708 Mon Sep 17 00:00:00 2001 From: Sean Preston Date: Fri, 8 Jul 2022 11:39:44 +0200 Subject: [PATCH 04/27] add identities in test data command --- create_test_data.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/create_test_data.py b/create_test_data.py index 2d6182ee8..3a000134e 100644 --- a/create_test_data.py +++ b/create_test_data.py @@ -186,6 +186,8 @@ def create_test_data(db: orm.Session) -> FidesUser: "origin": f"https://example.com/{status.value}/", "policy_id": policy.id, "client_id": policy.client_id, + "identity_email": "test@example.com", + "identity_phone_number": "+1 234 567 8910", }, ) From 38a6f3acb58b00375001a1db13b71fad18075f35 Mon Sep 17 00:00:00 2001 From: Sean Preston Date: Fri, 8 Jul 2022 11:40:13 +0200 Subject: [PATCH 05/27] store identities provided via the DRP creation endpoint --- .../api/v1/endpoints/drp_endpoints.py | 21 +++++++++++++------ tests/api/v1/endpoints/test_drp_endpoints.py | 13 +++++++++--- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/src/fidesops/api/v1/endpoints/drp_endpoints.py b/src/fidesops/api/v1/endpoints/drp_endpoints.py index ae1abefc7..e592d5aee 100644 --- a/src/fidesops/api/v1/endpoints/drp_endpoints.py +++ b/src/fidesops/api/v1/endpoints/drp_endpoints.py @@ -89,12 +89,6 @@ def create_drp_privacy_request( ) try: - privacy_request: PrivacyRequest = PrivacyRequest.create( - db=db, data=privacy_request_kwargs - ) - - logger.info(f"Decrypting identity for DRP privacy request {privacy_request.id}") - decrypted_identity: DrpIdentity = DrpIdentity( **jwt.decode(data.identity, jwt_key, algorithms=["HS256"]) ) @@ -103,6 +97,21 @@ def create_drp_privacy_request( drp_identity=decrypted_identity ) + if mapped_identity.email: + privacy_request_kwargs["identity_email"] = mapped_identity.email + + if mapped_identity.phone_number: + privacy_request_kwargs[ + "identity_phone_number" + ] = mapped_identity.phone_number + + privacy_request: PrivacyRequest = PrivacyRequest.create( + db=db, + data=privacy_request_kwargs, + ) + + logger.info(f"Decrypting identity for DRP privacy request {privacy_request.id}") + cache_data(privacy_request, policy, mapped_identity, None, data) queue_privacy_request(privacy_request.id) diff --git a/tests/api/v1/endpoints/test_drp_endpoints.py b/tests/api/v1/endpoints/test_drp_endpoints.py index 77d0ccfa3..78684b65b 100644 --- a/tests/api/v1/endpoints/test_drp_endpoints.py +++ b/tests/api/v1/endpoints/test_drp_endpoints.py @@ -43,8 +43,12 @@ def test_create_drp_privacy_request( policy_drp_action, cache, ): - - identity = {"email": "test@example.com"} + TEST_EMAIL = "test@example.com" + TEST_PHONE_NUMBER = "+1 234 567 8910" + identity = { + "email": TEST_EMAIL, + "phone_number": TEST_PHONE_NUMBER, + } encoded_identity: str = jwt.encode( identity, config.security.DRP_JWT_SECRET, algorithm="HS256" ) @@ -84,13 +88,16 @@ def test_create_drp_privacy_request( ) assert ( cache.get(identity_key) - == "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJlbWFpbCI6InRlc3RAZXhhbXBsZS5jb20ifQ.4I8XLWnTYp8oMHjN2ypP3Hpg45DIaGNAEmj1QCYONUI" + == "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJlbWFpbCI6InRlc3RAZXhhbXBsZS5jb20iLCJwaG9uZV9udW1iZXIiOiIrMSAyMzQgNTY3IDg5MTAifQ.kHV4ru6vxQR96Meae31oKIU7mMnTJgt1cnli6GLUBFk" ) fidesops_identity_key = get_identity_cache_key( privacy_request_id=pr.id, identity_attribute="email", ) assert cache.get(fidesops_identity_key) == identity["email"] + assert pr.identity_email == TEST_EMAIL + assert pr.identity_phone_number == TEST_PHONE_NUMBER + pr.delete(db=db) assert run_access_request_mock.called From eeb5721f7a7e7af89cbd91f96f49ac0764d5beda Mon Sep 17 00:00:00 2001 From: Sean Preston Date: Fri, 8 Jul 2022 11:44:43 +0200 Subject: [PATCH 06/27] black + isort --- ...68_add_identity_data_to_privacy_request.py | 39 +++++++++++++------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/src/fidesops/migrations/versions/cdaac83e0a68_add_identity_data_to_privacy_request.py b/src/fidesops/migrations/versions/cdaac83e0a68_add_identity_data_to_privacy_request.py index 1ffbe5fc8..6da3dfbc5 100644 --- a/src/fidesops/migrations/versions/cdaac83e0a68_add_identity_data_to_privacy_request.py +++ b/src/fidesops/migrations/versions/cdaac83e0a68_add_identity_data_to_privacy_request.py @@ -5,30 +5,45 @@ Create Date: 2022-07-08 09:02:11.717954 """ -from alembic import op import sqlalchemy as sa - +from alembic import op # revision identifiers, used by Alembic. -revision = 'cdaac83e0a68' -down_revision = 'fc90277bbcde' +revision = "cdaac83e0a68" +down_revision = "fc90277bbcde" branch_labels = None depends_on = None def upgrade(): # ### commands auto generated by Alembic - please adjust! ### - op.add_column('privacyrequest', sa.Column('identity_email', sa.String(), nullable=True)) - op.add_column('privacyrequest', sa.Column('identity_phone_number', sa.String(), nullable=True)) - op.create_index(op.f('ix_privacyrequest_identity_email'), 'privacyrequest', ['identity_email'], unique=False) - op.create_index(op.f('ix_privacyrequest_identity_phone_number'), 'privacyrequest', ['identity_phone_number'], unique=False) + op.add_column( + "privacyrequest", sa.Column("identity_email", sa.String(), nullable=True) + ) + op.add_column( + "privacyrequest", sa.Column("identity_phone_number", sa.String(), nullable=True) + ) + op.create_index( + op.f("ix_privacyrequest_identity_email"), + "privacyrequest", + ["identity_email"], + unique=False, + ) + op.create_index( + op.f("ix_privacyrequest_identity_phone_number"), + "privacyrequest", + ["identity_phone_number"], + unique=False, + ) # ### end Alembic commands ### def downgrade(): # ### commands auto generated by Alembic - please adjust! ### - op.drop_index(op.f('ix_privacyrequest_identity_phone_number'), table_name='privacyrequest') - op.drop_index(op.f('ix_privacyrequest_identity_email'), table_name='privacyrequest') - op.drop_column('privacyrequest', 'identity_phone_number') - op.drop_column('privacyrequest', 'identity_email') + op.drop_index( + op.f("ix_privacyrequest_identity_phone_number"), table_name="privacyrequest" + ) + op.drop_index(op.f("ix_privacyrequest_identity_email"), table_name="privacyrequest") + op.drop_column("privacyrequest", "identity_phone_number") + op.drop_column("privacyrequest", "identity_email") # ### end Alembic commands ### From 21bf006bb4f5cad0efa5ffa97bc1a66e9ec263c0 Mon Sep 17 00:00:00 2001 From: Sean Preston Date: Fri, 8 Jul 2022 11:50:52 +0200 Subject: [PATCH 07/27] store provided identity data in request creation from onetrust --- src/fidesops/service/privacy_request/onetrust_service.py | 3 +++ tests/service/privacy_request/onetrust_test.py | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/fidesops/service/privacy_request/onetrust_service.py b/src/fidesops/service/privacy_request/onetrust_service.py index 9cb3f715f..aa8886324 100644 --- a/src/fidesops/service/privacy_request/onetrust_service.py +++ b/src/fidesops/service/privacy_request/onetrust_service.py @@ -152,6 +152,9 @@ def _create_privacy_request( # pylint: disable=R0913 "client_id": onetrust_policy.client_id, "external_id": subtask_id, } + if identity.email: + kwargs["identity_email"] = identity.email + privacy_request: PrivacyRequest = PrivacyRequest.create(db=db, data=kwargs) privacy_request.cache_identity(identity) try: diff --git a/tests/service/privacy_request/onetrust_test.py b/tests/service/privacy_request/onetrust_test.py index eadeda6bd..51e2721e3 100644 --- a/tests/service/privacy_request/onetrust_test.py +++ b/tests/service/privacy_request/onetrust_test.py @@ -57,10 +57,11 @@ def test_intake_onetrust_requests_success( policy = _create_mock_policy(client, db, storage_config_onetrust) # mock onetrust auth mock_get_onetrust_access_token.return_value = "124-asdf-23412424" + TEST_EMAIL = "some-customer@mail.com" # mock onetrust requests mock_request_1: OneTrustRequest = OneTrustRequest() - mock_request_1.email = "some-customer@mail.com" + mock_request_1.email = TEST_EMAIL mock_request_1.requestQueueRefId = "23xrnqq3crwf" mock_request_1.dateCreated = "2021-08-09T12:49:47.983Z" mock_request_2: OneTrustRequest = OneTrustRequest() @@ -115,6 +116,7 @@ def test_intake_onetrust_requests_success( field="external_id", value=mock_subtask_1.subTaskId, ) + assert pr.identity_email == TEST_EMAIL assert pr is not None assert finish_processing_mock.called # clean up From cd173c9b84d0868585be280ecf62a822137fe9a9 Mon Sep 17 00:00:00 2001 From: Sean Preston Date: Fri, 8 Jul 2022 14:23:46 +0200 Subject: [PATCH 08/27] remove deprecated migration --- ...68_add_identity_data_to_privacy_request.py | 49 ------------------- 1 file changed, 49 deletions(-) delete mode 100644 src/fidesops/migrations/versions/cdaac83e0a68_add_identity_data_to_privacy_request.py diff --git a/src/fidesops/migrations/versions/cdaac83e0a68_add_identity_data_to_privacy_request.py b/src/fidesops/migrations/versions/cdaac83e0a68_add_identity_data_to_privacy_request.py deleted file mode 100644 index 6da3dfbc5..000000000 --- a/src/fidesops/migrations/versions/cdaac83e0a68_add_identity_data_to_privacy_request.py +++ /dev/null @@ -1,49 +0,0 @@ -"""add identity data to privacy_request - -Revision ID: cdaac83e0a68 -Revises: fc90277bbcde -Create Date: 2022-07-08 09:02:11.717954 - -""" -import sqlalchemy as sa -from alembic import op - -# revision identifiers, used by Alembic. -revision = "cdaac83e0a68" -down_revision = "fc90277bbcde" -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.add_column( - "privacyrequest", sa.Column("identity_email", sa.String(), nullable=True) - ) - op.add_column( - "privacyrequest", sa.Column("identity_phone_number", sa.String(), nullable=True) - ) - op.create_index( - op.f("ix_privacyrequest_identity_email"), - "privacyrequest", - ["identity_email"], - unique=False, - ) - op.create_index( - op.f("ix_privacyrequest_identity_phone_number"), - "privacyrequest", - ["identity_phone_number"], - unique=False, - ) - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.drop_index( - op.f("ix_privacyrequest_identity_phone_number"), table_name="privacyrequest" - ) - op.drop_index(op.f("ix_privacyrequest_identity_email"), table_name="privacyrequest") - op.drop_column("privacyrequest", "identity_phone_number") - op.drop_column("privacyrequest", "identity_email") - # ### end Alembic commands ### From e32326d79fe9234a776d6f09707a6c84001e87f0 Mon Sep 17 00:00:00 2001 From: Sean Preston Date: Fri, 8 Jul 2022 14:24:17 +0200 Subject: [PATCH 09/27] adds new provided identity table --- ...3465d_adds_provided_identity_table_for_.py | 75 ++++++++++ src/fidesops/models/privacy_request.py | 130 +++++++++++++++--- 2 files changed, 186 insertions(+), 19 deletions(-) create mode 100644 src/fidesops/migrations/versions/3c5e1253465d_adds_provided_identity_table_for_.py diff --git a/src/fidesops/migrations/versions/3c5e1253465d_adds_provided_identity_table_for_.py b/src/fidesops/migrations/versions/3c5e1253465d_adds_provided_identity_table_for_.py new file mode 100644 index 000000000..050d2a159 --- /dev/null +++ b/src/fidesops/migrations/versions/3c5e1253465d_adds_provided_identity_table_for_.py @@ -0,0 +1,75 @@ +"""adds provided identity table for identity storage and later identity search + +Revision ID: 3c5e1253465d +Revises: fc90277bbcde +Create Date: 2022-07-08 11:53:05.215848 + +""" +import sqlalchemy as sa +import sqlalchemy_utils +from alembic import op + + +# revision identifiers, used by Alembic. +revision = "3c5e1253465d" +down_revision = "fc90277bbcde" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "providedidentity", + sa.Column("id", sa.String(length=255), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=True, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=True, + ), + sa.Column("privacy_request_id", sa.String(), nullable=False), + sa.Column( + "field_name", + sa.Enum("email", "phone_number", name="providedidentitytype"), + nullable=False, + ), + sa.Column("hashed_value", sa.String(), nullable=True), + sa.Column("salt", sa.String(), nullable=False), + sa.Column( + "encrypted_value", + sqlalchemy_utils.types.encrypted.encrypted_type.StringEncryptedType(), + nullable=True, + ), + sa.ForeignKeyConstraint( + ["privacy_request_id"], + ["privacyrequest.id"], + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + op.f("ix_providedidentity_hashed_value"), + "providedidentity", + ["hashed_value"], + unique=False, + ) + op.create_index( + op.f("ix_providedidentity_id"), "providedidentity", ["id"], unique=False + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f("ix_providedidentity_id"), table_name="providedidentity") + op.drop_index( + op.f("ix_providedidentity_hashed_value"), table_name="providedidentity" + ) + op.drop_table("providedidentity") + # ### end Alembic commands ### diff --git a/src/fidesops/models/privacy_request.py b/src/fidesops/models/privacy_request.py index 0433cea91..9a4491e2e 100644 --- a/src/fidesops/models/privacy_request.py +++ b/src/fidesops/models/privacy_request.py @@ -6,6 +6,7 @@ from typing import Any, Dict, List, Optional from celery.result import AsyncResult +from fideslib.cryptography.cryptographic_util import generate_salt, hash_with_salt from fideslib.db.base import Base from fideslib.db.base_class import FidesBase from fideslib.models.audit_log import AuditLog @@ -17,12 +18,17 @@ from sqlalchemy import Enum as EnumColumn from sqlalchemy import ForeignKey, String from sqlalchemy.dialects.postgresql import JSONB -from sqlalchemy.ext.mutable import MutableList +from sqlalchemy.ext.mutable import MutableList, MutableDict from sqlalchemy.orm import Session, backref, relationship +from sqlalchemy_utils.types.encrypted.encrypted_type import ( + AesGcmEngine, + StringEncryptedType, +) from fidesops.api.v1.scope_registry import PRIVACY_REQUEST_CALLBACK_RESUME from fidesops.common_exceptions import PrivacyRequestPaused from fidesops.core.config import config +from fidesops.db.base_class import JSONTypeOverride from fidesops.graph.config import CollectionAddress from fidesops.models.policy import ( ActionType, @@ -190,22 +196,6 @@ class PrivacyRequest(Base): # pylint: disable=R0904 ) paused_at = Column(DateTime(timezone=True), nullable=True) - # Provided PII fields at request creation, stored to facilitate searching by identity in the UIs - # In the future these can be moved into a JSON field, but for now since there are only two fields - # supported as identities it's simpler for indexing and searching to leave them on the table - identity_email = Column( - String, - index=True, - unique=False, - nullable=True, - ) - identity_phone_number = Column( - String, - index=True, - unique=False, - nullable=True, - ) - @classmethod def create(cls, db: Session, *, data: Dict[str, Any]) -> FidesBase: """ @@ -218,13 +208,16 @@ def create(cls, db: Session, *, data: Dict[str, Any]) -> FidesBase: def delete(self, db: Session) -> None: """ - Clean up the cached data related to this privacy request before deleting this - object from the database + Clean up the cached and persisted data related to this privacy request before + deleting this object from the database """ cache: FidesopsRedis = get_cache() all_keys = get_all_cache_keys_for_privacy_request(privacy_request_id=self.id) for key in all_keys: cache.delete(key) + + for provided_identity in self.provided_identities: + provided_identity.delete(db=db) super().delete(db=db) def cache_identity(self, identity: PrivacyRequestIdentity) -> None: @@ -238,6 +231,40 @@ def cache_identity(self, identity: PrivacyRequestIdentity) -> None: value, ) + def persist_identity(self, db: Session, identity: PrivacyRequestIdentity) -> None: + """ + Stores the identity provided with the privacy request in a secure way, compatible with + blind indexing for later searching and audit purposes. + """ + identity_dict: Dict[str, Any] = dict(identity) + for key, value in identity_dict.items(): + if value is not None: + hashed_value, salt = ProvidedIdentity.hash_value(value) + ProvidedIdentity.create( + db=db, + data={ + "privacy_request_id": self.id, + "field_name": key, + # We don't need to manually encrypt this field, it's done at the ORM level + "encrypted_value": {"value": value}, + "hashed_value": hashed_value, + "salt": salt, + }, + ) + + def get_persisted_identity(self) -> PrivacyRequestIdentity: + """ + Retrieves persisted identity fields from the DB. + """ + schema = PrivacyRequestIdentity() + for field in self.provided_identities: + setattr( + schema, + field.field_name.value, + field.encrypted_value["value"], + ) + return schema + def cache_task_id(self, task_id: str) -> None: """Sets a task_id for this privacy request's asynchronous execution.""" cache: FidesopsRedis = get_cache() @@ -509,6 +536,71 @@ def error_processing(self, db: Session) -> None: ) +class ProvidedIdentityType(EnumType): + """Enum for privacy request identity types""" + + email = "email" + phone_number = "phone_number" + + +class ProvidedIdentity(Base): # pylint: disable=R0904 + + privacy_request_id = Column( + String, + ForeignKey(PrivacyRequest.id_field_path), + nullable=False, + ) + privacy_request = relationship( + PrivacyRequest, + backref="provided_identities", + ) # Which privacy request this identity belongs to + + # Provided PII fields at request creation, stored to facilitate searching by identity in the UIs + # In the future these can be moved into a JSON field, but for now since there are only two fields + # supported as identities it's simpler for indexing and searching to leave them on the table + field_name = Column( + EnumColumn(ProvidedIdentityType), + index=False, + nullable=False, + ) + hashed_value = Column( + String, + index=True, + unique=False, + nullable=True, + ) # This field is used as a blind index for exact match searches + salt = Column( + String, + index=False, + nullable=False, + ) + encrypted_value = Column( + MutableDict.as_mutable( + StringEncryptedType( + JSONTypeOverride, + config.security.APP_ENCRYPTION_KEY, + AesGcmEngine, + "pkcs5", + ) + ), + nullable=True, + ) # Type bytea in the db + + @classmethod + def hash_value( + cls, + value: str, + encoding: str = "UTF-8", + ) -> tuple[str, str]: + """Utility function to hash a user's password with a generated salt""" + salt = generate_salt() + hashed_value = hash_with_salt( + value.encode(encoding), + salt.encode(encoding), + ) + return hashed_value, salt + + # Unique text to separate a step from a collection address, so we can store two values in one. PAUSED_SEPARATOR = "__fidesops_paused_sep__" From 90c32cfafcf88f5127a111098916996d340c2fa6 Mon Sep 17 00:00:00 2001 From: Sean Preston Date: Fri, 8 Jul 2022 14:24:42 +0200 Subject: [PATCH 10/27] use new provided identity table --- .../api/v1/endpoints/privacy_request_endpoints.py | 9 +++------ tests/api/v1/endpoints/test_privacy_request_endpoints.py | 5 +++-- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/fidesops/api/v1/endpoints/privacy_request_endpoints.py b/src/fidesops/api/v1/endpoints/privacy_request_endpoints.py index 4b36d7fc7..ae98488a9 100644 --- a/src/fidesops/api/v1/endpoints/privacy_request_endpoints.py +++ b/src/fidesops/api/v1/endpoints/privacy_request_endpoints.py @@ -176,14 +176,11 @@ def create_privacy_request( if attr is not None: kwargs[field] = attr - if privacy_request_data.identity.email: - kwargs["identity_email"] = privacy_request_data.identity.email - - if privacy_request_data.identity.phone_number: - kwargs["identity_phone_number"] = privacy_request_data.identity.phone_number - try: privacy_request: PrivacyRequest = PrivacyRequest.create(db=db, data=kwargs) + privacy_request.persist_identity( + db=db, identity=privacy_request_data.identity + ) cache_data( privacy_request, diff --git a/tests/api/v1/endpoints/test_privacy_request_endpoints.py b/tests/api/v1/endpoints/test_privacy_request_endpoints.py index 7438c41aa..e7804db3a 100644 --- a/tests/api/v1/endpoints/test_privacy_request_endpoints.py +++ b/tests/api/v1/endpoints/test_privacy_request_endpoints.py @@ -130,8 +130,9 @@ def test_create_privacy_request_stores_identities( response_data = resp.json()["succeeded"] assert len(response_data) == 1 pr = PrivacyRequest.get(db=db, object_id=response_data[0]["id"]) - assert pr.identity_email == TEST_EMAIL - assert pr.identity_phone_number == TEST_PHONE_NUMBER + persisted_identity = pr.get_persisted_identity() + assert persisted_identity.email == TEST_EMAIL + assert persisted_identity.phone_number == TEST_PHONE_NUMBER pr.delete(db=db) assert run_access_request_mock.called From ddf0d66e4ee465063d3feaeffa8f37bff9a6db3e Mon Sep 17 00:00:00 2001 From: Sean Preston Date: Fri, 8 Jul 2022 14:26:32 +0200 Subject: [PATCH 11/27] add docstring, remove comment --- src/fidesops/models/privacy_request.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/fidesops/models/privacy_request.py b/src/fidesops/models/privacy_request.py index 9a4491e2e..0aa065f3d 100644 --- a/src/fidesops/models/privacy_request.py +++ b/src/fidesops/models/privacy_request.py @@ -544,6 +544,10 @@ class ProvidedIdentityType(EnumType): class ProvidedIdentity(Base): # pylint: disable=R0904 + """ + A table for storing identity fields and values provided at privacy request + creation time. + """ privacy_request_id = Column( String, @@ -555,9 +559,6 @@ class ProvidedIdentity(Base): # pylint: disable=R0904 backref="provided_identities", ) # Which privacy request this identity belongs to - # Provided PII fields at request creation, stored to facilitate searching by identity in the UIs - # In the future these can be moved into a JSON field, but for now since there are only two fields - # supported as identities it's simpler for indexing and searching to leave them on the table field_name = Column( EnumColumn(ProvidedIdentityType), index=False, From bb248621925d98abc89bb2ac8e9230597c6bbe3f Mon Sep 17 00:00:00 2001 From: Sean Preston Date: Fri, 8 Jul 2022 14:30:50 +0200 Subject: [PATCH 12/27] update DRP privacy request creation to use ProvidedIdentity model --- src/fidesops/api/v1/endpoints/drp_endpoints.py | 12 ++++-------- tests/api/v1/endpoints/test_drp_endpoints.py | 5 +++-- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/src/fidesops/api/v1/endpoints/drp_endpoints.py b/src/fidesops/api/v1/endpoints/drp_endpoints.py index e592d5aee..73f8350f9 100644 --- a/src/fidesops/api/v1/endpoints/drp_endpoints.py +++ b/src/fidesops/api/v1/endpoints/drp_endpoints.py @@ -97,18 +97,14 @@ def create_drp_privacy_request( drp_identity=decrypted_identity ) - if mapped_identity.email: - privacy_request_kwargs["identity_email"] = mapped_identity.email - - if mapped_identity.phone_number: - privacy_request_kwargs[ - "identity_phone_number" - ] = mapped_identity.phone_number - privacy_request: PrivacyRequest = PrivacyRequest.create( db=db, data=privacy_request_kwargs, ) + privacy_request.persist_identity( + db=db, + identity=mapped_identity, + ) logger.info(f"Decrypting identity for DRP privacy request {privacy_request.id}") diff --git a/tests/api/v1/endpoints/test_drp_endpoints.py b/tests/api/v1/endpoints/test_drp_endpoints.py index 78684b65b..06e7e44b0 100644 --- a/tests/api/v1/endpoints/test_drp_endpoints.py +++ b/tests/api/v1/endpoints/test_drp_endpoints.py @@ -95,8 +95,9 @@ def test_create_drp_privacy_request( identity_attribute="email", ) assert cache.get(fidesops_identity_key) == identity["email"] - assert pr.identity_email == TEST_EMAIL - assert pr.identity_phone_number == TEST_PHONE_NUMBER + persisted_identity = pr.get_persisted_identity() + assert persisted_identity.email == TEST_EMAIL + assert persisted_identity.phone_number == TEST_PHONE_NUMBER pr.delete(db=db) assert run_access_request_mock.called From 971c36b8559972c42e7e629b6ad6f28827f87fb8 Mon Sep 17 00:00:00 2001 From: Sean Preston Date: Fri, 8 Jul 2022 14:33:07 +0200 Subject: [PATCH 13/27] update identity creation in test data command --- create_test_data.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/create_test_data.py b/create_test_data.py index 3a000134e..657425c9c 100644 --- a/create_test_data.py +++ b/create_test_data.py @@ -21,6 +21,7 @@ from fidesops.models.policy import ActionType, Policy, Rule, RuleTarget from fidesops.models.privacy_request import PrivacyRequest, PrivacyRequestStatus from fidesops.models.storage import ResponseFormat, StorageConfig +from fidesops.schemas.redis_cache import PrivacyRequestIdentity from fidesops.schemas.storage.storage import FileNaming, StorageDetails, StorageType from fidesops.util.data_category import DataCategory @@ -176,7 +177,7 @@ def create_test_data(db: orm.Session) -> FidesUser: for policy in policies: for status in PrivacyRequestStatus.__members__.values(): - PrivacyRequest.create( + pr = PrivacyRequest.create( db=db, data={ "external_id": f"ext-{uuid4()}", @@ -186,10 +187,15 @@ def create_test_data(db: orm.Session) -> FidesUser: "origin": f"https://example.com/{status.value}/", "policy_id": policy.id, "client_id": policy.client_id, - "identity_email": "test@example.com", - "identity_phone_number": "+1 234 567 8910", }, ) + pr.persist_identity( + db=db, + identity=PrivacyRequestIdentity( + email="test@example.com", + phone_number="+1 234 567 8910", + ), + ) print("Adding connection configs") _create_connection_configs(db) From 3143de8bb70ce4c30f743dd4035efb986ba600fb Mon Sep 17 00:00:00 2001 From: Sean Preston Date: Fri, 8 Jul 2022 14:35:02 +0200 Subject: [PATCH 14/27] use persisted identity in OneTrust --- src/fidesops/service/privacy_request/onetrust_service.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/fidesops/service/privacy_request/onetrust_service.py b/src/fidesops/service/privacy_request/onetrust_service.py index aa8886324..efac307f4 100644 --- a/src/fidesops/service/privacy_request/onetrust_service.py +++ b/src/fidesops/service/privacy_request/onetrust_service.py @@ -152,10 +152,12 @@ def _create_privacy_request( # pylint: disable=R0913 "client_id": onetrust_policy.client_id, "external_id": subtask_id, } - if identity.email: - kwargs["identity_email"] = identity.email privacy_request: PrivacyRequest = PrivacyRequest.create(db=db, data=kwargs) + privacy_request.persist_identity( + db=db, + identity=PrivacyRequestIdentity(email=identity.email), + ) privacy_request.cache_identity(identity) try: queue_privacy_request(privacy_request_id=privacy_request.id) From 9ca5d973724edca285c6c2f1fb7f19fbfde429ad Mon Sep 17 00:00:00 2001 From: Sean Preston Date: Fri, 8 Jul 2022 14:35:17 +0200 Subject: [PATCH 15/27] update test to use persisted identity --- tests/service/privacy_request/onetrust_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/service/privacy_request/onetrust_test.py b/tests/service/privacy_request/onetrust_test.py index 51e2721e3..3346ef551 100644 --- a/tests/service/privacy_request/onetrust_test.py +++ b/tests/service/privacy_request/onetrust_test.py @@ -116,7 +116,8 @@ def test_intake_onetrust_requests_success( field="external_id", value=mock_subtask_1.subTaskId, ) - assert pr.identity_email == TEST_EMAIL + persisted_identity = pr.get_persisted_identity() + assert persisted_identity.email == TEST_EMAIL assert pr is not None assert finish_processing_mock.called # clean up From cf6915bdc3073cc1c766151114fa2c671bf17f30 Mon Sep 17 00:00:00 2001 From: Sean Preston Date: Fri, 8 Jul 2022 15:15:51 +0200 Subject: [PATCH 16/27] isort update --- .../versions/3c5e1253465d_adds_provided_identity_table_for_.py | 1 - src/fidesops/models/privacy_request.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/fidesops/migrations/versions/3c5e1253465d_adds_provided_identity_table_for_.py b/src/fidesops/migrations/versions/3c5e1253465d_adds_provided_identity_table_for_.py index 050d2a159..c51780643 100644 --- a/src/fidesops/migrations/versions/3c5e1253465d_adds_provided_identity_table_for_.py +++ b/src/fidesops/migrations/versions/3c5e1253465d_adds_provided_identity_table_for_.py @@ -9,7 +9,6 @@ import sqlalchemy_utils from alembic import op - # revision identifiers, used by Alembic. revision = "3c5e1253465d" down_revision = "fc90277bbcde" diff --git a/src/fidesops/models/privacy_request.py b/src/fidesops/models/privacy_request.py index 0aa065f3d..c2a4af05f 100644 --- a/src/fidesops/models/privacy_request.py +++ b/src/fidesops/models/privacy_request.py @@ -18,7 +18,7 @@ from sqlalchemy import Enum as EnumColumn from sqlalchemy import ForeignKey, String from sqlalchemy.dialects.postgresql import JSONB -from sqlalchemy.ext.mutable import MutableList, MutableDict +from sqlalchemy.ext.mutable import MutableDict, MutableList from sqlalchemy.orm import Session, backref, relationship from sqlalchemy_utils.types.encrypted.encrypted_type import ( AesGcmEngine, From 184b106ed454359da9d56175e93a14a2a0ab21cf Mon Sep 17 00:00:00 2001 From: Sean Preston Date: Sun, 10 Jul 2022 21:51:58 +0200 Subject: [PATCH 17/27] use enums --- src/fidesops/service/drp/drp_fidesops_mapper.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/fidesops/service/drp/drp_fidesops_mapper.py b/src/fidesops/service/drp/drp_fidesops_mapper.py index 0307605f1..405d74616 100644 --- a/src/fidesops/service/drp/drp_fidesops_mapper.py +++ b/src/fidesops/service/drp/drp_fidesops_mapper.py @@ -1,7 +1,7 @@ import logging from typing import Dict -from fidesops.models.privacy_request import PrivacyRequestStatus +from fidesops.models.privacy_request import PrivacyRequestStatus, ProvidedIdentityType from fidesops.schemas.drp_privacy_request import DrpIdentity from fidesops.schemas.privacy_request import PrivacyRequestDRPStatus from fidesops.schemas.redis_cache import PrivacyRequestIdentity @@ -23,8 +23,8 @@ def map_identity(drp_identity: DrpIdentity) -> PrivacyRequestIdentity: """ fidesops_identity_kwargs: Dict[str, str] = {} DRP_TO_FIDESOPS_SUPPORTED_IDENTITY_PROPS_MAP: Dict[str, str] = { - "email": "email", - "phone_number": "phone_number", + "email": ProvidedIdentityType.email.value, + "phone_number": ProvidedIdentityType.phone_number.value, } for attr, val in drp_identity.__dict__.items(): if attr not in DRP_TO_FIDESOPS_SUPPORTED_IDENTITY_PROPS_MAP: From d3068dbc1ecabaa24fe3a5d8ca0b51df7e118973 Mon Sep 17 00:00:00 2001 From: Sean Preston Date: Sun, 10 Jul 2022 21:56:43 +0200 Subject: [PATCH 18/27] optionally receive a salt in hash_value cmd --- src/fidesops/models/privacy_request.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/fidesops/models/privacy_request.py b/src/fidesops/models/privacy_request.py index c2a4af05f..02f9aa2de 100644 --- a/src/fidesops/models/privacy_request.py +++ b/src/fidesops/models/privacy_request.py @@ -592,9 +592,12 @@ def hash_value( cls, value: str, encoding: str = "UTF-8", + salt: Optional[str] = None, ) -> tuple[str, str]: """Utility function to hash a user's password with a generated salt""" - salt = generate_salt() + if not salt: + salt = generate_salt() + hashed_value = hash_with_salt( value.encode(encoding), salt.encode(encoding), From ecedcec1c20853f1946d2149f945c63224dfdd3a Mon Sep 17 00:00:00 2001 From: Sean Preston Date: Sun, 10 Jul 2022 22:05:24 +0200 Subject: [PATCH 19/27] use a constant salt for provided identity hashing --- ...253465d_adds_provided_identity_table_for_.py | 1 - src/fidesops/models/privacy_request.py | 17 ++++------------- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/src/fidesops/migrations/versions/3c5e1253465d_adds_provided_identity_table_for_.py b/src/fidesops/migrations/versions/3c5e1253465d_adds_provided_identity_table_for_.py index c51780643..06074880f 100644 --- a/src/fidesops/migrations/versions/3c5e1253465d_adds_provided_identity_table_for_.py +++ b/src/fidesops/migrations/versions/3c5e1253465d_adds_provided_identity_table_for_.py @@ -40,7 +40,6 @@ def upgrade(): nullable=False, ), sa.Column("hashed_value", sa.String(), nullable=True), - sa.Column("salt", sa.String(), nullable=False), sa.Column( "encrypted_value", sqlalchemy_utils.types.encrypted.encrypted_type.StringEncryptedType(), diff --git a/src/fidesops/models/privacy_request.py b/src/fidesops/models/privacy_request.py index 02f9aa2de..fdb872b1b 100644 --- a/src/fidesops/models/privacy_request.py +++ b/src/fidesops/models/privacy_request.py @@ -239,7 +239,7 @@ def persist_identity(self, db: Session, identity: PrivacyRequestIdentity) -> Non identity_dict: Dict[str, Any] = dict(identity) for key, value in identity_dict.items(): if value is not None: - hashed_value, salt = ProvidedIdentity.hash_value(value) + hashed_value = ProvidedIdentity.hash_value(value) ProvidedIdentity.create( db=db, data={ @@ -248,7 +248,6 @@ def persist_identity(self, db: Session, identity: PrivacyRequestIdentity) -> Non # We don't need to manually encrypt this field, it's done at the ORM level "encrypted_value": {"value": value}, "hashed_value": hashed_value, - "salt": salt, }, ) @@ -570,11 +569,6 @@ class ProvidedIdentity(Base): # pylint: disable=R0904 unique=False, nullable=True, ) # This field is used as a blind index for exact match searches - salt = Column( - String, - index=False, - nullable=False, - ) encrypted_value = Column( MutableDict.as_mutable( StringEncryptedType( @@ -592,17 +586,14 @@ def hash_value( cls, value: str, encoding: str = "UTF-8", - salt: Optional[str] = None, ) -> tuple[str, str]: """Utility function to hash a user's password with a generated salt""" - if not salt: - salt = generate_salt() - + SALT = "a-salt" hashed_value = hash_with_salt( value.encode(encoding), - salt.encode(encoding), + SALT.encode(encoding), ) - return hashed_value, salt + return hashed_value # Unique text to separate a step from a collection address, so we can store two values in one. From 6a8c046e1267e4a96340297af3943a2c4ba6faaa Mon Sep 17 00:00:00 2001 From: Sean Preston Date: Sun, 10 Jul 2022 23:06:02 +0200 Subject: [PATCH 20/27] remove import --- src/fidesops/models/privacy_request.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fidesops/models/privacy_request.py b/src/fidesops/models/privacy_request.py index fdb872b1b..2d5af2381 100644 --- a/src/fidesops/models/privacy_request.py +++ b/src/fidesops/models/privacy_request.py @@ -6,7 +6,7 @@ from typing import Any, Dict, List, Optional from celery.result import AsyncResult -from fideslib.cryptography.cryptographic_util import generate_salt, hash_with_salt +from fideslib.cryptography.cryptographic_util import hash_with_salt from fideslib.db.base import Base from fideslib.db.base_class import FidesBase from fideslib.models.audit_log import AuditLog From e0034ba4f03a559c273c9f78ff8d0484275c5138 Mon Sep 17 00:00:00 2001 From: Sean Preston Date: Sun, 10 Jul 2022 23:06:53 +0200 Subject: [PATCH 21/27] use typehints --- src/fidesops/service/drp/drp_fidesops_mapper.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/fidesops/service/drp/drp_fidesops_mapper.py b/src/fidesops/service/drp/drp_fidesops_mapper.py index 405d74616..d3beb4d4f 100644 --- a/src/fidesops/service/drp/drp_fidesops_mapper.py +++ b/src/fidesops/service/drp/drp_fidesops_mapper.py @@ -22,9 +22,11 @@ def map_identity(drp_identity: DrpIdentity) -> PrivacyRequestIdentity: This class also allows us to implement custom logic to handle "verified" id props. """ fidesops_identity_kwargs: Dict[str, str] = {} + email: str = ProvidedIdentityType.email.value + phone_number: str = ProvidedIdentityType.phone_number.value DRP_TO_FIDESOPS_SUPPORTED_IDENTITY_PROPS_MAP: Dict[str, str] = { - "email": ProvidedIdentityType.email.value, - "phone_number": ProvidedIdentityType.phone_number.value, + "email": email, + "phone_number": phone_number, } for attr, val in drp_identity.__dict__.items(): if attr not in DRP_TO_FIDESOPS_SUPPORTED_IDENTITY_PROPS_MAP: From fbd3f10870da37b3ff36238bdcc9c719f5b4e8b0 Mon Sep 17 00:00:00 2001 From: Sean Preston Date: Mon, 11 Jul 2022 11:00:00 +0200 Subject: [PATCH 22/27] update typedef --- src/fidesops/service/drp/drp_fidesops_mapper.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/fidesops/service/drp/drp_fidesops_mapper.py b/src/fidesops/service/drp/drp_fidesops_mapper.py index d3beb4d4f..4d5ad3217 100644 --- a/src/fidesops/service/drp/drp_fidesops_mapper.py +++ b/src/fidesops/service/drp/drp_fidesops_mapper.py @@ -22,11 +22,11 @@ def map_identity(drp_identity: DrpIdentity) -> PrivacyRequestIdentity: This class also allows us to implement custom logic to handle "verified" id props. """ fidesops_identity_kwargs: Dict[str, str] = {} - email: str = ProvidedIdentityType.email.value - phone_number: str = ProvidedIdentityType.phone_number.value - DRP_TO_FIDESOPS_SUPPORTED_IDENTITY_PROPS_MAP: Dict[str, str] = { - "email": email, - "phone_number": phone_number, + DRP_TO_FIDESOPS_SUPPORTED_IDENTITY_PROPS_MAP: Dict[ + str, ProvidedIdentityType + ] = { + "email": ProvidedIdentityType.email.value, + "phone_number": ProvidedIdentityType.phone_number.value, } for attr, val in drp_identity.__dict__.items(): if attr not in DRP_TO_FIDESOPS_SUPPORTED_IDENTITY_PROPS_MAP: From 7d86a6b4b757aeba51602dd96e4c031c6442b2f8 Mon Sep 17 00:00:00 2001 From: Sean Preston Date: Mon, 11 Jul 2022 12:56:02 +0200 Subject: [PATCH 23/27] use enum in dict --- src/fidesops/service/drp/drp_fidesops_mapper.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/fidesops/service/drp/drp_fidesops_mapper.py b/src/fidesops/service/drp/drp_fidesops_mapper.py index 4d5ad3217..e0dc2f9c7 100644 --- a/src/fidesops/service/drp/drp_fidesops_mapper.py +++ b/src/fidesops/service/drp/drp_fidesops_mapper.py @@ -25,8 +25,8 @@ def map_identity(drp_identity: DrpIdentity) -> PrivacyRequestIdentity: DRP_TO_FIDESOPS_SUPPORTED_IDENTITY_PROPS_MAP: Dict[ str, ProvidedIdentityType ] = { - "email": ProvidedIdentityType.email.value, - "phone_number": ProvidedIdentityType.phone_number.value, + "email": ProvidedIdentityType.email, + "phone_number": ProvidedIdentityType.phone_number, } for attr, val in drp_identity.__dict__.items(): if attr not in DRP_TO_FIDESOPS_SUPPORTED_IDENTITY_PROPS_MAP: @@ -34,7 +34,9 @@ def map_identity(drp_identity: DrpIdentity) -> PrivacyRequestIdentity: f"Identity attribute of {attr} is not supported by Fidesops at this time. Continuing to use other identity props, if provided." ) else: - fidesops_prop: str = DRP_TO_FIDESOPS_SUPPORTED_IDENTITY_PROPS_MAP[attr] + fidesops_prop: str = DRP_TO_FIDESOPS_SUPPORTED_IDENTITY_PROPS_MAP[ + attr + ].value fidesops_identity_kwargs[fidesops_prop] = val return PrivacyRequestIdentity(**fidesops_identity_kwargs) From 6c90bf48cacf8db255a214dcfe2761460f1b18a1 Mon Sep 17 00:00:00 2001 From: Sean Preston Date: Mon, 11 Jul 2022 18:14:38 +0200 Subject: [PATCH 24/27] test for exact match search --- .../test_privacy_request_endpoints.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/api/v1/endpoints/test_privacy_request_endpoints.py b/tests/api/v1/endpoints/test_privacy_request_endpoints.py index e7804db3a..36bf8aaab 100644 --- a/tests/api/v1/endpoints/test_privacy_request_endpoints.py +++ b/tests/api/v1/endpoints/test_privacy_request_endpoints.py @@ -5,6 +5,7 @@ from datetime import datetime from typing import List from unittest import mock +from fidesops.schemas.redis_cache import PrivacyRequestIdentity import pytest from dateutil.parser import parse @@ -731,6 +732,31 @@ def test_filter_privacy_requests_by_internal_id( assert len(resp["items"]) == 1 assert resp["items"][0]["id"] == privacy_request.id + def test_filter_privacy_requests_by_identity_exact( + self, + db, + api_client, + url, + generate_auth_header, + privacy_request, + ): + TEST_EMAIL = "test-12345678910@example.com" + privacy_request.persist_identity( + db=db, + identity=PrivacyRequestIdentity( + email=TEST_EMAIL, + ), + ) + auth_header = generate_auth_header(scopes=[PRIVACY_REQUEST_READ]) + response = api_client.get( + url + f"?identity={TEST_EMAIL}", + headers=auth_header, + ) + assert 200 == response.status_code + resp = response.json() + assert len(resp["items"]) == 1 + assert resp["items"][0]["id"] == privacy_request.id + def test_filter_privacy_requests_by_external_id( self, db, From 2ba24ddcb9cdcd90570424b54fa2230f462001cb Mon Sep 17 00:00:00 2001 From: Sean Preston Date: Mon, 11 Jul 2022 18:16:02 +0200 Subject: [PATCH 25/27] added exact match search to request status api --- .../v1/endpoints/privacy_request_endpoints.py | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/fidesops/api/v1/endpoints/privacy_request_endpoints.py b/src/fidesops/api/v1/endpoints/privacy_request_endpoints.py index ae98488a9..79b2f8dd6 100644 --- a/src/fidesops/api/v1/endpoints/privacy_request_endpoints.py +++ b/src/fidesops/api/v1/endpoints/privacy_request_endpoints.py @@ -5,7 +5,7 @@ import logging from collections import defaultdict from datetime import datetime -from typing import Any, Callable, DefaultDict, Dict, List, Optional, Union +from typing import Any, Callable, DefaultDict, Dict, List, Optional, Set, Union from fastapi import APIRouter, Body, Depends, HTTPException, Security from fastapi.params import Query as FastAPIQuery @@ -15,6 +15,7 @@ from fideslib.models.audit_log import AuditLog, AuditLogAction from fideslib.models.client import ClientDetail from pydantic import conlist +from sqlalchemy import column from sqlalchemy.orm import Query, Session from starlette.responses import StreamingResponse from starlette.status import ( @@ -59,6 +60,7 @@ ExecutionLog, PrivacyRequest, PrivacyRequestStatus, + ProvidedIdentity, ) from fidesops.schemas.dataset import CollectionAddressResponse, DryRunDatasetResponse from fidesops.schemas.external_https import PrivacyRequestResumeFormat @@ -292,8 +294,10 @@ def execution_logs_by_dataset_name( def _filter_privacy_request_queryset( + db: Session, query: Query, request_id: Optional[str] = None, + identity: Optional[str] = None, status: Optional[List[PrivacyRequestStatus]] = None, created_lt: Optional[datetime] = None, created_gt: Optional[datetime] = None, @@ -337,6 +341,17 @@ def _filter_privacy_request_queryset( detail=f"Value specified for {field_name}_lt: {end} must be after {field_name}_gt: {start}.", ) + if identity: + hashed_identity = ProvidedIdentity.hash_value(value=identity) + identities: Set[str] = { + identity[0] + for identity in ProvidedIdentity.filter( + db=db, + conditions=(ProvidedIdentity.hashed_value == hashed_identity), + ).values(column("privacy_request_id")) + } + query = query.filter(PrivacyRequest.id.in_(identities)) + # Further restrict all PrivacyRequests by query params if request_id: query = query.filter(PrivacyRequest.id.ilike(f"{request_id}%")) @@ -432,6 +447,7 @@ def get_request_status( db: Session = Depends(deps.get_db), params: Params = Depends(), request_id: Optional[str] = None, + identity: Optional[str] = None, status: Optional[List[PrivacyRequestStatus]] = FastAPIQuery( default=None ), # type:ignore @@ -457,8 +473,10 @@ def get_request_status( logger.info(f"Finding all request statuses with pagination params {params}") query = db.query(PrivacyRequest) query = _filter_privacy_request_queryset( + db, query, request_id, + identity, status, created_lt, created_gt, From 440cf5bcda865e074765c8337fe99157de3e5d41 Mon Sep 17 00:00:00 2001 From: Sean Preston Date: Mon, 11 Jul 2022 23:18:07 +0200 Subject: [PATCH 26/27] import order --- tests/api/v1/endpoints/test_privacy_request_endpoints.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/api/v1/endpoints/test_privacy_request_endpoints.py b/tests/api/v1/endpoints/test_privacy_request_endpoints.py index 36bf8aaab..edc78ca47 100644 --- a/tests/api/v1/endpoints/test_privacy_request_endpoints.py +++ b/tests/api/v1/endpoints/test_privacy_request_endpoints.py @@ -5,7 +5,6 @@ from datetime import datetime from typing import List from unittest import mock -from fidesops.schemas.redis_cache import PrivacyRequestIdentity import pytest from dateutil.parser import parse @@ -59,6 +58,7 @@ from fidesops.schemas.dataset import DryRunDatasetResponse from fidesops.schemas.masking.masking_secrets import SecretType from fidesops.schemas.policy import PolicyResponse +from fidesops.schemas.redis_cache import PrivacyRequestIdentity from fidesops.util.cache import ( get_encryption_cache_key, get_identity_cache_key, From 713d8127b3a6e4b4069a805983cc0743ef68fa3e Mon Sep 17 00:00:00 2001 From: Sean Preston Date: Tue, 12 Jul 2022 12:28:18 +0200 Subject: [PATCH 27/27] update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 265fbf3c0..ff5227e0d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ The types of changes are: * Adds endpoint to get available connectors (database and saas) [#768](https://github.com/ethyca/fidesops/pull/768) * Adds endpoint to get the secrets required for different connectors [#795](https://github.com/ethyca/fidesops/pull/795) * Store provided identity data in the privacy request table [#743](https://github.com/ethyca/fidesops/pull/834) +* Adds exact match identity search to the privacy request status endpoint [#765](https://github.com/ethyca/fidesops/pull/847/) ### Developer Experience * Replace user authentication routes with fideslib routes [#811](https://github.com/ethyca/fidesops/pull/811)