From c5688134636800474aa127a0e7cf5b57a0790804 Mon Sep 17 00:00:00 2001 From: Sean Preston Date: Tue, 10 May 2022 17:37:25 -0400 Subject: [PATCH] [#417] DRP status endpoint (#485) * schema for DRP privacy requests * URL for DRP status endpont * add new route, move fulfilment of queries into a util method * add tests, simplify endpoint to remove filtering * add mappings of privacyrequeststatus to DPR statuses * adds docstrings * remove unused import * fixes return type and adds docstring --- .../v1/endpoints/privacy_request_endpoints.py | 180 +++++++++++++----- src/fidesops/api/v1/urn_registry.py | 1 + src/fidesops/schemas/privacy_request.py | 30 +++ .../test_privacy_request_endpoints.py | 108 ++++++++++- tests/fixtures/application_fixtures.py | 25 ++- 5 files changed, 296 insertions(+), 48 deletions(-) diff --git a/src/fidesops/api/v1/endpoints/privacy_request_endpoints.py b/src/fidesops/api/v1/endpoints/privacy_request_endpoints.py index efe662a10..3af905d94 100644 --- a/src/fidesops/api/v1/endpoints/privacy_request_endpoints.py +++ b/src/fidesops/api/v1/endpoints/privacy_request_endpoints.py @@ -56,6 +56,8 @@ PrivacyRequestVerboseResponse, ReviewPrivacyRequestIds, DenyPrivacyRequests, + PrivacyRequestDRPStatusResponse, + PrivacyRequestDRPStatus, ) from fidesops.service.privacy_request.request_runner_service import PrivacyRequestRunner from fidesops.service.privacy_request.request_service import ( @@ -244,20 +246,32 @@ def privacy_request_csv_download( return response -@router.get( - urls.PRIVACY_REQUESTS, - dependencies=[Security(verify_oauth_client, scopes=[scopes.PRIVACY_REQUEST_READ])], - response_model=Page[ - Union[ - PrivacyRequestVerboseResponse, - PrivacyRequestResponse, - ] - ], -) -def get_request_status( - *, +def execution_logs_by_dataset_name( + self: PrivacyRequest, +) -> DefaultDict[str, List["ExecutionLog"]]: + """ + Returns a truncated list of ExecutionLogs for each dataset name associated with + a PrivacyRequest. Added as a conditional property to the PrivacyRequest class at runtime to + show optionally embedded execution logs. + + An example response might include your execution logs from your mongo db in one group, and execution logs from + your postgres db in a different group. + """ + + execution_logs: DefaultDict[str, List["ExecutionLog"]] = defaultdict(list) + + for log in self.execution_logs.order_by( + ExecutionLog.dataset_name, ExecutionLog.updated_at.asc() + ): + if len(execution_logs[log.dataset_name]) > EMBEDDED_EXECUTION_LOG_LIMIT - 1: + continue + execution_logs[log.dataset_name].append(log) + return execution_logs + + +def _filter_privacy_request_queryset( + query: Query, db: Session = Depends(deps.get_db), - params: Params = Depends(), id: Optional[str] = None, status: Optional[PrivacyRequestStatus] = None, created_lt: Optional[datetime] = None, @@ -269,16 +283,10 @@ def get_request_status( errored_lt: Optional[datetime] = None, errored_gt: Optional[datetime] = None, external_id: Optional[str] = None, - verbose: Optional[bool] = False, - include_identities: Optional[bool] = False, - download_csv: Optional[bool] = False, -) -> Union[StreamingResponse, AbstractPage[PrivacyRequest]]: - """Returns PrivacyRequest information. Supports a variety of optional query params. - - To fetch a single privacy request, use the id query param `?id=`. - To see individual execution logs, use the verbose query param `?verbose=True`. +) -> Query: + """ + Utility method to apply filters to our privacy request query """ - if any([completed_lt, completed_gt]) and any([errored_lt, errored_gt]): raise HTTPException( status_code=HTTP_400_BAD_REQUEST, @@ -304,8 +312,6 @@ def get_request_status( detail=f"Value specified for {field_name}_lt: {end} must be after {field_name}_gt: {start}.", ) - query = db.query(PrivacyRequest) - # Further restrict all PrivacyRequests by query params if id: query = query.filter(PrivacyRequest.id.ilike(f"{id}%")) @@ -342,6 +348,62 @@ def get_request_status( PrivacyRequest.finished_processing_at > errored_gt, ) + return query.order_by(PrivacyRequest.created_at.desc()) + + +@router.get( + urls.PRIVACY_REQUESTS, + dependencies=[Security(verify_oauth_client, scopes=[scopes.PRIVACY_REQUEST_READ])], + response_model=Page[ + Union[ + PrivacyRequestVerboseResponse, + PrivacyRequestResponse, + ] + ], +) +def get_request_status( + *, + db: Session = Depends(deps.get_db), + params: Params = Depends(), + id: Optional[str] = None, + status: Optional[PrivacyRequestStatus] = None, + created_lt: Optional[datetime] = None, + created_gt: Optional[datetime] = None, + started_lt: Optional[datetime] = None, + started_gt: Optional[datetime] = None, + completed_lt: Optional[datetime] = None, + completed_gt: Optional[datetime] = None, + errored_lt: Optional[datetime] = None, + errored_gt: Optional[datetime] = None, + external_id: Optional[str] = None, + verbose: Optional[bool] = False, + include_identities: Optional[bool] = False, + download_csv: Optional[bool] = False, +) -> Union[StreamingResponse, AbstractPage[PrivacyRequest]]: + """Returns PrivacyRequest information. Supports a variety of optional query params. + + To fetch a single privacy request, use the id query param `?id=`. + To see individual execution logs, use the verbose query param `?verbose=True`. + """ + + logger.info(f"Finding all request statuses with pagination params {params}") + query = db.query(PrivacyRequest) + query = _filter_privacy_request_queryset( + query, + db, + id, + status, + created_lt, + created_gt, + started_lt, + started_gt, + completed_lt, + completed_gt, + errored_lt, + errored_gt, + external_id, + ) + if download_csv: # Returning here if download_csv param was specified logger.info("Downloading privacy requests as csv") @@ -356,8 +418,6 @@ def get_request_status( else: PrivacyRequest.execution_logs_by_dataset = property(lambda self: None) - query = query.order_by(PrivacyRequest.created_at.desc()) - paginated = paginate(query, params) if include_identities: # Conditionally include the cached identity data in the response if @@ -365,31 +425,63 @@ def get_request_status( for item in paginated.items: item.identity = item.get_cached_identity_data() - logger.info(f"Finding all request statuses with pagination params {params}") return paginated -def execution_logs_by_dataset_name( - self: PrivacyRequest, -) -> DefaultDict[str, List["ExecutionLog"]]: - """ - Returns a truncated list of ExecutionLogs for each dataset name associated with - a PrivacyRequest. Added as a conditional property to the PrivacyRequest class at runtime to - show optionally embedded execution logs. +def _map_fidesops_status_to_drp_status( + status: PrivacyRequestStatus, +) -> PrivacyRequestDRPStatus: + PRIVACY_REQUEST_STATUS_TO_DRP_MAPPING: Dict[ + PrivacyRequestStatus, PrivacyRequestDRPStatus + ] = { + PrivacyRequestStatus.pending: PrivacyRequestDRPStatus.open, + PrivacyRequestStatus.approved: PrivacyRequestDRPStatus.in_progress, + PrivacyRequestStatus.denied: PrivacyRequestDRPStatus.denied, + PrivacyRequestStatus.in_processing: PrivacyRequestDRPStatus.in_progress, + PrivacyRequestStatus.complete: PrivacyRequestDRPStatus.fulfilled, + PrivacyRequestStatus.paused: PrivacyRequestDRPStatus.in_progress, + PrivacyRequestStatus.error: PrivacyRequestDRPStatus.expired, + } + try: + return PRIVACY_REQUEST_STATUS_TO_DRP_MAPPING[status] + except KeyError: + raise ValueError(f"Request has invalid DRP request status: {status.value}") - An example response might include your execution logs from your mongo db in one group, and execution logs from - your postgres db in a different group. + +@router.get( + urls.REQUEST_STATUS_DRP, + dependencies=[Security(verify_oauth_client, scopes=[scopes.PRIVACY_REQUEST_READ])], + response_model=PrivacyRequestDRPStatusResponse, +) +def get_request_status_drp( + privacy_request_id: str, + *, + db: Session = Depends(deps.get_db), +) -> PrivacyRequestDRPStatusResponse: + """ + Returns PrivacyRequest information where the respective privacy request is associated with + a policy that implements a Data Rights Protocol action. """ - execution_logs: DefaultDict[str, List["ExecutionLog"]] = defaultdict(list) + logger.info(f"Finding request for DRP with ID: {privacy_request_id}") + request = PrivacyRequest.get( + db=db, + id=privacy_request_id, + ) + if not request or not request.policy or not request.policy.drp_action: + # If no request is found with this ID, or that request has no policy, + # or that request's policy has no associated drp_action. + raise HTTPException( + status_code=HTTP_404_NOT_FOUND, + detail=f"Privacy request with ID {privacy_request_id} does not exist, or is not associated with a data rights protocol action.", + ) - for log in self.execution_logs.order_by( - ExecutionLog.dataset_name, ExecutionLog.updated_at.asc() - ): - if len(execution_logs[log.dataset_name]) > EMBEDDED_EXECUTION_LOG_LIMIT - 1: - continue - execution_logs[log.dataset_name].append(log) - return execution_logs + logger.info(f"Privacy request with ID: {privacy_request_id} found for DRP status.") + return PrivacyRequestDRPStatusResponse( + request_id=request.id, + received_at=request.requested_at, + status=_map_fidesops_status_to_drp_status(request.status), + ) @router.get( diff --git a/src/fidesops/api/v1/urn_registry.py b/src/fidesops/api/v1/urn_registry.py index be9b946b3..939fbd9e5 100644 --- a/src/fidesops/api/v1/urn_registry.py +++ b/src/fidesops/api/v1/urn_registry.py @@ -37,6 +37,7 @@ REQUEST_STATUS_LOGS = "/privacy-request/{privacy_request_id}/log" PRIVACY_REQUEST_RESUME = "/privacy-request/{privacy_request_id}/resume" REQUEST_PREVIEW = "/privacy-request/preview" +REQUEST_STATUS_DRP = "/privacy-request/{privacy_request_id}/drp" # Rule URLs RULE_LIST = "/policy/{policy_key}/rule" diff --git a/src/fidesops/schemas/privacy_request.py b/src/fidesops/schemas/privacy_request.py index 86c621217..352f9f276 100644 --- a/src/fidesops/schemas/privacy_request.py +++ b/src/fidesops/schemas/privacy_request.py @@ -1,4 +1,5 @@ from datetime import datetime +from enum import Enum as EnumType from typing import List, Optional, Dict from pydantic import Field, validator @@ -15,6 +16,35 @@ from fidesops.util.encryption.aes_gcm_encryption_scheme import verify_encryption_key +class PrivacyRequestDRPStatus(EnumType): + """A list of privacy request statuses specified by the Data Rights Protocol.""" + + open = "open" + in_progress = "in_progress" + fulfilled = "fulfilled" + revoked = "revoked" + denied = "denied" + expired = "expired" + + +class PrivacyRequestDRPStatusResponse(BaseSchema): + """A Fidesops PrivacyRequest updated to fit the Data Rights Protocol specification.""" + + request_id: str + received_at: datetime + expected_by: Optional[datetime] + processing_details: Optional[str] + status: PrivacyRequestDRPStatus + reason: Optional[str] + user_verification_url: Optional[str] + + class Config: + """Set orm_mode and use_enum_values""" + + orm_mode = True + use_enum_values = True + + class PrivacyRequestCreate(BaseSchema): """Data required to create a PrivacyRequest""" diff --git a/tests/api/v1/endpoints/test_privacy_request_endpoints.py b/tests/api/v1/endpoints/test_privacy_request_endpoints.py index 796985277..d68bd0292 100644 --- a/tests/api/v1/endpoints/test_privacy_request_endpoints.py +++ b/tests/api/v1/endpoints/test_privacy_request_endpoints.py @@ -6,12 +6,13 @@ import json from datetime import datetime from dateutil.parser import parse -from typing import List +from typing import Callable, List from unittest import mock from fastapi_pagination import Params import pytest from starlette.testclient import TestClient +from sqlalchemy.orm import Session from fidesops.api.v1.endpoints.privacy_request_endpoints import ( EMBEDDED_EXECUTION_LOG_LIMIT, @@ -24,6 +25,7 @@ DATASETS, PRIVACY_REQUEST_APPROVE, PRIVACY_REQUEST_DENY, + REQUEST_STATUS_DRP, ) from fidesops.api.v1.scope_registry import ( STORAGE_CREATE_OR_UPDATE, @@ -49,6 +51,7 @@ JWE_ISSUED_AT, ) from fidesops.schemas.masking.masking_secrets import SecretType +from fidesops.schemas.privacy_request import PrivacyRequestDRPStatus from fidesops.util.cache import ( get_identity_cache_key, get_encryption_cache_key, @@ -374,6 +377,109 @@ def test_create_privacy_request_no_identities( assert len(response_data) == 1 +class TestGetPrivacyRequestDRP: + """ + Tests for the endpoint retrieving privacy requests specific to the DRP. + """ + + @pytest.fixture(scope="function") + def url_for_privacy_request( + self, + privacy_request: PrivacyRequest, + ) -> str: + return V1_URL_PREFIX + REQUEST_STATUS_DRP.format( + privacy_request_id=privacy_request.id + ) + + @pytest.fixture(scope="function") + def url_for_privacy_request_with_drp_action( + self, + privacy_request_with_drp_action: PrivacyRequest, + ) -> str: + return V1_URL_PREFIX + REQUEST_STATUS_DRP.format( + privacy_request_id=privacy_request_with_drp_action.id + ) + + def test_get_privacy_requests_unauthenticated( + self, + api_client: TestClient, + url_for_privacy_request: str, + ): + response = api_client.get( + url_for_privacy_request, + headers={}, + ) + assert 401 == response.status_code + + def test_get_privacy_requests_wrong_scope( + self, + api_client: TestClient, + generate_auth_header: Callable, + url_for_privacy_request: str, + ): + auth_header = generate_auth_header(scopes=[STORAGE_CREATE_OR_UPDATE]) + response = api_client.get( + url_for_privacy_request, + headers=auth_header, + ) + assert 403 == response.status_code + + def test_get_non_drp_privacy_request( + self, + api_client: TestClient, + generate_auth_header: Callable, + url_for_privacy_request: str, + ): + auth_header = generate_auth_header(scopes=[PRIVACY_REQUEST_READ]) + response = api_client.get( + url_for_privacy_request, + headers=auth_header, + ) + assert 404 == response.status_code + privacy_request_id = url_for_privacy_request.split("/")[-2] + assert ( + response.json()["detail"] + == f"Privacy request with ID {privacy_request_id} does not exist, or is not associated with a data rights protocol action." + ) + + @pytest.mark.parametrize( + "privacy_request_status,expected_drp_status", + [ + (PrivacyRequestStatus.pending, PrivacyRequestDRPStatus.open), + (PrivacyRequestStatus.approved, PrivacyRequestDRPStatus.in_progress), + (PrivacyRequestStatus.denied, PrivacyRequestDRPStatus.denied), + (PrivacyRequestStatus.in_processing, PrivacyRequestDRPStatus.in_progress), + (PrivacyRequestStatus.complete, PrivacyRequestDRPStatus.fulfilled), + (PrivacyRequestStatus.paused, PrivacyRequestDRPStatus.in_progress), + (PrivacyRequestStatus.error, PrivacyRequestDRPStatus.expired), + ], + ) + def test_get_privacy_request_with_drp_action( + self, + api_client: TestClient, + db: Session, + generate_auth_header: Callable, + url_for_privacy_request_with_drp_action: str, + privacy_request_with_drp_action: PrivacyRequest, + privacy_request_status: PrivacyRequestStatus, + expected_drp_status: PrivacyRequestDRPStatus, + ): + privacy_request_with_drp_action.status = privacy_request_status + privacy_request_with_drp_action.save(db=db) + auth_header = generate_auth_header(scopes=[PRIVACY_REQUEST_READ]) + response = api_client.get( + url_for_privacy_request_with_drp_action, + headers=auth_header, + ) + assert 200 == response.status_code + assert expected_drp_status.value == response.json()["status"] + assert privacy_request_with_drp_action.id == response.json()["request_id"] + assert ( + privacy_request_with_drp_action.requested_at.isoformat() + == response.json()["received_at"] + ) + + class TestGetPrivacyRequests: @pytest.fixture(scope="function") def url(self, oauth_client: ClientDetail) -> str: diff --git a/tests/fixtures/application_fixtures.py b/tests/fixtures/application_fixtures.py index 11a99f0ef..3af2185ce 100644 --- a/tests/fixtures/application_fixtures.py +++ b/tests/fixtures/application_fixtures.py @@ -692,9 +692,11 @@ def privacy_requests(db: Session, policy: Policy) -> Generator: pr.delete(db) -@pytest.fixture(scope="function") -def privacy_request(db: Session, policy: Policy) -> PrivacyRequest: - privacy_request = PrivacyRequest.create( +def _create_privacy_request_for_policy( + db: Session, + policy: Policy, +) -> PrivacyRequest: + return PrivacyRequest.create( db=db, data={ "external_id": f"ext-{str(uuid4())}", @@ -724,6 +726,23 @@ def privacy_request(db: Session, policy: Policy) -> PrivacyRequest: "client_id": policy.client_id, }, ) + + +@pytest.fixture(scope="function") +def privacy_request(db: Session, policy: Policy) -> PrivacyRequest: + privacy_request = _create_privacy_request_for_policy(db, policy) + yield privacy_request + privacy_request.delete(db) + + +@pytest.fixture(scope="function") +def privacy_request_with_drp_action( + db: Session, policy_drp_action: Policy +) -> PrivacyRequest: + privacy_request = _create_privacy_request_for_policy( + db, + policy_drp_action, + ) yield privacy_request privacy_request.delete(db)