From ce471afd4ed19d0fd4e9fdaffd60cef999911231 Mon Sep 17 00:00:00 2001 From: Paul Sanders Date: Tue, 5 Jul 2022 16:46:56 -0400 Subject: [PATCH] Resolve issue with MyPy seeing files in fidesops as missing imports (#719) Co-authored-by: Paul Sanders --- CHANGELOG.md | 3 ++ MANIFEST.in | 1 + dev-requirements.txt | 1 + pyproject.toml | 25 ++++++++++++- .../api/v1/endpoints/connection_endpoints.py | 11 +++--- .../api/v1/endpoints/dataset_endpoints.py | 7 ++-- .../api/v1/endpoints/drp_endpoints.py | 9 +++-- .../api/v1/endpoints/encryption_endpoints.py | 2 +- .../api/v1/endpoints/oauth_endpoints.py | 6 ++-- .../v1/endpoints/policy_webhook_endpoints.py | 22 ++++++------ .../v1/endpoints/privacy_request_endpoints.py | 14 ++++---- .../api/v1/endpoints/saas_config_endpoints.py | 10 +++--- src/fidesops/api/v1/exception_handlers.py | 6 ++-- src/fidesops/graph/config.py | 6 ++-- src/fidesops/graph/graph.py | 6 ++-- src/fidesops/graph/traversal.py | 3 +- src/fidesops/models/datasetconfig.py | 7 ++-- src/fidesops/models/policy.py | 2 +- src/fidesops/models/privacy_request.py | 12 +++---- src/fidesops/models/storage.py | 4 +-- src/{__init__.py => fidesops/py.typed} | 0 .../connection_configuration/__init__.py | 8 +++-- src/fidesops/schemas/dataset.py | 2 +- src/fidesops/schemas/external_https.py | 2 +- src/fidesops/schemas/policy.py | 2 +- src/fidesops/schemas/privacy_request.py | 2 +- src/fidesops/schemas/saas/saas_config.py | 6 ++-- .../schemas/saas/strategy_configuration.py | 4 +-- .../authentication_strategy_basic.py | 7 ++-- .../authentication_strategy_bearer.py | 6 ++-- .../authentication_strategy_oauth2.py | 25 ++++++------- .../authentication_strategy_query_param.py | 6 ++-- src/fidesops/service/connectors/__init__.py | 2 +- .../service/connectors/base_connector.py | 2 +- .../service/connectors/manual_connector.py | 4 +-- .../service/connectors/query_config.py | 14 ++++---- .../connectors/saas/authenticated_client.py | 19 ++++++---- .../service/connectors/saas_connector.py | 35 +++++++++---------- .../service/connectors/saas_query_config.py | 12 ++++--- .../service/connectors/sql_connector.py | 4 +-- .../strategy/masking_strategy_aes_encrypt.py | 21 ++++++----- .../strategy/masking_strategy_factory.py | 6 ++-- .../masking/strategy/masking_strategy_hash.py | 9 +++-- .../masking/strategy/masking_strategy_hmac.py | 11 +++--- .../strategy/masking_strategy_nullify.py | 2 +- .../masking_strategy_random_string_rewrite.py | 2 +- .../masking_strategy_string_rewrite.py | 2 +- .../service/pagination/pagination_strategy.py | 8 +++-- .../pagination/pagination_strategy_cursor.py | 2 +- .../pagination/pagination_strategy_factory.py | 10 ++++-- .../pagination/pagination_strategy_link.py | 2 +- .../pagination/pagination_strategy_offset.py | 6 ++-- .../privacy_request/onetrust_service.py | 6 ++-- .../privacy_request/request_runner_service.py | 12 +++---- .../privacy_request/request_service.py | 10 ++---- .../post_processor_strategy_filter.py | 8 ++--- .../post_processor_strategy_unwrap.py | 2 +- .../storage/storage_uploader_service.py | 11 +++--- src/fidesops/task/graph_task.py | 20 +++++------ src/fidesops/task/task_resources.py | 2 +- src/fidesops/tasks/storage.py | 14 ++++---- src/fidesops/util/encryption/secrets_util.py | 6 ++-- src/fidesops/util/saas_util.py | 10 +++--- 63 files changed, 284 insertions(+), 217 deletions(-) rename src/{__init__.py => fidesops/py.typed} (100%) diff --git a/CHANGELOG.md b/CHANGELOG.md index fd63bc552..dba485b3e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,9 @@ The types of changes are: ## [Unreleased](https://github.com/ethyca/fidesops/compare/1.6.1...main) * Add support for multiple statuses to be selected for filtering subject requests [#660](https://github.com/ethyca/fidesops/pull/802) +### Fixed +* Resolve issue with MyPy seeing files in fidesops as missing imports [719](https://github.com/ethyca/fidesops/pull/719) + ## [1.6.1](https://github.com/ethyca/fidesops/compare/1.6.0...1.6.1) ### Added diff --git a/MANIFEST.in b/MANIFEST.in index 570a7a81e..010b6bde9 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -5,6 +5,7 @@ include dev-requirements.txt include versioneer.py include src/fidesops/alembic.ini include src/fidesops/_version.py +include src/fidesops/py.typed graft src/fidesops/migrations exclude src/fidesops/migrations/README exclude src/fidesops/migrations/script.py.mako diff --git a/dev-requirements.txt b/dev-requirements.txt index a8408dcf4..7cfbbff7d 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -15,3 +15,4 @@ types-PyYAML==6.0.9 types-redis==4.3.2 types-toml==0.10.7 types-ujson==5.2.0 +types-urllib3==1.26.15 diff --git a/pyproject.toml b/pyproject.toml index fe17e43b2..f20bab9fa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,6 @@ requires = ["setuptools", "wheel", "versioneer-518"] # PEP 508 specifications. [tool.mypy] exclude = ["fidesops.migrations.*"] warn_unused_configs = true -ignore_missing_imports = true pretty = true plugins = ["pydantic.mypy", "sqlmypy"] disallow_untyped_defs = true @@ -17,6 +16,30 @@ show_error_codes = true module = ["tests.*"] disallow_untyped_defs = false +[[tool.mypy.overrides]] +module = [ + "alembic.*", + "apscheduler.*", + "boto3.*", + "botocore.*", + "bson.*", + "celery.*", + "dask.*", + "fideslang.*", + "fideslib.*", + "fideslog.*", + "jose.*", + "jwt.*", + "multidimensional_urlencode.*", + "pandas.*", + "pydash.*", + "pymongo.*", + "snowflake.*", + "sqlalchemy_utils.*", + "uvicorn.*" +] +ignore_missing_imports = true + ####### # Black ####### diff --git a/src/fidesops/api/v1/endpoints/connection_endpoints.py b/src/fidesops/api/v1/endpoints/connection_endpoints.py index 0d579571f..796a28aef 100644 --- a/src/fidesops/api/v1/endpoints/connection_endpoints.py +++ b/src/fidesops/api/v1/endpoints/connection_endpoints.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import logging from typing import List, Optional @@ -246,7 +248,7 @@ def validate_secrets( ) try: - schema = get_connection_secrets_validator(connection_type.value, saas_config) + schema = get_connection_secrets_validator(connection_type.value, saas_config) # type: ignore logger.info( f"Validating secrets on connection config with key '{connection_config.key}'" ) @@ -266,7 +268,8 @@ def connection_status( connector = get_connector(connection_config) try: - status: ConnectionTestStatus = connector.test_connection() + status: ConnectionTestStatus | None = connector.test_connection() + except (ConnectionException, ClientUnsuccessfulException) as exc: logger.warning( "Connection test failed on %s: %s", @@ -282,8 +285,8 @@ def connection_status( failure_reason=str(exc), ) - logger.info(f"Connection test {status.value} on {connection_config.key}") - connection_config.update_test_status(test_status=status, db=db) + logger.info(f"Connection test {status.value} on {connection_config.key}") # type: ignore + connection_config.update_test_status(test_status=status, db=db) # type: ignore return TestStatusMessage( msg=msg, diff --git a/src/fidesops/api/v1/endpoints/dataset_endpoints.py b/src/fidesops/api/v1/endpoints/dataset_endpoints.py index cfe63a9f8..49aa626fe 100644 --- a/src/fidesops/api/v1/endpoints/dataset_endpoints.py +++ b/src/fidesops/api/v1/endpoints/dataset_endpoints.py @@ -102,15 +102,14 @@ def validate_dataset( try: # Attempt to generate a traversal for this dataset by providing an empty # dictionary of all unique identity keys - graph = convert_dataset_to_graph(dataset, connection_config.key) + graph = convert_dataset_to_graph(dataset, connection_config.key) # type: ignore # Datasets for SaaS connections need to be merged with a SaaS config to # be able to generate a valid traversal if connection_config.connection_type == ConnectionType.saas: _validate_saas_dataset(connection_config, dataset) graph = merge_datasets( - graph, - connection_config.get_saas_config().get_graph(), + graph, connection_config.get_saas_config().get_graph() # type: ignore ) complete_graph = DatasetGraph(graph) unique_identities = set(complete_graph.identity_keys.values()) @@ -247,7 +246,7 @@ def create_or_update_dataset( ) -> None: try: if connection_config.connection_type == ConnectionType.saas: - _validate_saas_dataset(connection_config, dataset) + _validate_saas_dataset(connection_config, dataset) # type: ignore # Try to find an existing DatasetConfig matching the given connection & key dataset_config = DatasetConfig.create_or_update(db, data=data) created_or_updated.append(dataset_config.dataset) diff --git a/src/fidesops/api/v1/endpoints/drp_endpoints.py b/src/fidesops/api/v1/endpoints/drp_endpoints.py index 3ea0bf2a5..ae1abefc7 100644 --- a/src/fidesops/api/v1/endpoints/drp_endpoints.py +++ b/src/fidesops/api/v1/endpoints/drp_endpoints.py @@ -110,7 +110,7 @@ def create_drp_privacy_request( return PrivacyRequestDRPStatusResponse( request_id=privacy_request.id, received_at=privacy_request.requested_at, - status=DrpFidesopsMapper.map_status(privacy_request.status), + status=DrpFidesopsMapper.map_status(privacy_request.status), # type: ignore ) except common_exceptions.RedisConnectionError as exc: @@ -174,7 +174,7 @@ def get_drp_data_rights(*, db: Session = Depends(deps.get_db)) -> DrpDataRightsR logger.info("Fetching available DRP data rights") actions: List[DrpAction] = [ - item.drp_action + item.drp_action # type: ignore for item in db.query(Policy.drp_action).filter(Policy.drp_action.isnot(None)) ] @@ -201,8 +201,7 @@ def revoke_request( if privacy_request.status != PrivacyRequestStatus.pending: raise HTTPException( status_code=HTTP_400_BAD_REQUEST, - detail=f"Invalid revoke request. Can only revoke `pending` requests. " - f"Privacy request '{privacy_request.id}' status = {privacy_request.status.value}.", + detail=f"Invalid revoke request. Can only revoke `pending` requests. Privacy request '{privacy_request.id}' status = {privacy_request.status.value}.", # type: ignore ) logger.info(f"Canceling privacy request '{privacy_request.id}'") @@ -211,6 +210,6 @@ def revoke_request( return PrivacyRequestDRPStatusResponse( request_id=privacy_request.id, received_at=privacy_request.requested_at, - status=DrpFidesopsMapper.map_status(privacy_request.status), + status=DrpFidesopsMapper.map_status(privacy_request.status), # type: ignore reason=data.reason, ) diff --git a/src/fidesops/api/v1/endpoints/encryption_endpoints.py b/src/fidesops/api/v1/endpoints/encryption_endpoints.py index 1370f7419..26e7f482f 100644 --- a/src/fidesops/api/v1/endpoints/encryption_endpoints.py +++ b/src/fidesops/api/v1/endpoints/encryption_endpoints.py @@ -56,7 +56,7 @@ def aes_encrypt(encryption_request: AesEncryptionRequest) -> AesEncryptionRespon encrypted_value: str = aes_gcm_encrypt( encryption_request.value, - encryption_request.key, + encryption_request.key, # type: ignore nonce, ) return AesEncryptionResponse( diff --git a/src/fidesops/api/v1/endpoints/oauth_endpoints.py b/src/fidesops/api/v1/endpoints/oauth_endpoints.py index 98b2a8eb9..0168e9e05 100644 --- a/src/fidesops/api/v1/endpoints/oauth_endpoints.py +++ b/src/fidesops/api/v1/endpoints/oauth_endpoints.py @@ -206,10 +206,10 @@ def oauth_callback(code: str, state: str, db: Session = Depends(get_db)) -> None try: authentication = ( - connection_config.get_saas_config().client_config.authentication + connection_config.get_saas_config().client_config.authentication # type: ignore ) - auth_strategy: OAuth2AuthenticationStrategy = get_strategy( - authentication.strategy, authentication.configuration + auth_strategy: OAuth2AuthenticationStrategy = get_strategy( # type: ignore + authentication.strategy, authentication.configuration # type: ignore ) auth_strategy.get_access_token(db, code, connection_config) except (OAuth2TokenException, FidesopsException) as exc: diff --git a/src/fidesops/api/v1/endpoints/policy_webhook_endpoints.py b/src/fidesops/api/v1/endpoints/policy_webhook_endpoints.py index 6231cd41c..4a238d60a 100644 --- a/src/fidesops/api/v1/endpoints/policy_webhook_endpoints.py +++ b/src/fidesops/api/v1/endpoints/policy_webhook_endpoints.py @@ -139,7 +139,9 @@ def put_webhooks( staged_webhook_keys = [webhook.key for webhook in staged_webhooks] webhooks_to_remove = getattr( policy, f"{webhook_cls.prefix}_execution_webhooks" - ).filter(webhook_cls.key.not_in(staged_webhook_keys)) + ).filter( + webhook_cls.key.not_in(staged_webhook_keys) # type: ignore + ) if webhooks_to_remove.count(): logger.info( @@ -177,7 +179,7 @@ def create_or_update_pre_execution_webhooks( All webhooks must be included in the request in the desired order. Any missing webhooks from the request body will be removed. """ - return put_webhooks(PolicyPreWebhook, policy_key, db, webhooks) + return put_webhooks(PolicyPreWebhook, policy_key, db, webhooks) # type: ignore @router.put( @@ -200,7 +202,7 @@ def create_or_update_post_execution_webhooks( All webhooks must be included in the request in the desired order. Any missing webhooks from the request body will be removed. """ - return put_webhooks(PolicyPostWebhook, policy_key, db, webhooks) + return put_webhooks(PolicyPostWebhook, policy_key, db, webhooks) # type: ignore def get_policy_webhook_or_error( @@ -246,7 +248,7 @@ def get_policy_pre_execution_webhook( Loads the given Pre-Execution Webhook on the Policy """ policy = get_policy_or_error(db, policy_key) - return get_policy_webhook_or_error(db, policy, pre_webhook_key, PolicyPreWebhook) + return get_policy_webhook_or_error(db, policy, pre_webhook_key, PolicyPreWebhook) # type: ignore @router.get( @@ -265,7 +267,7 @@ def get_policy_post_execution_webhook( Loads the given Post-Execution Webhook on the Policy """ policy = get_policy_or_error(db, policy_key) - return get_policy_webhook_or_error(db, policy, post_webhook_key, PolicyPostWebhook) + return get_policy_webhook_or_error(db, policy, post_webhook_key, PolicyPostWebhook) # type: ignore def _patch_webhook( @@ -286,7 +288,7 @@ def _patch_webhook( if data.get("connection_config_key"): connection_config = get_connection_config_or_error( - db, data.get("connection_config_key") + db, data.get("connection_config_key") # type: ignore ) data["connection_config_id"] = connection_config.id @@ -355,7 +357,7 @@ def update_pre_execution_webhook( policy_key=policy_key, webhook_key=pre_webhook_key, webhook_body=webhook_body, - webhook_cls=PolicyPreWebhook, + webhook_cls=PolicyPreWebhook, # type: ignore ) @@ -383,7 +385,7 @@ def update_post_execution_webhook( policy_key=policy_key, webhook_key=post_webhook_key, webhook_body=webhook_body, - webhook_cls=PolicyPostWebhook, + webhook_cls=PolicyPostWebhook, # type: ignore ) @@ -442,7 +444,7 @@ def delete_pre_execution_webhook( db=db, policy_key=policy_key, webhook_key=pre_webhook_key, - webhook_cls=PolicyPreWebhook, + webhook_cls=PolicyPreWebhook, # type: ignore ) @@ -463,5 +465,5 @@ def delete_post_execution_webhook( db=db, policy_key=policy_key, webhook_key=post_webhook_key, - webhook_cls=PolicyPostWebhook, + webhook_cls=PolicyPostWebhook, # type: ignore ) diff --git a/src/fidesops/api/v1/endpoints/privacy_request_endpoints.py b/src/fidesops/api/v1/endpoints/privacy_request_endpoints.py index 2d4725e8d..fd2007baf 100644 --- a/src/fidesops/api/v1/endpoints/privacy_request_endpoints.py +++ b/src/fidesops/api/v1/endpoints/privacy_request_endpoints.py @@ -400,9 +400,9 @@ def attach_resume_instructions(privacy_request: PrivacyRequest) -> None: resume_endpoint = PRIVACY_REQUEST_RETRY if stopped_collection_details: - stopped_collection_details.step = stopped_collection_details.step.value + stopped_collection_details.step = stopped_collection_details.step.value # type: ignore stopped_collection_details.collection = ( - stopped_collection_details.collection.value + stopped_collection_details.collection.value # type: ignore ) privacy_request.stopped_collection_details = stopped_collection_details @@ -613,12 +613,12 @@ def resume_privacy_request( ) -> PrivacyRequestResponse: """Resume running a privacy request after it was paused by a Pre-Execution webhook""" privacy_request = get_privacy_request_or_error(db, privacy_request_id) - privacy_request.cache_identity(webhook_callback.derived_identity) + privacy_request.cache_identity(webhook_callback.derived_identity) # type: ignore if privacy_request.status != PrivacyRequestStatus.paused: raise HTTPException( status_code=HTTP_400_BAD_REQUEST, - detail=f"Invalid resume request: privacy request '{privacy_request.id}' status = {privacy_request.status.value}.", + detail=f"Invalid resume request: privacy request '{privacy_request.id}' status = {privacy_request.status.value}.", # type: ignore ) logger.info( @@ -670,7 +670,7 @@ def resume_privacy_request_with_manual_input( if privacy_request.status != PrivacyRequestStatus.paused: raise HTTPException( status_code=HTTP_400_BAD_REQUEST, - detail=f"Invalid resume request: privacy request '{privacy_request.id}' " + detail=f"Invalid resume request: privacy request '{privacy_request.id}' " # type: ignore f"status = {privacy_request.status.value}. Privacy request is not paused.", ) @@ -715,7 +715,7 @@ def resume_privacy_request_with_manual_input( logger.info( f"Caching manually erased row count for privacy request '{privacy_request_id}', collection: '{paused_collection}'" ) - privacy_request.cache_manual_erasure_count(paused_collection, manual_count) + privacy_request.cache_manual_erasure_count(paused_collection, manual_count) # type: ignore logger.info( f"Resuming privacy request '{privacy_request_id}', {paused_step.value} step, from collection " @@ -811,7 +811,7 @@ def restart_privacy_request_from_failure( if privacy_request.status != PrivacyRequestStatus.error: raise HTTPException( status_code=HTTP_400_BAD_REQUEST, - detail=f"Cannot restart privacy request from failure: privacy request '{privacy_request.id}' status = {privacy_request.status.value}.", + detail=f"Cannot restart privacy request from failure: privacy request '{privacy_request.id}' status = {privacy_request.status.value}.", # type: ignore ) failed_details: Optional[ diff --git a/src/fidesops/api/v1/endpoints/saas_config_endpoints.py b/src/fidesops/api/v1/endpoints/saas_config_endpoints.py index 837267f1a..e5675b2ba 100644 --- a/src/fidesops/api/v1/endpoints/saas_config_endpoints.py +++ b/src/fidesops/api/v1/endpoints/saas_config_endpoints.py @@ -85,7 +85,7 @@ def verify_oauth_connection_config( detail="The connection config does not contain a SaaS config.", ) - authentication = connection_config.get_saas_config().client_config.authentication + authentication = saas_config.client_config.authentication if not authentication: raise HTTPException( status_code=HTTP_422_UNPROCESSABLE_ENTITY, @@ -145,7 +145,7 @@ def patch_saas_config( f"Updating SaaS config '{saas_config.fides_key}' on connection config '{connection_config.key}'" ) connection_config.update_saas_config(db, saas_config=saas_config) - return connection_config.saas_config + return connection_config.saas_config # type: ignore @router.get( @@ -165,7 +165,7 @@ def get_saas_config( status_code=HTTP_404_NOT_FOUND, detail=f"No SaaS config found for connection '{connection_config.key}'", ) - return connection_config.saas_config + return saas_config @router.delete( @@ -233,11 +233,11 @@ def authorize_connection( """Returns the authorization URL for the SaaS Connector (if available)""" verify_oauth_connection_config(connection_config) - authentication = connection_config.get_saas_config().client_config.authentication + authentication = connection_config.get_saas_config().client_config.authentication # type: ignore try: auth_strategy: OAuth2AuthenticationStrategy = get_strategy( - authentication.strategy, authentication.configuration + authentication.strategy, authentication.configuration # type: ignore ) return auth_strategy.get_authorization_url(db, connection_config) except FidesopsException as exc: diff --git a/src/fidesops/api/v1/exception_handlers.py b/src/fidesops/api/v1/exception_handlers.py index 9065e4bb1..ca2a01663 100644 --- a/src/fidesops/api/v1/exception_handlers.py +++ b/src/fidesops/api/v1/exception_handlers.py @@ -1,7 +1,7 @@ from typing import Callable, List from fastapi import Request -from fastapi.responses import JSONResponse, Response +from fastapi.responses import JSONResponse from starlette.status import HTTP_500_INTERNAL_SERVER_ERROR from fidesops.common_exceptions import FunctionalityNotConfigured @@ -17,5 +17,7 @@ def functionality_not_configured_handler( ) @classmethod - def get_handlers(cls) -> List[Callable[[Request, Exception], Response]]: + def get_handlers( + cls, + ) -> List[Callable[[Request, FunctionalityNotConfigured], JSONResponse]]: return [ExceptionHandlers.functionality_not_configured_handler] diff --git a/src/fidesops/graph/config.py b/src/fidesops/graph/config.py index a3d326ffd..e9fa1e4ee 100644 --- a/src/fidesops/graph/config.py +++ b/src/fidesops/graph/config.py @@ -344,7 +344,7 @@ def collect_matching(self, func: Callable[[Field], bool]) -> Dict[FieldPath, Fie *[field.collect_matching(func) for field in self.fields.values()] ) return merge_dicts( - base, + base, # type: ignore { field_path.prepend(self.name): field # pylint: disable=no-member for field_path, field in child_dicts.items() @@ -367,7 +367,7 @@ def generate_field( data_categories: Optional[List[str]], identity: Optional[str], data_type_name: str, - references: List[Tuple[FieldAddress, EdgeDirection]], + references: List[Tuple[FieldAddress, Optional[EdgeDirection]]], is_pk: bool, length: Optional[int], is_array: bool, @@ -452,7 +452,7 @@ def references( if field.references } - def identities(self) -> Dict[FieldPath, Tuple[str, ...]]: + def identities(self) -> Dict[FieldPath, str]: """return identity pointers included in the table""" return { field_path: field.identity # type: ignore diff --git a/src/fidesops/graph/graph.py b/src/fidesops/graph/graph.py index 38d29f27d..88dd4c4ba 100644 --- a/src/fidesops/graph/graph.py +++ b/src/fidesops/graph/graph.py @@ -218,9 +218,11 @@ def __init__(self, *datasets: Dataset) -> None: ) # collect all seed references - self.identity_keys: Dict[FieldAddress, SeedAddress] = { + self.identity_keys: dict[FieldAddress, SeedAddress] = { FieldAddress( - node.address.dataset, node.address.collection, *field_path.levels + node.address.dataset, + node.address.collection, + *field_path.levels, ): seed_address for node in nodes for field_path, seed_address in node.collection.identities().items() diff --git a/src/fidesops/graph/traversal.py b/src/fidesops/graph/traversal.py index 8d39199a1..2266d834f 100644 --- a/src/fidesops/graph/traversal.py +++ b/src/fidesops/graph/traversal.py @@ -104,7 +104,8 @@ def typed_filtered_values(self, input_data: Dict[str, List[Any]]) -> Dict[str, A out = {} for key, values in input_data.items(): path: FieldPath = FieldPath.parse(key) - field: Field = self.node.collection.field(path) + field: Field | None = self.node.collection.field(path) + if field and path in self.query_field_paths and isinstance(values, list): cast_values = [field.cast(v) for v in values] filtered = list(filter(lambda x: x is not None, cast_values)) diff --git a/src/fidesops/models/datasetconfig.py b/src/fidesops/models/datasetconfig.py index bbaea35ec..74461325e 100644 --- a/src/fidesops/models/datasetconfig.py +++ b/src/fidesops/models/datasetconfig.py @@ -77,16 +77,17 @@ def get_graph(self) -> Dataset: the corresponding SaaS config is merged in as well """ dataset_graph = convert_dataset_to_graph( - FidesopsDataset(**self.dataset), self.connection_config.key + FidesopsDataset(**self.dataset), self.connection_config.key # type: ignore ) if ( self.connection_config.connection_type == ConnectionType.saas and self.connection_config.saas_config is not None and self.connection_config.saas_config["fides_key"] == self.fides_key ): + dataset_graph = merge_datasets( dataset_graph, - self.connection_config.get_saas_config().get_graph(), + self.connection_config.get_saas_config().get_graph(), # type: ignore ) else: logger.debug( @@ -164,7 +165,7 @@ def to_graph_field( name=field.name, data_categories=field.data_categories, identity=identity, - data_type_name=data_type_name, + data_type_name=data_type_name, # type: ignore references=references, is_pk=is_pk, length=length, diff --git a/src/fidesops/models/policy.py b/src/fidesops/models/policy.py index 6c9327668..5732b185a 100644 --- a/src/fidesops/models/policy.py +++ b/src/fidesops/models/policy.py @@ -191,7 +191,7 @@ def _validate_rule_target_collection(target_categories: List[str]) -> None: for cat in target_categories: # Here we check that `cat` is not an ancestor of any other category within `target_categories` is_ancestor, ancestor_fides_key = _is_ancestor_of_contained_categories( - fides_key=cat, + fides_key=cat, # type: ignore data_categories=target_categories, ) if is_ancestor: diff --git a/src/fidesops/models/privacy_request.py b/src/fidesops/models/privacy_request.py index c7750c750..6d94cd9cb 100644 --- a/src/fidesops/models/privacy_request.py +++ b/src/fidesops/models/privacy_request.py @@ -71,7 +71,7 @@ class ManualAction(BaseSchema): @root_validator @classmethod - def get_or_update_details( + def get_or_update_details( # type: ignore cls: BaseSchema, values: Dict[str, Any] ) -> Dict[str, Any]: """Validate that 'get' or 'update' details are supplied.""" @@ -390,7 +390,7 @@ def get_manual_erasure_count(self, collection: CollectionAddress) -> Optional[in """ cache: FidesopsRedis = get_cache() prefix = f"MANUAL_MASK__{self.id}__{collection.value}" - value_dict: Optional[Dict[str, int]] = cache.get_encoded_objects_by_prefix( + value_dict: Optional[Dict[str, int]] = cache.get_encoded_objects_by_prefix( # type: ignore prefix ) return list(value_dict.values())[0] if value_dict else None @@ -405,10 +405,10 @@ def trigger_policy_webhook(self, webhook: WebhookTypes) -> None: # temp fix for circular dependency from fidesops.service.connectors import HTTPSConnector, get_connector - https_connector: HTTPSConnector = get_connector(webhook.connection_config) + https_connector: HTTPSConnector = get_connector(webhook.connection_config) # type: ignore request_body = SecondPartyRequestFormat( privacy_request_id=self.id, - direction=webhook.direction.value, + direction=webhook.direction.value, # type: ignore callback_type=webhook.prefix, identity=self.get_cached_identity_data(), ) @@ -423,7 +423,7 @@ def trigger_policy_webhook(self, webhook: WebhookTypes) -> None: } logger.info(f"Calling webhook {webhook.key} for privacy_request {self.id}") - response: Optional[SecondPartyResponseFormat] = https_connector.execute( + response: Optional[SecondPartyResponseFormat] = https_connector.execute( # type: ignore request_body.dict(), response_expected=response_expected, additional_headers=headers, @@ -431,7 +431,7 @@ def trigger_policy_webhook(self, webhook: WebhookTypes) -> None: if not response: return - response_body = SecondPartyResponseFormat(**response) + response_body = SecondPartyResponseFormat(**response) # type: ignore # Cache any new identities if response_body.derived_identity and any( diff --git a/src/fidesops/models/storage.py b/src/fidesops/models/storage.py index e83e276f6..6d4f2a9ca 100644 --- a/src/fidesops/models/storage.py +++ b/src/fidesops/models/storage.py @@ -44,7 +44,7 @@ def get_schema_for_secrets( ) try: - return schema.parse_obj(secrets) + return schema.parse_obj(secrets) # type: ignore except ValidationError as exc: # Pydantic requires validators raise either a ValueError, TypeError, or AssertionError # so this exception is cast into a `ValueError`. @@ -90,7 +90,7 @@ def set_secrets( try: get_schema_for_secrets( - storage_type=storage_type, + storage_type=storage_type, # type: ignore secrets=storage_secrets, ) except ( diff --git a/src/__init__.py b/src/fidesops/py.typed similarity index 100% rename from src/__init__.py rename to src/fidesops/py.typed diff --git a/src/fidesops/schemas/connection_configuration/__init__.py b/src/fidesops/schemas/connection_configuration/__init__.py index ce4833cb9..5d6ebbfbb 100644 --- a/src/fidesops/schemas/connection_configuration/__init__.py +++ b/src/fidesops/schemas/connection_configuration/__init__.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Union +from typing import Any, Dict, Optional, Union from fidesops.models.connectionconfig import ConnectionType from fidesops.schemas.connection_configuration.connection_secrets import ( @@ -60,7 +60,7 @@ def get_connection_secrets_validator( - connection_type: str, saas_config: SaaSConfig + connection_type: str, saas_config: Optional[SaaSConfig] ) -> ConnectionConfigSecretsSchema: """ Returns a validation schema for the connection "secrets" depending on the connection_type. @@ -68,6 +68,10 @@ def get_connection_secrets_validator( For example, a "postgres" connection_type would need to have its secrets validated against a PostgreSQL schema. """ + if connection_type == "saas" and not saas_config: + raise ValueError( + "A SaaS config to validate the secrets is required for a saas connection" + ) try: schema = ( SaaSSchemaFactory(saas_config).get_saas_schema() diff --git a/src/fidesops/schemas/dataset.py b/src/fidesops/schemas/dataset.py index 8f2417b92..13150bd12 100644 --- a/src/fidesops/schemas/dataset.py +++ b/src/fidesops/schemas/dataset.py @@ -33,7 +33,7 @@ def _valid_data_type(data_type_str: Optional[str]) -> Optional[str]: """If the data_type is provided ensure that it is a member of DataType.""" dt, _ = parse_data_type_string(data_type_str) - if not is_valid_data_type(dt): + if not is_valid_data_type(dt): # type: ignore raise InvalidDataTypeValidationError( f"The data type {data_type_str} is not supported." ) diff --git a/src/fidesops/schemas/external_https.py b/src/fidesops/schemas/external_https.py index ced7dcc36..f66f1ff7b 100644 --- a/src/fidesops/schemas/external_https.py +++ b/src/fidesops/schemas/external_https.py @@ -46,7 +46,7 @@ class Config: class PrivacyRequestResumeFormat(BaseModel): """Expected request body to resume a privacy request after it was paused by a webhook""" - derived_identity: Optional[PrivacyRequestIdentity] = {} + derived_identity: Optional[PrivacyRequestIdentity] = {} # type: ignore class Config: """Using enum values""" diff --git a/src/fidesops/schemas/policy.py b/src/fidesops/schemas/policy.py index 0f82bba23..b2b61d8ee 100644 --- a/src/fidesops/schemas/policy.py +++ b/src/fidesops/schemas/policy.py @@ -31,7 +31,7 @@ class RuleTarget(BaseSchema): name: Optional[str] key: Optional[FidesOpsKey] - data_category: DataCategory + data_category: DataCategory # type: ignore class Config: """Populate models with the raw value of enum fields, rather than the enum itself""" diff --git a/src/fidesops/schemas/privacy_request.py b/src/fidesops/schemas/privacy_request.py index 429179953..7ae1434ef 100644 --- a/src/fidesops/schemas/privacy_request.py +++ b/src/fidesops/schemas/privacy_request.py @@ -115,7 +115,7 @@ class RowCountRequest(BaseSchema): class StoppedCollectionDetails(StoppedCollection): - collection: Optional[str] = None + collection: Optional[str] = None # type: ignore class PrivacyRequestResponse(BaseSchema): diff --git a/src/fidesops/schemas/saas/saas_config.py b/src/fidesops/schemas/saas/saas_config.py index 9911b7b52..ad3db9b3c 100644 --- a/src/fidesops/schemas/saas/saas_config.py +++ b/src/fidesops/schemas/saas/saas_config.py @@ -113,8 +113,10 @@ def validate_request_for_pagination(cls, values: Dict[str, Any]) -> Dict[str, An before any field validation. """ - # delay import to avoid cyclic-dependency error - from fidesops.service.pagination.pagination_strategy_factory import get_strategy + # delay import to avoid cyclic-dependency error - We still ignore the pylint error + from fidesops.service.pagination.pagination_strategy_factory import ( # pylint: disable=R0401 + get_strategy, + ) pagination = values.get("pagination") if pagination is not None: diff --git a/src/fidesops/schemas/saas/strategy_configuration.py b/src/fidesops/schemas/saas/strategy_configuration.py index 69e00b73b..772911067 100644 --- a/src/fidesops/schemas/saas/strategy_configuration.py +++ b/src/fidesops/schemas/saas/strategy_configuration.py @@ -22,8 +22,8 @@ class FilterPostProcessorConfiguration(StrategyConfiguration): field: str value: Union[str, IdentityParamRef] - exact: Optional[bool] = True - case_sensitive: Optional[bool] = True + exact: bool = True + case_sensitive: bool = True class OffsetPaginationConfiguration(StrategyConfiguration): diff --git a/src/fidesops/service/authentication/authentication_strategy_basic.py b/src/fidesops/service/authentication/authentication_strategy_basic.py index 6fd9c1a96..5d5a4b966 100644 --- a/src/fidesops/service/authentication/authentication_strategy_basic.py +++ b/src/fidesops/service/authentication/authentication_strategy_basic.py @@ -28,14 +28,15 @@ def add_authentication( ) -> PreparedRequest: """Add basic authentication to the request""" secrets = connection_config.secrets + request.prepare_auth( auth=( - assign_placeholders(self.username, secrets), - assign_placeholders(self.password, secrets), + assign_placeholders(self.username, secrets), # type: ignore + assign_placeholders(self.password, secrets), # type: ignore ) ) return request @staticmethod def get_configuration_model() -> StrategyConfiguration: - return BasicAuthenticationConfiguration + return BasicAuthenticationConfiguration # type: ignore diff --git a/src/fidesops/service/authentication/authentication_strategy_bearer.py b/src/fidesops/service/authentication/authentication_strategy_bearer.py index a5eef0e6d..bbfea19bc 100644 --- a/src/fidesops/service/authentication/authentication_strategy_bearer.py +++ b/src/fidesops/service/authentication/authentication_strategy_bearer.py @@ -26,11 +26,11 @@ def add_authentication( self, request: PreparedRequest, connection_config: ConnectionConfig ) -> PreparedRequest: """Add bearer authentication to the request""" - request.headers["Authorization"] = "Bearer " + assign_placeholders( - self.token, connection_config.secrets + request.headers["Authorization"] = "Bearer " + assign_placeholders( # type: ignore + self.token, connection_config.secrets # type: ignore ) return request @staticmethod def get_configuration_model() -> StrategyConfiguration: - return BearerAuthenticationConfiguration + return BearerAuthenticationConfiguration # type: ignore diff --git a/src/fidesops/service/authentication/authentication_strategy_oauth2.py b/src/fidesops/service/authentication/authentication_strategy_oauth2.py index 7bb18c415..5fb968a7e 100644 --- a/src/fidesops/service/authentication/authentication_strategy_oauth2.py +++ b/src/fidesops/service/authentication/authentication_strategy_oauth2.py @@ -50,7 +50,7 @@ def add_authentication( # make sure required secrets have been provided self._check_required_secrets(connection_config) - access_token = connection_config.secrets.get("access_token") + access_token = connection_config.secrets.get("access_token") # type: ignore if not access_token: raise FidesopsException( f"OAuth2 access token not found for {connection_config.key}, please " @@ -63,7 +63,7 @@ def add_authentication( # https://datatracker.ietf.org/doc/html/rfc6749#section-5.1 if self.refresh_request: - expires_at = connection_config.secrets.get("expires_at") + expires_at = connection_config.secrets.get("expires_at") # type: ignore if self._close_to_expiration(expires_at, connection_config): refresh_response = self._call_token_request( "refresh", self.refresh_request, connection_config @@ -104,7 +104,7 @@ def _call_token_request( # get the client config from the token request or default to the # protocol and host specified by the root client config (no auth) - root_client_config = connection_config.get_saas_config().client_config + root_client_config = connection_config.get_saas_config().client_config # type: ignore oauth_client_config = ( token_request.client_config if token_request.client_config @@ -116,7 +116,7 @@ def _call_token_request( client = AuthenticatedClient( ( f"{oauth_client_config.protocol}://" - f"{assign_placeholders(oauth_client_config.host, connection_config.secrets)}" + f"{assign_placeholders(oauth_client_config.host, connection_config.secrets)}" # type: ignore ), connection_config, oauth_client_config, @@ -128,7 +128,7 @@ def _call_token_request( action, f"{connection_config.name} OAuth2", token_request, - connection_config.secrets, + connection_config.secrets, # type: ignore ) # ignore errors so we can return the error message in the response response = client.send(prepared_request, ignore_errors=True) @@ -203,7 +203,7 @@ def _validate_and_store_response( # get the session from the connection_config as a fallback if not db: db = Session.object_session(connection_config) - updated_secrets = {**connection_config.secrets, **data} + updated_secrets = {**connection_config.secrets, **data} # type: ignore connection_config.update(db, data={"secrets": updated_secrets}) logger.info( f"Successfully updated the OAuth2 token(s) for {connection_config.key}" @@ -228,14 +228,14 @@ def get_authorization_url( db, data={"connection_key": connection_config.key, "state": state} ) # add state to secrets - connection_config.secrets["state"] = state + connection_config.secrets["state"] = state # type: ignore # assign placeholders in the authorization request config prepared_authorization_request = map_param_values( "authorize", f"{connection_config.name} OAuth2", self.authorization_request, - connection_config.secrets, + connection_config.secrets, # type: ignore ) # get the client config from the authorization request or default @@ -243,12 +243,12 @@ def get_authorization_url( client_config = ( self.authorization_request.client_config if self.authorization_request.client_config - else connection_config.get_saas_config().client_config + else connection_config.get_saas_config().client_config # type: ignore ) # build the complete URL with query params return ( - f"{client_config.protocol}://{assign_placeholders(client_config.host, connection_config.secrets)}" + f"{client_config.protocol}://{assign_placeholders(client_config.host, connection_config.secrets)}" # type: ignore f"{prepared_authorization_request.path}" f"?{urlencode(prepared_authorization_request.query_params)}" ) @@ -263,7 +263,8 @@ def get_access_token( """ self._check_required_secrets(connection_config) - connection_config.secrets = {**connection_config.secrets, "code": code} + + connection_config.secrets = {**connection_config.secrets, "code": code} # type: ignore access_response = self._call_token_request( "access", self.token_request, connection_config ) @@ -288,4 +289,4 @@ def _check_required_secrets(connection_config: ConnectionConfig) -> None: @staticmethod def get_configuration_model() -> StrategyConfiguration: - return OAuth2AuthenticationConfiguration + return OAuth2AuthenticationConfiguration # type: ignore diff --git a/src/fidesops/service/authentication/authentication_strategy_query_param.py b/src/fidesops/service/authentication/authentication_strategy_query_param.py index 62dec1111..599e79ef9 100644 --- a/src/fidesops/service/authentication/authentication_strategy_query_param.py +++ b/src/fidesops/service/authentication/authentication_strategy_query_param.py @@ -29,12 +29,12 @@ def add_authentication( ) -> PreparedRequest: """Add token to the request as a query param""" request.url = set_query_parameter( - request.url, + request.url, # type: ignore self.name, - assign_placeholders(self.value, connection_config.secrets), + assign_placeholders(self.value, connection_config.secrets), # type: ignore ) return request @staticmethod def get_configuration_model() -> StrategyConfiguration: - return QueryParamAuthenticationConfiguration + return QueryParamAuthenticationConfiguration # type: ignore diff --git a/src/fidesops/service/connectors/__init__.py b/src/fidesops/service/connectors/__init__.py index ddc31a33e..ce48a0851 100644 --- a/src/fidesops/service/connectors/__init__.py +++ b/src/fidesops/service/connectors/__init__.py @@ -34,7 +34,7 @@ def get_connector(conn_config: ConnectionConfig) -> BaseConnector: """Return the Connector class corresponding to the connection_type.""" try: - return supported_connectors[conn_config.connection_type.value](conn_config) + return supported_connectors[conn_config.connection_type.value](conn_config) # type: ignore except KeyError: raise NotImplementedError( f"Add {conn_config.connection_type} to the 'supported_connectors' mapping." diff --git a/src/fidesops/service/connectors/base_connector.py b/src/fidesops/service/connectors/base_connector.py index 7bf38c362..9e10bb487 100644 --- a/src/fidesops/service/connectors/base_connector.py +++ b/src/fidesops/service/connectors/base_connector.py @@ -84,7 +84,7 @@ def mask_data( ) -> int: """Execute a masking request. Return the number of rows that have been updated""" - def dry_run_query(self, node: TraversalNode) -> str: + def dry_run_query(self, node: TraversalNode) -> Optional[str]: """Generate a dry-run query to display action that will be taken""" return self.query_config(node).dry_run_query() diff --git a/src/fidesops/service/connectors/manual_connector.py b/src/fidesops/service/connectors/manual_connector.py index 1bc4f51d1..3939fd07e 100644 --- a/src/fidesops/service/connectors/manual_connector.py +++ b/src/fidesops/service/connectors/manual_connector.py @@ -32,7 +32,7 @@ def test_connection(self) -> None: """No automated test_connection available for the Manual Connector""" return None - def retrieve_data( + def retrieve_data( # type: ignore self, node: TraversalNode, policy: Policy, @@ -67,7 +67,7 @@ def retrieve_data( f"Collection '{node.address.value}' waiting on manual data for privacy request '{privacy_request.id}'" ) - def mask_data( + def mask_data( # type: ignore self, node: TraversalNode, policy: Policy, diff --git a/src/fidesops/service/connectors/query_config.py b/src/fidesops/service/connectors/query_config.py index 16a938ce7..5abf6f183 100644 --- a/src/fidesops/service/connectors/query_config.py +++ b/src/fidesops/service/connectors/query_config.py @@ -71,7 +71,7 @@ def build_rule_target_field_paths( targeted_field_paths = [] collection_categories: Dict[ str, List[FieldPath] - ] = self.node.node.collection.field_paths_by_category + ] = self.node.node.collection.field_paths_by_category # type: ignore for rule_cat in rule_categories: for collection_cat, field_paths in collection_categories.items(): if collection_cat.startswith(rule_cat): @@ -125,7 +125,7 @@ def display_query_data(self) -> Dict[str, Any]: return data - def update_value_map( + def update_value_map( # pylint: disable=R0914 self, row: Row, policy: Policy, request: PrivacyRequest ) -> Dict[str, Any]: """Map the relevant field (as strings) to be updated on the row with their masked values from Policy Rules @@ -162,7 +162,7 @@ def update_value_map( masking_override, null_masking, strategy ): logger.warning( - f"Unable to generate a query for field {rule_field_path.string_path}: data_type is either not " + f"Unable to generate a query for field {rule_field_path.string_path}: data_type is either not " # type: ignore f"present on the field or not supported for the {strategy_config['strategy']} masking " f"strategy. Received data type: {masking_override.data_type_converter.name}" ) @@ -210,7 +210,7 @@ def _generate_masked_value( # pylint: disable=R0913 str_field_path: str, ) -> T: # masking API takes and returns lists, but here we are only leveraging single elements - masked_val = strategy.mask([val], request_id)[0] + masked_val = strategy.mask([val], request_id)[0] # type: ignore logger.debug( f"Generated the following masked val for field {str_field_path}: {masked_val}" @@ -226,7 +226,7 @@ def _generate_masked_value( # pylint: disable=R0913 f"of masked value to match, regardless of masking strategy" ) # for strategies other than null masking we assume that masked data type is the same as specified data type - masked_val = masking_override.data_type_converter.truncate( + masked_val = masking_override.data_type_converter.truncate( # type: ignore masking_override.length, masked_val ) return masked_val @@ -291,7 +291,7 @@ def generate_query( def query_to_str(self, t: T, input_data: Dict[str, List[Any]]) -> None: """Not used for ManualQueryConfig, we output the dry run query as a dictionary instead of a string""" - def dry_run_query(self) -> Optional[ManualAction]: + def dry_run_query(self) -> Optional[ManualAction]: # type: ignore """Displays the ManualAction needed with question marks instead of action data for the locators as a dry run query""" fake_data: Dict[str, Any] = self.display_query_data() @@ -718,7 +718,7 @@ def transform_query_pairs(pairs: Dict[str, Any]) -> Dict[str, Any]: return None def generate_update_stmt( - self, row: Row, policy: Policy, request: PrivacyRequest = None + self, row: Row, policy: Policy, request: PrivacyRequest ) -> Optional[MongoStatement]: """Generate a SQL update statement in the form of Mongo update statement components""" update_clauses = self.update_value_map(row, policy, request) diff --git a/src/fidesops/service/connectors/saas/authenticated_client.py b/src/fidesops/service/connectors/saas/authenticated_client.py index 6c5f3d51c..e652c7470 100644 --- a/src/fidesops/service/connectors/saas/authenticated_client.py +++ b/src/fidesops/service/connectors/saas/authenticated_client.py @@ -1,13 +1,17 @@ +from __future__ import annotations + import logging -from typing import Optional +from typing import TYPE_CHECKING, Optional from requests import PreparedRequest, Request, Response, Session from fidesops.common_exceptions import ClientUnsuccessfulException, ConnectionException from fidesops.core.config import config -from fidesops.models.connectionconfig import ConnectionConfig -from fidesops.schemas.saas.saas_config import ClientConfig -from fidesops.schemas.saas.shared_schemas import SaaSRequestParams + +if TYPE_CHECKING: + from fidesops.models.connectionconfig import ConnectionConfig + from fidesops.schemas.saas.saas_config import ClientConfig + from fidesops.schemas.saas.shared_schemas import SaaSRequestParams logger = logging.getLogger(__name__) @@ -23,8 +27,9 @@ def __init__( self, uri: str, configuration: ConnectionConfig, - request_client_config: ClientConfig = None, + request_client_config: Optional[ClientConfig] = None, ): + saas_config = configuration.get_saas_config() self.configuration = configuration self.session = Session() self.uri = uri @@ -32,7 +37,7 @@ def __init__( self.client_config = ( request_client_config if request_client_config - else configuration.get_saas_config().client_config + else saas_config.client_config # type: ignore ) self.secrets = configuration.secrets @@ -44,7 +49,7 @@ def get_authenticated_request( incoming path, headers, query, and body params. """ - from fidesops.service.authentication.authentication_strategy_factory import ( + from fidesops.service.authentication.authentication_strategy_factory import ( # pylint: disable=R0401 get_strategy, ) diff --git a/src/fidesops/service/connectors/saas_connector.py b/src/fidesops/service/connectors/saas_connector.py index 71c198029..87b227e2d 100644 --- a/src/fidesops/service/connectors/saas_connector.py +++ b/src/fidesops/service/connectors/saas_connector.py @@ -37,8 +37,8 @@ def __init__(self, configuration: ConnectionConfig): super().__init__(configuration) self.secrets = configuration.secrets self.saas_config = configuration.get_saas_config() - self.client_config = self.saas_config.client_config - self.endpoints = self.saas_config.top_level_endpoint_dict + self.client_config = self.saas_config.client_config # type: ignore + self.endpoints = self.saas_config.top_level_endpoint_dict # type: ignore self.collection_name: Optional[str] = None def query_config(self, node: TraversalNode) -> SaaSQueryConfig: @@ -49,12 +49,12 @@ def query_config(self, node: TraversalNode) -> SaaSQueryConfig: # store collection_name for logging purposes self.collection_name = node.address.collection return SaaSQueryConfig( - node, self.endpoints, self.secrets, self.saas_config.data_protection_request + node, self.endpoints, self.secrets, self.saas_config.data_protection_request # type: ignore ) def test_connection(self) -> Optional[ConnectionTestStatus]: """Generates and executes a test connection based on the SaaS config""" - test_request: SaaSRequest = self.saas_config.test_request + test_request: SaaSRequest = self.saas_config.test_request # type: ignore prepared_request: SaaSRequestParams = SaaSRequestParams( method=test_request.method, path=test_request.path ) @@ -65,9 +65,7 @@ def test_connection(self) -> Optional[ConnectionTestStatus]: def build_uri(self) -> str: """Build base URI for the given connector""" host = self.client_config.host - return ( - f"{self.client_config.protocol}://{assign_placeholders(host, self.secrets)}" - ) + return f"{self.client_config.protocol}://{assign_placeholders(host, self.secrets)}" # type: ignore def create_client(self) -> AuthenticatedClient: """Creates an authenticated request builder""" @@ -95,7 +93,7 @@ def create_client_from_request( if saas_request.client_config: return self._build_client_with_config(saas_request.client_config) - return self._build_client_with_config(self.saas_config.client_config) + return self._build_client_with_config(self.saas_config.client_config) # type: ignore def retrieve_data( self, @@ -105,13 +103,12 @@ def retrieve_data( input_data: Dict[str, List[Any]], ) -> List[Row]: """Retrieve data from SaaS APIs""" - # generate initial set of requests if read request is defined, otherwise raise an exception query_config: SaaSQueryConfig = self.query_config(node) read_request: Optional[SaaSRequest] = query_config.get_request_by_action("read") if not read_request: raise FidesopsException( - f"The 'read' action is not defined for the '{self.collection_name}' " + f"The 'read' action is not defined for the '{self.collection_name}' " # type: ignore f"endpoint in {self.saas_config.fides_key}" ) prepared_requests: List[SaaSRequestParams] = query_config.generate_requests( @@ -124,7 +121,7 @@ def retrieve_data( rows: List[Row] = [] for next_request in prepared_requests: while next_request: - processed_rows, next_request = self.execute_prepared_request( + processed_rows, next_request = self.execute_prepared_request( # type: ignore next_request, privacy_request.get_cached_identity_data(), read_request, @@ -143,7 +140,7 @@ def execute_prepared_request( Returns processed data and request_params for next page of data if available. """ client: AuthenticatedClient = self.create_client_from_request(saas_request) - response: Response = client.send(prepared_request, saas_request) + response: Response = client.send(prepared_request, saas_request.ignore_errors) response = self._handle_errored_response(saas_request, response) response_data = self._unwrap_response_data(saas_request, response) @@ -151,7 +148,7 @@ def execute_prepared_request( rows = self.process_response_data( response_data, identity_data, - saas_request.postprocessors, + saas_request.postprocessors, # type: ignore ) logger.info( @@ -166,12 +163,12 @@ def execute_prepared_request( saas_request.pagination.configuration, ) next_request = strategy.get_next_request( - prepared_request, self.secrets, response, saas_request.data_path + prepared_request, self.secrets, response, saas_request.data_path # type: ignore ) if next_request: logger.info( - f"Using '{saas_request.pagination.strategy}' " + f"Using '{saas_request.pagination.strategy}' " # type: ignore f"pagination strategy to get next page for '{self.collection_name}'." ) @@ -193,17 +190,17 @@ def process_response_data( processed_data = response_data for postprocessor in postprocessors or []: strategy: PostProcessorStrategy = get_postprocessor_strategy( - postprocessor.strategy, postprocessor.configuration + postprocessor.strategy, postprocessor.configuration # type: ignore ) logger.info( - f"Starting postprocessing of '{self.collection_name}' collection with " + f"Starting postprocessing of '{self.collection_name}' collection with " # type: ignore f"'{postprocessor.strategy}' strategy." ) try: processed_data = strategy.process(processed_data, identity_data) except Exception as exc: raise PostProcessingException( - f"Exception occurred during the '{postprocessor.strategy}' postprocessor " + f"Exception occurred during the '{postprocessor.strategy}' postprocessor " # type: ignore f"on the '{self.collection_name}' collection: {exc}" ) if not processed_data: @@ -251,7 +248,7 @@ def mask_data( rows = self.process_response_data( rows, privacy_request.get_cached_identity_data(), - masking_request.postprocessors, + masking_request.postprocessors, # type: ignore ) prepared_requests = [ diff --git a/src/fidesops/service/connectors/saas_query_config.py b/src/fidesops/service/connectors/saas_query_config.py index 160f541ea..864ccac7f 100644 --- a/src/fidesops/service/connectors/saas_query_config.py +++ b/src/fidesops/service/connectors/saas_query_config.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import json import logging from typing import Any, Dict, List, Optional, TypeVar @@ -48,7 +50,7 @@ def get_request_by_action(self, action: str) -> Optional[SaaSRequest]: # store action name for logging purposes self.action = action collection_name = self.node.address.collection - request = self.endpoints[collection_name].requests[action] + request = self.endpoints[collection_name].requests[action] # type: ignore logger.info( f"Found matching endpoint to {action} '{collection_name}' collection" ) @@ -128,7 +130,7 @@ def generate_query( query statement (select statement, where clause, limit, offset, etc.) """ - current_request: SaaSRequest = self.get_request_by_action("read") + current_request: SaaSRequest | None = self.get_request_by_action("read") if not current_request: raise FidesopsException( f"The 'read' action is not defined for the '{self.collection_name}' " @@ -152,7 +154,7 @@ def generate_query( # map param values to placeholders in path, headers, and query params saas_request_params: SaaSRequestParams = saas_util.map_param_values( - self.action, self.collection_name, current_request, param_values + self.action, self.collection_name, current_request, param_values # type: ignore ) logger.info(f"Populated request params for {current_request.path}") @@ -168,7 +170,7 @@ def generate_update_stmt( if specified by the body field of the masking request. """ - current_request: SaaSRequest = self.get_masking_request() + current_request: SaaSRequest = self.get_masking_request() # type: ignore collection_name: str = self.node.address.collection collection_values: Dict[str, Row] = {collection_name: row} identity_data: Dict[str, Any] = request.get_cached_identity_data() @@ -208,7 +210,7 @@ def generate_update_stmt( # map param values to placeholders in path, headers, and query params saas_request_params: SaaSRequestParams = saas_util.map_param_values( - self.action, self.collection_name, current_request, param_values + self.action, self.collection_name, current_request, param_values # type: ignore ) logger.info(f"Populated request params for {current_request.path}") diff --git a/src/fidesops/service/connectors/sql_connector.py b/src/fidesops/service/connectors/sql_connector.py index 2e0abaeda..786180ecd 100644 --- a/src/fidesops/service/connectors/sql_connector.py +++ b/src/fidesops/service/connectors/sql_connector.py @@ -93,11 +93,11 @@ def test_connection(self) -> Optional[ConnectionTestStatus]: connection.execute("select 1") except OperationalError: raise ConnectionException( - f"Operational Error connecting to {self.configuration.connection_type.value} db." + f"Operational Error connecting to {self.configuration.connection_type.value} db." # type: ignore ) except InternalError: raise ConnectionException( - f"Internal Error connecting to {self.configuration.connection_type.value} db." + f"Internal Error connecting to {self.configuration.connection_type.value} db." # type: ignore ) except Exception: raise ConnectionException("Connection error.") diff --git a/src/fidesops/service/masking/strategy/masking_strategy_aes_encrypt.py b/src/fidesops/service/masking/strategy/masking_strategy_aes_encrypt.py index 464f3eb99..a19c32cd2 100644 --- a/src/fidesops/service/masking/strategy/masking_strategy_aes_encrypt.py +++ b/src/fidesops/service/masking/strategy/masking_strategy_aes_encrypt.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from typing import Dict, List, Optional from fidesops.schemas.masking.masking_configuration import ( @@ -41,24 +43,25 @@ def mask( masking_meta: Dict[ SecretType, MaskingSecretMeta ] = self._build_masking_secret_meta() - key: bytes = SecretsUtil.get_or_generate_secret( + key: bytes | None = SecretsUtil.get_or_generate_secret( request_id, SecretType.key, masking_meta[SecretType.key] ) - key_hmac: str = SecretsUtil.get_or_generate_secret( + key_hmac: str | None = SecretsUtil.get_or_generate_secret( request_id, SecretType.key_hmac, masking_meta[SecretType.key_hmac], ) + # The nonce is generated deterministically such that the same input val will result in same nonce # and therefore the same masked val through the aes strategy. This is called convergent encryption, with this # implementation loosely based on https://www.vaultproject.io/docs/secrets/transit#convergent-encryption masked_values: List[str] = [] for value in values: - nonce: bytes = self._generate_nonce( - value, key_hmac, request_id, masking_meta + nonce: bytes | None = self._generate_nonce( + value, key_hmac, request_id, masking_meta # type: ignore ) - masked: str = encrypt(value, key, nonce) + masked: str = encrypt(value, key, nonce) # type: ignore if self.format_preservation is not None: formatter = FormatPreservation(self.format_preservation) masked = formatter.format(masked) @@ -79,7 +82,7 @@ def generate_secrets_for_cache(self) -> List[MaskingSecretCache]: @staticmethod def get_configuration_model() -> MaskingConfiguration: """Used to get the configuration model to configure the strategy""" - return AesEncryptionMaskingConfiguration + return AesEncryptionMaskingConfiguration # type: ignore @staticmethod def get_description() -> MaskingStrategyDescription: @@ -108,18 +111,18 @@ def data_type_supported(data_type: Optional[str]) -> bool: @staticmethod def _generate_nonce( - value: Optional[str], + value: str, key: str, privacy_request_id: Optional[str], masking_meta: Dict[SecretType, MaskingSecretMeta], ) -> bytes: - salt: str = SecretsUtil.get_or_generate_secret( + salt = SecretsUtil.get_or_generate_secret( privacy_request_id, SecretType.salt_hmac, masking_meta[SecretType.salt_hmac] ) # Trim to 12 bytes, which is recommended length from aes gcm lib: # https://cryptography.io/en/latest/hazmat/primitives/aead/#cryptography.hazmat.primitives.ciphers.aead.AESGCM.encrypt return hmac_encrypt_return_bytes( - value, key, salt, HmacMaskingConfiguration.Algorithm.sha_256 + value, key, salt, HmacMaskingConfiguration.Algorithm.sha_256 # type: ignore )[:12] @staticmethod diff --git a/src/fidesops/service/masking/strategy/masking_strategy_factory.py b/src/fidesops/service/masking/strategy/masking_strategy_factory.py index 7ca5eeac1..60fe7667d 100644 --- a/src/fidesops/service/masking/strategy/masking_strategy_factory.py +++ b/src/fidesops/service/masking/strategy/masking_strategy_factory.py @@ -52,12 +52,12 @@ def get_strategy( f"Strategy '{strategy_name}' does not exist. Valid strategies are [{cls.valid_strategies}]" ) try: - strategy_config = strategy.get_configuration_model()(**configuration) + strategy_config = strategy.get_configuration_model()(**configuration) # type: ignore except ValidationError as e: raise FidesopsValidationError(message=str(e)) - return strategy(configuration=strategy_config) + return strategy(configuration=strategy_config) # type: ignore @classmethod def get_strategies(cls) -> ValuesView[MaskingStrategy]: """Returns all supported masking strategies""" - return cls.registry.values() + return cls.registry.values() # type: ignore diff --git a/src/fidesops/service/masking/strategy/masking_strategy_hash.py b/src/fidesops/service/masking/strategy/masking_strategy_hash.py index 24fac311e..27a26ae97 100644 --- a/src/fidesops/service/masking/strategy/masking_strategy_hash.py +++ b/src/fidesops/service/masking/strategy/masking_strategy_hash.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import hashlib from typing import Dict, List, Optional @@ -50,14 +52,15 @@ def mask( masking_meta: Dict[ SecretType, MaskingSecretMeta ] = self._build_masking_secret_meta() - salt: str = SecretsUtil.get_or_generate_secret( + salt: str | None = SecretsUtil.get_or_generate_secret( request_id, SecretType.salt, masking_meta[SecretType.salt], ) + masked_values: List[str] = [] for value in values: - masked: str = self.algorithm_function(value, salt) + masked: str = self.algorithm_function(value, salt) # type: ignore if self.format_preservation is not None: formatter = FormatPreservation(self.format_preservation) masked = formatter.format(masked) @@ -75,7 +78,7 @@ def generate_secrets_for_cache(self) -> List[MaskingSecretCache]: @staticmethod def get_configuration_model() -> MaskingConfiguration: - return HashMaskingConfiguration + return HashMaskingConfiguration # type: ignore # MR Note - We will need a way to ensure that this does not fall out of date. Given that it # includes subjective instructions, this is not straightforward to automate diff --git a/src/fidesops/service/masking/strategy/masking_strategy_hmac.py b/src/fidesops/service/masking/strategy/masking_strategy_hmac.py index 1f6c4a70f..6b38e6415 100644 --- a/src/fidesops/service/masking/strategy/masking_strategy_hmac.py +++ b/src/fidesops/service/masking/strategy/masking_strategy_hmac.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from typing import Dict, List, Optional from fidesops.schemas.masking.masking_configuration import ( @@ -49,15 +51,16 @@ def mask( masking_meta: Dict[ SecretType, MaskingSecretMeta ] = self._build_masking_secret_meta() - key: str = SecretsUtil.get_or_generate_secret( + key: str | None = SecretsUtil.get_or_generate_secret( request_id, SecretType.key, masking_meta[SecretType.key] ) - salt: str = SecretsUtil.get_or_generate_secret( + salt: str | None = SecretsUtil.get_or_generate_secret( request_id, SecretType.salt, masking_meta[SecretType.salt] ) + masked_values: List[str] = [] for value in values: - masked: str = hmac_encrypt_return_str(value, key, salt, self.algorithm) + masked: str = hmac_encrypt_return_str(value, key, salt, self.algorithm) # type: ignore if self.format_preservation is not None: formatter = FormatPreservation(self.format_preservation) masked = formatter.format(masked) @@ -75,7 +78,7 @@ def generate_secrets_for_cache(self) -> List[MaskingSecretCache]: @staticmethod def get_configuration_model() -> MaskingConfiguration: - return HmacMaskingConfiguration + return HmacMaskingConfiguration # type: ignore @staticmethod def get_description() -> MaskingStrategyDescription: diff --git a/src/fidesops/service/masking/strategy/masking_strategy_nullify.py b/src/fidesops/service/masking/strategy/masking_strategy_nullify.py index f4335955c..a06a0411a 100644 --- a/src/fidesops/service/masking/strategy/masking_strategy_nullify.py +++ b/src/fidesops/service/masking/strategy/masking_strategy_nullify.py @@ -41,7 +41,7 @@ def secrets_required(self) -> bool: @staticmethod def get_configuration_model() -> MaskingConfiguration: - return NullMaskingConfiguration + return NullMaskingConfiguration # type: ignore @staticmethod def get_description() -> MaskingStrategyDescription: diff --git a/src/fidesops/service/masking/strategy/masking_strategy_random_string_rewrite.py b/src/fidesops/service/masking/strategy/masking_strategy_random_string_rewrite.py index cac2daf7d..6c1ceab1d 100644 --- a/src/fidesops/service/masking/strategy/masking_strategy_random_string_rewrite.py +++ b/src/fidesops/service/masking/strategy/masking_strategy_random_string_rewrite.py @@ -55,7 +55,7 @@ def secrets_required(self) -> bool: @staticmethod def get_configuration_model() -> MaskingConfiguration: - return RandomStringMaskingConfiguration + return RandomStringMaskingConfiguration # type: ignore @staticmethod def get_description() -> MaskingStrategyDescription: diff --git a/src/fidesops/service/masking/strategy/masking_strategy_string_rewrite.py b/src/fidesops/service/masking/strategy/masking_strategy_string_rewrite.py index afc4a684a..4bc19970e 100644 --- a/src/fidesops/service/masking/strategy/masking_strategy_string_rewrite.py +++ b/src/fidesops/service/masking/strategy/masking_strategy_string_rewrite.py @@ -49,7 +49,7 @@ def secrets_required(self) -> bool: @staticmethod def get_configuration_model() -> MaskingConfiguration: - return StringRewriteMaskingConfiguration + return StringRewriteMaskingConfiguration # type: ignore # MR Note - We will need a way to ensure that this does not fall out of date. Given that it # includes subjective instructions, this is not straightforward to automate diff --git a/src/fidesops/service/pagination/pagination_strategy.py b/src/fidesops/service/pagination/pagination_strategy.py index 6e0765237..de5e2e419 100644 --- a/src/fidesops/service/pagination/pagination_strategy.py +++ b/src/fidesops/service/pagination/pagination_strategy.py @@ -1,10 +1,14 @@ +from __future__ import annotations + from abc import ABC, abstractmethod -from typing import Any, Dict, Optional +from typing import TYPE_CHECKING, Any, Dict, Optional from requests import Response from fidesops.schemas.saas.shared_schemas import SaaSRequestParams -from fidesops.schemas.saas.strategy_configuration import StrategyConfiguration + +if TYPE_CHECKING: + from fidesops.schemas.saas.strategy_configuration import StrategyConfiguration class PaginationStrategy(ABC): diff --git a/src/fidesops/service/pagination/pagination_strategy_cursor.py b/src/fidesops/service/pagination/pagination_strategy_cursor.py index a3ca2df09..a9ad9f87f 100644 --- a/src/fidesops/service/pagination/pagination_strategy_cursor.py +++ b/src/fidesops/service/pagination/pagination_strategy_cursor.py @@ -54,4 +54,4 @@ def get_next_request( @staticmethod def get_configuration_model() -> StrategyConfiguration: - return CursorPaginationConfiguration + return CursorPaginationConfiguration # type: ignore diff --git a/src/fidesops/service/pagination/pagination_strategy_factory.py b/src/fidesops/service/pagination/pagination_strategy_factory.py index 137e79d62..8e5b8ab6b 100644 --- a/src/fidesops/service/pagination/pagination_strategy_factory.py +++ b/src/fidesops/service/pagination/pagination_strategy_factory.py @@ -1,13 +1,13 @@ +from __future__ import annotations + import logging from enum import Enum -from typing import Any, Dict, List +from typing import TYPE_CHECKING, Any, Dict, List from pydantic import ValidationError from fidesops.common_exceptions import NoSuchStrategyException from fidesops.common_exceptions import ValidationError as FidesopsValidationError -from fidesops.schemas.saas.strategy_configuration import StrategyConfiguration -from fidesops.service.pagination.pagination_strategy import PaginationStrategy from fidesops.service.pagination.pagination_strategy_cursor import ( CursorPaginationStrategy, ) @@ -16,6 +16,10 @@ OffsetPaginationStrategy, ) +if TYPE_CHECKING: + from fidesops.schemas.saas.strategy_configuration import StrategyConfiguration + from fidesops.service.pagination.pagination_strategy import PaginationStrategy + logger = logging.getLogger(__name__) diff --git a/src/fidesops/service/pagination/pagination_strategy_link.py b/src/fidesops/service/pagination/pagination_strategy_link.py index 8b64de08a..31a382c86 100644 --- a/src/fidesops/service/pagination/pagination_strategy_link.py +++ b/src/fidesops/service/pagination/pagination_strategy_link.py @@ -70,4 +70,4 @@ def get_next_request( @staticmethod def get_configuration_model() -> StrategyConfiguration: - return LinkPaginationConfiguration + return LinkPaginationConfiguration # type: ignore diff --git a/src/fidesops/service/pagination/pagination_strategy_offset.py b/src/fidesops/service/pagination/pagination_strategy_offset.py index 08643ebf9..c6fdbb49a 100644 --- a/src/fidesops/service/pagination/pagination_strategy_offset.py +++ b/src/fidesops/service/pagination/pagination_strategy_offset.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional +from typing import Any, Dict, Optional, Union import pydash from requests import Response @@ -48,7 +48,7 @@ def get_next_request( ) # increment param value and return None if limit has been reached to indicate there are no more pages - limit = self.limit + limit: Optional[Union[int, ConnectorParamRef]] = self.limit if isinstance(self.limit, ConnectorParamRef): limit = connector_params.get(self.limit.connector_param) if limit is None: @@ -70,7 +70,7 @@ def get_next_request( @staticmethod def get_configuration_model() -> StrategyConfiguration: - return OffsetPaginationConfiguration + return OffsetPaginationConfiguration # type: ignore def validate_request(self, request: Dict[str, Any]) -> None: """Ensures that the query param specified by 'incremental_param' exists in the request""" diff --git a/src/fidesops/service/privacy_request/onetrust_service.py b/src/fidesops/service/privacy_request/onetrust_service.py index 46afca4f7..9cb3f715f 100644 --- a/src/fidesops/service/privacy_request/onetrust_service.py +++ b/src/fidesops/service/privacy_request/onetrust_service.py @@ -94,14 +94,14 @@ def intake_onetrust_requests(config_key: FidesOpsKey) -> None: identity_kwargs = {"email": request.email} identity = PrivacyRequestIdentity(**identity_kwargs) fides_task: Optional[OneTrustSubtask] = OneTrustService._get_fides_subtask( - hostname, request.requestQueueRefId, access_token + hostname, request.requestQueueRefId, access_token # type: ignore ) if fides_task is None: # no fides task associated with this request continue OneTrustService._create_privacy_request( - fides_task.subTaskId, - request.dateCreated, + fides_task.subTaskId, # type: ignore + request.dateCreated, # type: ignore identity, onetrust_policy, hostname, diff --git a/src/fidesops/service/privacy_request/request_runner_service.py b/src/fidesops/service/privacy_request/request_runner_service.py index a58cea469..dee474850 100644 --- a/src/fidesops/service/privacy_request/request_runner_service.py +++ b/src/fidesops/service/privacy_request/request_runner_service.py @@ -54,7 +54,7 @@ def run_webhooks_and_report_status( Updates privacy request status if execution is paused/errored. Returns True if execution should proceed. """ - webhooks = db.query(webhook_cls).filter_by(policy_id=privacy_request.policy.id) + webhooks = db.query(webhook_cls).filter_by(policy_id=privacy_request.policy.id) # type: ignore if after_webhook_id: # Only run webhooks configured to run after this Pre-Execution webhook @@ -119,7 +119,7 @@ def upload_access_results( db=session, request_id=privacy_request.id, data=filtered_results, - storage_key=rule.storage_destination.key, + storage_key=rule.storage_destination.key, # type: ignore ) except common_exceptions.StorageUploadError as exc: logging.error( @@ -167,7 +167,7 @@ def run_privacy_request( if from_step is not None: # Re-cast `from_step` into an Enum to enforce the validation since unserializable objects # can't be passed into and between tasks - from_step = PausedStep(from_step) + from_step = PausedStep(from_step) # type: ignore SessionLocal = get_db_session(config) with SessionLocal() as session: @@ -186,7 +186,7 @@ def run_privacy_request( proceed = run_webhooks_and_report_status( session, privacy_request=privacy_request, - webhook_cls=PolicyPreWebhook, + webhook_cls=PolicyPreWebhook, # type: ignore after_webhook_id=from_webhook_id, ) if not proceed: @@ -209,7 +209,7 @@ def run_privacy_request( connection_configs = ConnectionConfig.all(db=session) if ( - not from_step == PausedStep.erasure + from_step != PausedStep.erasure ): # Skip if we're resuming from erasure step access_result: Dict[str, List[Row]] = run_access_request( privacy_request=privacy_request, @@ -257,7 +257,7 @@ def run_privacy_request( proceed = run_webhooks_and_report_status( db=session, privacy_request=privacy_request, - webhook_cls=PolicyPostWebhook, + webhook_cls=PolicyPostWebhook, # type: ignore ) if not proceed: session.close() diff --git a/src/fidesops/service/privacy_request/request_service.py b/src/fidesops/service/privacy_request/request_service.py index 65fabc8f5..75bdef9ff 100644 --- a/src/fidesops/service/privacy_request/request_service.py +++ b/src/fidesops/service/privacy_request/request_service.py @@ -5,9 +5,7 @@ from fidesops.models.policy import ActionType, Policy from fidesops.models.privacy_request import PrivacyRequest from fidesops.schemas.drp_privacy_request import DrpPrivacyRequestCreate -from fidesops.schemas.masking.masking_configuration import MaskingConfiguration from fidesops.schemas.masking.masking_secrets import MaskingSecretCache -from fidesops.schemas.policy import Rule from fidesops.schemas.redis_cache import PrivacyRequestIdentity from fidesops.service.masking.strategy.masking_strategy_factory import ( MaskingStrategyFactory, @@ -42,13 +40,11 @@ def cache_data( # Store masking secrets in the cache logger.info(f"Caching masking secrets for privacy request {privacy_request.id}") - erasure_rules: List[Rule] = policy.get_rules_for_action( - action_type=ActionType.erasure - ) + erasure_rules = policy.get_rules_for_action(action_type=ActionType.erasure) unique_masking_strategies_by_name: Set[str] = set() for rule in erasure_rules: - strategy_name: str = rule.masking_strategy["strategy"] - configuration: MaskingConfiguration = rule.masking_strategy["configuration"] + strategy_name: str = rule.masking_strategy["strategy"] # type: ignore + configuration = rule.masking_strategy["configuration"] # type: ignore if strategy_name in unique_masking_strategies_by_name: continue unique_masking_strategies_by_name.add(strategy_name) diff --git a/src/fidesops/service/processors/post_processor_strategy/post_processor_strategy_filter.py b/src/fidesops/service/processors/post_processor_strategy/post_processor_strategy_filter.py index 5c5449880..d4051c5d2 100644 --- a/src/fidesops/service/processors/post_processor_strategy/post_processor_strategy_filter.py +++ b/src/fidesops/service/processors/post_processor_strategy/post_processor_strategy_filter.py @@ -78,7 +78,7 @@ def process( f"strategy: {self.get_strategy_name()}" ) return [] - filter_value = identity_data.get(self.value.identity) + filter_value = identity_data.get(self.value.identity) # type: ignore try: if isinstance(data, list): @@ -88,7 +88,7 @@ def process( if self._matches( self.exact, self.case_sensitive, - filter_value, + filter_value, # type: ignore pydash.get(item, self.field), ) ] @@ -97,7 +97,7 @@ def process( if self._matches( self.exact, self.case_sensitive, - filter_value, + filter_value, # type: ignore pydash.get(data, self.field), ) else [] @@ -160,4 +160,4 @@ def _matches( @staticmethod def get_configuration_model() -> StrategyConfiguration: - return FilterPostProcessorConfiguration + return FilterPostProcessorConfiguration # type: ignore diff --git a/src/fidesops/service/processors/post_processor_strategy/post_processor_strategy_unwrap.py b/src/fidesops/service/processors/post_processor_strategy/post_processor_strategy_unwrap.py index 03a5db43f..8c0f1a5a2 100644 --- a/src/fidesops/service/processors/post_processor_strategy/post_processor_strategy_unwrap.py +++ b/src/fidesops/service/processors/post_processor_strategy/post_processor_strategy_unwrap.py @@ -82,4 +82,4 @@ def process( @staticmethod def get_configuration_model() -> StrategyConfiguration: - return UnwrapPostProcessorConfiguration + return UnwrapPostProcessorConfiguration # type: ignore diff --git a/src/fidesops/service/storage/storage_uploader_service.py b/src/fidesops/service/storage/storage_uploader_service.py index 62a4ce89d..f10db6ec5 100644 --- a/src/fidesops/service/storage/storage_uploader_service.py +++ b/src/fidesops/service/storage/storage_uploader_service.py @@ -32,7 +32,7 @@ def upload( if config.secrets is None and config.type != StorageType.local: logger.warning(f"Storage secrets not found: {storage_key}") raise StorageUploadError("Storage secrets not found") - uploader: Any = _get_uploader_from_config_type(config.type) + uploader: Any = _get_uploader_from_config_type(config.type) # type: ignore return uploader(db, config, data, request_id) @@ -59,7 +59,7 @@ def _construct_file_key(request_id: str, config: StorageConfig) -> str: if naming != FileNaming.request_id.value: raise ValueError(f"File naming of {naming} not supported") - return f"{request_id}.{get_extension(config.format)}" + return f"{request_id}.{get_extension(config.format)}" # type: ignore def _get_uploader_from_config_type(storage_type: StorageType) -> Any: @@ -77,7 +77,7 @@ def _s3_uploader(_: Session, config: StorageConfig, data: Dict, request_id: str) bucket_name = config.details[StorageDetails.BUCKET.value] return upload_to_s3( - config.secrets, data, bucket_name, file_key, config.format.value, request_id + config.secrets, data, bucket_name, file_key, config.format.value, request_id # type: ignore ) @@ -93,6 +93,7 @@ def _onetrust_uploader( raise StorageUploadError( f"Request could not be found for request_id: {request_id}" ) + payload_data = { "language": "en-us", "system": config.details[StorageDetails.SERVICE_NAME.value], @@ -100,8 +101,8 @@ def _onetrust_uploader( } return upload_to_onetrust( payload_data, - config.secrets, - request_details.external_id, + config.secrets, # type: ignore + request_details.external_id, # type: ignore ) diff --git a/src/fidesops/task/graph_task.py b/src/fidesops/task/graph_task.py index c01c44137..114cb8ee3 100644 --- a/src/fidesops/task/graph_task.py +++ b/src/fidesops/task/graph_task.py @@ -4,7 +4,7 @@ from abc import ABC from functools import wraps from time import sleep -from typing import Any, Callable, Dict, List, Optional, Set, Tuple +from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union import dask from dask.threaded import get @@ -62,7 +62,7 @@ def result(*args: Any, **kwargs: Any) -> List[Optional[Row]]: method_name = func.__name__ self = args[0] - raised_ex = None + raised_ex: Optional[Union[BaseException, Exception]] = None for attempt in range(config.execution.TASK_RETRY_COUNT + 1): try: self.skip_if_disabled() @@ -143,7 +143,7 @@ def __repr__(self) -> str: return f"{type(self)}:{self.key}" @property - def grouped_fields(self) -> Set[Optional[str]]: + def grouped_fields(self) -> Set[str]: """Convenience property - returns a set of fields that have been specified on the collection as dependent upon one another """ @@ -154,7 +154,7 @@ def dependent_identity_fields(self) -> bool: """If the current collection needs inputs from other collections, in addition to its seed data.""" collection = self.traversal_node.node.collection for field in self.grouped_fields: - if collection.field(FieldPath(field)).identity: + if collection.field(FieldPath(field)).identity: # type: ignore return True return False @@ -189,7 +189,7 @@ def field_map(keep: Callable) -> COLLECTION_FIELD_PATH_MAP: return field_map(lambda string_path: True), field_map(lambda string_path: False) - def generate_dry_run_query(self) -> str: + def generate_dry_run_query(self) -> Optional[str]: """Type-specific query generated for this traversal_node.""" return self.connector.dry_run_query(self.traversal_node) @@ -213,8 +213,8 @@ def _combine_seed_data( for (foreign_field_path, local_field_path) in dependent_field_mappings[ ROOT_COLLECTION_ADDRESS ]: - dependent_values: List = consolidate_query_matches( - row=seed_data, target_path=foreign_field_path + dependent_values = consolidate_query_matches( + row=seed_data, target_path=foreign_field_path # type: ignore ) grouped_data[local_field_path.string_path] = dependent_values return grouped_data @@ -387,7 +387,7 @@ def post_process_input_data( out: FieldPathNodeInput = {} for key, values in pre_processed_inputs.items(): path: FieldPath = FieldPath.parse(key) - field: Field = self.traversal_node.node.collection.field(path) + field: Optional[Field] = self.traversal_node.node.collection.field(path) if ( field and path in self.traversal_node.query_field_paths @@ -523,7 +523,7 @@ def collect_queries_fn( tn: TraversalNode, data: Dict[CollectionAddress, str] ) -> None: if not tn.is_root_node(): - data[tn.address] = GraphTask(tn, resources).generate_dry_run_query() + data[tn.address] = GraphTask(tn, resources).generate_dry_run_query() # type: ignore env: Dict[CollectionAddress, str] = {} traversal.traverse(env, collect_queries_fn) @@ -709,7 +709,7 @@ def build_affected_field_logs( collection_categories: Dict[ str, List[FieldPath] - ] = node.collection.field_paths_by_category + ] = node.collection.field_paths_by_category # type: ignore for rule_cat in rule_categories: for collection_cat, field_paths in collection_categories.items(): if collection_cat.startswith(rule_cat): diff --git a/src/fidesops/task/task_resources.py b/src/fidesops/task/task_resources.py index 808a57862..3610c4cda 100644 --- a/src/fidesops/task/task_resources.py +++ b/src/fidesops/task/task_resources.py @@ -146,7 +146,7 @@ def get_all_cached_erasures(self) -> Dict[str, int]: f"{self.request.id}__erasure_request" ) # extract request id to return a map of address:value - return {k.split("__")[-1]: v for k, v in value_dict.items()} + return {k.split("__")[-1]: v for k, v in value_dict.items()} # type: ignore def write_execution_log( # pylint: disable=too-many-arguments self, diff --git a/src/fidesops/tasks/storage.py b/src/fidesops/tasks/storage.py index 45d3b1f4f..df5c67c46 100644 --- a/src/fidesops/tasks/storage.py +++ b/src/fidesops/tasks/storage.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import json import logging import os @@ -40,7 +42,7 @@ def encrypt_access_request_results(data: Union[str, bytes], request_id: str) -> if isinstance(data, bytes): data = data.decode(config.security.ENCODING) - encryption_key: str = cache.get(encryption_cache_key) + encryption_key: str | None = cache.get(encryption_cache_key) if not encryption_key: return data @@ -106,9 +108,9 @@ def upload_to_s3( # pylint: disable=R0913 logger.info(f"Starting S3 Upload of {file_key}") try: my_session = get_s3_session( - aws_access_key_id=storage_secrets[StorageSecrets.AWS_ACCESS_KEY_ID.value], + aws_access_key_id=storage_secrets[StorageSecrets.AWS_ACCESS_KEY_ID.value], # type: ignore aws_secret_access_key=storage_secrets[ - StorageSecrets.AWS_SECRET_ACCESS_KEY.value + StorageSecrets.AWS_SECRET_ACCESS_KEY.value # type: ignore ], ) @@ -136,10 +138,10 @@ def upload_to_onetrust( """Uploads arbitrary data to onetrust returned from an access request""" logger.info(f"Starting OneTrust Upload for ref_id {ref_id}") - onetrust_hostname = storage_secrets[StorageSecrets.ONETRUST_HOSTNAME.value] + onetrust_hostname = storage_secrets[StorageSecrets.ONETRUST_HOSTNAME.value] # type: ignore access_token = get_onetrust_access_token( - client_id=storage_secrets[StorageSecrets.ONETRUST_CLIENT_ID.value], - client_secret=storage_secrets[StorageSecrets.ONETRUST_CLIENT_SECRET.value], + client_id=storage_secrets[StorageSecrets.ONETRUST_CLIENT_ID.value], # type: ignore + client_secret=storage_secrets[StorageSecrets.ONETRUST_CLIENT_SECRET.value], # type: ignore hostname=onetrust_hostname, ) headers = {"Authorization": f"Bearer {access_token}"} diff --git a/src/fidesops/util/encryption/secrets_util.py b/src/fidesops/util/encryption/secrets_util.py index aaa101bb8..94260540b 100644 --- a/src/fidesops/util/encryption/secrets_util.py +++ b/src/fidesops/util/encryption/secrets_util.py @@ -19,9 +19,9 @@ def get_or_generate_secret( privacy_request_id: Optional[str], secret_type: SecretType, masking_secret_meta: MaskingSecretMeta[T], - ) -> T: + ) -> Optional[T]: if privacy_request_id is not None: - secret: T = SecretsUtil._get_secret_from_cache( + secret = SecretsUtil._get_secret_from_cache( privacy_request_id, secret_type, masking_secret_meta ) if not secret: @@ -40,7 +40,7 @@ def _get_secret_from_cache( privacy_request_id: str, secret_type: SecretType, masking_secret_meta: MaskingSecretMeta[T], - ) -> T: + ) -> Optional[T]: cache = get_cache() masking_secret_cache_key: str = get_masking_secret_cache_key( privacy_request_id=privacy_request_id, diff --git a/src/fidesops/util/saas_util.py b/src/fidesops/util/saas_util.py index 27a2957e7..a075a7d99 100644 --- a/src/fidesops/util/saas_util.py +++ b/src/fidesops/util/saas_util.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import json import logging import re @@ -44,8 +46,8 @@ def get_collection_grouped_inputs( collections: List[Collection], name: str ) -> Optional[Set[str]]: """Get collection grouped inputs""" - collection: Collection = next( - (collect for collect in collections if collect.name == name), {} + collection: Collection | None = next( + (collect for collect in collections if collect.name == name), None ) if not collection: return set() @@ -56,8 +58,8 @@ def get_collection_after( collections: List[Collection], name: str ) -> Set[CollectionAddress]: """If specified, return the collections that need to run before the current collection for saas configs""" - collection: Collection = next( - (collect for collect in collections if collect.name == name), {} + collection: Collection | None = next( + (collect for collect in collections if collect.name == name), None ) if not collection: return set()