Resolve issue with MyPy seeing files in fidesops as missing imports (#…

…719)


Co-authored-by: Paul Sanders <[email protected]>

CHANGELOG.md

@@ -20,6 +20,9 @@ The types of changes are:
 ## [Unreleased](https://github.com/ethyca/fidesops/compare/1.6.1...main)
 * Add support for multiple statuses to be selected for filtering subject requests [#660](https://github.com/ethyca/fidesops/pull/802)
 
+### Fixed
+* Resolve issue with MyPy seeing files in fidesops as missing imports [719](https://github.com/ethyca/fidesops/pull/719)
+
 ## [1.6.1](https://github.com/ethyca/fidesops/compare/1.6.0...1.6.1)
 
 ### Added

MANIFEST.in

@@ -5,6 +5,7 @@ include dev-requirements.txt
 include versioneer.py
 include src/fidesops/alembic.ini
 include src/fidesops/_version.py
+include src/fidesops/py.typed
 graft src/fidesops/migrations
 exclude src/fidesops/migrations/README
 exclude src/fidesops/migrations/script.py.mako

dev-requirements.txt

@@ -15,3 +15,4 @@ types-PyYAML==6.0.9
 types-redis==4.3.2
 types-toml==0.10.7
 types-ujson==5.2.0
+types-urllib3==1.26.15

pyproject.toml

@@ -7,7 +7,6 @@ requires = ["setuptools", "wheel", "versioneer-518"]  # PEP 508 specifications.
 [tool.mypy]
 exclude = ["fidesops.migrations.*"]
 warn_unused_configs = true
-ignore_missing_imports = true
 pretty = true
 plugins = ["pydantic.mypy", "sqlmypy"]
 disallow_untyped_defs = true
@@ -17,6 +16,30 @@ show_error_codes = true
 module = ["tests.*"]
 disallow_untyped_defs = false
 
+[[tool.mypy.overrides]]
+module = [
+  "alembic.*",
+  "apscheduler.*",
+  "boto3.*",
+  "botocore.*",
+  "bson.*",
+  "celery.*",
+  "dask.*",
+  "fideslang.*",
+  "fideslib.*",
+  "fideslog.*",
+  "jose.*",
+  "jwt.*",
+  "multidimensional_urlencode.*",
+  "pandas.*",
+  "pydash.*",
+  "pymongo.*",
+  "snowflake.*",
+  "sqlalchemy_utils.*",
+  "uvicorn.*"
+]
+ignore_missing_imports = true
+
 #######
 # Black
 #######

src/fidesops/api/v1/endpoints/connection_endpoints.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import logging
 from typing import List, Optional
 
@@ -246,7 +248,7 @@ def validate_secrets(
         )
 
     try:
-        schema = get_connection_secrets_validator(connection_type.value, saas_config)
+        schema = get_connection_secrets_validator(connection_type.value, saas_config)  # type: ignore
         logger.info(
             f"Validating secrets on connection config with key '{connection_config.key}'"
         )
@@ -266,7 +268,8 @@ def connection_status(
 
     connector = get_connector(connection_config)
     try:
-        status: ConnectionTestStatus = connector.test_connection()
+        status: ConnectionTestStatus | None = connector.test_connection()
+
     except (ConnectionException, ClientUnsuccessfulException) as exc:
         logger.warning(
             "Connection test failed on %s: %s",
@@ -282,8 +285,8 @@ def connection_status(
             failure_reason=str(exc),
         )
 
-    logger.info(f"Connection test {status.value} on {connection_config.key}")
-    connection_config.update_test_status(test_status=status, db=db)
+    logger.info(f"Connection test {status.value} on {connection_config.key}")  # type: ignore
+    connection_config.update_test_status(test_status=status, db=db)  # type: ignore
 
     return TestStatusMessage(
         msg=msg,

src/fidesops/api/v1/endpoints/dataset_endpoints.py

@@ -102,15 +102,14 @@ def validate_dataset(
     try:
         # Attempt to generate a traversal for this dataset by providing an empty
         # dictionary of all unique identity keys
-        graph = convert_dataset_to_graph(dataset, connection_config.key)
+        graph = convert_dataset_to_graph(dataset, connection_config.key)  # type: ignore
 
         # Datasets for SaaS connections need to be merged with a SaaS config to
         # be able to generate a valid traversal
         if connection_config.connection_type == ConnectionType.saas:
             _validate_saas_dataset(connection_config, dataset)
             graph = merge_datasets(
-                graph,
-                connection_config.get_saas_config().get_graph(),
+                graph, connection_config.get_saas_config().get_graph()  # type: ignore
             )
         complete_graph = DatasetGraph(graph)
         unique_identities = set(complete_graph.identity_keys.values())
@@ -247,7 +246,7 @@ def create_or_update_dataset(
 ) -> None:
     try:
         if connection_config.connection_type == ConnectionType.saas:
-            _validate_saas_dataset(connection_config, dataset)
+            _validate_saas_dataset(connection_config, dataset)  # type: ignore
         # Try to find an existing DatasetConfig matching the given connection & key
         dataset_config = DatasetConfig.create_or_update(db, data=data)
         created_or_updated.append(dataset_config.dataset)

src/fidesops/api/v1/endpoints/drp_endpoints.py

@@ -110,7 +110,7 @@ def create_drp_privacy_request(
         return PrivacyRequestDRPStatusResponse(
             request_id=privacy_request.id,
             received_at=privacy_request.requested_at,
-            status=DrpFidesopsMapper.map_status(privacy_request.status),
+            status=DrpFidesopsMapper.map_status(privacy_request.status),  # type: ignore
         )
 
     except common_exceptions.RedisConnectionError as exc:
@@ -174,7 +174,7 @@ def get_drp_data_rights(*, db: Session = Depends(deps.get_db)) -> DrpDataRightsR
 
     logger.info("Fetching available DRP data rights")
     actions: List[DrpAction] = [
-        item.drp_action
+        item.drp_action  # type: ignore
         for item in db.query(Policy.drp_action).filter(Policy.drp_action.isnot(None))
     ]
 
@@ -201,8 +201,7 @@ def revoke_request(
     if privacy_request.status != PrivacyRequestStatus.pending:
         raise HTTPException(
             status_code=HTTP_400_BAD_REQUEST,
-            detail=f"Invalid revoke request. Can only revoke `pending` requests. "
-            f"Privacy request '{privacy_request.id}' status = {privacy_request.status.value}.",
+            detail=f"Invalid revoke request. Can only revoke `pending` requests. Privacy request '{privacy_request.id}' status = {privacy_request.status.value}.",  # type: ignore
         )
 
     logger.info(f"Canceling privacy request '{privacy_request.id}'")
@@ -211,6 +210,6 @@ def revoke_request(
     return PrivacyRequestDRPStatusResponse(
         request_id=privacy_request.id,
         received_at=privacy_request.requested_at,
-        status=DrpFidesopsMapper.map_status(privacy_request.status),
+        status=DrpFidesopsMapper.map_status(privacy_request.status),  # type: ignore
         reason=data.reason,
     )

src/fidesops/api/v1/endpoints/encryption_endpoints.py

@@ -56,7 +56,7 @@ def aes_encrypt(encryption_request: AesEncryptionRequest) -> AesEncryptionRespon
 
     encrypted_value: str = aes_gcm_encrypt(
         encryption_request.value,
-        encryption_request.key,
+        encryption_request.key,  # type: ignore
         nonce,
     )
     return AesEncryptionResponse(

src/fidesops/api/v1/endpoints/oauth_endpoints.py

@@ -206,10 +206,10 @@ def oauth_callback(code: str, state: str, db: Session = Depends(get_db)) -> None
 
     try:
         authentication = (
-            connection_config.get_saas_config().client_config.authentication
+            connection_config.get_saas_config().client_config.authentication  # type: ignore
         )
-        auth_strategy: OAuth2AuthenticationStrategy = get_strategy(
-            authentication.strategy, authentication.configuration
+        auth_strategy: OAuth2AuthenticationStrategy = get_strategy(  # type: ignore
+            authentication.strategy, authentication.configuration  # type: ignore
         )
         auth_strategy.get_access_token(db, code, connection_config)
     except (OAuth2TokenException, FidesopsException) as exc:

src/fidesops/api/v1/endpoints/policy_webhook_endpoints.py

@@ -139,7 +139,9 @@ def put_webhooks(
     staged_webhook_keys = [webhook.key for webhook in staged_webhooks]
     webhooks_to_remove = getattr(
         policy, f"{webhook_cls.prefix}_execution_webhooks"
-    ).filter(webhook_cls.key.not_in(staged_webhook_keys))
+    ).filter(
+        webhook_cls.key.not_in(staged_webhook_keys)  # type: ignore
+    )
 
     if webhooks_to_remove.count():
         logger.info(
@@ -177,7 +179,7 @@ def create_or_update_pre_execution_webhooks(
     All webhooks must be included in the request in the desired order. Any missing webhooks
     from the request body will be removed.
     """
-    return put_webhooks(PolicyPreWebhook, policy_key, db, webhooks)
+    return put_webhooks(PolicyPreWebhook, policy_key, db, webhooks)  # type: ignore
 
 
 @router.put(
@@ -200,7 +202,7 @@ def create_or_update_post_execution_webhooks(
     All webhooks must be included in the request in the desired order. Any missing webhooks
     from the request body will be removed.
     """
-    return put_webhooks(PolicyPostWebhook, policy_key, db, webhooks)
+    return put_webhooks(PolicyPostWebhook, policy_key, db, webhooks)  # type: ignore
 
 
 def get_policy_webhook_or_error(
@@ -246,7 +248,7 @@ def get_policy_pre_execution_webhook(
     Loads the given Pre-Execution Webhook on the Policy
     """
     policy = get_policy_or_error(db, policy_key)
-    return get_policy_webhook_or_error(db, policy, pre_webhook_key, PolicyPreWebhook)
+    return get_policy_webhook_or_error(db, policy, pre_webhook_key, PolicyPreWebhook)  # type: ignore
 
 
 @router.get(
@@ -265,7 +267,7 @@ def get_policy_post_execution_webhook(
     Loads the given Post-Execution Webhook on the Policy
     """
     policy = get_policy_or_error(db, policy_key)
-    return get_policy_webhook_or_error(db, policy, post_webhook_key, PolicyPostWebhook)
+    return get_policy_webhook_or_error(db, policy, post_webhook_key, PolicyPostWebhook)  # type: ignore
 
 
 def _patch_webhook(
@@ -286,7 +288,7 @@ def _patch_webhook(
 
     if data.get("connection_config_key"):
         connection_config = get_connection_config_or_error(
-            db, data.get("connection_config_key")
+            db, data.get("connection_config_key")  # type: ignore
         )
         data["connection_config_id"] = connection_config.id
 
@@ -355,7 +357,7 @@ def update_pre_execution_webhook(
         policy_key=policy_key,
         webhook_key=pre_webhook_key,
         webhook_body=webhook_body,
-        webhook_cls=PolicyPreWebhook,
+        webhook_cls=PolicyPreWebhook,  # type: ignore
     )
 
 
@@ -383,7 +385,7 @@ def update_post_execution_webhook(
         policy_key=policy_key,
         webhook_key=post_webhook_key,
         webhook_body=webhook_body,
-        webhook_cls=PolicyPostWebhook,
+        webhook_cls=PolicyPostWebhook,  # type: ignore
     )
 
 
@@ -442,7 +444,7 @@ def delete_pre_execution_webhook(
         db=db,
         policy_key=policy_key,
         webhook_key=pre_webhook_key,
-        webhook_cls=PolicyPreWebhook,
+        webhook_cls=PolicyPreWebhook,  # type: ignore
     )
 
 
@@ -463,5 +465,5 @@ def delete_post_execution_webhook(
         db=db,
         policy_key=policy_key,
         webhook_key=post_webhook_key,
-        webhook_cls=PolicyPostWebhook,
+        webhook_cls=PolicyPostWebhook,  # type: ignore
     )

src/fidesops/api/v1/endpoints/privacy_request_endpoints.py

@@ -400,9 +400,9 @@ def attach_resume_instructions(privacy_request: PrivacyRequest) -> None:
         resume_endpoint = PRIVACY_REQUEST_RETRY
 
     if stopped_collection_details:
-        stopped_collection_details.step = stopped_collection_details.step.value
+        stopped_collection_details.step = stopped_collection_details.step.value  # type: ignore
         stopped_collection_details.collection = (
-            stopped_collection_details.collection.value
+            stopped_collection_details.collection.value  # type: ignore
         )
 
     privacy_request.stopped_collection_details = stopped_collection_details
@@ -613,12 +613,12 @@ def resume_privacy_request(
 ) -> PrivacyRequestResponse:
     """Resume running a privacy request after it was paused by a Pre-Execution webhook"""
     privacy_request = get_privacy_request_or_error(db, privacy_request_id)
-    privacy_request.cache_identity(webhook_callback.derived_identity)
+    privacy_request.cache_identity(webhook_callback.derived_identity)  # type: ignore
 
     if privacy_request.status != PrivacyRequestStatus.paused:
         raise HTTPException(
             status_code=HTTP_400_BAD_REQUEST,
-            detail=f"Invalid resume request: privacy request '{privacy_request.id}' status = {privacy_request.status.value}.",
+            detail=f"Invalid resume request: privacy request '{privacy_request.id}' status = {privacy_request.status.value}.",  # type: ignore
         )
 
     logger.info(
@@ -670,7 +670,7 @@ def resume_privacy_request_with_manual_input(
     if privacy_request.status != PrivacyRequestStatus.paused:
         raise HTTPException(
             status_code=HTTP_400_BAD_REQUEST,
-            detail=f"Invalid resume request: privacy request '{privacy_request.id}' "
+            detail=f"Invalid resume request: privacy request '{privacy_request.id}' "  # type: ignore
             f"status = {privacy_request.status.value}. Privacy request is not paused.",
         )
 
@@ -715,7 +715,7 @@ def resume_privacy_request_with_manual_input(
         logger.info(
             f"Caching manually erased row count for privacy request '{privacy_request_id}', collection: '{paused_collection}'"
         )
-        privacy_request.cache_manual_erasure_count(paused_collection, manual_count)
+        privacy_request.cache_manual_erasure_count(paused_collection, manual_count)  # type: ignore
 
     logger.info(
         f"Resuming privacy request '{privacy_request_id}', {paused_step.value} step, from collection "
@@ -811,7 +811,7 @@ def restart_privacy_request_from_failure(
     if privacy_request.status != PrivacyRequestStatus.error:
         raise HTTPException(
             status_code=HTTP_400_BAD_REQUEST,
-            detail=f"Cannot restart privacy request from failure: privacy request '{privacy_request.id}' status = {privacy_request.status.value}.",
+            detail=f"Cannot restart privacy request from failure: privacy request '{privacy_request.id}' status = {privacy_request.status.value}.",  # type: ignore
         )
 
     failed_details: Optional[

src/fidesops/api/v1/endpoints/saas_config_endpoints.py

@@ -85,7 +85,7 @@ def verify_oauth_connection_config(
             detail="The connection config does not contain a SaaS config.",
         )
 
-    authentication = connection_config.get_saas_config().client_config.authentication
+    authentication = saas_config.client_config.authentication
     if not authentication:
         raise HTTPException(
             status_code=HTTP_422_UNPROCESSABLE_ENTITY,
@@ -145,7 +145,7 @@ def patch_saas_config(
         f"Updating SaaS config '{saas_config.fides_key}' on connection config '{connection_config.key}'"
     )
     connection_config.update_saas_config(db, saas_config=saas_config)
-    return connection_config.saas_config
+    return connection_config.saas_config  # type: ignore
 
 
 @router.get(
@@ -165,7 +165,7 @@ def get_saas_config(
             status_code=HTTP_404_NOT_FOUND,
             detail=f"No SaaS config found for connection '{connection_config.key}'",
         )
-    return connection_config.saas_config
+    return saas_config
 
 
 @router.delete(
@@ -233,11 +233,11 @@ def authorize_connection(
     """Returns the authorization URL for the SaaS Connector (if available)"""
 
     verify_oauth_connection_config(connection_config)
-    authentication = connection_config.get_saas_config().client_config.authentication
+    authentication = connection_config.get_saas_config().client_config.authentication  # type: ignore
 
     try:
         auth_strategy: OAuth2AuthenticationStrategy = get_strategy(
-            authentication.strategy, authentication.configuration
+            authentication.strategy, authentication.configuration  # type: ignore
         )
         return auth_strategy.get_authorization_url(db, connection_config)
     except FidesopsException as exc:

src/fidesops/api/v1/exception_handlers.py

@@ -1,7 +1,7 @@
 from typing import Callable, List
 
 from fastapi import Request
-from fastapi.responses import JSONResponse, Response
+from fastapi.responses import JSONResponse
 from starlette.status import HTTP_500_INTERNAL_SERVER_ERROR
 
 from fidesops.common_exceptions import FunctionalityNotConfigured
@@ -17,5 +17,7 @@ def functionality_not_configured_handler(
         )
 
     @classmethod
-    def get_handlers(cls) -> List[Callable[[Request, Exception], Response]]:
+    def get_handlers(
+        cls,
+    ) -> List[Callable[[Request, FunctionalityNotConfigured], JSONResponse]]:
         return [ExceptionHandlers.functionality_not_configured_handler]