diff --git a/.github/workflows/safe_pr_checks.yml b/.github/workflows/safe_pr_checks.yml index 2f790d65f..e8b1ae903 100644 --- a/.github/workflows/safe_pr_checks.yml +++ b/.github/workflows/safe_pr_checks.yml @@ -18,6 +18,9 @@ jobs: - name: Format run: make black-ci + - name: Import Sorting + run: make isort-ci + - name: Lint run: make pylint diff --git a/CHANGELOG.md b/CHANGELOG.md index dd84c324e..ced374f74 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,13 @@ The types of changes are: ## [Unreleased](https://github.com/ethyca/fidesops/compare/1.5.0...main) + +### Developer Experience + +* Import ordering is now enforced using [isort](https://pycqa.github.io/isort/) in CI [#533](https://github.com/ethyca/fidesops/pull/533) + ### Docs + * Updated documentation for the user management ui [#530](https://github.com/ethyca/fidesops/pull/530) ## [1.5.0](https://github.com/ethyca/fidesops/compare/1.4.2...1.5.0) - 2022-05-18 @@ -57,10 +63,10 @@ The types of changes are: * Publish Fidesops to Pypi [#491](https://github.com/ethyca/fidesops/pull/491) -### Changed +### Changed * Converted HTTP Status Codes to Starlette constant values [#438](https://github.com/ethyca/fidesops/pull/438) * SaasConnector.send behavior on ignore_errors now returns raw response [#462](https://github.com/ethyca/fidesops/pull/462) -* Seed user permissions in `create_superuser.py` script [#468](https://github.com/ethyca/fidesops/pull/468) +* Seed user permissions in `create_superuser.py` script [#468](https://github.com/ethyca/fidesops/pull/468) * User API Endpoints (update fields and reset user passwords) [#471](https://github.com/ethyca/fidesops/pull/471) * Format tests with `black` [#466](https://github.com/ethyca/fidesops/pull/466) * Extract privacy request endpoint logic into separate service for DRP [#470](https://github.com/ethyca/fidesops/pull/470) @@ -89,4 +95,4 @@ The types of changes are: ### Developer Experience * Add script to seed initial Privacy Request [#487](https://github.com/ethyca/fidesops/pull/487) -* Add first and last name to `createsuperuser` script [#486](https://github.com/ethyca/fidesops/pull/486) \ No newline at end of file +* Add first and last name to `createsuperuser` script [#486](https://github.com/ethyca/fidesops/pull/486) diff --git a/Makefile b/Makefile index 225e45640..3c67e2a83 100644 --- a/Makefile +++ b/Makefile @@ -77,7 +77,7 @@ docker-push: # CI #################### -check-all: black-ci pylint mypy check-migrations pytest pytest-integration +check-all: isort-ci black-ci pylint mypy check-migrations pytest pytest-integration black-ci: compose-build @echo "Running black checks..." @@ -93,6 +93,11 @@ check-migrations: compose-build from fidesops.core.config import config; \ check_missing_migrations(config.database.SQLALCHEMY_DATABASE_URI);" +isort-ci: + @echo "Running isort checks..." + @docker-compose run $(IMAGE_NAME) \ + isort src tests --check-only + pylint: compose-build @echo "Running pylint checks..." @docker-compose run $(IMAGE_NAME) \ @@ -155,6 +160,12 @@ compose-build: @docker-compose down --remove-orphans @docker-compose build --build-arg REQUIRE_MSSQL="true" +.PHONY: isort +isort: + @echo "Running isort checks..." + @docker-compose run $(IMAGE_NAME) \ + isort src tests + .PHONY: teardown teardown: @echo "Tearing down the dev environment..." diff --git a/dev-requirements.txt b/dev-requirements.txt index 7f4a86130..f17442192 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,12 +1,13 @@ -Faker==8.12.1 black==21.8b0 +Faker==8.12.1 +ipython +isort +mypy==0.812 packaging==20.9 +pdbpp pre-commit==2.9.3 pylint==2.6.0 -pytest==6.2.2 pytest-cov==2.11.1 -requests-mock==1.8.0 -mypy==0.812 -pdbpp -ipython pytest-env==0.6.2 +pytest==6.2.2 +requests-mock==1.8.0 diff --git a/pyproject.toml b/pyproject.toml index 3d54c778a..a4459312b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,6 +33,14 @@ exclude = ''' )/ ''' +####### +# isort +####### +[tool.isort] +profile = "black" +line_length = 88 +src_paths = ["src", "tests"] + ######## # Pylint ######## diff --git a/src/fidesops/api/v1/api.py b/src/fidesops/api/v1/api.py index 8e217dd3f..1189c2cc2 100644 --- a/src/fidesops/api/v1/api.py +++ b/src/fidesops/api/v1/api.py @@ -12,13 +12,12 @@ policy_endpoints, policy_webhook_endpoints, privacy_request_endpoints, - storage_endpoints, saas_config_endpoints, + storage_endpoints, user_endpoints, user_permission_endpoints, ) - api_router = APIRouter() api_router.include_router(config_endpoints.router) api_router.include_router(connection_endpoints.router) diff --git a/src/fidesops/api/v1/endpoints/config_endpoints.py b/src/fidesops/api/v1/endpoints/config_endpoints.py index ea7979318..7c052e854 100644 --- a/src/fidesops/api/v1/endpoints/config_endpoints.py +++ b/src/fidesops/api/v1/endpoints/config_endpoints.py @@ -1,22 +1,14 @@ import logging -from typing import ( - Any, - Dict, -) +from typing import Any, Dict from fastapi import APIRouter from fastapi.params import Security -from fidesops.api.v1 import ( - scope_registry as scopes, - urn_registry as urls, -) -from fidesops.core.config import ( - censored_config, -) +from fidesops.api.v1 import scope_registry as scopes +from fidesops.api.v1 import urn_registry as urls +from fidesops.core.config import censored_config from fidesops.util.oauth_util import verify_oauth_client - router = APIRouter(tags=["Config"], prefix=urls.V1_URL_PREFIX) logger = logging.getLogger(__name__) diff --git a/src/fidesops/api/v1/endpoints/dataset_endpoints.py b/src/fidesops/api/v1/endpoints/dataset_endpoints.py index 55fefdd96..f35215862 100644 --- a/src/fidesops/api/v1/endpoints/dataset_endpoints.py +++ b/src/fidesops/api/v1/endpoints/dataset_endpoints.py @@ -1,7 +1,7 @@ import logging from typing import List -import yaml +import yaml from fastapi import APIRouter, Depends, HTTPException, Request from fastapi.params import Security from fastapi_pagination import Page, Params @@ -27,8 +27,8 @@ DATASET_BY_KEY, DATASET_VALIDATE, DATASETS, - YAML_DATASETS, V1_URL_PREFIX, + YAML_DATASETS, ) from fidesops.common_exceptions import ( SaaSConfigNotFoundException, diff --git a/src/fidesops/api/v1/endpoints/drp_endpoints.py b/src/fidesops/api/v1/endpoints/drp_endpoints.py index 392391354..0448b5ee6 100644 --- a/src/fidesops/api/v1/endpoints/drp_endpoints.py +++ b/src/fidesops/api/v1/endpoints/drp_endpoints.py @@ -1,14 +1,14 @@ import logging -from typing import Dict, Any, Optional +from typing import Any, Dict, Optional import jwt -from fastapi import HTTPException, Depends, APIRouter, Security +from fastapi import APIRouter, Depends, HTTPException, Security from sqlalchemy.orm import Session from starlette.status import ( + HTTP_200_OK, HTTP_404_NOT_FOUND, HTTP_422_UNPROCESSABLE_ENTITY, HTTP_424_FAILED_DEPENDENCY, - HTTP_200_OK, HTTP_500_INTERNAL_SERVER_ERROR, ) @@ -19,7 +19,7 @@ from fidesops.core.config import config from fidesops.models.policy import Policy from fidesops.models.privacy_request import PrivacyRequest -from fidesops.schemas.drp_privacy_request import DrpPrivacyRequestCreate, DrpIdentity +from fidesops.schemas.drp_privacy_request import DrpIdentity, DrpPrivacyRequestCreate from fidesops.schemas.privacy_request import PrivacyRequestDRPStatusResponse from fidesops.schemas.redis_cache import PrivacyRequestIdentity from fidesops.service.drp.drp_fidesops_mapper import DrpFidesopsMapper diff --git a/src/fidesops/api/v1/endpoints/encryption_endpoints.py b/src/fidesops/api/v1/endpoints/encryption_endpoints.py index 0e02896f4..1370f7419 100644 --- a/src/fidesops/api/v1/endpoints/encryption_endpoints.py +++ b/src/fidesops/api/v1/endpoints/encryption_endpoints.py @@ -1,29 +1,29 @@ +import logging import secrets -import logging from fastapi import APIRouter, Security from fidesops.api.v1.scope_registry import ENCRYPTION_EXEC from fidesops.api.v1.urn_registry import ( + DECRYPT_AES, ENCRYPT_AES, ENCRYPTION_KEY, - DECRYPT_AES, V1_URL_PREFIX, ) from fidesops.core.config import config from fidesops.schemas.encryption_request import ( + AesDecryptionRequest, + AesDecryptionResponse, AesEncryptionRequest, AesEncryptionResponse, - AesDecryptionResponse, - AesDecryptionRequest, ) from fidesops.util import cryptographic_util -from fidesops.util.cryptographic_util import bytes_to_b64_str, b64_str_to_bytes +from fidesops.util.cryptographic_util import b64_str_to_bytes, bytes_to_b64_str from fidesops.util.encryption.aes_gcm_encryption_scheme import ( - encrypt_verify_secret_length as aes_gcm_encrypt, + decrypt as aes_gcm_decrypt, ) from fidesops.util.encryption.aes_gcm_encryption_scheme import ( - decrypt as aes_gcm_decrypt, + encrypt_verify_secret_length as aes_gcm_encrypt, ) from fidesops.util.oauth_util import verify_oauth_client diff --git a/src/fidesops/api/v1/endpoints/masking_endpoints.py b/src/fidesops/api/v1/endpoints/masking_endpoints.py index 574796c67..4d71222d4 100644 --- a/src/fidesops/api/v1/endpoints/masking_endpoints.py +++ b/src/fidesops/api/v1/endpoints/masking_endpoints.py @@ -2,19 +2,18 @@ from typing import List from fastapi import APIRouter, HTTPException -from starlette.status import HTTP_404_NOT_FOUND, HTTP_400_BAD_REQUEST +from starlette.status import HTTP_400_BAD_REQUEST, HTTP_404_NOT_FOUND from fidesops.api.v1.urn_registry import MASKING, MASKING_STRATEGY, V1_URL_PREFIX from fidesops.common_exceptions import ValidationError -from fidesops.schemas.masking.masking_api import MaskingAPIResponse, MaskingAPIRequest +from fidesops.schemas.masking.masking_api import MaskingAPIRequest, MaskingAPIResponse from fidesops.schemas.masking.masking_strategy_description import ( MaskingStrategyDescription, ) - from fidesops.service.masking.strategy.masking_strategy_factory import ( + NoSuchStrategyException, get_strategies, get_strategy, - NoSuchStrategyException, ) router = APIRouter(tags=["Masking"], prefix=V1_URL_PREFIX) diff --git a/src/fidesops/common_exceptions.py b/src/fidesops/common_exceptions.py index 1659a5326..26be8156e 100644 --- a/src/fidesops/common_exceptions.py +++ b/src/fidesops/common_exceptions.py @@ -1,10 +1,9 @@ from typing import List from fastapi import HTTPException - from starlette.status import ( - HTTP_401_UNAUTHORIZED, HTTP_400_BAD_REQUEST, + HTTP_401_UNAUTHORIZED, HTTP_403_FORBIDDEN, HTTP_404_NOT_FOUND, ) diff --git a/src/fidesops/core/config.py b/src/fidesops/core/config.py index 001c88843..f03fe72b9 100644 --- a/src/fidesops/core/config.py +++ b/src/fidesops/core/config.py @@ -3,17 +3,11 @@ import hashlib import logging import os -from typing import Dict, List, Optional, Union, Tuple, Any, MutableMapping +from typing import Any, Dict, List, MutableMapping, Optional, Tuple, Union import bcrypt import toml -from pydantic import ( - AnyHttpUrl, - BaseSettings, - PostgresDsn, - ValidationError, - validator, -) +from pydantic import AnyHttpUrl, BaseSettings, PostgresDsn, ValidationError, validator from pydantic.env_settings import SettingsSourceCallable from fidesops.common_exceptions import MissingConfig diff --git a/src/fidesops/db/base.py b/src/fidesops/db/base.py index 22fa76353..5f357f207 100644 --- a/src/fidesops/db/base.py +++ b/src/fidesops/db/base.py @@ -2,12 +2,12 @@ # Import all the models, so that Base has them before being # imported by Alembic from fidesops.db.base_class import Base +from fidesops.models.audit_log import AuditLog from fidesops.models.client import ClientDetail from fidesops.models.connectionconfig import ConnectionConfig from fidesops.models.datasetconfig import DatasetConfig +from fidesops.models.fidesops_user import FidesopsUser +from fidesops.models.fidesops_user_permissions import FidesopsUserPermissions from fidesops.models.policy import Policy, Rule, RuleTarget from fidesops.models.privacy_request import PrivacyRequest from fidesops.models.storage import StorageConfig -from fidesops.models.fidesops_user import FidesopsUser -from fidesops.models.audit_log import AuditLog -from fidesops.models.fidesops_user_permissions import FidesopsUserPermissions diff --git a/src/fidesops/db/base_class.py b/src/fidesops/db/base_class.py index b85128efa..8c9f09f9d 100644 --- a/src/fidesops/db/base_class.py +++ b/src/fidesops/db/base_class.py @@ -7,14 +7,10 @@ from uuid import uuid4 import six - -from sqlalchemy import Column, String, DateTime +from sqlalchemy import Column, DateTime, String from sqlalchemy.ext.declarative import declarative_base, declared_attr from sqlalchemy.ext.mutable import MutableDict -from sqlalchemy.orm import ( - Session, - Query, -) +from sqlalchemy.orm import Query, Session from sqlalchemy.sql import func from sqlalchemy_utils import JSONType diff --git a/src/fidesops/db/database.py b/src/fidesops/db/database.py index 5073653f8..29c32c64b 100644 --- a/src/fidesops/db/database.py +++ b/src/fidesops/db/database.py @@ -2,13 +2,14 @@ Contains all of the logic for spinning up/tearing down the database. """ import os -from pydantic import PostgresDsn from alembic import command from alembic.config import Config from alembic.migration import MigrationContext +from pydantic import PostgresDsn from fidesops.db.session import get_db_engine + from .base import Base diff --git a/src/fidesops/db/session.py b/src/fidesops/db/session.py index c83a55781..cd8f29c02 100644 --- a/src/fidesops/db/session.py +++ b/src/fidesops/db/session.py @@ -3,10 +3,8 @@ from sqlalchemy import create_engine from sqlalchemy.engine import Engine -from sqlalchemy.orm import ( - sessionmaker, - Session, -) +from sqlalchemy.orm import Session, sessionmaker + from fidesops.core.config import config logger = logging.getLogger(__name__) diff --git a/src/fidesops/graph/config.py b/src/fidesops/graph/config.py index b8ade6c6a..17fbe6e93 100644 --- a/src/fidesops/graph/config.py +++ b/src/fidesops/graph/config.py @@ -80,14 +80,15 @@ from abc import ABC, abstractmethod from collections import defaultdict from dataclasses import dataclass -from typing import List, Literal, Optional, Tuple, Set, Dict, Any, Callable +from typing import Any, Callable, Dict, List, Literal, Optional, Set, Tuple + from pydantic import BaseModel, validator from fidesops.common_exceptions import FidesopsException from fidesops.graph.data_type import ( + DataType, DataTypeConverter, get_data_type_converter, - DataType, ) from fidesops.schemas.shared_schemas import FidesOpsKey from fidesops.util.collection_util import merge_dicts diff --git a/src/fidesops/graph/data_type.py b/src/fidesops/graph/data_type.py index b03764441..3eb5560a1 100644 --- a/src/fidesops/graph/data_type.py +++ b/src/fidesops/graph/data_type.py @@ -1,7 +1,7 @@ import logging -from abc import abstractmethod, ABC +from abc import ABC, abstractmethod from enum import Enum -from typing import Generic, Optional, Any, TypeVar, Dict, Tuple, Set +from typing import Any, Dict, Generic, Optional, Set, Tuple, TypeVar from bson.errors import InvalidId from bson.objectid import ObjectId diff --git a/src/fidesops/graph/graph.py b/src/fidesops/graph/graph.py index 08c94392b..ed32f0d46 100644 --- a/src/fidesops/graph/graph.py +++ b/src/fidesops/graph/graph.py @@ -2,18 +2,18 @@ import logging from collections import defaultdict -from typing import Tuple, Set, Dict, Optional, List, Callable +from typing import Callable, Dict, List, Optional, Set, Tuple from fidesops.common_exceptions import ValidationError from fidesops.graph.config import ( - EdgeDirection, - FieldAddress, - CollectionAddress, - SeedAddress, Collection, + CollectionAddress, Dataset, + EdgeDirection, Field, + FieldAddress, FieldPath, + SeedAddress, ) from fidesops.schemas.shared_schemas import FidesOpsKey diff --git a/src/fidesops/graph/traversal.py b/src/fidesops/graph/traversal.py index 4a54edc1c..1786bf3a0 100644 --- a/src/fidesops/graph/traversal.py +++ b/src/fidesops/graph/traversal.py @@ -1,24 +1,24 @@ from __future__ import annotations import logging -from typing import List, Any, Tuple, Set, Dict, Callable, cast +from typing import Any, Callable, Dict, List, Set, Tuple, cast import pydash.collections from fidesops.common_exceptions import TraversalError from fidesops.graph.config import ( + ROOT_COLLECTION_ADDRESS, + Collection, CollectionAddress, - FieldAddress, Dataset, - Collection, - ROOT_COLLECTION_ADDRESS, - FieldPath, Field, + FieldAddress, + FieldPath, ) -from fidesops.graph.graph import Node, Edge, DatasetGraph +from fidesops.graph.graph import DatasetGraph, Edge, Node +from fidesops.util.collection_util import Row, append from fidesops.util.logger import NotPii from fidesops.util.matching_queue import MatchingQueue -from fidesops.util.collection_util import append, Row logger = logging.getLogger(__name__) diff --git a/src/fidesops/main.py b/src/fidesops/main.py index 58bf93ce6..2208e6501 100644 --- a/src/fidesops/main.py +++ b/src/fidesops/main.py @@ -1,12 +1,13 @@ import logging + import uvicorn from fastapi import FastAPI from starlette.middleware.cors import CORSMiddleware from fidesops.api.v1.api import api_router from fidesops.api.v1.urn_registry import V1_URL_PREFIX -from fidesops.db.database import init_db from fidesops.core.config import config +from fidesops.db.database import init_db from fidesops.tasks.scheduled.scheduler import scheduler from fidesops.tasks.scheduled.tasks import initiate_scheduled_request_intake from fidesops.util.logger import get_fides_log_record_factory diff --git a/src/fidesops/models/audit_log.py b/src/fidesops/models/audit_log.py index 8009525a9..04d759995 100644 --- a/src/fidesops/models/audit_log.py +++ b/src/fidesops/models/audit_log.py @@ -1,5 +1,8 @@ from enum import Enum as EnumType -from sqlalchemy import Column, String, Enum as EnumColumn + +from sqlalchemy import Column +from sqlalchemy import Enum as EnumColumn +from sqlalchemy import String from fidesops.db.base_class import Base diff --git a/src/fidesops/models/client.py b/src/fidesops/models/client.py index 404935943..9f5d0216f 100644 --- a/src/fidesops/models/client.py +++ b/src/fidesops/models/client.py @@ -1,8 +1,8 @@ import json from datetime import datetime -from typing import List, Any, Optional, Tuple +from typing import Any, List, Optional, Tuple -from sqlalchemy import Column, String, ARRAY, ForeignKey +from sqlalchemy import ARRAY, Column, ForeignKey, String from sqlalchemy.ext.declarative import declared_attr from sqlalchemy.orm import Session @@ -11,14 +11,14 @@ from fidesops.db.base_class import Base from fidesops.models.fidesops_user import FidesopsUser from fidesops.schemas.jwt import ( - JWE_PAYLOAD_SCOPES, - JWE_PAYLOAD_CLIENT_ID, JWE_ISSUED_AT, + JWE_PAYLOAD_CLIENT_ID, + JWE_PAYLOAD_SCOPES, ) from fidesops.util.cryptographic_util import ( generate_salt, - hash_with_salt, generate_secure_random_string, + hash_with_salt, ) DEFAULT_SCOPES: List[str] = [] diff --git a/src/fidesops/models/connectionconfig.py b/src/fidesops/models/connectionconfig.py index c5355d04a..f79283596 100644 --- a/src/fidesops/models/connectionconfig.py +++ b/src/fidesops/models/connectionconfig.py @@ -4,14 +4,7 @@ from datetime import datetime from typing import Optional -from sqlalchemy import ( - Column, - Enum, - String, - DateTime, - Boolean, -) - +from sqlalchemy import Boolean, Column, DateTime, Enum, String from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.ext.mutable import MutableDict from sqlalchemy.orm import Session @@ -20,12 +13,9 @@ StringEncryptedType, ) -from fidesops.schemas.saas.saas_config import SaaSConfig from fidesops.core.config import config -from fidesops.db.base_class import ( - Base, - JSONTypeOverride, -) +from fidesops.db.base_class import Base, JSONTypeOverride +from fidesops.schemas.saas.saas_config import SaaSConfig class ConnectionTestStatus(enum.Enum): diff --git a/src/fidesops/models/datasetconfig.py b/src/fidesops/models/datasetconfig.py index 326c68484..3e591ea5d 100644 --- a/src/fidesops/models/datasetconfig.py +++ b/src/fidesops/models/datasetconfig.py @@ -1,28 +1,23 @@ import logging -from typing import Dict, Any, Set, Optional +from typing import Any, Dict, Optional, Set from boto3 import Session -from sqlalchemy import ( - Column, - ForeignKey, - String, -) +from sqlalchemy import Column, ForeignKey, String from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.ext.mutable import MutableDict from sqlalchemy.orm import relationship -from fidesops.models.connectionconfig import ConnectionType from fidesops.db.base_class import Base from fidesops.graph.config import ( Collection, + CollectionAddress, + Dataset, Field, FieldAddress, - Dataset, - CollectionAddress, generate_field, ) from fidesops.graph.data_type import parse_data_type_string -from fidesops.models.connectionconfig import ConnectionConfig +from fidesops.models.connectionconfig import ConnectionConfig, ConnectionType from fidesops.schemas.dataset import FidesopsDataset, FidesopsDatasetField from fidesops.schemas.shared_schemas import FidesOpsKey from fidesops.util.logger import NotPii diff --git a/src/fidesops/models/fidesops_user.py b/src/fidesops/models/fidesops_user.py index b91bb277a..a92e2b6a5 100644 --- a/src/fidesops/models/fidesops_user.py +++ b/src/fidesops/models/fidesops_user.py @@ -1,7 +1,7 @@ from datetime import datetime -from typing import Dict, Any, Tuple +from typing import Any, Dict, Tuple -from sqlalchemy import Column, String, DateTime +from sqlalchemy import Column, DateTime, String from sqlalchemy.orm import Session, relationship from fidesops.core.config import config diff --git a/src/fidesops/models/fidesops_user_permissions.py b/src/fidesops/models/fidesops_user_permissions.py index c699aa72c..f72176ff4 100644 --- a/src/fidesops/models/fidesops_user_permissions.py +++ b/src/fidesops/models/fidesops_user_permissions.py @@ -1,9 +1,9 @@ -from sqlalchemy import Column, String, ARRAY, ForeignKey -from sqlalchemy.orm import relationship, backref +from sqlalchemy import ARRAY, Column, ForeignKey, String +from sqlalchemy.orm import backref, relationship +from fidesops.api.v1.scope_registry import PRIVACY_REQUEST_READ from fidesops.db.base_class import Base from fidesops.models.fidesops_user import FidesopsUser -from fidesops.api.v1.scope_registry import PRIVACY_REQUEST_READ class FidesopsUserPermissions(Base): diff --git a/src/fidesops/models/policy.py b/src/fidesops/models/policy.py index d86c35702..91b99f92b 100644 --- a/src/fidesops/models/policy.py +++ b/src/fidesops/models/policy.py @@ -1,33 +1,14 @@ # pylint: disable=E1101 from enum import Enum as EnumType - -from typing import ( - Any, - Dict, - List, - Optional, - Tuple, - Union, -) +from typing import Any, Dict, List, Optional, Tuple, Union from fideslang import DEFAULT_TAXONOMY from fideslang.models import DataCategory as FideslangDataCategory -from sqlalchemy import ( - Column, - Enum as EnumColumn, - ForeignKey, - String, - UniqueConstraint, - Integer, -) +from sqlalchemy import Column +from sqlalchemy import Enum as EnumColumn +from sqlalchemy import ForeignKey, Integer, String, UniqueConstraint from sqlalchemy.ext.mutable import MutableDict -from sqlalchemy.orm import ( - relationship, - Session, - declared_attr, - backref, -) - +from sqlalchemy.orm import Session, backref, declared_attr, relationship from sqlalchemy_utils.types.encrypted.encrypted_type import ( AesGcmEngine, StringEncryptedType, @@ -36,11 +17,7 @@ from fidesops import common_exceptions from fidesops.common_exceptions import WebhookOrderException from fidesops.core.config import config -from fidesops.db.base_class import ( - Base, - FidesopsBase, - JSONTypeOverride, -) +from fidesops.db.base_class import Base, FidesopsBase, JSONTypeOverride from fidesops.models.client import ClientDetail from fidesops.models.connectionconfig import ConnectionConfig from fidesops.models.storage import StorageConfig diff --git a/src/fidesops/models/privacy_request.py b/src/fidesops/models/privacy_request.py index f64656914..b32b01162 100644 --- a/src/fidesops/models/privacy_request.py +++ b/src/fidesops/models/privacy_request.py @@ -1,36 +1,26 @@ # pylint: disable=R0401 +import json import logging from datetime import datetime - -import json - +from enum import Enum as EnumType from typing import Any, Dict, Optional -from enum import Enum as EnumType +from sqlalchemy import Column, DateTime +from sqlalchemy import Enum as EnumColumn +from sqlalchemy import ForeignKey, String from sqlalchemy.dialects.postgresql import JSONB -from sqlalchemy import ( - Column, - DateTime, - Enum as EnumColumn, - ForeignKey, - String, -) - from sqlalchemy.ext.mutable import MutableList -from sqlalchemy.orm import relationship, Session, backref +from sqlalchemy.orm import Session, backref, relationship from fidesops.api.v1.scope_registry import PRIVACY_REQUEST_CALLBACK_RESUME from fidesops.common_exceptions import PrivacyRequestPaused -from fidesops.db.base_class import ( - Base, - FidesopsBase, -) +from fidesops.db.base_class import Base, FidesopsBase from fidesops.models.audit_log import AuditLog from fidesops.models.client import ClientDetail from fidesops.models.fidesops_user import FidesopsUser from fidesops.models.policy import ( - Policy, ActionType, + Policy, PolicyPreWebhook, WebhookDirection, WebhookTypes, @@ -44,13 +34,13 @@ from fidesops.schemas.masking.masking_secrets import MaskingSecretCache from fidesops.schemas.redis_cache import PrivacyRequestIdentity from fidesops.util.cache import ( + FidesopsRedis, get_all_cache_keys_for_privacy_request, get_cache, - get_identity_cache_key, - FidesopsRedis, + get_drp_request_body_cache_key, get_encryption_cache_key, + get_identity_cache_key, get_masking_secret_cache_key, - get_drp_request_body_cache_key, ) from fidesops.util.oauth_util import generate_jwe diff --git a/src/fidesops/models/storage.py b/src/fidesops/models/storage.py index 0770bc402..4d6ff2f11 100644 --- a/src/fidesops/models/storage.py +++ b/src/fidesops/models/storage.py @@ -1,31 +1,24 @@ import logging from pydantic import ValidationError -from sqlalchemy import ( - Column, - String, - Enum, -) +from sqlalchemy import Column, Enum, String from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.ext.mutable import MutableDict from sqlalchemy.orm import Session - from sqlalchemy_utils.types.encrypted.encrypted_type import ( AesGcmEngine, StringEncryptedType, ) -from fidesops.db.base_class import Base +from fidesops.core.config import config +from fidesops.db.base_class import Base, JSONTypeOverride from fidesops.schemas.storage.storage import ( - StorageType, - StorageSecretsS3, - StorageSecretsOnetrust, - ResponseFormat, SUPPORTED_STORAGE_SECRETS, + ResponseFormat, + StorageSecretsOnetrust, + StorageSecretsS3, + StorageType, ) - -from fidesops.core.config import config -from fidesops.db.base_class import JSONTypeOverride from fidesops.schemas.storage.storage_secrets_docs_only import possible_storage_secrets logger = logging.getLogger(__name__) diff --git a/src/fidesops/schemas/base_class.py b/src/fidesops/schemas/base_class.py index 4f74147f4..680a978a9 100644 --- a/src/fidesops/schemas/base_class.py +++ b/src/fidesops/schemas/base_class.py @@ -1,4 +1,4 @@ -from typing import List, Any +from typing import Any, List from pydantic import BaseModel diff --git a/src/fidesops/schemas/connection_configuration/__init__.py b/src/fidesops/schemas/connection_configuration/__init__.py index a4582c3e2..ce4833cb9 100644 --- a/src/fidesops/schemas/connection_configuration/__init__.py +++ b/src/fidesops/schemas/connection_configuration/__init__.py @@ -1,48 +1,48 @@ -from typing import Dict, Any, Union +from typing import Any, Dict, Union -from fidesops.schemas.connection_configuration.connection_secrets_mariadb import ( - MariaDBSchema, - MariaDBDocsSchema, +from fidesops.models.connectionconfig import ConnectionType +from fidesops.schemas.connection_configuration.connection_secrets import ( + ConnectionConfigSecretsSchema, ) from fidesops.schemas.connection_configuration.connection_secrets_bigquery import ( - BigQuerySchema, BigQueryDocsSchema, + BigQuerySchema, +) +from fidesops.schemas.connection_configuration.connection_secrets_mariadb import ( + MariaDBDocsSchema, + MariaDBSchema, ) from fidesops.schemas.connection_configuration.connection_secrets_mongodb import ( - MongoDBSchema, MongoDBDocsSchema, -) -from fidesops.schemas.connection_configuration.connection_secrets import ( - ConnectionConfigSecretsSchema, + MongoDBSchema, ) from fidesops.schemas.connection_configuration.connection_secrets_mssql import ( MicrosoftSQLServerSchema, MSSQLDocsSchema, ) from fidesops.schemas.connection_configuration.connection_secrets_mysql import ( - MySQLSchema, MySQLDocsSchema, + MySQLSchema, ) from fidesops.schemas.connection_configuration.connection_secrets_postgres import ( - PostgreSQLSchema, PostgreSQLDocsSchema, + PostgreSQLSchema, ) -from fidesops.models.connectionconfig import ConnectionType from fidesops.schemas.connection_configuration.connection_secrets_redshift import ( - RedshiftSchema, RedshiftDocsSchema, + RedshiftSchema, +) +from fidesops.schemas.connection_configuration.connection_secrets_saas import ( + SaaSSchema, + SaaSSchemaFactory, ) from fidesops.schemas.connection_configuration.connection_secrets_snowflake import ( - SnowflakeSchema, SnowflakeDocsSchema, + SnowflakeSchema, ) from fidesops.schemas.connection_configuration.connections_secrets_https import ( HttpsSchema, ) -from fidesops.schemas.connection_configuration.connection_secrets_saas import ( - SaaSSchema, - SaaSSchemaFactory, -) from fidesops.schemas.saas.saas_config import SaaSConfig secrets_validators: Dict[str, Any] = { diff --git a/src/fidesops/schemas/connection_configuration/connection_config.py b/src/fidesops/schemas/connection_configuration/connection_config.py index f8df6905b..aa060a4e3 100644 --- a/src/fidesops/schemas/connection_configuration/connection_config.py +++ b/src/fidesops/schemas/connection_configuration/connection_config.py @@ -1,11 +1,11 @@ from datetime import datetime -from typing import Optional, List +from typing import List, Optional -from pydantic import Extra, BaseModel +from pydantic import BaseModel, Extra +from fidesops.models.connectionconfig import AccessLevel, ConnectionType from fidesops.schemas.api import BulkResponse, BulkUpdateFailed from fidesops.schemas.shared_schemas import FidesOpsKey -from fidesops.models.connectionconfig import ConnectionType, AccessLevel class CreateConnectionConfiguration(BaseModel): diff --git a/src/fidesops/schemas/connection_configuration/connection_secrets.py b/src/fidesops/schemas/connection_configuration/connection_secrets.py index 9686ee6d1..4275be6dc 100644 --- a/src/fidesops/schemas/connection_configuration/connection_secrets.py +++ b/src/fidesops/schemas/connection_configuration/connection_secrets.py @@ -1,7 +1,7 @@ from __future__ import annotations import abc -from typing import Optional, Dict, Any, List +from typing import Any, Dict, List, Optional from pydantic import BaseModel, Extra, root_validator diff --git a/src/fidesops/schemas/connection_configuration/connection_secrets_bigquery.py b/src/fidesops/schemas/connection_configuration/connection_secrets_bigquery.py index f09b18107..27af4e66e 100644 --- a/src/fidesops/schemas/connection_configuration/connection_secrets_bigquery.py +++ b/src/fidesops/schemas/connection_configuration/connection_secrets_bigquery.py @@ -1,4 +1,4 @@ -from typing import Optional, List +from typing import List, Optional from pydantic.main import BaseModel diff --git a/src/fidesops/schemas/connection_configuration/connection_secrets_mariadb.py b/src/fidesops/schemas/connection_configuration/connection_secrets_mariadb.py index 240f8c515..5e2996762 100644 --- a/src/fidesops/schemas/connection_configuration/connection_secrets_mariadb.py +++ b/src/fidesops/schemas/connection_configuration/connection_secrets_mariadb.py @@ -1,4 +1,4 @@ -from typing import Optional, List +from typing import List, Optional from fidesops.schemas.base_class import NoValidationSchema from fidesops.schemas.connection_configuration.connection_secrets import ( diff --git a/src/fidesops/schemas/connection_configuration/connection_secrets_mongodb.py b/src/fidesops/schemas/connection_configuration/connection_secrets_mongodb.py index 20f32a673..1c22d1ec2 100644 --- a/src/fidesops/schemas/connection_configuration/connection_secrets_mongodb.py +++ b/src/fidesops/schemas/connection_configuration/connection_secrets_mongodb.py @@ -1,4 +1,4 @@ -from typing import Optional, List +from typing import List, Optional from fidesops.schemas.base_class import NoValidationSchema from fidesops.schemas.connection_configuration.connection_secrets import ( diff --git a/src/fidesops/schemas/connection_configuration/connection_secrets_mssql.py b/src/fidesops/schemas/connection_configuration/connection_secrets_mssql.py index 5cfaa0db3..f7435fbbd 100644 --- a/src/fidesops/schemas/connection_configuration/connection_secrets_mssql.py +++ b/src/fidesops/schemas/connection_configuration/connection_secrets_mssql.py @@ -1,4 +1,4 @@ -from typing import Optional, List +from typing import List, Optional from fidesops.schemas.base_class import NoValidationSchema from fidesops.schemas.connection_configuration.connection_secrets import ( diff --git a/src/fidesops/schemas/connection_configuration/connection_secrets_mysql.py b/src/fidesops/schemas/connection_configuration/connection_secrets_mysql.py index 1b5d7dff9..2f49d7058 100644 --- a/src/fidesops/schemas/connection_configuration/connection_secrets_mysql.py +++ b/src/fidesops/schemas/connection_configuration/connection_secrets_mysql.py @@ -1,4 +1,4 @@ -from typing import Optional, List +from typing import List, Optional from fidesops.schemas.base_class import NoValidationSchema from fidesops.schemas.connection_configuration.connection_secrets import ( diff --git a/src/fidesops/schemas/connection_configuration/connection_secrets_postgres.py b/src/fidesops/schemas/connection_configuration/connection_secrets_postgres.py index 9f6e95d57..1733e4ab3 100644 --- a/src/fidesops/schemas/connection_configuration/connection_secrets_postgres.py +++ b/src/fidesops/schemas/connection_configuration/connection_secrets_postgres.py @@ -1,4 +1,4 @@ -from typing import Optional, List +from typing import List, Optional from fidesops.schemas.base_class import NoValidationSchema from fidesops.schemas.connection_configuration.connection_secrets import ( diff --git a/src/fidesops/schemas/connection_configuration/connection_secrets_redshift.py b/src/fidesops/schemas/connection_configuration/connection_secrets_redshift.py index 4d4d2aaf6..4b189760d 100644 --- a/src/fidesops/schemas/connection_configuration/connection_secrets_redshift.py +++ b/src/fidesops/schemas/connection_configuration/connection_secrets_redshift.py @@ -1,4 +1,4 @@ -from typing import Optional, List +from typing import List, Optional from fidesops.schemas.base_class import NoValidationSchema from fidesops.schemas.connection_configuration.connection_secrets import ( diff --git a/src/fidesops/schemas/connection_configuration/connection_secrets_saas.py b/src/fidesops/schemas/connection_configuration/connection_secrets_saas.py index d58aac37a..e9b899205 100644 --- a/src/fidesops/schemas/connection_configuration/connection_secrets_saas.py +++ b/src/fidesops/schemas/connection_configuration/connection_secrets_saas.py @@ -1,6 +1,8 @@ import abc from typing import Any, Dict, Type -from pydantic import BaseModel, Extra, root_validator, create_model + +from pydantic import BaseModel, Extra, create_model, root_validator + from fidesops.schemas.saas.saas_config import SaaSConfig diff --git a/src/fidesops/schemas/connection_configuration/connection_secrets_snowflake.py b/src/fidesops/schemas/connection_configuration/connection_secrets_snowflake.py index 6eda747b0..4858e5084 100644 --- a/src/fidesops/schemas/connection_configuration/connection_secrets_snowflake.py +++ b/src/fidesops/schemas/connection_configuration/connection_secrets_snowflake.py @@ -1,4 +1,4 @@ -from typing import Optional, List +from typing import List, Optional from fidesops.schemas.base_class import NoValidationSchema from fidesops.schemas.connection_configuration.connection_secrets import ( diff --git a/src/fidesops/schemas/dataset.py b/src/fidesops/schemas/dataset.py index 6965e8b9f..d6e381a78 100644 --- a/src/fidesops/schemas/dataset.py +++ b/src/fidesops/schemas/dataset.py @@ -1,14 +1,14 @@ -from typing import List, Optional, Dict, Any -from pydantic import BaseModel, validator, ConstrainedStr +from typing import Any, Dict, List, Optional from fideslang.models import Dataset, DatasetCollection, DatasetField -from fidesops.graph.config import EdgeDirection +from pydantic import BaseModel, ConstrainedStr, validator from fidesops.common_exceptions import ( InvalidDataLengthValidationError, + InvalidDataTypeValidationError, ) -from fidesops.common_exceptions import InvalidDataTypeValidationError -from fidesops.graph.data_type import parse_data_type_string, is_valid_data_type +from fidesops.graph.config import EdgeDirection +from fidesops.graph.data_type import is_valid_data_type, parse_data_type_string from fidesops.schemas.api import BulkResponse, BulkUpdateFailed from fidesops.schemas.base_class import BaseSchema from fidesops.schemas.shared_schemas import FidesOpsKey diff --git a/src/fidesops/schemas/drp_privacy_request.py b/src/fidesops/schemas/drp_privacy_request.py index 6d0e1015e..d856b7b59 100644 --- a/src/fidesops/schemas/drp_privacy_request.py +++ b/src/fidesops/schemas/drp_privacy_request.py @@ -1,5 +1,5 @@ from enum import Enum -from typing import Optional, List +from typing import List, Optional from pydantic import validator diff --git a/src/fidesops/schemas/external_https.py b/src/fidesops/schemas/external_https.py index 40242d383..ced7dcc36 100644 --- a/src/fidesops/schemas/external_https.py +++ b/src/fidesops/schemas/external_https.py @@ -1,5 +1,5 @@ from enum import Enum -from typing import Optional, List +from typing import List, Optional from pydantic import BaseModel diff --git a/src/fidesops/schemas/masking/masking_api.py b/src/fidesops/schemas/masking/masking_api.py index 55a1988b0..e5f8ab930 100644 --- a/src/fidesops/schemas/masking/masking_api.py +++ b/src/fidesops/schemas/masking/masking_api.py @@ -1,4 +1,4 @@ -from typing import List, Any +from typing import Any, List from pydantic import BaseModel diff --git a/src/fidesops/schemas/masking/masking_secrets.py b/src/fidesops/schemas/masking/masking_secrets.py index 7e5c24505..4de744941 100644 --- a/src/fidesops/schemas/masking/masking_secrets.py +++ b/src/fidesops/schemas/masking/masking_secrets.py @@ -1,7 +1,6 @@ from dataclasses import dataclass from enum import Enum -from typing import TypeVar, Generic, Callable - +from typing import Callable, Generic, TypeVar T = TypeVar("T") diff --git a/src/fidesops/schemas/policy.py b/src/fidesops/schemas/policy.py index 8ee82a9b1..0f82bba23 100644 --- a/src/fidesops/schemas/policy.py +++ b/src/fidesops/schemas/policy.py @@ -1,13 +1,10 @@ from typing import Dict, List, Optional, Union -from fidesops.schemas.shared_schemas import FidesOpsKey -from fidesops.models.policy import ( - ActionType, - DrpAction, -) +from fidesops.models.policy import ActionType, DrpAction from fidesops.schemas.api import BulkResponse, BulkUpdateFailed from fidesops.schemas.base_class import BaseSchema from fidesops.schemas.masking.masking_configuration import FormatPreservationConfig +from fidesops.schemas.shared_schemas import FidesOpsKey from fidesops.schemas.storage.storage import StorageDestinationResponse from fidesops.util.data_category import DataCategory diff --git a/src/fidesops/schemas/policy_webhooks.py b/src/fidesops/schemas/policy_webhooks.py index 30291436f..fe5f97ca7 100644 --- a/src/fidesops/schemas/policy_webhooks.py +++ b/src/fidesops/schemas/policy_webhooks.py @@ -1,15 +1,12 @@ from typing import List, Optional +from fidesops.models.policy import WebhookDirection +from fidesops.schemas.base_class import BaseSchema from fidesops.schemas.connection_configuration.connection_config import ( ConnectionConfigurationResponse, ) from fidesops.schemas.shared_schemas import FidesOpsKey -from fidesops.models.policy import ( - WebhookDirection, -) -from fidesops.schemas.base_class import BaseSchema - class WebhookBase(BaseSchema): """Base schema for Webhooks""" diff --git a/src/fidesops/schemas/privacy_request.py b/src/fidesops/schemas/privacy_request.py index 352f9f276..0652d976a 100644 --- a/src/fidesops/schemas/privacy_request.py +++ b/src/fidesops/schemas/privacy_request.py @@ -1,18 +1,18 @@ from datetime import datetime from enum import Enum as EnumType -from typing import List, Optional, Dict +from typing import Dict, List, Optional from pydantic import Field, validator from fidesops.core.config import config from fidesops.models.policy import ActionType +from fidesops.models.privacy_request import ExecutionLogStatus, PrivacyRequestStatus from fidesops.schemas.api import BulkResponse, BulkUpdateFailed +from fidesops.schemas.base_class import BaseSchema from fidesops.schemas.policy import Policy as PolicySchema from fidesops.schemas.redis_cache import PrivacyRequestIdentity -from fidesops.schemas.base_class import BaseSchema from fidesops.schemas.shared_schemas import FidesOpsKey from fidesops.schemas.user import PrivacyRequestReviewer -from fidesops.models.privacy_request import PrivacyRequestStatus, ExecutionLogStatus from fidesops.util.encryption.aes_gcm_encryption_scheme import verify_encryption_key diff --git a/src/fidesops/schemas/redis_cache.py b/src/fidesops/schemas/redis_cache.py index dc9a0e0e0..c67f0d51a 100644 --- a/src/fidesops/schemas/redis_cache.py +++ b/src/fidesops/schemas/redis_cache.py @@ -1,4 +1,5 @@ from typing import Optional + from pydantic import Extra from fidesops.schemas.base_class import BaseSchema diff --git a/src/fidesops/schemas/saas/shared_schemas.py b/src/fidesops/schemas/saas/shared_schemas.py index 1c89e15ed..6510106cb 100644 --- a/src/fidesops/schemas/saas/shared_schemas.py +++ b/src/fidesops/schemas/saas/shared_schemas.py @@ -1,5 +1,6 @@ from enum import Enum from typing import Any, Dict, Optional + from pydantic import BaseModel diff --git a/src/fidesops/schemas/user.py b/src/fidesops/schemas/user.py index e1db6aed2..bc8674b02 100644 --- a/src/fidesops/schemas/user.py +++ b/src/fidesops/schemas/user.py @@ -1,8 +1,9 @@ -from datetime import datetime import re +from datetime import datetime from typing import Optional from pydantic import validator + from fidesops.schemas.base_class import BaseSchema from fidesops.schemas.oauth import AccessToken diff --git a/src/fidesops/schemas/user_permission.py b/src/fidesops/schemas/user_permission.py index a51d38cb1..f6a70116a 100644 --- a/src/fidesops/schemas/user_permission.py +++ b/src/fidesops/schemas/user_permission.py @@ -1,9 +1,11 @@ from typing import List -from pydantic import validator + from fastapi import HTTPException +from pydantic import validator from starlette.status import HTTP_422_UNPROCESSABLE_ENTITY -from fidesops.schemas.base_class import BaseSchema + from fidesops.api.v1.scope_registry import SCOPE_REGISTRY +from fidesops.schemas.base_class import BaseSchema class UserPermissionsCreate(BaseSchema): diff --git a/src/fidesops/service/connectors/__init__.py b/src/fidesops/service/connectors/__init__.py index 28849ad08..205d1ff31 100644 --- a/src/fidesops/service/connectors/__init__.py +++ b/src/fidesops/service/connectors/__init__.py @@ -1,18 +1,18 @@ -from typing import Dict, Any +from typing import Any, Dict -from fidesops.service.connectors.http_connector import HTTPSConnector -from fidesops.service.connectors.saas_connector import SaaSConnector -from fidesops.service.connectors.mongodb_connector import MongoDBConnector from fidesops.models.connectionconfig import ConnectionConfig, ConnectionType from fidesops.service.connectors.base_connector import BaseConnector +from fidesops.service.connectors.http_connector import HTTPSConnector +from fidesops.service.connectors.mongodb_connector import MongoDBConnector +from fidesops.service.connectors.saas_connector import SaaSConnector from fidesops.service.connectors.sql_connector import ( - PostgreSQLConnector, + BigQueryConnector, + MariaDBConnector, + MicrosoftSQLServerConnector, MySQLConnector, + PostgreSQLConnector, RedshiftConnector, SnowflakeConnector, - MicrosoftSQLServerConnector, - MariaDBConnector, - BigQueryConnector, ) supported_connectors: Dict[str, Any] = { diff --git a/src/fidesops/service/connectors/base_connector.py b/src/fidesops/service/connectors/base_connector.py index d301ab0c7..53325d62d 100644 --- a/src/fidesops/service/connectors/base_connector.py +++ b/src/fidesops/service/connectors/base_connector.py @@ -1,6 +1,6 @@ import logging -from abc import abstractmethod, ABC -from typing import Any, Dict, List, Optional, TypeVar, Generic +from abc import ABC, abstractmethod +from typing import Any, Dict, Generic, List, Optional, TypeVar from fidesops.core.config import config from fidesops.graph.traversal import TraversalNode diff --git a/src/fidesops/service/connectors/mongodb_connector.py b/src/fidesops/service/connectors/mongodb_connector.py index 74f9c0c1a..742321a2a 100644 --- a/src/fidesops/service/connectors/mongodb_connector.py +++ b/src/fidesops/service/connectors/mongodb_connector.py @@ -1,8 +1,8 @@ import logging -from typing import Dict, Any, List, Optional +from typing import Any, Dict, List, Optional from pymongo import MongoClient -from pymongo.errors import ServerSelectionTimeoutError, OperationFailure +from pymongo.errors import OperationFailure, ServerSelectionTimeoutError from fidesops.common_exceptions import ConnectionException from fidesops.graph.traversal import Row, TraversalNode @@ -12,10 +12,8 @@ from fidesops.schemas.connection_configuration.connection_secrets_mongodb import ( MongoDBSchema, ) -from fidesops.service.connectors.base_connector import ( - BaseConnector, -) -from fidesops.service.connectors.query_config import QueryConfig, MongoQueryConfig +from fidesops.service.connectors.base_connector import BaseConnector +from fidesops.service.connectors.query_config import MongoQueryConfig, QueryConfig from fidesops.util.logger import NotPii logger = logging.getLogger(__name__) diff --git a/src/fidesops/service/connectors/query_config.py b/src/fidesops/service/connectors/query_config.py index 77ab1a7b0..f36387f55 100644 --- a/src/fidesops/service/connectors/query_config.py +++ b/src/fidesops/service/connectors/query_config.py @@ -1,28 +1,26 @@ import logging import re from abc import ABC, abstractmethod -from typing import Dict, Any, List, Optional, Generic, TypeVar, Tuple +from typing import Any, Dict, Generic, List, Optional, Tuple, TypeVar import pydash -from sqlalchemy import text, Table, MetaData +from sqlalchemy import MetaData, Table, text from sqlalchemy.engine import Engine from sqlalchemy.sql import Executable, Update -from sqlalchemy.sql.elements import TextClause, ColumnElement +from sqlalchemy.sql.elements import ColumnElement, TextClause from fidesops.graph.config import ( ROOT_COLLECTION_ADDRESS, CollectionAddress, Field, - MaskingOverride, FieldPath, + MaskingOverride, ) -from fidesops.graph.traversal import TraversalNode, Row -from fidesops.models.policy import Policy, ActionType, Rule +from fidesops.graph.traversal import Row, TraversalNode +from fidesops.models.policy import ActionType, Policy, Rule from fidesops.models.privacy_request import PrivacyRequest from fidesops.service.masking.strategy.masking_strategy import MaskingStrategy -from fidesops.service.masking.strategy.masking_strategy_factory import ( - get_strategy, -) +from fidesops.service.masking.strategy.masking_strategy_factory import get_strategy from fidesops.service.masking.strategy.masking_strategy_nullify import NULL_REWRITE from fidesops.task.refine_target_path import ( build_refined_target_paths, diff --git a/src/fidesops/service/connectors/sql_connector.py b/src/fidesops/service/connectors/sql_connector.py index 96dd65147..74923572b 100644 --- a/src/fidesops/service/connectors/sql_connector.py +++ b/src/fidesops/service/connectors/sql_connector.py @@ -2,19 +2,19 @@ from abc import abstractmethod from typing import Any, Dict, List, Optional +from snowflake.sqlalchemy import URL as Snowflake_URL from sqlalchemy import Column, text from sqlalchemy.engine import ( URL, - Engine, - create_engine, + Connection, CursorResult, + Engine, LegacyCursorResult, - Connection, + create_engine, ) -from sqlalchemy.exc import OperationalError, InternalError +from sqlalchemy.exc import InternalError, OperationalError from sqlalchemy.sql import Executable from sqlalchemy.sql.elements import TextClause -from snowflake.sqlalchemy import URL as Snowflake_URL from fidesops.common_exceptions import ConnectionException from fidesops.graph.traversal import Row, TraversalNode @@ -22,27 +22,27 @@ from fidesops.models.policy import Policy from fidesops.models.privacy_request import PrivacyRequest from fidesops.schemas.connection_configuration import ( + MicrosoftSQLServerSchema, PostgreSQLSchema, RedshiftSchema, SnowflakeSchema, - MicrosoftSQLServerSchema, -) -from fidesops.schemas.connection_configuration.connection_secrets_mariadb import ( - MariaDBSchema, ) from fidesops.schemas.connection_configuration.connection_secrets_bigquery import ( BigQuerySchema, ) +from fidesops.schemas.connection_configuration.connection_secrets_mariadb import ( + MariaDBSchema, +) from fidesops.schemas.connection_configuration.connection_secrets_mysql import ( MySQLSchema, ) from fidesops.service.connectors.base_connector import BaseConnector from fidesops.service.connectors.query_config import ( + BigQueryQueryConfig, + MicrosoftSQLServerQueryConfig, + RedshiftQueryConfig, SnowflakeQueryConfig, SQLQueryConfig, - RedshiftQueryConfig, - MicrosoftSQLServerQueryConfig, - BigQueryQueryConfig, ) logger = logging.getLogger(__name__) diff --git a/src/fidesops/service/masking/strategy/masking_strategy.py b/src/fidesops/service/masking/strategy/masking_strategy.py index d8b082b5c..b37de258c 100644 --- a/src/fidesops/service/masking/strategy/masking_strategy.py +++ b/src/fidesops/service/masking/strategy/masking_strategy.py @@ -1,6 +1,6 @@ # MR Note - It would be nice to enforce this at compile time -from abc import abstractmethod, ABC -from typing import Optional, List, Any +from abc import ABC, abstractmethod +from typing import Any, List, Optional from fidesops.schemas.masking.masking_configuration import MaskingConfiguration from fidesops.schemas.masking.masking_secrets import MaskingSecretCache diff --git a/src/fidesops/service/privacy_request/onetrust_service.py b/src/fidesops/service/privacy_request/onetrust_service.py index 1a2e4ea64..f17b95af2 100644 --- a/src/fidesops/service/privacy_request/onetrust_service.py +++ b/src/fidesops/service/privacy_request/onetrust_service.py @@ -1,32 +1,31 @@ import logging import urllib.parse -from datetime import timedelta, datetime -from typing import Optional, Final, List, Dict, Union +from datetime import datetime, timedelta +from typing import Dict, Final, List, Optional, Union import requests from requests import Response from sqlalchemy.orm import Session from fidesops.common_exceptions import ( + AuthenticationException, PolicyNotFoundException, StorageConfigNotFoundException, - AuthenticationException, ) from fidesops.db.session import get_db_session from fidesops.models.policy import Policy -from fidesops.models.privacy_request import ( - PrivacyRequest, -) +from fidesops.models.privacy_request import PrivacyRequest from fidesops.models.storage import StorageConfig from fidesops.schemas.privacy_request import PrivacyRequestIdentity +from fidesops.schemas.shared_schemas import FidesOpsKey +from fidesops.schemas.storage.storage import StorageDetails, StorageSecrets from fidesops.schemas.third_party.onetrust import ( - OneTrustGetSubtasksResponse, - OneTrustSubtask, OneTrustGetRequestsResponse, + OneTrustGetSubtasksResponse, OneTrustRequest, + OneTrustSubtask, + OneTrustSubtaskStatus, ) -from fidesops.schemas.shared_schemas import FidesOpsKey -from fidesops.schemas.storage.storage import StorageDetails, StorageSecrets from fidesops.service.outbound_urn_registry import ( ONETRUST_GET_ALL_REQUESTS, ONETRUST_GET_SUBTASKS_BY_REF_ID, @@ -35,7 +34,6 @@ from fidesops.service.privacy_request.request_runner_service import PrivacyRequestRunner from fidesops.util.cache import get_cache from fidesops.util.storage_authenticator import get_onetrust_access_token -from fidesops.schemas.third_party.onetrust import OneTrustSubtaskStatus logger = logging.getLogger(__name__) diff --git a/src/fidesops/service/privacy_request/request_runner_service.py b/src/fidesops/service/privacy_request/request_runner_service.py index ec3366b55..170d22dee 100644 --- a/src/fidesops/service/privacy_request/request_runner_service.py +++ b/src/fidesops/service/privacy_request/request_runner_service.py @@ -1,30 +1,30 @@ import logging from datetime import datetime, timedelta -from typing import Set, Optional, Awaitable +from typing import Awaitable, Optional, Set from pydantic import ValidationError from sqlalchemy.orm import Session from fidesops import common_exceptions +from fidesops.common_exceptions import ClientUnsuccessfulException, PrivacyRequestPaused from fidesops.core.config import config from fidesops.db.session import get_db_session -from fidesops.common_exceptions import PrivacyRequestPaused, ClientUnsuccessfulException from fidesops.graph.graph import DatasetGraph from fidesops.models.connectionconfig import ConnectionConfig from fidesops.models.datasetconfig import DatasetConfig from fidesops.models.policy import ( ActionType, - WebhookTypes, - PolicyPreWebhook, PolicyPostWebhook, + PolicyPreWebhook, + WebhookTypes, ) from fidesops.models.privacy_request import PrivacyRequest, PrivacyRequestStatus from fidesops.service.storage.storage_uploader_service import upload from fidesops.task.filter_results import filter_data_categories from fidesops.task.graph_task import ( + get_cached_data_for_erasures, run_access_request, run_erasure, - get_cached_data_for_erasures, ) from fidesops.tasks.scheduled.scheduler import scheduler from fidesops.util.async_util import run_async diff --git a/src/fidesops/service/privacy_request/request_service.py b/src/fidesops/service/privacy_request/request_service.py index e8d9743b0..a610a79f3 100644 --- a/src/fidesops/service/privacy_request/request_service.py +++ b/src/fidesops/service/privacy_request/request_service.py @@ -1,8 +1,8 @@ import logging from datetime import datetime -from typing import Optional, Any, Dict, Set, List +from typing import Any, Dict, List, Optional, Set -from fidesops.models.policy import Policy, ActionType +from fidesops.models.policy import ActionType, Policy from fidesops.models.privacy_request import PrivacyRequest from fidesops.schemas.drp_privacy_request import DrpPrivacyRequestCreate from fidesops.schemas.masking.masking_configuration import MaskingConfiguration diff --git a/src/fidesops/service/processors/post_processor_strategy/post_processor_strategy_unwrap.py b/src/fidesops/service/processors/post_processor_strategy/post_processor_strategy_unwrap.py index 18ee8cfef..03a5db43f 100644 --- a/src/fidesops/service/processors/post_processor_strategy/post_processor_strategy_unwrap.py +++ b/src/fidesops/service/processors/post_processor_strategy/post_processor_strategy_unwrap.py @@ -2,6 +2,7 @@ from typing import Any, Dict, List, Union import pydash + from fidesops.schemas.saas.strategy_configuration import ( StrategyConfiguration, UnwrapPostProcessorConfiguration, diff --git a/src/fidesops/task/consolidate_query_matches.py b/src/fidesops/task/consolidate_query_matches.py index d01b5fdb7..2539e07c4 100644 --- a/src/fidesops/task/consolidate_query_matches.py +++ b/src/fidesops/task/consolidate_query_matches.py @@ -1,4 +1,4 @@ -from typing import Optional, List, Any +from typing import Any, List, Optional from fidesops.graph.config import FieldPath from fidesops.util.collection_util import Row diff --git a/src/fidesops/task/filter_element_match.py b/src/fidesops/task/filter_element_match.py index a38b621f8..f504a82c6 100644 --- a/src/fidesops/task/filter_element_match.py +++ b/src/fidesops/task/filter_element_match.py @@ -1,16 +1,15 @@ import copy import logging from collections import defaultdict - -from typing import List, Any, Dict +from typing import Any, Dict, List import pydash from fidesops.task.refine_target_path import ( - build_refined_target_paths, DetailedPath, - join_detailed_path, FieldPathNodeInput, + build_refined_target_paths, + join_detailed_path, ) from fidesops.util.collection_util import FIDESOPS_DO_NOT_MASK_INDEX, Row diff --git a/src/fidesops/task/filter_results.py b/src/fidesops/task/filter_results.py index c242b94ac..6ca1d5c1f 100644 --- a/src/fidesops/task/filter_results.py +++ b/src/fidesops/task/filter_results.py @@ -1,10 +1,9 @@ -from collections import defaultdict - import itertools import logging -from typing import List, Dict, Any, Union, Set, Optional +from collections import defaultdict +from typing import Any, Dict, List, Optional, Set, Union -from fidesops.graph.config import FieldPath, CollectionAddress +from fidesops.graph.config import CollectionAddress, FieldPath from fidesops.schemas.shared_schemas import FidesOpsKey from fidesops.util.collection_util import Row diff --git a/src/fidesops/task/graph_task.py b/src/fidesops/task/graph_task.py index f32df674e..8b9cbabcc 100644 --- a/src/fidesops/task/graph_task.py +++ b/src/fidesops/task/graph_task.py @@ -1,37 +1,35 @@ import copy - import logging import traceback from abc import ABC from functools import wraps - from time import sleep -from typing import List, Dict, Any, Tuple, Callable, Optional, Set +from typing import Any, Callable, Dict, List, Optional, Set, Tuple import dask from dask.threaded import get from fidesops.core.config import config from fidesops.graph.config import ( - CollectionAddress, ROOT_COLLECTION_ADDRESS, TERMINATOR_ADDRESS, - FieldPath, + CollectionAddress, Field, FieldAddress, + FieldPath, ) -from fidesops.graph.graph import Edge, DatasetGraph, Node -from fidesops.graph.traversal import TraversalNode, Traversal -from fidesops.models.connectionconfig import ConnectionConfig, AccessLevel +from fidesops.graph.graph import DatasetGraph, Edge, Node +from fidesops.graph.traversal import Traversal, TraversalNode +from fidesops.models.connectionconfig import AccessLevel, ConnectionConfig from fidesops.models.policy import ActionType, Policy -from fidesops.models.privacy_request import PrivacyRequest, ExecutionLogStatus +from fidesops.models.privacy_request import ExecutionLogStatus, PrivacyRequest from fidesops.service.connectors import BaseConnector from fidesops.task.consolidate_query_matches import consolidate_query_matches from fidesops.task.filter_element_match import filter_element_match from fidesops.task.refine_target_path import FieldPathNodeInput from fidesops.task.task_resources import TaskResources from fidesops.util.cache import get_cache -from fidesops.util.collection_util import partition, append, NodeInput, Row +from fidesops.util.collection_util import NodeInput, Row, append, partition from fidesops.util.logger import NotPii from fidesops.util.saas_util import FIDESOPS_GROUPED_INPUTS diff --git a/src/fidesops/task/refine_target_path.py b/src/fidesops/task/refine_target_path.py index 6e970bced..1029134c4 100644 --- a/src/fidesops/task/refine_target_path.py +++ b/src/fidesops/task/refine_target_path.py @@ -1,5 +1,5 @@ import logging -from typing import Dict, Any, List, Union, Optional +from typing import Any, Dict, List, Optional, Union from fidesops.graph.config import FieldPath from fidesops.util.collection_util import FIDESOPS_DO_NOT_MASK_INDEX, Row diff --git a/src/fidesops/task/task_resources.py b/src/fidesops/task/task_resources.py index 13ad2f6ae..5e0bcee6e 100644 --- a/src/fidesops/task/task_resources.py +++ b/src/fidesops/task/task_resources.py @@ -1,28 +1,28 @@ import logging -from typing import Dict, Any, Optional, List - -from fidesops.schemas.shared_schemas import FidesOpsKey +from typing import Any, Dict, List, Optional from fidesops.common_exceptions import ConnectorNotFoundException from fidesops.db.session import get_db_session -from fidesops.graph.config import ( - CollectionAddress, -) +from fidesops.graph.config import CollectionAddress from fidesops.models.connectionconfig import ConnectionConfig, ConnectionType from fidesops.models.policy import ActionType, Policy -from fidesops.models.privacy_request import ExecutionLog, ExecutionLogStatus -from fidesops.models.privacy_request import PrivacyRequest +from fidesops.models.privacy_request import ( + ExecutionLog, + ExecutionLogStatus, + PrivacyRequest, +) +from fidesops.schemas.shared_schemas import FidesOpsKey from fidesops.service.connectors import ( BaseConnector, + BigQueryConnector, + MariaDBConnector, + MicrosoftSQLServerConnector, MongoDBConnector, MySQLConnector, PostgreSQLConnector, - SnowflakeConnector, RedshiftConnector, - MicrosoftSQLServerConnector, - MariaDBConnector, - BigQueryConnector, SaaSConnector, + SnowflakeConnector, ) from fidesops.util.cache import get_cache diff --git a/src/fidesops/tasks/scheduled/scheduler.py b/src/fidesops/tasks/scheduled/scheduler.py index af5195a26..a01f0da68 100644 --- a/src/fidesops/tasks/scheduled/scheduler.py +++ b/src/fidesops/tasks/scheduled/scheduler.py @@ -1,4 +1,3 @@ from apscheduler.schedulers.background import BackgroundScheduler - scheduler = BackgroundScheduler() diff --git a/src/fidesops/tasks/scheduled/tasks.py b/src/fidesops/tasks/scheduled/tasks.py index 05cf11082..5ae7259ea 100644 --- a/src/fidesops/tasks/scheduled/tasks.py +++ b/src/fidesops/tasks/scheduled/tasks.py @@ -2,11 +2,11 @@ from typing import Optional from apscheduler.jobstores.base import JobLookupError -from fidesops.schemas.shared_schemas import FidesOpsKey from fidesops.db.session import get_db_session from fidesops.models.storage import StorageConfig -from fidesops.schemas.storage.storage import StorageType, StorageDetails +from fidesops.schemas.shared_schemas import FidesOpsKey +from fidesops.schemas.storage.storage import StorageDetails, StorageType from fidesops.service.privacy_request.onetrust_service import OneTrustService from fidesops.tasks.scheduled.scheduler import scheduler diff --git a/src/fidesops/tasks/storage.py b/src/fidesops/tasks/storage.py index e700c7d92..42898cf77 100644 --- a/src/fidesops/tasks/storage.py +++ b/src/fidesops/tasks/storage.py @@ -1,33 +1,27 @@ -import secrets - -from datetime import datetime - -import os - -import logging -from typing import Any, Dict, Union - import json +import logging +import os +import secrets import zipfile +from datetime import datetime from io import BytesIO -import requests +from typing import Any, Dict, Union import pandas as pd - +import requests from botocore.exceptions import ClientError, ParamValidationError from fidesops.core.config import config from fidesops.models.storage import ResponseFormat from fidesops.schemas.storage.storage import StorageSecrets -from fidesops.util.cache import get_encryption_cache_key, get_cache +from fidesops.util.cache import get_cache, get_encryption_cache_key from fidesops.util.cryptographic_util import bytes_to_b64_str from fidesops.util.encryption.aes_gcm_encryption_scheme import ( encrypt_to_bytes_verify_secrets_length, ) - from fidesops.util.storage_authenticator import ( - get_s3_session, get_onetrust_access_token, + get_s3_session, ) logger = logging.getLogger(__name__) diff --git a/src/fidesops/util/async_util.py b/src/fidesops/util/async_util.py index 7f5f18e9c..7d52793a6 100644 --- a/src/fidesops/util/async_util.py +++ b/src/fidesops/util/async_util.py @@ -1,9 +1,8 @@ import asyncio +import logging from asyncio import AbstractEventLoop from concurrent.futures import ThreadPoolExecutor -from typing import TypeVar, Callable, Any, Awaitable, Optional -import logging - +from typing import Any, Awaitable, Callable, Optional, TypeVar logger = logging.getLogger(__name__) T = TypeVar("T") diff --git a/src/fidesops/util/cache.py b/src/fidesops/util/cache.py index 53c607e2e..ce88594a0 100644 --- a/src/fidesops/util/cache.py +++ b/src/fidesops/util/cache.py @@ -1,14 +1,7 @@ import base64 import logging import pickle -from typing import ( - Any, - List, - Optional, - Set, - Union, - Dict, -) +from typing import Any, Dict, List, Optional, Set, Union from redis import Redis from redis.client import Script diff --git a/src/fidesops/util/collection_util.py b/src/fidesops/util/collection_util.py index 301893402..fff5839e6 100644 --- a/src/fidesops/util/collection_util.py +++ b/src/fidesops/util/collection_util.py @@ -1,5 +1,5 @@ from functools import reduce -from typing import List, Dict, TypeVar, Iterable, Callable, Any, Optional +from typing import Any, Callable, Dict, Iterable, List, Optional, TypeVar T = TypeVar("T") U = TypeVar("U") diff --git a/src/fidesops/util/cryptographic_util.py b/src/fidesops/util/cryptographic_util.py index efc11b0e7..3cdab96b1 100644 --- a/src/fidesops/util/cryptographic_util.py +++ b/src/fidesops/util/cryptographic_util.py @@ -1,8 +1,7 @@ -from base64 import b64encode, b64decode - - import hashlib import secrets +from base64 import b64decode, b64encode + import bcrypt from fidesops.core.config import config diff --git a/src/fidesops/util/data_category.py b/src/fidesops/util/data_category.py index 622e1f132..4eebe7bc3 100644 --- a/src/fidesops/util/data_category.py +++ b/src/fidesops/util/data_category.py @@ -1,6 +1,8 @@ from enum import Enum as EnumType from typing import Type + from fideslang import DEFAULT_TAXONOMY + from fidesops import common_exceptions diff --git a/src/fidesops/util/logger.py b/src/fidesops/util/logger.py index bb320f876..03689d526 100644 --- a/src/fidesops/util/logger.py +++ b/src/fidesops/util/logger.py @@ -1,7 +1,7 @@ import logging import os -from typing import Any, Mapping, Union from numbers import Number +from typing import Any, Mapping, Union MASKED = "MASKED" diff --git a/src/fidesops/util/matching_queue.py b/src/fidesops/util/matching_queue.py index e5b598172..ac870c2e8 100644 --- a/src/fidesops/util/matching_queue.py +++ b/src/fidesops/util/matching_queue.py @@ -1,5 +1,6 @@ from __future__ import annotations -from typing import Optional, Callable, TypeVar, Generic + +from typing import Callable, Generic, Optional, TypeVar T = TypeVar("T") diff --git a/src/fidesops/util/oauth_util.py b/src/fidesops/util/oauth_util.py index d55ad84b5..fa20cd7fd 100644 --- a/src/fidesops/util/oauth_util.py +++ b/src/fidesops/util/oauth_util.py @@ -1,29 +1,29 @@ import json from datetime import datetime from typing import Optional -from starlette.status import HTTP_404_NOT_FOUND -from pydantic import ValidationError -from fastapi import Depends, Security, HTTPException +from fastapi import Depends, HTTPException, Security from fastapi.security import SecurityScopes from jose import jwe from jose.constants import ALGORITHMS +from pydantic import ValidationError from sqlalchemy.orm import Session +from starlette.status import HTTP_404_NOT_FOUND from fidesops.api import deps -from fidesops.api.v1.urn_registry import V1_URL_PREFIX, TOKEN -from fidesops.common_exceptions import AuthorizationError, AuthenticationFailure +from fidesops.api.v1.urn_registry import TOKEN, V1_URL_PREFIX +from fidesops.common_exceptions import AuthenticationFailure, AuthorizationError from fidesops.core.config import config from fidesops.models.client import ClientDetail from fidesops.models.fidesops_user import FidesopsUser from fidesops.models.policy import PolicyPreWebhook from fidesops.schemas.external_https import WebhookJWE -from fidesops.schemas.oauth import OAuth2ClientCredentialsBearer from fidesops.schemas.jwt import ( + JWE_ISSUED_AT, JWE_PAYLOAD_CLIENT_ID, JWE_PAYLOAD_SCOPES, - JWE_ISSUED_AT, ) +from fidesops.schemas.oauth import OAuth2ClientCredentialsBearer __JWT_ENCRYPTION_ALGORITHM = ALGORITHMS.A256GCM diff --git a/src/fidesops/util/url_util.py b/src/fidesops/util/url_util.py index bccffb99d..74db48948 100644 --- a/src/fidesops/util/url_util.py +++ b/src/fidesops/util/url_util.py @@ -1,4 +1,4 @@ -from urllib.parse import urlsplit, parse_qs, urlencode, urlunsplit +from urllib.parse import parse_qs, urlencode, urlsplit, urlunsplit def set_query_parameter(url: str, param_name: str, param_value: str) -> str: diff --git a/src/migrations/env.py b/src/migrations/env.py index 60c2200dd..7fcced32d 100644 --- a/src/migrations/env.py +++ b/src/migrations/env.py @@ -10,10 +10,10 @@ sys.path = ["", ".."] + sys.path[1:] -from alembic import context +from logging.config import fileConfig +from alembic import context from sqlalchemy import engine_from_config, pool -from logging.config import fileConfig # this is the Alembic Config object, which provides # access to the values within the .ini file in use. diff --git a/src/migrations/versions/0210948a8147_initial.py b/src/migrations/versions/0210948a8147_initial.py index 5165fbac3..fbbb6f147 100644 --- a/src/migrations/versions/0210948a8147_initial.py +++ b/src/migrations/versions/0210948a8147_initial.py @@ -5,10 +5,10 @@ Create Date: 2021-10-29 11:51:19.265141 """ -from alembic import op import sqlalchemy as sa -from sqlalchemy.dialects import postgresql import sqlalchemy_utils +from alembic import op +from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "0210948a8147" diff --git a/src/migrations/versions/07014ff34eb2_add_mariadb.py b/src/migrations/versions/07014ff34eb2_add_mariadb.py index 0d51430d7..9ce3ba78f 100644 --- a/src/migrations/versions/07014ff34eb2_add_mariadb.py +++ b/src/migrations/versions/07014ff34eb2_add_mariadb.py @@ -5,9 +5,8 @@ Create Date: 2022-01-27 19:18:11.899734 """ -from alembic import op import sqlalchemy as sa - +from alembic import op # revision identifiers, used by Alembic. revision = "07014ff34eb2" diff --git a/src/migrations/versions/1dfc5a2d30e7_add_saas_config_to_connection_config.py b/src/migrations/versions/1dfc5a2d30e7_add_saas_config_to_connection_config.py index f0b80e86c..d7e499c74 100644 --- a/src/migrations/versions/1dfc5a2d30e7_add_saas_config_to_connection_config.py +++ b/src/migrations/versions/1dfc5a2d30e7_add_saas_config_to_connection_config.py @@ -5,11 +5,10 @@ Create Date: 2022-02-09 23:27:24.742938 """ -from alembic import op import sqlalchemy as sa +from alembic import op from sqlalchemy.dialects import postgresql - # revision identifiers, used by Alembic. revision = "1dfc5a2d30e7" down_revision = "e55a51b354e3" diff --git a/src/migrations/versions/29a7d707163a_adds_first_name_and_last_name_to_user_.py b/src/migrations/versions/29a7d707163a_adds_first_name_and_last_name_to_user_.py index d95205746..bd9b3f8fe 100644 --- a/src/migrations/versions/29a7d707163a_adds_first_name_and_last_name_to_user_.py +++ b/src/migrations/versions/29a7d707163a_adds_first_name_and_last_name_to_user_.py @@ -5,9 +5,8 @@ Create Date: 2022-05-05 13:41:28.807920 """ -from alembic import op import sqlalchemy as sa - +from alembic import op # revision identifiers, used by Alembic. revision = "29a7d707163a" diff --git a/src/migrations/versions/5078badb90b9_adds_drp_action_to_policy.py b/src/migrations/versions/5078badb90b9_adds_drp_action_to_policy.py index 8b5d282ea..fb7f6c876 100644 --- a/src/migrations/versions/5078badb90b9_adds_drp_action_to_policy.py +++ b/src/migrations/versions/5078badb90b9_adds_drp_action_to_policy.py @@ -5,9 +5,8 @@ Create Date: 2022-05-04 17:22:46.500067 """ -from alembic import op import sqlalchemy as sa - +from alembic import op # revision identifiers, used by Alembic. from sqlalchemy.dialects import postgresql diff --git a/src/migrations/versions/530fb8533ca4_test.py b/src/migrations/versions/530fb8533ca4_test.py index 211115695..07751dbef 100644 --- a/src/migrations/versions/530fb8533ca4_test.py +++ b/src/migrations/versions/530fb8533ca4_test.py @@ -5,8 +5,8 @@ Create Date: 2022-04-27 15:47:15.015128 """ -from alembic import op import sqlalchemy as sa +from alembic import op from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. diff --git a/src/migrations/versions/5a966cd643d7_fidesops_user_and_client_detail.py b/src/migrations/versions/5a966cd643d7_fidesops_user_and_client_detail.py index 1ba3e87f4..fffcf7fde 100644 --- a/src/migrations/versions/5a966cd643d7_fidesops_user_and_client_detail.py +++ b/src/migrations/versions/5a966cd643d7_fidesops_user_and_client_detail.py @@ -5,9 +5,8 @@ Create Date: 2022-03-08 22:05:18.352032 """ -from alembic import op import sqlalchemy as sa - +from alembic import op # revision identifiers, used by Alembic. revision = "5a966cd643d7" diff --git a/src/migrations/versions/90070db16d05_add_fidesops_user_permissions.py b/src/migrations/versions/90070db16d05_add_fidesops_user_permissions.py index 6956ea915..bb67738f2 100644 --- a/src/migrations/versions/90070db16d05_add_fidesops_user_permissions.py +++ b/src/migrations/versions/90070db16d05_add_fidesops_user_permissions.py @@ -5,9 +5,8 @@ Create Date: 2022-04-27 17:24:31.548916 """ -from alembic import op import sqlalchemy as sa - +from alembic import op # revision identifiers, used by Alembic. revision = "90070db16d05" diff --git a/src/migrations/versions/906d7198df28_privacy_request_approve.py b/src/migrations/versions/906d7198df28_privacy_request_approve.py index 809e25020..81bf950ee 100644 --- a/src/migrations/versions/906d7198df28_privacy_request_approve.py +++ b/src/migrations/versions/906d7198df28_privacy_request_approve.py @@ -5,8 +5,8 @@ Create Date: 2022-03-11 19:49:26.450054 """ -from alembic import op import sqlalchemy as sa +from alembic import op from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. diff --git a/src/migrations/versions/c5336b841d70_add_policy_webhooks.py b/src/migrations/versions/c5336b841d70_add_policy_webhooks.py index 91d3e4463..2f5088a86 100644 --- a/src/migrations/versions/c5336b841d70_add_policy_webhooks.py +++ b/src/migrations/versions/c5336b841d70_add_policy_webhooks.py @@ -5,9 +5,8 @@ Create Date: 2021-11-24 23:20:48.807643 """ -from alembic import op import sqlalchemy as sa - +from alembic import op # revision identifiers, used by Alembic. revision = "c5336b841d70" diff --git a/src/migrations/versions/c9759675b5d3_add_user_login_and_password_reset_dates.py b/src/migrations/versions/c9759675b5d3_add_user_login_and_password_reset_dates.py index 682e748eb..27692d6ba 100644 --- a/src/migrations/versions/c9759675b5d3_add_user_login_and_password_reset_dates.py +++ b/src/migrations/versions/c9759675b5d3_add_user_login_and_password_reset_dates.py @@ -5,9 +5,8 @@ Create Date: 2022-04-14 13:16:58.940571 """ -from alembic import op import sqlalchemy as sa - +from alembic import op # revision identifiers, used by Alembic. revision = "c9759675b5d3" diff --git a/src/migrations/versions/c98da12d76f8_add_audit_log.py b/src/migrations/versions/c98da12d76f8_add_audit_log.py index 982085e90..8ef0b3bc5 100644 --- a/src/migrations/versions/c98da12d76f8_add_audit_log.py +++ b/src/migrations/versions/c98da12d76f8_add_audit_log.py @@ -5,9 +5,8 @@ Create Date: 2022-05-04 20:16:39.246537 """ -from alembic import op import sqlalchemy as sa - +from alembic import op # revision identifiers, used by Alembic. revision = "c98da12d76f8" diff --git a/src/migrations/versions/d65e7e921814_add_privacy_request_status.py b/src/migrations/versions/d65e7e921814_add_privacy_request_status.py index e8914b9a2..848cf742d 100644 --- a/src/migrations/versions/d65e7e921814_add_privacy_request_status.py +++ b/src/migrations/versions/d65e7e921814_add_privacy_request_status.py @@ -5,9 +5,8 @@ Create Date: 2021-12-02 22:06:16.270442 """ -from alembic import op import sqlalchemy as sa - +from alembic import op # revision identifiers, used by Alembic. revision = "d65e7e921814" diff --git a/src/migrations/versions/e55a51b354e3_add_bigquery.py b/src/migrations/versions/e55a51b354e3_add_bigquery.py index c49a65ae0..9d28f0cf0 100644 --- a/src/migrations/versions/e55a51b354e3_add_bigquery.py +++ b/src/migrations/versions/e55a51b354e3_add_bigquery.py @@ -5,9 +5,8 @@ Create Date: 2022-02-01 18:02:05.023599 """ -from alembic import op import sqlalchemy as sa - +from alembic import op # revision identifiers, used by Alembic. revision = "e55a51b354e3" diff --git a/src/migrations/versions/f206d4e7574d_add_redshift_and_snowflake_support.py b/src/migrations/versions/f206d4e7574d_add_redshift_and_snowflake_support.py index fd771ca2d..eefcd7498 100644 --- a/src/migrations/versions/f206d4e7574d_add_redshift_and_snowflake_support.py +++ b/src/migrations/versions/f206d4e7574d_add_redshift_and_snowflake_support.py @@ -7,7 +7,6 @@ """ from alembic import op - # revision identifiers, used by Alembic. revision = "f206d4e7574d" down_revision = "0210948a8147" diff --git a/src/migrations/versions/f3841942d90c_add_mssql.py b/src/migrations/versions/f3841942d90c_add_mssql.py index 394ce195a..327f612a7 100644 --- a/src/migrations/versions/f3841942d90c_add_mssql.py +++ b/src/migrations/versions/f3841942d90c_add_mssql.py @@ -5,9 +5,8 @@ Create Date: 2021-12-13 22:15:25.043952 """ -from alembic import op import sqlalchemy as sa - +from alembic import op # revision identifiers, used by Alembic. revision = "f3841942d90c" diff --git a/tests/api/v1/endpoints/test_config_endpoints.py b/tests/api/v1/endpoints/test_config_endpoints.py index 8a61ed318..f4b0ec24a 100644 --- a/tests/api/v1/endpoints/test_config_endpoints.py +++ b/tests/api/v1/endpoints/test_config_endpoints.py @@ -1,10 +1,8 @@ import pytest from starlette.testclient import TestClient -from fidesops.api.v1 import ( - scope_registry as scopes, - urn_registry as urls, -) +from fidesops.api.v1 import scope_registry as scopes +from fidesops.api.v1 import urn_registry as urls class TestGetConnections: diff --git a/tests/api/v1/endpoints/test_connection_config_endpoints.py b/tests/api/v1/endpoints/test_connection_config_endpoints.py index 3462a3835..7cf44f090 100644 --- a/tests/api/v1/endpoints/test_connection_config_endpoints.py +++ b/tests/api/v1/endpoints/test_connection_config_endpoints.py @@ -6,20 +6,18 @@ import pytest from fastapi import HTTPException from fastapi_pagination import Params +from sqlalchemy.orm import Session from starlette.testclient import TestClient -from fidesops.models.client import ClientDetail -from fidesops.models.connectionconfig import ConnectionConfig from fidesops.api.v1.scope_registry import ( CONNECTION_CREATE_OR_UPDATE, - STORAGE_DELETE, - CONNECTION_READ, CONNECTION_DELETE, + CONNECTION_READ, + STORAGE_DELETE, ) - -from sqlalchemy.orm import Session - from fidesops.api.v1.urn_registry import CONNECTIONS, SAAS_CONFIG, V1_URL_PREFIX +from fidesops.models.client import ClientDetail +from fidesops.models.connectionconfig import ConnectionConfig page_size = Params().size diff --git a/tests/api/v1/endpoints/test_dataset_endpoints.py b/tests/api/v1/endpoints/test_dataset_endpoints.py index f542bc1cc..55f371dcd 100644 --- a/tests/api/v1/endpoints/test_dataset_endpoints.py +++ b/tests/api/v1/endpoints/test_dataset_endpoints.py @@ -4,27 +4,27 @@ from unittest.mock import Mock import pydash +import pytest from fastapi import HTTPException from fastapi_pagination import Params from pydash import filter_ -from starlette.testclient import TestClient from sqlalchemy.orm import Session -import pytest +from starlette.testclient import TestClient -from fidesops.models.datasetconfig import DatasetConfig -from fidesops.models.connectionconfig import ConnectionConfig from fidesops.api.v1.scope_registry import ( DATASET_CREATE_OR_UPDATE, - DATASET_READ, DATASET_DELETE, + DATASET_READ, ) from fidesops.api.v1.urn_registry import ( + DATASET_BY_KEY, DATASET_VALIDATE, DATASETS, - YAML_DATASETS, - DATASET_BY_KEY, V1_URL_PREFIX, + YAML_DATASETS, ) +from fidesops.models.connectionconfig import ConnectionConfig +from fidesops.models.datasetconfig import DatasetConfig def _reject_key(dict: Dict, key: str) -> Dict: diff --git a/tests/api/v1/endpoints/test_drp_endpoints.py b/tests/api/v1/endpoints/test_drp_endpoints.py index f1da6d01c..a89275889 100644 --- a/tests/api/v1/endpoints/test_drp_endpoints.py +++ b/tests/api/v1/endpoints/test_drp_endpoints.py @@ -10,17 +10,9 @@ PRIVACY_REQUEST_READ, STORAGE_CREATE_OR_UPDATE, ) -from fidesops.api.v1.urn_registry import ( - V1_URL_PREFIX, - DRP_EXERCISE, - DRP_STATUS, -) +from fidesops.api.v1.urn_registry import DRP_EXERCISE, DRP_STATUS, V1_URL_PREFIX from fidesops.core.config import config - -from fidesops.models.privacy_request import ( - PrivacyRequest, - PrivacyRequestStatus, -) +from fidesops.models.privacy_request import PrivacyRequest, PrivacyRequestStatus from fidesops.schemas.privacy_request import PrivacyRequestDRPStatus from fidesops.util.cache import get_drp_request_body_cache_key, get_identity_cache_key diff --git a/tests/api/v1/endpoints/test_encryption_endpoints.py b/tests/api/v1/endpoints/test_encryption_endpoints.py index f52aa8c5d..794212046 100644 --- a/tests/api/v1/endpoints/test_encryption_endpoints.py +++ b/tests/api/v1/endpoints/test_encryption_endpoints.py @@ -1,6 +1,6 @@ +import json from unittest import mock from unittest.mock import Mock -import json import pytest from starlette.testclient import TestClient @@ -8,8 +8,8 @@ from fidesops.api.v1.scope_registry import ENCRYPTION_EXEC, STORAGE_CREATE_OR_UPDATE from fidesops.api.v1.urn_registry import ( DECRYPT_AES, - ENCRYPTION_KEY, ENCRYPT_AES, + ENCRYPTION_KEY, V1_URL_PREFIX, ) from fidesops.core.config import config diff --git a/tests/api/v1/endpoints/test_masking_endpoints.py b/tests/api/v1/endpoints/test_masking_endpoints.py index 9ffbb4ffc..1d329cb05 100644 --- a/tests/api/v1/endpoints/test_masking_endpoints.py +++ b/tests/api/v1/endpoints/test_masking_endpoints.py @@ -3,10 +3,12 @@ from starlette.testclient import TestClient from fidesops.api.v1.urn_registry import MASKING, MASKING_STRATEGY, V1_URL_PREFIX +from fidesops.schemas.masking.masking_api import MaskingAPIResponse from fidesops.schemas.masking.masking_configuration import ( AesEncryptionMaskingConfiguration, ) from fidesops.service.masking.strategy.masking_strategy_aes_encrypt import AES_ENCRYPT +from fidesops.service.masking.strategy.masking_strategy_factory import get_strategies from fidesops.service.masking.strategy.masking_strategy_hash import HASH from fidesops.service.masking.strategy.masking_strategy_hmac import HMAC from fidesops.service.masking.strategy.masking_strategy_nullify import NULL_REWRITE @@ -16,8 +18,6 @@ from fidesops.service.masking.strategy.masking_strategy_string_rewrite import ( STRING_REWRITE, ) -from fidesops.schemas.masking.masking_api import MaskingAPIResponse -from fidesops.service.masking.strategy.masking_strategy_factory import get_strategies class TestGetMaskingStrategies: diff --git a/tests/api/v1/endpoints/test_oauth_endpoints.py b/tests/api/v1/endpoints/test_oauth_endpoints.py index c15a5c3fe..44e5e0670 100644 --- a/tests/api/v1/endpoints/test_oauth_endpoints.py +++ b/tests/api/v1/endpoints/test_oauth_endpoints.py @@ -1,31 +1,31 @@ -from datetime import datetime import json -import pytest +from datetime import datetime +import pytest from starlette.testclient import TestClient from fidesops.api.v1.scope_registry import ( - SCOPE_REGISTRY, + CLIENT_CREATE, CLIENT_DELETE, CLIENT_READ, - STORAGE_READ, - CLIENT_CREATE, - SCOPE_READ, CLIENT_UPDATE, + SCOPE_READ, + SCOPE_REGISTRY, + STORAGE_READ, ) from fidesops.api.v1.urn_registry import ( CLIENT, CLIENT_BY_ID, CLIENT_SCOPE, + SCOPE, TOKEN, V1_URL_PREFIX, - SCOPE, ) from fidesops.models.client import ClientDetail from fidesops.schemas.jwt import ( + JWE_ISSUED_AT, JWE_PAYLOAD_CLIENT_ID, JWE_PAYLOAD_SCOPES, - JWE_ISSUED_AT, ) from fidesops.util.oauth_util import extract_payload, generate_jwe diff --git a/tests/api/v1/endpoints/test_policy_endpoints.py b/tests/api/v1/endpoints/test_policy_endpoints.py index dc4d3d6a0..284a0cdc5 100644 --- a/tests/api/v1/endpoints/test_policy_endpoints.py +++ b/tests/api/v1/endpoints/test_policy_endpoints.py @@ -5,23 +5,17 @@ from starlette.testclient import TestClient from fidesops.api.v1 import scope_registry as scopes +from fidesops.api.v1.urn_registry import POLICY_DETAIL as POLICY_DETAIL_URI +from fidesops.api.v1.urn_registry import POLICY_LIST as POLICY_CREATE_URI +from fidesops.api.v1.urn_registry import RULE_DETAIL as RULE_DETAIL_URI +from fidesops.api.v1.urn_registry import RULE_LIST as RULE_CREATE_URI from fidesops.api.v1.urn_registry import ( - POLICY_LIST as POLICY_CREATE_URI, - POLICY_DETAIL as POLICY_DETAIL_URI, - RULE_LIST as RULE_CREATE_URI, - RULE_DETAIL as RULE_DETAIL_URI, - RULE_TARGET_LIST, RULE_TARGET_DETAIL, + RULE_TARGET_LIST, V1_URL_PREFIX, ) from fidesops.models.client import ClientDetail -from fidesops.models.policy import ( - ActionType, - Policy, - Rule, - RuleTarget, - DrpAction, -) +from fidesops.models.policy import ActionType, DrpAction, Policy, Rule, RuleTarget from fidesops.service.masking.strategy.masking_strategy_nullify import NULL_REWRITE from fidesops.util.data_category import DataCategory, generate_fides_data_categories diff --git a/tests/api/v1/endpoints/test_policy_webhook_endpoints.py b/tests/api/v1/endpoints/test_policy_webhook_endpoints.py index dbc53aae7..6da06f1d0 100644 --- a/tests/api/v1/endpoints/test_policy_webhook_endpoints.py +++ b/tests/api/v1/endpoints/test_policy_webhook_endpoints.py @@ -1,24 +1,23 @@ import json - from typing import Dict import pytest from fidesops.api.v1.scope_registry import ( - WEBHOOK_READ, - WEBHOOK_CREATE_OR_UPDATE, POLICY_READ, + WEBHOOK_CREATE_OR_UPDATE, WEBHOOK_DELETE, + WEBHOOK_READ, ) from fidesops.api.v1.urn_registry import ( - V1_URL_PREFIX, - POLICY_WEBHOOKS_PRE, - POLICY_WEBHOOKS_POST, - POLICY_PRE_WEBHOOK_DETAIL, POLICY_POST_WEBHOOK_DETAIL, + POLICY_PRE_WEBHOOK_DETAIL, + POLICY_WEBHOOKS_POST, + POLICY_WEBHOOKS_PRE, + V1_URL_PREFIX, ) from fidesops.models.connectionconfig import ConnectionConfig -from fidesops.models.policy import PolicyPreWebhook, PolicyPostWebhook +from fidesops.models.policy import PolicyPostWebhook, PolicyPreWebhook from tests.api.v1.endpoints.test_privacy_request_endpoints import stringify_date diff --git a/tests/api/v1/endpoints/test_saas_config_endpoints.py b/tests/api/v1/endpoints/test_saas_config_endpoints.py index 953412b11..d8fd03540 100644 --- a/tests/api/v1/endpoints/test_saas_config_endpoints.py +++ b/tests/api/v1/endpoints/test_saas_config_endpoints.py @@ -1,6 +1,10 @@ import json -import pytest from typing import Optional + +import pytest +from sqlalchemy.orm import Session +from starlette.testclient import TestClient + from fidesops.api.v1.scope_registry import ( SAAS_CONFIG_CREATE_OR_UPDATE, SAAS_CONFIG_DELETE, @@ -16,8 +20,6 @@ ConnectionConfig, ConnectionType, ) -from starlette.testclient import TestClient -from sqlalchemy.orm import Session from tests.api.v1.endpoints.test_dataset_endpoints import _reject_key diff --git a/tests/api/v1/endpoints/test_storage_endpoints.py b/tests/api/v1/endpoints/test_storage_endpoints.py index af7788563..08bd0b470 100644 --- a/tests/api/v1/endpoints/test_storage_endpoints.py +++ b/tests/api/v1/endpoints/test_storage_endpoints.py @@ -1,17 +1,17 @@ +import json from typing import Dict from unittest import mock from unittest.mock import Mock -import json import pytest +from fastapi_pagination import Params from sqlalchemy.orm import Session from starlette.testclient import TestClient -from fastapi_pagination import Params from fidesops.api.v1.scope_registry import ( STORAGE_CREATE_OR_UPDATE, - STORAGE_READ, STORAGE_DELETE, + STORAGE_READ, ) from fidesops.api.v1.urn_registry import ( STORAGE_BY_KEY, @@ -24,11 +24,11 @@ from fidesops.models.storage import StorageConfig from fidesops.schemas.storage.data_upload_location_response import DataUpload from fidesops.schemas.storage.storage import ( - StorageSecrets, + FileNaming, ResponseFormat, - StorageType, StorageDetails, - FileNaming, + StorageSecrets, + StorageType, ) PAGE_SIZE = Params().size diff --git a/tests/api/v1/endpoints/test_user_endpoints.py b/tests/api/v1/endpoints/test_user_endpoints.py index a3c212edf..0515e827c 100644 --- a/tests/api/v1/endpoints/test_user_endpoints.py +++ b/tests/api/v1/endpoints/test_user_endpoints.py @@ -1,50 +1,48 @@ +import json from datetime import datetime from email.mime import application from typing import List -import json import pytest - from fastapi_pagination import Params -from starlette.testclient import TestClient from starlette.status import ( HTTP_200_OK, HTTP_201_CREATED, HTTP_204_NO_CONTENT, - HTTP_403_FORBIDDEN, - HTTP_401_UNAUTHORIZED, HTTP_400_BAD_REQUEST, - HTTP_422_UNPROCESSABLE_ENTITY, + HTTP_401_UNAUTHORIZED, + HTTP_403_FORBIDDEN, HTTP_404_NOT_FOUND, + HTTP_422_UNPROCESSABLE_ENTITY, ) +from starlette.testclient import TestClient -from fidesops.api.v1.urn_registry import ( - V1_URL_PREFIX, - USERS, - LOGIN, - LOGOUT, - USER_DETAIL, -) -from fidesops.models.client import ClientDetail, ADMIN_UI_ROOT from fidesops.api.v1.scope_registry import ( + PRIVACY_REQUEST_READ, + SCOPE_REGISTRY, STORAGE_READ, USER_CREATE, - USER_READ, - USER_UPDATE, USER_DELETE, USER_PASSWORD_RESET, - SCOPE_REGISTRY, - PRIVACY_REQUEST_READ, + USER_READ, + USER_UPDATE, +) +from fidesops.api.v1.urn_registry import ( + LOGIN, + LOGOUT, + USER_DETAIL, + USERS, + V1_URL_PREFIX, ) +from fidesops.models.client import ADMIN_UI_ROOT, ClientDetail from fidesops.models.fidesops_user import FidesopsUser from fidesops.models.fidesops_user_permissions import FidesopsUserPermissions -from fidesops.util.oauth_util import generate_jwe, extract_payload from fidesops.schemas.jwt import ( + JWE_ISSUED_AT, JWE_PAYLOAD_CLIENT_ID, JWE_PAYLOAD_SCOPES, - JWE_ISSUED_AT, ) - +from fidesops.util.oauth_util import extract_payload, generate_jwe from tests.conftest import generate_auth_header_for_user page_size = Params().size diff --git a/tests/api/v1/endpoints/test_user_permission_endpoints.py b/tests/api/v1/endpoints/test_user_permission_endpoints.py index 3f2ae09ea..0934286dd 100644 --- a/tests/api/v1/endpoints/test_user_permission_endpoints.py +++ b/tests/api/v1/endpoints/test_user_permission_endpoints.py @@ -1,22 +1,23 @@ -import pytest import json + +import pytest from starlette.status import ( + HTTP_200_OK, + HTTP_201_CREATED, HTTP_401_UNAUTHORIZED, HTTP_403_FORBIDDEN, - HTTP_422_UNPROCESSABLE_ENTITY, - HTTP_201_CREATED, HTTP_404_NOT_FOUND, - HTTP_200_OK, + HTTP_422_UNPROCESSABLE_ENTITY, ) -from fidesops.api.v1.urn_registry import V1_URL_PREFIX, USER_PERMISSIONS from fidesops.api.v1.scope_registry import ( + PRIVACY_REQUEST_READ, + SAAS_CONFIG_READ, USER_PERMISSION_CREATE, USER_PERMISSION_READ, USER_PERMISSION_UPDATE, - SAAS_CONFIG_READ, - PRIVACY_REQUEST_READ, ) +from fidesops.api.v1.urn_registry import USER_PERMISSIONS, V1_URL_PREFIX from fidesops.models.client import ClientDetail from fidesops.models.fidesops_user import FidesopsUser from fidesops.models.fidesops_user_permissions import FidesopsUserPermissions diff --git a/tests/conftest.py b/tests/conftest.py index 33ed140bf..38d71e0e3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,49 +1,42 @@ import json import logging -import pytest - from typing import Generator +import pytest from fastapi.testclient import TestClient -from sqlalchemy_utils.functions import ( - create_database, - database_exists, - drop_database, -) +from sqlalchemy_utils.functions import create_database, database_exists, drop_database -from fidesops.core.config import ( - config, -) +from fidesops.api.v1.scope_registry import SCOPE_REGISTRY +from fidesops.core.config import config from fidesops.db.database import init_db -from fidesops.db.session import get_db_session, get_db_engine, Session +from fidesops.db.session import Session, get_db_engine, get_db_session from fidesops.main import app from fidesops.models.privacy_request import generate_request_callback_jwe from fidesops.schemas.jwt import ( + JWE_ISSUED_AT, JWE_PAYLOAD_CLIENT_ID, JWE_PAYLOAD_SCOPES, - JWE_ISSUED_AT, ) -from fidesops.api.v1.scope_registry import SCOPE_REGISTRY from fidesops.tasks.scheduled.scheduler import scheduler from fidesops.util.cache import get_cache from fidesops.util.oauth_util import generate_jwe from .fixtures.application_fixtures import * +from .fixtures.bigquery_fixtures import * from .fixtures.integration_fixtures import * from .fixtures.mariadb_fixtures import * from .fixtures.mongodb_fixtures import * -from .fixtures.mysql_fixtures import * from .fixtures.mssql_fixtures import * +from .fixtures.mysql_fixtures import * from .fixtures.postgres_fixtures import * from .fixtures.redshift_fixtures import * -from .fixtures.snowflake_fixtures import * -from .fixtures.bigquery_fixtures import * -from .fixtures.saas_example_fixtures import * +from .fixtures.saas.hubspot_fixtures import * from .fixtures.saas.mailchimp_fixtures import * +from .fixtures.saas.segment_fixtures import * from .fixtures.saas.sentry_fixtures import * from .fixtures.saas.stripe_fixtures import * -from .fixtures.saas.hubspot_fixtures import * -from .fixtures.saas.segment_fixtures import * +from .fixtures.saas_example_fixtures import * +from .fixtures.snowflake_fixtures import * logger = logging.getLogger(__name__) diff --git a/tests/fixtures/application_fixtures.py b/tests/fixtures/application_fixtures.py index 2f2b392ce..1a85fb177 100644 --- a/tests/fixtures/application_fixtures.py +++ b/tests/fixtures/application_fixtures.py @@ -4,11 +4,6 @@ from unittest import mock from uuid import uuid4 -from fidesops.api.v1.scope_registry import SCOPE_REGISTRY, PRIVACY_REQUEST_READ -from fidesops.models.fidesops_user import FidesopsUser -from fidesops.service.masking.strategy.masking_strategy_hmac import HMAC -from fidesops.util.data_category import DataCategory - import pydash import pytest import yaml @@ -16,41 +11,41 @@ from sqlalchemy.orm import Session from sqlalchemy.orm.exc import ObjectDeletedError +from fidesops.api.v1.scope_registry import PRIVACY_REQUEST_READ, SCOPE_REGISTRY from fidesops.core.config import load_file, load_toml from fidesops.models.client import ClientDetail from fidesops.models.connectionconfig import ( - ConnectionConfig, AccessLevel, + ConnectionConfig, ConnectionType, ) from fidesops.models.datasetconfig import DatasetConfig +from fidesops.models.fidesops_user import FidesopsUser +from fidesops.models.fidesops_user_permissions import FidesopsUserPermissions from fidesops.models.policy import ( ActionType, Policy, + PolicyPostWebhook, + PolicyPreWebhook, Rule, RuleTarget, - PolicyPreWebhook, - PolicyPostWebhook, -) - -from fidesops.models.privacy_request import ( - PrivacyRequest, - PrivacyRequestStatus, ) -from fidesops.models.storage import StorageConfig, ResponseFormat -from fidesops.models.fidesops_user_permissions import FidesopsUserPermissions +from fidesops.models.privacy_request import PrivacyRequest, PrivacyRequestStatus +from fidesops.models.storage import ResponseFormat, StorageConfig from fidesops.schemas.storage.storage import ( FileNaming, StorageDetails, StorageSecrets, StorageType, ) +from fidesops.service.masking.strategy.masking_strategy_hmac import HMAC from fidesops.service.masking.strategy.masking_strategy_nullify import NULL_REWRITE from fidesops.service.masking.strategy.masking_strategy_string_rewrite import ( STRING_REWRITE, ) from fidesops.service.privacy_request.request_runner_service import PrivacyRequestRunner from fidesops.util.cache import FidesopsRedis +from fidesops.util.data_category import DataCategory logging.getLogger("faker").setLevel(logging.ERROR) # disable verbose faker logging diff --git a/tests/fixtures/bigquery_fixtures.py b/tests/fixtures/bigquery_fixtures.py index cacf3a436..35fae0007 100644 --- a/tests/fixtures/bigquery_fixtures.py +++ b/tests/fixtures/bigquery_fixtures.py @@ -1,21 +1,21 @@ import ast import logging import os -from typing import Generator, List, Dict +from typing import Dict, Generator, List from uuid import uuid4 import pytest - from sqlalchemy.orm import Session from fidesops.models.connectionconfig import ( AccessLevel, - ConnectionType, ConnectionConfig, + ConnectionType, ) from fidesops.models.datasetconfig import DatasetConfig from fidesops.schemas.connection_configuration import BigQuerySchema from fidesops.service.connectors import BigQueryConnector, get_connector + from .application_fixtures import integration_config logger = logging.getLogger(__name__) diff --git a/tests/fixtures/integration_fixtures.py b/tests/fixtures/integration_fixtures.py index f2c8a27e1..5ee1dc13d 100644 --- a/tests/fixtures/integration_fixtures.py +++ b/tests/fixtures/integration_fixtures.py @@ -1,7 +1,7 @@ import logging import random from datetime import datetime -from typing import Dict, Any, List +from typing import Any, Dict, List from uuid import uuid4 import pytest @@ -10,11 +10,12 @@ from sqlalchemy.engine import Engine from fidesops.models.connectionconfig import ( + AccessLevel, ConnectionConfig, ConnectionType, - AccessLevel, ) -from fidesops.service.connectors import PostgreSQLConnector, MongoDBConnector +from fidesops.service.connectors import MongoDBConnector, PostgreSQLConnector + from .application_fixtures import faker, integration_secrets logger = logging.getLogger(__name__) diff --git a/tests/fixtures/mongodb_fixtures.py b/tests/fixtures/mongodb_fixtures.py index 53a975d4d..48167bdaf 100644 --- a/tests/fixtures/mongodb_fixtures.py +++ b/tests/fixtures/mongodb_fixtures.py @@ -1,12 +1,12 @@ -import pytest from typing import Generator from uuid import uuid4 +import pytest from sqlalchemy.orm import Session from fidesops.models.connectionconfig import ( - ConnectionConfig, AccessLevel, + ConnectionConfig, ConnectionType, ) from fidesops.models.policy import ActionType diff --git a/tests/fixtures/mssql_fixtures.py b/tests/fixtures/mssql_fixtures.py index 14143614c..9d4ef9201 100644 --- a/tests/fixtures/mssql_fixtures.py +++ b/tests/fixtures/mssql_fixtures.py @@ -1,14 +1,14 @@ import logging -import pytest from typing import Dict, Generator, List from uuid import uuid4 +import pytest from sqlalchemy.orm import Session -from fidesops.db.session import get_db_session, get_db_engine +from fidesops.db.session import get_db_engine, get_db_session from fidesops.models.connectionconfig import ( - ConnectionConfig, AccessLevel, + ConnectionConfig, ConnectionType, ) from fidesops.models.datasetconfig import DatasetConfig diff --git a/tests/fixtures/mysql_fixtures.py b/tests/fixtures/mysql_fixtures.py index 4ba7a6547..bca684cb3 100644 --- a/tests/fixtures/mysql_fixtures.py +++ b/tests/fixtures/mysql_fixtures.py @@ -1,14 +1,14 @@ import logging -import pytest from typing import Dict, Generator, List from uuid import uuid4 +import pytest from sqlalchemy.orm import Session -from fidesops.db.session import get_db_session, get_db_engine +from fidesops.db.session import get_db_engine, get_db_session from fidesops.models.connectionconfig import ( - ConnectionConfig, AccessLevel, + ConnectionConfig, ConnectionType, ) from fidesops.models.datasetconfig import DatasetConfig diff --git a/tests/fixtures/postgres_fixtures.py b/tests/fixtures/postgres_fixtures.py index fa1ed55c4..d91256456 100644 --- a/tests/fixtures/postgres_fixtures.py +++ b/tests/fixtures/postgres_fixtures.py @@ -1,22 +1,16 @@ import logging -import pytest from typing import Dict, Generator, List from uuid import uuid4 -from sqlalchemy.orm import ( - Session, -) +import pytest +from sqlalchemy.orm import Session from sqlalchemy.sql import text -from sqlalchemy_utils.functions import ( - create_database, - database_exists, - drop_database, -) +from sqlalchemy_utils.functions import create_database, database_exists, drop_database -from fidesops.db.session import get_db_session, get_db_engine +from fidesops.db.session import get_db_engine, get_db_session from fidesops.models.connectionconfig import ( - ConnectionConfig, AccessLevel, + ConnectionConfig, ConnectionType, ) from fidesops.models.datasetconfig import DatasetConfig diff --git a/tests/fixtures/redshift_fixtures.py b/tests/fixtures/redshift_fixtures.py index e178f39ba..52a9f5b2a 100644 --- a/tests/fixtures/redshift_fixtures.py +++ b/tests/fixtures/redshift_fixtures.py @@ -1,13 +1,13 @@ import os -import pytest from typing import Dict, Generator, List from uuid import uuid4 +import pytest from sqlalchemy.orm import Session from fidesops.models.connectionconfig import ( - ConnectionConfig, AccessLevel, + ConnectionConfig, ConnectionType, ) from fidesops.models.datasetconfig import DatasetConfig diff --git a/tests/fixtures/saas/mailchimp_fixtures.py b/tests/fixtures/saas/mailchimp_fixtures.py index 091814846..b25311894 100644 --- a/tests/fixtures/saas/mailchimp_fixtures.py +++ b/tests/fixtures/saas/mailchimp_fixtures.py @@ -1,4 +1,11 @@ import json +import os +from typing import Any, Dict, Generator + +import pydash +import pytest +from sqlalchemy.orm import Session + from fidesops.core.config import load_toml from fidesops.db import session from fidesops.models.connectionconfig import ( @@ -7,15 +14,10 @@ ConnectionType, ) from fidesops.models.datasetconfig import DatasetConfig -import pytest -import pydash -import os -from typing import Any, Dict, Generator from fidesops.schemas.saas.shared_schemas import HTTPMethod, SaaSRequestParams from fidesops.service.connectors.saas_connector import SaaSConnector from tests.fixtures.application_fixtures import load_dataset from tests.fixtures.saas_example_fixtures import load_config -from sqlalchemy.orm import Session saas_config = load_toml("saas_config.toml") diff --git a/tests/fixtures/saas_example_fixtures.py b/tests/fixtures/saas_example_fixtures.py index 33f8f0e47..0dfb04a17 100644 --- a/tests/fixtures/saas_example_fixtures.py +++ b/tests/fixtures/saas_example_fixtures.py @@ -1,9 +1,9 @@ -import pytest +from typing import Any, Dict, Generator + import pydash +import pytest import yaml - from sqlalchemy.orm import Session -from typing import Any, Dict, Generator from fidesops.core.config import load_file, load_toml from fidesops.models.connectionconfig import ( diff --git a/tests/fixtures/snowflake_fixtures.py b/tests/fixtures/snowflake_fixtures.py index ce5e80a9c..c4517a24a 100644 --- a/tests/fixtures/snowflake_fixtures.py +++ b/tests/fixtures/snowflake_fixtures.py @@ -1,13 +1,13 @@ import os -import pytest from typing import Dict, Generator, List from uuid import uuid4 +import pytest from sqlalchemy.orm import Session from fidesops.models.connectionconfig import ( - ConnectionConfig, AccessLevel, + ConnectionConfig, ConnectionType, ) from fidesops.models.datasetconfig import DatasetConfig diff --git a/tests/generator/sql_data_generator.py b/tests/generator/sql_data_generator.py index 721ea654a..ba5fe6974 100644 --- a/tests/generator/sql_data_generator.py +++ b/tests/generator/sql_data_generator.py @@ -1,8 +1,9 @@ import random -from typing import Dict, List, Any +from string import ascii_letters, digits +from typing import Any, Dict, List from faker import Faker -from sqlalchemy import Column, String, Integer, Float, Boolean +from sqlalchemy import Boolean, Column, Float, Integer, String from sqlalchemy.ext.declarative import declarative_base from fidesops.db.session import ENGINE @@ -14,17 +15,16 @@ ScalarField, ) from fidesops.graph.data_type import DataType -from fidesops.graph.graph import Edge, BidirectionalEdge, DatasetGraph -from fidesops.graph.traversal import Traversal, TraversalNode, Row -from tests.graph.graph_test_util import generate_graph_resources, field -from string import ascii_letters, digits +from fidesops.graph.graph import BidirectionalEdge, DatasetGraph, Edge +from fidesops.graph.traversal import Row, Traversal, TraversalNode +from tests.graph.graph_test_util import field, generate_graph_resources Base = declarative_base() faker = Faker(use_weighting=False) -from faker.providers.lorem.en_US import Provider as lorem from faker.providers import address +from faker.providers.lorem.en_US import Provider as lorem faker.add_provider(address) faker.add_provider(lorem) diff --git a/tests/generator/test_data_generator.py b/tests/generator/test_data_generator.py index 7c425ca3d..8feebffed 100644 --- a/tests/generator/test_data_generator.py +++ b/tests/generator/test_data_generator.py @@ -2,15 +2,14 @@ from fidesops.graph.graph import * - # ------------------------------------------- # graph object tests # ------------------------------------------- from fidesops.graph.traversal import Traversal from fidesops.models.datasetconfig import convert_dataset_to_graph from fidesops.schemas.dataset import FidesopsDataset -from . import sql_data_generator +from . import sql_data_generator f = """dataset: - fides_key: db diff --git a/tests/graph/graph_test_util.py b/tests/graph/graph_test_util.py index a6ca77e88..7fa402f16 100644 --- a/tests/graph/graph_test_util.py +++ b/tests/graph/graph_test_util.py @@ -1,7 +1,6 @@ import random from typing import Iterable - from sqlalchemy.engine import Engine from fidesops.db.base_class import FidesopsBase @@ -11,16 +10,16 @@ # to avoid having faker spam the logs from fidesops.models.connectionconfig import ConnectionConfig -from fidesops.models.policy import Policy, RuleTarget, Rule, ActionType +from fidesops.models.policy import ActionType, Policy, Rule, RuleTarget from fidesops.models.privacy_request import PrivacyRequest from fidesops.service.connectors import BaseConnector, MongoDBConnector from fidesops.service.connectors.sql_connector import SQLConnector - from fidesops.task.graph_task import GraphTask from fidesops.task.task_resources import TaskResources -from ..fixtures.application_fixtures import faker from fidesops.util.collection_util import Row +from ..fixtures.application_fixtures import faker + class MockResources(TaskResources): def __init__(self, request: PrivacyRequest): diff --git a/tests/graph/test_config.py b/tests/graph/test_config.py index 7cb2aad0c..9cb17c59c 100644 --- a/tests/graph/test_config.py +++ b/tests/graph/test_config.py @@ -4,11 +4,11 @@ from fidesops.common_exceptions import ValidationError from fidesops.graph.config import * from fidesops.graph.data_type import ( - IntTypeConverter, - StringTypeConverter, BooleanTypeConverter, + IntTypeConverter, NoOpTypeConverter, ObjectTypeConverter, + StringTypeConverter, ) diff --git a/tests/graph/test_data_types.py b/tests/graph/test_data_types.py index 99697aed4..32b7cb2ed 100644 --- a/tests/graph/test_data_types.py +++ b/tests/graph/test_data_types.py @@ -3,10 +3,10 @@ from fidesops.graph.data_type import ( DataType, NoOpTypeConverter, - get_data_type_converter, StringTypeConverter, - parse_data_type_string, + get_data_type_converter, is_valid_data_type, + parse_data_type_string, ) diff --git a/tests/graph/test_edge.py b/tests/graph/test_edge.py index c18cf94df..1651aa673 100644 --- a/tests/graph/test_edge.py +++ b/tests/graph/test_edge.py @@ -1,7 +1,8 @@ -from fidesops.graph.graph import * -from fidesops.common_exceptions import ValidationError import pytest +from fidesops.common_exceptions import ValidationError +from fidesops.graph.graph import * + class TestEdge: def test_eq(self) -> None: diff --git a/tests/graph/test_graph_traversal.py b/tests/graph/test_graph_traversal.py index 3cbef25d4..5fe6e3582 100644 --- a/tests/graph/test_graph_traversal.py +++ b/tests/graph/test_graph_traversal.py @@ -1,7 +1,8 @@ import pytest + from fidesops.graph.graph import * -from .graph_test_util import * +from .graph_test_util import * # ------------------------------------------- # graph object tests diff --git a/tests/graph/test_traversal_node.py b/tests/graph/test_traversal_node.py index 3b4ca1891..9d42c0773 100644 --- a/tests/graph/test_traversal_node.py +++ b/tests/graph/test_traversal_node.py @@ -1,4 +1,5 @@ from fidesops.graph.traversal import * + from .test_graph_traversal import generate_node diff --git a/tests/integration_tests/saas/test_hubspot_task.py b/tests/integration_tests/saas/test_hubspot_task.py index 5d0e841e0..937eccef9 100644 --- a/tests/integration_tests/saas/test_hubspot_task.py +++ b/tests/integration_tests/saas/test_hubspot_task.py @@ -1,20 +1,18 @@ import json +import random import pytest -import random from fidesops.graph.graph import DatasetGraph from fidesops.models.privacy_request import ExecutionLog, PrivacyRequest from fidesops.schemas.redis_cache import PrivacyRequestIdentity -from fidesops.schemas.saas.shared_schemas import SaaSRequestParams, HTTPMethod +from fidesops.schemas.saas.shared_schemas import HTTPMethod, SaaSRequestParams from fidesops.service.connectors import SaaSConnector - from fidesops.task import graph_task -from tests.graph.graph_test_util import assert_rows_match, records_matching_fields - from fidesops.task.filter_results import filter_data_categories from fidesops.task.graph_task import get_cached_data_for_erasures from fidesops.util.saas_util import format_body +from tests.graph.graph_test_util import assert_rows_match, records_matching_fields @pytest.mark.integration_saas diff --git a/tests/integration_tests/saas/test_mailchimp_task.py b/tests/integration_tests/saas/test_mailchimp_task.py index 7e6be20e7..ea91d544d 100644 --- a/tests/integration_tests/saas/test_mailchimp_task.py +++ b/tests/integration_tests/saas/test_mailchimp_task.py @@ -1,10 +1,10 @@ -import pytest import random +import pytest + from fidesops.graph.graph import DatasetGraph from fidesops.models.privacy_request import ExecutionLog, PrivacyRequest from fidesops.schemas.redis_cache import PrivacyRequestIdentity - from fidesops.task import graph_task from fidesops.task.graph_task import get_cached_data_for_erasures from tests.graph.graph_test_util import assert_rows_match, records_matching_fields diff --git a/tests/integration_tests/setup_scripts/mariadb_setup.py b/tests/integration_tests/setup_scripts/mariadb_setup.py index 198fc7bdc..41ce6a254 100644 --- a/tests/integration_tests/setup_scripts/mariadb_setup.py +++ b/tests/integration_tests/setup_scripts/mariadb_setup.py @@ -5,10 +5,10 @@ import sqlalchemy from fidesops.core.config import load_toml -from fidesops.db.session import get_db_session, get_db_engine +from fidesops.db.session import get_db_engine, get_db_session from fidesops.models.connectionconfig import ( - ConnectionConfig, AccessLevel, + ConnectionConfig, ConnectionType, ) from fidesops.service.connectors.sql_connector import MariaDBConnector diff --git a/tests/integration_tests/setup_scripts/postgres_setup.py b/tests/integration_tests/setup_scripts/postgres_setup.py index b0f6814a3..935e08820 100644 --- a/tests/integration_tests/setup_scripts/postgres_setup.py +++ b/tests/integration_tests/setup_scripts/postgres_setup.py @@ -2,17 +2,13 @@ import pydash import sqlalchemy -from sqlalchemy_utils.functions import ( - create_database, - database_exists, - drop_database, -) +from sqlalchemy_utils.functions import create_database, database_exists, drop_database from fidesops.core.config import load_toml -from fidesops.db.session import get_db_session, get_db_engine +from fidesops.db.session import get_db_engine, get_db_session from fidesops.models.connectionconfig import ( - ConnectionConfig, AccessLevel, + ConnectionConfig, ConnectionType, ) from fidesops.service.connectors.sql_connector import PostgreSQLConnector diff --git a/tests/integration_tests/test_connection_configuration_integration.py b/tests/integration_tests/test_connection_configuration_integration.py index e95ace5dd..df4b0aab0 100644 --- a/tests/integration_tests/test_connection_configuration_integration.py +++ b/tests/integration_tests/test_connection_configuration_integration.py @@ -1,31 +1,32 @@ -from fidesops.service.connectors.saas_connector import AuthenticatedClient -import pytest import json +import pytest from pymongo import MongoClient from sqlalchemy.engine import Engine from sqlalchemy.orm import Session from starlette.testclient import TestClient -from fidesops.models.client import ClientDetail -from fidesops.models.connectionconfig import ConnectionTestStatus -from fidesops.service.connectors import MongoDBConnector -from fidesops.service.connectors.sql_connector import ( - MySQLConnector, - MicrosoftSQLServerConnector, - MariaDBConnector, -) -from fidesops.common_exceptions import ConnectionException -from fidesops.service.connectors import PostgreSQLConnector -from fidesops.service.connectors import SaaSConnector -from fidesops.service.connectors import get_connector from fidesops.api.v1.scope_registry import ( CONNECTION_CREATE_OR_UPDATE, CONNECTION_READ, STORAGE_READ, ) - from fidesops.api.v1.urn_registry import CONNECTIONS, V1_URL_PREFIX +from fidesops.common_exceptions import ConnectionException +from fidesops.models.client import ClientDetail +from fidesops.models.connectionconfig import ConnectionTestStatus +from fidesops.service.connectors import ( + MongoDBConnector, + PostgreSQLConnector, + SaaSConnector, + get_connector, +) +from fidesops.service.connectors.saas_connector import AuthenticatedClient +from fidesops.service.connectors.sql_connector import ( + MariaDBConnector, + MicrosoftSQLServerConnector, + MySQLConnector, +) @pytest.mark.integration_postgres diff --git a/tests/integration_tests/test_external_database_connections.py b/tests/integration_tests/test_external_database_connections.py index fd2d0661e..5abe00181 100644 --- a/tests/integration_tests/test_external_database_connections.py +++ b/tests/integration_tests/test_external_database_connections.py @@ -1,17 +1,17 @@ +import logging import os +from typing import Generator -import logging import pytest from sqlalchemy import inspect -from typing import Generator from fidesops.core.config import load_toml from fidesops.models.connectionconfig import ConnectionConfig, ConnectionType from fidesops.schemas.connection_configuration import RedshiftSchema, SnowflakeSchema from fidesops.service.connectors import ( - get_connector, RedshiftConnector, SnowflakeConnector, + get_connector, ) logger = logging.getLogger(__name__) diff --git a/tests/integration_tests/test_integration_mssql_example.py b/tests/integration_tests/test_integration_mssql_example.py index 7344d378d..c7fea740b 100644 --- a/tests/integration_tests/test_integration_mssql_example.py +++ b/tests/integration_tests/test_integration_mssql_example.py @@ -4,7 +4,7 @@ import pytest from sqlalchemy import func, select, table -from fidesops.db.session import get_db_session, get_db_engine +from fidesops.db.session import get_db_engine, get_db_session logger = logging.getLogger(__name__) diff --git a/tests/integration_tests/test_integration_postgres_example.py b/tests/integration_tests/test_integration_postgres_example.py index 6732a96b6..1e6f00b40 100644 --- a/tests/integration_tests/test_integration_postgres_example.py +++ b/tests/integration_tests/test_integration_postgres_example.py @@ -1,4 +1,5 @@ import logging + import pytest from sqlalchemy import func, select, table diff --git a/tests/integration_tests/test_mongo_task.py b/tests/integration_tests/test_mongo_task.py index 38a24c16b..523d7e88c 100644 --- a/tests/integration_tests/test_mongo_task.py +++ b/tests/integration_tests/test_mongo_task.py @@ -8,22 +8,15 @@ import pytest from bson import ObjectId -from fidesops.graph.config import ( - FieldAddress, - ScalarField, - Collection, - Dataset, -) +from fidesops.graph.config import Collection, Dataset, FieldAddress, ScalarField from fidesops.graph.data_type import ( IntTypeConverter, - StringTypeConverter, ObjectIdTypeConverter, + StringTypeConverter, ) -from fidesops.graph.graph import DatasetGraph, Node, Edge +from fidesops.graph.graph import DatasetGraph, Edge, Node from fidesops.graph.traversal import TraversalNode -from fidesops.models.connectionconfig import ( - ConnectionConfig, -) +from fidesops.models.connectionconfig import ConnectionConfig from fidesops.models.datasetconfig import convert_dataset_to_graph from fidesops.models.policy import Policy from fidesops.models.privacy_request import PrivacyRequest @@ -31,15 +24,13 @@ from fidesops.service.connectors import get_connector from fidesops.task import graph_task from fidesops.task.filter_results import filter_data_categories -from fidesops.task.graph_task import ( - get_cached_data_for_erasures, -) +from fidesops.task.graph_task import get_cached_data_for_erasures from ..graph.graph_test_util import assert_rows_match, erasure_policy, field from ..task.traversal_data import ( + combined_mongo_postgresql_graph, integration_db_graph, integration_db_mongo_graph, - combined_mongo_postgresql_graph, ) dask.config.set(scheduler="processes") diff --git a/tests/integration_tests/test_sql_task.py b/tests/integration_tests/test_sql_task.py index 98fbe5126..bea98f4dd 100644 --- a/tests/integration_tests/test_sql_task.py +++ b/tests/integration_tests/test_sql_task.py @@ -9,29 +9,27 @@ import pytest from fidesops.core.config import config -from fidesops.graph.config import FieldAddress, Collection, ScalarField, Dataset +from fidesops.graph.config import Collection, Dataset, FieldAddress, ScalarField from fidesops.graph.data_type import DataType, StringTypeConverter from fidesops.graph.graph import DatasetGraph, Edge, Node from fidesops.graph.traversal import TraversalNode from fidesops.models.connectionconfig import ConnectionConfig from fidesops.models.datasetconfig import convert_dataset_to_graph -from fidesops.models.policy import Policy -from fidesops.models.policy import Rule, RuleTarget, ActionType +from fidesops.models.policy import ActionType, Policy, Rule, RuleTarget from fidesops.models.privacy_request import ExecutionLog, PrivacyRequest from fidesops.schemas.dataset import FidesopsDataset from fidesops.service.connectors import get_connector from fidesops.task import graph_task from fidesops.task.filter_results import filter_data_categories -from fidesops.task.graph_task import ( - get_cached_data_for_erasures, -) +from fidesops.task.graph_task import get_cached_data_for_erasures + from ..graph.graph_test_util import ( assert_rows_match, - records_matching_fields, - field, erasure_policy, + field, + records_matching_fields, ) -from ..task.traversal_data import integration_db_graph, integration_db_dataset +from ..task.traversal_data import integration_db_dataset, integration_db_graph dask.config.set(scheduler="processes") logger = logging.getLogger(__name__) diff --git a/tests/models/test_base.py b/tests/models/test_base.py index 87f214b82..26b395dae 100644 --- a/tests/models/test_base.py +++ b/tests/models/test_base.py @@ -2,8 +2,8 @@ from sqlalchemy.orm import Session from fidesops.common_exceptions import KeyValidationError -from fidesops.models.storage import StorageConfig from fidesops.db.base_class import get_key_from_data +from fidesops.models.storage import StorageConfig def test_get_key_from_data_method() -> None: diff --git a/tests/models/test_client.py b/tests/models/test_client.py index 3e2ffd91f..aa51f6126 100644 --- a/tests/models/test_client.py +++ b/tests/models/test_client.py @@ -3,9 +3,7 @@ from fidesops.api.v1.scope_registry import SCOPE_REGISTRY from fidesops.core.config import config from fidesops.models.client import ClientDetail -from fidesops.util.cryptographic_util import ( - hash_with_salt, -) +from fidesops.util.cryptographic_util import hash_with_salt class TestClientModel: diff --git a/tests/models/test_connectionconfig.py b/tests/models/test_connectionconfig.py index 5e723084a..c49e3b5ec 100644 --- a/tests/models/test_connectionconfig.py +++ b/tests/models/test_connectionconfig.py @@ -1,11 +1,11 @@ import pytest from sqlalchemy.orm import Session -from fidesops.db.base_class import KeyValidationError, KeyOrNameAlreadyExists +from fidesops.db.base_class import KeyOrNameAlreadyExists, KeyValidationError from fidesops.models.connectionconfig import ( + AccessLevel, ConnectionConfig, ConnectionType, - AccessLevel, ) from fidesops.util.text import to_snake_case diff --git a/tests/models/test_datasetconfig.py b/tests/models/test_datasetconfig.py index 73907e312..e4515d713 100644 --- a/tests/models/test_datasetconfig.py +++ b/tests/models/test_datasetconfig.py @@ -1,11 +1,12 @@ from sqlalchemy.orm import Session -from fidesops.schemas.dataset import FidesopsDataset from fidesops.models.datasetconfig import ( - convert_dataset_to_graph, DatasetConfig, FieldAddress, + convert_dataset_to_graph, ) +from fidesops.schemas.dataset import FidesopsDataset + from ..graph.graph_test_util import field diff --git a/tests/models/test_fidesopsuser.py b/tests/models/test_fidesopsuser.py index 337b75b10..ea42495c3 100644 --- a/tests/models/test_fidesopsuser.py +++ b/tests/models/test_fidesopsuser.py @@ -1,6 +1,6 @@ import pytest - from sqlalchemy.orm import Session + from fidesops.models.fidesops_user import FidesopsUser diff --git a/tests/models/test_fidesopsuserpermissions.py b/tests/models/test_fidesopsuserpermissions.py index 31b43e940..3026b1195 100644 --- a/tests/models/test_fidesopsuserpermissions.py +++ b/tests/models/test_fidesopsuserpermissions.py @@ -1,8 +1,7 @@ +import psycopg2.errors as db_errors import pytest - from sqlalchemy.orm import Session -import psycopg2.errors as db_errors from fidesops.api.v1.scope_registry import PRIVACY_REQUEST_READ from fidesops.models.fidesops_user import FidesopsUser from fidesops.models.fidesops_user_permissions import FidesopsUserPermissions diff --git a/tests/models/test_policy.py b/tests/models/test_policy.py index 2921dcfae..5b3d461a7 100644 --- a/tests/models/test_policy.py +++ b/tests/models/test_policy.py @@ -16,8 +16,8 @@ ) from fidesops.service.masking.strategy.masking_strategy_hash import HASH from fidesops.service.masking.strategy.masking_strategy_nullify import NULL_REWRITE -from fidesops.util.text import to_snake_case from fidesops.util.data_category import DataCategory +from fidesops.util.text import to_snake_case def test_policy_sets_slug( diff --git a/tests/models/test_policy_webhooks.py b/tests/models/test_policy_webhooks.py index bd08ce248..a02f7db6a 100644 --- a/tests/models/test_policy_webhooks.py +++ b/tests/models/test_policy_webhooks.py @@ -1,6 +1,7 @@ +from uuid import uuid4 + import pytest from sqlalchemy.orm import Session -from uuid import uuid4 from fidesops.common_exceptions import WebhookOrderException from fidesops.models.policy import PolicyPostWebhook diff --git a/tests/models/test_privacy_request.py b/tests/models/test_privacy_request.py index 892030d77..0215b5f78 100644 --- a/tests/models/test_privacy_request.py +++ b/tests/models/test_privacy_request.py @@ -1,22 +1,15 @@ -import pytest -import requests_mock -from datetime import ( - datetime, - timedelta, - timezone, -) -from pydantic import ValidationError +from datetime import datetime, timedelta, timezone from typing import List from uuid import uuid4 +import pytest +import requests_mock +from pydantic import ValidationError from sqlalchemy.orm import Session from fidesops.common_exceptions import ClientUnsuccessfulException, PrivacyRequestPaused from fidesops.models.policy import Policy -from fidesops.models.privacy_request import ( - PrivacyRequest, - PrivacyRequestStatus, -) +from fidesops.models.privacy_request import PrivacyRequest, PrivacyRequestStatus from fidesops.schemas.redis_cache import PrivacyRequestIdentity from fidesops.util.cache import FidesopsRedis, get_identity_cache_key diff --git a/tests/models/test_saasconfig.py b/tests/models/test_saasconfig.py index 775340481..67d8cc3dd 100644 --- a/tests/models/test_saasconfig.py +++ b/tests/models/test_saasconfig.py @@ -1,4 +1,5 @@ from typing import Dict + import pytest from pydantic import ValidationError diff --git a/tests/models/test_storage.py b/tests/models/test_storage.py index 49d001519..7be3e70d5 100644 --- a/tests/models/test_storage.py +++ b/tests/models/test_storage.py @@ -6,11 +6,11 @@ from fidesops.db.base_class import KeyOrNameAlreadyExists from fidesops.models.storage import StorageConfig from fidesops.schemas.storage.storage import ( + ResponseFormat, StorageDestination, - StorageType, StorageDetails, - ResponseFormat, StorageSecrets, + StorageType, ) diff --git a/tests/schemas/connection_configuration/test_connection_secrets_saas.py b/tests/schemas/connection_configuration/test_connection_secrets_saas.py index 16e2a07c1..08e2d787b 100644 --- a/tests/schemas/connection_configuration/test_connection_secrets_saas.py +++ b/tests/schemas/connection_configuration/test_connection_secrets_saas.py @@ -1,10 +1,11 @@ import pytest from pydantic import ValidationError -from fidesops.schemas.saas.saas_config import SaaSConfig + from fidesops.schemas.connection_configuration.connection_secrets_saas import ( SaaSSchema, SaaSSchemaFactory, ) +from fidesops.schemas.saas.saas_config import SaaSConfig @pytest.mark.unit_saas diff --git a/tests/schemas/test_user_permission.py b/tests/schemas/test_user_permission.py index 12d263d8f..e19b4a222 100644 --- a/tests/schemas/test_user_permission.py +++ b/tests/schemas/test_user_permission.py @@ -1,8 +1,9 @@ import pytest from fastapi import HTTPException from starlette.status import HTTP_422_UNPROCESSABLE_ENTITY + +from fidesops.api.v1.scope_registry import USER_DELETE, USER_PERMISSION_CREATE from fidesops.schemas.user_permission import UserPermissionsCreate -from fidesops.api.v1.scope_registry import USER_PERMISSION_CREATE, USER_DELETE class TestUserPermissionsCreate: diff --git a/tests/scripts/test_create_superuser.py b/tests/scripts/test_create_superuser.py index 3c5b4ebf2..566734b27 100644 --- a/tests/scripts/test_create_superuser.py +++ b/tests/scripts/test_create_superuser.py @@ -1,16 +1,14 @@ -import pytest from unittest import mock -from create_superuser import ( - collect_username_and_password, - create_user_and_client, -) +import pytest +from create_superuser import collect_username_and_password, create_user_and_client + +from fidesops.api.v1.scope_registry import CLIENT_CREATE from fidesops.common_exceptions import KeyOrNameAlreadyExists -from fidesops.models.client import ClientDetail, ADMIN_UI_ROOT +from fidesops.models.client import ADMIN_UI_ROOT, ClientDetail from fidesops.models.fidesops_user import FidesopsUser from fidesops.models.fidesops_user_permissions import FidesopsUserPermissions from fidesops.schemas.user import UserCreate -from fidesops.api.v1.scope_registry import CLIENT_CREATE class TestCreateSuperuserScript: diff --git a/tests/service/authentication/test_authentication_strategy_basic.py b/tests/service/authentication/test_authentication_strategy_basic.py index 746dd7ed1..e2668dc35 100644 --- a/tests/service/authentication/test_authentication_strategy_basic.py +++ b/tests/service/authentication/test_authentication_strategy_basic.py @@ -1,9 +1,9 @@ import pytest from requests import PreparedRequest, Request +from fidesops.common_exceptions import ValidationError as FidesopsValidationError from fidesops.service.authentication.authentication_strategy_factory import get_strategy from fidesops.util.cryptographic_util import bytes_to_b64_str -from fidesops.common_exceptions import ValidationError as FidesopsValidationError def test_basic_auth_with_username_and_password(): diff --git a/tests/service/authentication/test_authentication_strategy_bearer.py b/tests/service/authentication/test_authentication_strategy_bearer.py index 31e3bb82d..b5b61ae5d 100644 --- a/tests/service/authentication/test_authentication_strategy_bearer.py +++ b/tests/service/authentication/test_authentication_strategy_bearer.py @@ -1,9 +1,9 @@ import pytest from requests import PreparedRequest, Request +from fidesops.common_exceptions import ValidationError as FidesopsValidationError from fidesops.service.authentication.authentication_strategy_factory import get_strategy from fidesops.util.cryptographic_util import bytes_to_b64_str -from fidesops.common_exceptions import ValidationError as FidesopsValidationError def test_bearer_auth_with_token(): diff --git a/tests/service/authentication/test_authentication_strategy_query_param.py b/tests/service/authentication/test_authentication_strategy_query_param.py index fb39f037c..e58354ee3 100644 --- a/tests/service/authentication/test_authentication_strategy_query_param.py +++ b/tests/service/authentication/test_authentication_strategy_query_param.py @@ -1,8 +1,8 @@ import pytest from requests import PreparedRequest, Request -from fidesops.service.authentication.authentication_strategy_factory import get_strategy from fidesops.common_exceptions import ValidationError as FidesopsValidationError +from fidesops.service.authentication.authentication_strategy_factory import get_strategy def test_query_param_auth(): diff --git a/tests/service/connection_config/test_mariadb_connector.py b/tests/service/connection_config/test_mariadb_connector.py index 11976572c..ec5453a01 100644 --- a/tests/service/connection_config/test_mariadb_connector.py +++ b/tests/service/connection_config/test_mariadb_connector.py @@ -1,4 +1,5 @@ from sqlalchemy.orm import Session + from fidesops.service.connectors.sql_connector import MariaDBConnector diff --git a/tests/service/connection_config/test_mysql_connector.py b/tests/service/connection_config/test_mysql_connector.py index 7a62fe11a..9ccd21f40 100644 --- a/tests/service/connection_config/test_mysql_connector.py +++ b/tests/service/connection_config/test_mysql_connector.py @@ -1,4 +1,5 @@ from sqlalchemy.orm import Session + from fidesops.service.connectors.sql_connector import MySQLConnector diff --git a/tests/service/connectors/test_queryconfig.py b/tests/service/connectors/test_queryconfig.py index a7e3cc7f8..5acfceac8 100644 --- a/tests/service/connectors/test_queryconfig.py +++ b/tests/service/connectors/test_queryconfig.py @@ -1,16 +1,16 @@ import json -from typing import Dict, Any, Set, Optional +from typing import Any, Dict, Optional, Set import pytest +from fidesops.core.config import config from fidesops.graph.config import ( CollectionAddress, + FieldAddress, FieldPath, ObjectField, ScalarField, - FieldAddress, ) -from fidesops.core.config import config from fidesops.graph.graph import DatasetGraph, Edge from fidesops.graph.traversal import Traversal, TraversalNode from fidesops.models.datasetconfig import convert_dataset_to_graph @@ -18,22 +18,18 @@ from fidesops.schemas.dataset import FidesopsDataset from fidesops.schemas.masking.masking_configuration import HashMaskingConfiguration from fidesops.schemas.masking.masking_secrets import MaskingSecretCache, SecretType -from fidesops.schemas.saas.saas_config import SaaSConfig, ParamValue, SaaSRequest -from fidesops.schemas.saas.shared_schemas import SaaSRequestParams, HTTPMethod +from fidesops.schemas.saas.saas_config import ParamValue, SaaSConfig, SaaSRequest +from fidesops.schemas.saas.shared_schemas import HTTPMethod, SaaSRequestParams +from fidesops.service.connectors.query_config import MongoQueryConfig, SQLQueryConfig from fidesops.service.connectors.saas_query_config import SaaSQueryConfig -from fidesops.service.connectors.query_config import SQLQueryConfig, MongoQueryConfig - from fidesops.service.masking.strategy.masking_strategy_hash import ( - HashMaskingStrategy, HASH, + HashMaskingStrategy, ) from fidesops.util.data_category import DataCategory -from ...task.traversal_data import ( - integration_db_graph, - combined_mongo_postgresql_graph, -) -from ...test_helpers.cache_secrets_helper import clear_cache_secrets, cache_secret +from ...task.traversal_data import combined_mongo_postgresql_graph, integration_db_graph +from ...test_helpers.cache_secrets_helper import cache_secret, clear_cache_secrets # customers -> address, order # orders -> address, payment card diff --git a/tests/service/connectors/test_saas_connector.py b/tests/service/connectors/test_saas_connector.py index f03493ee0..a0d708fc6 100644 --- a/tests/service/connectors/test_saas_connector.py +++ b/tests/service/connectors/test_saas_connector.py @@ -1,14 +1,14 @@ -from requests import Response -from starlette.status import HTTP_404_NOT_FOUND, HTTP_200_OK -from sqlalchemy.orm import Session -import pytest import json +import pytest +from requests import Response +from sqlalchemy.orm import Session +from starlette.status import HTTP_200_OK, HTTP_404_NOT_FOUND -from fidesops.service.connectors.saas_connector import SaaSConnector -from fidesops.service.connectors import get_connector from fidesops.schemas.saas.saas_config import SaaSRequest from fidesops.schemas.saas.shared_schemas import HTTPMethod +from fidesops.service.connectors import get_connector +from fidesops.service.connectors.saas_connector import SaaSConnector @pytest.mark.unit_saas diff --git a/tests/service/masking/strategy/test_masking_strategy_aes_encrypt.py b/tests/service/masking/strategy/test_masking_strategy_aes_encrypt.py index 0918bbb34..e17fcce4f 100644 --- a/tests/service/masking/strategy/test_masking_strategy_aes_encrypt.py +++ b/tests/service/masking/strategy/test_masking_strategy_aes_encrypt.py @@ -6,10 +6,11 @@ ) from fidesops.schemas.masking.masking_secrets import MaskingSecretCache, SecretType from fidesops.service.masking.strategy.masking_strategy_aes_encrypt import ( - AesEncryptionMaskingStrategy, AES_ENCRYPT, + AesEncryptionMaskingStrategy, ) -from ....test_helpers.cache_secrets_helper import clear_cache_secrets, cache_secret + +from ....test_helpers.cache_secrets_helper import cache_secret, clear_cache_secrets request_id = "1345134" GCM_CONFIGURATION = AesEncryptionMaskingConfiguration( diff --git a/tests/service/masking/strategy/test_masking_strategy_factory.py b/tests/service/masking/strategy/test_masking_strategy_factory.py index 12f96adcb..15cc9832c 100644 --- a/tests/service/masking/strategy/test_masking_strategy_factory.py +++ b/tests/service/masking/strategy/test_masking_strategy_factory.py @@ -4,8 +4,8 @@ AesEncryptionMaskingStrategy, ) from fidesops.service.masking.strategy.masking_strategy_factory import ( - get_strategy, NoSuchStrategyException, + get_strategy, ) from fidesops.service.masking.strategy.masking_strategy_hash import HashMaskingStrategy from fidesops.service.masking.strategy.masking_strategy_string_rewrite import ( diff --git a/tests/service/masking/strategy/test_masking_strategy_hash.py b/tests/service/masking/strategy/test_masking_strategy_hash.py index 5a0a60c95..75be96955 100644 --- a/tests/service/masking/strategy/test_masking_strategy_hash.py +++ b/tests/service/masking/strategy/test_masking_strategy_hash.py @@ -1,10 +1,11 @@ from fidesops.schemas.masking.masking_configuration import HashMaskingConfiguration from fidesops.schemas.masking.masking_secrets import MaskingSecretCache, SecretType from fidesops.service.masking.strategy.masking_strategy_hash import ( - HashMaskingStrategy, HASH, + HashMaskingStrategy, ) -from ....test_helpers.cache_secrets_helper import clear_cache_secrets, cache_secret + +from ....test_helpers.cache_secrets_helper import cache_secret, clear_cache_secrets request_id = "1345134" diff --git a/tests/service/masking/strategy/test_masking_strategy_hmac.py b/tests/service/masking/strategy/test_masking_strategy_hmac.py index dfc6dae13..b45a36571 100644 --- a/tests/service/masking/strategy/test_masking_strategy_hmac.py +++ b/tests/service/masking/strategy/test_masking_strategy_hmac.py @@ -1,10 +1,11 @@ +from fidesops.schemas.masking.masking_configuration import HmacMaskingConfiguration from fidesops.schemas.masking.masking_secrets import MaskingSecretCache, SecretType from fidesops.service.masking.strategy.masking_strategy_hmac import ( - HmacMaskingStrategy, HMAC, + HmacMaskingStrategy, ) -from fidesops.schemas.masking.masking_configuration import HmacMaskingConfiguration -from ....test_helpers.cache_secrets_helper import clear_cache_secrets, cache_secret + +from ....test_helpers.cache_secrets_helper import cache_secret, clear_cache_secrets request_id = "1345134" diff --git a/tests/service/pagination/test_pagination_strategy_cursor.py b/tests/service/pagination/test_pagination_strategy_cursor.py index 14249076b..1ec0d6d8e 100644 --- a/tests/service/pagination/test_pagination_strategy_cursor.py +++ b/tests/service/pagination/test_pagination_strategy_cursor.py @@ -3,7 +3,8 @@ import pytest from requests import Response -from fidesops.schemas.saas.shared_schemas import SaaSRequestParams, HTTPMethod + +from fidesops.schemas.saas.shared_schemas import HTTPMethod, SaaSRequestParams from fidesops.schemas.saas.strategy_configuration import CursorPaginationConfiguration from fidesops.service.pagination.pagination_strategy_cursor import ( CursorPaginationStrategy, diff --git a/tests/service/pagination/test_pagination_strategy_factory.py b/tests/service/pagination/test_pagination_strategy_factory.py index f8bf2a99c..259af13ce 100644 --- a/tests/service/pagination/test_pagination_strategy_factory.py +++ b/tests/service/pagination/test_pagination_strategy_factory.py @@ -1,4 +1,5 @@ import pytest + from fidesops.common_exceptions import NoSuchStrategyException, ValidationError from fidesops.service.pagination.pagination_strategy_cursor import ( CursorPaginationStrategy, diff --git a/tests/service/pagination/test_pagination_strategy_link.py b/tests/service/pagination/test_pagination_strategy_link.py index 6e63b3ac0..ec65a4f5f 100644 --- a/tests/service/pagination/test_pagination_strategy_link.py +++ b/tests/service/pagination/test_pagination_strategy_link.py @@ -3,8 +3,9 @@ import pytest from requests import Response + +from fidesops.schemas.saas.shared_schemas import HTTPMethod, SaaSRequestParams from fidesops.schemas.saas.strategy_configuration import LinkPaginationConfiguration -from fidesops.schemas.saas.shared_schemas import SaaSRequestParams, HTTPMethod from fidesops.service.pagination.pagination_strategy_link import LinkPaginationStrategy diff --git a/tests/service/pagination/test_pagination_strategy_offset.py b/tests/service/pagination/test_pagination_strategy_offset.py index 26a4caf9f..59bef8e6d 100644 --- a/tests/service/pagination/test_pagination_strategy_offset.py +++ b/tests/service/pagination/test_pagination_strategy_offset.py @@ -3,12 +3,11 @@ import pytest from requests import Response -from fidesops.schemas.saas.saas_config import SaaSRequest -from fidesops.schemas.saas.shared_schemas import SaaSRequestParams, HTTPMethod + from fidesops.common_exceptions import FidesopsException -from fidesops.schemas.saas.strategy_configuration import ( - OffsetPaginationConfiguration, -) +from fidesops.schemas.saas.saas_config import SaaSRequest +from fidesops.schemas.saas.shared_schemas import HTTPMethod, SaaSRequestParams +from fidesops.schemas.saas.strategy_configuration import OffsetPaginationConfiguration from fidesops.service.pagination.pagination_strategy_offset import ( OffsetPaginationStrategy, ) diff --git a/tests/service/privacy_request/onetrust_test.py b/tests/service/privacy_request/onetrust_test.py index cf607e769..fdcc4ceaa 100644 --- a/tests/service/privacy_request/onetrust_test.py +++ b/tests/service/privacy_request/onetrust_test.py @@ -5,28 +5,23 @@ from sqlalchemy.orm import Session from fidesops.common_exceptions import ( + AuthenticationException, PolicyNotFoundException, StorageConfigNotFoundException, - AuthenticationException, ) from fidesops.models.client import ClientDetail -from fidesops.models.policy import ( - ActionType, - Policy, - Rule, - RuleTarget, -) +from fidesops.models.policy import ActionType, Policy, Rule, RuleTarget from fidesops.models.privacy_request import PrivacyRequest from fidesops.models.storage import StorageConfig +from fidesops.schemas.storage.storage import StorageDetails, StorageSecrets from fidesops.schemas.third_party.onetrust import ( OneTrustRequest, OneTrustSubtask, OneTrustSubtaskStatus, ) -from fidesops.schemas.storage.storage import StorageSecrets, StorageDetails from fidesops.service.privacy_request.onetrust_service import ( - ONETRUST_POLICY_KEY, FIDES_TASK, + ONETRUST_POLICY_KEY, OneTrustService, ) from fidesops.util.data_category import DataCategory diff --git a/tests/service/privacy_request/request_runner_service_test.py b/tests/service/privacy_request/request_runner_service_test.py index 42cad2411..77e5caaf0 100644 --- a/tests/service/privacy_request/request_runner_service_test.py +++ b/tests/service/privacy_request/request_runner_service_test.py @@ -1,4 +1,3 @@ -import pytest import time from typing import Any, Dict, List, Set from unittest import mock @@ -6,32 +5,32 @@ from uuid import uuid4 import pydash -from sqlalchemy import ( - column, - table, - select, -) -from sqlalchemy.orm import Session +import pytest from pydantic import ValidationError +from sqlalchemy import column, select, table +from sqlalchemy.orm import Session -from fidesops.common_exceptions import PrivacyRequestPaused, ClientUnsuccessfulException +from fidesops.common_exceptions import ClientUnsuccessfulException, PrivacyRequestPaused from fidesops.core.config import config -from fidesops.models.policy import PolicyPreWebhook, ActionType -from fidesops.models.privacy_request import PrivacyRequestStatus -from fidesops.schemas.external_https import SecondPartyResponseFormat from fidesops.db.session import get_db_session -from fidesops.models.privacy_request import PrivacyRequest, ExecutionLog +from fidesops.models.policy import ActionType, PolicyPreWebhook +from fidesops.models.privacy_request import ( + ExecutionLog, + PrivacyRequest, + PrivacyRequestStatus, +) +from fidesops.schemas.external_https import SecondPartyResponseFormat from fidesops.schemas.masking.masking_configuration import ( HmacMaskingConfiguration, MaskingConfiguration, ) from fidesops.schemas.masking.masking_secrets import MaskingSecretCache from fidesops.schemas.policy import Rule -from fidesops.schemas.saas.shared_schemas import SaaSRequestParams, HTTPMethod +from fidesops.schemas.saas.shared_schemas import HTTPMethod, SaaSRequestParams from fidesops.service.connectors.saas_connector import SaaSConnector from fidesops.service.connectors.sql_connector import ( - SnowflakeConnector, RedshiftConnector, + SnowflakeConnector, ) from fidesops.service.masking.strategy.masking_strategy_factory import get_strategy from fidesops.service.masking.strategy.masking_strategy_hmac import HmacMaskingStrategy diff --git a/tests/service/processors/post_processor_strategy/test_post_processor_strategy_filter.py b/tests/service/processors/post_processor_strategy/test_post_processor_strategy_filter.py index 62cf71e96..0cd07de0b 100644 --- a/tests/service/processors/post_processor_strategy/test_post_processor_strategy_filter.py +++ b/tests/service/processors/post_processor_strategy/test_post_processor_strategy_filter.py @@ -1,5 +1,7 @@ +from typing import Any, Dict + import pytest -from typing import Dict, Any + from fidesops.common_exceptions import FidesopsException from fidesops.schemas.saas.strategy_configuration import ( FilterPostProcessorConfiguration, diff --git a/tests/service/storage_uploader_service_test.py b/tests/service/storage_uploader_service_test.py index cebe513fc..97677627d 100644 --- a/tests/service/storage_uploader_service_test.py +++ b/tests/service/storage_uploader_service_test.py @@ -2,35 +2,32 @@ import os from datetime import datetime from io import BytesIO -from typing import Dict, Any, Generator +from typing import Any, Dict, Generator from unittest import mock from unittest.mock import Mock from zipfile import ZipFile -import pandas as pd +import pandas as pd import pytest from sqlalchemy.orm import Session +from fidesops.common_exceptions import StorageUploadError from fidesops.core.config import config from fidesops.models.privacy_request import PrivacyRequest from fidesops.models.storage import StorageConfig from fidesops.schemas.storage.storage import ( FileNaming, - StorageType, + ResponseFormat, StorageDetails, StorageSecrets, - ResponseFormat, -) -from fidesops.service.storage.storage_uploader_service import ( - upload, - get_extension, + StorageType, ) +from fidesops.service.storage.storage_uploader_service import get_extension, upload from fidesops.tasks.storage import ( - write_to_in_memory_buffer, LOCAL_FIDES_UPLOAD_DIRECTORY, encrypt_access_request_results, + write_to_in_memory_buffer, ) -from fidesops.common_exceptions import StorageUploadError from fidesops.util.encryption.aes_gcm_encryption_scheme import ( decrypt, decrypt_combined_nonce_and_message, diff --git a/tests/task/test_filter_element_match.py b/tests/task/test_filter_element_match.py index 3a5490627..33c70eaa5 100644 --- a/tests/task/test_filter_element_match.py +++ b/tests/task/test_filter_element_match.py @@ -5,8 +5,8 @@ from fidesops.graph.config import FieldPath from fidesops.task.filter_element_match import ( _expand_array_paths_to_preserve, - filter_element_match, _remove_paths_from_row, + filter_element_match, ) from fidesops.util.collection_util import FIDESOPS_DO_NOT_MASK_INDEX diff --git a/tests/task/test_filter_results.py b/tests/task/test_filter_results.py index 51ffbce94..760891ab0 100644 --- a/tests/task/test_filter_results.py +++ b/tests/task/test_filter_results.py @@ -3,11 +3,11 @@ from bson import ObjectId -from fidesops.graph.config import FieldPath, CollectionAddress +from fidesops.graph.config import CollectionAddress, FieldPath from fidesops.task.filter_results import ( - select_and_save_field, - remove_empty_containers, filter_data_categories, + remove_empty_containers, + select_and_save_field, ) diff --git a/tests/task/test_graph_task.py b/tests/task/test_graph_task.py index c08a814c5..1ea5ae81f 100644 --- a/tests/task/test_graph_task.py +++ b/tests/task/test_graph_task.py @@ -1,33 +1,25 @@ -import pytest - import dask +import pytest from bson import ObjectId -from fidesops.graph.config import ( - CollectionAddress, - FieldPath, -) +from fidesops.graph.config import CollectionAddress, FieldPath from fidesops.graph.graph import DatasetGraph from fidesops.graph.traversal import Traversal from fidesops.models.connectionconfig import ConnectionConfig, ConnectionType -from fidesops.models.policy import Policy, ActionType, RuleTarget, Rule +from fidesops.models.policy import ActionType, Policy, Rule, RuleTarget from fidesops.task.graph_task import ( - collect_queries, - TaskResources, EMPTY_REQUEST, + TaskResources, build_affected_field_logs, + collect_queries, ) + +from ..graph.graph_test_util import MockMongoTask, MockSqlTask, erasure_policy, field from .traversal_data import ( - sample_traversal, combined_mongo_postgresql_graph, + sample_traversal, traversal_paired_dependency, ) -from ..graph.graph_test_util import ( - MockSqlTask, - MockMongoTask, - field, - erasure_policy, -) dask.config.set(scheduler="processes") diff --git a/tests/task/test_refine_target_path.py b/tests/task/test_refine_target_path.py index f5ce00b66..d800a5381 100644 --- a/tests/task/test_refine_target_path.py +++ b/tests/task/test_refine_target_path.py @@ -1,9 +1,9 @@ from fidesops.graph.config import FieldPath from fidesops.task.refine_target_path import ( - refine_target_path, + _match_found, build_refined_target_paths, join_detailed_path, - _match_found, + refine_target_path, ) from fidesops.util.collection_util import FIDESOPS_DO_NOT_MASK_INDEX diff --git a/tests/task/traversal_data.py b/tests/task/traversal_data.py index 4815730fe..727a82e5a 100644 --- a/tests/task/traversal_data.py +++ b/tests/task/traversal_data.py @@ -2,19 +2,19 @@ from fidesops.graph.config import ( Collection, - ScalarField, - FieldAddress, CollectionAddress, Dataset, + FieldAddress, ObjectField, + ScalarField, ) from fidesops.graph.data_type import ( DataType, IntTypeConverter, - StringTypeConverter, + NoOpTypeConverter, ObjectIdTypeConverter, ObjectTypeConverter, - NoOpTypeConverter, + StringTypeConverter, ) from fidesops.graph.graph import DatasetGraph from fidesops.graph.traversal import Traversal diff --git a/tests/tasks/test_scheduled.py b/tests/tasks/test_scheduled.py index 36049d665..0483962ea 100644 --- a/tests/tasks/test_scheduled.py +++ b/tests/tasks/test_scheduled.py @@ -2,16 +2,14 @@ from apscheduler.triggers.date import DateTrigger from fidesops.models.privacy_request import PrivacyRequestStatus -from fidesops.schemas.storage.storage import ( - StorageDetails, -) +from fidesops.schemas.storage.storage import StorageDetails from fidesops.service.privacy_request.request_runner_service import ( initiate_paused_privacy_request_followup, ) from fidesops.tasks.scheduled.scheduler import scheduler from fidesops.tasks.scheduled.tasks import ( - initiate_scheduled_request_intake, ONETRUST_INTAKE_TASK, + initiate_scheduled_request_intake, ) diff --git a/tests/util/encryption/test_aes_gcm_encryption_scheme.py b/tests/util/encryption/test_aes_gcm_encryption_scheme.py index 4d60d62df..35f48209e 100644 --- a/tests/util/encryption/test_aes_gcm_encryption_scheme.py +++ b/tests/util/encryption/test_aes_gcm_encryption_scheme.py @@ -2,8 +2,8 @@ from fidesops.util import cryptographic_util from fidesops.util.encryption.aes_gcm_encryption_scheme import ( - encrypt_verify_secret_length, decrypt, + encrypt_verify_secret_length, ) NONCE = b"B\xab\x93&\x99u\x0c\xea\xe9\xb7\x8dU" diff --git a/tests/util/encryption/test_secrets_util.py b/tests/util/encryption/test_secrets_util.py index 7849013c2..c2df7e006 100644 --- a/tests/util/encryption/test_secrets_util.py +++ b/tests/util/encryption/test_secrets_util.py @@ -1,9 +1,9 @@ -from typing import List, Dict +from typing import Dict, List from fidesops.schemas.masking.masking_secrets import ( MaskingSecretCache, - SecretType, MaskingSecretMeta, + SecretType, ) from fidesops.service.masking.strategy.masking_strategy_aes_encrypt import AES_ENCRYPT from fidesops.service.masking.strategy.masking_strategy_hmac import ( @@ -11,6 +11,7 @@ HmacMaskingStrategy, ) from fidesops.util.encryption.secrets_util import SecretsUtil + from ...test_helpers.cache_secrets_helper import cache_secret, clear_cache_secrets request_id = "12345" diff --git a/tests/util/test_cache.py b/tests/util/test_cache.py index 5bf8d676d..4e53af3e1 100644 --- a/tests/util/test_cache.py +++ b/tests/util/test_cache.py @@ -1,8 +1,9 @@ import random -from typing import List, Any +from typing import Any, List from fidesops.core.config import config from fidesops.util.cache import FidesopsRedis + from ..fixtures.application_fixtures import faker diff --git a/tests/util/test_collection_util.py b/tests/util/test_collection_util.py index 055931e4b..e86923c35 100644 --- a/tests/util/test_collection_util.py +++ b/tests/util/test_collection_util.py @@ -2,9 +2,9 @@ from fidesops.util.collection_util import ( append, - partition, filter_nonempty_values, merge_dicts, + partition, ) diff --git a/tests/util/test_dataset_yaml.py b/tests/util/test_dataset_yaml.py index b73eab557..29c05c0d7 100644 --- a/tests/util/test_dataset_yaml.py +++ b/tests/util/test_dataset_yaml.py @@ -1,4 +1,4 @@ -from typing import Dict, Any +from typing import Any, Dict import pytest import yaml @@ -6,14 +6,15 @@ from fidesops.graph.config import ( CollectionAddress, - ScalarField, - ObjectField, FieldAddress, FieldPath, + ObjectField, + ScalarField, ) from fidesops.graph.graph import DatasetGraph, Edge from fidesops.models.datasetconfig import convert_dataset_to_graph from fidesops.schemas.dataset import FidesopsDataset + from ..graph.graph_test_util import field example_dataset_yaml = """dataset: diff --git a/tests/util/test_jwt_util.py b/tests/util/test_jwt_util.py index 5822e1848..e4d560ffd 100644 --- a/tests/util/test_jwt_util.py +++ b/tests/util/test_jwt_util.py @@ -4,11 +4,11 @@ from unittest.mock import Mock from fidesops.schemas.jwt import ( - JWE_PAYLOAD_CLIENT_ID, JWE_ISSUED_AT, + JWE_PAYLOAD_CLIENT_ID, JWE_PAYLOAD_SCOPES, ) -from fidesops.util.oauth_util import generate_jwe, extract_payload, is_token_expired +from fidesops.util.oauth_util import extract_payload, generate_jwe, is_token_expired def test_jwe_create_and_extract() -> None: diff --git a/tests/util/test_logger.py b/tests/util/test_logger.py index b520d37d7..2466d1bf3 100644 --- a/tests/util/test_logger.py +++ b/tests/util/test_logger.py @@ -1,8 +1,9 @@ import os + import pytest from fidesops.util import logger -from fidesops.util.logger import NotPii, MASKED +from fidesops.util.logger import MASKED, NotPii @pytest.fixture(scope="function") diff --git a/tests/util/test_saas_util.py b/tests/util/test_saas_util.py index 66e31a488..8dcad230f 100644 --- a/tests/util/test_saas_util.py +++ b/tests/util/test_saas_util.py @@ -1,6 +1,6 @@ -from fidesops.common_exceptions import FidesopsException import pytest +from fidesops.common_exceptions import FidesopsException from fidesops.graph.config import ( Collection, Dataset, @@ -9,7 +9,7 @@ ObjectField, ScalarField, ) -from fidesops.util.saas_util import assign_placeholders, unflatten_dict, merge_datasets +from fidesops.util.saas_util import assign_placeholders, merge_datasets, unflatten_dict class TestMergeDatasets: