From b018406db4fa4f84dd3da72794d9bde7d3ccff14 Mon Sep 17 00:00:00 2001 From: Thomas Date: Thu, 11 Aug 2022 07:58:36 -0700 Subject: [PATCH] Reorganize application code (#1058) * Reorganize application code * first round of moving everything and updating the import paths * checkpoint, more path updates * more path updates * fix imports * fix isort * fix mypy, isort and setup.py issues * fix unsafe checks build step running on any label * update the changelog * delete temp files * Update docs/fidesops/docs/development/contributing_details.md Co-authored-by: Paul Sanders * update more file references Co-authored-by: Paul Sanders --- .../workflows/fidesops_unsafe_pr_checks.yml | 1 + CHANGELOG.md | 1 + Dockerfile | 4 +- MANIFEST.in | 8 +- clients/ops/admin-ui/package.json | 2 +- .../docs/development/contributing_details.md | 31 +++--- .../docs/guides/masking_strategies.md | 64 ++++++------- docs/fidesops/docs/guides/oauth.md | 4 +- docs/fidesops/docs/guides/storage.md | 35 ++++--- noxfiles/ci_nox.py | 2 +- noxfiles/utils_nox.py | 2 +- pyproject.toml | 2 +- scripts/create_superuser.py | 6 +- scripts/create_test_data.py | 18 ++-- scripts/quickstart.py | 6 +- setup.py | 2 +- src/fidesops/db/base.py | 15 --- src/fidesops/main.py | 33 ++++--- src/fidesops/{api => ops}/__init__.py | 0 src/fidesops/{ => ops}/alembic.ini | 0 src/fidesops/{ => ops}/analytics.py | 2 +- src/fidesops/{api/v1 => ops/api}/__init__.py | 0 src/fidesops/{ => ops}/api/deps.py | 6 +- .../v1/endpoints => ops/api/v1}/__init__.py | 0 src/fidesops/{ => ops}/api/v1/api.py | 4 +- .../api/v1/endpoints}/__init__.py | 0 .../api/v1/endpoints/config_endpoints.py | 10 +- .../api/v1/endpoints/connection_endpoints.py | 34 ++++--- .../v1/endpoints/connection_type_endpoints.py | 16 ++-- .../api/v1/endpoints/dataset_endpoints.py | 26 ++--- .../api/v1/endpoints/drp_endpoints.py | 34 +++---- .../api/v1/endpoints/encryption_endpoints.py | 16 ++-- .../api/v1/endpoints/health_endpoints.py | 12 +-- .../api/v1/endpoints/masking_endpoints.py | 15 +-- .../api/v1/endpoints/oauth_endpoints.py | 28 +++--- .../api/v1/endpoints/policy_endpoints.py | 22 ++--- .../v1/endpoints/policy_webhook_endpoints.py | 24 ++--- .../v1/endpoints/privacy_request_endpoints.py | 55 ++++++----- .../api/v1/endpoints/saas_config_endpoints.py | 26 ++--- .../api/v1/endpoints/storage_endpoints.py | 38 ++++---- .../api/v1/endpoints/user_endpoints.py | 14 +-- .../v1/endpoints/user_permission_endpoints.py | 14 +-- .../{ => ops}/api/v1/exception_handlers.py | 2 +- .../{ => ops}/api/v1/scope_registry.py | 0 src/fidesops/{ => ops}/api/v1/urn_registry.py | 0 src/fidesops/{ => ops}/cli.py | 2 +- src/fidesops/{ => ops}/common_exceptions.py | 0 src/fidesops/{db => ops/core}/__init__.py | 0 src/fidesops/{ => ops}/core/config.py | 2 +- src/fidesops/{graph => ops/db}/__init__.py | 0 src/fidesops/ops/db/base.py | 15 +++ src/fidesops/{ => ops}/db/base_class.py | 0 src/fidesops/{ => ops}/db/database.py | 0 .../{migrations => ops/graph}/__init__.py | 0 .../{ => ops}/graph/analytics_events.py | 18 ++-- src/fidesops/{ => ops}/graph/config.py | 10 +- src/fidesops/{ => ops}/graph/data_type.py | 0 src/fidesops/{ => ops}/graph/graph.py | 6 +- .../{ => ops}/graph/graph_differences.py | 8 +- src/fidesops/{ => ops}/graph/traversal.py | 12 +-- src/fidesops/{ => ops}/migrations/README | 0 .../versions => ops/migrations}/__init__.py | 0 src/fidesops/{ => ops}/migrations/env.py | 2 +- .../{ => ops}/migrations/script.py.mako | 0 .../versions/0210948a8147_initial.py | 0 .../versions/07014ff34eb2_add_mariadb.py | 0 ...e7_add_saas_config_to_connection_config.py | 0 .../1ff88b7bd579_add_authorization_request.py | 0 .../27fe9da9d0f9_privacy_request_stopped.py | 0 ..._adds_first_name_and_last_name_to_user_.py | 0 .../3a7c5fb119c9_add_manual_connector_type.py | 0 ...3465d_adds_provided_identity_table_for_.py | 0 .../5078badb90b9_adds_drp_action_to_policy.py | 0 .../migrations/versions/530fb8533ca4_test.py | 0 .../55d61eb8ed12_add_default_policies.py | 16 ++-- ...cd643d7_fidesops_user_and_client_detail.py | 0 ...70db16d05_add_fidesops_user_permissions.py | 0 .../906d7198df28_privacy_request_approve.py | 0 .../migrations/versions}/__init__.py | 0 ...7c4a0_add_connection_config_description.py | 0 .../bab75915670a_add_finished_audit_log.py | 0 .../c3472d75c80e_connectionconfig_disabled.py | 0 .../c5336b841d70_add_policy_webhooks.py | 0 ...d4b_rename_user_name_tables_and_indexes.py | 0 ...add_user_login_and_password_reset_dates.py | 0 .../versions/c98da12d76f8_add_audit_log.py | 0 ...d65e7e921814_add_privacy_request_status.py | 0 .../versions/e55a51b354e3_add_bigquery.py | 0 .../ed1b00ff963d_cancel_privacy_request.py | 0 ...574d_add_redshift_and_snowflake_support.py | 0 .../versions/f3841942d90c_add_mssql.py | 0 ...bbcde_populate_saas_type_to_saas_config.py | 0 .../masking => ops/models}/__init__.py | 0 .../models/authentication_request.py | 0 .../{ => ops}/models/connectionconfig.py | 6 +- .../{ => ops}/models/datasetconfig.py | 14 +-- src/fidesops/{ => ops}/models/policy.py | 16 ++-- .../{ => ops}/models/privacy_request.py | 32 +++---- src/fidesops/{ => ops}/models/storage.py | 10 +- src/fidesops/{ => ops}/schemas/__init__.py | 0 src/fidesops/{ => ops}/schemas/analytics.py | 0 src/fidesops/{ => ops}/schemas/api.py | 0 src/fidesops/{ => ops}/schemas/base_class.py | 0 src/fidesops/{ => ops}/schemas/client.py | 2 +- .../connection_configuration/__init__.py | 26 ++--- .../connection_config.py | 8 +- .../connection_secrets.py | 4 +- .../connection_secrets_bigquery.py | 4 +- .../connection_secrets_mariadb.py | 4 +- .../connection_secrets_mongodb.py | 4 +- .../connection_secrets_mssql.py | 4 +- .../connection_secrets_mysql.py | 4 +- .../connection_secrets_postgres.py | 4 +- .../connection_secrets_redshift.py | 4 +- .../connection_secrets_saas.py | 2 +- .../connection_secrets_snowflake.py | 4 +- .../connections_secrets_https.py | 4 +- src/fidesops/{ => ops}/schemas/dataset.py | 14 +-- .../{ => ops}/schemas/drp_privacy_request.py | 4 +- .../{ => ops}/schemas/encryption_request.py | 4 +- .../{ => ops}/schemas/external_https.py | 4 +- .../saas => ops/schemas/masking}/__init__.py | 0 .../{ => ops}/schemas/masking/masking_api.py | 2 +- .../schemas/masking/masking_configuration.py | 0 .../schemas/masking/masking_secrets.py | 0 .../masking/masking_strategy_description.py | 0 src/fidesops/{ => ops}/schemas/msg.py | 0 src/fidesops/{ => ops}/schemas/policy.py | 14 +-- .../{ => ops}/schemas/policy_webhooks.py | 8 +- .../{ => ops}/schemas/privacy_request.py | 18 ++-- src/fidesops/{ => ops}/schemas/redis_cache.py | 2 +- .../storage => ops/schemas/saas}/__init__.py | 0 .../{ => ops}/schemas/saas/saas_config.py | 12 +-- .../{ => ops}/schemas/saas/shared_schemas.py | 0 .../schemas/saas/strategy_configuration.py | 4 +- .../{ => ops}/schemas/shared_schemas.py | 0 .../schemas/storage}/__init__.py | 0 .../storage/data_upload_location_response.py | 0 .../{ => ops}/schemas/storage/storage.py | 4 +- .../storage/storage_secrets_docs_only.py | 7 +- .../schemas/third_party}/__init__.py | 0 .../{ => ops}/schemas/third_party/onetrust.py | 2 +- .../{ => ops}/schemas/user_permission.py | 2 +- .../service}/__init__.py | 0 .../service/authentication}/__init__.py | 0 .../authentication/authentication_strategy.py | 4 +- .../authentication_strategy_basic.py | 8 +- .../authentication_strategy_bearer.py | 8 +- .../authentication_strategy_factory.py | 16 ++-- .../authentication_strategy_oauth2.py | 22 +++-- .../authentication_strategy_query_param.py | 10 +- .../{ => ops}/service/connectors/__init__.py | 14 +-- .../service/connectors/base_connector.py | 20 ++-- .../service/connectors/http_connector.py | 18 ++-- .../service/connectors/manual_connector.py | 14 +-- .../service/connectors/mongodb_connector.py | 18 ++-- .../service/connectors/query_config.py | 22 ++--- .../service/connectors/saas}/__init__.py | 0 .../connectors/saas/authenticated_client.py | 15 +-- .../service/connectors/saas_connector.py | 34 +++---- .../service/connectors/saas_query_config.py | 24 ++--- .../service/connectors/sql_connector.py | 22 ++--- .../strategy => ops/service/drp}/__init__.py | 0 .../service/drp/drp_fidesops_mapper.py | 11 ++- .../{ => ops}/service/masking/__init__.py | 2 +- .../service/masking/strategy}/__init__.py | 0 .../masking/strategy/format_preservation.py | 2 +- .../masking/strategy/masking_strategy.py | 6 +- .../strategy/masking_strategy_aes_encrypt.py | 20 ++-- .../strategy/masking_strategy_factory.py | 8 +- .../masking/strategy/masking_strategy_hash.py | 16 ++-- .../masking/strategy/masking_strategy_hmac.py | 16 ++-- .../strategy/masking_strategy_nullify.py | 8 +- .../masking_strategy_random_string_rewrite.py | 10 +- .../masking_strategy_string_rewrite.py | 10 +- .../service/outbound_urn_registry.py | 0 .../service/pagination}/__init__.py | 0 .../service/pagination/pagination_strategy.py | 4 +- .../pagination/pagination_strategy_cursor.py | 6 +- .../pagination/pagination_strategy_factory.py | 16 ++-- .../pagination/pagination_strategy_link.py | 6 +- .../pagination/pagination_strategy_offset.py | 8 +- .../service/privacy_request}/__init__.py | 0 .../privacy_request/onetrust_service.py | 24 ++--- .../privacy_request/request_runner_service.py | 37 ++++---- .../privacy_request/request_service.py | 12 +-- .../service/processors}/__init__.py | 0 .../post_processor_strategy}/__init__.py | 0 .../post_processor_strategy.py | 0 .../post_processor_strategy_factory.py | 12 +-- .../post_processor_strategy_filter.py | 8 +- .../post_processor_strategy_unwrap.py | 4 +- .../ops/service/saas_request/__init__.py | 3 + .../override_implementations}/__init__.py | 0 .../mailchimp_request_overrides.py | 17 ++-- .../saas_request_override_factory.py | 4 +- .../{task => ops/service/storage}/__init__.py | 0 .../storage/storage_authenticator_service.py | 4 +- .../storage/storage_uploader_service.py | 12 +-- .../{tasks/scheduled => ops/task}/__init__.py | 0 .../task/consolidate_query_matches.py | 4 +- .../{ => ops}/task/filter_element_match.py | 4 +- src/fidesops/{ => ops}/task/filter_results.py | 6 +- src/fidesops/{ => ops}/task/graph_task.py | 38 ++++---- .../{ => ops}/task/refine_target_path.py | 4 +- src/fidesops/{ => ops}/task/task_resources.py | 18 ++-- src/fidesops/{ => ops}/tasks/__init__.py | 10 +- .../{util => ops/tasks/scheduled}/__init__.py | 0 .../{ => ops}/tasks/scheduled/scheduler.py | 0 .../{ => ops}/tasks/scheduled/tasks.py | 12 +-- src/fidesops/{ => ops}/tasks/storage.py | 12 +-- .../{util/encryption => ops/util}/__init__.py | 0 src/fidesops/{ => ops}/util/api_router.py | 0 src/fidesops/{ => ops}/util/cache.py | 6 +- .../{ => ops}/util/collection_util.py | 0 src/fidesops/{ => ops}/util/data_category.py | 2 +- src/fidesops/ops/util/encryption/__init__.py | 0 .../encryption/aes_gcm_encryption_scheme.py | 2 +- .../util/encryption/hmac_encryption_scheme.py | 4 +- .../{ => ops}/util/encryption/secrets_util.py | 4 +- src/fidesops/{ => ops}/util/logger.py | 0 src/fidesops/{ => ops}/util/matching_queue.py | 0 src/fidesops/{ => ops}/util/oauth_util.py | 12 +-- src/fidesops/{ => ops}/util/querytoken.py | 0 src/fidesops/{ => ops}/util/saas_util.py | 8 +- .../{ => ops}/util/storage_authenticator.py | 2 +- src/fidesops/{ => ops}/util/text.py | 0 src/fidesops/{ => ops}/util/url_util.py | 0 src/fidesops/service/saas_request/__init__.py | 3 - tests/ops/api/test_deps.py | 6 +- .../api/v1/endpoints/test_config_endpoints.py | 4 +- .../test_connection_config_endpoints.py | 6 +- .../test_connection_template_endpoints.py | 10 +- .../v1/endpoints/test_dataset_endpoints.py | 12 +-- .../api/v1/endpoints/test_drp_endpoints.py | 21 ++-- .../v1/endpoints/test_encryption_endpoints.py | 10 +- .../api/v1/endpoints/test_health_endpoints.py | 2 +- .../v1/endpoints/test_masking_endpoints.py | 24 +++-- .../api/v1/endpoints/test_oauth_endpoints.py | 14 +-- .../api/v1/endpoints/test_policy_endpoints.py | 18 ++-- .../test_policy_webhook_endpoints.py | 8 +- .../test_privacy_request_endpoints.py | 76 +++++++-------- .../endpoints/test_saas_config_endpoints.py | 8 +- .../v1/endpoints/test_storage_endpoints.py | 34 +++---- .../api/v1/endpoints/test_user_endpoints.py | 6 +- .../test_user_permission_endpoints.py | 6 +- tests/ops/api/v1/test_exception_handlers.py | 11 ++- tests/ops/api/v1/test_main.py | 2 +- tests/ops/conftest.py | 16 ++-- tests/ops/core/test_config.py | 2 +- tests/ops/fixtures/application_fixtures.py | 28 +++--- tests/ops/fixtures/bigquery_fixtures.py | 8 +- tests/ops/fixtures/integration_fixtures.py | 4 +- tests/ops/fixtures/manual_fixtures.py | 4 +- tests/ops/fixtures/mariadb_fixtures.py | 8 +- tests/ops/fixtures/mongodb_fixtures.py | 6 +- tests/ops/fixtures/mssql_fixtures.py | 8 +- tests/ops/fixtures/mysql_fixtures.py | 8 +- tests/ops/fixtures/postgres_fixtures.py | 12 +-- tests/ops/fixtures/redshift_fixtures.py | 6 +- .../fixtures/saas/adobe_campaign_fixtures.py | 6 +- tests/ops/fixtures/saas/auth0_fixtures.py | 4 +- tests/ops/fixtures/saas/hubspot_fixtures.py | 10 +- tests/ops/fixtures/saas/mailchimp_fixtures.py | 10 +- tests/ops/fixtures/saas/outreach_fixtures.py | 6 +- .../mailchimp_override_fixtures.py | 10 +- .../ops/fixtures/saas/salesforce_fixtures.py | 6 +- tests/ops/fixtures/saas/segment_fixtures.py | 6 +- tests/ops/fixtures/saas/sendgrid_fixtures.py | 4 +- tests/ops/fixtures/saas/sentry_fixtures.py | 6 +- tests/ops/fixtures/saas/stripe_fixtures.py | 6 +- tests/ops/fixtures/saas/zendesk_fixtures.py | 6 +- tests/ops/fixtures/saas_example_fixtures.py | 10 +- tests/ops/fixtures/snowflake_fixtures.py | 6 +- tests/ops/generator/sql_data_generator.py | 6 +- tests/ops/generator/test_data_generator.py | 8 +- tests/ops/graph/graph_test_util.py | 22 ++--- tests/ops/graph/test_config.py | 6 +- tests/ops/graph/test_data_types.py | 2 +- tests/ops/graph/test_edge.py | 4 +- tests/ops/graph/test_graph.py | 12 +-- .../ops/graph/test_graph_analytics_events.py | 4 +- tests/ops/graph/test_graph_differences.py | 18 ++-- tests/ops/graph/test_graph_traversal.py | 2 +- tests/ops/graph/test_traversal_node.py | 2 +- .../test_mailchimp_override_task.py | 10 +- .../saas/test_adobe_campaign_task.py | 14 +-- .../integration_tests/saas/test_auth0_task.py | 12 +-- .../saas/test_hubspot_task.py | 18 ++-- .../saas/test_mailchimp_task.py | 10 +- .../saas/test_outreach_task.py | 14 +-- .../saas/test_salesforce_task.py | 14 +-- .../saas/test_segment_task.py | 14 +-- .../saas/test_sendgrid_task.py | 12 +-- .../saas/test_sentry_task.py | 12 +-- .../saas/test_stripe_task.py | 14 +-- .../saas/test_zendesk_task.py | 12 +-- .../setup_scripts/mariadb_setup.py | 6 +- .../setup_scripts/postgres_setup.py | 6 +- ...st_connection_configuration_integration.py | 16 ++-- tests/ops/integration_tests/test_execution.py | 32 ++++--- .../test_external_database_connections.py | 9 +- .../ops/integration_tests/test_manual_task.py | 12 +-- .../ops/integration_tests/test_mongo_task.py | 32 +++---- tests/ops/integration_tests/test_sql_task.py | 40 ++++---- tests/ops/models/test_base.py | 2 +- tests/ops/models/test_client.py | 4 +- tests/ops/models/test_connectionconfig.py | 6 +- tests/ops/models/test_datasetconfig.py | 4 +- .../models/test_fidesopsuserpermissions.py | 2 +- tests/ops/models/test_policy.py | 14 +-- tests/ops/models/test_policy_webhooks.py | 4 +- tests/ops/models/test_privacy_request.py | 17 ++-- tests/ops/models/test_saasconfig.py | 6 +- tests/ops/models/test_storage.py | 4 +- .../test_connection_secrets_saas.py | 4 +- tests/ops/schemas/test_user_permission.py | 4 +- tests/ops/scripts/test_create_superuser.py | 2 +- .../test_authentication_strategy_basic.py | 8 +- .../test_authentication_strategy_bearer.py | 8 +- .../test_authentication_strategy_factory.py | 12 ++- .../test_authentication_strategy_oauth2.py | 34 ++++--- ...est_authentication_strategy_query_param.py | 8 +- .../connection_config/test_http_connector.py | 4 +- .../test_mariadb_connector.py | 2 +- .../connection_config/test_mysql_connector.py | 2 +- .../test_postgres_connector.py | 4 +- .../service/connectors/test_queryconfig.py | 33 ++++--- .../service/connectors/test_saas_connector.py | 8 +- .../test_masking_strategy_aes_encrypt.py | 14 ++- .../strategy/test_masking_strategy_factory.py | 10 +- .../strategy/test_masking_strategy_hash.py | 6 +- .../strategy/test_masking_strategy_hmac.py | 6 +- .../strategy/test_masking_strategy_null.py | 4 +- ..._masking_strategy_random_string_rewrite.py | 4 +- .../test_masking_strategy_string_rewrite.py | 4 +- .../test_pagination_strategy_cursor.py | 8 +- .../test_pagination_strategy_factory.py | 12 ++- .../test_pagination_strategy_link.py | 8 +- .../test_pagination_strategy_offset.py | 12 ++- .../service/privacy_request/onetrust_test.py | 58 +++++------ .../request_runner_service_test.py | 95 +++++++++++-------- .../test_post_processor_strategy_factory.py | 8 +- .../test_post_processor_strategy_filter.py | 6 +- .../test_post_processor_strategy_unwrap.py | 4 +- .../test_saas_request_override_factory.py | 12 +-- .../service/storage_uploader_service_test.py | 28 +++--- .../task/test_consolidate_query_matches.py | 4 +- tests/ops/task/test_filter_element_match.py | 6 +- tests/ops/task/test_filter_results.py | 4 +- tests/ops/task/test_graph_task.py | 12 +-- tests/ops/task/test_refine_target_path.py | 6 +- tests/ops/task/test_task_resources.py | 2 +- tests/ops/task/traversal_data.py | 12 +-- tests/ops/tasks/test_celery.py | 4 +- tests/ops/tasks/test_scheduled.py | 10 +- .../ops/test_helpers/cache_secrets_helper.py | 8 +- tests/ops/test_helpers/dataset_utils.py | 18 ++-- tests/ops/test_helpers/vault_client.py | 2 +- .../test_aes_gcm_encryption_scheme.py | 2 +- .../encryption/test_hmac_encryption_scheme.py | 4 +- .../ops/util/encryption/test_secrets_util.py | 8 +- tests/ops/util/test_api_router.py | 4 +- tests/ops/util/test_cache.py | 4 +- tests/ops/util/test_collection_util.py | 2 +- tests/ops/util/test_dataset_yaml.py | 8 +- tests/ops/util/test_jwt_util.py | 2 +- tests/ops/util/test_logger.py | 4 +- tests/ops/util/test_matching_queue.py | 2 +- tests/ops/util/test_saas_util.py | 10 +- 370 files changed, 1668 insertions(+), 1532 deletions(-) delete mode 100644 src/fidesops/db/base.py rename src/fidesops/{api => ops}/__init__.py (100%) rename src/fidesops/{ => ops}/alembic.ini (100%) rename src/fidesops/{ => ops}/analytics.py (97%) rename src/fidesops/{api/v1 => ops/api}/__init__.py (100%) rename src/fidesops/{ => ops}/api/deps.py (83%) rename src/fidesops/{api/v1/endpoints => ops/api/v1}/__init__.py (100%) rename src/fidesops/{ => ops}/api/v1/api.py (93%) rename src/fidesops/{core => ops/api/v1/endpoints}/__init__.py (100%) rename src/fidesops/{ => ops}/api/v1/endpoints/config_endpoints.py (65%) rename src/fidesops/{ => ops}/api/v1/endpoints/connection_endpoints.py (92%) rename src/fidesops/{ => ops}/api/v1/endpoints/connection_type_endpoints.py (88%) rename src/fidesops/{ => ops}/api/v1/endpoints/dataset_endpoints.py (94%) rename src/fidesops/{ => ops}/api/v1/endpoints/drp_endpoints.py (86%) rename src/fidesops/{ => ops}/api/v1/endpoints/encryption_endpoints.py (82%) rename src/fidesops/{ => ops}/api/v1/endpoints/health_endpoints.py (86%) rename src/fidesops/{ => ops}/api/v1/endpoints/masking_endpoints.py (79%) rename src/fidesops/{ => ops}/api/v1/endpoints/oauth_endpoints.py (89%) rename src/fidesops/{ => ops}/api/v1/endpoints/policy_endpoints.py (95%) rename src/fidesops/{ => ops}/api/v1/endpoints/policy_webhook_endpoints.py (95%) rename src/fidesops/{ => ops}/api/v1/endpoints/privacy_request_endpoints.py (95%) rename src/fidesops/{ => ops}/api/v1/endpoints/saas_config_endpoints.py (91%) rename src/fidesops/{ => ops}/api/v1/endpoints/storage_endpoints.py (87%) rename src/fidesops/{ => ops}/api/v1/endpoints/user_endpoints.py (88%) rename src/fidesops/{ => ops}/api/v1/endpoints/user_permission_endpoints.py (88%) rename src/fidesops/{ => ops}/api/v1/exception_handlers.py (90%) rename src/fidesops/{ => ops}/api/v1/scope_registry.py (100%) rename src/fidesops/{ => ops}/api/v1/urn_registry.py (100%) rename src/fidesops/{ => ops}/cli.py (93%) rename src/fidesops/{ => ops}/common_exceptions.py (100%) rename src/fidesops/{db => ops/core}/__init__.py (100%) rename src/fidesops/{ => ops}/core/config.py (99%) rename src/fidesops/{graph => ops/db}/__init__.py (100%) create mode 100644 src/fidesops/ops/db/base.py rename src/fidesops/{ => ops}/db/base_class.py (100%) rename src/fidesops/{ => ops}/db/database.py (100%) rename src/fidesops/{migrations => ops/graph}/__init__.py (100%) rename src/fidesops/{ => ops}/graph/analytics_events.py (87%) rename src/fidesops/{ => ops}/graph/config.py (98%) rename src/fidesops/{ => ops}/graph/data_type.py (100%) rename src/fidesops/{ => ops}/graph/graph.py (98%) rename src/fidesops/{ => ops}/graph/graph_differences.py (97%) rename src/fidesops/{ => ops}/graph/traversal.py (97%) rename src/fidesops/{ => ops}/migrations/README (100%) rename src/fidesops/{migrations/versions => ops/migrations}/__init__.py (100%) rename src/fidesops/{ => ops}/migrations/env.py (97%) rename src/fidesops/{ => ops}/migrations/script.py.mako (100%) rename src/fidesops/{ => ops}/migrations/versions/0210948a8147_initial.py (100%) rename src/fidesops/{ => ops}/migrations/versions/07014ff34eb2_add_mariadb.py (100%) rename src/fidesops/{ => ops}/migrations/versions/1dfc5a2d30e7_add_saas_config_to_connection_config.py (100%) rename src/fidesops/{ => ops}/migrations/versions/1ff88b7bd579_add_authorization_request.py (100%) rename src/fidesops/{ => ops}/migrations/versions/27fe9da9d0f9_privacy_request_stopped.py (100%) rename src/fidesops/{ => ops}/migrations/versions/29a7d707163a_adds_first_name_and_last_name_to_user_.py (100%) rename src/fidesops/{ => ops}/migrations/versions/3a7c5fb119c9_add_manual_connector_type.py (100%) rename src/fidesops/{ => ops}/migrations/versions/3c5e1253465d_adds_provided_identity_table_for_.py (100%) rename src/fidesops/{ => ops}/migrations/versions/5078badb90b9_adds_drp_action_to_policy.py (100%) rename src/fidesops/{ => ops}/migrations/versions/530fb8533ca4_test.py (100%) rename src/fidesops/{ => ops}/migrations/versions/55d61eb8ed12_add_default_policies.py (96%) rename src/fidesops/{ => ops}/migrations/versions/5a966cd643d7_fidesops_user_and_client_detail.py (100%) rename src/fidesops/{ => ops}/migrations/versions/90070db16d05_add_fidesops_user_permissions.py (100%) rename src/fidesops/{ => ops}/migrations/versions/906d7198df28_privacy_request_approve.py (100%) rename src/fidesops/{models => ops/migrations/versions}/__init__.py (100%) rename src/fidesops/{ => ops}/migrations/versions/b3b68c87c4a0_add_connection_config_description.py (100%) rename src/fidesops/{ => ops}/migrations/versions/bab75915670a_add_finished_audit_log.py (100%) rename src/fidesops/{ => ops}/migrations/versions/c3472d75c80e_connectionconfig_disabled.py (100%) rename src/fidesops/{ => ops}/migrations/versions/c5336b841d70_add_policy_webhooks.py (100%) rename src/fidesops/{ => ops}/migrations/versions/c7cc36820d4b_rename_user_name_tables_and_indexes.py (100%) rename src/fidesops/{ => ops}/migrations/versions/c9759675b5d3_add_user_login_and_password_reset_dates.py (100%) rename src/fidesops/{ => ops}/migrations/versions/c98da12d76f8_add_audit_log.py (100%) rename src/fidesops/{ => ops}/migrations/versions/d65e7e921814_add_privacy_request_status.py (100%) rename src/fidesops/{ => ops}/migrations/versions/e55a51b354e3_add_bigquery.py (100%) rename src/fidesops/{ => ops}/migrations/versions/ed1b00ff963d_cancel_privacy_request.py (100%) rename src/fidesops/{ => ops}/migrations/versions/f206d4e7574d_add_redshift_and_snowflake_support.py (100%) rename src/fidesops/{ => ops}/migrations/versions/f3841942d90c_add_mssql.py (100%) rename src/fidesops/{ => ops}/migrations/versions/fc90277bbcde_populate_saas_type_to_saas_config.py (100%) rename src/fidesops/{schemas/masking => ops/models}/__init__.py (100%) rename src/fidesops/{ => ops}/models/authentication_request.py (100%) rename src/fidesops/{ => ops}/models/connectionconfig.py (96%) rename src/fidesops/{ => ops}/models/datasetconfig.py (94%) rename src/fidesops/{ => ops}/models/policy.py (97%) rename src/fidesops/{ => ops}/models/privacy_request.py (96%) rename src/fidesops/{ => ops}/models/storage.py (92%) rename src/fidesops/{ => ops}/schemas/__init__.py (100%) rename src/fidesops/{ => ops}/schemas/analytics.py (100%) rename src/fidesops/{ => ops}/schemas/api.py (100%) rename src/fidesops/{ => ops}/schemas/base_class.py (100%) rename src/fidesops/{ => ops}/schemas/client.py (69%) rename src/fidesops/{ => ops}/schemas/connection_configuration/__init__.py (67%) rename src/fidesops/{ => ops}/schemas/connection_configuration/connection_config.py (89%) rename src/fidesops/{ => ops}/schemas/connection_configuration/connection_secrets.py (94%) rename src/fidesops/{ => ops}/schemas/connection_configuration/connection_secrets_bigquery.py (86%) rename src/fidesops/{ => ops}/schemas/connection_configuration/connection_secrets_mariadb.py (81%) rename src/fidesops/{ => ops}/schemas/connection_configuration/connection_secrets_mongodb.py (79%) rename src/fidesops/{ => ops}/schemas/connection_configuration/connection_secrets_mssql.py (83%) rename src/fidesops/{ => ops}/schemas/connection_configuration/connection_secrets_mysql.py (80%) rename src/fidesops/{ => ops}/schemas/connection_configuration/connection_secrets_postgres.py (81%) rename src/fidesops/{ => ops}/schemas/connection_configuration/connection_secrets_redshift.py (81%) rename src/fidesops/{ => ops}/schemas/connection_configuration/connection_secrets_saas.py (97%) rename src/fidesops/{ => ops}/schemas/connection_configuration/connection_secrets_snowflake.py (85%) rename src/fidesops/{ => ops}/schemas/connection_configuration/connections_secrets_https.py (73%) rename src/fidesops/{ => ops}/schemas/dataset.py (95%) rename src/fidesops/{ => ops}/schemas/drp_privacy_request.py (94%) rename src/fidesops/{ => ops}/schemas/encryption_request.py (86%) rename src/fidesops/{ => ops}/schemas/external_https.py (92%) rename src/fidesops/{schemas/saas => ops/schemas/masking}/__init__.py (100%) rename src/fidesops/{ => ops}/schemas/masking/masking_api.py (86%) rename src/fidesops/{ => ops}/schemas/masking/masking_configuration.py (100%) rename src/fidesops/{ => ops}/schemas/masking/masking_secrets.py (100%) rename src/fidesops/{ => ops}/schemas/masking/masking_strategy_description.py (100%) rename src/fidesops/{ => ops}/schemas/msg.py (100%) rename src/fidesops/{ => ops}/schemas/policy.py (87%) rename src/fidesops/{ => ops}/schemas/policy_webhooks.py (89%) rename src/fidesops/{ => ops}/schemas/privacy_request.py (90%) rename src/fidesops/{ => ops}/schemas/redis_cache.py (85%) rename src/fidesops/{schemas/storage => ops/schemas/saas}/__init__.py (100%) rename src/fidesops/{ => ops}/schemas/saas/saas_config.py (96%) rename src/fidesops/{ => ops}/schemas/saas/shared_schemas.py (100%) rename src/fidesops/{ => ops}/schemas/saas/strategy_configuration.py (95%) rename src/fidesops/{ => ops}/schemas/shared_schemas.py (100%) rename src/fidesops/{schemas/third_party => ops/schemas/storage}/__init__.py (100%) rename src/fidesops/{ => ops}/schemas/storage/data_upload_location_response.py (100%) rename src/fidesops/{ => ops}/schemas/storage/storage.py (97%) rename src/fidesops/{ => ops}/schemas/storage/storage_secrets_docs_only.py (70%) rename src/fidesops/{service => ops/schemas/third_party}/__init__.py (100%) rename src/fidesops/{ => ops}/schemas/third_party/onetrust.py (98%) rename src/fidesops/{ => ops}/schemas/user_permission.py (95%) rename src/fidesops/{service/authentication => ops/service}/__init__.py (100%) rename src/fidesops/{service/connectors/saas => ops/service/authentication}/__init__.py (100%) rename src/fidesops/{ => ops}/service/authentication/authentication_strategy.py (78%) rename src/fidesops/{ => ops}/service/authentication/authentication_strategy_basic.py (81%) rename src/fidesops/{ => ops}/service/authentication/authentication_strategy_bearer.py (79%) rename src/fidesops/{ => ops}/service/authentication/authentication_strategy_factory.py (74%) rename src/fidesops/{ => ops}/service/authentication/authentication_strategy_oauth2.py (94%) rename src/fidesops/{ => ops}/service/authentication/authentication_strategy_query_param.py (77%) rename src/fidesops/{ => ops}/service/connectors/__init__.py (70%) rename src/fidesops/{ => ops}/service/connectors/base_connector.py (82%) rename src/fidesops/{ => ops}/service/connectors/http_connector.py (84%) rename src/fidesops/{ => ops}/service/connectors/manual_connector.py (89%) rename src/fidesops/{ => ops}/service/connectors/mongodb_connector.py (89%) rename src/fidesops/{ => ops}/service/connectors/query_config.py (97%) rename src/fidesops/{service/drp => ops/service/connectors/saas}/__init__.py (100%) rename src/fidesops/{ => ops}/service/connectors/saas/authenticated_client.py (89%) rename src/fidesops/{ => ops}/service/connectors/saas_connector.py (92%) rename src/fidesops/{ => ops}/service/connectors/saas_query_config.py (94%) rename src/fidesops/{ => ops}/service/connectors/sql_connector.py (95%) rename src/fidesops/{service/masking/strategy => ops/service/drp}/__init__.py (100%) rename src/fidesops/{ => ops}/service/drp/drp_fidesops_mapper.py (88%) rename src/fidesops/{ => ops}/service/masking/__init__.py (79%) rename src/fidesops/{service/pagination => ops/service/masking/strategy}/__init__.py (100%) rename src/fidesops/{ => ops}/service/masking/strategy/format_preservation.py (75%) rename src/fidesops/{ => ops}/service/masking/strategy/masking_strategy.py (85%) rename src/fidesops/{ => ops}/service/masking/strategy/masking_strategy_aes_encrypt.py (88%) rename src/fidesops/{ => ops}/service/masking/strategy/masking_strategy_factory.py (86%) rename src/fidesops/{ => ops}/service/masking/strategy/masking_strategy_hash.py (88%) rename src/fidesops/{ => ops}/service/masking/strategy/masking_strategy_hmac.py (87%) rename src/fidesops/{ => ops}/service/masking/strategy/masking_strategy_nullify.py (84%) rename src/fidesops/{ => ops}/service/masking/strategy/masking_strategy_random_string_rewrite.py (86%) rename src/fidesops/{ => ops}/service/masking/strategy/masking_strategy_string_rewrite.py (86%) rename src/fidesops/{ => ops}/service/outbound_urn_registry.py (100%) rename src/fidesops/{service/privacy_request => ops/service/pagination}/__init__.py (100%) rename src/fidesops/{ => ops}/service/pagination/pagination_strategy.py (87%) rename src/fidesops/{ => ops}/service/pagination/pagination_strategy_cursor.py (88%) rename src/fidesops/{ => ops}/service/pagination/pagination_strategy_factory.py (75%) rename src/fidesops/{ => ops}/service/pagination/pagination_strategy_link.py (91%) rename src/fidesops/{ => ops}/service/pagination/pagination_strategy_offset.py (91%) rename src/fidesops/{service/processors => ops/service/privacy_request}/__init__.py (100%) rename src/fidesops/{ => ops}/service/privacy_request/onetrust_service.py (92%) rename src/fidesops/{ => ops}/service/privacy_request/request_runner_service.py (92%) rename src/fidesops/{ => ops}/service/privacy_request/request_service.py (83%) rename src/fidesops/{service/processors/post_processor_strategy => ops/service/processors}/__init__.py (100%) rename src/fidesops/{service/saas_request/override_implementations => ops/service/processors/post_processor_strategy}/__init__.py (100%) rename src/fidesops/{ => ops}/service/processors/post_processor_strategy/post_processor_strategy.py (100%) rename src/fidesops/{ => ops}/service/processors/post_processor_strategy/post_processor_strategy_factory.py (76%) rename src/fidesops/{ => ops}/service/processors/post_processor_strategy/post_processor_strategy_filter.py (94%) rename src/fidesops/{ => ops}/service/processors/post_processor_strategy/post_processor_strategy_unwrap.py (94%) create mode 100644 src/fidesops/ops/service/saas_request/__init__.py rename src/fidesops/{service/storage => ops/service/saas_request/override_implementations}/__init__.py (100%) rename src/fidesops/{ => ops}/service/saas_request/override_implementations/mailchimp_request_overrides.py (91%) rename src/fidesops/{ => ops}/service/saas_request/saas_request_override_factory.py (98%) rename src/fidesops/{task => ops/service/storage}/__init__.py (100%) rename src/fidesops/{ => ops}/service/storage/storage_authenticator_service.py (94%) rename src/fidesops/{ => ops}/service/storage/storage_uploader_service.py (90%) rename src/fidesops/{tasks/scheduled => ops/task}/__init__.py (100%) rename src/fidesops/{ => ops}/task/consolidate_query_matches.py (93%) rename src/fidesops/{ => ops}/task/filter_element_match.py (97%) rename src/fidesops/{ => ops}/task/filter_results.py (97%) rename src/fidesops/{ => ops}/task/graph_task.py (96%) rename src/fidesops/{ => ops}/task/refine_target_path.py (98%) rename src/fidesops/{ => ops}/task/task_resources.py (93%) rename src/fidesops/{ => ops}/tasks/__init__.py (85%) rename src/fidesops/{util => ops/tasks/scheduled}/__init__.py (100%) rename src/fidesops/{ => ops}/tasks/scheduled/scheduler.py (100%) rename src/fidesops/{ => ops}/tasks/scheduled/tasks.py (82%) rename src/fidesops/{ => ops}/tasks/storage.py (94%) rename src/fidesops/{util/encryption => ops/util}/__init__.py (100%) rename src/fidesops/{ => ops}/util/api_router.py (100%) rename src/fidesops/{ => ops}/util/cache.py (97%) rename src/fidesops/{ => ops}/util/collection_util.py (100%) rename src/fidesops/{ => ops}/util/data_category.py (95%) create mode 100644 src/fidesops/ops/util/encryption/__init__.py rename src/fidesops/{ => ops}/util/encryption/aes_gcm_encryption_scheme.py (98%) rename src/fidesops/{ => ops}/util/encryption/hmac_encryption_scheme.py (93%) rename src/fidesops/{ => ops}/util/encryption/secrets_util.py (94%) rename src/fidesops/{ => ops}/util/logger.py (100%) rename src/fidesops/{ => ops}/util/matching_queue.py (100%) rename src/fidesops/{ => ops}/util/oauth_util.py (94%) rename src/fidesops/{ => ops}/util/querytoken.py (100%) rename src/fidesops/{ => ops}/util/saas_util.py (96%) rename src/fidesops/{ => ops}/util/storage_authenticator.py (93%) rename src/fidesops/{ => ops}/util/text.py (100%) rename src/fidesops/{ => ops}/util/url_util.py (100%) delete mode 100644 src/fidesops/service/saas_request/__init__.py diff --git a/.github/workflows/fidesops_unsafe_pr_checks.yml b/.github/workflows/fidesops_unsafe_pr_checks.yml index 748641cc0b..fe4b0de7e5 100644 --- a/.github/workflows/fidesops_unsafe_pr_checks.yml +++ b/.github/workflows/fidesops_unsafe_pr_checks.yml @@ -13,6 +13,7 @@ env: jobs: Build: runs-on: ubuntu-latest + if: contains(github.event.pull_request.labels.*.name, 'run unsafe ci checks') steps: - name: Checkout uses: actions/checkout@v3 diff --git a/CHANGELOG.md b/CHANGELOG.md index d2c32f61d7..a0aab263e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,7 @@ The types of changes are: ### Changed * Users should be able to click on the full field of a dropdown-type filter to open up the dropdown [#747](https://github.com/ethyca/fidesops/pull/903) +* Reorganize the core code logic to be easier to merge with fidesctl [#1058](https://github.com/ethyca/fidesops/pull/1058) * Updated the python docker base image from slim-buster to slim-bullseye [#928](https://github.com/ethyca/fidesops/pull/928) * Removed ipython from the docker install [#928](https://github.com/ethyca/fidesops/pull/928) * Run static nox checks outside of Docker [#1053](https://github.com/ethyca/fidesops/pull/1053) diff --git a/Dockerfile b/Dockerfile index 12ee6bbded..2bb2240ded 100644 --- a/Dockerfile +++ b/Dockerfile @@ -71,7 +71,7 @@ ENV PYTHONUNBUFFERED=TRUE ENV RUNNING_IN_DOCKER=true # Make a static files directory -RUN mkdir -p /fidesops/src/fidesops/build/static/ +RUN mkdir -p /fidesops/src/fidesops/ops/build/static/ EXPOSE 8080 CMD [ "fidesops", "webserver" ] @@ -93,4 +93,4 @@ RUN python setup.py sdist RUN pip install dist/fidesops-*.tar.gz # Copy frontend build over -COPY --from=frontend /fidesops/clients/ops/admin-ui/out/ /fidesops/src/fidesops/build/static/ +COPY --from=frontend /fidesops/clients/ops/admin-ui/out/ /fidesops/src/fidesops/ops/build/static/ diff --git a/MANIFEST.in b/MANIFEST.in index 010b6bde9f..009d6d4f8e 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,11 +3,11 @@ include README.md include requirements.txt include dev-requirements.txt include versioneer.py -include src/fidesops/alembic.ini +include src/fidesops/ops/alembic.ini include src/fidesops/_version.py include src/fidesops/py.typed -graft src/fidesops/migrations -exclude src/fidesops/migrations/README -exclude src/fidesops/migrations/script.py.mako +graft src/fidesops/ops/migrations +exclude src/fidesops/ops/migrations/README +exclude src/fidesops/ops/migrations/script.py.mako global-exclude *.pyc diff --git a/clients/ops/admin-ui/package.json b/clients/ops/admin-ui/package.json index 9c1be59297..ff6c62771b 100644 --- a/clients/ops/admin-ui/package.json +++ b/clients/ops/admin-ui/package.json @@ -16,7 +16,7 @@ "analyze:server": "cross-env BUNDLE_ANALYZE=server next build", "analyze:browser": "cross-env BUNDLE_ANALYZE=browser next build", "export": "next build && next export", - "copy-export": "rsync -a --delete out/ ../../../src/fidesops/build/static/", + "copy-export": "rsync -a --delete out/ ../../../src/fidesops/ops/build/static/", "prod-export": "npm run export && npm run copy-export" }, "dependencies": { diff --git a/docs/fidesops/docs/development/contributing_details.md b/docs/fidesops/docs/development/contributing_details.md index 2655972f56..2396ae4b37 100644 --- a/docs/fidesops/docs/development/contributing_details.md +++ b/docs/fidesops/docs/development/contributing_details.md @@ -5,13 +5,14 @@ ## API Endpoints ### Postman API Collection + Our [fidesops Postman Collection](../postman/Fidesops.postman_collection.json) can be used to test fidesops endpoints. Follow our [Using Postman](../postman/using_postman.md) guide to learn more about the how to use the collection. ### API URLs -We define API URLs for specific API versions as constants within `app.api.v1.urn_registry` (where `v1` can be substituted for that particular API version), then import those URLs into their specific API views. Since we are on the first version, there is no clear precedent set for overriding URLs between versions yet. The most likely change is that we'll override the `APIRouter` class instantiation with a different base path (ie. `/api/v2` instead of `/api/v1`). For example: +We define API URLs for specific API versions as constants within `fidesops.ops.api.v1.urn_registry` (where `v1` can be substituted for that particular API version), then import those URLs into their specific API views. Since we are on the first version, there is no clear precedent set for overriding URLs between versions yet. The most likely change is that we'll override the `APIRouter` class instantiation with a different base path (ie. `/api/v2` instead of `/api/v1`). For example: ``` PRIVACY_REQUEST = "/privacy-request" @@ -20,18 +21,17 @@ PRIVACY_REQUEST_DETAIL = "/privacy-request/{privacy_request_id}" would both resolve as `/api/v1/privacy-request` and `/api/v1/privacy-request/{privacy_request_id}` respectively. - ## Database and Models ### The ORM -- SQLAlchemy -SQLAlchemy is an Object Relational Mapper, allowing us to avoid writing direct database queries within our codebase, and access the database via Python code instead. The ORM provides an additional configuration layer allowing user-defined Python classes to be mapped to database tables and other constructs, as well as an object persistence mechanism known as the `Session`. Some common uses cases are listed below, for a more comprehensive guide see: https://docs.sqlalchemy.org/en/14/tutorial/index.html - +SQLAlchemy is an Object Relational Mapper, allowing us to avoid writing direct database queries within our codebase, and access the database via Python code instead. The ORM provides an additional configuration layer allowing user-defined Python classes to be mapped to database tables and other constructs, as well as an object persistence mechanism known as the `Session`. Some common uses cases are listed below, for a more comprehensive guide see: ### Adding models -Database tables are defined with model classes. Model files should live in `src/app/models/`. Individual model classes must inherit from our custom base class at `app.db.base_class.Base` to ensure uniformity within the database. Multiple models per file are encouraged so long as they fit the same logical delineation within the project. An example model declaration is added below. For a comprehensive guide see: https://docs.sqlalchemy.org/en/14/orm/mapping_styles.html#declarative-mapping +Database tables are defined with model classes. Model files should live in `src/app/models/`. Individual model classes must inherit from our custom base class at `app.db.base_class.Base` to ensure uniformity within the database. Multiple models per file are encouraged so long as they fit the same logical delineation within the project. An example model declaration is added below. For a comprehensive guide see: You should also import your model in src/fidesops/db/base.py so it is visible for alembic. + ``` class Book(Base): __tablename__ = 'book' @@ -41,20 +41,23 @@ class Book(Base): page_count = Column(Integer, nullable=True) author_id = Column(Integer, ForeignKey("author.id"), nullable=False) ``` -When models are added to the project, we must then add them to the database in a recordable and repeatable fashion using migrations. +When models are added to the project, we must then add them to the database in a recordable and repeatable fashion using migrations. ### Using the database via models -Once you've added database tables via project models, you're ready to read, write and update them via Python code. Some examples of common use cases here are listed below. Official documentation is here: https://docs.sqlalchemy.org/en/14/orm/query.html#sqlalchemy.orm.Query. +Once you've added database tables via project models, you're ready to read, write and update them via Python code. Some examples of common use cases here are listed below. Official documentation is here: . -- Import our application's database session: `from fidesops.db.session import get_db_session` +- Import our application's database session: `from fidesops.ops.db.session import get_db_session` - Instantiate the database interaction object: + ``` SessionLocal = get_db_session(config) db = SessionLocal() ``` + - Create a new row in a table: + ``` db_obj = User( email="admin@fidesops.app", @@ -66,10 +69,12 @@ db.add(db_obj) db.commit() db.refresh(db_obj) ``` + - Fetch all objects in a table: `users = db.query(User).all()` - Fetch all objects in a table that meet some criteria: `active_users = db.query(User).filter(User.is_active == True)` - Get a specific row in a table: `user = db.query(User).get(User.email == "admin@fidesops.app")` - Update a specific row in a table: + ``` user.email = "updated@fidesops.app" db.add(user) @@ -78,6 +83,7 @@ db.refresh() ``` ### Connecting to the database + When you run `nox -s dev`, the database will be spun up in a Docker container with port `5432` exposed on localhost. You can connect to it using the credentials found in `.fidesops.toml`, e.g. - Hostname: `localhost` @@ -85,10 +91,9 @@ When you run `nox -s dev`, the database will be spun up in a Docker container wi - Username: see `database.user` in `.fidesops.toml` - Password: see `database.password` in `.fidesops.toml` - ### Alembic migrations -Some common Alembic commands are listed below. For a comprehensive guide see: https://alembic.sqlalchemy.org/en/latest/tutorial.html. +Some common Alembic commands are listed below. For a comprehensive guide see: . The commands will need to be run inside a shell on your Docker containers, which can be opened with `nox -s dev -- shell`. @@ -99,7 +104,7 @@ In the `/src/fidesops` directory: - Automatically generate a new migration: `alembic revision --autogenerate -m ""` - Create a new migration file to manually fill out: `alembic revision -m ""` - Migrate your database to a specific state `alembic upgrade ` or `alembic downgrade `, (or if you want to be smart `alembic upgrade || alembic downgrade ` is handy when you don't know whether the target revision is an upgrade or downgrade) - - NB. You can find the `revision-id` inside each migration file in `alembic/versions/` on line 3 next to `Revision ID: ...` + - NB. You can find the `revision-id` inside each migration file in `alembic/versions/` on line 3 next to `Revision ID: ...` When working on a PR with a migration, ensure that `down_revision` in the generated migration file correctly references the previous migration before submitting/merging the PR. @@ -116,12 +121,10 @@ def some_method(): raise SomeException("a message") ``` - ## General debugging -- pdb The project uses `pdb` for debugging as a `dev-requirement`. You can set breakpoints with `pdb` in much the same way you'd set them using `debugger` in Javascript. Insert `import pdb; pdb.set_trace()` into the line where you want the breakpoint to set, then run your Python code. - ## Docker Occasionally when developing you'll run into issues where it's beneficial to remove all existing Docker instances in order to recreate them based on some updated spec. Some commands to do this are below: @@ -131,4 +134,4 @@ Occasionally when developing you'll run into issues where it's beneficial to rem - Delete all local Docker volumes: `docker volume rm $(docker volume ls -q)` - Remove temp. files, installed dependencies, all local Docker containers and all local Docker volumes: `nox -s clean` - Delete all stopped containers, all networks not used by a container, all dangling images, and all build cache: `docker system prune` -- Recreate the project: `nox -s "build(dev)"` \ No newline at end of file +- Recreate the project: `nox -s "build(dev)"` diff --git a/docs/fidesops/docs/guides/masking_strategies.md b/docs/fidesops/docs/guides/masking_strategies.md index d996f8b306..71bb1bfc0b 100644 --- a/docs/fidesops/docs/guides/masking_strategies.md +++ b/docs/fidesops/docs/guides/masking_strategies.md @@ -2,11 +2,11 @@ ## What is data masking? -Data masking is the process of obfuscating data in client systems, so it is no longer recognizable as PII (personally +Data masking is the process of obfuscating data in client systems, so it is no longer recognizable as PII (personally identifiable information.) For example, if a customer requests that your remove all information associated with their email, -`test@example.com`, you might choose to "mask" that email with a random string, `xgoi4301nkyi79fjfdopvyjc5lnbr9`, and +`test@example.com`, you might choose to "mask" that email with a random string, `xgoi4301nkyi79fjfdopvyjc5lnbr9`, and their associated address with another random string `2ab6jghdg37uhkaz3hpyavpss1dvg2`. It's important to remember that masking does not equal anonymization. Since records are not deleted, a masked dataset is (at best) @@ -15,27 +15,25 @@ pseudonymized in most cases, and (at worst) may still be identifiable if the mas In fidesops, your options to pseudonymize data are captured in "masking strategies". Fidesops supports a wide variety of masking strategies for different purposes when used directly as an API including HMAC, Hash, AES encryption, string rewrite, random string rewrite, and null rewrite. - ### Why mask instead of delete? -Deleting customer data may involve entirely deleting a whole record (all attributes of the entity) or permanent and +Deleting customer data may involve entirely deleting a whole record (all attributes of the entity) or permanent and irreversible anonymization of the record by updating specific fields within a record with masked values. -Using a masking strategy instead of straight deletion to obscure PII helps ensure referential integrity in your -database. For example, you might have an `orders` table with a foreign key to `user` without cascade delete. Say you first -deleted a user with email `test@example.com` without addressing their orders, you could potentially -have lingering orphans in the `orders` table. Using masking as a "soft delete" might be a safer strategy +Using a masking strategy instead of straight deletion to obscure PII helps ensure referential integrity in your +database. For example, you might have an `orders` table with a foreign key to `user` without cascade delete. Say you first +deleted a user with email `test@example.com` without addressing their orders, you could potentially +have lingering orphans in the `orders` table. Using masking as a "soft delete" might be a safer strategy depending on how your tables are defined. -In order to ensure referential integrity is retained, any values that represent foreign keys must be consistently +In order to ensure referential integrity is retained, any values that represent foreign keys must be consistently updated with the same masked values across all sources. Other reasons to mask instead of delete include legal requirements that have you retain certain data for a certain length of time. - ## Using fidesops as a masking service -If you just want to use fidesops as a masking service, you can send a `PUT` request to the masking endpoint with the +If you just want to use fidesops as a masking service, you can send a `PUT` request to the masking endpoint with the value(s) you'd like pseudonymized. This endpoint is also useful for getting a feel of how the different masking strategies work. ### Masking example @@ -66,7 +64,6 @@ The email has been replaced with a random string of 20 characters, while still p See the [masking values](/fidesops/api#operations-tag-Masking) API on how to use fidesops to as a masking service. - ## Configuration Erasure requests will mask data with the chosen masking strategy. @@ -112,7 +109,7 @@ Masks the input value with a default string value. - `rewrite_value`: `str` that will replace input values - `format_preservation` (optional): `Dict` with the following key/vals: - - `suffix`: `str` that specifies suffix to append to masked value + - `suffix`: `str` that specifies suffix to append to masked value ### Hash @@ -124,7 +121,7 @@ Masks the data by hashing the input before returning it. The hash is determinist - `algorithm` (optional): `str` that specifies Hash masking algorithm. Options include `SHA-512` or `SHA_256`. Default = `SHA_256` - `format_preservation` (optional): `Dict` with the following key/vals: - - `suffix`: `str` that specifies suffix to append to masked value + - `suffix`: `str` that specifies suffix to append to masked value ### Random string rewrite @@ -136,7 +133,7 @@ Masks the input value with a random string of a specified length. - `length` (optional): `int` that specifies length of randomly generated string. Default = `30` - `format_preservation` (optional): `Dict` with the following key/vals: - - `suffix`: `str` that specifies suffix to append to masked value + - `suffix`: `str` that specifies suffix to append to masked value ### AES encrypt @@ -148,7 +145,7 @@ Masks the data using AES encryption before returning it. The AES encryption stra - `mode` (optional): `str` that specifies AES encryption mode. Only supported option is `GCM`. Default = `GCM` - `format_preservation` (optional): `Dict` with the following key/vals: - - `suffix`: `str` that specifies suffix to append to masked value + - `suffix`: `str` that specifies suffix to append to masked value ### HMAC @@ -160,36 +157,32 @@ Masks the data using HMAC before returning it. The HMAC encryption strategy is d - `algorithm` (optional): `str` that specifies HMAC masking algorithm. Options include `SHA-512` or `SHA_256`. Default = `SHA_256` - `format_preservation` (optional): `Dict` with the following key/vals: - - `suffix`: `str` that specifies suffix to append to masked value + - `suffix`: `str` that specifies suffix to append to masked value - See the [Policy guide](policies.md) for more detailed instructions on creating Policies and Rules. - ## Getting masking options Issue a GET request to [`/api/v1/masking/strategy`](/fidesops/api#operations-Masking-list_masking_strategies_api_v1_masking_strategy_get) to preview the different masking -strategies available, along with their configuration options. - +strategies available, along with their configuration options. ## Extensibility -In fidesops, masking strategies are all built on top of an abstract base class - `MaskingStrategy`. -`MaskingStrategy` has five methods - `mask`, `secrets_required`, `get_configuration_model`, `get_description`, and `data_type_supported`. For more detail on these +In fidesops, masking strategies are all built on top of an abstract base class - `MaskingStrategy`. +`MaskingStrategy` has five methods - `mask`, `secrets_required`, `get_configuration_model`, `get_description`, and `data_type_supported`. For more detail on these methods, visit the class in the fidesops repository. For now, we will focus on the implementation of `RandomStringRewriteMaskingStrategy` below: - ```python import string from typing import Optional from secrets import choice -from fidesops.schemas.masking.masking_configuration import RandomStringMaskingConfiguration, MaskingConfiguration -from fidesops.schemas.masking.masking_strategy_description import MaskingStrategyDescription -from fidesops.service.masking.strategy.format_preservation import FormatPreservation -from fidesops.service.masking.strategy.masking_strategy import MaskingStrategy -from fidesops.service.masking.strategy.masking_strategy_factory import ( +from fidesops.ops.schemas.masking.masking_configuration import RandomStringMaskingConfiguration, MaskingConfiguration +from fidesops.ops.schemas.masking.masking_strategy_description import MaskingStrategyDescription +from fidesops.ops.service.masking.strategy.format_preservation import FormatPreservation +from fidesops.ops.service.masking.strategy.masking_strategy import MaskingStrategy +from fidesops.ops.service.masking.strategy.masking_strategy_factory import ( MaskingStrategyFactory, ) @@ -237,18 +230,17 @@ class RandomStringRewriteMaskingStrategy(MaskingStrategy): """Not covered in this example""" ``` -The `mask` method will be called with the list of values to be masked and the masked values will be the output. In this case, we want to replace the supplied values with a random mixture of ascii lowercase letters and digits of the -specified length. If format preservation is specified, for example, we still want to know that an email was an email, +The `mask` method will be called with the list of values to be masked and the masked values will be the output. In this case, we want to replace the supplied values with a random mixture of ascii lowercase letters and digits of the +specified length. If format preservation is specified, for example, we still want to know that an email was an email, we might tack on an email-like suffix. -Note the arguments to the __init__ method - there is a field configuration of type `RandomStringMaskingConfiguration`. -This is the configuration for the masking strategy. It is used to house the options specified by the client as well as -any defaults that should be applied in their absence. All configuration classes extend from the +Note the arguments to the __init__ method - there is a field configuration of type `RandomStringMaskingConfiguration`. +This is the configuration for the masking strategy. It is used to house the options specified by the client as well as +any defaults that should be applied in their absence. All configuration classes extend from the `MaskingConfiguration` class. - ### Integrate the masking strategy factory In order to leverage an implemented masking strategy, the `MaskingStrategy` subclass must be registered with the `MaskingStrategyFactory`. To register a new `MaskingStrategy`, use the `register` decorator on the `MaskingStrategy` subclass definition, as shown in the above example. -The value passed as the argument to the decorator must be the registered name of the `MaskingStrategy` subclass. This is the same value defined by [callers](#using-fidesops-as-a-masking-service) in the `"masking_strategy"."strategy"` field. \ No newline at end of file +The value passed as the argument to the decorator must be the registered name of the `MaskingStrategy` subclass. This is the same value defined by [callers](#using-fidesops-as-a-masking-service) in the `"masking_strategy"."strategy"` field. diff --git a/docs/fidesops/docs/guides/oauth.md b/docs/fidesops/docs/guides/oauth.md index bbb776b6cb..7a78f784a8 100644 --- a/docs/fidesops/docs/guides/oauth.md +++ b/docs/fidesops/docs/guides/oauth.md @@ -57,7 +57,7 @@ For this call, we have to populate the `Authorization` header. Notice that the h To add scopes to the client, the body of your request must contain an array of scope tokens. -You can retrieve the available scopes by calling [`GET /api/v1/oauth/scopes`](/fidesops/api#operations-OAuth-read_scopes_api_v1_oauth_scope_get), or you can look in [the scope registry file](https://github.com/ethyca/fidesops/blob/main/src/fidesops/api/v1/scope_registry.py). +You can retrieve the available scopes by calling [`GET /api/v1/oauth/scopes`](/fidesops/api#operations-OAuth-read_scopes_api_v1_oauth_scope_get), or you can look in [the scope registry file](https://github.com/ethyca/fidesops/blob/main/src/fidesops/ops/api/v1/scope_registry.py). If the call is successful, Fidesapi responds with a new client ID/client secret pair: @@ -85,5 +85,3 @@ If you call a fidesops API with an expired token, the call returns `401`. ### Other OAuth Calls Fidesops defines OAuth operations that let you delete a client, and read and write a client's scopes. See the [**OAuth** section of the **API** documentation](/fidesops/api#operations-tag-OAuth) for details. - - diff --git a/docs/fidesops/docs/guides/storage.md b/docs/fidesops/docs/guides/storage.md index 1a12dcb147..297ea9a4be 100644 --- a/docs/fidesops/docs/guides/storage.md +++ b/docs/fidesops/docs/guides/storage.md @@ -2,11 +2,11 @@ ## Overview -Access requests will produce a data package upon completion. This data will need to be uploaded to a storage destination (e.g. an S3 bucket). +Access requests will produce a data package upon completion. This data will need to be uploaded to a storage destination (e.g. an S3 bucket). Fidesops never stores privacy request results locally, so you’ll need to configure at least one storage destination if you wish to process Access requests. -Storage destinations are configured on Rules. +Storage destinations are configured on Rules. ![Storage Destinations](../img/storage_destinations.png "Storage Destinations") @@ -24,7 +24,7 @@ To configure a StorageConfig, you'll first need to choose a storage destination Let's get started. To create a new StorageConfig, use the following endpoint ([API docs here](/fidesops/api#operations-Storage-put_config_api_v1_storage_config_put)): -```json title="PATCH {host}/api/v1/storage/config" +```json title="PATCH {host}/api/v1/storage/config" { "destinations": [ { @@ -46,6 +46,7 @@ Let's get started. To create a new StorageConfig, use the following endpoint ([A } ``` + Params: - `name`: A unique user-friendly name for your storage destination. @@ -56,7 +57,7 @@ Params: Additional params needed for S3: - `bucket`: Name of bucket in S3. -- `naming`: This defines how the uploaded files will be named. Currently, fidesops only supports upload file naming by `request_id`. Use this value for all your storage destinations. +- `naming`: This defines how the uploaded files will be named. Currently, fidesops only supports upload file naming by `request_id`. Use this value for all your storage destinations. Additional params needed for OneTrust: @@ -68,8 +69,7 @@ Additional params needed for local: - `naming`: This defines how the uploaded files will be named. Currently, fidesops only supports upload file naming by `request_id`. Use this value for all your storage destinations. -On success, the response from the above endpoint will include a `storage_key` for each destination. - +On success, the response from the above endpoint will include a `storage_key` for each destination. ```json title="Example response" { @@ -98,7 +98,7 @@ Next, you'll need to authenticate secrets with the specific storage destination. Authentication is not needed for the `local` destination type. -Use the `storage_key` obtained from above in the following endpoint ([API docs here](/fidesops/api#operations-Storage-put_config_secrets_api_v1_storage_config__config_key__secret_put)): +Use the `storage_key` obtained from above in the following endpoint ([API docs here](/fidesops/api#operations-Storage-put_config_secrets_api_v1_storage_config__config_key__secret_put)): ```json title="PUT {host}/api/v1/storage/config/{storage_key}/secret" { @@ -115,14 +115,14 @@ Use the `storage_key` obtained from above in the following endpoint ([API docs h Params needed for S3: - - `aws_access_key_id`: AWS access key id, obtained from AWS console. - - `aws_secret_access_key`: AWS secret access key, obtained from AWS console. +- `aws_access_key_id`: AWS access key id, obtained from AWS console. +- `aws_secret_access_key`: AWS secret access key, obtained from AWS console. Params needed for OneTrust: - - `onetrust_hostname`: Your unique OneTrust hostname, used to call OneTrust REST APIs, e.g. `my-company.onetrust` - - `onetrust_client_id`: OneTrust client id, obtained from OneTrust portal. - - `onetrust_client_secret`: OneTrust client id, obtained from OneTrust portal. +- `onetrust_hostname`: Your unique OneTrust hostname, used to call OneTrust REST APIs, e.g. `my-company.onetrust` +- `onetrust_client_id`: OneTrust client id, obtained from OneTrust portal. +- `onetrust_client_secret`: OneTrust client id, obtained from OneTrust portal. Currently, we do not save the secrets if credentials fail authentication with the given storage destination. @@ -130,7 +130,7 @@ Currently, we do not save the secrets if credentials fail authentication with th To test that your storage destination works correctly, you may hit the upload endpoint directly, where `request_id` in the path is an arbitrary string. -Keep in mind that OneTrust destinations will need to be tested end-to-end, using the OneTrust interface to approve a test privacy request. +Keep in mind that OneTrust destinations will need to be tested end-to-end, using the OneTrust interface to approve a test privacy request. To upload data to a storage destination of choice ([api docs here](/fidesops/api#operations-Storage-upload_data_api_v1_storage__request_id__post)): @@ -149,11 +149,10 @@ Params: - `storage_key`: key associated with the storage destination - `data`: dict of arbitrary data you wish to upload to storage destination. - ## Extensibility Need a different storage destination? Fidesops can be extended to support additional storage destinations by: - -1. Add destination-specific enums in `src/fidesops/schemas/storage/storage.py` -2. Implement an authenticator in `src/fidesops/service/storage/storage_authenticator_service.py` -3. Implement the uploader in `src/fidesops/service/storage/storage_uploader_service.py` + +1. Add destination-specific enums in `src/fidesops/ops/schemas/storage/storage.py` +2. Implement an authenticator in `src/fidesops/ops/service/storage/storage_authenticator_service.py` +3. Implement the uploader in `src/fidesops/ops/service/storage/storage_uploader_service.py` diff --git a/noxfiles/ci_nox.py b/noxfiles/ci_nox.py index a50ed8cadd..ba5e021986 100644 --- a/noxfiles/ci_nox.py +++ b/noxfiles/ci_nox.py @@ -114,7 +114,7 @@ def check_migrations(session: nox.Session) -> None: check_migration_command = ( "python", "-c", - "from fidesops.db.database import check_missing_migrations; from fidesops.core.config import config; check_missing_migrations(config.database.sqlalchemy_database_uri);", + "from fidesops.ops.db.database import check_missing_migrations; from fidesops.ops.core.config import config; check_missing_migrations(config.database.sqlalchemy_database_uri);", ) session.run(*RUN, *check_migration_command, external=True) diff --git a/noxfiles/utils_nox.py b/noxfiles/utils_nox.py index a3bf25f6eb..3fe480216a 100644 --- a/noxfiles/utils_nox.py +++ b/noxfiles/utils_nox.py @@ -51,7 +51,7 @@ def db(session: nox.Session, db_command: str) -> None: init_command = ( "python", "-c", - "from fidesops.db.database import init_db; from fidesops.core.config import config; init_db(config.database.sqlalchemy_database_uri)", + "from fidesops.ops.db.database import init_db; from fidesops.ops.core.config import config; init_db(config.database.sqlalchemy_database_uri)", ) session.run(*RUN, *init_command, external=True) diff --git a/pyproject.toml b/pyproject.toml index ecea9bb0f3..f95d474ff1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ requires = ["setuptools", "wheel", "versioneer-518"] # PEP 508 specifications. check_untyped_defs = true disallow_untyped_defs = true files = ['src'] -exclude = ["fidesops.migrations.*"] +exclude = ["fidesops.ops.migrations.*"] plugins = ["pydantic.mypy", "sqlmypy"] pretty = true show_error_codes = true diff --git a/scripts/create_superuser.py b/scripts/create_superuser.py index 74559d6cd7..61d629c881 100644 --- a/scripts/create_superuser.py +++ b/scripts/create_superuser.py @@ -12,9 +12,9 @@ from fideslib.oauth.schemas.user import UserCreate from sqlalchemy.orm import Session -from fidesops.api.v1.scope_registry import CLIENT_CREATE, SCOPE_REGISTRY -from fidesops.core.config import config -from fidesops.db.database import init_db +from fidesops.ops.api.v1.scope_registry import CLIENT_CREATE, SCOPE_REGISTRY +from fidesops.ops.core.config import config +from fidesops.ops.db.database import init_db def get_username(prompt: str) -> str: diff --git a/scripts/create_test_data.py b/scripts/create_test_data.py index 5c55284590..026534522b 100644 --- a/scripts/create_test_data.py +++ b/scripts/create_test_data.py @@ -9,19 +9,19 @@ from fideslib.models.fides_user import FidesUser from sqlalchemy import orm -from fidesops.core.config import config -from fidesops.db.database import init_db -from fidesops.models.connectionconfig import ( +from fidesops.ops.core.config import config +from fidesops.ops.db.database import init_db +from fidesops.ops.models.connectionconfig import ( AccessLevel, ConnectionConfig, ConnectionType, ) -from fidesops.models.policy import ActionType, Policy, Rule, RuleTarget -from fidesops.models.privacy_request import PrivacyRequest, PrivacyRequestStatus -from fidesops.models.storage import ResponseFormat, StorageConfig -from fidesops.schemas.redis_cache import PrivacyRequestIdentity -from fidesops.schemas.storage.storage import FileNaming, StorageDetails, StorageType -from fidesops.util.data_category import DataCategory +from fidesops.ops.models.policy import ActionType, Policy, Rule, RuleTarget +from fidesops.ops.models.privacy_request import PrivacyRequest, PrivacyRequestStatus +from fidesops.ops.models.storage import ResponseFormat, StorageConfig +from fidesops.ops.schemas.redis_cache import PrivacyRequestIdentity +from fidesops.ops.schemas.storage.storage import FileNaming, StorageDetails, StorageType +from fidesops.ops.util.data_category import DataCategory def _create_policy( diff --git a/scripts/quickstart.py b/scripts/quickstart.py index 6f7ec110a1..d43fa131e9 100644 --- a/scripts/quickstart.py +++ b/scripts/quickstart.py @@ -13,9 +13,9 @@ import requests import yaml -from fidesops.core.config import config -from fidesops.models.connectionconfig import ConnectionType -from fidesops.models.policy import ActionType +from fidesops.ops.core.config import config +from fidesops.ops.models.connectionconfig import ConnectionType +from fidesops.ops.models.policy import ActionType logger = logging.getLogger(__name__) logging.basicConfig(level=logging.INFO) diff --git a/setup.py b/setup.py index 3d80ef0d33..9c6ba9de9f 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ long_description=long_description, long_description_content_type="text/markdown", url="https://github.com/ethyca/fidesops", - entry_points={"console_scripts": ["fidesops=fidesops.cli:cli"]}, + entry_points={"console_scripts": ["fidesops=fidesops.ops.cli:cli"]}, python_requires=">=3.7, <4", package_dir={"": "src"}, packages=find_packages(where="src"), diff --git a/src/fidesops/db/base.py b/src/fidesops/db/base.py deleted file mode 100644 index f5bb4e4458..0000000000 --- a/src/fidesops/db/base.py +++ /dev/null @@ -1,15 +0,0 @@ -# pylint: disable=W0611 -# Import all the models, so that Base has them before being -# imported by Alembic -from fideslib.db.base_class import Base -from fideslib.models.audit_log import AuditLog -from fideslib.models.client import ClientDetail -from fideslib.models.fides_user import FidesUser -from fideslib.models.fides_user_permissions import FidesUserPermissions - -from fidesops.models.authentication_request import AuthenticationRequest -from fidesops.models.connectionconfig import ConnectionConfig -from fidesops.models.datasetconfig import DatasetConfig -from fidesops.models.policy import Policy, Rule, RuleTarget -from fidesops.models.privacy_request import PrivacyRequest -from fidesops.models.storage import StorageConfig diff --git a/src/fidesops/main.py b/src/fidesops/main.py index f4f9394509..8e7d38a47e 100644 --- a/src/fidesops/main.py +++ b/src/fidesops/main.py @@ -18,23 +18,26 @@ from starlette.middleware.cors import CORSMiddleware from starlette.status import HTTP_404_NOT_FOUND -from fidesops.analytics import ( +from fidesops.ops.analytics import ( accessed_through_local_host, in_docker_container, send_analytics_event, ) -from fidesops.api.v1.api import api_router -from fidesops.api.v1.exception_handlers import ExceptionHandlers -from fidesops.api.v1.urn_registry import V1_URL_PREFIX -from fidesops.common_exceptions import FunctionalityNotConfigured, RedisConnectionError -from fidesops.core.config import config -from fidesops.db.database import init_db -from fidesops.schemas.analytics import Event, ExtraData -from fidesops.tasks.scheduled.scheduler import scheduler -from fidesops.tasks.scheduled.tasks import initiate_scheduled_request_intake -from fidesops.util.cache import get_cache -from fidesops.util.logger import get_fides_log_record_factory -from fidesops.util.oauth_util import get_db, verify_oauth_client +from fidesops.ops.api.v1.api import api_router +from fidesops.ops.api.v1.exception_handlers import ExceptionHandlers +from fidesops.ops.api.v1.urn_registry import V1_URL_PREFIX +from fidesops.ops.common_exceptions import ( + FunctionalityNotConfigured, + RedisConnectionError, +) +from fidesops.ops.core.config import config +from fidesops.ops.db.database import init_db +from fidesops.ops.schemas.analytics import Event, ExtraData +from fidesops.ops.tasks.scheduled.scheduler import scheduler +from fidesops.ops.tasks.scheduled.tasks import initiate_scheduled_request_intake +from fidesops.ops.util.cache import get_cache +from fidesops.ops.util.logger import get_fides_log_record_factory +from fidesops.ops.util.oauth_util import get_db, verify_oauth_client logging.basicConfig(level=config.security.log_level) logging.setLogRecordFactory(get_fides_log_record_factory()) @@ -126,7 +129,7 @@ def prepare_and_log_request( for handler in ExceptionHandlers.get_handlers(): app.add_exception_handler(FunctionalityNotConfigured, handler) -WEBAPP_DIRECTORY = Path("src/fidesops/build/static") +WEBAPP_DIRECTORY = Path("src/fidesops/ops/build/static") WEBAPP_INDEX = WEBAPP_DIRECTORY / "index.html" if config.admin_ui.enabled: @@ -138,7 +141,7 @@ def check_if_admin_ui_index_exists() -> None: with open( WEBAPP_DIRECTORY / "index.html", "w", encoding="utf-8" ) as index_file: - heading = "

No src/fidesops/build/static/index.html found

" + heading = "

No src/fidesops/ops/build/static/index.html found

" help_message = "

A docker-compose.yml volume may be overwriting the built in Admin UI files

" index_file.write(f"{heading}{help_message}") logger.info( diff --git a/src/fidesops/api/__init__.py b/src/fidesops/ops/__init__.py similarity index 100% rename from src/fidesops/api/__init__.py rename to src/fidesops/ops/__init__.py diff --git a/src/fidesops/alembic.ini b/src/fidesops/ops/alembic.ini similarity index 100% rename from src/fidesops/alembic.ini rename to src/fidesops/ops/alembic.ini diff --git a/src/fidesops/analytics.py b/src/fidesops/ops/analytics.py similarity index 97% rename from src/fidesops/analytics.py rename to src/fidesops/ops/analytics.py index 0533fd3164..7bb15a393e 100644 --- a/src/fidesops/analytics.py +++ b/src/fidesops/ops/analytics.py @@ -8,7 +8,7 @@ from fideslog.sdk.python.exceptions import AnalyticsError from fidesops import __version__ as fidesops_version -from fidesops.core.config import config +from fidesops.ops.core.config import config logger = logging.getLogger(__name__) diff --git a/src/fidesops/api/v1/__init__.py b/src/fidesops/ops/api/__init__.py similarity index 100% rename from src/fidesops/api/v1/__init__.py rename to src/fidesops/ops/api/__init__.py diff --git a/src/fidesops/api/deps.py b/src/fidesops/ops/api/deps.py similarity index 83% rename from src/fidesops/api/deps.py rename to src/fidesops/ops/api/deps.py index c9439ba0c5..40c2f00395 100644 --- a/src/fidesops/api/deps.py +++ b/src/fidesops/ops/api/deps.py @@ -2,9 +2,9 @@ from fideslib.db.session import get_db_session -from fidesops.common_exceptions import FunctionalityNotConfigured -from fidesops.core.config import config -from fidesops.util.cache import get_cache as get_redis_connection +from fidesops.ops.common_exceptions import FunctionalityNotConfigured +from fidesops.ops.core.config import config +from fidesops.ops.util.cache import get_cache as get_redis_connection def get_db() -> Generator: diff --git a/src/fidesops/api/v1/endpoints/__init__.py b/src/fidesops/ops/api/v1/__init__.py similarity index 100% rename from src/fidesops/api/v1/endpoints/__init__.py rename to src/fidesops/ops/api/v1/__init__.py diff --git a/src/fidesops/api/v1/api.py b/src/fidesops/ops/api/v1/api.py similarity index 93% rename from src/fidesops/api/v1/api.py rename to src/fidesops/ops/api/v1/api.py index d521447b58..92c52dcd5c 100644 --- a/src/fidesops/api/v1/api.py +++ b/src/fidesops/ops/api/v1/api.py @@ -1,4 +1,4 @@ -from fidesops.api.v1.endpoints import ( +from fidesops.ops.api.v1.endpoints import ( config_endpoints, connection_endpoints, connection_type_endpoints, @@ -16,7 +16,7 @@ user_endpoints, user_permission_endpoints, ) -from fidesops.util.api_router import APIRouter +from fidesops.ops.util.api_router import APIRouter api_router = APIRouter() api_router.include_router(config_endpoints.router) diff --git a/src/fidesops/core/__init__.py b/src/fidesops/ops/api/v1/endpoints/__init__.py similarity index 100% rename from src/fidesops/core/__init__.py rename to src/fidesops/ops/api/v1/endpoints/__init__.py diff --git a/src/fidesops/api/v1/endpoints/config_endpoints.py b/src/fidesops/ops/api/v1/endpoints/config_endpoints.py similarity index 65% rename from src/fidesops/api/v1/endpoints/config_endpoints.py rename to src/fidesops/ops/api/v1/endpoints/config_endpoints.py index 8fde37e8fb..c0db9442fe 100644 --- a/src/fidesops/api/v1/endpoints/config_endpoints.py +++ b/src/fidesops/ops/api/v1/endpoints/config_endpoints.py @@ -3,11 +3,11 @@ from fastapi.params import Security -from fidesops.api.v1 import scope_registry as scopes -from fidesops.api.v1 import urn_registry as urls -from fidesops.core.config import censored_config -from fidesops.util.api_router import APIRouter -from fidesops.util.oauth_util import verify_oauth_client +from fidesops.ops.api.v1 import scope_registry as scopes +from fidesops.ops.api.v1 import urn_registry as urls +from fidesops.ops.core.config import censored_config +from fidesops.ops.util.api_router import APIRouter +from fidesops.ops.util.oauth_util import verify_oauth_client router = APIRouter(tags=["Config"], prefix=urls.V1_URL_PREFIX) diff --git a/src/fidesops/api/v1/endpoints/connection_endpoints.py b/src/fidesops/ops/api/v1/endpoints/connection_endpoints.py similarity index 92% rename from src/fidesops/api/v1/endpoints/connection_endpoints.py rename to src/fidesops/ops/api/v1/endpoints/connection_endpoints.py index c628333068..37ad5ba60b 100644 --- a/src/fidesops/api/v1/endpoints/connection_endpoints.py +++ b/src/fidesops/ops/api/v1/endpoints/connection_endpoints.py @@ -20,13 +20,13 @@ HTTP_422_UNPROCESSABLE_ENTITY, ) -from fidesops.api import deps -from fidesops.api.v1.scope_registry import ( +from fidesops.ops.api import deps +from fidesops.ops.api.v1.scope_registry import ( CONNECTION_CREATE_OR_UPDATE, CONNECTION_DELETE, CONNECTION_READ, ) -from fidesops.api.v1.urn_registry import ( +from fidesops.ops.api.v1.urn_registry import ( CONNECTION_BY_KEY, CONNECTION_SECRETS, CONNECTION_TEST, @@ -34,30 +34,33 @@ SAAS_CONFIG, V1_URL_PREFIX, ) -from fidesops.common_exceptions import ClientUnsuccessfulException, ConnectionException -from fidesops.models.connectionconfig import ConnectionConfig, ConnectionType -from fidesops.schemas.api import BulkUpdateFailed -from fidesops.schemas.connection_configuration import ( +from fidesops.ops.common_exceptions import ( + ClientUnsuccessfulException, + ConnectionException, +) +from fidesops.ops.models.connectionconfig import ConnectionConfig, ConnectionType +from fidesops.ops.schemas.api import BulkUpdateFailed +from fidesops.ops.schemas.connection_configuration import ( connection_secrets_schemas, get_connection_secrets_validator, ) -from fidesops.schemas.connection_configuration.connection_config import ( +from fidesops.ops.schemas.connection_configuration.connection_config import ( BulkPutConnectionConfiguration, ConnectionConfigurationResponse, CreateConnectionConfiguration, SystemType, TestStatus, ) -from fidesops.schemas.connection_configuration.connection_secrets import ( +from fidesops.ops.schemas.connection_configuration.connection_secrets import ( ConnectionConfigSecretsSchema, ConnectionTestStatus, TestStatusMessage, ) -from fidesops.schemas.shared_schemas import FidesOpsKey -from fidesops.service.connectors import get_connector -from fidesops.util.api_router import APIRouter -from fidesops.util.logger import NotPii -from fidesops.util.oauth_util import verify_oauth_client +from fidesops.ops.schemas.shared_schemas import FidesOpsKey +from fidesops.ops.service.connectors import get_connector +from fidesops.ops.util.api_router import APIRouter +from fidesops.ops.util.logger import NotPii +from fidesops.ops.util.oauth_util import verify_oauth_client router = APIRouter(tags=["Connections"], prefix=V1_URL_PREFIX) @@ -101,7 +104,8 @@ def get_connections( Can also filter on disabled, connection_type, test_status, and system_type. Connection_type supports "or" filtering: - ?connection_type=postgres&connection_type=mongo will be translated into an "or" query. + ?connection_type=postgres&connection_type=mongo will be translated + into an "or" query. """ logger.info( f"Finding connection configurations with pagination params {params} and search query: '{search if search else ''}'." diff --git a/src/fidesops/api/v1/endpoints/connection_type_endpoints.py b/src/fidesops/ops/api/v1/endpoints/connection_type_endpoints.py similarity index 88% rename from src/fidesops/api/v1/endpoints/connection_type_endpoints.py rename to src/fidesops/ops/api/v1/endpoints/connection_type_endpoints.py index 553cbe8bc0..b3239c7b2d 100644 --- a/src/fidesops/api/v1/endpoints/connection_type_endpoints.py +++ b/src/fidesops/ops/api/v1/endpoints/connection_type_endpoints.py @@ -7,24 +7,24 @@ from fastapi_pagination.bases import AbstractPage from starlette.status import HTTP_404_NOT_FOUND -from fidesops.api.v1.scope_registry import CONNECTION_TYPE_READ -from fidesops.api.v1.urn_registry import ( +from fidesops.ops.api.v1.scope_registry import CONNECTION_TYPE_READ +from fidesops.ops.api.v1.urn_registry import ( CONNECTION_TYPE_SECRETS, CONNECTION_TYPES, V1_URL_PREFIX, ) -from fidesops.models.connectionconfig import ConnectionType -from fidesops.schemas.connection_configuration import ( +from fidesops.ops.models.connectionconfig import ConnectionType +from fidesops.ops.schemas.connection_configuration import ( SaaSSchemaFactory, secrets_validators, ) -from fidesops.schemas.connection_configuration.connection_config import ( +from fidesops.ops.schemas.connection_configuration.connection_config import ( ConnectionSystemTypeMap, SystemType, ) -from fidesops.schemas.saas.saas_config import SaaSConfig, SaaSType -from fidesops.util.oauth_util import verify_oauth_client -from fidesops.util.saas_util import load_config +from fidesops.ops.schemas.saas.saas_config import SaaSConfig, SaaSType +from fidesops.ops.util.oauth_util import verify_oauth_client +from fidesops.ops.util.saas_util import load_config router = APIRouter(tags=["Connection Types"], prefix=V1_URL_PREFIX) diff --git a/src/fidesops/api/v1/endpoints/dataset_endpoints.py b/src/fidesops/ops/api/v1/endpoints/dataset_endpoints.py similarity index 94% rename from src/fidesops/api/v1/endpoints/dataset_endpoints.py rename to src/fidesops/ops/api/v1/endpoints/dataset_endpoints.py index d479a02555..37bf8fd34b 100644 --- a/src/fidesops/api/v1/endpoints/dataset_endpoints.py +++ b/src/fidesops/ops/api/v1/endpoints/dataset_endpoints.py @@ -17,42 +17,42 @@ HTTP_415_UNSUPPORTED_MEDIA_TYPE, ) -from fidesops.api import deps -from fidesops.api.v1.scope_registry import ( +from fidesops.ops.api import deps +from fidesops.ops.api.v1.scope_registry import ( DATASET_CREATE_OR_UPDATE, DATASET_DELETE, DATASET_READ, ) -from fidesops.api.v1.urn_registry import ( +from fidesops.ops.api.v1.urn_registry import ( DATASET_BY_KEY, DATASET_VALIDATE, DATASETS, V1_URL_PREFIX, YAML_DATASETS, ) -from fidesops.common_exceptions import ( +from fidesops.ops.common_exceptions import ( SaaSConfigNotFoundException, TraversalError, ValidationError, ) -from fidesops.graph.traversal import DatasetGraph, Traversal -from fidesops.models.connectionconfig import ConnectionConfig, ConnectionType -from fidesops.models.datasetconfig import ( +from fidesops.ops.graph.traversal import DatasetGraph, Traversal +from fidesops.ops.models.connectionconfig import ConnectionConfig, ConnectionType +from fidesops.ops.models.datasetconfig import ( DatasetConfig, convert_dataset_to_graph, to_graph_field, ) -from fidesops.schemas.api import BulkUpdateFailed -from fidesops.schemas.dataset import ( +from fidesops.ops.schemas.api import BulkUpdateFailed +from fidesops.ops.schemas.dataset import ( BulkPutDataset, DatasetTraversalDetails, FidesopsDataset, ValidateDatasetResponse, ) -from fidesops.schemas.shared_schemas import FidesOpsKey -from fidesops.util.api_router import APIRouter -from fidesops.util.oauth_util import verify_oauth_client -from fidesops.util.saas_util import merge_datasets +from fidesops.ops.schemas.shared_schemas import FidesOpsKey +from fidesops.ops.util.api_router import APIRouter +from fidesops.ops.util.oauth_util import verify_oauth_client +from fidesops.ops.util.saas_util import merge_datasets X_YAML = "application/x-yaml" diff --git a/src/fidesops/api/v1/endpoints/drp_endpoints.py b/src/fidesops/ops/api/v1/endpoints/drp_endpoints.py similarity index 86% rename from src/fidesops/api/v1/endpoints/drp_endpoints.py rename to src/fidesops/ops/api/v1/endpoints/drp_endpoints.py index 6bc0a9a074..61256f87d7 100644 --- a/src/fidesops/api/v1/endpoints/drp_endpoints.py +++ b/src/fidesops/ops/api/v1/endpoints/drp_endpoints.py @@ -13,36 +13,36 @@ HTTP_500_INTERNAL_SERVER_ERROR, ) -from fidesops import common_exceptions -from fidesops.api import deps -from fidesops.api.v1 import scope_registry as scopes -from fidesops.api.v1 import urn_registry as urls -from fidesops.api.v1.endpoints.privacy_request_endpoints import ( +from fidesops.ops import common_exceptions +from fidesops.ops.api import deps +from fidesops.ops.api.v1 import scope_registry as scopes +from fidesops.ops.api.v1 import urn_registry as urls +from fidesops.ops.api.v1.endpoints.privacy_request_endpoints import ( get_privacy_request_or_error, ) -from fidesops.core.config import config -from fidesops.models.policy import DrpAction, Policy -from fidesops.models.privacy_request import PrivacyRequest, PrivacyRequestStatus -from fidesops.schemas.drp_privacy_request import ( +from fidesops.ops.core.config import config +from fidesops.ops.models.policy import DrpAction, Policy +from fidesops.ops.models.privacy_request import PrivacyRequest, PrivacyRequestStatus +from fidesops.ops.schemas.drp_privacy_request import ( DRP_VERSION, DrpDataRightsResponse, DrpIdentity, DrpPrivacyRequestCreate, DrpRevokeRequest, ) -from fidesops.schemas.privacy_request import PrivacyRequestDRPStatusResponse -from fidesops.schemas.redis_cache import PrivacyRequestIdentity -from fidesops.service.drp.drp_fidesops_mapper import DrpFidesopsMapper -from fidesops.service.privacy_request.request_runner_service import ( +from fidesops.ops.schemas.privacy_request import PrivacyRequestDRPStatusResponse +from fidesops.ops.schemas.redis_cache import PrivacyRequestIdentity +from fidesops.ops.service.drp.drp_fidesops_mapper import DrpFidesopsMapper +from fidesops.ops.service.privacy_request.request_runner_service import ( queue_privacy_request, ) -from fidesops.service.privacy_request.request_service import ( +from fidesops.ops.service.privacy_request.request_service import ( build_required_privacy_request_kwargs, cache_data, ) -from fidesops.util.api_router import APIRouter -from fidesops.util.cache import FidesopsRedis -from fidesops.util.oauth_util import verify_oauth_client +from fidesops.ops.util.api_router import APIRouter +from fidesops.ops.util.cache import FidesopsRedis +from fidesops.ops.util.oauth_util import verify_oauth_client logger = logging.getLogger(__name__) router = APIRouter(tags=["DRP"], prefix=urls.V1_URL_PREFIX) diff --git a/src/fidesops/api/v1/endpoints/encryption_endpoints.py b/src/fidesops/ops/api/v1/endpoints/encryption_endpoints.py similarity index 82% rename from src/fidesops/api/v1/endpoints/encryption_endpoints.py rename to src/fidesops/ops/api/v1/endpoints/encryption_endpoints.py index ac72020f11..4823aca5d4 100644 --- a/src/fidesops/api/v1/endpoints/encryption_endpoints.py +++ b/src/fidesops/ops/api/v1/endpoints/encryption_endpoints.py @@ -5,28 +5,28 @@ from fideslib.cryptography import cryptographic_util from fideslib.cryptography.cryptographic_util import b64_str_to_bytes, bytes_to_b64_str -from fidesops.api.v1.scope_registry import ENCRYPTION_EXEC -from fidesops.api.v1.urn_registry import ( +from fidesops.ops.api.v1.scope_registry import ENCRYPTION_EXEC +from fidesops.ops.api.v1.urn_registry import ( DECRYPT_AES, ENCRYPT_AES, ENCRYPTION_KEY, V1_URL_PREFIX, ) -from fidesops.core.config import config -from fidesops.schemas.encryption_request import ( +from fidesops.ops.core.config import config +from fidesops.ops.schemas.encryption_request import ( AesDecryptionRequest, AesDecryptionResponse, AesEncryptionRequest, AesEncryptionResponse, ) -from fidesops.util.api_router import APIRouter -from fidesops.util.encryption.aes_gcm_encryption_scheme import ( +from fidesops.ops.util.api_router import APIRouter +from fidesops.ops.util.encryption.aes_gcm_encryption_scheme import ( decrypt as aes_gcm_decrypt, ) -from fidesops.util.encryption.aes_gcm_encryption_scheme import ( +from fidesops.ops.util.encryption.aes_gcm_encryption_scheme import ( encrypt_verify_secret_length as aes_gcm_encrypt, ) -from fidesops.util.oauth_util import verify_oauth_client +from fidesops.ops.util.oauth_util import verify_oauth_client router = APIRouter(tags=["Encryption"], prefix=V1_URL_PREFIX) diff --git a/src/fidesops/api/v1/endpoints/health_endpoints.py b/src/fidesops/ops/api/v1/endpoints/health_endpoints.py similarity index 86% rename from src/fidesops/api/v1/endpoints/health_endpoints.py rename to src/fidesops/ops/api/v1/endpoints/health_endpoints.py index e29e6fd4d7..94a89ce25f 100644 --- a/src/fidesops/api/v1/endpoints/health_endpoints.py +++ b/src/fidesops/ops/api/v1/endpoints/health_endpoints.py @@ -5,12 +5,12 @@ from redis.exceptions import ResponseError from sqlalchemy import create_engine -from fidesops.api.v1.urn_registry import HEALTH -from fidesops.common_exceptions import RedisConnectionError -from fidesops.core.config import config -from fidesops.db.database import get_alembic_config -from fidesops.util.api_router import APIRouter -from fidesops.util.cache import get_cache +from fidesops.ops.api.v1.urn_registry import HEALTH +from fidesops.ops.common_exceptions import RedisConnectionError +from fidesops.ops.core.config import config +from fidesops.ops.db.database import get_alembic_config +from fidesops.ops.util.api_router import APIRouter +from fidesops.ops.util.cache import get_cache router = APIRouter(tags=["Public"]) diff --git a/src/fidesops/api/v1/endpoints/masking_endpoints.py b/src/fidesops/ops/api/v1/endpoints/masking_endpoints.py similarity index 79% rename from src/fidesops/api/v1/endpoints/masking_endpoints.py rename to src/fidesops/ops/api/v1/endpoints/masking_endpoints.py index d91c339e33..7ed78fee4e 100644 --- a/src/fidesops/api/v1/endpoints/masking_endpoints.py +++ b/src/fidesops/ops/api/v1/endpoints/masking_endpoints.py @@ -4,17 +4,20 @@ from fastapi import HTTPException from starlette.status import HTTP_400_BAD_REQUEST, HTTP_404_NOT_FOUND -from fidesops.api.v1.urn_registry import MASKING, MASKING_STRATEGY, V1_URL_PREFIX -from fidesops.common_exceptions import ValidationError -from fidesops.schemas.masking.masking_api import MaskingAPIRequest, MaskingAPIResponse -from fidesops.schemas.masking.masking_strategy_description import ( +from fidesops.ops.api.v1.urn_registry import MASKING, MASKING_STRATEGY, V1_URL_PREFIX +from fidesops.ops.common_exceptions import ValidationError +from fidesops.ops.schemas.masking.masking_api import ( + MaskingAPIRequest, + MaskingAPIResponse, +) +from fidesops.ops.schemas.masking.masking_strategy_description import ( MaskingStrategyDescription, ) -from fidesops.service.masking.strategy.masking_strategy_factory import ( +from fidesops.ops.service.masking.strategy.masking_strategy_factory import ( MaskingStrategyFactory, NoSuchStrategyException, ) -from fidesops.util.api_router import APIRouter +from fidesops.ops.util.api_router import APIRouter router = APIRouter(tags=["Masking"], prefix=V1_URL_PREFIX) diff --git a/src/fidesops/api/v1/endpoints/oauth_endpoints.py b/src/fidesops/ops/api/v1/endpoints/oauth_endpoints.py similarity index 89% rename from src/fidesops/api/v1/endpoints/oauth_endpoints.py rename to src/fidesops/ops/api/v1/endpoints/oauth_endpoints.py index 1f5a096395..d6e975d20e 100644 --- a/src/fidesops/api/v1/endpoints/oauth_endpoints.py +++ b/src/fidesops/ops/api/v1/endpoints/oauth_endpoints.py @@ -12,11 +12,11 @@ HTTP_422_UNPROCESSABLE_ENTITY, ) -from fidesops.api.deps import get_db -from fidesops.api.v1.endpoints.saas_config_endpoints import ( +from fidesops.ops.api.deps import get_db +from fidesops.ops.api.v1.endpoints.saas_config_endpoints import ( verify_oauth_connection_config, ) -from fidesops.api.v1.scope_registry import ( +from fidesops.ops.api.v1.scope_registry import ( CLIENT_CREATE, CLIENT_DELETE, CLIENT_READ, @@ -24,7 +24,7 @@ SCOPE_READ, SCOPE_REGISTRY, ) -from fidesops.api.v1.urn_registry import ( +from fidesops.ops.api.v1.urn_registry import ( CLIENT, CLIENT_BY_ID, CLIENT_SCOPE, @@ -33,21 +33,23 @@ TOKEN, V1_URL_PREFIX, ) -from fidesops.common_exceptions import ( +from fidesops.ops.common_exceptions import ( AuthenticationFailure, FidesopsException, OAuth2TokenException, ) -from fidesops.core.config import config -from fidesops.models.authentication_request import AuthenticationRequest -from fidesops.models.connectionconfig import ConnectionConfig -from fidesops.schemas.client import ClientCreatedResponse -from fidesops.service.authentication.authentication_strategy_factory import get_strategy -from fidesops.service.authentication.authentication_strategy_oauth2 import ( +from fidesops.ops.core.config import config +from fidesops.ops.models.authentication_request import AuthenticationRequest +from fidesops.ops.models.connectionconfig import ConnectionConfig +from fidesops.ops.schemas.client import ClientCreatedResponse +from fidesops.ops.service.authentication.authentication_strategy_factory import ( + get_strategy, +) +from fidesops.ops.service.authentication.authentication_strategy_oauth2 import ( OAuth2AuthenticationStrategy, ) -from fidesops.util.api_router import APIRouter -from fidesops.util.oauth_util import verify_oauth_client +from fidesops.ops.util.api_router import APIRouter +from fidesops.ops.util.oauth_util import verify_oauth_client router = APIRouter(tags=["OAuth"], prefix=V1_URL_PREFIX) diff --git a/src/fidesops/api/v1/endpoints/policy_endpoints.py b/src/fidesops/ops/api/v1/endpoints/policy_endpoints.py similarity index 95% rename from src/fidesops/api/v1/endpoints/policy_endpoints.py rename to src/fidesops/ops/api/v1/endpoints/policy_endpoints.py index bd46d77b71..d25fb5d966 100644 --- a/src/fidesops/api/v1/endpoints/policy_endpoints.py +++ b/src/fidesops/ops/api/v1/endpoints/policy_endpoints.py @@ -13,23 +13,23 @@ from starlette.exceptions import HTTPException from starlette.status import HTTP_200_OK, HTTP_204_NO_CONTENT, HTTP_404_NOT_FOUND -from fidesops.api import deps -from fidesops.api.v1 import scope_registry -from fidesops.api.v1 import urn_registry as urls -from fidesops.common_exceptions import ( +from fidesops.ops.api import deps +from fidesops.ops.api.v1 import scope_registry +from fidesops.ops.api.v1 import urn_registry as urls +from fidesops.ops.common_exceptions import ( DataCategoryNotSupported, DrpActionValidationError, PolicyValidationError, RuleTargetValidationError, RuleValidationError, ) -from fidesops.models.policy import ActionType, Policy, Rule, RuleTarget -from fidesops.models.storage import StorageConfig -from fidesops.schemas import policy as schemas -from fidesops.schemas.api import BulkUpdateFailed -from fidesops.schemas.shared_schemas import FidesOpsKey -from fidesops.util.api_router import APIRouter -from fidesops.util.oauth_util import verify_oauth_client +from fidesops.ops.models.policy import ActionType, Policy, Rule, RuleTarget +from fidesops.ops.models.storage import StorageConfig +from fidesops.ops.schemas import policy as schemas +from fidesops.ops.schemas.api import BulkUpdateFailed +from fidesops.ops.schemas.shared_schemas import FidesOpsKey +from fidesops.ops.util.api_router import APIRouter +from fidesops.ops.util.oauth_util import verify_oauth_client router = APIRouter(tags=["Policy"], prefix=urls.V1_URL_PREFIX) diff --git a/src/fidesops/api/v1/endpoints/policy_webhook_endpoints.py b/src/fidesops/ops/api/v1/endpoints/policy_webhook_endpoints.py similarity index 95% rename from src/fidesops/api/v1/endpoints/policy_webhook_endpoints.py rename to src/fidesops/ops/api/v1/endpoints/policy_webhook_endpoints.py index ddcd320488..51e4fa0c0b 100644 --- a/src/fidesops/api/v1/endpoints/policy_webhook_endpoints.py +++ b/src/fidesops/ops/api/v1/endpoints/policy_webhook_endpoints.py @@ -12,25 +12,25 @@ from starlette.exceptions import HTTPException from starlette.status import HTTP_200_OK, HTTP_400_BAD_REQUEST, HTTP_404_NOT_FOUND -from fidesops.api import deps -from fidesops.api.v1 import scope_registry as scopes -from fidesops.api.v1 import urn_registry as urls -from fidesops.api.v1.endpoints.connection_endpoints import ( +from fidesops.ops.api import deps +from fidesops.ops.api.v1 import scope_registry as scopes +from fidesops.ops.api.v1 import urn_registry as urls +from fidesops.ops.api.v1.endpoints.connection_endpoints import ( get_connection_config_or_error, ) -from fidesops.api.v1.endpoints.policy_endpoints import get_policy_or_error -from fidesops.common_exceptions import WebhookOrderException -from fidesops.models.policy import ( +from fidesops.ops.api.v1.endpoints.policy_endpoints import get_policy_or_error +from fidesops.ops.common_exceptions import WebhookOrderException +from fidesops.ops.models.policy import ( Policy, PolicyPostWebhook, PolicyPreWebhook, WebhookTypes, ) -from fidesops.schemas import policy_webhooks as schemas -from fidesops.schemas.policy_webhooks import PolicyWebhookDeleteResponse -from fidesops.schemas.shared_schemas import FidesOpsKey -from fidesops.util.api_router import APIRouter -from fidesops.util.oauth_util import verify_oauth_client +from fidesops.ops.schemas import policy_webhooks as schemas +from fidesops.ops.schemas.policy_webhooks import PolicyWebhookDeleteResponse +from fidesops.ops.schemas.shared_schemas import FidesOpsKey +from fidesops.ops.util.api_router import APIRouter +from fidesops.ops.util.oauth_util import verify_oauth_client router = APIRouter(tags=["Policy Webhooks"], prefix=urls.V1_URL_PREFIX) diff --git a/src/fidesops/api/v1/endpoints/privacy_request_endpoints.py b/src/fidesops/ops/api/v1/endpoints/privacy_request_endpoints.py similarity index 95% rename from src/fidesops/api/v1/endpoints/privacy_request_endpoints.py rename to src/fidesops/ops/api/v1/endpoints/privacy_request_endpoints.py index 46824d5fd4..3dcc5a2bfb 100644 --- a/src/fidesops/api/v1/endpoints/privacy_request_endpoints.py +++ b/src/fidesops/ops/api/v1/endpoints/privacy_request_endpoints.py @@ -26,16 +26,16 @@ HTTP_424_FAILED_DEPENDENCY, ) -from fidesops import common_exceptions -from fidesops.api import deps -from fidesops.api.v1 import scope_registry as scopes -from fidesops.api.v1 import urn_registry as urls -from fidesops.api.v1.scope_registry import ( +from fidesops.ops import common_exceptions +from fidesops.ops.api import deps +from fidesops.ops.api.v1 import scope_registry as scopes +from fidesops.ops.api.v1 import urn_registry as urls +from fidesops.ops.api.v1.scope_registry import ( PRIVACY_REQUEST_CALLBACK_RESUME, PRIVACY_REQUEST_READ, PRIVACY_REQUEST_REVIEW, ) -from fidesops.api.v1.urn_registry import ( +from fidesops.ops.api.v1.urn_registry import ( PRIVACY_REQUEST_APPROVE, PRIVACY_REQUEST_DENY, PRIVACY_REQUEST_MANUAL_ERASURE, @@ -44,27 +44,30 @@ PRIVACY_REQUEST_RETRY, REQUEST_PREVIEW, ) -from fidesops.common_exceptions import ( +from fidesops.ops.common_exceptions import ( FunctionalityNotConfigured, TraversalError, ValidationError, ) -from fidesops.core.config import config -from fidesops.graph.config import CollectionAddress -from fidesops.graph.graph import DatasetGraph, Node -from fidesops.graph.traversal import Traversal -from fidesops.models.connectionconfig import ConnectionConfig -from fidesops.models.datasetconfig import DatasetConfig -from fidesops.models.policy import PausedStep, Policy, PolicyPreWebhook -from fidesops.models.privacy_request import ( +from fidesops.ops.core.config import config +from fidesops.ops.graph.config import CollectionAddress +from fidesops.ops.graph.graph import DatasetGraph, Node +from fidesops.ops.graph.traversal import Traversal +from fidesops.ops.models.connectionconfig import ConnectionConfig +from fidesops.ops.models.datasetconfig import DatasetConfig +from fidesops.ops.models.policy import PausedStep, Policy, PolicyPreWebhook +from fidesops.ops.models.privacy_request import ( ExecutionLog, PrivacyRequest, PrivacyRequestStatus, ProvidedIdentity, ) -from fidesops.schemas.dataset import CollectionAddressResponse, DryRunDatasetResponse -from fidesops.schemas.external_https import PrivacyRequestResumeFormat -from fidesops.schemas.privacy_request import ( +from fidesops.ops.schemas.dataset import ( + CollectionAddressResponse, + DryRunDatasetResponse, +) +from fidesops.ops.schemas.external_https import PrivacyRequestResumeFormat +from fidesops.ops.schemas.privacy_request import ( BulkPostPrivacyRequests, BulkReviewResponse, DenyPrivacyRequests, @@ -76,19 +79,19 @@ RowCountRequest, StoppedCollection, ) -from fidesops.service.privacy_request.request_runner_service import ( +from fidesops.ops.service.privacy_request.request_runner_service import ( queue_privacy_request, ) -from fidesops.service.privacy_request.request_service import ( +from fidesops.ops.service.privacy_request.request_service import ( build_required_privacy_request_kwargs, cache_data, ) -from fidesops.task.graph_task import EMPTY_REQUEST, collect_queries -from fidesops.task.task_resources import TaskResources -from fidesops.util.api_router import APIRouter -from fidesops.util.cache import FidesopsRedis -from fidesops.util.collection_util import Row -from fidesops.util.oauth_util import verify_callback_oauth, verify_oauth_client +from fidesops.ops.task.graph_task import EMPTY_REQUEST, collect_queries +from fidesops.ops.task.task_resources import TaskResources +from fidesops.ops.util.api_router import APIRouter +from fidesops.ops.util.cache import FidesopsRedis +from fidesops.ops.util.collection_util import Row +from fidesops.ops.util.oauth_util import verify_callback_oauth, verify_oauth_client logger = logging.getLogger(__name__) router = APIRouter(tags=["Privacy Requests"], prefix=urls.V1_URL_PREFIX) diff --git a/src/fidesops/api/v1/endpoints/saas_config_endpoints.py b/src/fidesops/ops/api/v1/endpoints/saas_config_endpoints.py similarity index 91% rename from src/fidesops/api/v1/endpoints/saas_config_endpoints.py rename to src/fidesops/ops/api/v1/endpoints/saas_config_endpoints.py index f97bbafad7..a910a50e49 100644 --- a/src/fidesops/api/v1/endpoints/saas_config_endpoints.py +++ b/src/fidesops/ops/api/v1/endpoints/saas_config_endpoints.py @@ -12,34 +12,36 @@ HTTP_422_UNPROCESSABLE_ENTITY, ) -from fidesops.api import deps -from fidesops.api.v1.scope_registry import ( +from fidesops.ops.api import deps +from fidesops.ops.api.v1.scope_registry import ( CONNECTION_AUTHORIZE, SAAS_CONFIG_CREATE_OR_UPDATE, SAAS_CONFIG_DELETE, SAAS_CONFIG_READ, ) -from fidesops.api.v1.urn_registry import ( +from fidesops.ops.api.v1.urn_registry import ( AUTHORIZE, SAAS_CONFIG, SAAS_CONFIG_VALIDATE, V1_URL_PREFIX, ) -from fidesops.common_exceptions import FidesopsException -from fidesops.models.connectionconfig import ConnectionConfig, ConnectionType -from fidesops.models.datasetconfig import DatasetConfig -from fidesops.schemas.saas.saas_config import ( +from fidesops.ops.common_exceptions import FidesopsException +from fidesops.ops.models.connectionconfig import ConnectionConfig, ConnectionType +from fidesops.ops.models.datasetconfig import DatasetConfig +from fidesops.ops.schemas.saas.saas_config import ( SaaSConfig, SaaSConfigValidationDetails, ValidateSaaSConfigResponse, ) -from fidesops.schemas.shared_schemas import FidesOpsKey -from fidesops.service.authentication.authentication_strategy_factory import get_strategy -from fidesops.service.authentication.authentication_strategy_oauth2 import ( +from fidesops.ops.schemas.shared_schemas import FidesOpsKey +from fidesops.ops.service.authentication.authentication_strategy_factory import ( + get_strategy, +) +from fidesops.ops.service.authentication.authentication_strategy_oauth2 import ( OAuth2AuthenticationStrategy, ) -from fidesops.util.api_router import APIRouter -from fidesops.util.oauth_util import verify_oauth_client +from fidesops.ops.util.api_router import APIRouter +from fidesops.ops.util.oauth_util import verify_oauth_client router = APIRouter(tags=["SaaS Configs"], prefix=V1_URL_PREFIX) logger = logging.getLogger(__name__) diff --git a/src/fidesops/api/v1/endpoints/storage_endpoints.py b/src/fidesops/ops/api/v1/endpoints/storage_endpoints.py similarity index 87% rename from src/fidesops/api/v1/endpoints/storage_endpoints.py rename to src/fidesops/ops/api/v1/endpoints/storage_endpoints.py index d2c7097ed6..10557dfc4e 100644 --- a/src/fidesops/api/v1/endpoints/storage_endpoints.py +++ b/src/fidesops/ops/api/v1/endpoints/storage_endpoints.py @@ -19,40 +19,42 @@ HTTP_422_UNPROCESSABLE_ENTITY, ) -from fidesops.api import deps -from fidesops.api.v1.scope_registry import ( +from fidesops.ops.api import deps +from fidesops.ops.api.v1.scope_registry import ( STORAGE_CREATE_OR_UPDATE, STORAGE_DELETE, STORAGE_READ, ) -from fidesops.api.v1.urn_registry import ( +from fidesops.ops.api.v1.urn_registry import ( STORAGE_BY_KEY, STORAGE_CONFIG, STORAGE_SECRETS, STORAGE_UPLOAD, V1_URL_PREFIX, ) -from fidesops.common_exceptions import StorageUploadError -from fidesops.models.connectionconfig import ConnectionTestStatus -from fidesops.models.privacy_request import PrivacyRequest -from fidesops.models.storage import StorageConfig, get_schema_for_secrets -from fidesops.schemas.api import BulkUpdateFailed -from fidesops.schemas.connection_configuration.connection_secrets import ( +from fidesops.ops.common_exceptions import StorageUploadError +from fidesops.ops.models.connectionconfig import ConnectionTestStatus +from fidesops.ops.models.privacy_request import PrivacyRequest +from fidesops.ops.models.storage import StorageConfig, get_schema_for_secrets +from fidesops.ops.schemas.api import BulkUpdateFailed +from fidesops.ops.schemas.connection_configuration.connection_secrets import ( TestStatusMessage, ) -from fidesops.schemas.shared_schemas import FidesOpsKey -from fidesops.schemas.storage.data_upload_location_response import DataUpload -from fidesops.schemas.storage.storage import ( +from fidesops.ops.schemas.shared_schemas import FidesOpsKey +from fidesops.ops.schemas.storage.data_upload_location_response import DataUpload +from fidesops.ops.schemas.storage.storage import ( BulkPutStorageConfigResponse, StorageDestination, StorageDestinationResponse, ) -from fidesops.schemas.storage.storage_secrets_docs_only import possible_storage_secrets -from fidesops.service.storage.storage_authenticator_service import secrets_are_valid -from fidesops.service.storage.storage_uploader_service import upload -from fidesops.tasks.scheduled.tasks import initiate_scheduled_request_intake -from fidesops.util.api_router import APIRouter -from fidesops.util.oauth_util import verify_oauth_client +from fidesops.ops.schemas.storage.storage_secrets_docs_only import ( + possible_storage_secrets, +) +from fidesops.ops.service.storage.storage_authenticator_service import secrets_are_valid +from fidesops.ops.service.storage.storage_uploader_service import upload +from fidesops.ops.tasks.scheduled.tasks import initiate_scheduled_request_intake +from fidesops.ops.util.api_router import APIRouter +from fidesops.ops.util.oauth_util import verify_oauth_client router = APIRouter(tags=["Storage"], prefix=V1_URL_PREFIX) logger = logging.getLogger(__name__) diff --git a/src/fidesops/api/v1/endpoints/user_endpoints.py b/src/fidesops/ops/api/v1/endpoints/user_endpoints.py similarity index 88% rename from src/fidesops/api/v1/endpoints/user_endpoints.py rename to src/fidesops/ops/api/v1/endpoints/user_endpoints.py index 11bbae3a08..ecd3eabf0d 100644 --- a/src/fidesops/api/v1/endpoints/user_endpoints.py +++ b/src/fidesops/ops/api/v1/endpoints/user_endpoints.py @@ -13,13 +13,13 @@ HTTP_404_NOT_FOUND, ) -from fidesops.api import deps -from fidesops.api.v1 import urn_registry as urls -from fidesops.api.v1.scope_registry import USER_PASSWORD_RESET, USER_UPDATE -from fidesops.api.v1.urn_registry import V1_URL_PREFIX -from fidesops.core.config import config -from fidesops.util.api_router import APIRouter -from fidesops.util.oauth_util import get_current_user, verify_oauth_client +from fidesops.ops.api import deps +from fidesops.ops.api.v1 import urn_registry as urls +from fidesops.ops.api.v1.scope_registry import USER_PASSWORD_RESET, USER_UPDATE +from fidesops.ops.api.v1.urn_registry import V1_URL_PREFIX +from fidesops.ops.core.config import config +from fidesops.ops.util.api_router import APIRouter +from fidesops.ops.util.oauth_util import get_current_user, verify_oauth_client logger = logging.getLogger(__name__) router = APIRouter(tags=["Users"], prefix=V1_URL_PREFIX) diff --git a/src/fidesops/api/v1/endpoints/user_permission_endpoints.py b/src/fidesops/ops/api/v1/endpoints/user_permission_endpoints.py similarity index 88% rename from src/fidesops/api/v1/endpoints/user_permission_endpoints.py rename to src/fidesops/ops/api/v1/endpoints/user_permission_endpoints.py index 97dbbb27f5..644dfff059 100644 --- a/src/fidesops/api/v1/endpoints/user_permission_endpoints.py +++ b/src/fidesops/ops/api/v1/endpoints/user_permission_endpoints.py @@ -6,21 +6,21 @@ from sqlalchemy.orm import Session from starlette.status import HTTP_201_CREATED, HTTP_400_BAD_REQUEST, HTTP_404_NOT_FOUND -from fidesops.api import deps -from fidesops.api.v1 import urn_registry as urls -from fidesops.api.v1.scope_registry import ( +from fidesops.ops.api import deps +from fidesops.ops.api.v1 import urn_registry as urls +from fidesops.ops.api.v1.scope_registry import ( USER_PERMISSION_CREATE, USER_PERMISSION_READ, USER_PERMISSION_UPDATE, ) -from fidesops.api.v1.urn_registry import V1_URL_PREFIX -from fidesops.schemas.user_permission import ( +from fidesops.ops.api.v1.urn_registry import V1_URL_PREFIX +from fidesops.ops.schemas.user_permission import ( UserPermissionsCreate, UserPermissionsEdit, UserPermissionsResponse, ) -from fidesops.util.api_router import APIRouter -from fidesops.util.oauth_util import verify_oauth_client +from fidesops.ops.util.api_router import APIRouter +from fidesops.ops.util.oauth_util import verify_oauth_client logger = logging.getLogger(__name__) router = APIRouter(tags=["User Permissions"], prefix=V1_URL_PREFIX) diff --git a/src/fidesops/api/v1/exception_handlers.py b/src/fidesops/ops/api/v1/exception_handlers.py similarity index 90% rename from src/fidesops/api/v1/exception_handlers.py rename to src/fidesops/ops/api/v1/exception_handlers.py index ca2a016633..bd59602eaa 100644 --- a/src/fidesops/api/v1/exception_handlers.py +++ b/src/fidesops/ops/api/v1/exception_handlers.py @@ -4,7 +4,7 @@ from fastapi.responses import JSONResponse from starlette.status import HTTP_500_INTERNAL_SERVER_ERROR -from fidesops.common_exceptions import FunctionalityNotConfigured +from fidesops.ops.common_exceptions import FunctionalityNotConfigured class ExceptionHandlers: diff --git a/src/fidesops/api/v1/scope_registry.py b/src/fidesops/ops/api/v1/scope_registry.py similarity index 100% rename from src/fidesops/api/v1/scope_registry.py rename to src/fidesops/ops/api/v1/scope_registry.py diff --git a/src/fidesops/api/v1/urn_registry.py b/src/fidesops/ops/api/v1/urn_registry.py similarity index 100% rename from src/fidesops/api/v1/urn_registry.py rename to src/fidesops/ops/api/v1/urn_registry.py diff --git a/src/fidesops/cli.py b/src/fidesops/ops/cli.py similarity index 93% rename from src/fidesops/cli.py rename to src/fidesops/ops/cli.py index 757d940963..283d02fbab 100644 --- a/src/fidesops/cli.py +++ b/src/fidesops/ops/cli.py @@ -4,7 +4,7 @@ import click from fidesops.main import start_webserver -from fidesops.tasks import start_worker +from fidesops.ops.tasks import start_worker logger = logging.getLogger(__name__) diff --git a/src/fidesops/common_exceptions.py b/src/fidesops/ops/common_exceptions.py similarity index 100% rename from src/fidesops/common_exceptions.py rename to src/fidesops/ops/common_exceptions.py diff --git a/src/fidesops/db/__init__.py b/src/fidesops/ops/core/__init__.py similarity index 100% rename from src/fidesops/db/__init__.py rename to src/fidesops/ops/core/__init__.py diff --git a/src/fidesops/core/config.py b/src/fidesops/ops/core/config.py similarity index 99% rename from src/fidesops/core/config.py rename to src/fidesops/ops/core/config.py index 078a518742..72708c1930 100644 --- a/src/fidesops/core/config.py +++ b/src/fidesops/ops/core/config.py @@ -16,7 +16,7 @@ from fideslog.sdk.python.utils import FIDESOPS, generate_client_id from pydantic import validator -from fidesops.util.logger import NotPii +from fidesops.ops.util.logger import NotPii logger = logging.getLogger(__name__) diff --git a/src/fidesops/graph/__init__.py b/src/fidesops/ops/db/__init__.py similarity index 100% rename from src/fidesops/graph/__init__.py rename to src/fidesops/ops/db/__init__.py diff --git a/src/fidesops/ops/db/base.py b/src/fidesops/ops/db/base.py new file mode 100644 index 0000000000..bc76d5d320 --- /dev/null +++ b/src/fidesops/ops/db/base.py @@ -0,0 +1,15 @@ +# pylint: disable=W0611 +# Import all the models, so that Base has them before being +# imported by Alembic +from fideslib.db.base_class import Base +from fideslib.models.audit_log import AuditLog +from fideslib.models.client import ClientDetail +from fideslib.models.fides_user import FidesUser +from fideslib.models.fides_user_permissions import FidesUserPermissions + +from fidesops.ops.models.authentication_request import AuthenticationRequest +from fidesops.ops.models.connectionconfig import ConnectionConfig +from fidesops.ops.models.datasetconfig import DatasetConfig +from fidesops.ops.models.policy import Policy, Rule, RuleTarget +from fidesops.ops.models.privacy_request import PrivacyRequest +from fidesops.ops.models.storage import StorageConfig diff --git a/src/fidesops/db/base_class.py b/src/fidesops/ops/db/base_class.py similarity index 100% rename from src/fidesops/db/base_class.py rename to src/fidesops/ops/db/base_class.py diff --git a/src/fidesops/db/database.py b/src/fidesops/ops/db/database.py similarity index 100% rename from src/fidesops/db/database.py rename to src/fidesops/ops/db/database.py diff --git a/src/fidesops/migrations/__init__.py b/src/fidesops/ops/graph/__init__.py similarity index 100% rename from src/fidesops/migrations/__init__.py rename to src/fidesops/ops/graph/__init__.py diff --git a/src/fidesops/graph/analytics_events.py b/src/fidesops/ops/graph/analytics_events.py similarity index 87% rename from src/fidesops/graph/analytics_events.py rename to src/fidesops/ops/graph/analytics_events.py index 73d585f5b7..8ad6387d5c 100644 --- a/src/fidesops/graph/analytics_events.py +++ b/src/fidesops/ops/graph/analytics_events.py @@ -3,22 +3,22 @@ from fideslog.sdk.python.event import AnalyticsEvent -from fidesops.analytics import in_docker_container, send_analytics_event -from fidesops.core.config import config -from fidesops.graph.config import CollectionAddress -from fidesops.graph.graph_differences import ( +from fidesops.ops.analytics import in_docker_container, send_analytics_event +from fidesops.ops.core.config import config +from fidesops.ops.graph.config import CollectionAddress +from fidesops.ops.graph.graph_differences import ( GraphDiffSummary, GraphRepr, find_graph_differences_summary, format_graph_for_caching, ) -from fidesops.models.policy import ActionType -from fidesops.models.privacy_request import PrivacyRequest -from fidesops.task.task_resources import TaskResources -from fidesops.util.collection_util import Row +from fidesops.ops.models.policy import ActionType +from fidesops.ops.models.privacy_request import PrivacyRequest +from fidesops.ops.task.task_resources import TaskResources +from fidesops.ops.util.collection_util import Row if TYPE_CHECKING: - from fidesops.task.graph_task import GraphTask + from fidesops.ops.task.graph_task import GraphTask def fideslog_graph_failure(event: Optional[AnalyticsEvent]) -> None: diff --git a/src/fidesops/graph/config.py b/src/fidesops/ops/graph/config.py similarity index 98% rename from src/fidesops/graph/config.py rename to src/fidesops/ops/graph/config.py index e9fa1e4ee5..da6692e89f 100644 --- a/src/fidesops/graph/config.py +++ b/src/fidesops/ops/graph/config.py @@ -84,15 +84,15 @@ from pydantic import BaseModel, validator -from fidesops.common_exceptions import FidesopsException -from fidesops.graph.data_type import ( +from fidesops.ops.common_exceptions import FidesopsException +from fidesops.ops.graph.data_type import ( DataType, DataTypeConverter, get_data_type_converter, ) -from fidesops.schemas.shared_schemas import FidesOpsKey -from fidesops.util.collection_util import merge_dicts -from fidesops.util.querytoken import QueryToken +from fidesops.ops.schemas.shared_schemas import FidesOpsKey +from fidesops.ops.util.collection_util import merge_dicts +from fidesops.ops.util.querytoken import QueryToken DatasetAddress = str SeedAddress = str diff --git a/src/fidesops/graph/data_type.py b/src/fidesops/ops/graph/data_type.py similarity index 100% rename from src/fidesops/graph/data_type.py rename to src/fidesops/ops/graph/data_type.py diff --git a/src/fidesops/graph/graph.py b/src/fidesops/ops/graph/graph.py similarity index 98% rename from src/fidesops/graph/graph.py rename to src/fidesops/ops/graph/graph.py index 88dd4c4ba2..a0eeb04074 100644 --- a/src/fidesops/graph/graph.py +++ b/src/fidesops/ops/graph/graph.py @@ -4,8 +4,8 @@ from collections import defaultdict from typing import Callable, Dict, List, Optional, Set, Tuple -from fidesops.common_exceptions import ValidationError -from fidesops.graph.config import ( +from fidesops.ops.common_exceptions import ValidationError +from fidesops.ops.graph.config import ( Collection, CollectionAddress, Dataset, @@ -15,7 +15,7 @@ FieldPath, SeedAddress, ) -from fidesops.schemas.shared_schemas import FidesOpsKey +from fidesops.ops.schemas.shared_schemas import FidesOpsKey logger = logging.getLogger(__name__) diff --git a/src/fidesops/graph/graph_differences.py b/src/fidesops/ops/graph/graph_differences.py similarity index 97% rename from src/fidesops/graph/graph_differences.py rename to src/fidesops/ops/graph/graph_differences.py index ed72cbd71b..111efdc4fa 100644 --- a/src/fidesops/graph/graph_differences.py +++ b/src/fidesops/ops/graph/graph_differences.py @@ -1,15 +1,15 @@ from typing import TYPE_CHECKING, Dict, List, Optional, Set -from fidesops.graph.config import ( +from fidesops.ops.graph.config import ( ROOT_COLLECTION_ADDRESS, TERMINATOR_ADDRESS, CollectionAddress, ) -from fidesops.schemas.base_class import BaseSchema -from fidesops.util.collection_util import Row +from fidesops.ops.schemas.base_class import BaseSchema +from fidesops.ops.util.collection_util import Row if TYPE_CHECKING: - from fidesops.task.graph_task import GraphTask + from fidesops.ops.task.graph_task import GraphTask GraphRepr = Dict[str, Dict[str, List[str]]] diff --git a/src/fidesops/graph/traversal.py b/src/fidesops/ops/graph/traversal.py similarity index 97% rename from src/fidesops/graph/traversal.py rename to src/fidesops/ops/graph/traversal.py index 2266d834fd..6f2b620b81 100644 --- a/src/fidesops/graph/traversal.py +++ b/src/fidesops/ops/graph/traversal.py @@ -5,8 +5,8 @@ import pydash.collections -from fidesops.common_exceptions import TraversalError -from fidesops.graph.config import ( +from fidesops.ops.common_exceptions import TraversalError +from fidesops.ops.graph.config import ( ROOT_COLLECTION_ADDRESS, Collection, CollectionAddress, @@ -15,10 +15,10 @@ FieldAddress, FieldPath, ) -from fidesops.graph.graph import DatasetGraph, Edge, Node -from fidesops.util.collection_util import Row, append -from fidesops.util.logger import NotPii -from fidesops.util.matching_queue import MatchingQueue +from fidesops.ops.graph.graph import DatasetGraph, Edge, Node +from fidesops.ops.util.collection_util import Row, append +from fidesops.ops.util.logger import NotPii +from fidesops.ops.util.matching_queue import MatchingQueue logger = logging.getLogger(__name__) diff --git a/src/fidesops/migrations/README b/src/fidesops/ops/migrations/README similarity index 100% rename from src/fidesops/migrations/README rename to src/fidesops/ops/migrations/README diff --git a/src/fidesops/migrations/versions/__init__.py b/src/fidesops/ops/migrations/__init__.py similarity index 100% rename from src/fidesops/migrations/versions/__init__.py rename to src/fidesops/ops/migrations/__init__.py diff --git a/src/fidesops/migrations/env.py b/src/fidesops/ops/migrations/env.py similarity index 97% rename from src/fidesops/migrations/env.py rename to src/fidesops/ops/migrations/env.py index 31f561bef1..2d61b4b566 100644 --- a/src/fidesops/migrations/env.py +++ b/src/fidesops/ops/migrations/env.py @@ -23,7 +23,7 @@ # add your model's MetaData object here # for 'autogenerate' support -from fidesops.core.config import config as fides_config +from fidesops.ops.core.config import config as fides_config # Load the correct environment target_metadata = Base.metadata diff --git a/src/fidesops/migrations/script.py.mako b/src/fidesops/ops/migrations/script.py.mako similarity index 100% rename from src/fidesops/migrations/script.py.mako rename to src/fidesops/ops/migrations/script.py.mako diff --git a/src/fidesops/migrations/versions/0210948a8147_initial.py b/src/fidesops/ops/migrations/versions/0210948a8147_initial.py similarity index 100% rename from src/fidesops/migrations/versions/0210948a8147_initial.py rename to src/fidesops/ops/migrations/versions/0210948a8147_initial.py diff --git a/src/fidesops/migrations/versions/07014ff34eb2_add_mariadb.py b/src/fidesops/ops/migrations/versions/07014ff34eb2_add_mariadb.py similarity index 100% rename from src/fidesops/migrations/versions/07014ff34eb2_add_mariadb.py rename to src/fidesops/ops/migrations/versions/07014ff34eb2_add_mariadb.py diff --git a/src/fidesops/migrations/versions/1dfc5a2d30e7_add_saas_config_to_connection_config.py b/src/fidesops/ops/migrations/versions/1dfc5a2d30e7_add_saas_config_to_connection_config.py similarity index 100% rename from src/fidesops/migrations/versions/1dfc5a2d30e7_add_saas_config_to_connection_config.py rename to src/fidesops/ops/migrations/versions/1dfc5a2d30e7_add_saas_config_to_connection_config.py diff --git a/src/fidesops/migrations/versions/1ff88b7bd579_add_authorization_request.py b/src/fidesops/ops/migrations/versions/1ff88b7bd579_add_authorization_request.py similarity index 100% rename from src/fidesops/migrations/versions/1ff88b7bd579_add_authorization_request.py rename to src/fidesops/ops/migrations/versions/1ff88b7bd579_add_authorization_request.py diff --git a/src/fidesops/migrations/versions/27fe9da9d0f9_privacy_request_stopped.py b/src/fidesops/ops/migrations/versions/27fe9da9d0f9_privacy_request_stopped.py similarity index 100% rename from src/fidesops/migrations/versions/27fe9da9d0f9_privacy_request_stopped.py rename to src/fidesops/ops/migrations/versions/27fe9da9d0f9_privacy_request_stopped.py diff --git a/src/fidesops/migrations/versions/29a7d707163a_adds_first_name_and_last_name_to_user_.py b/src/fidesops/ops/migrations/versions/29a7d707163a_adds_first_name_and_last_name_to_user_.py similarity index 100% rename from src/fidesops/migrations/versions/29a7d707163a_adds_first_name_and_last_name_to_user_.py rename to src/fidesops/ops/migrations/versions/29a7d707163a_adds_first_name_and_last_name_to_user_.py diff --git a/src/fidesops/migrations/versions/3a7c5fb119c9_add_manual_connector_type.py b/src/fidesops/ops/migrations/versions/3a7c5fb119c9_add_manual_connector_type.py similarity index 100% rename from src/fidesops/migrations/versions/3a7c5fb119c9_add_manual_connector_type.py rename to src/fidesops/ops/migrations/versions/3a7c5fb119c9_add_manual_connector_type.py diff --git a/src/fidesops/migrations/versions/3c5e1253465d_adds_provided_identity_table_for_.py b/src/fidesops/ops/migrations/versions/3c5e1253465d_adds_provided_identity_table_for_.py similarity index 100% rename from src/fidesops/migrations/versions/3c5e1253465d_adds_provided_identity_table_for_.py rename to src/fidesops/ops/migrations/versions/3c5e1253465d_adds_provided_identity_table_for_.py diff --git a/src/fidesops/migrations/versions/5078badb90b9_adds_drp_action_to_policy.py b/src/fidesops/ops/migrations/versions/5078badb90b9_adds_drp_action_to_policy.py similarity index 100% rename from src/fidesops/migrations/versions/5078badb90b9_adds_drp_action_to_policy.py rename to src/fidesops/ops/migrations/versions/5078badb90b9_adds_drp_action_to_policy.py diff --git a/src/fidesops/migrations/versions/530fb8533ca4_test.py b/src/fidesops/ops/migrations/versions/530fb8533ca4_test.py similarity index 100% rename from src/fidesops/migrations/versions/530fb8533ca4_test.py rename to src/fidesops/ops/migrations/versions/530fb8533ca4_test.py diff --git a/src/fidesops/migrations/versions/55d61eb8ed12_add_default_policies.py b/src/fidesops/ops/migrations/versions/55d61eb8ed12_add_default_policies.py similarity index 96% rename from src/fidesops/migrations/versions/55d61eb8ed12_add_default_policies.py rename to src/fidesops/ops/migrations/versions/55d61eb8ed12_add_default_policies.py index 8284970136..82ac8a84d9 100644 --- a/src/fidesops/migrations/versions/55d61eb8ed12_add_default_policies.py +++ b/src/fidesops/ops/migrations/versions/55d61eb8ed12_add_default_policies.py @@ -24,16 +24,16 @@ from sqlalchemy_utils import StringEncryptedType from sqlalchemy_utils.types.encrypted.encrypted_type import AesGcmEngine -from fidesops.api.v1.scope_registry import SCOPE_REGISTRY -from fidesops.core.config import config -from fidesops.db.base import Policy, Rule, RuleTarget, StorageConfig -from fidesops.db.base_class import JSONTypeOverride -from fidesops.models.policy import ActionType, DrpAction -from fidesops.schemas.storage.storage import StorageType -from fidesops.service.masking.strategy.masking_strategy_string_rewrite import ( +from fidesops.ops.api.v1.scope_registry import SCOPE_REGISTRY +from fidesops.ops.core.config import config +from fidesops.ops.db.base import Policy, Rule, RuleTarget, StorageConfig +from fidesops.ops.db.base_class import JSONTypeOverride +from fidesops.ops.models.policy import ActionType, DrpAction +from fidesops.ops.schemas.storage.storage import StorageType +from fidesops.ops.service.masking.strategy.masking_strategy_string_rewrite import ( STRING_REWRITE_STRATEGY_NAME, ) -from fidesops.util.data_category import DataCategory +from fidesops.ops.util.data_category import DataCategory logging.basicConfig() logger = logging.getLogger(__name__) diff --git a/src/fidesops/migrations/versions/5a966cd643d7_fidesops_user_and_client_detail.py b/src/fidesops/ops/migrations/versions/5a966cd643d7_fidesops_user_and_client_detail.py similarity index 100% rename from src/fidesops/migrations/versions/5a966cd643d7_fidesops_user_and_client_detail.py rename to src/fidesops/ops/migrations/versions/5a966cd643d7_fidesops_user_and_client_detail.py diff --git a/src/fidesops/migrations/versions/90070db16d05_add_fidesops_user_permissions.py b/src/fidesops/ops/migrations/versions/90070db16d05_add_fidesops_user_permissions.py similarity index 100% rename from src/fidesops/migrations/versions/90070db16d05_add_fidesops_user_permissions.py rename to src/fidesops/ops/migrations/versions/90070db16d05_add_fidesops_user_permissions.py diff --git a/src/fidesops/migrations/versions/906d7198df28_privacy_request_approve.py b/src/fidesops/ops/migrations/versions/906d7198df28_privacy_request_approve.py similarity index 100% rename from src/fidesops/migrations/versions/906d7198df28_privacy_request_approve.py rename to src/fidesops/ops/migrations/versions/906d7198df28_privacy_request_approve.py diff --git a/src/fidesops/models/__init__.py b/src/fidesops/ops/migrations/versions/__init__.py similarity index 100% rename from src/fidesops/models/__init__.py rename to src/fidesops/ops/migrations/versions/__init__.py diff --git a/src/fidesops/migrations/versions/b3b68c87c4a0_add_connection_config_description.py b/src/fidesops/ops/migrations/versions/b3b68c87c4a0_add_connection_config_description.py similarity index 100% rename from src/fidesops/migrations/versions/b3b68c87c4a0_add_connection_config_description.py rename to src/fidesops/ops/migrations/versions/b3b68c87c4a0_add_connection_config_description.py diff --git a/src/fidesops/migrations/versions/bab75915670a_add_finished_audit_log.py b/src/fidesops/ops/migrations/versions/bab75915670a_add_finished_audit_log.py similarity index 100% rename from src/fidesops/migrations/versions/bab75915670a_add_finished_audit_log.py rename to src/fidesops/ops/migrations/versions/bab75915670a_add_finished_audit_log.py diff --git a/src/fidesops/migrations/versions/c3472d75c80e_connectionconfig_disabled.py b/src/fidesops/ops/migrations/versions/c3472d75c80e_connectionconfig_disabled.py similarity index 100% rename from src/fidesops/migrations/versions/c3472d75c80e_connectionconfig_disabled.py rename to src/fidesops/ops/migrations/versions/c3472d75c80e_connectionconfig_disabled.py diff --git a/src/fidesops/migrations/versions/c5336b841d70_add_policy_webhooks.py b/src/fidesops/ops/migrations/versions/c5336b841d70_add_policy_webhooks.py similarity index 100% rename from src/fidesops/migrations/versions/c5336b841d70_add_policy_webhooks.py rename to src/fidesops/ops/migrations/versions/c5336b841d70_add_policy_webhooks.py diff --git a/src/fidesops/migrations/versions/c7cc36820d4b_rename_user_name_tables_and_indexes.py b/src/fidesops/ops/migrations/versions/c7cc36820d4b_rename_user_name_tables_and_indexes.py similarity index 100% rename from src/fidesops/migrations/versions/c7cc36820d4b_rename_user_name_tables_and_indexes.py rename to src/fidesops/ops/migrations/versions/c7cc36820d4b_rename_user_name_tables_and_indexes.py diff --git a/src/fidesops/migrations/versions/c9759675b5d3_add_user_login_and_password_reset_dates.py b/src/fidesops/ops/migrations/versions/c9759675b5d3_add_user_login_and_password_reset_dates.py similarity index 100% rename from src/fidesops/migrations/versions/c9759675b5d3_add_user_login_and_password_reset_dates.py rename to src/fidesops/ops/migrations/versions/c9759675b5d3_add_user_login_and_password_reset_dates.py diff --git a/src/fidesops/migrations/versions/c98da12d76f8_add_audit_log.py b/src/fidesops/ops/migrations/versions/c98da12d76f8_add_audit_log.py similarity index 100% rename from src/fidesops/migrations/versions/c98da12d76f8_add_audit_log.py rename to src/fidesops/ops/migrations/versions/c98da12d76f8_add_audit_log.py diff --git a/src/fidesops/migrations/versions/d65e7e921814_add_privacy_request_status.py b/src/fidesops/ops/migrations/versions/d65e7e921814_add_privacy_request_status.py similarity index 100% rename from src/fidesops/migrations/versions/d65e7e921814_add_privacy_request_status.py rename to src/fidesops/ops/migrations/versions/d65e7e921814_add_privacy_request_status.py diff --git a/src/fidesops/migrations/versions/e55a51b354e3_add_bigquery.py b/src/fidesops/ops/migrations/versions/e55a51b354e3_add_bigquery.py similarity index 100% rename from src/fidesops/migrations/versions/e55a51b354e3_add_bigquery.py rename to src/fidesops/ops/migrations/versions/e55a51b354e3_add_bigquery.py diff --git a/src/fidesops/migrations/versions/ed1b00ff963d_cancel_privacy_request.py b/src/fidesops/ops/migrations/versions/ed1b00ff963d_cancel_privacy_request.py similarity index 100% rename from src/fidesops/migrations/versions/ed1b00ff963d_cancel_privacy_request.py rename to src/fidesops/ops/migrations/versions/ed1b00ff963d_cancel_privacy_request.py diff --git a/src/fidesops/migrations/versions/f206d4e7574d_add_redshift_and_snowflake_support.py b/src/fidesops/ops/migrations/versions/f206d4e7574d_add_redshift_and_snowflake_support.py similarity index 100% rename from src/fidesops/migrations/versions/f206d4e7574d_add_redshift_and_snowflake_support.py rename to src/fidesops/ops/migrations/versions/f206d4e7574d_add_redshift_and_snowflake_support.py diff --git a/src/fidesops/migrations/versions/f3841942d90c_add_mssql.py b/src/fidesops/ops/migrations/versions/f3841942d90c_add_mssql.py similarity index 100% rename from src/fidesops/migrations/versions/f3841942d90c_add_mssql.py rename to src/fidesops/ops/migrations/versions/f3841942d90c_add_mssql.py diff --git a/src/fidesops/migrations/versions/fc90277bbcde_populate_saas_type_to_saas_config.py b/src/fidesops/ops/migrations/versions/fc90277bbcde_populate_saas_type_to_saas_config.py similarity index 100% rename from src/fidesops/migrations/versions/fc90277bbcde_populate_saas_type_to_saas_config.py rename to src/fidesops/ops/migrations/versions/fc90277bbcde_populate_saas_type_to_saas_config.py diff --git a/src/fidesops/schemas/masking/__init__.py b/src/fidesops/ops/models/__init__.py similarity index 100% rename from src/fidesops/schemas/masking/__init__.py rename to src/fidesops/ops/models/__init__.py diff --git a/src/fidesops/models/authentication_request.py b/src/fidesops/ops/models/authentication_request.py similarity index 100% rename from src/fidesops/models/authentication_request.py rename to src/fidesops/ops/models/authentication_request.py diff --git a/src/fidesops/models/connectionconfig.py b/src/fidesops/ops/models/connectionconfig.py similarity index 96% rename from src/fidesops/models/connectionconfig.py rename to src/fidesops/ops/models/connectionconfig.py index f98ec50fd0..dedb1af8ab 100644 --- a/src/fidesops/models/connectionconfig.py +++ b/src/fidesops/ops/models/connectionconfig.py @@ -14,9 +14,9 @@ StringEncryptedType, ) -from fidesops.core.config import config -from fidesops.db.base_class import JSONTypeOverride -from fidesops.schemas.saas.saas_config import SaaSConfig +from fidesops.ops.core.config import config +from fidesops.ops.db.base_class import JSONTypeOverride +from fidesops.ops.schemas.saas.saas_config import SaaSConfig class ConnectionTestStatus(enum.Enum): diff --git a/src/fidesops/models/datasetconfig.py b/src/fidesops/ops/models/datasetconfig.py similarity index 94% rename from src/fidesops/models/datasetconfig.py rename to src/fidesops/ops/models/datasetconfig.py index 74461325ec..968522d0de 100644 --- a/src/fidesops/models/datasetconfig.py +++ b/src/fidesops/ops/models/datasetconfig.py @@ -8,7 +8,7 @@ from sqlalchemy.ext.mutable import MutableDict from sqlalchemy.orm import relationship -from fidesops.graph.config import ( +from fidesops.ops.graph.config import ( Collection, CollectionAddress, Dataset, @@ -16,12 +16,12 @@ FieldAddress, generate_field, ) -from fidesops.graph.data_type import parse_data_type_string -from fidesops.models.connectionconfig import ConnectionConfig, ConnectionType -from fidesops.schemas.dataset import FidesopsDataset, FidesopsDatasetField -from fidesops.schemas.shared_schemas import FidesOpsKey -from fidesops.util.logger import NotPii -from fidesops.util.saas_util import merge_datasets +from fidesops.ops.graph.data_type import parse_data_type_string +from fidesops.ops.models.connectionconfig import ConnectionConfig, ConnectionType +from fidesops.ops.schemas.dataset import FidesopsDataset, FidesopsDatasetField +from fidesops.ops.schemas.shared_schemas import FidesOpsKey +from fidesops.ops.util.logger import NotPii +from fidesops.ops.util.saas_util import merge_datasets logger = logging.getLogger(__name__) diff --git a/src/fidesops/models/policy.py b/src/fidesops/ops/models/policy.py similarity index 97% rename from src/fidesops/models/policy.py rename to src/fidesops/ops/models/policy.py index 226090f869..77a0ad7ca6 100644 --- a/src/fidesops/models/policy.py +++ b/src/fidesops/ops/models/policy.py @@ -16,14 +16,14 @@ StringEncryptedType, ) -from fidesops import common_exceptions -from fidesops.common_exceptions import WebhookOrderException -from fidesops.core.config import config -from fidesops.db.base_class import JSONTypeOverride -from fidesops.models.connectionconfig import ConnectionConfig -from fidesops.models.storage import StorageConfig -from fidesops.schemas.shared_schemas import FidesOpsKey -from fidesops.util.data_category import _validate_data_category +from fidesops.ops import common_exceptions +from fidesops.ops.common_exceptions import WebhookOrderException +from fidesops.ops.core.config import config +from fidesops.ops.db.base_class import JSONTypeOverride +from fidesops.ops.models.connectionconfig import ConnectionConfig +from fidesops.ops.models.storage import StorageConfig +from fidesops.ops.schemas.shared_schemas import FidesOpsKey +from fidesops.ops.util.data_category import _validate_data_category class PausedStep(EnumType): diff --git a/src/fidesops/models/privacy_request.py b/src/fidesops/ops/models/privacy_request.py similarity index 96% rename from src/fidesops/models/privacy_request.py rename to src/fidesops/ops/models/privacy_request.py index 0fb6d4c36b..377ae14917 100644 --- a/src/fidesops/models/privacy_request.py +++ b/src/fidesops/ops/models/privacy_request.py @@ -25,13 +25,13 @@ StringEncryptedType, ) -from fidesops.api.v1.scope_registry import PRIVACY_REQUEST_CALLBACK_RESUME -from fidesops.common_exceptions import PrivacyRequestPaused -from fidesops.core.config import config -from fidesops.db.base_class import JSONTypeOverride -from fidesops.graph.config import CollectionAddress -from fidesops.graph.graph_differences import GraphRepr -from fidesops.models.policy import ( +from fidesops.ops.api.v1.scope_registry import PRIVACY_REQUEST_CALLBACK_RESUME +from fidesops.ops.common_exceptions import PrivacyRequestPaused +from fidesops.ops.core.config import config +from fidesops.ops.db.base_class import JSONTypeOverride +from fidesops.ops.graph.config import CollectionAddress +from fidesops.ops.graph.graph_differences import GraphRepr +from fidesops.ops.models.policy import ( ActionType, PausedStep, Policy, @@ -39,17 +39,17 @@ WebhookDirection, WebhookTypes, ) -from fidesops.schemas.base_class import BaseSchema -from fidesops.schemas.drp_privacy_request import DrpPrivacyRequestCreate -from fidesops.schemas.external_https import ( +from fidesops.ops.schemas.base_class import BaseSchema +from fidesops.ops.schemas.drp_privacy_request import DrpPrivacyRequestCreate +from fidesops.ops.schemas.external_https import ( SecondPartyRequestFormat, SecondPartyResponseFormat, WebhookJWE, ) -from fidesops.schemas.masking.masking_secrets import MaskingSecretCache -from fidesops.schemas.redis_cache import PrivacyRequestIdentity -from fidesops.tasks import celery_app -from fidesops.util.cache import ( +from fidesops.ops.schemas.masking.masking_secrets import MaskingSecretCache +from fidesops.ops.schemas.redis_cache import PrivacyRequestIdentity +from fidesops.ops.tasks import celery_app +from fidesops.ops.util.cache import ( FidesopsRedis, get_all_cache_keys_for_privacy_request, get_async_task_tracking_cache_key, @@ -59,7 +59,7 @@ get_identity_cache_key, get_masking_secret_cache_key, ) -from fidesops.util.collection_util import Row +from fidesops.ops.util.collection_util import Row logger = logging.getLogger(__name__) @@ -459,7 +459,7 @@ def trigger_policy_webhook(self, webhook: WebhookTypes) -> None: to halt. To resume, they use send a request to the reply-to URL with the reply-to-token. """ # temp fix for circular dependency - from fidesops.service.connectors import HTTPSConnector, get_connector + from fidesops.ops.service.connectors import HTTPSConnector, get_connector https_connector: HTTPSConnector = get_connector(webhook.connection_config) # type: ignore request_body = SecondPartyRequestFormat( diff --git a/src/fidesops/models/storage.py b/src/fidesops/ops/models/storage.py similarity index 92% rename from src/fidesops/models/storage.py rename to src/fidesops/ops/models/storage.py index 6de285b57a..980296de12 100644 --- a/src/fidesops/models/storage.py +++ b/src/fidesops/ops/models/storage.py @@ -11,16 +11,18 @@ StringEncryptedType, ) -from fidesops.core.config import config -from fidesops.db.base_class import JSONTypeOverride -from fidesops.schemas.storage.storage import ( +from fidesops.ops.core.config import config +from fidesops.ops.db.base_class import JSONTypeOverride +from fidesops.ops.schemas.storage.storage import ( SUPPORTED_STORAGE_SECRETS, ResponseFormat, StorageSecretsOnetrust, StorageSecretsS3, StorageType, ) -from fidesops.schemas.storage.storage_secrets_docs_only import possible_storage_secrets +from fidesops.ops.schemas.storage.storage_secrets_docs_only import ( + possible_storage_secrets, +) logger = logging.getLogger(__name__) diff --git a/src/fidesops/schemas/__init__.py b/src/fidesops/ops/schemas/__init__.py similarity index 100% rename from src/fidesops/schemas/__init__.py rename to src/fidesops/ops/schemas/__init__.py diff --git a/src/fidesops/schemas/analytics.py b/src/fidesops/ops/schemas/analytics.py similarity index 100% rename from src/fidesops/schemas/analytics.py rename to src/fidesops/ops/schemas/analytics.py diff --git a/src/fidesops/schemas/api.py b/src/fidesops/ops/schemas/api.py similarity index 100% rename from src/fidesops/schemas/api.py rename to src/fidesops/ops/schemas/api.py diff --git a/src/fidesops/schemas/base_class.py b/src/fidesops/ops/schemas/base_class.py similarity index 100% rename from src/fidesops/schemas/base_class.py rename to src/fidesops/ops/schemas/base_class.py diff --git a/src/fidesops/schemas/client.py b/src/fidesops/ops/schemas/client.py similarity index 69% rename from src/fidesops/schemas/client.py rename to src/fidesops/ops/schemas/client.py index 7eb93e0f9b..7cfcf9a134 100644 --- a/src/fidesops/schemas/client.py +++ b/src/fidesops/ops/schemas/client.py @@ -1,4 +1,4 @@ -from fidesops.schemas.base_class import FidesopsSchema +from fidesops.ops.schemas.base_class import FidesopsSchema class ClientCreatedResponse(FidesopsSchema): diff --git a/src/fidesops/schemas/connection_configuration/__init__.py b/src/fidesops/ops/schemas/connection_configuration/__init__.py similarity index 67% rename from src/fidesops/schemas/connection_configuration/__init__.py rename to src/fidesops/ops/schemas/connection_configuration/__init__.py index 5d6ebbfbb1..aef9c381b6 100644 --- a/src/fidesops/schemas/connection_configuration/__init__.py +++ b/src/fidesops/ops/schemas/connection_configuration/__init__.py @@ -1,49 +1,49 @@ from typing import Any, Dict, Optional, Union -from fidesops.models.connectionconfig import ConnectionType -from fidesops.schemas.connection_configuration.connection_secrets import ( +from fidesops.ops.models.connectionconfig import ConnectionType +from fidesops.ops.schemas.connection_configuration.connection_secrets import ( ConnectionConfigSecretsSchema, ) -from fidesops.schemas.connection_configuration.connection_secrets_bigquery import ( +from fidesops.ops.schemas.connection_configuration.connection_secrets_bigquery import ( BigQueryDocsSchema, BigQuerySchema, ) -from fidesops.schemas.connection_configuration.connection_secrets_mariadb import ( +from fidesops.ops.schemas.connection_configuration.connection_secrets_mariadb import ( MariaDBDocsSchema, MariaDBSchema, ) -from fidesops.schemas.connection_configuration.connection_secrets_mongodb import ( +from fidesops.ops.schemas.connection_configuration.connection_secrets_mongodb import ( MongoDBDocsSchema, MongoDBSchema, ) -from fidesops.schemas.connection_configuration.connection_secrets_mssql import ( +from fidesops.ops.schemas.connection_configuration.connection_secrets_mssql import ( MicrosoftSQLServerSchema, MSSQLDocsSchema, ) -from fidesops.schemas.connection_configuration.connection_secrets_mysql import ( +from fidesops.ops.schemas.connection_configuration.connection_secrets_mysql import ( MySQLDocsSchema, MySQLSchema, ) -from fidesops.schemas.connection_configuration.connection_secrets_postgres import ( +from fidesops.ops.schemas.connection_configuration.connection_secrets_postgres import ( PostgreSQLDocsSchema, PostgreSQLSchema, ) -from fidesops.schemas.connection_configuration.connection_secrets_redshift import ( +from fidesops.ops.schemas.connection_configuration.connection_secrets_redshift import ( RedshiftDocsSchema, RedshiftSchema, ) -from fidesops.schemas.connection_configuration.connection_secrets_saas import ( +from fidesops.ops.schemas.connection_configuration.connection_secrets_saas import ( SaaSSchema, SaaSSchemaFactory, ) -from fidesops.schemas.connection_configuration.connection_secrets_snowflake import ( +from fidesops.ops.schemas.connection_configuration.connection_secrets_snowflake import ( SnowflakeDocsSchema, SnowflakeSchema, ) -from fidesops.schemas.connection_configuration.connections_secrets_https import ( +from fidesops.ops.schemas.connection_configuration.connections_secrets_https import ( HttpsSchema, ) -from fidesops.schemas.saas.saas_config import SaaSConfig +from fidesops.ops.schemas.saas.saas_config import SaaSConfig secrets_validators: Dict[str, Any] = { ConnectionType.postgres.value: PostgreSQLSchema, diff --git a/src/fidesops/schemas/connection_configuration/connection_config.py b/src/fidesops/ops/schemas/connection_configuration/connection_config.py similarity index 89% rename from src/fidesops/schemas/connection_configuration/connection_config.py rename to src/fidesops/ops/schemas/connection_configuration/connection_config.py index 0644fd4ea0..e56c62123c 100644 --- a/src/fidesops/schemas/connection_configuration/connection_config.py +++ b/src/fidesops/ops/schemas/connection_configuration/connection_config.py @@ -4,10 +4,10 @@ from pydantic import BaseModel, Extra -from fidesops.models.connectionconfig import AccessLevel, ConnectionType -from fidesops.schemas.api import BulkResponse, BulkUpdateFailed -from fidesops.schemas.saas.saas_config import SaaSConfigBase, SaaSType -from fidesops.schemas.shared_schemas import FidesOpsKey +from fidesops.ops.models.connectionconfig import AccessLevel, ConnectionType +from fidesops.ops.schemas.api import BulkResponse, BulkUpdateFailed +from fidesops.ops.schemas.saas.saas_config import SaaSConfigBase, SaaSType +from fidesops.ops.schemas.shared_schemas import FidesOpsKey class CreateConnectionConfiguration(BaseModel): diff --git a/src/fidesops/schemas/connection_configuration/connection_secrets.py b/src/fidesops/ops/schemas/connection_configuration/connection_secrets.py similarity index 94% rename from src/fidesops/schemas/connection_configuration/connection_secrets.py rename to src/fidesops/ops/schemas/connection_configuration/connection_secrets.py index 6d874030a3..087f521615 100644 --- a/src/fidesops/schemas/connection_configuration/connection_secrets.py +++ b/src/fidesops/ops/schemas/connection_configuration/connection_secrets.py @@ -5,8 +5,8 @@ from pydantic import BaseModel, Extra, root_validator -from fidesops.models.connectionconfig import ConnectionTestStatus -from fidesops.schemas import Msg +from fidesops.ops.models.connectionconfig import ConnectionTestStatus +from fidesops.ops.schemas import Msg class ConnectionConfigSecretsSchema(BaseModel, abc.ABC): diff --git a/src/fidesops/schemas/connection_configuration/connection_secrets_bigquery.py b/src/fidesops/ops/schemas/connection_configuration/connection_secrets_bigquery.py similarity index 86% rename from src/fidesops/schemas/connection_configuration/connection_secrets_bigquery.py rename to src/fidesops/ops/schemas/connection_configuration/connection_secrets_bigquery.py index 27af4e66e4..22d8e07b44 100644 --- a/src/fidesops/schemas/connection_configuration/connection_secrets_bigquery.py +++ b/src/fidesops/ops/schemas/connection_configuration/connection_secrets_bigquery.py @@ -2,8 +2,8 @@ from pydantic.main import BaseModel -from fidesops.schemas.base_class import NoValidationSchema -from fidesops.schemas.connection_configuration.connection_secrets import ( +from fidesops.ops.schemas.base_class import NoValidationSchema +from fidesops.ops.schemas.connection_configuration.connection_secrets import ( ConnectionConfigSecretsSchema, ) diff --git a/src/fidesops/schemas/connection_configuration/connection_secrets_mariadb.py b/src/fidesops/ops/schemas/connection_configuration/connection_secrets_mariadb.py similarity index 81% rename from src/fidesops/schemas/connection_configuration/connection_secrets_mariadb.py rename to src/fidesops/ops/schemas/connection_configuration/connection_secrets_mariadb.py index 5e2996762b..5e8877d3da 100644 --- a/src/fidesops/schemas/connection_configuration/connection_secrets_mariadb.py +++ b/src/fidesops/ops/schemas/connection_configuration/connection_secrets_mariadb.py @@ -1,7 +1,7 @@ from typing import List, Optional -from fidesops.schemas.base_class import NoValidationSchema -from fidesops.schemas.connection_configuration.connection_secrets import ( +from fidesops.ops.schemas.base_class import NoValidationSchema +from fidesops.ops.schemas.connection_configuration.connection_secrets import ( ConnectionConfigSecretsSchema, ) diff --git a/src/fidesops/schemas/connection_configuration/connection_secrets_mongodb.py b/src/fidesops/ops/schemas/connection_configuration/connection_secrets_mongodb.py similarity index 79% rename from src/fidesops/schemas/connection_configuration/connection_secrets_mongodb.py rename to src/fidesops/ops/schemas/connection_configuration/connection_secrets_mongodb.py index 1c22d1ec29..0f52462009 100644 --- a/src/fidesops/schemas/connection_configuration/connection_secrets_mongodb.py +++ b/src/fidesops/ops/schemas/connection_configuration/connection_secrets_mongodb.py @@ -1,7 +1,7 @@ from typing import List, Optional -from fidesops.schemas.base_class import NoValidationSchema -from fidesops.schemas.connection_configuration.connection_secrets import ( +from fidesops.ops.schemas.base_class import NoValidationSchema +from fidesops.ops.schemas.connection_configuration.connection_secrets import ( ConnectionConfigSecretsSchema, ) diff --git a/src/fidesops/schemas/connection_configuration/connection_secrets_mssql.py b/src/fidesops/ops/schemas/connection_configuration/connection_secrets_mssql.py similarity index 83% rename from src/fidesops/schemas/connection_configuration/connection_secrets_mssql.py rename to src/fidesops/ops/schemas/connection_configuration/connection_secrets_mssql.py index f7435fbbd3..5ac4a6bbf6 100644 --- a/src/fidesops/schemas/connection_configuration/connection_secrets_mssql.py +++ b/src/fidesops/ops/schemas/connection_configuration/connection_secrets_mssql.py @@ -1,7 +1,7 @@ from typing import List, Optional -from fidesops.schemas.base_class import NoValidationSchema -from fidesops.schemas.connection_configuration.connection_secrets import ( +from fidesops.ops.schemas.base_class import NoValidationSchema +from fidesops.ops.schemas.connection_configuration.connection_secrets import ( ConnectionConfigSecretsSchema, ) diff --git a/src/fidesops/schemas/connection_configuration/connection_secrets_mysql.py b/src/fidesops/ops/schemas/connection_configuration/connection_secrets_mysql.py similarity index 80% rename from src/fidesops/schemas/connection_configuration/connection_secrets_mysql.py rename to src/fidesops/ops/schemas/connection_configuration/connection_secrets_mysql.py index 2f49d7058a..ee7a93faa4 100644 --- a/src/fidesops/schemas/connection_configuration/connection_secrets_mysql.py +++ b/src/fidesops/ops/schemas/connection_configuration/connection_secrets_mysql.py @@ -1,7 +1,7 @@ from typing import List, Optional -from fidesops.schemas.base_class import NoValidationSchema -from fidesops.schemas.connection_configuration.connection_secrets import ( +from fidesops.ops.schemas.base_class import NoValidationSchema +from fidesops.ops.schemas.connection_configuration.connection_secrets import ( ConnectionConfigSecretsSchema, ) diff --git a/src/fidesops/schemas/connection_configuration/connection_secrets_postgres.py b/src/fidesops/ops/schemas/connection_configuration/connection_secrets_postgres.py similarity index 81% rename from src/fidesops/schemas/connection_configuration/connection_secrets_postgres.py rename to src/fidesops/ops/schemas/connection_configuration/connection_secrets_postgres.py index 1733e4ab36..2b54889fce 100644 --- a/src/fidesops/schemas/connection_configuration/connection_secrets_postgres.py +++ b/src/fidesops/ops/schemas/connection_configuration/connection_secrets_postgres.py @@ -1,7 +1,7 @@ from typing import List, Optional -from fidesops.schemas.base_class import NoValidationSchema -from fidesops.schemas.connection_configuration.connection_secrets import ( +from fidesops.ops.schemas.base_class import NoValidationSchema +from fidesops.ops.schemas.connection_configuration.connection_secrets import ( ConnectionConfigSecretsSchema, ) diff --git a/src/fidesops/schemas/connection_configuration/connection_secrets_redshift.py b/src/fidesops/ops/schemas/connection_configuration/connection_secrets_redshift.py similarity index 81% rename from src/fidesops/schemas/connection_configuration/connection_secrets_redshift.py rename to src/fidesops/ops/schemas/connection_configuration/connection_secrets_redshift.py index 4b189760d7..aa6f5f384e 100644 --- a/src/fidesops/schemas/connection_configuration/connection_secrets_redshift.py +++ b/src/fidesops/ops/schemas/connection_configuration/connection_secrets_redshift.py @@ -1,7 +1,7 @@ from typing import List, Optional -from fidesops.schemas.base_class import NoValidationSchema -from fidesops.schemas.connection_configuration.connection_secrets import ( +from fidesops.ops.schemas.base_class import NoValidationSchema +from fidesops.ops.schemas.connection_configuration.connection_secrets import ( ConnectionConfigSecretsSchema, ) diff --git a/src/fidesops/schemas/connection_configuration/connection_secrets_saas.py b/src/fidesops/ops/schemas/connection_configuration/connection_secrets_saas.py similarity index 97% rename from src/fidesops/schemas/connection_configuration/connection_secrets_saas.py rename to src/fidesops/ops/schemas/connection_configuration/connection_secrets_saas.py index 205e8b7578..173cf39edd 100644 --- a/src/fidesops/schemas/connection_configuration/connection_secrets_saas.py +++ b/src/fidesops/ops/schemas/connection_configuration/connection_secrets_saas.py @@ -3,7 +3,7 @@ from pydantic import BaseModel, Extra, create_model, root_validator -from fidesops.schemas.saas.saas_config import SaaSConfig +from fidesops.ops.schemas.saas.saas_config import SaaSConfig class SaaSSchema(BaseModel, abc.ABC): diff --git a/src/fidesops/schemas/connection_configuration/connection_secrets_snowflake.py b/src/fidesops/ops/schemas/connection_configuration/connection_secrets_snowflake.py similarity index 85% rename from src/fidesops/schemas/connection_configuration/connection_secrets_snowflake.py rename to src/fidesops/ops/schemas/connection_configuration/connection_secrets_snowflake.py index 4858e50840..ce607249cb 100644 --- a/src/fidesops/schemas/connection_configuration/connection_secrets_snowflake.py +++ b/src/fidesops/ops/schemas/connection_configuration/connection_secrets_snowflake.py @@ -1,7 +1,7 @@ from typing import List, Optional -from fidesops.schemas.base_class import NoValidationSchema -from fidesops.schemas.connection_configuration.connection_secrets import ( +from fidesops.ops.schemas.base_class import NoValidationSchema +from fidesops.ops.schemas.connection_configuration.connection_secrets import ( ConnectionConfigSecretsSchema, ) diff --git a/src/fidesops/schemas/connection_configuration/connections_secrets_https.py b/src/fidesops/ops/schemas/connection_configuration/connections_secrets_https.py similarity index 73% rename from src/fidesops/schemas/connection_configuration/connections_secrets_https.py rename to src/fidesops/ops/schemas/connection_configuration/connections_secrets_https.py index 9efe81020c..23c4712a44 100644 --- a/src/fidesops/schemas/connection_configuration/connections_secrets_https.py +++ b/src/fidesops/ops/schemas/connection_configuration/connections_secrets_https.py @@ -1,7 +1,7 @@ from typing import List -from fidesops.schemas.base_class import NoValidationSchema -from fidesops.schemas.connection_configuration.connection_secrets import ( +from fidesops.ops.schemas.base_class import NoValidationSchema +from fidesops.ops.schemas.connection_configuration.connection_secrets import ( ConnectionConfigSecretsSchema, ) diff --git a/src/fidesops/schemas/dataset.py b/src/fidesops/ops/schemas/dataset.py similarity index 95% rename from src/fidesops/schemas/dataset.py rename to src/fidesops/ops/schemas/dataset.py index 13150bd122..2787e75bf0 100644 --- a/src/fidesops/schemas/dataset.py +++ b/src/fidesops/ops/schemas/dataset.py @@ -3,16 +3,16 @@ from fideslang.models import Dataset, DatasetCollection, DatasetFieldBase from pydantic import BaseModel, ConstrainedStr, validator -from fidesops.common_exceptions import ( +from fidesops.ops.common_exceptions import ( InvalidDataLengthValidationError, InvalidDataTypeValidationError, ) -from fidesops.graph.config import EdgeDirection -from fidesops.graph.data_type import is_valid_data_type, parse_data_type_string -from fidesops.schemas.api import BulkResponse, BulkUpdateFailed -from fidesops.schemas.base_class import BaseSchema -from fidesops.schemas.shared_schemas import FidesOpsKey -from fidesops.util.data_category import _validate_data_category +from fidesops.ops.graph.config import EdgeDirection +from fidesops.ops.graph.data_type import is_valid_data_type, parse_data_type_string +from fidesops.ops.schemas.api import BulkResponse, BulkUpdateFailed +from fidesops.ops.schemas.base_class import BaseSchema +from fidesops.ops.schemas.shared_schemas import FidesOpsKey +from fidesops.ops.util.data_category import _validate_data_category def _valid_data_categories( diff --git a/src/fidesops/schemas/drp_privacy_request.py b/src/fidesops/ops/schemas/drp_privacy_request.py similarity index 94% rename from src/fidesops/schemas/drp_privacy_request.py rename to src/fidesops/ops/schemas/drp_privacy_request.py index e824a6e27a..feca521166 100644 --- a/src/fidesops/schemas/drp_privacy_request.py +++ b/src/fidesops/ops/schemas/drp_privacy_request.py @@ -3,8 +3,8 @@ from pydantic import validator -from fidesops.models.policy import DrpAction -from fidesops.schemas.base_class import BaseSchema +from fidesops.ops.models.policy import DrpAction +from fidesops.ops.schemas.base_class import BaseSchema DRP_VERSION = "0.5" diff --git a/src/fidesops/schemas/encryption_request.py b/src/fidesops/ops/schemas/encryption_request.py similarity index 86% rename from src/fidesops/schemas/encryption_request.py rename to src/fidesops/ops/schemas/encryption_request.py index a31a9cdd1d..c79567b884 100644 --- a/src/fidesops/schemas/encryption_request.py +++ b/src/fidesops/ops/schemas/encryption_request.py @@ -1,7 +1,7 @@ from pydantic import BaseModel, validator -from fidesops.core.config import config -from fidesops.util.encryption.aes_gcm_encryption_scheme import verify_encryption_key +from fidesops.ops.core.config import config +from fidesops.ops.util.encryption.aes_gcm_encryption_scheme import verify_encryption_key class AesEncryptionRequest(BaseModel): diff --git a/src/fidesops/schemas/external_https.py b/src/fidesops/ops/schemas/external_https.py similarity index 92% rename from src/fidesops/schemas/external_https.py rename to src/fidesops/ops/schemas/external_https.py index f66f1ff7bc..163e527746 100644 --- a/src/fidesops/schemas/external_https.py +++ b/src/fidesops/ops/schemas/external_https.py @@ -3,8 +3,8 @@ from pydantic import BaseModel -from fidesops.models.policy import WebhookDirection -from fidesops.schemas.redis_cache import PrivacyRequestIdentity +from fidesops.ops.models.policy import WebhookDirection +from fidesops.ops.schemas.redis_cache import PrivacyRequestIdentity class CallbackType(Enum): diff --git a/src/fidesops/schemas/saas/__init__.py b/src/fidesops/ops/schemas/masking/__init__.py similarity index 100% rename from src/fidesops/schemas/saas/__init__.py rename to src/fidesops/ops/schemas/masking/__init__.py diff --git a/src/fidesops/schemas/masking/masking_api.py b/src/fidesops/ops/schemas/masking/masking_api.py similarity index 86% rename from src/fidesops/schemas/masking/masking_api.py rename to src/fidesops/ops/schemas/masking/masking_api.py index e5f8ab9302..f77f8c04b8 100644 --- a/src/fidesops/schemas/masking/masking_api.py +++ b/src/fidesops/ops/schemas/masking/masking_api.py @@ -2,7 +2,7 @@ from pydantic import BaseModel -from fidesops.schemas.policy import PolicyMaskingSpec +from fidesops.ops.schemas.policy import PolicyMaskingSpec class MaskingAPIRequest(BaseModel): diff --git a/src/fidesops/schemas/masking/masking_configuration.py b/src/fidesops/ops/schemas/masking/masking_configuration.py similarity index 100% rename from src/fidesops/schemas/masking/masking_configuration.py rename to src/fidesops/ops/schemas/masking/masking_configuration.py diff --git a/src/fidesops/schemas/masking/masking_secrets.py b/src/fidesops/ops/schemas/masking/masking_secrets.py similarity index 100% rename from src/fidesops/schemas/masking/masking_secrets.py rename to src/fidesops/ops/schemas/masking/masking_secrets.py diff --git a/src/fidesops/schemas/masking/masking_strategy_description.py b/src/fidesops/ops/schemas/masking/masking_strategy_description.py similarity index 100% rename from src/fidesops/schemas/masking/masking_strategy_description.py rename to src/fidesops/ops/schemas/masking/masking_strategy_description.py diff --git a/src/fidesops/schemas/msg.py b/src/fidesops/ops/schemas/msg.py similarity index 100% rename from src/fidesops/schemas/msg.py rename to src/fidesops/ops/schemas/msg.py diff --git a/src/fidesops/schemas/policy.py b/src/fidesops/ops/schemas/policy.py similarity index 87% rename from src/fidesops/schemas/policy.py rename to src/fidesops/ops/schemas/policy.py index b2b61d8ee0..98341774d0 100644 --- a/src/fidesops/schemas/policy.py +++ b/src/fidesops/ops/schemas/policy.py @@ -1,12 +1,12 @@ from typing import Dict, List, Optional, Union -from fidesops.models.policy import ActionType, DrpAction -from fidesops.schemas.api import BulkResponse, BulkUpdateFailed -from fidesops.schemas.base_class import BaseSchema -from fidesops.schemas.masking.masking_configuration import FormatPreservationConfig -from fidesops.schemas.shared_schemas import FidesOpsKey -from fidesops.schemas.storage.storage import StorageDestinationResponse -from fidesops.util.data_category import DataCategory +from fidesops.ops.models.policy import ActionType, DrpAction +from fidesops.ops.schemas.api import BulkResponse, BulkUpdateFailed +from fidesops.ops.schemas.base_class import BaseSchema +from fidesops.ops.schemas.masking.masking_configuration import FormatPreservationConfig +from fidesops.ops.schemas.shared_schemas import FidesOpsKey +from fidesops.ops.schemas.storage.storage import StorageDestinationResponse +from fidesops.ops.util.data_category import DataCategory class PolicyMaskingSpec(BaseSchema): diff --git a/src/fidesops/schemas/policy_webhooks.py b/src/fidesops/ops/schemas/policy_webhooks.py similarity index 89% rename from src/fidesops/schemas/policy_webhooks.py rename to src/fidesops/ops/schemas/policy_webhooks.py index fe5f97ca73..a263e73b49 100644 --- a/src/fidesops/schemas/policy_webhooks.py +++ b/src/fidesops/ops/schemas/policy_webhooks.py @@ -1,11 +1,11 @@ from typing import List, Optional -from fidesops.models.policy import WebhookDirection -from fidesops.schemas.base_class import BaseSchema -from fidesops.schemas.connection_configuration.connection_config import ( +from fidesops.ops.models.policy import WebhookDirection +from fidesops.ops.schemas.base_class import BaseSchema +from fidesops.ops.schemas.connection_configuration.connection_config import ( ConnectionConfigurationResponse, ) -from fidesops.schemas.shared_schemas import FidesOpsKey +from fidesops.ops.schemas.shared_schemas import FidesOpsKey class WebhookBase(BaseSchema): diff --git a/src/fidesops/schemas/privacy_request.py b/src/fidesops/ops/schemas/privacy_request.py similarity index 90% rename from src/fidesops/schemas/privacy_request.py rename to src/fidesops/ops/schemas/privacy_request.py index 34dcd0848d..e401b08c6f 100644 --- a/src/fidesops/schemas/privacy_request.py +++ b/src/fidesops/ops/schemas/privacy_request.py @@ -5,19 +5,19 @@ from fideslib.oauth.schemas.user import PrivacyRequestReviewer from pydantic import Field, validator -from fidesops.core.config import config -from fidesops.models.policy import ActionType -from fidesops.models.privacy_request import ( +from fidesops.ops.core.config import config +from fidesops.ops.models.policy import ActionType +from fidesops.ops.models.privacy_request import ( ExecutionLogStatus, PrivacyRequestStatus, StoppedCollection, ) -from fidesops.schemas.api import BulkResponse, BulkUpdateFailed -from fidesops.schemas.base_class import BaseSchema -from fidesops.schemas.policy import PolicyResponse as PolicySchema -from fidesops.schemas.redis_cache import PrivacyRequestIdentity -from fidesops.schemas.shared_schemas import FidesOpsKey -from fidesops.util.encryption.aes_gcm_encryption_scheme import verify_encryption_key +from fidesops.ops.schemas.api import BulkResponse, BulkUpdateFailed +from fidesops.ops.schemas.base_class import BaseSchema +from fidesops.ops.schemas.policy import PolicyResponse as PolicySchema +from fidesops.ops.schemas.redis_cache import PrivacyRequestIdentity +from fidesops.ops.schemas.shared_schemas import FidesOpsKey +from fidesops.ops.util.encryption.aes_gcm_encryption_scheme import verify_encryption_key class PrivacyRequestDRPStatus(EnumType): diff --git a/src/fidesops/schemas/redis_cache.py b/src/fidesops/ops/schemas/redis_cache.py similarity index 85% rename from src/fidesops/schemas/redis_cache.py rename to src/fidesops/ops/schemas/redis_cache.py index c67f0d51a4..52937559a3 100644 --- a/src/fidesops/schemas/redis_cache.py +++ b/src/fidesops/ops/schemas/redis_cache.py @@ -2,7 +2,7 @@ from pydantic import Extra -from fidesops.schemas.base_class import BaseSchema +from fidesops.ops.schemas.base_class import BaseSchema class PrivacyRequestIdentity(BaseSchema): diff --git a/src/fidesops/schemas/storage/__init__.py b/src/fidesops/ops/schemas/saas/__init__.py similarity index 100% rename from src/fidesops/schemas/storage/__init__.py rename to src/fidesops/ops/schemas/saas/__init__.py diff --git a/src/fidesops/schemas/saas/saas_config.py b/src/fidesops/ops/schemas/saas/saas_config.py similarity index 96% rename from src/fidesops/schemas/saas/saas_config.py rename to src/fidesops/ops/schemas/saas/saas_config.py index d01c076a5b..b28c0c6cd9 100644 --- a/src/fidesops/schemas/saas/saas_config.py +++ b/src/fidesops/ops/schemas/saas/saas_config.py @@ -3,17 +3,17 @@ from pydantic import BaseModel, Extra, root_validator, validator -from fidesops.graph.config import ( +from fidesops.ops.graph.config import ( Collection, CollectionAddress, Dataset, FieldAddress, ScalarField, ) -from fidesops.schemas.base_class import BaseSchema -from fidesops.schemas.dataset import FidesCollectionKey, FidesopsDatasetReference -from fidesops.schemas.saas.shared_schemas import HTTPMethod -from fidesops.schemas.shared_schemas import FidesOpsKey +from fidesops.ops.schemas.base_class import BaseSchema +from fidesops.ops.schemas.dataset import FidesCollectionKey, FidesopsDatasetReference +from fidesops.ops.schemas.saas.shared_schemas import HTTPMethod +from fidesops.ops.schemas.shared_schemas import FidesOpsKey class ParamValue(BaseModel): @@ -116,7 +116,7 @@ def validate_request_for_pagination(cls, values: Dict[str, Any]) -> Dict[str, An """ # delay import to avoid cyclic-dependency error - We still ignore the pylint error - from fidesops.service.pagination.pagination_strategy_factory import ( # pylint: disable=R0401 + from fidesops.ops.service.pagination.pagination_strategy_factory import ( # pylint: disable=R0401 get_strategy, ) diff --git a/src/fidesops/schemas/saas/shared_schemas.py b/src/fidesops/ops/schemas/saas/shared_schemas.py similarity index 100% rename from src/fidesops/schemas/saas/shared_schemas.py rename to src/fidesops/ops/schemas/saas/shared_schemas.py diff --git a/src/fidesops/schemas/saas/strategy_configuration.py b/src/fidesops/ops/schemas/saas/strategy_configuration.py similarity index 95% rename from src/fidesops/schemas/saas/strategy_configuration.py rename to src/fidesops/ops/schemas/saas/strategy_configuration.py index 7729110670..6a795de818 100644 --- a/src/fidesops/schemas/saas/strategy_configuration.py +++ b/src/fidesops/ops/schemas/saas/strategy_configuration.py @@ -3,8 +3,8 @@ from pydantic import BaseModel, root_validator, validator -from fidesops.schemas.saas.saas_config import SaaSRequest -from fidesops.schemas.saas.shared_schemas import ConnectorParamRef, IdentityParamRef +from fidesops.ops.schemas.saas.saas_config import SaaSRequest +from fidesops.ops.schemas.saas.shared_schemas import ConnectorParamRef, IdentityParamRef class StrategyConfiguration(BaseModel): diff --git a/src/fidesops/schemas/shared_schemas.py b/src/fidesops/ops/schemas/shared_schemas.py similarity index 100% rename from src/fidesops/schemas/shared_schemas.py rename to src/fidesops/ops/schemas/shared_schemas.py diff --git a/src/fidesops/schemas/third_party/__init__.py b/src/fidesops/ops/schemas/storage/__init__.py similarity index 100% rename from src/fidesops/schemas/third_party/__init__.py rename to src/fidesops/ops/schemas/storage/__init__.py diff --git a/src/fidesops/schemas/storage/data_upload_location_response.py b/src/fidesops/ops/schemas/storage/data_upload_location_response.py similarity index 100% rename from src/fidesops/schemas/storage/data_upload_location_response.py rename to src/fidesops/ops/schemas/storage/data_upload_location_response.py diff --git a/src/fidesops/schemas/storage/storage.py b/src/fidesops/ops/schemas/storage/storage.py similarity index 97% rename from src/fidesops/schemas/storage/storage.py rename to src/fidesops/ops/schemas/storage/storage.py index 8014374ca0..c2b35b86ec 100644 --- a/src/fidesops/schemas/storage/storage.py +++ b/src/fidesops/ops/schemas/storage/storage.py @@ -5,8 +5,8 @@ from pydantic import Extra, ValidationError, root_validator, validator from pydantic.main import BaseModel -from fidesops.schemas.api import BulkResponse, BulkUpdateFailed -from fidesops.schemas.shared_schemas import FidesOpsKey +from fidesops.ops.schemas.api import BulkResponse, BulkUpdateFailed +from fidesops.ops.schemas.shared_schemas import FidesOpsKey logger = logging.getLogger(__name__) diff --git a/src/fidesops/schemas/storage/storage_secrets_docs_only.py b/src/fidesops/ops/schemas/storage/storage_secrets_docs_only.py similarity index 70% rename from src/fidesops/schemas/storage/storage_secrets_docs_only.py rename to src/fidesops/ops/schemas/storage/storage_secrets_docs_only.py index a69b6901af..a227d13d87 100644 --- a/src/fidesops/schemas/storage/storage_secrets_docs_only.py +++ b/src/fidesops/ops/schemas/storage/storage_secrets_docs_only.py @@ -1,7 +1,10 @@ from typing import Union -from fidesops.schemas.base_class import NoValidationSchema -from fidesops.schemas.storage.storage import StorageSecretsOnetrust, StorageSecretsS3 +from fidesops.ops.schemas.base_class import NoValidationSchema +from fidesops.ops.schemas.storage.storage import ( + StorageSecretsOnetrust, + StorageSecretsS3, +) class StorageSecretsS3Docs(StorageSecretsS3, NoValidationSchema): diff --git a/src/fidesops/service/__init__.py b/src/fidesops/ops/schemas/third_party/__init__.py similarity index 100% rename from src/fidesops/service/__init__.py rename to src/fidesops/ops/schemas/third_party/__init__.py diff --git a/src/fidesops/schemas/third_party/onetrust.py b/src/fidesops/ops/schemas/third_party/onetrust.py similarity index 98% rename from src/fidesops/schemas/third_party/onetrust.py rename to src/fidesops/ops/schemas/third_party/onetrust.py index cf11f26988..8af06fd437 100644 --- a/src/fidesops/schemas/third_party/onetrust.py +++ b/src/fidesops/ops/schemas/third_party/onetrust.py @@ -1,7 +1,7 @@ from enum import Enum from typing import Dict, List, Optional -from fidesops.schemas.base_class import BaseSchema +from fidesops.ops.schemas.base_class import BaseSchema class OneTrustSubtaskStatus(Enum): diff --git a/src/fidesops/schemas/user_permission.py b/src/fidesops/ops/schemas/user_permission.py similarity index 95% rename from src/fidesops/schemas/user_permission.py rename to src/fidesops/ops/schemas/user_permission.py index f3843c1793..971502bdfa 100644 --- a/src/fidesops/schemas/user_permission.py +++ b/src/fidesops/ops/schemas/user_permission.py @@ -13,7 +13,7 @@ from pydantic import validator from starlette.status import HTTP_422_UNPROCESSABLE_ENTITY -from fidesops.api.v1.scope_registry import SCOPE_REGISTRY +from fidesops.ops.api.v1.scope_registry import SCOPE_REGISTRY class UserPermissionsCreate(UserPermissionsCreateLib): diff --git a/src/fidesops/service/authentication/__init__.py b/src/fidesops/ops/service/__init__.py similarity index 100% rename from src/fidesops/service/authentication/__init__.py rename to src/fidesops/ops/service/__init__.py diff --git a/src/fidesops/service/connectors/saas/__init__.py b/src/fidesops/ops/service/authentication/__init__.py similarity index 100% rename from src/fidesops/service/connectors/saas/__init__.py rename to src/fidesops/ops/service/authentication/__init__.py diff --git a/src/fidesops/service/authentication/authentication_strategy.py b/src/fidesops/ops/service/authentication/authentication_strategy.py similarity index 78% rename from src/fidesops/service/authentication/authentication_strategy.py rename to src/fidesops/ops/service/authentication/authentication_strategy.py index a9a52c86ed..7934171f04 100644 --- a/src/fidesops/service/authentication/authentication_strategy.py +++ b/src/fidesops/ops/service/authentication/authentication_strategy.py @@ -2,8 +2,8 @@ from requests import PreparedRequest -from fidesops.models.connectionconfig import ConnectionConfig -from fidesops.schemas.saas.strategy_configuration import StrategyConfiguration +from fidesops.ops.models.connectionconfig import ConnectionConfig +from fidesops.ops.schemas.saas.strategy_configuration import StrategyConfiguration class AuthenticationStrategy(ABC): diff --git a/src/fidesops/service/authentication/authentication_strategy_basic.py b/src/fidesops/ops/service/authentication/authentication_strategy_basic.py similarity index 81% rename from src/fidesops/service/authentication/authentication_strategy_basic.py rename to src/fidesops/ops/service/authentication/authentication_strategy_basic.py index 5d5a4b9665..79f9716fbc 100644 --- a/src/fidesops/service/authentication/authentication_strategy_basic.py +++ b/src/fidesops/ops/service/authentication/authentication_strategy_basic.py @@ -1,14 +1,14 @@ from requests import PreparedRequest -from fidesops.models.connectionconfig import ConnectionConfig -from fidesops.schemas.saas.strategy_configuration import ( +from fidesops.ops.models.connectionconfig import ConnectionConfig +from fidesops.ops.schemas.saas.strategy_configuration import ( BasicAuthenticationConfiguration, StrategyConfiguration, ) -from fidesops.service.authentication.authentication_strategy import ( +from fidesops.ops.service.authentication.authentication_strategy import ( AuthenticationStrategy, ) -from fidesops.util.saas_util import assign_placeholders +from fidesops.ops.util.saas_util import assign_placeholders class BasicAuthenticationStrategy(AuthenticationStrategy): diff --git a/src/fidesops/service/authentication/authentication_strategy_bearer.py b/src/fidesops/ops/service/authentication/authentication_strategy_bearer.py similarity index 79% rename from src/fidesops/service/authentication/authentication_strategy_bearer.py rename to src/fidesops/ops/service/authentication/authentication_strategy_bearer.py index bbfea19bcd..c932a6b0e4 100644 --- a/src/fidesops/service/authentication/authentication_strategy_bearer.py +++ b/src/fidesops/ops/service/authentication/authentication_strategy_bearer.py @@ -1,14 +1,14 @@ from requests import PreparedRequest -from fidesops.models.connectionconfig import ConnectionConfig -from fidesops.schemas.saas.strategy_configuration import ( +from fidesops.ops.models.connectionconfig import ConnectionConfig +from fidesops.ops.schemas.saas.strategy_configuration import ( BearerAuthenticationConfiguration, StrategyConfiguration, ) -from fidesops.service.authentication.authentication_strategy import ( +from fidesops.ops.service.authentication.authentication_strategy import ( AuthenticationStrategy, ) -from fidesops.util.saas_util import assign_placeholders +from fidesops.ops.util.saas_util import assign_placeholders class BearerAuthenticationStrategy(AuthenticationStrategy): diff --git a/src/fidesops/service/authentication/authentication_strategy_factory.py b/src/fidesops/ops/service/authentication/authentication_strategy_factory.py similarity index 74% rename from src/fidesops/service/authentication/authentication_strategy_factory.py rename to src/fidesops/ops/service/authentication/authentication_strategy_factory.py index 7eba7d541a..9912f38816 100644 --- a/src/fidesops/service/authentication/authentication_strategy_factory.py +++ b/src/fidesops/ops/service/authentication/authentication_strategy_factory.py @@ -4,22 +4,22 @@ from pydantic import ValidationError -from fidesops.common_exceptions import NoSuchStrategyException -from fidesops.common_exceptions import ValidationError as FidesopsValidationError -from fidesops.schemas.saas.strategy_configuration import StrategyConfiguration -from fidesops.service.authentication.authentication_strategy import ( +from fidesops.ops.common_exceptions import NoSuchStrategyException +from fidesops.ops.common_exceptions import ValidationError as FidesopsValidationError +from fidesops.ops.schemas.saas.strategy_configuration import StrategyConfiguration +from fidesops.ops.service.authentication.authentication_strategy import ( AuthenticationStrategy, ) -from fidesops.service.authentication.authentication_strategy_basic import ( +from fidesops.ops.service.authentication.authentication_strategy_basic import ( BasicAuthenticationStrategy, ) -from fidesops.service.authentication.authentication_strategy_bearer import ( +from fidesops.ops.service.authentication.authentication_strategy_bearer import ( BearerAuthenticationStrategy, ) -from fidesops.service.authentication.authentication_strategy_oauth2 import ( +from fidesops.ops.service.authentication.authentication_strategy_oauth2 import ( OAuth2AuthenticationStrategy, ) -from fidesops.service.authentication.authentication_strategy_query_param import ( +from fidesops.ops.service.authentication.authentication_strategy_query_param import ( QueryParamAuthenticationStrategy, ) diff --git a/src/fidesops/service/authentication/authentication_strategy_oauth2.py b/src/fidesops/ops/service/authentication/authentication_strategy_oauth2.py similarity index 94% rename from src/fidesops/service/authentication/authentication_strategy_oauth2.py rename to src/fidesops/ops/service/authentication/authentication_strategy_oauth2.py index 710eb99601..1350b9c75c 100644 --- a/src/fidesops/service/authentication/authentication_strategy_oauth2.py +++ b/src/fidesops/ops/service/authentication/authentication_strategy_oauth2.py @@ -7,21 +7,23 @@ from requests import PreparedRequest from sqlalchemy.orm import Session -from fidesops.common_exceptions import FidesopsException, OAuth2TokenException -from fidesops.core.config import config -from fidesops.models.authentication_request import AuthenticationRequest -from fidesops.models.connectionconfig import ConnectionConfig -from fidesops.schemas.saas.saas_config import ClientConfig, SaaSRequest -from fidesops.schemas.saas.strategy_configuration import ( +from fidesops.ops.common_exceptions import FidesopsException, OAuth2TokenException +from fidesops.ops.core.config import config +from fidesops.ops.models.authentication_request import AuthenticationRequest +from fidesops.ops.models.connectionconfig import ConnectionConfig +from fidesops.ops.schemas.saas.saas_config import ClientConfig, SaaSRequest +from fidesops.ops.schemas.saas.strategy_configuration import ( OAuth2AuthenticationConfiguration, StrategyConfiguration, ) -from fidesops.service.authentication.authentication_strategy import ( +from fidesops.ops.service.authentication.authentication_strategy import ( AuthenticationStrategy, ) -from fidesops.service.connectors.saas.authenticated_client import AuthenticatedClient -from fidesops.util.logger import NotPii -from fidesops.util.saas_util import assign_placeholders, map_param_values +from fidesops.ops.service.connectors.saas.authenticated_client import ( + AuthenticatedClient, +) +from fidesops.ops.util.logger import NotPii +from fidesops.ops.util.saas_util import assign_placeholders, map_param_values logger = logging.getLogger(__name__) diff --git a/src/fidesops/service/authentication/authentication_strategy_query_param.py b/src/fidesops/ops/service/authentication/authentication_strategy_query_param.py similarity index 77% rename from src/fidesops/service/authentication/authentication_strategy_query_param.py rename to src/fidesops/ops/service/authentication/authentication_strategy_query_param.py index 599e79ef9a..5aaf5d1c38 100644 --- a/src/fidesops/service/authentication/authentication_strategy_query_param.py +++ b/src/fidesops/ops/service/authentication/authentication_strategy_query_param.py @@ -1,15 +1,15 @@ from requests import PreparedRequest -from fidesops.models.connectionconfig import ConnectionConfig -from fidesops.schemas.saas.strategy_configuration import ( +from fidesops.ops.models.connectionconfig import ConnectionConfig +from fidesops.ops.schemas.saas.strategy_configuration import ( QueryParamAuthenticationConfiguration, StrategyConfiguration, ) -from fidesops.service.authentication.authentication_strategy import ( +from fidesops.ops.service.authentication.authentication_strategy import ( AuthenticationStrategy, ) -from fidesops.util.saas_util import assign_placeholders -from fidesops.util.url_util import set_query_parameter +from fidesops.ops.util.saas_util import assign_placeholders +from fidesops.ops.util.url_util import set_query_parameter class QueryParamAuthenticationStrategy(AuthenticationStrategy): diff --git a/src/fidesops/service/connectors/__init__.py b/src/fidesops/ops/service/connectors/__init__.py similarity index 70% rename from src/fidesops/service/connectors/__init__.py rename to src/fidesops/ops/service/connectors/__init__.py index ce48a08517..155e0fb1b4 100644 --- a/src/fidesops/service/connectors/__init__.py +++ b/src/fidesops/ops/service/connectors/__init__.py @@ -1,12 +1,12 @@ from typing import Any, Dict -from fidesops.models.connectionconfig import ConnectionConfig, ConnectionType -from fidesops.service.connectors.base_connector import BaseConnector -from fidesops.service.connectors.http_connector import HTTPSConnector -from fidesops.service.connectors.manual_connector import ManualConnector -from fidesops.service.connectors.mongodb_connector import MongoDBConnector -from fidesops.service.connectors.saas_connector import SaaSConnector -from fidesops.service.connectors.sql_connector import ( +from fidesops.ops.models.connectionconfig import ConnectionConfig, ConnectionType +from fidesops.ops.service.connectors.base_connector import BaseConnector +from fidesops.ops.service.connectors.http_connector import HTTPSConnector +from fidesops.ops.service.connectors.manual_connector import ManualConnector +from fidesops.ops.service.connectors.mongodb_connector import MongoDBConnector +from fidesops.ops.service.connectors.saas_connector import SaaSConnector +from fidesops.ops.service.connectors.sql_connector import ( BigQueryConnector, MariaDBConnector, MicrosoftSQLServerConnector, diff --git a/src/fidesops/service/connectors/base_connector.py b/src/fidesops/ops/service/connectors/base_connector.py similarity index 82% rename from src/fidesops/service/connectors/base_connector.py rename to src/fidesops/ops/service/connectors/base_connector.py index 9e10bb4871..f012990f4b 100644 --- a/src/fidesops/service/connectors/base_connector.py +++ b/src/fidesops/ops/service/connectors/base_connector.py @@ -2,13 +2,13 @@ from abc import ABC, abstractmethod from typing import Any, Dict, Generic, List, Optional, TypeVar -from fidesops.core.config import config -from fidesops.graph.traversal import TraversalNode -from fidesops.models.connectionconfig import ConnectionConfig, ConnectionTestStatus -from fidesops.models.policy import Policy -from fidesops.models.privacy_request import PrivacyRequest -from fidesops.service.connectors.query_config import QueryConfig -from fidesops.util.collection_util import Row +from fidesops.ops.core.config import config +from fidesops.ops.graph.traversal import TraversalNode +from fidesops.ops.models.connectionconfig import ConnectionConfig, ConnectionTestStatus +from fidesops.ops.models.policy import Policy +from fidesops.ops.models.privacy_request import PrivacyRequest +from fidesops.ops.service.connectors.query_config import QueryConfig +from fidesops.ops.util.collection_util import Row logger = logging.getLogger(__name__) DB_CONNECTOR_TYPE = TypeVar("DB_CONNECTOR_TYPE") @@ -18,9 +18,9 @@ class BaseConnector(Generic[DB_CONNECTOR_TYPE], ABC): """Abstract BaseConnector class containing the methods to interact with your configured connection. How to use example: - from fidesops.db.session import get_db_session - from fidesops.models.connectionconfig import ConnectionConfig - from fidesops.service.connectors import get_connector + from fidesops.ops.db.session import get_db_session + from fidesops.ops.models.connectionconfig import ConnectionConfig + from fidesops.ops.service.connectors import get_connector SessionLocal = get_db_session(config) db = SessionLocal() diff --git a/src/fidesops/service/connectors/http_connector.py b/src/fidesops/ops/service/connectors/http_connector.py similarity index 84% rename from src/fidesops/service/connectors/http_connector.py rename to src/fidesops/ops/service/connectors/http_connector.py index 907fd1dd57..ff59fa0405 100644 --- a/src/fidesops/service/connectors/http_connector.py +++ b/src/fidesops/ops/service/connectors/http_connector.py @@ -5,15 +5,15 @@ import requests from starlette.status import HTTP_500_INTERNAL_SERVER_ERROR -from fidesops.common_exceptions import ClientUnsuccessfulException -from fidesops.graph.traversal import TraversalNode -from fidesops.models.connectionconfig import ConnectionTestStatus -from fidesops.models.policy import Policy -from fidesops.models.privacy_request import PrivacyRequest -from fidesops.schemas.connection_configuration import HttpsSchema -from fidesops.service.connectors.base_connector import BaseConnector -from fidesops.service.connectors.query_config import QueryConfig -from fidesops.util.collection_util import Row +from fidesops.ops.common_exceptions import ClientUnsuccessfulException +from fidesops.ops.graph.traversal import TraversalNode +from fidesops.ops.models.connectionconfig import ConnectionTestStatus +from fidesops.ops.models.policy import Policy +from fidesops.ops.models.privacy_request import PrivacyRequest +from fidesops.ops.schemas.connection_configuration import HttpsSchema +from fidesops.ops.service.connectors.base_connector import BaseConnector +from fidesops.ops.service.connectors.query_config import QueryConfig +from fidesops.ops.util.collection_util import Row logger = logging.getLogger(__name__) diff --git a/src/fidesops/service/connectors/manual_connector.py b/src/fidesops/ops/service/connectors/manual_connector.py similarity index 89% rename from src/fidesops/service/connectors/manual_connector.py rename to src/fidesops/ops/service/connectors/manual_connector.py index 3939fd07e5..fb2f45bf06 100644 --- a/src/fidesops/service/connectors/manual_connector.py +++ b/src/fidesops/ops/service/connectors/manual_connector.py @@ -1,13 +1,13 @@ import logging from typing import Any, Dict, List, Optional -from fidesops.common_exceptions import PrivacyRequestPaused -from fidesops.graph.traversal import TraversalNode -from fidesops.models.policy import PausedStep, Policy -from fidesops.models.privacy_request import ManualAction, PrivacyRequest -from fidesops.service.connectors.base_connector import BaseConnector -from fidesops.service.connectors.query_config import ManualQueryConfig -from fidesops.util.collection_util import Row +from fidesops.ops.common_exceptions import PrivacyRequestPaused +from fidesops.ops.graph.traversal import TraversalNode +from fidesops.ops.models.policy import PausedStep, Policy +from fidesops.ops.models.privacy_request import ManualAction, PrivacyRequest +from fidesops.ops.service.connectors.base_connector import BaseConnector +from fidesops.ops.service.connectors.query_config import ManualQueryConfig +from fidesops.ops.util.collection_util import Row logger = logging.getLogger(__name__) diff --git a/src/fidesops/service/connectors/mongodb_connector.py b/src/fidesops/ops/service/connectors/mongodb_connector.py similarity index 89% rename from src/fidesops/service/connectors/mongodb_connector.py rename to src/fidesops/ops/service/connectors/mongodb_connector.py index 742321a2a5..111845fce6 100644 --- a/src/fidesops/service/connectors/mongodb_connector.py +++ b/src/fidesops/ops/service/connectors/mongodb_connector.py @@ -4,17 +4,17 @@ from pymongo import MongoClient from pymongo.errors import OperationFailure, ServerSelectionTimeoutError -from fidesops.common_exceptions import ConnectionException -from fidesops.graph.traversal import Row, TraversalNode -from fidesops.models.connectionconfig import ConnectionTestStatus -from fidesops.models.policy import Policy -from fidesops.models.privacy_request import PrivacyRequest -from fidesops.schemas.connection_configuration.connection_secrets_mongodb import ( +from fidesops.ops.common_exceptions import ConnectionException +from fidesops.ops.graph.traversal import Row, TraversalNode +from fidesops.ops.models.connectionconfig import ConnectionTestStatus +from fidesops.ops.models.policy import Policy +from fidesops.ops.models.privacy_request import PrivacyRequest +from fidesops.ops.schemas.connection_configuration.connection_secrets_mongodb import ( MongoDBSchema, ) -from fidesops.service.connectors.base_connector import BaseConnector -from fidesops.service.connectors.query_config import MongoQueryConfig, QueryConfig -from fidesops.util.logger import NotPii +from fidesops.ops.service.connectors.base_connector import BaseConnector +from fidesops.ops.service.connectors.query_config import MongoQueryConfig, QueryConfig +from fidesops.ops.util.logger import NotPii logger = logging.getLogger(__name__) diff --git a/src/fidesops/service/connectors/query_config.py b/src/fidesops/ops/service/connectors/query_config.py similarity index 97% rename from src/fidesops/service/connectors/query_config.py rename to src/fidesops/ops/service/connectors/query_config.py index 5abf6f1839..314faff247 100644 --- a/src/fidesops/service/connectors/query_config.py +++ b/src/fidesops/ops/service/connectors/query_config.py @@ -9,29 +9,29 @@ from sqlalchemy.sql import Executable, Update # type: ignore from sqlalchemy.sql.elements import ColumnElement, TextClause -from fidesops.graph.config import ( +from fidesops.ops.graph.config import ( ROOT_COLLECTION_ADDRESS, CollectionAddress, Field, FieldPath, MaskingOverride, ) -from fidesops.graph.traversal import Row, TraversalNode -from fidesops.models.policy import ActionType, Policy, Rule -from fidesops.models.privacy_request import ManualAction, PrivacyRequest -from fidesops.service.masking.strategy.masking_strategy import MaskingStrategy -from fidesops.service.masking.strategy.masking_strategy_factory import ( +from fidesops.ops.graph.traversal import Row, TraversalNode +from fidesops.ops.models.policy import ActionType, Policy, Rule +from fidesops.ops.models.privacy_request import ManualAction, PrivacyRequest +from fidesops.ops.service.masking.strategy.masking_strategy import MaskingStrategy +from fidesops.ops.service.masking.strategy.masking_strategy_factory import ( MaskingStrategyFactory, ) -from fidesops.service.masking.strategy.masking_strategy_nullify import ( +from fidesops.ops.service.masking.strategy.masking_strategy_nullify import ( NULL_REWRITE_STRATEGY_NAME, ) -from fidesops.task.refine_target_path import ( +from fidesops.ops.task.refine_target_path import ( build_refined_target_paths, join_detailed_path, ) -from fidesops.util.collection_util import append, filter_nonempty_values -from fidesops.util.querytoken import QueryToken +from fidesops.ops.util.collection_util import append, filter_nonempty_values +from fidesops.ops.util.querytoken import QueryToken logger = logging.getLogger(__name__) T = TypeVar("T") @@ -58,7 +58,7 @@ def build_rule_target_field_paths( """ Return dictionary of rules mapped to update-able field paths on a given collection Example: - {: [FieldPath('name'), FieldPath('code'), FieldPath('ccn')]} + {: [FieldPath('name'), FieldPath('code'), FieldPath('ccn')]} """ rule_updates: Dict[Rule, List[FieldPath]] = {} for rule in policy.rules: diff --git a/src/fidesops/service/drp/__init__.py b/src/fidesops/ops/service/connectors/saas/__init__.py similarity index 100% rename from src/fidesops/service/drp/__init__.py rename to src/fidesops/ops/service/connectors/saas/__init__.py diff --git a/src/fidesops/service/connectors/saas/authenticated_client.py b/src/fidesops/ops/service/connectors/saas/authenticated_client.py similarity index 89% rename from src/fidesops/service/connectors/saas/authenticated_client.py rename to src/fidesops/ops/service/connectors/saas/authenticated_client.py index e652c74700..a61a1fde2a 100644 --- a/src/fidesops/service/connectors/saas/authenticated_client.py +++ b/src/fidesops/ops/service/connectors/saas/authenticated_client.py @@ -5,13 +5,16 @@ from requests import PreparedRequest, Request, Response, Session -from fidesops.common_exceptions import ClientUnsuccessfulException, ConnectionException -from fidesops.core.config import config +from fidesops.ops.common_exceptions import ( + ClientUnsuccessfulException, + ConnectionException, +) +from fidesops.ops.core.config import config if TYPE_CHECKING: - from fidesops.models.connectionconfig import ConnectionConfig - from fidesops.schemas.saas.saas_config import ClientConfig - from fidesops.schemas.saas.shared_schemas import SaaSRequestParams + from fidesops.ops.models.connectionconfig import ConnectionConfig + from fidesops.ops.schemas.saas.saas_config import ClientConfig + from fidesops.ops.schemas.saas.shared_schemas import SaaSRequestParams logger = logging.getLogger(__name__) @@ -49,7 +52,7 @@ def get_authenticated_request( incoming path, headers, query, and body params. """ - from fidesops.service.authentication.authentication_strategy_factory import ( # pylint: disable=R0401 + from fidesops.ops.service.authentication.authentication_strategy_factory import ( # pylint: disable=R0401 get_strategy, ) diff --git a/src/fidesops/service/connectors/saas_connector.py b/src/fidesops/ops/service/connectors/saas_connector.py similarity index 92% rename from src/fidesops/service/connectors/saas_connector.py rename to src/fidesops/ops/service/connectors/saas_connector.py index 6dd24659c2..20030bd133 100644 --- a/src/fidesops/service/connectors/saas_connector.py +++ b/src/fidesops/ops/service/connectors/saas_connector.py @@ -5,31 +5,33 @@ import pydash from requests import Response -from fidesops.common_exceptions import FidesopsException, PostProcessingException -from fidesops.graph.traversal import Row, TraversalNode -from fidesops.models.connectionconfig import ConnectionConfig, ConnectionTestStatus -from fidesops.models.policy import Policy -from fidesops.models.privacy_request import PrivacyRequest -from fidesops.schemas.saas.saas_config import ClientConfig, SaaSRequest -from fidesops.schemas.saas.shared_schemas import SaaSRequestParams -from fidesops.service.connectors.base_connector import BaseConnector -from fidesops.service.connectors.saas.authenticated_client import AuthenticatedClient -from fidesops.service.connectors.saas_query_config import SaaSQueryConfig -from fidesops.service.pagination.pagination_strategy import PaginationStrategy -from fidesops.service.pagination.pagination_strategy_factory import ( +from fidesops.ops.common_exceptions import FidesopsException, PostProcessingException +from fidesops.ops.graph.traversal import Row, TraversalNode +from fidesops.ops.models.connectionconfig import ConnectionConfig, ConnectionTestStatus +from fidesops.ops.models.policy import Policy +from fidesops.ops.models.privacy_request import PrivacyRequest +from fidesops.ops.schemas.saas.saas_config import ClientConfig, SaaSRequest +from fidesops.ops.schemas.saas.shared_schemas import SaaSRequestParams +from fidesops.ops.service.connectors.base_connector import BaseConnector +from fidesops.ops.service.connectors.saas.authenticated_client import ( + AuthenticatedClient, +) +from fidesops.ops.service.connectors.saas_query_config import SaaSQueryConfig +from fidesops.ops.service.pagination.pagination_strategy import PaginationStrategy +from fidesops.ops.service.pagination.pagination_strategy_factory import ( get_strategy as get_pagination_strategy, ) -from fidesops.service.processors.post_processor_strategy.post_processor_strategy import ( +from fidesops.ops.service.processors.post_processor_strategy.post_processor_strategy import ( PostProcessorStrategy, ) -from fidesops.service.processors.post_processor_strategy.post_processor_strategy_factory import ( +from fidesops.ops.service.processors.post_processor_strategy.post_processor_strategy_factory import ( get_strategy as get_postprocessor_strategy, ) -from fidesops.service.saas_request.saas_request_override_factory import ( +from fidesops.ops.service.saas_request.saas_request_override_factory import ( SaaSRequestOverrideFactory, SaaSRequestType, ) -from fidesops.util.saas_util import assign_placeholders, map_param_values +from fidesops.ops.util.saas_util import assign_placeholders, map_param_values logger = logging.getLogger(__name__) diff --git a/src/fidesops/service/connectors/saas_query_config.py b/src/fidesops/ops/service/connectors/saas_query_config.py similarity index 94% rename from src/fidesops/service/connectors/saas_query_config.py rename to src/fidesops/ops/service/connectors/saas_query_config.py index 7efccdc45a..c29757eb0d 100644 --- a/src/fidesops/service/connectors/saas_query_config.py +++ b/src/fidesops/ops/service/connectors/saas_query_config.py @@ -6,18 +6,18 @@ import pydash -from fidesops.common_exceptions import FidesopsException -from fidesops.core.config import config -from fidesops.graph.config import ScalarField -from fidesops.graph.traversal import TraversalNode -from fidesops.models.policy import Policy -from fidesops.models.privacy_request import PrivacyRequest -from fidesops.schemas.saas.saas_config import Endpoint, SaaSRequest -from fidesops.schemas.saas.shared_schemas import SaaSRequestParams -from fidesops.service.connectors.query_config import QueryConfig -from fidesops.util import saas_util -from fidesops.util.collection_util import Row, merge_dicts -from fidesops.util.saas_util import FIDESOPS_GROUPED_INPUTS, unflatten_dict +from fidesops.ops.common_exceptions import FidesopsException +from fidesops.ops.core.config import config +from fidesops.ops.graph.config import ScalarField +from fidesops.ops.graph.traversal import TraversalNode +from fidesops.ops.models.policy import Policy +from fidesops.ops.models.privacy_request import PrivacyRequest +from fidesops.ops.schemas.saas.saas_config import Endpoint, SaaSRequest +from fidesops.ops.schemas.saas.shared_schemas import SaaSRequestParams +from fidesops.ops.service.connectors.query_config import QueryConfig +from fidesops.ops.util import saas_util +from fidesops.ops.util.collection_util import Row, merge_dicts +from fidesops.ops.util.saas_util import FIDESOPS_GROUPED_INPUTS, unflatten_dict logger = logging.getLogger(__name__) diff --git a/src/fidesops/service/connectors/sql_connector.py b/src/fidesops/ops/service/connectors/sql_connector.py similarity index 95% rename from src/fidesops/service/connectors/sql_connector.py rename to src/fidesops/ops/service/connectors/sql_connector.py index 786180ecd5..0c2e12b7de 100644 --- a/src/fidesops/service/connectors/sql_connector.py +++ b/src/fidesops/ops/service/connectors/sql_connector.py @@ -16,28 +16,28 @@ from sqlalchemy.sql import Executable # type: ignore from sqlalchemy.sql.elements import TextClause -from fidesops.common_exceptions import ConnectionException -from fidesops.graph.traversal import Row, TraversalNode -from fidesops.models.connectionconfig import ConnectionTestStatus -from fidesops.models.policy import Policy -from fidesops.models.privacy_request import PrivacyRequest -from fidesops.schemas.connection_configuration import ( +from fidesops.ops.common_exceptions import ConnectionException +from fidesops.ops.graph.traversal import Row, TraversalNode +from fidesops.ops.models.connectionconfig import ConnectionTestStatus +from fidesops.ops.models.policy import Policy +from fidesops.ops.models.privacy_request import PrivacyRequest +from fidesops.ops.schemas.connection_configuration import ( MicrosoftSQLServerSchema, PostgreSQLSchema, RedshiftSchema, SnowflakeSchema, ) -from fidesops.schemas.connection_configuration.connection_secrets_bigquery import ( +from fidesops.ops.schemas.connection_configuration.connection_secrets_bigquery import ( BigQuerySchema, ) -from fidesops.schemas.connection_configuration.connection_secrets_mariadb import ( +from fidesops.ops.schemas.connection_configuration.connection_secrets_mariadb import ( MariaDBSchema, ) -from fidesops.schemas.connection_configuration.connection_secrets_mysql import ( +from fidesops.ops.schemas.connection_configuration.connection_secrets_mysql import ( MySQLSchema, ) -from fidesops.service.connectors.base_connector import BaseConnector -from fidesops.service.connectors.query_config import ( +from fidesops.ops.service.connectors.base_connector import BaseConnector +from fidesops.ops.service.connectors.query_config import ( BigQueryQueryConfig, MicrosoftSQLServerQueryConfig, RedshiftQueryConfig, diff --git a/src/fidesops/service/masking/strategy/__init__.py b/src/fidesops/ops/service/drp/__init__.py similarity index 100% rename from src/fidesops/service/masking/strategy/__init__.py rename to src/fidesops/ops/service/drp/__init__.py diff --git a/src/fidesops/service/drp/drp_fidesops_mapper.py b/src/fidesops/ops/service/drp/drp_fidesops_mapper.py similarity index 88% rename from src/fidesops/service/drp/drp_fidesops_mapper.py rename to src/fidesops/ops/service/drp/drp_fidesops_mapper.py index e0dc2f9c75..a2c0d3b26e 100644 --- a/src/fidesops/service/drp/drp_fidesops_mapper.py +++ b/src/fidesops/ops/service/drp/drp_fidesops_mapper.py @@ -1,10 +1,13 @@ import logging from typing import Dict -from fidesops.models.privacy_request import PrivacyRequestStatus, ProvidedIdentityType -from fidesops.schemas.drp_privacy_request import DrpIdentity -from fidesops.schemas.privacy_request import PrivacyRequestDRPStatus -from fidesops.schemas.redis_cache import PrivacyRequestIdentity +from fidesops.ops.models.privacy_request import ( + PrivacyRequestStatus, + ProvidedIdentityType, +) +from fidesops.ops.schemas.drp_privacy_request import DrpIdentity +from fidesops.ops.schemas.privacy_request import PrivacyRequestDRPStatus +from fidesops.ops.schemas.redis_cache import PrivacyRequestIdentity logger = logging.getLogger(__name__) diff --git a/src/fidesops/service/masking/__init__.py b/src/fidesops/ops/service/masking/__init__.py similarity index 79% rename from src/fidesops/service/masking/__init__.py rename to src/fidesops/ops/service/masking/__init__.py index 53adea3346..79f6a24e03 100644 --- a/src/fidesops/service/masking/__init__.py +++ b/src/fidesops/ops/service/masking/__init__.py @@ -1,4 +1,4 @@ -from fidesops.service.masking.strategy import ( +from fidesops.ops.service.masking.strategy import ( masking_strategy_aes_encrypt, masking_strategy_hash, masking_strategy_hmac, diff --git a/src/fidesops/service/pagination/__init__.py b/src/fidesops/ops/service/masking/strategy/__init__.py similarity index 100% rename from src/fidesops/service/pagination/__init__.py rename to src/fidesops/ops/service/masking/strategy/__init__.py diff --git a/src/fidesops/service/masking/strategy/format_preservation.py b/src/fidesops/ops/service/masking/strategy/format_preservation.py similarity index 75% rename from src/fidesops/service/masking/strategy/format_preservation.py rename to src/fidesops/ops/service/masking/strategy/format_preservation.py index 483d91c1b5..440e97c4fb 100644 --- a/src/fidesops/service/masking/strategy/format_preservation.py +++ b/src/fidesops/ops/service/masking/strategy/format_preservation.py @@ -1,4 +1,4 @@ -from fidesops.schemas.masking.masking_configuration import FormatPreservationConfig +from fidesops.ops.schemas.masking.masking_configuration import FormatPreservationConfig class FormatPreservation: diff --git a/src/fidesops/service/masking/strategy/masking_strategy.py b/src/fidesops/ops/service/masking/strategy/masking_strategy.py similarity index 85% rename from src/fidesops/service/masking/strategy/masking_strategy.py rename to src/fidesops/ops/service/masking/strategy/masking_strategy.py index b37de258c1..e40eb358b5 100644 --- a/src/fidesops/service/masking/strategy/masking_strategy.py +++ b/src/fidesops/ops/service/masking/strategy/masking_strategy.py @@ -2,9 +2,9 @@ from abc import ABC, abstractmethod from typing import Any, List, Optional -from fidesops.schemas.masking.masking_configuration import MaskingConfiguration -from fidesops.schemas.masking.masking_secrets import MaskingSecretCache -from fidesops.schemas.masking.masking_strategy_description import ( +from fidesops.ops.schemas.masking.masking_configuration import MaskingConfiguration +from fidesops.ops.schemas.masking.masking_secrets import MaskingSecretCache +from fidesops.ops.schemas.masking.masking_strategy_description import ( MaskingStrategyDescription, ) diff --git a/src/fidesops/service/masking/strategy/masking_strategy_aes_encrypt.py b/src/fidesops/ops/service/masking/strategy/masking_strategy_aes_encrypt.py similarity index 88% rename from src/fidesops/service/masking/strategy/masking_strategy_aes_encrypt.py rename to src/fidesops/ops/service/masking/strategy/masking_strategy_aes_encrypt.py index a19c32cd2e..fa1a34226b 100644 --- a/src/fidesops/service/masking/strategy/masking_strategy_aes_encrypt.py +++ b/src/fidesops/ops/service/masking/strategy/masking_strategy_aes_encrypt.py @@ -2,28 +2,30 @@ from typing import Dict, List, Optional -from fidesops.schemas.masking.masking_configuration import ( +from fidesops.ops.schemas.masking.masking_configuration import ( AesEncryptionMaskingConfiguration, HmacMaskingConfiguration, MaskingConfiguration, ) -from fidesops.schemas.masking.masking_secrets import ( +from fidesops.ops.schemas.masking.masking_secrets import ( MaskingSecretCache, MaskingSecretMeta, SecretType, ) -from fidesops.schemas.masking.masking_strategy_description import ( +from fidesops.ops.schemas.masking.masking_strategy_description import ( MaskingStrategyConfigurationDescription, MaskingStrategyDescription, ) -from fidesops.service.masking.strategy.format_preservation import FormatPreservation -from fidesops.service.masking.strategy.masking_strategy import MaskingStrategy -from fidesops.service.masking.strategy.masking_strategy_factory import ( +from fidesops.ops.service.masking.strategy.format_preservation import FormatPreservation +from fidesops.ops.service.masking.strategy.masking_strategy import MaskingStrategy +from fidesops.ops.service.masking.strategy.masking_strategy_factory import ( MaskingStrategyFactory, ) -from fidesops.util.encryption.aes_gcm_encryption_scheme import encrypt -from fidesops.util.encryption.hmac_encryption_scheme import hmac_encrypt_return_bytes -from fidesops.util.encryption.secrets_util import SecretsUtil +from fidesops.ops.util.encryption.aes_gcm_encryption_scheme import encrypt +from fidesops.ops.util.encryption.hmac_encryption_scheme import ( + hmac_encrypt_return_bytes, +) +from fidesops.ops.util.encryption.secrets_util import SecretsUtil AES_ENCRYPT_STRATEGY_NAME = "aes_encrypt" diff --git a/src/fidesops/service/masking/strategy/masking_strategy_factory.py b/src/fidesops/ops/service/masking/strategy/masking_strategy_factory.py similarity index 86% rename from src/fidesops/service/masking/strategy/masking_strategy_factory.py rename to src/fidesops/ops/service/masking/strategy/masking_strategy_factory.py index 60fe7667dc..33186f195a 100644 --- a/src/fidesops/service/masking/strategy/masking_strategy_factory.py +++ b/src/fidesops/ops/service/masking/strategy/masking_strategy_factory.py @@ -3,10 +3,10 @@ from pydantic import ValidationError -from fidesops.common_exceptions import NoSuchStrategyException -from fidesops.common_exceptions import ValidationError as FidesopsValidationError -from fidesops.schemas.masking.masking_configuration import FormatPreservationConfig -from fidesops.service.masking.strategy.masking_strategy import MaskingStrategy +from fidesops.ops.common_exceptions import NoSuchStrategyException +from fidesops.ops.common_exceptions import ValidationError as FidesopsValidationError +from fidesops.ops.schemas.masking.masking_configuration import FormatPreservationConfig +from fidesops.ops.service.masking.strategy.masking_strategy import MaskingStrategy logger = logging.getLogger(__name__) diff --git a/src/fidesops/service/masking/strategy/masking_strategy_hash.py b/src/fidesops/ops/service/masking/strategy/masking_strategy_hash.py similarity index 88% rename from src/fidesops/service/masking/strategy/masking_strategy_hash.py rename to src/fidesops/ops/service/masking/strategy/masking_strategy_hash.py index 15bec85754..3266577a5e 100644 --- a/src/fidesops/service/masking/strategy/masking_strategy_hash.py +++ b/src/fidesops/ops/service/masking/strategy/masking_strategy_hash.py @@ -3,26 +3,26 @@ import hashlib from typing import Dict, List, Optional -from fidesops.core.config import config -from fidesops.schemas.masking.masking_configuration import ( +from fidesops.ops.core.config import config +from fidesops.ops.schemas.masking.masking_configuration import ( HashMaskingConfiguration, MaskingConfiguration, ) -from fidesops.schemas.masking.masking_secrets import ( +from fidesops.ops.schemas.masking.masking_secrets import ( MaskingSecretCache, MaskingSecretMeta, SecretType, ) -from fidesops.schemas.masking.masking_strategy_description import ( +from fidesops.ops.schemas.masking.masking_strategy_description import ( MaskingStrategyConfigurationDescription, MaskingStrategyDescription, ) -from fidesops.service.masking.strategy.format_preservation import FormatPreservation -from fidesops.service.masking.strategy.masking_strategy import MaskingStrategy -from fidesops.service.masking.strategy.masking_strategy_factory import ( +from fidesops.ops.service.masking.strategy.format_preservation import FormatPreservation +from fidesops.ops.service.masking.strategy.masking_strategy import MaskingStrategy +from fidesops.ops.service.masking.strategy.masking_strategy_factory import ( MaskingStrategyFactory, ) -from fidesops.util.encryption.secrets_util import SecretsUtil +from fidesops.ops.util.encryption.secrets_util import SecretsUtil HASH_STRATEGY_NAME = "hash" diff --git a/src/fidesops/service/masking/strategy/masking_strategy_hmac.py b/src/fidesops/ops/service/masking/strategy/masking_strategy_hmac.py similarity index 87% rename from src/fidesops/service/masking/strategy/masking_strategy_hmac.py rename to src/fidesops/ops/service/masking/strategy/masking_strategy_hmac.py index 6b38e6415d..4e8b8f1287 100644 --- a/src/fidesops/service/masking/strategy/masking_strategy_hmac.py +++ b/src/fidesops/ops/service/masking/strategy/masking_strategy_hmac.py @@ -2,26 +2,26 @@ from typing import Dict, List, Optional -from fidesops.schemas.masking.masking_configuration import ( +from fidesops.ops.schemas.masking.masking_configuration import ( HmacMaskingConfiguration, MaskingConfiguration, ) -from fidesops.schemas.masking.masking_secrets import ( +from fidesops.ops.schemas.masking.masking_secrets import ( MaskingSecretCache, MaskingSecretMeta, SecretType, ) -from fidesops.schemas.masking.masking_strategy_description import ( +from fidesops.ops.schemas.masking.masking_strategy_description import ( MaskingStrategyConfigurationDescription, MaskingStrategyDescription, ) -from fidesops.service.masking.strategy.format_preservation import FormatPreservation -from fidesops.service.masking.strategy.masking_strategy import MaskingStrategy -from fidesops.service.masking.strategy.masking_strategy_factory import ( +from fidesops.ops.service.masking.strategy.format_preservation import FormatPreservation +from fidesops.ops.service.masking.strategy.masking_strategy import MaskingStrategy +from fidesops.ops.service.masking.strategy.masking_strategy_factory import ( MaskingStrategyFactory, ) -from fidesops.util.encryption.hmac_encryption_scheme import hmac_encrypt_return_str -from fidesops.util.encryption.secrets_util import SecretsUtil +from fidesops.ops.util.encryption.hmac_encryption_scheme import hmac_encrypt_return_str +from fidesops.ops.util.encryption.secrets_util import SecretsUtil HMAC_STRATEGY_NAME = "hmac" diff --git a/src/fidesops/service/masking/strategy/masking_strategy_nullify.py b/src/fidesops/ops/service/masking/strategy/masking_strategy_nullify.py similarity index 84% rename from src/fidesops/service/masking/strategy/masking_strategy_nullify.py rename to src/fidesops/ops/service/masking/strategy/masking_strategy_nullify.py index a06a0411a5..7c8c28d09c 100644 --- a/src/fidesops/service/masking/strategy/masking_strategy_nullify.py +++ b/src/fidesops/ops/service/masking/strategy/masking_strategy_nullify.py @@ -1,14 +1,14 @@ from typing import List, Optional -from fidesops.schemas.masking.masking_configuration import ( +from fidesops.ops.schemas.masking.masking_configuration import ( MaskingConfiguration, NullMaskingConfiguration, ) -from fidesops.schemas.masking.masking_strategy_description import ( +from fidesops.ops.schemas.masking.masking_strategy_description import ( MaskingStrategyDescription, ) -from fidesops.service.masking.strategy.masking_strategy import MaskingStrategy -from fidesops.service.masking.strategy.masking_strategy_factory import ( +from fidesops.ops.service.masking.strategy.masking_strategy import MaskingStrategy +from fidesops.ops.service.masking.strategy.masking_strategy_factory import ( MaskingStrategyFactory, ) diff --git a/src/fidesops/service/masking/strategy/masking_strategy_random_string_rewrite.py b/src/fidesops/ops/service/masking/strategy/masking_strategy_random_string_rewrite.py similarity index 86% rename from src/fidesops/service/masking/strategy/masking_strategy_random_string_rewrite.py rename to src/fidesops/ops/service/masking/strategy/masking_strategy_random_string_rewrite.py index 6c1ceab1d0..ce4de26859 100644 --- a/src/fidesops/service/masking/strategy/masking_strategy_random_string_rewrite.py +++ b/src/fidesops/ops/service/masking/strategy/masking_strategy_random_string_rewrite.py @@ -2,17 +2,17 @@ from secrets import choice from typing import List, Optional -from fidesops.schemas.masking.masking_configuration import ( +from fidesops.ops.schemas.masking.masking_configuration import ( MaskingConfiguration, RandomStringMaskingConfiguration, ) -from fidesops.schemas.masking.masking_strategy_description import ( +from fidesops.ops.schemas.masking.masking_strategy_description import ( MaskingStrategyConfigurationDescription, MaskingStrategyDescription, ) -from fidesops.service.masking.strategy.format_preservation import FormatPreservation -from fidesops.service.masking.strategy.masking_strategy import MaskingStrategy -from fidesops.service.masking.strategy.masking_strategy_factory import ( +from fidesops.ops.service.masking.strategy.format_preservation import FormatPreservation +from fidesops.ops.service.masking.strategy.masking_strategy import MaskingStrategy +from fidesops.ops.service.masking.strategy.masking_strategy_factory import ( MaskingStrategyFactory, ) diff --git a/src/fidesops/service/masking/strategy/masking_strategy_string_rewrite.py b/src/fidesops/ops/service/masking/strategy/masking_strategy_string_rewrite.py similarity index 86% rename from src/fidesops/service/masking/strategy/masking_strategy_string_rewrite.py rename to src/fidesops/ops/service/masking/strategy/masking_strategy_string_rewrite.py index 4bc19970ee..d851cad8ed 100644 --- a/src/fidesops/service/masking/strategy/masking_strategy_string_rewrite.py +++ b/src/fidesops/ops/service/masking/strategy/masking_strategy_string_rewrite.py @@ -1,16 +1,16 @@ from typing import List, Optional -from fidesops.schemas.masking.masking_configuration import ( +from fidesops.ops.schemas.masking.masking_configuration import ( MaskingConfiguration, StringRewriteMaskingConfiguration, ) -from fidesops.schemas.masking.masking_strategy_description import ( +from fidesops.ops.schemas.masking.masking_strategy_description import ( MaskingStrategyConfigurationDescription, MaskingStrategyDescription, ) -from fidesops.service.masking.strategy.format_preservation import FormatPreservation -from fidesops.service.masking.strategy.masking_strategy import MaskingStrategy -from fidesops.service.masking.strategy.masking_strategy_factory import ( +from fidesops.ops.service.masking.strategy.format_preservation import FormatPreservation +from fidesops.ops.service.masking.strategy.masking_strategy import MaskingStrategy +from fidesops.ops.service.masking.strategy.masking_strategy_factory import ( MaskingStrategyFactory, ) diff --git a/src/fidesops/service/outbound_urn_registry.py b/src/fidesops/ops/service/outbound_urn_registry.py similarity index 100% rename from src/fidesops/service/outbound_urn_registry.py rename to src/fidesops/ops/service/outbound_urn_registry.py diff --git a/src/fidesops/service/privacy_request/__init__.py b/src/fidesops/ops/service/pagination/__init__.py similarity index 100% rename from src/fidesops/service/privacy_request/__init__.py rename to src/fidesops/ops/service/pagination/__init__.py diff --git a/src/fidesops/service/pagination/pagination_strategy.py b/src/fidesops/ops/service/pagination/pagination_strategy.py similarity index 87% rename from src/fidesops/service/pagination/pagination_strategy.py rename to src/fidesops/ops/service/pagination/pagination_strategy.py index de5e2e4193..783965d286 100644 --- a/src/fidesops/service/pagination/pagination_strategy.py +++ b/src/fidesops/ops/service/pagination/pagination_strategy.py @@ -5,10 +5,10 @@ from requests import Response -from fidesops.schemas.saas.shared_schemas import SaaSRequestParams +from fidesops.ops.schemas.saas.shared_schemas import SaaSRequestParams if TYPE_CHECKING: - from fidesops.schemas.saas.strategy_configuration import StrategyConfiguration + from fidesops.ops.schemas.saas.strategy_configuration import StrategyConfiguration class PaginationStrategy(ABC): diff --git a/src/fidesops/service/pagination/pagination_strategy_cursor.py b/src/fidesops/ops/service/pagination/pagination_strategy_cursor.py similarity index 88% rename from src/fidesops/service/pagination/pagination_strategy_cursor.py rename to src/fidesops/ops/service/pagination/pagination_strategy_cursor.py index a9ad9f87f4..556ad4db90 100644 --- a/src/fidesops/service/pagination/pagination_strategy_cursor.py +++ b/src/fidesops/ops/service/pagination/pagination_strategy_cursor.py @@ -3,12 +3,12 @@ import pydash from requests import Response -from fidesops.schemas.saas.shared_schemas import SaaSRequestParams -from fidesops.schemas.saas.strategy_configuration import ( +from fidesops.ops.schemas.saas.shared_schemas import SaaSRequestParams +from fidesops.ops.schemas.saas.strategy_configuration import ( CursorPaginationConfiguration, StrategyConfiguration, ) -from fidesops.service.pagination.pagination_strategy import PaginationStrategy +from fidesops.ops.service.pagination.pagination_strategy import PaginationStrategy STRATEGY_NAME = "cursor" diff --git a/src/fidesops/service/pagination/pagination_strategy_factory.py b/src/fidesops/ops/service/pagination/pagination_strategy_factory.py similarity index 75% rename from src/fidesops/service/pagination/pagination_strategy_factory.py rename to src/fidesops/ops/service/pagination/pagination_strategy_factory.py index 8e5b8ab6b4..d686794618 100644 --- a/src/fidesops/service/pagination/pagination_strategy_factory.py +++ b/src/fidesops/ops/service/pagination/pagination_strategy_factory.py @@ -6,19 +6,21 @@ from pydantic import ValidationError -from fidesops.common_exceptions import NoSuchStrategyException -from fidesops.common_exceptions import ValidationError as FidesopsValidationError -from fidesops.service.pagination.pagination_strategy_cursor import ( +from fidesops.ops.common_exceptions import NoSuchStrategyException +from fidesops.ops.common_exceptions import ValidationError as FidesopsValidationError +from fidesops.ops.service.pagination.pagination_strategy_cursor import ( CursorPaginationStrategy, ) -from fidesops.service.pagination.pagination_strategy_link import LinkPaginationStrategy -from fidesops.service.pagination.pagination_strategy_offset import ( +from fidesops.ops.service.pagination.pagination_strategy_link import ( + LinkPaginationStrategy, +) +from fidesops.ops.service.pagination.pagination_strategy_offset import ( OffsetPaginationStrategy, ) if TYPE_CHECKING: - from fidesops.schemas.saas.strategy_configuration import StrategyConfiguration - from fidesops.service.pagination.pagination_strategy import PaginationStrategy + from fidesops.ops.schemas.saas.strategy_configuration import StrategyConfiguration + from fidesops.ops.service.pagination.pagination_strategy import PaginationStrategy logger = logging.getLogger(__name__) diff --git a/src/fidesops/service/pagination/pagination_strategy_link.py b/src/fidesops/ops/service/pagination/pagination_strategy_link.py similarity index 91% rename from src/fidesops/service/pagination/pagination_strategy_link.py rename to src/fidesops/ops/service/pagination/pagination_strategy_link.py index 31a382c86e..9848d80dd3 100644 --- a/src/fidesops/service/pagination/pagination_strategy_link.py +++ b/src/fidesops/ops/service/pagination/pagination_strategy_link.py @@ -6,13 +6,13 @@ import pydash from requests import Response -from fidesops.schemas.saas.shared_schemas import SaaSRequestParams -from fidesops.schemas.saas.strategy_configuration import ( +from fidesops.ops.schemas.saas.shared_schemas import SaaSRequestParams +from fidesops.ops.schemas.saas.strategy_configuration import ( LinkPaginationConfiguration, LinkSource, StrategyConfiguration, ) -from fidesops.service.pagination.pagination_strategy import PaginationStrategy +from fidesops.ops.service.pagination.pagination_strategy import PaginationStrategy STRATEGY_NAME = "link" diff --git a/src/fidesops/service/pagination/pagination_strategy_offset.py b/src/fidesops/ops/service/pagination/pagination_strategy_offset.py similarity index 91% rename from src/fidesops/service/pagination/pagination_strategy_offset.py rename to src/fidesops/ops/service/pagination/pagination_strategy_offset.py index c6fdbb49a6..d4192d2047 100644 --- a/src/fidesops/service/pagination/pagination_strategy_offset.py +++ b/src/fidesops/ops/service/pagination/pagination_strategy_offset.py @@ -3,14 +3,14 @@ import pydash from requests import Response -from fidesops.common_exceptions import FidesopsException -from fidesops.schemas.saas.shared_schemas import SaaSRequestParams -from fidesops.schemas.saas.strategy_configuration import ( +from fidesops.ops.common_exceptions import FidesopsException +from fidesops.ops.schemas.saas.shared_schemas import SaaSRequestParams +from fidesops.ops.schemas.saas.strategy_configuration import ( ConnectorParamRef, OffsetPaginationConfiguration, StrategyConfiguration, ) -from fidesops.service.pagination.pagination_strategy import PaginationStrategy +from fidesops.ops.service.pagination.pagination_strategy import PaginationStrategy STRATEGY_NAME = "offset" diff --git a/src/fidesops/service/processors/__init__.py b/src/fidesops/ops/service/privacy_request/__init__.py similarity index 100% rename from src/fidesops/service/processors/__init__.py rename to src/fidesops/ops/service/privacy_request/__init__.py diff --git a/src/fidesops/service/privacy_request/onetrust_service.py b/src/fidesops/ops/service/privacy_request/onetrust_service.py similarity index 92% rename from src/fidesops/service/privacy_request/onetrust_service.py rename to src/fidesops/ops/service/privacy_request/onetrust_service.py index efac307f48..4fb73e0fe5 100644 --- a/src/fidesops/service/privacy_request/onetrust_service.py +++ b/src/fidesops/ops/service/privacy_request/onetrust_service.py @@ -9,33 +9,33 @@ from requests import Response from sqlalchemy.orm import Session -from fidesops.common_exceptions import ( +from fidesops.ops.common_exceptions import ( PolicyNotFoundException, StorageConfigNotFoundException, ) -from fidesops.core.config import config -from fidesops.models.policy import Policy -from fidesops.models.privacy_request import PrivacyRequest -from fidesops.models.storage import StorageConfig -from fidesops.schemas.privacy_request import PrivacyRequestIdentity -from fidesops.schemas.shared_schemas import FidesOpsKey -from fidesops.schemas.storage.storage import StorageDetails, StorageSecrets -from fidesops.schemas.third_party.onetrust import ( +from fidesops.ops.core.config import config +from fidesops.ops.models.policy import Policy +from fidesops.ops.models.privacy_request import PrivacyRequest +from fidesops.ops.models.storage import StorageConfig +from fidesops.ops.schemas.privacy_request import PrivacyRequestIdentity +from fidesops.ops.schemas.shared_schemas import FidesOpsKey +from fidesops.ops.schemas.storage.storage import StorageDetails, StorageSecrets +from fidesops.ops.schemas.third_party.onetrust import ( OneTrustGetRequestsResponse, OneTrustGetSubtasksResponse, OneTrustRequest, OneTrustSubtask, OneTrustSubtaskStatus, ) -from fidesops.service.outbound_urn_registry import ( +from fidesops.ops.service.outbound_urn_registry import ( ONETRUST_GET_ALL_REQUESTS, ONETRUST_GET_SUBTASKS_BY_REF_ID, ONETRUST_PUT_SUBTASK_STATUS, ) -from fidesops.service.privacy_request.request_runner_service import ( +from fidesops.ops.service.privacy_request.request_runner_service import ( queue_privacy_request, ) -from fidesops.util.storage_authenticator import get_onetrust_access_token +from fidesops.ops.util.storage_authenticator import get_onetrust_access_token logger = logging.getLogger(__name__) diff --git a/src/fidesops/service/privacy_request/request_runner_service.py b/src/fidesops/ops/service/privacy_request/request_runner_service.py similarity index 92% rename from src/fidesops/service/privacy_request/request_runner_service.py rename to src/fidesops/ops/service/privacy_request/request_runner_service.py index 2a85b72753..3bdfcfd10c 100644 --- a/src/fidesops/service/privacy_request/request_runner_service.py +++ b/src/fidesops/ops/service/privacy_request/request_runner_service.py @@ -10,17 +10,20 @@ from redis.exceptions import DataError from sqlalchemy.orm import Session -from fidesops import common_exceptions -from fidesops.common_exceptions import ClientUnsuccessfulException, PrivacyRequestPaused -from fidesops.core.config import config -from fidesops.graph.analytics_events import ( +from fidesops.ops import common_exceptions +from fidesops.ops.common_exceptions import ( + ClientUnsuccessfulException, + PrivacyRequestPaused, +) +from fidesops.ops.core.config import config +from fidesops.ops.graph.analytics_events import ( failed_graph_analytics_event, fideslog_graph_failure, ) -from fidesops.graph.graph import DatasetGraph -from fidesops.models.connectionconfig import ConnectionConfig -from fidesops.models.datasetconfig import DatasetConfig -from fidesops.models.policy import ( +from fidesops.ops.graph.graph import DatasetGraph +from fidesops.ops.models.connectionconfig import ConnectionConfig +from fidesops.ops.models.datasetconfig import DatasetConfig +from fidesops.ops.models.policy import ( ActionType, PausedStep, Policy, @@ -28,23 +31,23 @@ PolicyPreWebhook, WebhookTypes, ) -from fidesops.models.privacy_request import PrivacyRequest, PrivacyRequestStatus -from fidesops.service.storage.storage_uploader_service import upload -from fidesops.task.filter_results import filter_data_categories -from fidesops.task.graph_task import ( +from fidesops.ops.models.privacy_request import PrivacyRequest, PrivacyRequestStatus +from fidesops.ops.service.storage.storage_uploader_service import upload +from fidesops.ops.task.filter_results import filter_data_categories +from fidesops.ops.task.graph_task import ( get_cached_data_for_erasures, run_access_request, run_erasure, ) -from fidesops.tasks import celery_app -from fidesops.tasks.scheduled.scheduler import scheduler -from fidesops.util.cache import ( +from fidesops.ops.tasks import celery_app +from fidesops.ops.tasks.scheduled.scheduler import scheduler +from fidesops.ops.util.cache import ( FidesopsRedis, get_async_task_tracking_cache_key, get_cache, ) -from fidesops.util.collection_util import Row -from fidesops.util.logger import _log_exception, _log_warning +from fidesops.ops.util.collection_util import Row +from fidesops.ops.util.logger import _log_exception, _log_warning logger = get_task_logger(__name__) diff --git a/src/fidesops/service/privacy_request/request_service.py b/src/fidesops/ops/service/privacy_request/request_service.py similarity index 83% rename from src/fidesops/service/privacy_request/request_service.py rename to src/fidesops/ops/service/privacy_request/request_service.py index 75bdef9ff5..a6517f4e29 100644 --- a/src/fidesops/service/privacy_request/request_service.py +++ b/src/fidesops/ops/service/privacy_request/request_service.py @@ -2,12 +2,12 @@ from datetime import datetime from typing import Any, Dict, List, Optional, Set -from fidesops.models.policy import ActionType, Policy -from fidesops.models.privacy_request import PrivacyRequest -from fidesops.schemas.drp_privacy_request import DrpPrivacyRequestCreate -from fidesops.schemas.masking.masking_secrets import MaskingSecretCache -from fidesops.schemas.redis_cache import PrivacyRequestIdentity -from fidesops.service.masking.strategy.masking_strategy_factory import ( +from fidesops.ops.models.policy import ActionType, Policy +from fidesops.ops.models.privacy_request import PrivacyRequest +from fidesops.ops.schemas.drp_privacy_request import DrpPrivacyRequestCreate +from fidesops.ops.schemas.masking.masking_secrets import MaskingSecretCache +from fidesops.ops.schemas.redis_cache import PrivacyRequestIdentity +from fidesops.ops.service.masking.strategy.masking_strategy_factory import ( MaskingStrategyFactory, ) diff --git a/src/fidesops/service/processors/post_processor_strategy/__init__.py b/src/fidesops/ops/service/processors/__init__.py similarity index 100% rename from src/fidesops/service/processors/post_processor_strategy/__init__.py rename to src/fidesops/ops/service/processors/__init__.py diff --git a/src/fidesops/service/saas_request/override_implementations/__init__.py b/src/fidesops/ops/service/processors/post_processor_strategy/__init__.py similarity index 100% rename from src/fidesops/service/saas_request/override_implementations/__init__.py rename to src/fidesops/ops/service/processors/post_processor_strategy/__init__.py diff --git a/src/fidesops/service/processors/post_processor_strategy/post_processor_strategy.py b/src/fidesops/ops/service/processors/post_processor_strategy/post_processor_strategy.py similarity index 100% rename from src/fidesops/service/processors/post_processor_strategy/post_processor_strategy.py rename to src/fidesops/ops/service/processors/post_processor_strategy/post_processor_strategy.py diff --git a/src/fidesops/service/processors/post_processor_strategy/post_processor_strategy_factory.py b/src/fidesops/ops/service/processors/post_processor_strategy/post_processor_strategy_factory.py similarity index 76% rename from src/fidesops/service/processors/post_processor_strategy/post_processor_strategy_factory.py rename to src/fidesops/ops/service/processors/post_processor_strategy/post_processor_strategy_factory.py index 3af31df6b7..a0c61aaeb3 100644 --- a/src/fidesops/service/processors/post_processor_strategy/post_processor_strategy_factory.py +++ b/src/fidesops/ops/service/processors/post_processor_strategy/post_processor_strategy_factory.py @@ -4,16 +4,16 @@ from pydantic import ValidationError -from fidesops.common_exceptions import NoSuchStrategyException -from fidesops.common_exceptions import ValidationError as FidesopsValidationError -from fidesops.schemas.saas.strategy_configuration import StrategyConfiguration -from fidesops.service.processors.post_processor_strategy.post_processor_strategy import ( +from fidesops.ops.common_exceptions import NoSuchStrategyException +from fidesops.ops.common_exceptions import ValidationError as FidesopsValidationError +from fidesops.ops.schemas.saas.strategy_configuration import StrategyConfiguration +from fidesops.ops.service.processors.post_processor_strategy.post_processor_strategy import ( PostProcessorStrategy, ) -from fidesops.service.processors.post_processor_strategy.post_processor_strategy_filter import ( +from fidesops.ops.service.processors.post_processor_strategy.post_processor_strategy_filter import ( FilterPostProcessorStrategy, ) -from fidesops.service.processors.post_processor_strategy.post_processor_strategy_unwrap import ( +from fidesops.ops.service.processors.post_processor_strategy.post_processor_strategy_unwrap import ( UnwrapPostProcessorStrategy, ) diff --git a/src/fidesops/service/processors/post_processor_strategy/post_processor_strategy_filter.py b/src/fidesops/ops/service/processors/post_processor_strategy/post_processor_strategy_filter.py similarity index 94% rename from src/fidesops/service/processors/post_processor_strategy/post_processor_strategy_filter.py rename to src/fidesops/ops/service/processors/post_processor_strategy/post_processor_strategy_filter.py index d4051c5d2b..4f5c9cea4d 100644 --- a/src/fidesops/service/processors/post_processor_strategy/post_processor_strategy_filter.py +++ b/src/fidesops/ops/service/processors/post_processor_strategy/post_processor_strategy_filter.py @@ -3,13 +3,13 @@ import pydash -from fidesops.common_exceptions import FidesopsException -from fidesops.schemas.saas.shared_schemas import IdentityParamRef -from fidesops.schemas.saas.strategy_configuration import ( +from fidesops.ops.common_exceptions import FidesopsException +from fidesops.ops.schemas.saas.shared_schemas import IdentityParamRef +from fidesops.ops.schemas.saas.strategy_configuration import ( FilterPostProcessorConfiguration, StrategyConfiguration, ) -from fidesops.service.processors.post_processor_strategy.post_processor_strategy import ( +from fidesops.ops.service.processors.post_processor_strategy.post_processor_strategy import ( PostProcessorStrategy, ) diff --git a/src/fidesops/service/processors/post_processor_strategy/post_processor_strategy_unwrap.py b/src/fidesops/ops/service/processors/post_processor_strategy/post_processor_strategy_unwrap.py similarity index 94% rename from src/fidesops/service/processors/post_processor_strategy/post_processor_strategy_unwrap.py rename to src/fidesops/ops/service/processors/post_processor_strategy/post_processor_strategy_unwrap.py index 8c0f1a5a20..b63b0c7562 100644 --- a/src/fidesops/service/processors/post_processor_strategy/post_processor_strategy_unwrap.py +++ b/src/fidesops/ops/service/processors/post_processor_strategy/post_processor_strategy_unwrap.py @@ -3,11 +3,11 @@ import pydash -from fidesops.schemas.saas.strategy_configuration import ( +from fidesops.ops.schemas.saas.strategy_configuration import ( StrategyConfiguration, UnwrapPostProcessorConfiguration, ) -from fidesops.service.processors.post_processor_strategy.post_processor_strategy import ( +from fidesops.ops.service.processors.post_processor_strategy.post_processor_strategy import ( PostProcessorStrategy, ) diff --git a/src/fidesops/ops/service/saas_request/__init__.py b/src/fidesops/ops/service/saas_request/__init__.py new file mode 100644 index 0000000000..fad5bf1f25 --- /dev/null +++ b/src/fidesops/ops/service/saas_request/__init__.py @@ -0,0 +1,3 @@ +from fidesops.ops.service.saas_request.override_implementations import ( + mailchimp_request_overrides, +) diff --git a/src/fidesops/service/storage/__init__.py b/src/fidesops/ops/service/saas_request/override_implementations/__init__.py similarity index 100% rename from src/fidesops/service/storage/__init__.py rename to src/fidesops/ops/service/saas_request/override_implementations/__init__.py diff --git a/src/fidesops/service/saas_request/override_implementations/mailchimp_request_overrides.py b/src/fidesops/ops/service/saas_request/override_implementations/mailchimp_request_overrides.py similarity index 91% rename from src/fidesops/service/saas_request/override_implementations/mailchimp_request_overrides.py rename to src/fidesops/ops/service/saas_request/override_implementations/mailchimp_request_overrides.py index 1dd27c1fbd..01e85a5f52 100644 --- a/src/fidesops/service/saas_request/override_implementations/mailchimp_request_overrides.py +++ b/src/fidesops/ops/service/saas_request/override_implementations/mailchimp_request_overrides.py @@ -4,16 +4,19 @@ import pydash from requests import get, put -from fidesops.common_exceptions import ClientUnsuccessfulException, ConnectionException -from fidesops.core.config import config -from fidesops.graph.traversal import TraversalNode -from fidesops.models.policy import Policy -from fidesops.models.privacy_request import PrivacyRequest -from fidesops.service.saas_request.saas_request_override_factory import ( +from fidesops.ops.common_exceptions import ( + ClientUnsuccessfulException, + ConnectionException, +) +from fidesops.ops.core.config import config +from fidesops.ops.graph.traversal import TraversalNode +from fidesops.ops.models.policy import Policy +from fidesops.ops.models.privacy_request import PrivacyRequest +from fidesops.ops.service.saas_request.saas_request_override_factory import ( SaaSRequestType, register, ) -from fidesops.util.collection_util import Row +from fidesops.ops.util.collection_util import Row @register("mailchimp_messages_access", [SaaSRequestType.READ]) diff --git a/src/fidesops/service/saas_request/saas_request_override_factory.py b/src/fidesops/ops/service/saas_request/saas_request_override_factory.py similarity index 98% rename from src/fidesops/service/saas_request/saas_request_override_factory.py rename to src/fidesops/ops/service/saas_request/saas_request_override_factory.py index 3883716487..2d18def5b7 100644 --- a/src/fidesops/service/saas_request/saas_request_override_factory.py +++ b/src/fidesops/ops/service/saas_request/saas_request_override_factory.py @@ -3,11 +3,11 @@ from inspect import Signature, signature from typing import Callable, Dict, List, Union -from fidesops.common_exceptions import ( +from fidesops.ops.common_exceptions import ( InvalidSaaSRequestOverrideException, NoSuchSaaSRequestOverrideException, ) -from fidesops.util.collection_util import Row +from fidesops.ops.util.collection_util import Row logger = logging.getLogger(__name__) diff --git a/src/fidesops/task/__init__.py b/src/fidesops/ops/service/storage/__init__.py similarity index 100% rename from src/fidesops/task/__init__.py rename to src/fidesops/ops/service/storage/__init__.py diff --git a/src/fidesops/service/storage/storage_authenticator_service.py b/src/fidesops/ops/service/storage/storage_authenticator_service.py similarity index 94% rename from src/fidesops/service/storage/storage_authenticator_service.py rename to src/fidesops/ops/service/storage/storage_authenticator_service.py index 506475bb29..8387190e3e 100644 --- a/src/fidesops/service/storage/storage_authenticator_service.py +++ b/src/fidesops/ops/service/storage/storage_authenticator_service.py @@ -4,13 +4,13 @@ from botocore.exceptions import ClientError from requests import RequestException -from fidesops.schemas.storage.storage import ( +from fidesops.ops.schemas.storage.storage import ( SUPPORTED_STORAGE_SECRETS, StorageSecretsOnetrust, StorageSecretsS3, StorageType, ) -from fidesops.util.storage_authenticator import ( +from fidesops.ops.util.storage_authenticator import ( get_onetrust_access_token, get_s3_session, ) diff --git a/src/fidesops/service/storage/storage_uploader_service.py b/src/fidesops/ops/service/storage/storage_uploader_service.py similarity index 90% rename from src/fidesops/service/storage/storage_uploader_service.py rename to src/fidesops/ops/service/storage/storage_uploader_service.py index f10db6ec5b..9483a9f827 100644 --- a/src/fidesops/service/storage/storage_uploader_service.py +++ b/src/fidesops/ops/service/storage/storage_uploader_service.py @@ -3,17 +3,17 @@ from sqlalchemy.orm import Session -from fidesops.common_exceptions import StorageUploadError -from fidesops.models.privacy_request import PrivacyRequest -from fidesops.models.storage import StorageConfig -from fidesops.schemas.shared_schemas import FidesOpsKey -from fidesops.schemas.storage.storage import ( +from fidesops.ops.common_exceptions import StorageUploadError +from fidesops.ops.models.privacy_request import PrivacyRequest +from fidesops.ops.models.storage import StorageConfig +from fidesops.ops.schemas.shared_schemas import FidesOpsKey +from fidesops.ops.schemas.storage.storage import ( FileNaming, ResponseFormat, StorageDetails, StorageType, ) -from fidesops.tasks.storage import upload_to_local, upload_to_onetrust, upload_to_s3 +from fidesops.ops.tasks.storage import upload_to_local, upload_to_onetrust, upload_to_s3 logger = logging.getLogger(__name__) diff --git a/src/fidesops/tasks/scheduled/__init__.py b/src/fidesops/ops/task/__init__.py similarity index 100% rename from src/fidesops/tasks/scheduled/__init__.py rename to src/fidesops/ops/task/__init__.py diff --git a/src/fidesops/task/consolidate_query_matches.py b/src/fidesops/ops/task/consolidate_query_matches.py similarity index 93% rename from src/fidesops/task/consolidate_query_matches.py rename to src/fidesops/ops/task/consolidate_query_matches.py index 2539e07c48..5ab7331a28 100644 --- a/src/fidesops/task/consolidate_query_matches.py +++ b/src/fidesops/ops/task/consolidate_query_matches.py @@ -1,7 +1,7 @@ from typing import Any, List, Optional -from fidesops.graph.config import FieldPath -from fidesops.util.collection_util import Row +from fidesops.ops.graph.config import FieldPath +from fidesops.ops.util.collection_util import Row def consolidate_query_matches( diff --git a/src/fidesops/task/filter_element_match.py b/src/fidesops/ops/task/filter_element_match.py similarity index 97% rename from src/fidesops/task/filter_element_match.py rename to src/fidesops/ops/task/filter_element_match.py index f504a82c66..fa46cd1e0e 100644 --- a/src/fidesops/task/filter_element_match.py +++ b/src/fidesops/ops/task/filter_element_match.py @@ -5,13 +5,13 @@ import pydash -from fidesops.task.refine_target_path import ( +from fidesops.ops.task.refine_target_path import ( DetailedPath, FieldPathNodeInput, build_refined_target_paths, join_detailed_path, ) -from fidesops.util.collection_util import FIDESOPS_DO_NOT_MASK_INDEX, Row +from fidesops.ops.util.collection_util import FIDESOPS_DO_NOT_MASK_INDEX, Row logger = logging.getLogger(__name__) diff --git a/src/fidesops/task/filter_results.py b/src/fidesops/ops/task/filter_results.py similarity index 97% rename from src/fidesops/task/filter_results.py rename to src/fidesops/ops/task/filter_results.py index 4ee2d2d1bd..983e83fbec 100644 --- a/src/fidesops/task/filter_results.py +++ b/src/fidesops/ops/task/filter_results.py @@ -3,9 +3,9 @@ from collections import defaultdict from typing import Any, Dict, List, Optional, Set, Union -from fidesops.graph.config import CollectionAddress, FieldPath -from fidesops.schemas.shared_schemas import FidesOpsKey -from fidesops.util.collection_util import Row +from fidesops.ops.graph.config import CollectionAddress, FieldPath +from fidesops.ops.schemas.shared_schemas import FidesOpsKey +from fidesops.ops.util.collection_util import Row logger = logging.getLogger(__name__) diff --git a/src/fidesops/task/graph_task.py b/src/fidesops/ops/task/graph_task.py similarity index 96% rename from src/fidesops/task/graph_task.py rename to src/fidesops/ops/task/graph_task.py index a536507ad7..2efec6dce3 100644 --- a/src/fidesops/task/graph_task.py +++ b/src/fidesops/ops/task/graph_task.py @@ -10,13 +10,13 @@ from dask.threaded import get from sqlalchemy.orm import Session -from fidesops.common_exceptions import CollectionDisabled, PrivacyRequestPaused -from fidesops.core.config import config -from fidesops.graph.analytics_events import ( +from fidesops.ops.common_exceptions import CollectionDisabled, PrivacyRequestPaused +from fidesops.ops.core.config import config +from fidesops.ops.graph.analytics_events import ( fideslog_graph_rerun, prepare_rerun_graph_analytics_event, ) -from fidesops.graph.config import ( +from fidesops.ops.graph.config import ( ROOT_COLLECTION_ADDRESS, TERMINATOR_ADDRESS, CollectionAddress, @@ -24,21 +24,21 @@ FieldAddress, FieldPath, ) -from fidesops.graph.graph import DatasetGraph, Edge, Node -from fidesops.graph.graph_differences import format_graph_for_caching -from fidesops.graph.traversal import Traversal, TraversalNode -from fidesops.models.connectionconfig import AccessLevel, ConnectionConfig -from fidesops.models.policy import ActionType, Policy -from fidesops.models.privacy_request import ExecutionLogStatus, PrivacyRequest -from fidesops.service.connectors import BaseConnector -from fidesops.task.consolidate_query_matches import consolidate_query_matches -from fidesops.task.filter_element_match import filter_element_match -from fidesops.task.refine_target_path import FieldPathNodeInput -from fidesops.task.task_resources import TaskResources -from fidesops.util.cache import get_cache -from fidesops.util.collection_util import NodeInput, Row, append, partition -from fidesops.util.logger import NotPii -from fidesops.util.saas_util import FIDESOPS_GROUPED_INPUTS +from fidesops.ops.graph.graph import DatasetGraph, Edge, Node +from fidesops.ops.graph.graph_differences import format_graph_for_caching +from fidesops.ops.graph.traversal import Traversal, TraversalNode +from fidesops.ops.models.connectionconfig import AccessLevel, ConnectionConfig +from fidesops.ops.models.policy import ActionType, Policy +from fidesops.ops.models.privacy_request import ExecutionLogStatus, PrivacyRequest +from fidesops.ops.service.connectors import BaseConnector +from fidesops.ops.task.consolidate_query_matches import consolidate_query_matches +from fidesops.ops.task.filter_element_match import filter_element_match +from fidesops.ops.task.refine_target_path import FieldPathNodeInput +from fidesops.ops.task.task_resources import TaskResources +from fidesops.ops.util.cache import get_cache +from fidesops.ops.util.collection_util import NodeInput, Row, append, partition +from fidesops.ops.util.logger import NotPii +from fidesops.ops.util.saas_util import FIDESOPS_GROUPED_INPUTS logger = logging.getLogger(__name__) diff --git a/src/fidesops/task/refine_target_path.py b/src/fidesops/ops/task/refine_target_path.py similarity index 98% rename from src/fidesops/task/refine_target_path.py rename to src/fidesops/ops/task/refine_target_path.py index 1029134c45..9ffe2edd96 100644 --- a/src/fidesops/task/refine_target_path.py +++ b/src/fidesops/ops/task/refine_target_path.py @@ -1,8 +1,8 @@ import logging from typing import Any, Dict, List, Optional, Union -from fidesops.graph.config import FieldPath -from fidesops.util.collection_util import FIDESOPS_DO_NOT_MASK_INDEX, Row +from fidesops.ops.graph.config import FieldPath +from fidesops.ops.util.collection_util import FIDESOPS_DO_NOT_MASK_INDEX, Row Level = Union[ str, int diff --git a/src/fidesops/task/task_resources.py b/src/fidesops/ops/task/task_resources.py similarity index 93% rename from src/fidesops/task/task_resources.py rename to src/fidesops/ops/task/task_resources.py index 474ba289c5..f2b28158e1 100644 --- a/src/fidesops/task/task_resources.py +++ b/src/fidesops/ops/task/task_resources.py @@ -3,17 +3,17 @@ from sqlalchemy.orm import Session -from fidesops.common_exceptions import ConnectorNotFoundException -from fidesops.graph.config import CollectionAddress -from fidesops.models.connectionconfig import ConnectionConfig, ConnectionType -from fidesops.models.policy import ActionType, Policy -from fidesops.models.privacy_request import ( +from fidesops.ops.common_exceptions import ConnectorNotFoundException +from fidesops.ops.graph.config import CollectionAddress +from fidesops.ops.models.connectionconfig import ConnectionConfig, ConnectionType +from fidesops.ops.models.policy import ActionType, Policy +from fidesops.ops.models.privacy_request import ( ExecutionLog, ExecutionLogStatus, PrivacyRequest, ) -from fidesops.schemas.shared_schemas import FidesOpsKey -from fidesops.service.connectors import ( +from fidesops.ops.schemas.shared_schemas import FidesOpsKey +from fidesops.ops.service.connectors import ( BaseConnector, BigQueryConnector, ManualConnector, @@ -26,8 +26,8 @@ SaaSConnector, SnowflakeConnector, ) -from fidesops.util.cache import get_cache -from fidesops.util.collection_util import Row +from fidesops.ops.util.cache import get_cache +from fidesops.ops.util.collection_util import Row logger = logging.getLogger(__name__) diff --git a/src/fidesops/tasks/__init__.py b/src/fidesops/ops/tasks/__init__.py similarity index 85% rename from src/fidesops/tasks/__init__.py rename to src/fidesops/ops/tasks/__init__.py index 84b4c10a50..061674a180 100644 --- a/src/fidesops/tasks/__init__.py +++ b/src/fidesops/ops/tasks/__init__.py @@ -4,8 +4,8 @@ from celery.utils.log import get_task_logger from fideslib.core.config import load_toml -from fidesops.core.config import config -from fidesops.util.logger import NotPii +from fidesops.ops.core.config import config +from fidesops.ops.util.logger import NotPii logger = get_task_logger(__name__) @@ -35,9 +35,9 @@ def _create_celery(config_path: str = config.execution.celery_config_path) -> Ce logger.info("Autodiscovering tasks...") app.autodiscover_tasks( [ - "fidesops.tasks", - "fidesops.tasks.scheduled", - "fidesops.service.privacy_request", + "fidesops.ops.tasks", + "fidesops.ops.tasks.scheduled", + "fidesops.ops.service.privacy_request", ] ) return app diff --git a/src/fidesops/util/__init__.py b/src/fidesops/ops/tasks/scheduled/__init__.py similarity index 100% rename from src/fidesops/util/__init__.py rename to src/fidesops/ops/tasks/scheduled/__init__.py diff --git a/src/fidesops/tasks/scheduled/scheduler.py b/src/fidesops/ops/tasks/scheduled/scheduler.py similarity index 100% rename from src/fidesops/tasks/scheduled/scheduler.py rename to src/fidesops/ops/tasks/scheduled/scheduler.py diff --git a/src/fidesops/tasks/scheduled/tasks.py b/src/fidesops/ops/tasks/scheduled/tasks.py similarity index 82% rename from src/fidesops/tasks/scheduled/tasks.py rename to src/fidesops/ops/tasks/scheduled/tasks.py index 664fff8f5f..e8106d78cd 100644 --- a/src/fidesops/tasks/scheduled/tasks.py +++ b/src/fidesops/ops/tasks/scheduled/tasks.py @@ -4,12 +4,12 @@ from apscheduler.jobstores.base import JobLookupError from fideslib.db.session import get_db_session -from fidesops.core.config import config -from fidesops.models.storage import StorageConfig -from fidesops.schemas.shared_schemas import FidesOpsKey -from fidesops.schemas.storage.storage import StorageDetails, StorageType -from fidesops.service.privacy_request.onetrust_service import OneTrustService -from fidesops.tasks.scheduled.scheduler import scheduler +from fidesops.ops.core.config import config +from fidesops.ops.models.storage import StorageConfig +from fidesops.ops.schemas.shared_schemas import FidesOpsKey +from fidesops.ops.schemas.storage.storage import StorageDetails, StorageType +from fidesops.ops.service.privacy_request.onetrust_service import OneTrustService +from fidesops.ops.tasks.scheduled.scheduler import scheduler logger = logging.getLogger(__name__) diff --git a/src/fidesops/tasks/storage.py b/src/fidesops/ops/tasks/storage.py similarity index 94% rename from src/fidesops/tasks/storage.py rename to src/fidesops/ops/tasks/storage.py index 7018feea35..cc4c7c685a 100644 --- a/src/fidesops/tasks/storage.py +++ b/src/fidesops/ops/tasks/storage.py @@ -14,14 +14,14 @@ from botocore.exceptions import ClientError, ParamValidationError from fideslib.cryptography.cryptographic_util import bytes_to_b64_str -from fidesops.core.config import config -from fidesops.models.storage import ResponseFormat -from fidesops.schemas.storage.storage import StorageSecrets -from fidesops.util.cache import get_cache, get_encryption_cache_key -from fidesops.util.encryption.aes_gcm_encryption_scheme import ( +from fidesops.ops.core.config import config +from fidesops.ops.models.storage import ResponseFormat +from fidesops.ops.schemas.storage.storage import StorageSecrets +from fidesops.ops.util.cache import get_cache, get_encryption_cache_key +from fidesops.ops.util.encryption.aes_gcm_encryption_scheme import ( encrypt_to_bytes_verify_secrets_length, ) -from fidesops.util.storage_authenticator import ( +from fidesops.ops.util.storage_authenticator import ( get_onetrust_access_token, get_s3_session, ) diff --git a/src/fidesops/util/encryption/__init__.py b/src/fidesops/ops/util/__init__.py similarity index 100% rename from src/fidesops/util/encryption/__init__.py rename to src/fidesops/ops/util/__init__.py diff --git a/src/fidesops/util/api_router.py b/src/fidesops/ops/util/api_router.py similarity index 100% rename from src/fidesops/util/api_router.py rename to src/fidesops/ops/util/api_router.py diff --git a/src/fidesops/util/cache.py b/src/fidesops/ops/util/cache.py similarity index 97% rename from src/fidesops/util/cache.py rename to src/fidesops/ops/util/cache.py index 2e099cb295..861e41f995 100644 --- a/src/fidesops/util/cache.py +++ b/src/fidesops/ops/util/cache.py @@ -6,9 +6,9 @@ from redis import Redis from redis.client import Script # type: ignore -from fidesops import common_exceptions -from fidesops.core.config import config -from fidesops.schemas.masking.masking_secrets import SecretType +from fidesops.ops import common_exceptions +from fidesops.ops.core.config import config +from fidesops.ops.schemas.masking.masking_secrets import SecretType logger = logging.getLogger(__name__) diff --git a/src/fidesops/util/collection_util.py b/src/fidesops/ops/util/collection_util.py similarity index 100% rename from src/fidesops/util/collection_util.py rename to src/fidesops/ops/util/collection_util.py diff --git a/src/fidesops/util/data_category.py b/src/fidesops/ops/util/data_category.py similarity index 95% rename from src/fidesops/util/data_category.py rename to src/fidesops/ops/util/data_category.py index b42b576425..338bb65149 100644 --- a/src/fidesops/util/data_category.py +++ b/src/fidesops/ops/util/data_category.py @@ -3,7 +3,7 @@ from fideslang import DEFAULT_TAXONOMY -from fidesops import common_exceptions +from fidesops.ops import common_exceptions def generate_fides_data_categories() -> Type[EnumType]: diff --git a/src/fidesops/ops/util/encryption/__init__.py b/src/fidesops/ops/util/encryption/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/fidesops/util/encryption/aes_gcm_encryption_scheme.py b/src/fidesops/ops/util/encryption/aes_gcm_encryption_scheme.py similarity index 98% rename from src/fidesops/util/encryption/aes_gcm_encryption_scheme.py rename to src/fidesops/ops/util/encryption/aes_gcm_encryption_scheme.py index e1604f26b5..48a8b960bb 100644 --- a/src/fidesops/util/encryption/aes_gcm_encryption_scheme.py +++ b/src/fidesops/ops/util/encryption/aes_gcm_encryption_scheme.py @@ -4,7 +4,7 @@ from cryptography.hazmat.primitives.ciphers.aead import AESGCM from fideslib.cryptography.cryptographic_util import bytes_to_b64_str -from fidesops.core.config import config +from fidesops.ops.core.config import config def encrypt_to_bytes_verify_secrets_length( diff --git a/src/fidesops/util/encryption/hmac_encryption_scheme.py b/src/fidesops/ops/util/encryption/hmac_encryption_scheme.py similarity index 93% rename from src/fidesops/util/encryption/hmac_encryption_scheme.py rename to src/fidesops/ops/util/encryption/hmac_encryption_scheme.py index 0fb8ec8e55..14e364d358 100644 --- a/src/fidesops/util/encryption/hmac_encryption_scheme.py +++ b/src/fidesops/ops/util/encryption/hmac_encryption_scheme.py @@ -2,8 +2,8 @@ import hmac from typing import Callable -from fidesops.core.config import config -from fidesops.schemas.masking.masking_configuration import HmacMaskingConfiguration +from fidesops.ops.core.config import config +from fidesops.ops.schemas.masking.masking_configuration import HmacMaskingConfiguration def hmac_encrypt_return_bytes( diff --git a/src/fidesops/util/encryption/secrets_util.py b/src/fidesops/ops/util/encryption/secrets_util.py similarity index 94% rename from src/fidesops/util/encryption/secrets_util.py rename to src/fidesops/ops/util/encryption/secrets_util.py index 94260540ba..46370aadfa 100644 --- a/src/fidesops/util/encryption/secrets_util.py +++ b/src/fidesops/ops/util/encryption/secrets_util.py @@ -2,12 +2,12 @@ import secrets from typing import Dict, List, Optional, TypeVar -from fidesops.schemas.masking.masking_secrets import ( +from fidesops.ops.schemas.masking.masking_secrets import ( MaskingSecretCache, MaskingSecretMeta, SecretType, ) -from fidesops.util.cache import get_cache, get_masking_secret_cache_key +from fidesops.ops.util.cache import get_cache, get_masking_secret_cache_key T = TypeVar("T") logger = logging.getLogger(__name__) diff --git a/src/fidesops/util/logger.py b/src/fidesops/ops/util/logger.py similarity index 100% rename from src/fidesops/util/logger.py rename to src/fidesops/ops/util/logger.py diff --git a/src/fidesops/util/matching_queue.py b/src/fidesops/ops/util/matching_queue.py similarity index 100% rename from src/fidesops/util/matching_queue.py rename to src/fidesops/ops/util/matching_queue.py diff --git a/src/fidesops/util/oauth_util.py b/src/fidesops/ops/util/oauth_util.py similarity index 94% rename from src/fidesops/util/oauth_util.py rename to src/fidesops/ops/util/oauth_util.py index 0f95b17758..3b35179f71 100644 --- a/src/fidesops/util/oauth_util.py +++ b/src/fidesops/ops/util/oauth_util.py @@ -20,12 +20,12 @@ from sqlalchemy.orm import Session from starlette.status import HTTP_404_NOT_FOUND -from fidesops.api.deps import get_db -from fidesops.api.v1.scope_registry import SCOPE_REGISTRY -from fidesops.api.v1.urn_registry import TOKEN, V1_URL_PREFIX -from fidesops.core.config import config -from fidesops.models.policy import PolicyPreWebhook -from fidesops.schemas.external_https import WebhookJWE +from fidesops.ops.api.deps import get_db +from fidesops.ops.api.v1.scope_registry import SCOPE_REGISTRY +from fidesops.ops.api.v1.urn_registry import TOKEN, V1_URL_PREFIX +from fidesops.ops.core.config import config +from fidesops.ops.models.policy import PolicyPreWebhook +from fidesops.ops.schemas.external_https import WebhookJWE JWT_ENCRYPTION_ALGORITHM = ALGORITHMS.A256GCM diff --git a/src/fidesops/util/querytoken.py b/src/fidesops/ops/util/querytoken.py similarity index 100% rename from src/fidesops/util/querytoken.py rename to src/fidesops/ops/util/querytoken.py diff --git a/src/fidesops/util/saas_util.py b/src/fidesops/ops/util/saas_util.py similarity index 96% rename from src/fidesops/util/saas_util.py rename to src/fidesops/ops/util/saas_util.py index 941c7cb552..9510395d30 100644 --- a/src/fidesops/util/saas_util.py +++ b/src/fidesops/ops/util/saas_util.py @@ -11,10 +11,10 @@ from fideslib.core.config import load_file from multidimensional_urlencode import urlencode as multidimensional_urlencode -from fidesops.common_exceptions import FidesopsException -from fidesops.graph.config import Collection, CollectionAddress, Dataset, Field -from fidesops.schemas.saas.saas_config import SaaSRequest -from fidesops.schemas.saas.shared_schemas import SaaSRequestParams +from fidesops.ops.common_exceptions import FidesopsException +from fidesops.ops.graph.config import Collection, CollectionAddress, Dataset, Field +from fidesops.ops.schemas.saas.saas_config import SaaSRequest +from fidesops.ops.schemas.saas.shared_schemas import SaaSRequestParams logger = logging.getLogger(__name__) diff --git a/src/fidesops/util/storage_authenticator.py b/src/fidesops/ops/util/storage_authenticator.py similarity index 93% rename from src/fidesops/util/storage_authenticator.py rename to src/fidesops/ops/util/storage_authenticator.py index 409129eb45..4dd02112df 100644 --- a/src/fidesops/util/storage_authenticator.py +++ b/src/fidesops/ops/util/storage_authenticator.py @@ -3,7 +3,7 @@ from boto3 import Session from requests import Response -from fidesops.schemas.third_party.onetrust import OneTrustOAuthResponse +from fidesops.ops.schemas.third_party.onetrust import OneTrustOAuthResponse def get_s3_session(aws_access_key_id: str, aws_secret_access_key: str) -> Session: diff --git a/src/fidesops/util/text.py b/src/fidesops/ops/util/text.py similarity index 100% rename from src/fidesops/util/text.py rename to src/fidesops/ops/util/text.py diff --git a/src/fidesops/util/url_util.py b/src/fidesops/ops/util/url_util.py similarity index 100% rename from src/fidesops/util/url_util.py rename to src/fidesops/ops/util/url_util.py diff --git a/src/fidesops/service/saas_request/__init__.py b/src/fidesops/service/saas_request/__init__.py deleted file mode 100644 index 5558cb092b..0000000000 --- a/src/fidesops/service/saas_request/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from fidesops.service.saas_request.override_implementations import ( - mailchimp_request_overrides, -) diff --git a/tests/ops/api/test_deps.py b/tests/ops/api/test_deps.py index 0851380f7b..8a1f5e59d3 100644 --- a/tests/ops/api/test_deps.py +++ b/tests/ops/api/test_deps.py @@ -1,8 +1,8 @@ import pytest -from fidesops.api.deps import get_cache, get_db -from fidesops.common_exceptions import FunctionalityNotConfigured -from fidesops.core import config +from fidesops.ops.api.deps import get_cache, get_db +from fidesops.ops.common_exceptions import FunctionalityNotConfigured +from fidesops.ops.core import config @pytest.fixture diff --git a/tests/ops/api/v1/endpoints/test_config_endpoints.py b/tests/ops/api/v1/endpoints/test_config_endpoints.py index 1ab97a5870..95286f1c51 100644 --- a/tests/ops/api/v1/endpoints/test_config_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_config_endpoints.py @@ -1,8 +1,8 @@ import pytest from starlette.testclient import TestClient -from fidesops.api.v1 import scope_registry as scopes -from fidesops.api.v1 import urn_registry as urls +from fidesops.ops.api.v1 import scope_registry as scopes +from fidesops.ops.api.v1 import urn_registry as urls class TestGetConnections: diff --git a/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py b/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py index 6d8c44cb60..8b7cedd153 100644 --- a/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py @@ -11,14 +11,14 @@ from sqlalchemy.orm import Session from starlette.testclient import TestClient -from fidesops.api.v1.scope_registry import ( +from fidesops.ops.api.v1.scope_registry import ( CONNECTION_CREATE_OR_UPDATE, CONNECTION_DELETE, CONNECTION_READ, STORAGE_DELETE, ) -from fidesops.api.v1.urn_registry import CONNECTIONS, SAAS_CONFIG, V1_URL_PREFIX -from fidesops.models.connectionconfig import ConnectionConfig +from fidesops.ops.api.v1.urn_registry import CONNECTIONS, SAAS_CONFIG, V1_URL_PREFIX +from fidesops.ops.models.connectionconfig import ConnectionConfig page_size = Params().size diff --git a/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py b/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py index 919a22269a..a60fc6902a 100644 --- a/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py @@ -4,18 +4,18 @@ from fideslib.models.client import ClientDetail from starlette.testclient import TestClient -from fidesops.api.v1.scope_registry import CONNECTION_READ, CONNECTION_TYPE_READ -from fidesops.api.v1.urn_registry import ( +from fidesops.ops.api.v1.scope_registry import CONNECTION_READ, CONNECTION_TYPE_READ +from fidesops.ops.api.v1.urn_registry import ( CONNECTION_TYPE_SECRETS, CONNECTION_TYPES, V1_URL_PREFIX, ) -from fidesops.models.connectionconfig import ConnectionType -from fidesops.schemas.connection_configuration.connection_config import ( +from fidesops.ops.models.connectionconfig import ConnectionType +from fidesops.ops.schemas.connection_configuration.connection_config import ( ConnectionSystemTypeMap, SystemType, ) -from fidesops.schemas.saas.saas_config import SaaSType +from fidesops.ops.schemas.saas.saas_config import SaaSType class TestGetConnections: diff --git a/tests/ops/api/v1/endpoints/test_dataset_endpoints.py b/tests/ops/api/v1/endpoints/test_dataset_endpoints.py index b942995d3d..2018d5d825 100644 --- a/tests/ops/api/v1/endpoints/test_dataset_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_dataset_endpoints.py @@ -11,20 +11,20 @@ from sqlalchemy.orm import Session from starlette.testclient import TestClient -from fidesops.api.v1.scope_registry import ( +from fidesops.ops.api.v1.scope_registry import ( DATASET_CREATE_OR_UPDATE, DATASET_DELETE, DATASET_READ, ) -from fidesops.api.v1.urn_registry import ( +from fidesops.ops.api.v1.urn_registry import ( DATASET_BY_KEY, DATASET_VALIDATE, DATASETS, V1_URL_PREFIX, YAML_DATASETS, ) -from fidesops.models.connectionconfig import ConnectionConfig -from fidesops.models.datasetconfig import DatasetConfig +from fidesops.ops.models.connectionconfig import ConnectionConfig +from fidesops.ops.models.datasetconfig import DatasetConfig def _reject_key(dict: Dict, key: str) -> Dict: @@ -800,7 +800,7 @@ def test_patch_datasets_fides_key_mismatch( "'saas_connector_example' of the connection config" ) - @mock.patch("fidesops.models.datasetconfig.DatasetConfig.create_or_update") + @mock.patch("fidesops.ops.models.datasetconfig.DatasetConfig.create_or_update") def test_patch_datasets_failed_response( self, mock_create: Mock, @@ -894,7 +894,7 @@ def test_patch_dataset_invalid_content( ) assert response.status_code == 400 - @mock.patch("fidesops.models.datasetconfig.DatasetConfig.create_or_update") + @mock.patch("fidesops.ops.models.datasetconfig.DatasetConfig.create_or_update") def test_patch_datasets_failed_response( self, mock_create: Mock, diff --git a/tests/ops/api/v1/endpoints/test_drp_endpoints.py b/tests/ops/api/v1/endpoints/test_drp_endpoints.py index 626bec1139..10f5afe40a 100644 --- a/tests/ops/api/v1/endpoints/test_drp_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_drp_endpoints.py @@ -6,24 +6,27 @@ from sqlalchemy.orm import Session from starlette.testclient import TestClient -from fidesops.api.v1.scope_registry import ( +from fidesops.ops.api.v1.scope_registry import ( POLICY_READ, PRIVACY_REQUEST_READ, PRIVACY_REQUEST_REVIEW, STORAGE_CREATE_OR_UPDATE, ) -from fidesops.api.v1.urn_registry import ( +from fidesops.ops.api.v1.urn_registry import ( DRP_DATA_RIGHTS, DRP_EXERCISE, DRP_REVOKE, DRP_STATUS, V1_URL_PREFIX, ) -from fidesops.core.config import config -from fidesops.models.policy import DrpAction -from fidesops.models.privacy_request import PrivacyRequest, PrivacyRequestStatus -from fidesops.schemas.privacy_request import PrivacyRequestDRPStatus -from fidesops.util.cache import get_drp_request_body_cache_key, get_identity_cache_key +from fidesops.ops.core.config import config +from fidesops.ops.models.policy import DrpAction +from fidesops.ops.models.privacy_request import PrivacyRequest, PrivacyRequestStatus +from fidesops.ops.schemas.privacy_request import PrivacyRequestDRPStatus +from fidesops.ops.util.cache import ( + get_drp_request_body_cache_key, + get_identity_cache_key, +) class TestCreateDrpPrivacyRequest: @@ -32,7 +35,7 @@ def url(self) -> str: return V1_URL_PREFIX + DRP_EXERCISE @mock.patch( - "fidesops.service.privacy_request.request_runner_service.run_privacy_request.delay" + "fidesops.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" ) def test_create_drp_privacy_request( self, @@ -103,7 +106,7 @@ def test_create_drp_privacy_request( assert run_access_request_mock.called @mock.patch( - "fidesops.service.privacy_request.request_runner_service.run_privacy_request.delay" + "fidesops.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" ) def test_create_drp_privacy_request_unsupported_identity_props( self, diff --git a/tests/ops/api/v1/endpoints/test_encryption_endpoints.py b/tests/ops/api/v1/endpoints/test_encryption_endpoints.py index 1b0519820b..624b811773 100644 --- a/tests/ops/api/v1/endpoints/test_encryption_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_encryption_endpoints.py @@ -6,15 +6,15 @@ from fideslib.cryptography.cryptographic_util import b64_str_to_bytes, bytes_to_b64_str from starlette.testclient import TestClient -from fidesops.api.v1.scope_registry import ENCRYPTION_EXEC, STORAGE_CREATE_OR_UPDATE -from fidesops.api.v1.urn_registry import ( +from fidesops.ops.api.v1.scope_registry import ENCRYPTION_EXEC, STORAGE_CREATE_OR_UPDATE +from fidesops.ops.api.v1.urn_registry import ( DECRYPT_AES, ENCRYPT_AES, ENCRYPTION_KEY, V1_URL_PREFIX, ) -from fidesops.core.config import config -from fidesops.util.encryption.aes_gcm_encryption_scheme import ( +from fidesops.ops.core.config import config +from fidesops.ops.util.encryption.aes_gcm_encryption_scheme import ( decrypt, encrypt_verify_secret_length, ) @@ -39,7 +39,7 @@ def test_get_encryption_key_wrong_scope( assert response.status_code == 403 @mock.patch( - "fidesops.api.v1.endpoints.encryption_endpoints.cryptographic_util.generate_secure_random_string" + "fidesops.ops.api.v1.endpoints.encryption_endpoints.cryptographic_util.generate_secure_random_string" ) def test_get_encryption_key( self, diff --git a/tests/ops/api/v1/endpoints/test_health_endpoints.py b/tests/ops/api/v1/endpoints/test_health_endpoints.py index df23fb13a3..43deb8fb90 100644 --- a/tests/ops/api/v1/endpoints/test_health_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_health_endpoints.py @@ -1,6 +1,6 @@ from starlette.testclient import TestClient -from fidesops.api.v1.urn_registry import HEALTH +from fidesops.ops.api.v1.urn_registry import HEALTH def test_health(api_client: TestClient) -> None: diff --git a/tests/ops/api/v1/endpoints/test_masking_endpoints.py b/tests/ops/api/v1/endpoints/test_masking_endpoints.py index 6c48d3d595..b8f9240929 100644 --- a/tests/ops/api/v1/endpoints/test_masking_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_masking_endpoints.py @@ -2,26 +2,30 @@ from starlette.testclient import TestClient -from fidesops.api.v1.urn_registry import MASKING, MASKING_STRATEGY, V1_URL_PREFIX -from fidesops.schemas.masking.masking_api import MaskingAPIResponse -from fidesops.schemas.masking.masking_configuration import ( +from fidesops.ops.api.v1.urn_registry import MASKING, MASKING_STRATEGY, V1_URL_PREFIX +from fidesops.ops.schemas.masking.masking_api import MaskingAPIResponse +from fidesops.ops.schemas.masking.masking_configuration import ( AesEncryptionMaskingConfiguration, ) -from fidesops.service.masking.strategy.masking_strategy_aes_encrypt import ( +from fidesops.ops.service.masking.strategy.masking_strategy_aes_encrypt import ( AES_ENCRYPT_STRATEGY_NAME, ) -from fidesops.service.masking.strategy.masking_strategy_factory import ( +from fidesops.ops.service.masking.strategy.masking_strategy_factory import ( MaskingStrategyFactory, ) -from fidesops.service.masking.strategy.masking_strategy_hash import HASH_STRATEGY_NAME -from fidesops.service.masking.strategy.masking_strategy_hmac import HMAC_STRATEGY_NAME -from fidesops.service.masking.strategy.masking_strategy_nullify import ( +from fidesops.ops.service.masking.strategy.masking_strategy_hash import ( + HASH_STRATEGY_NAME, +) +from fidesops.ops.service.masking.strategy.masking_strategy_hmac import ( + HMAC_STRATEGY_NAME, +) +from fidesops.ops.service.masking.strategy.masking_strategy_nullify import ( NULL_REWRITE_STRATEGY_NAME, ) -from fidesops.service.masking.strategy.masking_strategy_random_string_rewrite import ( +from fidesops.ops.service.masking.strategy.masking_strategy_random_string_rewrite import ( RANDOM_STRING_REWRITE_STRATEGY_NAME, ) -from fidesops.service.masking.strategy.masking_strategy_string_rewrite import ( +from fidesops.ops.service.masking.strategy.masking_strategy_string_rewrite import ( STRING_REWRITE_STRATEGY_NAME, ) diff --git a/tests/ops/api/v1/endpoints/test_oauth_endpoints.py b/tests/ops/api/v1/endpoints/test_oauth_endpoints.py index 5ecd0cc452..a5be15154b 100644 --- a/tests/ops/api/v1/endpoints/test_oauth_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_oauth_endpoints.py @@ -14,7 +14,7 @@ from fideslib.oauth.oauth_util import extract_payload from starlette.testclient import TestClient -from fidesops.api.v1.scope_registry import ( +from fidesops.ops.api.v1.scope_registry import ( CLIENT_CREATE, CLIENT_DELETE, CLIENT_READ, @@ -23,7 +23,7 @@ SCOPE_REGISTRY, STORAGE_READ, ) -from fidesops.api.v1.urn_registry import ( +from fidesops.ops.api.v1.urn_registry import ( CLIENT, CLIENT_BY_ID, CLIENT_SCOPE, @@ -32,9 +32,9 @@ TOKEN, V1_URL_PREFIX, ) -from fidesops.common_exceptions import OAuth2TokenException -from fidesops.core.config import config -from fidesops.models.authentication_request import AuthenticationRequest +from fidesops.ops.common_exceptions import OAuth2TokenException +from fidesops.ops.core.config import config +from fidesops.ops.models.authentication_request import AuthenticationRequest class TestCreateClient: @@ -458,7 +458,7 @@ def test_callback_for_missing_state(self, db, api_client: TestClient, callback_u } @mock.patch( - "fidesops.api.v1.endpoints.saas_config_endpoints.OAuth2AuthenticationStrategy.get_access_token" + "fidesops.ops.api.v1.endpoints.saas_config_endpoints.OAuth2AuthenticationStrategy.get_access_token" ) def test_callback_for_valid_state( self, @@ -485,7 +485,7 @@ def test_callback_for_valid_state( authentication_request.delete(db) @mock.patch( - "fidesops.api.v1.endpoints.saas_config_endpoints.OAuth2AuthenticationStrategy.get_access_token" + "fidesops.ops.api.v1.endpoints.saas_config_endpoints.OAuth2AuthenticationStrategy.get_access_token" ) def test_callback_for_valid_state_with_token_error( self, diff --git a/tests/ops/api/v1/endpoints/test_policy_endpoints.py b/tests/ops/api/v1/endpoints/test_policy_endpoints.py index 1ea239dccd..4c0b7bda00 100644 --- a/tests/ops/api/v1/endpoints/test_policy_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_policy_endpoints.py @@ -5,21 +5,21 @@ from fideslib.models.client import ClientDetail from starlette.testclient import TestClient -from fidesops.api.v1 import scope_registry as scopes -from fidesops.api.v1.urn_registry import POLICY_DETAIL as POLICY_DETAIL_URI -from fidesops.api.v1.urn_registry import POLICY_LIST as POLICY_CREATE_URI -from fidesops.api.v1.urn_registry import RULE_DETAIL as RULE_DETAIL_URI -from fidesops.api.v1.urn_registry import RULE_LIST as RULE_CREATE_URI -from fidesops.api.v1.urn_registry import ( +from fidesops.ops.api.v1 import scope_registry as scopes +from fidesops.ops.api.v1.urn_registry import POLICY_DETAIL as POLICY_DETAIL_URI +from fidesops.ops.api.v1.urn_registry import POLICY_LIST as POLICY_CREATE_URI +from fidesops.ops.api.v1.urn_registry import RULE_DETAIL as RULE_DETAIL_URI +from fidesops.ops.api.v1.urn_registry import RULE_LIST as RULE_CREATE_URI +from fidesops.ops.api.v1.urn_registry import ( RULE_TARGET_DETAIL, RULE_TARGET_LIST, V1_URL_PREFIX, ) -from fidesops.models.policy import ActionType, DrpAction, Policy, Rule, RuleTarget -from fidesops.service.masking.strategy.masking_strategy_nullify import ( +from fidesops.ops.models.policy import ActionType, DrpAction, Policy, Rule, RuleTarget +from fidesops.ops.service.masking.strategy.masking_strategy_nullify import ( NULL_REWRITE_STRATEGY_NAME, ) -from fidesops.util.data_category import DataCategory, generate_fides_data_categories +from fidesops.ops.util.data_category import DataCategory, generate_fides_data_categories class TestGetPolicies: diff --git a/tests/ops/api/v1/endpoints/test_policy_webhook_endpoints.py b/tests/ops/api/v1/endpoints/test_policy_webhook_endpoints.py index 41707501ca..05e86e12e9 100644 --- a/tests/ops/api/v1/endpoints/test_policy_webhook_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_policy_webhook_endpoints.py @@ -3,21 +3,21 @@ import pytest -from fidesops.api.v1.scope_registry import ( +from fidesops.ops.api.v1.scope_registry import ( POLICY_READ, WEBHOOK_CREATE_OR_UPDATE, WEBHOOK_DELETE, WEBHOOK_READ, ) -from fidesops.api.v1.urn_registry import ( +from fidesops.ops.api.v1.urn_registry import ( POLICY_POST_WEBHOOK_DETAIL, POLICY_PRE_WEBHOOK_DETAIL, POLICY_WEBHOOKS_POST, POLICY_WEBHOOKS_PRE, V1_URL_PREFIX, ) -from fidesops.models.connectionconfig import ConnectionConfig -from fidesops.models.policy import PolicyPostWebhook, PolicyPreWebhook +from fidesops.ops.models.connectionconfig import ConnectionConfig +from fidesops.ops.models.policy import PolicyPostWebhook, PolicyPreWebhook from tests.ops.api.v1.endpoints.test_privacy_request_endpoints import stringify_date diff --git a/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py b/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py index 77b2acdffc..d04a08660d 100644 --- a/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py @@ -20,18 +20,18 @@ from fideslib.oauth.jwt import generate_jwe from starlette.testclient import TestClient -from fidesops.api.v1.endpoints.privacy_request_endpoints import ( +from fidesops.ops.api.v1.endpoints.privacy_request_endpoints import ( EMBEDDED_EXECUTION_LOG_LIMIT, validate_manual_input, ) -from fidesops.api.v1.scope_registry import ( +from fidesops.ops.api.v1.scope_registry import ( DATASET_CREATE_OR_UPDATE, PRIVACY_REQUEST_CALLBACK_RESUME, PRIVACY_REQUEST_READ, PRIVACY_REQUEST_REVIEW, STORAGE_CREATE_OR_UPDATE, ) -from fidesops.api.v1.urn_registry import ( +from fidesops.ops.api.v1.urn_registry import ( DATASETS, PRIVACY_REQUEST_APPROVE, PRIVACY_REQUEST_DENY, @@ -43,23 +43,23 @@ REQUEST_PREVIEW, V1_URL_PREFIX, ) -from fidesops.core.config import config -from fidesops.graph.config import CollectionAddress -from fidesops.graph.graph import DatasetGraph -from fidesops.models.datasetconfig import DatasetConfig -from fidesops.models.policy import ActionType, PausedStep -from fidesops.models.privacy_request import ( +from fidesops.ops.core.config import config +from fidesops.ops.graph.config import CollectionAddress +from fidesops.ops.graph.graph import DatasetGraph +from fidesops.ops.models.datasetconfig import DatasetConfig +from fidesops.ops.models.policy import ActionType, PausedStep +from fidesops.ops.models.privacy_request import ( ExecutionLog, ExecutionLogStatus, ManualAction, PrivacyRequest, PrivacyRequestStatus, ) -from fidesops.schemas.dataset import DryRunDatasetResponse -from fidesops.schemas.masking.masking_secrets import SecretType -from fidesops.schemas.policy import PolicyResponse -from fidesops.schemas.redis_cache import PrivacyRequestIdentity -from fidesops.util.cache import ( +from fidesops.ops.schemas.dataset import DryRunDatasetResponse +from fidesops.ops.schemas.masking.masking_secrets import SecretType +from fidesops.ops.schemas.policy import PolicyResponse +from fidesops.ops.schemas.redis_cache import PrivacyRequestIdentity +from fidesops.ops.util.cache import ( get_encryption_cache_key, get_identity_cache_key, get_masking_secret_cache_key, @@ -78,7 +78,7 @@ def url(self, oauth_client: ClientDetail, policy) -> str: return V1_URL_PREFIX + PRIVACY_REQUESTS @mock.patch( - "fidesops.service.privacy_request.request_runner_service.run_privacy_request.delay" + "fidesops.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" ) def test_create_privacy_request( self, @@ -104,7 +104,7 @@ def test_create_privacy_request( assert run_access_request_mock.called @mock.patch( - "fidesops.service.privacy_request.request_runner_service.run_privacy_request.delay" + "fidesops.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" ) def test_create_privacy_request_stores_identities( self, @@ -138,7 +138,7 @@ def test_create_privacy_request_stores_identities( assert run_access_request_mock.called @mock.patch( - "fidesops.service.privacy_request.request_runner_service.run_privacy_request.delay" + "fidesops.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" ) def test_create_privacy_request_require_manual_approval( self, @@ -169,7 +169,7 @@ def test_create_privacy_request_require_manual_approval( config.execution.require_manual_request_approval = False @mock.patch( - "fidesops.service.privacy_request.request_runner_service.run_privacy_request.delay" + "fidesops.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" ) def test_create_privacy_request_with_masking_configuration( self, @@ -195,7 +195,7 @@ def test_create_privacy_request_with_masking_configuration( assert run_access_request_mock.called @mock.patch( - "fidesops.service.privacy_request.request_runner_service.run_access_request" + "fidesops.ops.service.privacy_request.request_runner_service.run_access_request" ) def test_create_privacy_request_limit_exceeded( self, @@ -224,7 +224,7 @@ def test_create_privacy_request_limit_exceeded( ) @mock.patch( - "fidesops.service.privacy_request.request_runner_service.run_privacy_request.delay" + "fidesops.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" ) def test_create_privacy_request_starts_processing( self, @@ -249,7 +249,7 @@ def test_create_privacy_request_starts_processing( pr.delete(db=db) @mock.patch( - "fidesops.service.privacy_request.request_runner_service.run_privacy_request.delay" + "fidesops.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" ) def test_create_privacy_request_with_external_id( self, @@ -279,7 +279,7 @@ def test_create_privacy_request_with_external_id( assert run_access_request_mock.called @mock.patch( - "fidesops.service.privacy_request.request_runner_service.run_privacy_request.delay" + "fidesops.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" ) def test_create_privacy_request_caches_identity( self, @@ -312,7 +312,7 @@ def test_create_privacy_request_caches_identity( assert run_access_request_mock.called @mock.patch( - "fidesops.service.privacy_request.request_runner_service.run_privacy_request.delay" + "fidesops.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" ) def test_create_privacy_request_caches_masking_secrets( self, @@ -361,7 +361,7 @@ def test_create_privacy_request_invalid_encryption_values( assert resp.json()["detail"][0]["msg"] == "Encryption key must be 16 bytes long" @mock.patch( - "fidesops.service.privacy_request.request_runner_service.run_privacy_request.delay" + "fidesops.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" ) def test_create_privacy_request_caches_encryption_keys( self, @@ -439,7 +439,7 @@ def test_create_privacy_request_registers_async_task( pr.delete(db=db) @mock.patch( - "fidesops.service.privacy_request.request_runner_service.run_privacy_request.delay" + "fidesops.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" ) def test_create_privacy_request_creates_system_audit_log( self, @@ -1535,7 +1535,7 @@ def test_approve_privacy_request_bad_scopes( assert response.status_code == 403 @mock.patch( - "fidesops.service.privacy_request.request_runner_service.run_privacy_request.delay" + "fidesops.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" ) def test_approve_privacy_request_does_not_exist( self, submit_mock, db, url, api_client, generate_auth_header, privacy_request @@ -1560,7 +1560,7 @@ def test_approve_privacy_request_does_not_exist( [PrivacyRequestStatus.complete, PrivacyRequestStatus.canceled], ) @mock.patch( - "fidesops.service.privacy_request.request_runner_service.run_privacy_request.delay" + "fidesops.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" ) def test_approve_privacy_request_in_non_pending_state( self, @@ -1590,7 +1590,7 @@ def test_approve_privacy_request_in_non_pending_state( assert not submit_mock.called @mock.patch( - "fidesops.service.privacy_request.request_runner_service.run_privacy_request.delay" + "fidesops.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" ) def test_approve_privacy_request_no_user_on_client( self, @@ -1621,7 +1621,7 @@ def test_approve_privacy_request_no_user_on_client( assert submit_mock.called @mock.patch( - "fidesops.service.privacy_request.request_runner_service.run_privacy_request.delay" + "fidesops.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" ) def test_approve_privacy_request( self, @@ -1664,7 +1664,7 @@ def test_approve_privacy_request( privacy_request.delete(db) @mock.patch( - "fidesops.service.privacy_request.request_runner_service.run_privacy_request.delay" + "fidesops.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" ) def test_approve_privacy_request_creates_audit_log( self, @@ -1720,7 +1720,7 @@ def test_deny_privacy_request_bad_scopes( assert response.status_code == 403 @mock.patch( - "fidesops.service.privacy_request.request_runner_service.run_privacy_request.delay" + "fidesops.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" ) def test_deny_privacy_request_does_not_exist( self, submit_mock, db, url, api_client, generate_auth_header, privacy_request @@ -1741,7 +1741,7 @@ def test_deny_privacy_request_does_not_exist( assert not submit_mock.called @mock.patch( - "fidesops.service.privacy_request.request_runner_service.run_privacy_request.delay" + "fidesops.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" ) def test_deny_completed_privacy_request( self, submit_mock, db, url, api_client, generate_auth_header, privacy_request @@ -1762,7 +1762,7 @@ def test_deny_completed_privacy_request( assert not submit_mock.called @mock.patch( - "fidesops.service.privacy_request.request_runner_service.run_privacy_request.delay" + "fidesops.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" ) def test_deny_privacy_request_without_denial_reason( self, @@ -1813,7 +1813,7 @@ def test_deny_privacy_request_without_denial_reason( privacy_request.delete(db) @mock.patch( - "fidesops.service.privacy_request.request_runner_service.run_privacy_request.delay" + "fidesops.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" ) def test_deny_privacy_request_with_denial_reason( self, @@ -1969,7 +1969,7 @@ def test_resume_privacy_request_not_paused( privacy_request.delete(db) @mock.patch( - "fidesops.service.privacy_request.request_runner_service.run_privacy_request.delay" + "fidesops.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" ) def test_resume_privacy_request( self, @@ -2119,7 +2119,7 @@ def test_resume_with_manual_input_invalid_data( privacy_request.delete(db) @mock.patch( - "fidesops.service.privacy_request.request_runner_service.run_privacy_request.delay" + "fidesops.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" ) @pytest.mark.usefixtures( "postgres_example_test_dataset_config", "manual_dataset_config" @@ -2300,7 +2300,7 @@ def test_resume_still_paused_at_access_request( "postgres_example_test_dataset_config", "manual_dataset_config" ) @mock.patch( - "fidesops.service.privacy_request.request_runner_service.run_privacy_request.delay" + "fidesops.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" ) def test_resume_with_manual_count( self, @@ -2380,7 +2380,7 @@ def test_restart_from_failure_no_stopped_collection( privacy_request.delete(db) @mock.patch( - "fidesops.service.privacy_request.request_runner_service.run_privacy_request.delay" + "fidesops.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" ) def test_restart_from_failure( self, submit_mock, api_client, url, generate_auth_header, db, privacy_request diff --git a/tests/ops/api/v1/endpoints/test_saas_config_endpoints.py b/tests/ops/api/v1/endpoints/test_saas_config_endpoints.py index fe80ff5f98..4819ac0c78 100644 --- a/tests/ops/api/v1/endpoints/test_saas_config_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_saas_config_endpoints.py @@ -7,20 +7,20 @@ from sqlalchemy.orm import Session from starlette.testclient import TestClient -from fidesops.api.v1.scope_registry import ( +from fidesops.ops.api.v1.scope_registry import ( CLIENT_READ, CONNECTION_AUTHORIZE, SAAS_CONFIG_CREATE_OR_UPDATE, SAAS_CONFIG_DELETE, SAAS_CONFIG_READ, ) -from fidesops.api.v1.urn_registry import ( +from fidesops.ops.api.v1.urn_registry import ( AUTHORIZE, SAAS_CONFIG, SAAS_CONFIG_VALIDATE, V1_URL_PREFIX, ) -from fidesops.models.connectionconfig import ( +from fidesops.ops.models.connectionconfig import ( AccessLevel, ConnectionConfig, ConnectionType, @@ -435,7 +435,7 @@ def test_client_wrong_scope( assert 403 == response.status_code @mock.patch( - "fidesops.api.v1.endpoints.saas_config_endpoints.OAuth2AuthenticationStrategy.get_authorization_url" + "fidesops.ops.api.v1.endpoints.saas_config_endpoints.OAuth2AuthenticationStrategy.get_authorization_url" ) def test_get_authorize_url( self, diff --git a/tests/ops/api/v1/endpoints/test_storage_endpoints.py b/tests/ops/api/v1/endpoints/test_storage_endpoints.py index ba5bdabf03..89fa111085 100644 --- a/tests/ops/api/v1/endpoints/test_storage_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_storage_endpoints.py @@ -9,21 +9,21 @@ from sqlalchemy.orm import Session from starlette.testclient import TestClient -from fidesops.api.v1.scope_registry import ( +from fidesops.ops.api.v1.scope_registry import ( STORAGE_CREATE_OR_UPDATE, STORAGE_DELETE, STORAGE_READ, ) -from fidesops.api.v1.urn_registry import ( +from fidesops.ops.api.v1.urn_registry import ( STORAGE_BY_KEY, STORAGE_CONFIG, STORAGE_SECRETS, STORAGE_UPLOAD, V1_URL_PREFIX, ) -from fidesops.models.storage import StorageConfig -from fidesops.schemas.storage.data_upload_location_response import DataUpload -from fidesops.schemas.storage.storage import ( +from fidesops.ops.models.storage import StorageConfig +from fidesops.ops.schemas.storage.data_upload_location_response import DataUpload +from fidesops.ops.schemas.storage.storage import ( FileNaming, ResponseFormat, StorageDetails, @@ -69,7 +69,7 @@ def test_invalid_privacy_request( response = api_client.post(url, headers=auth_header, json=payload) assert 404 == response.status_code - @mock.patch("fidesops.api.v1.endpoints.storage_endpoints.upload") + @mock.patch("fidesops.ops.api.v1.endpoints.storage_endpoints.upload") def test_post_upload_data( self, mock_post_upload_data: Mock, @@ -118,7 +118,7 @@ def payload(self): ] @mock.patch( - "fidesops.api.v1.endpoints.storage_endpoints.initiate_scheduled_request_intake" + "fidesops.ops.api.v1.endpoints.storage_endpoints.initiate_scheduled_request_intake" ) def test_patch_storage_config_not_authenticated( self, mock_scheduled_task, api_client: TestClient, payload, url @@ -129,7 +129,7 @@ def test_patch_storage_config_not_authenticated( mock_scheduled_task.assert_not_called() @mock.patch( - "fidesops.api.v1.endpoints.storage_endpoints.initiate_scheduled_request_intake" + "fidesops.ops.api.v1.endpoints.storage_endpoints.initiate_scheduled_request_intake" ) def test_patch_storage_config_incorrect_scope( self, @@ -173,7 +173,7 @@ def test_patch_storage_config_with_onetrust_format_conflict( ) @mock.patch( - "fidesops.api.v1.endpoints.storage_endpoints.initiate_scheduled_request_intake" + "fidesops.ops.api.v1.endpoints.storage_endpoints.initiate_scheduled_request_intake" ) def test_patch_storage_config_with_no_key( self, @@ -213,7 +213,7 @@ def test_put_storage_config_with_invalid_key( ) @mock.patch( - "fidesops.api.v1.endpoints.storage_endpoints.initiate_scheduled_request_intake" + "fidesops.ops.api.v1.endpoints.storage_endpoints.initiate_scheduled_request_intake" ) def test_patch_storage_configs_limits_exceeded( self, @@ -251,7 +251,7 @@ def test_patch_storage_configs_limits_exceeded( ) @mock.patch( - "fidesops.api.v1.endpoints.storage_endpoints.initiate_scheduled_request_intake" + "fidesops.ops.api.v1.endpoints.storage_endpoints.initiate_scheduled_request_intake" ) def test_patch_storage_config_with_key( self, @@ -294,7 +294,7 @@ def test_patch_storage_config_with_key( storage_config.delete(db) @mock.patch( - "fidesops.api.v1.endpoints.storage_endpoints.initiate_scheduled_request_intake" + "fidesops.ops.api.v1.endpoints.storage_endpoints.initiate_scheduled_request_intake" ) def test_patch_config_response_format_not_specified( self, @@ -466,7 +466,7 @@ def test_put_config_secrets_without_verifying( == "23451345834789" ) - @mock.patch("fidesops.api.v1.endpoints.storage_endpoints.secrets_are_valid") + @mock.patch("fidesops.ops.api.v1.endpoints.storage_endpoints.secrets_are_valid") def test_put_config_secrets_and_verify( self, mock_valid: Mock, @@ -507,7 +507,9 @@ def test_put_config_secrets_and_verify( "failure_reason": None, } - @mock.patch("fidesops.service.storage.storage_authenticator_service.get_s3_session") + @mock.patch( + "fidesops.ops.service.storage.storage_authenticator_service.get_s3_session" + ) def test_put_s3_config_secrets_and_verify( self, get_s3_session_mock: Mock, @@ -522,7 +524,7 @@ def test_put_s3_config_secrets_and_verify( get_s3_session_mock.assert_called_once_with(**payload) @mock.patch( - "fidesops.service.storage.storage_authenticator_service.get_onetrust_access_token" + "fidesops.ops.service.storage.storage_authenticator_service.get_onetrust_access_token" ) def test_put_onetrust_config_secrets_and_verify( self, @@ -615,7 +617,7 @@ def test_put_config_secrets_without_verifying( == "peanutbutter.onetrust" ) - @mock.patch("fidesops.api.v1.endpoints.storage_endpoints.secrets_are_valid") + @mock.patch("fidesops.ops.api.v1.endpoints.storage_endpoints.secrets_are_valid") def test_put_config_secrets_and_verify( self, mock_valid: Mock, diff --git a/tests/ops/api/v1/endpoints/test_user_endpoints.py b/tests/ops/api/v1/endpoints/test_user_endpoints.py index 960789c9cc..e00501f775 100644 --- a/tests/ops/api/v1/endpoints/test_user_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_user_endpoints.py @@ -27,7 +27,7 @@ ) from starlette.testclient import TestClient -from fidesops.api.v1.scope_registry import ( +from fidesops.ops.api.v1.scope_registry import ( PRIVACY_REQUEST_READ, STORAGE_READ, USER_CREATE, @@ -36,14 +36,14 @@ USER_READ, USER_UPDATE, ) -from fidesops.api.v1.urn_registry import ( +from fidesops.ops.api.v1.urn_registry import ( LOGIN, LOGOUT, USER_DETAIL, USERS, V1_URL_PREFIX, ) -from fidesops.core.config import config +from fidesops.ops.core.config import config from tests.ops.conftest import generate_auth_header_for_user page_size = Params().size diff --git a/tests/ops/api/v1/endpoints/test_user_permission_endpoints.py b/tests/ops/api/v1/endpoints/test_user_permission_endpoints.py index fb5ee7e205..6956661509 100644 --- a/tests/ops/api/v1/endpoints/test_user_permission_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_user_permission_endpoints.py @@ -13,15 +13,15 @@ HTTP_422_UNPROCESSABLE_ENTITY, ) -from fidesops.api.v1.scope_registry import ( +from fidesops.ops.api.v1.scope_registry import ( PRIVACY_REQUEST_READ, SAAS_CONFIG_READ, USER_PERMISSION_CREATE, USER_PERMISSION_READ, USER_PERMISSION_UPDATE, ) -from fidesops.api.v1.urn_registry import USER_PERMISSIONS, V1_URL_PREFIX -from fidesops.core.config import config +from fidesops.ops.api.v1.urn_registry import USER_PERMISSIONS, V1_URL_PREFIX +from fidesops.ops.core.config import config class TestCreateUserPermissions: diff --git a/tests/ops/api/v1/test_exception_handlers.py b/tests/ops/api/v1/test_exception_handlers.py index d9e52a6e27..71e510ebbe 100644 --- a/tests/ops/api/v1/test_exception_handlers.py +++ b/tests/ops/api/v1/test_exception_handlers.py @@ -3,9 +3,14 @@ import pytest from starlette.testclient import TestClient -from fidesops.api.v1.scope_registry import CLIENT_CREATE -from fidesops.api.v1.urn_registry import CLIENT, HEALTH, PRIVACY_REQUESTS, V1_URL_PREFIX -from fidesops.core import config +from fidesops.ops.api.v1.scope_registry import CLIENT_CREATE +from fidesops.ops.api.v1.urn_registry import ( + CLIENT, + HEALTH, + PRIVACY_REQUESTS, + V1_URL_PREFIX, +) +from fidesops.ops.core import config @pytest.fixture diff --git a/tests/ops/api/v1/test_main.py b/tests/ops/api/v1/test_main.py index 9ff8fa45ce..fbf6d3e7e2 100644 --- a/tests/ops/api/v1/test_main.py +++ b/tests/ops/api/v1/test_main.py @@ -1,6 +1,6 @@ from starlette.testclient import TestClient -from fidesops.api.v1.urn_registry import V1_URL_PREFIX +from fidesops.ops.api.v1.urn_registry import V1_URL_PREFIX class CustomTestException(BaseException): diff --git a/tests/ops/conftest.py b/tests/ops/conftest.py index f6030672bd..2afec4cfe5 100644 --- a/tests/ops/conftest.py +++ b/tests/ops/conftest.py @@ -18,14 +18,14 @@ from sqlalchemy.exc import IntegrityError from sqlalchemy_utils.functions import create_database, database_exists, drop_database -from fidesops.api.v1.scope_registry import SCOPE_REGISTRY -from fidesops.core.config import config -from fidesops.db.base import Base -from fidesops.db.database import init_db from fidesops.main import app -from fidesops.models.privacy_request import generate_request_callback_jwe -from fidesops.tasks.scheduled.scheduler import scheduler -from fidesops.util.cache import get_cache +from fidesops.ops.api.v1.scope_registry import SCOPE_REGISTRY +from fidesops.ops.core.config import config +from fidesops.ops.db.base import Base +from fidesops.ops.db.database import init_db +from fidesops.ops.models.privacy_request import generate_request_callback_jwe +from fidesops.ops.tasks.scheduled.scheduler import scheduler +from fidesops.ops.util.cache import get_cache from .fixtures.application_fixtures import * from .fixtures.bigquery_fixtures import * @@ -218,7 +218,7 @@ def run_privacy_request_task(celery_session_app): registered to the `celery_app` fixture which uses the virtualised `celery_worker` """ yield celery_session_app.tasks[ - "fidesops.service.privacy_request.request_runner_service.run_privacy_request" + "fidesops.ops.service.privacy_request.request_runner_service.run_privacy_request" ] diff --git a/tests/ops/core/test_config.py b/tests/ops/core/test_config.py index 34308fae2d..c5bdfbe262 100644 --- a/tests/ops/core/test_config.py +++ b/tests/ops/core/test_config.py @@ -6,7 +6,7 @@ from fideslib.core.config import get_config from pydantic import ValidationError -from fidesops.core.config import FidesopsConfig +from fidesops.ops.core.config import FidesopsConfig def test_config_from_default() -> None: diff --git a/tests/ops/fixtures/application_fixtures.py b/tests/ops/fixtures/application_fixtures.py index 7bbdfdcd11..fe3fa9c2b4 100644 --- a/tests/ops/fixtures/application_fixtures.py +++ b/tests/ops/fixtures/application_fixtures.py @@ -15,14 +15,14 @@ from sqlalchemy.orm import Session from sqlalchemy.orm.exc import ObjectDeletedError -from fidesops.api.v1.scope_registry import PRIVACY_REQUEST_READ, SCOPE_REGISTRY -from fidesops.models.connectionconfig import ( +from fidesops.ops.api.v1.scope_registry import PRIVACY_REQUEST_READ, SCOPE_REGISTRY +from fidesops.ops.models.connectionconfig import ( AccessLevel, ConnectionConfig, ConnectionType, ) -from fidesops.models.datasetconfig import DatasetConfig -from fidesops.models.policy import ( +from fidesops.ops.models.datasetconfig import DatasetConfig +from fidesops.ops.models.policy import ( ActionType, Policy, PolicyPostWebhook, @@ -30,23 +30,25 @@ Rule, RuleTarget, ) -from fidesops.models.privacy_request import PrivacyRequest, PrivacyRequestStatus -from fidesops.models.storage import ResponseFormat, StorageConfig -from fidesops.schemas.redis_cache import PrivacyRequestIdentity -from fidesops.schemas.storage.storage import ( +from fidesops.ops.models.privacy_request import PrivacyRequest, PrivacyRequestStatus +from fidesops.ops.models.storage import ResponseFormat, StorageConfig +from fidesops.ops.schemas.redis_cache import PrivacyRequestIdentity +from fidesops.ops.schemas.storage.storage import ( FileNaming, StorageDetails, StorageSecrets, StorageType, ) -from fidesops.service.masking.strategy.masking_strategy_hmac import HMAC_STRATEGY_NAME -from fidesops.service.masking.strategy.masking_strategy_nullify import ( +from fidesops.ops.service.masking.strategy.masking_strategy_hmac import ( + HMAC_STRATEGY_NAME, +) +from fidesops.ops.service.masking.strategy.masking_strategy_nullify import ( NULL_REWRITE_STRATEGY_NAME, ) -from fidesops.service.masking.strategy.masking_strategy_string_rewrite import ( +from fidesops.ops.service.masking.strategy.masking_strategy_string_rewrite import ( STRING_REWRITE_STRATEGY_NAME, ) -from fidesops.util.data_category import DataCategory +from fidesops.ops.util.data_category import DataCategory logging.getLogger("faker").setLevel(logging.ERROR) # disable verbose faker logging @@ -99,7 +101,7 @@ @pytest.fixture(scope="session", autouse=True) def mock_upload_logic() -> Generator: with mock.patch( - "fidesops.service.storage.storage_uploader_service.upload_to_s3" + "fidesops.ops.service.storage.storage_uploader_service.upload_to_s3" ) as _fixture: yield _fixture diff --git a/tests/ops/fixtures/bigquery_fixtures.py b/tests/ops/fixtures/bigquery_fixtures.py index 35fae00072..8ab51d3273 100644 --- a/tests/ops/fixtures/bigquery_fixtures.py +++ b/tests/ops/fixtures/bigquery_fixtures.py @@ -7,14 +7,14 @@ import pytest from sqlalchemy.orm import Session -from fidesops.models.connectionconfig import ( +from fidesops.ops.models.connectionconfig import ( AccessLevel, ConnectionConfig, ConnectionType, ) -from fidesops.models.datasetconfig import DatasetConfig -from fidesops.schemas.connection_configuration import BigQuerySchema -from fidesops.service.connectors import BigQueryConnector, get_connector +from fidesops.ops.models.datasetconfig import DatasetConfig +from fidesops.ops.schemas.connection_configuration import BigQuerySchema +from fidesops.ops.service.connectors import BigQueryConnector, get_connector from .application_fixtures import integration_config diff --git a/tests/ops/fixtures/integration_fixtures.py b/tests/ops/fixtures/integration_fixtures.py index 44cf2e9884..ab9277d831 100644 --- a/tests/ops/fixtures/integration_fixtures.py +++ b/tests/ops/fixtures/integration_fixtures.py @@ -9,12 +9,12 @@ from sqlalchemy import text from sqlalchemy.engine import Engine -from fidesops.models.connectionconfig import ( +from fidesops.ops.models.connectionconfig import ( AccessLevel, ConnectionConfig, ConnectionType, ) -from fidesops.service.connectors import MongoDBConnector +from fidesops.ops.service.connectors import MongoDBConnector from .application_fixtures import faker, integration_secrets diff --git a/tests/ops/fixtures/manual_fixtures.py b/tests/ops/fixtures/manual_fixtures.py index 677251874e..32b893606b 100644 --- a/tests/ops/fixtures/manual_fixtures.py +++ b/tests/ops/fixtures/manual_fixtures.py @@ -4,12 +4,12 @@ import pytest from sqlalchemy.orm import Session -from fidesops.models.connectionconfig import ( +from fidesops.ops.models.connectionconfig import ( AccessLevel, ConnectionConfig, ConnectionType, ) -from fidesops.models.datasetconfig import DatasetConfig +from fidesops.ops.models.datasetconfig import DatasetConfig @pytest.fixture(scope="function") diff --git a/tests/ops/fixtures/mariadb_fixtures.py b/tests/ops/fixtures/mariadb_fixtures.py index 89485320f2..3c394a43e0 100644 --- a/tests/ops/fixtures/mariadb_fixtures.py +++ b/tests/ops/fixtures/mariadb_fixtures.py @@ -7,14 +7,14 @@ from fideslib.db.session import get_db_engine, get_db_session from sqlalchemy.orm import Session -from fidesops.core.config import config -from fidesops.models.connectionconfig import ( +from fidesops.ops.core.config import config +from fidesops.ops.models.connectionconfig import ( AccessLevel, ConnectionConfig, ConnectionType, ) -from fidesops.models.datasetconfig import DatasetConfig -from fidesops.service.connectors import MariaDBConnector +from fidesops.ops.models.datasetconfig import DatasetConfig +from fidesops.ops.service.connectors import MariaDBConnector from .application_fixtures import integration_secrets diff --git a/tests/ops/fixtures/mongodb_fixtures.py b/tests/ops/fixtures/mongodb_fixtures.py index 48167bdaf9..44f8470532 100644 --- a/tests/ops/fixtures/mongodb_fixtures.py +++ b/tests/ops/fixtures/mongodb_fixtures.py @@ -4,13 +4,13 @@ import pytest from sqlalchemy.orm import Session -from fidesops.models.connectionconfig import ( +from fidesops.ops.models.connectionconfig import ( AccessLevel, ConnectionConfig, ConnectionType, ) -from fidesops.models.policy import ActionType -from fidesops.models.privacy_request import ( +from fidesops.ops.models.policy import ActionType +from fidesops.ops.models.privacy_request import ( ExecutionLog, ExecutionLogStatus, PrivacyRequest, diff --git a/tests/ops/fixtures/mssql_fixtures.py b/tests/ops/fixtures/mssql_fixtures.py index 1d9888f505..b771da4dcc 100644 --- a/tests/ops/fixtures/mssql_fixtures.py +++ b/tests/ops/fixtures/mssql_fixtures.py @@ -6,14 +6,14 @@ from fideslib.db.session import get_db_engine, get_db_session from sqlalchemy.orm import Session -from fidesops.core.config import config -from fidesops.models.connectionconfig import ( +from fidesops.ops.core.config import config +from fidesops.ops.models.connectionconfig import ( AccessLevel, ConnectionConfig, ConnectionType, ) -from fidesops.models.datasetconfig import DatasetConfig -from fidesops.service.connectors import MicrosoftSQLServerConnector +from fidesops.ops.models.datasetconfig import DatasetConfig +from fidesops.ops.service.connectors import MicrosoftSQLServerConnector from .application_fixtures import integration_secrets diff --git a/tests/ops/fixtures/mysql_fixtures.py b/tests/ops/fixtures/mysql_fixtures.py index 31726235ff..248bcddd50 100644 --- a/tests/ops/fixtures/mysql_fixtures.py +++ b/tests/ops/fixtures/mysql_fixtures.py @@ -6,14 +6,14 @@ from fideslib.db.session import get_db_engine, get_db_session from sqlalchemy.orm import Session -from fidesops.core.config import config -from fidesops.models.connectionconfig import ( +from fidesops.ops.core.config import config +from fidesops.ops.models.connectionconfig import ( AccessLevel, ConnectionConfig, ConnectionType, ) -from fidesops.models.datasetconfig import DatasetConfig -from fidesops.service.connectors import MySQLConnector +from fidesops.ops.models.datasetconfig import DatasetConfig +from fidesops.ops.service.connectors import MySQLConnector from .application_fixtures import integration_secrets diff --git a/tests/ops/fixtures/postgres_fixtures.py b/tests/ops/fixtures/postgres_fixtures.py index d487be6e91..0c0a7642a4 100644 --- a/tests/ops/fixtures/postgres_fixtures.py +++ b/tests/ops/fixtures/postgres_fixtures.py @@ -8,20 +8,20 @@ from sqlalchemy.sql import text from sqlalchemy_utils.functions import create_database, database_exists, drop_database -from fidesops.core.config import config -from fidesops.models.connectionconfig import ( +from fidesops.ops.core.config import config +from fidesops.ops.models.connectionconfig import ( AccessLevel, ConnectionConfig, ConnectionType, ) -from fidesops.models.datasetconfig import DatasetConfig -from fidesops.models.policy import ActionType -from fidesops.models.privacy_request import ( +from fidesops.ops.models.datasetconfig import DatasetConfig +from fidesops.ops.models.policy import ActionType +from fidesops.ops.models.privacy_request import ( ExecutionLog, ExecutionLogStatus, PrivacyRequest, ) -from fidesops.service.connectors import PostgreSQLConnector +from fidesops.ops.service.connectors import PostgreSQLConnector from .application_fixtures import integration_secrets diff --git a/tests/ops/fixtures/redshift_fixtures.py b/tests/ops/fixtures/redshift_fixtures.py index 52a9f5b2ab..787b19249a 100644 --- a/tests/ops/fixtures/redshift_fixtures.py +++ b/tests/ops/fixtures/redshift_fixtures.py @@ -5,13 +5,13 @@ import pytest from sqlalchemy.orm import Session -from fidesops.models.connectionconfig import ( +from fidesops.ops.models.connectionconfig import ( AccessLevel, ConnectionConfig, ConnectionType, ) -from fidesops.models.datasetconfig import DatasetConfig -from fidesops.schemas.connection_configuration.connection_secrets_redshift import ( +from fidesops.ops.models.datasetconfig import DatasetConfig +from fidesops.ops.schemas.connection_configuration.connection_secrets_redshift import ( RedshiftSchema, ) diff --git a/tests/ops/fixtures/saas/adobe_campaign_fixtures.py b/tests/ops/fixtures/saas/adobe_campaign_fixtures.py index 74db619856..4b7c359d7f 100644 --- a/tests/ops/fixtures/saas/adobe_campaign_fixtures.py +++ b/tests/ops/fixtures/saas/adobe_campaign_fixtures.py @@ -6,13 +6,13 @@ from fideslib.cryptography import cryptographic_util from sqlalchemy.orm import Session -from fidesops.models.connectionconfig import ( +from fidesops.ops.models.connectionconfig import ( AccessLevel, ConnectionConfig, ConnectionType, ) -from fidesops.models.datasetconfig import DatasetConfig -from fidesops.util.saas_util import load_config +from fidesops.ops.models.datasetconfig import DatasetConfig +from fidesops.ops.util.saas_util import load_config from tests.ops.fixtures.application_fixtures import load_dataset from tests.ops.test_helpers.vault_client import get_secrets diff --git a/tests/ops/fixtures/saas/auth0_fixtures.py b/tests/ops/fixtures/saas/auth0_fixtures.py index 010f250ccd..f252323cd4 100644 --- a/tests/ops/fixtures/saas/auth0_fixtures.py +++ b/tests/ops/fixtures/saas/auth0_fixtures.py @@ -8,12 +8,12 @@ from sqlalchemy.orm import Session from starlette.status import HTTP_204_NO_CONTENT -from fidesops.models.connectionconfig import ( +from fidesops.ops.models.connectionconfig import ( AccessLevel, ConnectionConfig, ConnectionType, ) -from fidesops.models.datasetconfig import DatasetConfig +from fidesops.ops.models.datasetconfig import DatasetConfig from tests.ops.fixtures.application_fixtures import load_dataset from tests.ops.fixtures.saas_example_fixtures import load_config from tests.ops.test_helpers.saas_test_utils import poll_for_existence diff --git a/tests/ops/fixtures/saas/hubspot_fixtures.py b/tests/ops/fixtures/saas/hubspot_fixtures.py index 9510dffc53..6b46325946 100644 --- a/tests/ops/fixtures/saas/hubspot_fixtures.py +++ b/tests/ops/fixtures/saas/hubspot_fixtures.py @@ -6,15 +6,15 @@ from fideslib.cryptography import cryptographic_util from sqlalchemy.orm import Session -from fidesops.models.connectionconfig import ( +from fidesops.ops.models.connectionconfig import ( AccessLevel, ConnectionConfig, ConnectionType, ) -from fidesops.models.datasetconfig import DatasetConfig -from fidesops.schemas.saas.shared_schemas import HTTPMethod, SaaSRequestParams -from fidesops.service.connectors import SaaSConnector -from fidesops.util.saas_util import format_body, load_config +from fidesops.ops.models.datasetconfig import DatasetConfig +from fidesops.ops.schemas.saas.shared_schemas import HTTPMethod, SaaSRequestParams +from fidesops.ops.service.connectors import SaaSConnector +from fidesops.ops.util.saas_util import format_body, load_config from tests.ops.fixtures.application_fixtures import load_dataset from tests.ops.test_helpers.saas_test_utils import poll_for_existence from tests.ops.test_helpers.vault_client import get_secrets diff --git a/tests/ops/fixtures/saas/mailchimp_fixtures.py b/tests/ops/fixtures/saas/mailchimp_fixtures.py index d455feef1d..beaaf23b48 100644 --- a/tests/ops/fixtures/saas/mailchimp_fixtures.py +++ b/tests/ops/fixtures/saas/mailchimp_fixtures.py @@ -6,15 +6,15 @@ from fideslib.db import session from sqlalchemy.orm import Session -from fidesops.models.connectionconfig import ( +from fidesops.ops.models.connectionconfig import ( AccessLevel, ConnectionConfig, ConnectionType, ) -from fidesops.models.datasetconfig import DatasetConfig -from fidesops.schemas.saas.shared_schemas import HTTPMethod, SaaSRequestParams -from fidesops.service.connectors.saas_connector import SaaSConnector -from fidesops.util.saas_util import load_config +from fidesops.ops.models.datasetconfig import DatasetConfig +from fidesops.ops.schemas.saas.shared_schemas import HTTPMethod, SaaSRequestParams +from fidesops.ops.service.connectors.saas_connector import SaaSConnector +from fidesops.ops.util.saas_util import load_config from tests.ops.fixtures.application_fixtures import load_dataset from tests.ops.test_helpers.vault_client import get_secrets diff --git a/tests/ops/fixtures/saas/outreach_fixtures.py b/tests/ops/fixtures/saas/outreach_fixtures.py index 14a32bbd76..3815773958 100644 --- a/tests/ops/fixtures/saas/outreach_fixtures.py +++ b/tests/ops/fixtures/saas/outreach_fixtures.py @@ -7,13 +7,13 @@ from fideslib.db import session from sqlalchemy.orm import Session -from fidesops.models.connectionconfig import ( +from fidesops.ops.models.connectionconfig import ( AccessLevel, ConnectionConfig, ConnectionType, ) -from fidesops.models.datasetconfig import DatasetConfig -from fidesops.util.saas_util import load_config +from fidesops.ops.models.datasetconfig import DatasetConfig +from fidesops.ops.util.saas_util import load_config from tests.ops.fixtures.application_fixtures import load_dataset from tests.ops.test_helpers.vault_client import get_secrets diff --git a/tests/ops/fixtures/saas/request_override/mailchimp_override_fixtures.py b/tests/ops/fixtures/saas/request_override/mailchimp_override_fixtures.py index f02239ffde..b48acf5842 100644 --- a/tests/ops/fixtures/saas/request_override/mailchimp_override_fixtures.py +++ b/tests/ops/fixtures/saas/request_override/mailchimp_override_fixtures.py @@ -5,15 +5,15 @@ from fideslib.db import session from sqlalchemy.orm import Session -from fidesops.models.connectionconfig import ( +from fidesops.ops.models.connectionconfig import ( AccessLevel, ConnectionConfig, ConnectionType, ) -from fidesops.models.datasetconfig import DatasetConfig -from fidesops.schemas.saas.shared_schemas import HTTPMethod, SaaSRequestParams -from fidesops.service.connectors.saas_connector import SaaSConnector -from fidesops.util.saas_util import load_config +from fidesops.ops.models.datasetconfig import DatasetConfig +from fidesops.ops.schemas.saas.shared_schemas import HTTPMethod, SaaSRequestParams +from fidesops.ops.service.connectors.saas_connector import SaaSConnector +from fidesops.ops.util.saas_util import load_config from tests.ops.fixtures.application_fixtures import load_dataset diff --git a/tests/ops/fixtures/saas/salesforce_fixtures.py b/tests/ops/fixtures/saas/salesforce_fixtures.py index 6e6f7a6eca..fb3b4304a6 100644 --- a/tests/ops/fixtures/saas/salesforce_fixtures.py +++ b/tests/ops/fixtures/saas/salesforce_fixtures.py @@ -8,13 +8,13 @@ from sqlalchemy.orm import Session from starlette.status import HTTP_204_NO_CONTENT, HTTP_404_NOT_FOUND -from fidesops.models.connectionconfig import ( +from fidesops.ops.models.connectionconfig import ( AccessLevel, ConnectionConfig, ConnectionType, ) -from fidesops.models.datasetconfig import DatasetConfig -from fidesops.util.saas_util import load_config +from fidesops.ops.models.datasetconfig import DatasetConfig +from fidesops.ops.util.saas_util import load_config from tests.ops.fixtures.application_fixtures import load_dataset from tests.ops.test_helpers.vault_client import get_secrets diff --git a/tests/ops/fixtures/saas/segment_fixtures.py b/tests/ops/fixtures/saas/segment_fixtures.py index 898b687674..7fbb850abc 100644 --- a/tests/ops/fixtures/saas/segment_fixtures.py +++ b/tests/ops/fixtures/saas/segment_fixtures.py @@ -9,13 +9,13 @@ from fideslib.db import session from sqlalchemy.orm import Session -from fidesops.models.connectionconfig import ( +from fidesops.ops.models.connectionconfig import ( AccessLevel, ConnectionConfig, ConnectionType, ) -from fidesops.models.datasetconfig import DatasetConfig -from fidesops.util.saas_util import load_config +from fidesops.ops.models.datasetconfig import DatasetConfig +from fidesops.ops.util.saas_util import load_config from tests.ops.fixtures.application_fixtures import load_dataset from tests.ops.test_helpers.saas_test_utils import poll_for_existence from tests.ops.test_helpers.vault_client import get_secrets diff --git a/tests/ops/fixtures/saas/sendgrid_fixtures.py b/tests/ops/fixtures/saas/sendgrid_fixtures.py index 6d3407f08f..50a64f32de 100644 --- a/tests/ops/fixtures/saas/sendgrid_fixtures.py +++ b/tests/ops/fixtures/saas/sendgrid_fixtures.py @@ -8,12 +8,12 @@ from sqlalchemy.orm import Session from starlette.status import HTTP_202_ACCEPTED -from fidesops.models.connectionconfig import ( +from fidesops.ops.models.connectionconfig import ( AccessLevel, ConnectionConfig, ConnectionType, ) -from fidesops.models.datasetconfig import DatasetConfig +from fidesops.ops.models.datasetconfig import DatasetConfig from tests.ops.fixtures.application_fixtures import load_dataset from tests.ops.fixtures.saas_example_fixtures import load_config from tests.ops.test_helpers.saas_test_utils import poll_for_existence diff --git a/tests/ops/fixtures/saas/sentry_fixtures.py b/tests/ops/fixtures/saas/sentry_fixtures.py index f4f4365717..9f4eee0cdf 100644 --- a/tests/ops/fixtures/saas/sentry_fixtures.py +++ b/tests/ops/fixtures/saas/sentry_fixtures.py @@ -5,13 +5,13 @@ from fideslib.db import session from sqlalchemy.orm import Session -from fidesops.models.connectionconfig import ( +from fidesops.ops.models.connectionconfig import ( AccessLevel, ConnectionConfig, ConnectionType, ) -from fidesops.models.datasetconfig import DatasetConfig -from fidesops.util.saas_util import load_config +from fidesops.ops.models.datasetconfig import DatasetConfig +from fidesops.ops.util.saas_util import load_config from tests.ops.fixtures.application_fixtures import load_dataset from tests.ops.test_helpers.vault_client import get_secrets diff --git a/tests/ops/fixtures/saas/stripe_fixtures.py b/tests/ops/fixtures/saas/stripe_fixtures.py index ab4471ece7..4ea8e4888a 100644 --- a/tests/ops/fixtures/saas/stripe_fixtures.py +++ b/tests/ops/fixtures/saas/stripe_fixtures.py @@ -7,13 +7,13 @@ from multidimensional_urlencode import urlencode as multidimensional_urlencode from sqlalchemy.orm import Session -from fidesops.models.connectionconfig import ( +from fidesops.ops.models.connectionconfig import ( AccessLevel, ConnectionConfig, ConnectionType, ) -from fidesops.models.datasetconfig import DatasetConfig -from fidesops.util.saas_util import load_config +from fidesops.ops.models.datasetconfig import DatasetConfig +from fidesops.ops.util.saas_util import load_config from tests.ops.fixtures.application_fixtures import load_dataset from tests.ops.test_helpers.vault_client import get_secrets diff --git a/tests/ops/fixtures/saas/zendesk_fixtures.py b/tests/ops/fixtures/saas/zendesk_fixtures.py index d73afc6d4c..b67121efaa 100644 --- a/tests/ops/fixtures/saas/zendesk_fixtures.py +++ b/tests/ops/fixtures/saas/zendesk_fixtures.py @@ -6,13 +6,13 @@ from fideslib.cryptography import cryptographic_util from sqlalchemy.orm import Session -from fidesops.models.connectionconfig import ( +from fidesops.ops.models.connectionconfig import ( AccessLevel, ConnectionConfig, ConnectionType, ) -from fidesops.models.datasetconfig import DatasetConfig -from fidesops.util.saas_util import load_config +from fidesops.ops.models.datasetconfig import DatasetConfig +from fidesops.ops.util.saas_util import load_config from tests.ops.fixtures.application_fixtures import load_dataset from tests.ops.test_helpers.vault_client import get_secrets diff --git a/tests/ops/fixtures/saas_example_fixtures.py b/tests/ops/fixtures/saas_example_fixtures.py index e3c044e053..e6b69cf498 100644 --- a/tests/ops/fixtures/saas_example_fixtures.py +++ b/tests/ops/fixtures/saas_example_fixtures.py @@ -5,17 +5,17 @@ from fideslib.core.config import load_toml from sqlalchemy.orm import Session -from fidesops.models.connectionconfig import ( +from fidesops.ops.models.connectionconfig import ( AccessLevel, ConnectionConfig, ConnectionType, ) -from fidesops.models.datasetconfig import DatasetConfig -from fidesops.schemas.saas.strategy_configuration import ( +from fidesops.ops.models.datasetconfig import DatasetConfig +from fidesops.ops.schemas.saas.strategy_configuration import ( OAuth2AuthenticationConfiguration, ) -from fidesops.util.logger import NotPii -from fidesops.util.saas_util import load_config +from fidesops.ops.util.logger import NotPii +from fidesops.ops.util.saas_util import load_config from tests.ops.fixtures.application_fixtures import load_dataset logger = logging.getLogger(__name__) diff --git a/tests/ops/fixtures/snowflake_fixtures.py b/tests/ops/fixtures/snowflake_fixtures.py index c4517a24a6..e7efbd9fce 100644 --- a/tests/ops/fixtures/snowflake_fixtures.py +++ b/tests/ops/fixtures/snowflake_fixtures.py @@ -5,13 +5,13 @@ import pytest from sqlalchemy.orm import Session -from fidesops.models.connectionconfig import ( +from fidesops.ops.models.connectionconfig import ( AccessLevel, ConnectionConfig, ConnectionType, ) -from fidesops.models.datasetconfig import DatasetConfig -from fidesops.schemas.connection_configuration.connection_secrets_snowflake import ( +from fidesops.ops.models.datasetconfig import DatasetConfig +from fidesops.ops.schemas.connection_configuration.connection_secrets_snowflake import ( SnowflakeSchema, ) diff --git a/tests/ops/generator/sql_data_generator.py b/tests/ops/generator/sql_data_generator.py index eeb19d9e7d..b1d971ba4e 100644 --- a/tests/ops/generator/sql_data_generator.py +++ b/tests/ops/generator/sql_data_generator.py @@ -6,9 +6,9 @@ from sqlalchemy import Boolean, Column, Float, Integer, String from sqlalchemy.ext.declarative import declarative_base -from fidesops.graph.config import CollectionAddress, Field, ScalarField -from fidesops.graph.data_type import DataType -from fidesops.graph.traversal import Row, Traversal, TraversalNode +from fidesops.ops.graph.config import CollectionAddress, Field, ScalarField +from fidesops.ops.graph.data_type import DataType +from fidesops.ops.graph.traversal import Row, Traversal, TraversalNode Base = declarative_base() diff --git a/tests/ops/generator/test_data_generator.py b/tests/ops/generator/test_data_generator.py index 8feebffed1..422a755a18 100644 --- a/tests/ops/generator/test_data_generator.py +++ b/tests/ops/generator/test_data_generator.py @@ -1,13 +1,13 @@ import yaml -from fidesops.graph.graph import * +from fidesops.ops.graph.graph import * # ------------------------------------------- # graph object tests # ------------------------------------------- -from fidesops.graph.traversal import Traversal -from fidesops.models.datasetconfig import convert_dataset_to_graph -from fidesops.schemas.dataset import FidesopsDataset +from fidesops.ops.graph.traversal import Traversal +from fidesops.ops.models.datasetconfig import convert_dataset_to_graph +from fidesops.ops.schemas.dataset import FidesopsDataset from . import sql_data_generator diff --git a/tests/ops/graph/graph_test_util.py b/tests/ops/graph/graph_test_util.py index 16f590df50..8b56302702 100644 --- a/tests/ops/graph/graph_test_util.py +++ b/tests/ops/graph/graph_test_util.py @@ -4,19 +4,19 @@ from fideslib.db.base_class import FidesBase from sqlalchemy.engine import Engine -from fidesops.graph.config import * -from fidesops.graph.traversal import * -from fidesops.graph.traversal import Traversal, TraversalNode +from fidesops.ops.graph.config import * +from fidesops.ops.graph.traversal import * +from fidesops.ops.graph.traversal import Traversal, TraversalNode # to avoid having faker spam the logs -from fidesops.models.connectionconfig import ConnectionConfig -from fidesops.models.policy import ActionType, Policy, Rule, RuleTarget -from fidesops.models.privacy_request import PrivacyRequest -from fidesops.service.connectors import BaseConnector, MongoDBConnector -from fidesops.service.connectors.sql_connector import SQLConnector -from fidesops.task.graph_task import GraphTask -from fidesops.task.task_resources import TaskResources -from fidesops.util.collection_util import Row +from fidesops.ops.models.connectionconfig import ConnectionConfig +from fidesops.ops.models.policy import ActionType, Policy, Rule, RuleTarget +from fidesops.ops.models.privacy_request import PrivacyRequest +from fidesops.ops.service.connectors import BaseConnector, MongoDBConnector +from fidesops.ops.service.connectors.sql_connector import SQLConnector +from fidesops.ops.task.graph_task import GraphTask +from fidesops.ops.task.task_resources import TaskResources +from fidesops.ops.util.collection_util import Row from ..fixtures.application_fixtures import faker diff --git a/tests/ops/graph/test_config.py b/tests/ops/graph/test_config.py index 9cb17c59c2..45a2a87912 100644 --- a/tests/ops/graph/test_config.py +++ b/tests/ops/graph/test_config.py @@ -1,9 +1,9 @@ import pydantic import pytest -from fidesops.common_exceptions import ValidationError -from fidesops.graph.config import * -from fidesops.graph.data_type import ( +from fidesops.ops.common_exceptions import ValidationError +from fidesops.ops.graph.config import * +from fidesops.ops.graph.data_type import ( BooleanTypeConverter, IntTypeConverter, NoOpTypeConverter, diff --git a/tests/ops/graph/test_data_types.py b/tests/ops/graph/test_data_types.py index 32b7cb2ed7..2591cc1a43 100644 --- a/tests/ops/graph/test_data_types.py +++ b/tests/ops/graph/test_data_types.py @@ -1,6 +1,6 @@ from bson import ObjectId -from fidesops.graph.data_type import ( +from fidesops.ops.graph.data_type import ( DataType, NoOpTypeConverter, StringTypeConverter, diff --git a/tests/ops/graph/test_edge.py b/tests/ops/graph/test_edge.py index 1651aa673d..f8c5406ee8 100644 --- a/tests/ops/graph/test_edge.py +++ b/tests/ops/graph/test_edge.py @@ -1,7 +1,7 @@ import pytest -from fidesops.common_exceptions import ValidationError -from fidesops.graph.graph import * +from fidesops.ops.common_exceptions import ValidationError +from fidesops.ops.graph.graph import * class TestEdge: diff --git a/tests/ops/graph/test_graph.py b/tests/ops/graph/test_graph.py index f3af6ee0de..a13ea7700d 100644 --- a/tests/ops/graph/test_graph.py +++ b/tests/ops/graph/test_graph.py @@ -1,11 +1,11 @@ import pytest -from fidesops.core.config import config -from fidesops.graph.config import * -from fidesops.graph.traversal import * -from fidesops.models.policy import ActionType -from fidesops.task.graph_task import retry -from fidesops.task.task_resources import TaskResources +from fidesops.ops.core.config import config +from fidesops.ops.graph.config import * +from fidesops.ops.graph.traversal import * +from fidesops.ops.models.policy import ActionType +from fidesops.ops.task.graph_task import retry +from fidesops.ops.task.task_resources import TaskResources from tests.ops.task.traversal_data import integration_db_graph t1 = Collection( diff --git a/tests/ops/graph/test_graph_analytics_events.py b/tests/ops/graph/test_graph_analytics_events.py index 8854c045dc..d5451cc2bf 100644 --- a/tests/ops/graph/test_graph_analytics_events.py +++ b/tests/ops/graph/test_graph_analytics_events.py @@ -1,5 +1,5 @@ -from fidesops.common_exceptions import FidesopsException -from fidesops.graph.analytics_events import failed_graph_analytics_event +from fidesops.ops.common_exceptions import FidesopsException +from fidesops.ops.graph.analytics_events import failed_graph_analytics_event class TestFailedGraphAnalyticsEvent: diff --git a/tests/ops/graph/test_graph_differences.py b/tests/ops/graph/test_graph_differences.py index 79ba43432e..78ef722c15 100644 --- a/tests/ops/graph/test_graph_differences.py +++ b/tests/ops/graph/test_graph_differences.py @@ -2,25 +2,25 @@ import pytest -from fidesops.graph.analytics_events import prepare_rerun_graph_analytics_event -from fidesops.graph.config import ( +from fidesops.ops.graph.analytics_events import prepare_rerun_graph_analytics_event +from fidesops.ops.graph.config import ( ROOT_COLLECTION_ADDRESS, CollectionAddress, FieldAddress, ) -from fidesops.graph.graph import Edge -from fidesops.graph.graph_differences import ( +from fidesops.ops.graph.graph import Edge +from fidesops.ops.graph.graph_differences import ( GraphDiff, GraphDiffSummary, _find_graph_differences, find_graph_differences_summary, format_graph_for_caching, ) -from fidesops.graph.traversal import TraversalNode, artificial_traversal_node -from fidesops.models.connectionconfig import ConnectionConfig, ConnectionType -from fidesops.models.policy import ActionType, Policy -from fidesops.task.graph_task import EMPTY_REQUEST, GraphTask -from fidesops.task.task_resources import TaskResources +from fidesops.ops.graph.traversal import TraversalNode, artificial_traversal_node +from fidesops.ops.models.connectionconfig import ConnectionConfig, ConnectionType +from fidesops.ops.models.policy import ActionType, Policy +from fidesops.ops.task.graph_task import EMPTY_REQUEST, GraphTask +from fidesops.ops.task.task_resources import TaskResources from ..graph.graph_test_util import generate_node diff --git a/tests/ops/graph/test_graph_traversal.py b/tests/ops/graph/test_graph_traversal.py index 5fe6e3582e..7d94afec8e 100644 --- a/tests/ops/graph/test_graph_traversal.py +++ b/tests/ops/graph/test_graph_traversal.py @@ -1,6 +1,6 @@ import pytest -from fidesops.graph.graph import * +from fidesops.ops.graph.graph import * from .graph_test_util import * diff --git a/tests/ops/graph/test_traversal_node.py b/tests/ops/graph/test_traversal_node.py index 9d42c0773e..2299807c64 100644 --- a/tests/ops/graph/test_traversal_node.py +++ b/tests/ops/graph/test_traversal_node.py @@ -1,4 +1,4 @@ -from fidesops.graph.traversal import * +from fidesops.ops.graph.traversal import * from .test_graph_traversal import generate_node diff --git a/tests/ops/integration_tests/saas/request_override/test_mailchimp_override_task.py b/tests/ops/integration_tests/saas/request_override/test_mailchimp_override_task.py index 3b292a2c26..8ae1ef1fd8 100644 --- a/tests/ops/integration_tests/saas/request_override/test_mailchimp_override_task.py +++ b/tests/ops/integration_tests/saas/request_override/test_mailchimp_override_task.py @@ -2,11 +2,11 @@ import pytest -from fidesops.graph.graph import DatasetGraph -from fidesops.models.privacy_request import ExecutionLog, PrivacyRequest -from fidesops.schemas.redis_cache import PrivacyRequestIdentity -from fidesops.task import graph_task -from fidesops.task.graph_task import get_cached_data_for_erasures +from fidesops.ops.graph.graph import DatasetGraph +from fidesops.ops.models.privacy_request import ExecutionLog, PrivacyRequest +from fidesops.ops.schemas.redis_cache import PrivacyRequestIdentity +from fidesops.ops.task import graph_task +from fidesops.ops.task.graph_task import get_cached_data_for_erasures from tests.ops.graph.graph_test_util import assert_rows_match, records_matching_fields """ diff --git a/tests/ops/integration_tests/saas/test_adobe_campaign_task.py b/tests/ops/integration_tests/saas/test_adobe_campaign_task.py index c550dd0b5d..bc79fe659a 100644 --- a/tests/ops/integration_tests/saas/test_adobe_campaign_task.py +++ b/tests/ops/integration_tests/saas/test_adobe_campaign_task.py @@ -2,13 +2,13 @@ import pytest -from fidesops.core.config import config -from fidesops.graph.graph import DatasetGraph -from fidesops.models.privacy_request import PrivacyRequest -from fidesops.schemas.redis_cache import PrivacyRequestIdentity -from fidesops.service.connectors import get_connector -from fidesops.task import graph_task -from fidesops.task.graph_task import get_cached_data_for_erasures +from fidesops.ops.core.config import config +from fidesops.ops.graph.graph import DatasetGraph +from fidesops.ops.models.privacy_request import PrivacyRequest +from fidesops.ops.schemas.redis_cache import PrivacyRequestIdentity +from fidesops.ops.service.connectors import get_connector +from fidesops.ops.task import graph_task +from fidesops.ops.task.graph_task import get_cached_data_for_erasures from tests.ops.graph.graph_test_util import assert_rows_match diff --git a/tests/ops/integration_tests/saas/test_auth0_task.py b/tests/ops/integration_tests/saas/test_auth0_task.py index 6e6c824d6f..756c803a80 100644 --- a/tests/ops/integration_tests/saas/test_auth0_task.py +++ b/tests/ops/integration_tests/saas/test_auth0_task.py @@ -3,12 +3,12 @@ import pytest import requests -from fidesops.core.config import config -from fidesops.graph.graph import DatasetGraph -from fidesops.models.privacy_request import PrivacyRequest -from fidesops.schemas.redis_cache import PrivacyRequestIdentity -from fidesops.task import graph_task -from fidesops.task.graph_task import get_cached_data_for_erasures +from fidesops.ops.core.config import config +from fidesops.ops.graph.graph import DatasetGraph +from fidesops.ops.models.privacy_request import PrivacyRequest +from fidesops.ops.schemas.redis_cache import PrivacyRequestIdentity +from fidesops.ops.task import graph_task +from fidesops.ops.task.graph_task import get_cached_data_for_erasures from tests.ops.graph.graph_test_util import assert_rows_match diff --git a/tests/ops/integration_tests/saas/test_hubspot_task.py b/tests/ops/integration_tests/saas/test_hubspot_task.py index 97346f39d0..f8e806c5c8 100644 --- a/tests/ops/integration_tests/saas/test_hubspot_task.py +++ b/tests/ops/integration_tests/saas/test_hubspot_task.py @@ -3,15 +3,15 @@ import pytest -from fidesops.graph.graph import DatasetGraph -from fidesops.models.privacy_request import ExecutionLog, PrivacyRequest -from fidesops.schemas.redis_cache import PrivacyRequestIdentity -from fidesops.schemas.saas.shared_schemas import HTTPMethod, SaaSRequestParams -from fidesops.service.connectors import SaaSConnector -from fidesops.task import graph_task -from fidesops.task.filter_results import filter_data_categories -from fidesops.task.graph_task import get_cached_data_for_erasures -from fidesops.util.saas_util import format_body +from fidesops.ops.graph.graph import DatasetGraph +from fidesops.ops.models.privacy_request import ExecutionLog, PrivacyRequest +from fidesops.ops.schemas.redis_cache import PrivacyRequestIdentity +from fidesops.ops.schemas.saas.shared_schemas import HTTPMethod, SaaSRequestParams +from fidesops.ops.service.connectors import SaaSConnector +from fidesops.ops.task import graph_task +from fidesops.ops.task.filter_results import filter_data_categories +from fidesops.ops.task.graph_task import get_cached_data_for_erasures +from fidesops.ops.util.saas_util import format_body from tests.ops.graph.graph_test_util import assert_rows_match, records_matching_fields diff --git a/tests/ops/integration_tests/saas/test_mailchimp_task.py b/tests/ops/integration_tests/saas/test_mailchimp_task.py index 45124df9b2..2e188a4292 100644 --- a/tests/ops/integration_tests/saas/test_mailchimp_task.py +++ b/tests/ops/integration_tests/saas/test_mailchimp_task.py @@ -2,11 +2,11 @@ import pytest -from fidesops.graph.graph import DatasetGraph -from fidesops.models.privacy_request import ExecutionLog, PrivacyRequest -from fidesops.schemas.redis_cache import PrivacyRequestIdentity -from fidesops.task import graph_task -from fidesops.task.graph_task import get_cached_data_for_erasures +from fidesops.ops.graph.graph import DatasetGraph +from fidesops.ops.models.privacy_request import ExecutionLog, PrivacyRequest +from fidesops.ops.schemas.redis_cache import PrivacyRequestIdentity +from fidesops.ops.task import graph_task +from fidesops.ops.task.graph_task import get_cached_data_for_erasures from tests.ops.graph.graph_test_util import assert_rows_match, records_matching_fields diff --git a/tests/ops/integration_tests/saas/test_outreach_task.py b/tests/ops/integration_tests/saas/test_outreach_task.py index 178f955548..72da773016 100644 --- a/tests/ops/integration_tests/saas/test_outreach_task.py +++ b/tests/ops/integration_tests/saas/test_outreach_task.py @@ -2,13 +2,13 @@ import pytest -from fidesops.core.config import config -from fidesops.graph.graph import DatasetGraph -from fidesops.models.privacy_request import PrivacyRequest -from fidesops.schemas.redis_cache import PrivacyRequestIdentity -from fidesops.task import graph_task -from fidesops.task.filter_results import filter_data_categories -from fidesops.task.graph_task import get_cached_data_for_erasures +from fidesops.ops.core.config import config +from fidesops.ops.graph.graph import DatasetGraph +from fidesops.ops.models.privacy_request import PrivacyRequest +from fidesops.ops.schemas.redis_cache import PrivacyRequestIdentity +from fidesops.ops.task import graph_task +from fidesops.ops.task.filter_results import filter_data_categories +from fidesops.ops.task.graph_task import get_cached_data_for_erasures from tests.ops.graph.graph_test_util import assert_rows_match diff --git a/tests/ops/integration_tests/saas/test_salesforce_task.py b/tests/ops/integration_tests/saas/test_salesforce_task.py index 3bf37dba80..fdc3fdccf8 100644 --- a/tests/ops/integration_tests/saas/test_salesforce_task.py +++ b/tests/ops/integration_tests/saas/test_salesforce_task.py @@ -3,13 +3,13 @@ import pytest import requests -from fidesops.core.config import config -from fidesops.graph.graph import DatasetGraph -from fidesops.models.privacy_request import PrivacyRequest -from fidesops.schemas.redis_cache import PrivacyRequestIdentity -from fidesops.service.connectors import get_connector -from fidesops.task import graph_task -from fidesops.task.graph_task import get_cached_data_for_erasures +from fidesops.ops.core.config import config +from fidesops.ops.graph.graph import DatasetGraph +from fidesops.ops.models.privacy_request import PrivacyRequest +from fidesops.ops.schemas.redis_cache import PrivacyRequestIdentity +from fidesops.ops.service.connectors import get_connector +from fidesops.ops.task import graph_task +from fidesops.ops.task.graph_task import get_cached_data_for_erasures from tests.ops.graph.graph_test_util import assert_rows_match diff --git a/tests/ops/integration_tests/saas/test_segment_task.py b/tests/ops/integration_tests/saas/test_segment_task.py index 755efeab73..0673aa35b9 100644 --- a/tests/ops/integration_tests/saas/test_segment_task.py +++ b/tests/ops/integration_tests/saas/test_segment_task.py @@ -2,13 +2,13 @@ import pytest -from fidesops.core.config import config -from fidesops.graph.graph import DatasetGraph -from fidesops.models.privacy_request import PrivacyRequest -from fidesops.schemas.redis_cache import PrivacyRequestIdentity -from fidesops.task import graph_task -from fidesops.task.filter_results import filter_data_categories -from fidesops.task.graph_task import get_cached_data_for_erasures +from fidesops.ops.core.config import config +from fidesops.ops.graph.graph import DatasetGraph +from fidesops.ops.models.privacy_request import PrivacyRequest +from fidesops.ops.schemas.redis_cache import PrivacyRequestIdentity +from fidesops.ops.task import graph_task +from fidesops.ops.task.filter_results import filter_data_categories +from fidesops.ops.task.graph_task import get_cached_data_for_erasures from tests.ops.graph.graph_test_util import assert_rows_match diff --git a/tests/ops/integration_tests/saas/test_sendgrid_task.py b/tests/ops/integration_tests/saas/test_sendgrid_task.py index 7891437e0d..bd098ba74f 100644 --- a/tests/ops/integration_tests/saas/test_sendgrid_task.py +++ b/tests/ops/integration_tests/saas/test_sendgrid_task.py @@ -2,12 +2,12 @@ import pytest -from fidesops.core.config import config -from fidesops.graph.graph import DatasetGraph -from fidesops.models.privacy_request import PrivacyRequest -from fidesops.schemas.redis_cache import PrivacyRequestIdentity -from fidesops.task import graph_task -from fidesops.task.graph_task import get_cached_data_for_erasures +from fidesops.ops.core.config import config +from fidesops.ops.graph.graph import DatasetGraph +from fidesops.ops.models.privacy_request import PrivacyRequest +from fidesops.ops.schemas.redis_cache import PrivacyRequestIdentity +from fidesops.ops.task import graph_task +from fidesops.ops.task.graph_task import get_cached_data_for_erasures from tests.ops.fixtures.saas.sendgrid_fixtures import contact_exists from tests.ops.graph.graph_test_util import assert_rows_match from tests.ops.test_helpers.saas_test_utils import poll_for_existence diff --git a/tests/ops/integration_tests/saas/test_sentry_task.py b/tests/ops/integration_tests/saas/test_sentry_task.py index aae7b38042..33c384b337 100644 --- a/tests/ops/integration_tests/saas/test_sentry_task.py +++ b/tests/ops/integration_tests/saas/test_sentry_task.py @@ -5,12 +5,12 @@ import pytest import requests -from fidesops.graph.graph import DatasetGraph -from fidesops.models.privacy_request import PrivacyRequest -from fidesops.schemas.redis_cache import PrivacyRequestIdentity -from fidesops.task import graph_task -from fidesops.task.filter_results import filter_data_categories -from fidesops.task.graph_task import get_cached_data_for_erasures +from fidesops.ops.graph.graph import DatasetGraph +from fidesops.ops.models.privacy_request import PrivacyRequest +from fidesops.ops.schemas.redis_cache import PrivacyRequestIdentity +from fidesops.ops.task import graph_task +from fidesops.ops.task.filter_results import filter_data_categories +from fidesops.ops.task.graph_task import get_cached_data_for_erasures from tests.ops.graph.graph_test_util import assert_rows_match from tests.ops.test_helpers.saas_test_utils import poll_for_existence diff --git a/tests/ops/integration_tests/saas/test_stripe_task.py b/tests/ops/integration_tests/saas/test_stripe_task.py index cb7c9a8bd1..a3399c2971 100644 --- a/tests/ops/integration_tests/saas/test_stripe_task.py +++ b/tests/ops/integration_tests/saas/test_stripe_task.py @@ -4,13 +4,13 @@ import pytest import requests -from fidesops.core.config import config -from fidesops.graph.graph import DatasetGraph -from fidesops.models.privacy_request import PrivacyRequest -from fidesops.schemas.redis_cache import PrivacyRequestIdentity -from fidesops.task import graph_task -from fidesops.task.filter_results import filter_data_categories -from fidesops.task.graph_task import get_cached_data_for_erasures +from fidesops.ops.core.config import config +from fidesops.ops.graph.graph import DatasetGraph +from fidesops.ops.models.privacy_request import PrivacyRequest +from fidesops.ops.schemas.redis_cache import PrivacyRequestIdentity +from fidesops.ops.task import graph_task +from fidesops.ops.task.filter_results import filter_data_categories +from fidesops.ops.task.graph_task import get_cached_data_for_erasures from tests.ops.graph.graph_test_util import assert_rows_match diff --git a/tests/ops/integration_tests/saas/test_zendesk_task.py b/tests/ops/integration_tests/saas/test_zendesk_task.py index d2f5c9d5c4..bb25b8b58b 100644 --- a/tests/ops/integration_tests/saas/test_zendesk_task.py +++ b/tests/ops/integration_tests/saas/test_zendesk_task.py @@ -4,12 +4,12 @@ import pytest import requests -from fidesops.core.config import config -from fidesops.graph.graph import DatasetGraph -from fidesops.models.privacy_request import PrivacyRequest -from fidesops.schemas.redis_cache import PrivacyRequestIdentity -from fidesops.task import graph_task -from fidesops.task.graph_task import get_cached_data_for_erasures +from fidesops.ops.core.config import config +from fidesops.ops.graph.graph import DatasetGraph +from fidesops.ops.models.privacy_request import PrivacyRequest +from fidesops.ops.schemas.redis_cache import PrivacyRequestIdentity +from fidesops.ops.task import graph_task +from fidesops.ops.task.graph_task import get_cached_data_for_erasures from tests.ops.graph.graph_test_util import assert_rows_match diff --git a/tests/ops/integration_tests/setup_scripts/mariadb_setup.py b/tests/ops/integration_tests/setup_scripts/mariadb_setup.py index 237ad6a584..2c1bd3a710 100644 --- a/tests/ops/integration_tests/setup_scripts/mariadb_setup.py +++ b/tests/ops/integration_tests/setup_scripts/mariadb_setup.py @@ -5,13 +5,13 @@ from fideslib.core.config import load_toml from fideslib.db.session import get_db_engine, get_db_session -from fidesops.core.config import config -from fidesops.models.connectionconfig import ( +from fidesops.ops.core.config import config +from fidesops.ops.models.connectionconfig import ( AccessLevel, ConnectionConfig, ConnectionType, ) -from fidesops.service.connectors.sql_connector import MariaDBConnector +from fidesops.ops.service.connectors.sql_connector import MariaDBConnector integration_config = load_toml(["fidesops-integration.toml"]) diff --git a/tests/ops/integration_tests/setup_scripts/postgres_setup.py b/tests/ops/integration_tests/setup_scripts/postgres_setup.py index 682b04cea3..43e5cab7c6 100644 --- a/tests/ops/integration_tests/setup_scripts/postgres_setup.py +++ b/tests/ops/integration_tests/setup_scripts/postgres_setup.py @@ -6,13 +6,13 @@ from fideslib.db.session import get_db_engine, get_db_session from sqlalchemy_utils.functions import create_database, database_exists, drop_database -from fidesops.core.config import config -from fidesops.models.connectionconfig import ( +from fidesops.ops.core.config import config +from fidesops.ops.models.connectionconfig import ( AccessLevel, ConnectionConfig, ConnectionType, ) -from fidesops.service.connectors.sql_connector import PostgreSQLConnector +from fidesops.ops.service.connectors.sql_connector import PostgreSQLConnector integration_config = load_toml(["fidesops-integration.toml"]) diff --git a/tests/ops/integration_tests/test_connection_configuration_integration.py b/tests/ops/integration_tests/test_connection_configuration_integration.py index 532d1ccacb..2b1977a9ad 100644 --- a/tests/ops/integration_tests/test_connection_configuration_integration.py +++ b/tests/ops/integration_tests/test_connection_configuration_integration.py @@ -7,22 +7,24 @@ from sqlalchemy.orm import Session from starlette.testclient import TestClient -from fidesops.api.v1.scope_registry import ( +from fidesops.ops.api.v1.scope_registry import ( CONNECTION_CREATE_OR_UPDATE, CONNECTION_READ, STORAGE_READ, ) -from fidesops.api.v1.urn_registry import CONNECTIONS, V1_URL_PREFIX -from fidesops.common_exceptions import ConnectionException -from fidesops.models.connectionconfig import ConnectionTestStatus -from fidesops.service.connectors import ( +from fidesops.ops.api.v1.urn_registry import CONNECTIONS, V1_URL_PREFIX +from fidesops.ops.common_exceptions import ConnectionException +from fidesops.ops.models.connectionconfig import ConnectionTestStatus +from fidesops.ops.service.connectors import ( MongoDBConnector, PostgreSQLConnector, SaaSConnector, get_connector, ) -from fidesops.service.connectors.saas.authenticated_client import AuthenticatedClient -from fidesops.service.connectors.sql_connector import ( +from fidesops.ops.service.connectors.saas.authenticated_client import ( + AuthenticatedClient, +) +from fidesops.ops.service.connectors.sql_connector import ( MariaDBConnector, MicrosoftSQLServerConnector, MySQLConnector, diff --git a/tests/ops/integration_tests/test_execution.py b/tests/ops/integration_tests/test_execution.py index e238bebd68..d2c723de18 100644 --- a/tests/ops/integration_tests/test_execution.py +++ b/tests/ops/integration_tests/test_execution.py @@ -6,24 +6,24 @@ from pydantic import ValidationError from sqlalchemy.exc import InvalidRequestError -from fidesops.core.config import config -from fidesops.graph.config import CollectionAddress -from fidesops.graph.graph import DatasetGraph -from fidesops.models.connectionconfig import ( +from fidesops.ops.core.config import config +from fidesops.ops.graph.config import CollectionAddress +from fidesops.ops.graph.graph import DatasetGraph +from fidesops.ops.models.connectionconfig import ( AccessLevel, ConnectionConfig, ConnectionType, ) -from fidesops.models.datasetconfig import convert_dataset_to_graph -from fidesops.models.policy import PausedStep -from fidesops.models.privacy_request import ( +from fidesops.ops.models.datasetconfig import convert_dataset_to_graph +from fidesops.ops.models.policy import PausedStep +from fidesops.ops.models.privacy_request import ( ExecutionLog, PrivacyRequest, StoppedCollection, ) -from fidesops.schemas.dataset import FidesopsDataset -from fidesops.task import graph_task -from fidesops.task.graph_task import get_cached_data_for_erasures +from fidesops.ops.schemas.dataset import FidesopsDataset +from fidesops.ops.task import graph_task +from fidesops.ops.task.graph_task import get_cached_data_for_erasures from ..fixtures.application_fixtures import integration_secrets from ..service.privacy_request.request_runner_service_test import ( @@ -96,7 +96,7 @@ def test_delete_collection_before_new_request( ) assert pr.get_results() == {} - @mock.patch("fidesops.task.graph_task.GraphTask.log_start") + @mock.patch("fidesops.ops.task.graph_task.GraphTask.log_start") def test_delete_collection_while_in_progress( self, mocked_log_start, @@ -358,7 +358,7 @@ def test_skip_collection_new_request( assert mongo_logs.count() == 9 assert mongo_logs.filter_by(status="skipped").count() == 9 - @mock.patch("fidesops.task.graph_task.GraphTask.log_start") + @mock.patch("fidesops.ops.task.graph_task.GraphTask.log_start") def test_run_disabled_collections_in_progress( self, mocked_log_start, @@ -642,7 +642,9 @@ def test_restart_graph_from_failure( integration_mongodb_config.save(db) # Rerun access request using cached results - with mock.patch("fidesops.task.graph_task.fideslog_graph_rerun") as mock_log_event: + with mock.patch( + "fidesops.ops.task.graph_task.fideslog_graph_rerun" + ) as mock_log_event: graph_task.run_access_request( privacy_request, policy, @@ -770,7 +772,9 @@ def test_restart_graph_from_failure_during_erasure( integration_postgres_config.save(db) # Rerun erasure portion of request using cached results - with mock.patch("fidesops.task.graph_task.fideslog_graph_rerun") as mock_log_event: + with mock.patch( + "fidesops.ops.task.graph_task.fideslog_graph_rerun" + ) as mock_log_event: graph_task.run_erasure( privacy_request, policy, diff --git a/tests/ops/integration_tests/test_external_database_connections.py b/tests/ops/integration_tests/test_external_database_connections.py index 9299d882b0..b58c2fa821 100644 --- a/tests/ops/integration_tests/test_external_database_connections.py +++ b/tests/ops/integration_tests/test_external_database_connections.py @@ -6,9 +6,12 @@ from fideslib.core.config import load_toml from sqlalchemy import inspect -from fidesops.models.connectionconfig import ConnectionConfig, ConnectionType -from fidesops.schemas.connection_configuration import RedshiftSchema, SnowflakeSchema -from fidesops.service.connectors import ( +from fidesops.ops.models.connectionconfig import ConnectionConfig, ConnectionType +from fidesops.ops.schemas.connection_configuration import ( + RedshiftSchema, + SnowflakeSchema, +) +from fidesops.ops.service.connectors import ( RedshiftConnector, SnowflakeConnector, get_connector, diff --git a/tests/ops/integration_tests/test_manual_task.py b/tests/ops/integration_tests/test_manual_task.py index c3343474a8..1e11994102 100644 --- a/tests/ops/integration_tests/test_manual_task.py +++ b/tests/ops/integration_tests/test_manual_task.py @@ -2,16 +2,16 @@ import pytest -from fidesops.common_exceptions import PrivacyRequestPaused -from fidesops.core.config import config -from fidesops.graph.config import CollectionAddress -from fidesops.models.policy import PausedStep -from fidesops.models.privacy_request import ( +from fidesops.ops.common_exceptions import PrivacyRequestPaused +from fidesops.ops.core.config import config +from fidesops.ops.graph.config import CollectionAddress +from fidesops.ops.models.policy import PausedStep +from fidesops.ops.models.privacy_request import ( ExecutionLog, ExecutionLogStatus, PrivacyRequest, ) -from fidesops.task import graph_task +from fidesops.ops.task import graph_task from ..graph.graph_test_util import assert_rows_match from ..task.traversal_data import postgres_and_manual_nodes diff --git a/tests/ops/integration_tests/test_mongo_task.py b/tests/ops/integration_tests/test_mongo_task.py index d667c81c8f..47e37df1ca 100644 --- a/tests/ops/integration_tests/test_mongo_task.py +++ b/tests/ops/integration_tests/test_mongo_task.py @@ -7,23 +7,23 @@ import pytest from bson import ObjectId -from fidesops.graph.config import Collection, Dataset, FieldAddress, ScalarField -from fidesops.graph.data_type import ( +from fidesops.ops.graph.config import Collection, Dataset, FieldAddress, ScalarField +from fidesops.ops.graph.data_type import ( IntTypeConverter, ObjectIdTypeConverter, StringTypeConverter, ) -from fidesops.graph.graph import DatasetGraph, Edge, Node -from fidesops.graph.traversal import TraversalNode -from fidesops.models.connectionconfig import ConnectionConfig -from fidesops.models.datasetconfig import convert_dataset_to_graph -from fidesops.models.policy import Policy -from fidesops.models.privacy_request import PrivacyRequest -from fidesops.schemas.dataset import FidesopsDataset -from fidesops.service.connectors import get_connector -from fidesops.task import graph_task -from fidesops.task.filter_results import filter_data_categories -from fidesops.task.graph_task import get_cached_data_for_erasures +from fidesops.ops.graph.graph import DatasetGraph, Edge, Node +from fidesops.ops.graph.traversal import TraversalNode +from fidesops.ops.models.connectionconfig import ConnectionConfig +from fidesops.ops.models.datasetconfig import convert_dataset_to_graph +from fidesops.ops.models.policy import Policy +from fidesops.ops.models.privacy_request import PrivacyRequest +from fidesops.ops.schemas.dataset import FidesopsDataset +from fidesops.ops.service.connectors import get_connector +from fidesops.ops.task import graph_task +from fidesops.ops.task.filter_results import filter_data_categories +from fidesops.ops.task.graph_task import get_cached_data_for_erasures from ..graph.graph_test_util import assert_rows_match, erasure_policy, field from ..task.traversal_data import ( @@ -1049,7 +1049,7 @@ def traversal_node(self, example_datasets, integration_mongodb_config): traversal_node = TraversalNode(node) return traversal_node - @mock.patch("fidesops.graph.traversal.TraversalNode.incoming_edges") + @mock.patch("fidesops.ops.graph.traversal.TraversalNode.incoming_edges") def test_retrieving_data( self, mock_incoming_edges: Mock, @@ -1071,7 +1071,7 @@ def test_retrieving_data( assert results[0]["customer_id"] == 1 - @mock.patch("fidesops.graph.traversal.TraversalNode.incoming_edges") + @mock.patch("fidesops.ops.graph.traversal.TraversalNode.incoming_edges") def test_retrieving_data_no_input( self, mock_incoming_edges: Mock, @@ -1109,7 +1109,7 @@ def test_retrieving_data_no_input( ) assert results == [] - @mock.patch("fidesops.graph.traversal.TraversalNode.incoming_edges") + @mock.patch("fidesops.ops.graph.traversal.TraversalNode.incoming_edges") def test_retrieving_data_input_not_in_table( self, mock_incoming_edges: Mock, diff --git a/tests/ops/integration_tests/test_sql_task.py b/tests/ops/integration_tests/test_sql_task.py index 44225384e9..ef4aea55d4 100644 --- a/tests/ops/integration_tests/test_sql_task.py +++ b/tests/ops/integration_tests/test_sql_task.py @@ -7,26 +7,26 @@ import pytest -from fidesops.core.config import config -from fidesops.graph.config import ( +from fidesops.ops.core.config import config +from fidesops.ops.graph.config import ( Collection, CollectionAddress, Dataset, FieldAddress, ScalarField, ) -from fidesops.graph.data_type import DataType, StringTypeConverter -from fidesops.graph.graph import DatasetGraph, Edge, Node -from fidesops.graph.traversal import TraversalNode -from fidesops.models.connectionconfig import ConnectionConfig -from fidesops.models.datasetconfig import convert_dataset_to_graph -from fidesops.models.policy import ActionType, Policy, Rule, RuleTarget -from fidesops.models.privacy_request import ExecutionLog, PrivacyRequest -from fidesops.schemas.dataset import FidesopsDataset -from fidesops.service.connectors import get_connector -from fidesops.task import graph_task -from fidesops.task.filter_results import filter_data_categories -from fidesops.task.graph_task import get_cached_data_for_erasures +from fidesops.ops.graph.data_type import DataType, StringTypeConverter +from fidesops.ops.graph.graph import DatasetGraph, Edge, Node +from fidesops.ops.graph.traversal import TraversalNode +from fidesops.ops.models.connectionconfig import ConnectionConfig +from fidesops.ops.models.datasetconfig import convert_dataset_to_graph +from fidesops.ops.models.policy import ActionType, Policy, Rule, RuleTarget +from fidesops.ops.models.privacy_request import ExecutionLog, PrivacyRequest +from fidesops.ops.schemas.dataset import FidesopsDataset +from fidesops.ops.service.connectors import get_connector +from fidesops.ops.task import graph_task +from fidesops.ops.task.filter_results import filter_data_categories +from fidesops.ops.task.graph_task import get_cached_data_for_erasures from ..graph.graph_test_util import ( assert_rows_match, @@ -859,7 +859,7 @@ def traversal_node(self, example_datasets, integration_postgres_config): traversal_node = TraversalNode(node) return traversal_node - @mock.patch("fidesops.graph.traversal.TraversalNode.incoming_edges") + @mock.patch("fidesops.ops.graph.traversal.TraversalNode.incoming_edges") def test_retrieving_data( self, mock_incoming_edges: Mock, @@ -893,7 +893,7 @@ def test_retrieving_data( } ] - @mock.patch("fidesops.graph.traversal.TraversalNode.incoming_edges") + @mock.patch("fidesops.ops.graph.traversal.TraversalNode.incoming_edges") def test_retrieving_data_no_input( self, mock_incoming_edges: Mock, @@ -929,7 +929,7 @@ def test_retrieving_data_no_input( traversal_node, Policy(), privacy_request, {"email": None} ) - @mock.patch("fidesops.graph.traversal.TraversalNode.incoming_edges") + @mock.patch("fidesops.ops.graph.traversal.TraversalNode.incoming_edges") def test_retrieving_data_input_not_in_table( self, mock_incoming_edges: Mock, @@ -959,7 +959,9 @@ def test_retrieving_data_input_not_in_table( @pytest.mark.integration_postgres @pytest.mark.integration class TestRetryIntegration: - @mock.patch("fidesops.service.connectors.sql_connector.SQLConnector.retrieve_data") + @mock.patch( + "fidesops.ops.service.connectors.sql_connector.SQLConnector.retrieve_data" + ) def test_retry_access_request( self, mock_retrieve, @@ -1011,7 +1013,7 @@ def test_retry_access_request( ("postgres_example_test_dataset:employee", "error"), ] - @mock.patch("fidesops.service.connectors.sql_connector.SQLConnector.mask_data") + @mock.patch("fidesops.ops.service.connectors.sql_connector.SQLConnector.mask_data") def test_retry_erasure( self, mock_mask: Mock, diff --git a/tests/ops/models/test_base.py b/tests/ops/models/test_base.py index 2f85fdd569..e75f214b6c 100644 --- a/tests/ops/models/test_base.py +++ b/tests/ops/models/test_base.py @@ -8,7 +8,7 @@ from fideslib.exceptions import KeyValidationError from sqlalchemy.orm import Session -from fidesops.models.storage import StorageConfig +from fidesops.ops.models.storage import StorageConfig def test_get_key_from_data_method_invalid_key() -> None: diff --git a/tests/ops/models/test_client.py b/tests/ops/models/test_client.py index 2a7e062ebe..d1b4798bc8 100644 --- a/tests/ops/models/test_client.py +++ b/tests/ops/models/test_client.py @@ -2,8 +2,8 @@ from fideslib.models.client import ClientDetail from sqlalchemy.orm import Session -from fidesops.api.v1.scope_registry import SCOPE_REGISTRY -from fidesops.core.config import config +from fidesops.ops.api.v1.scope_registry import SCOPE_REGISTRY +from fidesops.ops.core.config import config class TestClientModel: diff --git a/tests/ops/models/test_connectionconfig.py b/tests/ops/models/test_connectionconfig.py index 89227d7297..f57124c933 100644 --- a/tests/ops/models/test_connectionconfig.py +++ b/tests/ops/models/test_connectionconfig.py @@ -2,13 +2,13 @@ from fideslib.db.base_class import KeyOrNameAlreadyExists, KeyValidationError from sqlalchemy.orm import Session -from fidesops.models.connectionconfig import ( +from fidesops.ops.models.connectionconfig import ( AccessLevel, ConnectionConfig, ConnectionType, ) -from fidesops.schemas.saas.saas_config import SaaSConfig -from fidesops.util.text import to_snake_case +from fidesops.ops.schemas.saas.saas_config import SaaSConfig +from fidesops.ops.util.text import to_snake_case class TestConnectionConfigModel: diff --git a/tests/ops/models/test_datasetconfig.py b/tests/ops/models/test_datasetconfig.py index e4515d713f..4ded2f4ac2 100644 --- a/tests/ops/models/test_datasetconfig.py +++ b/tests/ops/models/test_datasetconfig.py @@ -1,11 +1,11 @@ from sqlalchemy.orm import Session -from fidesops.models.datasetconfig import ( +from fidesops.ops.models.datasetconfig import ( DatasetConfig, FieldAddress, convert_dataset_to_graph, ) -from fidesops.schemas.dataset import FidesopsDataset +from fidesops.ops.schemas.dataset import FidesopsDataset from ..graph.graph_test_util import field diff --git a/tests/ops/models/test_fidesopsuserpermissions.py b/tests/ops/models/test_fidesopsuserpermissions.py index 42010e1ebf..7b80bdde5a 100644 --- a/tests/ops/models/test_fidesopsuserpermissions.py +++ b/tests/ops/models/test_fidesopsuserpermissions.py @@ -3,7 +3,7 @@ from fideslib.models.fides_user_permissions import FidesUserPermissions from sqlalchemy.orm import Session -from fidesops.api.v1.scope_registry import PRIVACY_REQUEST_READ +from fidesops.ops.api.v1.scope_registry import PRIVACY_REQUEST_READ class TestFidesUserPermissions: diff --git a/tests/ops/models/test_policy.py b/tests/ops/models/test_policy.py index abdf6d4b67..e10b75170f 100644 --- a/tests/ops/models/test_policy.py +++ b/tests/ops/models/test_policy.py @@ -2,24 +2,26 @@ from fideslib.models.client import ClientDetail from sqlalchemy.orm import Session -from fidesops.common_exceptions import ( +from fidesops.ops.common_exceptions import ( DataCategoryNotSupported, PolicyValidationError, RuleValidationError, ) -from fidesops.models.policy import ( +from fidesops.ops.models.policy import ( ActionType, Policy, Rule, RuleTarget, _is_ancestor_of_contained_categories, ) -from fidesops.service.masking.strategy.masking_strategy_hash import HASH_STRATEGY_NAME -from fidesops.service.masking.strategy.masking_strategy_nullify import ( +from fidesops.ops.service.masking.strategy.masking_strategy_hash import ( + HASH_STRATEGY_NAME, +) +from fidesops.ops.service.masking.strategy.masking_strategy_nullify import ( NULL_REWRITE_STRATEGY_NAME, ) -from fidesops.util.data_category import DataCategory -from fidesops.util.text import to_snake_case +from fidesops.ops.util.data_category import DataCategory +from fidesops.ops.util.text import to_snake_case def test_policy_sets_slug( diff --git a/tests/ops/models/test_policy_webhooks.py b/tests/ops/models/test_policy_webhooks.py index a02f7db6a3..3bebae53b8 100644 --- a/tests/ops/models/test_policy_webhooks.py +++ b/tests/ops/models/test_policy_webhooks.py @@ -3,8 +3,8 @@ import pytest from sqlalchemy.orm import Session -from fidesops.common_exceptions import WebhookOrderException -from fidesops.models.policy import PolicyPostWebhook +from fidesops.ops.common_exceptions import WebhookOrderException +from fidesops.ops.models.policy import PolicyPostWebhook def test_reorder_webhooks(db: Session, policy, https_connection_config): diff --git a/tests/ops/models/test_privacy_request.py b/tests/ops/models/test_privacy_request.py index 56fe1d0e94..debaa36408 100644 --- a/tests/ops/models/test_privacy_request.py +++ b/tests/ops/models/test_privacy_request.py @@ -7,13 +7,16 @@ from pydantic import ValidationError from sqlalchemy.orm import Session -from fidesops.common_exceptions import ClientUnsuccessfulException, PrivacyRequestPaused -from fidesops.graph.config import CollectionAddress -from fidesops.models.policy import PausedStep, Policy -from fidesops.models.privacy_request import PrivacyRequest, PrivacyRequestStatus -from fidesops.schemas.redis_cache import PrivacyRequestIdentity -from fidesops.service.connectors.manual_connector import ManualAction -from fidesops.util.cache import FidesopsRedis, get_identity_cache_key +from fidesops.ops.common_exceptions import ( + ClientUnsuccessfulException, + PrivacyRequestPaused, +) +from fidesops.ops.graph.config import CollectionAddress +from fidesops.ops.models.policy import PausedStep, Policy +from fidesops.ops.models.privacy_request import PrivacyRequest, PrivacyRequestStatus +from fidesops.ops.schemas.redis_cache import PrivacyRequestIdentity +from fidesops.ops.service.connectors.manual_connector import ManualAction +from fidesops.ops.util.cache import FidesopsRedis, get_identity_cache_key paused_location = CollectionAddress("test_dataset", "test_collection") diff --git a/tests/ops/models/test_saasconfig.py b/tests/ops/models/test_saasconfig.py index d566c26206..f4e25f1edf 100644 --- a/tests/ops/models/test_saasconfig.py +++ b/tests/ops/models/test_saasconfig.py @@ -3,9 +3,9 @@ import pytest from pydantic import ValidationError -from fidesops.graph.config import CollectionAddress, FieldAddress -from fidesops.schemas.dataset import FidesopsDatasetReference -from fidesops.schemas.saas.saas_config import ParamValue, SaaSConfig, SaaSRequest +from fidesops.ops.graph.config import CollectionAddress, FieldAddress +from fidesops.ops.schemas.dataset import FidesopsDatasetReference +from fidesops.ops.schemas.saas.saas_config import ParamValue, SaaSConfig, SaaSRequest @pytest.mark.unit_saas diff --git a/tests/ops/models/test_storage.py b/tests/ops/models/test_storage.py index 77d8caec13..8ff5514103 100644 --- a/tests/ops/models/test_storage.py +++ b/tests/ops/models/test_storage.py @@ -4,8 +4,8 @@ from fideslib.db.base_class import KeyOrNameAlreadyExists from sqlalchemy.orm import Session -from fidesops.models.storage import StorageConfig -from fidesops.schemas.storage.storage import ( +from fidesops.ops.models.storage import StorageConfig +from fidesops.ops.schemas.storage.storage import ( ResponseFormat, StorageDestination, StorageDetails, diff --git a/tests/ops/schemas/connection_configuration/test_connection_secrets_saas.py b/tests/ops/schemas/connection_configuration/test_connection_secrets_saas.py index adf188bf56..1aad40d025 100644 --- a/tests/ops/schemas/connection_configuration/test_connection_secrets_saas.py +++ b/tests/ops/schemas/connection_configuration/test_connection_secrets_saas.py @@ -3,11 +3,11 @@ import pytest from pydantic import ValidationError -from fidesops.schemas.connection_configuration.connection_secrets_saas import ( +from fidesops.ops.schemas.connection_configuration.connection_secrets_saas import ( SaaSSchema, SaaSSchemaFactory, ) -from fidesops.schemas.saas.saas_config import SaaSConfig +from fidesops.ops.schemas.saas.saas_config import SaaSConfig @pytest.mark.unit_saas diff --git a/tests/ops/schemas/test_user_permission.py b/tests/ops/schemas/test_user_permission.py index e19b4a222f..3f11818fe3 100644 --- a/tests/ops/schemas/test_user_permission.py +++ b/tests/ops/schemas/test_user_permission.py @@ -2,8 +2,8 @@ from fastapi import HTTPException from starlette.status import HTTP_422_UNPROCESSABLE_ENTITY -from fidesops.api.v1.scope_registry import USER_DELETE, USER_PERMISSION_CREATE -from fidesops.schemas.user_permission import UserPermissionsCreate +from fidesops.ops.api.v1.scope_registry import USER_DELETE, USER_PERMISSION_CREATE +from fidesops.ops.schemas.user_permission import UserPermissionsCreate class TestUserPermissionsCreate: diff --git a/tests/ops/scripts/test_create_superuser.py b/tests/ops/scripts/test_create_superuser.py index 797f18360f..24785174f4 100644 --- a/tests/ops/scripts/test_create_superuser.py +++ b/tests/ops/scripts/test_create_superuser.py @@ -14,7 +14,7 @@ from fideslib.models.fides_user_permissions import FidesUserPermissions from fideslib.oauth.schemas.user import UserCreate -from fidesops.api.v1.scope_registry import CLIENT_CREATE +from fidesops.ops.api.v1.scope_registry import CLIENT_CREATE class TestCreateSuperuserScript: diff --git a/tests/ops/service/authentication/test_authentication_strategy_basic.py b/tests/ops/service/authentication/test_authentication_strategy_basic.py index edc896179f..06deec2c7a 100644 --- a/tests/ops/service/authentication/test_authentication_strategy_basic.py +++ b/tests/ops/service/authentication/test_authentication_strategy_basic.py @@ -2,9 +2,11 @@ from fideslib.cryptography.cryptographic_util import bytes_to_b64_str from requests import PreparedRequest, Request -from fidesops.common_exceptions import ValidationError as FidesopsValidationError -from fidesops.models.connectionconfig import ConnectionConfig -from fidesops.service.authentication.authentication_strategy_factory import get_strategy +from fidesops.ops.common_exceptions import ValidationError as FidesopsValidationError +from fidesops.ops.models.connectionconfig import ConnectionConfig +from fidesops.ops.service.authentication.authentication_strategy_factory import ( + get_strategy, +) def test_basic_auth_with_username_and_password(): diff --git a/tests/ops/service/authentication/test_authentication_strategy_bearer.py b/tests/ops/service/authentication/test_authentication_strategy_bearer.py index 523eb1035f..2bf53867e0 100644 --- a/tests/ops/service/authentication/test_authentication_strategy_bearer.py +++ b/tests/ops/service/authentication/test_authentication_strategy_bearer.py @@ -1,9 +1,11 @@ import pytest from requests import PreparedRequest, Request -from fidesops.common_exceptions import ValidationError as FidesopsValidationError -from fidesops.models.connectionconfig import ConnectionConfig -from fidesops.service.authentication.authentication_strategy_factory import get_strategy +from fidesops.ops.common_exceptions import ValidationError as FidesopsValidationError +from fidesops.ops.models.connectionconfig import ConnectionConfig +from fidesops.ops.service.authentication.authentication_strategy_factory import ( + get_strategy, +) def test_bearer_auth_with_token(): diff --git a/tests/ops/service/authentication/test_authentication_strategy_factory.py b/tests/ops/service/authentication/test_authentication_strategy_factory.py index fb78814225..0ed62657f8 100644 --- a/tests/ops/service/authentication/test_authentication_strategy_factory.py +++ b/tests/ops/service/authentication/test_authentication_strategy_factory.py @@ -1,14 +1,16 @@ import pytest -from fidesops.common_exceptions import NoSuchStrategyException -from fidesops.service.authentication.authentication_strategy_basic import ( +from fidesops.ops.common_exceptions import NoSuchStrategyException +from fidesops.ops.service.authentication.authentication_strategy_basic import ( BasicAuthenticationStrategy, ) -from fidesops.service.authentication.authentication_strategy_bearer import ( +from fidesops.ops.service.authentication.authentication_strategy_bearer import ( BearerAuthenticationStrategy, ) -from fidesops.service.authentication.authentication_strategy_factory import get_strategy -from fidesops.service.authentication.authentication_strategy_query_param import ( +from fidesops.ops.service.authentication.authentication_strategy_factory import ( + get_strategy, +) +from fidesops.ops.service.authentication.authentication_strategy_query_param import ( QueryParamAuthenticationStrategy, ) diff --git a/tests/ops/service/authentication/test_authentication_strategy_oauth2.py b/tests/ops/service/authentication/test_authentication_strategy_oauth2.py index 47a4fd092a..97cddfcbd5 100644 --- a/tests/ops/service/authentication/test_authentication_strategy_oauth2.py +++ b/tests/ops/service/authentication/test_authentication_strategy_oauth2.py @@ -6,9 +6,11 @@ from requests import PreparedRequest, Request from sqlalchemy.orm import Session -from fidesops.common_exceptions import FidesopsException, OAuth2TokenException -from fidesops.service.authentication.authentication_strategy_factory import get_strategy -from fidesops.service.authentication.authentication_strategy_oauth2 import ( +from fidesops.ops.common_exceptions import FidesopsException, OAuth2TokenException +from fidesops.ops.service.authentication.authentication_strategy_factory import ( + get_strategy, +) +from fidesops.ops.service.authentication.authentication_strategy_oauth2 import ( OAuth2AuthenticationStrategy, ) @@ -84,8 +86,10 @@ def test_oauth2_authentication_missing_secrets( ) # access token expired, call refresh request - @mock.patch("fidesops.models.connectionconfig.ConnectionConfig.update") - @mock.patch("fidesops.service.connectors.saas_connector.AuthenticatedClient.send") + @mock.patch("fidesops.ops.models.connectionconfig.ConnectionConfig.update") + @mock.patch( + "fidesops.ops.service.connectors.saas_connector.AuthenticatedClient.send" + ) def test_oauth2_authentication_successful_refresh( self, mock_send: Mock, @@ -143,7 +147,9 @@ def test_oauth2_authentication_no_refresh( ) # access token expired, unable to refresh - @mock.patch("fidesops.service.connectors.saas_connector.AuthenticatedClient.send") + @mock.patch( + "fidesops.ops.service.connectors.saas_connector.AuthenticatedClient.send" + ) def test_oauth2_authentication_failed_refresh( self, mock_send: Mock, oauth2_connection_config, oauth2_configuration ): @@ -167,10 +173,10 @@ def test_oauth2_authentication_failed_refresh( class TestAuthorizationUrl: @mock.patch( - "fidesops.service.authentication.authentication_strategy_oauth2.OAuth2AuthenticationStrategy._generate_state" + "fidesops.ops.service.authentication.authentication_strategy_oauth2.OAuth2AuthenticationStrategy._generate_state" ) @mock.patch( - "fidesops.models.authentication_request.AuthenticationRequest.create_or_update" + "fidesops.ops.models.authentication_request.AuthenticationRequest.create_or_update" ) def test_get_authorization_url( self, @@ -218,8 +224,10 @@ def test_get_authorization_url_missing_secrets( class TestAccessTokenRequest: @mock.patch("datetime.datetime") - @mock.patch("fidesops.models.connectionconfig.ConnectionConfig.update") - @mock.patch("fidesops.service.connectors.saas_connector.AuthenticatedClient.send") + @mock.patch("fidesops.ops.models.connectionconfig.ConnectionConfig.update") + @mock.patch( + "fidesops.ops.service.connectors.saas_connector.AuthenticatedClient.send" + ) def test_get_access_token( self, mock_send: Mock, @@ -265,8 +273,10 @@ def test_get_access_token( # make sure we can use the expires_in value in the config if no expires_in is provided # in the access token response @mock.patch("datetime.datetime") - @mock.patch("fidesops.models.connectionconfig.ConnectionConfig.update") - @mock.patch("fidesops.service.connectors.saas_connector.AuthenticatedClient.send") + @mock.patch("fidesops.ops.models.connectionconfig.ConnectionConfig.update") + @mock.patch( + "fidesops.ops.service.connectors.saas_connector.AuthenticatedClient.send" + ) def test_get_access_token_no_expires_in( self, mock_send: Mock, diff --git a/tests/ops/service/authentication/test_authentication_strategy_query_param.py b/tests/ops/service/authentication/test_authentication_strategy_query_param.py index 1a7229c19a..5e4b7ed01e 100644 --- a/tests/ops/service/authentication/test_authentication_strategy_query_param.py +++ b/tests/ops/service/authentication/test_authentication_strategy_query_param.py @@ -1,9 +1,11 @@ import pytest from requests import PreparedRequest, Request -from fidesops.common_exceptions import ValidationError as FidesopsValidationError -from fidesops.models.connectionconfig import ConnectionConfig -from fidesops.service.authentication.authentication_strategy_factory import get_strategy +from fidesops.ops.common_exceptions import ValidationError as FidesopsValidationError +from fidesops.ops.models.connectionconfig import ConnectionConfig +from fidesops.ops.service.authentication.authentication_strategy_factory import ( + get_strategy, +) def test_query_param_auth(): diff --git a/tests/ops/service/connection_config/test_http_connector.py b/tests/ops/service/connection_config/test_http_connector.py index 974e76b654..1909c69945 100644 --- a/tests/ops/service/connection_config/test_http_connector.py +++ b/tests/ops/service/connection_config/test_http_connector.py @@ -3,8 +3,8 @@ import pytest import requests_mock -from fidesops.common_exceptions import ClientUnsuccessfulException -from fidesops.service.connectors import HTTPSConnector +from fidesops.ops.common_exceptions import ClientUnsuccessfulException +from fidesops.ops.service.connectors import HTTPSConnector class TestHttpConnectorMethods: diff --git a/tests/ops/service/connection_config/test_mariadb_connector.py b/tests/ops/service/connection_config/test_mariadb_connector.py index ec5453a01b..cb2c3243e8 100644 --- a/tests/ops/service/connection_config/test_mariadb_connector.py +++ b/tests/ops/service/connection_config/test_mariadb_connector.py @@ -1,6 +1,6 @@ from sqlalchemy.orm import Session -from fidesops.service.connectors.sql_connector import MariaDBConnector +from fidesops.ops.service.connectors.sql_connector import MariaDBConnector def test_mariadb_connector_build_uri(connection_config_mariadb, db: Session): diff --git a/tests/ops/service/connection_config/test_mysql_connector.py b/tests/ops/service/connection_config/test_mysql_connector.py index 9ccd21f405..2fc4eb81de 100644 --- a/tests/ops/service/connection_config/test_mysql_connector.py +++ b/tests/ops/service/connection_config/test_mysql_connector.py @@ -1,6 +1,6 @@ from sqlalchemy.orm import Session -from fidesops.service.connectors.sql_connector import MySQLConnector +from fidesops.ops.service.connectors.sql_connector import MySQLConnector def test_mysql_connector_build_uri(connection_config_mysql, db: Session): diff --git a/tests/ops/service/connection_config/test_postgres_connector.py b/tests/ops/service/connection_config/test_postgres_connector.py index 10a9a1ffa1..436bf6d60a 100644 --- a/tests/ops/service/connection_config/test_postgres_connector.py +++ b/tests/ops/service/connection_config/test_postgres_connector.py @@ -1,7 +1,7 @@ from sqlalchemy.orm import Session -from fidesops.models.connectionconfig import ConnectionConfig -from fidesops.service.connectors import PostgreSQLConnector +from fidesops.ops.models.connectionconfig import ConnectionConfig +from fidesops.ops.service.connectors import PostgreSQLConnector def test_postgres_connector_build_uri(connection_config: ConnectionConfig, db: Session): diff --git a/tests/ops/service/connectors/test_queryconfig.py b/tests/ops/service/connectors/test_queryconfig.py index 451dd34422..2dcfd76752 100644 --- a/tests/ops/service/connectors/test_queryconfig.py +++ b/tests/ops/service/connectors/test_queryconfig.py @@ -3,30 +3,33 @@ import pytest -from fidesops.core.config import config -from fidesops.graph.config import ( +from fidesops.ops.core.config import config +from fidesops.ops.graph.config import ( CollectionAddress, FieldAddress, FieldPath, ObjectField, ScalarField, ) -from fidesops.graph.graph import DatasetGraph, Edge -from fidesops.graph.traversal import Traversal, TraversalNode -from fidesops.models.datasetconfig import convert_dataset_to_graph -from fidesops.models.privacy_request import PrivacyRequest -from fidesops.schemas.dataset import FidesopsDataset -from fidesops.schemas.masking.masking_configuration import HashMaskingConfiguration -from fidesops.schemas.masking.masking_secrets import MaskingSecretCache, SecretType -from fidesops.schemas.saas.saas_config import ParamValue, SaaSConfig, SaaSRequest -from fidesops.schemas.saas.shared_schemas import HTTPMethod, SaaSRequestParams -from fidesops.service.connectors.query_config import MongoQueryConfig, SQLQueryConfig -from fidesops.service.connectors.saas_query_config import SaaSQueryConfig -from fidesops.service.masking.strategy.masking_strategy_hash import ( +from fidesops.ops.graph.graph import DatasetGraph, Edge +from fidesops.ops.graph.traversal import Traversal, TraversalNode +from fidesops.ops.models.datasetconfig import convert_dataset_to_graph +from fidesops.ops.models.privacy_request import PrivacyRequest +from fidesops.ops.schemas.dataset import FidesopsDataset +from fidesops.ops.schemas.masking.masking_configuration import HashMaskingConfiguration +from fidesops.ops.schemas.masking.masking_secrets import MaskingSecretCache, SecretType +from fidesops.ops.schemas.saas.saas_config import ParamValue, SaaSConfig, SaaSRequest +from fidesops.ops.schemas.saas.shared_schemas import HTTPMethod, SaaSRequestParams +from fidesops.ops.service.connectors.query_config import ( + MongoQueryConfig, + SQLQueryConfig, +) +from fidesops.ops.service.connectors.saas_query_config import SaaSQueryConfig +from fidesops.ops.service.masking.strategy.masking_strategy_hash import ( HASH_STRATEGY_NAME, HashMaskingStrategy, ) -from fidesops.util.data_category import DataCategory +from fidesops.ops.util.data_category import DataCategory from ...task.traversal_data import combined_mongo_postgresql_graph, integration_db_graph from ...test_helpers.cache_secrets_helper import cache_secret, clear_cache_secrets diff --git a/tests/ops/service/connectors/test_saas_connector.py b/tests/ops/service/connectors/test_saas_connector.py index 81a8696a96..831fc557ad 100644 --- a/tests/ops/service/connectors/test_saas_connector.py +++ b/tests/ops/service/connectors/test_saas_connector.py @@ -5,10 +5,10 @@ from sqlalchemy.orm import Session from starlette.status import HTTP_200_OK, HTTP_404_NOT_FOUND -from fidesops.schemas.saas.saas_config import SaaSRequest -from fidesops.schemas.saas.shared_schemas import HTTPMethod -from fidesops.service.connectors import get_connector -from fidesops.service.connectors.saas_connector import SaaSConnector +from fidesops.ops.schemas.saas.saas_config import SaaSRequest +from fidesops.ops.schemas.saas.shared_schemas import HTTPMethod +from fidesops.ops.service.connectors import get_connector +from fidesops.ops.service.connectors.saas_connector import SaaSConnector @pytest.mark.unit_saas diff --git a/tests/ops/service/masking/strategy/test_masking_strategy_aes_encrypt.py b/tests/ops/service/masking/strategy/test_masking_strategy_aes_encrypt.py index 3761b6508c..7ddf98d4b2 100644 --- a/tests/ops/service/masking/strategy/test_masking_strategy_aes_encrypt.py +++ b/tests/ops/service/masking/strategy/test_masking_strategy_aes_encrypt.py @@ -1,11 +1,11 @@ from unittest import mock from unittest.mock import Mock -from fidesops.schemas.masking.masking_configuration import ( +from fidesops.ops.schemas.masking.masking_configuration import ( AesEncryptionMaskingConfiguration, ) -from fidesops.schemas.masking.masking_secrets import MaskingSecretCache, SecretType -from fidesops.service.masking.strategy.masking_strategy_aes_encrypt import ( +from fidesops.ops.schemas.masking.masking_secrets import MaskingSecretCache, SecretType +from fidesops.ops.service.masking.strategy.masking_strategy_aes_encrypt import ( AES_ENCRYPT_STRATEGY_NAME, AesEncryptionMaskingStrategy, ) @@ -19,7 +19,9 @@ AES_STRATEGY = AesEncryptionMaskingStrategy(configuration=GCM_CONFIGURATION) -@mock.patch("fidesops.service.masking.strategy.masking_strategy_aes_encrypt.encrypt") +@mock.patch( + "fidesops.ops.service.masking.strategy.masking_strategy_aes_encrypt.encrypt" +) def test_mask_gcm_happypath(mock_encrypt: Mock): mock_encrypt.return_value = "encrypted" @@ -34,7 +36,9 @@ def test_mask_gcm_happypath(mock_encrypt: Mock): clear_cache_secrets(request_id) -@mock.patch("fidesops.service.masking.strategy.masking_strategy_aes_encrypt.encrypt") +@mock.patch( + "fidesops.ops.service.masking.strategy.masking_strategy_aes_encrypt.encrypt" +) def test_mask_all_aes_modes(mock_encrypt: Mock): cache_secrets() for mode in AesEncryptionMaskingConfiguration.Mode: diff --git a/tests/ops/service/masking/strategy/test_masking_strategy_factory.py b/tests/ops/service/masking/strategy/test_masking_strategy_factory.py index f2acf84cc7..67d4628e2f 100644 --- a/tests/ops/service/masking/strategy/test_masking_strategy_factory.py +++ b/tests/ops/service/masking/strategy/test_masking_strategy_factory.py @@ -1,14 +1,16 @@ import pytest -from fidesops.service.masking.strategy.masking_strategy_aes_encrypt import ( +from fidesops.ops.service.masking.strategy.masking_strategy_aes_encrypt import ( AesEncryptionMaskingStrategy, ) -from fidesops.service.masking.strategy.masking_strategy_factory import ( +from fidesops.ops.service.masking.strategy.masking_strategy_factory import ( MaskingStrategyFactory, NoSuchStrategyException, ) -from fidesops.service.masking.strategy.masking_strategy_hash import HashMaskingStrategy -from fidesops.service.masking.strategy.masking_strategy_string_rewrite import ( +from fidesops.ops.service.masking.strategy.masking_strategy_hash import ( + HashMaskingStrategy, +) +from fidesops.ops.service.masking.strategy.masking_strategy_string_rewrite import ( StringRewriteMaskingStrategy, ) diff --git a/tests/ops/service/masking/strategy/test_masking_strategy_hash.py b/tests/ops/service/masking/strategy/test_masking_strategy_hash.py index b5da396d32..eda1805a2b 100644 --- a/tests/ops/service/masking/strategy/test_masking_strategy_hash.py +++ b/tests/ops/service/masking/strategy/test_masking_strategy_hash.py @@ -1,6 +1,6 @@ -from fidesops.schemas.masking.masking_configuration import HashMaskingConfiguration -from fidesops.schemas.masking.masking_secrets import MaskingSecretCache, SecretType -from fidesops.service.masking.strategy.masking_strategy_hash import ( +from fidesops.ops.schemas.masking.masking_configuration import HashMaskingConfiguration +from fidesops.ops.schemas.masking.masking_secrets import MaskingSecretCache, SecretType +from fidesops.ops.service.masking.strategy.masking_strategy_hash import ( HASH_STRATEGY_NAME, HashMaskingStrategy, ) diff --git a/tests/ops/service/masking/strategy/test_masking_strategy_hmac.py b/tests/ops/service/masking/strategy/test_masking_strategy_hmac.py index 822ac1efa7..502d6cfb41 100644 --- a/tests/ops/service/masking/strategy/test_masking_strategy_hmac.py +++ b/tests/ops/service/masking/strategy/test_masking_strategy_hmac.py @@ -1,6 +1,6 @@ -from fidesops.schemas.masking.masking_configuration import HmacMaskingConfiguration -from fidesops.schemas.masking.masking_secrets import MaskingSecretCache, SecretType -from fidesops.service.masking.strategy.masking_strategy_hmac import ( +from fidesops.ops.schemas.masking.masking_configuration import HmacMaskingConfiguration +from fidesops.ops.schemas.masking.masking_secrets import MaskingSecretCache, SecretType +from fidesops.ops.service.masking.strategy.masking_strategy_hmac import ( HMAC_STRATEGY_NAME, HmacMaskingStrategy, ) diff --git a/tests/ops/service/masking/strategy/test_masking_strategy_null.py b/tests/ops/service/masking/strategy/test_masking_strategy_null.py index 505cf45bff..22e117848c 100644 --- a/tests/ops/service/masking/strategy/test_masking_strategy_null.py +++ b/tests/ops/service/masking/strategy/test_masking_strategy_null.py @@ -1,5 +1,5 @@ -from fidesops.schemas.masking.masking_configuration import NullMaskingConfiguration -from fidesops.service.masking.strategy.masking_strategy_nullify import ( +from fidesops.ops.schemas.masking.masking_configuration import NullMaskingConfiguration +from fidesops.ops.service.masking.strategy.masking_strategy_nullify import ( NullMaskingStrategy, ) diff --git a/tests/ops/service/masking/strategy/test_masking_strategy_random_string_rewrite.py b/tests/ops/service/masking/strategy/test_masking_strategy_random_string_rewrite.py index f06a5f88da..8d2888b648 100644 --- a/tests/ops/service/masking/strategy/test_masking_strategy_random_string_rewrite.py +++ b/tests/ops/service/masking/strategy/test_masking_strategy_random_string_rewrite.py @@ -1,7 +1,7 @@ -from fidesops.schemas.masking.masking_configuration import ( +from fidesops.ops.schemas.masking.masking_configuration import ( RandomStringMaskingConfiguration, ) -from fidesops.service.masking.strategy.masking_strategy_random_string_rewrite import ( +from fidesops.ops.service.masking.strategy.masking_strategy_random_string_rewrite import ( RandomStringRewriteMaskingStrategy, ) diff --git a/tests/ops/service/masking/strategy/test_masking_strategy_string_rewrite.py b/tests/ops/service/masking/strategy/test_masking_strategy_string_rewrite.py index 211cadd3b4..f100723cb9 100644 --- a/tests/ops/service/masking/strategy/test_masking_strategy_string_rewrite.py +++ b/tests/ops/service/masking/strategy/test_masking_strategy_string_rewrite.py @@ -1,7 +1,7 @@ -from fidesops.schemas.masking.masking_configuration import ( +from fidesops.ops.schemas.masking.masking_configuration import ( StringRewriteMaskingConfiguration, ) -from fidesops.service.masking.strategy.masking_strategy_string_rewrite import ( +from fidesops.ops.service.masking.strategy.masking_strategy_string_rewrite import ( StringRewriteMaskingStrategy, ) diff --git a/tests/ops/service/pagination/test_pagination_strategy_cursor.py b/tests/ops/service/pagination/test_pagination_strategy_cursor.py index 1ec0d6d8e3..66c6b6f109 100644 --- a/tests/ops/service/pagination/test_pagination_strategy_cursor.py +++ b/tests/ops/service/pagination/test_pagination_strategy_cursor.py @@ -4,9 +4,11 @@ import pytest from requests import Response -from fidesops.schemas.saas.shared_schemas import HTTPMethod, SaaSRequestParams -from fidesops.schemas.saas.strategy_configuration import CursorPaginationConfiguration -from fidesops.service.pagination.pagination_strategy_cursor import ( +from fidesops.ops.schemas.saas.shared_schemas import HTTPMethod, SaaSRequestParams +from fidesops.ops.schemas.saas.strategy_configuration import ( + CursorPaginationConfiguration, +) +from fidesops.ops.service.pagination.pagination_strategy_cursor import ( CursorPaginationStrategy, ) diff --git a/tests/ops/service/pagination/test_pagination_strategy_factory.py b/tests/ops/service/pagination/test_pagination_strategy_factory.py index 259af13ce9..f8466ce326 100644 --- a/tests/ops/service/pagination/test_pagination_strategy_factory.py +++ b/tests/ops/service/pagination/test_pagination_strategy_factory.py @@ -1,12 +1,14 @@ import pytest -from fidesops.common_exceptions import NoSuchStrategyException, ValidationError -from fidesops.service.pagination.pagination_strategy_cursor import ( +from fidesops.ops.common_exceptions import NoSuchStrategyException, ValidationError +from fidesops.ops.service.pagination.pagination_strategy_cursor import ( CursorPaginationStrategy, ) -from fidesops.service.pagination.pagination_strategy_factory import get_strategy -from fidesops.service.pagination.pagination_strategy_link import LinkPaginationStrategy -from fidesops.service.pagination.pagination_strategy_offset import ( +from fidesops.ops.service.pagination.pagination_strategy_factory import get_strategy +from fidesops.ops.service.pagination.pagination_strategy_link import ( + LinkPaginationStrategy, +) +from fidesops.ops.service.pagination.pagination_strategy_offset import ( OffsetPaginationStrategy, ) diff --git a/tests/ops/service/pagination/test_pagination_strategy_link.py b/tests/ops/service/pagination/test_pagination_strategy_link.py index ec65a4f5fe..53190d4e41 100644 --- a/tests/ops/service/pagination/test_pagination_strategy_link.py +++ b/tests/ops/service/pagination/test_pagination_strategy_link.py @@ -4,9 +4,11 @@ import pytest from requests import Response -from fidesops.schemas.saas.shared_schemas import HTTPMethod, SaaSRequestParams -from fidesops.schemas.saas.strategy_configuration import LinkPaginationConfiguration -from fidesops.service.pagination.pagination_strategy_link import LinkPaginationStrategy +from fidesops.ops.schemas.saas.shared_schemas import HTTPMethod, SaaSRequestParams +from fidesops.ops.schemas.saas.strategy_configuration import LinkPaginationConfiguration +from fidesops.ops.service.pagination.pagination_strategy_link import ( + LinkPaginationStrategy, +) @pytest.fixture(scope="function") diff --git a/tests/ops/service/pagination/test_pagination_strategy_offset.py b/tests/ops/service/pagination/test_pagination_strategy_offset.py index 59bef8e6d4..c1ab80abff 100644 --- a/tests/ops/service/pagination/test_pagination_strategy_offset.py +++ b/tests/ops/service/pagination/test_pagination_strategy_offset.py @@ -4,11 +4,13 @@ import pytest from requests import Response -from fidesops.common_exceptions import FidesopsException -from fidesops.schemas.saas.saas_config import SaaSRequest -from fidesops.schemas.saas.shared_schemas import HTTPMethod, SaaSRequestParams -from fidesops.schemas.saas.strategy_configuration import OffsetPaginationConfiguration -from fidesops.service.pagination.pagination_strategy_offset import ( +from fidesops.ops.common_exceptions import FidesopsException +from fidesops.ops.schemas.saas.saas_config import SaaSRequest +from fidesops.ops.schemas.saas.shared_schemas import HTTPMethod, SaaSRequestParams +from fidesops.ops.schemas.saas.strategy_configuration import ( + OffsetPaginationConfiguration, +) +from fidesops.ops.service.pagination.pagination_strategy_offset import ( OffsetPaginationStrategy, ) diff --git a/tests/ops/service/privacy_request/onetrust_test.py b/tests/ops/service/privacy_request/onetrust_test.py index 3346ef5518..540c6a07d9 100644 --- a/tests/ops/service/privacy_request/onetrust_test.py +++ b/tests/ops/service/privacy_request/onetrust_test.py @@ -6,41 +6,41 @@ from fideslib.models.client import ClientDetail from sqlalchemy.orm import Session -from fidesops.common_exceptions import ( +from fidesops.ops.common_exceptions import ( PolicyNotFoundException, StorageConfigNotFoundException, ) -from fidesops.models.policy import ActionType, Policy, Rule, RuleTarget -from fidesops.models.privacy_request import PrivacyRequest -from fidesops.models.storage import StorageConfig -from fidesops.schemas.storage.storage import StorageDetails, StorageSecrets -from fidesops.schemas.third_party.onetrust import ( +from fidesops.ops.models.policy import ActionType, Policy, Rule, RuleTarget +from fidesops.ops.models.privacy_request import PrivacyRequest +from fidesops.ops.models.storage import StorageConfig +from fidesops.ops.schemas.storage.storage import StorageDetails, StorageSecrets +from fidesops.ops.schemas.third_party.onetrust import ( OneTrustRequest, OneTrustSubtask, OneTrustSubtaskStatus, ) -from fidesops.service.privacy_request.onetrust_service import ( +from fidesops.ops.service.privacy_request.onetrust_service import ( FIDES_TASK, ONETRUST_POLICY_KEY, OneTrustService, ) -from fidesops.util.data_category import DataCategory +from fidesops.ops.util.data_category import DataCategory @mock.patch( - "fidesops.service.privacy_request.onetrust_service.OneTrustService.transition_status" + "fidesops.ops.service.privacy_request.onetrust_service.OneTrustService.transition_status" ) @mock.patch( - "fidesops.service.privacy_request.onetrust_service.get_onetrust_access_token" + "fidesops.ops.service.privacy_request.onetrust_service.get_onetrust_access_token" ) @mock.patch( - "fidesops.service.privacy_request.onetrust_service.OneTrustService._get_all_requests" + "fidesops.ops.service.privacy_request.onetrust_service.OneTrustService._get_all_requests" ) @mock.patch( - "fidesops.service.privacy_request.onetrust_service.OneTrustService._get_all_subtasks" + "fidesops.ops.service.privacy_request.onetrust_service.OneTrustService._get_all_subtasks" ) @mock.patch( - "fidesops.service.privacy_request.request_runner_service.run_privacy_request.delay" + "fidesops.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" ) def test_intake_onetrust_requests_success( finish_processing_mock: Mock, @@ -127,16 +127,16 @@ def test_intake_onetrust_requests_success( @mock.patch( - "fidesops.service.privacy_request.onetrust_service.get_onetrust_access_token" + "fidesops.ops.service.privacy_request.onetrust_service.get_onetrust_access_token" ) @mock.patch( - "fidesops.service.privacy_request.onetrust_service.OneTrustService._get_all_requests" + "fidesops.ops.service.privacy_request.onetrust_service.OneTrustService._get_all_requests" ) @mock.patch( - "fidesops.service.privacy_request.onetrust_service.OneTrustService._get_all_subtasks" + "fidesops.ops.service.privacy_request.onetrust_service.OneTrustService._get_all_subtasks" ) @mock.patch( - "fidesops.service.privacy_request.request_runner_service.run_privacy_request.delay" + "fidesops.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" ) def test_intake_onetrust_requests_no_config( finish_processing_mock: Mock, @@ -184,16 +184,16 @@ def test_intake_onetrust_requests_no_config( @mock.patch( - "fidesops.service.privacy_request.onetrust_service.get_onetrust_access_token" + "fidesops.ops.service.privacy_request.onetrust_service.get_onetrust_access_token" ) @mock.patch( - "fidesops.service.privacy_request.onetrust_service.OneTrustService._get_all_requests" + "fidesops.ops.service.privacy_request.onetrust_service.OneTrustService._get_all_requests" ) @mock.patch( - "fidesops.service.privacy_request.onetrust_service.OneTrustService._get_all_subtasks" + "fidesops.ops.service.privacy_request.onetrust_service.OneTrustService._get_all_subtasks" ) @mock.patch( - "fidesops.service.privacy_request.request_runner_service.run_privacy_request.delay" + "fidesops.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" ) def test_intake_onetrust_requests_no_policy( finish_processing_mock: Mock, @@ -243,16 +243,16 @@ def test_intake_onetrust_requests_no_policy( @mock.patch( - "fidesops.service.privacy_request.onetrust_service.get_onetrust_access_token" + "fidesops.ops.service.privacy_request.onetrust_service.get_onetrust_access_token" ) @mock.patch( - "fidesops.service.privacy_request.onetrust_service.OneTrustService._get_all_requests" + "fidesops.ops.service.privacy_request.onetrust_service.OneTrustService._get_all_requests" ) @mock.patch( - "fidesops.service.privacy_request.onetrust_service.OneTrustService._get_all_subtasks" + "fidesops.ops.service.privacy_request.onetrust_service.OneTrustService._get_all_subtasks" ) @mock.patch( - "fidesops.service.privacy_request.request_runner_service.run_privacy_request.delay" + "fidesops.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" ) def test_intake_onetrust_requests_auth_fail( finish_processing_mock: Mock, @@ -304,16 +304,16 @@ def test_intake_onetrust_requests_auth_fail( @mock.patch( - "fidesops.service.privacy_request.onetrust_service.get_onetrust_access_token" + "fidesops.ops.service.privacy_request.onetrust_service.get_onetrust_access_token" ) @mock.patch( - "fidesops.service.privacy_request.onetrust_service.OneTrustService._get_all_requests" + "fidesops.ops.service.privacy_request.onetrust_service.OneTrustService._get_all_requests" ) @mock.patch( - "fidesops.service.privacy_request.onetrust_service.OneTrustService._get_all_subtasks" + "fidesops.ops.service.privacy_request.onetrust_service.OneTrustService._get_all_subtasks" ) @mock.patch( - "fidesops.service.privacy_request.request_runner_service.run_privacy_request.delay" + "fidesops.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" ) def test_intake_onetrust_requests_no_fides_tasks( finish_processing_mock: Mock, diff --git a/tests/ops/service/privacy_request/request_runner_service_test.py b/tests/ops/service/privacy_request/request_runner_service_test.py index b507d208c0..9d2b91de8a 100644 --- a/tests/ops/service/privacy_request/request_runner_service_test.py +++ b/tests/ops/service/privacy_request/request_runner_service_test.py @@ -11,44 +11,49 @@ from sqlalchemy import column, select, table from sqlalchemy.orm import Session -from fidesops.common_exceptions import ClientUnsuccessfulException, PrivacyRequestPaused -from fidesops.core.config import config -from fidesops.models.policy import PausedStep, PolicyPostWebhook -from fidesops.models.privacy_request import ( +from fidesops.ops.common_exceptions import ( + ClientUnsuccessfulException, + PrivacyRequestPaused, +) +from fidesops.ops.core.config import config +from fidesops.ops.models.policy import PausedStep, PolicyPostWebhook +from fidesops.ops.models.privacy_request import ( ActionType, ExecutionLog, PolicyPreWebhook, PrivacyRequest, PrivacyRequestStatus, ) -from fidesops.schemas.external_https import SecondPartyResponseFormat -from fidesops.schemas.masking.masking_configuration import ( +from fidesops.ops.schemas.external_https import SecondPartyResponseFormat +from fidesops.ops.schemas.masking.masking_configuration import ( HmacMaskingConfiguration, MaskingConfiguration, ) -from fidesops.schemas.masking.masking_secrets import MaskingSecretCache -from fidesops.schemas.policy import Rule -from fidesops.schemas.saas.shared_schemas import HTTPMethod, SaaSRequestParams -from fidesops.service.connectors.saas_connector import SaaSConnector -from fidesops.service.connectors.sql_connector import ( +from fidesops.ops.schemas.masking.masking_secrets import MaskingSecretCache +from fidesops.ops.schemas.policy import Rule +from fidesops.ops.schemas.saas.shared_schemas import HTTPMethod, SaaSRequestParams +from fidesops.ops.service.connectors.saas_connector import SaaSConnector +from fidesops.ops.service.connectors.sql_connector import ( RedshiftConnector, SnowflakeConnector, ) -from fidesops.service.masking.strategy.masking_strategy_factory import ( +from fidesops.ops.service.masking.strategy.masking_strategy_factory import ( MaskingStrategyFactory, ) -from fidesops.service.masking.strategy.masking_strategy_hmac import HmacMaskingStrategy -from fidesops.service.privacy_request.request_runner_service import ( +from fidesops.ops.service.masking.strategy.masking_strategy_hmac import ( + HmacMaskingStrategy, +) +from fidesops.ops.service.privacy_request.request_runner_service import ( run_webhooks_and_report_status, ) -from fidesops.util.data_category import DataCategory +from fidesops.ops.util.data_category import DataCategory PRIVACY_REQUEST_TASK_TIMEOUT = 5 # External services take much longer to return PRIVACY_REQUEST_TASK_TIMEOUT_EXTERNAL = 30 -@mock.patch("fidesops.service.privacy_request.request_runner_service.upload") +@mock.patch("fidesops.ops.service.privacy_request.request_runner_service.upload") def test_policy_upload_called( upload_mock: Mock, privacy_request_status_pending: PrivacyRequest, @@ -95,7 +100,7 @@ def test_start_processing_doesnt_overwrite_started_processing_at( @mock.patch( - "fidesops.service.privacy_request.request_runner_service.upload_access_results" + "fidesops.ops.service.privacy_request.request_runner_service.upload_access_results" ) def test_halts_proceeding_if_cancelled( upload_access_results_mock, @@ -117,12 +122,12 @@ def test_halts_proceeding_if_cancelled( @mock.patch( - "fidesops.service.privacy_request.request_runner_service.run_webhooks_and_report_status", + "fidesops.ops.service.privacy_request.request_runner_service.run_webhooks_and_report_status", ) @mock.patch( - "fidesops.service.privacy_request.request_runner_service.run_access_request" + "fidesops.ops.service.privacy_request.request_runner_service.run_access_request" ) -@mock.patch("fidesops.service.privacy_request.request_runner_service.run_erasure") +@mock.patch("fidesops.ops.service.privacy_request.request_runner_service.run_erasure") def test_from_graph_resume_does_not_run_pre_webhooks( run_erasure, run_access, @@ -155,12 +160,12 @@ def test_from_graph_resume_does_not_run_pre_webhooks( @mock.patch( - "fidesops.service.privacy_request.request_runner_service.run_webhooks_and_report_status", + "fidesops.ops.service.privacy_request.request_runner_service.run_webhooks_and_report_status", ) @mock.patch( - "fidesops.service.privacy_request.request_runner_service.run_access_request" + "fidesops.ops.service.privacy_request.request_runner_service.run_access_request" ) -@mock.patch("fidesops.service.privacy_request.request_runner_service.run_erasure") +@mock.patch("fidesops.ops.service.privacy_request.request_runner_service.run_erasure") def test_resume_privacy_request_from_erasure( run_erasure, run_access, @@ -248,7 +253,7 @@ def get_privacy_request_results( @pytest.mark.integration_postgres @pytest.mark.integration -@mock.patch("fidesops.models.privacy_request.PrivacyRequest.trigger_policy_webhook") +@mock.patch("fidesops.ops.models.privacy_request.PrivacyRequest.trigger_policy_webhook") def test_create_and_process_access_request( trigger_webhook_mock, postgres_example_test_dataset_config_read_access, @@ -316,7 +321,7 @@ def test_create_and_process_access_request( @pytest.mark.integration -@mock.patch("fidesops.models.privacy_request.PrivacyRequest.trigger_policy_webhook") +@mock.patch("fidesops.ops.models.privacy_request.PrivacyRequest.trigger_policy_webhook") def test_create_and_process_access_request_mssql( trigger_webhook_mock, mssql_example_test_dataset_config, @@ -362,7 +367,7 @@ def test_create_and_process_access_request_mssql( @pytest.mark.integration -@mock.patch("fidesops.models.privacy_request.PrivacyRequest.trigger_policy_webhook") +@mock.patch("fidesops.ops.models.privacy_request.PrivacyRequest.trigger_policy_webhook") def test_create_and_process_access_request_mysql( trigger_webhook_mock, mysql_example_test_dataset_config, @@ -409,7 +414,7 @@ def test_create_and_process_access_request_mysql( @pytest.mark.integration_mariadb @pytest.mark.integration -@mock.patch("fidesops.models.privacy_request.PrivacyRequest.trigger_policy_webhook") +@mock.patch("fidesops.ops.models.privacy_request.PrivacyRequest.trigger_policy_webhook") def test_create_and_process_access_request_mariadb( trigger_webhook_mock, mariadb_example_test_dataset_config, @@ -456,7 +461,7 @@ def test_create_and_process_access_request_mariadb( @pytest.mark.integration_saas @pytest.mark.integration_mailchimp -@mock.patch("fidesops.models.privacy_request.PrivacyRequest.trigger_policy_webhook") +@mock.patch("fidesops.ops.models.privacy_request.PrivacyRequest.trigger_policy_webhook") def test_create_and_process_access_request_saas_mailchimp( trigger_webhook_mock, mailchimp_connection_config, @@ -502,7 +507,7 @@ def test_create_and_process_access_request_saas_mailchimp( @pytest.mark.integration_saas @pytest.mark.integration_mailchimp -@mock.patch("fidesops.models.privacy_request.PrivacyRequest.trigger_policy_webhook") +@mock.patch("fidesops.ops.models.privacy_request.PrivacyRequest.trigger_policy_webhook") def test_create_and_process_erasure_request_saas( _, mailchimp_connection_config, @@ -561,7 +566,7 @@ def test_create_and_process_erasure_request_saas( @pytest.mark.integration_saas @pytest.mark.integration_hubspot -@mock.patch("fidesops.models.privacy_request.PrivacyRequest.trigger_policy_webhook") +@mock.patch("fidesops.ops.models.privacy_request.PrivacyRequest.trigger_policy_webhook") def test_create_and_process_access_request_saas_hubspot( trigger_webhook_mock, connection_config_hubspot, @@ -1376,7 +1381,9 @@ def test_create_and_process_erasure_request_bigquery( class TestRunPrivacyRequestRunsWebhooks: - @mock.patch("fidesops.models.privacy_request.PrivacyRequest.trigger_policy_webhook") + @mock.patch( + "fidesops.ops.models.privacy_request.PrivacyRequest.trigger_policy_webhook" + ) def test_run_webhooks_halt_received( self, mock_trigger_policy_webhook, @@ -1394,7 +1401,9 @@ def test_run_webhooks_halt_received( assert privacy_request.status == PrivacyRequestStatus.paused assert privacy_request.paused_at is not None - @mock.patch("fidesops.models.privacy_request.PrivacyRequest.trigger_policy_webhook") + @mock.patch( + "fidesops.ops.models.privacy_request.PrivacyRequest.trigger_policy_webhook" + ) def test_run_webhooks_ap_scheduler_cleanup( self, mock_trigger_policy_webhook, @@ -1419,7 +1428,9 @@ def test_run_webhooks_ap_scheduler_cleanup( assert privacy_request.finished_processing_at is not None assert privacy_request.paused_at is not None - @mock.patch("fidesops.models.privacy_request.PrivacyRequest.trigger_policy_webhook") + @mock.patch( + "fidesops.ops.models.privacy_request.PrivacyRequest.trigger_policy_webhook" + ) def test_run_webhooks_client_error( self, mock_trigger_policy_webhook, @@ -1437,7 +1448,9 @@ def test_run_webhooks_client_error( assert privacy_request.finished_processing_at is not None assert privacy_request.paused_at is None - @mock.patch("fidesops.models.privacy_request.PrivacyRequest.trigger_policy_webhook") + @mock.patch( + "fidesops.ops.models.privacy_request.PrivacyRequest.trigger_policy_webhook" + ) def test_run_webhooks_validation_error( self, mock_trigger_policy_webhook, @@ -1455,7 +1468,9 @@ def test_run_webhooks_validation_error( assert privacy_request.status == PrivacyRequestStatus.error assert privacy_request.paused_at is None - @mock.patch("fidesops.models.privacy_request.PrivacyRequest.trigger_policy_webhook") + @mock.patch( + "fidesops.ops.models.privacy_request.PrivacyRequest.trigger_policy_webhook" + ) def test_run_webhooks( self, mock_trigger_policy_webhook, @@ -1470,7 +1485,9 @@ def test_run_webhooks( assert privacy_request.finished_processing_at is None assert mock_trigger_policy_webhook.call_count == 2 - @mock.patch("fidesops.models.privacy_request.PrivacyRequest.trigger_policy_webhook") + @mock.patch( + "fidesops.ops.models.privacy_request.PrivacyRequest.trigger_policy_webhook" + ) def test_run_webhooks_after_webhook( self, mock_trigger_policy_webhook, @@ -1491,9 +1508,9 @@ def test_run_webhooks_after_webhook( @pytest.mark.integration_postgres @pytest.mark.integration @mock.patch( - "fidesops.service.privacy_request.request_runner_service.run_access_request" + "fidesops.ops.service.privacy_request.request_runner_service.run_access_request" ) -@mock.patch("fidesops.models.privacy_request.PrivacyRequest.trigger_policy_webhook") +@mock.patch("fidesops.ops.models.privacy_request.PrivacyRequest.trigger_policy_webhook") def test_privacy_request_log_failure( _, run_access_request_mock, @@ -1515,7 +1532,7 @@ def test_privacy_request_log_failure( } with mock.patch( - "fidesops.service.privacy_request.request_runner_service.fideslog_graph_failure" + "fidesops.ops.service.privacy_request.request_runner_service.fideslog_graph_failure" ) as mock_log_event: pr = get_privacy_request_results( db, diff --git a/tests/ops/service/processors/post_processor_strategy/test_post_processor_strategy_factory.py b/tests/ops/service/processors/post_processor_strategy/test_post_processor_strategy_factory.py index 546da6c735..d2bd60ee22 100644 --- a/tests/ops/service/processors/post_processor_strategy/test_post_processor_strategy_factory.py +++ b/tests/ops/service/processors/post_processor_strategy/test_post_processor_strategy_factory.py @@ -1,13 +1,13 @@ import pytest -from fidesops.common_exceptions import NoSuchStrategyException, ValidationError -from fidesops.service.processors.post_processor_strategy.post_processor_strategy_factory import ( +from fidesops.ops.common_exceptions import NoSuchStrategyException, ValidationError +from fidesops.ops.service.processors.post_processor_strategy.post_processor_strategy_factory import ( get_strategy, ) -from fidesops.service.processors.post_processor_strategy.post_processor_strategy_filter import ( +from fidesops.ops.service.processors.post_processor_strategy.post_processor_strategy_filter import ( FilterPostProcessorStrategy, ) -from fidesops.service.processors.post_processor_strategy.post_processor_strategy_unwrap import ( +from fidesops.ops.service.processors.post_processor_strategy.post_processor_strategy_unwrap import ( UnwrapPostProcessorStrategy, ) diff --git a/tests/ops/service/processors/post_processor_strategy/test_post_processor_strategy_filter.py b/tests/ops/service/processors/post_processor_strategy/test_post_processor_strategy_filter.py index 80c639bf6f..25037a98f5 100644 --- a/tests/ops/service/processors/post_processor_strategy/test_post_processor_strategy_filter.py +++ b/tests/ops/service/processors/post_processor_strategy/test_post_processor_strategy_filter.py @@ -2,11 +2,11 @@ import pytest -from fidesops.common_exceptions import FidesopsException -from fidesops.schemas.saas.strategy_configuration import ( +from fidesops.ops.common_exceptions import FidesopsException +from fidesops.ops.schemas.saas.strategy_configuration import ( FilterPostProcessorConfiguration, ) -from fidesops.service.processors.post_processor_strategy.post_processor_strategy_filter import ( +from fidesops.ops.service.processors.post_processor_strategy.post_processor_strategy_filter import ( FilterPostProcessorStrategy, ) diff --git a/tests/ops/service/processors/post_processor_strategy/test_post_processor_strategy_unwrap.py b/tests/ops/service/processors/post_processor_strategy/test_post_processor_strategy_unwrap.py index d7d5151832..270593965c 100644 --- a/tests/ops/service/processors/post_processor_strategy/test_post_processor_strategy_unwrap.py +++ b/tests/ops/service/processors/post_processor_strategy/test_post_processor_strategy_unwrap.py @@ -1,7 +1,7 @@ -from fidesops.schemas.saas.strategy_configuration import ( +from fidesops.ops.schemas.saas.strategy_configuration import ( UnwrapPostProcessorConfiguration, ) -from fidesops.service.processors.post_processor_strategy.post_processor_strategy_unwrap import ( +from fidesops.ops.service.processors.post_processor_strategy.post_processor_strategy_unwrap import ( UnwrapPostProcessorStrategy, ) diff --git a/tests/ops/service/saas_request/test_saas_request_override_factory.py b/tests/ops/service/saas_request/test_saas_request_override_factory.py index 8ac0869445..ad28914000 100644 --- a/tests/ops/service/saas_request/test_saas_request_override_factory.py +++ b/tests/ops/service/saas_request/test_saas_request_override_factory.py @@ -3,19 +3,19 @@ import pytest -from fidesops.common_exceptions import ( +from fidesops.ops.common_exceptions import ( InvalidSaaSRequestOverrideException, NoSuchSaaSRequestOverrideException, ) -from fidesops.graph.traversal import TraversalNode -from fidesops.models.policy import Policy -from fidesops.models.privacy_request import PrivacyRequest -from fidesops.service.saas_request.saas_request_override_factory import ( +from fidesops.ops.graph.traversal import TraversalNode +from fidesops.ops.models.policy import Policy +from fidesops.ops.models.privacy_request import PrivacyRequest +from fidesops.ops.service.saas_request.saas_request_override_factory import ( SaaSRequestOverrideFactory, SaaSRequestType, register, ) -from fidesops.util.collection_util import Row +from fidesops.ops.util.collection_util import Row def uuid(): diff --git a/tests/ops/service/storage_uploader_service_test.py b/tests/ops/service/storage_uploader_service_test.py index c3d2b1ac02..24931b1ec1 100644 --- a/tests/ops/service/storage_uploader_service_test.py +++ b/tests/ops/service/storage_uploader_service_test.py @@ -11,30 +11,30 @@ import pytest from sqlalchemy.orm import Session -from fidesops.common_exceptions import StorageUploadError -from fidesops.core.config import config -from fidesops.models.privacy_request import PrivacyRequest -from fidesops.models.storage import StorageConfig -from fidesops.schemas.storage.storage import ( +from fidesops.ops.common_exceptions import StorageUploadError +from fidesops.ops.core.config import config +from fidesops.ops.models.privacy_request import PrivacyRequest +from fidesops.ops.models.storage import StorageConfig +from fidesops.ops.schemas.storage.storage import ( FileNaming, ResponseFormat, StorageDetails, StorageSecrets, StorageType, ) -from fidesops.service.storage.storage_uploader_service import get_extension, upload -from fidesops.tasks.storage import ( +from fidesops.ops.service.storage.storage_uploader_service import get_extension, upload +from fidesops.ops.tasks.storage import ( LOCAL_FIDES_UPLOAD_DIRECTORY, encrypt_access_request_results, write_to_in_memory_buffer, ) -from fidesops.util.encryption.aes_gcm_encryption_scheme import ( +from fidesops.ops.util.encryption.aes_gcm_encryption_scheme import ( decrypt, decrypt_combined_nonce_and_message, ) -@mock.patch("fidesops.service.storage.storage_uploader_service.upload_to_s3") +@mock.patch("fidesops.ops.service.storage.storage_uploader_service.upload_to_s3") def test_uploader_s3_success( mock_upload_to_s3: Mock, db: Session, privacy_request ) -> None: @@ -81,7 +81,7 @@ def test_uploader_s3_success( storage_config.delete(db) -@mock.patch("fidesops.service.storage.storage_uploader_service.upload_to_s3") +@mock.patch("fidesops.ops.service.storage.storage_uploader_service.upload_to_s3") def test_uploader_s3_invalid_file_naming(mock_upload_to_s3: Mock, db: Session) -> None: request_id = "214513r" @@ -116,7 +116,7 @@ def test_uploader_s3_invalid_file_naming(mock_upload_to_s3: Mock, db: Session) - sc.delete(db) -@mock.patch("fidesops.service.storage.storage_uploader_service.upload_to_s3") +@mock.patch("fidesops.ops.service.storage.storage_uploader_service.upload_to_s3") def test_uploader_no_config(mock_upload_to_s3: Mock, db: Session) -> None: request_id = "214513r" storage_key = "s3_key" @@ -129,8 +129,8 @@ def test_uploader_no_config(mock_upload_to_s3: Mock, db: Session) -> None: mock_upload_to_s3.assert_not_called() -@mock.patch("fidesops.service.storage.storage_uploader_service.upload_to_onetrust") -@mock.patch("fidesops.models.privacy_request.PrivacyRequest.get") +@mock.patch("fidesops.ops.service.storage.storage_uploader_service.upload_to_onetrust") +@mock.patch("fidesops.ops.models.privacy_request.PrivacyRequest.get") def test_uploader_onetrust_success( mock_get_request_details: Mock, mock_upload_to_onetrust: Mock, @@ -189,7 +189,7 @@ def test_uploader_onetrust_success( config.delete(db) -@mock.patch("fidesops.service.storage.storage_uploader_service.upload_to_onetrust") +@mock.patch("fidesops.ops.service.storage.storage_uploader_service.upload_to_onetrust") def test_uploader_onetrust_request_details_not_found( mock_upload_to_onetrust: Mock, db: Session, diff --git a/tests/ops/task/test_consolidate_query_matches.py b/tests/ops/task/test_consolidate_query_matches.py index 91d35a034a..aa5bc1240a 100644 --- a/tests/ops/task/test_consolidate_query_matches.py +++ b/tests/ops/task/test_consolidate_query_matches.py @@ -1,5 +1,5 @@ -from fidesops.graph.config import FieldPath -from fidesops.task.consolidate_query_matches import consolidate_query_matches +from fidesops.ops.graph.config import FieldPath +from fidesops.ops.task.consolidate_query_matches import consolidate_query_matches def test_consolidate_query_matches(): diff --git a/tests/ops/task/test_filter_element_match.py b/tests/ops/task/test_filter_element_match.py index 33c70eaa5f..947ea84e06 100644 --- a/tests/ops/task/test_filter_element_match.py +++ b/tests/ops/task/test_filter_element_match.py @@ -2,13 +2,13 @@ import pytest -from fidesops.graph.config import FieldPath -from fidesops.task.filter_element_match import ( +from fidesops.ops.graph.config import FieldPath +from fidesops.ops.task.filter_element_match import ( _expand_array_paths_to_preserve, _remove_paths_from_row, filter_element_match, ) -from fidesops.util.collection_util import FIDESOPS_DO_NOT_MASK_INDEX +from fidesops.ops.util.collection_util import FIDESOPS_DO_NOT_MASK_INDEX class TestFilterElementMatch: diff --git a/tests/ops/task/test_filter_results.py b/tests/ops/task/test_filter_results.py index 760891ab0e..10ef5f9210 100644 --- a/tests/ops/task/test_filter_results.py +++ b/tests/ops/task/test_filter_results.py @@ -3,8 +3,8 @@ from bson import ObjectId -from fidesops.graph.config import CollectionAddress, FieldPath -from fidesops.task.filter_results import ( +from fidesops.ops.graph.config import CollectionAddress, FieldPath +from fidesops.ops.task.filter_results import ( filter_data_categories, remove_empty_containers, select_and_save_field, diff --git a/tests/ops/task/test_graph_task.py b/tests/ops/task/test_graph_task.py index d0cc81ddfc..c7ad927792 100644 --- a/tests/ops/task/test_graph_task.py +++ b/tests/ops/task/test_graph_task.py @@ -2,12 +2,12 @@ import pytest from bson import ObjectId -from fidesops.graph.config import CollectionAddress, FieldPath -from fidesops.graph.graph import DatasetGraph -from fidesops.graph.traversal import Traversal -from fidesops.models.connectionconfig import ConnectionConfig, ConnectionType -from fidesops.models.policy import ActionType, Policy, Rule, RuleTarget -from fidesops.task.graph_task import ( +from fidesops.ops.graph.config import CollectionAddress, FieldPath +from fidesops.ops.graph.graph import DatasetGraph +from fidesops.ops.graph.traversal import Traversal +from fidesops.ops.models.connectionconfig import ConnectionConfig, ConnectionType +from fidesops.ops.models.policy import ActionType, Policy, Rule, RuleTarget +from fidesops.ops.task.graph_task import ( EMPTY_REQUEST, TaskResources, build_affected_field_logs, diff --git a/tests/ops/task/test_refine_target_path.py b/tests/ops/task/test_refine_target_path.py index d800a53811..2530ef1f43 100644 --- a/tests/ops/task/test_refine_target_path.py +++ b/tests/ops/task/test_refine_target_path.py @@ -1,11 +1,11 @@ -from fidesops.graph.config import FieldPath -from fidesops.task.refine_target_path import ( +from fidesops.ops.graph.config import FieldPath +from fidesops.ops.task.refine_target_path import ( _match_found, build_refined_target_paths, join_detailed_path, refine_target_path, ) -from fidesops.util.collection_util import FIDESOPS_DO_NOT_MASK_INDEX +from fidesops.ops.util.collection_util import FIDESOPS_DO_NOT_MASK_INDEX class TestRefineTargetPathToValue: diff --git a/tests/ops/task/test_task_resources.py b/tests/ops/task/test_task_resources.py index c8096c3182..22472815db 100644 --- a/tests/ops/task/test_task_resources.py +++ b/tests/ops/task/test_task_resources.py @@ -1,4 +1,4 @@ -from fidesops.task.task_resources import TaskResources +from fidesops.ops.task.task_resources import TaskResources class TestTaskResources: diff --git a/tests/ops/task/traversal_data.py b/tests/ops/task/traversal_data.py index 2d4a233bd9..6d353e33c7 100644 --- a/tests/ops/task/traversal_data.py +++ b/tests/ops/task/traversal_data.py @@ -1,6 +1,6 @@ from typing import Optional, Tuple -from fidesops.graph.config import ( +from fidesops.ops.graph.config import ( Collection, CollectionAddress, Dataset, @@ -8,7 +8,7 @@ ObjectField, ScalarField, ) -from fidesops.graph.data_type import ( +from fidesops.ops.graph.data_type import ( DataType, IntTypeConverter, NoOpTypeConverter, @@ -16,10 +16,10 @@ ObjectTypeConverter, StringTypeConverter, ) -from fidesops.graph.graph import DatasetGraph -from fidesops.graph.traversal import Traversal -from fidesops.models.connectionconfig import ConnectionConfig -from fidesops.schemas.shared_schemas import FidesOpsKey +from fidesops.ops.graph.graph import DatasetGraph +from fidesops.ops.graph.traversal import Traversal +from fidesops.ops.models.connectionconfig import ConnectionConfig +from fidesops.ops.schemas.shared_schemas import FidesOpsKey str_converter = DataType.string.value bool_converter = DataType.boolean.value diff --git a/tests/ops/tasks/test_celery.py b/tests/ops/tasks/test_celery.py index c6f0b28e77..8aa8f2c4a3 100644 --- a/tests/ops/tasks/test_celery.py +++ b/tests/ops/tasks/test_celery.py @@ -1,5 +1,5 @@ -from fidesops.core.config import config -from fidesops.tasks import _create_celery +from fidesops.ops.core.config import config +from fidesops.ops.tasks import _create_celery def test_create_task(celery_session_app, celery_session_worker): diff --git a/tests/ops/tasks/test_scheduled.py b/tests/ops/tasks/test_scheduled.py index 0483962eaa..fd19c11b68 100644 --- a/tests/ops/tasks/test_scheduled.py +++ b/tests/ops/tasks/test_scheduled.py @@ -1,13 +1,13 @@ from apscheduler.triggers.cron import CronTrigger from apscheduler.triggers.date import DateTrigger -from fidesops.models.privacy_request import PrivacyRequestStatus -from fidesops.schemas.storage.storage import StorageDetails -from fidesops.service.privacy_request.request_runner_service import ( +from fidesops.ops.models.privacy_request import PrivacyRequestStatus +from fidesops.ops.schemas.storage.storage import StorageDetails +from fidesops.ops.service.privacy_request.request_runner_service import ( initiate_paused_privacy_request_followup, ) -from fidesops.tasks.scheduled.scheduler import scheduler -from fidesops.tasks.scheduled.tasks import ( +from fidesops.ops.tasks.scheduled.scheduler import scheduler +from fidesops.ops.tasks.scheduled.tasks import ( ONETRUST_INTAKE_TASK, initiate_scheduled_request_intake, ) diff --git a/tests/ops/test_helpers/cache_secrets_helper.py b/tests/ops/test_helpers/cache_secrets_helper.py index 639ce65ce0..9216d91188 100644 --- a/tests/ops/test_helpers/cache_secrets_helper.py +++ b/tests/ops/test_helpers/cache_secrets_helper.py @@ -1,5 +1,9 @@ -from fidesops.schemas.masking.masking_secrets import MaskingSecretCache -from fidesops.util.cache import FidesopsRedis, get_cache, get_masking_secret_cache_key +from fidesops.ops.schemas.masking.masking_secrets import MaskingSecretCache +from fidesops.ops.util.cache import ( + FidesopsRedis, + get_cache, + get_masking_secret_cache_key, +) def cache_secret(masking_secret_cache: MaskingSecretCache, request_id: str) -> None: diff --git a/tests/ops/test_helpers/dataset_utils.py b/tests/ops/test_helpers/dataset_utils.py index bc6d256b50..81797c3574 100644 --- a/tests/ops/test_helpers/dataset_utils.py +++ b/tests/ops/test_helpers/dataset_utils.py @@ -3,12 +3,18 @@ import yaml -from fidesops.graph.config import Collection, Field, FieldPath, ObjectField, ScalarField -from fidesops.graph.data_type import DataType, get_data_type, to_data_type_string -from fidesops.models.connectionconfig import ConnectionConfig -from fidesops.models.datasetconfig import DatasetConfig, convert_dataset_to_graph -from fidesops.schemas.dataset import FidesopsDataset -from fidesops.util.collection_util import Row +from fidesops.ops.graph.config import ( + Collection, + Field, + FieldPath, + ObjectField, + ScalarField, +) +from fidesops.ops.graph.data_type import DataType, get_data_type, to_data_type_string +from fidesops.ops.models.connectionconfig import ConnectionConfig +from fidesops.ops.models.datasetconfig import DatasetConfig, convert_dataset_to_graph +from fidesops.ops.schemas.dataset import FidesopsDataset +from fidesops.ops.util.collection_util import Row SAAS_DATASET_DIRECTORY = "data/saas/dataset/" diff --git a/tests/ops/test_helpers/vault_client.py b/tests/ops/test_helpers/vault_client.py index da14453c03..391e69bda2 100644 --- a/tests/ops/test_helpers/vault_client.py +++ b/tests/ops/test_helpers/vault_client.py @@ -4,7 +4,7 @@ from hvac import Client -from fidesops.common_exceptions import FidesopsException +from fidesops.ops.common_exceptions import FidesopsException logger = logging.getLogger(__name__) diff --git a/tests/ops/util/encryption/test_aes_gcm_encryption_scheme.py b/tests/ops/util/encryption/test_aes_gcm_encryption_scheme.py index 7109f9553e..23e9733aa1 100644 --- a/tests/ops/util/encryption/test_aes_gcm_encryption_scheme.py +++ b/tests/ops/util/encryption/test_aes_gcm_encryption_scheme.py @@ -1,7 +1,7 @@ import pytest from fideslib.cryptography import cryptographic_util -from fidesops.util.encryption.aes_gcm_encryption_scheme import ( +from fidesops.ops.util.encryption.aes_gcm_encryption_scheme import ( decrypt, encrypt_verify_secret_length, ) diff --git a/tests/ops/util/encryption/test_hmac_encryption_scheme.py b/tests/ops/util/encryption/test_hmac_encryption_scheme.py index 78edc9237a..40759044d5 100644 --- a/tests/ops/util/encryption/test_hmac_encryption_scheme.py +++ b/tests/ops/util/encryption/test_hmac_encryption_scheme.py @@ -1,5 +1,5 @@ -from fidesops.schemas.masking.masking_configuration import HmacMaskingConfiguration -from fidesops.util.encryption.hmac_encryption_scheme import ( +from fidesops.ops.schemas.masking.masking_configuration import HmacMaskingConfiguration +from fidesops.ops.util.encryption.hmac_encryption_scheme import ( hmac_encrypt_return_bytes, hmac_encrypt_return_str, ) diff --git a/tests/ops/util/encryption/test_secrets_util.py b/tests/ops/util/encryption/test_secrets_util.py index 606b926519..0ce0acdbbe 100644 --- a/tests/ops/util/encryption/test_secrets_util.py +++ b/tests/ops/util/encryption/test_secrets_util.py @@ -1,18 +1,18 @@ from typing import Dict, List -from fidesops.schemas.masking.masking_secrets import ( +from fidesops.ops.schemas.masking.masking_secrets import ( MaskingSecretCache, MaskingSecretMeta, SecretType, ) -from fidesops.service.masking.strategy.masking_strategy_aes_encrypt import ( +from fidesops.ops.service.masking.strategy.masking_strategy_aes_encrypt import ( AES_ENCRYPT_STRATEGY_NAME, ) -from fidesops.service.masking.strategy.masking_strategy_hmac import ( +from fidesops.ops.service.masking.strategy.masking_strategy_hmac import ( HMAC_STRATEGY_NAME, HmacMaskingStrategy, ) -from fidesops.util.encryption.secrets_util import SecretsUtil +from fidesops.ops.util.encryption.secrets_util import SecretsUtil from ...test_helpers.cache_secrets_helper import cache_secret, clear_cache_secrets diff --git a/tests/ops/util/test_api_router.py b/tests/ops/util/test_api_router.py index 151007eb79..9b41b2daeb 100644 --- a/tests/ops/util/test_api_router.py +++ b/tests/ops/util/test_api_router.py @@ -3,8 +3,8 @@ from starlette.status import HTTP_200_OK, HTTP_404_NOT_FOUND from starlette.testclient import TestClient -from fidesops.api.v1.scope_registry import PRIVACY_REQUEST_READ -from fidesops.api.v1.urn_registry import PRIVACY_REQUESTS, V1_URL_PREFIX +from fidesops.ops.api.v1.scope_registry import PRIVACY_REQUEST_READ +from fidesops.ops.api.v1.urn_registry import PRIVACY_REQUESTS, V1_URL_PREFIX class TestApiRouter: diff --git a/tests/ops/util/test_cache.py b/tests/ops/util/test_cache.py index d3441924c5..c25b1b4d3e 100644 --- a/tests/ops/util/test_cache.py +++ b/tests/ops/util/test_cache.py @@ -1,8 +1,8 @@ import random from typing import Any, List -from fidesops.core.config import config -from fidesops.util.cache import FidesopsRedis +from fidesops.ops.core.config import config +from fidesops.ops.util.cache import FidesopsRedis from ..fixtures.application_fixtures import faker diff --git a/tests/ops/util/test_collection_util.py b/tests/ops/util/test_collection_util.py index e86923c351..b7908246f0 100644 --- a/tests/ops/util/test_collection_util.py +++ b/tests/ops/util/test_collection_util.py @@ -1,6 +1,6 @@ from typing import Dict, List -from fidesops.util.collection_util import ( +from fidesops.ops.util.collection_util import ( append, filter_nonempty_values, merge_dicts, diff --git a/tests/ops/util/test_dataset_yaml.py b/tests/ops/util/test_dataset_yaml.py index 62e53116c7..988e4c5268 100644 --- a/tests/ops/util/test_dataset_yaml.py +++ b/tests/ops/util/test_dataset_yaml.py @@ -4,16 +4,16 @@ import yaml from pydantic import ValidationError -from fidesops.graph.config import ( +from fidesops.ops.graph.config import ( CollectionAddress, FieldAddress, FieldPath, ObjectField, ScalarField, ) -from fidesops.graph.graph import DatasetGraph, Edge -from fidesops.models.datasetconfig import convert_dataset_to_graph -from fidesops.schemas.dataset import FidesopsDataset +from fidesops.ops.graph.graph import DatasetGraph, Edge +from fidesops.ops.models.datasetconfig import convert_dataset_to_graph +from fidesops.ops.schemas.dataset import FidesopsDataset from ..graph.graph_test_util import field diff --git a/tests/ops/util/test_jwt_util.py b/tests/ops/util/test_jwt_util.py index b5d4c6764c..c62196a350 100644 --- a/tests/ops/util/test_jwt_util.py +++ b/tests/ops/util/test_jwt_util.py @@ -9,7 +9,7 @@ from fideslib.oauth.jwt import generate_jwe from fideslib.oauth.oauth_util import extract_payload, is_token_expired -from fidesops.core.config import config +from fidesops.ops.core.config import config def test_jwe_create_and_extract() -> None: diff --git a/tests/ops/util/test_logger.py b/tests/ops/util/test_logger.py index 2466d1bf30..ca80c44d0b 100644 --- a/tests/ops/util/test_logger.py +++ b/tests/ops/util/test_logger.py @@ -2,8 +2,8 @@ import pytest -from fidesops.util import logger -from fidesops.util.logger import MASKED, NotPii +from fidesops.ops.util import logger +from fidesops.ops.util.logger import MASKED, NotPii @pytest.fixture(scope="function") diff --git a/tests/ops/util/test_matching_queue.py b/tests/ops/util/test_matching_queue.py index a46a45ca17..f33f2de835 100644 --- a/tests/ops/util/test_matching_queue.py +++ b/tests/ops/util/test_matching_queue.py @@ -1,4 +1,4 @@ -from fidesops.util.matching_queue import * +from fidesops.ops.util.matching_queue import * def test_queue() -> None: diff --git a/tests/ops/util/test_saas_util.py b/tests/ops/util/test_saas_util.py index 0c6fc80f4a..811781e8ab 100644 --- a/tests/ops/util/test_saas_util.py +++ b/tests/ops/util/test_saas_util.py @@ -1,7 +1,7 @@ import pytest -from fidesops.common_exceptions import FidesopsException -from fidesops.graph.config import ( +from fidesops.ops.common_exceptions import FidesopsException +from fidesops.ops.graph.config import ( Collection, Dataset, FieldAddress, @@ -9,7 +9,11 @@ ObjectField, ScalarField, ) -from fidesops.util.saas_util import assign_placeholders, merge_datasets, unflatten_dict +from fidesops.ops.util.saas_util import ( + assign_placeholders, + merge_datasets, + unflatten_dict, +) class TestMergeDatasets: