From 51c33cbbaa245d261307fd49dfa247b3d7f83056 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Sun, 7 Aug 2022 11:13:20 -0700 Subject: [PATCH 01/17] Starting point for SaaS connector templates --- data/saas/config/mailchimp_config.yml | 8 +- data/saas/dataset/mailchimp_dataset.yml | 11 +- data/saas/icon/adobe.svg | 5 + data/saas/icon/default.svg | 4 + data/saas/icon/hubspot.svg | 4 + data/saas/icon/mailchimp.svg | 10 ++ data/saas/icon/outreach.svg | 4 + data/saas/icon/salesforce.svg | 4 + data/saas/icon/segment.svg | 9 ++ data/saas/icon/sentry.svg | 4 + data/saas/icon/stripe.svg | 31 +++++ data/saas/icon/zendesk.svg | 7 ++ saas_connector_registry.toml | 4 + .../api/v1/endpoints/saas_config_endpoints.py | 58 ++++++++- src/fidesops/api/v1/scope_registry.py | 2 + src/fidesops/api/v1/urn_registry.py | 1 + src/fidesops/main.py | 10 ++ .../saas/connector_registry_service.py | 112 ++++++++++++++++++ src/fidesops/util/saas_util.py | 12 ++ 19 files changed, 290 insertions(+), 10 deletions(-) create mode 100644 data/saas/icon/adobe.svg create mode 100644 data/saas/icon/default.svg create mode 100644 data/saas/icon/hubspot.svg create mode 100644 data/saas/icon/mailchimp.svg create mode 100644 data/saas/icon/outreach.svg create mode 100644 data/saas/icon/salesforce.svg create mode 100644 data/saas/icon/segment.svg create mode 100644 data/saas/icon/sentry.svg create mode 100644 data/saas/icon/stripe.svg create mode 100644 data/saas/icon/zendesk.svg create mode 100644 saas_connector_registry.toml create mode 100644 src/fidesops/service/connectors/saas/connector_registry_service.py diff --git a/data/saas/config/mailchimp_config.yml b/data/saas/config/mailchimp_config.yml index e73d0f7b5..20370d047 100644 --- a/data/saas/config/mailchimp_config.yml +++ b/data/saas/config/mailchimp_config.yml @@ -1,5 +1,5 @@ saas_config: - fides_key: mailchimp_connector_example + fides_key: instance_fides_key name: Mailchimp SaaS Config type: mailchimp description: A sample schema representing the Mailchimp connector for Fidesops @@ -32,7 +32,7 @@ saas_config: param_values: - name: conversation_id references: - - dataset: mailchimp_connector_example + - dataset: instance_fides_key field: conversations.id direction: from data_path: conversation_messages @@ -80,12 +80,12 @@ saas_config: param_values: - name: list_id references: - - dataset: mailchimp_connector_example + - dataset: instance_fides_key field: member.list_id direction: from - name: subscriber_hash references: - - dataset: mailchimp_connector_example + - dataset: instance_fides_key field: member.id direction: from body: | diff --git a/data/saas/dataset/mailchimp_dataset.yml b/data/saas/dataset/mailchimp_dataset.yml index 80293f57a..6a6256733 100644 --- a/data/saas/dataset/mailchimp_dataset.yml +++ b/data/saas/dataset/mailchimp_dataset.yml @@ -1,5 +1,5 @@ dataset: - - fides_key: mailchimp_connector_example + - fides_key: instance_fides_key name: Mailchimp Dataset description: A sample dataset representing the Mailchimp connector for Fidesops collections: @@ -90,15 +90,18 @@ dataset: fidesops_meta: data_type: string - name: zip - data_categories: [user.provided.identifiable.contact.postal_code] + data_categories: + [user.provided.identifiable.contact.postal_code] fidesops_meta: data_type: string - name: country - data_categories: [user.provided.identifiable.contact.country] + data_categories: + [user.provided.identifiable.contact.country] fidesops_meta: data_type: string - name: PHONE - data_categories: [user.provided.identifiable.contact.phone_number] + data_categories: + [user.provided.identifiable.contact.phone_number] fidesops_meta: data_type: string - name: BIRTHDAY diff --git a/data/saas/icon/adobe.svg b/data/saas/icon/adobe.svg new file mode 100644 index 000000000..fadef181c --- /dev/null +++ b/data/saas/icon/adobe.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/data/saas/icon/default.svg b/data/saas/icon/default.svg new file mode 100644 index 000000000..32e4384db --- /dev/null +++ b/data/saas/icon/default.svg @@ -0,0 +1,4 @@ + + + + diff --git a/data/saas/icon/hubspot.svg b/data/saas/icon/hubspot.svg new file mode 100644 index 000000000..e7afe860e --- /dev/null +++ b/data/saas/icon/hubspot.svg @@ -0,0 +1,4 @@ + + + + diff --git a/data/saas/icon/mailchimp.svg b/data/saas/icon/mailchimp.svg new file mode 100644 index 000000000..28edc6cf3 --- /dev/null +++ b/data/saas/icon/mailchimp.svg @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/data/saas/icon/outreach.svg b/data/saas/icon/outreach.svg new file mode 100644 index 000000000..769fbb2c5 --- /dev/null +++ b/data/saas/icon/outreach.svg @@ -0,0 +1,4 @@ + + + + diff --git a/data/saas/icon/salesforce.svg b/data/saas/icon/salesforce.svg new file mode 100644 index 000000000..d10da329d --- /dev/null +++ b/data/saas/icon/salesforce.svg @@ -0,0 +1,4 @@ + + + + diff --git a/data/saas/icon/segment.svg b/data/saas/icon/segment.svg new file mode 100644 index 000000000..a84f3677d --- /dev/null +++ b/data/saas/icon/segment.svg @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/data/saas/icon/sentry.svg b/data/saas/icon/sentry.svg new file mode 100644 index 000000000..e2e3be20e --- /dev/null +++ b/data/saas/icon/sentry.svg @@ -0,0 +1,4 @@ + + + + diff --git a/data/saas/icon/stripe.svg b/data/saas/icon/stripe.svg new file mode 100644 index 000000000..e41146fff --- /dev/null +++ b/data/saas/icon/stripe.svg @@ -0,0 +1,31 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/data/saas/icon/zendesk.svg b/data/saas/icon/zendesk.svg new file mode 100644 index 000000000..198243423 --- /dev/null +++ b/data/saas/icon/zendesk.svg @@ -0,0 +1,7 @@ + + + + + + + diff --git a/saas_connector_registry.toml b/saas_connector_registry.toml new file mode 100644 index 000000000..a5efff4fb --- /dev/null +++ b/saas_connector_registry.toml @@ -0,0 +1,4 @@ +[mailchimp] +config = "data/saas/config/mailchimp_config.yml" +dataset = "data/saas/dataset/mailchimp_dataset.yml" +icon = "data/saas/icon/mailchimp.svg" \ No newline at end of file diff --git a/src/fidesops/api/v1/endpoints/saas_config_endpoints.py b/src/fidesops/api/v1/endpoints/saas_config_endpoints.py index f97bbafad..9b7bace86 100644 --- a/src/fidesops/api/v1/endpoints/saas_config_endpoints.py +++ b/src/fidesops/api/v1/endpoints/saas_config_endpoints.py @@ -1,8 +1,10 @@ +import json import logging -from typing import Optional +from typing import Any, Dict, Optional -from fastapi import Depends, HTTPException +from fastapi import Depends, HTTPException, Request from fastapi.params import Security +from fastapi.responses import JSONResponse from sqlalchemy.orm import Session from starlette.status import ( HTTP_200_OK, @@ -15,12 +17,14 @@ from fidesops.api import deps from fidesops.api.v1.scope_registry import ( CONNECTION_AUTHORIZE, + CONNECTION_INSTANTIATE, SAAS_CONFIG_CREATE_OR_UPDATE, SAAS_CONFIG_DELETE, SAAS_CONFIG_READ, ) from fidesops.api.v1.urn_registry import ( AUTHORIZE, + INSTANTIATE, SAAS_CONFIG, SAAS_CONFIG_VALIDATE, V1_URL_PREFIX, @@ -38,8 +42,15 @@ from fidesops.service.authentication.authentication_strategy_oauth2 import ( OAuth2AuthenticationStrategy, ) +from fidesops.service.connectors.saas.connector_registry_service import ( + ConnectorTemplate, + connector_types, + get_connector_template, + instantiate_connector_template, +) from fidesops.util.api_router import APIRouter from fidesops.util.oauth_util import verify_oauth_client +from fidesops.util.saas_util import load_yaml_as_string router = APIRouter(tags=["SaaS Configs"], prefix=V1_URL_PREFIX) logger = logging.getLogger(__name__) @@ -243,3 +254,46 @@ def authorize_connection( return auth_strategy.get_authorization_url(db, connection_config) except FidesopsException as exc: raise HTTPException(status_code=HTTP_400_BAD_REQUEST, detail=str(exc)) + + +@router.post( + INSTANTIATE, + dependencies=[Security(verify_oauth_client, scopes=[CONNECTION_INSTANTIATE])], + response_model=str, +) +async def instantiate_connection( + saas_connector_type: str, + request: Request, + db: Session = Depends(deps.get_db), +) -> JSONResponse: + """ + Looks up the connector type in the SaaS connector registry and, if all required + fields are provided, persists the associated config and dataset to the database. + """ + + # verify connector type is registered + if saas_connector_type not in connector_types(): + raise HTTPException( + status_code=HTTP_400_BAD_REQUEST, + detail=f"SaaS connector type '{saas_connector_type}' is not registered.", + ) + + # verify instance_key is provided + payload = await request.json() + instance_key = payload.pop("instance_key", None) + if not instance_key: + raise HTTPException( + status_code=HTTP_400_BAD_REQUEST, + detail=f"The instance_key was not specified.", + ) + + # verify instance_key is not already in use + + # let the service do the heavy lifting + instantiate_connector_template(saas_connector_type, instance_key, payload) + + return JSONResponse( + content={ + "message": f"A new {saas_connector_type} connector with fides_key '{instance_key}' was successfully instantiated" + } + ) diff --git a/src/fidesops/api/v1/scope_registry.py b/src/fidesops/api/v1/scope_registry.py index 91a4e4b7f..3c72bd33d 100644 --- a/src/fidesops/api/v1/scope_registry.py +++ b/src/fidesops/api/v1/scope_registry.py @@ -15,6 +15,7 @@ CONNECTION_READ = "connection:read" CONNECTION_DELETE = "connection:delete" CONNECTION_AUTHORIZE = "connection:authorize" +CONNECTION_INSTANTIATE = "connection:instantiate" PRIVACY_REQUEST_READ = "privacy-request:read" PRIVACY_REQUEST_DELETE = "privacy-request:delete" @@ -67,6 +68,7 @@ CONNECTION_CREATE_OR_UPDATE, CONNECTION_DELETE, CONNECTION_AUTHORIZE, + CONNECTION_INSTANTIATE, CONNECTION_TYPE_READ, DATASET_CREATE_OR_UPDATE, DATASET_DELETE, diff --git a/src/fidesops/api/v1/urn_registry.py b/src/fidesops/api/v1/urn_registry.py index f0a2f84d8..484b1b28d 100644 --- a/src/fidesops/api/v1/urn_registry.py +++ b/src/fidesops/api/v1/urn_registry.py @@ -83,6 +83,7 @@ # SaaS Config URLs SAAS_CONFIG_VALIDATE = CONNECTION_BY_KEY + "/validate_saas_config" SAAS_CONFIG = CONNECTION_BY_KEY + "/saas_config" +INSTANTIATE = "/connection/instantiate/{saas_connector_type}" # User URLs diff --git a/src/fidesops/main.py b/src/fidesops/main.py index f4f939450..0b1762311 100644 --- a/src/fidesops/main.py +++ b/src/fidesops/main.py @@ -30,6 +30,10 @@ from fidesops.core.config import config from fidesops.db.database import init_db from fidesops.schemas.analytics import Event, ExtraData +from fidesops.service.connectors.saas.connector_registry_service import ( + load_registry, + update_connector_instances, +) from fidesops.tasks.scheduled.scheduler import scheduler from fidesops.tasks.scheduled.tasks import initiate_scheduled_request_intake from fidesops.util.cache import get_cache @@ -177,6 +181,9 @@ def start_webserver() -> None: ) config.log_all_config_values() + logger.info("Validating SaaS connector templates...") + load_registry("saas_connector_registry.toml") + if config.database.enabled: logger.info("Running any pending DB migrations...") try: @@ -185,6 +192,9 @@ def start_webserver() -> None: logger.error(f"Connection to database failed: {error}") return + logger.info("Updating SaaS connector instances...") + update_connector_instances() + if config.redis.enabled: logger.info("Running Redis connection test...") try: diff --git a/src/fidesops/service/connectors/saas/connector_registry_service.py b/src/fidesops/service/connectors/saas/connector_registry_service.py new file mode 100644 index 000000000..6bfbcc7f6 --- /dev/null +++ b/src/fidesops/service/connectors/saas/connector_registry_service.py @@ -0,0 +1,112 @@ +from __future__ import annotations + +from asyncio.log import logger +from os.path import exists +from typing import Any, Dict, List, Optional + +from fideslib.core.config import load_toml +from pydantic import BaseModel, validator + +from fidesops.schemas.dataset import FidesopsDataset +from fidesops.schemas.saas.saas_config import SaaSConfig +from fidesops.util.saas_util import load_config, load_dataset, load_yaml_as_string + +_registry: ConnectorRegistry = {} + + +class ConnectorTemplate(BaseModel): + """ + A collection of references to artifacts that make up + a complete SaaS connector (SaaS config, dataset, etc.) + """ + + config: str + dataset: str + icon: str + + @validator("config") + def validate_config(cls, config: str) -> str: + """Validates the config at the given path""" + SaaSConfig(**load_config(config)) + return config + + @validator("dataset") + def validate_dataset(cls, dataset: str) -> str: + """Validates the dataset at the given path""" + FidesopsDataset(**load_dataset(dataset)[0]) + return dataset + + @validator("icon") + def validate_icon(cls, icon: str) -> str: + """Validates the icon at the given path""" + if not exists(icon): + raise ValueError(f"Icon file {icon} was not found") + return icon + + +class ConnectorRegistry(BaseModel): + """A map of SaaS connector templates""" + + __root__: Dict[str, ConnectorTemplate] + + def keys(self): + return list(self.__root__) + + def get(self, item): + try: + return self.__root__[item] + except: + return None + + +def connector_types() -> List[str]: + """List of registered SaaS connector types""" + return _registry.keys() + + +def get_connector_template(connector_type: str) -> Optional[ConnectorTemplate]: + """ + Returns an object containing the references to the various SaaS connector artifacts + """ + return _registry.get(connector_type) + + +def instantiate_connector_template( + connector_type: str, instance_key: str, secrets: Dict[str, Any] +) -> None: + """Creates a SaaS connection config and associates the SaaS config and dataset with the new connection""" + + template = _registry.get(connector_type) + # validate secrets + # create connection config + + # add saas config + saas_config = load_yaml_as_string(template.config).replace( + "instance_fides_key", instance_key + ) + logger.info(saas_config) + + # add dataset + saas_dataset = load_yaml_as_string(template.dataset).replace( + "instance_fides_key", instance_key + ) + logger.info(saas_dataset) + + +def update_connector_instances() -> None: + """ + Updates every SaaS connection config and dataset with the latest version + defined in the SaaS connector registry + """ + # get all saas connection configs + # get version number from SaaS config (figure out versioning) + # if registry version is newer + # update saas config + # update dataset + pass + + +def load_registry(config_file: str) -> None: + """Loads a SaaS connector registry from the given config file.""" + global _registry + _registry = ConnectorRegistry.parse_obj(load_toml([config_file])) diff --git a/src/fidesops/util/saas_util.py b/src/fidesops/util/saas_util.py index 941c7cb55..c0b978e4a 100644 --- a/src/fidesops/util/saas_util.py +++ b/src/fidesops/util/saas_util.py @@ -21,6 +21,12 @@ FIDESOPS_GROUPED_INPUTS = "fidesops_grouped_inputs" +def load_yaml_as_string(filename: str) -> str: + yaml_file = load_file([filename]) + with open(yaml_file, "r") as file: + return file.read() + + def load_config(filename: str) -> Dict: """Loads the saas config from the yaml file""" yaml_file = load_file([filename]) @@ -28,6 +34,12 @@ def load_config(filename: str) -> Dict: return yaml.safe_load(file).get("saas_config", []) +def load_dataset(filename: str) -> Dict: + yaml_file = load_file([filename]) + with open(yaml_file, "r") as file: + return yaml.safe_load(file).get("dataset", []) + + def merge_fields(target: Field, source: Field) -> Field: """Replaces source references and identities if they are available from the target""" if source.references is not None: From 36c98a050cb63e934d87d29ec78d67dbb6bb61e3 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Mon, 15 Aug 2022 11:40:01 -0500 Subject: [PATCH 02/17] Fix imports from restructuring. --- src/fidesops/main.py | 5 ++++- .../ops/api/v1/endpoints/saas_config_endpoints.py | 6 ++++-- .../service/connectors/saas/connector_registry_service.py | 8 ++++---- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/fidesops/main.py b/src/fidesops/main.py index ab4330303..7adc5ad4d 100644 --- a/src/fidesops/main.py +++ b/src/fidesops/main.py @@ -33,12 +33,15 @@ from fidesops.ops.core.config import config from fidesops.ops.db.database import init_db from fidesops.ops.schemas.analytics import Event, ExtraData +from fidesops.ops.service.connectors.saas.connector_registry_service import ( + load_registry, + update_connector_instances, +) from fidesops.ops.tasks.scheduled.scheduler import scheduler from fidesops.ops.tasks.scheduled.tasks import initiate_scheduled_request_intake from fidesops.ops.util.cache import get_cache from fidesops.ops.util.logger import get_fides_log_record_factory from fidesops.ops.util.oauth_util import get_db, verify_oauth_client -from fidesops.ops.service.connectors.saas.connector_registry_service import load_registry, update_connector_instances logging.basicConfig(level=config.security.log_level) logging.setLogRecordFactory(get_fides_log_record_factory()) diff --git a/src/fidesops/ops/api/v1/endpoints/saas_config_endpoints.py b/src/fidesops/ops/api/v1/endpoints/saas_config_endpoints.py index 383d06968..86e8501af 100644 --- a/src/fidesops/ops/api/v1/endpoints/saas_config_endpoints.py +++ b/src/fidesops/ops/api/v1/endpoints/saas_config_endpoints.py @@ -43,10 +43,12 @@ from fidesops.ops.service.authentication.authentication_strategy_oauth2 import ( OAuth2AuthenticationStrategy, ) +from fidesops.ops.service.connectors.saas.connector_registry_service import ( + connector_types, + instantiate_connector_template, +) from fidesops.ops.util.api_router import APIRouter from fidesops.ops.util.oauth_util import verify_oauth_client -from fidesops.ops.service.connectors.saas.connector_registry_service import connector_types, \ - instantiate_connector_template router = APIRouter(tags=["SaaS Configs"], prefix=V1_URL_PREFIX) logger = logging.getLogger(__name__) diff --git a/src/fidesops/ops/service/connectors/saas/connector_registry_service.py b/src/fidesops/ops/service/connectors/saas/connector_registry_service.py index 9969cc2c7..3018040f9 100644 --- a/src/fidesops/ops/service/connectors/saas/connector_registry_service.py +++ b/src/fidesops/ops/service/connectors/saas/connector_registry_service.py @@ -7,9 +7,9 @@ from fideslib.core.config import load_toml from pydantic import BaseModel, validator -from fidesops.schemas.dataset import FidesopsDataset -from fidesops.schemas.saas.saas_config import SaaSConfig -from fidesops.util.saas_util import load_config, load_dataset, load_yaml_as_string +from fidesops.ops.schemas.dataset import FidesopsDataset +from fidesops.ops.schemas.saas.saas_config import SaaSConfig +from fidesops.ops.util.saas_util import load_config, load_dataset, load_yaml_as_string _registry: ConnectorRegistry = {} @@ -88,7 +88,7 @@ def instantiate_connector_template( # add dataset saas_dataset = load_yaml_as_string(template.dataset).replace( - "instance_fides_key", instance_keyx + "instance_fides_key", instance_key ) logger.info(saas_dataset) From e69c94a643f1a325b4cf78db038727f5f0f9b30f Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Mon, 15 Aug 2022 17:06:53 -0500 Subject: [PATCH 03/17] Get happy path working for instantiate connector from template endpoint. --- src/fidesops/main.py | 3 +- .../api/v1/endpoints/saas_config_endpoints.py | 66 ++++++---- src/fidesops/ops/api/v1/urn_registry.py | 2 +- .../connection_config.py | 11 ++ .../saas/connector_registry_service.py | 123 +++++++++++------- src/fidesops/ops/util/saas_util.py | 20 +++ .../test_connection_template_endpoints.py | 92 ++++++++++++- .../test_connector_registry_service.py | 21 +++ 8 files changed, 263 insertions(+), 75 deletions(-) create mode 100644 tests/ops/service/connectors/test_connector_registry_service.py diff --git a/src/fidesops/main.py b/src/fidesops/main.py index 7adc5ad4d..6a4f82e5d 100644 --- a/src/fidesops/main.py +++ b/src/fidesops/main.py @@ -35,6 +35,7 @@ from fidesops.ops.schemas.analytics import Event, ExtraData from fidesops.ops.service.connectors.saas.connector_registry_service import ( load_registry, + registry_file, update_connector_instances, ) from fidesops.ops.tasks.scheduled.scheduler import scheduler @@ -185,7 +186,7 @@ def start_webserver() -> None: config.log_all_config_values() logger.info("Validating SaaS connector templates...") - load_registry("saas_connector_registry.toml") + load_registry(registry_file) if config.database.enabled: logger.info("Running any pending DB migrations...") diff --git a/src/fidesops/ops/api/v1/endpoints/saas_config_endpoints.py b/src/fidesops/ops/api/v1/endpoints/saas_config_endpoints.py index 86e8501af..cef167c8a 100644 --- a/src/fidesops/ops/api/v1/endpoints/saas_config_endpoints.py +++ b/src/fidesops/ops/api/v1/endpoints/saas_config_endpoints.py @@ -1,9 +1,8 @@ import logging from typing import Optional -from fastapi import Depends, HTTPException, Request +from fastapi import Depends, HTTPException from fastapi.params import Security -from fastapi.responses import JSONResponse from sqlalchemy.orm import Session from starlette.status import ( HTTP_200_OK, @@ -14,6 +13,7 @@ ) from fidesops.ops.api import deps +from fidesops.ops.api.v1.endpoints.connection_endpoints import validate_secrets from fidesops.ops.api.v1.scope_registry import ( CONNECTION_AUTHORIZE, CONNECTION_INSTANTIATE, @@ -23,14 +23,18 @@ ) from fidesops.ops.api.v1.urn_registry import ( AUTHORIZE, - INSTANTIATE, SAAS_CONFIG, SAAS_CONFIG_VALIDATE, + SAAS_CONNECTOR_FROM_TEMPLATE, V1_URL_PREFIX, ) from fidesops.ops.common_exceptions import FidesopsException from fidesops.ops.models.connectionconfig import ConnectionConfig, ConnectionType from fidesops.ops.models.datasetconfig import DatasetConfig +from fidesops.ops.schemas.connection_configuration.connection_config import ( + SaasConnectionTemplateValues, +) +from fidesops.ops.schemas.dataset import FidesopsDataset from fidesops.ops.schemas.saas.saas_config import ( SaaSConfig, SaaSConfigValidationDetails, @@ -44,8 +48,12 @@ OAuth2AuthenticationStrategy, ) from fidesops.ops.service.connectors.saas.connector_registry_service import ( - connector_types, - instantiate_connector_template, + ConnectorRegistry, + ConnectorTemplate, + create_connection_config_from_template, + create_dataset_config_from_template, + load_registry, + registry_file, ) from fidesops.ops.util.api_router import APIRouter from fidesops.ops.util.oauth_util import verify_oauth_client @@ -255,43 +263,51 @@ def authorize_connection( @router.post( - INSTANTIATE, + SAAS_CONNECTOR_FROM_TEMPLATE, dependencies=[Security(verify_oauth_client, scopes=[CONNECTION_INSTANTIATE])], - response_model=str, + response_model=FidesopsDataset, ) -async def instantiate_connection( +def instantiate_connection( saas_connector_type: str, - request: Request, + template_values: SaasConnectionTemplateValues, db: Session = Depends(deps.get_db), -) -> JSONResponse: +) -> FidesopsDataset: """ Looks up the connector type in the SaaS connector registry and, if all required - fields are provided, persists the associated config and dataset to the database. + fields are provided, persists the associated connection config and dataset to the database. """ # verify connector type is registered - if saas_connector_type not in connector_types(): + registry: ConnectorRegistry = load_registry(registry_file) + connector_template: Optional[ConnectorTemplate] = registry.get_connector_template( + saas_connector_type + ) + if not connector_template: raise HTTPException( status_code=HTTP_400_BAD_REQUEST, detail=f"SaaS connector type '{saas_connector_type}' is not registered.", ) - # verify instance_key is provided - payload = await request.json() - instance_key = payload.pop("instance_key", None) - if not instance_key: + if DatasetConfig.filter( + db=db, + conditions=((DatasetConfig.fides_key == template_values.instance_key)), + ).count(): raise HTTPException( status_code=HTTP_400_BAD_REQUEST, - detail=f"The instance_key was not specified.", + detail=f"SaaS connector instance key '{template_values.instance_key}' already exists.", ) - # verify instance_key is not already in use - - # let the service do the heavy lifting - instantiate_connector_template(saas_connector_type, instance_key, payload) + connection_config: ConnectionConfig = create_connection_config_from_template( + db, connector_template, template_values + ) + connection_config.secrets = validate_secrets( + template_values.secrets, connection_config + ).dict() + connection_config.save(db=db) - return JSONResponse( - content={ - "message": f"A new {saas_connector_type} connector with fides_key '{instance_key}' was successfully instantiated" - } + dataset_config: DatasetConfig = create_dataset_config_from_template( + db, connection_config, connector_template, template_values ) + # TODO Roll back if there's an error. + + return dataset_config.dataset diff --git a/src/fidesops/ops/api/v1/urn_registry.py b/src/fidesops/ops/api/v1/urn_registry.py index 484b1b28d..88b02d3d6 100644 --- a/src/fidesops/ops/api/v1/urn_registry.py +++ b/src/fidesops/ops/api/v1/urn_registry.py @@ -83,7 +83,7 @@ # SaaS Config URLs SAAS_CONFIG_VALIDATE = CONNECTION_BY_KEY + "/validate_saas_config" SAAS_CONFIG = CONNECTION_BY_KEY + "/saas_config" -INSTANTIATE = "/connection/instantiate/{saas_connector_type}" +SAAS_CONNECTOR_FROM_TEMPLATE = "/connection/instantiate/{saas_connector_type}" # User URLs diff --git a/src/fidesops/ops/schemas/connection_configuration/connection_config.py b/src/fidesops/ops/schemas/connection_configuration/connection_config.py index e56c62123..0de31357f 100644 --- a/src/fidesops/ops/schemas/connection_configuration/connection_config.py +++ b/src/fidesops/ops/schemas/connection_configuration/connection_config.py @@ -6,6 +6,7 @@ from fidesops.ops.models.connectionconfig import AccessLevel, ConnectionType from fidesops.ops.schemas.api import BulkResponse, BulkUpdateFailed +from fidesops.ops.schemas.connection_configuration import connection_secrets_schemas from fidesops.ops.schemas.saas.saas_config import SaaSConfigBase, SaaSType from fidesops.ops.schemas.shared_schemas import FidesOpsKey @@ -98,3 +99,13 @@ class BulkPutConnectionConfiguration(BulkResponse): succeeded: List[ConnectionConfigurationResponse] failed: List[BulkUpdateFailed] + + +class SaasConnectionTemplateValues(BaseModel): + """Schema with values to create both a Saas ConnectionConfig and DatasetConfig from a template""" + + name: str # For ConnectionConfig + key: Optional[FidesOpsKey] # For ConnectionConfig + description: Optional[str] # For ConnectionConfig + secrets: connection_secrets_schemas # For ConnectionConfig + instance_key: FidesOpsKey # For DatasetConfig.fides_key diff --git a/src/fidesops/ops/service/connectors/saas/connector_registry_service.py b/src/fidesops/ops/service/connectors/saas/connector_registry_service.py index 3018040f9..00831444c 100644 --- a/src/fidesops/ops/service/connectors/saas/connector_registry_service.py +++ b/src/fidesops/ops/service/connectors/saas/connector_registry_service.py @@ -1,22 +1,37 @@ from __future__ import annotations -from asyncio.log import logger from os.path import exists -from typing import Any, Dict, List, Optional +from typing import Dict, List, Optional from fideslib.core.config import load_toml from pydantic import BaseModel, validator - +from sqlalchemy.orm import Session + +from fidesops.ops.models.connectionconfig import ( + AccessLevel, + ConnectionConfig, + ConnectionType, +) +from fidesops.ops.models.datasetconfig import DatasetConfig +from fidesops.ops.schemas.connection_configuration.connection_config import ( + SaasConnectionTemplateValues, +) from fidesops.ops.schemas.dataset import FidesopsDataset from fidesops.ops.schemas.saas.saas_config import SaaSConfig -from fidesops.ops.util.saas_util import load_config, load_dataset, load_yaml_as_string +from fidesops.ops.util.saas_util import ( + load_config, + load_config_with_replacement, + load_dataset, + load_dataset_with_replacement, +) -_registry: ConnectorRegistry = {} +_registry: Optional[ConnectorRegistry] = None +registry_file = "saas_connector_registry.toml" class ConnectorTemplate(BaseModel): """ - A collection of references to artifacts that make up + A collection of paths to artifacts that make up a complete SaaS connector (SaaS config, dataset, etc.) """ @@ -49,48 +64,63 @@ class ConnectorRegistry(BaseModel): __root__: Dict[str, ConnectorTemplate] - def keys(self): + def connector_types(self) -> List[str]: + """List of registered SaaS connector types""" return list(self.__root__) - def get(self, item): - try: - return self.__root__[item] - except: - return None - - -def connector_types() -> List[str]: - """List of registered SaaS connector types""" - return _registry.keys() - - -def get_connector_template(connector_type: str) -> Optional[ConnectorTemplate]: - """ - Returns an object containing the references to the various SaaS connector artifacts - """ - return _registry.get(connector_type) - + def get_connector_template( + self, connector_type: str + ) -> Optional[ConnectorTemplate]: + """ + Returns an object containing the references to the various SaaS connector artifacts + """ + return self.__root__.get(connector_type) + + +def create_connection_config_from_template( + db: Session, + connector_template: ConnectorTemplate, + template_values: SaasConnectionTemplateValues, +) -> ConnectionConfig: + """Creates a SaaS connection config from a template""" + # Load saas config from template + config_from_template: Dict = load_config_with_replacement( + connector_template.config, "instance_fides_key", template_values.instance_key + ) -def instantiate_connector_template( - connector_type: str, instance_key: str, secrets: Dict[str, Any] -) -> None: - """Creates a SaaS connection config and associates the SaaS config and dataset with the new connection""" + # Create SaaS ConnectionConfig + connection_config = ConnectionConfig.create_or_update( + db, + data={ + "name": template_values.name, + "key": template_values.key, + "description": template_values.description, + "connection_type": ConnectionType.saas, + "access": AccessLevel.write, + "saas_config": config_from_template, + }, + ) - template = _registry.get(connector_type) - # validate secrets - # create connection config + return connection_config - # add saas config - saas_config = load_yaml_as_string(template.config).replace( - "instance_fides_key", instance_key - ) - logger.info(saas_config) - # add dataset - saas_dataset = load_yaml_as_string(template.dataset).replace( - "instance_fides_key", instance_key - ) - logger.info(saas_dataset) +def create_dataset_config_from_template( + db: Session, + connection_config: ConnectionConfig, + connector_template: ConnectorTemplate, + template_values: SaasConnectionTemplateValues, +) -> DatasetConfig: + """Creates a DatasetConfig from a template and associates it with a ConnectionConfig""" + dataset_from_template: Dict = load_dataset_with_replacement( + connector_template.dataset, "instance_fides_key", template_values.instance_key + )[0] + data = { + "connection_config_id": connection_config.id, + "fides_key": template_values.instance_key, + "dataset": dataset_from_template, + } + dataset_config = DatasetConfig.create_or_update(db, data=data) + return dataset_config def update_connector_instances() -> None: @@ -103,10 +133,11 @@ def update_connector_instances() -> None: # if registry version is newer # update saas config # update dataset - pass -def load_registry(config_file: str) -> None: +def load_registry(config_file: str) -> ConnectorRegistry: """Loads a SaaS connector registry from the given config file.""" - global _registry - _registry = ConnectorRegistry.parse_obj(load_toml([config_file])) + global _registry # pylint: disable=W0603 + if _registry is None: + _registry = ConnectorRegistry.parse_obj(load_toml([config_file])) + return _registry diff --git a/src/fidesops/ops/util/saas_util.py b/src/fidesops/ops/util/saas_util.py index 135b08e93..a7bf82dee 100644 --- a/src/fidesops/ops/util/saas_util.py +++ b/src/fidesops/ops/util/saas_util.py @@ -34,12 +34,32 @@ def load_config(filename: str) -> Dict: return yaml.safe_load(file).get("saas_config", []) +def load_config_with_replacement( + filename: str, string_to_replace: str, replacement: str +) -> Dict: + """Loads the saas config from the yaml file and replaces any string with the given value""" + yaml_str: str = load_yaml_as_string(filename).replace( + string_to_replace, replacement + ) + return yaml.safe_load(yaml_str).get("saas_config", []) + + def load_dataset(filename: str) -> Dict: yaml_file = load_file([filename]) with open(yaml_file, "r") as file: return yaml.safe_load(file).get("dataset", []) +def load_dataset_with_replacement( + filename: str, string_to_replace: str, replacement: str +) -> Dict: + """Loads the dataset from the yaml file and replaces any string with the given value""" + yaml_str: str = load_yaml_as_string(filename).replace( + string_to_replace, replacement + ) + return yaml.safe_load(yaml_str).get("dataset", []) + + def merge_fields(target: Field, source: Field) -> Field: """Replaces source references and identities if they are available from the target""" if source.references is not None: diff --git a/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py b/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py index ee2aefcd6..f95b3a756 100644 --- a/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py @@ -4,13 +4,23 @@ from fideslib.models.client import ClientDetail from starlette.testclient import TestClient -from fidesops.ops.api.v1.scope_registry import CONNECTION_READ, CONNECTION_TYPE_READ +from fidesops.ops.api.v1.scope_registry import ( + CONNECTION_INSTANTIATE, + CONNECTION_READ, + CONNECTION_TYPE_READ, +) from fidesops.ops.api.v1.urn_registry import ( CONNECTION_TYPE_SECRETS, CONNECTION_TYPES, + SAAS_CONNECTOR_FROM_TEMPLATE, V1_URL_PREFIX, ) -from fidesops.ops.models.connectionconfig import ConnectionType +from fidesops.ops.models.connectionconfig import ( + AccessLevel, + ConnectionConfig, + ConnectionType, +) +from fidesops.ops.models.datasetconfig import DatasetConfig from fidesops.ops.schemas.connection_configuration.connection_config import ( ConnectionSystemTypeMap, SystemType, @@ -196,3 +206,81 @@ def test_get_connection_secret_schema_hubspot( "required": ["hapikey"], "additionalProperties": False, } + + +class TestInstantiateConnectionFromTemplate: + @pytest.fixture(scope="function") + def base_url(self) -> str: + return V1_URL_PREFIX + SAAS_CONNECTOR_FROM_TEMPLATE + + def test_instantiate_connection_not_authenticated(self, api_client, base_url): + resp = api_client.post( + base_url.format(saas_connector_type="mailchimp"), headers={} + ) + assert resp.status_code == 401 + + def test_instantiate_connection_wrong_scope( + self, generate_auth_header, api_client, base_url + ): + auth_header = generate_auth_header(scopes=[CONNECTION_READ]) + resp = api_client.post( + base_url.format(saas_connector_type="mailchimp"), headers=auth_header + ) + assert resp.status_code == 403 + + def test_instantiate_mail_chimp_connection_from_template( + self, db, generate_auth_header, api_client, base_url + ): + connection_config = ConnectionConfig.filter( + db=db, conditions=(ConnectionConfig.key == "mailchimp_connection_config") + ).first() + assert connection_config is None + + dataset_config = DatasetConfig.filter( + db=db, conditions=(DatasetConfig.fides_key == "primary_mailchimp_instance") + ).first() + assert dataset_config is None + + auth_header = generate_auth_header(scopes=[CONNECTION_INSTANTIATE]) + request_body = { + "instance_key": "secondary_mailchimp_instance", + "secrets": { + "domain": "test_mailchimp_domain", + "username": "test_mailchimp_username", + "api_key": "test_mailchimp_api_key", + }, + "name": "Mailchimp Connector", + "description": "Mailchimp ConnectionConfig description", + "key": "mailchimp_connection_config", + } + resp = api_client.post( + base_url.format(saas_connector_type="mailchimp"), + headers=auth_header, + json=request_body, + ) + + assert resp.status_code == 200 + assert resp.json()["fides_key"] == "secondary_mailchimp_instance" + + connection_config = ConnectionConfig.filter( + db=db, conditions=(ConnectionConfig.key == "mailchimp_connection_config") + ).first() + dataset_config = DatasetConfig.filter( + db=db, + conditions=(DatasetConfig.fides_key == "secondary_mailchimp_instance"), + ).first() + + assert connection_config is not None + assert dataset_config is not None + assert connection_config.name == "Mailchimp Connector" + assert connection_config.description == "Mailchimp ConnectionConfig description" + + assert connection_config.access == AccessLevel.write + assert connection_config.connection_type == ConnectionType.saas + assert connection_config.saas_config is not None + + assert dataset_config.connection_config_id == connection_config.id + assert dataset_config.dataset is not None + + dataset_config.delete(db) + connection_config.delete(db) diff --git a/tests/ops/service/connectors/test_connector_registry_service.py b/tests/ops/service/connectors/test_connector_registry_service.py new file mode 100644 index 000000000..8c482ac82 --- /dev/null +++ b/tests/ops/service/connectors/test_connector_registry_service.py @@ -0,0 +1,21 @@ +from fidesops.ops.service.connectors.saas.connector_registry_service import ( + ConnectorTemplate, + load_registry, + registry_file, +) + + +class TestConnectionRegistry: + def test_get_connector_template(self): + registry = load_registry(registry_file) + + assert "mailchimp" in registry.connector_types() + + assert registry.get_connector_template("bad_key") is None + mailchimp_registry = registry.get_connector_template("mailchimp") + + assert mailchimp_registry == ConnectorTemplate( + config="data/saas/config/mailchimp_config.yml", + dataset="data/saas/dataset/mailchimp_dataset.yml", + icon="data/saas/icon/mailchimp.svg", + ) From 5f378200d41ee6e8735a3c986fb5181125cbcb1e Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Tue, 16 Aug 2022 08:42:51 -0500 Subject: [PATCH 04/17] Remove updating connector instances for now - out of scope. --- src/fidesops/main.py | 4 ---- .../connectors/saas/connector_registry_service.py | 12 ------------ 2 files changed, 16 deletions(-) diff --git a/src/fidesops/main.py b/src/fidesops/main.py index 6a4f82e5d..30de5f49b 100644 --- a/src/fidesops/main.py +++ b/src/fidesops/main.py @@ -36,7 +36,6 @@ from fidesops.ops.service.connectors.saas.connector_registry_service import ( load_registry, registry_file, - update_connector_instances, ) from fidesops.ops.tasks.scheduled.scheduler import scheduler from fidesops.ops.tasks.scheduled.tasks import initiate_scheduled_request_intake @@ -196,9 +195,6 @@ def start_webserver() -> None: logger.error(f"Connection to database failed: {error}") return - logger.info("Updating SaaS connector instances...") - update_connector_instances() - if config.redis.enabled: logger.info("Running Redis connection test...") try: diff --git a/src/fidesops/ops/service/connectors/saas/connector_registry_service.py b/src/fidesops/ops/service/connectors/saas/connector_registry_service.py index 00831444c..9ef382ce7 100644 --- a/src/fidesops/ops/service/connectors/saas/connector_registry_service.py +++ b/src/fidesops/ops/service/connectors/saas/connector_registry_service.py @@ -123,18 +123,6 @@ def create_dataset_config_from_template( return dataset_config -def update_connector_instances() -> None: - """ - Updates every SaaS connection config and dataset with the latest version - defined in the SaaS connector registry - """ - # get all saas connection configs - # get version number from SaaS config (figure out versioning) - # if registry version is newer - # update saas config - # update dataset - - def load_registry(config_file: str) -> ConnectorRegistry: """Loads a SaaS connector registry from the given config file.""" global _registry # pylint: disable=W0603 From 2f109578820015460ccbc08860535a5e3e1ed70e Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Tue, 16 Aug 2022 08:43:27 -0500 Subject: [PATCH 05/17] Test nonexistent templates, secrets validation, instance key / fides key already exists. --- .../api/v1/endpoints/saas_config_endpoints.py | 4 +- .../saas/connector_registry_service.py | 8 +- src/fidesops/ops/util/saas_util.py | 4 +- .../test_connection_template_endpoints.py | 84 ++++++++++++++++++- 4 files changed, 91 insertions(+), 9 deletions(-) diff --git a/src/fidesops/ops/api/v1/endpoints/saas_config_endpoints.py b/src/fidesops/ops/api/v1/endpoints/saas_config_endpoints.py index cef167c8a..bc0a96e8c 100644 --- a/src/fidesops/ops/api/v1/endpoints/saas_config_endpoints.py +++ b/src/fidesops/ops/api/v1/endpoints/saas_config_endpoints.py @@ -284,13 +284,13 @@ def instantiate_connection( ) if not connector_template: raise HTTPException( - status_code=HTTP_400_BAD_REQUEST, + status_code=HTTP_404_NOT_FOUND, detail=f"SaaS connector type '{saas_connector_type}' is not registered.", ) if DatasetConfig.filter( db=db, - conditions=((DatasetConfig.fides_key == template_values.instance_key)), + conditions=(DatasetConfig.fides_key == template_values.instance_key), ).count(): raise HTTPException( status_code=HTTP_400_BAD_REQUEST, diff --git a/src/fidesops/ops/service/connectors/saas/connector_registry_service.py b/src/fidesops/ops/service/connectors/saas/connector_registry_service.py index 9ef382ce7..363355737 100644 --- a/src/fidesops/ops/service/connectors/saas/connector_registry_service.py +++ b/src/fidesops/ops/service/connectors/saas/connector_registry_service.py @@ -79,13 +79,13 @@ def get_connector_template( def create_connection_config_from_template( db: Session, - connector_template: ConnectorTemplate, + template: ConnectorTemplate, template_values: SaasConnectionTemplateValues, ) -> ConnectionConfig: """Creates a SaaS connection config from a template""" # Load saas config from template config_from_template: Dict = load_config_with_replacement( - connector_template.config, "instance_fides_key", template_values.instance_key + template.config, "instance_fides_key", template_values.instance_key ) # Create SaaS ConnectionConfig @@ -107,12 +107,12 @@ def create_connection_config_from_template( def create_dataset_config_from_template( db: Session, connection_config: ConnectionConfig, - connector_template: ConnectorTemplate, + template: ConnectorTemplate, template_values: SaasConnectionTemplateValues, ) -> DatasetConfig: """Creates a DatasetConfig from a template and associates it with a ConnectionConfig""" dataset_from_template: Dict = load_dataset_with_replacement( - connector_template.dataset, "instance_fides_key", template_values.instance_key + template.dataset, "instance_fides_key", template_values.instance_key )[0] data = { "connection_config_id": connection_config.id, diff --git a/src/fidesops/ops/util/saas_util.py b/src/fidesops/ops/util/saas_util.py index a7bf82dee..be4be17aa 100644 --- a/src/fidesops/ops/util/saas_util.py +++ b/src/fidesops/ops/util/saas_util.py @@ -23,7 +23,7 @@ def load_yaml_as_string(filename: str) -> str: yaml_file = load_file([filename]) - with open(yaml_file, "r") as file: + with open(yaml_file, "r", encoding="utf-8") as file: return file.read() @@ -46,7 +46,7 @@ def load_config_with_replacement( def load_dataset(filename: str) -> Dict: yaml_file = load_file([filename]) - with open(yaml_file, "r") as file: + with open(yaml_file, "r", encoding="utf-8") as file: return yaml.safe_load(file).get("dataset", []) diff --git a/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py b/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py index f95b3a756..b036f320c 100644 --- a/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py @@ -228,7 +228,89 @@ def test_instantiate_connection_wrong_scope( ) assert resp.status_code == 403 - def test_instantiate_mail_chimp_connection_from_template( + def test_instantiate_nonexistent_template( + self, generate_auth_header, api_client, base_url + ): + auth_header = generate_auth_header(scopes=[CONNECTION_INSTANTIATE]) + request_body = { + "instance_key": "test_instance_key", + "secrets": {}, + "name": "Unsupported Connector", + "description": "Unsupported connector description", + "key": "unsupported_connector", + } + resp = api_client.post( + base_url.format(saas_connector_type="does_not_exist"), + headers=auth_header, + json=request_body, + ) + assert resp.status_code == 404 + assert ( + resp.json()["detail"] + == f"SaaS connector type '{'does_not_exist'}' is not registered." + ) + + def test_instance_key_already_exists( + self, generate_auth_header, api_client, base_url, dataset_config + ): + auth_header = generate_auth_header(scopes=[CONNECTION_INSTANTIATE]) + request_body = { + "instance_key": dataset_config.fides_key, + "secrets": { + "domain": "test_mailchimp_domain", + "username": "test_mailchimp_username", + "api_key": "test_mailchimp_api_key", + }, + "name": "Mailchimp Connector", + "description": "Mailchimp ConnectionConfig description", + "key": "mailchimp_connection_config", + } + resp = api_client.post( + base_url.format(saas_connector_type="mailchimp"), + headers=auth_header, + json=request_body, + ) + assert resp.status_code == 400 + assert ( + resp.json()["detail"] + == f"SaaS connector instance key '{dataset_config.fides_key}' already exists." + ) + + def test_template_secrets_validation( + self, generate_auth_header, api_client, base_url + ): + auth_header = generate_auth_header(scopes=[CONNECTION_INSTANTIATE]) + # Secrets have one field missing, one field extra + request_body = { + "instance_key": "secondary_mailchimp_instance", + "secrets": { + "bad_mailchimp_secret_key": "bad_key", + "username": "test_mailchimp_username", + "api_key": "test_mailchimp_api_key", + }, + "name": "Mailchimp Connector", + "description": "Mailchimp ConnectionConfig description", + "key": "mailchimp_connection_config", + } + resp = api_client.post( + base_url.format(saas_connector_type="mailchimp"), + headers=auth_header, + json=request_body, + ) + + assert resp.status_code == 422 + assert resp.json()["detail"][0] == { + "loc": ["domain"], + "msg": "field required", + "type": "value_error.missing", + } + assert resp.json()["detail"][1] == { + "loc": ["bad_mailchimp_secret_key"], + "msg": "extra fields not permitted", + "type": "value_error.extra", + } + + def test_instantiate_mailchimp_connection_from_template( self, db, generate_auth_header, api_client, base_url ): connection_config = ConnectionConfig.filter( From ea306d4cc1e70418203bd8be4d0eb33f6f81b010 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Tue, 16 Aug 2022 09:37:45 -0500 Subject: [PATCH 06/17] Create DatasetConfigs and ConnectionConfigs instead of create_or_update in the template endpoint. Don't save ConnectionConfig until secrets are validated. --- .../api/v1/endpoints/saas_config_endpoints.py | 17 ++++-- src/fidesops/ops/models/connectionconfig.py | 31 +++++++++- .../saas/connector_registry_service.py | 4 +- .../test_connection_template_endpoints.py | 59 ++++++++++++++++++- 4 files changed, 102 insertions(+), 9 deletions(-) diff --git a/src/fidesops/ops/api/v1/endpoints/saas_config_endpoints.py b/src/fidesops/ops/api/v1/endpoints/saas_config_endpoints.py index bc0a96e8c..7c8a92a64 100644 --- a/src/fidesops/ops/api/v1/endpoints/saas_config_endpoints.py +++ b/src/fidesops/ops/api/v1/endpoints/saas_config_endpoints.py @@ -3,6 +3,7 @@ from fastapi import Depends, HTTPException from fastapi.params import Security +from fideslib.exceptions import KeyOrNameAlreadyExists from sqlalchemy.orm import Session from starlette.status import ( HTTP_200_OK, @@ -277,7 +278,6 @@ def instantiate_connection( fields are provided, persists the associated connection config and dataset to the database. """ - # verify connector type is registered registry: ConnectorRegistry = load_registry(registry_file) connector_template: Optional[ConnectorTemplate] = registry.get_connector_template( saas_connector_type @@ -297,13 +297,20 @@ def instantiate_connection( detail=f"SaaS connector instance key '{template_values.instance_key}' already exists.", ) - connection_config: ConnectionConfig = create_connection_config_from_template( - db, connector_template, template_values - ) + try: + connection_config: ConnectionConfig = create_connection_config_from_template( + db, connector_template, template_values + ) + except KeyOrNameAlreadyExists as exc: + raise HTTPException( + status_code=HTTP_400_BAD_REQUEST, + detail=exc.args[0], + ) + connection_config.secrets = validate_secrets( template_values.secrets, connection_config ).dict() - connection_config.save(db=db) + connection_config.save(db=db) # Not persisted to db until secrets are validated dataset_config: DatasetConfig = create_dataset_config_from_template( db, connection_config, connector_template, template_values diff --git a/src/fidesops/ops/models/connectionconfig.py b/src/fidesops/ops/models/connectionconfig.py index dedb1af8a..0cefa4673 100644 --- a/src/fidesops/ops/models/connectionconfig.py +++ b/src/fidesops/ops/models/connectionconfig.py @@ -2,9 +2,11 @@ import enum from datetime import datetime -from typing import Any, Optional +from typing import Any, Optional, Type, TypeVar from fideslib.db.base import Base +from fideslib.db.base_class import get_key_from_data +from fideslib.exceptions import KeyOrNameAlreadyExists from sqlalchemy import Boolean, Column, DateTime, Enum, String, event from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.ext.mutable import MutableDict @@ -88,6 +90,33 @@ class ConnectionConfig(Base): MutableDict.as_mutable(JSONB), index=False, unique=False, nullable=True ) + @classmethod + def create_without_saving( + cls: Type[ConnectionConfig], db: Session, *, data: dict[str, Any] + ) -> ConnectionConfig: + """Create a ConnectionConfig without persisting to the database""" + # Build properly formatted key/name for ConnectionConfig. + # Borrowed from OrmWrappedFidesBase.create + if hasattr(cls, "key"): + data["key"] = get_key_from_data(data, cls.__name__) + if db.query(cls).filter_by(key=data["key"]).first(): + raise KeyOrNameAlreadyExists( + f"Key {data['key']} already exists in {cls.__name__}. Keys will be snake-cased names if not provided. " + f"If you are seeing this error without providing a key, please provide a key or a different name." + "" + ) + + if hasattr(cls, "name"): + data["name"] = data.get("name") + if db.query(cls).filter_by(name=data["name"]).first(): + raise KeyOrNameAlreadyExists( + f"Name {data['name']} already exists in {cls.__name__}." + ) + + # Create + db_obj = cls(**data) # type: ignore + return db_obj + def get_saas_config(self) -> Optional[SaaSConfig]: """Returns a SaaSConfig object from a yaml config""" return SaaSConfig(**self.saas_config) if self.saas_config else None diff --git a/src/fidesops/ops/service/connectors/saas/connector_registry_service.py b/src/fidesops/ops/service/connectors/saas/connector_registry_service.py index 363355737..f78cbe3a3 100644 --- a/src/fidesops/ops/service/connectors/saas/connector_registry_service.py +++ b/src/fidesops/ops/service/connectors/saas/connector_registry_service.py @@ -89,7 +89,7 @@ def create_connection_config_from_template( ) # Create SaaS ConnectionConfig - connection_config = ConnectionConfig.create_or_update( + connection_config = ConnectionConfig.create_without_saving( db, data={ "name": template_values.name, @@ -119,7 +119,7 @@ def create_dataset_config_from_template( "fides_key": template_values.instance_key, "dataset": dataset_from_template, } - dataset_config = DatasetConfig.create_or_update(db, data=data) + dataset_config = DatasetConfig.create(db, data=data) return dataset_config diff --git a/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py b/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py index b036f320c..752d7d844 100644 --- a/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py @@ -277,7 +277,7 @@ def test_instance_key_already_exists( ) def test_template_secrets_validation( - self, generate_auth_header, api_client, base_url + self, generate_auth_header, api_client, base_url, db ): auth_header = generate_auth_header(scopes=[CONNECTION_INSTANTIATE]) # Secrets have one field missing, one field extra @@ -310,6 +310,63 @@ def test_template_secrets_validation( "type": "value_error.extra", } + connection_config = ConnectionConfig.filter( + db=db, conditions=(ConnectionConfig.key == "mailchimp_connection_config") + ).first() + assert connection_config is None, "ConnectionConfig not persisted" + + def test_connection_config_key_already_exists( + self, db, generate_auth_header, api_client, base_url, connection_config + ): + auth_header = generate_auth_header(scopes=[CONNECTION_INSTANTIATE]) + request_body = { + "instance_key": "secondary_mailchimp_instance", + "secrets": { + "domain": "test_mailchimp_domain", + "username": "test_mailchimp_username", + "api_key": "test_mailchimp_api_key", + }, + "name": connection_config.name, + "description": "Mailchimp ConnectionConfig description", + "key": connection_config.key, + } + resp = api_client.post( + base_url.format(saas_connector_type="mailchimp"), + headers=auth_header, + json=request_body, + ) + assert resp.status_code == 400 + assert ( + f"Key {connection_config.key} already exists in ConnectionConfig" + in resp.json()["detail"] + ) + + def test_connection_config_name_already_exists( + self, db, generate_auth_header, api_client, base_url, connection_config + ): + auth_header = generate_auth_header(scopes=[CONNECTION_INSTANTIATE]) + request_body = { + "instance_key": "secondary_mailchimp_instance", + "secrets": { + "domain": "test_mailchimp_domain", + "username": "test_mailchimp_username", + "api_key": "test_mailchimp_api_key", + }, + "name": connection_config.name, + "description": "Mailchimp ConnectionConfig description", + "key": "brand_new_key", + } + resp = api_client.post( + base_url.format(saas_connector_type="mailchimp"), + headers=auth_header, + json=request_body, + ) + assert resp.status_code == 400 + assert ( + f"Name {connection_config.name} already exists in ConnectionConfig" + in resp.json()["detail"] + ) + def test_instantiate_mailchimp_connection_from_template( self, db, generate_auth_header, api_client, base_url ): From c50f59f4fb5a590b8e3536b9b8de51065f15bf1c Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Tue, 16 Aug 2022 16:03:59 -0500 Subject: [PATCH 07/17] Add the other saas connectors to the registry and update their configs and datasets with instance_fides_key. - Fix datadog yaml so it can be included in the saas connector registry. There was an error in how the saas config was formatted. --- data/saas/config/adobe_campaign_config.yml | 2 +- data/saas/config/auth0_config.yml | 6 +- data/saas/config/datadog_config.yml | 108 +++++++++--------- data/saas/config/hubspot_config.yml | 6 +- data/saas/config/logi_id_config.yml | 4 +- data/saas/config/outreach_config.yml | 2 +- data/saas/config/salesforce_config.yml | 24 ++-- data/saas/config/segment_config.yml | 8 +- data/saas/config/sendgrid_config.yml | 4 +- data/saas/config/sentry_config.yml | 20 ++-- data/saas/config/stripe_config.yml | 50 ++++---- data/saas/config/zendesk_config.yml | 12 +- data/saas/dataset/adobe_campaign_dataset.yml | 2 +- data/saas/dataset/auth0_dataset.yml | 2 +- data/saas/dataset/datadog_dataset.yml | 2 +- data/saas/dataset/hubspot_dataset.yml | 2 +- data/saas/dataset/logi_id_dataset.yml | 2 +- data/saas/dataset/outreach_dataset.yml | 2 +- data/saas/dataset/saas_example_dataset.yml | 2 +- data/saas/dataset/salesforce_dataset.yml | 2 +- data/saas/dataset/segment_dataset.yml | 2 +- data/saas/dataset/sendgrid_dataset.yml | 2 +- data/saas/dataset/sentry_dataset.yml | 2 +- data/saas/dataset/stripe_dataset.yml | 2 +- data/saas/dataset/zendesk_dataset.yml | 2 +- saas_connector_registry.toml | 62 +++++++++- .../api/v1/endpoints/saas_config_endpoints.py | 17 +-- src/fidesops/ops/api/v1/scope_registry.py | 4 +- src/fidesops/ops/models/connectionconfig.py | 2 +- .../saas/connector_registry_service.py | 4 +- .../test_connection_template_endpoints.py | 52 +++++++-- 31 files changed, 256 insertions(+), 157 deletions(-) diff --git a/data/saas/config/adobe_campaign_config.yml b/data/saas/config/adobe_campaign_config.yml index 082dd9dc8..f41500b48 100644 --- a/data/saas/config/adobe_campaign_config.yml +++ b/data/saas/config/adobe_campaign_config.yml @@ -1,5 +1,5 @@ saas_config: - fides_key: adobe_campaign_connector_example + fides_key: instance_fides_key name: Adobe Campaign SaaS Config type: adobe_campaign description: A schema representing the Adobe Campaign connector for Fidesops diff --git a/data/saas/config/auth0_config.yml b/data/saas/config/auth0_config.yml index 8f1241701..ba6f2b558 100644 --- a/data/saas/config/auth0_config.yml +++ b/data/saas/config/auth0_config.yml @@ -1,5 +1,5 @@ saas_config: - fides_key: auth0_connector_example + fides_key: instance_fides_key name: Auth0 SaaS Config type: auth0 description: A sample schema representing the Auth0 connector for Fidesops @@ -46,7 +46,7 @@ saas_config: param_values: - name: user_id references: - - dataset: auth0_connector_example + - dataset: instance_fides_key field: users.user_id direction: from - name: user_logs @@ -57,6 +57,6 @@ saas_config: param_values: - name: user_id references: - - dataset: auth0_connector_example + - dataset: instance_fides_key field: users.user_id direction: from \ No newline at end of file diff --git a/data/saas/config/datadog_config.yml b/data/saas/config/datadog_config.yml index bfc1c3235..00a6de8f1 100644 --- a/data/saas/config/datadog_config.yml +++ b/data/saas/config/datadog_config.yml @@ -1,59 +1,59 @@ saas_config: - - fides_key: datadog_connector_example - name: Datadog SaaS Config - type: datadog - description: A sample schema representing the Datadog connector for Fidesops - version: 0.0.1 + fides_key: instance_fides_key + name: Datadog SaaS Config + type: datadog + description: A sample schema representing the Datadog connector for Fidesops + version: 0.0.1 - connector_params: - - name: domain - - name: api_key - - name: app_key - - name: page_size + connector_params: + - name: domain + - name: api_key + - name: app_key + - name: page_size - client_config: - protocol: https - host: + client_config: + protocol: https + host: - test_request: - method: GET - path: /api/v2/logs/events - headers: - - name: DD-APPLICATION-KEY - value: - - name: DD-API-KEY - value: + test_request: + method: GET + path: /api/v2/logs/events + headers: + - name: DD-APPLICATION-KEY + value: + - name: DD-API-KEY + value: - endpoints: - - name: events - requests: - read: - method: GET - path: /api/v2/logs/events - headers: - - name: DD-APPLICATION-KEY - value: - - name: DD-API-KEY - value: - query_params: - - name: filter[query] - value: - - name: filter[from] - value: 0 - - name: filter[to] - value: now - - name: page[limit] - value: - param_values: - - name: app_key - connector_param: app_key - - name: api_key - connector_param: api_key - - name: email - identity: email - data_path: data - pagination: - strategy: link - configuration: - source: body - path: links.next + endpoints: + - name: events + requests: + read: + method: GET + path: /api/v2/logs/events + headers: + - name: DD-APPLICATION-KEY + value: + - name: DD-API-KEY + value: + query_params: + - name: filter[query] + value: + - name: filter[from] + value: 0 + - name: filter[to] + value: now + - name: page[limit] + value: + param_values: + - name: app_key + connector_param: app_key + - name: api_key + connector_param: api_key + - name: email + identity: email + data_path: data + pagination: + strategy: link + configuration: + source: body + path: links.next diff --git a/data/saas/config/hubspot_config.yml b/data/saas/config/hubspot_config.yml index 4bcc8b08e..efb69ec01 100644 --- a/data/saas/config/hubspot_config.yml +++ b/data/saas/config/hubspot_config.yml @@ -1,5 +1,5 @@ saas_config: - fides_key: hubspot_connector_example + fides_key: instance_fides_key name: Hubspot SaaS Config type: hubspot description: A sample schema representing the Hubspot connector for Fidesops @@ -64,7 +64,7 @@ saas_config: param_values: - name: contactId references: - - dataset: hubspot_connector_example + - dataset: instance_fides_key field: contacts.id direction: from - name: owners @@ -113,7 +113,7 @@ saas_config: identity: email - name: subscriptionId references: - - dataset: hubspot_connector_example + - dataset: instance_fides_key field: subscription_preferences.id direction: from postprocessors: diff --git a/data/saas/config/logi_id_config.yml b/data/saas/config/logi_id_config.yml index 0c4b729e6..0479f6b3c 100644 --- a/data/saas/config/logi_id_config.yml +++ b/data/saas/config/logi_id_config.yml @@ -1,5 +1,5 @@ saas_config: - fides_key: logi_id_connector_example + fides_key: instance_fides_key name: Logi ID SaaS Config type: logi_id description: A sample schema representing the Logi ID connector for Fidesops @@ -44,7 +44,7 @@ saas_config: param_values: - name: user_id references: - - dataset: logi_id_connector_example + - dataset: instance_fides_key field: users.id direction: from - name: user_claims diff --git a/data/saas/config/outreach_config.yml b/data/saas/config/outreach_config.yml index 91c7f6897..c427cdeff 100644 --- a/data/saas/config/outreach_config.yml +++ b/data/saas/config/outreach_config.yml @@ -1,5 +1,5 @@ saas_config: - fides_key: outreach_connector_example + fides_key: instance_fides_key name: Outreach Example Config type: outreach description: A sample schema representing the Outreach connector for Fidesops diff --git a/data/saas/config/salesforce_config.yml b/data/saas/config/salesforce_config.yml index b13e76920..43d9188d8 100644 --- a/data/saas/config/salesforce_config.yml +++ b/data/saas/config/salesforce_config.yml @@ -1,5 +1,5 @@ saas_config: - fides_key: salesforce_connector_example + fides_key: instance_fides_key name: Salesforce SaaS Config type: salesforce description: A sample schema representing the Salesforce connector for Fidesops @@ -87,7 +87,7 @@ saas_config: param_values: - name: contact_id references: - - dataset: salesforce_connector_example + - dataset: instance_fides_key field: contact_list.Id direction: from update: @@ -100,7 +100,7 @@ saas_config: param_values: - name: contact_id references: - - dataset: salesforce_connector_example + - dataset: instance_fides_key field: contacts.Id direction: from - name: case_list @@ -114,7 +114,7 @@ saas_config: param_values: - name: contact_id references: - - dataset: salesforce_connector_example + - dataset: instance_fides_key field: contact_list.Id direction: from data_path: records @@ -126,7 +126,7 @@ saas_config: param_values: - name: case_id references: - - dataset: salesforce_connector_example + - dataset: instance_fides_key field: case_list.Id direction: from update: @@ -139,7 +139,7 @@ saas_config: param_values: - name: case_id references: - - dataset: salesforce_connector_example + - dataset: instance_fides_key field: cases.Id direction: from - name: lead_list @@ -162,7 +162,7 @@ saas_config: param_values: - name: lead_id references: - - dataset: salesforce_connector_example + - dataset: instance_fides_key field: lead_list.Id direction: from update: @@ -175,7 +175,7 @@ saas_config: param_values: - name: lead_id references: - - dataset: salesforce_connector_example + - dataset: instance_fides_key field: leads.Id direction: from - name: accounts @@ -186,7 +186,7 @@ saas_config: param_values: - name: account_id references: - - dataset: salesforce_connector_example + - dataset: instance_fides_key field: contacts.AccountId update: method: PATCH @@ -198,7 +198,7 @@ saas_config: param_values: - name: account_id references: - - dataset: salesforce_connector_example + - dataset: instance_fides_key field: accounts.Id direction: from - name: campaign_member_list @@ -221,7 +221,7 @@ saas_config: param_values: - name: campaign_member_id references: - - dataset: salesforce_connector_example + - dataset: instance_fides_key field: campaign_member_list.Id direction: from update: @@ -234,6 +234,6 @@ saas_config: param_values: - name: campaign_member_id references: - - dataset: salesforce_connector_example + - dataset: instance_fides_key field: campaign_members.Id direction: from diff --git a/data/saas/config/segment_config.yml b/data/saas/config/segment_config.yml index 72ed99ec2..02c5c71b5 100644 --- a/data/saas/config/segment_config.yml +++ b/data/saas/config/segment_config.yml @@ -1,5 +1,5 @@ saas_config: - fides_key: segment_connector_example + fides_key: instance_fides_key name: Segment SaaS Config type: segment description: A sample schema representing the Segment connector for Fidesops @@ -55,7 +55,7 @@ saas_config: connector_param: namespace_id - name: segment_id references: - - dataset: segment_connector_example + - dataset: instance_fides_key field: segment_user.segment_id direction: from data_path: data @@ -84,7 +84,7 @@ saas_config: connector_param: namespace_id - name: segment_id references: - - dataset: segment_connector_example + - dataset: instance_fides_key field: segment_user.segment_id direction: from data_path: traits @@ -110,7 +110,7 @@ saas_config: connector_param: namespace_id - name: segment_id references: - - dataset: segment_connector_example + - dataset: instance_fides_key field: segment_user.segment_id direction: from data_path: data diff --git a/data/saas/config/sendgrid_config.yml b/data/saas/config/sendgrid_config.yml index 4fb3efe46..db3ed2bbc 100644 --- a/data/saas/config/sendgrid_config.yml +++ b/data/saas/config/sendgrid_config.yml @@ -1,5 +1,5 @@ saas_config: - fides_key: sendgrid_connector_example + fides_key: instance_fides_key name: Sendgrid SaaS Config type: sendgrid description: A sample schema representing the Sendgrid connector for Fidesops @@ -41,6 +41,6 @@ saas_config: param_values: - name: contact_id references: - - dataset: sendgrid_connector_example + - dataset: instance_fides_key field: contacts.id direction: from diff --git a/data/saas/config/sentry_config.yml b/data/saas/config/sentry_config.yml index 2ada95e8c..7cbc1fe1f 100644 --- a/data/saas/config/sentry_config.yml +++ b/data/saas/config/sentry_config.yml @@ -1,5 +1,5 @@ saas_config: - fides_key: sentry_connector + fides_key: instance_fides_key name: Sentry SaaS Config type: sentry description: A sample schema representing the Sentry connector for Fidesops @@ -44,7 +44,7 @@ saas_config: param_values: - name: organization_slug references: - - dataset: sentry_connector + - dataset: instance_fides_key field: organizations.slug direction: from postprocessors: @@ -77,7 +77,7 @@ saas_config: param_values: - name: issue_id references: - - dataset: sentry_connector + - dataset: instance_fides_key field: issues.id direction: from body: | @@ -94,12 +94,12 @@ saas_config: param_values: - name: organization_slug references: - - dataset: sentry_connector + - dataset: instance_fides_key field: projects.organization.slug direction: from - name: project_slug references: - - dataset: sentry_connector + - dataset: instance_fides_key field: projects.slug direction: from - name: query @@ -118,12 +118,12 @@ saas_config: param_values: - name: organization_slug references: - - dataset: sentry_connector + - dataset: instance_fides_key field: projects.organization.slug direction: from - name: project_slug references: - - dataset: sentry_connector + - dataset: instance_fides_key field: projects.slug direction: from postprocessors: @@ -138,7 +138,7 @@ saas_config: source: headers rel: next - name: person - after: [sentry_connector.projects] + after: [instance_fides_key.projects] requests: read: method: GET @@ -151,12 +151,12 @@ saas_config: param_values: - name: organization_slug references: - - dataset: sentry_connector + - dataset: instance_fides_key field: projects.organization.slug direction: from - name: project_slug references: - - dataset: sentry_connector + - dataset: instance_fides_key field: projects.slug direction: from - name: query diff --git a/data/saas/config/stripe_config.yml b/data/saas/config/stripe_config.yml index f29911533..9edbe02d6 100644 --- a/data/saas/config/stripe_config.yml +++ b/data/saas/config/stripe_config.yml @@ -1,5 +1,5 @@ saas_config: - fides_key: stripe_connector_example + fides_key: instance_fides_key name: Stripe SaaS Config type: stripe description: A sample schema representing the Stripe connector for Fidesops @@ -46,7 +46,7 @@ saas_config: param_values: - name: customer_id references: - - dataset: stripe_connector_example + - dataset: instance_fides_key field: customer.id direction: from body: | @@ -66,7 +66,7 @@ saas_config: param_values: - name: customer_id references: - - dataset: stripe_connector_example + - dataset: instance_fides_key field: customer.id direction: from - name: limit @@ -92,12 +92,12 @@ saas_config: param_values: - name: charge_id references: - - dataset: stripe_connector_example + - dataset: instance_fides_key field: charge.id direction: from - name: payment_intent_id references: - - dataset: stripe_connector_example + - dataset: instance_fides_key field: payment_intent.id direction: from - name: limit @@ -121,7 +121,7 @@ saas_config: param_values: - name: customer_id references: - - dataset: stripe_connector_example + - dataset: instance_fides_key field: customer.id direction: from - name: limit @@ -145,7 +145,7 @@ saas_config: param_values: - name: customer_id references: - - dataset: stripe_connector_example + - dataset: instance_fides_key field: customer.id direction: from - name: type @@ -167,7 +167,7 @@ saas_config: param_values: - name: payment_method_id references: - - dataset: stripe_connector_example + - dataset: instance_fides_key field: payment_method.id direction: from body: | @@ -187,7 +187,7 @@ saas_config: param_values: - name: customer_id references: - - dataset: stripe_connector_example + - dataset: instance_fides_key field: customer.id direction: from - name: limit @@ -207,12 +207,12 @@ saas_config: param_values: - name: customer_id references: - - dataset: stripe_connector_example + - dataset: instance_fides_key field: bank_account.customer direction: from - name: bank_account_id references: - - dataset: stripe_connector_example + - dataset: instance_fides_key field: bank_account.id direction: from body: | @@ -232,7 +232,7 @@ saas_config: param_values: - name: customer_id references: - - dataset: stripe_connector_example + - dataset: instance_fides_key field: customer.id direction: from - name: limit @@ -252,12 +252,12 @@ saas_config: param_values: - name: customer_id references: - - dataset: stripe_connector_example + - dataset: instance_fides_key field: card.customer direction: from - name: card_id references: - - dataset: stripe_connector_example + - dataset: instance_fides_key field: card.id direction: from body: | @@ -277,7 +277,7 @@ saas_config: param_values: - name: customer_id references: - - dataset: stripe_connector_example + - dataset: instance_fides_key field: customer.id direction: from - name: limit @@ -299,7 +299,7 @@ saas_config: param_values: - name: customer_id references: - - dataset: stripe_connector_example + - dataset: instance_fides_key field: customer.id direction: from - name: limit @@ -321,7 +321,7 @@ saas_config: param_values: - name: customer_id references: - - dataset: stripe_connector_example + - dataset: instance_fides_key field: customer.id direction: from - name: limit @@ -338,12 +338,12 @@ saas_config: param_values: - name: customer_id references: - - dataset: stripe_connector_example + - dataset: instance_fides_key field: tax_id.customer direction: from - name: tax_id references: - - dataset: stripe_connector_example + - dataset: instance_fides_key field: tax_id.id direction: from - name: invoice @@ -359,7 +359,7 @@ saas_config: param_values: - name: customer_id references: - - dataset: stripe_connector_example + - dataset: instance_fides_key field: customer.id direction: from - name: limit @@ -377,7 +377,7 @@ saas_config: param_values: - name: invoice_id references: - - dataset: stripe_connector_example + - dataset: instance_fides_key field: invoice.id direction: from - name: invoice_item @@ -393,7 +393,7 @@ saas_config: param_values: - name: customer_id references: - - dataset: stripe_connector_example + - dataset: instance_fides_key field: customer.id direction: from - name: limit @@ -411,7 +411,7 @@ saas_config: param_values: - name: invoice_item_id references: - - dataset: stripe_connector_example + - dataset: instance_fides_key field: invoice_item.id direction: from - name: subscription @@ -427,7 +427,7 @@ saas_config: param_values: - name: customer_id references: - - dataset: stripe_connector_example + - dataset: instance_fides_key field: customer.id direction: from - name: limit @@ -444,6 +444,6 @@ saas_config: param_values: - name: subscription_id references: - - dataset: stripe_connector_example + - dataset: instance_fides_key field: subscription.id direction: from diff --git a/data/saas/config/zendesk_config.yml b/data/saas/config/zendesk_config.yml index d52b1e895..b546e4f1c 100644 --- a/data/saas/config/zendesk_config.yml +++ b/data/saas/config/zendesk_config.yml @@ -1,5 +1,5 @@ saas_config: - fides_key: zendesk_connector_example + fides_key: instance_fides_key name: Zendesk SaaS Config type: zendesk description: A sample schema representing the Zendesk connector for Fidesops @@ -46,7 +46,7 @@ saas_config: param_values: - name: user_id references: - - dataset: zendesk_connector_example + - dataset: instance_fides_key field: users.id direction: from - name: user_identities @@ -60,7 +60,7 @@ saas_config: param_values: - name: user_id references: - - dataset: zendesk_connector_example + - dataset: instance_fides_key field: users.id direction: from - name: page_size @@ -82,7 +82,7 @@ saas_config: param_values: - name: user_id references: - - dataset: zendesk_connector_example + - dataset: instance_fides_key field: users.id direction: from - name: page_size @@ -99,7 +99,7 @@ saas_config: param_values: - name: ticket_id references: - - dataset: zendesk_connector_example + - dataset: instance_fides_key field: tickets.id direction: from - name: ticket_comments @@ -113,7 +113,7 @@ saas_config: param_values: - name: ticket_id references: - - dataset: zendesk_connector_example + - dataset: instance_fides_key field: tickets.id direction: from - name: page_size diff --git a/data/saas/dataset/adobe_campaign_dataset.yml b/data/saas/dataset/adobe_campaign_dataset.yml index 9eb427d7f..7b64a2cbd 100644 --- a/data/saas/dataset/adobe_campaign_dataset.yml +++ b/data/saas/dataset/adobe_campaign_dataset.yml @@ -1,5 +1,5 @@ dataset: - - fides_key: adobe_campaign_connector_example + - fides_key: instance_fides_key name: Adobe Campaign Dataset description: A dataset representing the Adobe Campaign connector for Fidesops collections: diff --git a/data/saas/dataset/auth0_dataset.yml b/data/saas/dataset/auth0_dataset.yml index 13fede1bb..0cae7177c 100644 --- a/data/saas/dataset/auth0_dataset.yml +++ b/data/saas/dataset/auth0_dataset.yml @@ -1,5 +1,5 @@ dataset: - - fides_key: auth0_connector_example + - fides_key: instance_fides_key name: Auth0 Dataset description: A sample dataset representing the Auth0 connector for Fidesops collections: diff --git a/data/saas/dataset/datadog_dataset.yml b/data/saas/dataset/datadog_dataset.yml index 56cc24833..15cc20aa0 100644 --- a/data/saas/dataset/datadog_dataset.yml +++ b/data/saas/dataset/datadog_dataset.yml @@ -1,5 +1,5 @@ dataset: - - fides_key: datadog_connector_example + - fides_key: instance_fides_key name: Datadog Dataset description: A sample dataset representing the Datadog connector for Fidesops collections: diff --git a/data/saas/dataset/hubspot_dataset.yml b/data/saas/dataset/hubspot_dataset.yml index 2ebd528ac..1e801612f 100644 --- a/data/saas/dataset/hubspot_dataset.yml +++ b/data/saas/dataset/hubspot_dataset.yml @@ -1,5 +1,5 @@ dataset: - - fides_key: hubspot_connector_example + - fides_key: instance_fides_key name: Hubspot Dataset description: A sample dataset representing the Hubspot connector for Fidesops collections: diff --git a/data/saas/dataset/logi_id_dataset.yml b/data/saas/dataset/logi_id_dataset.yml index aa57e9597..9f1da8a3d 100644 --- a/data/saas/dataset/logi_id_dataset.yml +++ b/data/saas/dataset/logi_id_dataset.yml @@ -1,5 +1,5 @@ dataset: - - fides_key: logi_id_connector_example + - fides_key: instance_fides_key name: Logi ID Example Dataset description: A sample dataset representing the Logi ID connector for Fidesops collections: diff --git a/data/saas/dataset/outreach_dataset.yml b/data/saas/dataset/outreach_dataset.yml index df51a3b7b..e79d34f74 100644 --- a/data/saas/dataset/outreach_dataset.yml +++ b/data/saas/dataset/outreach_dataset.yml @@ -1,5 +1,5 @@ dataset: - - fides_key: outreach_connector_example + - fides_key: instance_fides_key name: Outreach Dataset description: A sample dataset representing the Outreach connector for Fidesops collections: diff --git a/data/saas/dataset/saas_example_dataset.yml b/data/saas/dataset/saas_example_dataset.yml index 1974cb279..dd9a0f116 100644 --- a/data/saas/dataset/saas_example_dataset.yml +++ b/data/saas/dataset/saas_example_dataset.yml @@ -1,5 +1,5 @@ dataset: - - fides_key: saas_connector_example + - fides_key: instance_fides_key name: SaaS Example Dataset description: A sample dataset representing a SaaS connector for Fidesops collections: diff --git a/data/saas/dataset/salesforce_dataset.yml b/data/saas/dataset/salesforce_dataset.yml index b1292a5ec..d80d96589 100644 --- a/data/saas/dataset/salesforce_dataset.yml +++ b/data/saas/dataset/salesforce_dataset.yml @@ -1,5 +1,5 @@ dataset: - - fides_key: salesforce_connector_example + - fides_key: instance_fides_key name: Salesforce Example Dataset description: A sample dataset representing the Salesforce connector for Fidesops collections: diff --git a/data/saas/dataset/segment_dataset.yml b/data/saas/dataset/segment_dataset.yml index d7ef65e66..578620eaf 100644 --- a/data/saas/dataset/segment_dataset.yml +++ b/data/saas/dataset/segment_dataset.yml @@ -1,5 +1,5 @@ dataset: - - fides_key: segment_connector_example + - fides_key: instance_fides_key name: Segment Dataset description: A sample dataset representing the Segment connector for Fidesops collections: diff --git a/data/saas/dataset/sendgrid_dataset.yml b/data/saas/dataset/sendgrid_dataset.yml index 94554c17a..d51846066 100644 --- a/data/saas/dataset/sendgrid_dataset.yml +++ b/data/saas/dataset/sendgrid_dataset.yml @@ -1,5 +1,5 @@ dataset: - - fides_key: sendgrid_connector_example + - fides_key: instance_fides_key name: Sendgrid Dataset description: A sample dataset representing the Sendgrid connector for Fidesops collections: diff --git a/data/saas/dataset/sentry_dataset.yml b/data/saas/dataset/sentry_dataset.yml index efd765187..729d2ba8b 100644 --- a/data/saas/dataset/sentry_dataset.yml +++ b/data/saas/dataset/sentry_dataset.yml @@ -1,5 +1,5 @@ dataset: - - fides_key: sentry_connector + - fides_key: instance_fides_key name: Sentry Dataset description: A sample dataset representing the Sentry connector for Fidesops collections: diff --git a/data/saas/dataset/stripe_dataset.yml b/data/saas/dataset/stripe_dataset.yml index 045cfb615..3ad759755 100644 --- a/data/saas/dataset/stripe_dataset.yml +++ b/data/saas/dataset/stripe_dataset.yml @@ -1,5 +1,5 @@ dataset: - - fides_key: stripe_connector_example + - fides_key: instance_fides_key name: Stripe Dataset description: A sample dataset representing the Stripe connector for Fidesops collections: diff --git a/data/saas/dataset/zendesk_dataset.yml b/data/saas/dataset/zendesk_dataset.yml index 95dd961b3..1980c144b 100644 --- a/data/saas/dataset/zendesk_dataset.yml +++ b/data/saas/dataset/zendesk_dataset.yml @@ -1,5 +1,5 @@ dataset: - - fides_key: zendesk_connector_example + - fides_key: instance_fides_key name: Zendesk Dataset description: A sample dataset representing the Zendesk connector for Fidesops collections: diff --git a/saas_connector_registry.toml b/saas_connector_registry.toml index a5efff4fb..1851fed5b 100644 --- a/saas_connector_registry.toml +++ b/saas_connector_registry.toml @@ -1,4 +1,64 @@ +[adobe_campaign] +config = "data/saas/config/adobe_campaign_config.yml" +dataset = "data/saas/dataset/adobe_campaign_dataset.yml" +icon = "data/saas/icon/adobe.svg" + +[auth0] +config = "data/saas/config/auth0_config.yml" +dataset = "data/saas/dataset/auth0_dataset.yml" +icon = "data/saas/icon/default.svg" + +[datadog] +config = "data/saas/config/datadog_config.yml" +dataset = "data/saas/dataset/datadog_dataset.yml" +icon = "data/saas/icon/default.svg" + +[hubspot] +config = "data/saas/config/hubspot_config.yml" +dataset = "data/saas/dataset/hubspot_dataset.yml" +icon = "data/saas/icon/hubspot.svg" + +[logi_id] +config = "data/saas/config/logi_id_config.yml" +dataset = "data/saas/dataset/logi_id_dataset.yml" +icon = "data/saas/icon/default.svg" + [mailchimp] config = "data/saas/config/mailchimp_config.yml" dataset = "data/saas/dataset/mailchimp_dataset.yml" -icon = "data/saas/icon/mailchimp.svg" \ No newline at end of file +icon = "data/saas/icon/mailchimp.svg" + +[outreach] +config = "data/saas/config/outreach_config.yml" +dataset = "data/saas/dataset/outreach_dataset.yml" +icon = "data/saas/icon/outreach.svg" + +[salesforce] +config = "data/saas/config/salesforce_config.yml" +dataset = "data/saas/dataset/salesforce_dataset.yml" +icon = "data/saas/icon/salesforce.svg" + +[segment] +config = "data/saas/config/segment_config.yml" +dataset = "data/saas/dataset/segment_dataset.yml" +icon = "data/saas/icon/segment.svg" + +[sendgrid] +config = "data/saas/config/sendgrid_config.yml" +dataset = "data/saas/dataset/sendgrid_dataset.yml" +icon = "data/saas/icon/default.svg" + +[sentry] +config = "data/saas/config/sentry_config.yml" +dataset = "data/saas/dataset/sentry_dataset.yml" +icon = "data/saas/icon/sentry.svg" + +[stripe] +config = "data/saas/config/stripe_config.yml" +dataset = "data/saas/dataset/stripe_dataset.yml" +icon = "data/saas/icon/stripe.svg" + +[zendesk] +config = "data/saas/config/zendesk_config.yml" +dataset = "data/saas/dataset/zendesk_dataset.yml" +icon = "data/saas/icon/zendesk.svg" \ No newline at end of file diff --git a/src/fidesops/ops/api/v1/endpoints/saas_config_endpoints.py b/src/fidesops/ops/api/v1/endpoints/saas_config_endpoints.py index 7c8a92a64..cbc89e9e4 100644 --- a/src/fidesops/ops/api/v1/endpoints/saas_config_endpoints.py +++ b/src/fidesops/ops/api/v1/endpoints/saas_config_endpoints.py @@ -17,10 +17,10 @@ from fidesops.ops.api.v1.endpoints.connection_endpoints import validate_secrets from fidesops.ops.api.v1.scope_registry import ( CONNECTION_AUTHORIZE, - CONNECTION_INSTANTIATE, SAAS_CONFIG_CREATE_OR_UPDATE, SAAS_CONFIG_DELETE, SAAS_CONFIG_READ, + SAAS_CONNECTION_INSTANTIATE, ) from fidesops.ops.api.v1.urn_registry import ( AUTHORIZE, @@ -51,7 +51,7 @@ from fidesops.ops.service.connectors.saas.connector_registry_service import ( ConnectorRegistry, ConnectorTemplate, - create_connection_config_from_template, + create_connection_config_from_template_no_save, create_dataset_config_from_template, load_registry, registry_file, @@ -265,15 +265,17 @@ def authorize_connection( @router.post( SAAS_CONNECTOR_FROM_TEMPLATE, - dependencies=[Security(verify_oauth_client, scopes=[CONNECTION_INSTANTIATE])], + dependencies=[Security(verify_oauth_client, scopes=[SAAS_CONNECTION_INSTANTIATE])], response_model=FidesopsDataset, ) -def instantiate_connection( +def instantiate_connection_from_template( saas_connector_type: str, template_values: SaasConnectionTemplateValues, db: Session = Depends(deps.get_db), ) -> FidesopsDataset: """ + Creates a SaaS Connector and a SaaS Dataset from a template. + Looks up the connector type in the SaaS connector registry and, if all required fields are provided, persists the associated connection config and dataset to the database. """ @@ -298,8 +300,10 @@ def instantiate_connection( ) try: - connection_config: ConnectionConfig = create_connection_config_from_template( - db, connector_template, template_values + connection_config: ConnectionConfig = ( + create_connection_config_from_template_no_save( + db, connector_template, template_values + ) ) except KeyOrNameAlreadyExists as exc: raise HTTPException( @@ -315,6 +319,5 @@ def instantiate_connection( dataset_config: DatasetConfig = create_dataset_config_from_template( db, connection_config, connector_template, template_values ) - # TODO Roll back if there's an error. return dataset_config.dataset diff --git a/src/fidesops/ops/api/v1/scope_registry.py b/src/fidesops/ops/api/v1/scope_registry.py index 3c72bd33d..4e6122ffe 100644 --- a/src/fidesops/ops/api/v1/scope_registry.py +++ b/src/fidesops/ops/api/v1/scope_registry.py @@ -15,7 +15,7 @@ CONNECTION_READ = "connection:read" CONNECTION_DELETE = "connection:delete" CONNECTION_AUTHORIZE = "connection:authorize" -CONNECTION_INSTANTIATE = "connection:instantiate" +SAAS_CONNECTION_INSTANTIATE = "connection:instantiate" PRIVACY_REQUEST_READ = "privacy-request:read" PRIVACY_REQUEST_DELETE = "privacy-request:delete" @@ -68,7 +68,7 @@ CONNECTION_CREATE_OR_UPDATE, CONNECTION_DELETE, CONNECTION_AUTHORIZE, - CONNECTION_INSTANTIATE, + SAAS_CONNECTION_INSTANTIATE, CONNECTION_TYPE_READ, DATASET_CREATE_OR_UPDATE, DATASET_DELETE, diff --git a/src/fidesops/ops/models/connectionconfig.py b/src/fidesops/ops/models/connectionconfig.py index 0cefa4673..094de5e92 100644 --- a/src/fidesops/ops/models/connectionconfig.py +++ b/src/fidesops/ops/models/connectionconfig.py @@ -2,7 +2,7 @@ import enum from datetime import datetime -from typing import Any, Optional, Type, TypeVar +from typing import Any, Optional, Type from fideslib.db.base import Base from fideslib.db.base_class import get_key_from_data diff --git a/src/fidesops/ops/service/connectors/saas/connector_registry_service.py b/src/fidesops/ops/service/connectors/saas/connector_registry_service.py index f78cbe3a3..e8f1ed40a 100644 --- a/src/fidesops/ops/service/connectors/saas/connector_registry_service.py +++ b/src/fidesops/ops/service/connectors/saas/connector_registry_service.py @@ -77,12 +77,12 @@ def get_connector_template( return self.__root__.get(connector_type) -def create_connection_config_from_template( +def create_connection_config_from_template_no_save( db: Session, template: ConnectorTemplate, template_values: SaasConnectionTemplateValues, ) -> ConnectionConfig: - """Creates a SaaS connection config from a template""" + """Creates a SaaS connection config from a template without saving it.""" # Load saas config from template config_from_template: Dict = load_config_with_replacement( template.config, "instance_fides_key", template_values.instance_key diff --git a/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py b/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py index 752d7d844..7a7ad3f37 100644 --- a/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py @@ -5,9 +5,9 @@ from starlette.testclient import TestClient from fidesops.ops.api.v1.scope_registry import ( - CONNECTION_INSTANTIATE, CONNECTION_READ, CONNECTION_TYPE_READ, + SAAS_CONNECTION_INSTANTIATE, ) from fidesops.ops.api.v1.urn_registry import ( CONNECTION_TYPE_SECRETS, @@ -231,7 +231,7 @@ def test_instantiate_connection_wrong_scope( def test_instantiate_nonexistent_template( self, generate_auth_header, api_client, base_url ): - auth_header = generate_auth_header(scopes=[CONNECTION_INSTANTIATE]) + auth_header = generate_auth_header(scopes=[SAAS_CONNECTION_INSTANTIATE]) request_body = { "instance_key": "test_instance_key", "secrets": {}, @@ -253,7 +253,7 @@ def test_instantiate_nonexistent_template( def test_instance_key_already_exists( self, generate_auth_header, api_client, base_url, dataset_config ): - auth_header = generate_auth_header(scopes=[CONNECTION_INSTANTIATE]) + auth_header = generate_auth_header(scopes=[SAAS_CONNECTION_INSTANTIATE]) request_body = { "instance_key": dataset_config.fides_key, "secrets": { @@ -279,7 +279,7 @@ def test_instance_key_already_exists( def test_template_secrets_validation( self, generate_auth_header, api_client, base_url, db ): - auth_header = generate_auth_header(scopes=[CONNECTION_INSTANTIATE]) + auth_header = generate_auth_header(scopes=[SAAS_CONNECTION_INSTANTIATE]) # Secrets have one field missing, one field extra request_body = { "instance_key": "secondary_mailchimp_instance", @@ -318,7 +318,7 @@ def test_template_secrets_validation( def test_connection_config_key_already_exists( self, db, generate_auth_header, api_client, base_url, connection_config ): - auth_header = generate_auth_header(scopes=[CONNECTION_INSTANTIATE]) + auth_header = generate_auth_header(scopes=[SAAS_CONNECTION_INSTANTIATE]) request_body = { "instance_key": "secondary_mailchimp_instance", "secrets": { @@ -344,7 +344,7 @@ def test_connection_config_key_already_exists( def test_connection_config_name_already_exists( self, db, generate_auth_header, api_client, base_url, connection_config ): - auth_header = generate_auth_header(scopes=[CONNECTION_INSTANTIATE]) + auth_header = generate_auth_header(scopes=[SAAS_CONNECTION_INSTANTIATE]) request_body = { "instance_key": "secondary_mailchimp_instance", "secrets": { @@ -367,7 +367,43 @@ def test_connection_config_name_already_exists( in resp.json()["detail"] ) - def test_instantiate_mailchimp_connection_from_template( + def test_create_connection_from_template_without_supplying_connection_key( + self, db, generate_auth_header, api_client, base_url + ): + auth_header = generate_auth_header(scopes=[SAAS_CONNECTION_INSTANTIATE]) + request_body = { + "instance_key": "secondary_mailchimp_instance", + "secrets": { + "domain": "test_mailchimp_domain", + "username": "test_mailchimp_username", + "api_key": "test_mailchimp_api_key", + }, + "name": "Mailchimp Connector", + "description": "Mailchimp ConnectionConfig description", + } + resp = api_client.post( + base_url.format(saas_connector_type="mailchimp"), + headers=auth_header, + json=request_body, + ) + assert resp.status_code == 200 + + connection_config = ConnectionConfig.filter( + db=db, conditions=(ConnectionConfig.name == "Mailchimp Connector") + ).first() + dataset_config = DatasetConfig.filter( + db=db, + conditions=(DatasetConfig.fides_key == "secondary_mailchimp_instance"), + ).first() + + assert connection_config is not None + assert dataset_config is not None + + assert connection_config.key == "mailchimp_connector" + dataset_config.delete(db) + connection_config.delete(db) + + def test_instantiate_connection_from_template( self, db, generate_auth_header, api_client, base_url ): connection_config = ConnectionConfig.filter( @@ -380,7 +416,7 @@ def test_instantiate_mailchimp_connection_from_template( ).first() assert dataset_config is None - auth_header = generate_auth_header(scopes=[CONNECTION_INSTANTIATE]) + auth_header = generate_auth_header(scopes=[SAAS_CONNECTION_INSTANTIATE]) request_body = { "instance_key": "secondary_mailchimp_instance", "secrets": { From 7d593883482afe570de2a98294b86b3747ae24b4 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Tue, 16 Aug 2022 16:26:54 -0500 Subject: [PATCH 08/17] Update the fides_keys in the existing saas configs and dataset yamls to have brackets around the "instance_fides_key" to indicate these will be replaced. Update the fides_key definition to allow "" with brackets specifically to pass validation. --- data/saas/config/adobe_campaign_config.yml | 2 +- data/saas/config/auth0_config.yml | 6 +-- data/saas/config/datadog_config.yml | 2 +- data/saas/config/hubspot_config.yml | 6 +-- data/saas/config/logi_id_config.yml | 4 +- data/saas/config/mailchimp_config.yml | 8 +-- data/saas/config/outreach_config.yml | 2 +- data/saas/config/salesforce_config.yml | 24 ++++----- data/saas/config/segment_config.yml | 8 +-- data/saas/config/sendgrid_config.yml | 4 +- data/saas/config/sentry_config.yml | 20 ++++---- data/saas/config/stripe_config.yml | 50 +++++++++---------- data/saas/config/zendesk_config.yml | 12 ++--- data/saas/dataset/adobe_campaign_dataset.yml | 2 +- data/saas/dataset/auth0_dataset.yml | 2 +- data/saas/dataset/datadog_dataset.yml | 2 +- data/saas/dataset/hubspot_dataset.yml | 2 +- data/saas/dataset/logi_id_dataset.yml | 2 +- data/saas/dataset/mailchimp_dataset.yml | 2 +- data/saas/dataset/outreach_dataset.yml | 2 +- data/saas/dataset/saas_example_dataset.yml | 2 +- data/saas/dataset/salesforce_dataset.yml | 2 +- data/saas/dataset/segment_dataset.yml | 2 +- data/saas/dataset/sendgrid_dataset.yml | 2 +- data/saas/dataset/sentry_dataset.yml | 2 +- data/saas/dataset/stripe_dataset.yml | 2 +- data/saas/dataset/zendesk_dataset.yml | 2 +- src/fidesops/ops/schemas/dataset.py | 6 ++- src/fidesops/ops/schemas/shared_schemas.py | 3 ++ .../saas/connector_registry_service.py | 4 +- 30 files changed, 98 insertions(+), 91 deletions(-) diff --git a/data/saas/config/adobe_campaign_config.yml b/data/saas/config/adobe_campaign_config.yml index f41500b48..4cb2d12df 100644 --- a/data/saas/config/adobe_campaign_config.yml +++ b/data/saas/config/adobe_campaign_config.yml @@ -1,5 +1,5 @@ saas_config: - fides_key: instance_fides_key + fides_key: name: Adobe Campaign SaaS Config type: adobe_campaign description: A schema representing the Adobe Campaign connector for Fidesops diff --git a/data/saas/config/auth0_config.yml b/data/saas/config/auth0_config.yml index ba6f2b558..dfbe68486 100644 --- a/data/saas/config/auth0_config.yml +++ b/data/saas/config/auth0_config.yml @@ -1,5 +1,5 @@ saas_config: - fides_key: instance_fides_key + fides_key: name: Auth0 SaaS Config type: auth0 description: A sample schema representing the Auth0 connector for Fidesops @@ -46,7 +46,7 @@ saas_config: param_values: - name: user_id references: - - dataset: instance_fides_key + - dataset: field: users.user_id direction: from - name: user_logs @@ -57,6 +57,6 @@ saas_config: param_values: - name: user_id references: - - dataset: instance_fides_key + - dataset: field: users.user_id direction: from \ No newline at end of file diff --git a/data/saas/config/datadog_config.yml b/data/saas/config/datadog_config.yml index 00a6de8f1..b50f00609 100644 --- a/data/saas/config/datadog_config.yml +++ b/data/saas/config/datadog_config.yml @@ -1,5 +1,5 @@ saas_config: - fides_key: instance_fides_key + fides_key: name: Datadog SaaS Config type: datadog description: A sample schema representing the Datadog connector for Fidesops diff --git a/data/saas/config/hubspot_config.yml b/data/saas/config/hubspot_config.yml index efb69ec01..6fab80034 100644 --- a/data/saas/config/hubspot_config.yml +++ b/data/saas/config/hubspot_config.yml @@ -1,5 +1,5 @@ saas_config: - fides_key: instance_fides_key + fides_key: name: Hubspot SaaS Config type: hubspot description: A sample schema representing the Hubspot connector for Fidesops @@ -64,7 +64,7 @@ saas_config: param_values: - name: contactId references: - - dataset: instance_fides_key + - dataset: field: contacts.id direction: from - name: owners @@ -113,7 +113,7 @@ saas_config: identity: email - name: subscriptionId references: - - dataset: instance_fides_key + - dataset: field: subscription_preferences.id direction: from postprocessors: diff --git a/data/saas/config/logi_id_config.yml b/data/saas/config/logi_id_config.yml index 0479f6b3c..051a4b4e8 100644 --- a/data/saas/config/logi_id_config.yml +++ b/data/saas/config/logi_id_config.yml @@ -1,5 +1,5 @@ saas_config: - fides_key: instance_fides_key + fides_key: name: Logi ID SaaS Config type: logi_id description: A sample schema representing the Logi ID connector for Fidesops @@ -44,7 +44,7 @@ saas_config: param_values: - name: user_id references: - - dataset: instance_fides_key + - dataset: field: users.id direction: from - name: user_claims diff --git a/data/saas/config/mailchimp_config.yml b/data/saas/config/mailchimp_config.yml index 20370d047..644a904b2 100644 --- a/data/saas/config/mailchimp_config.yml +++ b/data/saas/config/mailchimp_config.yml @@ -1,5 +1,5 @@ saas_config: - fides_key: instance_fides_key + fides_key: name: Mailchimp SaaS Config type: mailchimp description: A sample schema representing the Mailchimp connector for Fidesops @@ -32,7 +32,7 @@ saas_config: param_values: - name: conversation_id references: - - dataset: instance_fides_key + - dataset: field: conversations.id direction: from data_path: conversation_messages @@ -80,12 +80,12 @@ saas_config: param_values: - name: list_id references: - - dataset: instance_fides_key + - dataset: field: member.list_id direction: from - name: subscriber_hash references: - - dataset: instance_fides_key + - dataset: field: member.id direction: from body: | diff --git a/data/saas/config/outreach_config.yml b/data/saas/config/outreach_config.yml index c427cdeff..d02508f49 100644 --- a/data/saas/config/outreach_config.yml +++ b/data/saas/config/outreach_config.yml @@ -1,5 +1,5 @@ saas_config: - fides_key: instance_fides_key + fides_key: name: Outreach Example Config type: outreach description: A sample schema representing the Outreach connector for Fidesops diff --git a/data/saas/config/salesforce_config.yml b/data/saas/config/salesforce_config.yml index 43d9188d8..aa1accd33 100644 --- a/data/saas/config/salesforce_config.yml +++ b/data/saas/config/salesforce_config.yml @@ -1,5 +1,5 @@ saas_config: - fides_key: instance_fides_key + fides_key: name: Salesforce SaaS Config type: salesforce description: A sample schema representing the Salesforce connector for Fidesops @@ -87,7 +87,7 @@ saas_config: param_values: - name: contact_id references: - - dataset: instance_fides_key + - dataset: field: contact_list.Id direction: from update: @@ -100,7 +100,7 @@ saas_config: param_values: - name: contact_id references: - - dataset: instance_fides_key + - dataset: field: contacts.Id direction: from - name: case_list @@ -114,7 +114,7 @@ saas_config: param_values: - name: contact_id references: - - dataset: instance_fides_key + - dataset: field: contact_list.Id direction: from data_path: records @@ -126,7 +126,7 @@ saas_config: param_values: - name: case_id references: - - dataset: instance_fides_key + - dataset: field: case_list.Id direction: from update: @@ -139,7 +139,7 @@ saas_config: param_values: - name: case_id references: - - dataset: instance_fides_key + - dataset: field: cases.Id direction: from - name: lead_list @@ -162,7 +162,7 @@ saas_config: param_values: - name: lead_id references: - - dataset: instance_fides_key + - dataset: field: lead_list.Id direction: from update: @@ -175,7 +175,7 @@ saas_config: param_values: - name: lead_id references: - - dataset: instance_fides_key + - dataset: field: leads.Id direction: from - name: accounts @@ -186,7 +186,7 @@ saas_config: param_values: - name: account_id references: - - dataset: instance_fides_key + - dataset: field: contacts.AccountId update: method: PATCH @@ -198,7 +198,7 @@ saas_config: param_values: - name: account_id references: - - dataset: instance_fides_key + - dataset: field: accounts.Id direction: from - name: campaign_member_list @@ -221,7 +221,7 @@ saas_config: param_values: - name: campaign_member_id references: - - dataset: instance_fides_key + - dataset: field: campaign_member_list.Id direction: from update: @@ -234,6 +234,6 @@ saas_config: param_values: - name: campaign_member_id references: - - dataset: instance_fides_key + - dataset: field: campaign_members.Id direction: from diff --git a/data/saas/config/segment_config.yml b/data/saas/config/segment_config.yml index 02c5c71b5..f2a884a85 100644 --- a/data/saas/config/segment_config.yml +++ b/data/saas/config/segment_config.yml @@ -1,5 +1,5 @@ saas_config: - fides_key: instance_fides_key + fides_key: name: Segment SaaS Config type: segment description: A sample schema representing the Segment connector for Fidesops @@ -55,7 +55,7 @@ saas_config: connector_param: namespace_id - name: segment_id references: - - dataset: instance_fides_key + - dataset: field: segment_user.segment_id direction: from data_path: data @@ -84,7 +84,7 @@ saas_config: connector_param: namespace_id - name: segment_id references: - - dataset: instance_fides_key + - dataset: field: segment_user.segment_id direction: from data_path: traits @@ -110,7 +110,7 @@ saas_config: connector_param: namespace_id - name: segment_id references: - - dataset: instance_fides_key + - dataset: field: segment_user.segment_id direction: from data_path: data diff --git a/data/saas/config/sendgrid_config.yml b/data/saas/config/sendgrid_config.yml index db3ed2bbc..58f33cecc 100644 --- a/data/saas/config/sendgrid_config.yml +++ b/data/saas/config/sendgrid_config.yml @@ -1,5 +1,5 @@ saas_config: - fides_key: instance_fides_key + fides_key: name: Sendgrid SaaS Config type: sendgrid description: A sample schema representing the Sendgrid connector for Fidesops @@ -41,6 +41,6 @@ saas_config: param_values: - name: contact_id references: - - dataset: instance_fides_key + - dataset: field: contacts.id direction: from diff --git a/data/saas/config/sentry_config.yml b/data/saas/config/sentry_config.yml index 7cbc1fe1f..75de54823 100644 --- a/data/saas/config/sentry_config.yml +++ b/data/saas/config/sentry_config.yml @@ -1,5 +1,5 @@ saas_config: - fides_key: instance_fides_key + fides_key: name: Sentry SaaS Config type: sentry description: A sample schema representing the Sentry connector for Fidesops @@ -44,7 +44,7 @@ saas_config: param_values: - name: organization_slug references: - - dataset: instance_fides_key + - dataset: field: organizations.slug direction: from postprocessors: @@ -77,7 +77,7 @@ saas_config: param_values: - name: issue_id references: - - dataset: instance_fides_key + - dataset: field: issues.id direction: from body: | @@ -94,12 +94,12 @@ saas_config: param_values: - name: organization_slug references: - - dataset: instance_fides_key + - dataset: field: projects.organization.slug direction: from - name: project_slug references: - - dataset: instance_fides_key + - dataset: field: projects.slug direction: from - name: query @@ -118,12 +118,12 @@ saas_config: param_values: - name: organization_slug references: - - dataset: instance_fides_key + - dataset: field: projects.organization.slug direction: from - name: project_slug references: - - dataset: instance_fides_key + - dataset: field: projects.slug direction: from postprocessors: @@ -138,7 +138,7 @@ saas_config: source: headers rel: next - name: person - after: [instance_fides_key.projects] + after: [.projects] requests: read: method: GET @@ -151,12 +151,12 @@ saas_config: param_values: - name: organization_slug references: - - dataset: instance_fides_key + - dataset: field: projects.organization.slug direction: from - name: project_slug references: - - dataset: instance_fides_key + - dataset: field: projects.slug direction: from - name: query diff --git a/data/saas/config/stripe_config.yml b/data/saas/config/stripe_config.yml index 9edbe02d6..2a8b00d48 100644 --- a/data/saas/config/stripe_config.yml +++ b/data/saas/config/stripe_config.yml @@ -1,5 +1,5 @@ saas_config: - fides_key: instance_fides_key + fides_key: name: Stripe SaaS Config type: stripe description: A sample schema representing the Stripe connector for Fidesops @@ -46,7 +46,7 @@ saas_config: param_values: - name: customer_id references: - - dataset: instance_fides_key + - dataset: field: customer.id direction: from body: | @@ -66,7 +66,7 @@ saas_config: param_values: - name: customer_id references: - - dataset: instance_fides_key + - dataset: field: customer.id direction: from - name: limit @@ -92,12 +92,12 @@ saas_config: param_values: - name: charge_id references: - - dataset: instance_fides_key + - dataset: field: charge.id direction: from - name: payment_intent_id references: - - dataset: instance_fides_key + - dataset: field: payment_intent.id direction: from - name: limit @@ -121,7 +121,7 @@ saas_config: param_values: - name: customer_id references: - - dataset: instance_fides_key + - dataset: field: customer.id direction: from - name: limit @@ -145,7 +145,7 @@ saas_config: param_values: - name: customer_id references: - - dataset: instance_fides_key + - dataset: field: customer.id direction: from - name: type @@ -167,7 +167,7 @@ saas_config: param_values: - name: payment_method_id references: - - dataset: instance_fides_key + - dataset: field: payment_method.id direction: from body: | @@ -187,7 +187,7 @@ saas_config: param_values: - name: customer_id references: - - dataset: instance_fides_key + - dataset: field: customer.id direction: from - name: limit @@ -207,12 +207,12 @@ saas_config: param_values: - name: customer_id references: - - dataset: instance_fides_key + - dataset: field: bank_account.customer direction: from - name: bank_account_id references: - - dataset: instance_fides_key + - dataset: field: bank_account.id direction: from body: | @@ -232,7 +232,7 @@ saas_config: param_values: - name: customer_id references: - - dataset: instance_fides_key + - dataset: field: customer.id direction: from - name: limit @@ -252,12 +252,12 @@ saas_config: param_values: - name: customer_id references: - - dataset: instance_fides_key + - dataset: field: card.customer direction: from - name: card_id references: - - dataset: instance_fides_key + - dataset: field: card.id direction: from body: | @@ -277,7 +277,7 @@ saas_config: param_values: - name: customer_id references: - - dataset: instance_fides_key + - dataset: field: customer.id direction: from - name: limit @@ -299,7 +299,7 @@ saas_config: param_values: - name: customer_id references: - - dataset: instance_fides_key + - dataset: field: customer.id direction: from - name: limit @@ -321,7 +321,7 @@ saas_config: param_values: - name: customer_id references: - - dataset: instance_fides_key + - dataset: field: customer.id direction: from - name: limit @@ -338,12 +338,12 @@ saas_config: param_values: - name: customer_id references: - - dataset: instance_fides_key + - dataset: field: tax_id.customer direction: from - name: tax_id references: - - dataset: instance_fides_key + - dataset: field: tax_id.id direction: from - name: invoice @@ -359,7 +359,7 @@ saas_config: param_values: - name: customer_id references: - - dataset: instance_fides_key + - dataset: field: customer.id direction: from - name: limit @@ -377,7 +377,7 @@ saas_config: param_values: - name: invoice_id references: - - dataset: instance_fides_key + - dataset: field: invoice.id direction: from - name: invoice_item @@ -393,7 +393,7 @@ saas_config: param_values: - name: customer_id references: - - dataset: instance_fides_key + - dataset: field: customer.id direction: from - name: limit @@ -411,7 +411,7 @@ saas_config: param_values: - name: invoice_item_id references: - - dataset: instance_fides_key + - dataset: field: invoice_item.id direction: from - name: subscription @@ -427,7 +427,7 @@ saas_config: param_values: - name: customer_id references: - - dataset: instance_fides_key + - dataset: field: customer.id direction: from - name: limit @@ -444,6 +444,6 @@ saas_config: param_values: - name: subscription_id references: - - dataset: instance_fides_key + - dataset: field: subscription.id direction: from diff --git a/data/saas/config/zendesk_config.yml b/data/saas/config/zendesk_config.yml index b546e4f1c..b0bccfc38 100644 --- a/data/saas/config/zendesk_config.yml +++ b/data/saas/config/zendesk_config.yml @@ -1,5 +1,5 @@ saas_config: - fides_key: instance_fides_key + fides_key: name: Zendesk SaaS Config type: zendesk description: A sample schema representing the Zendesk connector for Fidesops @@ -46,7 +46,7 @@ saas_config: param_values: - name: user_id references: - - dataset: instance_fides_key + - dataset: field: users.id direction: from - name: user_identities @@ -60,7 +60,7 @@ saas_config: param_values: - name: user_id references: - - dataset: instance_fides_key + - dataset: field: users.id direction: from - name: page_size @@ -82,7 +82,7 @@ saas_config: param_values: - name: user_id references: - - dataset: instance_fides_key + - dataset: field: users.id direction: from - name: page_size @@ -99,7 +99,7 @@ saas_config: param_values: - name: ticket_id references: - - dataset: instance_fides_key + - dataset: field: tickets.id direction: from - name: ticket_comments @@ -113,7 +113,7 @@ saas_config: param_values: - name: ticket_id references: - - dataset: instance_fides_key + - dataset: field: tickets.id direction: from - name: page_size diff --git a/data/saas/dataset/adobe_campaign_dataset.yml b/data/saas/dataset/adobe_campaign_dataset.yml index 7b64a2cbd..d1ef51538 100644 --- a/data/saas/dataset/adobe_campaign_dataset.yml +++ b/data/saas/dataset/adobe_campaign_dataset.yml @@ -1,5 +1,5 @@ dataset: - - fides_key: instance_fides_key + - fides_key: name: Adobe Campaign Dataset description: A dataset representing the Adobe Campaign connector for Fidesops collections: diff --git a/data/saas/dataset/auth0_dataset.yml b/data/saas/dataset/auth0_dataset.yml index 0cae7177c..b6b087caa 100644 --- a/data/saas/dataset/auth0_dataset.yml +++ b/data/saas/dataset/auth0_dataset.yml @@ -1,5 +1,5 @@ dataset: - - fides_key: instance_fides_key + - fides_key: name: Auth0 Dataset description: A sample dataset representing the Auth0 connector for Fidesops collections: diff --git a/data/saas/dataset/datadog_dataset.yml b/data/saas/dataset/datadog_dataset.yml index 15cc20aa0..6e08ac53a 100644 --- a/data/saas/dataset/datadog_dataset.yml +++ b/data/saas/dataset/datadog_dataset.yml @@ -1,5 +1,5 @@ dataset: - - fides_key: instance_fides_key + - fides_key: name: Datadog Dataset description: A sample dataset representing the Datadog connector for Fidesops collections: diff --git a/data/saas/dataset/hubspot_dataset.yml b/data/saas/dataset/hubspot_dataset.yml index 1e801612f..4d8bc9aa1 100644 --- a/data/saas/dataset/hubspot_dataset.yml +++ b/data/saas/dataset/hubspot_dataset.yml @@ -1,5 +1,5 @@ dataset: - - fides_key: instance_fides_key + - fides_key: name: Hubspot Dataset description: A sample dataset representing the Hubspot connector for Fidesops collections: diff --git a/data/saas/dataset/logi_id_dataset.yml b/data/saas/dataset/logi_id_dataset.yml index 9f1da8a3d..bb0cf1a6b 100644 --- a/data/saas/dataset/logi_id_dataset.yml +++ b/data/saas/dataset/logi_id_dataset.yml @@ -1,5 +1,5 @@ dataset: - - fides_key: instance_fides_key + - fides_key: name: Logi ID Example Dataset description: A sample dataset representing the Logi ID connector for Fidesops collections: diff --git a/data/saas/dataset/mailchimp_dataset.yml b/data/saas/dataset/mailchimp_dataset.yml index 84ff945c2..ac27b0931 100644 --- a/data/saas/dataset/mailchimp_dataset.yml +++ b/data/saas/dataset/mailchimp_dataset.yml @@ -1,5 +1,5 @@ dataset: - - fides_key: instance_fides_key + - fides_key: name: Mailchimp Dataset description: A sample dataset representing the Mailchimp connector for Fidesops collections: diff --git a/data/saas/dataset/outreach_dataset.yml b/data/saas/dataset/outreach_dataset.yml index e79d34f74..cfbc732e1 100644 --- a/data/saas/dataset/outreach_dataset.yml +++ b/data/saas/dataset/outreach_dataset.yml @@ -1,5 +1,5 @@ dataset: - - fides_key: instance_fides_key + - fides_key: name: Outreach Dataset description: A sample dataset representing the Outreach connector for Fidesops collections: diff --git a/data/saas/dataset/saas_example_dataset.yml b/data/saas/dataset/saas_example_dataset.yml index dd9a0f116..00fa9bf2d 100644 --- a/data/saas/dataset/saas_example_dataset.yml +++ b/data/saas/dataset/saas_example_dataset.yml @@ -1,5 +1,5 @@ dataset: - - fides_key: instance_fides_key + - fides_key: name: SaaS Example Dataset description: A sample dataset representing a SaaS connector for Fidesops collections: diff --git a/data/saas/dataset/salesforce_dataset.yml b/data/saas/dataset/salesforce_dataset.yml index d80d96589..6302a5e44 100644 --- a/data/saas/dataset/salesforce_dataset.yml +++ b/data/saas/dataset/salesforce_dataset.yml @@ -1,5 +1,5 @@ dataset: - - fides_key: instance_fides_key + - fides_key: name: Salesforce Example Dataset description: A sample dataset representing the Salesforce connector for Fidesops collections: diff --git a/data/saas/dataset/segment_dataset.yml b/data/saas/dataset/segment_dataset.yml index 578620eaf..473d93dd5 100644 --- a/data/saas/dataset/segment_dataset.yml +++ b/data/saas/dataset/segment_dataset.yml @@ -1,5 +1,5 @@ dataset: - - fides_key: instance_fides_key + - fides_key: name: Segment Dataset description: A sample dataset representing the Segment connector for Fidesops collections: diff --git a/data/saas/dataset/sendgrid_dataset.yml b/data/saas/dataset/sendgrid_dataset.yml index d51846066..dcb625bf8 100644 --- a/data/saas/dataset/sendgrid_dataset.yml +++ b/data/saas/dataset/sendgrid_dataset.yml @@ -1,5 +1,5 @@ dataset: - - fides_key: instance_fides_key + - fides_key: name: Sendgrid Dataset description: A sample dataset representing the Sendgrid connector for Fidesops collections: diff --git a/data/saas/dataset/sentry_dataset.yml b/data/saas/dataset/sentry_dataset.yml index 729d2ba8b..41617b09c 100644 --- a/data/saas/dataset/sentry_dataset.yml +++ b/data/saas/dataset/sentry_dataset.yml @@ -1,5 +1,5 @@ dataset: - - fides_key: instance_fides_key + - fides_key: name: Sentry Dataset description: A sample dataset representing the Sentry connector for Fidesops collections: diff --git a/data/saas/dataset/stripe_dataset.yml b/data/saas/dataset/stripe_dataset.yml index 3ad759755..212e5f08e 100644 --- a/data/saas/dataset/stripe_dataset.yml +++ b/data/saas/dataset/stripe_dataset.yml @@ -1,5 +1,5 @@ dataset: - - fides_key: instance_fides_key + - fides_key: name: Stripe Dataset description: A sample dataset representing the Stripe connector for Fidesops collections: diff --git a/data/saas/dataset/zendesk_dataset.yml b/data/saas/dataset/zendesk_dataset.yml index 1980c144b..fdbc5a3e9 100644 --- a/data/saas/dataset/zendesk_dataset.yml +++ b/data/saas/dataset/zendesk_dataset.yml @@ -1,5 +1,5 @@ dataset: - - fides_key: instance_fides_key + - fides_key: name: Zendesk Dataset description: A sample dataset representing the Zendesk connector for Fidesops collections: diff --git a/src/fidesops/ops/schemas/dataset.py b/src/fidesops/ops/schemas/dataset.py index 2787e75bf..1e0babfea 100644 --- a/src/fidesops/ops/schemas/dataset.py +++ b/src/fidesops/ops/schemas/dataset.py @@ -1,7 +1,8 @@ from typing import Any, Dict, List, Optional from fideslang.models import Dataset, DatasetCollection, DatasetFieldBase -from pydantic import BaseModel, ConstrainedStr, validator +from fideslang.validation import FidesKey +from pydantic import BaseModel, ConstrainedStr, Field, validator from fidesops.ops.common_exceptions import ( InvalidDataLengthValidationError, @@ -201,6 +202,9 @@ def valid_data_categories( class FidesopsDataset(Dataset): """Overrides fideslang Collection model with additional Fidesops annotations""" + fides_key: FidesOpsKey = Field( + description="A unique key used to identify this resource." + ) fidesops_meta: Optional[FidesopsDatasetMeta] collections: List[FidesopsDatasetCollection] """Overrides fideslang.models.Collection.collections""" diff --git a/src/fidesops/ops/schemas/shared_schemas.py b/src/fidesops/ops/schemas/shared_schemas.py index b45a590d2..afa4d60e3 100644 --- a/src/fidesops/ops/schemas/shared_schemas.py +++ b/src/fidesops/ops/schemas/shared_schemas.py @@ -11,6 +11,9 @@ class FidesOpsKey(FidesKey): @classmethod def validate(cls, value: Optional[str]) -> Optional[str]: """Throws ValueError if val is not a valid FidesKey""" + if value == "": + return value + if value is not None and not cls.regex.match(value): raise ValueError( "FidesKey must only contain alphanumeric characters, '.', '_' or '-'." diff --git a/src/fidesops/ops/service/connectors/saas/connector_registry_service.py b/src/fidesops/ops/service/connectors/saas/connector_registry_service.py index e8f1ed40a..728a95184 100644 --- a/src/fidesops/ops/service/connectors/saas/connector_registry_service.py +++ b/src/fidesops/ops/service/connectors/saas/connector_registry_service.py @@ -85,7 +85,7 @@ def create_connection_config_from_template_no_save( """Creates a SaaS connection config from a template without saving it.""" # Load saas config from template config_from_template: Dict = load_config_with_replacement( - template.config, "instance_fides_key", template_values.instance_key + template.config, "", template_values.instance_key ) # Create SaaS ConnectionConfig @@ -112,7 +112,7 @@ def create_dataset_config_from_template( ) -> DatasetConfig: """Creates a DatasetConfig from a template and associates it with a ConnectionConfig""" dataset_from_template: Dict = load_dataset_with_replacement( - template.dataset, "instance_fides_key", template_values.instance_key + template.dataset, "", template_values.instance_key )[0] data = { "connection_config_id": connection_config.id, From 70d6404f02acb36df8b9da8abb4b00af1c01869a Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Tue, 16 Aug 2022 16:40:58 -0500 Subject: [PATCH 09/17] Fix a side effect on a separate endpoint that returns the types of secrets that should be supplied for a given connector. Use the saas config type instead of the fides key for the model title. Add test verifying that fides key /instance key validation works as expected. --- .../connection_secrets_saas.py | 4 +-- .../test_connection_template_endpoints.py | 26 ++++++++++++++++++- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/src/fidesops/ops/schemas/connection_configuration/connection_secrets_saas.py b/src/fidesops/ops/schemas/connection_configuration/connection_secrets_saas.py index 173cf39ed..0780a7288 100644 --- a/src/fidesops/ops/schemas/connection_configuration/connection_secrets_saas.py +++ b/src/fidesops/ops/schemas/connection_configuration/connection_secrets_saas.py @@ -58,9 +58,9 @@ def get_saas_schema(self) -> Type[SaaSSchema]: if connector_param.default_value else (str, ...) ) - SaaSSchema.__doc__ = f"{str(self.saas_config.type).capitalize()} secrets schema" # Dynamically override the docstring + SaaSSchema.__doc__ = f"{str(self.saas_config.type).capitalize()} secrets schema" # Dynamically override the docstring to create a description model: Type[SaaSSchema] = create_model( - f"{self.saas_config.fides_key}_schema", + f"{self.saas_config.type}_schema", **field_definitions, __base__=SaaSSchema, ) diff --git a/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py b/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py index 7a7ad3f37..400b2c590 100644 --- a/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py @@ -192,7 +192,7 @@ def test_get_connection_secret_schema_hubspot( ) assert resp.json() == { - "title": "hubspot_connector_example_schema", + "title": "hubspot_schema", "description": "Hubspot secrets schema", "type": "object", "properties": { @@ -403,6 +403,30 @@ def test_create_connection_from_template_without_supplying_connection_key( dataset_config.delete(db) connection_config.delete(db) + def test_invalid_instance_key(self, db, generate_auth_header, api_client, base_url): + auth_header = generate_auth_header(scopes=[SAAS_CONNECTION_INSTANTIATE]) + request_body = { + "instance_key": "< this is an invalid key! >", + "secrets": { + "domain": "test_mailchimp_domain", + "username": "test_mailchimp_username", + "api_key": "test_mailchimp_api_key", + }, + "name": "Mailchimp Connector", + "description": "Mailchimp ConnectionConfig description", + "key": "mailchimp_connection_config", + } + resp = api_client.post( + base_url.format(saas_connector_type="mailchimp"), + headers=auth_header, + json=request_body, + ) + assert resp.json()["detail"][0] == { + "loc": ["body", "instance_key"], + "msg": "FidesKey must only contain alphanumeric characters, '.', '_' or '-'.", + "type": "value_error", + } + def test_instantiate_connection_from_template( self, db, generate_auth_header, api_client, base_url ): From 6239ccca5f3b0ebc6fb74bcf3f29c89790e33c91 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Tue, 16 Aug 2022 17:37:39 -0500 Subject: [PATCH 10/17] - Update CHANGELOG - Add new endpoint to postman collection - Add drafts doc. - Update old response body in docs for connection types. --- CHANGELOG.md | 1 + data/saas/dataset/saas_example_dataset.yml | 2 +- docs/fidesops/docs/guides/connection_types.md | 131 +++++++++++++++--- .../postman/Fidesops.postman_collection.json | 49 ++++++- src/fidesops/ops/schemas/dataset.py | 1 - .../test_connection_template_endpoints.py | 4 + .../test_connection_secrets_saas.py | 4 +- 7 files changed, 166 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 66bb45570..b10a9b2f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ The types of changes are: * Access support for Datadog Logs [#1060](https://github.com/ethyca/fidesops/pull/1060) * Access and erasure support for Logi ID [#1074](https://github.com/ethyca/fidesops/pull/1074) +* Add an endpoint that allows you to create a Saas connector and all supporting resources with a single request [#1076](https://github.com/ethyca/fidesops/pull/1076) ### Developer Experience diff --git a/data/saas/dataset/saas_example_dataset.yml b/data/saas/dataset/saas_example_dataset.yml index 00fa9bf2d..1974cb279 100644 --- a/data/saas/dataset/saas_example_dataset.yml +++ b/data/saas/dataset/saas_example_dataset.yml @@ -1,5 +1,5 @@ dataset: - - fides_key: + - fides_key: saas_connector_example name: SaaS Example Dataset description: A sample dataset representing a SaaS connector for Fidesops collections: diff --git a/docs/fidesops/docs/guides/connection_types.md b/docs/fidesops/docs/guides/connection_types.md index cc64fa446..9d411d333 100644 --- a/docs/fidesops/docs/guides/connection_types.md +++ b/docs/fidesops/docs/guides/connection_types.md @@ -10,24 +10,92 @@ database options and third party API services with which fidesops can communicat ```json title="GET /api/v1/connection_type" { "items": [ - "bigquery", - "hubspot", - "mailchimp", - "mariadb", - "mongodb", - "mssql", - "mysql", - "outreach", - "postgres", - "redshift", - "salesforce", - "segment", - "sentry", - "snowflake", - "stripe", - "zendesk" + { + "identifier": "bigquery", + "type": "database" + }, + { + "identifier": "mariadb", + "type": "database" + }, + { + "identifier": "mongodb", + "type": "database" + }, + { + "identifier": "mssql", + "type": "database" + }, + { + "identifier": "mysql", + "type": "database" + }, + { + "identifier": "postgres", + "type": "database" + }, + { + "identifier": "redshift", + "type": "database" + }, + { + "identifier": "snowflake", + "type": "database" + }, + { + "identifier": "adobe_campaign", + "type": "saas" + }, + { + "identifier": "auth0", + "type": "saas" + }, + { + "identifier": "datadog", + "type": "saas" + }, + { + "identifier": "hubspot", + "type": "saas" + }, + { + "identifier": "logi_id", + "type": "saas" + }, + { + "identifier": "mailchimp", + "type": "saas" + }, + { + "identifier": "outreach", + "type": "saas" + }, + { + "identifier": "salesforce", + "type": "saas" + }, + { + "identifier": "segment", + "type": "saas" + }, + { + "identifier": "sendgrid", + "type": "saas" + }, + { + "identifier": "sentry", + "type": "saas" + }, + { + "identifier": "stripe", + "type": "saas" + }, + { + "identifier": "zendesk", + "type": "saas" + } ], - "total": 15, + "total": 21, "page": 1, "size": 50 } @@ -40,7 +108,7 @@ To view the secrets needed to authenticate with a given connection, visit `GET / ### Example ```json title="GET /api/v1/connection_type/sentry/secret" { - "title": "sentry_connector_schema", + "title": "sentry_schema", "description": "Sentry secrets schema", "type": "object", "properties": { @@ -59,4 +127,29 @@ To view the secrets needed to authenticate with a given connection, visit `GET / ], "additionalProperties": false } -``` \ No newline at end of file +``` + +## Setting up a SaaS Connector from a Template + +To create all the resources necessary to set up a SaaS Connector in one request, you can create a connector from +a template. + +This creates a `saas` ConnectionConfig for you with your supplied name and description, with your supplied `secrets`. +In the example below, we're creating a `mailchimp` saas connector, so you should supply the relevant mailchimp `secrets`. +Your `instance_key` will become the identifier for the related `DatasetConfig` resource. By default, the saas connection config +is enabled, with write access. + + +```json title="POST /connection/instantiate/mailchimp" +{ + "name": "My Mailchimp connector", + "description": "Production Mailchimp Instance", + "secrets": { + "domain": "{{mailchimp_domain}}", + "api_key": "{{mailchimp_api_key}}", + "username": "{{mailchimp_username}}" + }, + "instance_key": "primary_mailchimp", +} +``` + diff --git a/docs/fidesops/docs/postman/Fidesops.postman_collection.json b/docs/fidesops/docs/postman/Fidesops.postman_collection.json index 67b81aa1a..df3e5f95e 100644 --- a/docs/fidesops/docs/postman/Fidesops.postman_collection.json +++ b/docs/fidesops/docs/postman/Fidesops.postman_collection.json @@ -1,6 +1,6 @@ { "info": { - "_postman_id": "2244490f-dd6c-4811-bcd7-40b560cec834", + "_postman_id": "645bae7d-d9af-4b08-84cf-b98d9ae26014", "name": "Fidesops", "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" }, @@ -72,7 +72,7 @@ "header": [], "body": { "mode": "raw", - "raw": "[\n \"client:create\",\n \"client:update\",\n \"client:read\",\n \"client:delete\",\n \"config:read\",\n \"connection_type:read\",\n \"connection:read\",\n \"connection:create_or_update\",\n \"connection:delete\",\n \"dataset:create_or_update\",\n \"dataset:delete\",\n \"dataset:read\",\n \"encryption:exec\",\n \"policy:create_or_update\",\n \"policy:read\",\n \"policy:delete\",\n \"privacy-request:read\",\n \"privacy-request:delete\",\n \"rule:create_or_update\",\n \"rule:read\",\n \"rule:delete\",\n \"scope:read\",\n \"storage:create_or_update\",\n \"storage:delete\",\n \"storage:read\",\n \"privacy-request:resume\",\n \"webhook:create_or_update\",\n \"webhook:read\",\n \"webhook:delete\",\n \"saas_config:create_or_update\",\n \"saas_config:read\",\n \"saas_config:delete\",\n \"privacy-request:review\",\n \"user:create\",\n \"user:delete\"\n]", + "raw": "[\n \"client:create\",\n \"client:update\",\n \"client:read\",\n \"client:delete\",\n \"config:read\",\n \"connection_type:read\",\n \"connection:read\",\n \"connection:create_or_update\",\n \"connection:delete\",\n \"connection:instantiate\",\n \"dataset:create_or_update\",\n \"dataset:delete\",\n \"dataset:read\",\n \"encryption:exec\",\n \"policy:create_or_update\",\n \"policy:read\",\n \"policy:delete\",\n \"privacy-request:read\",\n \"privacy-request:delete\",\n \"rule:create_or_update\",\n \"rule:read\",\n \"rule:delete\",\n \"scope:read\",\n \"storage:create_or_update\",\n \"storage:delete\",\n \"storage:read\",\n \"privacy-request:resume\",\n \"webhook:create_or_update\",\n \"webhook:read\",\n \"webhook:delete\",\n \"saas_config:create_or_update\",\n \"saas_config:read\",\n \"saas_config:delete\",\n \"privacy-request:review\",\n \"user:create\",\n \"user:delete\"\n]", "options": { "raw": { "language": "json" @@ -3281,6 +3281,44 @@ } }, "response": [] + }, + { + "name": "Create From Template", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"name\": \"{{saas_connector_type}} connector\",\n \"instance_key\": \"primary_{{saas_connector_type}}\",\n \"secrets\": {\n \"domain\": \"{{mailchimp_domain}}\",\n \"api_key\": \"{{mailchimp_api_key}}\",\n \"username\": \"{{mailchimp_username}}\"\n }\n\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/connection/instantiate/{{saas_connector_type}}", + "host": [ + "{{host}}" + ], + "path": [ + "connection", + "instantiate", + "{{saas_connector_type}}" + ] + } + }, + "response": [] } ] }, @@ -4138,6 +4176,11 @@ { "key": "manual_connector", "value": "manual_key" + }, + { + "key": "saas_connector_type", + "value": "mailchimp", + "type": "string" } ] -} +} \ No newline at end of file diff --git a/src/fidesops/ops/schemas/dataset.py b/src/fidesops/ops/schemas/dataset.py index 1e0babfea..b26039139 100644 --- a/src/fidesops/ops/schemas/dataset.py +++ b/src/fidesops/ops/schemas/dataset.py @@ -1,7 +1,6 @@ from typing import Any, Dict, List, Optional from fideslang.models import Dataset, DatasetCollection, DatasetFieldBase -from fideslang.validation import FidesKey from pydantic import BaseModel, ConstrainedStr, Field, validator from fidesops.ops.common_exceptions import ( diff --git a/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py b/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py index 400b2c590..a30412752 100644 --- a/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py @@ -477,6 +477,10 @@ def test_instantiate_connection_from_template( assert connection_config.access == AccessLevel.write assert connection_config.connection_type == ConnectionType.saas assert connection_config.saas_config is not None + assert connection_config.disabled is False + assert connection_config.disabled_at is None + assert connection_config.last_test_timestamp is None + assert connection_config.last_test_succeeded is None assert dataset_config.connection_config_id == connection_config.id assert dataset_config.dataset is not None diff --git a/tests/ops/schemas/connection_configuration/test_connection_secrets_saas.py b/tests/ops/schemas/connection_configuration/test_connection_secrets_saas.py index 1aad40d02..45a2a1de9 100644 --- a/tests/ops/schemas/connection_configuration/test_connection_secrets_saas.py +++ b/tests/ops/schemas/connection_configuration/test_connection_secrets_saas.py @@ -22,7 +22,7 @@ def test_get_saas_schema(self, saas_config): that the schema is a subclass of SaaSSchema """ schema = SaaSSchemaFactory(saas_config).get_saas_schema() - assert schema.__name__ == f"{saas_config.fides_key}_schema" + assert schema.__name__ == f"{saas_config.type}_schema" assert issubclass(schema.__base__, SaaSSchema) def test_validation( @@ -43,7 +43,7 @@ def test_missing_fields(self, saas_config: SaaSConfig): if not connector_param.default_value ] assert ( - f"{saas_config.fides_key}_schema must be supplied all of: " + f"{saas_config.type}_schema must be supplied all of: " f"[{', '.join(required_fields)}]." in str(exc.value) ) From 8acd9a05b5ddf9f1ecb715bf3800d24dc71bc211 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Wed, 17 Aug 2022 09:05:49 -0500 Subject: [PATCH 11/17] Replace the with a properly formatted fides_key in the saas fixtures. --- .../saas/connector_registry_service.py | 5 ++++- .../fixtures/saas/adobe_campaign_fixtures.py | 17 +++++++++++---- tests/ops/fixtures/saas/auth0_fixtures.py | 14 +++++++++---- tests/ops/fixtures/saas/datadog_fixtures.py | 18 ++++++++++++---- tests/ops/fixtures/saas/hubspot_fixtures.py | 21 ++++++++++++++----- tests/ops/fixtures/saas/logi_id_fixtures.py | 18 ++++++++++++---- tests/ops/fixtures/saas/mailchimp_fixtures.py | 18 ++++++++++++---- tests/ops/fixtures/saas/outreach_fixtures.py | 18 ++++++++++++---- .../ops/fixtures/saas/salesforce_fixtures.py | 18 ++++++++++++---- tests/ops/fixtures/saas/segment_fixtures.py | 18 ++++++++++++---- tests/ops/fixtures/saas/sendgrid_fixtures.py | 18 ++++++++++++---- tests/ops/fixtures/saas/sentry_fixtures.py | 14 +++++++++---- tests/ops/fixtures/saas/stripe_fixtures.py | 14 +++++++++---- tests/ops/fixtures/saas/zendesk_fixtures.py | 18 ++++++++++++---- 14 files changed, 175 insertions(+), 54 deletions(-) diff --git a/src/fidesops/ops/service/connectors/saas/connector_registry_service.py b/src/fidesops/ops/service/connectors/saas/connector_registry_service.py index 728a95184..fa22e9fc0 100644 --- a/src/fidesops/ops/service/connectors/saas/connector_registry_service.py +++ b/src/fidesops/ops/service/connectors/saas/connector_registry_service.py @@ -83,7 +83,8 @@ def create_connection_config_from_template_no_save( template_values: SaasConnectionTemplateValues, ) -> ConnectionConfig: """Creates a SaaS connection config from a template without saving it.""" - # Load saas config from template + # Load saas config from template and replace every instance of "" with the fides_key + # the user has chosen config_from_template: Dict = load_config_with_replacement( template.config, "", template_values.instance_key ) @@ -111,6 +112,8 @@ def create_dataset_config_from_template( template_values: SaasConnectionTemplateValues, ) -> DatasetConfig: """Creates a DatasetConfig from a template and associates it with a ConnectionConfig""" + # Load the dataset config from template and replace every instance of "" with the fides_key + # the user has chosen dataset_from_template: Dict = load_dataset_with_replacement( template.dataset, "", template_values.instance_key )[0] diff --git a/tests/ops/fixtures/saas/adobe_campaign_fixtures.py b/tests/ops/fixtures/saas/adobe_campaign_fixtures.py index 4b7c359d7..d032bd012 100644 --- a/tests/ops/fixtures/saas/adobe_campaign_fixtures.py +++ b/tests/ops/fixtures/saas/adobe_campaign_fixtures.py @@ -12,8 +12,10 @@ ConnectionType, ) from fidesops.ops.models.datasetconfig import DatasetConfig -from fidesops.ops.util.saas_util import load_config -from tests.ops.fixtures.application_fixtures import load_dataset +from fidesops.ops.util.saas_util import ( + load_config_with_replacement, + load_dataset_with_replacement, +) from tests.ops.test_helpers.vault_client import get_secrets secrets = get_secrets("adobe_campaign") @@ -51,12 +53,19 @@ def adobe_campaign_erasure_identity_email() -> str: @pytest.fixture def adobe_campaign_config() -> Dict[str, Any]: - return load_config("data/saas/config/adobe_campaign_config.yml") + return load_config_with_replacement( + "data/saas/config/adobe_campaign_config.yml", + "", + "adobe_campaign_instance", + ) @pytest.fixture def adobe_campaign_dataset() -> Dict[str, Any]: - return load_dataset("data/saas/dataset/adobe_campaign_dataset.yml")[0] + return load_dataset_with_replacement( + "data/saas/dataset/adobe_campaign_dataset.yml" "", + "adobe_campaign_instance", + )[0] @pytest.fixture(scope="function") diff --git a/tests/ops/fixtures/saas/auth0_fixtures.py b/tests/ops/fixtures/saas/auth0_fixtures.py index f252323cd..c796949a9 100644 --- a/tests/ops/fixtures/saas/auth0_fixtures.py +++ b/tests/ops/fixtures/saas/auth0_fixtures.py @@ -14,8 +14,10 @@ ConnectionType, ) from fidesops.ops.models.datasetconfig import DatasetConfig -from tests.ops.fixtures.application_fixtures import load_dataset -from tests.ops.fixtures.saas_example_fixtures import load_config +from fidesops.ops.util.saas_util import ( + load_config_with_replacement, + load_dataset_with_replacement, +) from tests.ops.test_helpers.saas_test_utils import poll_for_existence from tests.ops.test_helpers.vault_client import get_secrets @@ -43,12 +45,16 @@ def auth0_erasure_identity_email(): @pytest.fixture def auth0_config() -> Dict[str, Any]: - return load_config("data/saas/config/auth0_config.yml") + return load_config_with_replacement( + "data/saas/config/auth0_config.yml", "", "auth_0_instance" + ) @pytest.fixture def auth0_dataset() -> Dict[str, Any]: - return load_dataset("data/saas/dataset/auth0_dataset.yml")[0] + return load_dataset_with_replacement( + "data/saas/dataset/auth0_dataset.yml", "", "auth_0_instance" + )[0] @pytest.fixture(scope="function") diff --git a/tests/ops/fixtures/saas/datadog_fixtures.py b/tests/ops/fixtures/saas/datadog_fixtures.py index de9d23e4a..5210fe4a3 100644 --- a/tests/ops/fixtures/saas/datadog_fixtures.py +++ b/tests/ops/fixtures/saas/datadog_fixtures.py @@ -11,8 +11,10 @@ ConnectionType, ) from fidesops.ops.models.datasetconfig import DatasetConfig -from fidesops.ops.util.saas_util import load_config -from tests.ops.fixtures.application_fixtures import load_dataset +from fidesops.ops.util.saas_util import ( + load_config_with_replacement, + load_dataset_with_replacement, +) from tests.ops.test_helpers.vault_client import get_secrets secrets = get_secrets("datadog") @@ -36,12 +38,20 @@ def datadog_identity_email(saas_config): @pytest.fixture def datadog_config() -> Dict[str, Any]: - return load_config("data/saas/config/datadog_config.yml")[0] + return load_config_with_replacement( + "data/saas/config/datadog_config.yml", + "", + "datadog_instance", + )[0] @pytest.fixture def datadog_dataset() -> Dict[str, Any]: - return load_dataset("data/saas/dataset/datadog_dataset.yml")[0] + return load_dataset_with_replacement( + "data/saas/dataset/datadog_dataset.yml", + "", + "datadog_instance", + )[0] @pytest.fixture(scope="function") diff --git a/tests/ops/fixtures/saas/hubspot_fixtures.py b/tests/ops/fixtures/saas/hubspot_fixtures.py index 6b4632594..0844f053d 100644 --- a/tests/ops/fixtures/saas/hubspot_fixtures.py +++ b/tests/ops/fixtures/saas/hubspot_fixtures.py @@ -14,8 +14,11 @@ from fidesops.ops.models.datasetconfig import DatasetConfig from fidesops.ops.schemas.saas.shared_schemas import HTTPMethod, SaaSRequestParams from fidesops.ops.service.connectors import SaaSConnector -from fidesops.ops.util.saas_util import format_body, load_config -from tests.ops.fixtures.application_fixtures import load_dataset +from fidesops.ops.util.saas_util import ( + format_body, + load_config_with_replacement, + load_dataset_with_replacement, +) from tests.ops.test_helpers.saas_test_utils import poll_for_existence from tests.ops.test_helpers.vault_client import get_secrets @@ -46,12 +49,20 @@ def hubspot_erasure_identity_email(): @pytest.fixture def hubspot_config() -> Dict[str, Any]: - return load_config("data/saas/config/hubspot_config.yml") + return load_config_with_replacement( + "data/saas/config/hubspot_config.yml", + "", + "hubspot_instance", + ) @pytest.fixture def hubspot_dataset() -> Dict[str, Any]: - return load_dataset("data/saas/dataset/hubspot_dataset.yml")[0] + return load_dataset_with_replacement( + "data/saas/dataset/hubspot_dataset.yml", + "", + "hubspot_instance", + )[0] @pytest.fixture(scope="function") @@ -60,7 +71,7 @@ def connection_config_hubspot( hubspot_config, hubspot_secrets, ) -> Generator: - fides_key = hubspot_config["fides_key"] + fides_key = "hubspot_instance" connection_config = ConnectionConfig.create( db=db, data={ diff --git a/tests/ops/fixtures/saas/logi_id_fixtures.py b/tests/ops/fixtures/saas/logi_id_fixtures.py index b36d19436..516ac63c3 100644 --- a/tests/ops/fixtures/saas/logi_id_fixtures.py +++ b/tests/ops/fixtures/saas/logi_id_fixtures.py @@ -14,8 +14,10 @@ ConnectionType, ) from fidesops.ops.models.datasetconfig import DatasetConfig -from tests.ops.fixtures.application_fixtures import load_dataset -from tests.ops.fixtures.saas_example_fixtures import load_config +from fidesops.ops.util.saas_util import ( + load_config_with_replacement, + load_dataset_with_replacement, +) from tests.ops.test_helpers.saas_test_utils import poll_for_existence from tests.ops.test_helpers.vault_client import get_secrets @@ -47,12 +49,20 @@ def logi_id_erasure_identity_email(): @pytest.fixture def logi_id_config() -> Dict[str, Any]: - return load_config("data/saas/config/logi_id_config.yml") + return load_config_with_replacement( + "data/saas/config/logi_id_config.yml", + "", + "logi_id_instance", + ) @pytest.fixture def logi_id_dataset() -> Dict[str, Any]: - return load_dataset("data/saas/dataset/logi_id_dataset.yml")[0] + return load_dataset_with_replacement( + "data/saas/dataset/logi_id_dataset.yml", + "", + "logi_id_instance", + )[0] @pytest.fixture(scope="function") diff --git a/tests/ops/fixtures/saas/mailchimp_fixtures.py b/tests/ops/fixtures/saas/mailchimp_fixtures.py index beaaf23b4..087f87479 100644 --- a/tests/ops/fixtures/saas/mailchimp_fixtures.py +++ b/tests/ops/fixtures/saas/mailchimp_fixtures.py @@ -14,8 +14,10 @@ from fidesops.ops.models.datasetconfig import DatasetConfig from fidesops.ops.schemas.saas.shared_schemas import HTTPMethod, SaaSRequestParams from fidesops.ops.service.connectors.saas_connector import SaaSConnector -from fidesops.ops.util.saas_util import load_config -from tests.ops.fixtures.application_fixtures import load_dataset +from fidesops.ops.util.saas_util import ( + load_config_with_replacement, + load_dataset_with_replacement, +) from tests.ops.test_helpers.vault_client import get_secrets secrets = get_secrets("mailchimp") @@ -40,12 +42,20 @@ def mailchimp_identity_email(saas_config): @pytest.fixture def mailchimp_config() -> Dict[str, Any]: - return load_config("data/saas/config/mailchimp_config.yml") + return load_config_with_replacement( + "data/saas/config/mailchimp_config.yml", + "", + "mailchimp_instance", + ) @pytest.fixture def mailchimp_dataset() -> Dict[str, Any]: - return load_dataset("data/saas/dataset/mailchimp_dataset.yml")[0] + return load_dataset_with_replacement( + "data/saas/dataset/mailchimp_dataset.yml", + "", + "mailchimp_instance", + )[0] @pytest.fixture(scope="function") diff --git a/tests/ops/fixtures/saas/outreach_fixtures.py b/tests/ops/fixtures/saas/outreach_fixtures.py index 381577395..fd46067d2 100644 --- a/tests/ops/fixtures/saas/outreach_fixtures.py +++ b/tests/ops/fixtures/saas/outreach_fixtures.py @@ -13,8 +13,10 @@ ConnectionType, ) from fidesops.ops.models.datasetconfig import DatasetConfig -from fidesops.ops.util.saas_util import load_config -from tests.ops.fixtures.application_fixtures import load_dataset +from fidesops.ops.util.saas_util import ( + load_config_with_replacement, + load_dataset_with_replacement, +) from tests.ops.test_helpers.vault_client import get_secrets secrets = get_secrets("outreach") @@ -51,12 +53,20 @@ def outreach_erasure_identity_email() -> str: @pytest.fixture def outreach_config() -> Dict[str, Any]: - return load_config("data/saas/config/outreach_config.yml") + return load_config_with_replacement( + "data/saas/config/outreach_config.yml", + "", + "outreach_instance", + ) @pytest.fixture def outreach_dataset() -> Dict[str, Any]: - return load_dataset("data/saas/dataset/outreach_dataset.yml")[0] + return load_dataset_with_replacement( + "data/saas/dataset/outreach_dataset.yml", + "", + "outreach_dataset", + )[0] @pytest.fixture(scope="function") diff --git a/tests/ops/fixtures/saas/salesforce_fixtures.py b/tests/ops/fixtures/saas/salesforce_fixtures.py index fb3b4304a..832434235 100644 --- a/tests/ops/fixtures/saas/salesforce_fixtures.py +++ b/tests/ops/fixtures/saas/salesforce_fixtures.py @@ -14,8 +14,10 @@ ConnectionType, ) from fidesops.ops.models.datasetconfig import DatasetConfig -from fidesops.ops.util.saas_util import load_config -from tests.ops.fixtures.application_fixtures import load_dataset +from fidesops.ops.util.saas_util import ( + load_config_with_replacement, + load_dataset_with_replacement, +) from tests.ops.test_helpers.vault_client import get_secrets secrets = get_secrets("salesforce") @@ -68,12 +70,20 @@ def salesforce_token(salesforce_secrets) -> str: @pytest.fixture def salesforce_config() -> Dict[str, Any]: - return load_config("data/saas/config/salesforce_config.yml") + return load_config_with_replacement( + "data/saas/config/salesforce_config.yml", + "", + "salesforce_instance", + ) @pytest.fixture def salesforce_dataset() -> Dict[str, Any]: - return load_dataset("data/saas/dataset/salesforce_dataset.yml")[0] + return load_dataset_with_replacement( + "data/saas/dataset/salesforce_dataset.yml", + "", + "salesforce_dataset", + )[0] @pytest.fixture(scope="function") diff --git a/tests/ops/fixtures/saas/segment_fixtures.py b/tests/ops/fixtures/saas/segment_fixtures.py index 7fbb850ab..7e7afd6f9 100644 --- a/tests/ops/fixtures/saas/segment_fixtures.py +++ b/tests/ops/fixtures/saas/segment_fixtures.py @@ -15,8 +15,10 @@ ConnectionType, ) from fidesops.ops.models.datasetconfig import DatasetConfig -from fidesops.ops.util.saas_util import load_config -from tests.ops.fixtures.application_fixtures import load_dataset +from fidesops.ops.util.saas_util import ( + load_config_with_replacement, + load_dataset_with_replacement, +) from tests.ops.test_helpers.saas_test_utils import poll_for_existence from tests.ops.test_helpers.vault_client import get_secrets @@ -53,12 +55,20 @@ def segment_identity_email(saas_config): @pytest.fixture def segment_config() -> Dict[str, Any]: - return load_config("data/saas/config/segment_config.yml") + return load_config_with_replacement( + "data/saas/config/segment_config.yml", + "", + "segment_instance", + ) @pytest.fixture def segment_dataset() -> Dict[str, Any]: - return load_dataset("data/saas/dataset/segment_dataset.yml")[0] + return load_dataset_with_replacement( + "data/saas/dataset/segment_dataset.yml", + "", + "segment_instance", + )[0] @pytest.fixture(scope="function") diff --git a/tests/ops/fixtures/saas/sendgrid_fixtures.py b/tests/ops/fixtures/saas/sendgrid_fixtures.py index 50a64f32d..48c3c4a1b 100644 --- a/tests/ops/fixtures/saas/sendgrid_fixtures.py +++ b/tests/ops/fixtures/saas/sendgrid_fixtures.py @@ -14,8 +14,10 @@ ConnectionType, ) from fidesops.ops.models.datasetconfig import DatasetConfig -from tests.ops.fixtures.application_fixtures import load_dataset -from tests.ops.fixtures.saas_example_fixtures import load_config +from fidesops.ops.util.saas_util import ( + load_config_with_replacement, + load_dataset_with_replacement, +) from tests.ops.test_helpers.saas_test_utils import poll_for_existence from tests.ops.test_helpers.vault_client import get_secrets @@ -46,12 +48,20 @@ def sendgrid_erasure_identity_email(): @pytest.fixture def sendgrid_config() -> Dict[str, Any]: - return load_config("data/saas/config/sendgrid_config.yml") + return load_config_with_replacement( + "data/saas/config/sendgrid_config.yml", + "", + "sendgrid_instance", + ) @pytest.fixture def sendgrid_dataset() -> Dict[str, Any]: - return load_dataset("data/saas/dataset/sendgrid_dataset.yml")[0] + return load_dataset_with_replacement( + "data/saas/dataset/sendgrid_dataset.yml", + "", + "sendgrid_instance", + )[0] @pytest.fixture(scope="function") diff --git a/tests/ops/fixtures/saas/sentry_fixtures.py b/tests/ops/fixtures/saas/sentry_fixtures.py index 9f4eee0cd..b100bbbd4 100644 --- a/tests/ops/fixtures/saas/sentry_fixtures.py +++ b/tests/ops/fixtures/saas/sentry_fixtures.py @@ -11,8 +11,10 @@ ConnectionType, ) from fidesops.ops.models.datasetconfig import DatasetConfig -from fidesops.ops.util.saas_util import load_config -from tests.ops.fixtures.application_fixtures import load_dataset +from fidesops.ops.util.saas_util import ( + load_config_with_replacement, + load_dataset_with_replacement, +) from tests.ops.test_helpers.vault_client import get_secrets secrets = get_secrets("sentry") @@ -44,12 +46,16 @@ def sentry_identity_email(saas_config): @pytest.fixture def sentry_config() -> Dict[str, Any]: - return load_config("data/saas/config/sentry_config.yml") + return load_config_with_replacement( + "data/saas/config/sentry_config.yml", "", "sentry_instance" + ) @pytest.fixture def sentry_dataset() -> Dict[str, Any]: - return load_dataset("data/saas/dataset/sentry_dataset.yml")[0] + return load_dataset_with_replacement( + "data/saas/dataset/sentry_dataset.yml", "", "sentry_dataset" + )[0] @pytest.fixture(scope="function") diff --git a/tests/ops/fixtures/saas/stripe_fixtures.py b/tests/ops/fixtures/saas/stripe_fixtures.py index 4ea8e4888..0a24e8878 100644 --- a/tests/ops/fixtures/saas/stripe_fixtures.py +++ b/tests/ops/fixtures/saas/stripe_fixtures.py @@ -13,8 +13,10 @@ ConnectionType, ) from fidesops.ops.models.datasetconfig import DatasetConfig -from fidesops.ops.util.saas_util import load_config -from tests.ops.fixtures.application_fixtures import load_dataset +from fidesops.ops.util.saas_util import ( + load_config_with_replacement, + load_dataset_with_replacement, +) from tests.ops.test_helpers.vault_client import get_secrets secrets = get_secrets("stripe") @@ -44,12 +46,16 @@ def stripe_erasure_identity_email(): @pytest.fixture def stripe_config() -> Dict[str, Any]: - return load_config("data/saas/config/stripe_config.yml") + return load_config_with_replacement( + "data/saas/config/stripe_config.yml", "", "stripe_instance" + ) @pytest.fixture def stripe_dataset() -> Dict[str, Any]: - return load_dataset("data/saas/dataset/stripe_dataset.yml")[0] + return load_dataset_with_replacement( + "data/saas/dataset/stripe_dataset.yml", "", "strip_dataset" + )[0] @pytest.fixture(scope="function") diff --git a/tests/ops/fixtures/saas/zendesk_fixtures.py b/tests/ops/fixtures/saas/zendesk_fixtures.py index b67121efa..db13496c1 100644 --- a/tests/ops/fixtures/saas/zendesk_fixtures.py +++ b/tests/ops/fixtures/saas/zendesk_fixtures.py @@ -12,8 +12,10 @@ ConnectionType, ) from fidesops.ops.models.datasetconfig import DatasetConfig -from fidesops.ops.util.saas_util import load_config -from tests.ops.fixtures.application_fixtures import load_dataset +from fidesops.ops.util.saas_util import ( + load_config_with_replacement, + load_dataset_with_replacement, +) from tests.ops.test_helpers.vault_client import get_secrets secrets = get_secrets("zendesk") @@ -42,12 +44,20 @@ def zendesk_erasure_identity_email() -> str: @pytest.fixture def zendesk_config() -> Dict[str, Any]: - return load_config("data/saas/config/zendesk_config.yml") + return load_config_with_replacement( + "data/saas/config/zendesk_config.yml", + "", + "zendesk_instance", + ) @pytest.fixture def zendesk_dataset() -> Dict[str, Any]: - return load_dataset("data/saas/dataset/zendesk_dataset.yml")[0] + return load_dataset_with_replacement( + "data/saas/dataset/zendesk_dataset.yml", + "", + "zendesk_instance", + )[0] @pytest.fixture(scope="function") From 96c4990d96868fc38b92d593bcdc4d918f1899e8 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Wed, 17 Aug 2022 09:41:20 -0500 Subject: [PATCH 12/17] If DatasetConfig creation fails, delete the recently created ConnectionConfig. --- .../api/v1/endpoints/saas_config_endpoints.py | 16 +++++-- .../test_connection_template_endpoints.py | 46 ++++++++++++++++++- 2 files changed, 58 insertions(+), 4 deletions(-) diff --git a/src/fidesops/ops/api/v1/endpoints/saas_config_endpoints.py b/src/fidesops/ops/api/v1/endpoints/saas_config_endpoints.py index cbc89e9e4..8606fcecd 100644 --- a/src/fidesops/ops/api/v1/endpoints/saas_config_endpoints.py +++ b/src/fidesops/ops/api/v1/endpoints/saas_config_endpoints.py @@ -11,6 +11,7 @@ HTTP_400_BAD_REQUEST, HTTP_404_NOT_FOUND, HTTP_422_UNPROCESSABLE_ENTITY, + HTTP_500_INTERNAL_SERVER_ERROR, ) from fidesops.ops.api import deps @@ -316,8 +317,17 @@ def instantiate_connection_from_template( ).dict() connection_config.save(db=db) # Not persisted to db until secrets are validated - dataset_config: DatasetConfig = create_dataset_config_from_template( - db, connection_config, connector_template, template_values + try: + dataset_config: DatasetConfig = create_dataset_config_from_template( + db, connection_config, connector_template, template_values + ) + except Exception: + connection_config.delete(db) + raise HTTPException( + status_code=HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"SaaS Connector could not be created from the '{saas_connector_type}' template at this time.", + ) + logger.info( + f"SaaS Connector and Dataset {template_values.instance_key} successfully created from '{saas_connector_type}' template." ) - return dataset_config.dataset diff --git a/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py b/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py index a30412752..1bfffbecb 100644 --- a/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py @@ -1,4 +1,5 @@ from typing import List +from unittest import mock import pytest from fideslib.models.client import ClientDetail @@ -427,6 +428,48 @@ def test_invalid_instance_key(self, db, generate_auth_header, api_client, base_u "type": "value_error", } + @mock.patch( + "fidesops.ops.api.v1.endpoints.saas_config_endpoints.create_dataset_config_from_template" + ) + def test_dataset_config_saving_fails( + self, mock_create_dataset, db, generate_auth_header, api_client, base_url + ): + mock_create_dataset.side_effect = Exception("KeyError") + + auth_header = generate_auth_header(scopes=[SAAS_CONNECTION_INSTANTIATE]) + request_body = { + "instance_key": "secondary_mailchimp_instance", + "secrets": { + "domain": "test_mailchimp_domain", + "username": "test_mailchimp_username", + "api_key": "test_mailchimp_api_key", + }, + "name": "Mailchimp Connector", + "description": "Mailchimp ConnectionConfig description", + "key": "mailchimp_connection_config", + } + resp = api_client.post( + base_url.format(saas_connector_type="mailchimp"), + headers=auth_header, + json=request_body, + ) + assert resp.status_code == 500 + assert ( + resp.json()["detail"] + == "SaaS Connector could not be created from the 'mailchimp' template at this time." + ) + + connection_config = ConnectionConfig.filter( + db=db, conditions=(ConnectionConfig.key == "mailchimp_connection_config") + ).first() + assert connection_config is None + + dataset_config = DatasetConfig.filter( + db=db, + conditions=(DatasetConfig.fides_key == "secondary_mailchimp_instance"), + ).first() + assert dataset_config is None + def test_instantiate_connection_from_template( self, db, generate_auth_header, api_client, base_url ): @@ -436,7 +479,8 @@ def test_instantiate_connection_from_template( assert connection_config is None dataset_config = DatasetConfig.filter( - db=db, conditions=(DatasetConfig.fides_key == "primary_mailchimp_instance") + db=db, + conditions=(DatasetConfig.fides_key == "secondary_mailchimp_instance"), ).first() assert dataset_config is None From d8f7d2174b1c385afaeaf592ce16bc0b1e8e86a1 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Wed, 17 Aug 2022 09:47:27 -0500 Subject: [PATCH 13/17] Address some of the saas integration tests where I've changed the fides_key. --- .../mailchimp_request_overrides.py | 2 +- .../saas/test_adobe_campaign_task.py | 4 ++-- .../ops/integration_tests/saas/test_hubspot_task.py | 6 +++--- .../ops/integration_tests/saas/test_logi_id_task.py | 4 ++-- .../ops/integration_tests/saas/test_segment_task.py | 8 ++++---- .../ops/integration_tests/saas/test_sendgrid_task.py | 2 +- tests/ops/integration_tests/saas/test_sentry_task.py | 12 ++++++------ .../privacy_request/request_runner_service_test.py | 4 ++-- 8 files changed, 21 insertions(+), 21 deletions(-) diff --git a/src/fidesops/ops/service/saas_request/override_implementations/mailchimp_request_overrides.py b/src/fidesops/ops/service/saas_request/override_implementations/mailchimp_request_overrides.py index 01e85a5f5..5fcbcacf3 100644 --- a/src/fidesops/ops/service/saas_request/override_implementations/mailchimp_request_overrides.py +++ b/src/fidesops/ops/service/saas_request/override_implementations/mailchimp_request_overrides.py @@ -40,7 +40,7 @@ def mailchimp_messages_access( - name: conversation_id type: path references: - - dataset: mailchimp_connector_example + - dataset: mailchimp_instance field: conversations.id direction: from data_path: conversation_messages diff --git a/tests/ops/integration_tests/saas/test_adobe_campaign_task.py b/tests/ops/integration_tests/saas/test_adobe_campaign_task.py index bc79fe659..fabd5200f 100644 --- a/tests/ops/integration_tests/saas/test_adobe_campaign_task.py +++ b/tests/ops/integration_tests/saas/test_adobe_campaign_task.py @@ -305,8 +305,8 @@ def test_adobe_campaign_saas_erasure_request_task( # Assert erasure request made to adobe_campaign_user assert x == { - "adobe_campaign_connector_example:profile": 1, - "adobe_campaign_connector_example:marketing_history": 0, + "adobe_instance:profile": 1, + "adobe_instance:marketing_history": 0, } config.execution.masking_strict = masking_strict # Reset diff --git a/tests/ops/integration_tests/saas/test_hubspot_task.py b/tests/ops/integration_tests/saas/test_hubspot_task.py index fe68cc17f..f1c5201b6 100644 --- a/tests/ops/integration_tests/saas/test_hubspot_task.py +++ b/tests/ops/integration_tests/saas/test_hubspot_task.py @@ -175,9 +175,9 @@ def test_saas_erasure_request_task( # Masking request only issued to "contacts" and "subscription_preferences" endpoints assert erasure == { - "hubspot_connector_example:contacts": 1, - "hubspot_connector_example:owners": 0, - "hubspot_connector_example:subscription_preferences": 1, + "husbspot_instance:contacts": 1, + "husbspot_instance:owners": 0, + "husbspot_instance:subscription_preferences": 1, } connector = SaaSConnector(connection_config_hubspot) diff --git a/tests/ops/integration_tests/saas/test_logi_id_task.py b/tests/ops/integration_tests/saas/test_logi_id_task.py index 1b8bc6ae2..0fbe031e2 100644 --- a/tests/ops/integration_tests/saas/test_logi_id_task.py +++ b/tests/ops/integration_tests/saas/test_logi_id_task.py @@ -154,8 +154,8 @@ def test_logi_id_erasure_request_task( db, ) assert erasure == { - "logi_id_connector_example:user_claims": 0, - "logi_id_connector_example:users": 1, + "logi_id_instance:user_claims": 0, + "logi_id_instance:users": 1, } # Verifying user is deleted diff --git a/tests/ops/integration_tests/saas/test_segment_task.py b/tests/ops/integration_tests/saas/test_segment_task.py index 0673aa35b..fb03b8573 100644 --- a/tests/ops/integration_tests/saas/test_segment_task.py +++ b/tests/ops/integration_tests/saas/test_segment_task.py @@ -222,10 +222,10 @@ def test_segment_saas_erasure_request_task( # Assert erasure request made to segment_user - cannot verify success immediately as this can take # days, weeks to process assert x == { - "segment_connector_example:segment_user": 1, - "segment_connector_example:traits": 0, - "segment_connector_example:external_ids": 0, - "segment_connector_example:track_events": 0, + "segment_instance:segment_user": 1, + "segment_instance:traits": 0, + "segment_instance:external_ids": 0, + "segment_instance:track_events": 0, } config.execution.masking_strict = True # Reset diff --git a/tests/ops/integration_tests/saas/test_sendgrid_task.py b/tests/ops/integration_tests/saas/test_sendgrid_task.py index bd098ba74..254f2b21a 100644 --- a/tests/ops/integration_tests/saas/test_sendgrid_task.py +++ b/tests/ops/integration_tests/saas/test_sendgrid_task.py @@ -135,7 +135,7 @@ def test_sendgrid_erasure_request_task( get_cached_data_for_erasures(privacy_request.id), db, ) - assert erasure == {"sendgrid_connector_example:contacts": 1} + assert erasure == {"sendgrid_instance:contacts": 1} error_message = f"Contact with email {sendgrid_erasure_identity_email} could not be deleted in Sendgrid" poll_for_existence( contact_exists, diff --git a/tests/ops/integration_tests/saas/test_sentry_task.py b/tests/ops/integration_tests/saas/test_sentry_task.py index ef7c9bc9e..d261aee59 100644 --- a/tests/ops/integration_tests/saas/test_sentry_task.py +++ b/tests/ops/integration_tests/saas/test_sentry_task.py @@ -376,12 +376,12 @@ def test_sentry_erasure_request_task( # Masking request only issued to "issues" endpoint assert x == { - "sentry_connector:projects": 0, - "sentry_connector:person": 0, - "sentry_connector:issues": 1, - "sentry_connector:organizations": 0, - "sentry_connector:user_feedback": 0, - "sentry_connector:employees": 0, + "sentry_instance:projects": 0, + "sentry_instance:person": 0, + "sentry_instance:issues": 1, + "sentry_instance:organizations": 0, + "sentry_instance:user_feedback": 0, + "sentry_instance:employees": 0, } # Verify the user has been assigned to None diff --git a/tests/ops/service/privacy_request/request_runner_service_test.py b/tests/ops/service/privacy_request/request_runner_service_test.py index 884c71cb1..754a7eda2 100644 --- a/tests/ops/service/privacy_request/request_runner_service_test.py +++ b/tests/ops/service/privacy_request/request_runner_service_test.py @@ -495,7 +495,7 @@ def test_create_and_process_access_request_saas_mailchimp( assert results[key] is not None assert results[key] != {} - result_key_prefix = f"EN_{pr.id}__access_request__mailchimp_connector_example:" + result_key_prefix = f"EN_{pr.id}__access_request__mailchimp_instance:" member_key = result_key_prefix + "member" assert results[member_key][0]["email_address"] == customer_email @@ -600,7 +600,7 @@ def test_create_and_process_access_request_saas_hubspot( assert results[key] is not None assert results[key] != {} - result_key_prefix = f"EN_{pr.id}__access_request__hubspot_connector_example:" + result_key_prefix = f"EN_{pr.id}__access_request__husbspot_instance:" contacts_key = result_key_prefix + "contacts" assert results[contacts_key][0]["properties"]["email"] == customer_email From 23cde59c1340928db825756e3f277eac71ac9b1c Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Wed, 17 Aug 2022 10:14:37 -0500 Subject: [PATCH 14/17] Fix typos. --- tests/ops/fixtures/saas/datadog_fixtures.py | 2 +- tests/ops/fixtures/saas/stripe_fixtures.py | 2 +- tests/ops/integration_tests/saas/test_hubspot_task.py | 6 +++--- .../service/privacy_request/request_runner_service_test.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/ops/fixtures/saas/datadog_fixtures.py b/tests/ops/fixtures/saas/datadog_fixtures.py index 5210fe4a3..230e52075 100644 --- a/tests/ops/fixtures/saas/datadog_fixtures.py +++ b/tests/ops/fixtures/saas/datadog_fixtures.py @@ -42,7 +42,7 @@ def datadog_config() -> Dict[str, Any]: "data/saas/config/datadog_config.yml", "", "datadog_instance", - )[0] + ) @pytest.fixture diff --git a/tests/ops/fixtures/saas/stripe_fixtures.py b/tests/ops/fixtures/saas/stripe_fixtures.py index 0a24e8878..459082209 100644 --- a/tests/ops/fixtures/saas/stripe_fixtures.py +++ b/tests/ops/fixtures/saas/stripe_fixtures.py @@ -54,7 +54,7 @@ def stripe_config() -> Dict[str, Any]: @pytest.fixture def stripe_dataset() -> Dict[str, Any]: return load_dataset_with_replacement( - "data/saas/dataset/stripe_dataset.yml", "", "strip_dataset" + "data/saas/dataset/stripe_dataset.yml", "", "stripe_dataset" )[0] diff --git a/tests/ops/integration_tests/saas/test_hubspot_task.py b/tests/ops/integration_tests/saas/test_hubspot_task.py index f1c5201b6..148170bdb 100644 --- a/tests/ops/integration_tests/saas/test_hubspot_task.py +++ b/tests/ops/integration_tests/saas/test_hubspot_task.py @@ -175,9 +175,9 @@ def test_saas_erasure_request_task( # Masking request only issued to "contacts" and "subscription_preferences" endpoints assert erasure == { - "husbspot_instance:contacts": 1, - "husbspot_instance:owners": 0, - "husbspot_instance:subscription_preferences": 1, + "hubspot_instance:contacts": 1, + "hubspot_instance:owners": 0, + "hubspot_instance:subscription_preferences": 1, } connector = SaaSConnector(connection_config_hubspot) diff --git a/tests/ops/service/privacy_request/request_runner_service_test.py b/tests/ops/service/privacy_request/request_runner_service_test.py index 754a7eda2..b0ff85ccb 100644 --- a/tests/ops/service/privacy_request/request_runner_service_test.py +++ b/tests/ops/service/privacy_request/request_runner_service_test.py @@ -600,7 +600,7 @@ def test_create_and_process_access_request_saas_hubspot( assert results[key] is not None assert results[key] != {} - result_key_prefix = f"EN_{pr.id}__access_request__husbspot_instance:" + result_key_prefix = f"EN_{pr.id}__access_request__hubspot_instance:" contacts_key = result_key_prefix + "contacts" assert results[contacts_key][0]["properties"]["email"] == customer_email From 33c85b236670a91fb111741f8825c5962fa7cb63 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Wed, 17 Aug 2022 10:44:57 -0500 Subject: [PATCH 15/17] Fix typo. --- tests/ops/fixtures/saas/hubspot_fixtures.py | 2 +- tests/ops/fixtures/saas/stripe_fixtures.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/ops/fixtures/saas/hubspot_fixtures.py b/tests/ops/fixtures/saas/hubspot_fixtures.py index 0844f053d..a8a95d0bb 100644 --- a/tests/ops/fixtures/saas/hubspot_fixtures.py +++ b/tests/ops/fixtures/saas/hubspot_fixtures.py @@ -71,7 +71,7 @@ def connection_config_hubspot( hubspot_config, hubspot_secrets, ) -> Generator: - fides_key = "hubspot_instance" + fides_key = hubspot_config["fides_key"] connection_config = ConnectionConfig.create( db=db, data={ diff --git a/tests/ops/fixtures/saas/stripe_fixtures.py b/tests/ops/fixtures/saas/stripe_fixtures.py index 459082209..f08441101 100644 --- a/tests/ops/fixtures/saas/stripe_fixtures.py +++ b/tests/ops/fixtures/saas/stripe_fixtures.py @@ -54,7 +54,9 @@ def stripe_config() -> Dict[str, Any]: @pytest.fixture def stripe_dataset() -> Dict[str, Any]: return load_dataset_with_replacement( - "data/saas/dataset/stripe_dataset.yml", "", "stripe_dataset" + "data/saas/dataset/stripe_dataset.yml", + "", + "stripe_instance", )[0] From a4b2dd776d484e466d4075438c5920869a15f4cf Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Wed, 17 Aug 2022 11:09:47 -0500 Subject: [PATCH 16/17] Fix unrelated bug where hubspot dataset has new datacategories with user-* data categories after the fideslang update, so they would show up if the user picked a "user" data category. --- tests/ops/integration_tests/saas/test_hubspot_task.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/ops/integration_tests/saas/test_hubspot_task.py b/tests/ops/integration_tests/saas/test_hubspot_task.py index 148170bdb..18f2cb355 100644 --- a/tests/ops/integration_tests/saas/test_hubspot_task.py +++ b/tests/ops/integration_tests/saas/test_hubspot_task.py @@ -70,7 +70,12 @@ def test_saas_access_request_task( f"{dataset_name}:contacts", f"{dataset_name}:subscription_preferences", } - assert set(filtered_results[f"{dataset_name}:contacts"][0].keys()) == {"properties"} + assert set(filtered_results[f"{dataset_name}:contacts"][0].keys()) == { + "id", + "createdAt", + "updatedAt", + "properties", + } assert set( filtered_results[f"{dataset_name}:contacts"][0]["properties"].keys() From cec0d027f37054f441e0a8aab550c67b753a3575 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Wed, 17 Aug 2022 11:47:44 -0500 Subject: [PATCH 17/17] Respond to CR. --- .../saas/saas_connector_registry.toml | 0 src/fidesops/ops/api/v1/endpoints/saas_config_endpoints.py | 3 ++- src/fidesops/ops/schemas/shared_schemas.py | 2 ++ .../ops/service/connectors/saas/connector_registry_service.py | 2 +- .../ops/api/v1/endpoints/test_connection_template_endpoints.py | 2 +- 5 files changed, 6 insertions(+), 3 deletions(-) rename saas_connector_registry.toml => data/saas/saas_connector_registry.toml (100%) diff --git a/saas_connector_registry.toml b/data/saas/saas_connector_registry.toml similarity index 100% rename from saas_connector_registry.toml rename to data/saas/saas_connector_registry.toml diff --git a/src/fidesops/ops/api/v1/endpoints/saas_config_endpoints.py b/src/fidesops/ops/api/v1/endpoints/saas_config_endpoints.py index 8606fcecd..77a5e2e1a 100644 --- a/src/fidesops/ops/api/v1/endpoints/saas_config_endpoints.py +++ b/src/fidesops/ops/api/v1/endpoints/saas_config_endpoints.py @@ -25,6 +25,7 @@ ) from fidesops.ops.api.v1.urn_registry import ( AUTHORIZE, + CONNECTION_TYPES, SAAS_CONFIG, SAAS_CONFIG_VALIDATE, SAAS_CONNECTOR_FROM_TEMPLATE, @@ -288,7 +289,7 @@ def instantiate_connection_from_template( if not connector_template: raise HTTPException( status_code=HTTP_404_NOT_FOUND, - detail=f"SaaS connector type '{saas_connector_type}' is not registered.", + detail=f"SaaS connector type '{saas_connector_type}' is not yet available in Fidesops. For a list of available SaaS connectors, refer to {CONNECTION_TYPES}.", ) if DatasetConfig.filter( diff --git a/src/fidesops/ops/schemas/shared_schemas.py b/src/fidesops/ops/schemas/shared_schemas.py index afa4d60e3..8f78c2502 100644 --- a/src/fidesops/ops/schemas/shared_schemas.py +++ b/src/fidesops/ops/schemas/shared_schemas.py @@ -12,6 +12,8 @@ class FidesOpsKey(FidesKey): def validate(cls, value: Optional[str]) -> Optional[str]: """Throws ValueError if val is not a valid FidesKey""" if value == "": + # Ignore in saas templates. This value will be replaced with a + # user-specified value. return value if value is not None and not cls.regex.match(value): diff --git a/src/fidesops/ops/service/connectors/saas/connector_registry_service.py b/src/fidesops/ops/service/connectors/saas/connector_registry_service.py index fa22e9fc0..6d745dffa 100644 --- a/src/fidesops/ops/service/connectors/saas/connector_registry_service.py +++ b/src/fidesops/ops/service/connectors/saas/connector_registry_service.py @@ -26,7 +26,7 @@ ) _registry: Optional[ConnectorRegistry] = None -registry_file = "saas_connector_registry.toml" +registry_file = "data/saas/saas_connector_registry.toml" class ConnectorTemplate(BaseModel): diff --git a/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py b/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py index 1bfffbecb..5851c6f92 100644 --- a/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py @@ -248,7 +248,7 @@ def test_instantiate_nonexistent_template( assert resp.status_code == 404 assert ( resp.json()["detail"] - == f"SaaS connector type '{'does_not_exist'}' is not registered." + == f"SaaS connector type 'does_not_exist' is not yet available in Fidesops. For a list of available SaaS connectors, refer to /connection_type." ) def test_instance_key_already_exists(