diff --git a/CHANGELOG.md b/CHANGELOG.md index 5409e33df..4c2fa59b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ The types of changes are: * Access support for Datadog Logs [#1060](https://github.com/ethyca/fidesops/pull/1060) * Access and erasure support for Logi ID [#1074](https://github.com/ethyca/fidesops/pull/1074) * Adds infra for email config and dispatch [#1059](https://github.com/ethyca/fidesops/pull/1059) +* Add an endpoint that allows you to create a Saas connector and all supporting resources with a single request [#1076](https://github.com/ethyca/fidesops/pull/1076) ### Developer Experience diff --git a/data/saas/config/adobe_campaign_config.yml b/data/saas/config/adobe_campaign_config.yml index 082dd9dc8..4cb2d12df 100644 --- a/data/saas/config/adobe_campaign_config.yml +++ b/data/saas/config/adobe_campaign_config.yml @@ -1,5 +1,5 @@ saas_config: - fides_key: adobe_campaign_connector_example + fides_key: name: Adobe Campaign SaaS Config type: adobe_campaign description: A schema representing the Adobe Campaign connector for Fidesops diff --git a/data/saas/config/auth0_config.yml b/data/saas/config/auth0_config.yml index 8f1241701..dfbe68486 100644 --- a/data/saas/config/auth0_config.yml +++ b/data/saas/config/auth0_config.yml @@ -1,5 +1,5 @@ saas_config: - fides_key: auth0_connector_example + fides_key: name: Auth0 SaaS Config type: auth0 description: A sample schema representing the Auth0 connector for Fidesops @@ -46,7 +46,7 @@ saas_config: param_values: - name: user_id references: - - dataset: auth0_connector_example + - dataset: field: users.user_id direction: from - name: user_logs @@ -57,6 +57,6 @@ saas_config: param_values: - name: user_id references: - - dataset: auth0_connector_example + - dataset: field: users.user_id direction: from \ No newline at end of file diff --git a/data/saas/config/datadog_config.yml b/data/saas/config/datadog_config.yml index bfc1c3235..b50f00609 100644 --- a/data/saas/config/datadog_config.yml +++ b/data/saas/config/datadog_config.yml @@ -1,59 +1,59 @@ saas_config: - - fides_key: datadog_connector_example - name: Datadog SaaS Config - type: datadog - description: A sample schema representing the Datadog connector for Fidesops - version: 0.0.1 + fides_key: + name: Datadog SaaS Config + type: datadog + description: A sample schema representing the Datadog connector for Fidesops + version: 0.0.1 - connector_params: - - name: domain - - name: api_key - - name: app_key - - name: page_size + connector_params: + - name: domain + - name: api_key + - name: app_key + - name: page_size - client_config: - protocol: https - host: + client_config: + protocol: https + host: - test_request: - method: GET - path: /api/v2/logs/events - headers: - - name: DD-APPLICATION-KEY - value: - - name: DD-API-KEY - value: + test_request: + method: GET + path: /api/v2/logs/events + headers: + - name: DD-APPLICATION-KEY + value: + - name: DD-API-KEY + value: - endpoints: - - name: events - requests: - read: - method: GET - path: /api/v2/logs/events - headers: - - name: DD-APPLICATION-KEY - value: - - name: DD-API-KEY - value: - query_params: - - name: filter[query] - value: - - name: filter[from] - value: 0 - - name: filter[to] - value: now - - name: page[limit] - value: - param_values: - - name: app_key - connector_param: app_key - - name: api_key - connector_param: api_key - - name: email - identity: email - data_path: data - pagination: - strategy: link - configuration: - source: body - path: links.next + endpoints: + - name: events + requests: + read: + method: GET + path: /api/v2/logs/events + headers: + - name: DD-APPLICATION-KEY + value: + - name: DD-API-KEY + value: + query_params: + - name: filter[query] + value: + - name: filter[from] + value: 0 + - name: filter[to] + value: now + - name: page[limit] + value: + param_values: + - name: app_key + connector_param: app_key + - name: api_key + connector_param: api_key + - name: email + identity: email + data_path: data + pagination: + strategy: link + configuration: + source: body + path: links.next diff --git a/data/saas/config/hubspot_config.yml b/data/saas/config/hubspot_config.yml index 4bcc8b08e..6fab80034 100644 --- a/data/saas/config/hubspot_config.yml +++ b/data/saas/config/hubspot_config.yml @@ -1,5 +1,5 @@ saas_config: - fides_key: hubspot_connector_example + fides_key: name: Hubspot SaaS Config type: hubspot description: A sample schema representing the Hubspot connector for Fidesops @@ -64,7 +64,7 @@ saas_config: param_values: - name: contactId references: - - dataset: hubspot_connector_example + - dataset: field: contacts.id direction: from - name: owners @@ -113,7 +113,7 @@ saas_config: identity: email - name: subscriptionId references: - - dataset: hubspot_connector_example + - dataset: field: subscription_preferences.id direction: from postprocessors: diff --git a/data/saas/config/logi_id_config.yml b/data/saas/config/logi_id_config.yml index 0c4b729e6..051a4b4e8 100644 --- a/data/saas/config/logi_id_config.yml +++ b/data/saas/config/logi_id_config.yml @@ -1,5 +1,5 @@ saas_config: - fides_key: logi_id_connector_example + fides_key: name: Logi ID SaaS Config type: logi_id description: A sample schema representing the Logi ID connector for Fidesops @@ -44,7 +44,7 @@ saas_config: param_values: - name: user_id references: - - dataset: logi_id_connector_example + - dataset: field: users.id direction: from - name: user_claims diff --git a/data/saas/config/mailchimp_config.yml b/data/saas/config/mailchimp_config.yml index e73d0f7b5..644a904b2 100644 --- a/data/saas/config/mailchimp_config.yml +++ b/data/saas/config/mailchimp_config.yml @@ -1,5 +1,5 @@ saas_config: - fides_key: mailchimp_connector_example + fides_key: name: Mailchimp SaaS Config type: mailchimp description: A sample schema representing the Mailchimp connector for Fidesops @@ -32,7 +32,7 @@ saas_config: param_values: - name: conversation_id references: - - dataset: mailchimp_connector_example + - dataset: field: conversations.id direction: from data_path: conversation_messages @@ -80,12 +80,12 @@ saas_config: param_values: - name: list_id references: - - dataset: mailchimp_connector_example + - dataset: field: member.list_id direction: from - name: subscriber_hash references: - - dataset: mailchimp_connector_example + - dataset: field: member.id direction: from body: | diff --git a/data/saas/config/outreach_config.yml b/data/saas/config/outreach_config.yml index 91c7f6897..d02508f49 100644 --- a/data/saas/config/outreach_config.yml +++ b/data/saas/config/outreach_config.yml @@ -1,5 +1,5 @@ saas_config: - fides_key: outreach_connector_example + fides_key: name: Outreach Example Config type: outreach description: A sample schema representing the Outreach connector for Fidesops diff --git a/data/saas/config/salesforce_config.yml b/data/saas/config/salesforce_config.yml index b13e76920..aa1accd33 100644 --- a/data/saas/config/salesforce_config.yml +++ b/data/saas/config/salesforce_config.yml @@ -1,5 +1,5 @@ saas_config: - fides_key: salesforce_connector_example + fides_key: name: Salesforce SaaS Config type: salesforce description: A sample schema representing the Salesforce connector for Fidesops @@ -87,7 +87,7 @@ saas_config: param_values: - name: contact_id references: - - dataset: salesforce_connector_example + - dataset: field: contact_list.Id direction: from update: @@ -100,7 +100,7 @@ saas_config: param_values: - name: contact_id references: - - dataset: salesforce_connector_example + - dataset: field: contacts.Id direction: from - name: case_list @@ -114,7 +114,7 @@ saas_config: param_values: - name: contact_id references: - - dataset: salesforce_connector_example + - dataset: field: contact_list.Id direction: from data_path: records @@ -126,7 +126,7 @@ saas_config: param_values: - name: case_id references: - - dataset: salesforce_connector_example + - dataset: field: case_list.Id direction: from update: @@ -139,7 +139,7 @@ saas_config: param_values: - name: case_id references: - - dataset: salesforce_connector_example + - dataset: field: cases.Id direction: from - name: lead_list @@ -162,7 +162,7 @@ saas_config: param_values: - name: lead_id references: - - dataset: salesforce_connector_example + - dataset: field: lead_list.Id direction: from update: @@ -175,7 +175,7 @@ saas_config: param_values: - name: lead_id references: - - dataset: salesforce_connector_example + - dataset: field: leads.Id direction: from - name: accounts @@ -186,7 +186,7 @@ saas_config: param_values: - name: account_id references: - - dataset: salesforce_connector_example + - dataset: field: contacts.AccountId update: method: PATCH @@ -198,7 +198,7 @@ saas_config: param_values: - name: account_id references: - - dataset: salesforce_connector_example + - dataset: field: accounts.Id direction: from - name: campaign_member_list @@ -221,7 +221,7 @@ saas_config: param_values: - name: campaign_member_id references: - - dataset: salesforce_connector_example + - dataset: field: campaign_member_list.Id direction: from update: @@ -234,6 +234,6 @@ saas_config: param_values: - name: campaign_member_id references: - - dataset: salesforce_connector_example + - dataset: field: campaign_members.Id direction: from diff --git a/data/saas/config/segment_config.yml b/data/saas/config/segment_config.yml index 72ed99ec2..f2a884a85 100644 --- a/data/saas/config/segment_config.yml +++ b/data/saas/config/segment_config.yml @@ -1,5 +1,5 @@ saas_config: - fides_key: segment_connector_example + fides_key: name: Segment SaaS Config type: segment description: A sample schema representing the Segment connector for Fidesops @@ -55,7 +55,7 @@ saas_config: connector_param: namespace_id - name: segment_id references: - - dataset: segment_connector_example + - dataset: field: segment_user.segment_id direction: from data_path: data @@ -84,7 +84,7 @@ saas_config: connector_param: namespace_id - name: segment_id references: - - dataset: segment_connector_example + - dataset: field: segment_user.segment_id direction: from data_path: traits @@ -110,7 +110,7 @@ saas_config: connector_param: namespace_id - name: segment_id references: - - dataset: segment_connector_example + - dataset: field: segment_user.segment_id direction: from data_path: data diff --git a/data/saas/config/sendgrid_config.yml b/data/saas/config/sendgrid_config.yml index 4fb3efe46..58f33cecc 100644 --- a/data/saas/config/sendgrid_config.yml +++ b/data/saas/config/sendgrid_config.yml @@ -1,5 +1,5 @@ saas_config: - fides_key: sendgrid_connector_example + fides_key: name: Sendgrid SaaS Config type: sendgrid description: A sample schema representing the Sendgrid connector for Fidesops @@ -41,6 +41,6 @@ saas_config: param_values: - name: contact_id references: - - dataset: sendgrid_connector_example + - dataset: field: contacts.id direction: from diff --git a/data/saas/config/sentry_config.yml b/data/saas/config/sentry_config.yml index 2ada95e8c..75de54823 100644 --- a/data/saas/config/sentry_config.yml +++ b/data/saas/config/sentry_config.yml @@ -1,5 +1,5 @@ saas_config: - fides_key: sentry_connector + fides_key: name: Sentry SaaS Config type: sentry description: A sample schema representing the Sentry connector for Fidesops @@ -44,7 +44,7 @@ saas_config: param_values: - name: organization_slug references: - - dataset: sentry_connector + - dataset: field: organizations.slug direction: from postprocessors: @@ -77,7 +77,7 @@ saas_config: param_values: - name: issue_id references: - - dataset: sentry_connector + - dataset: field: issues.id direction: from body: | @@ -94,12 +94,12 @@ saas_config: param_values: - name: organization_slug references: - - dataset: sentry_connector + - dataset: field: projects.organization.slug direction: from - name: project_slug references: - - dataset: sentry_connector + - dataset: field: projects.slug direction: from - name: query @@ -118,12 +118,12 @@ saas_config: param_values: - name: organization_slug references: - - dataset: sentry_connector + - dataset: field: projects.organization.slug direction: from - name: project_slug references: - - dataset: sentry_connector + - dataset: field: projects.slug direction: from postprocessors: @@ -138,7 +138,7 @@ saas_config: source: headers rel: next - name: person - after: [sentry_connector.projects] + after: [.projects] requests: read: method: GET @@ -151,12 +151,12 @@ saas_config: param_values: - name: organization_slug references: - - dataset: sentry_connector + - dataset: field: projects.organization.slug direction: from - name: project_slug references: - - dataset: sentry_connector + - dataset: field: projects.slug direction: from - name: query diff --git a/data/saas/config/stripe_config.yml b/data/saas/config/stripe_config.yml index f29911533..2a8b00d48 100644 --- a/data/saas/config/stripe_config.yml +++ b/data/saas/config/stripe_config.yml @@ -1,5 +1,5 @@ saas_config: - fides_key: stripe_connector_example + fides_key: name: Stripe SaaS Config type: stripe description: A sample schema representing the Stripe connector for Fidesops @@ -46,7 +46,7 @@ saas_config: param_values: - name: customer_id references: - - dataset: stripe_connector_example + - dataset: field: customer.id direction: from body: | @@ -66,7 +66,7 @@ saas_config: param_values: - name: customer_id references: - - dataset: stripe_connector_example + - dataset: field: customer.id direction: from - name: limit @@ -92,12 +92,12 @@ saas_config: param_values: - name: charge_id references: - - dataset: stripe_connector_example + - dataset: field: charge.id direction: from - name: payment_intent_id references: - - dataset: stripe_connector_example + - dataset: field: payment_intent.id direction: from - name: limit @@ -121,7 +121,7 @@ saas_config: param_values: - name: customer_id references: - - dataset: stripe_connector_example + - dataset: field: customer.id direction: from - name: limit @@ -145,7 +145,7 @@ saas_config: param_values: - name: customer_id references: - - dataset: stripe_connector_example + - dataset: field: customer.id direction: from - name: type @@ -167,7 +167,7 @@ saas_config: param_values: - name: payment_method_id references: - - dataset: stripe_connector_example + - dataset: field: payment_method.id direction: from body: | @@ -187,7 +187,7 @@ saas_config: param_values: - name: customer_id references: - - dataset: stripe_connector_example + - dataset: field: customer.id direction: from - name: limit @@ -207,12 +207,12 @@ saas_config: param_values: - name: customer_id references: - - dataset: stripe_connector_example + - dataset: field: bank_account.customer direction: from - name: bank_account_id references: - - dataset: stripe_connector_example + - dataset: field: bank_account.id direction: from body: | @@ -232,7 +232,7 @@ saas_config: param_values: - name: customer_id references: - - dataset: stripe_connector_example + - dataset: field: customer.id direction: from - name: limit @@ -252,12 +252,12 @@ saas_config: param_values: - name: customer_id references: - - dataset: stripe_connector_example + - dataset: field: card.customer direction: from - name: card_id references: - - dataset: stripe_connector_example + - dataset: field: card.id direction: from body: | @@ -277,7 +277,7 @@ saas_config: param_values: - name: customer_id references: - - dataset: stripe_connector_example + - dataset: field: customer.id direction: from - name: limit @@ -299,7 +299,7 @@ saas_config: param_values: - name: customer_id references: - - dataset: stripe_connector_example + - dataset: field: customer.id direction: from - name: limit @@ -321,7 +321,7 @@ saas_config: param_values: - name: customer_id references: - - dataset: stripe_connector_example + - dataset: field: customer.id direction: from - name: limit @@ -338,12 +338,12 @@ saas_config: param_values: - name: customer_id references: - - dataset: stripe_connector_example + - dataset: field: tax_id.customer direction: from - name: tax_id references: - - dataset: stripe_connector_example + - dataset: field: tax_id.id direction: from - name: invoice @@ -359,7 +359,7 @@ saas_config: param_values: - name: customer_id references: - - dataset: stripe_connector_example + - dataset: field: customer.id direction: from - name: limit @@ -377,7 +377,7 @@ saas_config: param_values: - name: invoice_id references: - - dataset: stripe_connector_example + - dataset: field: invoice.id direction: from - name: invoice_item @@ -393,7 +393,7 @@ saas_config: param_values: - name: customer_id references: - - dataset: stripe_connector_example + - dataset: field: customer.id direction: from - name: limit @@ -411,7 +411,7 @@ saas_config: param_values: - name: invoice_item_id references: - - dataset: stripe_connector_example + - dataset: field: invoice_item.id direction: from - name: subscription @@ -427,7 +427,7 @@ saas_config: param_values: - name: customer_id references: - - dataset: stripe_connector_example + - dataset: field: customer.id direction: from - name: limit @@ -444,6 +444,6 @@ saas_config: param_values: - name: subscription_id references: - - dataset: stripe_connector_example + - dataset: field: subscription.id direction: from diff --git a/data/saas/config/zendesk_config.yml b/data/saas/config/zendesk_config.yml index d52b1e895..b0bccfc38 100644 --- a/data/saas/config/zendesk_config.yml +++ b/data/saas/config/zendesk_config.yml @@ -1,5 +1,5 @@ saas_config: - fides_key: zendesk_connector_example + fides_key: name: Zendesk SaaS Config type: zendesk description: A sample schema representing the Zendesk connector for Fidesops @@ -46,7 +46,7 @@ saas_config: param_values: - name: user_id references: - - dataset: zendesk_connector_example + - dataset: field: users.id direction: from - name: user_identities @@ -60,7 +60,7 @@ saas_config: param_values: - name: user_id references: - - dataset: zendesk_connector_example + - dataset: field: users.id direction: from - name: page_size @@ -82,7 +82,7 @@ saas_config: param_values: - name: user_id references: - - dataset: zendesk_connector_example + - dataset: field: users.id direction: from - name: page_size @@ -99,7 +99,7 @@ saas_config: param_values: - name: ticket_id references: - - dataset: zendesk_connector_example + - dataset: field: tickets.id direction: from - name: ticket_comments @@ -113,7 +113,7 @@ saas_config: param_values: - name: ticket_id references: - - dataset: zendesk_connector_example + - dataset: field: tickets.id direction: from - name: page_size diff --git a/data/saas/dataset/adobe_campaign_dataset.yml b/data/saas/dataset/adobe_campaign_dataset.yml index 9eb427d7f..d1ef51538 100644 --- a/data/saas/dataset/adobe_campaign_dataset.yml +++ b/data/saas/dataset/adobe_campaign_dataset.yml @@ -1,5 +1,5 @@ dataset: - - fides_key: adobe_campaign_connector_example + - fides_key: name: Adobe Campaign Dataset description: A dataset representing the Adobe Campaign connector for Fidesops collections: diff --git a/data/saas/dataset/auth0_dataset.yml b/data/saas/dataset/auth0_dataset.yml index 13fede1bb..b6b087caa 100644 --- a/data/saas/dataset/auth0_dataset.yml +++ b/data/saas/dataset/auth0_dataset.yml @@ -1,5 +1,5 @@ dataset: - - fides_key: auth0_connector_example + - fides_key: name: Auth0 Dataset description: A sample dataset representing the Auth0 connector for Fidesops collections: diff --git a/data/saas/dataset/datadog_dataset.yml b/data/saas/dataset/datadog_dataset.yml index 56cc24833..6e08ac53a 100644 --- a/data/saas/dataset/datadog_dataset.yml +++ b/data/saas/dataset/datadog_dataset.yml @@ -1,5 +1,5 @@ dataset: - - fides_key: datadog_connector_example + - fides_key: name: Datadog Dataset description: A sample dataset representing the Datadog connector for Fidesops collections: diff --git a/data/saas/dataset/hubspot_dataset.yml b/data/saas/dataset/hubspot_dataset.yml index 2ebd528ac..4d8bc9aa1 100644 --- a/data/saas/dataset/hubspot_dataset.yml +++ b/data/saas/dataset/hubspot_dataset.yml @@ -1,5 +1,5 @@ dataset: - - fides_key: hubspot_connector_example + - fides_key: name: Hubspot Dataset description: A sample dataset representing the Hubspot connector for Fidesops collections: diff --git a/data/saas/dataset/logi_id_dataset.yml b/data/saas/dataset/logi_id_dataset.yml index aa57e9597..bb0cf1a6b 100644 --- a/data/saas/dataset/logi_id_dataset.yml +++ b/data/saas/dataset/logi_id_dataset.yml @@ -1,5 +1,5 @@ dataset: - - fides_key: logi_id_connector_example + - fides_key: name: Logi ID Example Dataset description: A sample dataset representing the Logi ID connector for Fidesops collections: diff --git a/data/saas/dataset/mailchimp_dataset.yml b/data/saas/dataset/mailchimp_dataset.yml index 5f3e91e1d..ac27b0931 100644 --- a/data/saas/dataset/mailchimp_dataset.yml +++ b/data/saas/dataset/mailchimp_dataset.yml @@ -1,5 +1,5 @@ dataset: - - fides_key: mailchimp_connector_example + - fides_key: name: Mailchimp Dataset description: A sample dataset representing the Mailchimp connector for Fidesops collections: diff --git a/data/saas/dataset/outreach_dataset.yml b/data/saas/dataset/outreach_dataset.yml index df51a3b7b..cfbc732e1 100644 --- a/data/saas/dataset/outreach_dataset.yml +++ b/data/saas/dataset/outreach_dataset.yml @@ -1,5 +1,5 @@ dataset: - - fides_key: outreach_connector_example + - fides_key: name: Outreach Dataset description: A sample dataset representing the Outreach connector for Fidesops collections: diff --git a/data/saas/dataset/salesforce_dataset.yml b/data/saas/dataset/salesforce_dataset.yml index b1292a5ec..6302a5e44 100644 --- a/data/saas/dataset/salesforce_dataset.yml +++ b/data/saas/dataset/salesforce_dataset.yml @@ -1,5 +1,5 @@ dataset: - - fides_key: salesforce_connector_example + - fides_key: name: Salesforce Example Dataset description: A sample dataset representing the Salesforce connector for Fidesops collections: diff --git a/data/saas/dataset/segment_dataset.yml b/data/saas/dataset/segment_dataset.yml index d7ef65e66..473d93dd5 100644 --- a/data/saas/dataset/segment_dataset.yml +++ b/data/saas/dataset/segment_dataset.yml @@ -1,5 +1,5 @@ dataset: - - fides_key: segment_connector_example + - fides_key: name: Segment Dataset description: A sample dataset representing the Segment connector for Fidesops collections: diff --git a/data/saas/dataset/sendgrid_dataset.yml b/data/saas/dataset/sendgrid_dataset.yml index 94554c17a..dcb625bf8 100644 --- a/data/saas/dataset/sendgrid_dataset.yml +++ b/data/saas/dataset/sendgrid_dataset.yml @@ -1,5 +1,5 @@ dataset: - - fides_key: sendgrid_connector_example + - fides_key: name: Sendgrid Dataset description: A sample dataset representing the Sendgrid connector for Fidesops collections: diff --git a/data/saas/dataset/sentry_dataset.yml b/data/saas/dataset/sentry_dataset.yml index efd765187..41617b09c 100644 --- a/data/saas/dataset/sentry_dataset.yml +++ b/data/saas/dataset/sentry_dataset.yml @@ -1,5 +1,5 @@ dataset: - - fides_key: sentry_connector + - fides_key: name: Sentry Dataset description: A sample dataset representing the Sentry connector for Fidesops collections: diff --git a/data/saas/dataset/stripe_dataset.yml b/data/saas/dataset/stripe_dataset.yml index 045cfb615..212e5f08e 100644 --- a/data/saas/dataset/stripe_dataset.yml +++ b/data/saas/dataset/stripe_dataset.yml @@ -1,5 +1,5 @@ dataset: - - fides_key: stripe_connector_example + - fides_key: name: Stripe Dataset description: A sample dataset representing the Stripe connector for Fidesops collections: diff --git a/data/saas/dataset/zendesk_dataset.yml b/data/saas/dataset/zendesk_dataset.yml index 95dd961b3..fdbc5a3e9 100644 --- a/data/saas/dataset/zendesk_dataset.yml +++ b/data/saas/dataset/zendesk_dataset.yml @@ -1,5 +1,5 @@ dataset: - - fides_key: zendesk_connector_example + - fides_key: name: Zendesk Dataset description: A sample dataset representing the Zendesk connector for Fidesops collections: diff --git a/data/saas/icon/adobe.svg b/data/saas/icon/adobe.svg new file mode 100644 index 000000000..fadef181c --- /dev/null +++ b/data/saas/icon/adobe.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/data/saas/icon/default.svg b/data/saas/icon/default.svg new file mode 100644 index 000000000..32e4384db --- /dev/null +++ b/data/saas/icon/default.svg @@ -0,0 +1,4 @@ + + + + diff --git a/data/saas/icon/hubspot.svg b/data/saas/icon/hubspot.svg new file mode 100644 index 000000000..e7afe860e --- /dev/null +++ b/data/saas/icon/hubspot.svg @@ -0,0 +1,4 @@ + + + + diff --git a/data/saas/icon/mailchimp.svg b/data/saas/icon/mailchimp.svg new file mode 100644 index 000000000..28edc6cf3 --- /dev/null +++ b/data/saas/icon/mailchimp.svg @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/data/saas/icon/outreach.svg b/data/saas/icon/outreach.svg new file mode 100644 index 000000000..769fbb2c5 --- /dev/null +++ b/data/saas/icon/outreach.svg @@ -0,0 +1,4 @@ + + + + diff --git a/data/saas/icon/salesforce.svg b/data/saas/icon/salesforce.svg new file mode 100644 index 000000000..d10da329d --- /dev/null +++ b/data/saas/icon/salesforce.svg @@ -0,0 +1,4 @@ + + + + diff --git a/data/saas/icon/segment.svg b/data/saas/icon/segment.svg new file mode 100644 index 000000000..a84f3677d --- /dev/null +++ b/data/saas/icon/segment.svg @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/data/saas/icon/sentry.svg b/data/saas/icon/sentry.svg new file mode 100644 index 000000000..e2e3be20e --- /dev/null +++ b/data/saas/icon/sentry.svg @@ -0,0 +1,4 @@ + + + + diff --git a/data/saas/icon/stripe.svg b/data/saas/icon/stripe.svg new file mode 100644 index 000000000..e41146fff --- /dev/null +++ b/data/saas/icon/stripe.svg @@ -0,0 +1,31 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/data/saas/icon/zendesk.svg b/data/saas/icon/zendesk.svg new file mode 100644 index 000000000..198243423 --- /dev/null +++ b/data/saas/icon/zendesk.svg @@ -0,0 +1,7 @@ + + + + + + + diff --git a/data/saas/saas_connector_registry.toml b/data/saas/saas_connector_registry.toml new file mode 100644 index 000000000..1851fed5b --- /dev/null +++ b/data/saas/saas_connector_registry.toml @@ -0,0 +1,64 @@ +[adobe_campaign] +config = "data/saas/config/adobe_campaign_config.yml" +dataset = "data/saas/dataset/adobe_campaign_dataset.yml" +icon = "data/saas/icon/adobe.svg" + +[auth0] +config = "data/saas/config/auth0_config.yml" +dataset = "data/saas/dataset/auth0_dataset.yml" +icon = "data/saas/icon/default.svg" + +[datadog] +config = "data/saas/config/datadog_config.yml" +dataset = "data/saas/dataset/datadog_dataset.yml" +icon = "data/saas/icon/default.svg" + +[hubspot] +config = "data/saas/config/hubspot_config.yml" +dataset = "data/saas/dataset/hubspot_dataset.yml" +icon = "data/saas/icon/hubspot.svg" + +[logi_id] +config = "data/saas/config/logi_id_config.yml" +dataset = "data/saas/dataset/logi_id_dataset.yml" +icon = "data/saas/icon/default.svg" + +[mailchimp] +config = "data/saas/config/mailchimp_config.yml" +dataset = "data/saas/dataset/mailchimp_dataset.yml" +icon = "data/saas/icon/mailchimp.svg" + +[outreach] +config = "data/saas/config/outreach_config.yml" +dataset = "data/saas/dataset/outreach_dataset.yml" +icon = "data/saas/icon/outreach.svg" + +[salesforce] +config = "data/saas/config/salesforce_config.yml" +dataset = "data/saas/dataset/salesforce_dataset.yml" +icon = "data/saas/icon/salesforce.svg" + +[segment] +config = "data/saas/config/segment_config.yml" +dataset = "data/saas/dataset/segment_dataset.yml" +icon = "data/saas/icon/segment.svg" + +[sendgrid] +config = "data/saas/config/sendgrid_config.yml" +dataset = "data/saas/dataset/sendgrid_dataset.yml" +icon = "data/saas/icon/default.svg" + +[sentry] +config = "data/saas/config/sentry_config.yml" +dataset = "data/saas/dataset/sentry_dataset.yml" +icon = "data/saas/icon/sentry.svg" + +[stripe] +config = "data/saas/config/stripe_config.yml" +dataset = "data/saas/dataset/stripe_dataset.yml" +icon = "data/saas/icon/stripe.svg" + +[zendesk] +config = "data/saas/config/zendesk_config.yml" +dataset = "data/saas/dataset/zendesk_dataset.yml" +icon = "data/saas/icon/zendesk.svg" \ No newline at end of file diff --git a/docs/fidesops/docs/guides/connection_types.md b/docs/fidesops/docs/guides/connection_types.md index cc64fa446..9d411d333 100644 --- a/docs/fidesops/docs/guides/connection_types.md +++ b/docs/fidesops/docs/guides/connection_types.md @@ -10,24 +10,92 @@ database options and third party API services with which fidesops can communicat ```json title="GET /api/v1/connection_type" { "items": [ - "bigquery", - "hubspot", - "mailchimp", - "mariadb", - "mongodb", - "mssql", - "mysql", - "outreach", - "postgres", - "redshift", - "salesforce", - "segment", - "sentry", - "snowflake", - "stripe", - "zendesk" + { + "identifier": "bigquery", + "type": "database" + }, + { + "identifier": "mariadb", + "type": "database" + }, + { + "identifier": "mongodb", + "type": "database" + }, + { + "identifier": "mssql", + "type": "database" + }, + { + "identifier": "mysql", + "type": "database" + }, + { + "identifier": "postgres", + "type": "database" + }, + { + "identifier": "redshift", + "type": "database" + }, + { + "identifier": "snowflake", + "type": "database" + }, + { + "identifier": "adobe_campaign", + "type": "saas" + }, + { + "identifier": "auth0", + "type": "saas" + }, + { + "identifier": "datadog", + "type": "saas" + }, + { + "identifier": "hubspot", + "type": "saas" + }, + { + "identifier": "logi_id", + "type": "saas" + }, + { + "identifier": "mailchimp", + "type": "saas" + }, + { + "identifier": "outreach", + "type": "saas" + }, + { + "identifier": "salesforce", + "type": "saas" + }, + { + "identifier": "segment", + "type": "saas" + }, + { + "identifier": "sendgrid", + "type": "saas" + }, + { + "identifier": "sentry", + "type": "saas" + }, + { + "identifier": "stripe", + "type": "saas" + }, + { + "identifier": "zendesk", + "type": "saas" + } ], - "total": 15, + "total": 21, "page": 1, "size": 50 } @@ -40,7 +108,7 @@ To view the secrets needed to authenticate with a given connection, visit `GET / ### Example ```json title="GET /api/v1/connection_type/sentry/secret" { - "title": "sentry_connector_schema", + "title": "sentry_schema", "description": "Sentry secrets schema", "type": "object", "properties": { @@ -59,4 +127,29 @@ To view the secrets needed to authenticate with a given connection, visit `GET / ], "additionalProperties": false } -``` \ No newline at end of file +``` + +## Setting up a SaaS Connector from a Template + +To create all the resources necessary to set up a SaaS Connector in one request, you can create a connector from +a template. + +This creates a `saas` ConnectionConfig for you with your supplied name and description, with your supplied `secrets`. +In the example below, we're creating a `mailchimp` saas connector, so you should supply the relevant mailchimp `secrets`. +Your `instance_key` will become the identifier for the related `DatasetConfig` resource. By default, the saas connection config +is enabled, with write access. + + +```json title="POST /connection/instantiate/mailchimp" +{ + "name": "My Mailchimp connector", + "description": "Production Mailchimp Instance", + "secrets": { + "domain": "{{mailchimp_domain}}", + "api_key": "{{mailchimp_api_key}}", + "username": "{{mailchimp_username}}" + }, + "instance_key": "primary_mailchimp", +} +``` + diff --git a/docs/fidesops/docs/postman/Fidesops.postman_collection.json b/docs/fidesops/docs/postman/Fidesops.postman_collection.json index 21311c090..5316e1c95 100644 --- a/docs/fidesops/docs/postman/Fidesops.postman_collection.json +++ b/docs/fidesops/docs/postman/Fidesops.postman_collection.json @@ -1,6 +1,6 @@ { "info": { - "_postman_id": "c2e8bb17-9afa-4e3a-8cc0-e294b35956b5", + "_postman_id": "645bae7d-d9af-4b08-84cf-b98d9ae26014", "name": "Fidesops", "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json", "_exporter_id": "13396647" @@ -73,7 +73,7 @@ "header": [], "body": { "mode": "raw", - "raw": "[\n \"client:create\",\n \"client:update\",\n \"client:read\",\n \"client:delete\",\n \"config:read\",\n \"connection_type:read\",\n \"connection:read\",\n \"connection:create_or_update\",\n \"connection:delete\",\n \"dataset:create_or_update\",\n \"dataset:delete\",\n \"dataset:read\",\n \"encryption:exec\",\n \"policy:create_or_update\",\n \"policy:read\",\n \"policy:delete\",\n \"privacy-request:read\",\n \"privacy-request:delete\",\n \"rule:create_or_update\",\n \"rule:read\",\n \"rule:delete\",\n \"scope:read\",\n \"storage:create_or_update\",\n \"storage:delete\",\n \"storage:read\",\n \"privacy-request:resume\",\n \"webhook:create_or_update\",\n \"webhook:read\",\n \"webhook:delete\",\n \"saas_config:create_or_update\",\n \"saas_config:read\",\n \"saas_config:delete\",\n \"privacy-request:review\",\n \"user:create\",\n \"user:delete\"\n]", + "raw": "[\n \"client:create\",\n \"client:update\",\n \"client:read\",\n \"client:delete\",\n \"config:read\",\n \"connection_type:read\",\n \"connection:read\",\n \"connection:create_or_update\",\n \"connection:delete\",\n \"connection:instantiate\",\n \"dataset:create_or_update\",\n \"dataset:delete\",\n \"dataset:read\",\n \"encryption:exec\",\n \"policy:create_or_update\",\n \"policy:read\",\n \"policy:delete\",\n \"privacy-request:read\",\n \"privacy-request:delete\",\n \"rule:create_or_update\",\n \"rule:read\",\n \"rule:delete\",\n \"scope:read\",\n \"storage:create_or_update\",\n \"storage:delete\",\n \"storage:read\",\n \"privacy-request:resume\",\n \"webhook:create_or_update\",\n \"webhook:read\",\n \"webhook:delete\",\n \"saas_config:create_or_update\",\n \"saas_config:read\",\n \"saas_config:delete\",\n \"privacy-request:review\",\n \"user:create\",\n \"user:delete\"\n]", "options": { "raw": { "language": "json" @@ -3282,6 +3282,44 @@ } }, "response": [] + }, + { + "name": "Create From Template", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"name\": \"{{saas_connector_type}} connector\",\n \"instance_key\": \"primary_{{saas_connector_type}}\",\n \"secrets\": {\n \"domain\": \"{{mailchimp_domain}}\",\n \"api_key\": \"{{mailchimp_api_key}}\",\n \"username\": \"{{mailchimp_username}}\"\n }\n\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/connection/instantiate/{{saas_connector_type}}", + "host": [ + "{{host}}" + ], + "path": [ + "connection", + "instantiate", + "{{saas_connector_type}}" + ] + } + }, + "response": [] } ] }, @@ -4360,6 +4398,11 @@ "key": "email_config_key", "value": "my_email_config", "type": "string" + }, + { + "key": "saas_connector_type", + "value": "mailchimp", + "type": "string" } ] } \ No newline at end of file diff --git a/src/fidesops/main.py b/src/fidesops/main.py index 8e7d38a47..30de5f49b 100644 --- a/src/fidesops/main.py +++ b/src/fidesops/main.py @@ -33,6 +33,10 @@ from fidesops.ops.core.config import config from fidesops.ops.db.database import init_db from fidesops.ops.schemas.analytics import Event, ExtraData +from fidesops.ops.service.connectors.saas.connector_registry_service import ( + load_registry, + registry_file, +) from fidesops.ops.tasks.scheduled.scheduler import scheduler from fidesops.ops.tasks.scheduled.tasks import initiate_scheduled_request_intake from fidesops.ops.util.cache import get_cache @@ -180,6 +184,9 @@ def start_webserver() -> None: ) config.log_all_config_values() + logger.info("Validating SaaS connector templates...") + load_registry(registry_file) + if config.database.enabled: logger.info("Running any pending DB migrations...") try: diff --git a/src/fidesops/ops/api/v1/endpoints/saas_config_endpoints.py b/src/fidesops/ops/api/v1/endpoints/saas_config_endpoints.py index a910a50e4..77a5e2e1a 100644 --- a/src/fidesops/ops/api/v1/endpoints/saas_config_endpoints.py +++ b/src/fidesops/ops/api/v1/endpoints/saas_config_endpoints.py @@ -3,6 +3,7 @@ from fastapi import Depends, HTTPException from fastapi.params import Security +from fideslib.exceptions import KeyOrNameAlreadyExists from sqlalchemy.orm import Session from starlette.status import ( HTTP_200_OK, @@ -10,24 +11,33 @@ HTTP_400_BAD_REQUEST, HTTP_404_NOT_FOUND, HTTP_422_UNPROCESSABLE_ENTITY, + HTTP_500_INTERNAL_SERVER_ERROR, ) from fidesops.ops.api import deps +from fidesops.ops.api.v1.endpoints.connection_endpoints import validate_secrets from fidesops.ops.api.v1.scope_registry import ( CONNECTION_AUTHORIZE, SAAS_CONFIG_CREATE_OR_UPDATE, SAAS_CONFIG_DELETE, SAAS_CONFIG_READ, + SAAS_CONNECTION_INSTANTIATE, ) from fidesops.ops.api.v1.urn_registry import ( AUTHORIZE, + CONNECTION_TYPES, SAAS_CONFIG, SAAS_CONFIG_VALIDATE, + SAAS_CONNECTOR_FROM_TEMPLATE, V1_URL_PREFIX, ) from fidesops.ops.common_exceptions import FidesopsException from fidesops.ops.models.connectionconfig import ConnectionConfig, ConnectionType from fidesops.ops.models.datasetconfig import DatasetConfig +from fidesops.ops.schemas.connection_configuration.connection_config import ( + SaasConnectionTemplateValues, +) +from fidesops.ops.schemas.dataset import FidesopsDataset from fidesops.ops.schemas.saas.saas_config import ( SaaSConfig, SaaSConfigValidationDetails, @@ -40,6 +50,14 @@ from fidesops.ops.service.authentication.authentication_strategy_oauth2 import ( OAuth2AuthenticationStrategy, ) +from fidesops.ops.service.connectors.saas.connector_registry_service import ( + ConnectorRegistry, + ConnectorTemplate, + create_connection_config_from_template_no_save, + create_dataset_config_from_template, + load_registry, + registry_file, +) from fidesops.ops.util.api_router import APIRouter from fidesops.ops.util.oauth_util import verify_oauth_client @@ -245,3 +263,72 @@ def authorize_connection( return auth_strategy.get_authorization_url(db, connection_config) except FidesopsException as exc: raise HTTPException(status_code=HTTP_400_BAD_REQUEST, detail=str(exc)) + + +@router.post( + SAAS_CONNECTOR_FROM_TEMPLATE, + dependencies=[Security(verify_oauth_client, scopes=[SAAS_CONNECTION_INSTANTIATE])], + response_model=FidesopsDataset, +) +def instantiate_connection_from_template( + saas_connector_type: str, + template_values: SaasConnectionTemplateValues, + db: Session = Depends(deps.get_db), +) -> FidesopsDataset: + """ + Creates a SaaS Connector and a SaaS Dataset from a template. + + Looks up the connector type in the SaaS connector registry and, if all required + fields are provided, persists the associated connection config and dataset to the database. + """ + + registry: ConnectorRegistry = load_registry(registry_file) + connector_template: Optional[ConnectorTemplate] = registry.get_connector_template( + saas_connector_type + ) + if not connector_template: + raise HTTPException( + status_code=HTTP_404_NOT_FOUND, + detail=f"SaaS connector type '{saas_connector_type}' is not yet available in Fidesops. For a list of available SaaS connectors, refer to {CONNECTION_TYPES}.", + ) + + if DatasetConfig.filter( + db=db, + conditions=(DatasetConfig.fides_key == template_values.instance_key), + ).count(): + raise HTTPException( + status_code=HTTP_400_BAD_REQUEST, + detail=f"SaaS connector instance key '{template_values.instance_key}' already exists.", + ) + + try: + connection_config: ConnectionConfig = ( + create_connection_config_from_template_no_save( + db, connector_template, template_values + ) + ) + except KeyOrNameAlreadyExists as exc: + raise HTTPException( + status_code=HTTP_400_BAD_REQUEST, + detail=exc.args[0], + ) + + connection_config.secrets = validate_secrets( + template_values.secrets, connection_config + ).dict() + connection_config.save(db=db) # Not persisted to db until secrets are validated + + try: + dataset_config: DatasetConfig = create_dataset_config_from_template( + db, connection_config, connector_template, template_values + ) + except Exception: + connection_config.delete(db) + raise HTTPException( + status_code=HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"SaaS Connector could not be created from the '{saas_connector_type}' template at this time.", + ) + logger.info( + f"SaaS Connector and Dataset {template_values.instance_key} successfully created from '{saas_connector_type}' template." + ) + return dataset_config.dataset diff --git a/src/fidesops/ops/api/v1/scope_registry.py b/src/fidesops/ops/api/v1/scope_registry.py index 4aaed52db..b628d95b5 100644 --- a/src/fidesops/ops/api/v1/scope_registry.py +++ b/src/fidesops/ops/api/v1/scope_registry.py @@ -15,6 +15,7 @@ CONNECTION_READ = "connection:read" CONNECTION_DELETE = "connection:delete" CONNECTION_AUTHORIZE = "connection:authorize" +SAAS_CONNECTION_INSTANTIATE = "connection:instantiate" PRIVACY_REQUEST_READ = "privacy-request:read" PRIVACY_REQUEST_DELETE = "privacy-request:delete" @@ -71,6 +72,7 @@ CONNECTION_CREATE_OR_UPDATE, CONNECTION_DELETE, CONNECTION_AUTHORIZE, + SAAS_CONNECTION_INSTANTIATE, CONNECTION_TYPE_READ, DATASET_CREATE_OR_UPDATE, DATASET_DELETE, diff --git a/src/fidesops/ops/api/v1/urn_registry.py b/src/fidesops/ops/api/v1/urn_registry.py index 2cf400c9d..9a6fa8a79 100644 --- a/src/fidesops/ops/api/v1/urn_registry.py +++ b/src/fidesops/ops/api/v1/urn_registry.py @@ -88,6 +88,7 @@ # SaaS Config URLs SAAS_CONFIG_VALIDATE = CONNECTION_BY_KEY + "/validate_saas_config" SAAS_CONFIG = CONNECTION_BY_KEY + "/saas_config" +SAAS_CONNECTOR_FROM_TEMPLATE = "/connection/instantiate/{saas_connector_type}" # User URLs diff --git a/src/fidesops/ops/models/connectionconfig.py b/src/fidesops/ops/models/connectionconfig.py index dedb1af8a..094de5e92 100644 --- a/src/fidesops/ops/models/connectionconfig.py +++ b/src/fidesops/ops/models/connectionconfig.py @@ -2,9 +2,11 @@ import enum from datetime import datetime -from typing import Any, Optional +from typing import Any, Optional, Type from fideslib.db.base import Base +from fideslib.db.base_class import get_key_from_data +from fideslib.exceptions import KeyOrNameAlreadyExists from sqlalchemy import Boolean, Column, DateTime, Enum, String, event from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.ext.mutable import MutableDict @@ -88,6 +90,33 @@ class ConnectionConfig(Base): MutableDict.as_mutable(JSONB), index=False, unique=False, nullable=True ) + @classmethod + def create_without_saving( + cls: Type[ConnectionConfig], db: Session, *, data: dict[str, Any] + ) -> ConnectionConfig: + """Create a ConnectionConfig without persisting to the database""" + # Build properly formatted key/name for ConnectionConfig. + # Borrowed from OrmWrappedFidesBase.create + if hasattr(cls, "key"): + data["key"] = get_key_from_data(data, cls.__name__) + if db.query(cls).filter_by(key=data["key"]).first(): + raise KeyOrNameAlreadyExists( + f"Key {data['key']} already exists in {cls.__name__}. Keys will be snake-cased names if not provided. " + f"If you are seeing this error without providing a key, please provide a key or a different name." + "" + ) + + if hasattr(cls, "name"): + data["name"] = data.get("name") + if db.query(cls).filter_by(name=data["name"]).first(): + raise KeyOrNameAlreadyExists( + f"Name {data['name']} already exists in {cls.__name__}." + ) + + # Create + db_obj = cls(**data) # type: ignore + return db_obj + def get_saas_config(self) -> Optional[SaaSConfig]: """Returns a SaaSConfig object from a yaml config""" return SaaSConfig(**self.saas_config) if self.saas_config else None diff --git a/src/fidesops/ops/schemas/connection_configuration/connection_config.py b/src/fidesops/ops/schemas/connection_configuration/connection_config.py index e56c62123..0de31357f 100644 --- a/src/fidesops/ops/schemas/connection_configuration/connection_config.py +++ b/src/fidesops/ops/schemas/connection_configuration/connection_config.py @@ -6,6 +6,7 @@ from fidesops.ops.models.connectionconfig import AccessLevel, ConnectionType from fidesops.ops.schemas.api import BulkResponse, BulkUpdateFailed +from fidesops.ops.schemas.connection_configuration import connection_secrets_schemas from fidesops.ops.schemas.saas.saas_config import SaaSConfigBase, SaaSType from fidesops.ops.schemas.shared_schemas import FidesOpsKey @@ -98,3 +99,13 @@ class BulkPutConnectionConfiguration(BulkResponse): succeeded: List[ConnectionConfigurationResponse] failed: List[BulkUpdateFailed] + + +class SaasConnectionTemplateValues(BaseModel): + """Schema with values to create both a Saas ConnectionConfig and DatasetConfig from a template""" + + name: str # For ConnectionConfig + key: Optional[FidesOpsKey] # For ConnectionConfig + description: Optional[str] # For ConnectionConfig + secrets: connection_secrets_schemas # For ConnectionConfig + instance_key: FidesOpsKey # For DatasetConfig.fides_key diff --git a/src/fidesops/ops/schemas/connection_configuration/connection_secrets_saas.py b/src/fidesops/ops/schemas/connection_configuration/connection_secrets_saas.py index 173cf39ed..0780a7288 100644 --- a/src/fidesops/ops/schemas/connection_configuration/connection_secrets_saas.py +++ b/src/fidesops/ops/schemas/connection_configuration/connection_secrets_saas.py @@ -58,9 +58,9 @@ def get_saas_schema(self) -> Type[SaaSSchema]: if connector_param.default_value else (str, ...) ) - SaaSSchema.__doc__ = f"{str(self.saas_config.type).capitalize()} secrets schema" # Dynamically override the docstring + SaaSSchema.__doc__ = f"{str(self.saas_config.type).capitalize()} secrets schema" # Dynamically override the docstring to create a description model: Type[SaaSSchema] = create_model( - f"{self.saas_config.fides_key}_schema", + f"{self.saas_config.type}_schema", **field_definitions, __base__=SaaSSchema, ) diff --git a/src/fidesops/ops/schemas/dataset.py b/src/fidesops/ops/schemas/dataset.py index 2787e75bf..b26039139 100644 --- a/src/fidesops/ops/schemas/dataset.py +++ b/src/fidesops/ops/schemas/dataset.py @@ -1,7 +1,7 @@ from typing import Any, Dict, List, Optional from fideslang.models import Dataset, DatasetCollection, DatasetFieldBase -from pydantic import BaseModel, ConstrainedStr, validator +from pydantic import BaseModel, ConstrainedStr, Field, validator from fidesops.ops.common_exceptions import ( InvalidDataLengthValidationError, @@ -201,6 +201,9 @@ def valid_data_categories( class FidesopsDataset(Dataset): """Overrides fideslang Collection model with additional Fidesops annotations""" + fides_key: FidesOpsKey = Field( + description="A unique key used to identify this resource." + ) fidesops_meta: Optional[FidesopsDatasetMeta] collections: List[FidesopsDatasetCollection] """Overrides fideslang.models.Collection.collections""" diff --git a/src/fidesops/ops/schemas/shared_schemas.py b/src/fidesops/ops/schemas/shared_schemas.py index b45a590d2..8f78c2502 100644 --- a/src/fidesops/ops/schemas/shared_schemas.py +++ b/src/fidesops/ops/schemas/shared_schemas.py @@ -11,6 +11,11 @@ class FidesOpsKey(FidesKey): @classmethod def validate(cls, value: Optional[str]) -> Optional[str]: """Throws ValueError if val is not a valid FidesKey""" + if value == "": + # Ignore in saas templates. This value will be replaced with a + # user-specified value. + return value + if value is not None and not cls.regex.match(value): raise ValueError( "FidesKey must only contain alphanumeric characters, '.', '_' or '-'." diff --git a/src/fidesops/ops/service/connectors/saas/connector_registry_service.py b/src/fidesops/ops/service/connectors/saas/connector_registry_service.py new file mode 100644 index 000000000..6d745dffa --- /dev/null +++ b/src/fidesops/ops/service/connectors/saas/connector_registry_service.py @@ -0,0 +1,134 @@ +from __future__ import annotations + +from os.path import exists +from typing import Dict, List, Optional + +from fideslib.core.config import load_toml +from pydantic import BaseModel, validator +from sqlalchemy.orm import Session + +from fidesops.ops.models.connectionconfig import ( + AccessLevel, + ConnectionConfig, + ConnectionType, +) +from fidesops.ops.models.datasetconfig import DatasetConfig +from fidesops.ops.schemas.connection_configuration.connection_config import ( + SaasConnectionTemplateValues, +) +from fidesops.ops.schemas.dataset import FidesopsDataset +from fidesops.ops.schemas.saas.saas_config import SaaSConfig +from fidesops.ops.util.saas_util import ( + load_config, + load_config_with_replacement, + load_dataset, + load_dataset_with_replacement, +) + +_registry: Optional[ConnectorRegistry] = None +registry_file = "data/saas/saas_connector_registry.toml" + + +class ConnectorTemplate(BaseModel): + """ + A collection of paths to artifacts that make up + a complete SaaS connector (SaaS config, dataset, etc.) + """ + + config: str + dataset: str + icon: str + + @validator("config") + def validate_config(cls, config: str) -> str: + """Validates the config at the given path""" + SaaSConfig(**load_config(config)) + return config + + @validator("dataset") + def validate_dataset(cls, dataset: str) -> str: + """Validates the dataset at the given path""" + FidesopsDataset(**load_dataset(dataset)[0]) + return dataset + + @validator("icon") + def validate_icon(cls, icon: str) -> str: + """Validates the icon at the given path""" + if not exists(icon): + raise ValueError(f"Icon file {icon} was not found") + return icon + + +class ConnectorRegistry(BaseModel): + """A map of SaaS connector templates""" + + __root__: Dict[str, ConnectorTemplate] + + def connector_types(self) -> List[str]: + """List of registered SaaS connector types""" + return list(self.__root__) + + def get_connector_template( + self, connector_type: str + ) -> Optional[ConnectorTemplate]: + """ + Returns an object containing the references to the various SaaS connector artifacts + """ + return self.__root__.get(connector_type) + + +def create_connection_config_from_template_no_save( + db: Session, + template: ConnectorTemplate, + template_values: SaasConnectionTemplateValues, +) -> ConnectionConfig: + """Creates a SaaS connection config from a template without saving it.""" + # Load saas config from template and replace every instance of "" with the fides_key + # the user has chosen + config_from_template: Dict = load_config_with_replacement( + template.config, "", template_values.instance_key + ) + + # Create SaaS ConnectionConfig + connection_config = ConnectionConfig.create_without_saving( + db, + data={ + "name": template_values.name, + "key": template_values.key, + "description": template_values.description, + "connection_type": ConnectionType.saas, + "access": AccessLevel.write, + "saas_config": config_from_template, + }, + ) + + return connection_config + + +def create_dataset_config_from_template( + db: Session, + connection_config: ConnectionConfig, + template: ConnectorTemplate, + template_values: SaasConnectionTemplateValues, +) -> DatasetConfig: + """Creates a DatasetConfig from a template and associates it with a ConnectionConfig""" + # Load the dataset config from template and replace every instance of "" with the fides_key + # the user has chosen + dataset_from_template: Dict = load_dataset_with_replacement( + template.dataset, "", template_values.instance_key + )[0] + data = { + "connection_config_id": connection_config.id, + "fides_key": template_values.instance_key, + "dataset": dataset_from_template, + } + dataset_config = DatasetConfig.create(db, data=data) + return dataset_config + + +def load_registry(config_file: str) -> ConnectorRegistry: + """Loads a SaaS connector registry from the given config file.""" + global _registry # pylint: disable=W0603 + if _registry is None: + _registry = ConnectorRegistry.parse_obj(load_toml([config_file])) + return _registry diff --git a/src/fidesops/ops/service/saas_request/override_implementations/mailchimp_request_overrides.py b/src/fidesops/ops/service/saas_request/override_implementations/mailchimp_request_overrides.py index 01e85a5f5..5fcbcacf3 100644 --- a/src/fidesops/ops/service/saas_request/override_implementations/mailchimp_request_overrides.py +++ b/src/fidesops/ops/service/saas_request/override_implementations/mailchimp_request_overrides.py @@ -40,7 +40,7 @@ def mailchimp_messages_access( - name: conversation_id type: path references: - - dataset: mailchimp_connector_example + - dataset: mailchimp_instance field: conversations.id direction: from data_path: conversation_messages diff --git a/src/fidesops/ops/util/saas_util.py b/src/fidesops/ops/util/saas_util.py index 9510395d3..be4be17aa 100644 --- a/src/fidesops/ops/util/saas_util.py +++ b/src/fidesops/ops/util/saas_util.py @@ -21,6 +21,12 @@ FIDESOPS_GROUPED_INPUTS = "fidesops_grouped_inputs" +def load_yaml_as_string(filename: str) -> str: + yaml_file = load_file([filename]) + with open(yaml_file, "r", encoding="utf-8") as file: + return file.read() + + def load_config(filename: str) -> Dict: """Loads the saas config from the yaml file""" yaml_file = load_file([filename]) @@ -28,6 +34,32 @@ def load_config(filename: str) -> Dict: return yaml.safe_load(file).get("saas_config", []) +def load_config_with_replacement( + filename: str, string_to_replace: str, replacement: str +) -> Dict: + """Loads the saas config from the yaml file and replaces any string with the given value""" + yaml_str: str = load_yaml_as_string(filename).replace( + string_to_replace, replacement + ) + return yaml.safe_load(yaml_str).get("saas_config", []) + + +def load_dataset(filename: str) -> Dict: + yaml_file = load_file([filename]) + with open(yaml_file, "r", encoding="utf-8") as file: + return yaml.safe_load(file).get("dataset", []) + + +def load_dataset_with_replacement( + filename: str, string_to_replace: str, replacement: str +) -> Dict: + """Loads the dataset from the yaml file and replaces any string with the given value""" + yaml_str: str = load_yaml_as_string(filename).replace( + string_to_replace, replacement + ) + return yaml.safe_load(yaml_str).get("dataset", []) + + def merge_fields(target: Field, source: Field) -> Field: """Replaces source references and identities if they are available from the target""" if source.references is not None: diff --git a/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py b/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py index ee2aefcd6..5851c6f92 100644 --- a/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py @@ -1,16 +1,27 @@ from typing import List +from unittest import mock import pytest from fideslib.models.client import ClientDetail from starlette.testclient import TestClient -from fidesops.ops.api.v1.scope_registry import CONNECTION_READ, CONNECTION_TYPE_READ +from fidesops.ops.api.v1.scope_registry import ( + CONNECTION_READ, + CONNECTION_TYPE_READ, + SAAS_CONNECTION_INSTANTIATE, +) from fidesops.ops.api.v1.urn_registry import ( CONNECTION_TYPE_SECRETS, CONNECTION_TYPES, + SAAS_CONNECTOR_FROM_TEMPLATE, V1_URL_PREFIX, ) -from fidesops.ops.models.connectionconfig import ConnectionType +from fidesops.ops.models.connectionconfig import ( + AccessLevel, + ConnectionConfig, + ConnectionType, +) +from fidesops.ops.models.datasetconfig import DatasetConfig from fidesops.ops.schemas.connection_configuration.connection_config import ( ConnectionSystemTypeMap, SystemType, @@ -182,7 +193,7 @@ def test_get_connection_secret_schema_hubspot( ) assert resp.json() == { - "title": "hubspot_connector_example_schema", + "title": "hubspot_schema", "description": "Hubspot secrets schema", "type": "object", "properties": { @@ -196,3 +207,327 @@ def test_get_connection_secret_schema_hubspot( "required": ["hapikey"], "additionalProperties": False, } + + +class TestInstantiateConnectionFromTemplate: + @pytest.fixture(scope="function") + def base_url(self) -> str: + return V1_URL_PREFIX + SAAS_CONNECTOR_FROM_TEMPLATE + + def test_instantiate_connection_not_authenticated(self, api_client, base_url): + resp = api_client.post( + base_url.format(saas_connector_type="mailchimp"), headers={} + ) + assert resp.status_code == 401 + + def test_instantiate_connection_wrong_scope( + self, generate_auth_header, api_client, base_url + ): + auth_header = generate_auth_header(scopes=[CONNECTION_READ]) + resp = api_client.post( + base_url.format(saas_connector_type="mailchimp"), headers=auth_header + ) + assert resp.status_code == 403 + + def test_instantiate_nonexistent_template( + self, generate_auth_header, api_client, base_url + ): + auth_header = generate_auth_header(scopes=[SAAS_CONNECTION_INSTANTIATE]) + request_body = { + "instance_key": "test_instance_key", + "secrets": {}, + "name": "Unsupported Connector", + "description": "Unsupported connector description", + "key": "unsupported_connector", + } + resp = api_client.post( + base_url.format(saas_connector_type="does_not_exist"), + headers=auth_header, + json=request_body, + ) + assert resp.status_code == 404 + assert ( + resp.json()["detail"] + == f"SaaS connector type 'does_not_exist' is not yet available in Fidesops. For a list of available SaaS connectors, refer to /connection_type." + ) + + def test_instance_key_already_exists( + self, generate_auth_header, api_client, base_url, dataset_config + ): + auth_header = generate_auth_header(scopes=[SAAS_CONNECTION_INSTANTIATE]) + request_body = { + "instance_key": dataset_config.fides_key, + "secrets": { + "domain": "test_mailchimp_domain", + "username": "test_mailchimp_username", + "api_key": "test_mailchimp_api_key", + }, + "name": "Mailchimp Connector", + "description": "Mailchimp ConnectionConfig description", + "key": "mailchimp_connection_config", + } + resp = api_client.post( + base_url.format(saas_connector_type="mailchimp"), + headers=auth_header, + json=request_body, + ) + assert resp.status_code == 400 + assert ( + resp.json()["detail"] + == f"SaaS connector instance key '{dataset_config.fides_key}' already exists." + ) + + def test_template_secrets_validation( + self, generate_auth_header, api_client, base_url, db + ): + auth_header = generate_auth_header(scopes=[SAAS_CONNECTION_INSTANTIATE]) + # Secrets have one field missing, one field extra + request_body = { + "instance_key": "secondary_mailchimp_instance", + "secrets": { + "bad_mailchimp_secret_key": "bad_key", + "username": "test_mailchimp_username", + "api_key": "test_mailchimp_api_key", + }, + "name": "Mailchimp Connector", + "description": "Mailchimp ConnectionConfig description", + "key": "mailchimp_connection_config", + } + resp = api_client.post( + base_url.format(saas_connector_type="mailchimp"), + headers=auth_header, + json=request_body, + ) + + assert resp.status_code == 422 + assert resp.json()["detail"][0] == { + "loc": ["domain"], + "msg": "field required", + "type": "value_error.missing", + } + assert resp.json()["detail"][1] == { + "loc": ["bad_mailchimp_secret_key"], + "msg": "extra fields not permitted", + "type": "value_error.extra", + } + + connection_config = ConnectionConfig.filter( + db=db, conditions=(ConnectionConfig.key == "mailchimp_connection_config") + ).first() + assert connection_config is None, "ConnectionConfig not persisted" + + def test_connection_config_key_already_exists( + self, db, generate_auth_header, api_client, base_url, connection_config + ): + auth_header = generate_auth_header(scopes=[SAAS_CONNECTION_INSTANTIATE]) + request_body = { + "instance_key": "secondary_mailchimp_instance", + "secrets": { + "domain": "test_mailchimp_domain", + "username": "test_mailchimp_username", + "api_key": "test_mailchimp_api_key", + }, + "name": connection_config.name, + "description": "Mailchimp ConnectionConfig description", + "key": connection_config.key, + } + resp = api_client.post( + base_url.format(saas_connector_type="mailchimp"), + headers=auth_header, + json=request_body, + ) + assert resp.status_code == 400 + assert ( + f"Key {connection_config.key} already exists in ConnectionConfig" + in resp.json()["detail"] + ) + + def test_connection_config_name_already_exists( + self, db, generate_auth_header, api_client, base_url, connection_config + ): + auth_header = generate_auth_header(scopes=[SAAS_CONNECTION_INSTANTIATE]) + request_body = { + "instance_key": "secondary_mailchimp_instance", + "secrets": { + "domain": "test_mailchimp_domain", + "username": "test_mailchimp_username", + "api_key": "test_mailchimp_api_key", + }, + "name": connection_config.name, + "description": "Mailchimp ConnectionConfig description", + "key": "brand_new_key", + } + resp = api_client.post( + base_url.format(saas_connector_type="mailchimp"), + headers=auth_header, + json=request_body, + ) + assert resp.status_code == 400 + assert ( + f"Name {connection_config.name} already exists in ConnectionConfig" + in resp.json()["detail"] + ) + + def test_create_connection_from_template_without_supplying_connection_key( + self, db, generate_auth_header, api_client, base_url + ): + auth_header = generate_auth_header(scopes=[SAAS_CONNECTION_INSTANTIATE]) + request_body = { + "instance_key": "secondary_mailchimp_instance", + "secrets": { + "domain": "test_mailchimp_domain", + "username": "test_mailchimp_username", + "api_key": "test_mailchimp_api_key", + }, + "name": "Mailchimp Connector", + "description": "Mailchimp ConnectionConfig description", + } + resp = api_client.post( + base_url.format(saas_connector_type="mailchimp"), + headers=auth_header, + json=request_body, + ) + assert resp.status_code == 200 + + connection_config = ConnectionConfig.filter( + db=db, conditions=(ConnectionConfig.name == "Mailchimp Connector") + ).first() + dataset_config = DatasetConfig.filter( + db=db, + conditions=(DatasetConfig.fides_key == "secondary_mailchimp_instance"), + ).first() + + assert connection_config is not None + assert dataset_config is not None + + assert connection_config.key == "mailchimp_connector" + dataset_config.delete(db) + connection_config.delete(db) + + def test_invalid_instance_key(self, db, generate_auth_header, api_client, base_url): + auth_header = generate_auth_header(scopes=[SAAS_CONNECTION_INSTANTIATE]) + request_body = { + "instance_key": "< this is an invalid key! >", + "secrets": { + "domain": "test_mailchimp_domain", + "username": "test_mailchimp_username", + "api_key": "test_mailchimp_api_key", + }, + "name": "Mailchimp Connector", + "description": "Mailchimp ConnectionConfig description", + "key": "mailchimp_connection_config", + } + resp = api_client.post( + base_url.format(saas_connector_type="mailchimp"), + headers=auth_header, + json=request_body, + ) + assert resp.json()["detail"][0] == { + "loc": ["body", "instance_key"], + "msg": "FidesKey must only contain alphanumeric characters, '.', '_' or '-'.", + "type": "value_error", + } + + @mock.patch( + "fidesops.ops.api.v1.endpoints.saas_config_endpoints.create_dataset_config_from_template" + ) + def test_dataset_config_saving_fails( + self, mock_create_dataset, db, generate_auth_header, api_client, base_url + ): + mock_create_dataset.side_effect = Exception("KeyError") + + auth_header = generate_auth_header(scopes=[SAAS_CONNECTION_INSTANTIATE]) + request_body = { + "instance_key": "secondary_mailchimp_instance", + "secrets": { + "domain": "test_mailchimp_domain", + "username": "test_mailchimp_username", + "api_key": "test_mailchimp_api_key", + }, + "name": "Mailchimp Connector", + "description": "Mailchimp ConnectionConfig description", + "key": "mailchimp_connection_config", + } + resp = api_client.post( + base_url.format(saas_connector_type="mailchimp"), + headers=auth_header, + json=request_body, + ) + assert resp.status_code == 500 + assert ( + resp.json()["detail"] + == "SaaS Connector could not be created from the 'mailchimp' template at this time." + ) + + connection_config = ConnectionConfig.filter( + db=db, conditions=(ConnectionConfig.key == "mailchimp_connection_config") + ).first() + assert connection_config is None + + dataset_config = DatasetConfig.filter( + db=db, + conditions=(DatasetConfig.fides_key == "secondary_mailchimp_instance"), + ).first() + assert dataset_config is None + + def test_instantiate_connection_from_template( + self, db, generate_auth_header, api_client, base_url + ): + connection_config = ConnectionConfig.filter( + db=db, conditions=(ConnectionConfig.key == "mailchimp_connection_config") + ).first() + assert connection_config is None + + dataset_config = DatasetConfig.filter( + db=db, + conditions=(DatasetConfig.fides_key == "secondary_mailchimp_instance"), + ).first() + assert dataset_config is None + + auth_header = generate_auth_header(scopes=[SAAS_CONNECTION_INSTANTIATE]) + request_body = { + "instance_key": "secondary_mailchimp_instance", + "secrets": { + "domain": "test_mailchimp_domain", + "username": "test_mailchimp_username", + "api_key": "test_mailchimp_api_key", + }, + "name": "Mailchimp Connector", + "description": "Mailchimp ConnectionConfig description", + "key": "mailchimp_connection_config", + } + resp = api_client.post( + base_url.format(saas_connector_type="mailchimp"), + headers=auth_header, + json=request_body, + ) + + assert resp.status_code == 200 + assert resp.json()["fides_key"] == "secondary_mailchimp_instance" + + connection_config = ConnectionConfig.filter( + db=db, conditions=(ConnectionConfig.key == "mailchimp_connection_config") + ).first() + dataset_config = DatasetConfig.filter( + db=db, + conditions=(DatasetConfig.fides_key == "secondary_mailchimp_instance"), + ).first() + + assert connection_config is not None + assert dataset_config is not None + assert connection_config.name == "Mailchimp Connector" + assert connection_config.description == "Mailchimp ConnectionConfig description" + + assert connection_config.access == AccessLevel.write + assert connection_config.connection_type == ConnectionType.saas + assert connection_config.saas_config is not None + assert connection_config.disabled is False + assert connection_config.disabled_at is None + assert connection_config.last_test_timestamp is None + assert connection_config.last_test_succeeded is None + + assert dataset_config.connection_config_id == connection_config.id + assert dataset_config.dataset is not None + + dataset_config.delete(db) + connection_config.delete(db) diff --git a/tests/ops/fixtures/saas/adobe_campaign_fixtures.py b/tests/ops/fixtures/saas/adobe_campaign_fixtures.py index 4b7c359d7..d032bd012 100644 --- a/tests/ops/fixtures/saas/adobe_campaign_fixtures.py +++ b/tests/ops/fixtures/saas/adobe_campaign_fixtures.py @@ -12,8 +12,10 @@ ConnectionType, ) from fidesops.ops.models.datasetconfig import DatasetConfig -from fidesops.ops.util.saas_util import load_config -from tests.ops.fixtures.application_fixtures import load_dataset +from fidesops.ops.util.saas_util import ( + load_config_with_replacement, + load_dataset_with_replacement, +) from tests.ops.test_helpers.vault_client import get_secrets secrets = get_secrets("adobe_campaign") @@ -51,12 +53,19 @@ def adobe_campaign_erasure_identity_email() -> str: @pytest.fixture def adobe_campaign_config() -> Dict[str, Any]: - return load_config("data/saas/config/adobe_campaign_config.yml") + return load_config_with_replacement( + "data/saas/config/adobe_campaign_config.yml", + "", + "adobe_campaign_instance", + ) @pytest.fixture def adobe_campaign_dataset() -> Dict[str, Any]: - return load_dataset("data/saas/dataset/adobe_campaign_dataset.yml")[0] + return load_dataset_with_replacement( + "data/saas/dataset/adobe_campaign_dataset.yml" "", + "adobe_campaign_instance", + )[0] @pytest.fixture(scope="function") diff --git a/tests/ops/fixtures/saas/auth0_fixtures.py b/tests/ops/fixtures/saas/auth0_fixtures.py index f252323cd..c796949a9 100644 --- a/tests/ops/fixtures/saas/auth0_fixtures.py +++ b/tests/ops/fixtures/saas/auth0_fixtures.py @@ -14,8 +14,10 @@ ConnectionType, ) from fidesops.ops.models.datasetconfig import DatasetConfig -from tests.ops.fixtures.application_fixtures import load_dataset -from tests.ops.fixtures.saas_example_fixtures import load_config +from fidesops.ops.util.saas_util import ( + load_config_with_replacement, + load_dataset_with_replacement, +) from tests.ops.test_helpers.saas_test_utils import poll_for_existence from tests.ops.test_helpers.vault_client import get_secrets @@ -43,12 +45,16 @@ def auth0_erasure_identity_email(): @pytest.fixture def auth0_config() -> Dict[str, Any]: - return load_config("data/saas/config/auth0_config.yml") + return load_config_with_replacement( + "data/saas/config/auth0_config.yml", "", "auth_0_instance" + ) @pytest.fixture def auth0_dataset() -> Dict[str, Any]: - return load_dataset("data/saas/dataset/auth0_dataset.yml")[0] + return load_dataset_with_replacement( + "data/saas/dataset/auth0_dataset.yml", "", "auth_0_instance" + )[0] @pytest.fixture(scope="function") diff --git a/tests/ops/fixtures/saas/datadog_fixtures.py b/tests/ops/fixtures/saas/datadog_fixtures.py index de9d23e4a..230e52075 100644 --- a/tests/ops/fixtures/saas/datadog_fixtures.py +++ b/tests/ops/fixtures/saas/datadog_fixtures.py @@ -11,8 +11,10 @@ ConnectionType, ) from fidesops.ops.models.datasetconfig import DatasetConfig -from fidesops.ops.util.saas_util import load_config -from tests.ops.fixtures.application_fixtures import load_dataset +from fidesops.ops.util.saas_util import ( + load_config_with_replacement, + load_dataset_with_replacement, +) from tests.ops.test_helpers.vault_client import get_secrets secrets = get_secrets("datadog") @@ -36,12 +38,20 @@ def datadog_identity_email(saas_config): @pytest.fixture def datadog_config() -> Dict[str, Any]: - return load_config("data/saas/config/datadog_config.yml")[0] + return load_config_with_replacement( + "data/saas/config/datadog_config.yml", + "", + "datadog_instance", + ) @pytest.fixture def datadog_dataset() -> Dict[str, Any]: - return load_dataset("data/saas/dataset/datadog_dataset.yml")[0] + return load_dataset_with_replacement( + "data/saas/dataset/datadog_dataset.yml", + "", + "datadog_instance", + )[0] @pytest.fixture(scope="function") diff --git a/tests/ops/fixtures/saas/hubspot_fixtures.py b/tests/ops/fixtures/saas/hubspot_fixtures.py index 6b4632594..a8a95d0bb 100644 --- a/tests/ops/fixtures/saas/hubspot_fixtures.py +++ b/tests/ops/fixtures/saas/hubspot_fixtures.py @@ -14,8 +14,11 @@ from fidesops.ops.models.datasetconfig import DatasetConfig from fidesops.ops.schemas.saas.shared_schemas import HTTPMethod, SaaSRequestParams from fidesops.ops.service.connectors import SaaSConnector -from fidesops.ops.util.saas_util import format_body, load_config -from tests.ops.fixtures.application_fixtures import load_dataset +from fidesops.ops.util.saas_util import ( + format_body, + load_config_with_replacement, + load_dataset_with_replacement, +) from tests.ops.test_helpers.saas_test_utils import poll_for_existence from tests.ops.test_helpers.vault_client import get_secrets @@ -46,12 +49,20 @@ def hubspot_erasure_identity_email(): @pytest.fixture def hubspot_config() -> Dict[str, Any]: - return load_config("data/saas/config/hubspot_config.yml") + return load_config_with_replacement( + "data/saas/config/hubspot_config.yml", + "", + "hubspot_instance", + ) @pytest.fixture def hubspot_dataset() -> Dict[str, Any]: - return load_dataset("data/saas/dataset/hubspot_dataset.yml")[0] + return load_dataset_with_replacement( + "data/saas/dataset/hubspot_dataset.yml", + "", + "hubspot_instance", + )[0] @pytest.fixture(scope="function") diff --git a/tests/ops/fixtures/saas/logi_id_fixtures.py b/tests/ops/fixtures/saas/logi_id_fixtures.py index b36d19436..516ac63c3 100644 --- a/tests/ops/fixtures/saas/logi_id_fixtures.py +++ b/tests/ops/fixtures/saas/logi_id_fixtures.py @@ -14,8 +14,10 @@ ConnectionType, ) from fidesops.ops.models.datasetconfig import DatasetConfig -from tests.ops.fixtures.application_fixtures import load_dataset -from tests.ops.fixtures.saas_example_fixtures import load_config +from fidesops.ops.util.saas_util import ( + load_config_with_replacement, + load_dataset_with_replacement, +) from tests.ops.test_helpers.saas_test_utils import poll_for_existence from tests.ops.test_helpers.vault_client import get_secrets @@ -47,12 +49,20 @@ def logi_id_erasure_identity_email(): @pytest.fixture def logi_id_config() -> Dict[str, Any]: - return load_config("data/saas/config/logi_id_config.yml") + return load_config_with_replacement( + "data/saas/config/logi_id_config.yml", + "", + "logi_id_instance", + ) @pytest.fixture def logi_id_dataset() -> Dict[str, Any]: - return load_dataset("data/saas/dataset/logi_id_dataset.yml")[0] + return load_dataset_with_replacement( + "data/saas/dataset/logi_id_dataset.yml", + "", + "logi_id_instance", + )[0] @pytest.fixture(scope="function") diff --git a/tests/ops/fixtures/saas/mailchimp_fixtures.py b/tests/ops/fixtures/saas/mailchimp_fixtures.py index beaaf23b4..087f87479 100644 --- a/tests/ops/fixtures/saas/mailchimp_fixtures.py +++ b/tests/ops/fixtures/saas/mailchimp_fixtures.py @@ -14,8 +14,10 @@ from fidesops.ops.models.datasetconfig import DatasetConfig from fidesops.ops.schemas.saas.shared_schemas import HTTPMethod, SaaSRequestParams from fidesops.ops.service.connectors.saas_connector import SaaSConnector -from fidesops.ops.util.saas_util import load_config -from tests.ops.fixtures.application_fixtures import load_dataset +from fidesops.ops.util.saas_util import ( + load_config_with_replacement, + load_dataset_with_replacement, +) from tests.ops.test_helpers.vault_client import get_secrets secrets = get_secrets("mailchimp") @@ -40,12 +42,20 @@ def mailchimp_identity_email(saas_config): @pytest.fixture def mailchimp_config() -> Dict[str, Any]: - return load_config("data/saas/config/mailchimp_config.yml") + return load_config_with_replacement( + "data/saas/config/mailchimp_config.yml", + "", + "mailchimp_instance", + ) @pytest.fixture def mailchimp_dataset() -> Dict[str, Any]: - return load_dataset("data/saas/dataset/mailchimp_dataset.yml")[0] + return load_dataset_with_replacement( + "data/saas/dataset/mailchimp_dataset.yml", + "", + "mailchimp_instance", + )[0] @pytest.fixture(scope="function") diff --git a/tests/ops/fixtures/saas/outreach_fixtures.py b/tests/ops/fixtures/saas/outreach_fixtures.py index 381577395..fd46067d2 100644 --- a/tests/ops/fixtures/saas/outreach_fixtures.py +++ b/tests/ops/fixtures/saas/outreach_fixtures.py @@ -13,8 +13,10 @@ ConnectionType, ) from fidesops.ops.models.datasetconfig import DatasetConfig -from fidesops.ops.util.saas_util import load_config -from tests.ops.fixtures.application_fixtures import load_dataset +from fidesops.ops.util.saas_util import ( + load_config_with_replacement, + load_dataset_with_replacement, +) from tests.ops.test_helpers.vault_client import get_secrets secrets = get_secrets("outreach") @@ -51,12 +53,20 @@ def outreach_erasure_identity_email() -> str: @pytest.fixture def outreach_config() -> Dict[str, Any]: - return load_config("data/saas/config/outreach_config.yml") + return load_config_with_replacement( + "data/saas/config/outreach_config.yml", + "", + "outreach_instance", + ) @pytest.fixture def outreach_dataset() -> Dict[str, Any]: - return load_dataset("data/saas/dataset/outreach_dataset.yml")[0] + return load_dataset_with_replacement( + "data/saas/dataset/outreach_dataset.yml", + "", + "outreach_dataset", + )[0] @pytest.fixture(scope="function") diff --git a/tests/ops/fixtures/saas/salesforce_fixtures.py b/tests/ops/fixtures/saas/salesforce_fixtures.py index fb3b4304a..832434235 100644 --- a/tests/ops/fixtures/saas/salesforce_fixtures.py +++ b/tests/ops/fixtures/saas/salesforce_fixtures.py @@ -14,8 +14,10 @@ ConnectionType, ) from fidesops.ops.models.datasetconfig import DatasetConfig -from fidesops.ops.util.saas_util import load_config -from tests.ops.fixtures.application_fixtures import load_dataset +from fidesops.ops.util.saas_util import ( + load_config_with_replacement, + load_dataset_with_replacement, +) from tests.ops.test_helpers.vault_client import get_secrets secrets = get_secrets("salesforce") @@ -68,12 +70,20 @@ def salesforce_token(salesforce_secrets) -> str: @pytest.fixture def salesforce_config() -> Dict[str, Any]: - return load_config("data/saas/config/salesforce_config.yml") + return load_config_with_replacement( + "data/saas/config/salesforce_config.yml", + "", + "salesforce_instance", + ) @pytest.fixture def salesforce_dataset() -> Dict[str, Any]: - return load_dataset("data/saas/dataset/salesforce_dataset.yml")[0] + return load_dataset_with_replacement( + "data/saas/dataset/salesforce_dataset.yml", + "", + "salesforce_dataset", + )[0] @pytest.fixture(scope="function") diff --git a/tests/ops/fixtures/saas/segment_fixtures.py b/tests/ops/fixtures/saas/segment_fixtures.py index 7fbb850ab..7e7afd6f9 100644 --- a/tests/ops/fixtures/saas/segment_fixtures.py +++ b/tests/ops/fixtures/saas/segment_fixtures.py @@ -15,8 +15,10 @@ ConnectionType, ) from fidesops.ops.models.datasetconfig import DatasetConfig -from fidesops.ops.util.saas_util import load_config -from tests.ops.fixtures.application_fixtures import load_dataset +from fidesops.ops.util.saas_util import ( + load_config_with_replacement, + load_dataset_with_replacement, +) from tests.ops.test_helpers.saas_test_utils import poll_for_existence from tests.ops.test_helpers.vault_client import get_secrets @@ -53,12 +55,20 @@ def segment_identity_email(saas_config): @pytest.fixture def segment_config() -> Dict[str, Any]: - return load_config("data/saas/config/segment_config.yml") + return load_config_with_replacement( + "data/saas/config/segment_config.yml", + "", + "segment_instance", + ) @pytest.fixture def segment_dataset() -> Dict[str, Any]: - return load_dataset("data/saas/dataset/segment_dataset.yml")[0] + return load_dataset_with_replacement( + "data/saas/dataset/segment_dataset.yml", + "", + "segment_instance", + )[0] @pytest.fixture(scope="function") diff --git a/tests/ops/fixtures/saas/sendgrid_fixtures.py b/tests/ops/fixtures/saas/sendgrid_fixtures.py index 50a64f32d..48c3c4a1b 100644 --- a/tests/ops/fixtures/saas/sendgrid_fixtures.py +++ b/tests/ops/fixtures/saas/sendgrid_fixtures.py @@ -14,8 +14,10 @@ ConnectionType, ) from fidesops.ops.models.datasetconfig import DatasetConfig -from tests.ops.fixtures.application_fixtures import load_dataset -from tests.ops.fixtures.saas_example_fixtures import load_config +from fidesops.ops.util.saas_util import ( + load_config_with_replacement, + load_dataset_with_replacement, +) from tests.ops.test_helpers.saas_test_utils import poll_for_existence from tests.ops.test_helpers.vault_client import get_secrets @@ -46,12 +48,20 @@ def sendgrid_erasure_identity_email(): @pytest.fixture def sendgrid_config() -> Dict[str, Any]: - return load_config("data/saas/config/sendgrid_config.yml") + return load_config_with_replacement( + "data/saas/config/sendgrid_config.yml", + "", + "sendgrid_instance", + ) @pytest.fixture def sendgrid_dataset() -> Dict[str, Any]: - return load_dataset("data/saas/dataset/sendgrid_dataset.yml")[0] + return load_dataset_with_replacement( + "data/saas/dataset/sendgrid_dataset.yml", + "", + "sendgrid_instance", + )[0] @pytest.fixture(scope="function") diff --git a/tests/ops/fixtures/saas/sentry_fixtures.py b/tests/ops/fixtures/saas/sentry_fixtures.py index 9f4eee0cd..b100bbbd4 100644 --- a/tests/ops/fixtures/saas/sentry_fixtures.py +++ b/tests/ops/fixtures/saas/sentry_fixtures.py @@ -11,8 +11,10 @@ ConnectionType, ) from fidesops.ops.models.datasetconfig import DatasetConfig -from fidesops.ops.util.saas_util import load_config -from tests.ops.fixtures.application_fixtures import load_dataset +from fidesops.ops.util.saas_util import ( + load_config_with_replacement, + load_dataset_with_replacement, +) from tests.ops.test_helpers.vault_client import get_secrets secrets = get_secrets("sentry") @@ -44,12 +46,16 @@ def sentry_identity_email(saas_config): @pytest.fixture def sentry_config() -> Dict[str, Any]: - return load_config("data/saas/config/sentry_config.yml") + return load_config_with_replacement( + "data/saas/config/sentry_config.yml", "", "sentry_instance" + ) @pytest.fixture def sentry_dataset() -> Dict[str, Any]: - return load_dataset("data/saas/dataset/sentry_dataset.yml")[0] + return load_dataset_with_replacement( + "data/saas/dataset/sentry_dataset.yml", "", "sentry_dataset" + )[0] @pytest.fixture(scope="function") diff --git a/tests/ops/fixtures/saas/stripe_fixtures.py b/tests/ops/fixtures/saas/stripe_fixtures.py index 4ea8e4888..f08441101 100644 --- a/tests/ops/fixtures/saas/stripe_fixtures.py +++ b/tests/ops/fixtures/saas/stripe_fixtures.py @@ -13,8 +13,10 @@ ConnectionType, ) from fidesops.ops.models.datasetconfig import DatasetConfig -from fidesops.ops.util.saas_util import load_config -from tests.ops.fixtures.application_fixtures import load_dataset +from fidesops.ops.util.saas_util import ( + load_config_with_replacement, + load_dataset_with_replacement, +) from tests.ops.test_helpers.vault_client import get_secrets secrets = get_secrets("stripe") @@ -44,12 +46,18 @@ def stripe_erasure_identity_email(): @pytest.fixture def stripe_config() -> Dict[str, Any]: - return load_config("data/saas/config/stripe_config.yml") + return load_config_with_replacement( + "data/saas/config/stripe_config.yml", "", "stripe_instance" + ) @pytest.fixture def stripe_dataset() -> Dict[str, Any]: - return load_dataset("data/saas/dataset/stripe_dataset.yml")[0] + return load_dataset_with_replacement( + "data/saas/dataset/stripe_dataset.yml", + "", + "stripe_instance", + )[0] @pytest.fixture(scope="function") diff --git a/tests/ops/fixtures/saas/zendesk_fixtures.py b/tests/ops/fixtures/saas/zendesk_fixtures.py index b67121efa..db13496c1 100644 --- a/tests/ops/fixtures/saas/zendesk_fixtures.py +++ b/tests/ops/fixtures/saas/zendesk_fixtures.py @@ -12,8 +12,10 @@ ConnectionType, ) from fidesops.ops.models.datasetconfig import DatasetConfig -from fidesops.ops.util.saas_util import load_config -from tests.ops.fixtures.application_fixtures import load_dataset +from fidesops.ops.util.saas_util import ( + load_config_with_replacement, + load_dataset_with_replacement, +) from tests.ops.test_helpers.vault_client import get_secrets secrets = get_secrets("zendesk") @@ -42,12 +44,20 @@ def zendesk_erasure_identity_email() -> str: @pytest.fixture def zendesk_config() -> Dict[str, Any]: - return load_config("data/saas/config/zendesk_config.yml") + return load_config_with_replacement( + "data/saas/config/zendesk_config.yml", + "", + "zendesk_instance", + ) @pytest.fixture def zendesk_dataset() -> Dict[str, Any]: - return load_dataset("data/saas/dataset/zendesk_dataset.yml")[0] + return load_dataset_with_replacement( + "data/saas/dataset/zendesk_dataset.yml", + "", + "zendesk_instance", + )[0] @pytest.fixture(scope="function") diff --git a/tests/ops/integration_tests/saas/test_adobe_campaign_task.py b/tests/ops/integration_tests/saas/test_adobe_campaign_task.py index bc79fe659..fabd5200f 100644 --- a/tests/ops/integration_tests/saas/test_adobe_campaign_task.py +++ b/tests/ops/integration_tests/saas/test_adobe_campaign_task.py @@ -305,8 +305,8 @@ def test_adobe_campaign_saas_erasure_request_task( # Assert erasure request made to adobe_campaign_user assert x == { - "adobe_campaign_connector_example:profile": 1, - "adobe_campaign_connector_example:marketing_history": 0, + "adobe_instance:profile": 1, + "adobe_instance:marketing_history": 0, } config.execution.masking_strict = masking_strict # Reset diff --git a/tests/ops/integration_tests/saas/test_hubspot_task.py b/tests/ops/integration_tests/saas/test_hubspot_task.py index fe68cc17f..18f2cb355 100644 --- a/tests/ops/integration_tests/saas/test_hubspot_task.py +++ b/tests/ops/integration_tests/saas/test_hubspot_task.py @@ -70,7 +70,12 @@ def test_saas_access_request_task( f"{dataset_name}:contacts", f"{dataset_name}:subscription_preferences", } - assert set(filtered_results[f"{dataset_name}:contacts"][0].keys()) == {"properties"} + assert set(filtered_results[f"{dataset_name}:contacts"][0].keys()) == { + "id", + "createdAt", + "updatedAt", + "properties", + } assert set( filtered_results[f"{dataset_name}:contacts"][0]["properties"].keys() @@ -175,9 +180,9 @@ def test_saas_erasure_request_task( # Masking request only issued to "contacts" and "subscription_preferences" endpoints assert erasure == { - "hubspot_connector_example:contacts": 1, - "hubspot_connector_example:owners": 0, - "hubspot_connector_example:subscription_preferences": 1, + "hubspot_instance:contacts": 1, + "hubspot_instance:owners": 0, + "hubspot_instance:subscription_preferences": 1, } connector = SaaSConnector(connection_config_hubspot) diff --git a/tests/ops/integration_tests/saas/test_logi_id_task.py b/tests/ops/integration_tests/saas/test_logi_id_task.py index 1b8bc6ae2..0fbe031e2 100644 --- a/tests/ops/integration_tests/saas/test_logi_id_task.py +++ b/tests/ops/integration_tests/saas/test_logi_id_task.py @@ -154,8 +154,8 @@ def test_logi_id_erasure_request_task( db, ) assert erasure == { - "logi_id_connector_example:user_claims": 0, - "logi_id_connector_example:users": 1, + "logi_id_instance:user_claims": 0, + "logi_id_instance:users": 1, } # Verifying user is deleted diff --git a/tests/ops/integration_tests/saas/test_segment_task.py b/tests/ops/integration_tests/saas/test_segment_task.py index 0673aa35b..fb03b8573 100644 --- a/tests/ops/integration_tests/saas/test_segment_task.py +++ b/tests/ops/integration_tests/saas/test_segment_task.py @@ -222,10 +222,10 @@ def test_segment_saas_erasure_request_task( # Assert erasure request made to segment_user - cannot verify success immediately as this can take # days, weeks to process assert x == { - "segment_connector_example:segment_user": 1, - "segment_connector_example:traits": 0, - "segment_connector_example:external_ids": 0, - "segment_connector_example:track_events": 0, + "segment_instance:segment_user": 1, + "segment_instance:traits": 0, + "segment_instance:external_ids": 0, + "segment_instance:track_events": 0, } config.execution.masking_strict = True # Reset diff --git a/tests/ops/integration_tests/saas/test_sendgrid_task.py b/tests/ops/integration_tests/saas/test_sendgrid_task.py index bd098ba74..254f2b21a 100644 --- a/tests/ops/integration_tests/saas/test_sendgrid_task.py +++ b/tests/ops/integration_tests/saas/test_sendgrid_task.py @@ -135,7 +135,7 @@ def test_sendgrid_erasure_request_task( get_cached_data_for_erasures(privacy_request.id), db, ) - assert erasure == {"sendgrid_connector_example:contacts": 1} + assert erasure == {"sendgrid_instance:contacts": 1} error_message = f"Contact with email {sendgrid_erasure_identity_email} could not be deleted in Sendgrid" poll_for_existence( contact_exists, diff --git a/tests/ops/integration_tests/saas/test_sentry_task.py b/tests/ops/integration_tests/saas/test_sentry_task.py index ef7c9bc9e..d261aee59 100644 --- a/tests/ops/integration_tests/saas/test_sentry_task.py +++ b/tests/ops/integration_tests/saas/test_sentry_task.py @@ -376,12 +376,12 @@ def test_sentry_erasure_request_task( # Masking request only issued to "issues" endpoint assert x == { - "sentry_connector:projects": 0, - "sentry_connector:person": 0, - "sentry_connector:issues": 1, - "sentry_connector:organizations": 0, - "sentry_connector:user_feedback": 0, - "sentry_connector:employees": 0, + "sentry_instance:projects": 0, + "sentry_instance:person": 0, + "sentry_instance:issues": 1, + "sentry_instance:organizations": 0, + "sentry_instance:user_feedback": 0, + "sentry_instance:employees": 0, } # Verify the user has been assigned to None diff --git a/tests/ops/schemas/connection_configuration/test_connection_secrets_saas.py b/tests/ops/schemas/connection_configuration/test_connection_secrets_saas.py index 1aad40d02..45a2a1de9 100644 --- a/tests/ops/schemas/connection_configuration/test_connection_secrets_saas.py +++ b/tests/ops/schemas/connection_configuration/test_connection_secrets_saas.py @@ -22,7 +22,7 @@ def test_get_saas_schema(self, saas_config): that the schema is a subclass of SaaSSchema """ schema = SaaSSchemaFactory(saas_config).get_saas_schema() - assert schema.__name__ == f"{saas_config.fides_key}_schema" + assert schema.__name__ == f"{saas_config.type}_schema" assert issubclass(schema.__base__, SaaSSchema) def test_validation( @@ -43,7 +43,7 @@ def test_missing_fields(self, saas_config: SaaSConfig): if not connector_param.default_value ] assert ( - f"{saas_config.fides_key}_schema must be supplied all of: " + f"{saas_config.type}_schema must be supplied all of: " f"[{', '.join(required_fields)}]." in str(exc.value) ) diff --git a/tests/ops/service/connectors/test_connector_registry_service.py b/tests/ops/service/connectors/test_connector_registry_service.py new file mode 100644 index 000000000..8c482ac82 --- /dev/null +++ b/tests/ops/service/connectors/test_connector_registry_service.py @@ -0,0 +1,21 @@ +from fidesops.ops.service.connectors.saas.connector_registry_service import ( + ConnectorTemplate, + load_registry, + registry_file, +) + + +class TestConnectionRegistry: + def test_get_connector_template(self): + registry = load_registry(registry_file) + + assert "mailchimp" in registry.connector_types() + + assert registry.get_connector_template("bad_key") is None + mailchimp_registry = registry.get_connector_template("mailchimp") + + assert mailchimp_registry == ConnectorTemplate( + config="data/saas/config/mailchimp_config.yml", + dataset="data/saas/dataset/mailchimp_dataset.yml", + icon="data/saas/icon/mailchimp.svg", + ) diff --git a/tests/ops/service/privacy_request/request_runner_service_test.py b/tests/ops/service/privacy_request/request_runner_service_test.py index 884c71cb1..b0ff85ccb 100644 --- a/tests/ops/service/privacy_request/request_runner_service_test.py +++ b/tests/ops/service/privacy_request/request_runner_service_test.py @@ -495,7 +495,7 @@ def test_create_and_process_access_request_saas_mailchimp( assert results[key] is not None assert results[key] != {} - result_key_prefix = f"EN_{pr.id}__access_request__mailchimp_connector_example:" + result_key_prefix = f"EN_{pr.id}__access_request__mailchimp_instance:" member_key = result_key_prefix + "member" assert results[member_key][0]["email_address"] == customer_email @@ -600,7 +600,7 @@ def test_create_and_process_access_request_saas_hubspot( assert results[key] is not None assert results[key] != {} - result_key_prefix = f"EN_{pr.id}__access_request__hubspot_connector_example:" + result_key_prefix = f"EN_{pr.id}__access_request__hubspot_instance:" contacts_key = result_key_prefix + "contacts" assert results[contacts_key][0]["properties"]["email"] == customer_email