From 2bf7481a891bc5a20a2b3e612723dabbc0f70d18 Mon Sep 17 00:00:00 2001 From: Catherine Smith Date: Mon, 18 Jul 2022 10:04:07 -0500 Subject: [PATCH] Feat: Vault for secrets (#869) * Replace config/gh secrets with secrets from vault * sorting and update changelog * fix path for importing test helpers * more sorting * run ci again with empty commit * update unsafe_pr_checks * allow for no vault vars for unit tests, adds back back for saas config toml vars * check for client * sort * init client to None * fix imports * fix pytest markers to better indicate which tests rely on actual secrets --- .github/workflows/unsafe_pr_checks.yml | 53 +++++-------------- CHANGELOG.md | 1 + Makefile | 10 +--- requirements.txt | 1 + .../test_connection_config_endpoints.py | 2 + tests/fixtures/saas/hubspot_fixtures.py | 15 +++--- tests/fixtures/saas/mailchimp_fixtures.py | 19 ++++--- tests/fixtures/saas/outreach_fixtures.py | 22 ++++---- tests/fixtures/saas/salesforce_fixtures.py | 21 ++++---- tests/fixtures/saas/segment_fixtures.py | 26 ++++----- tests/fixtures/saas/sentry_fixtures.py | 24 ++++----- tests/fixtures/saas/stripe_fixtures.py | 21 ++++---- tests/fixtures/saas/zendesk_fixtures.py | 20 ++++--- .../saas/test_salesforce_task.py | 2 +- .../service/connectors/test_saas_connector.py | 3 +- tests/test_helpers/vault_client.py | 40 ++++++++++++++ 16 files changed, 143 insertions(+), 137 deletions(-) create mode 100644 tests/test_helpers/vault_client.py diff --git a/.github/workflows/unsafe_pr_checks.yml b/.github/workflows/unsafe_pr_checks.yml index f114f3e16b..0b723fbd2f 100644 --- a/.github/workflows/unsafe_pr_checks.yml +++ b/.github/workflows/unsafe_pr_checks.yml @@ -27,49 +27,24 @@ jobs: External-SaaS-Connectors: runs-on: ubuntu-latest if: contains(github.event.pull_request.labels.*.name, 'run unsafe ci checks') + permissions: + contents: read + id-token: write steps: - name: Checkout uses: actions/checkout@v3 + - name: Get Vault Token + uses: hashicorp/vault-action@v2.4.1 + with: + url: ${{ secrets.VAULT_ADDR }} + namespace: ${{ secrets.VAULT_NAMESPACE }} + method: jwt + role: ${{ secrets.VAULT_ROLE }} + exportToken: True + - name: SaaS Connector Tests env: - MAILCHIMP_DOMAIN: ${{ secrets.MAILCHIMP_DOMAIN }} - MAILCHIMP_USERNAME: ${{ secrets.MAILCHIMP_USERNAME }} - MAILCHIMP_API_KEY: ${{ secrets.MAILCHIMP_API_KEY }} - MAILCHIMP_IDENTITY_EMAIL: ${{ secrets.MAILCHIMP_IDENTITY_EMAIL }} - SENTRY_DOMAIN: ${{ secrets.SENTRY_DOMAIN }} - SENTRY_ACCESS_TOKEN: ${{ secrets.SENTRY_ACCESS_TOKEN }} - SENTRY_ERASURE_TOKEN: ${{ secrets.SENTRY_ERASURE_TOKEN }} - SENTRY_IDENTITY_EMAIL: ${{ secrets.SENTRY_IDENTITY_EMAIL }} - SENTRY_ERASURE_IDENTITY: ${{ secrets.SENTRY_ERASURE_IDENTITY }} - SENTRY_USER_ID: ${{ secrets.SENTRY_USER_ID }} - SENTRY_ISSUE_URL: ${{ secrets.SENTRY_ISSUE_URL }} - STRIPE_DOMAIN: ${{ secrets.STRIPE_DOMAIN }} - STRIPE_API_KEY: ${{ secrets.STRIPE_API_KEY }} - STRIPE_PAYMENT_TYPES: ${{ secrets.STRIPE_PAYMENT_TYPES }} - STRIPE_ITEMS_PER_PAGE: ${{ secrets.STRIPE_ITEMS_PER_PAGE }} - STRIPE_IDENTITY_EMAIL: ${{ secrets.STRIPE_IDENTITY_EMAIL }} - HUBSPOT_DOMAIN: ${{ secrets.HUBSPOT_DOMAIN }} - HUBSPOT_HAPIKEY: ${{ secrets.HUBSPOT_HAPIKEY }} - HUBSPOT_IDENTITY_EMAIL: ${{ secrets.HUBSPOT_IDENTITY_EMAIL }} - SEGMENT_DOMAIN: ${{ secrets.SEGMENT_DOMAIN }} - SEGMENT_API_DOMAIN: ${{ secrets.SEGMENT_API_DOMAIN }} - SEGMENT_USER_TOKEN: ${{ secrets.SEGMENT_USER_TOKEN }} - SEGMENT_PERSONAS_DOMAIN: ${{ secrets.SEGMENT_PERSONAS_DOMAIN }} - SEGMENT_WORKSPACE: ${{ secrets.SEGMENT_WORKSPACE }} - SEGMENT_ACCESS_TOKEN: ${{ secrets.SEGMENT_ACCESS_TOKEN }} - SEGMENT_NAMESPACE_ID: ${{ secrets.SEGMENT_NAMESPACE_ID }} - SEGMENT_ACCESS_SECRET: ${{ secrets.SEGMENT_ACCESS_SECRET }} - SEGMENT_IDENTITY_EMAIL: ${{ secrets.SEGMENT_IDENTITY_EMAIL }} - SALESFORCE_DOMAIN: ${{ secrets.SALESFORCE_DOMAIN }} - SALESFORCE_CLIENT_ID: ${{ secrets.SALESFORCE_CLIENT_ID }} - SALESFORCE_CLIENT_SECRET: ${{ secrets.SALESFORCE_CLIENT_SECRET }} - SALESFORCE_ACCESS_TOKEN: ${{ secrets.SALESFORCE_ACCESS_TOKEN }} - SALESFORCE_USERNAME: ${{ secrets.SALESFORCE_USERNAME }} - SALESFORCE_PASSWORD: ${{ secrets.SALESFORCE_PASSWORD }} - SALESFORCE_IDENTITY_EMAIL: ${{ secrets.SALESFORCE_IDENTITY_EMAIL }} - ZENDESK_DOMAIN: ${{ secrets.ZENDESK_DOMAIN }} - ZENDESK_USERNAME: ${{ secrets.ZENDESK_USERNAME }} - ZENDESK_API_KEY: ${{ secrets.ZENDESK_API_KEY }} - ZENDESK_IDENTITY_EMAIL: ${{ secrets.ZENDESK_IDENTITY_EMAIL }} + VAULT_ADDR: ${{ secrets.VAULT_ADDR }} + VAULT_NAMESPACE: ${{ secrets.VAULT_NAMESPACE }} run: make pytest-saas diff --git a/CHANGELOG.md b/CHANGELOG.md index d6ff704632..1e4d767015 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,7 @@ The types of changes are: * Adds endpoint to get available connectors (database and saas) [#768](https://github.com/ethyca/fidesops/pull/768) * Erasure support for Zendesk [#775](https://github.com/ethyca/fidesops/pull/775) * Adds endpoint to get the secrets required for different connectors [#795](https://github.com/ethyca/fidesops/pull/795) +* Adds Vault for secrets management [#688](https://github.com/ethyca/fidesops/pull/869) ### Changed diff --git a/Makefile b/Makefile index cf6a171608..570c348b63 100644 --- a/Makefile +++ b/Makefile @@ -59,7 +59,7 @@ server-no-db: compose-build -e FIDESOPS__REDIS__ENABLED=false \ --no-deps \ --service-ports \ - webserver + webserver server-shell: compose-build @docker-compose run $(COMPOSE_SERVICE_NAME) /bin/bash @@ -166,13 +166,7 @@ pytest-saas: compose-build @echo "Running integration tests for SaaS connectors" @docker-compose run \ -e ANALYTICS_OPT_OUT \ - -e MAILCHIMP_DOMAIN -e MAILCHIMP_USERNAME -e MAILCHIMP_API_KEY -e MAILCHIMP_IDENTITY_EMAIL \ - -e SENTRY_DOMAIN -e SENTRY_ACCESS_TOKEN -e SENTRY_IDENTITY_EMAIL -e SENTRY_ERASURE_TOKEN -e SENTRY_ERASURE_IDENTITY -e SENTRY_USER_ID -e SENTRY_ISSUE_URL \ - -e HUBSPOT_DOMAIN -e HUBSPOT_HAPIKEY -e HUBSPOT_IDENTITY_EMAIL \ - -e ZENDESK_DOMAIN -e ZENDESK_USERNAME -e ZENDESK_API_KEY -e ZENDESK_IDENTITY_EMAIL \ - -e SEGMENT_DOMAIN -e SEGMENT_PERSONAS_DOMAIN -e SEGMENT_WORKSPACE -e SEGMENT_ACCESS_TOKEN -e SEGMENT_API_DOMAIN -e SEGMENT_NAMESPACE_ID -e SEGMENT_ACCESS_SECRET -e SEGMENT_USER_TOKEN -e SEGMENT_IDENTITY_EMAIL \ - -e STRIPE_DOMAIN -e STRIPE_API_KEY -e STRIPE_PAYMENT_TYPES -e STRIPE_PAGE_SIZE -e STRIPE_IDENTITY_EMAIL \ - -e SALESFORCE_DOMAIN -e SALESFORCE_CLIENT_ID -e SALESFORCE_CLIENT_SECRET -e SALESFORCE_ACCESS_TOKEN -e SALESFORCE_USERNAME -e SALESFORCE_PASSWORD -e SALESFORCE_IDENTITY_EMAIL \ + -e VAULT_ADDR -e VAULT_NAMESPACE -e VAULT_TOKEN \ $(COMPOSE_SERVICE_NAME) pytest $(pytestpath) -m "integration_saas" @make teardown diff --git a/requirements.txt b/requirements.txt index ba2bd50c0a..4789181544 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,6 +12,7 @@ fastapi[all]==0.78.0 fideslang==1.0.0 fideslib==3.0.0 fideslog==1.2.1 +hvac==0.11.2 multidimensional_urlencode==0.0.4 pandas==1.3.3 passlib[bcrypt]==1.7.4 diff --git a/tests/api/v1/endpoints/test_connection_config_endpoints.py b/tests/api/v1/endpoints/test_connection_config_endpoints.py index d145d4be6f..1d317ca52d 100644 --- a/tests/api/v1/endpoints/test_connection_config_endpoints.py +++ b/tests/api/v1/endpoints/test_connection_config_endpoints.py @@ -588,6 +588,8 @@ def test_filter_test_status( assert len(items) == 3 assert [item["last_test_succeeded"] is None for item in items] + @pytest.mark.integration_saas + @pytest.mark.integration_stripe def test_filter_system_type( self, db, diff --git a/tests/fixtures/saas/hubspot_fixtures.py b/tests/fixtures/saas/hubspot_fixtures.py index e95a254532..0519950674 100644 --- a/tests/fixtures/saas/hubspot_fixtures.py +++ b/tests/fixtures/saas/hubspot_fixtures.py @@ -1,5 +1,4 @@ import json -import os from typing import Any, Dict, Generator import pydash @@ -19,26 +18,26 @@ from fidesops.util.saas_util import format_body, load_config from tests.fixtures.application_fixtures import load_dataset from tests.test_helpers.saas_test_utils import poll_for_existence +from tests.test_helpers.vault_client import get_secrets saas_config = load_toml(["saas_config.toml"]) +secrets = get_secrets("hubspot") HUBSPOT_FIRSTNAME = "SomeoneFirstname" -@pytest.fixture(scope="function") +@pytest.fixture(scope="session") def hubspot_secrets(): return { - "domain": pydash.get(saas_config, "hubspot.domain") - or os.environ.get("HUBSPOT_DOMAIN"), - "hapikey": pydash.get(saas_config, "hubspot.hapikey") - or os.environ.get("HUBSPOT_HAPIKEY"), + "domain": pydash.get(saas_config, "hubspot.domain") or secrets["domain"], + "hapikey": pydash.get(saas_config, "hubspot.hapikey") or secrets["hapikey"], } @pytest.fixture(scope="function") def hubspot_identity_email(): - return pydash.get(saas_config, "hubspot.identity_email") or os.environ.get( - "HUBSPOT_IDENTITY_EMAIL" + return ( + pydash.get(saas_config, "hubspot.identity_email") or secrets["identity_email"] ) diff --git a/tests/fixtures/saas/mailchimp_fixtures.py b/tests/fixtures/saas/mailchimp_fixtures.py index f73503a4e6..f76aff6624 100644 --- a/tests/fixtures/saas/mailchimp_fixtures.py +++ b/tests/fixtures/saas/mailchimp_fixtures.py @@ -1,5 +1,4 @@ import json -import os from typing import Any, Dict, Generator import pydash @@ -18,26 +17,26 @@ from fidesops.service.connectors.saas_connector import SaaSConnector from fidesops.util.saas_util import load_config from tests.fixtures.application_fixtures import load_dataset +from tests.test_helpers.vault_client import get_secrets saas_config = load_toml(["saas_config.toml"]) +secrets = get_secrets("mailchimp") -@pytest.fixture(scope="function") +@pytest.fixture(scope="session") def mailchimp_secrets(): return { - "domain": pydash.get(saas_config, "mailchimp.domain") - or os.environ.get("MAILCHIMP_DOMAIN"), + "domain": pydash.get(saas_config, "mailchimp.domain") or secrets["domain"], "username": pydash.get(saas_config, "mailchimp.username") - or os.environ.get("MAILCHIMP_USERNAME"), - "api_key": pydash.get(saas_config, "mailchimp.api_key") - or os.environ.get("MAILCHIMP_API_KEY"), + or secrets["username"], + "api_key": pydash.get(saas_config, "mailchimp.api_key") or secrets["api_key"], } -@pytest.fixture(scope="function") +@pytest.fixture(scope="session") def mailchimp_identity_email(): - return pydash.get(saas_config, "mailchimp.identity_email") or os.environ.get( - "MAILCHIMP_IDENTITY_EMAIL" + return ( + pydash.get(saas_config, "mailchimp.identity_email") or secrets["identity_email"] ) diff --git a/tests/fixtures/saas/outreach_fixtures.py b/tests/fixtures/saas/outreach_fixtures.py index 3f59de5a1e..904757aa93 100644 --- a/tests/fixtures/saas/outreach_fixtures.py +++ b/tests/fixtures/saas/outreach_fixtures.py @@ -1,4 +1,3 @@ -import os from typing import Any, Dict, Generator import pydash @@ -17,32 +16,33 @@ from fidesops.models.datasetconfig import DatasetConfig from fidesops.util.saas_util import load_config from tests.fixtures.application_fixtures import load_dataset +from tests.test_helpers.vault_client import get_secrets saas_config = load_toml(["saas_config.toml"]) +secrets = get_secrets("outreach") -@pytest.fixture(scope="function") +@pytest.fixture(scope="session") def outreach_secrets(): return { - "domain": pydash.get(saas_config, "outreach.domain") - or os.environ.get("OUTREACH_DOMAIN"), + "domain": pydash.get(saas_config, "outreach.domain") or secrets["domain"], "requester_email": pydash.get(saas_config, "outreach.requester_email") - or os.environ.get("OUTREACH_REQUESTER_EMAIL"), + or secrets["requester_email"], "client_id": pydash.get(saas_config, "outreach.client_id") - or os.environ.get("OUTREACH_CLIENT_ID"), + or secrets["client_id"], "client_secret": pydash.get(saas_config, "outreach.client_secret") - or os.environ.get("OUTREACH_CLIENT_SECRET"), + or secrets["client_secret"], "redirect_uri": pydash.get(saas_config, "outreach.redirect_uri") - or os.environ.get("OUTREACH_REDIRECT_URI"), + or secrets["redirect_uri"], "page_size": pydash.get(saas_config, "outreach.page_size") - or os.environ.get("OUTREACH_PAGE_SIZE"), + or secrets["page_size"], } @pytest.fixture(scope="session") def outreach_identity_email(): - return pydash.get(saas_config, "outreach.identity_email") or os.environ.get( - "OUTREACH_IDENTITY_EMAIL" + return ( + pydash.get(saas_config, "outreach.identity_email") or secrets["identity_email"] ) diff --git a/tests/fixtures/saas/salesforce_fixtures.py b/tests/fixtures/saas/salesforce_fixtures.py index 98bce17c82..0302787720 100644 --- a/tests/fixtures/saas/salesforce_fixtures.py +++ b/tests/fixtures/saas/salesforce_fixtures.py @@ -1,4 +1,3 @@ -import os from typing import Any, Dict, Generator import pydash @@ -17,32 +16,34 @@ from fidesops.models.datasetconfig import DatasetConfig from fidesops.util.saas_util import load_config from tests.fixtures.application_fixtures import load_dataset +from tests.test_helpers.vault_client import get_secrets saas_config = load_toml(["saas_config.toml"]) +secrets = get_secrets("salesforce") @pytest.fixture(scope="session") def salesforce_secrets(): return { - "domain": pydash.get(saas_config, "salesforce.domain") - or os.environ.get("SALESFORCE_DOMAIN"), + "domain": pydash.get(saas_config, "salesforce.domain") or secrets["domain"], "username": pydash.get(saas_config, "salesforce.username") - or os.environ.get("SALESFORCE_USERNAME"), + or secrets["username"], "password": pydash.get(saas_config, "salesforce.password") - or os.environ.get("SALESFORCE_PASSWORD"), + or secrets["password"], "client_id": pydash.get(saas_config, "salesforce.client_id") - or os.environ.get("SALESFORCE_CLIENT_ID"), + or secrets["client_id"], "client_secret": pydash.get(saas_config, "salesforce.client_secret") - or os.environ.get("SALESFORCE_CLIENT_SECRET"), + or secrets["client_secret"], "access_token": pydash.get(saas_config, "salesforce.access_token") - or os.environ.get("SALESFORCE_ACCESS_TOKEN"), + or secrets["access_token"], } @pytest.fixture(scope="session") def salesforce_identity_email(): - return pydash.get(saas_config, "salesforce.identity_email") or os.environ.get( - "SALESFORCE_IDENTITY_EMAIL" + return ( + pydash.get(saas_config, "salesforce.identity_email") + or secrets["identity_email"] ) diff --git a/tests/fixtures/saas/segment_fixtures.py b/tests/fixtures/saas/segment_fixtures.py index bf8bc0efeb..73ad702593 100644 --- a/tests/fixtures/saas/segment_fixtures.py +++ b/tests/fixtures/saas/segment_fixtures.py @@ -1,4 +1,3 @@ -import os import random import time from typing import Any, Dict, Generator @@ -20,36 +19,37 @@ from fidesops.util.saas_util import load_config from tests.fixtures.application_fixtures import load_dataset from tests.test_helpers.saas_test_utils import poll_for_existence +from tests.test_helpers.vault_client import get_secrets saas_config = load_toml(["saas_config.toml"]) +secrets = get_secrets("segment") -@pytest.fixture(scope="function") +@pytest.fixture(scope="session") def segment_secrets(): return { - "domain": pydash.get(saas_config, "segment.domain") - or os.environ.get("SEGMENT_DOMAIN"), + "domain": pydash.get(saas_config, "segment.domain") or secrets["domain"], "personas_domain": pydash.get(saas_config, "segment.personas_domain") - or os.environ.get("SEGMENT_PERSONAS_DOMAIN"), + or secrets["personas_domain"], "workspace": pydash.get(saas_config, "segment.workspace") - or os.environ.get("SEGMENT_WORKSPACE"), + or secrets["workspace"], "access_token": pydash.get(saas_config, "segment.access_token") - or os.environ.get("SEGMENT_ACCESS_TOKEN"), + or secrets["access_token"], "namespace_id": pydash.get(saas_config, "segment.namespace_id") - or os.environ.get("SEGMENT_NAMESPACE_ID"), + or secrets["namespace_id"], "access_secret": pydash.get(saas_config, "segment.access_secret") - or os.environ.get("SEGMENT_ACCESS_SECRET"), + or secrets["access_secret"], "api_domain": pydash.get(saas_config, "segment.api_domain") - or os.environ.get("SEGMENT_API_DOMAIN"), + or secrets["api_domain"], "user_token": pydash.get(saas_config, "segment.user_token") - or os.environ.get("SEGMENT_USER_TOKEN"), + or secrets["user_token"], } @pytest.fixture(scope="session") def segment_identity_email(): - return pydash.get(saas_config, "segment.identity_email") or os.environ.get( - "SEGMENT_IDENTITY_EMAIL" + return ( + pydash.get(saas_config, "segment.identity_email") or secrets["identity_email"] ) diff --git a/tests/fixtures/saas/sentry_fixtures.py b/tests/fixtures/saas/sentry_fixtures.py index 2dc04092a4..7fc21ed295 100644 --- a/tests/fixtures/saas/sentry_fixtures.py +++ b/tests/fixtures/saas/sentry_fixtures.py @@ -1,4 +1,3 @@ -import os from typing import Any, Dict, Generator import pydash @@ -15,35 +14,34 @@ from fidesops.models.datasetconfig import DatasetConfig from fidesops.util.saas_util import load_config from tests.fixtures.application_fixtures import load_dataset +from tests.test_helpers.vault_client import get_secrets saas_config = load_toml(["saas_config.toml"]) +secrets = get_secrets("sentry") -@pytest.fixture(scope="function") +@pytest.fixture(scope="session") def sentry_secrets(): return { - "domain": pydash.get(saas_config, "sentry.domain") - or os.environ.get("SENTRY_DOMAIN"), + "domain": pydash.get(saas_config, "sentry.domain") or secrets["sentry.domain"], "access_token": pydash.get(saas_config, "sentry.access_token") - or os.environ.get("SENTRY_ACCESS_TOKEN"), + or secrets["access_token"], "erasure_access_token": pydash.get(saas_config, "sentry.erasure_access_token") - or os.environ.get("SENTRY_ERASURE_TOKEN"), + or secrets["erasure_access_token"], "erasure_identity_email": pydash.get( saas_config, "sentry.erasure_identity_email" ) - or os.environ.get("SENTRY_ERASURE_IDENTITY"), + or secrets["erasure_identity_email"], "user_id_erasure": pydash.get(saas_config, "sentry.user_id_erasure") - or os.environ.get("SENTRY_USER_ID"), + or secrets["user_id_erasure"], "issue_url": pydash.get(saas_config, "sentry.issue_url") - or os.environ.get("SENTRY_ISSUE_URL"), + or secrets["issue_url"], } -@pytest.fixture(scope="function") +@pytest.fixture(scope="session") def sentry_identity_email(): - return pydash.get(saas_config, "sentry.identity_email") or os.environ.get( - "SENTRY_IDENTITY_EMAIL" - ) + return pydash.get(saas_config, "sentry.identity_email") or secrets["identity_email"] @pytest.fixture diff --git a/tests/fixtures/saas/stripe_fixtures.py b/tests/fixtures/saas/stripe_fixtures.py index 465f144cd7..f485cff82b 100644 --- a/tests/fixtures/saas/stripe_fixtures.py +++ b/tests/fixtures/saas/stripe_fixtures.py @@ -1,4 +1,3 @@ -import os from typing import Any, Dict, Generator import pydash @@ -17,29 +16,27 @@ from fidesops.models.datasetconfig import DatasetConfig from fidesops.util.saas_util import load_config from tests.fixtures.application_fixtures import load_dataset +from tests.test_helpers.vault_client import get_secrets saas_config = load_toml(["saas_config.toml"]) +secrets = get_secrets("stripe") -@pytest.fixture(scope="function") +@pytest.fixture(scope="session") def stripe_secrets(): return { - "domain": pydash.get(saas_config, "stripe.domain") - or os.environ.get("STRIPE_DOMAIN"), - "api_key": pydash.get(saas_config, "stripe.api_key") - or os.environ.get("STRIPE_API_KEY"), + "domain": pydash.get(saas_config, "stripe.domain") or secrets["domain"], + "api_key": pydash.get(saas_config, "stripe.api_key") or secrets["api_key"], "payment_types": pydash.get(saas_config, "stripe.payment_types") - or os.environ.get("STRIPE_PAYMENT_TYPES"), + or secrets["payment_types"], "page_size": pydash.get(saas_config, "stripe.page_size") - or os.environ.get("STRIPE_PAGE_SIZE"), + or secrets["page_size"], } -@pytest.fixture(scope="function") +@pytest.fixture(scope="session") def stripe_identity_email(): - return pydash.get(saas_config, "stripe.identity_email") or os.environ.get( - "STRIPE_IDENTITY_EMAIL" - ) + return pydash.get(saas_config, "stripe.identity_email") or secrets["identity_email"] @pytest.fixture(scope="function") diff --git a/tests/fixtures/saas/zendesk_fixtures.py b/tests/fixtures/saas/zendesk_fixtures.py index fc119afe92..4506f47a59 100644 --- a/tests/fixtures/saas/zendesk_fixtures.py +++ b/tests/fixtures/saas/zendesk_fixtures.py @@ -1,4 +1,3 @@ -import os from typing import Any, Dict, Generator import pydash @@ -16,26 +15,25 @@ from fidesops.models.datasetconfig import DatasetConfig from fidesops.util.saas_util import load_config from tests.fixtures.application_fixtures import load_dataset +from tests.test_helpers.vault_client import get_secrets saas_config = load_toml(["saas_config.toml"]) +secrets = get_secrets("zendesk") -@pytest.fixture(scope="function") +@pytest.fixture(scope="session") def zendesk_secrets(): return { - "domain": pydash.get(saas_config, "zendesk.domain") - or os.environ.get("ZENDESK_DOMAIN"), - "username": pydash.get(saas_config, "zendesk.username") - or os.environ.get("ZENDESK_USERNAME"), - "api_key": pydash.get(saas_config, "zendesk.api_key") - or os.environ.get("ZENDESK_API_KEY"), + "domain": pydash.get(saas_config, "zendesk.domain") or secrets["domain"], + "username": pydash.get(saas_config, "zendesk.username") or secrets["username"], + "api_key": pydash.get(saas_config, "zendesk.api_key") or secrets["api_key"], } -@pytest.fixture(scope="function") +@pytest.fixture(scope="session") def zendesk_identity_email(): - return pydash.get(saas_config, "zendesk.identity_email") or os.environ.get( - "ZENDESK_IDENTITY_EMAIL" + return ( + pydash.get(saas_config, "zendesk.identity_email") or secrets["identity_email"] ) diff --git a/tests/integration_tests/saas/test_salesforce_task.py b/tests/integration_tests/saas/test_salesforce_task.py index e2d5e16119..66f7fb4ad3 100644 --- a/tests/integration_tests/saas/test_salesforce_task.py +++ b/tests/integration_tests/saas/test_salesforce_task.py @@ -11,7 +11,7 @@ @pytest.mark.integration_saas -@pytest.mark.integration_saas +@pytest.mark.integration_salesforce def test_salesforce_connection_test(salesforce_connection_config) -> None: get_connector(salesforce_connection_config).test_connection() diff --git a/tests/service/connectors/test_saas_connector.py b/tests/service/connectors/test_saas_connector.py index a0d708fc65..81a8696a96 100644 --- a/tests/service/connectors/test_saas_connector.py +++ b/tests/service/connectors/test_saas_connector.py @@ -118,7 +118,8 @@ def test_unwrap_response_data_no_data_path(self): assert response_body == unwrapped -@pytest.mark.unit_saas +@pytest.mark.integration_saas +@pytest.mark.integration_segment class TestSaaSConnectorMethods: def test_create_client_from_request( self, db: Session, segment_connection_config, segment_dataset_config diff --git a/tests/test_helpers/vault_client.py b/tests/test_helpers/vault_client.py new file mode 100644 index 0000000000..4454eefed6 --- /dev/null +++ b/tests/test_helpers/vault_client.py @@ -0,0 +1,40 @@ +import logging +import os +from typing import Any, Dict, Optional + +from hvac import Client + +from fidesops.common_exceptions import FidesopsException + +logger = logging.getLogger(__name__) + +params = { + "VAULT_ADDR": os.environ.get("VAULT_ADDR"), + "VAULT_NAMESPACE": os.environ.get("VAULT_NAMESPACE"), + "VAULT_TOKEN": os.environ.get("VAULT_TOKEN"), +} +_environment = os.environ.get("VAULT_ENVIRONMENT", "development") + + +_client = None +if all(params.values()): + try: + _client = Client( + url=params["VAULT_ADDR"], + namespace=params["VAULT_NAMESPACE"], + token=params["VAULT_TOKEN"], + ) + except Exception as exc: + raise FidesopsException(f"Unable to create Vault client: {str(exc)}") + + +def get_secrets(connector: str) -> Optional[Dict[str, Any]]: + """Returns a map of secrets for the given connector.""" + if not _client: + return + secrets = _client.secrets.kv.v2.read_secret_version( + mount_point=_environment, path=connector + ) + secrets_map = secrets["data"]["data"] + logger.info(f'Loading secrets for {connector}: {", ".join(secrets_map.keys())}') + return secrets_map