Feat: Vault for secrets (#869)

* Replace config/gh secrets with secrets from vault

* sorting and update changelog

* fix path for importing test helpers

* more sorting

* run ci again with empty commit

* update unsafe_pr_checks

* allow for no vault vars for unit tests, adds back back for saas config toml vars

* check for client

* sort

* init client to None

* fix imports

* fix pytest markers to better indicate which tests rely on actual secrets

.github/workflows/unsafe_pr_checks.yml

@@ -27,49 +27,24 @@ jobs:
   External-SaaS-Connectors:
     runs-on: ubuntu-latest
     if: contains(github.event.pull_request.labels.*.name, 'run unsafe ci checks')
+    permissions:
+      contents: read
+      id-token: write
     steps:
       - name: Checkout
         uses: actions/checkout@v3
 
+      - name: Get Vault Token
+        uses: hashicorp/[email protected]
+        with:
+          url: ${{ secrets.VAULT_ADDR }}
+          namespace: ${{ secrets.VAULT_NAMESPACE }}
+          method: jwt
+          role: ${{ secrets.VAULT_ROLE }}
+          exportToken: True
+
       - name: SaaS Connector Tests
         env:
-          MAILCHIMP_DOMAIN: ${{ secrets.MAILCHIMP_DOMAIN }}
-          MAILCHIMP_USERNAME: ${{ secrets.MAILCHIMP_USERNAME }}
-          MAILCHIMP_API_KEY: ${{ secrets.MAILCHIMP_API_KEY }}
-          MAILCHIMP_IDENTITY_EMAIL: ${{ secrets.MAILCHIMP_IDENTITY_EMAIL }}
-          SENTRY_DOMAIN: ${{ secrets.SENTRY_DOMAIN }}
-          SENTRY_ACCESS_TOKEN: ${{ secrets.SENTRY_ACCESS_TOKEN }}
-          SENTRY_ERASURE_TOKEN: ${{ secrets.SENTRY_ERASURE_TOKEN }}
-          SENTRY_IDENTITY_EMAIL: ${{ secrets.SENTRY_IDENTITY_EMAIL }}
-          SENTRY_ERASURE_IDENTITY: ${{ secrets.SENTRY_ERASURE_IDENTITY }}
-          SENTRY_USER_ID: ${{ secrets.SENTRY_USER_ID }}
-          SENTRY_ISSUE_URL: ${{ secrets.SENTRY_ISSUE_URL }}
-          STRIPE_DOMAIN: ${{ secrets.STRIPE_DOMAIN }}
-          STRIPE_API_KEY: ${{ secrets.STRIPE_API_KEY }}
-          STRIPE_PAYMENT_TYPES: ${{ secrets.STRIPE_PAYMENT_TYPES }}
-          STRIPE_ITEMS_PER_PAGE: ${{ secrets.STRIPE_ITEMS_PER_PAGE }}
-          STRIPE_IDENTITY_EMAIL: ${{ secrets.STRIPE_IDENTITY_EMAIL }}
-          HUBSPOT_DOMAIN: ${{ secrets.HUBSPOT_DOMAIN }}
-          HUBSPOT_HAPIKEY: ${{ secrets.HUBSPOT_HAPIKEY }}
-          HUBSPOT_IDENTITY_EMAIL: ${{ secrets.HUBSPOT_IDENTITY_EMAIL }}
-          SEGMENT_DOMAIN: ${{ secrets.SEGMENT_DOMAIN }}
-          SEGMENT_API_DOMAIN: ${{ secrets.SEGMENT_API_DOMAIN }}
-          SEGMENT_USER_TOKEN: ${{ secrets.SEGMENT_USER_TOKEN }}
-          SEGMENT_PERSONAS_DOMAIN: ${{ secrets.SEGMENT_PERSONAS_DOMAIN }}
-          SEGMENT_WORKSPACE: ${{ secrets.SEGMENT_WORKSPACE }}
-          SEGMENT_ACCESS_TOKEN: ${{ secrets.SEGMENT_ACCESS_TOKEN }}
-          SEGMENT_NAMESPACE_ID: ${{ secrets.SEGMENT_NAMESPACE_ID }}
-          SEGMENT_ACCESS_SECRET: ${{ secrets.SEGMENT_ACCESS_SECRET }}
-          SEGMENT_IDENTITY_EMAIL: ${{ secrets.SEGMENT_IDENTITY_EMAIL }}
-          SALESFORCE_DOMAIN: ${{ secrets.SALESFORCE_DOMAIN }}
-          SALESFORCE_CLIENT_ID: ${{ secrets.SALESFORCE_CLIENT_ID }}
-          SALESFORCE_CLIENT_SECRET: ${{ secrets.SALESFORCE_CLIENT_SECRET }}
-          SALESFORCE_ACCESS_TOKEN: ${{ secrets.SALESFORCE_ACCESS_TOKEN }}
-          SALESFORCE_USERNAME: ${{ secrets.SALESFORCE_USERNAME }}
-          SALESFORCE_PASSWORD: ${{ secrets.SALESFORCE_PASSWORD }}
-          SALESFORCE_IDENTITY_EMAIL: ${{ secrets.SALESFORCE_IDENTITY_EMAIL }}
-          ZENDESK_DOMAIN: ${{ secrets.ZENDESK_DOMAIN }}
-          ZENDESK_USERNAME: ${{ secrets.ZENDESK_USERNAME }}
-          ZENDESK_API_KEY: ${{ secrets.ZENDESK_API_KEY }}
-          ZENDESK_IDENTITY_EMAIL: ${{ secrets.ZENDESK_IDENTITY_EMAIL }}
+          VAULT_ADDR: ${{ secrets.VAULT_ADDR }}
+          VAULT_NAMESPACE: ${{ secrets.VAULT_NAMESPACE }}
         run: make pytest-saas

CHANGELOG.md

@@ -45,6 +45,7 @@ The types of changes are:
 * Adds endpoint to get available connectors (database and saas) [#768](https://github.com/ethyca/fidesops/pull/768)
 * Erasure support for Zendesk [#775](https://github.com/ethyca/fidesops/pull/775)
 * Adds endpoint to get the secrets required for different connectors [#795](https://github.com/ethyca/fidesops/pull/795)
+* Adds Vault for secrets management [#688](https://github.com/ethyca/fidesops/pull/869)
 
 ### Changed
 

Makefile

@@ -59,7 +59,7 @@ server-no-db: compose-build
     -e FIDESOPS__REDIS__ENABLED=false \
 	--no-deps \
 	--service-ports \
-	webserver	
+	webserver
 
 server-shell: compose-build
 	@docker-compose run $(COMPOSE_SERVICE_NAME) /bin/bash
@@ -166,13 +166,7 @@ pytest-saas: compose-build
 	@echo "Running integration tests for SaaS connectors"
 	@docker-compose run \
 		-e ANALYTICS_OPT_OUT \
-		-e MAILCHIMP_DOMAIN -e MAILCHIMP_USERNAME -e MAILCHIMP_API_KEY -e MAILCHIMP_IDENTITY_EMAIL \
-		-e SENTRY_DOMAIN -e SENTRY_ACCESS_TOKEN -e SENTRY_IDENTITY_EMAIL -e SENTRY_ERASURE_TOKEN -e SENTRY_ERASURE_IDENTITY -e SENTRY_USER_ID -e SENTRY_ISSUE_URL  \
-		-e HUBSPOT_DOMAIN -e HUBSPOT_HAPIKEY -e HUBSPOT_IDENTITY_EMAIL \
-		-e ZENDESK_DOMAIN -e ZENDESK_USERNAME -e ZENDESK_API_KEY -e ZENDESK_IDENTITY_EMAIL \
-		-e SEGMENT_DOMAIN -e SEGMENT_PERSONAS_DOMAIN -e SEGMENT_WORKSPACE -e SEGMENT_ACCESS_TOKEN -e SEGMENT_API_DOMAIN -e SEGMENT_NAMESPACE_ID -e SEGMENT_ACCESS_SECRET -e SEGMENT_USER_TOKEN -e SEGMENT_IDENTITY_EMAIL \
-		-e STRIPE_DOMAIN -e STRIPE_API_KEY -e STRIPE_PAYMENT_TYPES -e STRIPE_PAGE_SIZE -e STRIPE_IDENTITY_EMAIL \
-		-e SALESFORCE_DOMAIN -e SALESFORCE_CLIENT_ID -e SALESFORCE_CLIENT_SECRET -e SALESFORCE_ACCESS_TOKEN -e SALESFORCE_USERNAME -e SALESFORCE_PASSWORD -e SALESFORCE_IDENTITY_EMAIL \
+		-e VAULT_ADDR -e VAULT_NAMESPACE -e VAULT_TOKEN \
 		$(COMPOSE_SERVICE_NAME) pytest $(pytestpath) -m "integration_saas"
 	@make teardown
 

requirements.txt

@@ -12,6 +12,7 @@ fastapi[all]==0.78.0
 fideslang==1.0.0
 fideslib==3.0.0
 fideslog==1.2.1
+hvac==0.11.2
 multidimensional_urlencode==0.0.4
 pandas==1.3.3
 passlib[bcrypt]==1.7.4

tests/api/v1/endpoints/test_connection_config_endpoints.py

@@ -588,6 +588,8 @@ def test_filter_test_status(
         assert len(items) == 3
         assert [item["last_test_succeeded"] is None for item in items]
 
+    @pytest.mark.integration_saas
+    @pytest.mark.integration_stripe
     def test_filter_system_type(
         self,
         db,

tests/fixtures/saas/hubspot_fixtures.py

@@ -1,5 +1,4 @@
 import json
-import os
 from typing import Any, Dict, Generator
 
 import pydash
@@ -19,26 +18,26 @@
 from fidesops.util.saas_util import format_body, load_config
 from tests.fixtures.application_fixtures import load_dataset
 from tests.test_helpers.saas_test_utils import poll_for_existence
+from tests.test_helpers.vault_client import get_secrets
 
 saas_config = load_toml(["saas_config.toml"])
+secrets = get_secrets("hubspot")
 
 HUBSPOT_FIRSTNAME = "SomeoneFirstname"
 
 
-@pytest.fixture(scope="function")
+@pytest.fixture(scope="session")
 def hubspot_secrets():
     return {
-        "domain": pydash.get(saas_config, "hubspot.domain")
-        or os.environ.get("HUBSPOT_DOMAIN"),
-        "hapikey": pydash.get(saas_config, "hubspot.hapikey")
-        or os.environ.get("HUBSPOT_HAPIKEY"),
+        "domain": pydash.get(saas_config, "hubspot.domain") or secrets["domain"],
+        "hapikey": pydash.get(saas_config, "hubspot.hapikey") or secrets["hapikey"],
     }
 
 
 @pytest.fixture(scope="function")
 def hubspot_identity_email():
-    return pydash.get(saas_config, "hubspot.identity_email") or os.environ.get(
-        "HUBSPOT_IDENTITY_EMAIL"
+    return (
+        pydash.get(saas_config, "hubspot.identity_email") or secrets["identity_email"]
     )
 
 

tests/fixtures/saas/mailchimp_fixtures.py

@@ -1,5 +1,4 @@
 import json
-import os
 from typing import Any, Dict, Generator
 
 import pydash
@@ -18,26 +17,26 @@
 from fidesops.service.connectors.saas_connector import SaaSConnector
 from fidesops.util.saas_util import load_config
 from tests.fixtures.application_fixtures import load_dataset
+from tests.test_helpers.vault_client import get_secrets
 
 saas_config = load_toml(["saas_config.toml"])
+secrets = get_secrets("mailchimp")
 
 
-@pytest.fixture(scope="function")
+@pytest.fixture(scope="session")
 def mailchimp_secrets():
     return {
-        "domain": pydash.get(saas_config, "mailchimp.domain")
-        or os.environ.get("MAILCHIMP_DOMAIN"),
+        "domain": pydash.get(saas_config, "mailchimp.domain") or secrets["domain"],
         "username": pydash.get(saas_config, "mailchimp.username")
-        or os.environ.get("MAILCHIMP_USERNAME"),
-        "api_key": pydash.get(saas_config, "mailchimp.api_key")
-        or os.environ.get("MAILCHIMP_API_KEY"),
+        or secrets["username"],
+        "api_key": pydash.get(saas_config, "mailchimp.api_key") or secrets["api_key"],
     }
 
 
-@pytest.fixture(scope="function")
+@pytest.fixture(scope="session")
 def mailchimp_identity_email():
-    return pydash.get(saas_config, "mailchimp.identity_email") or os.environ.get(
-        "MAILCHIMP_IDENTITY_EMAIL"
+    return (
+        pydash.get(saas_config, "mailchimp.identity_email") or secrets["identity_email"]
     )
 
 

tests/fixtures/saas/outreach_fixtures.py

@@ -1,4 +1,3 @@
-import os
 from typing import Any, Dict, Generator
 
 import pydash
@@ -17,32 +16,33 @@
 from fidesops.models.datasetconfig import DatasetConfig
 from fidesops.util.saas_util import load_config
 from tests.fixtures.application_fixtures import load_dataset
+from tests.test_helpers.vault_client import get_secrets
 
 saas_config = load_toml(["saas_config.toml"])
+secrets = get_secrets("outreach")
 
 
-@pytest.fixture(scope="function")
+@pytest.fixture(scope="session")
 def outreach_secrets():
     return {
-        "domain": pydash.get(saas_config, "outreach.domain")
-        or os.environ.get("OUTREACH_DOMAIN"),
+        "domain": pydash.get(saas_config, "outreach.domain") or secrets["domain"],
         "requester_email": pydash.get(saas_config, "outreach.requester_email")
-        or os.environ.get("OUTREACH_REQUESTER_EMAIL"),
+        or secrets["requester_email"],
         "client_id": pydash.get(saas_config, "outreach.client_id")
-        or os.environ.get("OUTREACH_CLIENT_ID"),
+        or secrets["client_id"],
         "client_secret": pydash.get(saas_config, "outreach.client_secret")
-        or os.environ.get("OUTREACH_CLIENT_SECRET"),
+        or secrets["client_secret"],
         "redirect_uri": pydash.get(saas_config, "outreach.redirect_uri")
-        or os.environ.get("OUTREACH_REDIRECT_URI"),
+        or secrets["redirect_uri"],
         "page_size": pydash.get(saas_config, "outreach.page_size")
-        or os.environ.get("OUTREACH_PAGE_SIZE"),
+        or secrets["page_size"],
     }
 
 
 @pytest.fixture(scope="session")
 def outreach_identity_email():
-    return pydash.get(saas_config, "outreach.identity_email") or os.environ.get(
-        "OUTREACH_IDENTITY_EMAIL"
+    return (
+        pydash.get(saas_config, "outreach.identity_email") or secrets["identity_email"]
     )
 
 

tests/fixtures/saas/salesforce_fixtures.py

@@ -1,4 +1,3 @@
-import os
 from typing import Any, Dict, Generator
 
 import pydash
@@ -17,32 +16,34 @@
 from fidesops.models.datasetconfig import DatasetConfig
 from fidesops.util.saas_util import load_config
 from tests.fixtures.application_fixtures import load_dataset
+from tests.test_helpers.vault_client import get_secrets
 
 saas_config = load_toml(["saas_config.toml"])
+secrets = get_secrets("salesforce")
 
 
 @pytest.fixture(scope="session")
 def salesforce_secrets():
     return {
-        "domain": pydash.get(saas_config, "salesforce.domain")
-        or os.environ.get("SALESFORCE_DOMAIN"),
+        "domain": pydash.get(saas_config, "salesforce.domain") or secrets["domain"],
         "username": pydash.get(saas_config, "salesforce.username")
-        or os.environ.get("SALESFORCE_USERNAME"),
+        or secrets["username"],
         "password": pydash.get(saas_config, "salesforce.password")
-        or os.environ.get("SALESFORCE_PASSWORD"),
+        or secrets["password"],
         "client_id": pydash.get(saas_config, "salesforce.client_id")
-        or os.environ.get("SALESFORCE_CLIENT_ID"),
+        or secrets["client_id"],
         "client_secret": pydash.get(saas_config, "salesforce.client_secret")
-        or os.environ.get("SALESFORCE_CLIENT_SECRET"),
+        or secrets["client_secret"],
         "access_token": pydash.get(saas_config, "salesforce.access_token")
-        or os.environ.get("SALESFORCE_ACCESS_TOKEN"),
+        or secrets["access_token"],
     }
 
 
 @pytest.fixture(scope="session")
 def salesforce_identity_email():
-    return pydash.get(saas_config, "salesforce.identity_email") or os.environ.get(
-        "SALESFORCE_IDENTITY_EMAIL"
+    return (
+        pydash.get(saas_config, "salesforce.identity_email")
+        or secrets["identity_email"]
     )
 
 

tests/fixtures/saas/segment_fixtures.py

@@ -1,4 +1,3 @@
-import os
 import random
 import time
 from typing import Any, Dict, Generator
@@ -20,36 +19,37 @@
 from fidesops.util.saas_util import load_config
 from tests.fixtures.application_fixtures import load_dataset
 from tests.test_helpers.saas_test_utils import poll_for_existence
+from tests.test_helpers.vault_client import get_secrets
 
 saas_config = load_toml(["saas_config.toml"])
+secrets = get_secrets("segment")
 
 
-@pytest.fixture(scope="function")
+@pytest.fixture(scope="session")
 def segment_secrets():
     return {
-        "domain": pydash.get(saas_config, "segment.domain")
-        or os.environ.get("SEGMENT_DOMAIN"),
+        "domain": pydash.get(saas_config, "segment.domain") or secrets["domain"],
         "personas_domain": pydash.get(saas_config, "segment.personas_domain")
-        or os.environ.get("SEGMENT_PERSONAS_DOMAIN"),
+        or secrets["personas_domain"],
         "workspace": pydash.get(saas_config, "segment.workspace")
-        or os.environ.get("SEGMENT_WORKSPACE"),
+        or secrets["workspace"],
         "access_token": pydash.get(saas_config, "segment.access_token")
-        or os.environ.get("SEGMENT_ACCESS_TOKEN"),
+        or secrets["access_token"],
         "namespace_id": pydash.get(saas_config, "segment.namespace_id")
-        or os.environ.get("SEGMENT_NAMESPACE_ID"),
+        or secrets["namespace_id"],
         "access_secret": pydash.get(saas_config, "segment.access_secret")
-        or os.environ.get("SEGMENT_ACCESS_SECRET"),
+        or secrets["access_secret"],
         "api_domain": pydash.get(saas_config, "segment.api_domain")
-        or os.environ.get("SEGMENT_API_DOMAIN"),
+        or secrets["api_domain"],
         "user_token": pydash.get(saas_config, "segment.user_token")
-        or os.environ.get("SEGMENT_USER_TOKEN"),
+        or secrets["user_token"],
     }
 
 
 @pytest.fixture(scope="session")
 def segment_identity_email():
-    return pydash.get(saas_config, "segment.identity_email") or os.environ.get(
-        "SEGMENT_IDENTITY_EMAIL"
+    return (
+        pydash.get(saas_config, "segment.identity_email") or secrets["identity_email"]
     )
 
 

tests/fixtures/saas/sentry_fixtures.py

@@ -1,4 +1,3 @@
-import os
 from typing import Any, Dict, Generator
 
 import pydash
@@ -15,35 +14,34 @@
 from fidesops.models.datasetconfig import DatasetConfig
 from fidesops.util.saas_util import load_config
 from tests.fixtures.application_fixtures import load_dataset
+from tests.test_helpers.vault_client import get_secrets
 
 saas_config = load_toml(["saas_config.toml"])
+secrets = get_secrets("sentry")
 
 
-@pytest.fixture(scope="function")
+@pytest.fixture(scope="session")
 def sentry_secrets():
     return {
-        "domain": pydash.get(saas_config, "sentry.domain")
-        or os.environ.get("SENTRY_DOMAIN"),
+        "domain": pydash.get(saas_config, "sentry.domain") or secrets["sentry.domain"],
         "access_token": pydash.get(saas_config, "sentry.access_token")
-        or os.environ.get("SENTRY_ACCESS_TOKEN"),
+        or secrets["access_token"],
         "erasure_access_token": pydash.get(saas_config, "sentry.erasure_access_token")
-        or os.environ.get("SENTRY_ERASURE_TOKEN"),
+        or secrets["erasure_access_token"],
         "erasure_identity_email": pydash.get(
             saas_config, "sentry.erasure_identity_email"
         )
-        or os.environ.get("SENTRY_ERASURE_IDENTITY"),
+        or secrets["erasure_identity_email"],
         "user_id_erasure": pydash.get(saas_config, "sentry.user_id_erasure")
-        or os.environ.get("SENTRY_USER_ID"),
+        or secrets["user_id_erasure"],
         "issue_url": pydash.get(saas_config, "sentry.issue_url")
-        or os.environ.get("SENTRY_ISSUE_URL"),
+        or secrets["issue_url"],
     }
 
 
-@pytest.fixture(scope="function")
+@pytest.fixture(scope="session")
 def sentry_identity_email():
-    return pydash.get(saas_config, "sentry.identity_email") or os.environ.get(
-        "SENTRY_IDENTITY_EMAIL"
-    )
+    return pydash.get(saas_config, "sentry.identity_email") or secrets["identity_email"]
 
 
 @pytest.fixture