Merge branch 'main' into klaviyo-access-vivek

.fides/fides.toml

@@ -19,10 +19,12 @@ server_port = 8080
 
 [user]
 analytics_opt_out = false
+username = "root_user"
+password = "Testpassword1!"
 
 [redis]
 host = "redis"
-password = "testpassword"
+password = "redispassword"
 port = 6379
 charset = "utf8"
 default_ttl_seconds = 604800
@@ -39,7 +41,7 @@ app_encryption_key = "OLMkv91j8DHiDAULnK5Lxx3kSCov30b3"
 oauth_root_client_id = "fidesadmin"
 oauth_root_client_secret = "fidesadminsecret"
 drp_jwt_secret = "secret"
-env = "dev"
+env = "prod"
 
 [execution]
 masking_strict = false

.fides/systems.yml

@@ -28,7 +28,7 @@ system:
           - fides_db
 
   # System Info
-  - fides_key: privacy_request_fullfillment
+  - fides_key: privacy_request_fulfillment
     name: Fides Privacy Request Fulfillment
     organization_fides_key: default_organization
     description: Privacy request fufillment.

.github/pull_request_template.md

@@ -17,6 +17,7 @@ Closes #<issue>
 * [ ] Issue Requirements are Met
 * [ ] Relevant Follow-Up Issues Created
 * [ ] Update `CHANGELOG.md`
+* [ ] For API changes, the [Postman collection](https://github.com/ethyca/fides/blob/main/docs/fides/docs/development/postman/Fides.postman_collection.json) has been updated
 
 ### Description Of Changes
 

.github/workflows/backend_checks.yml

@@ -4,13 +4,14 @@ on:
   pull_request:
     paths-ignore:
       - "**.md"
+      - "clients/**"
   push:
     branches:
       - "main"
 
 env:
   IMAGE: ethyca/fides:local
-  DEFAULT_PYTHON_VERSION: "3.10.7"
+  DEFAULT_PYTHON_VERSION: "3.10.11"
 
 jobs:
   ###############
@@ -39,7 +40,7 @@ jobs:
     strategy:
       matrix:
         # NOTE: These are the currently supported/tested Python Versions
-        python_version: ["3.8.14", "3.9.14", "3.10.7"]
+        python_version: ["3.8.16", "3.9.16", "3.10.11"]
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -141,15 +142,16 @@ jobs:
     needs: Build
     strategy:
       matrix:
-        python_version: ["3.8.14", "3.9.14", "3.10.7"]
+        python_version: ["3.8.16", "3.9.16", "3.10.11"]
         test_selection:
           - "ctl-not-external"
           - "ops-unit"
           - "ops-integration"
           - "lib"
+          - "nox"
 
     runs-on: ubuntu-latest
-    timeout-minutes: 15
+    timeout-minutes: 20
     continue-on-error: true
     steps:
       - name: Download container
@@ -192,9 +194,9 @@ jobs:
     strategy:
       max-parallel: 1 # This prevents collisions in shared external resources
       matrix:
-        python_version: ["3.8.14", "3.9.14", "3.10.7"]
+        python_version: ["3.8.16", "3.9.16", "3.10.11"]
     runs-on: ubuntu-latest
-    timeout-minutes: 15
+    timeout-minutes: 20
     # In PRs run with the "unsafe" label, or run on a "push" event to main
     if: contains(github.event.pull_request.labels.*.name, 'run unsafe ci checks') || github.event_name == 'push'
     steps:
@@ -229,9 +231,9 @@ jobs:
     strategy:
       max-parallel: 1 # This prevents collisions in shared external resources
       matrix:
-        python_version: ["3.8.14", "3.9.14", "3.10.7"]
+        python_version: ["3.8.16", "3.9.16", "3.10.11"]
     runs-on: ubuntu-latest
-    timeout-minutes: 15
+    timeout-minutes: 20
     # In PRs run with the "unsafe" label, or run on a "push" event to main
     if: contains(github.event.pull_request.labels.*.name, 'run unsafe ci checks') || github.event_name == 'push'
     steps:
@@ -262,7 +264,7 @@ jobs:
   External-SaaS-Connectors:
     needs: Build
     runs-on: ubuntu-latest
-    timeout-minutes: 15
+    timeout-minutes: 30
     # In PRs run with the "unsafe" label, or run on a "push" event to main
     if: contains(github.event.pull_request.labels.*.name, 'run unsafe ci checks') || github.event_name == 'push'
     permissions:
@@ -271,7 +273,7 @@ jobs:
     strategy:
       max-parallel: 1 # This prevents collisions in shared external resources
       matrix:
-        python_version: ["3.8.14", "3.9.14", "3.10.7"]
+        python_version: ["3.8.16", "3.9.16", "3.10.11"]
     steps:
       - name: Download container
         uses: actions/download-artifact@v3
@@ -289,7 +291,7 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Get Vault Token
-        uses: hashicorp/vault-action@v2.4.2
+        uses: hashicorp/vault-action@v2.5.0
         with:
           url: ${{ secrets.VAULT_ADDR }}
           namespace: ${{ secrets.VAULT_NAMESPACE }}

.github/workflows/cli_checks.yml

@@ -8,28 +8,17 @@ on:
   pull_request:
     paths-ignore:
       - "**.md"
+      - "clients/**"
   push:
     branches:
       - "main"
 
 env:
-  DEFAULT_PYTHON_VERSION: "3.10.7"
+  DEFAULT_PYTHON_VERSION: "3.10.11"
 
 jobs:
-  Test-Envs:
-    runs-on: ubuntu-latest
-    timeout-minutes: 25
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-
-      - name: Install Nox
-        run: pip install nox>=2022
-
-      - name: Initialize the test environment
-        run: nox -s "fides_env(test)" -- test
-
-  Fides-Deploy:
+  # Basic smoke test of a local install of the fides Python CLI
+  Fides-Install:
     runs-on: ubuntu-latest
     timeout-minutes: 20
     steps:
@@ -44,14 +33,8 @@ jobs:
       - name: Install Nox
         run: pip install nox>=2022
 
-      - name: Build the sample image
-        run: nox -s "build(sample)"
-
       - name: Install fides
         run: pip install .
 
-      - name: Start the sample application
-        run: fides deploy up --no-pull --no-init
-
-      - name: Stop the sample application
-        run: fides deploy down
+      - name: Run `fides --version`
+        run: fides --version

.github/workflows/cypress_e2e.yml

@@ -0,0 +1,54 @@
+name: Cypress E2E Tests
+
+on:
+  pull_request:
+    paths-ignore:
+      - "**.md"
+  push:
+    branches:
+      - "main"
+
+env:
+  CI: true
+
+jobs:
+  Cypress-E2E:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node-version: [16.x]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Install Nox
+        run: pip install nox>=2022
+
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node-version }}
+
+      - name: Start test environment in the background
+        run: nox -s "fides_env(test)" -- keep_alive
+
+      - name: Install dependencies
+        run: |
+          cd clients/cypress-e2e
+          npm install
+
+      - name: Cypress E2E tests
+        uses: cypress-io/github-action@v5
+        with:
+          working-directory: clients/cypress-e2e
+          install: false
+          wait-on: "http://localhost:8080, http://localhost:3001"
+          record: true
+        env:
+          # pass the Cypress Cloud record key as an environment variable
+          CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
+          # pass GitHub token to allow accurately detecting a build vs a re-run build
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Teardown
+        run: nox -s teardown

.github/workflows/frontend_checks.yml

@@ -52,7 +52,7 @@ jobs:
         node-version: [16.x]
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Use Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v3
@@ -71,7 +71,7 @@ jobs:
           install: false
           start: npm run cy:start
           wait-on: "http://localhost:3000"
-          wait-on-timeout: 120
+          wait-on-timeout: 180
 
       - uses: actions/upload-artifact@v3
         if: failure()
@@ -105,20 +105,20 @@ jobs:
       - name: Format
         run: npm run format:ci
 
-      - name: Jest test
-        run: npm run test:ci
-
       - name: Build
         run: npm run build
 
+      - name: Jest test
+        run: npm run test:ci
+
   Privacy-Center-Cypress:
     runs-on: ubuntu-latest
     strategy:
       matrix:
         node-version: [16.x]
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Use Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v3

.github/workflows/publish_docker.yaml

@@ -39,7 +39,15 @@ jobs:
         run: |
           if [[ ${{ github.event.ref }} =~ ^refs/tags/[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
               echo ::set-output name=match::true
+          else
+              echo ::set-output name=match::false
           fi
       - name: Push Fides Prod Tags
         if: steps.check-tag.outputs.match == 'true'
         run: nox -s "push(prod)"
+
+      # if not a prod tag, then we run the git-tag job to publish images with a git tag
+      # if one exists on the current commit. the job is a no-op if the commit hasn't been tagged
+      - name: Push Fides Commit Tags
+        if: steps.check-tag.outputs.match == 'false'
+        run: nox -s "push(git-tag)"

.github/workflows/publish_docs.yaml

@@ -31,15 +31,15 @@ jobs:
 
       - name: Install Docs Requirements
         run: pip install -r docs/fides/requirements.txt
-
+        
       - name: Install fides
         run: pip install -e ./[all]
 
       - name: Checkout the gh-pages branches
         run: git fetch origin gh-pages --depth=1
 
       - name: Build docs
-        run: nox -s "docs_build(local)"
+        run: nox -s generate_docs
 
       # This is required by "mike"
       - name: Set the Git Username & Email

.github/workflows/publish_package.yaml

@@ -48,18 +48,27 @@ jobs:
       - name: Build the wheel
         run: python setup.py bdist_wheel
 
+      # If the tag matches either a release tag or a beta tag, allow publishing to PyPi, e.g.:
+      #   2.10.0 --> match (production release tag)
+      #   2.10.0b1 --> match (beta release tag, used on main)
+      #   2.10.0.a0 --> no match (alpha tag, used on feature branches)
+      #   2.10.0.dev0 --> no match (arbitrary development tag)
+      - name: Check Prod Tag
+        id: check-tag
+        run: |
+          if [[ ${{ github.event.ref }} =~ ^refs/tags/[0-9]+\.[0-9]+\.[0-9]+(b[0-9]*)?$ ]]; then
+              echo ::set-output name=match::true
+          fi
+
+      # We don't upload to Test PyPI if it is a real release
       - name: Upload to test pypi
+        if: steps.check-tag.outputs.match != 'true'
         run: twine upload --repository testpypi dist/*
         env:
           TWINE_USERNAME: __token__
           TWINE_PASSWORD: ${{ secrets.TEST_PYPI_TOKEN }}
 
-      - name: Check Prod Tag
-        id: check-tag
-        run: |
-          if [[ ${{ github.event.ref }} =~ ^refs/tags/[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
-              echo ::set-output name=match::true
-          fi
+
       - name: Upload to pypi
         if: steps.check-tag.outputs.match == 'true'
         run: twine upload dist/*

.gitignore

@@ -25,8 +25,7 @@ tmp/*
 # docs
 docs/fides/site/
 docs/fides/docs/api/openapi.json
-docs/fidesops/docs/api/openapi.json
-docs/fidesops/site
+docs/fides/docs/config/fides.toml
 
 # python specific
 *.pyc
@@ -257,5 +256,5 @@ fides_uploads
 saas_config.toml
 
 # Frontend files
-clients/privacy-center/node_modules
-clients/privacy-center/.next
+**/node_modules
+**/.next