diff --git a/.fides/db_dataset.yml b/.fides/db_dataset.yml index 515b7b4512..2ef5f7f68d 100644 --- a/.fides/db_dataset.yml +++ b/.fides/db_dataset.yml @@ -313,6 +313,10 @@ dataset: data_categories: - system.operations data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: fidesctl_meta + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified - name: id data_categories: - system.operations diff --git a/.fides/systems.yml b/.fides/systems.yml index c8dbef883c..e5d36896f2 100644 --- a/.fides/systems.yml +++ b/.fides/systems.yml @@ -1,12 +1,20 @@ system: + # System Info - fides_key: privacy_annotations name: Fides Privacy Annotations organization_fides_key: default_organization description: Privacy annotations as code. system_type: Service + # RoPA Info - these need to exist to pass an audit data_responsibility_title: Controller data_protection_impact_assessment: is_required: false + progress: null + link: null + joint_controller: null + third_country_transfers: null + administrating_department: Not defined + # Privacy Declarations privacy_declarations: - name: Store fidesctl annotation data. data_categories: @@ -19,13 +27,22 @@ system: dataset_references: - fides_db + # System Info - fides_key: privacy_request_fullfillment name: Fides Privacy Request Fulfillment organization_fides_key: default_organization description: Privacy request fufillment. system_type: Application + # RoPA Info - these need to exist to pass an audit + data_responsibility_title: Controller data_protection_impact_assessment: is_required: false + progress: null + link: null + joint_controller: null + third_country_transfers: null + administrating_department: Not defined + # Privacy Declarations privacy_declarations: - name: Manage privacy request Fufillment data_categories: [] # comprehensive list of data categories in `fides_db` @@ -37,22 +54,32 @@ system: - fides_db - fides_cache + # System Info - fides_key: admin_ui name: Fides Administration UI organization_fides_key: default_organization description: Fides administration UI. system_type: Application + # RoPA Info - these need to exist to pass an audit + data_responsibility_title: Controller data_protection_impact_assessment: is_required: false + progress: null + link: null + joint_controller: null + third_country_transfers: null + administrating_department: Not defined + # Privacy Declarations privacy_declarations: - name: Manage privacy request Fufillment data_categories: [] # comprehensive list of data categories in `fides_db` - data_use: system.operations + data_use: provide.service data_subjects: - employee data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified dataset_references: - fides_db + # Specific tables for reference # - fides_db.client # - fides_db.connectionconfig # - fides_db.fidesuser diff --git a/.github/workflows/code_checks.yml b/.github/workflows/code_checks.yml index e62e4b325d..da2836ec7c 100644 --- a/.github/workflows/code_checks.yml +++ b/.github/workflows/code_checks.yml @@ -159,7 +159,7 @@ jobs: - name: Check that the docs can build run: nox -s docs_check - Fidesctl: + Fides-Checks: needs: Build runs-on: ubuntu-latest steps: @@ -184,23 +184,24 @@ jobs: run: pip install nox>=2022 - name: Check fides installation - run: nox -s check_install -- docker + run: nox -s check_install - name: Run fides evaluation - run: nox -s fidesctl -- docker + run: nox -s check_fides_annotations -- docker - name: Scan fides db - run: nox -s fidesctl_db_scan -- docker + run: nox -s fides_db_scan -########### -## Tests ## -########### +################ +## Safe Tests ## +################ Pytest-Ctl-Not-External: needs: Build strategy: matrix: python_version: ["3.8.14", "3.9.14", "3.10.6"] runs-on: ubuntu-latest + continue-on-error: true steps: - name: Download container uses: actions/download-artifact@v3 @@ -220,12 +221,71 @@ jobs: - name: Run non-external test suite run: nox -s "pytest(not-external)" + + Pytest-Unit-Ops: + needs: Build + strategy: + matrix: + python_version: ["3.8.14", "3.9.14", "3.10.6"] + runs-on: ubuntu-latest + continue-on-error: true + steps: + - name: Download container + uses: actions/download-artifact@v3 + with: + name: python-${{ matrix.python_version }} + path: /tmp/ + + - name: Load image + run: docker load --input /tmp/python-${{ matrix.python_version }}.tar + + - name: Checkout + uses: actions/checkout@v3 + + - name: Install Nox + run: pip install nox>=2022 + + - name: Run unit test suite + run: nox -s pytest_unit + + Pytest-Integration-Ops: + needs: Build + strategy: + matrix: + python_version: ["3.8.14", "3.9.14", "3.10.6"] + runs-on: ubuntu-latest + continue-on-error: true + steps: + - name: Download container + uses: actions/download-artifact@v3 + with: + name: python-${{ matrix.python_version }} + path: /tmp/ + + - name: Load image + run: docker load --input /tmp/python-${{ matrix.python_version }}.tar + + - name: Checkout + uses: actions/checkout@v3 + + - name: Install Nox + run: pip install nox>=2022 + + - name: Run integration test suite + run: nox -s pytest_integration + +################## +## Unsafe Tests ## +################## Pytest-Ctl-External: needs: Build strategy: + max-parallel: 1 # This prevents collisions in shared external resources matrix: python_version: ["3.8.14", "3.9.14", "3.10.6"] runs-on: ubuntu-latest + if: contains(github.event.pull_request.labels.*.name, 'run unsafe ci checks') + continue-on-error: true steps: - name: Download container uses: actions/download-artifact@v3 @@ -253,12 +313,15 @@ jobs: AWS_DEFAULT_REGION: us-east-1 BIGQUERY_CONFIG: ${{ secrets.BIGQUERY_CONFIG }} - Pytest-Unit-Ops: + External-Datastores: needs: Build strategy: + max-parallel: 1 # This prevents collisions in shared external resources matrix: python_version: ["3.8.14", "3.9.14", "3.10.6"] runs-on: ubuntu-latest + if: contains(github.event.pull_request.labels.*.name, 'run unsafe ci checks') + continue-on-error: true steps: - name: Download container uses: actions/download-artifact@v3 @@ -275,15 +338,27 @@ jobs: - name: Install Nox run: pip install nox>=2022 - - name: Run unit test suite - run: nox -s pytest_unit - - Pytest-Integration-Ops: + - name: Integration Tests (External) + env: + REDSHIFT_TEST_URI: ${{ secrets.REDSHIFT_TEST_URI }} + REDSHIFT_TEST_DB_SCHEMA: fidesops_test + BIGQUERY_KEYFILE_CREDS: ${{ secrets.BIGQUERY_KEYFILE_CREDS }} + BIGQUERY_DATASET: fidesopstest + SNOWFLAKE_TEST_URI: ${{ secrets.SNOWFLAKE_TEST_URI }} + run: nox -s pytest_integration_external + + External-SaaS-Connectors: needs: Build + runs-on: ubuntu-latest + continue-on-error: true + if: contains(github.event.pull_request.labels.*.name, 'run unsafe ci checks') + permissions: + contents: read + id-token: write strategy: + max-parallel: 1 # This prevents collisions in shared external resources matrix: python_version: ["3.8.14", "3.9.14", "3.10.6"] - runs-on: ubuntu-latest steps: - name: Download container uses: actions/download-artifact@v3 @@ -294,11 +369,23 @@ jobs: - name: Load image run: docker load --input /tmp/python-${{ matrix.python_version }}.tar + - name: Install Nox + run: pip install nox>=2022 + - name: Checkout uses: actions/checkout@v3 - - name: Install Nox - run: pip install nox>=2022 + - name: Get Vault Token + uses: hashicorp/vault-action@v2.4.2 + with: + url: ${{ secrets.VAULT_ADDR }} + namespace: ${{ secrets.VAULT_NAMESPACE }} + method: jwt + role: ${{ secrets.VAULT_ROLE }} + exportToken: True - - name: Run integration test suite - run: nox -s pytest_integration + - name: SaaS Connector Tests + env: + VAULT_ADDR: ${{ secrets.VAULT_ADDR }} + VAULT_NAMESPACE: ${{ secrets.VAULT_NAMESPACE }} + run: nox -s pytest_saas diff --git a/.github/workflows/unsafe_code_checks.yml b/.github/workflows/unsafe_code_checks.yml deleted file mode 100644 index aca1bcfde2..0000000000 --- a/.github/workflows/unsafe_code_checks.yml +++ /dev/null @@ -1,107 +0,0 @@ -name: Unsafe Code Checks -on: - push: - branches: - - main - pull_request: - types: [labeled] - -env: - CONTAINER: fides-local - IMAGE: ethyca/fides:local - -jobs: - Build: - runs-on: ubuntu-latest - if: contains(github.event.pull_request.labels.*.name, 'run unsafe ci checks') - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v2 - - - name: Build container - uses: docker/build-push-action@v3 - with: - builder: ${{ steps.buildx.outputs.name }} - context: . - target: prod - outputs: type=docker,dest=/tmp/${{ env.CONTAINER }}.tar - push: false - tags: ${{ env.IMAGE }} - - - name: Upload container - uses: actions/upload-artifact@v3 - with: - name: ${{ env.CONTAINER }} - path: /tmp/${{ env.CONTAINER }}.tar - retention-days: 1 - - External-Datastores: - needs: Build - runs-on: ubuntu-latest - if: contains(github.event.pull_request.labels.*.name, 'run unsafe ci checks') - steps: - - name: Download container - uses: actions/download-artifact@v3 - with: - name: ${{ env.CONTAINER }} - path: /tmp/ - - - name: Load image - run: docker load --input /tmp/${{ env.CONTAINER }}.tar - - - name: Checkout - uses: actions/checkout@v3 - - - name: Install Nox - run: pip install nox>=2022 - - - name: Integration Tests (External) - env: - REDSHIFT_TEST_URI: ${{ secrets.REDSHIFT_TEST_URI }} - REDSHIFT_TEST_DB_SCHEMA: fidesops_test - BIGQUERY_KEYFILE_CREDS: ${{ secrets.BIGQUERY_KEYFILE_CREDS }} - BIGQUERY_DATASET: fidesopstest - SNOWFLAKE_TEST_URI: ${{ secrets.SNOWFLAKE_TEST_URI }} - run: nox -s pytest_integration_external - - External-SaaS-Connectors: - needs: Build - runs-on: ubuntu-latest - if: contains(github.event.pull_request.labels.*.name, 'run unsafe ci checks') - permissions: - contents: read - id-token: write - steps: - - name: Download container - uses: actions/download-artifact@v3 - with: - name: ${{ env.CONTAINER }} - path: /tmp/ - - - name: Load image - run: docker load --input /tmp/${{ env.CONTAINER }}.tar - - - name: Install Nox - run: pip install nox>=2022 - - - name: Checkout - uses: actions/checkout@v3 - - - name: Get Vault Token - uses: hashicorp/vault-action@v2.4.2 - with: - url: ${{ secrets.VAULT_ADDR }} - namespace: ${{ secrets.VAULT_NAMESPACE }} - method: jwt - role: ${{ secrets.VAULT_ROLE }} - exportToken: True - - - name: SaaS Connector Tests - env: - VAULT_ADDR: ${{ secrets.VAULT_ADDR }} - VAULT_NAMESPACE: ${{ secrets.VAULT_NAMESPACE }} - run: nox -s pytest_saas diff --git a/noxfiles/ci_nox.py b/noxfiles/ci_nox.py index 4d35964fca..b5a8c60860 100644 --- a/noxfiles/ci_nox.py +++ b/noxfiles/ci_nox.py @@ -1,6 +1,4 @@ """Contains the nox sessions used during CI checks.""" -from time import sleep - import nox from constants_nox import ( @@ -118,17 +116,17 @@ def check_install(session: nox.Session) -> None: @nox.session() -def fidesctl(session: nox.Session) -> None: +def check_fides_annotations(session: nox.Session) -> None: """Run a fidesctl evaluation.""" if session.posargs == ["docker"]: - run_command = (*RUN_STATIC_ANALYSIS, "fidesctl") + run_command = (*RUN_STATIC_ANALYSIS, "check_fides_annotations") else: run_command = ("fides", "--local", *(WITH_TEST_CONFIG), "evaluate") session.run(*run_command, external=True) @nox.session() -def fidesctl_db_scan(session: nox.Session) -> None: +def fides_db_scan(session: nox.Session) -> None: """Scan the fidesctl application database to check for dataset discrepancies.""" session.notify("teardown") session.run(*START_APP, external=True)