Daily DB Publisher R2 #139
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: 'Daily DB Publisher R2' | |
| on: | |
| # allow for kicking off DB builds manually | |
| workflow_dispatch: | |
| inputs: | |
| publish-databases: | |
| description: "build new databases and upload to S3" | |
| type: boolean | |
| required: true | |
| default: true | |
| publish-listing: | |
| description: "use S3 state to update and publish listing file" | |
| type: boolean | |
| required: true | |
| default: true | |
| # run 4 AM (UTC) daily | |
| schedule: | |
| - cron: '0 4 * * *' | |
| env: | |
| CGO_ENABLED: "0" | |
| SLACK_NOTIFICATIONS: true | |
| FORCE_COLOR: true | |
| jobs: | |
| discover-schema-versions: | |
| # note about workflow dispatch inputs and booleans: | |
| # a) booleans come across as string types :( | |
| # b) if not using workflow_dispatch the default values are empty, which means we want these to effectively evaluate to true (so only check the negative case) | |
| if: ${{ github.event.inputs.publish-databases != 'false' }} | |
| name: "Pull vulnerability data" | |
| runs-on: ubuntu-20.04 | |
| outputs: | |
| schema-versions: ${{ steps.read-schema-versions.outputs.schema-versions }} | |
| pull-date: ${{ steps.timestamp.outputs.date }} | |
| # set the permissions granted to the github token to read the pull cache from ghcr.io | |
| permissions: | |
| contents: read | |
| packages: read | |
| steps: | |
| - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2 | |
| - name: Read supported schema versions | |
| id: read-schema-versions | |
| run: | | |
| content=`cat manager/src/grype_db_manager/data/schema-info.json | jq -c '[.available[] | select(.supported == true) | .schema]'` | |
| echo "schema-versions=$content" >> $GITHUB_OUTPUT | |
| generate-and-publish-dbs: | |
| # note about workflow dispatch inputs and booleans: | |
| # a) booleans come across as string types :( | |
| # b) if not using workflow_dispatch the default values are empty, which means we want these to effectively evaluate to true (so only check the negative case) | |
| if: ${{ github.event.inputs.publish-databases != 'false' }} | |
| name: "Generate and publish DBs" | |
| needs: discover-schema-versions | |
| runs-on: ubuntu-22.04-4core-16gb | |
| strategy: | |
| matrix: | |
| schema-version: ${{fromJson(needs.discover-schema-versions.outputs.schema-versions)}} | |
| # set the permissions granted to the github token to read the pull cache from ghcr.io | |
| permissions: | |
| contents: read | |
| packages: read | |
| steps: | |
| - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2 | |
| with: | |
| submodules: true | |
| - name: Bootstrap environment | |
| uses: ./.github/actions/bootstrap | |
| - name: Login to ghcr.io | |
| run: | | |
| echo ${{ secrets.GITHUB_TOKEN }} | oras login ghcr.io --username ${{ github.actor }} --password-stdin | |
| - name: Pull vulnerability data | |
| run: make download-all-provider-cache | |
| - name: Generate and upload DB (schema ${{ matrix.schema-version }}) | |
| run: | | |
| poetry run \ | |
| grype-db-manager \ | |
| -c ./config/grype-db-manager/publish-production-r2.yaml \ | |
| db build-and-upload \ | |
| --schema-version ${{ matrix.schema-version }} \ | |
| -vvv | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.TOOLBOX_CLOUDFLARE_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.TOOLBOX_CLOUDFLARE_SECRET_ACCESS_KEY }} | |
| GRYPE_DB_MANAGER_DISTRIBUTION_S3_ENDPOINT_URL: ${{ secrets.TOOLBOX_CLOUDFLARE_R2_ENDPOINT }} | |
| - uses: 8398a7/action-slack@28ba43ae48961b90635b50953d216767a6bea486 #v3.16.2 | |
| with: | |
| status: ${{ job.status }} | |
| fields: workflow,eventName,job | |
| text: Publishing the Grype DB has failed (schema ${{ matrix.schema-version }}) | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_TOOLBOX_WEBHOOK_URL }} | |
| if: ${{ failure() && env.SLACK_NOTIFICATIONS == 'true' }} | |
| publish-listing-file: | |
| # fun! https://github.com/actions/runner/issues/491#issuecomment-850884422 | |
| # essentially even if the workflow dispatch job is skipping steps, we still want to run this step. | |
| # however, if not running from a workflow dispatch then we want the job ordering to be honored. | |
| # also... | |
| # note about workflow dispatch inputs and booleans: | |
| # a) booleans come across as string types :( | |
| # b) if not using workflow_dispatch the default values are empty, which means we want these to effectively evaluate to true (so only check the negative case) | |
| if: | | |
| always() && | |
| (needs.generate-and-publish-dbs.result == 'success' || needs.generate-and-publish-dbs.result == 'skipped') && | |
| github.event.inputs.publish-listing != 'false' | |
| name: "Publish listing file" | |
| needs: generate-and-publish-dbs | |
| runs-on: ubuntu-22.04-4core-16gb | |
| steps: | |
| - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2 | |
| with: | |
| submodules: true | |
| - name: Bootstrap environment | |
| uses: ./.github/actions/bootstrap | |
| - name: Publish listing file | |
| run: | | |
| poetry run \ | |
| grype-db-manager \ | |
| -c ./config/grype-db-manager/publish-production-r2.yaml \ | |
| listing update | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.TOOLBOX_CLOUDFLARE_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.TOOLBOX_CLOUDFLARE_SECRET_ACCESS_KEY }} | |
| GRYPE_DB_MANAGER_DISTRIBUTION_S3_ENDPOINT_URL: ${{ secrets.TOOLBOX_CLOUDFLARE_R2_ENDPOINT }} | |
| - uses: 8398a7/action-slack@28ba43ae48961b90635b50953d216767a6bea486 #v3.16.2 | |
| with: | |
| status: ${{ job.status }} | |
| fields: workflow,eventName,job | |
| text: Publishing the Grype DB listing file has failed | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_TOOLBOX_WEBHOOK_URL }} | |
| if: ${{ failure() && env.SLACK_NOTIFICATIONS == 'true' }} | |
| sync-listing-file-to-s3: | |
| name: "Sync listing file to S3" | |
| needs: | |
| - publish-listing-file | |
| uses: ./.github/workflows/copy-listing-from-r2.yaml | |
| secrets: inherit |