From eaf429c1b74b9b62b57ce8d4a820d98ec7931b5f Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Wed, 20 Nov 2024 18:54:22 +0100 Subject: [PATCH 01/51] ci: change s3 backend to decrease costs in tests-rs-package --- .github/actions/docker/action.yaml | 13 +++++-- .github/actions/librocksdb/action.yaml | 4 -- .../s3-layer-cache-settings/action.yaml | 20 ++++++---- .github/workflows/tests-rs-package.yml | 39 ++++++++++++------- Dockerfile | 2 + 5 files changed, 48 insertions(+), 30 deletions(-) diff --git a/.github/actions/docker/action.yaml b/.github/actions/docker/action.yaml index 1e12427e9a..075eb6e8b9 100644 --- a/.github/actions/docker/action.yaml +++ b/.github/actions/docker/action.yaml @@ -27,10 +27,13 @@ inputs: default: dev bucket: description: S3 bucket to use for caching, must match runner define in `runs-on` - default: multi-runner-cache-x1xibo9c + default: ${{ env.AWS_S3_BUCKET }} region: description: S3 bucket region required: true + endpoint: + description: S3 endpoint to use for caching + required: true aws_access_key_id: description: AWS access key ID required: true @@ -38,8 +41,8 @@ inputs: description: AWS secret access key required: true cache_to_name: - description: 'Save cache to name manifest (should be used only on default branch)' - default: 'false' + description: "Save cache to name manifest (should be used only on default branch)" + default: "false" outputs: digest: value: ${{ steps.docker_build.outputs.digest }} @@ -82,6 +85,7 @@ runs: name: ${{ inputs.image_name }} region: ${{ inputs.region }} bucket: ${{ inputs.bucket }} + endpoint: ${{ inputs.endpoint }} cache_to_name: ${{ inputs.cache_to_name }} - name: Set HOME variable to github context @@ -133,7 +137,7 @@ runs: id: arch uses: actions/github-script@v6 with: - result-encoding: 'string' + result-encoding: "string" script: return '${{ inputs.platform }}'.replace('linux/', ''); - name: Inject cargo cache into docker @@ -164,6 +168,7 @@ runs: RUSTC_WRAPPER=sccache SCCACHE_BUCKET=${{ inputs.bucket }} SCCACHE_REGION=${{ inputs.region }} + SCCACHE_ENDPOINT=${{ inputs.endpoint }} SCCACHE_S3_KEY_PREFIX=${{ runner.os }}/sccache AWS_ACCESS_KEY_ID=${{ inputs.aws_access_key_id }} AWS_SECRET_ACCESS_KEY=${{ inputs.aws_secret_access_key }} diff --git a/.github/actions/librocksdb/action.yaml b/.github/actions/librocksdb/action.yaml index a73666584e..217e2745eb 100644 --- a/.github/actions/librocksdb/action.yaml +++ b/.github/actions/librocksdb/action.yaml @@ -9,10 +9,6 @@ inputs: description: RocksDB version, eg. "8.10.2" required: false default: "8.10.2" - bucket: - description: S3 bucket to use for caching - required: false - default: multi-runner-cache-x1xibo9c force: description: Force rebuild required: false diff --git a/.github/actions/s3-layer-cache-settings/action.yaml b/.github/actions/s3-layer-cache-settings/action.yaml index 49cdaeef66..2529733c87 100644 --- a/.github/actions/s3-layer-cache-settings/action.yaml +++ b/.github/actions/s3-layer-cache-settings/action.yaml @@ -1,4 +1,4 @@ -name: 'Get S3 Docker Layer Cache settings' +name: "Get S3 Docker Layer Cache settings" description: | This action generates string with s3-based cache configuration for docker buildx. It defines three manifests: @@ -8,10 +8,10 @@ description: | inputs: name: - description: 'Cache key name will be used as a prefix for all docker image manifests' + description: "Cache key name will be used as a prefix for all docker image manifests" required: true head_ref: - description: 'Head ref for an additional manifest to hit all builds for this head' + description: "Head ref for an additional manifest to hit all builds for this head" default: ${{ github.ref }} region: description: S3 region @@ -19,22 +19,25 @@ inputs: bucket: description: S3 bucket name required: true + endpoint: + description: S3 endpoint to use for caching + required: false prefix: description: S3 key prefix - default: 'cache-layers/' + default: "cache-layers/" mode: description: Cache mode default: max cache_to_name: - description: 'Save cache to name manifest (should be used only on default branch)' - default: 'false' + description: "Save cache to name manifest (should be used only on default branch)" + default: "false" outputs: cache_to: - description: 'String with s3-based cache configuration for docker buildx cache-to option' + description: "String with s3-based cache configuration for docker buildx cache-to option" value: ${{ steps.script.outputs.cache_to }} cache_from: - description: 'String with s3-based cache configuration for docker buildx cache-from option' + description: "String with s3-based cache configuration for docker buildx cache-from option" value: ${{ steps.script.outputs.cache_from }} runs: @@ -49,6 +52,7 @@ runs: region: '${{ inputs.region }}', bucket: '${{ inputs.bucket }}', prefix: '${{ inputs.prefix }}', + endpoint_url: '${{ inputs.endpoint }}', }; const settingsString = Object.entries(settings) diff --git a/.github/workflows/tests-rs-package.yml b/.github/workflows/tests-rs-package.yml index b37d7b9a3f..c1e3dfd093 100644 --- a/.github/workflows/tests-rs-package.yml +++ b/.github/workflows/tests-rs-package.yml @@ -13,6 +13,7 @@ on: jobs: lint: name: Linting + environment: backblaze runs-on: ubuntu-24.04 permissions: id-token: write @@ -27,7 +28,7 @@ jobs: with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_REGION }} + aws-region: ${{ env.AWS_REGION }} - name: Setup Rust uses: ./.github/actions/rust @@ -42,8 +43,9 @@ jobs: args: --package ${{ inputs.package }} --all-features --locked -- --no-deps env: RUSTC_WRAPPER: sccache - SCCACHE_BUCKET: multi-runner-cache-x1xibo9c - SCCACHE_REGION: ${{ secrets.AWS_REGION }} + SCCACHE_BUCKET: ${{ env.AWS_S3_BUCKET }} + SCCACHE_REGION: ${{ env.AWS_REGION }} + SCCACHE_ENDPOINT: ${{ env.AWS_S3_ENDPOINT }} SCCACHE_S3_KEY_PREFIX: ${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu ROCKSDB_STATIC: "/opt/rocksdb/usr/local/lib/librocksdb.a" ROCKSDB_LIB_DIR: "/opt/rocksdb/usr/local/lib" @@ -51,6 +53,7 @@ jobs: SNAPPY_LIB_DIR: "/usr/lib/x86_64-linux-gnu" formatting: name: Formatting + environment: backblaze runs-on: ubuntu-24.04 timeout-minutes: 5 steps: @@ -68,13 +71,15 @@ jobs: - name: Check formatting env: RUSTC_WRAPPER: sccache - SCCACHE_BUCKET: multi-runner-cache-x1xibo9c - SCCACHE_REGION: ${{ secrets.AWS_REGION }} + SCCACHE_BUCKET: ${{ env.AWS_S3_BUCKET }} + SCCACHE_REGION: ${{ env.AWS_REGION }} + SCCACHE_ENDPOINT: ${{ env.AWS_S3_ENDPOINT }} SCCACHE_S3_KEY_PREFIX: ${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu run: cargo fmt --check --package=${{ inputs.package }} unused_deps: name: Unused dependencies + environment: backblaze runs-on: ubuntu-24.04 permissions: id-token: write @@ -87,7 +92,7 @@ jobs: - name: Configure AWS credentials and bucket region uses: aws-actions/configure-aws-credentials@v4 with: - aws-region: ${{ secrets.AWS_REGION }} + aws-region: ${{ env.AWS_REGION }} aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} @@ -107,8 +112,9 @@ jobs: uses: lklimek/cargo-machete@feat/workdir env: RUSTC_WRAPPER: sccache - SCCACHE_BUCKET: multi-runner-cache-x1xibo9c - SCCACHE_REGION: ${{ secrets.AWS_REGION }} + SCCACHE_BUCKET: ${{ env.AWS_S3_BUCKET }} + SCCACHE_REGION: ${{ env.AWS_REGION }} + SCCACHE_ENDPOINT: ${{ env.AWS_S3_ENDPOINT }} SCCACHE_S3_KEY_PREFIX: ${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu ROCKSDB_STATIC: "/opt/rocksdb/usr/local/lib/librocksdb.a" ROCKSDB_LIB_DIR: "/opt/rocksdb/usr/local/lib" @@ -119,6 +125,7 @@ jobs: detect_structure_changes: name: Detect immutable structure changes + environment: backblaze runs-on: ubuntu-24.04 # FIXME: as we use `gh pr view` below, this check can only # run on pull requests. We should find a way to run it @@ -180,6 +187,7 @@ jobs: test: name: Tests + environment: backblaze runs-on: ubuntu-24.04 timeout-minutes: 25 steps: @@ -189,7 +197,7 @@ jobs: - name: Configure AWS credentials and bucket region uses: aws-actions/configure-aws-credentials@v4 with: - aws-region: ${{ secrets.AWS_REGION }} + aws-region: ${{ env.AWS_REGION }} aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} @@ -203,8 +211,9 @@ jobs: run: cargo test --package=${{ inputs.package }} --all-features --locked env: RUSTC_WRAPPER: sccache - SCCACHE_BUCKET: multi-runner-cache-x1xibo9c - SCCACHE_REGION: ${{ secrets.AWS_REGION }} + SCCACHE_BUCKET: ${{ env.AWS_S3_BUCKET }} + SCCACHE_REGION: ${{ env.AWS_REGION }} + SCCACHE_ENDPOINT: ${{ env.AWS_S3_ENDPOINT }} SCCACHE_S3_KEY_PREFIX: ${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu ROCKSDB_STATIC: "/opt/rocksdb/usr/local/lib/librocksdb.a" ROCKSDB_LIB_DIR: "/opt/rocksdb/usr/local/lib" @@ -213,6 +222,7 @@ jobs: check_each_feature: name: Check each feature + environment: backblaze runs-on: ubuntu-24.04 timeout-minutes: 10 if: ${{ inputs.check-each-feature }} @@ -223,7 +233,7 @@ jobs: - name: Configure AWS credentials and bucket region uses: aws-actions/configure-aws-credentials@v4 with: - aws-region: ${{ secrets.AWS_REGION }} + aws-region: ${{ env.AWS_REGION }} aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} @@ -242,8 +252,9 @@ jobs: - name: Check each feature in ${{ inputs.package }} env: RUSTC_WRAPPER: sccache - SCCACHE_BUCKET: multi-runner-cache-x1xibo9c - SCCACHE_REGION: ${{ secrets.AWS_REGION }} + SCCACHE_BUCKET: ${{ env.AWS_S3_BUCKET }} + SCCACHE_REGION: ${{ env.AWS_REGION }} + SCCACHE_ENDPOINT: ${{ env.AWS_S3_ENDPOINT }} SCCACHE_S3_KEY_PREFIX: ${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu ROCKSDB_STATIC: "/opt/rocksdb/usr/local/lib/librocksdb.a" ROCKSDB_LIB_DIR: "/opt/rocksdb/usr/local/lib" diff --git a/Dockerfile b/Dockerfile index cdd7211918..7370109472 100644 --- a/Dockerfile +++ b/Dockerfile @@ -157,6 +157,7 @@ ARG AWS_ACCESS_KEY_ID ARG AWS_REGION ARG SCCACHE_REGION ARG SCCACHE_S3_KEY_PREFIX +ARG SCCACHE_ENDPOINT # Generate sccache configuration variables and save them to /root/env # @@ -181,6 +182,7 @@ RUN <> /root/env # AWS_SECRET_ACCESS_KEY is a secret so we load it using ONBUILD ARG later on echo "export SCCACHE_BUCKET='${SCCACHE_BUCKET}'" >> /root/env + echo "export SCCACHE_ENDPOINT='${SCCACHE_ENDPOINT}'" >> /root/env echo "export SCCACHE_S3_KEY_PREFIX='${SCCACHE_S3_KEY_PREFIX}/${TARGETARCH}/linux-musl'" >> /root/env elif [ -n "${SCCACHE_MEMCACHED}" ]; then # memcached From 69be7f64bc104bd94ed4de103ff8cc73792b921a Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Thu, 21 Nov 2024 10:47:07 +0100 Subject: [PATCH 02/51] test: comment out some tests to speed up testing - to be reverted --- .github/workflows/tests.yml | 256 ++++++++++++++++++------------------ 1 file changed, 130 insertions(+), 126 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5092d32ede..795cefe955 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -53,12 +53,13 @@ jobs: - name: Drive image_name: drive target: drive-abci - - name: DAPI - image_name: dapi - target: dapi - - name: Dashmate helper - image_name: dashmate-helper - target: dashmate-helper + # TODO: uncomment after testing + # - name: DAPI + # image_name: dapi + # target: dapi + # - name: Dashmate helper + # image_name: dashmate-helper + # target: dashmate-helper uses: ./.github/workflows/tests-build-image.yml with: name: ${{ matrix.name }} @@ -73,7 +74,10 @@ jobs: strategy: fail-fast: false matrix: - rs-package: ${{ fromJson(needs.changes.outputs.rs-packages) }} + # TODO: uncomment after testing + # rs-package: ${{ fromJson(needs.changes.outputs.rs-packages) }} + rs-package: + - dpp uses: ./.github/workflows/tests-rs-package.yml with: package: ${{ matrix.rs-package }} @@ -91,122 +95,122 @@ jobs: uses: rustsec/audit-check@v1 with: token: ${{ secrets.GITHUB_TOKEN }} - - js-packages: - name: JS packages - needs: - - changes - - build-js - secrets: inherit - strategy: - fail-fast: false - matrix: - js-package: ${{ fromJson(needs.changes.outputs.js-packages) }} - uses: ./.github/workflows/tests-js-package.yml - with: - package: ${{ matrix.js-package }} - test-command: ${{ matrix.js-package == 'dashmate' && 'test:unit' || 'test' }} - skip-tests: ${{ contains(matrix.js-package, 'platform-test-suite') }} - - js-deps-versions: - name: JS dependency versions check - if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || !github.event.pull_request.draft }} - runs-on: ubuntu-24.04 - steps: - - name: Check out repo - uses: actions/checkout@v4 - - - name: Setup Node.JS - uses: actions/setup-node@v4 - with: - node-version: "20" - - - name: Enable corepack - run: corepack enable - - - name: Validate workspaces - run: yarn constraints - - js-npm-security: - name: JS NPM security audit - if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || !github.event.pull_request.draft }} - runs-on: ubuntu-24.04 - steps: - - name: Check out repo - uses: actions/checkout@v4 - - - name: Enable corepack - run: corepack enable - - - name: Audit NPM - run: yarn npm audit --environment production --all --recursive - - js-codeql: - name: JS code security audit - needs: build-js - secrets: inherit - uses: ./.github/workflows/tests-codeql.yml - - dashmate-e2e-tests: - name: Dashmate E2E tests - secrets: inherit - needs: - - changes - - build-js - - build-images - strategy: - fail-fast: false - matrix: - include: - - name: Local network - test-pattern: test/e2e/localNetwork.spec.js - restore_local_network_data: true - - name: Testnet fullnode - test-pattern: test/e2e/testnetFullnode.spec.js - restore_local_network_data: false - - name: Testnet Evonode - test-pattern: test/e2e/testnetEvonode.spec.js - restore_local_network_data: false - uses: ./.github/workflows/tests-dashmate.yml - with: - name: ${{ matrix.name }} - test-pattern: ${{ matrix.test-pattern }} - restore_local_network_data: ${{ matrix.restore_local_network_data }} - if: contains(needs.changes.outputs.js-packages, 'dashmate') - - test-suite: - name: Test Suite - needs: - - build-js - - build-images - secrets: inherit - strategy: - fail-fast: false - matrix: - include: - - name: Test Suite - command: test:suite - batch_index: 0 - batch_total: 0 - - name: Test Suite in browser (1) - command: test:browsers - batch_index: 0 - batch_total: 2 - - name: Test Suite in browser (2) - command: test:browsers - batch_index: 1 - batch_total: 2 - uses: ./.github/workflows/tests-test-suite.yml - with: - name: ${{ matrix.name }} - command: ${{ matrix.command }} - batch_total: ${{ matrix.batch_total }} - batch_index: ${{ matrix.batch_index }} - - test-functional: - name: Packages functional tests - needs: - - build-js - - build-images - secrets: inherit - uses: ./.github/workflows/tests-packges-functional.yml + # TODO: uncomment after testing + # js-packages: + # name: JS packages + # needs: + # - changes + # - build-js + # secrets: inherit + # strategy: + # fail-fast: false + # matrix: + # js-package: ${{ fromJson(needs.changes.outputs.js-packages) }} + # uses: ./.github/workflows/tests-js-package.yml + # with: + # package: ${{ matrix.js-package }} + # test-command: ${{ matrix.js-package == 'dashmate' && 'test:unit' || 'test' }} + # skip-tests: ${{ contains(matrix.js-package, 'platform-test-suite') }} + + # js-deps-versions: + # name: JS dependency versions check + # if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || !github.event.pull_request.draft }} + # runs-on: ubuntu-24.04 + # steps: + # - name: Check out repo + # uses: actions/checkout@v4 + + # - name: Setup Node.JS + # uses: actions/setup-node@v4 + # with: + # node-version: "20" + + # - name: Enable corepack + # run: corepack enable + + # - name: Validate workspaces + # run: yarn constraints + + # js-npm-security: + # name: JS NPM security audit + # if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || !github.event.pull_request.draft }} + # runs-on: ubuntu-24.04 + # steps: + # - name: Check out repo + # uses: actions/checkout@v4 + + # - name: Enable corepack + # run: corepack enable + + # - name: Audit NPM + # run: yarn npm audit --environment production --all --recursive + + # js-codeql: + # name: JS code security audit + # needs: build-js + # secrets: inherit + # uses: ./.github/workflows/tests-codeql.yml + + # dashmate-e2e-tests: + # name: Dashmate E2E tests + # secrets: inherit + # needs: + # - changes + # - build-js + # - build-images + # strategy: + # fail-fast: false + # matrix: + # include: + # - name: Local network + # test-pattern: test/e2e/localNetwork.spec.js + # restore_local_network_data: true + # - name: Testnet fullnode + # test-pattern: test/e2e/testnetFullnode.spec.js + # restore_local_network_data: false + # - name: Testnet Evonode + # test-pattern: test/e2e/testnetEvonode.spec.js + # restore_local_network_data: false + # uses: ./.github/workflows/tests-dashmate.yml + # with: + # name: ${{ matrix.name }} + # test-pattern: ${{ matrix.test-pattern }} + # restore_local_network_data: ${{ matrix.restore_local_network_data }} + # if: contains(needs.changes.outputs.js-packages, 'dashmate') + + # test-suite: + # name: Test Suite + # needs: + # - build-js + # - build-images + # secrets: inherit + # strategy: + # fail-fast: false + # matrix: + # include: + # - name: Test Suite + # command: test:suite + # batch_index: 0 + # batch_total: 0 + # - name: Test Suite in browser (1) + # command: test:browsers + # batch_index: 0 + # batch_total: 2 + # - name: Test Suite in browser (2) + # command: test:browsers + # batch_index: 1 + # batch_total: 2 + # uses: ./.github/workflows/tests-test-suite.yml + # with: + # name: ${{ matrix.name }} + # command: ${{ matrix.command }} + # batch_total: ${{ matrix.batch_total }} + # batch_index: ${{ matrix.batch_index }} + + # test-functional: + # name: Packages functional tests + # needs: + # - build-js + # - build-images + # secrets: inherit + # uses: ./.github/workflows/tests-packges-functional.yml From 18fe6ab6176fc3bb2d6a958a5bbdd020f14d6e8c Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Thu, 21 Nov 2024 11:19:53 +0100 Subject: [PATCH 03/51] chore: rename S3 vars to CACHE_* --- .github/actions/docker/action.yaml | 5 +-- .github/workflows/tests-rs-package.yml | 42 +++++++++++++------------- Dockerfile | 11 +++---- 3 files changed, 28 insertions(+), 30 deletions(-) diff --git a/.github/actions/docker/action.yaml b/.github/actions/docker/action.yaml index 075eb6e8b9..811da111f0 100644 --- a/.github/actions/docker/action.yaml +++ b/.github/actions/docker/action.yaml @@ -27,7 +27,7 @@ inputs: default: dev bucket: description: S3 bucket to use for caching, must match runner define in `runs-on` - default: ${{ env.AWS_S3_BUCKET }} + required: true region: description: S3 bucket region required: true @@ -163,6 +163,8 @@ runs: push: ${{ inputs.push_tags }} tags: ${{ inputs.push_tags == 'true' && steps.docker_meta.outputs.tags || '' }} platforms: ${{ inputs.platform }} + secrets: | + AWS_SECRET_ACCESS_KEY=${{ inputs.aws_secret_access_key }} build-args: | CARGO_BUILD_PROFILE=${{ inputs.cargo_profile }} RUSTC_WRAPPER=sccache @@ -171,7 +173,6 @@ runs: SCCACHE_ENDPOINT=${{ inputs.endpoint }} SCCACHE_S3_KEY_PREFIX=${{ runner.os }}/sccache AWS_ACCESS_KEY_ID=${{ inputs.aws_access_key_id }} - AWS_SECRET_ACCESS_KEY=${{ inputs.aws_secret_access_key }} cache-from: ${{ steps.layer_cache_settings.outputs.cache_from }} cache-to: ${{ steps.layer_cache_settings.outputs.cache_to }} outputs: type=image,name=${{ inputs.image_org }}/${{ inputs.image_name }},push-by-digest=${{ inputs.push_tags != 'true' }},name-canonical=true,push=true diff --git a/.github/workflows/tests-rs-package.yml b/.github/workflows/tests-rs-package.yml index c1e3dfd093..2077bbbc4e 100644 --- a/.github/workflows/tests-rs-package.yml +++ b/.github/workflows/tests-rs-package.yml @@ -13,7 +13,7 @@ on: jobs: lint: name: Linting - environment: backblaze + environment: test runs-on: ubuntu-24.04 permissions: id-token: write @@ -43,9 +43,9 @@ jobs: args: --package ${{ inputs.package }} --all-features --locked -- --no-deps env: RUSTC_WRAPPER: sccache - SCCACHE_BUCKET: ${{ env.AWS_S3_BUCKET }} - SCCACHE_REGION: ${{ env.AWS_REGION }} - SCCACHE_ENDPOINT: ${{ env.AWS_S3_ENDPOINT }} + SCCACHE_BUCKET: ${{ env.CACHE_S3_BUCKET }} + SCCACHE_REGION: ${{ env.CACHE_REGION }} + SCCACHE_ENDPOINT: ${{ env.CACHE_S3_ENDPOINT }} SCCACHE_S3_KEY_PREFIX: ${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu ROCKSDB_STATIC: "/opt/rocksdb/usr/local/lib/librocksdb.a" ROCKSDB_LIB_DIR: "/opt/rocksdb/usr/local/lib" @@ -53,7 +53,7 @@ jobs: SNAPPY_LIB_DIR: "/usr/lib/x86_64-linux-gnu" formatting: name: Formatting - environment: backblaze + environment: test runs-on: ubuntu-24.04 timeout-minutes: 5 steps: @@ -71,15 +71,15 @@ jobs: - name: Check formatting env: RUSTC_WRAPPER: sccache - SCCACHE_BUCKET: ${{ env.AWS_S3_BUCKET }} - SCCACHE_REGION: ${{ env.AWS_REGION }} - SCCACHE_ENDPOINT: ${{ env.AWS_S3_ENDPOINT }} + SCCACHE_BUCKET: ${{ env.CACHE_S3_BUCKET }} + SCCACHE_REGION: ${{ env.CACHE_REGION }} + SCCACHE_ENDPOINT: ${{ env.CACHE_S3_ENDPOINT }} SCCACHE_S3_KEY_PREFIX: ${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu run: cargo fmt --check --package=${{ inputs.package }} unused_deps: name: Unused dependencies - environment: backblaze + environment: test runs-on: ubuntu-24.04 permissions: id-token: write @@ -112,9 +112,9 @@ jobs: uses: lklimek/cargo-machete@feat/workdir env: RUSTC_WRAPPER: sccache - SCCACHE_BUCKET: ${{ env.AWS_S3_BUCKET }} - SCCACHE_REGION: ${{ env.AWS_REGION }} - SCCACHE_ENDPOINT: ${{ env.AWS_S3_ENDPOINT }} + SCCACHE_BUCKET: ${{ env.CACHE_S3_BUCKET }} + SCCACHE_REGION: ${{ env.CACHE_REGION }} + SCCACHE_ENDPOINT: ${{ env.CACHE_S3_ENDPOINT }} SCCACHE_S3_KEY_PREFIX: ${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu ROCKSDB_STATIC: "/opt/rocksdb/usr/local/lib/librocksdb.a" ROCKSDB_LIB_DIR: "/opt/rocksdb/usr/local/lib" @@ -125,7 +125,7 @@ jobs: detect_structure_changes: name: Detect immutable structure changes - environment: backblaze + environment: test runs-on: ubuntu-24.04 # FIXME: as we use `gh pr view` below, this check can only # run on pull requests. We should find a way to run it @@ -187,7 +187,7 @@ jobs: test: name: Tests - environment: backblaze + environment: test runs-on: ubuntu-24.04 timeout-minutes: 25 steps: @@ -211,9 +211,9 @@ jobs: run: cargo test --package=${{ inputs.package }} --all-features --locked env: RUSTC_WRAPPER: sccache - SCCACHE_BUCKET: ${{ env.AWS_S3_BUCKET }} - SCCACHE_REGION: ${{ env.AWS_REGION }} - SCCACHE_ENDPOINT: ${{ env.AWS_S3_ENDPOINT }} + SCCACHE_BUCKET: ${{ env.CACHE_S3_BUCKET }} + SCCACHE_REGION: ${{ env.CACHE_REGION }} + SCCACHE_ENDPOINT: ${{ env.CACHE_S3_ENDPOINT }} SCCACHE_S3_KEY_PREFIX: ${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu ROCKSDB_STATIC: "/opt/rocksdb/usr/local/lib/librocksdb.a" ROCKSDB_LIB_DIR: "/opt/rocksdb/usr/local/lib" @@ -222,7 +222,7 @@ jobs: check_each_feature: name: Check each feature - environment: backblaze + environment: test runs-on: ubuntu-24.04 timeout-minutes: 10 if: ${{ inputs.check-each-feature }} @@ -252,9 +252,9 @@ jobs: - name: Check each feature in ${{ inputs.package }} env: RUSTC_WRAPPER: sccache - SCCACHE_BUCKET: ${{ env.AWS_S3_BUCKET }} - SCCACHE_REGION: ${{ env.AWS_REGION }} - SCCACHE_ENDPOINT: ${{ env.AWS_S3_ENDPOINT }} + SCCACHE_BUCKET: ${{ env.CACHE_S3_BUCKET }} + SCCACHE_REGION: ${{ env.CACHE_REGION }} + SCCACHE_ENDPOINT: ${{ env.CACHE_S3_ENDPOINT }} SCCACHE_S3_KEY_PREFIX: ${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu ROCKSDB_STATIC: "/opt/rocksdb/usr/local/lib/librocksdb.a" ROCKSDB_LIB_DIR: "/opt/rocksdb/usr/local/lib" diff --git a/Dockerfile b/Dockerfile index 7370109472..ac780a7f17 100644 --- a/Dockerfile +++ b/Dockerfile @@ -169,7 +169,8 @@ RUN <> /root/env echo "export ACTIONS_CACHE_URL=${ACTIONS_CACHE_URL}" >> /root/env - # ACTIONS_RUNTIME_TOKEN is a secret so we load it using ONBUILD ARG later on + # ACTIONS_RUNTIME_TOKEN is a secret so we load it on demand + echo 'export ACTIONS_RUNTIME_TOKEN="$(cat /run/secrets/ACTIONS_RUNTIME_TOKEN)"' >> /root/env elif [ -n "${SCCACHE_BUCKET}" ]; then # AWS S3 if [ -z "${SCCACHE_REGION}" ] ; then @@ -180,7 +181,8 @@ RUN <> /root/env echo "export SCCACHE_REGION='${SCCACHE_REGION}'" >> /root/env echo "export AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}" >> /root/env - # AWS_SECRET_ACCESS_KEY is a secret so we load it using ONBUILD ARG later on + # AWS_SECRET_ACCESS_KEY is a secret so we load it on demand + echo 'export AWS_SECRET_ACCESS_KEY="$(cat /run/secrets/AWS_SECRET_ACCESS_KEY)"' >> /root/env echo "export SCCACHE_BUCKET='${SCCACHE_BUCKET}'" >> /root/env echo "export SCCACHE_ENDPOINT='${SCCACHE_ENDPOINT}'" >> /root/env echo "export SCCACHE_S3_KEY_PREFIX='${SCCACHE_S3_KEY_PREFIX}/${TARGETARCH}/linux-musl'" >> /root/env @@ -199,11 +201,6 @@ RUN < Date: Thu, 21 Nov 2024 13:09:01 +0100 Subject: [PATCH 04/51] ci: fix region --- .github/workflows/tests-build-image.yml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/tests-build-image.yml b/.github/workflows/tests-build-image.yml index 564feff055..aad6904868 100644 --- a/.github/workflows/tests-build-image.yml +++ b/.github/workflows/tests-build-image.yml @@ -17,6 +17,7 @@ on: jobs: build-image: name: Build ${{ inputs.name }} image + environment: test runs-on: ubuntu-24.04 steps: - name: Check out repo @@ -29,23 +30,25 @@ jobs: with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_REGION }} + aws-region: ${{ env.AWS_REGION }} - name: Login to ECR - run: aws ecr get-login-password --region ${{ secrets.AWS_REGION }} | docker login --username AWS --password-stdin ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com + run: aws ecr get-login-password --region ${{ env.AWS_REGION }} | docker login --username AWS --password-stdin ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ env.AWS_REGION }}.amazonaws.com shell: bash - name: Build and push by SHA uses: ./.github/actions/docker with: image_name: ${{ inputs.image_name }} - image_org: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com + image_org: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ env.AWS_REGION }}.amazonaws.com target: ${{ inputs.target }} platform: linux/amd64 push_tags: true dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }} dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }} - region: ${{ secrets.AWS_REGION }} - aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + region: ${{ env.CACHE_REGION }} + bucket: ${{ env.CACHE_S3_BUCKET }} + endpoint: ${{ env.CACHE_S3_ENDPOINT }} + aws_access_key_id: ${{ secrets.CACHE_KEY_ID }} + aws_secret_access_key: ${{ secrets.CACHE_SECRET_KEY }} cache_to_name: ${{ github.event_name == 'push' && 'true' || 'false' }} From edb98c1017fd809774e0cdc70443d49b3a841edd Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Thu, 21 Nov 2024 14:28:14 +0100 Subject: [PATCH 05/51] chore: sccache action --- .github/actions/docker/action.yaml | 16 +++++---- .github/actions/sccache/action.yaml | 48 ++++++++++++++++++++++++++ .github/workflows/tests-rs-package.yml | 38 +++++++++++++++++++- Dockerfile | 16 +++++---- 4 files changed, 104 insertions(+), 14 deletions(-) create mode 100644 .github/actions/sccache/action.yaml diff --git a/.github/actions/docker/action.yaml b/.github/actions/docker/action.yaml index 811da111f0..02426bc6f8 100644 --- a/.github/actions/docker/action.yaml +++ b/.github/actions/docker/action.yaml @@ -163,16 +163,18 @@ runs: push: ${{ inputs.push_tags }} tags: ${{ inputs.push_tags == 'true' && steps.docker_meta.outputs.tags || '' }} platforms: ${{ inputs.platform }} - secrets: | - AWS_SECRET_ACCESS_KEY=${{ inputs.aws_secret_access_key }} + secret-files: + - "${{ env.HOME }}/credentials=AWS" + # secrets: | + # AWS_SECRET_ACCESS_KEY=${{ inputs.aws_secret_access_key }} build-args: | CARGO_BUILD_PROFILE=${{ inputs.cargo_profile }} RUSTC_WRAPPER=sccache - SCCACHE_BUCKET=${{ inputs.bucket }} - SCCACHE_REGION=${{ inputs.region }} - SCCACHE_ENDPOINT=${{ inputs.endpoint }} - SCCACHE_S3_KEY_PREFIX=${{ runner.os }}/sccache - AWS_ACCESS_KEY_ID=${{ inputs.aws_access_key_id }} + SCCACHE_BUCKET=${{ env.SCCACHE_BUCKET }} + SCCACHE_REGION=${{ env.SCCACHE_REGION }} + SCCACHE_ENDPOINT=${{ env.CACHE_S3_ENDPOINT }} + SCCACHE_S3_KEY_PREFIX=${{ env.SCCACHE_S3_KEY_PREFIX }} + AWS_PROFILE=sccache cache-from: ${{ steps.layer_cache_settings.outputs.cache_from }} cache-to: ${{ steps.layer_cache_settings.outputs.cache_to }} outputs: type=image,name=${{ inputs.image_org }}/${{ inputs.image_name }},push-by-digest=${{ inputs.push_tags != 'true' }},name-canonical=true,push=true diff --git a/.github/actions/sccache/action.yaml b/.github/actions/sccache/action.yaml new file mode 100644 index 0000000000..2ab2e09804 --- /dev/null +++ b/.github/actions/sccache/action.yaml @@ -0,0 +1,48 @@ +--- +name: "sccache" +description: "Configure sccache caching" +inputs: + bucket: + description: S3 bucket to use for caching + required: true + region: + description: S3 bucket region + required: true + endpoint: + description: S3 endpoint to use for caching + required: true + aws_access_key_id: + description: AWS access key ID + required: true + aws_secret_access_key: + description: AWS secret access key + required: true + +# TODO: Cache deps here to save 1 minute +runs: + using: composite + steps: + - name: Install sccache-cache + uses: mozilla-actions/sccache-action@v0.0.6 + with: + version: "v0.8.2" # Must be the same as in Dockerfile + + - name: Configure AWS credentials + shell: bash + run: | + echo "[sccache]" >> ${HOME}/.aws/credentials + echo "aws_access_key_id=${{ inputs.aws_access_key_id }}" >> ${HOME}/.aws/credentials + echo "aws_secret_access_key=${{ inputs.aws_secret_access_key }}" >> ${HOME}/.aws/credentials + chmod 600 ${HOME}/.aws/credentials + + - name: Configure sccache + shell: bash + run: | + echo "CARGO_INCREMENTAL=0" >> $GITHUB_ENV + echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV + echo "SCCACHE_BUCKET=${{ inputs.bucket }}" >> $GITHUB_ENV + echo "SCCACHE_REGION=${{ inputs.region }}" >> $GITHUB_ENV + echo "SCCACHE_ENDPOINT=${{ inputs.endpoint }}" >> $GITHUB_ENV + echo "SCCACHE_S3_KEY_PREFIX='${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu'" >> $GITHUB_ENV + echo "AWS_PROFILE=sccache" >> $GITHUB_ENV + echo "AWS_SHARED_CREDENTIALS_FILE=${HOME}/.aws/credentials" >> $GITHUB_ENV diff --git a/.github/workflows/tests-rs-package.yml b/.github/workflows/tests-rs-package.yml index 2077bbbc4e..677e5768e0 100644 --- a/.github/workflows/tests-rs-package.yml +++ b/.github/workflows/tests-rs-package.yml @@ -35,6 +35,15 @@ jobs: with: components: clippy + - name: Setup sccache + uses: ./.github/actions/sccache + with: + bucket: ${{ env.CACHE_S3_BUCKET }} + region: ${{ env.CACHE_REGION }} + endpoint: ${{ env.CACHE_S3_ENDPOINT }} + aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + - name: Install librocksdb uses: ./.github/actions/librocksdb @@ -66,7 +75,8 @@ jobs: components: rustfmt cache: false - # This step doesn't need librocksdb, so we don't install it + # We don't use cache for this step, nothing to cache here + # This step doesn't need librocksdb, so we don't install it - name: Check formatting env: @@ -99,6 +109,14 @@ jobs: - name: Setup Rust uses: ./.github/actions/rust + - name: Setup sccache + uses: ./.github/actions/sccache + with: + bucket: ${{ env.CACHE_S3_BUCKET }} + region: ${{ env.CACHE_REGION }} + endpoint: ${{ env.CACHE_S3_ENDPOINT }} + aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - name: Install librocksdb uses: ./.github/actions/librocksdb @@ -204,6 +222,15 @@ jobs: - name: Setup Rust uses: ./.github/actions/rust + - name: Setup sccache + uses: ./.github/actions/sccache + with: + bucket: ${{ env.CACHE_S3_BUCKET }} + region: ${{ env.CACHE_REGION }} + endpoint: ${{ env.CACHE_S3_ENDPOINT }} + aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + - name: Install librocksdb uses: ./.github/actions/librocksdb @@ -240,6 +267,15 @@ jobs: - name: Setup Rust uses: ./.github/actions/rust + - name: Setup sccache + uses: ./.github/actions/sccache + with: + bucket: ${{ env.CACHE_S3_BUCKET }} + region: ${{ env.CACHE_REGION }} + endpoint: ${{ env.CACHE_S3_ENDPOINT }} + aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + - name: Install librocksdb uses: ./.github/actions/librocksdb diff --git a/Dockerfile b/Dockerfile index ac780a7f17..fbf9312291 100644 --- a/Dockerfile +++ b/Dockerfile @@ -154,6 +154,7 @@ ARG SCCACHE_MEMCACHED # S3 storage ARG SCCACHE_BUCKET ARG AWS_ACCESS_KEY_ID +ARG AWS_PROFILE ARG AWS_REGION ARG SCCACHE_REGION ARG SCCACHE_S3_KEY_PREFIX @@ -162,7 +163,7 @@ ARG SCCACHE_ENDPOINT # Generate sccache configuration variables and save them to /root/env # # We only enable one cache at a time. Setting env variables belonging to multiple cache backends may fail the build. -RUN <> /root/env echo "export SCCACHE_REGION='${SCCACHE_REGION}'" >> /root/env - echo "export AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}" >> /root/env - # AWS_SECRET_ACCESS_KEY is a secret so we load it on demand - echo 'export AWS_SECRET_ACCESS_KEY="$(cat /run/secrets/AWS_SECRET_ACCESS_KEY)"' >> /root/env + + [ -n "${AWS_REGION}" ] && echo "export AWS_REGION='${AWS_REGION}'" >> /root/env + [ -n "${AWS_PROFILE}"] && echo "export AWS_PROFILE='${AWS_PROFILE}'" >> /root/env + echo "export AWS_SHARED_CREDENTIALS_FILE=/run/secrets/AWS" >> /root/env echo "export SCCACHE_BUCKET='${SCCACHE_BUCKET}'" >> /root/env echo "export SCCACHE_ENDPOINT='${SCCACHE_ENDPOINT}'" >> /root/env echo "export SCCACHE_S3_KEY_PREFIX='${SCCACHE_S3_KEY_PREFIX}/${TARGETARCH}/linux-musl'" >> /root/env @@ -340,6 +340,7 @@ COPY --from=build-planner /platform/recipe.json /platform/.cargo /platform/ RUN --mount=type=cache,sharing=shared,id=cargo_registry_index,target=${CARGO_HOME}/registry/index \ --mount=type=cache,sharing=shared,id=cargo_registry_cache,target=${CARGO_HOME}/registry/cache \ --mount=type=cache,sharing=shared,id=cargo_git,target=${CARGO_HOME}/git/db \ + --mount=type=secret,id=AWS \ set -ex; \ if [[ "${CARGO_BUILD_PROFILE}" == "release" ]] ; then \ mv .cargo/config-release.toml .cargo/config.toml; \ @@ -397,6 +398,7 @@ RUN mkdir /artifacts RUN --mount=type=cache,sharing=shared,id=cargo_registry_index,target=${CARGO_HOME}/registry/index \ --mount=type=cache,sharing=shared,id=cargo_registry_cache,target=${CARGO_HOME}/registry/cache \ --mount=type=cache,sharing=shared,id=cargo_git,target=${CARGO_HOME}/git/db \ + --mount=type=secret,id=AWS \ set -ex; \ source $HOME/.cargo/env && \ source /root/env && \ @@ -432,6 +434,7 @@ COPY --from=build-planner /platform/recipe.json recipe.json RUN --mount=type=cache,sharing=shared,id=cargo_registry_index,target=${CARGO_HOME}/registry/index \ --mount=type=cache,sharing=shared,id=cargo_registry_cache,target=${CARGO_HOME}/registry/cache \ --mount=type=cache,sharing=shared,id=cargo_git,target=${CARGO_HOME}/git/db \ + --mount=type=secret,id=AWS \ source $HOME/.cargo/env && \ source /root/env && \ cargo chef cook \ @@ -483,6 +486,7 @@ RUN --mount=type=cache,sharing=shared,id=cargo_registry_index,target=${CARGO_HOM --mount=type=cache,sharing=shared,id=cargo_registry_cache,target=${CARGO_HOME}/registry/cache \ --mount=type=cache,sharing=shared,id=cargo_git,target=${CARGO_HOME}/git/db \ --mount=type=cache,sharing=shared,id=unplugged_${TARGETARCH},target=/tmp/unplugged \ + --mount=type=secret,id=AWS \ source $HOME/.cargo/env && \ source /root/env && \ cp -R /tmp/unplugged /platform/.yarn/ && \ From 4edf3093f2f8bacb1ae3e25499531a6d1909d062 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Thu, 21 Nov 2024 14:49:29 +0100 Subject: [PATCH 06/51] chore: s/env/vars --- .github/actions/docker/action.yaml | 19 +++++++-- .github/actions/sccache/action.yaml | 6 ++- .github/workflows/tests-rs-package.yml | 57 ++++++++------------------ 3 files changed, 36 insertions(+), 46 deletions(-) diff --git a/.github/actions/docker/action.yaml b/.github/actions/docker/action.yaml index 02426bc6f8..10728d01b4 100644 --- a/.github/actions/docker/action.yaml +++ b/.github/actions/docker/action.yaml @@ -50,6 +50,7 @@ outputs: runs: using: composite + steps: - name: Login to DockerHub uses: docker/login-action@v3 @@ -152,6 +153,16 @@ runs: } skip-extraction: ${{ steps.yarn-cache.outputs.cache-hit }} + - name: Setup sccache vars + uses: ./.github/actions/sccache + with: + bucket: ${{ inputs.bucket }} + region: ${{ inputs.region }} + endpoint: ${{ inputs.endpoint }} + aws_access_key_id: ${{ inputs.aws_access_key_id }} + aws_secret_access_key: ${{ inputs.aws_secret_access_key }} + install: false + - name: Build and push Docker image ${{ inputs.image }} id: docker_build uses: docker/build-push-action@v6 @@ -170,10 +181,10 @@ runs: build-args: | CARGO_BUILD_PROFILE=${{ inputs.cargo_profile }} RUSTC_WRAPPER=sccache - SCCACHE_BUCKET=${{ env.SCCACHE_BUCKET }} - SCCACHE_REGION=${{ env.SCCACHE_REGION }} - SCCACHE_ENDPOINT=${{ env.CACHE_S3_ENDPOINT }} - SCCACHE_S3_KEY_PREFIX=${{ env.SCCACHE_S3_KEY_PREFIX }} + SCCACHE_BUCKET=${{ inputs.bucket }} + SCCACHE_REGION=${{ inputs.region }} + SCCACHE_ENDPOINT=${{ inputs.endpoint }} + SCCACHE_S3_KEY_PREFIX=${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu AWS_PROFILE=sccache cache-from: ${{ steps.layer_cache_settings.outputs.cache_from }} cache-to: ${{ steps.layer_cache_settings.outputs.cache_to }} diff --git a/.github/actions/sccache/action.yaml b/.github/actions/sccache/action.yaml index 2ab2e09804..dbbf845d0c 100644 --- a/.github/actions/sccache/action.yaml +++ b/.github/actions/sccache/action.yaml @@ -17,12 +17,16 @@ inputs: aws_secret_access_key: description: AWS secret access key required: true + install: + description: "Install sccache" + default: "true" # TODO: Cache deps here to save 1 minute runs: using: composite steps: - - name: Install sccache-cache + - name: Install sccache binary + if: ${{ inputs.install == 'true' }} uses: mozilla-actions/sccache-action@v0.0.6 with: version: "v0.8.2" # Must be the same as in Dockerfile diff --git a/.github/workflows/tests-rs-package.yml b/.github/workflows/tests-rs-package.yml index 677e5768e0..a2305cea8b 100644 --- a/.github/workflows/tests-rs-package.yml +++ b/.github/workflows/tests-rs-package.yml @@ -28,7 +28,7 @@ jobs: with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ env.AWS_REGION }} + aws-region: ${{ vars.AWS_REGION }} - name: Setup Rust uses: ./.github/actions/rust @@ -38,9 +38,9 @@ jobs: - name: Setup sccache uses: ./.github/actions/sccache with: - bucket: ${{ env.CACHE_S3_BUCKET }} - region: ${{ env.CACHE_REGION }} - endpoint: ${{ env.CACHE_S3_ENDPOINT }} + bucket: ${{ vars.CACHE_S3_BUCKET }} + region: ${{ vars.CACHE_REGION }} + endpoint: ${{ vars.CACHE_S3_ENDPOINT }} aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} @@ -51,11 +51,6 @@ jobs: with: args: --package ${{ inputs.package }} --all-features --locked -- --no-deps env: - RUSTC_WRAPPER: sccache - SCCACHE_BUCKET: ${{ env.CACHE_S3_BUCKET }} - SCCACHE_REGION: ${{ env.CACHE_REGION }} - SCCACHE_ENDPOINT: ${{ env.CACHE_S3_ENDPOINT }} - SCCACHE_S3_KEY_PREFIX: ${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu ROCKSDB_STATIC: "/opt/rocksdb/usr/local/lib/librocksdb.a" ROCKSDB_LIB_DIR: "/opt/rocksdb/usr/local/lib" SNAPPY_STATIC: "/usr/lib/x86_64-linux-gnu/libsnappy.a" @@ -79,12 +74,6 @@ jobs: # This step doesn't need librocksdb, so we don't install it - name: Check formatting - env: - RUSTC_WRAPPER: sccache - SCCACHE_BUCKET: ${{ env.CACHE_S3_BUCKET }} - SCCACHE_REGION: ${{ env.CACHE_REGION }} - SCCACHE_ENDPOINT: ${{ env.CACHE_S3_ENDPOINT }} - SCCACHE_S3_KEY_PREFIX: ${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu run: cargo fmt --check --package=${{ inputs.package }} unused_deps: @@ -102,7 +91,7 @@ jobs: - name: Configure AWS credentials and bucket region uses: aws-actions/configure-aws-credentials@v4 with: - aws-region: ${{ env.AWS_REGION }} + aws-region: ${{ vars.AWS_REGION }} aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} @@ -112,9 +101,9 @@ jobs: - name: Setup sccache uses: ./.github/actions/sccache with: - bucket: ${{ env.CACHE_S3_BUCKET }} - region: ${{ env.CACHE_REGION }} - endpoint: ${{ env.CACHE_S3_ENDPOINT }} + bucket: ${{ vars.CACHE_S3_BUCKET }} + region: ${{ vars.CACHE_REGION }} + endpoint: ${{ vars.CACHE_S3_ENDPOINT }} aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - name: Install librocksdb @@ -129,11 +118,6 @@ jobs: - name: Find unused dependencies uses: lklimek/cargo-machete@feat/workdir env: - RUSTC_WRAPPER: sccache - SCCACHE_BUCKET: ${{ env.CACHE_S3_BUCKET }} - SCCACHE_REGION: ${{ env.CACHE_REGION }} - SCCACHE_ENDPOINT: ${{ env.CACHE_S3_ENDPOINT }} - SCCACHE_S3_KEY_PREFIX: ${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu ROCKSDB_STATIC: "/opt/rocksdb/usr/local/lib/librocksdb.a" ROCKSDB_LIB_DIR: "/opt/rocksdb/usr/local/lib" SNAPPY_STATIC: "/usr/lib/x86_64-linux-gnu/libsnappy.a" @@ -215,7 +199,7 @@ jobs: - name: Configure AWS credentials and bucket region uses: aws-actions/configure-aws-credentials@v4 with: - aws-region: ${{ env.AWS_REGION }} + aws-region: ${{ vars.AWS_REGION }} aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} @@ -225,9 +209,9 @@ jobs: - name: Setup sccache uses: ./.github/actions/sccache with: - bucket: ${{ env.CACHE_S3_BUCKET }} - region: ${{ env.CACHE_REGION }} - endpoint: ${{ env.CACHE_S3_ENDPOINT }} + bucket: ${{ vars.CACHE_S3_BUCKET }} + region: ${{ vars.CACHE_REGION }} + endpoint: ${{ vars.CACHE_S3_ENDPOINT }} aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} @@ -237,10 +221,6 @@ jobs: - name: Run tests run: cargo test --package=${{ inputs.package }} --all-features --locked env: - RUSTC_WRAPPER: sccache - SCCACHE_BUCKET: ${{ env.CACHE_S3_BUCKET }} - SCCACHE_REGION: ${{ env.CACHE_REGION }} - SCCACHE_ENDPOINT: ${{ env.CACHE_S3_ENDPOINT }} SCCACHE_S3_KEY_PREFIX: ${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu ROCKSDB_STATIC: "/opt/rocksdb/usr/local/lib/librocksdb.a" ROCKSDB_LIB_DIR: "/opt/rocksdb/usr/local/lib" @@ -260,7 +240,7 @@ jobs: - name: Configure AWS credentials and bucket region uses: aws-actions/configure-aws-credentials@v4 with: - aws-region: ${{ env.AWS_REGION }} + aws-region: ${{ vars.AWS_REGION }} aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} @@ -270,9 +250,9 @@ jobs: - name: Setup sccache uses: ./.github/actions/sccache with: - bucket: ${{ env.CACHE_S3_BUCKET }} - region: ${{ env.CACHE_REGION }} - endpoint: ${{ env.CACHE_S3_ENDPOINT }} + bucket: ${{ vars.CACHE_S3_BUCKET }} + region: ${{ vars.CACHE_REGION }} + endpoint: ${{ vars.CACHE_S3_ENDPOINT }} aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} @@ -287,11 +267,6 @@ jobs: - name: Check each feature in ${{ inputs.package }} env: - RUSTC_WRAPPER: sccache - SCCACHE_BUCKET: ${{ env.CACHE_S3_BUCKET }} - SCCACHE_REGION: ${{ env.CACHE_REGION }} - SCCACHE_ENDPOINT: ${{ env.CACHE_S3_ENDPOINT }} - SCCACHE_S3_KEY_PREFIX: ${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu ROCKSDB_STATIC: "/opt/rocksdb/usr/local/lib/librocksdb.a" ROCKSDB_LIB_DIR: "/opt/rocksdb/usr/local/lib" SNAPPY_STATIC: "/usr/lib/x86_64-linux-gnu/libsnappy.a" From 3aa361c5276d5c52c85b0fb9e28ce1b1b06a5b03 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Thu, 21 Nov 2024 14:56:44 +0100 Subject: [PATCH 07/51] chore: env/vars --- .github/actions/sccache/action.yaml | 1 + .github/workflows/tests-build-image.yml | 12 ++++++------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/actions/sccache/action.yaml b/.github/actions/sccache/action.yaml index dbbf845d0c..6787fbeab4 100644 --- a/.github/actions/sccache/action.yaml +++ b/.github/actions/sccache/action.yaml @@ -42,6 +42,7 @@ runs: - name: Configure sccache shell: bash run: | + mkdir -p "${HOME}/.aws" echo "CARGO_INCREMENTAL=0" >> $GITHUB_ENV echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV echo "SCCACHE_BUCKET=${{ inputs.bucket }}" >> $GITHUB_ENV diff --git a/.github/workflows/tests-build-image.yml b/.github/workflows/tests-build-image.yml index aad6904868..4e86735f71 100644 --- a/.github/workflows/tests-build-image.yml +++ b/.github/workflows/tests-build-image.yml @@ -30,25 +30,25 @@ jobs: with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ env.AWS_REGION }} + aws-region: ${{ vars.AWS_REGION }} - name: Login to ECR - run: aws ecr get-login-password --region ${{ env.AWS_REGION }} | docker login --username AWS --password-stdin ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ env.AWS_REGION }}.amazonaws.com + run: aws ecr get-login-password --region ${{ vars.AWS_REGION }} | docker login --username AWS --password-stdin ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_REGION }}.amazonaws.com shell: bash - name: Build and push by SHA uses: ./.github/actions/docker with: image_name: ${{ inputs.image_name }} - image_org: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ env.AWS_REGION }}.amazonaws.com + image_org: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_REGION }}.amazonaws.com target: ${{ inputs.target }} platform: linux/amd64 push_tags: true dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }} dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }} - region: ${{ env.CACHE_REGION }} - bucket: ${{ env.CACHE_S3_BUCKET }} - endpoint: ${{ env.CACHE_S3_ENDPOINT }} + region: ${{ vars.CACHE_REGION }} + bucket: ${{ vars.CACHE_S3_BUCKET }} + endpoint: ${{ vars.CACHE_S3_ENDPOINT }} aws_access_key_id: ${{ secrets.CACHE_KEY_ID }} aws_secret_access_key: ${{ secrets.CACHE_SECRET_KEY }} cache_to_name: ${{ github.event_name == 'push' && 'true' || 'false' }} From 524b9018fbcb349f2832cbe19bc5d35efd85a73b Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Thu, 21 Nov 2024 15:16:02 +0100 Subject: [PATCH 08/51] chore: minor fixes --- .github/actions/docker/action.yaml | 4 ++-- .github/actions/sccache/action.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/docker/action.yaml b/.github/actions/docker/action.yaml index 10728d01b4..0a4e331272 100644 --- a/.github/actions/docker/action.yaml +++ b/.github/actions/docker/action.yaml @@ -174,8 +174,8 @@ runs: push: ${{ inputs.push_tags }} tags: ${{ inputs.push_tags == 'true' && steps.docker_meta.outputs.tags || '' }} platforms: ${{ inputs.platform }} - secret-files: - - "${{ env.HOME }}/credentials=AWS" + secret-files: | + ${{ env.HOME }}/credentials=AWS # secrets: | # AWS_SECRET_ACCESS_KEY=${{ inputs.aws_secret_access_key }} build-args: | diff --git a/.github/actions/sccache/action.yaml b/.github/actions/sccache/action.yaml index 6787fbeab4..648340e504 100644 --- a/.github/actions/sccache/action.yaml +++ b/.github/actions/sccache/action.yaml @@ -34,6 +34,7 @@ runs: - name: Configure AWS credentials shell: bash run: | + mkdir --mode=0700 -p "${HOME}/.aws" echo "[sccache]" >> ${HOME}/.aws/credentials echo "aws_access_key_id=${{ inputs.aws_access_key_id }}" >> ${HOME}/.aws/credentials echo "aws_secret_access_key=${{ inputs.aws_secret_access_key }}" >> ${HOME}/.aws/credentials @@ -42,7 +43,6 @@ runs: - name: Configure sccache shell: bash run: | - mkdir -p "${HOME}/.aws" echo "CARGO_INCREMENTAL=0" >> $GITHUB_ENV echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV echo "SCCACHE_BUCKET=${{ inputs.bucket }}" >> $GITHUB_ENV From dbacec6b84931f35dbf47073af6c67180fde0e2c Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Thu, 21 Nov 2024 15:27:30 +0100 Subject: [PATCH 09/51] chore: dockerfile typo --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index fbf9312291..7a84865afa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -181,7 +181,7 @@ RUN --mount=type=secret,id=AWS <> /root/env [ -n "${AWS_REGION}" ] && echo "export AWS_REGION='${AWS_REGION}'" >> /root/env - [ -n "${AWS_PROFILE}"] && echo "export AWS_PROFILE='${AWS_PROFILE}'" >> /root/env + [ -n "${AWS_PROFILE}" ] && echo "export AWS_PROFILE='${AWS_PROFILE}'" >> /root/env echo "export AWS_SHARED_CREDENTIALS_FILE=/run/secrets/AWS" >> /root/env echo "export SCCACHE_BUCKET='${SCCACHE_BUCKET}'" >> /root/env echo "export SCCACHE_ENDPOINT='${SCCACHE_ENDPOINT}'" >> /root/env From c6ed0384a66ef8a10ef16d2d6f981a8231af0347 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Thu, 21 Nov 2024 15:42:37 +0100 Subject: [PATCH 10/51] chore: some secrets fixes --- .github/actions/docker/action.yaml | 2 +- .github/actions/sccache/action.yaml | 6 +++--- Dockerfile | 15 ++++++++++++--- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/.github/actions/docker/action.yaml b/.github/actions/docker/action.yaml index 0a4e331272..588164b41f 100644 --- a/.github/actions/docker/action.yaml +++ b/.github/actions/docker/action.yaml @@ -175,7 +175,7 @@ runs: tags: ${{ inputs.push_tags == 'true' && steps.docker_meta.outputs.tags || '' }} platforms: ${{ inputs.platform }} secret-files: | - ${{ env.HOME }}/credentials=AWS + AWS=${{ env.HOME }}/.aws/credentials # secrets: | # AWS_SECRET_ACCESS_KEY=${{ inputs.aws_secret_access_key }} build-args: | diff --git a/.github/actions/sccache/action.yaml b/.github/actions/sccache/action.yaml index 648340e504..ac9b218aff 100644 --- a/.github/actions/sccache/action.yaml +++ b/.github/actions/sccache/action.yaml @@ -34,11 +34,11 @@ runs: - name: Configure AWS credentials shell: bash run: | - mkdir --mode=0700 -p "${HOME}/.aws" + mkdir -p "${HOME}/.aws" echo "[sccache]" >> ${HOME}/.aws/credentials echo "aws_access_key_id=${{ inputs.aws_access_key_id }}" >> ${HOME}/.aws/credentials echo "aws_secret_access_key=${{ inputs.aws_secret_access_key }}" >> ${HOME}/.aws/credentials - chmod 600 ${HOME}/.aws/credentials + chmod -R go-rwx ${HOME}/.aws - name: Configure sccache shell: bash @@ -48,6 +48,6 @@ runs: echo "SCCACHE_BUCKET=${{ inputs.bucket }}" >> $GITHUB_ENV echo "SCCACHE_REGION=${{ inputs.region }}" >> $GITHUB_ENV echo "SCCACHE_ENDPOINT=${{ inputs.endpoint }}" >> $GITHUB_ENV - echo "SCCACHE_S3_KEY_PREFIX='${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu'" >> $GITHUB_ENV + echo "SCCACHE_S3_KEY_PREFIX=${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu" >> $GITHUB_ENV echo "AWS_PROFILE=sccache" >> $GITHUB_ENV echo "AWS_SHARED_CREDENTIALS_FILE=${HOME}/.aws/credentials" >> $GITHUB_ENV diff --git a/Dockerfile b/Dockerfile index 7a84865afa..ba5a8bf253 100644 --- a/Dockerfile +++ b/Dockerfile @@ -166,14 +166,16 @@ ARG SCCACHE_ENDPOINT RUN --mount=type=secret,id=AWS <> /root/env echo "export ACTIONS_CACHE_URL=${ACTIONS_CACHE_URL}" >> /root/env # ACTIONS_RUNTIME_TOKEN is a secret so we load it on demand echo 'export ACTIONS_RUNTIME_TOKEN="$(cat /run/secrets/ACTIONS_RUNTIME_TOKEN)"' >> /root/env + + ### AWS S3 ### elif [ -n "${SCCACHE_BUCKET}" ]; then - # AWS S3 if [ -z "${SCCACHE_REGION}" ] ; then # Default to AWS_REGION if not set export SCCACHE_REGION=${AWS_REGION} @@ -182,10 +184,15 @@ RUN --mount=type=secret,id=AWS <> /root/env [ -n "${AWS_PROFILE}" ] && echo "export AWS_PROFILE='${AWS_PROFILE}'" >> /root/env - echo "export AWS_SHARED_CREDENTIALS_FILE=/run/secrets/AWS" >> /root/env echo "export SCCACHE_BUCKET='${SCCACHE_BUCKET}'" >> /root/env echo "export SCCACHE_ENDPOINT='${SCCACHE_ENDPOINT}'" >> /root/env echo "export SCCACHE_S3_KEY_PREFIX='${SCCACHE_S3_KEY_PREFIX}/${TARGETARCH}/linux-musl'" >> /root/env + + echo "export AWS_SHARED_CREDENTIALS_FILE=/run/secrets/AWS" >> /root/env + # Check if AWS credentials file is mounted correctly, eg. --mount=type=secret,id=AWS + echo '[ -r "${AWS_SHARED_CREDENTIALS_FILE}" ] || echo "Cannot read ${AWS_SHARED_CREDENTIALS_FILE}"' >> /root/env + + ### memcached ### elif [ -n "${SCCACHE_MEMCACHED}" ]; then # memcached echo "export SCCACHE_MEMCACHED='${SCCACHE_MEMCACHED}'" >> /root/env @@ -199,6 +206,8 @@ RUN --mount=type=secret,id=AWS < Date: Thu, 21 Nov 2024 16:44:08 +0100 Subject: [PATCH 11/51] build: dockerfile improvements --- .github/actions/docker/action.yaml | 1 - Dockerfile | 104 +++++++++++++++++++---------- 2 files changed, 69 insertions(+), 36 deletions(-) diff --git a/.github/actions/docker/action.yaml b/.github/actions/docker/action.yaml index 588164b41f..f566883006 100644 --- a/.github/actions/docker/action.yaml +++ b/.github/actions/docker/action.yaml @@ -180,7 +180,6 @@ runs: # AWS_SECRET_ACCESS_KEY=${{ inputs.aws_secret_access_key }} build-args: | CARGO_BUILD_PROFILE=${{ inputs.cargo_profile }} - RUSTC_WRAPPER=sccache SCCACHE_BUCKET=${{ inputs.bucket }} SCCACHE_REGION=${{ inputs.region }} SCCACHE_ENDPOINT=${{ inputs.endpoint }} diff --git a/Dockerfile b/Dockerfile index ba5a8bf253..3d5bd4a55e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,16 +15,26 @@ # The following build arguments can be provided using --build-arg: # - CARGO_BUILD_PROFILE - set to `release` to build final binary, without debugging information # - NODE_ENV - node.js environment name to use to build the library -# - RUSTC_WRAPPER - set to `sccache` to enable sccache support and make the following variables available: -# - SCCACHE_GHA_ENABLED, ACTIONS_CACHE_URL, ACTIONS_RUNTIME_TOKEN - store sccache caches inside github actions -# - SCCACHE_BUCKET, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_REGION, SCCACHE_S3_KEY_PREFIX - store caches in S3 -# - SCCACHE_MEMCACHED - set to memcache server URI (eg. tcp://172.17.0.1:11211) to enable sccache memcached backend # - ALPINE_VERSION - use different version of Alpine base image; requires also rust:apline... # image to be available # - USERNAME, USER_UID, USER_GID - specification of user used to run the binary # +# # sccache cache backends # -# +# To enable sccache support and make the following variables available: +# 1. For S3 buckets: +# - SCCACHE_BUCKET - S3 bucket name +# - AWS_PROFILE +# - SCCACHE_REGION +# - SCCACHE_S3_KEY_PREFIX +# - SCCACHE_ENDPOINT +# - also, AWS credentials file ($HOME/.aws/credentials) should be provided as a secret file with id=AWS +# 2. For Github Actions: +# - SCCACHE_GHA_ENABLED, ACTIONS_CACHE_URL +# - also, Github Actions token should be provided as a secret file with id=GHA +# 3. For memcached: +# - SCCACHE_MEMCACHED - set to memcache server URI (eg. tcp://172.17.0.1:11211) to enable sccache memcached backend + # # BUILD PROCESS # @@ -35,7 +45,17 @@ # 3. Configuration variables are shared between runs using /root/env file. ARG ALPINE_VERSION=3.18 -ARG RUSTC_WRAPPER + +# deps-${RUSTC_WRAPPER:-base} +# If one of SCCACHE_GHA_ENABLED, SCCACHE_BUCKET, SCCACHE_MEMCACHED is set, then deps-sccache is used, otherwise deps-base +ARG SCCACHE_GHA_ENABLED +ARG SCCACHE_BUCKET +ARG SCCACHE_MEMCACHED + +# Determine if we have sccache enabled; if yes, use deps-sccache, otherwise use deps-base as a dependency image +ARG DEPS_IMAGE=${SCCACHE_GHA_ENABLED}${SCCACHE_BUCKET}${SCCACHE_MEMCACHED} +ARG DEPS_IMAGE=${DEPS_IMAGE:+sccache} +ARG DEPS_IMAGE=deps-${DEPS_IMAGE:-base} # # DEPS: INSTALL AND CACHE DEPENDENCIES @@ -119,10 +139,10 @@ ENV NODE_ENV=${NODE_ENV} # # This stage is used to install sccache and configure it. # Later on, one should source /root/env before building to use sccache. - +# # Note that, due to security concerns, each stage needs to declare variables containing authentication secrets, like -# ACTIONS_RUNTIME_TOKEN, AWS_SECRET_ACCESS_KEY. It is done using ONBUILD directive, so the secrets are not stored in the -# final image. +# ACTIONS_RUNTIME_TOKEN, AWS_SECRET_ACCESS_KEY. This is to prevent leaking secrets to the final image. The secrets are +# loaded using docker buildx `--secret` flag and need to be explicitly mounted with `--mount=type=secret,id=SECRET_ID`. FROM deps-base AS deps-sccache @@ -139,7 +159,6 @@ RUN if [[ "$TARGETARCH" == "arm64" ]] ; then export SCC_ARCH=aarch64; else expor # # Configure sccache # -ARG RUSTC_WRAPPER # Disable incremental builds, not supported by sccache RUN echo 'export CARGO_INCREMENTAL=false' >> /root/env @@ -153,9 +172,7 @@ ARG SCCACHE_MEMCACHED # S3 storage ARG SCCACHE_BUCKET -ARG AWS_ACCESS_KEY_ID ARG AWS_PROFILE -ARG AWS_REGION ARG SCCACHE_REGION ARG SCCACHE_S3_KEY_PREFIX ARG SCCACHE_ENDPOINT @@ -166,52 +183,51 @@ ARG SCCACHE_ENDPOINT RUN --mount=type=secret,id=AWS <> /root/env echo "export ACTIONS_CACHE_URL=${ACTIONS_CACHE_URL}" >> /root/env # ACTIONS_RUNTIME_TOKEN is a secret so we load it on demand - echo 'export ACTIONS_RUNTIME_TOKEN="$(cat /run/secrets/ACTIONS_RUNTIME_TOKEN)"' >> /root/env + echo 'export ACTIONS_RUNTIME_TOKEN="$(cat /run/secrets/GHA)"' >> /root/env ### AWS S3 ### elif [ -n "${SCCACHE_BUCKET}" ]; then - if [ -z "${SCCACHE_REGION}" ] ; then - # Default to AWS_REGION if not set - export SCCACHE_REGION=${AWS_REGION} - fi + echo "export SCCACHE_BUCKET='${SCCACHE_BUCKET}'" >> /root/env echo "export SCCACHE_REGION='${SCCACHE_REGION}'" >> /root/env - - [ -n "${AWS_REGION}" ] && echo "export AWS_REGION='${AWS_REGION}'" >> /root/env [ -n "${AWS_PROFILE}" ] && echo "export AWS_PROFILE='${AWS_PROFILE}'" >> /root/env - echo "export SCCACHE_BUCKET='${SCCACHE_BUCKET}'" >> /root/env echo "export SCCACHE_ENDPOINT='${SCCACHE_ENDPOINT}'" >> /root/env echo "export SCCACHE_S3_KEY_PREFIX='${SCCACHE_S3_KEY_PREFIX}/${TARGETARCH}/linux-musl'" >> /root/env - echo "export AWS_SHARED_CREDENTIALS_FILE=/run/secrets/AWS" >> /root/env + # Configure AWS credentials + mkdir --mode=0700 -p "$HOME/.aws" + ln -s /run/secrets/AWS "$HOME/.aws/credentials" + echo "export AWS_SHARED_CREDENTIALS_FILE=$HOME/.aws/credentials" >> /root/env + # Check if AWS credentials file is mounted correctly, eg. --mount=type=secret,id=AWS - echo '[ -r "${AWS_SHARED_CREDENTIALS_FILE}" ] || echo "Cannot read ${AWS_SHARED_CREDENTIALS_FILE}"' >> /root/env + echo '[ -e "${AWS_SHARED_CREDENTIALS_FILE}" ] || { echo "$(id -u): Cannot read ${AWS_SHARED_CREDENTIALS_FILE}"; exit 1; }' >> /root/env + echo '[ -e "${AWS_SHARED_CREDENTIALS_FILE}" ] || ls -lR $HOME/.aws' >> /root/env ### memcached ### elif [ -n "${SCCACHE_MEMCACHED}" ]; then # memcached echo "export SCCACHE_MEMCACHED='${SCCACHE_MEMCACHED}'" >> /root/env + else + echo "Error: cannot determine sccache cache backend" >&2 + exit 1 fi - - if [ -n "${RUSTC_WRAPPER}" ]; then - echo "export CXX='${RUSTC_WRAPPER} clang++'" >> /root/env - echo "export CC='${RUSTC_WRAPPER} clang'" >> /root/env - echo "export RUSTC_WRAPPER='${RUSTC_WRAPPER}'" >> /root/env - echo "export SCCACHE_SERVER_PORT=$((RANDOM+1025))" >> /root/env - fi + + # Configure compilers to use sccache + echo "export CXX='sccache clang++'" >> /root/env + echo "export CC='sccache clang'" >> /root/env + echo "export RUSTC_WRAPPER=sccache" >> /root/env + echo "export SCCACHE_SERVER_PORT=$((RANDOM+1025))" >> /root/env + # for debugging, we display what we generated cat /root/env - - stat /run/secrets/AWS EOS # Image containing compolation dependencies; used to overcome lack of interpolation in COPY --from -FROM deps-${RUSTC_WRAPPER:-base} AS deps-compilation +FROM ${DEPS_IMAGE} AS deps-compilation # Stage intentionally left empty # @@ -223,7 +239,24 @@ FROM deps-compilation AS deps-rocksdb RUN mkdir -p /tmp/rocksdb WORKDIR /tmp/rocksdb -RUN < a.c +# sccache clang -o a.o -c a.c +# cd - + +# sccache -s +# EOS + +RUN --mount=type=secret,id=AWS < Date: Thu, 21 Nov 2024 16:52:16 +0100 Subject: [PATCH 12/51] chore: remove aws-actions/configure-aws-credentials@v4 --- .github/workflows/tests-rs-package.yml | 28 -------------------------- 1 file changed, 28 deletions(-) diff --git a/.github/workflows/tests-rs-package.yml b/.github/workflows/tests-rs-package.yml index a2305cea8b..373a05698c 100644 --- a/.github/workflows/tests-rs-package.yml +++ b/.github/workflows/tests-rs-package.yml @@ -23,13 +23,6 @@ jobs: - name: Check out repo uses: actions/checkout@v4 - - name: Configure AWS credentials and bucket region - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ vars.AWS_REGION }} - - name: Setup Rust uses: ./.github/actions/rust with: @@ -88,13 +81,6 @@ jobs: - name: Check out repo uses: actions/checkout@v4 - - name: Configure AWS credentials and bucket region - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-region: ${{ vars.AWS_REGION }} - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - - name: Setup Rust uses: ./.github/actions/rust @@ -196,13 +182,6 @@ jobs: - name: Check out repo uses: actions/checkout@v4 - - name: Configure AWS credentials and bucket region - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-region: ${{ vars.AWS_REGION }} - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - - name: Setup Rust uses: ./.github/actions/rust @@ -237,13 +216,6 @@ jobs: - name: Check out repo uses: actions/checkout@v4 - - name: Configure AWS credentials and bucket region - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-region: ${{ vars.AWS_REGION }} - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - - name: Setup Rust uses: ./.github/actions/rust From 3337f5999e7503a1a3833f1aa1278219a752c634 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Thu, 21 Nov 2024 17:30:54 +0100 Subject: [PATCH 13/51] chore: change how we configure ecr credentials --- .github/actions/aws_credentials/action.yaml | 31 +++++++++++++++++++++ .github/actions/sccache/action.yaml | 14 ++++------ .github/workflows/tests-build-image.yml | 10 +++++-- 3 files changed, 43 insertions(+), 12 deletions(-) create mode 100644 .github/actions/aws_credentials/action.yaml diff --git a/.github/actions/aws_credentials/action.yaml b/.github/actions/aws_credentials/action.yaml new file mode 100644 index 0000000000..9c890d11e5 --- /dev/null +++ b/.github/actions/aws_credentials/action.yaml @@ -0,0 +1,31 @@ +--- +name: "aws_credentials" +description: "Configure AWS credentials" +inputs: + aws_access_key_id: + description: AWS access key ID + required: true + aws_secret_access_key: + description: AWS secret access key + required: true + profile: + description: AWS profile to use + default: "default" + +runs: + using: composite + steps: + - name: Configure AWS credentials + shell: bash + run: | + mkdir -p "${HOME}/.aws" + echo "[${{ inputs.profile }}]" >> ${HOME}/.aws/credentials + echo "aws_access_key_id=${{ inputs.aws_access_key_id }}" >> ${HOME}/.aws/credentials + echo "aws_secret_access_key=${{ inputs.aws_secret_access_key }}" >> ${HOME}/.aws/credentials + chmod -R go-rwx ${HOME}/.aws + + - name: Set env variables + shell: bash + run: | + echo "AWS_PROFILE=sccache" >> $GITHUB_ENV + echo "AWS_SHARED_CREDENTIALS_FILE=${HOME}/.aws/credentials" >> $GITHUB_ENV diff --git a/.github/actions/sccache/action.yaml b/.github/actions/sccache/action.yaml index ac9b218aff..e1cdd29ca0 100644 --- a/.github/actions/sccache/action.yaml +++ b/.github/actions/sccache/action.yaml @@ -32,13 +32,11 @@ runs: version: "v0.8.2" # Must be the same as in Dockerfile - name: Configure AWS credentials - shell: bash - run: | - mkdir -p "${HOME}/.aws" - echo "[sccache]" >> ${HOME}/.aws/credentials - echo "aws_access_key_id=${{ inputs.aws_access_key_id }}" >> ${HOME}/.aws/credentials - echo "aws_secret_access_key=${{ inputs.aws_secret_access_key }}" >> ${HOME}/.aws/credentials - chmod -R go-rwx ${HOME}/.aws + uses: ./.github/actions/aws_credentials + with: + aws_access_key_id: ${{ inputs.aws_access_key_id }} + aws_secret_access_key: ${{ inputs.aws_secret_access_key }} + profile: "sccache" - name: Configure sccache shell: bash @@ -49,5 +47,3 @@ runs: echo "SCCACHE_REGION=${{ inputs.region }}" >> $GITHUB_ENV echo "SCCACHE_ENDPOINT=${{ inputs.endpoint }}" >> $GITHUB_ENV echo "SCCACHE_S3_KEY_PREFIX=${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu" >> $GITHUB_ENV - echo "AWS_PROFILE=sccache" >> $GITHUB_ENV - echo "AWS_SHARED_CREDENTIALS_FILE=${HOME}/.aws/credentials" >> $GITHUB_ENV diff --git a/.github/workflows/tests-build-image.yml b/.github/workflows/tests-build-image.yml index 4e86735f71..93508010ad 100644 --- a/.github/workflows/tests-build-image.yml +++ b/.github/workflows/tests-build-image.yml @@ -26,14 +26,18 @@ jobs: fetch-depth: 0 - name: Configure AWS credentials and bucket region - uses: aws-actions/configure-aws-credentials@v4 + uses: ./.github/actions/aws_credentials with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ vars.AWS_REGION }} + profile: ecr - name: Login to ECR - run: aws ecr get-login-password --region ${{ vars.AWS_REGION }} | docker login --username AWS --password-stdin ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_REGION }}.amazonaws.com + run: | + aws ecr get-login-password \ + --region ${{ vars.AWS_REGION }} \ + --profile ecr | docker login --username AWS --password-stdin ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_REGION }}.amazonaws.com + shell: bash - name: Build and push by SHA From 275fac12ca8c02fc26a0cf59166ac617394bd997 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Thu, 21 Nov 2024 17:33:22 +0100 Subject: [PATCH 14/51] chore: add some debug, to be reverted --- .github/workflows/tests-build-image.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/tests-build-image.yml b/.github/workflows/tests-build-image.yml index 93508010ad..8329c9f889 100644 --- a/.github/workflows/tests-build-image.yml +++ b/.github/workflows/tests-build-image.yml @@ -25,6 +25,16 @@ jobs: with: fetch-depth: 0 + - name: Check region + shell: bash + run: | + if [ "${{ vars.AWS_REGION }}" == "${{ secrets.AWS_REGION }}"]; then + echo "AWS_REGION is OK"; + else + echo "Wrong region"; + exit 1; + fi + - name: Configure AWS credentials and bucket region uses: ./.github/actions/aws_credentials with: From 7d3a679c35b5db550954b113546429c2286d0eb8 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Thu, 21 Nov 2024 17:40:09 +0100 Subject: [PATCH 15/51] ci: rs-crates-security sccache --- .github/workflows/tests.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 795cefe955..ea1df9b1c5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -85,12 +85,22 @@ jobs: rs-crates-security: name: Rust crates security audit + environment: test if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || !github.event.pull_request.draft }} runs-on: ubuntu-24.04 steps: - name: Check out repo uses: actions/checkout@v4 + - name: Setup sccache + uses: ./.github/actions/sccache + with: + bucket: ${{ vars.CACHE_S3_BUCKET }} + region: ${{ vars.CACHE_REGION }} + endpoint: ${{ vars.CACHE_S3_ENDPOINT }} + aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + - name: Audit crates uses: rustsec/audit-check@v1 with: From 73fb938ef6bbb17118e9707e8e9d097c625f5f86 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Thu, 21 Nov 2024 17:41:08 +0100 Subject: [PATCH 16/51] chore: typo --- .github/workflows/tests-build-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests-build-image.yml b/.github/workflows/tests-build-image.yml index 8329c9f889..59103c480d 100644 --- a/.github/workflows/tests-build-image.yml +++ b/.github/workflows/tests-build-image.yml @@ -28,7 +28,7 @@ jobs: - name: Check region shell: bash run: | - if [ "${{ vars.AWS_REGION }}" == "${{ secrets.AWS_REGION }}"]; then + if [ "${{ vars.AWS_REGION }}" == "${{ secrets.AWS_REGION }}" ]; then echo "AWS_REGION is OK"; else echo "Wrong region"; From 4dbff395adfe2e4ef23e36f355b4639930c391b2 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Thu, 21 Nov 2024 17:46:32 +0100 Subject: [PATCH 17/51] chore: debug sccache in gha --- .github/workflows/tests-rs-package.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/tests-rs-package.yml b/.github/workflows/tests-rs-package.yml index 373a05698c..095227edc2 100644 --- a/.github/workflows/tests-rs-package.yml +++ b/.github/workflows/tests-rs-package.yml @@ -246,6 +246,10 @@ jobs: run: | echo Verify all features disabled set -ex + set | grep AWS + cat $AWS_SHARED_CREDENTIALS_FILE + sccache --start-server + features="${{ steps.crate_info.outputs.features }}" fails="" RUSTFLAGS="-D warnings" From 6e8b28d2e4065e53edcf8c16b1c0870915121c03 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Thu, 21 Nov 2024 18:01:05 +0100 Subject: [PATCH 18/51] chore: fix keys --- .github/actions/aws_credentials/action.yaml | 8 ++++---- .github/actions/docker/action.yaml | 14 ++++++-------- .github/actions/sccache/action.yaml | 4 ++-- .github/workflows/tests-build-image.yml | 19 ++++--------------- .github/workflows/tests-rs-package.yml | 16 ++++++++-------- .github/workflows/tests.yml | 4 ++-- 6 files changed, 26 insertions(+), 39 deletions(-) diff --git a/.github/actions/aws_credentials/action.yaml b/.github/actions/aws_credentials/action.yaml index 9c890d11e5..a730b000b2 100644 --- a/.github/actions/aws_credentials/action.yaml +++ b/.github/actions/aws_credentials/action.yaml @@ -2,10 +2,10 @@ name: "aws_credentials" description: "Configure AWS credentials" inputs: - aws_access_key_id: + access_key_id: description: AWS access key ID required: true - aws_secret_access_key: + secret_access_key: description: AWS secret access key required: true profile: @@ -20,8 +20,8 @@ runs: run: | mkdir -p "${HOME}/.aws" echo "[${{ inputs.profile }}]" >> ${HOME}/.aws/credentials - echo "aws_access_key_id=${{ inputs.aws_access_key_id }}" >> ${HOME}/.aws/credentials - echo "aws_secret_access_key=${{ inputs.aws_secret_access_key }}" >> ${HOME}/.aws/credentials + echo "aws_access_key_id=${{ inputs.access_key_id }}" >> ${HOME}/.aws/credentials + echo "aws_secret_access_key=${{ inputs.secret_access_key }}" >> ${HOME}/.aws/credentials chmod -R go-rwx ${HOME}/.aws - name: Set env variables diff --git a/.github/actions/docker/action.yaml b/.github/actions/docker/action.yaml index f566883006..39646cf5cd 100644 --- a/.github/actions/docker/action.yaml +++ b/.github/actions/docker/action.yaml @@ -34,11 +34,11 @@ inputs: endpoint: description: S3 endpoint to use for caching required: true - aws_access_key_id: - description: AWS access key ID + cache_access_key_id: + description: Access key ID for s3 cache required: true - aws_secret_access_key: - description: AWS secret access key + cache_secret_access_key: + description: Secret access key for s3 cache required: true cache_to_name: description: "Save cache to name manifest (should be used only on default branch)" @@ -159,8 +159,8 @@ runs: bucket: ${{ inputs.bucket }} region: ${{ inputs.region }} endpoint: ${{ inputs.endpoint }} - aws_access_key_id: ${{ inputs.aws_access_key_id }} - aws_secret_access_key: ${{ inputs.aws_secret_access_key }} + access_key_id: ${{ inputs.cache_access_key_id }} + secret_access_key: ${{ inputs.cache_secret_access_key }} install: false - name: Build and push Docker image ${{ inputs.image }} @@ -176,8 +176,6 @@ runs: platforms: ${{ inputs.platform }} secret-files: | AWS=${{ env.HOME }}/.aws/credentials - # secrets: | - # AWS_SECRET_ACCESS_KEY=${{ inputs.aws_secret_access_key }} build-args: | CARGO_BUILD_PROFILE=${{ inputs.cargo_profile }} SCCACHE_BUCKET=${{ inputs.bucket }} diff --git a/.github/actions/sccache/action.yaml b/.github/actions/sccache/action.yaml index e1cdd29ca0..f4ccef5bf5 100644 --- a/.github/actions/sccache/action.yaml +++ b/.github/actions/sccache/action.yaml @@ -34,8 +34,8 @@ runs: - name: Configure AWS credentials uses: ./.github/actions/aws_credentials with: - aws_access_key_id: ${{ inputs.aws_access_key_id }} - aws_secret_access_key: ${{ inputs.aws_secret_access_key }} + access_key_id: ${{ inputs.aws_access_key_id }} + secret_access_key: ${{ inputs.aws_secret_access_key }} profile: "sccache" - name: Configure sccache diff --git a/.github/workflows/tests-build-image.yml b/.github/workflows/tests-build-image.yml index 59103c480d..ecfeac71b1 100644 --- a/.github/workflows/tests-build-image.yml +++ b/.github/workflows/tests-build-image.yml @@ -25,29 +25,18 @@ jobs: with: fetch-depth: 0 - - name: Check region - shell: bash - run: | - if [ "${{ vars.AWS_REGION }}" == "${{ secrets.AWS_REGION }}" ]; then - echo "AWS_REGION is OK"; - else - echo "Wrong region"; - exit 1; - fi - - name: Configure AWS credentials and bucket region - uses: ./.github/actions/aws_credentials + uses: aws-actions/configure-aws-credentials@v4 with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - profile: ecr + aws-region: ${{ secrets.AWS_REGION }} - name: Login to ECR run: | aws ecr get-login-password \ --region ${{ vars.AWS_REGION }} \ --profile ecr | docker login --username AWS --password-stdin ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_REGION }}.amazonaws.com - shell: bash - name: Build and push by SHA @@ -63,6 +52,6 @@ jobs: region: ${{ vars.CACHE_REGION }} bucket: ${{ vars.CACHE_S3_BUCKET }} endpoint: ${{ vars.CACHE_S3_ENDPOINT }} - aws_access_key_id: ${{ secrets.CACHE_KEY_ID }} - aws_secret_access_key: ${{ secrets.CACHE_SECRET_KEY }} + cache_access_key_id: ${{ secrets.CACHE_KEY_ID }} + cache_secret_access_key: ${{ secrets.CACHE_SECRET_KEY }} cache_to_name: ${{ github.event_name == 'push' && 'true' || 'false' }} diff --git a/.github/workflows/tests-rs-package.yml b/.github/workflows/tests-rs-package.yml index 095227edc2..240af6f1e6 100644 --- a/.github/workflows/tests-rs-package.yml +++ b/.github/workflows/tests-rs-package.yml @@ -34,8 +34,8 @@ jobs: bucket: ${{ vars.CACHE_S3_BUCKET }} region: ${{ vars.CACHE_REGION }} endpoint: ${{ vars.CACHE_S3_ENDPOINT }} - aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws_access_key_id: ${{ secrets.CACHE_KEY_ID }} + aws_secret_access_key: ${{ secrets.CACHE_SECRET_KEY }} - name: Install librocksdb uses: ./.github/actions/librocksdb @@ -90,8 +90,8 @@ jobs: bucket: ${{ vars.CACHE_S3_BUCKET }} region: ${{ vars.CACHE_REGION }} endpoint: ${{ vars.CACHE_S3_ENDPOINT }} - aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws_access_key_id: ${{ secrets.CACHE_KEY_ID }} + aws_secret_access_key: ${{ secrets.CACHE_SECRET_KEY }} - name: Install librocksdb uses: ./.github/actions/librocksdb @@ -191,8 +191,8 @@ jobs: bucket: ${{ vars.CACHE_S3_BUCKET }} region: ${{ vars.CACHE_REGION }} endpoint: ${{ vars.CACHE_S3_ENDPOINT }} - aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws_access_key_id: ${{ secrets.CACHE_KEY_ID }} + aws_secret_access_key: ${{ secrets.CACHE_SECRET_KEY }} - name: Install librocksdb uses: ./.github/actions/librocksdb @@ -225,8 +225,8 @@ jobs: bucket: ${{ vars.CACHE_S3_BUCKET }} region: ${{ vars.CACHE_REGION }} endpoint: ${{ vars.CACHE_S3_ENDPOINT }} - aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws_access_key_id: ${{ secrets.CACHE_KEY_ID }} + aws_secret_access_key: ${{ secrets.CACHE_SECRET_KEY }} - name: Install librocksdb uses: ./.github/actions/librocksdb diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ea1df9b1c5..d072c9fb4f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -98,8 +98,8 @@ jobs: bucket: ${{ vars.CACHE_S3_BUCKET }} region: ${{ vars.CACHE_REGION }} endpoint: ${{ vars.CACHE_S3_ENDPOINT }} - aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws_access_key_id: ${{ secrets.CACHE_KEY_ID }} + aws_secret_access_key: ${{ secrets.CACHE_SECRET_KEY }} - name: Audit crates uses: rustsec/audit-check@v1 From c3a7e5dbfd3b95446c1ce72f9fe9d3f961e3592d Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Thu, 21 Nov 2024 18:11:36 +0100 Subject: [PATCH 19/51] chore: testing --- .github/workflows/tests-build-image.yml | 3 +-- .github/workflows/tests-rs-package.yml | 5 +++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests-build-image.yml b/.github/workflows/tests-build-image.yml index ecfeac71b1..83f0e198aa 100644 --- a/.github/workflows/tests-build-image.yml +++ b/.github/workflows/tests-build-image.yml @@ -35,8 +35,7 @@ jobs: - name: Login to ECR run: | aws ecr get-login-password \ - --region ${{ vars.AWS_REGION }} \ - --profile ecr | docker login --username AWS --password-stdin ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_REGION }}.amazonaws.com + --region ${{ vars.AWS_REGION }} | docker login --username AWS --password-stdin ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_REGION }}.amazonaws.com shell: bash - name: Build and push by SHA diff --git a/.github/workflows/tests-rs-package.yml b/.github/workflows/tests-rs-package.yml index 240af6f1e6..f6a53508b4 100644 --- a/.github/workflows/tests-rs-package.yml +++ b/.github/workflows/tests-rs-package.yml @@ -247,8 +247,9 @@ jobs: echo Verify all features disabled set -ex set | grep AWS - cat $AWS_SHARED_CREDENTIALS_FILE - sccache --start-server + cat $HOME/.aws/credentials + SCCACHE_LOG=debug sccache --start-server + sccache --stop-server features="${{ steps.crate_info.outputs.features }}" fails="" From e77787e971fcb9e76f02e8810b54d2418b404d1b Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Fri, 22 Nov 2024 13:04:22 +0100 Subject: [PATCH 20/51] chore: sccache input rename --- .github/actions/sccache/action.yaml | 12 ++++++------ .github/workflows/tests-rs-package.yml | 16 ++++++++-------- .github/workflows/tests.yml | 4 ++-- Dockerfile | 1 - 4 files changed, 16 insertions(+), 17 deletions(-) diff --git a/.github/actions/sccache/action.yaml b/.github/actions/sccache/action.yaml index f4ccef5bf5..7844ea5edd 100644 --- a/.github/actions/sccache/action.yaml +++ b/.github/actions/sccache/action.yaml @@ -11,11 +11,11 @@ inputs: endpoint: description: S3 endpoint to use for caching required: true - aws_access_key_id: - description: AWS access key ID + access_key_id: + description: S3 endpoint access key ID required: true - aws_secret_access_key: - description: AWS secret access key + secret_access_key: + description: S3 endpoint secret access key required: true install: description: "Install sccache" @@ -34,8 +34,8 @@ runs: - name: Configure AWS credentials uses: ./.github/actions/aws_credentials with: - access_key_id: ${{ inputs.aws_access_key_id }} - secret_access_key: ${{ inputs.aws_secret_access_key }} + access_key_id: ${{ inputs.access_key_id }} + secret_access_key: ${{ inputs.secret_access_key }} profile: "sccache" - name: Configure sccache diff --git a/.github/workflows/tests-rs-package.yml b/.github/workflows/tests-rs-package.yml index f6a53508b4..7726986e5c 100644 --- a/.github/workflows/tests-rs-package.yml +++ b/.github/workflows/tests-rs-package.yml @@ -34,8 +34,8 @@ jobs: bucket: ${{ vars.CACHE_S3_BUCKET }} region: ${{ vars.CACHE_REGION }} endpoint: ${{ vars.CACHE_S3_ENDPOINT }} - aws_access_key_id: ${{ secrets.CACHE_KEY_ID }} - aws_secret_access_key: ${{ secrets.CACHE_SECRET_KEY }} + access_key_id: ${{ secrets.CACHE_KEY_ID }} + secret_access_key: ${{ secrets.CACHE_SECRET_KEY }} - name: Install librocksdb uses: ./.github/actions/librocksdb @@ -90,8 +90,8 @@ jobs: bucket: ${{ vars.CACHE_S3_BUCKET }} region: ${{ vars.CACHE_REGION }} endpoint: ${{ vars.CACHE_S3_ENDPOINT }} - aws_access_key_id: ${{ secrets.CACHE_KEY_ID }} - aws_secret_access_key: ${{ secrets.CACHE_SECRET_KEY }} + access_key_id: ${{ secrets.CACHE_KEY_ID }} + secret_access_key: ${{ secrets.CACHE_SECRET_KEY }} - name: Install librocksdb uses: ./.github/actions/librocksdb @@ -191,8 +191,8 @@ jobs: bucket: ${{ vars.CACHE_S3_BUCKET }} region: ${{ vars.CACHE_REGION }} endpoint: ${{ vars.CACHE_S3_ENDPOINT }} - aws_access_key_id: ${{ secrets.CACHE_KEY_ID }} - aws_secret_access_key: ${{ secrets.CACHE_SECRET_KEY }} + access_key_id: ${{ secrets.CACHE_KEY_ID }} + secret_access_key: ${{ secrets.CACHE_SECRET_KEY }} - name: Install librocksdb uses: ./.github/actions/librocksdb @@ -225,8 +225,8 @@ jobs: bucket: ${{ vars.CACHE_S3_BUCKET }} region: ${{ vars.CACHE_REGION }} endpoint: ${{ vars.CACHE_S3_ENDPOINT }} - aws_access_key_id: ${{ secrets.CACHE_KEY_ID }} - aws_secret_access_key: ${{ secrets.CACHE_SECRET_KEY }} + access_key_id: ${{ secrets.CACHE_KEY_ID }} + secret_access_key: ${{ secrets.CACHE_SECRET_KEY }} - name: Install librocksdb uses: ./.github/actions/librocksdb diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d072c9fb4f..164f45d4e1 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -98,8 +98,8 @@ jobs: bucket: ${{ vars.CACHE_S3_BUCKET }} region: ${{ vars.CACHE_REGION }} endpoint: ${{ vars.CACHE_S3_ENDPOINT }} - aws_access_key_id: ${{ secrets.CACHE_KEY_ID }} - aws_secret_access_key: ${{ secrets.CACHE_SECRET_KEY }} + access_key_id: ${{ secrets.CACHE_KEY_ID }} + secret_access_key: ${{ secrets.CACHE_SECRET_KEY }} - name: Audit crates uses: rustsec/audit-check@v1 diff --git a/Dockerfile b/Dockerfile index 3d5bd4a55e..d8a9167a30 100644 --- a/Dockerfile +++ b/Dockerfile @@ -205,7 +205,6 @@ RUN --mount=type=secret,id=AWS <> /root/env - echo '[ -e "${AWS_SHARED_CREDENTIALS_FILE}" ] || ls -lR $HOME/.aws' >> /root/env ### memcached ### elif [ -n "${SCCACHE_MEMCACHED}" ]; then From 9c524d26e443ce8bb302b2e720d0d9f005014b4b Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Fri, 22 Nov 2024 13:26:06 +0100 Subject: [PATCH 21/51] chore: fix layers push --- .github/actions/aws_credentials/action.yaml | 4 +++- .github/actions/s3-layer-cache-settings/action.yaml | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/actions/aws_credentials/action.yaml b/.github/actions/aws_credentials/action.yaml index a730b000b2..da4219f0c8 100644 --- a/.github/actions/aws_credentials/action.yaml +++ b/.github/actions/aws_credentials/action.yaml @@ -1,6 +1,8 @@ --- +# This file contains configuration of aws credentials file. +# Its primary use is to prepare a credentials file that will be used as a secrets mount when building Docker images. name: "aws_credentials" -description: "Configure AWS credentials" +description: "Configure .aws/credentials ${{ inputs.profile }}" inputs: access_key_id: description: AWS access key ID diff --git a/.github/actions/s3-layer-cache-settings/action.yaml b/.github/actions/s3-layer-cache-settings/action.yaml index 2529733c87..b4a0ade592 100644 --- a/.github/actions/s3-layer-cache-settings/action.yaml +++ b/.github/actions/s3-layer-cache-settings/action.yaml @@ -63,7 +63,7 @@ runs: const sanitizedHeadRef = '${{ inputs.head_ref }}'.replace(/[^a-zA-Z0-9]/g, '-'); const shaManifestName = '${{ inputs.name }}_sha_${{ github.sha }}'; - const headRefManifestName = '${{ inputs.name }}_tag_${ sanitizedHeadRef }'; + const headRefManifestName = '${{ inputs.name }}_tag_${{ sanitizedHeadRef }}'; const cacheFromManifestNames = [ shaManifestName, From e899b18840096341fca5b46960b53aaf3b99771a Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Fri, 22 Nov 2024 13:33:14 +0100 Subject: [PATCH 22/51] chore: fix --- .github/actions/aws_credentials/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/aws_credentials/action.yaml b/.github/actions/aws_credentials/action.yaml index da4219f0c8..c9d0eac7a0 100644 --- a/.github/actions/aws_credentials/action.yaml +++ b/.github/actions/aws_credentials/action.yaml @@ -2,7 +2,7 @@ # This file contains configuration of aws credentials file. # Its primary use is to prepare a credentials file that will be used as a secrets mount when building Docker images. name: "aws_credentials" -description: "Configure .aws/credentials ${{ inputs.profile }}" +description: "Configure .aws/credentials" inputs: access_key_id: description: AWS access key ID From edb040cac9a82b9e24411d6d2ad186c84e7f6a0a Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Fri, 22 Nov 2024 13:44:19 +0100 Subject: [PATCH 23/51] chore: typo --- .github/actions/aws_credentials/action.yaml | 6 +++--- .github/actions/docker/action.yaml | 2 +- .github/actions/s3-layer-cache-settings/action.yaml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/actions/aws_credentials/action.yaml b/.github/actions/aws_credentials/action.yaml index c9d0eac7a0..5084f6af10 100644 --- a/.github/actions/aws_credentials/action.yaml +++ b/.github/actions/aws_credentials/action.yaml @@ -5,10 +5,10 @@ name: "aws_credentials" description: "Configure .aws/credentials" inputs: access_key_id: - description: AWS access key ID + description: Access key ID required: true secret_access_key: - description: AWS secret access key + description: Secret access key required: true profile: description: AWS profile to use @@ -29,5 +29,5 @@ runs: - name: Set env variables shell: bash run: | - echo "AWS_PROFILE=sccache" >> $GITHUB_ENV + echo "AWS_PROFILE=${{ inputs.profile }}" >> $GITHUB_ENV echo "AWS_SHARED_CREDENTIALS_FILE=${HOME}/.aws/credentials" >> $GITHUB_ENV diff --git a/.github/actions/docker/action.yaml b/.github/actions/docker/action.yaml index 39646cf5cd..5d620ca461 100644 --- a/.github/actions/docker/action.yaml +++ b/.github/actions/docker/action.yaml @@ -26,7 +26,7 @@ inputs: description: Cargo build profile, i.e release or dev default: dev bucket: - description: S3 bucket to use for caching, must match runner define in `runs-on` + description: S3 bucket to use for caching (both sccache and layer cache) required: true region: description: S3 bucket region diff --git a/.github/actions/s3-layer-cache-settings/action.yaml b/.github/actions/s3-layer-cache-settings/action.yaml index b4a0ade592..dc375eebb3 100644 --- a/.github/actions/s3-layer-cache-settings/action.yaml +++ b/.github/actions/s3-layer-cache-settings/action.yaml @@ -63,7 +63,7 @@ runs: const sanitizedHeadRef = '${{ inputs.head_ref }}'.replace(/[^a-zA-Z0-9]/g, '-'); const shaManifestName = '${{ inputs.name }}_sha_${{ github.sha }}'; - const headRefManifestName = '${{ inputs.name }}_tag_${{ sanitizedHeadRef }}'; + const headRefManifestName = '${{ inputs.name }}_tag_' + sanitizedHeadRef; const cacheFromManifestNames = [ shaManifestName, From 806225f21e28b9c312b9b581b3a22c4060720642 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Fri, 22 Nov 2024 13:47:02 +0100 Subject: [PATCH 24/51] build: Dockerfile cargo incremental --- Dockerfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index d8a9167a30..5d658d9ebd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -160,9 +160,6 @@ RUN if [[ "$TARGETARCH" == "arm64" ]] ; then export SCC_ARCH=aarch64; else expor # Configure sccache # -# Disable incremental builds, not supported by sccache -RUN echo 'export CARGO_INCREMENTAL=false' >> /root/env - # Set args below to use Github Actions cache; see https://github.com/mozilla/sccache/blob/main/docs/GHA.md ARG SCCACHE_GHA_ENABLED ARG ACTIONS_CACHE_URL @@ -215,12 +212,15 @@ RUN --mount=type=secret,id=AWS <> /root/env + # Configure compilers to use sccache echo "export CXX='sccache clang++'" >> /root/env echo "export CC='sccache clang'" >> /root/env echo "export RUSTC_WRAPPER=sccache" >> /root/env - echo "export SCCACHE_SERVER_PORT=$((RANDOM+1025))" >> /root/env - + # Disable Rust incremental builds, not supported by sccache + echo 'export CARGO_INCREMENTAL=0' >> /root/env + # for debugging, we display what we generated cat /root/env EOS From 12fe70b11c1daad744a2afa05e540f5b96924089 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Mon, 25 Nov 2024 09:25:56 +0100 Subject: [PATCH 25/51] refactor(docker): some renames --- .github/actions/docker/action.yaml | 24 ++++++++++++------------ .github/workflows/tests-build-image.yml | 6 +++--- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/actions/docker/action.yaml b/.github/actions/docker/action.yaml index 5d620ca461..97e2f0995f 100644 --- a/.github/actions/docker/action.yaml +++ b/.github/actions/docker/action.yaml @@ -25,13 +25,13 @@ inputs: cargo_profile: description: Cargo build profile, i.e release or dev default: dev - bucket: + cache_bucket: description: S3 bucket to use for caching (both sccache and layer cache) required: true - region: + cache_region: description: S3 bucket region required: true - endpoint: + cache_endpoint: description: S3 endpoint to use for caching required: true cache_access_key_id: @@ -84,9 +84,9 @@ runs: id: layer_cache_settings with: name: ${{ inputs.image_name }} - region: ${{ inputs.region }} - bucket: ${{ inputs.bucket }} - endpoint: ${{ inputs.endpoint }} + region: ${{ inputs.cache_region }} + bucket: ${{ inputs.cache_bucket }} + endpoint: ${{ inputs.cache_endpoint }} cache_to_name: ${{ inputs.cache_to_name }} - name: Set HOME variable to github context @@ -156,9 +156,9 @@ runs: - name: Setup sccache vars uses: ./.github/actions/sccache with: - bucket: ${{ inputs.bucket }} - region: ${{ inputs.region }} - endpoint: ${{ inputs.endpoint }} + bucket: ${{ inputs.cache_bucket }} + region: ${{ inputs.cache_region }} + endpoint: ${{ inputs.cache_endpoint }} access_key_id: ${{ inputs.cache_access_key_id }} secret_access_key: ${{ inputs.cache_secret_access_key }} install: false @@ -178,9 +178,9 @@ runs: AWS=${{ env.HOME }}/.aws/credentials build-args: | CARGO_BUILD_PROFILE=${{ inputs.cargo_profile }} - SCCACHE_BUCKET=${{ inputs.bucket }} - SCCACHE_REGION=${{ inputs.region }} - SCCACHE_ENDPOINT=${{ inputs.endpoint }} + SCCACHE_BUCKET=${{ inputs.cache_bucket }} + SCCACHE_REGION=${{ inputs.cache_region }} + SCCACHE_ENDPOINT=${{ inputs.cache_endpoint }} SCCACHE_S3_KEY_PREFIX=${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu AWS_PROFILE=sccache cache-from: ${{ steps.layer_cache_settings.outputs.cache_from }} diff --git a/.github/workflows/tests-build-image.yml b/.github/workflows/tests-build-image.yml index 83f0e198aa..a7f1945347 100644 --- a/.github/workflows/tests-build-image.yml +++ b/.github/workflows/tests-build-image.yml @@ -48,9 +48,9 @@ jobs: push_tags: true dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }} dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }} - region: ${{ vars.CACHE_REGION }} - bucket: ${{ vars.CACHE_S3_BUCKET }} - endpoint: ${{ vars.CACHE_S3_ENDPOINT }} + cache_region: ${{ vars.CACHE_REGION }} + cache_bucket: ${{ vars.CACHE_S3_BUCKET }} + cache_endpoint: ${{ vars.CACHE_S3_ENDPOINT }} cache_access_key_id: ${{ secrets.CACHE_KEY_ID }} cache_secret_access_key: ${{ secrets.CACHE_SECRET_KEY }} cache_to_name: ${{ github.event_name == 'push' && 'true' || 'false' }} From a42f95df1cacb4aed6b1dcc0f8cec18439579e18 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Mon, 25 Nov 2024 09:41:05 +0100 Subject: [PATCH 26/51] chore: try to unset credentials --- .github/workflows/tests-build-image.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/tests-build-image.yml b/.github/workflows/tests-build-image.yml index a7f1945347..9eb48cdcad 100644 --- a/.github/workflows/tests-build-image.yml +++ b/.github/workflows/tests-build-image.yml @@ -38,6 +38,11 @@ jobs: --region ${{ vars.AWS_REGION }} | docker login --username AWS --password-stdin ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_REGION }}.amazonaws.com shell: bash + - name: Unset AWS credentials to avoid conflicts + uses: aws-actions/configure-aws-credentials@v4 + with: + unset-current-credentials: true + - name: Build and push by SHA uses: ./.github/actions/docker with: From 517a83f24b18dc8980a8a223083aeb6c7f57a8a9 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Mon, 25 Nov 2024 09:44:36 +0100 Subject: [PATCH 27/51] chore: unset aws creds --- .github/workflows/tests-build-image.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests-build-image.yml b/.github/workflows/tests-build-image.yml index 9eb48cdcad..106174aa50 100644 --- a/.github/workflows/tests-build-image.yml +++ b/.github/workflows/tests-build-image.yml @@ -39,9 +39,11 @@ jobs: shell: bash - name: Unset AWS credentials to avoid conflicts - uses: aws-actions/configure-aws-credentials@v4 - with: - unset-current-credentials: true + shell: bash + run: | + set | grep AWS + unset AWS_ACCESS_KEY_ID + unset AWS_SECRET_ACCESS_KEY - name: Build and push by SHA uses: ./.github/actions/docker From a470e8eeae8a6f27919ec9896621a48f97791344 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Mon, 25 Nov 2024 09:53:35 +0100 Subject: [PATCH 28/51] chore: unset aws creds, continued --- .github/workflows/tests-build-image.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests-build-image.yml b/.github/workflows/tests-build-image.yml index 106174aa50..5d7716bde2 100644 --- a/.github/workflows/tests-build-image.yml +++ b/.github/workflows/tests-build-image.yml @@ -41,9 +41,10 @@ jobs: - name: Unset AWS credentials to avoid conflicts shell: bash run: | - set | grep AWS - unset AWS_ACCESS_KEY_ID - unset AWS_SECRET_ACCESS_KEY + echo AWS_DEFAULT_REGION='' >> $GITHUB_ENV + echo AWS_REGION='' >> $GITHUB_ENV + echo AWS_ACCESS_KEY_ID='' >> $GITHUB_ENV + echo AWS_SECRET_ACCESS_KEY='' >> $GITHUB_ENV - name: Build and push by SHA uses: ./.github/actions/docker From e6b4de8e426c8b4164572e55cb8d1f3735f239a1 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Mon, 25 Nov 2024 10:02:30 +0100 Subject: [PATCH 29/51] chore: unset creds moved to docker --- .github/actions/docker/action.yaml | 9 +++++++++ .github/workflows/tests-build-image.yml | 8 -------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/actions/docker/action.yaml b/.github/actions/docker/action.yaml index 97e2f0995f..3d43b62124 100644 --- a/.github/actions/docker/action.yaml +++ b/.github/actions/docker/action.yaml @@ -163,6 +163,15 @@ runs: secret_access_key: ${{ inputs.cache_secret_access_key }} install: false + # Unset AWS credentials to avoid conflicts, as we use credentials in ~/.aws/credentials generated in sccache action. + - name: Unset AWS credentials to avoid conflicts + shell: bash + run: | + echo AWS_DEFAULT_REGION='' >> $GITHUB_ENV + echo AWS_REGION='' >> $GITHUB_ENV + echo AWS_ACCESS_KEY_ID='' >> $GITHUB_ENV + echo AWS_SECRET_ACCESS_KEY='' >> $GITHUB_ENV + - name: Build and push Docker image ${{ inputs.image }} id: docker_build uses: docker/build-push-action@v6 diff --git a/.github/workflows/tests-build-image.yml b/.github/workflows/tests-build-image.yml index 5d7716bde2..a7f1945347 100644 --- a/.github/workflows/tests-build-image.yml +++ b/.github/workflows/tests-build-image.yml @@ -38,14 +38,6 @@ jobs: --region ${{ vars.AWS_REGION }} | docker login --username AWS --password-stdin ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_REGION }}.amazonaws.com shell: bash - - name: Unset AWS credentials to avoid conflicts - shell: bash - run: | - echo AWS_DEFAULT_REGION='' >> $GITHUB_ENV - echo AWS_REGION='' >> $GITHUB_ENV - echo AWS_ACCESS_KEY_ID='' >> $GITHUB_ENV - echo AWS_SECRET_ACCESS_KEY='' >> $GITHUB_ENV - - name: Build and push by SHA uses: ./.github/actions/docker with: From 4046a2f6017ccdd4d2b04d96535f6514bc6e5216 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Mon, 25 Nov 2024 10:30:59 +0100 Subject: [PATCH 30/51] chore: update all other files --- .github/workflows/release-docker-image.yml | 15 ++++------- .github/workflows/release.yml | 26 ++++++++++--------- .github/workflows/tests-build-image.yml | 1 + .github/workflows/tests-build-js.yml | 19 +++++++++----- .github/workflows/tests-codeql.yml | 3 ++- .github/workflows/tests-dashmate.yml | 7 +++-- .github/workflows/tests-js-package.yml | 5 ++-- .../workflows/tests-packges-functional.yml | 6 ++--- .github/workflows/tests-test-suite.yml | 10 +++---- 9 files changed, 48 insertions(+), 44 deletions(-) diff --git a/.github/workflows/release-docker-image.yml b/.github/workflows/release-docker-image.yml index 54e4cb465a..6b0c1adb7b 100644 --- a/.github/workflows/release-docker-image.yml +++ b/.github/workflows/release-docker-image.yml @@ -47,13 +47,6 @@ jobs: with: fetch-depth: 0 - - name: Configure AWS credentials and bucket region - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_REGION }} - - name: Build and push by digest uses: ./.github/actions/docker id: docker_build @@ -66,9 +59,11 @@ jobs: cargo_profile: ${{ inputs.cargo_profile }} dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }} dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }} - region: ${{ secrets.AWS_REGION }} - aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + cache_region: ${{ vars.CACHE_REGION }} + cache_bucket: ${{ vars.CACHE_S3_BUCKET }} + cache_endpoint: ${{ vars.CACHE_S3_ENDPOINT }} + cache_access_key_id: ${{ secrets.CACHE_KEY_ID }} + cache_secret_access_key: ${{ secrets.CACHE_SECRET_KEY }} - name: Export digest run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 86476d082a..c74070294b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -33,13 +33,6 @@ jobs: env: TAG_PREFIX: v - - name: Configure AWS credentials and bucket region - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_REGION }} - - uses: softwareforgood/check-artifact-v4-existence@v0 id: check-artifact with: @@ -58,6 +51,17 @@ jobs: target: wasm32-unknown-unknown if: ${{ steps.check-artifact.outputs.exists != 'true' }} + - name: Setup sccache + uses: ./.github/actions/sccache + with: + bucket: ${{ vars.CACHE_S3_BUCKET }} + region: ${{ vars.AWS_REGION }} + endpoint: ${{ vars.CACHE_S3_ENDPOINT }} + access_key_id: ${{ secrets.CACHE_KEY_ID }} + secret_access_key: ${{ secrets.CACHE_SECRET_KEY }} + + if: ${{ steps.check-artifact.outputs.exists != 'true' }} + - name: Setup Node.JS uses: ./.github/actions/nodejs @@ -73,10 +77,7 @@ jobs: run: yarn build env: CARGO_BUILD_PROFILE: release - RUSTC_WRAPPER: sccache - SCCACHE_BUCKET: multi-runner-cache-x1xibo9c - SCCACHE_REGION: ${{ secrets.AWS_REGION }} - SCCACHE_S3_KEY_PREFIX: ${{ runner.os }}/sccache/wasm/wasm32 + if: ${{ steps.check-artifact.outputs.exists != 'true' }} - name: Set suffix @@ -230,10 +231,11 @@ jobs: with: fetch-depth: 0 + # TODO: Do we still need this? - name: Configure AWS credentials and bucket region uses: aws-actions/configure-aws-credentials@v4 with: - aws-region: ${{ secrets.AWS_REGION }} + aws-region: ${{ vars.AWS_REGION }} aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/tests-build-image.yml b/.github/workflows/tests-build-image.yml index a7f1945347..3e92319cc4 100644 --- a/.github/workflows/tests-build-image.yml +++ b/.github/workflows/tests-build-image.yml @@ -25,6 +25,7 @@ jobs: with: fetch-depth: 0 + # AWS credentials only needed for ECR login - name: Configure AWS credentials and bucket region uses: aws-actions/configure-aws-credentials@v4 with: diff --git a/.github/workflows/tests-build-js.yml b/.github/workflows/tests-build-js.yml index 83909e0c31..0c726c5797 100644 --- a/.github/workflows/tests-build-js.yml +++ b/.github/workflows/tests-build-js.yml @@ -11,7 +11,7 @@ jobs: with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_REGION }} + aws-region: ${{ vars.AWS_REGION }} - uses: softwareforgood/check-artifact-v4-existence@v0 id: check-artifact @@ -41,6 +41,16 @@ jobs: target: wasm32-unknown-unknown if: ${{ steps.check-artifact.outputs.exists != 'true' }} + - name: Setup sccache + uses: ./.github/actions/sccache + with: + bucket: ${{ vars.CACHE_S3_BUCKET }} + region: ${{ vars.CACHE_REGION }} + endpoint: ${{ vars.CACHE_S3_ENDPOINT }} + access_key_id: ${{ secrets.CACHE_KEY_ID }} + secret_access_key: ${{ secrets.CACHE_SECRET_KEY }} + if: ${{ steps.check-artifact.outputs.exists != 'true' }} + - name: Install Cargo binstall uses: cargo-bins/cargo-binstall@v1.3.1 if: ${{ steps.check-artifact.outputs.exists != 'true' }} @@ -51,11 +61,7 @@ jobs: - name: Build JS packages run: yarn build - env: - RUSTC_WRAPPER: sccache - SCCACHE_BUCKET: multi-runner-cache-x1xibo9c - SCCACHE_REGION: ${{ secrets.AWS_REGION }} - SCCACHE_S3_KEY_PREFIX: ${{ runner.os }}/sccache/wasm/wasm32 + if: ${{ steps.check-artifact.outputs.exists != 'true' }} - name: Ignore only already cached artifacts @@ -87,4 +93,3 @@ jobs: if-no-files-found: error include-hidden-files: true if: ${{ steps.check-artifact.outputs.exists != 'true' }} - diff --git a/.github/workflows/tests-codeql.yml b/.github/workflows/tests-codeql.yml index 034fd545b8..2e39f97ef5 100644 --- a/.github/workflows/tests-codeql.yml +++ b/.github/workflows/tests-codeql.yml @@ -20,12 +20,13 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} + # TODO do we still need this? - name: Configure AWS credentials and bucket region uses: aws-actions/configure-aws-credentials@v4 with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_REGION }} + aws-region: ${{ vars.AWS_REGION }} - name: Setup Node.JS uses: ./.github/actions/nodejs diff --git a/.github/workflows/tests-dashmate.yml b/.github/workflows/tests-dashmate.yml index a451381b9c..6c681ae45c 100644 --- a/.github/workflows/tests-dashmate.yml +++ b/.github/workflows/tests-dashmate.yml @@ -36,7 +36,7 @@ jobs: with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_REGION }} + aws-region: ${{ vars.CACHE_REGION }} - name: Setup Node.JS uses: ./.github/actions/nodejs @@ -52,8 +52,8 @@ jobs: set -x # Login to ECR - DOCKER_HUB_ORG="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com" - aws ecr get-login-password --region ${{ secrets.AWS_REGION }} | docker login --username AWS --password-stdin $DOCKER_HUB_ORG + DOCKER_HUB_ORG="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_REGION }}.amazonaws.com" + aws ecr get-login-password --region ${{ vars.AWS_REGION }} | docker login --username AWS --password-stdin $DOCKER_HUB_ORG SHA_TAG=sha-${{ github.sha }} @@ -115,4 +115,3 @@ jobs: - name: Show Docker logs if: ${{ failure() }} uses: jwalton/gh-docker-logs@v2 - diff --git a/.github/workflows/tests-js-package.yml b/.github/workflows/tests-js-package.yml index e71d9d85d1..c403618d3e 100644 --- a/.github/workflows/tests-js-package.yml +++ b/.github/workflows/tests-js-package.yml @@ -30,7 +30,7 @@ jobs: with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_REGION }} + aws-region: ${{ vars.AWS_REGION }} - name: Setup Node.JS uses: ./.github/actions/nodejs @@ -57,10 +57,11 @@ jobs: with: fetch-depth: 0 + # TODO: Do we still need this? - name: Configure AWS credentials and bucket region uses: aws-actions/configure-aws-credentials@v4 with: - aws-region: ${{ secrets.AWS_REGION }} + aws-region: ${{ vars.AWS_REGION }} aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/tests-packges-functional.yml b/.github/workflows/tests-packges-functional.yml index 39db41b53f..811b3a7690 100644 --- a/.github/workflows/tests-packges-functional.yml +++ b/.github/workflows/tests-packges-functional.yml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 15 env: - ECR_HOST: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com + ECR_HOST: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_REGION }}.amazonaws.com steps: - name: Check out repo uses: actions/checkout@v4 @@ -25,10 +25,10 @@ jobs: with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_REGION }} + aws-region: ${{ vars.AWS_REGION }} - name: Login to ECR - run: aws ecr get-login-password --region ${{ secrets.AWS_REGION }} | docker login --username AWS --password-stdin ${{ env.ECR_HOST }} + run: aws ecr get-login-password --region ${{ vars.AWS_REGION }} | docker login --username AWS --password-stdin ${{ env.ECR_HOST }} - name: Start local network uses: ./.github/actions/local-network diff --git a/.github/workflows/tests-test-suite.yml b/.github/workflows/tests-test-suite.yml index e9107b5e60..e5c8babba5 100644 --- a/.github/workflows/tests-test-suite.yml +++ b/.github/workflows/tests-test-suite.yml @@ -25,7 +25,7 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 15 env: - ECR_HOST: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com + ECR_HOST: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_REGION }}.amazonaws.com steps: - name: Check out repo uses: actions/checkout@v4 @@ -43,10 +43,10 @@ jobs: with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_REGION }} + aws-region: ${{ vars.AWS_REGION }} - name: Login to ECR - run: aws ecr get-login-password --region ${{ secrets.AWS_REGION }} | docker login --username AWS --password-stdin ${{ env.ECR_HOST }} + run: aws ecr get-login-password --region ${{ vars.AWS_REGION }} | docker login --username AWS --password-stdin ${{ env.ECR_HOST }} - name: Start local network uses: ./.github/actions/local-network @@ -56,8 +56,8 @@ jobs: - name: Run test suite run: yarn workspace @dashevo/platform-test-suite ${{ inputs.command }} env: - BROWSER_TEST_BATCH_INDEX: ${{ inputs.batch_index }} - BROWSER_TEST_BATCH_TOTAL: ${{ inputs.batch_total }} + BROWSER_TEST_BATCH_INDEX: ${{ inputs.batch_index }} + BROWSER_TEST_BATCH_TOTAL: ${{ inputs.batch_total }} - name: Show Docker logs if: ${{ failure() }} From 88c2f0a103b6dd39bb81435ba86b781d65d624cb Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Mon, 25 Nov 2024 10:31:45 +0100 Subject: [PATCH 31/51] Revert "test: comment out some tests to speed up testing - to be reverted" This reverts commit 69be7f64bc104bd94ed4de103ff8cc73792b921a. --- .github/workflows/tests.yml | 256 ++++++++++++++++++------------------ 1 file changed, 126 insertions(+), 130 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 164f45d4e1..488ce8f59d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -53,13 +53,12 @@ jobs: - name: Drive image_name: drive target: drive-abci - # TODO: uncomment after testing - # - name: DAPI - # image_name: dapi - # target: dapi - # - name: Dashmate helper - # image_name: dashmate-helper - # target: dashmate-helper + - name: DAPI + image_name: dapi + target: dapi + - name: Dashmate helper + image_name: dashmate-helper + target: dashmate-helper uses: ./.github/workflows/tests-build-image.yml with: name: ${{ matrix.name }} @@ -74,10 +73,7 @@ jobs: strategy: fail-fast: false matrix: - # TODO: uncomment after testing - # rs-package: ${{ fromJson(needs.changes.outputs.rs-packages) }} - rs-package: - - dpp + rs-package: ${{ fromJson(needs.changes.outputs.rs-packages) }} uses: ./.github/workflows/tests-rs-package.yml with: package: ${{ matrix.rs-package }} @@ -105,122 +101,122 @@ jobs: uses: rustsec/audit-check@v1 with: token: ${{ secrets.GITHUB_TOKEN }} - # TODO: uncomment after testing - # js-packages: - # name: JS packages - # needs: - # - changes - # - build-js - # secrets: inherit - # strategy: - # fail-fast: false - # matrix: - # js-package: ${{ fromJson(needs.changes.outputs.js-packages) }} - # uses: ./.github/workflows/tests-js-package.yml - # with: - # package: ${{ matrix.js-package }} - # test-command: ${{ matrix.js-package == 'dashmate' && 'test:unit' || 'test' }} - # skip-tests: ${{ contains(matrix.js-package, 'platform-test-suite') }} - - # js-deps-versions: - # name: JS dependency versions check - # if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || !github.event.pull_request.draft }} - # runs-on: ubuntu-24.04 - # steps: - # - name: Check out repo - # uses: actions/checkout@v4 - - # - name: Setup Node.JS - # uses: actions/setup-node@v4 - # with: - # node-version: "20" - - # - name: Enable corepack - # run: corepack enable - - # - name: Validate workspaces - # run: yarn constraints - - # js-npm-security: - # name: JS NPM security audit - # if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || !github.event.pull_request.draft }} - # runs-on: ubuntu-24.04 - # steps: - # - name: Check out repo - # uses: actions/checkout@v4 - - # - name: Enable corepack - # run: corepack enable - - # - name: Audit NPM - # run: yarn npm audit --environment production --all --recursive - - # js-codeql: - # name: JS code security audit - # needs: build-js - # secrets: inherit - # uses: ./.github/workflows/tests-codeql.yml - - # dashmate-e2e-tests: - # name: Dashmate E2E tests - # secrets: inherit - # needs: - # - changes - # - build-js - # - build-images - # strategy: - # fail-fast: false - # matrix: - # include: - # - name: Local network - # test-pattern: test/e2e/localNetwork.spec.js - # restore_local_network_data: true - # - name: Testnet fullnode - # test-pattern: test/e2e/testnetFullnode.spec.js - # restore_local_network_data: false - # - name: Testnet Evonode - # test-pattern: test/e2e/testnetEvonode.spec.js - # restore_local_network_data: false - # uses: ./.github/workflows/tests-dashmate.yml - # with: - # name: ${{ matrix.name }} - # test-pattern: ${{ matrix.test-pattern }} - # restore_local_network_data: ${{ matrix.restore_local_network_data }} - # if: contains(needs.changes.outputs.js-packages, 'dashmate') - - # test-suite: - # name: Test Suite - # needs: - # - build-js - # - build-images - # secrets: inherit - # strategy: - # fail-fast: false - # matrix: - # include: - # - name: Test Suite - # command: test:suite - # batch_index: 0 - # batch_total: 0 - # - name: Test Suite in browser (1) - # command: test:browsers - # batch_index: 0 - # batch_total: 2 - # - name: Test Suite in browser (2) - # command: test:browsers - # batch_index: 1 - # batch_total: 2 - # uses: ./.github/workflows/tests-test-suite.yml - # with: - # name: ${{ matrix.name }} - # command: ${{ matrix.command }} - # batch_total: ${{ matrix.batch_total }} - # batch_index: ${{ matrix.batch_index }} - - # test-functional: - # name: Packages functional tests - # needs: - # - build-js - # - build-images - # secrets: inherit - # uses: ./.github/workflows/tests-packges-functional.yml + + js-packages: + name: JS packages + needs: + - changes + - build-js + secrets: inherit + strategy: + fail-fast: false + matrix: + js-package: ${{ fromJson(needs.changes.outputs.js-packages) }} + uses: ./.github/workflows/tests-js-package.yml + with: + package: ${{ matrix.js-package }} + test-command: ${{ matrix.js-package == 'dashmate' && 'test:unit' || 'test' }} + skip-tests: ${{ contains(matrix.js-package, 'platform-test-suite') }} + + js-deps-versions: + name: JS dependency versions check + if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || !github.event.pull_request.draft }} + runs-on: ubuntu-24.04 + steps: + - name: Check out repo + uses: actions/checkout@v4 + + - name: Setup Node.JS + uses: actions/setup-node@v4 + with: + node-version: "20" + + - name: Enable corepack + run: corepack enable + + - name: Validate workspaces + run: yarn constraints + + js-npm-security: + name: JS NPM security audit + if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || !github.event.pull_request.draft }} + runs-on: ubuntu-24.04 + steps: + - name: Check out repo + uses: actions/checkout@v4 + + - name: Enable corepack + run: corepack enable + + - name: Audit NPM + run: yarn npm audit --environment production --all --recursive + + js-codeql: + name: JS code security audit + needs: build-js + secrets: inherit + uses: ./.github/workflows/tests-codeql.yml + + dashmate-e2e-tests: + name: Dashmate E2E tests + secrets: inherit + needs: + - changes + - build-js + - build-images + strategy: + fail-fast: false + matrix: + include: + - name: Local network + test-pattern: test/e2e/localNetwork.spec.js + restore_local_network_data: true + - name: Testnet fullnode + test-pattern: test/e2e/testnetFullnode.spec.js + restore_local_network_data: false + - name: Testnet Evonode + test-pattern: test/e2e/testnetEvonode.spec.js + restore_local_network_data: false + uses: ./.github/workflows/tests-dashmate.yml + with: + name: ${{ matrix.name }} + test-pattern: ${{ matrix.test-pattern }} + restore_local_network_data: ${{ matrix.restore_local_network_data }} + if: contains(needs.changes.outputs.js-packages, 'dashmate') + + test-suite: + name: Test Suite + needs: + - build-js + - build-images + secrets: inherit + strategy: + fail-fast: false + matrix: + include: + - name: Test Suite + command: test:suite + batch_index: 0 + batch_total: 0 + - name: Test Suite in browser (1) + command: test:browsers + batch_index: 0 + batch_total: 2 + - name: Test Suite in browser (2) + command: test:browsers + batch_index: 1 + batch_total: 2 + uses: ./.github/workflows/tests-test-suite.yml + with: + name: ${{ matrix.name }} + command: ${{ matrix.command }} + batch_total: ${{ matrix.batch_total }} + batch_index: ${{ matrix.batch_index }} + + test-functional: + name: Packages functional tests + needs: + - build-js + - build-images + secrets: inherit + uses: ./.github/workflows/tests-packges-functional.yml From aeb7cb371f14ab897f72f80e9338d04cda00ca94 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Mon, 25 Nov 2024 10:45:30 +0100 Subject: [PATCH 32/51] chore: remove deprecated set-output --- .github/actions/rust/action.yaml | 2 +- .github/workflows/tests-build-js.yml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/actions/rust/action.yaml b/.github/actions/rust/action.yaml index 3b74e2102b..722e7df55f 100644 --- a/.github/actions/rust/action.yaml +++ b/.github/actions/rust/action.yaml @@ -31,7 +31,7 @@ runs: fi echo "TOOLCHAIN_VERSION=$TOOLCHAIN_VERSION" >> $GITHUB_ENV - echo "::set-output name=version::$TOOLCHAIN_VERSION" + echo "version=$TOOLCHAIN_VERSION" >> $GITHUB_OUTPUT - uses: dtolnay/rust-toolchain@master name: Install Rust toolchain diff --git a/.github/workflows/tests-build-js.yml b/.github/workflows/tests-build-js.yml index 0c726c5797..ddff6cab3d 100644 --- a/.github/workflows/tests-build-js.yml +++ b/.github/workflows/tests-build-js.yml @@ -61,7 +61,6 @@ jobs: - name: Build JS packages run: yarn build - if: ${{ steps.check-artifact.outputs.exists != 'true' }} - name: Ignore only already cached artifacts From 9486d0fa9b33918e945d78824bfc08ece4ac4518 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Mon, 25 Nov 2024 10:49:36 +0100 Subject: [PATCH 33/51] chore: comment out aws creds from "Build JS" step --- .github/workflows/tests-build-js.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/tests-build-js.yml b/.github/workflows/tests-build-js.yml index ddff6cab3d..18cd1523a8 100644 --- a/.github/workflows/tests-build-js.yml +++ b/.github/workflows/tests-build-js.yml @@ -6,12 +6,12 @@ jobs: name: Build JS runs-on: ubuntu-24.04 steps: - - name: Configure AWS credentials and bucket region - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ vars.AWS_REGION }} + # - name: Configure AWS credentials and bucket region + # uses: aws-actions/configure-aws-credentials@v4 + # with: + # aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + # aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + # aws-region: ${{ vars.AWS_REGION }} - uses: softwareforgood/check-artifact-v4-existence@v0 id: check-artifact From bf4bb7a6c45c1728fddbd2761ebb13938766d774 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Mon, 25 Nov 2024 11:22:20 +0100 Subject: [PATCH 34/51] fix: invalid region --- .github/workflows/tests-dashmate.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests-dashmate.yml b/.github/workflows/tests-dashmate.yml index 6c681ae45c..965cd2d017 100644 --- a/.github/workflows/tests-dashmate.yml +++ b/.github/workflows/tests-dashmate.yml @@ -36,7 +36,7 @@ jobs: with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ vars.CACHE_REGION }} + aws-region: ${{ vars.AWS_REGION }} - name: Setup Node.JS uses: ./.github/actions/nodejs From e007e8392a5c4d65334b3dd561a0d44efecb73fb Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Mon, 25 Nov 2024 11:25:46 +0100 Subject: [PATCH 35/51] chore: remove environment --- .github/workflows/tests-build-image.yml | 1 - .github/workflows/tests-rs-package.yml | 6 ------ .github/workflows/tests.yml | 1 - 3 files changed, 8 deletions(-) diff --git a/.github/workflows/tests-build-image.yml b/.github/workflows/tests-build-image.yml index 3e92319cc4..6f35895f34 100644 --- a/.github/workflows/tests-build-image.yml +++ b/.github/workflows/tests-build-image.yml @@ -17,7 +17,6 @@ on: jobs: build-image: name: Build ${{ inputs.name }} image - environment: test runs-on: ubuntu-24.04 steps: - name: Check out repo diff --git a/.github/workflows/tests-rs-package.yml b/.github/workflows/tests-rs-package.yml index 7726986e5c..3df3d254d7 100644 --- a/.github/workflows/tests-rs-package.yml +++ b/.github/workflows/tests-rs-package.yml @@ -13,7 +13,6 @@ on: jobs: lint: name: Linting - environment: test runs-on: ubuntu-24.04 permissions: id-token: write @@ -50,7 +49,6 @@ jobs: SNAPPY_LIB_DIR: "/usr/lib/x86_64-linux-gnu" formatting: name: Formatting - environment: test runs-on: ubuntu-24.04 timeout-minutes: 5 steps: @@ -71,7 +69,6 @@ jobs: unused_deps: name: Unused dependencies - environment: test runs-on: ubuntu-24.04 permissions: id-token: write @@ -113,7 +110,6 @@ jobs: detect_structure_changes: name: Detect immutable structure changes - environment: test runs-on: ubuntu-24.04 # FIXME: as we use `gh pr view` below, this check can only # run on pull requests. We should find a way to run it @@ -175,7 +171,6 @@ jobs: test: name: Tests - environment: test runs-on: ubuntu-24.04 timeout-minutes: 25 steps: @@ -208,7 +203,6 @@ jobs: check_each_feature: name: Check each feature - environment: test runs-on: ubuntu-24.04 timeout-minutes: 10 if: ${{ inputs.check-each-feature }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5bd4a3a64f..4cf511cfbb 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -87,7 +87,6 @@ jobs: rs-crates-security: name: Rust crates security audit - environment: test if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || !github.event.pull_request.draft }} runs-on: ubuntu-24.04 steps: From 601c907d4b5e214854aa314a465794528dfe4636 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Mon, 25 Nov 2024 11:37:14 +0100 Subject: [PATCH 36/51] chore: remove test code --- .github/workflows/tests-build-image.yml | 2 +- .github/workflows/tests-rs-package.yml | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/tests-build-image.yml b/.github/workflows/tests-build-image.yml index 6f35895f34..b8b39065a7 100644 --- a/.github/workflows/tests-build-image.yml +++ b/.github/workflows/tests-build-image.yml @@ -30,7 +30,7 @@ jobs: with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_REGION }} + aws-region: ${{ vars.AWS_REGION }} - name: Login to ECR run: | diff --git a/.github/workflows/tests-rs-package.yml b/.github/workflows/tests-rs-package.yml index 3df3d254d7..3696e7e9db 100644 --- a/.github/workflows/tests-rs-package.yml +++ b/.github/workflows/tests-rs-package.yml @@ -240,10 +240,6 @@ jobs: run: | echo Verify all features disabled set -ex - set | grep AWS - cat $HOME/.aws/credentials - SCCACHE_LOG=debug sccache --start-server - sccache --stop-server features="${{ steps.crate_info.outputs.features }}" fails="" From 2893ac05277b7de4c3a53900e4d35166ddbe40e2 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Mon, 25 Nov 2024 11:44:39 +0100 Subject: [PATCH 37/51] chore: trigger full workflow run --- packages/rs-dpp/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/rs-dpp/src/lib.rs b/packages/rs-dpp/src/lib.rs index a5d4ec177e..5bdd20f575 100644 --- a/packages/rs-dpp/src/lib.rs +++ b/packages/rs-dpp/src/lib.rs @@ -1,5 +1,5 @@ #![cfg_attr(docsrs, feature(doc_cfg))] -// Coding conventions +// Coding conventions. #![forbid(unsafe_code)] //#![deny(missing_docs)] #![allow(dead_code)] From 4148f7c5dff8878199a6073be8608f0d1890254e Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Mon, 25 Nov 2024 12:44:55 +0100 Subject: [PATCH 38/51] chore: trigger rebuild --- packages/rs-dpp/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/rs-dpp/src/lib.rs b/packages/rs-dpp/src/lib.rs index 5bdd20f575..168a9ad284 100644 --- a/packages/rs-dpp/src/lib.rs +++ b/packages/rs-dpp/src/lib.rs @@ -1,5 +1,5 @@ #![cfg_attr(docsrs, feature(doc_cfg))] -// Coding conventions. +// Coding conventions . #![forbid(unsafe_code)] //#![deny(missing_docs)] #![allow(dead_code)] From 5f3eb51ab12d3102b4d8a84dba8ac852f31c1ff4 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Mon, 25 Nov 2024 14:00:26 +0100 Subject: [PATCH 39/51] chore: rabbit's feedback --- .github/actions/aws_credentials/action.yaml | 16 +++++++++++++--- .github/actions/docker/action.yaml | 1 + .github/actions/sccache/action.yaml | 8 +++++++- .github/workflows/tests-test-suite.yml | 2 +- Dockerfile | 7 +++++-- 5 files changed, 27 insertions(+), 7 deletions(-) diff --git a/.github/actions/aws_credentials/action.yaml b/.github/actions/aws_credentials/action.yaml index 5084f6af10..34bf11d913 100644 --- a/.github/actions/aws_credentials/action.yaml +++ b/.github/actions/aws_credentials/action.yaml @@ -21,13 +21,23 @@ runs: shell: bash run: | mkdir -p "${HOME}/.aws" - echo "[${{ inputs.profile }}]" >> ${HOME}/.aws/credentials - echo "aws_access_key_id=${{ inputs.access_key_id }}" >> ${HOME}/.aws/credentials - echo "aws_secret_access_key=${{ inputs.secret_access_key }}" >> ${HOME}/.aws/credentials + cat >> ${HOME}/.aws/credentials << EOF + [${{ inputs.profile }}] + aws_access_key_id=${{ inputs.access_key_id }} + aws_secret_access_key=${{ inputs.secret_access_key }} + EOF chmod -R go-rwx ${HOME}/.aws - name: Set env variables shell: bash run: | + # Exit on any error + set -euo pipefail + # Validate AWS_PROFILE is not empty + if [ -z "${{ inputs.profile }}" ]; then + echo "Error: AWS_PROFILE cannot be empty" + exit 1 + fi + # Export variables echo "AWS_PROFILE=${{ inputs.profile }}" >> $GITHUB_ENV echo "AWS_SHARED_CREDENTIALS_FILE=${HOME}/.aws/credentials" >> $GITHUB_ENV diff --git a/.github/actions/docker/action.yaml b/.github/actions/docker/action.yaml index 3d43b62124..f1e8a9ac5f 100644 --- a/.github/actions/docker/action.yaml +++ b/.github/actions/docker/action.yaml @@ -191,6 +191,7 @@ runs: SCCACHE_REGION=${{ inputs.cache_region }} SCCACHE_ENDPOINT=${{ inputs.cache_endpoint }} SCCACHE_S3_KEY_PREFIX=${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu + SCCACHE_VERSION=${{ env.SCCACHE_VERSION }} AWS_PROFILE=sccache cache-from: ${{ steps.layer_cache_settings.outputs.cache_from }} cache-to: ${{ steps.layer_cache_settings.outputs.cache_to }} diff --git a/.github/actions/sccache/action.yaml b/.github/actions/sccache/action.yaml index 7844ea5edd..d64e719273 100644 --- a/.github/actions/sccache/action.yaml +++ b/.github/actions/sccache/action.yaml @@ -20,6 +20,10 @@ inputs: install: description: "Install sccache" default: "true" + version: + description: "sccache version" + default: "0.8.2" + required: false # TODO: Cache deps here to save 1 minute runs: @@ -29,7 +33,7 @@ runs: if: ${{ inputs.install == 'true' }} uses: mozilla-actions/sccache-action@v0.0.6 with: - version: "v0.8.2" # Must be the same as in Dockerfile + version: "v${{ inputs.version }}" - name: Configure AWS credentials uses: ./.github/actions/aws_credentials @@ -47,3 +51,5 @@ runs: echo "SCCACHE_REGION=${{ inputs.region }}" >> $GITHUB_ENV echo "SCCACHE_ENDPOINT=${{ inputs.endpoint }}" >> $GITHUB_ENV echo "SCCACHE_S3_KEY_PREFIX=${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu" >> $GITHUB_ENV + # "SCCACHE_VERSION" is used inside Docker to install the same version of sccache + echo "SCCACHE_VERSION=${{ inputs.version }}" >> $GITHUB_ENV diff --git a/.github/workflows/tests-test-suite.yml b/.github/workflows/tests-test-suite.yml index e5c8babba5..23c2f7915d 100644 --- a/.github/workflows/tests-test-suite.yml +++ b/.github/workflows/tests-test-suite.yml @@ -46,7 +46,7 @@ jobs: aws-region: ${{ vars.AWS_REGION }} - name: Login to ECR - run: aws ecr get-login-password --region ${{ vars.AWS_REGION }} | docker login --username AWS --password-stdin ${{ env.ECR_HOST }} + run: aws ecr get-login-password --region ${{ vars.AWS_REGION }} | docker login --username AWS --password-stdin ${{ env.ECR_HOST }} - name: Start local network uses: ./.github/actions/local-network diff --git a/Dockerfile b/Dockerfile index bf3ca79384..3b509c4fef 100644 --- a/Dockerfile +++ b/Dockerfile @@ -184,7 +184,7 @@ RUN --mount=type=secret,id=AWS <> /root/env echo "export ACTIONS_CACHE_URL=${ACTIONS_CACHE_URL}" >> /root/env - # ACTIONS_RUNTIME_TOKEN is a secret so we load it on demand + # ACTIONS_RUNTIME_TOKEN is a secret so we quote it here, and it will be loaded when `source /root/env` is run echo 'export ACTIONS_RUNTIME_TOKEN="$(cat /run/secrets/GHA)"' >> /root/env ### AWS S3 ### @@ -201,7 +201,10 @@ RUN --mount=type=secret,id=AWS <> /root/env # Check if AWS credentials file is mounted correctly, eg. --mount=type=secret,id=AWS - echo '[ -e "${AWS_SHARED_CREDENTIALS_FILE}" ] || { echo "$(id -u): Cannot read ${AWS_SHARED_CREDENTIALS_FILE}"; exit 1; }' >> /root/env + echo '[ -e "${AWS_SHARED_CREDENTIALS_FILE}" ] || { + echo "$(id -u): Cannot read ${AWS_SHARED_CREDENTIALS_FILE}; did you use RUN --mount=type=secret,id=AWS ?"; + exit 1; + }' >> /root/env ### memcached ### elif [ -n "${SCCACHE_MEMCACHED}" ]; then From 2fa40cd644066026911483584f92d6280665bc10 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Mon, 25 Nov 2024 14:15:58 +0100 Subject: [PATCH 40/51] chore: apply rabbit feedback --- .github/actions/docker/action.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/actions/docker/action.yaml b/.github/actions/docker/action.yaml index f1e8a9ac5f..11f1ced49f 100644 --- a/.github/actions/docker/action.yaml +++ b/.github/actions/docker/action.yaml @@ -50,7 +50,6 @@ outputs: runs: using: composite - steps: - name: Login to DockerHub uses: docker/login-action@v3 @@ -167,6 +166,10 @@ runs: - name: Unset AWS credentials to avoid conflicts shell: bash run: | + if [ ! -f "$HOME/.aws/credentials" ]; then + echo "Error: AWS credentials file not found" + exit 1 + fi echo AWS_DEFAULT_REGION='' >> $GITHUB_ENV echo AWS_REGION='' >> $GITHUB_ENV echo AWS_ACCESS_KEY_ID='' >> $GITHUB_ENV From 9a0a8ce48ea54da5d5d676633b19ad373cfbd9b6 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Mon, 25 Nov 2024 16:33:37 +0100 Subject: [PATCH 41/51] chore: cleanup ECR login logic --- .github/actions/aws_ecr_login/action.yaml | 42 +++++++++++++++++++ .github/actions/docker/action.yaml | 13 ------ .github/workflows/release.yml | 8 ---- .github/workflows/tests-build-image.yml | 18 +++----- .github/workflows/tests-build-js.yml | 7 ---- .github/workflows/tests-codeql.yml | 8 ---- .github/workflows/tests-dashmate.yml | 12 +++--- .github/workflows/tests-js-package.yml | 15 ------- .../workflows/tests-packges-functional.yml | 14 +++---- .github/workflows/tests-test-suite.yml | 14 +++---- 10 files changed, 66 insertions(+), 85 deletions(-) create mode 100644 .github/actions/aws_ecr_login/action.yaml diff --git a/.github/actions/aws_ecr_login/action.yaml b/.github/actions/aws_ecr_login/action.yaml new file mode 100644 index 0000000000..fc93942222 --- /dev/null +++ b/.github/actions/aws_ecr_login/action.yaml @@ -0,0 +1,42 @@ +--- +# Login to AWS ECR +name: "aws_ecr_login" +description: "Login to AWS ECR to store Docker containers" +inputs: + aws_account_id: + description: AWS account ID (AWS_ACCOUNT_ID) + required: true + aws_access_key_id: + description: Access key ID (AWS_ACCESS_KEY_ID) + required: true + aws_secret_access_key: + description: Secret access key (AWS_SECRET_ACCESS_KEY) + required: true + aws_region: + description: AWS region to use (AWS_REGION) + required: true + +runs: + using: composite + steps: + - name: Configure AWS credentials and bucket region + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ inputs.aws_access_key_id }} + aws-secret-access-key: ${{ inputs.aws_secret_access_key }} + aws-region: ${{ inputs.aws_region }} + + - name: Login to ECR + run: | + aws ecr get-login-password \ + --region ${{ inputs.aws_region }} | docker login --username AWS --password-stdin ${{ inputs.aws_account_id }}.dkr.ecr.${{ inputs.aws_region }}.amazonaws.com + shell: bash + + # Unset AWS credentials to avoid conflicts, as we prefer credentials from ~/.aws/credentials to authenticate + - name: Unset AWS credentials to avoid conflicts + shell: bash + run: | + echo AWS_DEFAULT_REGION='' >> $GITHUB_ENV + echo AWS_REGION='' >> $GITHUB_ENV + echo AWS_ACCESS_KEY_ID='' >> $GITHUB_ENV + echo AWS_SECRET_ACCESS_KEY='' >> $GITHUB_ENV diff --git a/.github/actions/docker/action.yaml b/.github/actions/docker/action.yaml index 11f1ced49f..e9a9a35212 100644 --- a/.github/actions/docker/action.yaml +++ b/.github/actions/docker/action.yaml @@ -162,19 +162,6 @@ runs: secret_access_key: ${{ inputs.cache_secret_access_key }} install: false - # Unset AWS credentials to avoid conflicts, as we use credentials in ~/.aws/credentials generated in sccache action. - - name: Unset AWS credentials to avoid conflicts - shell: bash - run: | - if [ ! -f "$HOME/.aws/credentials" ]; then - echo "Error: AWS credentials file not found" - exit 1 - fi - echo AWS_DEFAULT_REGION='' >> $GITHUB_ENV - echo AWS_REGION='' >> $GITHUB_ENV - echo AWS_ACCESS_KEY_ID='' >> $GITHUB_ENV - echo AWS_SECRET_ACCESS_KEY='' >> $GITHUB_ENV - - name: Build and push Docker image ${{ inputs.image }} id: docker_build uses: docker/build-push-action@v6 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c74070294b..546141327a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -231,14 +231,6 @@ jobs: with: fetch-depth: 0 - # TODO: Do we still need this? - - name: Configure AWS credentials and bucket region - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-region: ${{ vars.AWS_REGION }} - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - - name: Download JS build artifacts uses: actions/download-artifact@v4 with: diff --git a/.github/workflows/tests-build-image.yml b/.github/workflows/tests-build-image.yml index b8b39065a7..dc6efffa47 100644 --- a/.github/workflows/tests-build-image.yml +++ b/.github/workflows/tests-build-image.yml @@ -24,19 +24,13 @@ jobs: with: fetch-depth: 0 - # AWS credentials only needed for ECR login - - name: Configure AWS credentials and bucket region - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ vars.AWS_REGION }} - - name: Login to ECR - run: | - aws ecr get-login-password \ - --region ${{ vars.AWS_REGION }} | docker login --username AWS --password-stdin ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_REGION }}.amazonaws.com - shell: bash + uses: ./.github/actions/aws_ecr_login + with: + aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws_region: ${{ vars.AWS_REGION }} + aws_account_id: ${{ secrets.AWS_ACCOUNT_ID }} - name: Build and push by SHA uses: ./.github/actions/docker diff --git a/.github/workflows/tests-build-js.yml b/.github/workflows/tests-build-js.yml index 18cd1523a8..8c83c8b556 100644 --- a/.github/workflows/tests-build-js.yml +++ b/.github/workflows/tests-build-js.yml @@ -6,13 +6,6 @@ jobs: name: Build JS runs-on: ubuntu-24.04 steps: - # - name: Configure AWS credentials and bucket region - # uses: aws-actions/configure-aws-credentials@v4 - # with: - # aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - # aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - # aws-region: ${{ vars.AWS_REGION }} - - uses: softwareforgood/check-artifact-v4-existence@v0 id: check-artifact with: diff --git a/.github/workflows/tests-codeql.yml b/.github/workflows/tests-codeql.yml index 2e39f97ef5..ed972e6d9c 100644 --- a/.github/workflows/tests-codeql.yml +++ b/.github/workflows/tests-codeql.yml @@ -20,14 +20,6 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - # TODO do we still need this? - - name: Configure AWS credentials and bucket region - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ vars.AWS_REGION }} - - name: Setup Node.JS uses: ./.github/actions/nodejs diff --git a/.github/workflows/tests-dashmate.yml b/.github/workflows/tests-dashmate.yml index 965cd2d017..9df6684332 100644 --- a/.github/workflows/tests-dashmate.yml +++ b/.github/workflows/tests-dashmate.yml @@ -31,12 +31,13 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Configure AWS credentials and bucket region - uses: aws-actions/configure-aws-credentials@v4 + - name: Login to ECR + uses: ./.github/actions/aws_ecr_login with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ vars.AWS_REGION }} + aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws_region: ${{ vars.AWS_REGION }} + aws_account_id: ${{ secrets.AWS_ACCOUNT_ID }} - name: Setup Node.JS uses: ./.github/actions/nodejs @@ -53,7 +54,6 @@ jobs: # Login to ECR DOCKER_HUB_ORG="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_REGION }}.amazonaws.com" - aws ecr get-login-password --region ${{ vars.AWS_REGION }} | docker login --username AWS --password-stdin $DOCKER_HUB_ORG SHA_TAG=sha-${{ github.sha }} diff --git a/.github/workflows/tests-js-package.yml b/.github/workflows/tests-js-package.yml index c403618d3e..681c27b560 100644 --- a/.github/workflows/tests-js-package.yml +++ b/.github/workflows/tests-js-package.yml @@ -25,13 +25,6 @@ jobs: - name: Check out repo uses: actions/checkout@v4 - - name: Configure AWS credentials and bucket region - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ vars.AWS_REGION }} - - name: Setup Node.JS uses: ./.github/actions/nodejs @@ -57,14 +50,6 @@ jobs: with: fetch-depth: 0 - # TODO: Do we still need this? - - name: Configure AWS credentials and bucket region - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-region: ${{ vars.AWS_REGION }} - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - - name: Setup Node.JS uses: ./.github/actions/nodejs diff --git a/.github/workflows/tests-packges-functional.yml b/.github/workflows/tests-packges-functional.yml index 811b3a7690..81d0083e53 100644 --- a/.github/workflows/tests-packges-functional.yml +++ b/.github/workflows/tests-packges-functional.yml @@ -20,15 +20,13 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Configure AWS credentials and bucket region - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ vars.AWS_REGION }} - - name: Login to ECR - run: aws ecr get-login-password --region ${{ vars.AWS_REGION }} | docker login --username AWS --password-stdin ${{ env.ECR_HOST }} + uses: ./.github/actions/aws_ecr_login + with: + aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws_region: ${{ vars.AWS_REGION }} + aws_account_id: ${{ secrets.AWS_ACCOUNT_ID }} - name: Start local network uses: ./.github/actions/local-network diff --git a/.github/workflows/tests-test-suite.yml b/.github/workflows/tests-test-suite.yml index 23c2f7915d..698e5ea153 100644 --- a/.github/workflows/tests-test-suite.yml +++ b/.github/workflows/tests-test-suite.yml @@ -38,15 +38,13 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Configure AWS credentials and bucket region - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ vars.AWS_REGION }} - - name: Login to ECR - run: aws ecr get-login-password --region ${{ vars.AWS_REGION }} | docker login --username AWS --password-stdin ${{ env.ECR_HOST }} + uses: ./.github/actions/aws_ecr_login + with: + aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws_region: ${{ vars.AWS_REGION }} + aws_account_id: ${{ secrets.AWS_ACCOUNT_ID }} - name: Start local network uses: ./.github/actions/local-network From ecc73967456f703cc4484e48110b4cc822e319c7 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Mon, 25 Nov 2024 16:46:09 +0100 Subject: [PATCH 42/51] chore: minor fixes from rabbit --- .github/workflows/release-docker-image.yml | 2 +- .github/workflows/release.yml | 2 +- .github/workflows/tests-build-js.yml | 2 +- .github/workflows/tests-dashmate.yml | 4 ++-- .github/workflows/tests-packges-functional.yml | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release-docker-image.yml b/.github/workflows/release-docker-image.yml index d24935f8f3..728bd7e94c 100644 --- a/.github/workflows/release-docker-image.yml +++ b/.github/workflows/release-docker-image.yml @@ -63,7 +63,7 @@ jobs: cargo_profile: ${{ inputs.cargo_profile }} dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }} dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }} - cache_region: ${{ vars.CACHE_REGION }} + cache_region: ${{ vars.CACHE_REGION }} cache_bucket: ${{ vars.CACHE_S3_BUCKET }} cache_endpoint: ${{ vars.CACHE_S3_ENDPOINT }} cache_access_key_id: ${{ secrets.CACHE_KEY_ID }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 546141327a..bb165d3918 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -55,7 +55,7 @@ jobs: uses: ./.github/actions/sccache with: bucket: ${{ vars.CACHE_S3_BUCKET }} - region: ${{ vars.AWS_REGION }} + region: ${{ vars.AWS_REGION }} endpoint: ${{ vars.CACHE_S3_ENDPOINT }} access_key_id: ${{ secrets.CACHE_KEY_ID }} secret_access_key: ${{ secrets.CACHE_SECRET_KEY }} diff --git a/.github/workflows/tests-build-js.yml b/.github/workflows/tests-build-js.yml index 8c83c8b556..1c73612e22 100644 --- a/.github/workflows/tests-build-js.yml +++ b/.github/workflows/tests-build-js.yml @@ -38,7 +38,7 @@ jobs: uses: ./.github/actions/sccache with: bucket: ${{ vars.CACHE_S3_BUCKET }} - region: ${{ vars.CACHE_REGION }} + region: ${{ vars.CACHE_REGION }} endpoint: ${{ vars.CACHE_S3_ENDPOINT }} access_key_id: ${{ secrets.CACHE_KEY_ID }} secret_access_key: ${{ secrets.CACHE_SECRET_KEY }} diff --git a/.github/workflows/tests-dashmate.yml b/.github/workflows/tests-dashmate.yml index 9df6684332..170006c22a 100644 --- a/.github/workflows/tests-dashmate.yml +++ b/.github/workflows/tests-dashmate.yml @@ -50,10 +50,10 @@ jobs: - name: Replace with pre-built images run: | - set -x + set -e # Login to ECR - DOCKER_HUB_ORG="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_REGION }}.amazonaws.com" + DOCKER_HUB_ORG="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_REGION }}.amazonaws.com" SHA_TAG=sha-${{ github.sha }} diff --git a/.github/workflows/tests-packges-functional.yml b/.github/workflows/tests-packges-functional.yml index 81d0083e53..dcd99f7581 100644 --- a/.github/workflows/tests-packges-functional.yml +++ b/.github/workflows/tests-packges-functional.yml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 15 env: - ECR_HOST: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_REGION }}.amazonaws.com + ECR_HOST: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_REGION }}.amazonaws.com steps: - name: Check out repo uses: actions/checkout@v4 From f7ffa9a9c919a9cb3762a7ab3a3bbb6fe0d405bb Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Mon, 25 Nov 2024 17:07:36 +0100 Subject: [PATCH 43/51] fix: multi-platform docker build cache misses due to overwriting other platforms --- .github/actions/docker/action.yaml | 2 ++ .github/actions/sccache/action.yaml | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/actions/docker/action.yaml b/.github/actions/docker/action.yaml index e9a9a35212..8bf1df906c 100644 --- a/.github/actions/docker/action.yaml +++ b/.github/actions/docker/action.yaml @@ -86,6 +86,7 @@ runs: region: ${{ inputs.cache_region }} bucket: ${{ inputs.cache_bucket }} endpoint: ${{ inputs.cache_endpoint }} + prefix: "cache-layers/${{ inputs.platform }}/" cache_to_name: ${{ inputs.cache_to_name }} - name: Set HOME variable to github context @@ -160,6 +161,7 @@ runs: endpoint: ${{ inputs.cache_endpoint }} access_key_id: ${{ inputs.cache_access_key_id }} secret_access_key: ${{ inputs.cache_secret_access_key }} + platform: ${{ inputs.platform }} install: false - name: Build and push Docker image ${{ inputs.image }} diff --git a/.github/actions/sccache/action.yaml b/.github/actions/sccache/action.yaml index d64e719273..2984abc6c0 100644 --- a/.github/actions/sccache/action.yaml +++ b/.github/actions/sccache/action.yaml @@ -17,6 +17,9 @@ inputs: secret_access_key: description: S3 endpoint secret access key required: true + platform: + description: "Platform and architecture to use when caching; helps to avoid invalid cache on different arch" + required: true install: description: "Install sccache" default: "true" @@ -50,6 +53,6 @@ runs: echo "SCCACHE_BUCKET=${{ inputs.bucket }}" >> $GITHUB_ENV echo "SCCACHE_REGION=${{ inputs.region }}" >> $GITHUB_ENV echo "SCCACHE_ENDPOINT=${{ inputs.endpoint }}" >> $GITHUB_ENV - echo "SCCACHE_S3_KEY_PREFIX=${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu" >> $GITHUB_ENV + echo "SCCACHE_S3_KEY_PREFIX=${{ runner.os }}/sccache/${{ inputs.platform }}/" >> $GITHUB_ENV # "SCCACHE_VERSION" is used inside Docker to install the same version of sccache echo "SCCACHE_VERSION=${{ inputs.version }}" >> $GITHUB_ENV From e8bb7485ed65c7fe35b92b7b8fb1819a7a483497 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Mon, 25 Nov 2024 17:29:15 +0100 Subject: [PATCH 44/51] chore: further platform fixes --- .github/actions/docker/action.yaml | 2 +- .github/actions/rust/action.yaml | 6 ------ .github/actions/sccache/action.yaml | 7 ++++--- 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/.github/actions/docker/action.yaml b/.github/actions/docker/action.yaml index 8bf1df906c..cfd35a19ec 100644 --- a/.github/actions/docker/action.yaml +++ b/.github/actions/docker/action.yaml @@ -182,7 +182,7 @@ runs: SCCACHE_BUCKET=${{ inputs.cache_bucket }} SCCACHE_REGION=${{ inputs.cache_region }} SCCACHE_ENDPOINT=${{ inputs.cache_endpoint }} - SCCACHE_S3_KEY_PREFIX=${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu + SCCACHE_S3_KEY_PREFIX=${{ env.SCCACHE_S3_KEY_PREFIX }} SCCACHE_VERSION=${{ env.SCCACHE_VERSION }} AWS_PROFILE=sccache cache-from: ${{ steps.layer_cache_settings.outputs.cache_from }} diff --git a/.github/actions/rust/action.yaml b/.github/actions/rust/action.yaml index 722e7df55f..ff26db3fad 100644 --- a/.github/actions/rust/action.yaml +++ b/.github/actions/rust/action.yaml @@ -82,12 +82,6 @@ runs: echo "PROTOC=${HOME}/.local/bin/protoc" >> $GITHUB_ENV export PATH="${PATH}:${HOME}/.local/bin" - - name: Install sccache-cache - uses: mozilla-actions/sccache-action@v0.0.6 - with: - version: "v0.8.2" # Must be the same as in Dockerfile - if: inputs.cache == 'true' - - name: Set HOME variable to github context shell: bash run: echo "HOME=$HOME" >> $GITHUB_ENV diff --git a/.github/actions/sccache/action.yaml b/.github/actions/sccache/action.yaml index 2984abc6c0..ac9e42da98 100644 --- a/.github/actions/sccache/action.yaml +++ b/.github/actions/sccache/action.yaml @@ -18,8 +18,9 @@ inputs: description: S3 endpoint secret access key required: true platform: - description: "Platform and architecture to use when caching; helps to avoid invalid cache on different arch" - required: true + description: "Platform and architecture to use when caching; defaults to linux/amd64" + required: false + default: "linux/amd64" install: description: "Install sccache" default: "true" @@ -53,6 +54,6 @@ runs: echo "SCCACHE_BUCKET=${{ inputs.bucket }}" >> $GITHUB_ENV echo "SCCACHE_REGION=${{ inputs.region }}" >> $GITHUB_ENV echo "SCCACHE_ENDPOINT=${{ inputs.endpoint }}" >> $GITHUB_ENV - echo "SCCACHE_S3_KEY_PREFIX=${{ runner.os }}/sccache/${{ inputs.platform }}/" >> $GITHUB_ENV + echo "SCCACHE_S3_KEY_PREFIX=sccache/${{ inputs.platform }}/" >> $GITHUB_ENV # "SCCACHE_VERSION" is used inside Docker to install the same version of sccache echo "SCCACHE_VERSION=${{ inputs.version }}" >> $GITHUB_ENV From 646c83008a320086142101ece6360850e4b39b06 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Mon, 25 Nov 2024 17:39:07 +0100 Subject: [PATCH 45/51] fix: use the same sccache prefix in gha and docker --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 9e7b67bfff..6e102863f9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -193,7 +193,7 @@ RUN --mount=type=secret,id=AWS <> /root/env [ -n "${AWS_PROFILE}" ] && echo "export AWS_PROFILE='${AWS_PROFILE}'" >> /root/env echo "export SCCACHE_ENDPOINT='${SCCACHE_ENDPOINT}'" >> /root/env - echo "export SCCACHE_S3_KEY_PREFIX='${SCCACHE_S3_KEY_PREFIX}/${TARGETARCH}/linux-musl'" >> /root/env + echo "export SCCACHE_S3_KEY_PREFIX='${SCCACHE_S3_KEY_PREFIX}'" >> /root/env # Configure AWS credentials mkdir --mode=0700 -p "$HOME/.aws" From f33c8c76c9d1de7bef9855cc85a8944186f57772 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Tue, 26 Nov 2024 09:02:01 +0100 Subject: [PATCH 46/51] chore: double-check dockerfile secrets mounts --- Dockerfile | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6e102863f9..0236a9e20e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -103,7 +103,6 @@ ARG TARGETARCH WORKDIR /platform -# TODO: It doesn't sharing PATH between stages, so we need "source $HOME/.cargo/env" everywhere COPY rust-toolchain.toml . RUN TOOLCHAIN_VERSION="$(grep channel rust-toolchain.toml | awk '{print $3}' | tr -d '"')" && \ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- \ @@ -115,6 +114,10 @@ RUN TOOLCHAIN_VERSION="$(grep channel rust-toolchain.toml | awk '{print $3}' | t ONBUILD ENV HOME=/root ONBUILD ENV CARGO_HOME=$HOME/.cargo +# Configure Rust toolchain +# It doesn't sharing PATH between stages, so we need "source $HOME/.cargo/env" everywhere +RUN echo 'source $HOME/.cargo/env' >> /root/env + # Install protoc - protobuf compiler # The one shipped with Alpine does not work ARG PROTOC_VERSION=27.3 @@ -296,7 +299,9 @@ WORKDIR /platform # Download and install cargo-binstall ENV BINSTALL_VERSION=1.10.11 -RUN set -ex; \ +RUN --mount=type=secret,id=AWS \ + set -ex; \ + source /root/env; \ if [ "$TARGETARCH" = "amd64" ]; then \ CARGO_BINSTALL_ARCH="x86_64-unknown-linux-musl"; \ elif [ "$TARGETARCH" = "arm64" ]; then \ @@ -310,10 +315,10 @@ RUN set -ex; \ curl -A "Mozilla/5.0 (X11; Linux x86_64; rv:60.0) Gecko/20100101 Firefox/81.0" -L --proto '=https' --tlsv1.2 -sSf "$DOWNLOAD_URL" | tar -xvzf -; \ ./cargo-binstall -y --force cargo-binstall@${BINSTALL_VERSION}; \ rm ./cargo-binstall; \ - source $HOME/.cargo/env; \ cargo binstall -V -RUN source $HOME/.cargo/env; \ +RUN --mount=type=secret,id=AWS \ + source /root/env; \ cargo binstall wasm-bindgen-cli@0.2.86 cargo-chef@0.1.67 \ --locked \ --no-discover-github-token \ @@ -363,7 +368,6 @@ COPY --parents \ /platform/ RUN --mount=type=secret,id=AWS \ - source $HOME/.cargo/env && \ source /root/env && \ cargo chef prepare $RELEASE --recipe-path recipe.json @@ -385,13 +389,12 @@ RUN --mount=type=cache,sharing=shared,id=cargo_registry_index,target=${CARGO_HOM --mount=type=cache,sharing=shared,id=cargo_git,target=${CARGO_HOME}/git/db \ --mount=type=secret,id=AWS \ set -ex; \ + source /root/env && \ if [[ "${CARGO_BUILD_PROFILE}" == "release" ]] ; then \ mv .cargo/config-release.toml .cargo/config.toml; \ else \ export FEATURES_FLAG="--features=console,grovedbg" ; \ fi && \ - source $HOME/.cargo/env && \ - source /root/env && \ cargo chef cook \ --recipe-path recipe.json \ --profile "$CARGO_BUILD_PROFILE" \ @@ -444,7 +447,6 @@ RUN --mount=type=cache,sharing=shared,id=cargo_registry_index,target=${CARGO_HOM --mount=type=cache,sharing=shared,id=cargo_git,target=${CARGO_HOME}/git/db \ --mount=type=secret,id=AWS \ set -ex; \ - source $HOME/.cargo/env && \ source /root/env && \ if [[ "${CARGO_BUILD_PROFILE}" == "release" ]] ; then \ mv .cargo/config-release.toml .cargo/config.toml && \ @@ -479,7 +481,6 @@ RUN --mount=type=cache,sharing=shared,id=cargo_registry_index,target=${CARGO_HOM --mount=type=cache,sharing=shared,id=cargo_registry_cache,target=${CARGO_HOME}/registry/cache \ --mount=type=cache,sharing=shared,id=cargo_git,target=${CARGO_HOME}/git/db \ --mount=type=secret,id=AWS \ - source $HOME/.cargo/env && \ source /root/env && \ cargo chef cook \ --recipe-path recipe.json \ @@ -532,7 +533,6 @@ RUN --mount=type=cache,sharing=shared,id=cargo_registry_index,target=${CARGO_HOM --mount=type=cache,sharing=shared,id=cargo_git,target=${CARGO_HOME}/git/db \ --mount=type=cache,sharing=shared,id=unplugged_${TARGETARCH},target=/tmp/unplugged \ --mount=type=secret,id=AWS \ - source $HOME/.cargo/env && \ source /root/env && \ cp -R /tmp/unplugged /platform/.yarn/ && \ yarn install --inline-builds && \ From ba6507aafd3af21cda12917afaa3636a6cb78155 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Tue, 26 Nov 2024 12:38:38 +0100 Subject: [PATCH 47/51] refactor: improve readability of aws creds --- .github/actions/docker/action.yaml | 15 ++++++--------- .../s3-layer-cache-settings/action.yaml | 18 ++++++++++++++++++ .github/actions/sccache/action.yaml | 16 ++++++++++++++++ 3 files changed, 40 insertions(+), 9 deletions(-) diff --git a/.github/actions/docker/action.yaml b/.github/actions/docker/action.yaml index cfd35a19ec..7ffeb7fd2c 100644 --- a/.github/actions/docker/action.yaml +++ b/.github/actions/docker/action.yaml @@ -88,6 +88,8 @@ runs: endpoint: ${{ inputs.cache_endpoint }} prefix: "cache-layers/${{ inputs.platform }}/" cache_to_name: ${{ inputs.cache_to_name }} + s3_access_key_id: ${{ inputs.cache_access_key_id }} + s3_secret_access_key: ${{ inputs.cache_secret_access_key }} - name: Set HOME variable to github context shell: bash @@ -153,8 +155,9 @@ runs: } skip-extraction: ${{ steps.yarn-cache.outputs.cache-hit }} - - name: Setup sccache vars + - name: Configure sccache settings uses: ./.github/actions/sccache + id: sccache with: bucket: ${{ inputs.cache_bucket }} region: ${{ inputs.cache_region }} @@ -167,6 +170,7 @@ runs: - name: Build and push Docker image ${{ inputs.image }} id: docker_build uses: docker/build-push-action@v6 + env: ${{ steps.layer_cache_settings.outputs.env_vars }} with: context: . builder: ${{ steps.buildx.outputs.name }} @@ -177,14 +181,7 @@ runs: platforms: ${{ inputs.platform }} secret-files: | AWS=${{ env.HOME }}/.aws/credentials - build-args: | - CARGO_BUILD_PROFILE=${{ inputs.cargo_profile }} - SCCACHE_BUCKET=${{ inputs.cache_bucket }} - SCCACHE_REGION=${{ inputs.cache_region }} - SCCACHE_ENDPOINT=${{ inputs.cache_endpoint }} - SCCACHE_S3_KEY_PREFIX=${{ env.SCCACHE_S3_KEY_PREFIX }} - SCCACHE_VERSION=${{ env.SCCACHE_VERSION }} - AWS_PROFILE=sccache + build-args: ${{ steps.sccache.outputs.env_vars }} cache-from: ${{ steps.layer_cache_settings.outputs.cache_from }} cache-to: ${{ steps.layer_cache_settings.outputs.cache_to }} outputs: type=image,name=${{ inputs.image_org }}/${{ inputs.image_name }},push-by-digest=${{ inputs.push_tags != 'true' }},name-canonical=true,push=true diff --git a/.github/actions/s3-layer-cache-settings/action.yaml b/.github/actions/s3-layer-cache-settings/action.yaml index dc375eebb3..d163e59b57 100644 --- a/.github/actions/s3-layer-cache-settings/action.yaml +++ b/.github/actions/s3-layer-cache-settings/action.yaml @@ -25,6 +25,12 @@ inputs: prefix: description: S3 key prefix default: "cache-layers/" + s3_access_key_id: + description: Access key ID for S3 cache + required: true + s3_secret_access_key: + description: Secret access key for S3 cache + required: true mode: description: Cache mode default: max @@ -39,10 +45,22 @@ outputs: cache_from: description: "String with s3-based cache configuration for docker buildx cache-from option" value: ${{ steps.script.outputs.cache_from }} + env_vars: + description: "Environment variables to set before running docker buildx" + value: | + AWS_PROFILE=docker-layers + AWS_SHARED_CREDENTIALS_FILE=${HOME}/.aws/credentials runs: using: composite steps: + - name: Configure AWS credentials for s3 layers + uses: ./.github/actions/aws_credentials + with: + access_key_id: ${{ inputs.access_key_id }} + secret_access_key: ${{ inputs.secret_access_key }} + profile: "docker-layers" + - uses: actions/github-script@v6 id: script with: diff --git a/.github/actions/sccache/action.yaml b/.github/actions/sccache/action.yaml index ac9e42da98..ded663e562 100644 --- a/.github/actions/sccache/action.yaml +++ b/.github/actions/sccache/action.yaml @@ -28,6 +28,20 @@ inputs: description: "sccache version" default: "0.8.2" required: false +outputs: + env_vars: + description: "Environment variables to set" + value: | + AWS_PROFILE=sccache + CARGO_INCREMENTAL=0 + RUSTC_WRAPPER=sccache + SCCACHE_BUCKET=${{ inputs.bucket }} + SCCACHE_REGION=${{ inputs.region }} + SCCACHE_ENDPOINT=${{ inputs.endpoint }} + SCCACHE_S3_KEY_PREFIX=sccache/${{ inputs.platform }}/ + SCCACHE_VERSION=${{ inputs.version }} + CC="sccache cc" + CXX="sccache c++" # TODO: Cache deps here to save 1 minute runs: @@ -57,3 +71,5 @@ runs: echo "SCCACHE_S3_KEY_PREFIX=sccache/${{ inputs.platform }}/" >> $GITHUB_ENV # "SCCACHE_VERSION" is used inside Docker to install the same version of sccache echo "SCCACHE_VERSION=${{ inputs.version }}" >> $GITHUB_ENV + echo "CC=\"sccache cc\"" >> $GITHUB_ENV + echo "CXX=\"sccache c++\"" >> $GITHUB_ENV From d100822c2862d292e40d34ad4ab1f3fcbe46b912 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Tue, 26 Nov 2024 12:41:52 +0100 Subject: [PATCH 48/51] chore: use clang --- .github/actions/sccache/action.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/sccache/action.yaml b/.github/actions/sccache/action.yaml index ded663e562..9b1b76317b 100644 --- a/.github/actions/sccache/action.yaml +++ b/.github/actions/sccache/action.yaml @@ -71,5 +71,5 @@ runs: echo "SCCACHE_S3_KEY_PREFIX=sccache/${{ inputs.platform }}/" >> $GITHUB_ENV # "SCCACHE_VERSION" is used inside Docker to install the same version of sccache echo "SCCACHE_VERSION=${{ inputs.version }}" >> $GITHUB_ENV - echo "CC=\"sccache cc\"" >> $GITHUB_ENV - echo "CXX=\"sccache c++\"" >> $GITHUB_ENV + echo "CC=\"sccache clang\"" >> $GITHUB_ENV + echo "CXX=\"sccache clang++\"" >> $GITHUB_ENV From 5fbe5810cc9578714d690eb7e59ee38d8523a3e7 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Tue, 26 Nov 2024 12:55:27 +0100 Subject: [PATCH 49/51] chore: layert cache profile fix --- .github/actions/docker/action.yaml | 4 +++- .github/actions/s3-layer-cache-settings/action.yaml | 8 +++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/actions/docker/action.yaml b/.github/actions/docker/action.yaml index 7ffeb7fd2c..4af07c3d2b 100644 --- a/.github/actions/docker/action.yaml +++ b/.github/actions/docker/action.yaml @@ -170,7 +170,9 @@ runs: - name: Build and push Docker image ${{ inputs.image }} id: docker_build uses: docker/build-push-action@v6 - env: ${{ steps.layer_cache_settings.outputs.env_vars }} + env: + # AWS profile to be used by layer cache; sccache settings are passed via build-args + AWS_PROFILE: ${{ steps.layer_cache_settings.outputs.aws_profile }} with: context: . builder: ${{ steps.buildx.outputs.name }} diff --git a/.github/actions/s3-layer-cache-settings/action.yaml b/.github/actions/s3-layer-cache-settings/action.yaml index d163e59b57..f8a6ad1616 100644 --- a/.github/actions/s3-layer-cache-settings/action.yaml +++ b/.github/actions/s3-layer-cache-settings/action.yaml @@ -45,11 +45,9 @@ outputs: cache_from: description: "String with s3-based cache configuration for docker buildx cache-from option" value: ${{ steps.script.outputs.cache_from }} - env_vars: - description: "Environment variables to set before running docker buildx" - value: | - AWS_PROFILE=docker-layers - AWS_SHARED_CREDENTIALS_FILE=${HOME}/.aws/credentials + aws_profile: + description: "AWS profile to use for s3 cache, to set inside AWS_PROFILE env var" + value: docker-layers runs: using: composite From 6c1bb7a23c54e40525f05b862c742cf7f53735c6 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Tue, 26 Nov 2024 13:24:33 +0100 Subject: [PATCH 50/51] chore: fix docker --- .github/actions/aws_credentials/action.yaml | 14 ++++++++++---- .github/actions/docker/action.yaml | 9 ++++++++- .../actions/s3-layer-cache-settings/action.yaml | 10 ++++++---- .github/actions/sccache/action.yaml | 17 +++++++++++++---- 4 files changed, 37 insertions(+), 13 deletions(-) diff --git a/.github/actions/aws_credentials/action.yaml b/.github/actions/aws_credentials/action.yaml index 34bf11d913..afef5fba5c 100644 --- a/.github/actions/aws_credentials/action.yaml +++ b/.github/actions/aws_credentials/action.yaml @@ -1,8 +1,14 @@ --- -# This file contains configuration of aws credentials file. -# Its primary use is to prepare a credentials file that will be used as a secrets mount when building Docker images. name: "aws_credentials" -description: "Configure .aws/credentials" +description: | + Configure .aws/credentials file with provided access key ID and secret access key. + + This action creates a credentials file in ${HOME}/.aws/credentials with the provided access key ID and secret access key. + It also sets AWS_PROFILE and AWS_SHARED_CREDENTIALS_FILE environment variables to use this profile. + + It can conflict with other actions that define AWS credentials or set AWS_PROFILE env variable. + Explicitly set AWS_PROFILE=sccache and unset AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in case + of conflicting settings. inputs: access_key_id: description: Access key ID @@ -11,7 +17,7 @@ inputs: description: Secret access key required: true profile: - description: AWS profile to use + description: AWS profile to use; set AWS_PROFILE env variable to use this profile default: "default" runs: diff --git a/.github/actions/docker/action.yaml b/.github/actions/docker/action.yaml index 4af07c3d2b..495222f724 100644 --- a/.github/actions/docker/action.yaml +++ b/.github/actions/docker/action.yaml @@ -1,6 +1,13 @@ --- name: "Build and push docker image" -description: "Build and push docker image by digest with Rust caching" +description: | + Build and push docker image by digest with extensive caching. + + This action builds and pushes a Docker image to Docker Hub. + It uses caching for Rust dependencies and Docker layers. + It also provides sccache settings to the docker builder for caching Rust compilation. + + Layers cache and sccache will use the same credentials and S3 bucket, but different prefixes. inputs: image_name: description: Name of image in Docker Hub, like `drive` diff --git a/.github/actions/s3-layer-cache-settings/action.yaml b/.github/actions/s3-layer-cache-settings/action.yaml index f8a6ad1616..4260ee0659 100644 --- a/.github/actions/s3-layer-cache-settings/action.yaml +++ b/.github/actions/s3-layer-cache-settings/action.yaml @@ -6,6 +6,8 @@ description: | - name and head ref to hit all builds for this branch with this name - just name to hit all builds for this name + To correcly use caching, ensure buildx has AWS_PROFILE environment set to value of `aws_profile` output. + inputs: name: description: "Cache key name will be used as a prefix for all docker image manifests" @@ -47,7 +49,7 @@ outputs: value: ${{ steps.script.outputs.cache_from }} aws_profile: description: "AWS profile to use for s3 cache, to set inside AWS_PROFILE env var" - value: docker-layers + value: layers runs: using: composite @@ -55,9 +57,9 @@ runs: - name: Configure AWS credentials for s3 layers uses: ./.github/actions/aws_credentials with: - access_key_id: ${{ inputs.access_key_id }} - secret_access_key: ${{ inputs.secret_access_key }} - profile: "docker-layers" + access_key_id: ${{ inputs.s3_access_key_id }} + secret_access_key: ${{ inputs.s3_secret_access_key }} + profile: "layers" - uses: actions/github-script@v6 id: script diff --git a/.github/actions/sccache/action.yaml b/.github/actions/sccache/action.yaml index 9b1b76317b..84ab07f0f5 100644 --- a/.github/actions/sccache/action.yaml +++ b/.github/actions/sccache/action.yaml @@ -1,6 +1,14 @@ --- name: "sccache" -description: "Configure sccache caching" +description: | + Configure sccache caching. + + This action installs sccache and configures it to use an S3 bucket for caching. + It also sets environment variables to use when building Rust projects. + + It can conflict with other actions that define AWS credentials or set AWS_PROFILE env variable. + Manually set AWS_PROFILE=sccache and unset AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in case + of conflicting settings. inputs: bucket: description: S3 bucket to use for caching @@ -30,7 +38,7 @@ inputs: required: false outputs: env_vars: - description: "Environment variables to set" + description: "Environment variables set by this action" value: | AWS_PROFILE=sccache CARGO_INCREMENTAL=0 @@ -40,8 +48,8 @@ outputs: SCCACHE_ENDPOINT=${{ inputs.endpoint }} SCCACHE_S3_KEY_PREFIX=sccache/${{ inputs.platform }}/ SCCACHE_VERSION=${{ inputs.version }} - CC="sccache cc" - CXX="sccache c++" + CC="sccache clang" + CXX="sccache clang++" # TODO: Cache deps here to save 1 minute runs: @@ -63,6 +71,7 @@ runs: - name: Configure sccache shell: bash run: | + echo "AWS_PROFILE=sccache" >> $GITHUB_ENV echo "CARGO_INCREMENTAL=0" >> $GITHUB_ENV echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV echo "SCCACHE_BUCKET=${{ inputs.bucket }}" >> $GITHUB_ENV From b9102560a054c8a677c0c440f955524c898d721c Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Tue, 26 Nov 2024 13:27:52 +0100 Subject: [PATCH 51/51] chore: fix quotes --- .github/actions/sccache/action.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/sccache/action.yaml b/.github/actions/sccache/action.yaml index 84ab07f0f5..14954b1f20 100644 --- a/.github/actions/sccache/action.yaml +++ b/.github/actions/sccache/action.yaml @@ -80,5 +80,5 @@ runs: echo "SCCACHE_S3_KEY_PREFIX=sccache/${{ inputs.platform }}/" >> $GITHUB_ENV # "SCCACHE_VERSION" is used inside Docker to install the same version of sccache echo "SCCACHE_VERSION=${{ inputs.version }}" >> $GITHUB_ENV - echo "CC=\"sccache clang\"" >> $GITHUB_ENV - echo "CXX=\"sccache clang++\"" >> $GITHUB_ENV + echo "CC=sccache clang" >> $GITHUB_ENV + echo "CXX=sccache clang++" >> $GITHUB_ENV