diff --git a/.github/actions/aws_credentials/action.yaml b/.github/actions/aws_credentials/action.yaml new file mode 100644 index 0000000000..afef5fba5c --- /dev/null +++ b/.github/actions/aws_credentials/action.yaml @@ -0,0 +1,49 @@ +--- +name: "aws_credentials" +description: | + Configure .aws/credentials file with provided access key ID and secret access key. + + This action creates a credentials file in ${HOME}/.aws/credentials with the provided access key ID and secret access key. + It also sets AWS_PROFILE and AWS_SHARED_CREDENTIALS_FILE environment variables to use this profile. + + It can conflict with other actions that define AWS credentials or set AWS_PROFILE env variable. + Explicitly set AWS_PROFILE=sccache and unset AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in case + of conflicting settings. +inputs: + access_key_id: + description: Access key ID + required: true + secret_access_key: + description: Secret access key + required: true + profile: + description: AWS profile to use; set AWS_PROFILE env variable to use this profile + default: "default" + +runs: + using: composite + steps: + - name: Configure AWS credentials + shell: bash + run: | + mkdir -p "${HOME}/.aws" + cat >> ${HOME}/.aws/credentials << EOF + [${{ inputs.profile }}] + aws_access_key_id=${{ inputs.access_key_id }} + aws_secret_access_key=${{ inputs.secret_access_key }} + EOF + chmod -R go-rwx ${HOME}/.aws + + - name: Set env variables + shell: bash + run: | + # Exit on any error + set -euo pipefail + # Validate AWS_PROFILE is not empty + if [ -z "${{ inputs.profile }}" ]; then + echo "Error: AWS_PROFILE cannot be empty" + exit 1 + fi + # Export variables + echo "AWS_PROFILE=${{ inputs.profile }}" >> $GITHUB_ENV + echo "AWS_SHARED_CREDENTIALS_FILE=${HOME}/.aws/credentials" >> $GITHUB_ENV diff --git a/.github/actions/aws_ecr_login/action.yaml b/.github/actions/aws_ecr_login/action.yaml new file mode 100644 index 0000000000..fc93942222 --- /dev/null +++ b/.github/actions/aws_ecr_login/action.yaml @@ -0,0 +1,42 @@ +--- +# Login to AWS ECR +name: "aws_ecr_login" +description: "Login to AWS ECR to store Docker containers" +inputs: + aws_account_id: + description: AWS account ID (AWS_ACCOUNT_ID) + required: true + aws_access_key_id: + description: Access key ID (AWS_ACCESS_KEY_ID) + required: true + aws_secret_access_key: + description: Secret access key (AWS_SECRET_ACCESS_KEY) + required: true + aws_region: + description: AWS region to use (AWS_REGION) + required: true + +runs: + using: composite + steps: + - name: Configure AWS credentials and bucket region + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ inputs.aws_access_key_id }} + aws-secret-access-key: ${{ inputs.aws_secret_access_key }} + aws-region: ${{ inputs.aws_region }} + + - name: Login to ECR + run: | + aws ecr get-login-password \ + --region ${{ inputs.aws_region }} | docker login --username AWS --password-stdin ${{ inputs.aws_account_id }}.dkr.ecr.${{ inputs.aws_region }}.amazonaws.com + shell: bash + + # Unset AWS credentials to avoid conflicts, as we prefer credentials from ~/.aws/credentials to authenticate + - name: Unset AWS credentials to avoid conflicts + shell: bash + run: | + echo AWS_DEFAULT_REGION='' >> $GITHUB_ENV + echo AWS_REGION='' >> $GITHUB_ENV + echo AWS_ACCESS_KEY_ID='' >> $GITHUB_ENV + echo AWS_SECRET_ACCESS_KEY='' >> $GITHUB_ENV diff --git a/.github/actions/docker/action.yaml b/.github/actions/docker/action.yaml index 1e12427e9a..495222f724 100644 --- a/.github/actions/docker/action.yaml +++ b/.github/actions/docker/action.yaml @@ -1,6 +1,13 @@ --- name: "Build and push docker image" -description: "Build and push docker image by digest with Rust caching" +description: | + Build and push docker image by digest with extensive caching. + + This action builds and pushes a Docker image to Docker Hub. + It uses caching for Rust dependencies and Docker layers. + It also provides sccache settings to the docker builder for caching Rust compilation. + + Layers cache and sccache will use the same credentials and S3 bucket, but different prefixes. inputs: image_name: description: Name of image in Docker Hub, like `drive` @@ -25,21 +32,24 @@ inputs: cargo_profile: description: Cargo build profile, i.e release or dev default: dev - bucket: - description: S3 bucket to use for caching, must match runner define in `runs-on` - default: multi-runner-cache-x1xibo9c - region: + cache_bucket: + description: S3 bucket to use for caching (both sccache and layer cache) + required: true + cache_region: description: S3 bucket region required: true - aws_access_key_id: - description: AWS access key ID + cache_endpoint: + description: S3 endpoint to use for caching + required: true + cache_access_key_id: + description: Access key ID for s3 cache required: true - aws_secret_access_key: - description: AWS secret access key + cache_secret_access_key: + description: Secret access key for s3 cache required: true cache_to_name: - description: 'Save cache to name manifest (should be used only on default branch)' - default: 'false' + description: "Save cache to name manifest (should be used only on default branch)" + default: "false" outputs: digest: value: ${{ steps.docker_build.outputs.digest }} @@ -80,9 +90,13 @@ runs: id: layer_cache_settings with: name: ${{ inputs.image_name }} - region: ${{ inputs.region }} - bucket: ${{ inputs.bucket }} + region: ${{ inputs.cache_region }} + bucket: ${{ inputs.cache_bucket }} + endpoint: ${{ inputs.cache_endpoint }} + prefix: "cache-layers/${{ inputs.platform }}/" cache_to_name: ${{ inputs.cache_to_name }} + s3_access_key_id: ${{ inputs.cache_access_key_id }} + s3_secret_access_key: ${{ inputs.cache_secret_access_key }} - name: Set HOME variable to github context shell: bash @@ -133,7 +147,7 @@ runs: id: arch uses: actions/github-script@v6 with: - result-encoding: 'string' + result-encoding: "string" script: return '${{ inputs.platform }}'.replace('linux/', ''); - name: Inject cargo cache into docker @@ -148,9 +162,24 @@ runs: } skip-extraction: ${{ steps.yarn-cache.outputs.cache-hit }} + - name: Configure sccache settings + uses: ./.github/actions/sccache + id: sccache + with: + bucket: ${{ inputs.cache_bucket }} + region: ${{ inputs.cache_region }} + endpoint: ${{ inputs.cache_endpoint }} + access_key_id: ${{ inputs.cache_access_key_id }} + secret_access_key: ${{ inputs.cache_secret_access_key }} + platform: ${{ inputs.platform }} + install: false + - name: Build and push Docker image ${{ inputs.image }} id: docker_build uses: docker/build-push-action@v6 + env: + # AWS profile to be used by layer cache; sccache settings are passed via build-args + AWS_PROFILE: ${{ steps.layer_cache_settings.outputs.aws_profile }} with: context: . builder: ${{ steps.buildx.outputs.name }} @@ -159,14 +188,9 @@ runs: push: ${{ inputs.push_tags }} tags: ${{ inputs.push_tags == 'true' && steps.docker_meta.outputs.tags || '' }} platforms: ${{ inputs.platform }} - build-args: | - CARGO_BUILD_PROFILE=${{ inputs.cargo_profile }} - RUSTC_WRAPPER=sccache - SCCACHE_BUCKET=${{ inputs.bucket }} - SCCACHE_REGION=${{ inputs.region }} - SCCACHE_S3_KEY_PREFIX=${{ runner.os }}/sccache - AWS_ACCESS_KEY_ID=${{ inputs.aws_access_key_id }} - AWS_SECRET_ACCESS_KEY=${{ inputs.aws_secret_access_key }} + secret-files: | + AWS=${{ env.HOME }}/.aws/credentials + build-args: ${{ steps.sccache.outputs.env_vars }} cache-from: ${{ steps.layer_cache_settings.outputs.cache_from }} cache-to: ${{ steps.layer_cache_settings.outputs.cache_to }} outputs: type=image,name=${{ inputs.image_org }}/${{ inputs.image_name }},push-by-digest=${{ inputs.push_tags != 'true' }},name-canonical=true,push=true diff --git a/.github/actions/librocksdb/action.yaml b/.github/actions/librocksdb/action.yaml index a73666584e..217e2745eb 100644 --- a/.github/actions/librocksdb/action.yaml +++ b/.github/actions/librocksdb/action.yaml @@ -9,10 +9,6 @@ inputs: description: RocksDB version, eg. "8.10.2" required: false default: "8.10.2" - bucket: - description: S3 bucket to use for caching - required: false - default: multi-runner-cache-x1xibo9c force: description: Force rebuild required: false diff --git a/.github/actions/rust/action.yaml b/.github/actions/rust/action.yaml index 3b74e2102b..ff26db3fad 100644 --- a/.github/actions/rust/action.yaml +++ b/.github/actions/rust/action.yaml @@ -31,7 +31,7 @@ runs: fi echo "TOOLCHAIN_VERSION=$TOOLCHAIN_VERSION" >> $GITHUB_ENV - echo "::set-output name=version::$TOOLCHAIN_VERSION" + echo "version=$TOOLCHAIN_VERSION" >> $GITHUB_OUTPUT - uses: dtolnay/rust-toolchain@master name: Install Rust toolchain @@ -82,12 +82,6 @@ runs: echo "PROTOC=${HOME}/.local/bin/protoc" >> $GITHUB_ENV export PATH="${PATH}:${HOME}/.local/bin" - - name: Install sccache-cache - uses: mozilla-actions/sccache-action@v0.0.6 - with: - version: "v0.8.2" # Must be the same as in Dockerfile - if: inputs.cache == 'true' - - name: Set HOME variable to github context shell: bash run: echo "HOME=$HOME" >> $GITHUB_ENV diff --git a/.github/actions/s3-layer-cache-settings/action.yaml b/.github/actions/s3-layer-cache-settings/action.yaml index 49cdaeef66..4260ee0659 100644 --- a/.github/actions/s3-layer-cache-settings/action.yaml +++ b/.github/actions/s3-layer-cache-settings/action.yaml @@ -1,4 +1,4 @@ -name: 'Get S3 Docker Layer Cache settings' +name: "Get S3 Docker Layer Cache settings" description: | This action generates string with s3-based cache configuration for docker buildx. It defines three manifests: @@ -6,12 +6,14 @@ description: | - name and head ref to hit all builds for this branch with this name - just name to hit all builds for this name + To correcly use caching, ensure buildx has AWS_PROFILE environment set to value of `aws_profile` output. + inputs: name: - description: 'Cache key name will be used as a prefix for all docker image manifests' + description: "Cache key name will be used as a prefix for all docker image manifests" required: true head_ref: - description: 'Head ref for an additional manifest to hit all builds for this head' + description: "Head ref for an additional manifest to hit all builds for this head" default: ${{ github.ref }} region: description: S3 region @@ -19,27 +21,46 @@ inputs: bucket: description: S3 bucket name required: true + endpoint: + description: S3 endpoint to use for caching + required: false prefix: description: S3 key prefix - default: 'cache-layers/' + default: "cache-layers/" + s3_access_key_id: + description: Access key ID for S3 cache + required: true + s3_secret_access_key: + description: Secret access key for S3 cache + required: true mode: description: Cache mode default: max cache_to_name: - description: 'Save cache to name manifest (should be used only on default branch)' - default: 'false' + description: "Save cache to name manifest (should be used only on default branch)" + default: "false" outputs: cache_to: - description: 'String with s3-based cache configuration for docker buildx cache-to option' + description: "String with s3-based cache configuration for docker buildx cache-to option" value: ${{ steps.script.outputs.cache_to }} cache_from: - description: 'String with s3-based cache configuration for docker buildx cache-from option' + description: "String with s3-based cache configuration for docker buildx cache-from option" value: ${{ steps.script.outputs.cache_from }} + aws_profile: + description: "AWS profile to use for s3 cache, to set inside AWS_PROFILE env var" + value: layers runs: using: composite steps: + - name: Configure AWS credentials for s3 layers + uses: ./.github/actions/aws_credentials + with: + access_key_id: ${{ inputs.s3_access_key_id }} + secret_access_key: ${{ inputs.s3_secret_access_key }} + profile: "layers" + - uses: actions/github-script@v6 id: script with: @@ -49,6 +70,7 @@ runs: region: '${{ inputs.region }}', bucket: '${{ inputs.bucket }}', prefix: '${{ inputs.prefix }}', + endpoint_url: '${{ inputs.endpoint }}', }; const settingsString = Object.entries(settings) @@ -59,7 +81,7 @@ runs: const sanitizedHeadRef = '${{ inputs.head_ref }}'.replace(/[^a-zA-Z0-9]/g, '-'); const shaManifestName = '${{ inputs.name }}_sha_${{ github.sha }}'; - const headRefManifestName = '${{ inputs.name }}_tag_${ sanitizedHeadRef }'; + const headRefManifestName = '${{ inputs.name }}_tag_' + sanitizedHeadRef; const cacheFromManifestNames = [ shaManifestName, diff --git a/.github/actions/sccache/action.yaml b/.github/actions/sccache/action.yaml new file mode 100644 index 0000000000..14954b1f20 --- /dev/null +++ b/.github/actions/sccache/action.yaml @@ -0,0 +1,84 @@ +--- +name: "sccache" +description: | + Configure sccache caching. + + This action installs sccache and configures it to use an S3 bucket for caching. + It also sets environment variables to use when building Rust projects. + + It can conflict with other actions that define AWS credentials or set AWS_PROFILE env variable. + Manually set AWS_PROFILE=sccache and unset AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in case + of conflicting settings. +inputs: + bucket: + description: S3 bucket to use for caching + required: true + region: + description: S3 bucket region + required: true + endpoint: + description: S3 endpoint to use for caching + required: true + access_key_id: + description: S3 endpoint access key ID + required: true + secret_access_key: + description: S3 endpoint secret access key + required: true + platform: + description: "Platform and architecture to use when caching; defaults to linux/amd64" + required: false + default: "linux/amd64" + install: + description: "Install sccache" + default: "true" + version: + description: "sccache version" + default: "0.8.2" + required: false +outputs: + env_vars: + description: "Environment variables set by this action" + value: | + AWS_PROFILE=sccache + CARGO_INCREMENTAL=0 + RUSTC_WRAPPER=sccache + SCCACHE_BUCKET=${{ inputs.bucket }} + SCCACHE_REGION=${{ inputs.region }} + SCCACHE_ENDPOINT=${{ inputs.endpoint }} + SCCACHE_S3_KEY_PREFIX=sccache/${{ inputs.platform }}/ + SCCACHE_VERSION=${{ inputs.version }} + CC="sccache clang" + CXX="sccache clang++" + +# TODO: Cache deps here to save 1 minute +runs: + using: composite + steps: + - name: Install sccache binary + if: ${{ inputs.install == 'true' }} + uses: mozilla-actions/sccache-action@v0.0.6 + with: + version: "v${{ inputs.version }}" + + - name: Configure AWS credentials + uses: ./.github/actions/aws_credentials + with: + access_key_id: ${{ inputs.access_key_id }} + secret_access_key: ${{ inputs.secret_access_key }} + profile: "sccache" + + - name: Configure sccache + shell: bash + run: | + echo "AWS_PROFILE=sccache" >> $GITHUB_ENV + echo "CARGO_INCREMENTAL=0" >> $GITHUB_ENV + echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV + echo "SCCACHE_BUCKET=${{ inputs.bucket }}" >> $GITHUB_ENV + echo "SCCACHE_REGION=${{ inputs.region }}" >> $GITHUB_ENV + echo "SCCACHE_ENDPOINT=${{ inputs.endpoint }}" >> $GITHUB_ENV + echo "SCCACHE_S3_KEY_PREFIX=sccache/${{ inputs.platform }}/" >> $GITHUB_ENV + # "SCCACHE_VERSION" is used inside Docker to install the same version of sccache + echo "SCCACHE_VERSION=${{ inputs.version }}" >> $GITHUB_ENV + echo "CC=sccache clang" >> $GITHUB_ENV + echo "CXX=sccache clang++" >> $GITHUB_ENV diff --git a/.github/workflows/release-docker-image.yml b/.github/workflows/release-docker-image.yml index 425b8a5f44..728bd7e94c 100644 --- a/.github/workflows/release-docker-image.yml +++ b/.github/workflows/release-docker-image.yml @@ -47,13 +47,6 @@ jobs: with: fetch-depth: 0 - - name: Configure AWS credentials and bucket region - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_REGION }} - - name: Set up QEMU uses: docker/setup-qemu-action@v3 if: ${{ matrix.platform == 'linux/arm64' }} @@ -70,9 +63,11 @@ jobs: cargo_profile: ${{ inputs.cargo_profile }} dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }} dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }} - region: ${{ secrets.AWS_REGION }} - aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + cache_region: ${{ vars.CACHE_REGION }} + cache_bucket: ${{ vars.CACHE_S3_BUCKET }} + cache_endpoint: ${{ vars.CACHE_S3_ENDPOINT }} + cache_access_key_id: ${{ secrets.CACHE_KEY_ID }} + cache_secret_access_key: ${{ secrets.CACHE_SECRET_KEY }} - name: Export digest run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 86476d082a..bb165d3918 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -33,13 +33,6 @@ jobs: env: TAG_PREFIX: v - - name: Configure AWS credentials and bucket region - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_REGION }} - - uses: softwareforgood/check-artifact-v4-existence@v0 id: check-artifact with: @@ -58,6 +51,17 @@ jobs: target: wasm32-unknown-unknown if: ${{ steps.check-artifact.outputs.exists != 'true' }} + - name: Setup sccache + uses: ./.github/actions/sccache + with: + bucket: ${{ vars.CACHE_S3_BUCKET }} + region: ${{ vars.AWS_REGION }} + endpoint: ${{ vars.CACHE_S3_ENDPOINT }} + access_key_id: ${{ secrets.CACHE_KEY_ID }} + secret_access_key: ${{ secrets.CACHE_SECRET_KEY }} + + if: ${{ steps.check-artifact.outputs.exists != 'true' }} + - name: Setup Node.JS uses: ./.github/actions/nodejs @@ -73,10 +77,7 @@ jobs: run: yarn build env: CARGO_BUILD_PROFILE: release - RUSTC_WRAPPER: sccache - SCCACHE_BUCKET: multi-runner-cache-x1xibo9c - SCCACHE_REGION: ${{ secrets.AWS_REGION }} - SCCACHE_S3_KEY_PREFIX: ${{ runner.os }}/sccache/wasm/wasm32 + if: ${{ steps.check-artifact.outputs.exists != 'true' }} - name: Set suffix @@ -230,13 +231,6 @@ jobs: with: fetch-depth: 0 - - name: Configure AWS credentials and bucket region - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-region: ${{ secrets.AWS_REGION }} - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - - name: Download JS build artifacts uses: actions/download-artifact@v4 with: diff --git a/.github/workflows/tests-build-image.yml b/.github/workflows/tests-build-image.yml index 564feff055..dc6efffa47 100644 --- a/.github/workflows/tests-build-image.yml +++ b/.github/workflows/tests-build-image.yml @@ -24,28 +24,27 @@ jobs: with: fetch-depth: 0 - - name: Configure AWS credentials and bucket region - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_REGION }} - - name: Login to ECR - run: aws ecr get-login-password --region ${{ secrets.AWS_REGION }} | docker login --username AWS --password-stdin ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com - shell: bash + uses: ./.github/actions/aws_ecr_login + with: + aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws_region: ${{ vars.AWS_REGION }} + aws_account_id: ${{ secrets.AWS_ACCOUNT_ID }} - name: Build and push by SHA uses: ./.github/actions/docker with: image_name: ${{ inputs.image_name }} - image_org: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com + image_org: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_REGION }}.amazonaws.com target: ${{ inputs.target }} platform: linux/amd64 push_tags: true dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }} dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }} - region: ${{ secrets.AWS_REGION }} - aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + cache_region: ${{ vars.CACHE_REGION }} + cache_bucket: ${{ vars.CACHE_S3_BUCKET }} + cache_endpoint: ${{ vars.CACHE_S3_ENDPOINT }} + cache_access_key_id: ${{ secrets.CACHE_KEY_ID }} + cache_secret_access_key: ${{ secrets.CACHE_SECRET_KEY }} cache_to_name: ${{ github.event_name == 'push' && 'true' || 'false' }} diff --git a/.github/workflows/tests-build-js.yml b/.github/workflows/tests-build-js.yml index 83909e0c31..1c73612e22 100644 --- a/.github/workflows/tests-build-js.yml +++ b/.github/workflows/tests-build-js.yml @@ -6,13 +6,6 @@ jobs: name: Build JS runs-on: ubuntu-24.04 steps: - - name: Configure AWS credentials and bucket region - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_REGION }} - - uses: softwareforgood/check-artifact-v4-existence@v0 id: check-artifact with: @@ -41,6 +34,16 @@ jobs: target: wasm32-unknown-unknown if: ${{ steps.check-artifact.outputs.exists != 'true' }} + - name: Setup sccache + uses: ./.github/actions/sccache + with: + bucket: ${{ vars.CACHE_S3_BUCKET }} + region: ${{ vars.CACHE_REGION }} + endpoint: ${{ vars.CACHE_S3_ENDPOINT }} + access_key_id: ${{ secrets.CACHE_KEY_ID }} + secret_access_key: ${{ secrets.CACHE_SECRET_KEY }} + if: ${{ steps.check-artifact.outputs.exists != 'true' }} + - name: Install Cargo binstall uses: cargo-bins/cargo-binstall@v1.3.1 if: ${{ steps.check-artifact.outputs.exists != 'true' }} @@ -51,11 +54,6 @@ jobs: - name: Build JS packages run: yarn build - env: - RUSTC_WRAPPER: sccache - SCCACHE_BUCKET: multi-runner-cache-x1xibo9c - SCCACHE_REGION: ${{ secrets.AWS_REGION }} - SCCACHE_S3_KEY_PREFIX: ${{ runner.os }}/sccache/wasm/wasm32 if: ${{ steps.check-artifact.outputs.exists != 'true' }} - name: Ignore only already cached artifacts @@ -87,4 +85,3 @@ jobs: if-no-files-found: error include-hidden-files: true if: ${{ steps.check-artifact.outputs.exists != 'true' }} - diff --git a/.github/workflows/tests-codeql.yml b/.github/workflows/tests-codeql.yml index 034fd545b8..ed972e6d9c 100644 --- a/.github/workflows/tests-codeql.yml +++ b/.github/workflows/tests-codeql.yml @@ -20,13 +20,6 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Configure AWS credentials and bucket region - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_REGION }} - - name: Setup Node.JS uses: ./.github/actions/nodejs diff --git a/.github/workflows/tests-dashmate.yml b/.github/workflows/tests-dashmate.yml index a451381b9c..170006c22a 100644 --- a/.github/workflows/tests-dashmate.yml +++ b/.github/workflows/tests-dashmate.yml @@ -31,12 +31,13 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Configure AWS credentials and bucket region - uses: aws-actions/configure-aws-credentials@v4 + - name: Login to ECR + uses: ./.github/actions/aws_ecr_login with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_REGION }} + aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws_region: ${{ vars.AWS_REGION }} + aws_account_id: ${{ secrets.AWS_ACCOUNT_ID }} - name: Setup Node.JS uses: ./.github/actions/nodejs @@ -49,11 +50,10 @@ jobs: - name: Replace with pre-built images run: | - set -x + set -e # Login to ECR - DOCKER_HUB_ORG="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com" - aws ecr get-login-password --region ${{ secrets.AWS_REGION }} | docker login --username AWS --password-stdin $DOCKER_HUB_ORG + DOCKER_HUB_ORG="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_REGION }}.amazonaws.com" SHA_TAG=sha-${{ github.sha }} @@ -115,4 +115,3 @@ jobs: - name: Show Docker logs if: ${{ failure() }} uses: jwalton/gh-docker-logs@v2 - diff --git a/.github/workflows/tests-js-package.yml b/.github/workflows/tests-js-package.yml index e71d9d85d1..681c27b560 100644 --- a/.github/workflows/tests-js-package.yml +++ b/.github/workflows/tests-js-package.yml @@ -25,13 +25,6 @@ jobs: - name: Check out repo uses: actions/checkout@v4 - - name: Configure AWS credentials and bucket region - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_REGION }} - - name: Setup Node.JS uses: ./.github/actions/nodejs @@ -57,13 +50,6 @@ jobs: with: fetch-depth: 0 - - name: Configure AWS credentials and bucket region - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-region: ${{ secrets.AWS_REGION }} - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - - name: Setup Node.JS uses: ./.github/actions/nodejs diff --git a/.github/workflows/tests-packges-functional.yml b/.github/workflows/tests-packges-functional.yml index 39db41b53f..dcd99f7581 100644 --- a/.github/workflows/tests-packges-functional.yml +++ b/.github/workflows/tests-packges-functional.yml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 15 env: - ECR_HOST: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com + ECR_HOST: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_REGION }}.amazonaws.com steps: - name: Check out repo uses: actions/checkout@v4 @@ -20,15 +20,13 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Configure AWS credentials and bucket region - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_REGION }} - - name: Login to ECR - run: aws ecr get-login-password --region ${{ secrets.AWS_REGION }} | docker login --username AWS --password-stdin ${{ env.ECR_HOST }} + uses: ./.github/actions/aws_ecr_login + with: + aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws_region: ${{ vars.AWS_REGION }} + aws_account_id: ${{ secrets.AWS_ACCOUNT_ID }} - name: Start local network uses: ./.github/actions/local-network diff --git a/.github/workflows/tests-rs-package.yml b/.github/workflows/tests-rs-package.yml index b37d7b9a3f..3696e7e9db 100644 --- a/.github/workflows/tests-rs-package.yml +++ b/.github/workflows/tests-rs-package.yml @@ -22,18 +22,20 @@ jobs: - name: Check out repo uses: actions/checkout@v4 - - name: Configure AWS credentials and bucket region - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_REGION }} - - name: Setup Rust uses: ./.github/actions/rust with: components: clippy + - name: Setup sccache + uses: ./.github/actions/sccache + with: + bucket: ${{ vars.CACHE_S3_BUCKET }} + region: ${{ vars.CACHE_REGION }} + endpoint: ${{ vars.CACHE_S3_ENDPOINT }} + access_key_id: ${{ secrets.CACHE_KEY_ID }} + secret_access_key: ${{ secrets.CACHE_SECRET_KEY }} + - name: Install librocksdb uses: ./.github/actions/librocksdb @@ -41,10 +43,6 @@ jobs: with: args: --package ${{ inputs.package }} --all-features --locked -- --no-deps env: - RUSTC_WRAPPER: sccache - SCCACHE_BUCKET: multi-runner-cache-x1xibo9c - SCCACHE_REGION: ${{ secrets.AWS_REGION }} - SCCACHE_S3_KEY_PREFIX: ${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu ROCKSDB_STATIC: "/opt/rocksdb/usr/local/lib/librocksdb.a" ROCKSDB_LIB_DIR: "/opt/rocksdb/usr/local/lib" SNAPPY_STATIC: "/usr/lib/x86_64-linux-gnu/libsnappy.a" @@ -63,14 +61,10 @@ jobs: components: rustfmt cache: false - # This step doesn't need librocksdb, so we don't install it + # We don't use cache for this step, nothing to cache here + # This step doesn't need librocksdb, so we don't install it - name: Check formatting - env: - RUSTC_WRAPPER: sccache - SCCACHE_BUCKET: multi-runner-cache-x1xibo9c - SCCACHE_REGION: ${{ secrets.AWS_REGION }} - SCCACHE_S3_KEY_PREFIX: ${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu run: cargo fmt --check --package=${{ inputs.package }} unused_deps: @@ -84,16 +78,17 @@ jobs: - name: Check out repo uses: actions/checkout@v4 - - name: Configure AWS credentials and bucket region - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-region: ${{ secrets.AWS_REGION }} - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - - name: Setup Rust uses: ./.github/actions/rust + - name: Setup sccache + uses: ./.github/actions/sccache + with: + bucket: ${{ vars.CACHE_S3_BUCKET }} + region: ${{ vars.CACHE_REGION }} + endpoint: ${{ vars.CACHE_S3_ENDPOINT }} + access_key_id: ${{ secrets.CACHE_KEY_ID }} + secret_access_key: ${{ secrets.CACHE_SECRET_KEY }} - name: Install librocksdb uses: ./.github/actions/librocksdb @@ -106,10 +101,6 @@ jobs: - name: Find unused dependencies uses: lklimek/cargo-machete@feat/workdir env: - RUSTC_WRAPPER: sccache - SCCACHE_BUCKET: multi-runner-cache-x1xibo9c - SCCACHE_REGION: ${{ secrets.AWS_REGION }} - SCCACHE_S3_KEY_PREFIX: ${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu ROCKSDB_STATIC: "/opt/rocksdb/usr/local/lib/librocksdb.a" ROCKSDB_LIB_DIR: "/opt/rocksdb/usr/local/lib" SNAPPY_STATIC: "/usr/lib/x86_64-linux-gnu/libsnappy.a" @@ -186,25 +177,24 @@ jobs: - name: Check out repo uses: actions/checkout@v4 - - name: Configure AWS credentials and bucket region - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-region: ${{ secrets.AWS_REGION }} - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - - name: Setup Rust uses: ./.github/actions/rust + - name: Setup sccache + uses: ./.github/actions/sccache + with: + bucket: ${{ vars.CACHE_S3_BUCKET }} + region: ${{ vars.CACHE_REGION }} + endpoint: ${{ vars.CACHE_S3_ENDPOINT }} + access_key_id: ${{ secrets.CACHE_KEY_ID }} + secret_access_key: ${{ secrets.CACHE_SECRET_KEY }} + - name: Install librocksdb uses: ./.github/actions/librocksdb - name: Run tests run: cargo test --package=${{ inputs.package }} --all-features --locked env: - RUSTC_WRAPPER: sccache - SCCACHE_BUCKET: multi-runner-cache-x1xibo9c - SCCACHE_REGION: ${{ secrets.AWS_REGION }} SCCACHE_S3_KEY_PREFIX: ${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu ROCKSDB_STATIC: "/opt/rocksdb/usr/local/lib/librocksdb.a" ROCKSDB_LIB_DIR: "/opt/rocksdb/usr/local/lib" @@ -220,16 +210,18 @@ jobs: - name: Check out repo uses: actions/checkout@v4 - - name: Configure AWS credentials and bucket region - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-region: ${{ secrets.AWS_REGION }} - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - - name: Setup Rust uses: ./.github/actions/rust + - name: Setup sccache + uses: ./.github/actions/sccache + with: + bucket: ${{ vars.CACHE_S3_BUCKET }} + region: ${{ vars.CACHE_REGION }} + endpoint: ${{ vars.CACHE_S3_ENDPOINT }} + access_key_id: ${{ secrets.CACHE_KEY_ID }} + secret_access_key: ${{ secrets.CACHE_SECRET_KEY }} + - name: Install librocksdb uses: ./.github/actions/librocksdb @@ -241,10 +233,6 @@ jobs: - name: Check each feature in ${{ inputs.package }} env: - RUSTC_WRAPPER: sccache - SCCACHE_BUCKET: multi-runner-cache-x1xibo9c - SCCACHE_REGION: ${{ secrets.AWS_REGION }} - SCCACHE_S3_KEY_PREFIX: ${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu ROCKSDB_STATIC: "/opt/rocksdb/usr/local/lib/librocksdb.a" ROCKSDB_LIB_DIR: "/opt/rocksdb/usr/local/lib" SNAPPY_STATIC: "/usr/lib/x86_64-linux-gnu/libsnappy.a" @@ -252,6 +240,7 @@ jobs: run: | echo Verify all features disabled set -ex + features="${{ steps.crate_info.outputs.features }}" fails="" RUSTFLAGS="-D warnings" diff --git a/.github/workflows/tests-test-suite.yml b/.github/workflows/tests-test-suite.yml index e9107b5e60..698e5ea153 100644 --- a/.github/workflows/tests-test-suite.yml +++ b/.github/workflows/tests-test-suite.yml @@ -25,7 +25,7 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 15 env: - ECR_HOST: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com + ECR_HOST: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_REGION }}.amazonaws.com steps: - name: Check out repo uses: actions/checkout@v4 @@ -38,15 +38,13 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Configure AWS credentials and bucket region - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_REGION }} - - name: Login to ECR - run: aws ecr get-login-password --region ${{ secrets.AWS_REGION }} | docker login --username AWS --password-stdin ${{ env.ECR_HOST }} + uses: ./.github/actions/aws_ecr_login + with: + aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws_region: ${{ vars.AWS_REGION }} + aws_account_id: ${{ secrets.AWS_ACCOUNT_ID }} - name: Start local network uses: ./.github/actions/local-network @@ -56,8 +54,8 @@ jobs: - name: Run test suite run: yarn workspace @dashevo/platform-test-suite ${{ inputs.command }} env: - BROWSER_TEST_BATCH_INDEX: ${{ inputs.batch_index }} - BROWSER_TEST_BATCH_TOTAL: ${{ inputs.batch_total }} + BROWSER_TEST_BATCH_INDEX: ${{ inputs.batch_index }} + BROWSER_TEST_BATCH_TOTAL: ${{ inputs.batch_total }} - name: Show Docker logs if: ${{ failure() }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c26e8e5289..4cf511cfbb 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -93,6 +93,15 @@ jobs: - name: Check out repo uses: actions/checkout@v4 + - name: Setup sccache + uses: ./.github/actions/sccache + with: + bucket: ${{ vars.CACHE_S3_BUCKET }} + region: ${{ vars.CACHE_REGION }} + endpoint: ${{ vars.CACHE_S3_ENDPOINT }} + access_key_id: ${{ secrets.CACHE_KEY_ID }} + secret_access_key: ${{ secrets.CACHE_SECRET_KEY }} + - name: Audit crates uses: rustsec/audit-check@v1 with: diff --git a/Dockerfile b/Dockerfile index c729f240d2..0236a9e20e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,16 +15,26 @@ # The following build arguments can be provided using --build-arg: # - CARGO_BUILD_PROFILE - set to `release` to build final binary, without debugging information # - NODE_ENV - node.js environment name to use to build the library -# - RUSTC_WRAPPER - set to `sccache` to enable sccache support and make the following variables available: -# - SCCACHE_GHA_ENABLED, ACTIONS_CACHE_URL, ACTIONS_RUNTIME_TOKEN - store sccache caches inside github actions -# - SCCACHE_BUCKET, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_REGION, SCCACHE_S3_KEY_PREFIX - store caches in S3 -# - SCCACHE_MEMCACHED - set to memcache server URI (eg. tcp://172.17.0.1:11211) to enable sccache memcached backend # - ALPINE_VERSION - use different version of Alpine base image; requires also rust:apline... # image to be available # - USERNAME, USER_UID, USER_GID - specification of user used to run the binary # +# # sccache cache backends # -# +# To enable sccache support and make the following variables available: +# 1. For S3 buckets: +# - SCCACHE_BUCKET - S3 bucket name +# - AWS_PROFILE +# - SCCACHE_REGION +# - SCCACHE_S3_KEY_PREFIX +# - SCCACHE_ENDPOINT +# - also, AWS credentials file ($HOME/.aws/credentials) should be provided as a secret file with id=AWS +# 2. For Github Actions: +# - SCCACHE_GHA_ENABLED, ACTIONS_CACHE_URL +# - also, Github Actions token should be provided as a secret file with id=GHA +# 3. For memcached: +# - SCCACHE_MEMCACHED - set to memcache server URI (eg. tcp://172.17.0.1:11211) to enable sccache memcached backend + # # BUILD PROCESS # @@ -35,7 +45,17 @@ # 3. Configuration variables are shared between runs using /root/env file. ARG ALPINE_VERSION=3.18 -ARG RUSTC_WRAPPER + +# deps-${RUSTC_WRAPPER:-base} +# If one of SCCACHE_GHA_ENABLED, SCCACHE_BUCKET, SCCACHE_MEMCACHED is set, then deps-sccache is used, otherwise deps-base +ARG SCCACHE_GHA_ENABLED +ARG SCCACHE_BUCKET +ARG SCCACHE_MEMCACHED + +# Determine if we have sccache enabled; if yes, use deps-sccache, otherwise use deps-base as a dependency image +ARG DEPS_IMAGE=${SCCACHE_GHA_ENABLED}${SCCACHE_BUCKET}${SCCACHE_MEMCACHED} +ARG DEPS_IMAGE=${DEPS_IMAGE:+sccache} +ARG DEPS_IMAGE=deps-${DEPS_IMAGE:-base} # # DEPS: INSTALL AND CACHE DEPENDENCIES @@ -83,7 +103,6 @@ ARG TARGETARCH WORKDIR /platform -# TODO: It doesn't sharing PATH between stages, so we need "source $HOME/.cargo/env" everywhere COPY rust-toolchain.toml . RUN TOOLCHAIN_VERSION="$(grep channel rust-toolchain.toml | awk '{print $3}' | tr -d '"')" && \ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- \ @@ -95,6 +114,10 @@ RUN TOOLCHAIN_VERSION="$(grep channel rust-toolchain.toml | awk '{print $3}' | t ONBUILD ENV HOME=/root ONBUILD ENV CARGO_HOME=$HOME/.cargo +# Configure Rust toolchain +# It doesn't sharing PATH between stages, so we need "source $HOME/.cargo/env" everywhere +RUN echo 'source $HOME/.cargo/env' >> /root/env + # Install protoc - protobuf compiler # The one shipped with Alpine does not work ARG PROTOC_VERSION=27.3 @@ -119,10 +142,10 @@ ENV NODE_ENV=${NODE_ENV} # # This stage is used to install sccache and configure it. # Later on, one should source /root/env before building to use sccache. - +# # Note that, due to security concerns, each stage needs to declare variables containing authentication secrets, like -# ACTIONS_RUNTIME_TOKEN, AWS_SECRET_ACCESS_KEY. It is done using ONBUILD directive, so the secrets are not stored in the -# final image. +# ACTIONS_RUNTIME_TOKEN, AWS_SECRET_ACCESS_KEY. This is to prevent leaking secrets to the final image. The secrets are +# loaded using docker buildx `--secret` flag and need to be explicitly mounted with `--mount=type=secret,id=SECRET_ID`. FROM deps-base AS deps-sccache @@ -139,10 +162,6 @@ RUN if [[ "$TARGETARCH" == "arm64" ]] ; then export SCC_ARCH=aarch64; else expor # # Configure sccache # -ARG RUSTC_WRAPPER - -# Disable incremental builds, not supported by sccache -RUN echo 'export CARGO_INCREMENTAL=false' >> /root/env # Set args below to use Github Actions cache; see https://github.com/mozilla/sccache/blob/main/docs/GHA.md ARG SCCACHE_GHA_ENABLED @@ -153,57 +172,67 @@ ARG SCCACHE_MEMCACHED # S3 storage ARG SCCACHE_BUCKET -ARG AWS_ACCESS_KEY_ID -ARG AWS_REGION +ARG AWS_PROFILE ARG SCCACHE_REGION ARG SCCACHE_S3_KEY_PREFIX +ARG SCCACHE_ENDPOINT # Generate sccache configuration variables and save them to /root/env # # We only enable one cache at a time. Setting env variables belonging to multiple cache backends may fail the build. -RUN <> /root/env echo "export ACTIONS_CACHE_URL=${ACTIONS_CACHE_URL}" >> /root/env - # ACTIONS_RUNTIME_TOKEN is a secret so we load it using ONBUILD ARG later on + # ACTIONS_RUNTIME_TOKEN is a secret so we quote it here, and it will be loaded when `source /root/env` is run + echo 'export ACTIONS_RUNTIME_TOKEN="$(cat /run/secrets/GHA)"' >> /root/env + + ### AWS S3 ### elif [ -n "${SCCACHE_BUCKET}" ]; then - # AWS S3 - if [ -z "${SCCACHE_REGION}" ] ; then - # Default to AWS_REGION if not set - export SCCACHE_REGION=${AWS_REGION} - fi - - echo "export AWS_REGION='${AWS_REGION}'" >> /root/env - echo "export SCCACHE_REGION='${SCCACHE_REGION}'" >> /root/env - echo "export AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}" >> /root/env - # AWS_SECRET_ACCESS_KEY is a secret so we load it using ONBUILD ARG later on echo "export SCCACHE_BUCKET='${SCCACHE_BUCKET}'" >> /root/env - echo "export SCCACHE_S3_KEY_PREFIX='${SCCACHE_S3_KEY_PREFIX}/${TARGETARCH}/linux-musl'" >> /root/env + echo "export SCCACHE_REGION='${SCCACHE_REGION}'" >> /root/env + [ -n "${AWS_PROFILE}" ] && echo "export AWS_PROFILE='${AWS_PROFILE}'" >> /root/env + echo "export SCCACHE_ENDPOINT='${SCCACHE_ENDPOINT}'" >> /root/env + echo "export SCCACHE_S3_KEY_PREFIX='${SCCACHE_S3_KEY_PREFIX}'" >> /root/env + + # Configure AWS credentials + mkdir --mode=0700 -p "$HOME/.aws" + ln -s /run/secrets/AWS "$HOME/.aws/credentials" + echo "export AWS_SHARED_CREDENTIALS_FILE=$HOME/.aws/credentials" >> /root/env + + # Check if AWS credentials file is mounted correctly, eg. --mount=type=secret,id=AWS + echo '[ -e "${AWS_SHARED_CREDENTIALS_FILE}" ] || { + echo "$(id -u): Cannot read ${AWS_SHARED_CREDENTIALS_FILE}; did you use RUN --mount=type=secret,id=AWS ?"; + exit 1; + }' >> /root/env + + ### memcached ### elif [ -n "${SCCACHE_MEMCACHED}" ]; then # memcached echo "export SCCACHE_MEMCACHED='${SCCACHE_MEMCACHED}'" >> /root/env + else + echo "Error: cannot determine sccache cache backend" >&2 + exit 1 fi + + echo "export SCCACHE_SERVER_PORT=$((RANDOM+1025))" >> /root/env + + # Configure compilers to use sccache + echo "export CXX='sccache clang++'" >> /root/env + echo "export CC='sccache clang'" >> /root/env + echo "export RUSTC_WRAPPER=sccache" >> /root/env + # Disable Rust incremental builds, not supported by sccache + echo 'export CARGO_INCREMENTAL=0' >> /root/env - if [ -n "${RUSTC_WRAPPER}" ]; then - echo "export CXX='${RUSTC_WRAPPER} clang++'" >> /root/env - echo "export CC='${RUSTC_WRAPPER} clang'" >> /root/env - echo "export RUSTC_WRAPPER='${RUSTC_WRAPPER}'" >> /root/env - echo "export SCCACHE_SERVER_PORT=$((RANDOM+1025))" >> /root/env - fi # for debugging, we display what we generated cat /root/env EOS -# We provide secrets using ONBUILD ARG mechanism, to avoid putting them into a file and potentialy leaking them -# to the final image or to layer cache -ONBUILD ARG ACTIONS_RUNTIME_TOKEN -ONBUILD ARG AWS_SECRET_ACCESS_KEY - # Image containing compolation dependencies; used to overcome lack of interpolation in COPY --from -FROM deps-${RUSTC_WRAPPER:-base} AS deps-compilation +FROM ${DEPS_IMAGE} AS deps-compilation # Stage intentionally left empty # @@ -215,7 +244,24 @@ FROM deps-compilation AS deps-rocksdb RUN mkdir -p /tmp/rocksdb WORKDIR /tmp/rocksdb -RUN < a.c +# sccache clang -o a.o -c a.c +# cd - + +# sccache -s +# EOS + +RUN --mount=type=secret,id=AWS <