Update from upstream

matter-labs · Sep 20, 2024 · 25de844 · 25de844
2 parents acb11a3 + e9d1d90
commit 25de844
Show file tree

Hide file tree

Showing 215 changed files with 3,096 additions and 988 deletions.
diff --git a/.env b/.env
diff --git a/.github/release-please/manifest.json b/.github/release-please/manifest.json
@@ -1,5 +1,5 @@
 {
-  "core": "24.24.0",
+  "core": "24.25.0",
   "prover": "16.5.0",
   "zk_toolbox": "0.1.2"
 }
diff --git a/.github/scripts/rate_limit_check.sh b/.github/scripts/rate_limit_check.sh
@@ -0,0 +1,43 @@
+#!/bin/bash
+
+set -o errexit
+set -o pipefail
+
+
+api_endpoint="https://api.github.com/users/zksync-era-bot"
+wait_time=60
+max_retries=60
+retry_count=0
+
+while [[ $retry_count -lt $max_retries ]]; do
+  response=$(run_retried curl -s -w "%{http_code}" -o temp.json "$api_endpoint")
+  http_code=$(echo "$response" | tail -n1)
+
+  if [[ "$http_code" == "200" ]]; then
+    echo "Request successful. Not rate-limited."
+    cat temp.json
+    rm temp.json
+    exit 0
+  elif [[ "$http_code" == "403" ]]; then
+    rate_limit_exceeded=$(jq -r '.message' temp.json | grep -i "API rate limit exceeded")
+    if [[ -n "$rate_limit_exceeded" ]]; then
+      retry_count=$((retry_count+1))
+      echo "API rate limit exceeded. Retry $retry_count of $max_retries. Retrying in $wait_time seconds..."
+      sleep $wait_time
+    else
+      echo "Request failed with HTTP status $http_code."
+      cat temp.json
+      rm temp.json
+      exit 1
+    fi
+  else
+    echo "Request failed with HTTP status $http_code."
+    cat temp.json
+    rm temp.json
+    exit 1
+  fi
+done
+
+echo "Reached the maximum number of retries ($max_retries). Exiting."
+rm temp.json
+exit 1
diff --git a/.github/workflows/build-base.yml b/.github/workflows/build-base.yml
@@ -0,0 +1,159 @@
+name: Build zksync-build-base Docker image
+on:
+  workflow_dispatch:
+    inputs:
+      repo_ref:
+        description: "git reference of the zksync-era to build"
+        required: true
+        default: main
+jobs:
+  build-images:
+    name: Build and Push Docker Images
+    runs-on: ${{ fromJSON('["matterlabs-ci-runner-high-performance", "matterlabs-ci-runner-arm"]')[contains(matrix.arch, 'arm')] }}
+    outputs:
+      image_tag_sha: ${{ steps.get-sha.outputs.image_tag_sha }}
+    # Needed to push to Gihub Package Registry
+    permissions:
+      packages: write
+      contents: read
+    env:
+      DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
+      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+      REPO_REF: ${{ github.event.inputs.repo_ref }}
+    strategy:
+      matrix:
+        name: [ build-base ]
+        repository: [ zksync-build-base ]
+        arch: [ amd64, arm64 ]
+
+    steps:
+      - uses: actions/checkout@6d193bf28034eafb982f37bd894289fe649468fc # v4.1.7
+        with:
+          submodules: "recursive"
+
+      - name: Login to google container registry
+        run: |
+          gcloud auth configure-docker us-docker.pkg.dev -q
+
+      - name: Login to DockerHub
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
+        with:
+          username: ${{ secrets.DOCKERHUB_USER }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Get tag
+        id: get-sha
+        run: |
+          echo IMAGE_TAG_SHA=$(git rev-parse --short HEAD) >> $GITHUB_ENV
+          echo image_tag_sha=$(git rev-parse --short HEAD) >> $GITHUB_OUTPUT
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db # v3.6.1
+
+      - name: Build and push
+        uses: docker/build-push-action@5cd11c3a4ced054e52742c5fd54dca954e0edd85 # v6.7.0
+        with:
+          push: true
+          context: .
+          file: docker/build-base/Dockerfile
+          labels: |
+            org.opencontainers.image.source=https://github.com/matter-labs/zksync-era
+            org.opencontainers.image.licenses="MIT OR Apache-2.0"
+          tags: |
+            matterlabs/zksync-build-base:${{ steps.get-sha.outputs.image_tag_sha }}-${{ matrix.arch }}
+            us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/zksync-build-base:${{ steps.get-sha.outputs.image_tag_sha }}-${{ matrix.arch }}
+            ghcr.io/${{ github.repository_owner }}/zksync-build-base:${{ steps.get-sha.outputs.image_tag_sha }}-${{ matrix.arch }}
+
+  multiarch_manifest:
+    # Needed to push to Gihub Package Registry
+    permissions:
+      packages: write
+      contents: read
+    needs: [ build-images ]
+    env:
+      IMAGE_TAG_SUFFIX: ${{ needs.build-images.outputs.image_tag_sha }}
+    runs-on: [ matterlabs-ci-runner-high-performance ]
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db # v3.6.1
+
+      - name: Login to google container registry
+        run: |
+          gcloud auth configure-docker us-docker.pkg.dev -q
+
+      - name: Login to DockerHub
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
+        with:
+          username: ${{ secrets.DOCKERHUB_USER }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Create and push multi-arch manifests for Dockerhub
+        shell: bash
+        run: |
+          images=("zksync-build-base")
+          archs=("amd64" "arm64")
+
+          for img in "${images[@]}"; do
+            multiarch_tag="matterlabs/zksync-build-base:latest"
+            individual_images=()
+
+            for arch in "${archs[@]}"; do
+              TAG="$IMAGE_TAG_SUFFIX"
+              docker pull matterlabs/zksync-build-base:${TAG}-${arch} --platform linux/${arch}
+              individual_images+=("matterlabs/zksync-build-base:${TAG}-${arch}")
+            done
+
+            docker buildx imagetools create --tag "${multiarch_tag}" "${individual_images[@]}"
+          done
+
+      - name: Create and push multi-arch manifests for GitHub Container Registry
+        shell: bash
+        run: |
+          images=("zksync-build-base")
+          archs=("amd64" "arm64")
+
+          for img in "${images[@]}"; do
+            multiarch_tag="ghcr.io/${{ github.repository_owner }}/zksync-build-base:latest"
+            individual_images=()
+
+            for arch in "${archs[@]}"; do
+              TAG="$IMAGE_TAG_SUFFIX"
+              docker pull ghcr.io/${{ github.repository_owner }}/zksync-build-base:${TAG}-${arch} --platform linux/${arch}
+              individual_images+=("ghcr.io/${{ github.repository_owner }}/zksync-build-base:${TAG}-${arch}")
+            done
+
+            docker buildx imagetools create --tag "${multiarch_tag}" "${individual_images[@]}"
+          done
+
+      - name: Create and push multi-arch manifests for Google Artifact Registry
+        shell: bash
+        run: |
+          images=("zksync-build-base")
+          archs=("amd64" "arm64")
+
+          for img in "${images[@]}"; do
+            multiarch_tag="us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/zksync-build-base:latest"
+            individual_images=()
+
+            for arch in "${archs[@]}"; do
+              TAG="$IMAGE_TAG_SUFFIX"
+              docker pull us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/zksync-build-base:${TAG}-${arch} --platform linux/${arch}
+              individual_images+=("us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/zksync-build-base:${TAG}-${arch}")
+            done
+
+            docker buildx imagetools create --tag "${multiarch_tag}" "${individual_images[@]}"
+          done
diff --git a/.github/workflows/build-runtime-base.yml b/.github/workflows/build-runtime-base.yml
@@ -0,0 +1,66 @@
+name: Build zksync-runtime-base Docker image
+on:
+  workflow_dispatch:
+    inputs:
+      repo_ref:
+        description: "git reference of the zksync-era to build"
+        required: true
+        default: main
+jobs:
+  build-images:
+    name: Build and Push Docker Images
+    runs-on: matterlabs-ci-runner-high-performance
+    outputs:
+      image_tag_sha: ${{ steps.get-sha.outputs.image_tag_sha }}
+    # Needed to push to Gihub Package Registry
+    permissions:
+      packages: write
+      contents: read
+    env:
+      REPO_REF: ${{ github.event.inputs.repo_ref }}
+    strategy:
+      matrix:
+        name: [ runtime-base ]
+        image_name: [ zksync-runtime-base ]
+
+    steps:
+      - uses: actions/checkout@6d193bf28034eafb982f37bd894289fe649468fc # v4.1.7
+        with:
+          submodules: "recursive"
+
+      - name: Login to google container registry
+        run: |
+          gcloud auth configure-docker us-docker.pkg.dev -q
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Get tag
+        id: get-sha
+        run: |
+          echo IMAGE_TAG_SHA=$(git rev-parse --short HEAD) >> $GITHUB_ENV
+          echo image_tag_sha=$(git rev-parse --short HEAD) >> $GITHUB_OUTPUT
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db # v3.6.1
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v3.2.0
+
+      - name: Build and push
+        uses: docker/build-push-action@5cd11c3a4ced054e52742c5fd54dca954e0edd85 # v6.7.0
+        with:
+          push: true
+          context: .
+          platforms: arm64, amd64
+          file: docker/${{ matrix.name }}/Dockerfile
+          labels: |
+            org.opencontainers.image.source=https://github.com/matter-labs/zksync-era
+            org.opencontainers.image.licenses="MIT OR Apache-2.0"
+          tags: |
+            us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.image_name }}:latest
+            ghcr.io/${{ github.repository_owner }}/${{ matrix.image_name }}:latest
diff --git a/.github/workflows/ci-common-reusable.yml b/.github/workflows/ci-common-reusable.yml
@@ -18,13 +18,16 @@ jobs:
           echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV
           echo $(pwd)/bin >> $GITHUB_PATH
           echo IN_DOCKER=1 >> .env
+          echo "SCCACHE_GCS_BUCKET=matterlabs-infra-sccache-storage" >> .env
+          echo "SCCACHE_GCS_SERVICE_ACCOUNT=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com" >> .env
+          echo "SCCACHE_GCS_RW_MODE=READ_WRITE" >> .env
+          echo "RUSTC_WRAPPER=sccache" >> .env
 
       - name: Start services
         run: |
           run_retried docker-compose -f ${RUNNER_COMPOSE_FILE} pull
           mkdir -p ./volumes/postgres
           docker-compose -f ${RUNNER_COMPOSE_FILE} up --build -d zk postgres
-          ci_run sccache --start-server
 
       - name: Init
         run: |

diff --git a/.github/workflows/ci-core-lint-reusable.yml b/.github/workflows/ci-core-lint-reusable.yml
@@ -15,13 +15,16 @@ jobs:
           echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV
           echo $(pwd)/bin >> $GITHUB_PATH
           echo IN_DOCKER=1 >> .env
+          echo "SCCACHE_GCS_BUCKET=matterlabs-infra-sccache-storage" >> .env
+          echo "SCCACHE_GCS_SERVICE_ACCOUNT=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com" >> .env
+          echo "SCCACHE_GCS_RW_MODE=READ_WRITE" >> .env
+          echo "RUSTC_WRAPPER=sccache" >> .env
           echo "prover_url=postgres://postgres:notsecurepassword@localhost:5432/zksync_local_prover" >> $GITHUB_ENV
           echo "core_url=postgres://postgres:notsecurepassword@localhost:5432/zksync_local" >> $GITHUB_ENV
 
       - name: Start services
         run: |
           ci_localnet_up
-          ci_run sccache --start-server
 
       - name: Build
         run: |