From 2bb7349193596cdb5d4f069df67fdcec336d2e6b Mon Sep 17 00:00:00 2001 From: EmilLuta Date: Mon, 25 Nov 2024 14:09:29 +0100 Subject: [PATCH 1/8] offcycle: Release circuit provers for v24.2 try 2 Same as previous [try](https://github.com/matter-labs/zksync-era/releases/tag/prover-v16.7.0-rc.1), but rebasing on 17.1.0. Non-mandatory release, internal interests only. --- prover/crates/lib/prover_fri_types/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/prover/crates/lib/prover_fri_types/src/lib.rs b/prover/crates/lib/prover_fri_types/src/lib.rs index 37e004d54ecc..4a8a1b3e4064 100644 --- a/prover/crates/lib/prover_fri_types/src/lib.rs +++ b/prover/crates/lib/prover_fri_types/src/lib.rs @@ -28,8 +28,8 @@ pub mod keys; pub mod queue; // THESE VALUES SHOULD BE UPDATED ON ANY PROTOCOL UPGRADE OF PROVERS -pub const PROVER_PROTOCOL_VERSION: ProtocolVersionId = ProtocolVersionId::Version25; -pub const PROVER_PROTOCOL_PATCH: VersionPatch = VersionPatch(0); +pub const PROVER_PROTOCOL_VERSION: ProtocolVersionId = ProtocolVersionId::Version24; +pub const PROVER_PROTOCOL_PATCH: VersionPatch = VersionPatch(2); pub const PROVER_PROTOCOL_SEMANTIC_VERSION: ProtocolSemanticVersion = ProtocolSemanticVersion { minor: PROVER_PROTOCOL_VERSION, patch: PROVER_PROTOCOL_PATCH, From fd6616dd6013a57d8e3bd91c224ae1e90a5d4e29 Mon Sep 17 00:00:00 2001 From: EmilLuta Date: Thu, 19 Dec 2024 12:10:48 +0100 Subject: [PATCH 2/8] Troubleshooting for GCS implementation for circuit provers --- core/lib/object_store/src/gcs.rs | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/core/lib/object_store/src/gcs.rs b/core/lib/object_store/src/gcs.rs index cad37715edb6..be0aa0c753fa 100644 --- a/core/lib/object_store/src/gcs.rs +++ b/core/lib/object_store/src/gcs.rs @@ -170,10 +170,20 @@ impl ObjectStore for GoogleCloudStore { object: filename, ..GetObjectRequest::default() }; - self.client + let response = self + .client .download_object(&request, &Range::default()) - .await - .map_err(Into::into) + .await; + match response { + Err(err) => { + tracing::info!("Errored with: {err:?}"); + Err(err.into()) + } + Ok(data) => { + tracing::info!("received {} bytes of data", data.len()); + Ok(data) + } + } } async fn put_raw( From e0d83132babd59f5a0347736d82c78819ce0acab Mon Sep 17 00:00:00 2001 From: EmilLuta Date: Thu, 19 Dec 2024 12:11:30 +0100 Subject: [PATCH 3/8] Print GCS file extraction --- core/lib/object_store/src/gcs.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/lib/object_store/src/gcs.rs b/core/lib/object_store/src/gcs.rs index be0aa0c753fa..4fc6f769b8f3 100644 --- a/core/lib/object_store/src/gcs.rs +++ b/core/lib/object_store/src/gcs.rs @@ -160,7 +160,7 @@ impl From for ObjectStoreError { impl ObjectStore for GoogleCloudStore { async fn get_raw(&self, bucket: Bucket, key: &str) -> Result, ObjectStoreError> { let filename = Self::filename(bucket.as_str(), key); - tracing::trace!( + tracing::info!( "Fetching data from GCS for key {filename} from bucket {}", self.bucket_prefix ); From 3188bb40490397a02e5cf000576e01cd2829a307 Mon Sep 17 00:00:00 2001 From: Yury Akudovich Date: Thu, 19 Dec 2024 12:19:19 +0100 Subject: [PATCH 4/8] Build only circuit-prover-gpu --- .github/workflows/build-docker-from-tag.yml | 104 +++++++++--------- .../workflows/new-build-prover-template.yml | 15 ++- 2 files changed, 59 insertions(+), 60 deletions(-) diff --git a/.github/workflows/build-docker-from-tag.yml b/.github/workflows/build-docker-from-tag.yml index 206e15bd195f..991b53d48df7 100644 --- a/.github/workflows/build-docker-from-tag.yml +++ b/.github/workflows/build-docker-from-tag.yml @@ -23,7 +23,7 @@ concurrency: docker-build jobs: setup: name: Setup - runs-on: [ ubuntu-latest ] + runs-on: [ubuntu-latest] outputs: image_tag_suffix: ${{ steps.set.outputs.image_tag_suffix }} prover_fri_gpu_key_id: ${{ steps.extract-prover-fri-setup-key-ids.outputs.gpu_short_commit_sha }} @@ -46,44 +46,44 @@ jobs: run: | ./prover/extract-setup-data-keys.sh >> $GITHUB_OUTPUT - build-push-core-images: - name: Build and push image - needs: [ setup ] - uses: ./.github/workflows/build-core-template.yml - if: contains(github.ref_name, 'core') - secrets: - DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - with: - image_tag_suffix: ${{ needs.setup.outputs.image_tag_suffix }} - en_alpha_release: true - - build-push-tee-prover-images: - name: Build and push images - needs: [ setup ] - uses: ./.github/workflows/build-tee-prover-template.yml - if: contains(github.ref_name, 'core') - secrets: - DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - ATTIC_TOKEN: ${{ secrets.ATTIC_TOKEN }} - with: - image_tag_suffix: ${{ needs.setup.outputs.image_tag_suffix }} - - build-push-contract-verifier: - name: Build and push image - needs: [ setup ] - uses: ./.github/workflows/build-contract-verifier-template.yml - if: contains(github.ref_name, 'contract_verifier') - secrets: - DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - with: - image_tag_suffix: ${{ needs.setup.outputs.image_tag_suffix }} - + # build-push-core-images: + # name: Build and push image + # needs: [ setup ] + # uses: ./.github/workflows/build-core-template.yml + # if: contains(github.ref_name, 'core') + # secrets: + # DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} + # DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + # with: + # image_tag_suffix: ${{ needs.setup.outputs.image_tag_suffix }} + # en_alpha_release: true + # + # build-push-tee-prover-images: + # name: Build and push images + # needs: [ setup ] + # uses: ./.github/workflows/build-tee-prover-template.yml + # if: contains(github.ref_name, 'core') + # secrets: + # DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} + # DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + # ATTIC_TOKEN: ${{ secrets.ATTIC_TOKEN }} + # with: + # image_tag_suffix: ${{ needs.setup.outputs.image_tag_suffix }} + # + # build-push-contract-verifier: + # name: Build and push image + # needs: [ setup ] + # uses: ./.github/workflows/build-contract-verifier-template.yml + # if: contains(github.ref_name, 'contract_verifier') + # secrets: + # DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} + # DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + # with: + # image_tag_suffix: ${{ needs.setup.outputs.image_tag_suffix }} + # build-push-prover-images: name: Build and push image - needs: [ setup ] + needs: [setup] uses: ./.github/workflows/build-prover-template.yml if: contains(github.ref_name, 'prover') with: @@ -94,23 +94,23 @@ jobs: DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - build-push-witness-generator-image-avx512: - name: Build and push image - needs: [ setup ] - uses: ./.github/workflows/build-witness-generator-template.yml - if: contains(github.ref_name, 'prover') - with: - image_tag_suffix: ${{ needs.setup.outputs.image_tag_suffix }}-avx512 - ERA_BELLMAN_CUDA_RELEASE: ${{ vars.ERA_BELLMAN_CUDA_RELEASE }} - CUDA_ARCH: "60;70;75;80;89" - WITNESS_GENERATOR_RUST_FLAGS: "-Ctarget_feature=+avx512bw,+avx512cd,+avx512dq,+avx512f,+avx512vl" - secrets: - DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - + # build-push-witness-generator-image-avx512: + # name: Build and push image + # needs: [ setup ] + # uses: ./.github/workflows/build-witness-generator-template.yml + # if: contains(github.ref_name, 'prover') + # with: + # image_tag_suffix: ${{ needs.setup.outputs.image_tag_suffix }}-avx512 + # ERA_BELLMAN_CUDA_RELEASE: ${{ vars.ERA_BELLMAN_CUDA_RELEASE }} + # CUDA_ARCH: "60;70;75;80;89" + # WITNESS_GENERATOR_RUST_FLAGS: "-Ctarget_feature=+avx512bw,+avx512cd,+avx512dq,+avx512f,+avx512vl" + # secrets: + # DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} + # DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + # build-gar-prover-fri-gpu-and-circuit-prover-gpu-gar: name: Build GAR prover FRI GPU - needs: [ setup, build-push-prover-images ] + needs: [setup, build-push-prover-images] uses: ./.github/workflows/build-prover-fri-gpu-gar-and-circuit-prover-gpu-gar.yml if: contains(github.ref_name, 'prover') with: diff --git a/.github/workflows/new-build-prover-template.yml b/.github/workflows/new-build-prover-template.yml index cb254f602fc5..69dcab5b6ea1 100644 --- a/.github/workflows/new-build-prover-template.yml +++ b/.github/workflows/new-build-prover-template.yml @@ -90,13 +90,13 @@ jobs: strategy: matrix: components: - - witness-generator - - prover-gpu-fri - - witness-vector-generator - - prover-fri-gateway - - prover-job-monitor - - proof-fri-gpu-compressor - - prover-autoscaler + # - witness-generator + # - prover-gpu-fri + # - witness-vector-generator + # - prover-fri-gateway + # - prover-job-monitor + # - proof-fri-gpu-compressor + # - prover-autoscaler - circuit-prover-gpu steps: - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 @@ -210,7 +210,6 @@ jobs: --tag asia-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.component }}:2.0-${{ inputs.image_tag_suffix }} \ us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.component }}:2.0-${{ inputs.image_tag_suffix }} - - name: Login and push to Europe GAR run: | gcloud auth print-access-token --lifetime=7200 --impersonate-service-account=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com | docker login -u oauth2accesstoken --password-stdin https://europe-docker.pkg.dev From 55177e0c2f5cff8a229df6b5296d2980b6222eaa Mon Sep 17 00:00:00 2001 From: Yury Akudovich Date: Thu, 19 Dec 2024 12:35:44 +0100 Subject: [PATCH 5/8] Build only circuit-prover-gpu in old build CI too --- .github/workflows/build-prover-template.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-prover-template.yml b/.github/workflows/build-prover-template.yml index 91de5dd51ecf..0fc8eb087e3c 100644 --- a/.github/workflows/build-prover-template.yml +++ b/.github/workflows/build-prover-template.yml @@ -49,14 +49,14 @@ jobs: strategy: matrix: component: - - witness-generator - - prover-gpu-fri - - witness-vector-generator + # - witness-generator + # - prover-gpu-fri + # - witness-vector-generator - circuit-prover-gpu - - prover-fri-gateway - - prover-job-monitor - - proof-fri-gpu-compressor - - prover-autoscaler + # - prover-fri-gateway + # - prover-job-monitor + # - proof-fri-gpu-compressor + # - prover-autoscaler outputs: protocol_version: ${{ steps.protocolversion.outputs.protocol_version }} steps: From 8f823e1415f01240e7a81ce293b2ecd4c893ac8d Mon Sep 17 00:00:00 2001 From: Yury Akudovich Date: Thu, 19 Dec 2024 12:37:27 +0100 Subject: [PATCH 6/8] Don't build prover-gpu-fri-gar --- ...fri-gpu-gar-and-circuit-prover-gpu-gar.yml | 50 +++++++++---------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/.github/workflows/build-prover-fri-gpu-gar-and-circuit-prover-gpu-gar.yml b/.github/workflows/build-prover-fri-gpu-gar-and-circuit-prover-gpu-gar.yml index 30990889caf6..53d875107885 100644 --- a/.github/workflows/build-prover-fri-gpu-gar-and-circuit-prover-gpu-gar.yml +++ b/.github/workflows/build-prover-fri-gpu-gar-and-circuit-prover-gpu-gar.yml @@ -47,31 +47,31 @@ jobs: run: | gcloud auth print-access-token --lifetime=7200 --impersonate-service-account=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com | docker login -u oauth2accesstoken --password-stdin https://europe-docker.pkg.dev - - name: Build and push prover-gpu-fri-gar - uses: docker/build-push-action@5cd11c3a4ced054e52742c5fd54dca954e0edd85 # v6.7.0 - with: - context: docker/prover-gpu-fri-gar - build-args: | - PROVER_IMAGE=${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }} - push: true - tags: | - us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }} - - - name: Build and push prover-gpu-fri-gar to Asia GAR - run: | - docker buildx imagetools create \ - --tag asia-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }} \ - us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }} - - - name: Build and push prover-gpu-fri-gar to Europe GAR - run: | - docker buildx imagetools create \ - --tag europe-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }} \ - us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }} - - - name: Move Setup data from prover-gpu-fri-gar to circuit-prover-gpu-gar - run: | - mv -v docker/prover-gpu-fri-gar/*.bin docker/circuit-prover-gpu-gar/ + # - name: Build and push prover-gpu-fri-gar + # uses: docker/build-push-action@5cd11c3a4ced054e52742c5fd54dca954e0edd85 # v6.7.0 + # with: + # context: docker/prover-gpu-fri-gar + # build-args: | + # PROVER_IMAGE=${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }} + # push: true + # tags: | + # us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }} + # + # - name: Build and push prover-gpu-fri-gar to Asia GAR + # run: | + # docker buildx imagetools create \ + # --tag asia-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }} \ + # us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }} + # + # - name: Build and push prover-gpu-fri-gar to Europe GAR + # run: | + # docker buildx imagetools create \ + # --tag europe-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }} \ + # us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }} + # + # - name: Move Setup data from prover-gpu-fri-gar to circuit-prover-gpu-gar + # run: | + # mv -v docker/prover-gpu-fri-gar/*.bin docker/circuit-prover-gpu-gar/ - name: Build and push circuit-prover-gpu-gar uses: docker/build-push-action@5cd11c3a4ced054e52742c5fd54dca954e0edd85 # v6.7.0 From 3ce45526d475400c9cfa06e8d869e118d2ee470d Mon Sep 17 00:00:00 2001 From: Yury Akudovich Date: Thu, 19 Dec 2024 14:09:20 +0100 Subject: [PATCH 7/8] Use new build workflow --- .github/workflows/build-docker-from-tag.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-docker-from-tag.yml b/.github/workflows/build-docker-from-tag.yml index 991b53d48df7..cb07259eb3aa 100644 --- a/.github/workflows/build-docker-from-tag.yml +++ b/.github/workflows/build-docker-from-tag.yml @@ -84,12 +84,12 @@ jobs: build-push-prover-images: name: Build and push image needs: [setup] - uses: ./.github/workflows/build-prover-template.yml + uses: ./.github/workflows/new-build-prover-template.yml if: contains(github.ref_name, 'prover') with: image_tag_suffix: ${{ needs.setup.outputs.image_tag_suffix }} ERA_BELLMAN_CUDA_RELEASE: ${{ vars.ERA_BELLMAN_CUDA_RELEASE }} - CUDA_ARCH: "60;70;75;80;89" + action: "push" secrets: DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} From 05d65624de33c40e7760e8cdf97f1f8a1c2d12a7 Mon Sep 17 00:00:00 2001 From: Yury Akudovich Date: Thu, 19 Dec 2024 14:11:49 +0100 Subject: [PATCH 8/8] And build all needed --- .github/workflows/new-build-prover-template.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/new-build-prover-template.yml b/.github/workflows/new-build-prover-template.yml index 69dcab5b6ea1..c38d5fb41314 100644 --- a/.github/workflows/new-build-prover-template.yml +++ b/.github/workflows/new-build-prover-template.yml @@ -30,8 +30,12 @@ on: CUDA_ARCH: description: "CUDA Arch to build" type: string - default: "89" + default: "75;80;89" required: false + # Details: https://arnon.dk/matching-sm-architectures-arch-and-gencode-for-various-nvidia-cards/ + # L4: 89 + # T4: 75 + # A100: 80 outputs: protocol_version: description: "Protocol version of the binary"