chore: publish native blockifier artifacts via CI (#2001)

* chore: publish native blockifier artifacts via CI

Signed-off-by: Dori Medini <[email protected]>

* chore: build_native_blockifier.sh to allow local builds

Co-Authored-By: alon.dotan <[email protected]>

.cargo/config.toml

@@ -1,4 +1,6 @@
 [env]
+# Enforce native_blockifier linking with pypy3.9.
+PYO3_PYTHON = "/usr/local/bin/pypy3.9"
 # Increase Rust stack size.
 # This should be large enough for `MAX_ENTRY_POINT_RECURSION_DEPTH` recursive entry point calls.
 RUST_MIN_STACK = "4194304" #  4 MiB
@@ -7,7 +9,7 @@ RUST_MIN_STACK = "4194304" #  4 MiB
 rustflags = [
   "-C", "link-arg=-undefined",
   "-C", "link-arg=dynamic_lookup",
-]
+  ]
 
 [target.aarch64-apple-darwin]
 rustflags = [

.github/workflows/ci.yml

@@ -82,6 +82,50 @@ jobs:
       - run: cargo test
       - run: cargo test --features concurrency
 
+  native-blockifier-artifacts-push:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+
+      - uses: Swatinem/rust-cache@v2
+
+      - name: Build native blockifier
+        run: ./build_native_blockifier.sh
+
+        # Commit hash on pull request event would be the head commit of the branch.
+      - name: Get commit hash prefix for PR update
+        if: ${{ github.event_name == 'pull_request' }}
+        env:
+          COMMIT_SHA: ${{ github.event.pull_request.head.sha }}
+        run: echo "SHORT_HASH=${COMMIT_SHA:0:7}" >> $GITHUB_ENV
+
+        # On push event (to main, for example) we should take the commit post-push.
+      - name: Get commit hash prefix for merge
+        if: ${{ github.event_name != 'pull_request' }}
+        env:
+          COMMIT_SHA: ${{ github.event.after }}
+        run: echo "SHORT_HASH=${COMMIT_SHA:0:7}" >> $GITHUB_ENV
+
+      # Rename is required; see https://pyo3.rs/v0.19.2/building_and_distribution#manual-builds.
+      - name: Rename shared object
+        run: |
+          mv \
+            target/release/libnative_blockifier.so \
+            target/release/native_blockifier.pypy39-pp73-x86_64-linux-gnu.so
+
+      - name: Authenticate with GCS
+        uses: "google-github-actions/auth@v2"
+        with:
+          credentials_json: ${{ secrets.SA_NATIVE_BLOCKIFIER_ARTIFACTS_BUCKET_WRITER_ACCESS_KEY }}
+
+      - name: Upload binary to GCP
+        id: upload_file
+        uses: "google-github-actions/upload-cloud-storage@v2"
+        with:
+          path: "target/release/native_blockifier.pypy39-pp73-x86_64-linux-gnu.so"
+          destination: "native_blockifier_artifacts/${{ env.SHORT_HASH }}/release/"
+
   udeps:
     runs-on: ubuntu-latest
     steps:
@@ -90,6 +134,14 @@ jobs:
         name: "Rust Toolchain Setup"
         with:
           toolchain: nightly-2024-01-12
+
+      # Setup pypy and link to the location expected by .cargo/config.toml.
+      - uses: actions/setup-python@v5
+        id: setup-pypy
+        with:
+          python-version: 'pypy3.9'
+      - run: ln -s '${{ steps.setup-pypy.outputs.python-path }}' /usr/local/bin/pypy3.9
+
       - uses: Swatinem/rust-cache@v2
         id: "cache-cargo"
       - if: ${{ steps.cache-cargo.outputs.cache-hit != 'true' }}

BUILD

@@ -0,0 +1,2 @@
+# Export the built artifact to allow local builds.
+exports_files(["target/release/libnative_blockifier.so"])

Dockerfile

@@ -0,0 +1,14 @@
+FROM ubuntu:20.04
+
+RUN apt update && apt -y install \
+    build-essential \
+    clang \
+    curl \
+    python3-dev
+
+ENV RUSTUP_HOME=/opt/rust
+ENV CARGO_HOME=/opt/rust
+ENV PATH=$PATH:/opt/rust/bin
+
+COPY scripts/install_build_tools.sh .
+RUN bash install_build_tools.sh

build_native_blockifier.sh

@@ -0,0 +1,16 @@
+#!/bin/env bash
+set -e
+
+docker_image_name=blockifier-ci
+docker build . -t ${docker_image_name}
+
+docker run \
+    --rm \
+    --net host \
+    -e CARGO_HOME=${HOME}/.cargo \
+    -u $UID \
+    -v /tmp:/tmp \
+    -v "${HOME}:${HOME}" \
+    --workdir ${PWD} \
+    ${docker_image_name} \
+    cargo build --release -p native_blockifier --features "testing"

crates/native_blockifier/Cargo.toml

@@ -33,7 +33,7 @@ indexmap.workspace = true
 log.workspace = true
 num-bigint.workspace = true
 papyrus_storage = { workspace = true, features = ["testing"] }
-pyo3 = { workspace = true, features = ["num-bigint", "hashbrown"] }
+pyo3 = { workspace = true, features = ["hashbrown", "num-bigint"] }
 pyo3-log.workspace = true
 serde = { workspace = true, features = ["derive"] }
 serde_json = { workspace = true, features = ["arbitrary_precision"] }

scripts/install_build_tools.sh

@@ -0,0 +1,36 @@
+#!/bin/env bash
+
+set -e
+
+function install_pypy() {
+  pushd /opt
+  $USE_SUDO bash -c '
+  curl -Lo pypy3.9-v7.3.11-linux64.tar.bz2 https://downloads.python.org/pypy/pypy3.9-v7.3.11-linux64.tar.bz2
+  tar -xf pypy3.9-v7.3.11-linux64.tar.bz2
+  rm pypy3.9-v7.3.11-linux64.tar.bz2
+  chmod +x pypy3.9-v7.3.11-linux64/bin/pypy3
+
+  if [ -L /usr/local/bin/pypy3.9 ]; then
+      unlink /usr/local/bin/pypy3.9
+  fi
+
+  ln -s /opt/pypy3.9-v7.3.11-linux64/bin/pypy3 /usr/local/bin/pypy3.9
+
+  if [ -L /opt/pypy3.9 ]; then
+      unlink /opt/pypy3.9
+  fi
+
+  ln -s /opt/pypy3.9-v7.3.11-linux64 /opt/pypy3.9
+  pypy3.9 -m ensurepip
+  pypy3.9 -m pip install wheel
+  '
+  popd
+}
+
+function install_rust () {
+    curl https://sh.rustup.rs -sSf | sh -s -- -y --no-modify-path
+}
+
+install_pypy &
+install_rust &
+wait