20230920 compile context optimizer #291
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Thanks: clvm_rs' github actions. | |
| name: Build | |
| on: | |
| push: | |
| branches: | |
| - base | |
| - dev | |
| release: | |
| types: [published] | |
| pull_request: | |
| branches: | |
| - '**' | |
| permissions: | |
| id-token: write | |
| contents: read | |
| jobs: | |
| build_wheels: | |
| name: Wheel on ${{ matrix.os }} py-${{ matrix.python }} | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [macos-latest, ubuntu-latest, windows-latest] | |
| python: [3.8, 3.9, '3.10', 3.11] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - uses: actions/setup-python@v4 | |
| name: Install Python ${{ matrix.python }} | |
| with: | |
| python-version: ${{ matrix.python }} | |
| - name: Update pip | |
| run: | | |
| python -m pip install --upgrade pip | |
| - name: Set up rust | |
| uses: actions-rs/toolchain@v1 | |
| with: | |
| toolchain: stable | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install maturin==1.1.0 | |
| - name: Build MacOs with maturin on Python ${{ matrix.python }} | |
| if: startsWith(matrix.os, 'macos') | |
| env: | |
| MACOSX_DEPLOYMENT_TARGET: '11.0' | |
| run: | | |
| python -m venv venv | |
| ln -s venv/bin/activate | |
| . ./activate | |
| maturin build -i python --release --strip | |
| - name: Build Linux in manylinux2010 with maturin on Python ${{ matrix.python }} | |
| if: startsWith(matrix.os, 'ubuntu') | |
| run: | | |
| docker run --rm \ | |
| -v ${{ github.workspace }}:/ws --workdir=/ws \ | |
| ghcr.io/chia-network/build-images/centos-pypa-rust-x86_64 \ | |
| bash -exc '\ | |
| yum -y install libc6 openssl-devel && \ | |
| source $HOME/.cargo/env && \ | |
| rustup target add x86_64-unknown-linux-musl && \ | |
| rm -rf venv && \ | |
| PY_VERSION=${{ matrix.python }} | |
| PY_VERSION=${PY_VERSION/.} && \ | |
| echo "Python version with dot removed is $PY_VERSION" && \ | |
| if [ "$PY_VERSION" = "37" ]; \ | |
| then export SCND_VERSION="${PY_VERSION}m"; \ | |
| else export SCND_VERSION="$PY_VERSION"; fi && \ | |
| echo "Exporting path /opt/python/cp$PY_VERSION-cp$SCND_VERSION/bin" && \ | |
| export PATH=/opt/python/cp$PY_VERSION-cp$SCND_VERSION/bin/:$PATH && \ | |
| /opt/python/cp38-cp38/bin/python -m venv venv && \ | |
| if [ ! -f "activate" ]; then ln -s venv/bin/activate; fi && \ | |
| . ./activate && \ | |
| pip install --upgrade pip | |
| ' | |
| docker run --rm -v $(pwd):/io ghcr.io/pyo3/maturin:v1.1.0 build --release --strip --manylinux 2014 | |
| # Refresh in case any ownerships changed. | |
| mv target target.docker && cp -r target.docker target | |
| # Ensure an empty .cargo-lock file exists. | |
| touch target/release/.cargo-lock | |
| - name: Build Windows with maturin on Python ${{ matrix.python }} | |
| if: startsWith(matrix.os, 'windows') | |
| run: | | |
| python -m venv venv | |
| echo ". .\venv\Scripts\Activate.ps1" > activate.ps1 | |
| . ./activate | |
| maturin build -i python --release --strip | |
| # Find and install the newly built wheel | |
| python support/wheelname.py | |
| - name: Install clvm_tools_rs wheel | |
| if: ${{ !startsWith(matrix.os, 'windows') }} | |
| run: | | |
| . ./activate | |
| ls target/wheels/ | |
| # this mess puts the name of the `.whl` file into `$WHEEL_PATH` | |
| # remove the dot, use the `glob` lib to grab the file from the directory | |
| export WHEEL_PATH=$(echo ${{ matrix.python }} | python -c 'DOTLESS=input().replace(".", ""); import glob; print(" ".join(filter(lambda x: "musl" not in x, glob.glob("target/wheels/clvm_tools_rs-*-cp*-*.whl"))))' ) | |
| echo ${WHEEL_PATH} | |
| pip install ${WHEEL_PATH} | |
| - name: Install other wheels | |
| run: | | |
| . ./activate | |
| python -m pip install pytest | |
| python -m pip install blspy | |
| - name: install clvm & clvm_tools | |
| run: | | |
| . ./activate | |
| git clone https://github.com/Chia-Network/clvm.git --branch=main --single-branch | |
| python -m pip install ./clvm | |
| echo "installing clvm_rs via pip" | |
| pip install clvm_rs | |
| echo "installing clvm_tools for clvm tests" | |
| # Ensure clvm_tools is installed from its own repo. | |
| git clone https://github.com/Chia-Network/clvm_tools.git --branch=main --single-branch | |
| python -m pip install ./clvm_tools | |
| - name: Ensure clvm, clvm_rs, clvm_tools_rs are installed | |
| run: | | |
| . ./activate | |
| python -c 'import clvm' | |
| python -c 'import clvm; print(clvm.__file__)' | |
| python -c 'import clvm_rs; print(clvm_rs.__file__)' | |
| python -c 'import clvm_tools_rs; print(clvm_tools_rs.__file__)' | |
| echo "CLVM_TOOLS_RS_VERSION=$(python -c 'import clvm_tools_rs; print(clvm_tools_rs.get_version())')" >> "$GITHUB_ENV" | |
| - name: Verify recompilation of old sources match with new compiler | |
| if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.python, '3.8') | |
| run: | | |
| set -x | |
| . ./activate | |
| # Build cmd line tools | |
| PYO3_PYTHON=`which python` cargo build --no-default-features --release | |
| # Grab chia-blockchain | |
| rm -rf chia-blockchain | |
| git clone https://github.com/Chia-Network/chia-blockchain | |
| # Check that recompiling deployed puzzles match with their deployed hashes | |
| cp support/install_deps.sh support/verify_compiler_version.sh chia-blockchain | |
| (cd chia-blockchain && python -m venv venv && . venv/bin/activate && pip install --upgrade pip && \ | |
| python -m pip install maturin==1.1.0 && \ | |
| cd .. && python support/wheelname.py && \ | |
| cd chia-blockchain && \ | |
| # deps for manage_clvm.py | |
| pip install click typing_extensions chia_rs clvm && \ | |
| export PYTHONPATH=${PYTHONPATH}:$(pwd) && \ | |
| ./verify_compiler_version.sh ${CLVM_TOOLS_RS_VERSION} && ./activated.py python tools/manage_clvm.py check) | |
| - name: Test Classic command line tools with pytest | |
| if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.python, '3.8') | |
| run: | | |
| python -m pip install pytest | |
| # This script installs the wheel built during this workflow. | |
| python support/wheelname.py | |
| # TODO: bring in the other tests in resources/tests/cmdline/tests besides cmds_test.py | |
| (cd resources/tests/cmdline/tests && py.test cmds_test.py ) | |
| - name: Verify recompilation of cl21 sources | |
| if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.python, '3.8') | |
| run: | | |
| . ./activate | |
| # We need chia-rs for the following. | |
| pip install chia-rs | |
| # Ensure we're using the sources we have. This is a hedge against | |
| # changes made by any other step doing operations on pip. | |
| # This script installs the wheel built during this workflow. | |
| python support/wheelname.py | |
| (cd resources/tests && python check-recompiles.py) | |
| - name: Verify recompilation follows date and modification rules | |
| if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.python, '3.8') | |
| run: | | |
| python support/wheelname.py | |
| python resources/tests/test-clvm-recompile-behavior.py | |
| - name: Install pytest | |
| run: | | |
| . ./activate | |
| pip install pytest pytest-asyncio | |
| - name: Run tests from clvm | |
| run: | | |
| . ./activate | |
| cd clvm | |
| pytest tests | |
| - name: Run tests from clvm_tools | |
| run: | | |
| . ./activate | |
| cd clvm_tools | |
| pytest tests | |
| - name: Run tests | |
| if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.python, '3.8') | |
| run: cargo test --no-default-features | |
| - name: Exhaustive assign tests | |
| if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.python, '3.8') | |
| run: cargo test -- --include-ignored assign | |
| - name: Check coverage | |
| if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.python, '3.8') | |
| run: | | |
| cargo install grcov | |
| rustup component add llvm-tools-preview | |
| python ./resources/coverage/run_coverage.py --require-percent 60 | |
| - name: Upload artifacts | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: wheels | |
| path: ./target/wheels/ | |
| - name: Test for secrets access | |
| id: check_secrets | |
| shell: bash | |
| run: | | |
| unset HAS_AWS_SECRET | |
| if [ -n "$AWS_SECRET" ]; then HAS_AWS_SECRET='true' ; fi | |
| echo HAS_AWS_SECRET=${HAS_AWS_SECRET} >>$GITHUB_OUTPUT | |
| env: | |
| AWS_SECRET: "${{ secrets.CHIA_AWS_ACCOUNT_ID }}" | |
| - name: Set Env | |
| uses: Chia-Network/actions/setjobenv@main | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: publish (PyPi) | |
| if: env.RELEASE == 'true' | |
| uses: pypa/gh-action-pypi-publish@release/v1 | |
| with: | |
| packages-dir: target/wheels/ | |
| skip-existing: true | |
| - name: publish (Test PyPi) | |
| if: env.PRE_RELEASE == 'true' | |
| uses: pypa/gh-action-pypi-publish@release/v1 | |
| with: | |
| repository-url: https://test.pypi.org/legacy/ | |
| packages-dir: target/wheels/ | |
| skip-existing: true | |
| - name: Configure AWS credentials | |
| if: steps.check_secrets.outputs.HAS_AWS_SECRET | |
| uses: aws-actions/configure-aws-credentials@v2 | |
| with: | |
| role-to-assume: arn:aws:iam::${{ secrets.CHIA_AWS_ACCOUNT_ID }}:role/installer-upload | |
| aws-region: us-west-2 | |
| - name: Publish Dev | |
| if: steps.check_secrets.outputs.HAS_AWS_SECRET && github.ref == 'refs/heads/dev' | |
| shell: bash | |
| working-directory: ./target/wheels | |
| run: | | |
| FILES=$(find . -type f -name '*.whl') | |
| while IFS= read -r file; do | |
| filename=$(basename $file) | |
| aws --no-progress s3 cp "$file" "s3://download.chia.net/simple-dev/clvm-tools-rs/$filename" | |
| done <<< "$FILES" | |
| - name: Build alpine wheel via docker | |
| if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.python, '3.8') | |
| run: | | |
| cd resources/alpine && docker build -t clvm-tools-rs-alpine . | |
| docker run -v ${GITHUB_WORKSPACE}:/root/clvm_tools_rs -t clvm-tools-rs-alpine sh /root/build-alpine.sh | |
| fmt: | |
| runs-on: ubuntu-20.04 | |
| name: cargo fmt | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 1 | |
| - name: Install rust | |
| uses: actions-rs/toolchain@v1 | |
| with: | |
| toolchain: stable | |
| override: true | |
| components: rustfmt, clippy | |
| - name: fmt | |
| run: cargo fmt -- --files-with-diff --check | |
| clippy: | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions-rs/toolchain@v1 | |
| with: | |
| toolchain: stable | |
| components: clippy | |
| override: true | |
| - name: clippy | |
| run: cargo clippy --all -- -D warnings | |
| - uses: giraffate/clippy-action@v1 | |
| with: | |
| reporter: 'github-pr-review' | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| unit_tests: | |
| runs-on: ubuntu-20.04 | |
| name: Unit tests | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 1 | |
| - name: Install rust | |
| uses: actions-rs/toolchain@v1 | |
| with: | |
| toolchain: stable | |
| components: rustfmt, clippy | |
| - name: cargo test | |
| run: cargo test | |
| coverage: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions-rs/toolchain@v1 | |
| with: | |
| toolchain: stable | |
| - name: Run for coverage | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install lcov -y | |
| rustup component add llvm-tools-preview | |
| cargo install grcov | |
| export RUSTFLAGS="-Cinstrument-coverage" | |
| export LLVM_PROFILE_FILE=$(pwd)/target/clvm_tools_rs-%p-%m.profraw | |
| export CARGO_TARGET_DIR=$(pwd)/target | |
| cargo test --release --workspace | |
| python -m venv venv | |
| source venv/bin/activate | |
| git clone https://github.com/Chia-Network/clvm_tools.git --branch=main --single-branch | |
| pip install ./clvm_tools | |
| pip install maturin pytest | |
| maturin develop --release | |
| (cd resources/tests/cmdline/tests && pytest) | |
| grcov . --binary-path target -s . --branch --ignore-not-existing --ignore='*/.cargo/*' --ignore='*/tests/*' -o rust_cov.info | |
| python -c 'with open("rust_cov.info") as f: lines = [l for l in f if not (l.startswith("DA:") and int(l.split(",")[1].strip()) >= 2**63)]; open("lcov.info", "w").writelines(lines)' | |
| - name: Upload to Coveralls | |
| uses: coverallsapp/github-action@v2 | |
| if: always() | |
| env: | |
| COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} | |
| with: | |
| path-to-lcov: './lcov.info' |