Merge in base with now-merged embed-file

.github/workflows/build-arm64-wheels.yml

@@ -38,8 +38,8 @@ jobs:
         /opt/python/cp38-cp38/bin/python -m venv venv
         if [ ! -f "activate" ]; then ln -s venv/bin/activate; fi
         . ./activate
-        pip install maturin==0.12.20
-        CC=gcc maturin build --release --strip --manylinux 2014 --no-sdist --cargo-extra-args=--all-features
+        pip install maturin==1.1.0
+        CC=gcc maturin build --release --strip --manylinux 2014
     - name: Upload artifacts
       uses: actions/upload-artifact@v3
       with:
@@ -75,10 +75,16 @@ jobs:
       shell: bash
       run: |
         unset HAS_SECRET
+        unset HAS_AWS_SECRET
+
         if [ -n "$SECRET" ]; then HAS_SECRET='true' ; fi
         echo "HAS_SECRET=${HAS_SECRET}" >>$GITHUB_OUTPUT
+
+        if [ -n "$AWS_SECRET" ]; then HAS_AWS_SECRET='true' ; fi
+        echo HAS_AWS_SECRET=${HAS_AWS_SECRET} >>$GITHUB_OUTPUT
       env:
         SECRET: "${{ secrets.test_pypi_password }}"
+        AWS_SECRET: "${{ secrets.INSTALLER_UPLOAD_KEY }}"
 
     - name: publish (PyPi)
       if: startsWith(github.event.ref, 'refs/tags') && steps.check_secrets.outputs.HAS_SECRET
@@ -99,3 +105,16 @@ jobs:
       run: |
         . ./activate
         twine upload --non-interactive --skip-existing --verbose 'target/wheels/*'
+
+    - name: Publish Dev
+      if: steps.check_secrets.outputs.HAS_AWS_SECRET && github.ref == 'refs/heads/dev'
+      env:
+        AWS_ACCESS_KEY_ID: "${{ secrets.INSTALLER_UPLOAD_KEY }}"
+        AWS_SECRET_ACCESS_KEY: "${{ secrets.INSTALLER_UPLOAD_SECRET }}"
+        AWS_REGION: us-west-2
+      run: |
+        FILES=$(find ${{ github.workspace }}/target/wheels -type f -name '*.whl')
+        while IFS= read -r file; do
+          filename=$(basename $file)
+          aws --no-progress s3 cp "$file" "s3://download.chia.net/simple-dev/clvm-tools-rs/$filename"
+        done <<< "$FILES"

.github/workflows/build-m1-wheel.yml

@@ -32,17 +32,25 @@ jobs:
 
     - name: Set up rust
       run: |
-        curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > rust.sh
-        arch -arm64 sh rust.sh -y
+        curl https://static.rust-lang.org/rustup/dist/aarch64-apple-darwin/rustup-init.sha256 | awk '{print $1 " *rustup-init"}' > checksum.txt
+        curl -O https://static.rust-lang.org/rustup/dist/aarch64-apple-darwin/rustup-init
+        cat checksum.txt
+        shasum -a 256 -c checksum.txt
+
+    - name: Install rust
+      run: |
+        chmod +x rustup-init
+        ./rustup-init -y || (echo "Rust is already installed. Exiting..." && exit 2)
+        rm rustup-init checksum.txt
 
     - name: Build m1 wheels
       run: |
         arch -arm64 python3 -m venv venv
         . ./venv/bin/activate
         export PATH=~/.cargo/bin:$PATH
-        arch -arm64 pip install maturin==0.12.20
-        arch -arm64 maturin build --no-sdist -i python --release --strip --cargo-extra-args="--all-features"
-        arch -arm64 cargo test --no-default-features
+        arch -arm64 pip install maturin==1.1.0
+        arch -arm64 maturin build -i python --release --strip
+        arch -arm64 cargo test
 
     - name: Install clvm_tools_rs wheel
       run: |
@@ -75,6 +83,11 @@ jobs:
         arch -arm64 python -c 'import clvm_tools; print(clvm_tools.__file__)'
         arch -arm64 python -c 'import clvm_tools_rs; print(clvm_tools_rs.__file__)'
 
+    - name: Install pytest
+      run: |
+        . ./venv/bin/activate
+        pip install pytest pytest-asyncio
+
 # Cost tests are currently failing.
 #    - name: Run tests from clvm
 #      run: |
@@ -83,11 +96,10 @@ jobs:
 #        arch -arm64 python -m py.test tests
 
     - name: Run tests from clvm_tools
-      continue-on-error: true
       run: |
         . ./venv/bin/activate
-        cd clvm_tools_rs
-        arch -arm64 cargo test
+        cd clvm_tools
+        arch -arm64 pytest
 
     - name: Upload wheels
       uses: actions/upload-artifact@v3
@@ -100,10 +112,16 @@ jobs:
       shell: bash
       run: |
         unset HAS_SECRET
+        unset HAS_AWS_SECRET
+
         if [ -n "$SECRET" ]; then HAS_SECRET='true' ; fi
         echo "HAS_SECRET=${HAS_SECRET}" >>$GITHUB_OUTPUT
+
+        if [ -n "$AWS_SECRET" ]; then HAS_AWS_SECRET='true' ; fi
+        echo HAS_AWS_SECRET=${HAS_AWS_SECRET} >>$GITHUB_OUTPUT
       env:
         SECRET: "${{ secrets.test_pypi_password }}"
+        AWS_SECRET: "${{ secrets.INSTALLER_UPLOAD_KEY }}"
 
     - name: Install twine
       run: arch -arm64 pip install twine
@@ -124,3 +142,16 @@ jobs:
         TWINE_NON_INTERACTIVE: 1
         TWINE_PASSWORD: ${{ secrets.test_pypi_password }}
       run: arch -arm64 twine upload --non-interactive --skip-existing --verbose 'target/wheels/*'
+
+    - name: Publish Dev
+      if: steps.check_secrets.outputs.HAS_AWS_SECRET && github.ref == 'refs/heads/dev'
+      env:
+        AWS_ACCESS_KEY_ID: "${{ secrets.INSTALLER_UPLOAD_KEY }}"
+        AWS_SECRET_ACCESS_KEY: "${{ secrets.INSTALLER_UPLOAD_SECRET }}"
+        AWS_REGION: us-west-2
+      run: |
+        FILES=$(find ${{ github.workspace }}/target/wheels -type f -name '*.whl')
+        while IFS= read -r file; do
+          filename=$(basename $file)
+          aws --no-progress s3 cp "$file" "s3://download.chia.net/simple-dev/clvm-tools-rs/$filename"
+        done <<< "$FILES"

.github/workflows/build-test.yml

@@ -20,7 +20,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [macos-latest, ubuntu-latest, windows-latest]
-        python: [3.7, 3.8, 3.9, '3.10']
+        python: [3.8, 3.9, '3.10', 3.11]
 
     steps:
     - uses: actions/checkout@v3
@@ -43,10 +43,12 @@ jobs:
 
     - name: Install dependencies
       run: |
-          python -m pip install maturin==0.12.20
+          python -m pip install maturin==1.1.0
 
     - name: Build MacOs with maturin on Python ${{ matrix.python }}
       if: startsWith(matrix.os, 'macos')
+      env:
+        MACOSX_DEPLOYMENT_TARGET: '11.0'
       run: |
         python -m venv venv
         ln -s venv/bin/activate
@@ -56,8 +58,8 @@ jobs:
     - name: Build Linux in manylinux2010 with maturin on Python ${{ matrix.python }}
       if: startsWith(matrix.os, 'ubuntu')
       run: |
-        podman run --rm=true \
-          -v ${{ github.workspace }}:/ws:rw --workdir=/ws \
+        docker run --rm \
+          -v ${{ github.workspace }}:/ws --workdir=/ws \
           ghcr.io/chia-network/build-images/centos-pypa-rust-x86_64 \
           bash -exc '\
             yum -y install libc6 openssl-devel && \
@@ -77,14 +79,14 @@ jobs:
             . ./activate && \
             pip install --upgrade pip
           '
-        docker run --rm -v $(pwd):/io ghcr.io/pyo3/maturin:v0.13.1 build --release --strip --manylinux 2010
+        docker run --rm -v $(pwd):/io ghcr.io/pyo3/maturin:v1.1.0 build --release --strip --manylinux 2014
         # Refresh in case any ownerships changed.
         mv target target.docker && cp -r target.docker target
         # Ensure an empty .cargo-lock file exists.
         touch target/release/.cargo-lock
 
     - name: Build alpine wheel via docker
-      if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.python, '3.7')
+      if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.python, '3.8')
       run: |
         cd resources/alpine && docker build -t clvm-tools-rs-alpine .
         docker run -v ${GITHUB_WORKSPACE}:/root/clvm_tools_rs -t clvm-tools-rs-alpine sh /root/build-alpine.sh
@@ -95,7 +97,7 @@ jobs:
         python -m venv venv
         echo ". .\venv\Scripts\Activate.ps1" > activate.ps1
         . ./activate
-        maturin build --no-sdist -i python --release --strip
+        maturin build -i python --release --strip
         # Find and install the newly built wheel
         python support/wheelname.py
 
@@ -126,8 +128,10 @@ jobs:
         pip install clvm_rs
 
         echo "installing clvm_tools for clvm tests"
-        # clvm tools is required to run the tests is clvm
-        python -m pip install clvm_tools
+
+        # Ensure clvm_tools is installed from its own repo.
+        git clone https://github.com/Chia-Network/clvm_tools.git --branch=main --single-branch
+        python -m pip install ./clvm_tools
 
     - name: Ensure clvm, clvm_rs, clvm_tools_rs are installed
       run: |
@@ -138,7 +142,7 @@ jobs:
         python -c 'import clvm_tools_rs; print(clvm_tools_rs.__file__)'
 
     - name: Verify recompilation of old sources match
-      if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.python, '3.7')
+      if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.python, '3.8')
       run: |
         . ./activate
         # Build cmd line tools
@@ -153,7 +157,7 @@ jobs:
         (cd chia-blockchain && python recompile_check.py)
 
     - name: Test Classic command line tools with pytest
-      if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.python, '3.7')
+      if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.python, '3.8')
       run: |
         python -m pip install pytest
         # This script installs the wheel built during this workflow.
@@ -162,7 +166,7 @@ jobs:
         (cd resources/tests/cmdline/tests && py.test cmds_test.py )
 
     - name: Verify recompilation of cl21 sources
-      if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.python, '3.7')
+      if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.python, '3.8')
       run: |
         . ./activate
         # We need chia-rs for the following.
@@ -173,19 +177,35 @@ jobs:
         python support/wheelname.py
         (cd resources/tests && python check-recompiles.py)
 
+    - name: Verify recompilation follows date and modification rules
+      if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.python, '3.8')
+      run: |
+        python support/wheelname.py
+        python resources/tests/test-clvm-recompile-behavior.py
+
+    - name: Install pytest
+      run: |
+        . ./activate
+        pip install pytest pytest-asyncio
+
     - name: Run tests from clvm
       run: |
         . ./activate
         cd clvm
-        pip install pytest pytest-asyncio
+        pytest tests
+
+    - name: Run tests from clvm_tools
+      run: |
+        . ./activate
+        cd clvm_tools
         pytest tests
 
     - name: Run tests
-      if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.python, '3.7')
+      if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.python, '3.8')
       run: cargo test --no-default-features
 
     - name: Check coverage
-      if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.python, '3.7')
+      if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.python, '3.8')
       run: |
         cargo install grcov
         rustup component add llvm-tools-preview
@@ -205,10 +225,17 @@ jobs:
       shell: bash
       run: |
         unset HAS_SECRET
+        unset HAS_AWS_SECRET
+
         if [ -n "$SECRET" ]; then HAS_SECRET='true' ; fi
         echo "HAS_SECRET=${HAS_SECRET}" >>$GITHUB_OUTPUT
+
+        if [ -n "$AWS_SECRET" ]; then HAS_AWS_SECRET='true' ; fi
+        echo HAS_AWS_SECRET=${HAS_AWS_SECRET} >>$GITHUB_OUTPUT
+
       env:
         SECRET: "${{ secrets.test_pypi_password }}"
+        AWS_SECRET: "${{ secrets.INSTALLER_UPLOAD_KEY }}"
 
     - name: publish (PyPi)
       if: startsWith(github.event.ref, 'refs/tags') && steps.check_secrets.outputs.HAS_SECRET
@@ -227,6 +254,21 @@ jobs:
         TWINE_PASSWORD: ${{ secrets.test_pypi_password }}
       run: twine upload --non-interactive --skip-existing --verbose 'target/wheels/*'
 
+    - name: Publish Dev
+      if: steps.check_secrets.outputs.HAS_AWS_SECRET && github.ref == 'refs/heads/dev'
+      shell: bash
+      working-directory: ./target/wheels
+      env:
+        AWS_ACCESS_KEY_ID: "${{ secrets.INSTALLER_UPLOAD_KEY }}"
+        AWS_SECRET_ACCESS_KEY: "${{ secrets.INSTALLER_UPLOAD_SECRET }}"
+        AWS_REGION: us-west-2
+      run: |
+        FILES=$(find . -type f -name '*.whl')
+        while IFS= read -r file; do
+          filename=$(basename $file)
+          aws --no-progress s3 cp "$file" "s3://download.chia.net/simple-dev/clvm-tools-rs/$filename"
+        done <<< "$FILES"
+
   fmt:
     runs-on: ubuntu-20.04
     name: cargo fmt

.github/workflows/npm-publish.yml

@@ -30,7 +30,7 @@ jobs:
           components: rustfmt, clippy
 
       - name: install wasm-pack
-        run: cargo install wasm-pack
+        run: cargo install --version 0.11.1 wasm-pack
 
       - name: wasm-pack build and pack
         run: wasm-pack build --release --target=nodejs wasm && wasm-pack pack wasm

CHANGELOG.md

@@ -1,6 +1,22 @@
 # clvm_tools_rs Changelog
 
-## 0.0.31 Chia CLVM Tools Rust 2023-04-17
+## 0.1.34
+### Fixed
+- Fixed chialisp compilation issues
+
+
+## 0.1.33
+### Changed
+- Set macOS deployment target to 10.14
+- Ensure we flush streams in case the runtime system doesn't get a chance
+### Fixed
+- Fix erroneous detection of recursion when two similar inline siblings
+
+
+## 0.1.32
+Skipped
+
+## 0.1.31 Chia CLVM Tools Rust 2023-04-17
 
 ### Added