From 12d2c8dcb0d75a6a8be55571db0fb741d01b8ea0 Mon Sep 17 00:00:00 2001 From: messense Date: Wed, 26 Oct 2022 13:58:55 +0800 Subject: [PATCH 1/6] Configure bors --- .github/bors.toml | 12 +++ .github/workflows/test.yml | 166 ++++++++++++++++++++++++++----------- 2 files changed, 131 insertions(+), 47 deletions(-) create mode 100644 .github/bors.toml diff --git a/.github/bors.toml b/.github/bors.toml new file mode 100644 index 000000000..d6f8af7a6 --- /dev/null +++ b/.github/bors.toml @@ -0,0 +1,12 @@ +delete_merged_branches = true +required_approvals = 0 +use_codeowners = false +status = [ + # GitHub Actions + "conclusion", + # Cirrus CI + "Test (x86_64 FreeBSD)", + "Test (arm64 macOS)", + "Test (arm64 Linux)" +] +timeout_sec = 21600 \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 72d5c1546..8bec5b035 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -3,7 +3,10 @@ name: Test on: push: branches: - - main + # for bors r+ + - staging + # for bors try + - trying pull_request: workflow_dispatch: @@ -12,12 +15,86 @@ concurrency: cancel-in-progress: true jobs: + generate-matrix: + name: Generate Matrix + runs-on: ubuntu-latest + outputs: + python-version: ${{ steps.generate-matrix.outputs.python-version }} + fail-fast: ${{ steps.generate-matrix.outputs.fail-fast }} + steps: + - uses: actions/setup-node@v2 + with: + node-version: 16 + - run: npm install js-yaml + - name: Generate matrix + id: generate-matrix + uses: actions/github-script@v6 + with: + script: | + const yaml = require('js-yaml') + const PYTHON_VERSIONS = yaml.load(process.env.PYTHON_VERSION) + + if (context.eventName == 'workflow_dispatch') { + // Run all of them on workflow dispatch + core.setOutput('python-version', PYTHON_VERSIONS) + core.setOutput('fail-fast', 'false') + } else if (context.eventName == 'push') { + const commitMessage = process.env.COMMIT_MESSAGE.trim() + const matches = commitMessage.match(/(Try|Merge) #([0-9]+):/) + if (matches) { + const prNumber = matches[2] + const { data: { labels: labels } } = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: prNumber + }) + const labelNames = labels.map(label => label.name) + if (labelNames.includes('CI-no-fail-fast')) { + core.setOutput('fail-fast', 'false') + } + } + core.setOutput('python-version', PYTHON_VERSIONS) + } else if (context.eventName == 'pull_request') { + const { data: { labels: labels } } = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.payload.pull_request.number + }) + const labelNames = labels.map(label => label.name) + if (labelNames.includes('CI-no-fail-fast')) { + core.setOutput('fail-fast', 'false') + } + // Only run latest CPython and PyPy tests on pull requests + const firstPyPy = PYTHON_VERSIONS.findIndex(version => version.startsWith('pypy')) + const pythonVersions = [PYTHON_VERSIONS[firstPyPy - 1], PYTHON_VERSIONS[PYTHON_VERSIONS.length - 1]] + core.setOutput('python-version', pythonVersions) + } + env: + COMMIT_MESSAGE: > + ${{ + (( + (startsWith(github.event.head_commit.message, 'Try #') || startsWith(github.event.head_commit.message, 'Merge #')) && + github.event.head_commit.author.username == 'bors[bot]' + ) && github.event.head_commit.message) || '' + }} + PYTHON_VERSION: | + - '3.7' + - '3.8' + - '3.9' + - '3.10' + - '3.11' + - 'pypy3.7' + - 'pypy3.8' + - 'pypy3.9' + test: name: Test + needs: [ generate-matrix ] strategy: - fail-fast: ${{ !contains(github.event.pull_request.labels.*.name, 'CI-no-fail-fast') }} + fail-fast: ${{ needs.generate-matrix.outputs.fail-fast != 'false' }} matrix: os: [ ubuntu-latest, macos-latest, windows-latest ] + python-version: ${{ fromJson(needs.generate-matrix.outputs.python-version) }} runs-on: ${{ matrix.os }} env: RUST_BACKTRACE: '1' @@ -29,18 +106,13 @@ jobs: activate-environment: '' - uses: actions/setup-python@v4 with: - python-version: "3.7" - - uses: actions/setup-python@v4 - with: - python-version: "3.8" - - uses: actions/setup-python@v4 - with: - python-version: "3.9" - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - - name: Install cffi and virtualenv - run: pip install cffi virtualenv ziglang~=0.9.0 twine + python-version: ${{ matrix.python-version }} + architecture: 'x64' + - name: Install cffi + if: ${{ !contains(matrix.python-version, 'pypy') }} + run: pip install cffi + - name: Install python packages + run: pip install virtualenv ziglang~=0.9.0 twine - uses: actions-rs/toolchain@v1 id: rustup with: @@ -84,33 +156,8 @@ jobs: with: path: test-crates/targets key: test-crates-${{ runner.os }}-${{ steps.rustup.outputs.rustc_hash }}-${{ hashFiles('test-crates/*/Cargo.lock') }} - - name: cargo test run: cargo nextest run --features password-storage - - uses: actions/setup-python@v4 - with: - python-version: "pypy-3.7" - - name: test build pypy wheel - shell: bash - run: | - set -ex - cargo run -- build -i pypy3 -m test-crates/pyo3-mixed-submodule/Cargo.toml - pypy3 -m pip install --force-reinstall --no-index --find-links test-crates/pyo3-mixed-submodule/target/wheels pyo3-mixed-submodule - pypy3 -m pip install pytest - pypy3 -m pytest test-crates/pyo3-mixed-submodule/tests/ - - name: test build pypy wheel with abi3 - shell: bash - run: | - set -ex - cargo run -- build -i pypy3 -m test-crates/pyo3-pure/Cargo.toml -vv - pypy3 -m pip install --force-reinstall --no-index --find-links test-crates/pyo3-pure/target/wheels pyo3-pure - pypy3 -m pip install pytest - pypy3 -m pytest test-crates/pyo3-pure/tests - - cargo run -- pep517 build-wheel -i pypy3 -m test-crates/pyo3-pure/Cargo.toml -vv - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - name: test cross compiling with zig shell: bash run: | @@ -120,19 +167,23 @@ jobs: rustup target add aarch64-apple-darwin # abi3 - cargo run -- build -i python -m test-crates/pyo3-pure/Cargo.toml --target aarch64-unknown-linux-gnu --zig - cargo run -- build -i python -m test-crates/pyo3-pure/Cargo.toml --target aarch64-unknown-linux-musl --zig - cargo run -- build -i python -m test-crates/pyo3-pure/Cargo.toml --target aarch64-apple-darwin --zig + cargo run -- build -i ${{ matrix.python-version }} -m test-crates/pyo3-pure/Cargo.toml --target aarch64-unknown-linux-gnu --zig + cargo run -- build -i ${{ matrix.python-version }} -m test-crates/pyo3-pure/Cargo.toml --target aarch64-unknown-linux-musl --zig + if [[ "${{ matrix.python-version }}" != "pypy"* ]]; then + cargo run -- build -i ${{ matrix.python-version }} -m test-crates/pyo3-pure/Cargo.toml --target aarch64-apple-darwin --zig + fi # Check wheels with twine twine check --strict test-crates/pyo3-pure/target/wheels/*.whl # non-abi3 - cargo run -- build -i python3.9 -m test-crates/pyo3-mixed/Cargo.toml --target aarch64-unknown-linux-gnu --zig - cargo run -- build -i python3.9 -m test-crates/pyo3-mixed/Cargo.toml --target aarch64-apple-darwin --zig + cargo run -- build -i ${{ matrix.python-version }} -m test-crates/pyo3-mixed/Cargo.toml --target aarch64-unknown-linux-gnu --zig + if [[ "${{ matrix.python-version }}" != "pypy"* ]]; then + cargo run -- build -i ${{ matrix.python-version }} -m test-crates/pyo3-mixed/Cargo.toml --target aarch64-apple-darwin --zig + fi # Check wheels with twine twine check --strict test-crates/pyo3-mixed/target/wheels/*.whl - name: test cross compiling windows wheel - if: matrix.os == 'ubuntu-latest' + if: ${{ matrix.os == 'ubuntu-latest' && !contains(matrix.python-version, 'pypy') }} run: | set -ex sudo apt-get install -y mingw-w64 @@ -145,7 +196,7 @@ jobs: cargo run -- build -m test-crates/pyo3-pure/Cargo.toml --target x86_64-pc-windows-msvc # no-abi3 - cargo run -- build -i python3.9 -m test-crates/pyo3-mixed/Cargo.toml --target x86_64-pc-windows-msvc + cargo run -- build -i python${{ matrix.python-version }} -m test-crates/pyo3-mixed/Cargo.toml --target x86_64-pc-windows-msvc - name: test compiling with PYO3_CONFIG_FILE shell: bash run: | @@ -235,7 +286,7 @@ jobs: fail-fast: ${{ !contains(github.event.pull_request.labels.*.name, 'CI-no-fail-fast') }} matrix: # manylinux2014 fails with `/lib64/libc.so.6: version `GLIBC_2.18'` not found recently - # could be a upstream Rust issue, disable it for now + # could be an upstream Rust issue, disable it for now # # manylinux: [ 'manylinux2014', 'manylinux_2_24' ] manylinux: [ 'manylinux_2_24' ] @@ -444,3 +495,24 @@ jobs: uses: Swatinem/rust-cache@v1 - name: cargo check run: cargo check --target ${{ matrix.platform.target }} + + conclusion: + needs: + - test + - test-alpine + - test-auditwheel + - test-docker + - test-cross-compile + - test-bootstrap + - test-msrv + - check + if: always() + runs-on: ubuntu-latest + steps: + - name: Result + run: | + jq -C <<< "${needs}" + # Check if all needs were successful or skipped. + "$(jq -r 'all(.result as $result | (["success", "skipped"] | contains([$result])))' <<< "${needs}")" + env: + needs: ${{ toJson(needs) }} From af58b0b1ed98b5190aa8c54c12256ce91c872bf4 Mon Sep 17 00:00:00 2001 From: messense Date: Mon, 31 Oct 2022 11:52:52 +0800 Subject: [PATCH 2/6] test: ignore develop_pyo3_pure_conda for now --- tests/run.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/run.rs b/tests/run.rs index 1a3cab6dc..b5eaf75d5 100644 --- a/tests/run.rs +++ b/tests/run.rs @@ -21,6 +21,7 @@ fn develop_pyo3_pure() { } #[test] +#[ignore] fn develop_pyo3_pure_conda() { // Only run on GitHub Actions for now if std::env::var("GITHUB_ACTIONS").is_ok() { From 937d7a98ed72201db24eac5ad57e72c053a071fa Mon Sep 17 00:00:00 2001 From: messense Date: Mon, 31 Oct 2022 12:08:24 +0800 Subject: [PATCH 3/6] CI: Add Python version to test crates cache key --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8bec5b035..1bf34c06f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -155,7 +155,7 @@ jobs: uses: actions/cache@v2 with: path: test-crates/targets - key: test-crates-${{ runner.os }}-${{ steps.rustup.outputs.rustc_hash }}-${{ hashFiles('test-crates/*/Cargo.lock') }} + key: test-crates-${{ runner.os }}-${{ steps.rustup.outputs.rustc_hash }}-${{ matrix.python-version }}-${{ hashFiles('test-crates/*/Cargo.lock') }} - name: cargo test run: cargo nextest run --features password-storage - name: test cross compiling with zig From 5c0739fa85d8a50f6242f6b762af970ec5ca19f0 Mon Sep 17 00:00:00 2001 From: messense Date: Mon, 31 Oct 2022 15:06:18 +0800 Subject: [PATCH 4/6] CI: `MATURIN_TEST_PYTHON` set to python3 for PyPy jobs --- .github/workflows/test.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1bf34c06f..35da88874 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -156,6 +156,10 @@ jobs: with: path: test-crates/targets key: test-crates-${{ runner.os }}-${{ steps.rustup.outputs.rustc_hash }}-${{ matrix.python-version }}-${{ hashFiles('test-crates/*/Cargo.lock') }} + - name: Set MATURIN_TEST_PYTHON for PyPy + shell: bash + if: contains(matrix.python-version, 'pypy') + run: echo "MATURIN_TEST_PYTHON=pypy3" >> $GITHUB_ENV - name: cargo test run: cargo nextest run --features password-storage - name: test cross compiling with zig From dbc9d5c8793c3523a6733b03527324a042363991 Mon Sep 17 00:00:00 2001 From: messense Date: Mon, 31 Oct 2022 15:25:28 +0800 Subject: [PATCH 5/6] Only run pyo3_no_extension_module test with CPython --- tests/run.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/run.rs b/tests/run.rs index b5eaf75d5..1be545377 100644 --- a/tests/run.rs +++ b/tests/run.rs @@ -349,7 +349,14 @@ fn abi3_without_version() { #[test] #[cfg(all(target_os = "linux", target_env = "gnu"))] fn pyo3_no_extension_module() { - handle_result(errors::pyo3_no_extension_module()) + let python = test_python_path().map(PathBuf::from).unwrap_or_else(|| { + let target = Target::from_target_triple(None).unwrap(); + target.get_python() + }); + let python_implementation = get_python_implementation(&python).unwrap(); + if python_implementation == "cpython" { + handle_result(errors::pyo3_no_extension_module()) + } } #[test] From 49f11fd662aadcd425439d395c140df9c45f5449 Mon Sep 17 00:00:00 2001 From: messense Date: Tue, 1 Nov 2022 09:27:40 +0800 Subject: [PATCH 6/6] Use `PyModule::filename` patch from pyo3 main branch --- test-crates/pyo3-mixed-py-subdir/Cargo.lock | 15 +++++---------- test-crates/pyo3-mixed-py-subdir/Cargo.toml | 4 ++++ 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/test-crates/pyo3-mixed-py-subdir/Cargo.lock b/test-crates/pyo3-mixed-py-subdir/Cargo.lock index ee6f69a34..56ca2d6d4 100644 --- a/test-crates/pyo3-mixed-py-subdir/Cargo.lock +++ b/test-crates/pyo3-mixed-py-subdir/Cargo.lock @@ -92,8 +92,7 @@ dependencies = [ [[package]] name = "pyo3" version = "0.17.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "201b6887e5576bf2f945fe65172c1fcbf3fcf285b23e4d71eb171d9736e38d32" +source = "git+https://github.com/PyO3/pyo3.git?branch=main#24dabc5c08279d90673c1642b2b63f91c8a03693" dependencies = [ "cfg-if", "indoc", @@ -109,8 +108,7 @@ dependencies = [ [[package]] name = "pyo3-build-config" version = "0.17.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf0708c9ed01692635cbf056e286008e5a2927ab1a5e48cdd3aeb1ba5a6fef47" +source = "git+https://github.com/PyO3/pyo3.git?branch=main#24dabc5c08279d90673c1642b2b63f91c8a03693" dependencies = [ "once_cell", "target-lexicon", @@ -119,8 +117,7 @@ dependencies = [ [[package]] name = "pyo3-ffi" version = "0.17.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90352dea4f486932b72ddf776264d293f85b79a1d214de1d023927b41461132d" +source = "git+https://github.com/PyO3/pyo3.git?branch=main#24dabc5c08279d90673c1642b2b63f91c8a03693" dependencies = [ "libc", "pyo3-build-config", @@ -129,8 +126,7 @@ dependencies = [ [[package]] name = "pyo3-macros" version = "0.17.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7eb24b804a2d9e88bfcc480a5a6dd76f006c1e3edaf064e8250423336e2cd79d" +source = "git+https://github.com/PyO3/pyo3.git?branch=main#24dabc5c08279d90673c1642b2b63f91c8a03693" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -141,8 +137,7 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" version = "0.17.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f22bb49f6a7348c253d7ac67a6875f2dc65f36c2ae64a82c381d528972bea6d6" +source = "git+https://github.com/PyO3/pyo3.git?branch=main#24dabc5c08279d90673c1642b2b63f91c8a03693" dependencies = [ "proc-macro2", "quote", diff --git a/test-crates/pyo3-mixed-py-subdir/Cargo.toml b/test-crates/pyo3-mixed-py-subdir/Cargo.toml index 239d1d233..26a8b8abd 100644 --- a/test-crates/pyo3-mixed-py-subdir/Cargo.toml +++ b/test-crates/pyo3-mixed-py-subdir/Cargo.toml @@ -16,3 +16,7 @@ crate-type = ["cdylib"] [package.metadata.maturin] python-source = "python" name = "pyo3_mixed_py_subdir._pyo3_mixed" + +[patch.crates-io] +pyo3-ffi = { git = "https://github.com/PyO3/pyo3.git", branch = "main" } +pyo3 = { git = "https://github.com/PyO3/pyo3.git", branch = "main" }