tests #92
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# This file has transitioning to run almost everything, with rules defined in | |
# this file rather than across lots of workflow files. | |
name: tests | |
on: | |
pull_request: | |
# Add `labeled`, so we can trigger a new run by adding a `pr-nightly` | |
# label, which we then use to trigger a `nightly` run. | |
types: [opened, reopened, synchronize, labeled] | |
push: | |
branches: | |
- main | |
schedule: | |
# Pick a random time, something that others won't pick, to reduce GH's demand variance. | |
- cron: "49 10 * * *" | |
workflow_dispatch: | |
workflow_call: | |
concurrency: | |
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} | |
cancel-in-progress: true | |
env: | |
CARGO_TERM_COLOR: always | |
CLICOLOR_FORCE: 1 | |
RUSTFLAGS: "-C debuginfo=0" | |
RUSTDOCFLAGS: "-Dwarnings" | |
jobs: | |
# This assesses whether we need to run jobs. Some of them are defined only by | |
# the changes in PR, others also define a set of other criteria, such as | |
# whether a label has been added, or we're on `main` branch. | |
rules: | |
runs-on: ubuntu-latest | |
permissions: | |
pull-requests: read | |
outputs: | |
book: ${{ steps.changes.outputs.book }} | |
dotnet: ${{ steps.changes.outputs.dotnet }} | |
devcontainer-push: ${{ steps.devcontainer-push.outputs.run }} | |
devcontainer-build: ${{ steps.devcontainer-build.outputs.run }} | |
elixir: ${{ steps.changes.outputs.elixir }} | |
grammars: ${{ steps.changes.outputs.grammars }} | |
java: ${{ steps.changes.outputs.java }} | |
js: ${{ steps.changes.outputs.js }} | |
lib: ${{ steps.changes.outputs.lib }} | |
# Run tests such as rust tests for all-OSs, and bindings tests on ubuntu. | |
# Somewhat a tradeoff between coverage and ensuring our CI queues stay | |
# short. | |
main: ${{ steps.main.outputs.run }} | |
# Run all tests | |
nightly: ${{ steps.nightly.outputs.run }} | |
# Really run all tests (e.g. taskfile & devcontainer) | |
nightly-schedule: ${{ steps.nightly-schedule.outputs.run }} | |
php: ${{ steps.changes.outputs.php }} | |
python: ${{ steps.changes.outputs.python }} | |
rust: ${{ steps.changes.outputs.rust }} | |
taskfile: ${{ steps.changes.outputs.taskfile }} | |
web: ${{ steps.changes.outputs.web }} | |
steps: | |
- name: 📂 Checkout code | |
uses: actions/checkout@v4 | |
- uses: dorny/paths-filter@v2 | |
id: changes | |
with: | |
filters: | | |
book: | |
- .github/workflows/check-links-book.yaml | |
- web/book/** | |
dotnet: | |
- prqlc/bindings/prql-dotnet/** | |
- prqlc/bindings/clib/** | |
- .github/workflows/test-dotnet.yaml | |
devcontainer-push: | |
- .devcontainer/**/*Dockerfile | |
- Taskfile.yml | |
devcontainer-build: | |
- .devcontainer/**/*Dockerfile | |
- .github/workflows/build-devcontainer.yaml | |
- Taskfile.yml | |
grammars: | |
- grammars/** | |
elixir: | |
- prqlc/bindings/elixir/** | |
- prqlc/bindings/clib/** | |
- .github/workflows/test-elixir.yaml | |
java: | |
- prqlc/bindings/java/** | |
- prqlc/bindings/clib/** | |
- .github/workflows/test-java.yaml | |
js: | |
- prqlc/bindings/js/** | |
- .github/workflows/test-js.yaml | |
clib: | |
- prqlc/bindings/clib/** | |
- .github/workflows/test-clib.yaml | |
main: | |
- "**/Cargo.*" | |
- .github/** | |
nightly: | |
- .github/workflows/nightly.yaml | |
- .github/workflows/release.yaml | |
- Cargo.lock | |
- rust-toolchain.toml | |
- .cargo/** | |
php: | |
- prqlc/bindings/php/** | |
- prqlc/bindings/clib/** | |
- .github/workflows/test-php.yaml | |
python: | |
- prqlc/bindings/python/** | |
- .github/workflows/test-python.yaml | |
rust: | |
- "**/*.rs" | |
- prqlc/** | |
- web/book/** | |
- .github/workflows/test-rust.yaml | |
taskfile: | |
- Taskfile.yml | |
web: | |
- "web/**" | |
- ".github/workflows/build-web.yaml" | |
- "**.md" | |
# We put a few of the more complex rules as steps here, rather than having | |
# them inline. There's no strict delineation between logic here vs. inline. | |
- id: nightly | |
# TODO: actionlint annoyingly blocks this — try and find a way of getting | |
# it back without too much trouble... | |
# contains(github.event.pull_request.title, '!') || | |
run: | |
echo "run=${{ steps.changes.outputs.nightly == 'true' || | |
contains(github.event.pull_request.labels.*.name, 'pr-nightly') || | |
github.event.schedule }}" >>"$GITHUB_OUTPUT" | |
- id: nightly-schedule | |
run: | |
echo "run=${{ github.event.schedule == 'true' }}" >>"$GITHUB_OUTPUT" | |
- id: main | |
run: | |
echo "run=${{ steps.changes.outputs.main == 'true' || github.ref == | |
'refs/heads/main' || steps.nightly.outputs.run == 'true' }}" >> | |
"$GITHUB_OUTPUT" | |
- id: devcontainer-push | |
# We push the devcontainer if the files have changed, and we've merged | |
# to main. | |
run: | |
echo "run=${{ steps.changes.outputs.devcontainer-push == 'true' && | |
github.ref == 'refs/heads/main' && github.event_name == 'push' }}" >> | |
"$GITHUB_OUTPUT" | |
- id: devcontainer-build | |
run: | |
echo "run=${{ steps.devcontainer-push.outputs.run == 'true' || | |
steps.changes.outputs.devcontainer-build == 'true' || | |
steps.changes.outputs.nightly-schedule == 'true' }}" >> | |
"$GITHUB_OUTPUT" | |
test-rust: | |
needs: rules | |
if: needs.rules.outputs.rust == 'true' || needs.rules.outputs.main == 'true' | |
uses: ./.github/workflows/test-rust.yaml | |
strategy: | |
matrix: | |
include: | |
- target: x86_64-unknown-linux-gnu | |
os: ubuntu-latest | |
features: test-dbs-external | |
# Only run wasm on ubuntu, given it's the same rust target. (There is a | |
# possibility of having a failure on just one platform, but it's quite | |
# unlikely. If we do observe this, we can expand, or introduce a | |
# `test-actually-all.yaml` which accounts for these corner cases without | |
# using runners & cache space) | |
- target: wasm32-unknown-unknown | |
os: ubuntu-latest | |
features: "" | |
with: | |
os: ubuntu-latest | |
target: ${{ matrix.target }} | |
features: ${{ matrix.features }} | |
nightly: ${{ needs.rules.outputs.nightly == 'true' }} | |
test-python: | |
needs: rules | |
if: | |
needs.rules.outputs.python == 'true' || needs.rules.outputs.main == 'true' | |
uses: ./.github/workflows/test-python.yaml | |
with: | |
# Only run on ubuntu unless there's a lang-specific change or we're | |
# running nightly. | |
# | |
# An alternative to these somewhat horrible expressions would be | |
# `test-python` & `test-python-more` workflows; though it would use up our | |
# 20 workflow limit. | |
oss: | |
${{ (needs.rules.outputs.python == 'true' || needs.rules.outputs.nightly | |
== 'true') && '["ubuntu-latest", "macos-latest", "windows-latest"]' || | |
'["ubuntu-latest"]' }} | |
test-js: | |
needs: rules | |
if: needs.rules.outputs.js == 'true' || needs.rules.outputs.main == 'true' | |
uses: ./.github/workflows/test-js.yaml | |
with: | |
# Only run on ubuntu unless there's a lang-specific change or we're running nightly. | |
oss: | |
${{ (needs.rules.outputs.js == 'true' || needs.rules.outputs.nightly == | |
'true') && '["ubuntu-latest", "macos-latest", "windows-latest"]' || | |
'["ubuntu-latest"]' }} | |
test-dotnet: | |
needs: rules | |
if: | |
needs.rules.outputs.dotnet == 'true' || needs.rules.outputs.main == 'true' | |
uses: ./.github/workflows/test-dotnet.yaml | |
test-php: | |
needs: rules | |
if: needs.rules.outputs.php == 'true' || needs.rules.outputs.main == 'true' | |
uses: ./.github/workflows/test-php.yaml | |
test-java: | |
needs: rules | |
if: needs.rules.outputs.java == 'true' || needs.rules.outputs.main == 'true' | |
uses: ./.github/workflows/test-java.yaml | |
with: | |
# Currently we never run windows | |
oss: | |
${{ (needs.rules.outputs.java == 'true' || needs.rules.outputs.nightly | |
== 'true') && '["ubuntu-latest", "macos-latest"]' || '["ubuntu-latest"]' | |
}} | |
test-elixir: | |
needs: rules | |
if: | |
needs.rules.outputs.elixir == 'true' || needs.rules.outputs.main == 'true' | |
uses: ./.github/workflows/test-elixir.yaml | |
with: | |
# Currently we never run Mac, see prql-elixir docs for details | |
oss: | |
${{ (needs.rules.outputs.elixir == 'true' || needs.rules.outputs.nightly | |
== 'true') && '["ubuntu-latest", "windows-latest"]' || | |
'["ubuntu-latest"]' }} | |
test-clib: | |
needs: rules | |
if: needs.rules.outputs.lib == 'true' || needs.rules.outputs.main == 'true' | |
uses: ./.github/workflows/test-clib.yaml | |
test-taskfile: | |
needs: rules | |
if: | |
# We only run on nightly scheduled, since this is very expensive and we | |
# don't want to have to run it on, for example, every dependency change. | |
needs.rules.outputs.taskfile == 'true' || | |
needs.rules.outputs.nightly-schedule == 'true' | |
runs-on: macos-latest | |
steps: | |
- name: 📂 Checkout code | |
uses: actions/checkout@v4 | |
- run: ./.github/workflows/scripts/set_version.sh | |
- name: 💰 Cache | |
uses: Swatinem/rust-cache@v2 | |
with: | |
prefix-key: ${{ env.version }} | |
# The mac rust cache key. It's not _that_ useful since this will build | |
# much more, but it's better than nothing. We can't have our own | |
# cache, since we're out of cache space and this workflow takes 1.5GB. | |
shared-key: rust-x86_64-apple-darwin-test-dbs | |
save-if: false | |
- uses: actions/setup-python@v4 | |
with: | |
python-version: "3.10" | |
- name: Install Task | |
uses: arduino/setup-task@v1 | |
with: | |
repo-token: ${{ secrets.GITHUB_TOKEN }} | |
# Required because of https://github.com/cargo-bins/cargo-binstall/issues/1254 | |
- run: brew install bash | |
- run: task install-brew-dependencies | |
- run: task setup-dev | |
# This also encompasses `build-all` | |
- run: task test-all | |
- run: task test-rust-fast | |
- run: task test-lint | |
test-rust-main: | |
needs: rules | |
if: needs.rules.outputs.main == 'true' | |
strategy: | |
matrix: | |
include: | |
- os: macos-latest | |
target: x86_64-apple-darwin | |
features: test-dbs | |
- os: windows-latest | |
target: x86_64-pc-windows-msvc | |
# We'd like to reenable integration tests on Windows, ref https://github.com/wangfenjin/duckdb-rs/issues/179. | |
features: "" | |
# Note that we run wasm in the standard `test-rust` job. | |
# TODO: potentially enable these | |
# - os: ubuntu-latest | |
# target: aarch64-unknown-linux-musl | |
# - os: macos-latest | |
# target: aarch64-apple-darwin | |
uses: ./.github/workflows/test-rust.yaml | |
with: | |
os: ${{ matrix.os }} | |
target: ${{ matrix.target }} | |
features: ${{ matrix.features }} | |
build-web: | |
needs: rules | |
if: needs.rules.outputs.web == 'true' || needs.rules.outputs.main == 'true' | |
uses: ./.github/workflows/build-web.yaml | |
lint-megalinter: | |
uses: ./.github/workflows/lint-megalinter.yaml | |
publish-web: | |
uses: ./.github/workflows/publish-web.yaml | |
if: contains(github.event.pull_request.labels.*.name, 'pr-publish-web') | |
nightly: | |
needs: rules | |
uses: ./.github/workflows/nightly.yaml | |
if: needs.rules.outputs.nightly == 'true' | |
check-links-markdown: | |
needs: rules | |
# Another option is https://github.com/lycheeverse/lychee, but it was | |
# weirdly difficult to exclude a directory, and I managed to get | |
# rate-limited by GH because of it scanning node_modules. | |
runs-on: ubuntu-latest | |
steps: | |
- name: 📂 Checkout code | |
uses: actions/checkout@v4 | |
- uses: gaurav-nelson/github-action-markdown-link-check@v1 | |
with: | |
# Run on all files in nightly schedule, but only on local files | |
# otherwise (and only changed files in PRs) | |
config-file: | |
${{ needs.rules.outputs.nightly-schedule == 'true' && | |
'.config/.markdown-link-check-all.json' || | |
'.config/.markdown-link-check-local.json' }} | |
base-branch: main | |
check-modified-files-only: | |
${{ needs.rules.outputs.nightly == 'true' && 'no' || 'yes' }} | |
check-links-book: | |
# We also have a check-links-markdown job, however it will not spot mdbook | |
# mistakes such as forgetting to list an .md file in SUMMARY.md. | |
# Running a link checker on the generated HTML is more reliable. | |
needs: rules | |
if: | |
needs.rules.outputs.book == 'true' || needs.rules.outputs.nightly == | |
'true' | |
runs-on: ubuntu-latest | |
steps: | |
- name: 📂 Checkout code | |
uses: actions/checkout@v4 | |
- uses: baptiste0928/cargo-install@v2 | |
with: | |
crate: mdbook | |
# the link checker | |
- uses: baptiste0928/cargo-install@v2 | |
with: | |
crate: hyperlink | |
- run: ./.github/workflows/scripts/set_version.sh | |
- name: Cache | |
uses: Swatinem/rust-cache@v2 | |
with: | |
prefix-key: ${{ env.version }} | |
shared-key: web | |
# Created by `build-web` | |
save-if: false | |
# Only build the book — rather than `build-web` which also builds the playground | |
- name: Build the mdbook | |
run: mdbook build web/book/ | |
- name: Check links | |
run: hyperlink web/book/book/ | |
measure-code-cov: | |
runs-on: ubuntu-latest | |
needs: rules | |
# TODO: Would be great to have this running on every PR, but | |
# waiting on https://github.com/PRQL/prql/issues/2870. We could enable it | |
# but not block merging on it? | |
if: needs.rules.outputs.main == 'true' | |
steps: | |
- name: 📂 Checkout code | |
uses: actions/checkout@v4 | |
- uses: baptiste0928/cargo-install@v2 | |
with: | |
crate: cargo-tarpaulin | |
- run: ./.github/workflows/scripts/set_version.sh | |
shell: bash | |
- name: 💰 Cache | |
uses: Swatinem/rust-cache@v2 | |
with: | |
save-if: ${{ github.ref == 'refs/heads/main' }} | |
prefix-key: ${{ env.version }} | |
- run: | |
cargo tarpaulin --skip-clean --all-targets --features=test-dbs | |
--out=xml | |
- name: Upload to codecov.io | |
uses: codecov/codecov-action@v3 | |
- name: Upload code coverage results | |
uses: actions/upload-artifact@v3 | |
with: | |
name: code-coverage-report | |
path: cobertura.xml | |
test-grammars: | |
# Currently tests lezer grammars. We could split that out into a separate | |
# job if we want when we add more. | |
runs-on: ubuntu-latest | |
needs: rules | |
if: | |
needs.rules.outputs.grammars == 'true' || needs.rules.outputs.main == | |
'true' | |
steps: | |
- name: 📂 Checkout code | |
uses: actions/checkout@v4 | |
- name: 🧅 Setup Bun | |
uses: oven-sh/setup-bun@v1 | |
with: | |
bun-version: latest | |
- name: Install dependencies | |
working-directory: grammars/prql-lezer/ | |
run: bun install | |
- name: Build grammar | |
working-directory: grammars/prql-lezer/ | |
run: bun run build | |
- name: Test grammar | |
working-directory: grammars/prql-lezer/ | |
run: bun run test | |
build-devcontainer: | |
needs: rules | |
if: needs.rules.outputs.devcontainer-build == 'true' | |
uses: ./.github/workflows/build-devcontainer.yaml | |
# One problem with this setup is that if another commit is merged to main, | |
# this workflow will cancel existing jobs, and so this won't get pushed. We | |
# have another workflow which runs on each release, so the image should get | |
# pushed eventually. The alternative is to have a separate workflow, but | |
# then we can't use the nice logic of when to run the workflow that we've | |
# built up here. | |
with: | |
# This needs to compare to the string `'true'`, because of GHA awkwardness | |
push: ${{ needs.rules.outputs.devcontainer-push == 'true' }} | |
time-compilation: | |
runs-on: ubuntu-latest | |
needs: rules | |
if: needs.rules.outputs.nightly == 'true' | |
steps: | |
- name: 📂 Checkout code | |
uses: actions/checkout@v4 | |
- uses: ./.github/actions/time-compilation | |
with: | |
use_cache: true | |
test-msrv: | |
runs-on: ubuntu-latest | |
needs: rules | |
if: needs.rules.outputs.nightly == 'true' | |
steps: | |
- name: 📂 Checkout code | |
uses: actions/checkout@v4 | |
- uses: baptiste0928/cargo-install@v2 | |
with: | |
crate: cargo-msrv | |
# Note this currently uses a manually maintained key in | |
# `prqlc/prql-compiler/Cargo.toml` (and `prqlc/prqlc/Cargo.toml` below), because of | |
# https://github.com/foresterre/cargo-msrv/issues/590 | |
- name: Verify minimum rust version — prqlc | |
# Ideally we'd check all crates, ref https://github.com/foresterre/cargo-msrv/issues/295 | |
working-directory: prqlc/prql-compiler | |
run: cargo msrv verify | |
- name: Verify minimum rust version — prqlc CLI | |
working-directory: prqlc/prqlc | |
run: cargo msrv verify | |
test-deps-min-versions: | |
runs-on: ubuntu-latest | |
needs: rules | |
if: needs.rules.outputs.nightly == 'true' | |
steps: | |
- name: 📂 Checkout code | |
uses: actions/checkout@v4 | |
- run: rustup toolchain add nightly | |
- uses: baptiste0928/cargo-install@v2 | |
with: | |
crate: cargo-hack | |
- uses: baptiste0928/cargo-install@v2 | |
with: | |
crate: cargo-minimal-versions | |
- name: 💰 Cache | |
uses: Swatinem/rust-cache@v2 | |
with: | |
save-if: ${{ github.ref == 'refs/heads/main' }} | |
- name: Verify minimum rust version | |
run: cargo minimal-versions test | |
check-ok-to-merge: | |
# This indicates to GitHub whether everything in this workflow has passed | |
# and (unlike if we included each task in the branch's GitHub required | |
# tests) will pass when a task is skipped. | |
if: always() | |
needs: | |
- build-devcontainer | |
- build-web | |
- check-links-book | |
- check-links-markdown | |
- lint-megalinter | |
- nightly | |
- publish-web | |
- test-msrv | |
- test-dotnet | |
- test-elixir | |
- test-java | |
- test-js | |
- test-grammars | |
- test-clib | |
- test-php | |
- test-python | |
- test-rust | |
- test-rust-main | |
- test-taskfile | |
runs-on: ubuntu-latest | |
steps: | |
- name: Decide whether the needed jobs succeeded or failed | |
# https://github.com/re-actors/alls-green/issues/23 | |
uses: re-actors/alls-green@cf9edfcf932a0ed6b431433fa183829c68b30e3f | |
with: | |
jobs: ${{ toJSON(needs) }} | |
# We don't include `check-links-markdown`, since occasionally we'll want to merge | |
# something which temporarily fails that, such as if we're changing the | |
# location of a file in this repo which is linked to. | |
# | |
# We're currently including `nightly` because I'm not sure whether | |
# it's always reliable; e.g. `cargo-audit` | |
allowed-failures: | | |
[ | |
"check-links-markdown", | |
"nightly" | |
] | |
# We skip jobs deliberately, so we are OK if any are skipped. | |
# | |
# Copy-pasted from `needs`, since it needs to be a json list, so `${{ | |
# toJSON(needs) }}` (which is a map) doesn't work. | |
# https://github.com/re-actors/alls-green/issues/23 | |
allowed-skips: | | |
[ | |
"build-devcontainer", | |
"build-web", | |
"check-links-book", | |
"check-links-markdown", | |
"lint-megalinter", | |
"nightly", | |
"publish-web", | |
"test-msrv", | |
"test-dotnet", | |
"test-elixir", | |
"test-java", | |
"test-js", | |
"test-clib", | |
"test-grammars", | |
"test-php", | |
"test-python", | |
"test-rust", | |
"test-rust-main", | |
"test-taskfile" | |
] | |
build-prqlc: | |
runs-on: ${{ matrix.os }} | |
needs: rules | |
if: needs.rules.outputs.rust == 'true' || needs.rules.outputs.main == 'true' | |
strategy: | |
fail-fast: false | |
matrix: | |
include: | |
# Match the features with the available caches from tests | |
- os: ubuntu-latest | |
target: x86_64-unknown-linux-musl | |
features: "" | |
# TODO: Until we have tests for these, we don't have a cache for them. | |
# If we can add tests, then re-enable them. They run on `release.yaml` | |
# regardless. | |
# | |
# - os: ubuntu-latest | |
# target: aarch64-unknown-linux-musl | |
# - os: macos-latest | |
# target: aarch64-apple-darwin | |
- os: macos-latest | |
target: x86_64-apple-darwin | |
features: test-dbs | |
- os: windows-latest | |
target: x86_64-pc-windows-msvc | |
features: "" | |
steps: | |
- name: 📂 Checkout code | |
uses: actions/checkout@v4 | |
- uses: ./.github/actions/build-prqlc | |
with: | |
target: ${{ matrix.target }} | |
profile: dev | |
features: ${{ matrix.features }} | |
# These are the same env variables as in `test-rust.yaml`. Custom actions | |
# don't allow setting env variables for the whole job, so we do it here. | |
env: | |
CARGO_TERM_COLOR: always | |
CLICOLOR_FORCE: 1 | |
RUSTFLAGS: "-C debuginfo=0" | |
RUSTDOCFLAGS: "-Dwarnings" | |
create-issue-on-nightly-failure: | |
runs-on: ubuntu-latest | |
needs: | |
- check-ok-to-merge | |
if: | |
# We care that it's on a schedule as well as it running on nightly — we | |
# don't want to trigger just on a `pr-nightly` label | |
always() && github.event.schedule && contains(needs.*.result, 'failure') | |
permissions: | |
contents: read | |
issues: write | |
steps: | |
- name: 📂 Checkout code | |
uses: actions/checkout@v4 | |
- uses: JasonEtco/create-an-issue@v2 | |
env: | |
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
LINK: | |
${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ | |
github.run_id }} | |
with: | |
filename: .github/nightly-failure.md | |
update_existing: true | |
search_existing: open |