Skip to content

tests

tests #444

Workflow file for this run

# This file has transitioning to run almost everything, with rules defined in
# this file rather than across lots of workflow files.
name: tests
on:
pull_request:
# Add `labeled`, so we can trigger a new run by adding a `pr-nightly`
# label, which we then use to trigger a `nightly` run.
types: [opened, reopened, synchronize, labeled]
branches:
- "*"
push:
branches:
- main
schedule:
# Pick a random time, something that others won't pick, to be good citizens
# and reduce GH's demand variance.
- cron: "49 10 * * *"
workflow_dispatch:
workflow_call:
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
cancel-in-progress: true
# We need consistent env vars across all workflows for the cache to work
env:
CARGO_TERM_COLOR: always
CLICOLOR_FORCE: 1
RUSTFLAGS: "-C debuginfo=0"
RUSTDOCFLAGS: "-Dwarnings"
jobs:
# This assesses whether we need to run jobs. Some of them are defined only by
# the changes in PR, others also define a set of other criteria, such as
# whether a label has been added, or we're on `main` branch.
rules:
runs-on: ubuntu-latest
permissions:
pull-requests: read
outputs:
book: ${{ steps.changes.outputs.book }}
dotnet: ${{ steps.changes.outputs.dotnet }}
devcontainer-push: ${{ steps.devcontainer-push.outputs.run }}
devcontainer-build: ${{ steps.devcontainer-build.outputs.run }}
elixir: ${{ steps.changes.outputs.elixir }}
grammars: ${{ steps.changes.outputs.grammars }}
java: ${{ steps.changes.outputs.java }}
js: ${{ steps.changes.outputs.js }}
prqlc-c: ${{ steps.changes.outputs.prqlc-c }}
# Run tests such as rust tests for all-OSs, and bindings tests on ubuntu.
# Somewhat a tradeoff between coverage and ensuring our CI queues stay
# short.
main: ${{ steps.main.outputs.run }}
# Run all tests
nightly: ${{ steps.nightly.outputs.run }}
# For tasks which are very expensive or can only run on
# the main repo, such as pushing devcontainer or creating issues
nightly-upstream: ${{ steps.nightly-upstream.outputs.run }}
php: ${{ steps.changes.outputs.php }}
python: ${{ steps.changes.outputs.python }}
rust: ${{ steps.changes.outputs.rust }}
taskfile: ${{ steps.changes.outputs.taskfile }}
web: ${{ steps.changes.outputs.web }}
steps:
- name: 📂 Checkout code
uses: actions/checkout@v4
- uses: dorny/paths-filter@v3
id: changes
with:
filters: |
book:
- .github/workflows/check-links-book.yaml
- web/book/**
dotnet:
- prqlc/bindings/prql-dotnet/**
- prqlc/bindings/prqlc-c/**
- .github/workflows/test-dotnet.yaml
devcontainer-push:
- .devcontainer/**/*Dockerfile
- Taskfile.yaml
devcontainer-build:
- .devcontainer/**/*Dockerfile
- .github/workflows/build-devcontainer.yaml
- Taskfile.yaml
grammars:
- grammars/**
elixir:
- prqlc/bindings/elixir/**
- prqlc/bindings/prqlc-c/**
- .github/workflows/test-elixir.yaml
java:
- prqlc/bindings/java/**
- prqlc/bindings/prqlc-c/**
- .github/workflows/test-java.yaml
js:
- prqlc/bindings/js/**
- .github/workflows/test-js.yaml
prqlc-c:
- prqlc/bindings/prqlc-c/**
- .github/workflows/test-prqlc-c.yaml
main:
- "**/Cargo.*"
- .github/**
- .config/**
nightly:
- .github/workflows/nightly.yaml
- .github/workflows/release.yaml
- Cargo.lock
- rust-toolchain.toml
- .cargo/**
php:
- prqlc/bindings/php/**
- prqlc/bindings/prqlc-c/**
- .github/workflows/test-php.yaml
python:
- prqlc/bindings/prqlc-python/**
- .github/workflows/test-python.yaml
rust:
- "**/*.rs"
- prqlc/**
- web/book/**
- .github/workflows/test-rust.yaml
taskfile:
# Run taskfile test on any Taskfile change, since the tasks pull in tasks from
# other taskfiles. (But we don't run the container rebuilds, since those are
# much heavier)
- "**/Taskfile.yaml"
web:
- "web/**"
- ".github/workflows/build-web.yaml"
- "**.md"
# We put a few of the more complex rules as steps here, rather than having
# them inline. There's no strict delineation between logic here vs. inline.
- id: nightly
# TODO: actionlint annoyingly blocks this — try and find a way of getting
# it back without too much trouble...
# contains(github.event.pull_request.title, '!') ||
run:
echo "run=${{ steps.changes.outputs.nightly == 'true' ||
contains(github.event.pull_request.labels.*.name, 'pr-nightly') ||
github.event_name == 'schedule' }}" >>"$GITHUB_OUTPUT"
- id: nightly-upstream
run:
echo "run=${{ github.event_name == 'schedule' &&
github.repository_owner == 'prql' }}" >>"$GITHUB_OUTPUT"
- id: main
run:
echo "run=${{ steps.changes.outputs.main == 'true' || github.ref ==
'refs/heads/main' || steps.nightly.outputs.run == 'true' }}" >>
"$GITHUB_OUTPUT"
- id: devcontainer-push
# We push the devcontainer if the files have changed, and we've merged
# to main.
run:
echo "run=${{ steps.changes.outputs.devcontainer-push == 'true' &&
github.ref == 'refs/heads/main' && github.event_name == 'push' }}" >>
"$GITHUB_OUTPUT"
- id: devcontainer-build
run:
echo "run=${{ steps.devcontainer-push.outputs.run == 'true' ||
steps.changes.outputs.devcontainer-build == 'true' ||
steps.changes.outputs.nightly-upstream == 'true' }}" >>
"$GITHUB_OUTPUT"
test-rust:
needs: rules
if: needs.rules.outputs.rust == 'true' || needs.rules.outputs.main == 'true'
uses: ./.github/workflows/test-rust.yaml
strategy:
matrix:
include:
- target: x86_64-unknown-linux-gnu
os: ubuntu-latest
features: default,test-dbs-external
# Only run wasm on ubuntu, given it's the same rust target. (There is
# a possibility of having a failure on just one platform, but it's
# quite unlikely. If we do observe this, we can add those tests them
# to nightly.
- target: wasm32-unknown-unknown
os: ubuntu-latest
features: default
with:
os: ubuntu-latest
target: ${{ matrix.target }}
features: ${{ matrix.features }}
nightly: ${{ needs.rules.outputs.nightly == 'true' }}
test-python:
needs: rules
if:
needs.rules.outputs.python == 'true' || needs.rules.outputs.main == 'true'
uses: ./.github/workflows/test-python.yaml
with:
# Only run on ubuntu unless there's a lang-specific change or we're
# running nightly.
#
# An alternative to these somewhat horrible expressions would be
# `test-python` & `test-python-more` workflows; though it would use up our
# 20 workflow limit.
oss:
${{ (needs.rules.outputs.python == 'true' || needs.rules.outputs.nightly
== 'true') && '["ubuntu-latest", "macos-14", "windows-latest"]' ||
'["ubuntu-latest"]' }}
test-js:
needs: rules
if: needs.rules.outputs.js == 'true' || needs.rules.outputs.main == 'true'
uses: ./.github/workflows/test-js.yaml
with:
# Only run on ubuntu unless there's a lang-specific change or we're running nightly.
oss:
${{ (needs.rules.outputs.js == 'true' || needs.rules.outputs.nightly ==
'true') && '["ubuntu-latest", "macos-14", "windows-latest"]' ||
'["ubuntu-latest"]' }}
test-dotnet:
needs: rules
if:
needs.rules.outputs.dotnet == 'true' || needs.rules.outputs.main == 'true'
uses: ./.github/workflows/test-dotnet.yaml
test-php:
needs: rules
if: needs.rules.outputs.php == 'true' || needs.rules.outputs.main == 'true'
uses: ./.github/workflows/test-php.yaml
test-java:
needs: rules
if: needs.rules.outputs.java == 'true' || needs.rules.outputs.main == 'true'
uses: ./.github/workflows/test-java.yaml
with:
# Currently we never run windows
oss:
${{ (needs.rules.outputs.java == 'true' || needs.rules.outputs.nightly
== 'true') && '["ubuntu-latest", "macos-14"]' || '["ubuntu-latest"]' }}
test-elixir:
needs: rules
if:
needs.rules.outputs.elixir == 'true' || needs.rules.outputs.main == 'true'
uses: ./.github/workflows/test-elixir.yaml
with:
# Currently we never run Mac, see prql-elixir docs for details
oss:
${{ (needs.rules.outputs.elixir == 'true' || needs.rules.outputs.nightly
== 'true') && '["ubuntu-latest", "windows-latest"]' ||
'["ubuntu-latest"]' }}
test-prqlc-c:
needs: rules
if:
needs.rules.outputs.prqlc-c == 'true' || needs.rules.outputs.main ==
'true'
uses: ./.github/workflows/test-prqlc-c.yaml
test-taskfile:
needs: rules
if:
# We only run on nightly scheduled, since this is very expensive and we
# don't want to have to run it on, for example, every dependency change.
needs.rules.outputs.taskfile == 'true' ||
needs.rules.outputs.nightly-upstream == 'true'
runs-on: macos-14
steps:
- name: 📂 Checkout code
uses: actions/checkout@v4
- run: ./.github/workflows/scripts/set_version.sh
- name: 💰 Cache
uses: Swatinem/rust-cache@v2
with:
prefix-key: ${{ env.version }}
# The mac rust cache key. It's not _that_ useful since this will build
# much more, but it's better than nothing. This task can't have our own
# cache, since we're out of cache space and this workflow takes 1.5GB.
shared-key: rust-aarch64-apple-darwin
save-if: false
- uses: actions/setup-python@v5
with:
python-version: "3.10"
- name: Install Task
uses: arduino/setup-task@v2
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
# Required because of https://github.com/cargo-bins/cargo-binstall/issues/1254
- run: brew install bash
- run: task install-brew-dependencies
- run: task setup-dev
# This also encompasses `build-all`
- run: task test-all
- run: task test-rust-fast
- run: task test-lint
test-rust-main:
needs: rules
if: needs.rules.outputs.main == 'true'
strategy:
matrix:
include:
- os: macos-14
target: aarch64-apple-darwin
features: default,test-dbs
- os: windows-latest
target: x86_64-pc-windows-msvc
# We'd like to reenable integration tests on Windows, ref https://github.com/wangfenjin/duckdb-rs/issues/179.
features: default
- os: ubuntu-latest
target: x86_64-unknown-linux-gnu
# One test with no features
features: ""
# TODO: potentially enable these
# - os: ubuntu-latest
# target: aarch64-unknown-linux-musl
uses: ./.github/workflows/test-rust.yaml
with:
os: ${{ matrix.os }}
target: ${{ matrix.target }}
features: ${{ matrix.features }}
build-web:
needs: rules
if: needs.rules.outputs.web == 'true' || needs.rules.outputs.main == 'true'
uses: ./.github/workflows/build-web.yaml
lint-megalinter:
uses: ./.github/workflows/lint-megalinter.yaml
publish-web:
uses: ./.github/workflows/publish-web.yaml
if: contains(github.event.pull_request.labels.*.name, 'pr-publish-web')
nightly:
needs: rules
uses: ./.github/workflows/nightly.yaml
if: needs.rules.outputs.nightly == 'true'
secrets: inherit
check-links-markdown:
needs: rules
# Another option is https://github.com/lycheeverse/lychee, but it was
# weirdly difficult to exclude a directory, and I managed to get
# rate-limited by GH because of it scanning node_modules.
runs-on: ubuntu-latest
steps:
- name: 📂 Checkout code
uses: actions/checkout@v4
- uses: gaurav-nelson/github-action-markdown-link-check@v1
with:
# Run on:
# - All links on nightly schedule
# - Links within files which have been changed PRs
config-file:
${{ needs.rules.outputs.nightly-upstream == 'true' &&
'.config/.markdown-link-check-all.json' ||
'.config/.markdown-link-check-local.json' }}
base-branch: main
check-modified-files-only:
${{ needs.rules.outputs.nightly == 'true' && 'no' || 'yes' }}
check-links-book:
# We also have a check-links-markdown job, however it will not spot mdbook
# mistakes such as forgetting to list an .md file in SUMMARY.md.
# Running a link checker on the generated HTML is more reliable.
needs: rules
if:
needs.rules.outputs.book == 'true' || needs.rules.outputs.nightly ==
'true'
runs-on: ubuntu-latest
steps:
- name: 📂 Checkout code
uses: actions/checkout@v4
- uses: baptiste0928/cargo-install@v3
with:
crate: mdbook
# the link checker
- uses: baptiste0928/cargo-install@v3
with:
crate: hyperlink
- run: ./.github/workflows/scripts/set_version.sh
- name: Cache
uses: Swatinem/rust-cache@v2
with:
prefix-key: ${{ env.version }}
shared-key: web
# Created by `build-web`
save-if: false
# Only build the book — rather than `build-web` which also builds the playground
- name: Build the mdbook
run: mdbook build web/book/
- name: Check links
run: hyperlink web/book/book/
measure-code-cov:
runs-on: ubuntu-latest
needs: rules
steps:
- name: 📂 Checkout code
uses: actions/checkout@v4
- run: ./.github/workflows/scripts/set_version.sh
- uses: baptiste0928/cargo-install@v3
with:
crate: cargo-llvm-cov
- name: 💰 Cache
uses: Swatinem/rust-cache@v2
with:
save-if: ${{ github.ref == 'refs/heads/main' }}
prefix-key: ${{ env.version }}
# Ensure nothing remains from caching
- run: cargo llvm-cov clean --workspace
- run:
# We considered moving to using `codecov.json` with
# `--codecov --output-path=codecov.json` since that has branch & region
# coverage. But the coverage is lower, in a way that doesn't represent
# what is useful coverage
cargo llvm-cov --cobertura --output-path=cobertura.xml
--no-default-features --features=default,test-dbs
- name: Upload code coverage results
uses: actions/upload-artifact@v4
with:
name: code-coverage-report
path: cobertura.xml
- name: Upload to codecov.io
if:
# This action raises an error on forks. It allows running on PRs to
# the main repo, which is important. Rarely do we need this uploading
# from forks so while we can reenable running from forks if it works,
# it's not that important.
#
# As of 2024-06, codecov was still working through how they handle
# forks / tokens on PRs given rate limits, expect some failures for a
# bit.
#
# As of 2024-06, we're also seeing that uploading on schedule can
# measure very slightly different coverage, which can then cause PRs
# based off that base to show reduced coverage, and show a failure. So
# we disable it on schedule. Not sure that's a perfect solution — is
# it giving different coverage _because_ it's on schedule, or is it
# random such that limiting to running on main will sometimes show
# reduced coverage? Because we're making comparisons, reproducible
# accuracy is important.
${{ github.repository_owner == 'prql' && github.event_name !=
'schedule' }}
uses: codecov/codecov-action@v4
with:
files: cobertura.xml
fail_ci_if_error: true
# As discussed in
# https://community.codecov.com/t/upload-issues-unable-to-locate-build-via-github-actions-api/3954,
# without this the upload has a fairly high failure rate. The only
# thing the token allows is uploading coverage, so there are
# apparently no security risks.
#
# Edit: actually no luck, waiting on
# https://github.com/codecov/codecov-action/issues/1469
token: cab4ace5-4f10-4027-8b5c-d79722234571
test-grammars:
# Currently tests lezer grammars. We could split that out into a separate
# job if we want when we add more.
runs-on: ubuntu-latest
needs: rules
if:
needs.rules.outputs.grammars == 'true' || needs.rules.outputs.main ==
'true'
steps:
- name: 📂 Checkout code
uses: actions/checkout@v4
- name: 🧅 Setup Bun
uses: oven-sh/setup-bun@v2
with:
bun-version: latest
- name: Install dependencies
working-directory: grammars/prql-lezer/
run: bun install
- name: Build grammar
working-directory: grammars/prql-lezer/
run: bun run build
- name: Test grammar
working-directory: grammars/prql-lezer/
run: bun run test
build-devcontainer:
needs: rules
if: needs.rules.outputs.devcontainer-build == 'true'
uses: ./.github/workflows/build-devcontainer.yaml
# One problem with this setup is that if another commit is merged to main,
# this workflow will cancel existing jobs, and so this won't get pushed. We
# have another workflow which runs on each release, so the image should get
# pushed eventually. The alternative is to have a separate workflow, but
# then we can't use the nice logic of when to run the workflow that we've
# built up here.
with:
# This needs to compare to the string `'true'`, because of GHA awkwardness
push: ${{ needs.rules.outputs.devcontainer-push == 'true' }}
test-msrv:
runs-on: ubuntu-latest
needs: rules
if: needs.rules.outputs.nightly == 'true'
steps:
- name: 📂 Checkout code
uses: actions/checkout@v4
- uses: baptiste0928/cargo-install@v3
with:
crate: cargo-msrv
# Note this currently uses a manually maintained key in
# `prqlc/prqlc/Cargo.toml`, because of
# https://github.com/foresterre/cargo-msrv/issues/590
- name: Verify minimum rust version — prqlc
# Ideally we'd check all crates, ref https://github.com/foresterre/cargo-msrv/issues/295
working-directory: prqlc/prqlc
run: cargo msrv verify
test-deps-min-versions:
runs-on: ubuntu-latest
needs: rules
if: needs.rules.outputs.nightly == 'true'
steps:
- name: 📂 Checkout code
uses: actions/checkout@v4
- run: rustup override set nightly-2024-06-30
- uses: baptiste0928/cargo-install@v3
with:
crate: cargo-hack
- uses: baptiste0928/cargo-install@v3
with:
crate: cargo-minimal-versions
- run: ./.github/workflows/scripts/set_version.sh
- name: 💰 Cache
uses: Swatinem/rust-cache@v2
with:
save-if: ${{ github.ref == 'refs/heads/main' }}
prefix-key: ${{ env.version }}
- name: Verify minimum rust version
run: cargo minimal-versions test --direct
check-ok-to-merge:
# This indicates to GitHub whether everything in this workflow has passed
# and (unlike if we included each task in the branch's GitHub required
# tests) will pass when a task is skipped.
if: always()
needs:
- build-devcontainer
- build-web
- check-links-book
- check-links-markdown
- lint-megalinter
- nightly
- publish-web
- test-deps-min-versions
- test-dotnet
- test-elixir
- test-grammars
- test-java
- test-js
- test-msrv
- test-php
- test-prqlc-c
- test-python
- test-rust
- test-rust-main
- test-taskfile
runs-on: ubuntu-latest
steps:
- name: Decide whether the needed jobs succeeded or failed
# https://github.com/re-actors/alls-green/issues/23
uses: re-actors/alls-green@cf9edfcf932a0ed6b431433fa183829c68b30e3f
with:
jobs: ${{ toJSON(needs) }}
# We don't include `check-links-markdown`, since occasionally we'll want to merge
# something which temporarily fails that, such as if we're changing the
# location of a file in this repo which is linked to.
#
# We're currently including `nightly` because I'm not sure whether
# it's always reliable; e.g. `cargo-audit`
allowed-failures: |
[
"check-links-markdown",
"nightly"
]
# We skip jobs deliberately, so we are OK if any are skipped.
#
# Copy-pasted from `needs`, since it needs to be a json list, so `${{
# toJSON(needs) }}` (which is a map) doesn't work.
# https://github.com/re-actors/alls-green/issues/23
allowed-skips: |
[
"build-devcontainer",
"build-web",
"check-links-book",
"check-links-markdown",
"lint-megalinter",
"measure-code-cov",
"nightly",
"publish-web",
"test-deps-min-versions",
"test-dotnet",
"test-elixir",
"test-grammars",
"test-java",
"test-js",
"test-msrv",
"test-php",
"test-prqlc-c",
"test-python",
"test-rust",
"test-rust-main",
"test-taskfile",
"time-compilation"
]
build-prqlc:
runs-on: ${{ matrix.os }}
needs: rules
if: needs.rules.outputs.rust == 'true' || needs.rules.outputs.main == 'true'
strategy:
fail-fast: false
matrix:
include:
# Match the features with the available caches from tests
- os: ubuntu-latest
target: x86_64-unknown-linux-musl
features: default
# TODO: Until we have tests for these, we don't have a cache for them.
# If we can add tests, then re-enable them. They run on `release.yaml`
# regardless.
#
# - os: ubuntu-latest
# target: aarch64-unknown-linux-musl
- os: macos-14
target: aarch64-apple-darwin
features: default,test-dbs
- os: windows-latest
target: x86_64-pc-windows-msvc
features: default
steps:
- name: 📂 Checkout code
uses: actions/checkout@v4
- uses: ./.github/actions/build-prqlc
with:
target: ${{ matrix.target }}
profile: dev
features: ${{ matrix.features }}
# We need consistent env vars across all workflows for the cache to work
env:
CARGO_TERM_COLOR: always
CLICOLOR_FORCE: 1
RUSTFLAGS: "-C debuginfo=0"
RUSTDOCFLAGS: "-Dwarnings"
build-prqlc-c:
runs-on: ${{ matrix.os }}
needs: rules
if: needs.rules.outputs.main == 'true'
strategy:
fail-fast: false
matrix:
include:
# Match the features with the available caches from tests
- os: ubuntu-latest
target: x86_64-unknown-linux-musl
features: default
- os: macos-14
target: aarch64-apple-darwin
features: default,test-dbs
- os: windows-latest
target: x86_64-pc-windows-msvc
features: default
steps:
- name: 📂 Checkout code
uses: actions/checkout@v4
- uses: ./.github/actions/build-prqlc-c
with:
target: ${{ matrix.target }}
profile: dev
features: ${{ matrix.features }}
# We need consistent env vars across all workflows for the cache to work
env:
CARGO_TERM_COLOR: always
CLICOLOR_FORCE: 1
RUSTFLAGS: "-C debuginfo=0"
RUSTDOCFLAGS: "-Dwarnings"
create-issue-on-nightly-failure:
runs-on: ubuntu-latest
needs:
- check-ok-to-merge
- rules
if:
${{ always() && contains(needs.*.result, 'failure') &&
needs.rules.outputs.nightly-upstream == 'true' }}
permissions:
contents: read
issues: write
steps:
- name: 📂 Checkout code
uses: actions/checkout@v4
- uses: JasonEtco/create-an-issue@v2
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
LINK:
${{ github.server_url }}/${{ github.repository }}/actions/runs/${{
github.run_id }}
with:
filename: .github/nightly-failure.md
update_existing: true
search_existing: open
update-rust-toolchain:
runs-on: ubuntu-latest
needs: rules
if: ${{ needs.rules.outputs.nightly-upstream == 'true' }}
# Note that this doesn't change the minimum supported version, only the
# default toolchain to run on. The minimum is defined by Cargo.toml's
# metadata.msrv and is updated manually based on when build environments
# such as debian & winget are updated.
steps:
- name: 📂 Checkout code
uses: actions/checkout@v4
- uses: a-kenji/update-rust-toolchain@main
with:
# Discussion in #1561
minor-version-delta: 1
toolchain-path: "./rust-toolchain.toml"
pr-title: "build: Update rust toolchain version"
token: ${{ secrets.PRQL_BOT_GITHUB_TOKEN }}