Skip to content

tests

tests #92

Workflow file for this run

# This file has transitioning to run almost everything, with rules defined in
# this file rather than across lots of workflow files.
name: tests
on:
pull_request:
# Add `labeled`, so we can trigger a new run by adding a `pr-nightly`
# label, which we then use to trigger a `nightly` run.
types: [opened, reopened, synchronize, labeled]
push:
branches:
- main
schedule:
# Pick a random time, something that others won't pick, to reduce GH's demand variance.
- cron: "49 10 * * *"
workflow_dispatch:
workflow_call:
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
cancel-in-progress: true
env:
CARGO_TERM_COLOR: always
CLICOLOR_FORCE: 1
RUSTFLAGS: "-C debuginfo=0"
RUSTDOCFLAGS: "-Dwarnings"
jobs:
# This assesses whether we need to run jobs. Some of them are defined only by
# the changes in PR, others also define a set of other criteria, such as
# whether a label has been added, or we're on `main` branch.
rules:
runs-on: ubuntu-latest
permissions:
pull-requests: read
outputs:
book: ${{ steps.changes.outputs.book }}
dotnet: ${{ steps.changes.outputs.dotnet }}
devcontainer-push: ${{ steps.devcontainer-push.outputs.run }}
devcontainer-build: ${{ steps.devcontainer-build.outputs.run }}
elixir: ${{ steps.changes.outputs.elixir }}
grammars: ${{ steps.changes.outputs.grammars }}
java: ${{ steps.changes.outputs.java }}
js: ${{ steps.changes.outputs.js }}
lib: ${{ steps.changes.outputs.lib }}
# Run tests such as rust tests for all-OSs, and bindings tests on ubuntu.
# Somewhat a tradeoff between coverage and ensuring our CI queues stay
# short.
main: ${{ steps.main.outputs.run }}
# Run all tests
nightly: ${{ steps.nightly.outputs.run }}
# Really run all tests (e.g. taskfile & devcontainer)
nightly-schedule: ${{ steps.nightly-schedule.outputs.run }}
php: ${{ steps.changes.outputs.php }}
python: ${{ steps.changes.outputs.python }}
rust: ${{ steps.changes.outputs.rust }}
taskfile: ${{ steps.changes.outputs.taskfile }}
web: ${{ steps.changes.outputs.web }}
steps:
- name: 📂 Checkout code
uses: actions/checkout@v4
- uses: dorny/paths-filter@v2
id: changes
with:
filters: |
book:
- .github/workflows/check-links-book.yaml
- web/book/**
dotnet:
- prqlc/bindings/prql-dotnet/**
- prqlc/bindings/clib/**
- .github/workflows/test-dotnet.yaml
devcontainer-push:
- .devcontainer/**/*Dockerfile
- Taskfile.yml
devcontainer-build:
- .devcontainer/**/*Dockerfile
- .github/workflows/build-devcontainer.yaml
- Taskfile.yml
grammars:
- grammars/**
elixir:
- prqlc/bindings/elixir/**
- prqlc/bindings/clib/**
- .github/workflows/test-elixir.yaml
java:
- prqlc/bindings/java/**
- prqlc/bindings/clib/**
- .github/workflows/test-java.yaml
js:
- prqlc/bindings/js/**
- .github/workflows/test-js.yaml
clib:
- prqlc/bindings/clib/**
- .github/workflows/test-clib.yaml
main:
- "**/Cargo.*"
- .github/**
nightly:
- .github/workflows/nightly.yaml
- .github/workflows/release.yaml
- Cargo.lock
- rust-toolchain.toml
- .cargo/**
php:
- prqlc/bindings/php/**
- prqlc/bindings/clib/**
- .github/workflows/test-php.yaml
python:
- prqlc/bindings/python/**
- .github/workflows/test-python.yaml
rust:
- "**/*.rs"
- prqlc/**
- web/book/**
- .github/workflows/test-rust.yaml
taskfile:
- Taskfile.yml
web:
- "web/**"
- ".github/workflows/build-web.yaml"
- "**.md"
# We put a few of the more complex rules as steps here, rather than having
# them inline. There's no strict delineation between logic here vs. inline.
- id: nightly
# TODO: actionlint annoyingly blocks this — try and find a way of getting
# it back without too much trouble...
# contains(github.event.pull_request.title, '!') ||
run:
echo "run=${{ steps.changes.outputs.nightly == 'true' ||
contains(github.event.pull_request.labels.*.name, 'pr-nightly') ||
github.event.schedule }}" >>"$GITHUB_OUTPUT"
- id: nightly-schedule
run:
echo "run=${{ github.event.schedule == 'true' }}" >>"$GITHUB_OUTPUT"
- id: main
run:
echo "run=${{ steps.changes.outputs.main == 'true' || github.ref ==
'refs/heads/main' || steps.nightly.outputs.run == 'true' }}" >>
"$GITHUB_OUTPUT"
- id: devcontainer-push
# We push the devcontainer if the files have changed, and we've merged
# to main.
run:
echo "run=${{ steps.changes.outputs.devcontainer-push == 'true' &&
github.ref == 'refs/heads/main' && github.event_name == 'push' }}" >>
"$GITHUB_OUTPUT"
- id: devcontainer-build
run:
echo "run=${{ steps.devcontainer-push.outputs.run == 'true' ||
steps.changes.outputs.devcontainer-build == 'true' ||
steps.changes.outputs.nightly-schedule == 'true' }}" >>
"$GITHUB_OUTPUT"
test-rust:
needs: rules
if: needs.rules.outputs.rust == 'true' || needs.rules.outputs.main == 'true'
uses: ./.github/workflows/test-rust.yaml
strategy:
matrix:
include:
- target: x86_64-unknown-linux-gnu
os: ubuntu-latest
features: test-dbs-external
# Only run wasm on ubuntu, given it's the same rust target. (There is a
# possibility of having a failure on just one platform, but it's quite
# unlikely. If we do observe this, we can expand, or introduce a
# `test-actually-all.yaml` which accounts for these corner cases without
# using runners & cache space)
- target: wasm32-unknown-unknown
os: ubuntu-latest
features: ""
with:
os: ubuntu-latest
target: ${{ matrix.target }}
features: ${{ matrix.features }}
nightly: ${{ needs.rules.outputs.nightly == 'true' }}
test-python:
needs: rules
if:
needs.rules.outputs.python == 'true' || needs.rules.outputs.main == 'true'
uses: ./.github/workflows/test-python.yaml
with:
# Only run on ubuntu unless there's a lang-specific change or we're
# running nightly.
#
# An alternative to these somewhat horrible expressions would be
# `test-python` & `test-python-more` workflows; though it would use up our
# 20 workflow limit.
oss:
${{ (needs.rules.outputs.python == 'true' || needs.rules.outputs.nightly
== 'true') && '["ubuntu-latest", "macos-latest", "windows-latest"]' ||
'["ubuntu-latest"]' }}
test-js:
needs: rules
if: needs.rules.outputs.js == 'true' || needs.rules.outputs.main == 'true'
uses: ./.github/workflows/test-js.yaml
with:
# Only run on ubuntu unless there's a lang-specific change or we're running nightly.
oss:
${{ (needs.rules.outputs.js == 'true' || needs.rules.outputs.nightly ==
'true') && '["ubuntu-latest", "macos-latest", "windows-latest"]' ||
'["ubuntu-latest"]' }}
test-dotnet:
needs: rules
if:
needs.rules.outputs.dotnet == 'true' || needs.rules.outputs.main == 'true'
uses: ./.github/workflows/test-dotnet.yaml
test-php:
needs: rules
if: needs.rules.outputs.php == 'true' || needs.rules.outputs.main == 'true'
uses: ./.github/workflows/test-php.yaml
test-java:
needs: rules
if: needs.rules.outputs.java == 'true' || needs.rules.outputs.main == 'true'
uses: ./.github/workflows/test-java.yaml
with:
# Currently we never run windows
oss:
${{ (needs.rules.outputs.java == 'true' || needs.rules.outputs.nightly
== 'true') && '["ubuntu-latest", "macos-latest"]' || '["ubuntu-latest"]'
}}
test-elixir:
needs: rules
if:
needs.rules.outputs.elixir == 'true' || needs.rules.outputs.main == 'true'
uses: ./.github/workflows/test-elixir.yaml
with:
# Currently we never run Mac, see prql-elixir docs for details
oss:
${{ (needs.rules.outputs.elixir == 'true' || needs.rules.outputs.nightly
== 'true') && '["ubuntu-latest", "windows-latest"]' ||
'["ubuntu-latest"]' }}
test-clib:
needs: rules
if: needs.rules.outputs.lib == 'true' || needs.rules.outputs.main == 'true'
uses: ./.github/workflows/test-clib.yaml
test-taskfile:
needs: rules
if:
# We only run on nightly scheduled, since this is very expensive and we
# don't want to have to run it on, for example, every dependency change.
needs.rules.outputs.taskfile == 'true' ||
needs.rules.outputs.nightly-schedule == 'true'
runs-on: macos-latest
steps:
- name: 📂 Checkout code
uses: actions/checkout@v4
- run: ./.github/workflows/scripts/set_version.sh
- name: 💰 Cache
uses: Swatinem/rust-cache@v2
with:
prefix-key: ${{ env.version }}
# The mac rust cache key. It's not _that_ useful since this will build
# much more, but it's better than nothing. We can't have our own
# cache, since we're out of cache space and this workflow takes 1.5GB.
shared-key: rust-x86_64-apple-darwin-test-dbs
save-if: false
- uses: actions/setup-python@v4
with:
python-version: "3.10"
- name: Install Task
uses: arduino/setup-task@v1
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
# Required because of https://github.com/cargo-bins/cargo-binstall/issues/1254
- run: brew install bash
- run: task install-brew-dependencies
- run: task setup-dev
# This also encompasses `build-all`
- run: task test-all
- run: task test-rust-fast
- run: task test-lint
test-rust-main:
needs: rules
if: needs.rules.outputs.main == 'true'
strategy:
matrix:
include:
- os: macos-latest
target: x86_64-apple-darwin
features: test-dbs
- os: windows-latest
target: x86_64-pc-windows-msvc
# We'd like to reenable integration tests on Windows, ref https://github.com/wangfenjin/duckdb-rs/issues/179.
features: ""
# Note that we run wasm in the standard `test-rust` job.
# TODO: potentially enable these
# - os: ubuntu-latest
# target: aarch64-unknown-linux-musl
# - os: macos-latest
# target: aarch64-apple-darwin
uses: ./.github/workflows/test-rust.yaml
with:
os: ${{ matrix.os }}
target: ${{ matrix.target }}
features: ${{ matrix.features }}
build-web:
needs: rules
if: needs.rules.outputs.web == 'true' || needs.rules.outputs.main == 'true'
uses: ./.github/workflows/build-web.yaml
lint-megalinter:
uses: ./.github/workflows/lint-megalinter.yaml
publish-web:
uses: ./.github/workflows/publish-web.yaml
if: contains(github.event.pull_request.labels.*.name, 'pr-publish-web')
nightly:
needs: rules
uses: ./.github/workflows/nightly.yaml
if: needs.rules.outputs.nightly == 'true'
check-links-markdown:
needs: rules
# Another option is https://github.com/lycheeverse/lychee, but it was
# weirdly difficult to exclude a directory, and I managed to get
# rate-limited by GH because of it scanning node_modules.
runs-on: ubuntu-latest
steps:
- name: 📂 Checkout code
uses: actions/checkout@v4
- uses: gaurav-nelson/github-action-markdown-link-check@v1
with:
# Run on all files in nightly schedule, but only on local files
# otherwise (and only changed files in PRs)
config-file:
${{ needs.rules.outputs.nightly-schedule == 'true' &&
'.config/.markdown-link-check-all.json' ||
'.config/.markdown-link-check-local.json' }}
base-branch: main
check-modified-files-only:
${{ needs.rules.outputs.nightly == 'true' && 'no' || 'yes' }}
check-links-book:
# We also have a check-links-markdown job, however it will not spot mdbook
# mistakes such as forgetting to list an .md file in SUMMARY.md.
# Running a link checker on the generated HTML is more reliable.
needs: rules
if:
needs.rules.outputs.book == 'true' || needs.rules.outputs.nightly ==
'true'
runs-on: ubuntu-latest
steps:
- name: 📂 Checkout code
uses: actions/checkout@v4
- uses: baptiste0928/cargo-install@v2
with:
crate: mdbook
# the link checker
- uses: baptiste0928/cargo-install@v2
with:
crate: hyperlink
- run: ./.github/workflows/scripts/set_version.sh
- name: Cache
uses: Swatinem/rust-cache@v2
with:
prefix-key: ${{ env.version }}
shared-key: web
# Created by `build-web`
save-if: false
# Only build the book — rather than `build-web` which also builds the playground
- name: Build the mdbook
run: mdbook build web/book/
- name: Check links
run: hyperlink web/book/book/
measure-code-cov:
runs-on: ubuntu-latest
needs: rules
# TODO: Would be great to have this running on every PR, but
# waiting on https://github.com/PRQL/prql/issues/2870. We could enable it
# but not block merging on it?
if: needs.rules.outputs.main == 'true'
steps:
- name: 📂 Checkout code
uses: actions/checkout@v4
- uses: baptiste0928/cargo-install@v2
with:
crate: cargo-tarpaulin
- run: ./.github/workflows/scripts/set_version.sh
shell: bash
- name: 💰 Cache
uses: Swatinem/rust-cache@v2
with:
save-if: ${{ github.ref == 'refs/heads/main' }}
prefix-key: ${{ env.version }}
- run:
cargo tarpaulin --skip-clean --all-targets --features=test-dbs
--out=xml
- name: Upload to codecov.io
uses: codecov/codecov-action@v3
- name: Upload code coverage results
uses: actions/upload-artifact@v3
with:
name: code-coverage-report
path: cobertura.xml
test-grammars:
# Currently tests lezer grammars. We could split that out into a separate
# job if we want when we add more.
runs-on: ubuntu-latest
needs: rules
if:
needs.rules.outputs.grammars == 'true' || needs.rules.outputs.main ==
'true'
steps:
- name: 📂 Checkout code
uses: actions/checkout@v4
- name: 🧅 Setup Bun
uses: oven-sh/setup-bun@v1
with:
bun-version: latest
- name: Install dependencies
working-directory: grammars/prql-lezer/
run: bun install
- name: Build grammar
working-directory: grammars/prql-lezer/
run: bun run build
- name: Test grammar
working-directory: grammars/prql-lezer/
run: bun run test
build-devcontainer:
needs: rules
if: needs.rules.outputs.devcontainer-build == 'true'
uses: ./.github/workflows/build-devcontainer.yaml
# One problem with this setup is that if another commit is merged to main,
# this workflow will cancel existing jobs, and so this won't get pushed. We
# have another workflow which runs on each release, so the image should get
# pushed eventually. The alternative is to have a separate workflow, but
# then we can't use the nice logic of when to run the workflow that we've
# built up here.
with:
# This needs to compare to the string `'true'`, because of GHA awkwardness
push: ${{ needs.rules.outputs.devcontainer-push == 'true' }}
time-compilation:
runs-on: ubuntu-latest
needs: rules
if: needs.rules.outputs.nightly == 'true'
steps:
- name: 📂 Checkout code
uses: actions/checkout@v4
- uses: ./.github/actions/time-compilation
with:
use_cache: true
test-msrv:
runs-on: ubuntu-latest
needs: rules
if: needs.rules.outputs.nightly == 'true'
steps:
- name: 📂 Checkout code
uses: actions/checkout@v4
- uses: baptiste0928/cargo-install@v2
with:
crate: cargo-msrv
# Note this currently uses a manually maintained key in
# `prqlc/prql-compiler/Cargo.toml` (and `prqlc/prqlc/Cargo.toml` below), because of
# https://github.com/foresterre/cargo-msrv/issues/590
- name: Verify minimum rust version — prqlc
# Ideally we'd check all crates, ref https://github.com/foresterre/cargo-msrv/issues/295
working-directory: prqlc/prql-compiler
run: cargo msrv verify
- name: Verify minimum rust version — prqlc CLI
working-directory: prqlc/prqlc
run: cargo msrv verify
test-deps-min-versions:
runs-on: ubuntu-latest
needs: rules
if: needs.rules.outputs.nightly == 'true'
steps:
- name: 📂 Checkout code
uses: actions/checkout@v4
- run: rustup toolchain add nightly
- uses: baptiste0928/cargo-install@v2
with:
crate: cargo-hack
- uses: baptiste0928/cargo-install@v2
with:
crate: cargo-minimal-versions
- name: 💰 Cache
uses: Swatinem/rust-cache@v2
with:
save-if: ${{ github.ref == 'refs/heads/main' }}
- name: Verify minimum rust version
run: cargo minimal-versions test
check-ok-to-merge:
# This indicates to GitHub whether everything in this workflow has passed
# and (unlike if we included each task in the branch's GitHub required
# tests) will pass when a task is skipped.
if: always()
needs:
- build-devcontainer
- build-web
- check-links-book
- check-links-markdown
- lint-megalinter
- nightly
- publish-web
- test-msrv
- test-dotnet
- test-elixir
- test-java
- test-js
- test-grammars
- test-clib
- test-php
- test-python
- test-rust
- test-rust-main
- test-taskfile
runs-on: ubuntu-latest
steps:
- name: Decide whether the needed jobs succeeded or failed
# https://github.com/re-actors/alls-green/issues/23
uses: re-actors/alls-green@cf9edfcf932a0ed6b431433fa183829c68b30e3f
with:
jobs: ${{ toJSON(needs) }}
# We don't include `check-links-markdown`, since occasionally we'll want to merge
# something which temporarily fails that, such as if we're changing the
# location of a file in this repo which is linked to.
#
# We're currently including `nightly` because I'm not sure whether
# it's always reliable; e.g. `cargo-audit`
allowed-failures: |
[
"check-links-markdown",
"nightly"
]
# We skip jobs deliberately, so we are OK if any are skipped.
#
# Copy-pasted from `needs`, since it needs to be a json list, so `${{
# toJSON(needs) }}` (which is a map) doesn't work.
# https://github.com/re-actors/alls-green/issues/23
allowed-skips: |
[
"build-devcontainer",
"build-web",
"check-links-book",
"check-links-markdown",
"lint-megalinter",
"nightly",
"publish-web",
"test-msrv",
"test-dotnet",
"test-elixir",
"test-java",
"test-js",
"test-clib",
"test-grammars",
"test-php",
"test-python",
"test-rust",
"test-rust-main",
"test-taskfile"
]
build-prqlc:
runs-on: ${{ matrix.os }}
needs: rules
if: needs.rules.outputs.rust == 'true' || needs.rules.outputs.main == 'true'
strategy:
fail-fast: false
matrix:
include:
# Match the features with the available caches from tests
- os: ubuntu-latest
target: x86_64-unknown-linux-musl
features: ""
# TODO: Until we have tests for these, we don't have a cache for them.
# If we can add tests, then re-enable them. They run on `release.yaml`
# regardless.
#
# - os: ubuntu-latest
# target: aarch64-unknown-linux-musl
# - os: macos-latest
# target: aarch64-apple-darwin
- os: macos-latest
target: x86_64-apple-darwin
features: test-dbs
- os: windows-latest
target: x86_64-pc-windows-msvc
features: ""
steps:
- name: 📂 Checkout code
uses: actions/checkout@v4
- uses: ./.github/actions/build-prqlc
with:
target: ${{ matrix.target }}
profile: dev
features: ${{ matrix.features }}
# These are the same env variables as in `test-rust.yaml`. Custom actions
# don't allow setting env variables for the whole job, so we do it here.
env:
CARGO_TERM_COLOR: always
CLICOLOR_FORCE: 1
RUSTFLAGS: "-C debuginfo=0"
RUSTDOCFLAGS: "-Dwarnings"
create-issue-on-nightly-failure:
runs-on: ubuntu-latest
needs:
- check-ok-to-merge
if:
# We care that it's on a schedule as well as it running on nightly — we
# don't want to trigger just on a `pr-nightly` label
always() && github.event.schedule && contains(needs.*.result, 'failure')
permissions:
contents: read
issues: write
steps:
- name: 📂 Checkout code
uses: actions/checkout@v4
- uses: JasonEtco/create-an-issue@v2
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
LINK:
${{ github.server_url }}/${{ github.repository }}/actions/runs/${{
github.run_id }}
with:
filename: .github/nightly-failure.md
update_existing: true
search_existing: open