diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 168ce546..b54c23c6 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -8,6 +8,8 @@ on: - version-0.4 name: tests +env: + CARGO_TERM_COLOR: always jobs: ci: @@ -85,3 +87,27 @@ jobs: export PATH=$HOME/.wasmtime/bin/:$PATH cargo install cargo-wasi cargo wasi bench --no-default-features -- --test + + nextest-compat: + name: Check compatibility with nextest + runs-on: ubuntu-latest + strategy: + matrix: + rust: + - stable + + steps: + - uses: actions/checkout@v2 + + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: ${{ matrix.rust }} + override: true + components: rustfmt, clippy + + - uses: Swatinem/rust-cache@v1 + + - uses: taiki-e/install-action@nextest + + - run: ci/nextest-compat.sh diff --git a/ci/nextest-compat.sh b/ci/nextest-compat.sh new file mode 100755 index 00000000..fed9f2f9 --- /dev/null +++ b/ci/nextest-compat.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -ex -o pipefail + +CARGO=${CARGO:-cargo} + +cd "$(git rev-parse --show-toplevel)" + +echo "Checking benches/bench_main..." + +$CARGO nextest list --benches +$CARGO nextest run --benches diff --git a/src/benchmark_group.rs b/src/benchmark_group.rs index 5ca0ab3b..687fb2f2 100644 --- a/src/benchmark_group.rs +++ b/src/benchmark_group.rs @@ -328,9 +328,9 @@ impl<'a, M: Measurement> BenchmarkGroup<'a, M> { ); } } - Mode::List => { + Mode::List(_) => { if do_run { - println!("{}: bench", id); + println!("{}: benchmark", id); } } Mode::Test => { @@ -392,7 +392,7 @@ impl<'a, M: Measurement> Drop for BenchmarkGroup<'a, M> { self.criterion.measurement.formatter(), ); } - if self.any_matched { + if self.any_matched && !self.criterion.mode.is_terse() { self.criterion.report.group_separator(); } } diff --git a/src/lib.rs b/src/lib.rs index 02c202f5..4bd772ca 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -108,8 +108,8 @@ static DEFAULT_PLOTTING_BACKEND: Lazy = Lazy::new(|| match &*GN #[cfg(feature = "plotters")] Err(e) => { match e { - VersionError::Exec(_) => println!("Gnuplot not found, using plotters backend"), - e => println!( + VersionError::Exec(_) => eprintln!("Gnuplot not found, using plotters backend"), + e => eprintln!( "Gnuplot not found or not usable, using plotters backend\n{}", e ), @@ -303,7 +303,7 @@ pub(crate) enum Mode { /// Run benchmarks normally. Benchmark, /// List all benchmarks but do not run them. - List, + List(ListFormat), /// Run benchmarks once to verify that they work, but otherwise do not measure them. Test, /// Iterate benchmarks for a given length of time but do not analyze or report on them. @@ -313,6 +313,39 @@ impl Mode { pub fn is_benchmark(&self) -> bool { matches!(self, Mode::Benchmark) } + + pub fn is_terse(&self) -> bool { + matches!(self, Mode::List(ListFormat::Terse)) + } +} + +#[derive(Debug, Clone)] +/// Enum representing the list format. +pub(crate) enum ListFormat { + /// The regular, default format. + Pretty, + /// The terse format, where nothing other than the name of the test and ": benchmark" at the end + /// is printed out. + Terse, +} + +impl Default for ListFormat { + fn default() -> Self { + Self::Pretty + } +} + +/// Benchmark filtering support. +#[derive(Clone, Debug)] +pub enum BenchmarkFilter { + /// Run all benchmarks. + AcceptAll, + /// Run benchmarks matching this regex. + Regex(Regex), + /// Run the benchmark matching this string exactly. + Exact(String), + /// Do not run any benchmarks. + RejectAll, } /// The benchmark manager @@ -331,7 +364,7 @@ impl Mode { /// benchmark. pub struct Criterion { config: BenchmarkConfig, - filter: Option, + filter: BenchmarkFilter, report: Reports, output_directory: PathBuf, baseline_directory: String, @@ -399,7 +432,7 @@ impl Default for Criterion { sampling_mode: SamplingMode::Auto, quick_mode: false, }, - filter: None, + filter: BenchmarkFilter::AcceptAll, report: reports, baseline_directory: "base".to_owned(), baseline: Baseline::Save, @@ -660,6 +693,8 @@ impl Criterion { #[must_use] /// Filters the benchmarks. Only benchmarks with names that contain the /// given string will be executed. + /// + /// This overwrites [`Self::with_benchmark_filter`]. pub fn with_filter>(mut self, filter: S) -> Criterion { let filter_text = filter.into(); let filter = Regex::new(&filter_text).unwrap_or_else(|err| { @@ -668,7 +703,16 @@ impl Criterion { filter_text, err ) }); - self.filter = Some(filter); + self.filter = BenchmarkFilter::Regex(filter); + + self + } + + /// Only run benchmarks specified by the given filter. + /// + /// This overwrites [`Self::with_filter`]. + pub fn with_benchmark_filter(mut self, filter: BenchmarkFilter) -> Criterion { + self.filter = filter; self } @@ -771,6 +815,19 @@ impl Criterion { .long("list") .help("List all benchmarks") .conflicts_with_all(&["test", "profile-time"])) + .arg(Arg::new("format") + .long("format") + .possible_values(["pretty", "terse"]) + .default_value("pretty") + // Note that libtest's --format also works during test execution, but criterion + // doesn't support that at the moment. + .help("Output formatting")) + .arg(Arg::new("ignored") + .long("ignored") + .help("List or run ignored benchmarks (currently means skip all benchmarks)")) + .arg(Arg::new("exact") + .long("exact") + .help("Run benchmarks that exactly match the provided filter")) .arg(Arg::new("profile-time") .long("profile-time") .takes_value(true) @@ -901,10 +958,21 @@ https://bheisler.github.io/criterion.rs/book/faq.html (false, _) => true, // cargo test --benches should run tests }; - self.mode = if test_mode { + self.mode = if matches.is_present("list") { + let list_format = match matches + .value_of("format") + .expect("a default value was provided for this") + { + "pretty" => ListFormat::Pretty, + "terse" => ListFormat::Terse, + other => unreachable!( + "unrecognized value for --format that isn't part of possible-values: {}", + other + ), + }; + Mode::List(list_format) + } else if test_mode { Mode::Test - } else if matches.is_present("list") { - Mode::List } else if matches.is_present("profile-time") { let num_seconds = matches.value_of_t_or_exit("profile-time"); @@ -923,9 +991,25 @@ https://bheisler.github.io/criterion.rs/book/faq.html self.connection = None; } - if let Some(filter) = matches.value_of("FILTER") { - self = self.with_filter(filter); - } + let filter = if matches.is_present("ignored") { + // --ignored overwrites any name-based filters passed in. + BenchmarkFilter::RejectAll + } else if let Some(filter) = matches.value_of("FILTER") { + if matches.is_present("exact") { + BenchmarkFilter::Exact(filter.to_owned()) + } else { + let regex = Regex::new(filter).unwrap_or_else(|err| { + panic!( + "Unable to parse '{}' as a regular expression: {}", + filter, err + ) + }); + BenchmarkFilter::Regex(regex) + } + } else { + BenchmarkFilter::AcceptAll + }; + self = self.with_benchmark_filter(filter); match matches.value_of("plotting-backend") { // Use plotting_backend() here to re-use the panic behavior if Gnuplot is not available. @@ -1061,8 +1145,10 @@ https://bheisler.github.io/criterion.rs/book/faq.html fn filter_matches(&self, id: &str) -> bool { match &self.filter { - Some(regex) => regex.is_match(id), - None => true, + BenchmarkFilter::AcceptAll => true, + BenchmarkFilter::Regex(regex) => regex.is_match(id), + BenchmarkFilter::Exact(exact) => id == exact, + BenchmarkFilter::RejectAll => false, } }