From 5d1ef2f2016573e5a662136000570346473e668d Mon Sep 17 00:00:00 2001 From: Tobias Pfeiffer Date: Mon, 12 Mar 2018 20:50:01 +0100 Subject: [PATCH 1/3] First shot at an appveyor configuration --- appveyor.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 appveyor.yml diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 00000000..336cf11b --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,16 @@ +version: build{build} + +install: + - cinst elixir + - refreshenv + +shallow_clone: true + +build_script: + - mix local.hex --force + - mix deps.get + - mix local.rebar --force + - mix compile --warnings-as-errors + +test_script: + - mix test From 304563858dcd77ae4eb95a03dcc1dba1f43c37ba Mon Sep 17 00:00:00 2001 From: Tobias Pfeiffer Date: Tue, 13 Mar 2018 20:26:54 +0100 Subject: [PATCH 2/3] Get better error messages for regex test failures * =~ seems to print the text that was matched against while Regex.match? doesn't --- test/benchee_test.exs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/benchee_test.exs b/test/benchee_test.exs index 40a5069a..35b5b488 100644 --- a/test/benchee_test.exs +++ b/test/benchee_test.exs @@ -144,7 +144,7 @@ defmodule BencheeTest do time: 0.001, warmup: 0, print: [fast_warning: false]) end - refute Regex.match? ~r/fast/, output + refute output =~ ~r/fast/ end test "integration comparison report can be deactivated" do @@ -318,9 +318,9 @@ defmodule BencheeTest do }, configuration) end - assert Regex.match?(@header_regex, output) - assert Regex.match? ~r/fast/, output - assert Regex.match? ~r/unreliable/, output + assert output =~ @header_regex + assert output =~ ~r/fast/ + assert output =~ ~r/unreliable/ assert String.contains? output, ["number_one", "symbol_one"] occurences = Regex.scan body_regex("identity"), output @@ -487,8 +487,8 @@ defmodule BencheeTest do time: 0.01, warmup: 0.005, measure_memory: true) end - assert Regex.match?(~r/Memory usage statistics:/, output) - assert Regex.match?(~r/To List\s+[0-9.]{3,} K*B{1}/, output) + assert output =~ ~r/Memory usage statistics:/ + assert output =~ ~r/To List\s+[0-9.]{3,} K*B{1}/ end end @@ -503,7 +503,7 @@ defmodule BencheeTest do assert output =~ ~r/^flat_map#{tag_regex}\s+\d+(\.\d+)?\s*.?(#{@slower_regex})?$/m assert output =~ ~r/#{@slower_regex}/m - refute Regex.match?(~r/fast/i, output) + refute output =~ ~r/fast/i end defp body_regex(benchmark_name, tag_regex \\ "") do From 955d9af526719d211efa2f5fdee29a3ea610c5ed Mon Sep 17 00:00:00 2001 From: Tobias Pfeiffer Date: Sun, 1 Apr 2018 15:05:16 +0200 Subject: [PATCH 3/3] Exclude tests/assertions that dont run on Windows because of millisecond measurement --- test/benchee/benchmark/runner_test.exs | 5 +++++ test/benchee_test.exs | 12 +++++++++++- test/test_helper.exs | 6 ++++++ 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/test/benchee/benchmark/runner_test.exs b/test/benchee/benchmark/runner_test.exs index 9a6734b5..571bf14f 100644 --- a/test/benchee/benchmark/runner_test.exs +++ b/test/benchee/benchmark/runner_test.exs @@ -40,6 +40,7 @@ defmodule Benchee.Benchmark.RunnerTest do end describe ".run_scenarios" do + @tag :performance test "runs a benchmark suite and enriches it with measurements" do retrying(fn -> suite = test_suite(%Suite{configuration: %{time: 60_000, warmup: 10_000}}) @@ -57,6 +58,7 @@ defmodule Benchee.Benchmark.RunnerTest do end) end + @tag :performance test "runs a suite with multiple jobs and gathers results" do retrying(fn -> suite = test_suite(%Suite{configuration: %{time: 100_000, warmup: 10_000}}) @@ -159,6 +161,7 @@ defmodule Benchee.Benchmark.RunnerTest do assert average < 10 end + @tag :performance test "doesn't take longer than advertised for very fast funs" do retrying(fn -> time = 20_000 @@ -262,6 +265,7 @@ defmodule Benchee.Benchmark.RunnerTest do end) end + @tag :performance test "populates results for all inputs" do retrying(fn -> inputs = %{ @@ -584,6 +588,7 @@ defmodule Benchee.Benchmark.RunnerTest do ]) end + @tag :performance test "each triggers for every invocation, scenario once" do me = self() diff --git a/test/benchee_test.exs b/test/benchee_test.exs index 35b5b488..89f2812a 100644 --- a/test/benchee_test.exs +++ b/test/benchee_test.exs @@ -228,6 +228,8 @@ defmodule BencheeTest do end @rough_10_milli_s "((8|9|10|11|12|13|14)\\.\\d{2} ms)" + + @tag :performance test "formatters have full access to the suite data, values in assigns" do retrying fn -> formatter_one = fn(suite) -> @@ -503,10 +505,18 @@ defmodule BencheeTest do assert output =~ ~r/^flat_map#{tag_regex}\s+\d+(\.\d+)?\s*.?(#{@slower_regex})?$/m assert output =~ ~r/#{@slower_regex}/m - refute output =~ ~r/fast/i + # In windows time resolution seems to be milliseconds, hence even + # standard examples produce a fast warning. + # So we skip this basic is everything going fine test on windows + unless windows?(), do: refute output =~ ~r/fast/i end defp body_regex(benchmark_name, tag_regex \\ "") do ~r/^#{benchmark_name}#{tag_regex}\s+\d+.+\s+\d+\.?\d*.+\s+.+\d+\.?\d*.+\s+\d+\.?\d*.+/m end + + defp windows? do + {_, os} = :os.type() + os == :nt + end end diff --git a/test/test_helper.exs b/test/test_helper.exs index 10af6e48..a3dce69e 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -1,4 +1,10 @@ +# OTP 18 doesn't support the memory measurement things we need otp_release = List.to_integer(:erlang.system_info(:otp_release)) exclusions = if otp_release > 18, do: [], else: [memory_measure: true] +# On Windows we have by far worse time measurements (millisecond level) +# see: https://github.com/PragTob/benchee/pull/195#issuecomment-377010006 +{_, os} = :os.type() +exclusions = if os == :nt, do: [{:performance, true} | exclusions], else: exclusions + ExUnit.start(exclude: exclusions)