First shot at an appveyor configuration

appveyor.yml

@@ -0,0 +1,16 @@
+version: build{build}
+
+install:
+  - cinst elixir
+  - refreshenv
+
+shallow_clone: true
+
+build_script:
+  - mix local.hex --force
+  - mix deps.get
+  - mix local.rebar --force
+  - mix compile --warnings-as-errors
+
+test_script:
+  - mix test

test/benchee/benchmark/runner_test.exs

@@ -40,6 +40,7 @@ defmodule Benchee.Benchmark.RunnerTest do
   end
 
   describe ".run_scenarios" do
+    @tag :performance
     test "runs a benchmark suite and enriches it with measurements" do
       retrying(fn ->
         suite = test_suite(%Suite{configuration: %{time: 60_000, warmup: 10_000}})
@@ -57,6 +58,7 @@ defmodule Benchee.Benchmark.RunnerTest do
       end)
     end
 
+    @tag :performance
     test "runs a suite with multiple jobs and gathers results" do
       retrying(fn ->
         suite = test_suite(%Suite{configuration: %{time: 100_000, warmup: 10_000}})
@@ -159,6 +161,7 @@ defmodule Benchee.Benchmark.RunnerTest do
       assert average < 10
     end
 
+    @tag :performance
     test "doesn't take longer than advertised for very fast funs" do
       retrying(fn ->
         time = 20_000
@@ -262,6 +265,7 @@ defmodule Benchee.Benchmark.RunnerTest do
       end)
     end
 
+    @tag :performance
     test "populates results for all inputs" do
       retrying(fn ->
         inputs = %{
@@ -584,6 +588,7 @@ defmodule Benchee.Benchmark.RunnerTest do
       ])
     end
 
+    @tag :performance
     test "each triggers for every invocation, scenario once" do
       me = self()
 

test/benchee_test.exs

@@ -144,7 +144,7 @@ defmodule BencheeTest do
                   time: 0.001, warmup: 0, print: [fast_warning: false])
     end
 
-    refute Regex.match? ~r/fast/, output
+    refute output =~ ~r/fast/
   end
 
   test "integration comparison report can be deactivated" do
@@ -228,6 +228,8 @@ defmodule BencheeTest do
   end
 
   @rough_10_milli_s "((8|9|10|11|12|13|14)\\.\\d{2} ms)"
+
+  @tag :performance
   test "formatters have full access to the suite data, values in assigns" do
     retrying fn ->
       formatter_one = fn(suite) ->
@@ -318,9 +320,9 @@ defmodule BencheeTest do
       }, configuration)
     end
 
-    assert Regex.match?(@header_regex, output)
-    assert Regex.match? ~r/fast/, output
-    assert Regex.match? ~r/unreliable/, output
+    assert output =~ @header_regex
+    assert output =~ ~r/fast/
+    assert output =~ ~r/unreliable/
 
     assert String.contains? output, ["number_one", "symbol_one"]
     occurences = Regex.scan body_regex("identity"), output
@@ -487,8 +489,8 @@ defmodule BencheeTest do
                     time: 0.01, warmup: 0.005, measure_memory: true)
       end
 
-      assert Regex.match?(~r/Memory usage statistics:/, output)
-      assert Regex.match?(~r/To List\s+[0-9.]{3,} K*B{1}/, output)
+      assert output =~ ~r/Memory usage statistics:/
+      assert output =~ ~r/To List\s+[0-9.]{3,} K*B{1}/
     end
   end
 
@@ -503,10 +505,18 @@ defmodule BencheeTest do
     assert output =~ ~r/^flat_map#{tag_regex}\s+\d+(\.\d+)?\s*.?(#{@slower_regex})?$/m
     assert output =~ ~r/#{@slower_regex}/m
 
-    refute Regex.match?(~r/fast/i, output)
+    # In windows time resolution seems to be milliseconds, hence even
+    # standard examples produce a fast warning.
+    # So we skip this basic is everything going fine test on windows
+    unless windows?(), do: refute output =~ ~r/fast/i
   end
 
   defp body_regex(benchmark_name, tag_regex \\ "") do
     ~r/^#{benchmark_name}#{tag_regex}\s+\d+.+\s+\d+\.?\d*.+\s+.+\d+\.?\d*.+\s+\d+\.?\d*.+/m
   end
+
+  defp windows? do
+    {_, os} = :os.type()
+    os == :nt
+  end
 end

test/test_helper.exs

@@ -1,4 +1,10 @@
+# OTP 18 doesn't support the memory measurement things we need
 otp_release = List.to_integer(:erlang.system_info(:otp_release))
 exclusions = if otp_release > 18, do: [], else: [memory_measure: true]
 
+# On Windows we have by far worse time measurements (millisecond level)
+# see: https://github.com/PragTob/benchee/pull/195#issuecomment-377010006
+{_, os} = :os.type()
+exclusions = if os == :nt, do: [{:performance, true} | exclusions], else: exclusions
+
 ExUnit.start(exclude: exclusions)