From c5be2d4323bc1e0aaab711a1c004aeb144f442ce Mon Sep 17 00:00:00 2001
From: Mayeul d'Avezac <mdavezac@gmail.com>
Date: Mon, 3 Oct 2016 22:44:50 +0100
Subject: [PATCH 1/4] Add functions to add and analyze benchmarks

---
 test/benchmarks.jl | 39 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)
 create mode 100644 test/benchmarks.jl

diff --git a/test/benchmarks.jl b/test/benchmarks.jl
new file mode 100644
index 00000000..721694cb
--- /dev/null
+++ b/test/benchmarks.jl
@@ -0,0 +1,39 @@
+using Unitful
+using BenchmarkTools
+using DataFrames
+
+function benchmark!(suite::BenchmarkGroup, expression::Expr)
+    @assert expression.head == :call
+    op = expression.args[1]
+    a = eval(expression.args[2])
+    b = eval(expression.args[3])
+    if op ∉ keys(suite)
+      suite[op] = BenchmarkGroup()
+    end
+    name = "$a $op $b"
+    if name ∉ keys(suite[op])
+        suite[op][name] = BenchmarkGroup([op])
+    end
+    suite[op][name][:units] = @benchmarkable $op($a, $b)
+    suite[op][name][:nounits] =
+        @benchmarkable $op($(ustrip(a)), $(ustrip(b)))
+    suite
+end
+benchmark(expression::Expr) =
+    benchmark!(BenchmarkGroup(["Unitful"]), expression)
+
+function performance_summary(suite)
+    benchs = run(suite)
+    result = DataFrame(op=Symbol[], expression=String[], performance=Float64[])
+    for op in keys(benchs)
+        for expression in keys(benchs[op])
+            units = benchs[op][expression][:units]
+            nounits = benchs[op][expression][:nounits]
+            push!(
+                result,
+                [op, expression, ratio(median(units), median(nounits)).time]
+            )
+        end
+    end
+    result
+end

From 2bda342f92e751d3250b9f462bd2ab0eebf3b8a4 Mon Sep 17 00:00:00 2001
From: Mayeul d'Avezac <mdavezac@gmail.com>
Date: Tue, 4 Oct 2016 20:55:08 +0100
Subject: [PATCH 2/4] Adding default benchmarks

---
 test/benchmarks.jl | 24 ++++++++++++++++++++----
 1 file changed, 20 insertions(+), 4 deletions(-)

diff --git a/test/benchmarks.jl b/test/benchmarks.jl
index 721694cb..0a868750 100644
--- a/test/benchmarks.jl
+++ b/test/benchmarks.jl
@@ -23,17 +23,33 @@ benchmark(expression::Expr) =
     benchmark!(BenchmarkGroup(["Unitful"]), expression)
 
 function performance_summary(suite)
+    tune!(suite)
     benchs = run(suite)
-    result = DataFrame(op=Symbol[], expression=String[], performance=Float64[])
+    result = DataFrame(op=Symbol[], expression=String[],
+                       ratio=Float64[], time=Any[], memory=Any[])
     for op in keys(benchs)
         for expression in keys(benchs[op])
-            units = benchs[op][expression][:units]
-            nounits = benchs[op][expression][:nounits]
+            units = median(benchs[op][expression][:units])
+            nounits = median(benchs[op][expression][:nounits])
+            rt = ratio(units, nounits).time
+            judgment = judge(units, nounits)
             push!(
                 result,
-                [op, expression, ratio(median(units), median(nounits)).time]
+                [op, expression, rt, judgment.time, judgment.memory]
             )
         end
     end
     result
 end
+
+function benchmark()
+    suite = BenchmarkGroup(["Unitful"])
+    for a in (2u"m", 2.1u"m"), b in (3u"kg", 3.5u"kg"), op in (:*, :/)
+      benchmark!(suite, :($op($a, $b)))
+    end
+    # for a in (2u"m", 1//2u"m"), b in (1//3u"kg"), op in (:*, :+, :-, ://)
+    #     benchmark!(suite, :($op($a, $b)))
+    # end
+    performance_summary(suite)
+end
+

From 3dbbc405807364264b248fb6fb5f63ec1ea4e0a2 Mon Sep 17 00:00:00 2001
From: Mayeul d'Avezac <mdavezac@gmail.com>
Date: Sat, 8 Oct 2016 20:54:16 +0100
Subject: [PATCH 3/4] Run benchmarks as tests

---
 test/benchmarks.jl | 83 ++++++++++++++++++++++------------------------
 1 file changed, 40 insertions(+), 43 deletions(-)

diff --git a/test/benchmarks.jl b/test/benchmarks.jl
index 0a868750..fec442e7 100644
--- a/test/benchmarks.jl
+++ b/test/benchmarks.jl
@@ -1,55 +1,52 @@
 using Unitful
 using BenchmarkTools
+using Base.Test
 using DataFrames
 
-function benchmark!(suite::BenchmarkGroup, expression::Expr)
-    @assert expression.head == :call
-    op = expression.args[1]
-    a = eval(expression.args[2])
-    b = eval(expression.args[3])
-    if op ∉ keys(suite)
-      suite[op] = BenchmarkGroup()
+function benchmark()
+    suite = BenchmarkGroup(["Unitful"])
+    bterms = (3u"kg", 3.5u"kg", 2u"m^-1")
+    for a in (2u"m", 2.1u"m"), b in bterms, op in (:*, :/)
+      benchmark!(suite, :($op($a, $b)))
     end
-    name = "$a $op $b"
-    if name ∉ keys(suite[op])
-        suite[op][name] = BenchmarkGroup([op])
+    for a in (2u"m", 2.1u"m"), b in (3u"m", 3.5u"m"), op in (:+, :-)
+        benchmark!(suite, :($op($a, $b)))
     end
-    suite[op][name][:units] = @benchmarkable $op($a, $b)
-    suite[op][name][:nounits] =
-        @benchmarkable $op($(ustrip(a)), $(ustrip(b)))
-    suite
+
+    performance_summary(suite)
 end
-benchmark(expression::Expr) =
-    benchmark!(BenchmarkGroup(["Unitful"]), expression)
 
-function performance_summary(suite)
-    tune!(suite)
-    benchs = run(suite)
-    result = DataFrame(op=Symbol[], expression=String[],
-                       ratio=Float64[], time=Any[], memory=Any[])
-    for op in keys(benchs)
-        for expression in keys(benchs[op])
-            units = median(benchs[op][expression][:units])
-            nounits = median(benchs[op][expression][:nounits])
-            rt = ratio(units, nounits).time
-            judgment = judge(units, nounits)
-            push!(
-                result,
-                [op, expression, rt, judgment.time, judgment.memory]
-            )
-        end
-    end
-    result
+function judge_unit_benchmark(units::Expr, nounits::Expr; kwargs...)
+    bench_units = @benchmarkable $units
+    bench_nounits = @benchmarkable $nounits
+    tune!(bench_units)
+    tune!(bench_nounits)
+    judge(
+        median(run(bench_units; kwargs...)),
+        median(run(bench_nounits; kwargs...))
+    )
 end
 
-function benchmark()
-    suite = BenchmarkGroup(["Unitful"])
-    for a in (2u"m", 2.1u"m"), b in (3u"kg", 3.5u"kg"), op in (:*, :/)
-      benchmark!(suite, :($op($a, $b)))
-    end
-    # for a in (2u"m", 1//2u"m"), b in (1//3u"kg"), op in (:*, :+, :-, ://)
-    #     benchmark!(suite, :($op($a, $b)))
-    # end
-    performance_summary(suite)
+function judge_unit_benchmark(units::Expr; kwargs...)
+    op = units.args[1]
+    a = ustrip(eval(units.args[2]))
+    b = ustrip(eval(units.args[3]))
+    judge_unit_benchmark(units, :($op($a, $b)); kwargs...)
+end
+
+function test_benchmark(unit::Expr...; kwargs...)
+    j = judge_unit_benchmark(unit...; kwargs...)
+    j.time == :invariant && j.memory == :invariant
 end
 
+@testset "Benchmarks" begin
+    as, ops, bs = (2u"m", 2.1u"m"), (:*, :/), (3u"kg", 3.5u"kg", 2u"m^-1")
+    # @testset "$a $op $b" for a = as, b = bs, op = ops
+    #     @test test_benchmark(:($op($a, $b)))
+    # end
+
+    @testset "dimensionless to unitless" begin
+        @test test_benchmark(:(1u"m" / 2u"m"))
+        @test test_benchmark(:(1u"m" * 2u"m^-1"))
+    end
+end

From 2850c9631369efdb1baf64476b2c23a4bafd3eb8 Mon Sep 17 00:00:00 2001
From: Mayeul d'Avezac <mdavezac@gmail.com>
Date: Sun, 9 Oct 2016 19:48:14 +0100
Subject: [PATCH 4/4] More benchmarks + broken tests

Broken tests are those expression that do not evaluate as fast as their
unit-less counterpart
---
 test/benchmarks.jl | 44 ++++++++++++++++++++++++++++++++++----------
 1 file changed, 34 insertions(+), 10 deletions(-)

diff --git a/test/benchmarks.jl b/test/benchmarks.jl
index fec442e7..6fb0f557 100644
--- a/test/benchmarks.jl
+++ b/test/benchmarks.jl
@@ -7,7 +7,7 @@ function benchmark()
     suite = BenchmarkGroup(["Unitful"])
     bterms = (3u"kg", 3.5u"kg", 2u"m^-1")
     for a in (2u"m", 2.1u"m"), b in bterms, op in (:*, :/)
-      benchmark!(suite, :($op($a, $b)))
+        benchmark!(suite, :($op($a, $b)))
     end
     for a in (2u"m", 2.1u"m"), b in (3u"m", 3.5u"m"), op in (:+, :-)
         benchmark!(suite, :($op($a, $b)))
@@ -17,7 +17,10 @@ function benchmark()
 end
 
 function judge_unit_benchmark(units::Expr, nounits::Expr; kwargs...)
-    bench_units = @benchmarkable $units
+    op = units.args[1]
+    a = eval(units.args[2])
+    b = eval(units.args[3])
+    bench_units = @benchmarkable $op($a, $b)
     bench_nounits = @benchmarkable $nounits
     tune!(bench_units)
     tune!(bench_nounits)
@@ -40,13 +43,34 @@ function test_benchmark(unit::Expr...; kwargs...)
 end
 
 @testset "Benchmarks" begin
-    as, ops, bs = (2u"m", 2.1u"m"), (:*, :/), (3u"kg", 3.5u"kg", 2u"m^-1")
-    # @testset "$a $op $b" for a = as, b = bs, op = ops
-    #     @test test_benchmark(:($op($a, $b)))
-    # end
-
-    @testset "dimensionless to unitless" begin
-        @test test_benchmark(:(1u"m" / 2u"m"))
-        @test test_benchmark(:(1u"m" * 2u"m^-1"))
+    as, ops, bs = (2u"m", 2.1u"m"), (:*, :/), (3u"kg", 3.5u"kg")
+    @testset "$a $op $b" for a = as, b = bs, op = ops
+        @test test_benchmark(:($op($a, $b)), time_tolerance=0.01)
+    end
+
+    @testset "unit to unitless" begin
+        @test test_benchmark(:((1u"m") / (2u"m")), time_tolerance=0.01)
+        @test test_benchmark(:(1u"m" * 2u"m^-1"), time_tolerance=0.01)
+        @test test_benchmark(:(1u"m" / 2u"km"), time_tolerance=0.01)
+    end
+
+
+    a  = 1//2 * u"m"
+    bs = (1u"kg", 2//3 * u"kg")
+    @testset "rational numbers: $a $op $b" for b = bs, op in (://, :*)
+        @test_broken test_benchmark(:($op($a, $b)))
+    end
+
+    @testset "summations" begin
+        @test test_benchmark(:(1u"m" + 2u"m"), time_tolerance=0.01)
+        @test_broken test_benchmark(:(1u"km" + 2u"m"), time_tolerance=0.01)
+    end
+
+    @testset "Arrays" begin
+        as, ops, bs = ([2, 1]u"m", [2.1, 3.1]u"m"), (:.*, :./, :.+, :.-),
+                      (3u"kg", [3, 2]u"kg")
+        @testset "$a $op $b" for a=as, b=bs, op=ops
+            @test_broken test_benchmark(:($op($a, $b)), time_tolerance=0.01)
+        end
     end
 end