Replace benchmark rows with maps

explorable-viz · Oct 9, 2023 · bb481bf · bb481bf
1 parent cd2fd12
commit bb481bf
Show file tree

Hide file tree

Showing 4 changed files with 98 additions and 91 deletions.
diff --git a/Benchmarks/benchmarks.csv b/Benchmarks/benchmarks.csv
@@ -1,64 +1,67 @@
-Test-Name, Trace-Eval, Trace-Bwd, Trace-Fwd, Graph-Eval, Graph-Bwd, Graph-BwdDual, Graph-BwdAll, Graph-Fwd, Graph-FwdDual, Graph-FwdAsDeMorgan
-arithmetic,1.6,20.6,1.6,3.2,0.2,0.0,0.2,0.2,0.0,7.0
-array,3.0,15.4,4.0,6.8,0.2,0.2,0.2,0.0,0.0,3.0
-compose,0.4,19.4,0.4,2.6,0.0,0.0,0.0,0.0,0.0,3.6
-dicts,1.8,13.6,1.4,5.2,0.4,0.2,0.6,0.0,0.0,3.0
-div-mod-quot-rem,2.0,16.4,1.2,9.6,0.2,0.2,2.4,0.0,0.0,3.0
-factorial,6.2,22.4,5.6,9.2,0.0,0.0,23.2,0.0,0.0,109.6
-filter,2.4,16.8,4.0,6.6,0.0,0.4,0.0,0.0,0.0,2.2
-first-class-constr,2.0,16.0,1.8,7.2,0.0,0.0,0.4,0.0,0.0,2.8
-flatten,5.4,16.6,10.6,10.4,0.2,0.4,1.2,0.0,0.0,5.6
-foldr_sumSquares,2.6,18.8,2.4,9.2,0.0,0.0,0.0,0.0,0.0,2.6
-lexicalScoping,2.6,15.8,1.4,4.2,0.2,0.0,0.2,0.0,0.0,6.0
-length,1.4,13.8,1.2,2.8,0.2,0.0,0.2,0.0,0.2,3.2
-lookup,3.8,18.6,5.0,11.4,0.4,0.0,0.2,0.0,0.0,3.2
-map,2.8,17.6,2.6,7.6,0.0,0.0,1.0,0.0,0.0,2.4
-mergeSort,16.4,24.2,16.8,24.0,0.0,0.0,1.2,0.0,0.0,5.2
-normalise,2.2,20.2,1.6,10.0,0.4,0.0,1.8,0.0,0.2,5.6
-pattern-match,4.0,18.2,8.0,11.2,0.0,0.0,0.8,0.0,0.0,3.8
-range,17.4,30.4,24.0,23.0,0.0,0.0,1.0,0.0,0.0,5.8
-records,5.0,17.8,5.0,9.2,0.0,0.0,0.6,0.0,0.0,3.2
-reverse,1.2,17.6,1.4,5.2,0.0,0.0,0.4,0.0,0.0,3.0
-desugar/list-comp-1,17.2,26.8,16.2,27.6,0.0,0.0,1.4,0.0,0.0,5.8
-desugar/list-comp-2,62.4,71.0,50.4,94.6,0.2,0.0,8.4,0.0,0.0,18.4
-desugar/list-comp-3,14.2,24.0,13.0,26.4,0.2,0.2,0.4,0.0,0.0,5.0
-desugar/list-comp-4,6.0,15.0,4.6,9.8,0.2,0.0,0.2,0.2,0.0,3.0
-desugar/list-comp-5,3.4,14.0,2.8,8.2,0.0,0.0,0.8,0.0,0.2,2.2
-desugar/list-comp-6,0.0,13.4,3.2,1.2,0.0,0.0,0.2,0.0,0.0,2.4
-desugar/list-comp-7,5.2,14.0,4.0,8.2,0.0,0.0,0.2,0.0,0.2,1.8
-desugar/list-enum,2.6,16.0,2.4,7.4,0.0,0.2,4.4,0.0,0.0,5.8
-add,0.0,13.2,0.0,1.8,0.0,0.0,0.0,0.2,0.2,3.0
-array/lookup,17.8,22.8,19.0,35.4,0.0,0.2,0.0,0.0,0.0,11.4
-array/dims,1.6,15.2,1.8,2.6,0.4,0.0,0.2,0.0,0.2,3.4
-convolution/edgeDetect,1545.2,1143.8,1560.8,3243.8,0.6,0.6,10.6,6.6,311.4,2868.0
-convolution/emboss,1348.2,905.4,1313.0,2625.0,1.0,0.6,9.0,2.2,354.0,2566.0
-convolution/gaussian,1247.2,921.6,1325.8,2525.4,0.2,0.2,10.0,1.8,313.4,2486.0
-dict/create,0.6,14.8,0.0,3.0,0.4,0.4,0.6,0.2,0.0,5.0
-dict/difference,1.4,13.6,0.4,3.0,0.0,0.0,0.2,0.2,0.2,3.2
-dict/disjointUnion,0.2,12.4,0.4,2.2,0.2,0.2,0.0,0.0,0.0,3.0
-dict/foldl,6.4,24.2,7.4,11.8,0.0,0.0,0.0,0.0,0.0,3.0
-dict/intersectionWith,1.2,13.2,1.0,2.6,0.0,0.2,0.0,0.2,0.0,2.6
-dict/fromRecord,0.4,16.0,0.0,0.8,0.2,0.0,0.4,0.0,0.0,2.4
-dict/get,0.4,15.6,0.6,1.6,0.0,0.2,0.0,0.0,0.0,3.4
-dict/map,3.6,19.0,3.2,7.6,0.2,0.2,0.0,0.0,0.2,2.6
-divide,0.0,14.0,0.2,1.4,0.2,0.4,0.0,0.0,0.0,2.6
-filter,4.4,17.6,2.6,6.4,0.0,0.0,0.8,0.0,0.0,2.2
-intersperse,1.6,19.2,1.8,10.8,0.2,0.0,0.8,0.4,0.2,2.2
-intersperse,1.2,18.6,1.6,8.6,0.2,0.0,0.6,0.4,0.8,2.0
-length,1.2,17.6,2.4,4.4,0.2,2.0,0.4,0.0,0.6,1.8
-list-comp,25.0,31.8,23.8,41.8,0.0,0.0,0.6,1.4,2.2,5.4
-list-comp,23.4,27.8,23.4,37.2,0.2,0.0,0.6,1.8,1.4,6.0
-lookup,7.2,17.0,5.8,10.6,0.4,0.6,0.4,0.8,0.6,4.0
-map,1.2,16.8,2.2,3.4,0.0,0.2,0.2,0.0,0.2,3.2
-multiply,0.0,12.4,0.0,3.2,0.0,1.4,0.0,0.0,0.0,1.8
-nth,3.0,16.0,2.4,4.8,0.4,0.2,0.0,0.0,0.0,2.4
-section-5-example,30.4,36.4,26.6,58.4,0.0,0.6,0.4,1.4,1.4,6.2
-section-5-example,24.2,29.8,23.0,46.4,0.4,0.0,0.6,0.2,0.4,7.4
-section-5-example,25.4,28.6,27.0,51.2,0.0,0.2,0.8,0.4,0.6,7.6
-zeros,0.6,15.2,1.6,2.8,0.6,0.0,0.0,0.0,0.4,2.2
-zeros,0.8,16.0,3.0,4.2,0.2,0.0,0.0,0.2,0.0,3.4
-zipWith,4.0,15.8,4.4,7.8,0.0,0.2,0.4,0.2,0.4,2.2
-graphics/background,1.8,15.4,1.6,7.2,0.0,0.0,1.8,0.0,0.0,12.8
-graphics/grouped-bar-chart,145.2,126.0,148.8,255.2,0.8,0.2,775.0,0.0,0.4,2186.4
-graphics/line-chart,201.8,146.2,193.8,393.0,1.6,0.2,1728.0,0.0,0.2,2987.2
-graphics/stacked-bar-chart,81.6,67.0,77.2,159.0,0.2,0.6,208.2,0.0,0.4,142.0
+Test-Name,Trace-Bwd,Trace-Eval,Trace-Fwd,Graph-Bwd,Graph-BwdAll,Graph-BwdDual,Graph-Eval,Graph-Fwd,Graph-FwdAsDeMorgan,Graph-FwdDual
+arithmetic,17.0,10.0,2.0,1.0,1.0,1.0,4.0,0.0,5.0,0.0
+array,10.0,11.0,4.0,1.0,2.0,0.0,10.0,0.0,4.0,0.0
+compose,14.0,23.0,1.0,1.0,0.0,0.0,2.0,0.0,5.0,0.0
+dicts,8.0,7.0,2.0,1.0,2.0,0.0,8.0,0.0,5.0,0.0
+div-mod-quot-rem,8.0,7.0,5.0,0.0,3.0,1.0,5.0,2.0,4.0,0.0
+factorial,13.0,9.0,8.0,0.0,32.0,0.0,7.0,0.0,158.0,0.0
+filter,16.0,14.0,3.0,1.0,1.0,0.0,5.0,6.0,3.0,0.0
+first-class-constr,10.0,11.0,3.0,0.0,2.0,0.0,5.0,1.0,3.0,0.0
+flatten,10.0,10.0,3.0,2.0,3.0,1.0,8.0,0.0,8.0,0.0
+foldr_sumSquares,9.0,10.0,3.0,0.0,2.0,0.0,6.0,0.0,2.0,0.0
+lexicalScoping,8.0,7.0,1.0,1.0,0.0,0.0,1.0,0.0,11.0,0.0
+length,10.0,8.0,1.0,1.0,3.0,0.0,2.0,0.0,2.0,0.0
+lookup,7.0,11.0,5.0,1.0,1.0,0.0,6.0,0.0,3.0,0.0
+map,8.0,10.0,3.0,1.0,2.0,1.0,6.0,0.0,3.0,0.0
+mergeSort,18.0,18.0,16.0,0.0,2.0,0.0,18.0,0.0,6.0,0.0
+normalise,14.0,17.0,2.0,0.0,1.0,0.0,3.0,0.0,10.0,0.0
+pattern-match,9.0,15.0,6.0,0.0,1.0,0.0,8.0,0.0,6.0,0.0
+range,26.0,18.0,13.0,1.0,2.0,0.0,15.0,0.0,8.0,0.0
+records,10.0,8.0,3.0,0.0,1.0,0.0,8.0,0.0,4.0,0.0
+reverse,7.0,13.0,1.0,0.0,0.0,0.0,6.0,0.0,3.0,0.0
+dtw/next-indices,62.0,80.0,57.0,2.0,151.0,2.0,84.0,6.0,768.0,0.0
+dtw/cost-matrix,64.0,55.0,39.0,2.0,8.0,1.0,84.0,3.0,108.0,0.0
+dtw/matrix-update,11.0,7.0,2.0,1.0,1.0,0.0,3.0,0.0,3.0,0.0
+desugar/list-comp-1,16.0,24.0,16.0,1.0,2.0,0.0,20.0,0.0,7.0,0.0
+desugar/list-comp-2,53.0,56.0,59.0,1.0,9.0,1.0,73.0,2.0,27.0,0.0
+desugar/list-comp-3,19.0,18.0,13.0,1.0,1.0,0.0,17.0,0.0,8.0,0.0
+desugar/list-comp-4,10.0,11.0,4.0,1.0,1.0,0.0,9.0,0.0,4.0,0.0
+desugar/list-comp-5,7.0,15.0,4.0,0.0,1.0,0.0,5.0,1.0,3.0,0.0
+desugar/list-comp-6,11.0,9.0,0.0,0.0,0.0,0.0,1.0,0.0,2.0,0.0
+desugar/list-comp-7,12.0,15.0,6.0,0.0,0.0,0.0,7.0,1.0,3.0,0.0
+desugar/list-enum,10.0,12.0,2.0,2.0,6.0,0.0,6.0,0.0,8.0,0.0
+add,10.0,5.0,0.0,0.0,0.0,1.0,1.0,1.0,2.0,0.0
+array/lookup,19.0,22.0,18.0,1.0,0.0,0.0,31.0,0.0,17.0,0.0
+array/dims,11.0,16.0,2.0,0.0,1.0,0.0,12.0,1.0,3.0,0.0
+convolution/edgeDetect,1082.0,1427.0,1619.0,30.0,38.0,0.0,2479.0,9.0,4066.0,305.0
+convolution/emboss,901.0,1188.0,1282.0,25.0,32.0,0.0,1958.0,2.0,3739.0,362.0
+convolution/gaussian,996.0,1264.0,1357.0,26.0,34.0,1.0,2052.0,2.0,3828.0,372.0
+dict/create,8.0,6.0,0.0,1.0,3.0,0.0,1.0,0.0,2.0,0.0
+dict/difference,6.0,7.0,1.0,1.0,1.0,0.0,3.0,0.0,6.0,0.0
+dict/disjointUnion,6.0,5.0,0.0,1.0,1.0,0.0,1.0,0.0,6.0,0.0
+dict/foldl,32.0,41.0,18.0,2.0,1.0,0.0,22.0,1.0,10.0,0.0
+dict/intersectionWith,11.0,8.0,2.0,1.0,1.0,1.0,4.0,1.0,10.0,0.0
+dict/fromRecord,6.0,9.0,1.0,1.0,3.0,1.0,1.0,0.0,2.0,0.0
+dict/get,8.0,5.0,1.0,5.0,1.0,0.0,2.0,0.0,2.0,0.0
+dict/map,23.0,12.0,3.0,0.0,1.0,0.0,5.0,1.0,9.0,0.0
+divide,11.0,7.0,0.0,0.0,0.0,0.0,2.0,0.0,3.0,0.0
+filter,12.0,9.0,3.0,1.0,1.0,0.0,5.0,0.0,3.0,0.0
+intersperse,7.0,6.0,7.0,0.0,2.0,1.0,3.0,0.0,5.0,1.0
+intersperse,6.0,9.0,2.0,1.0,1.0,0.0,5.0,1.0,3.0,3.0
+length,12.0,7.0,1.0,1.0,1.0,0.0,2.0,0.0,7.0,1.0
+list-comp,30.0,31.0,29.0,1.0,3.0,5.0,37.0,2.0,21.0,3.0
+list-comp,33.0,45.0,28.0,1.0,2.0,0.0,36.0,3.0,9.0,6.0
+lookup,9.0,10.0,7.0,2.0,0.0,1.0,6.0,1.0,2.0,4.0
+map,12.0,10.0,1.0,1.0,3.0,1.0,2.0,1.0,2.0,0.0
+multiply,11.0,15.0,0.0,0.0,1.0,0.0,6.0,1.0,2.0,0.0
+nth,7.0,11.0,1.0,0.0,1.0,0.0,2.0,0.0,3.0,0.0
+section-5-example,25.0,30.0,28.0,1.0,6.0,0.0,39.0,2.0,12.0,4.0
+section-5-example,41.0,43.0,28.0,1.0,3.0,0.0,37.0,1.0,13.0,6.0
+section-5-example,25.0,30.0,23.0,1.0,2.0,0.0,36.0,3.0,11.0,7.0
+zeros,22.0,15.0,1.0,2.0,1.0,1.0,18.0,0.0,5.0,1.0
+zeros,25.0,17.0,2.0,1.0,7.0,1.0,3.0,1.0,5.0,1.0
+zipWith,7.0,9.0,5.0,1.0,2.0,0.0,4.0,2.0,3.0,3.0
+graphics/background,7.0,8.0,1.0,2.0,4.0,5.0,9.0,1.0,19.0,0.0
+graphics/grouped-bar-chart,108.0,143.0,148.0,6.0,1492.0,1.0,247.0,406.0,3997.0,0.0
+graphics/line-chart,195.0,231.0,213.0,10.0,2052.0,2.0,452.0,868.0,3877.0,0.0
+graphics/stacked-bar-chart,71.0,96.0,86.0,5.0,261.0,0.0,161.0,158.0,208.0,0.0
diff --git a/test/Benchmark.purs b/test/Benchmark.purs
@@ -23,7 +23,7 @@ main = launchAff_ do
       iter = 1
       arr = concat ([ bench_misc, bench_desugaring, bench_bwd, bench_graphics ] <#> ((#) (iter × true)))
    outs <- sequence $ map (\(str × row) -> lift2 Tuple (pure str) row) arr
-   logShow $ BenchAcc outs
+   logShow (BenchAcc outs)
 
 bench_desugaring :: (Int × Boolean) -> Array (String × Aff BenchRow)
 bench_desugaring = many desugar_cases

diff --git a/test/Benchmark/Util.purs b/test/Benchmark/Util.purs
@@ -3,9 +3,9 @@ module Benchmark.Util where
 import Prelude
 
 import Control.Monad.Writer (WriterT, runWriterT)
-import Data.Array (intersperse)
-import Data.Foldable (fold)
-import Data.Map (Map, unionWith, fromFoldable)
+import Data.Array (intersperse, fromFoldable) as A
+import Data.List (fold)
+import Data.Map (Map, unionWith, fromFoldable, keys, values)
 import Effect.Class (class MonadEffect, liftEffect)
 import Test.Spec.Microtime (microtime)
 import Util (type (×), (×))
@@ -23,28 +23,35 @@ derive newtype instance Semigroup BenchAcc
 derive newtype instance Monoid BenchAcc
 
 type TraceRow = Map String Number
-   -- { tEval :: Number
-   -- , tBwd :: Number
-   -- , tFwd :: Number
-   -- }
+-- { tEval :: Number
+-- , tBwd :: Number
+-- , tFwd :: Number
+-- }
 
 type GraphRow = Map String Number
-   -- { tEval :: Number
-   -- , tBwd :: Number
-   -- , tBwdDual :: Number
-   -- , tBwdAll :: Number
-   -- , tFwd :: Number
-   -- , tFwdDual :: Number
-   -- , tFwdAsDemorgan :: Number
-   -- }
+
+-- { tEval :: Number
+-- , tBwd :: Number
+-- , tBwdDual :: Number
+-- , tBwdAll :: Number
+-- , tFwd :: Number
+-- , tFwdDual :: Number
+-- , tFwdAsDemorgan :: Number
+-- }
 
 instance Show BenchAcc where
    show (BenchAcc rows) =
-      "Test-Name, Trace-Eval, Trace-Bwd, Trace-Fwd, Graph-Eval, Graph-Bwd, Graph-BwdDual, Graph-BwdAll, Graph-Fwd, Graph-FwdDual, Graph-FwdAsDeMorgan\n"
-         <> (fold $ intersperse "\n" $ rowShow <$> rows)
+      fold $ A.intersperse "\n" ([ showHeader ] <> (showRow <$> rows))
+      where
+      BenchRow tr_empty gr_empty = mempty
+
+      showHeader :: String
+      showHeader =
+         fold $ A.intersperse "," ([ "Test-Name" ] <> A.fromFoldable (keys tr_empty) <> A.fromFoldable (keys gr_empty))
 
-rowShow :: String × BenchRow -> String
-rowShow (str × row) = str <> "," <> show row
+      showRow :: String × BenchRow -> String
+      showRow (test_name × (BenchRow trRow grRow)) =
+         fold $ A.intersperse "," ([ test_name ] <> (show <$> A.fromFoldable (values trRow <> values grRow)))
 
 instance Semigroup BenchRow where
    append (BenchRow trRow1 gRow1) (BenchRow trRow2 gRow2) =
@@ -54,12 +61,9 @@ instance Semigroup BenchRow where
 
 instance Monoid BenchRow where
    mempty = BenchRow
-      (fromFoldable [ ("Trace-Eval" × 0.0), ("Trace-Bwd" × 0.0), ("Trace-Fwd" × 0.0)])
+      (fromFoldable [ ("Trace-Eval" × 0.0), ("Trace-Bwd" × 0.0), ("Trace-Fwd" × 0.0) ])
       (fromFoldable [ ("Graph-Eval" × 0.0), ("Graph-Bwd" × 0.0), ("Graph-Fwd" × 0.0), ("Graph-BwdDual" × 0.0), ("Graph-BwdAll" × 0.0), ("Graph-FwdDual" × 0.0), ("Graph-FwdAsDeMorgan" × 0.0) ])
 
-instance Show BenchRow where
-   show (BenchRow trRow grRow) = show trRow <> show grRow
-
 tdiff :: Number -> Number -> Number
 tdiff x y = sub y x
 

diff --git a/test/Util.purs b/test/Util.purs
@@ -94,7 +94,7 @@ testTrace s γ { δv, bwd_expect, fwd_expect } = do
       when logging $ log (prettyP v𝔹)
       checkPretty "Trace-based value" fwd_expect v𝔹
 
-   pure (fromFoldable ["Trace-Eval" × t_eval, "Trace-Bwd" × t_bwd , "Trace-Fwd" × t_fwd])
+   pure (fromFoldable [ "Trace-Eval" × t_eval, "Trace-Bwd" × t_bwd, "Trace-Fwd" × t_fwd ])
 
 testGraph :: forall m. MonadAff m => MonadError Error m => Raw SE.Expr -> GraphConfig GraphImpl -> TestConfig -> Boolean -> m GraphRow
 testGraph s gconfig { δv, bwd_expect, fwd_expect } is_bench = do
@@ -130,7 +130,7 @@ testGraph s gconfig { δv, bwd_expect, fwd_expect } is_bench = do
    αs_out `shouldSatisfy "fwd ⚬ bwd round-tripping property"`
       (flip subset αs_out')
 
-   let benchmarks = fromFoldable ["Graph-Eval" × t_eval, "Graph-Bwd" × t_bwd , "Graph-Fwd" × t_fwd]
+   let benchmarks = fromFoldable [ "Graph-Eval" × t_eval, "Graph-Bwd" × t_bwd, "Graph-Fwd" × t_fwd ]
 
    if not is_bench then pure benchmarks
    else do
@@ -166,7 +166,7 @@ testGraph s gconfig { δv, bwd_expect, fwd_expect } is_bench = do
          log (prettyP v𝔹_dual)
 
       pure $ union benchmarks
-                   (fromFoldable [("Graph-BwdDual" × t_bwdDual), ("Graph-BwdAll" × t_bwdAll), ("Graph-FwdDual" × t_fwdDual),  ("Graph-FwdAsDeMorgan" × t_fwdAsDeMorgan)])
+         (fromFoldable [ ("Graph-BwdDual" × t_bwdDual), ("Graph-BwdAll" × t_bwdAll), ("Graph-FwdDual" × t_fwdDual), ("Graph-FwdAsDeMorgan" × t_fwdAsDeMorgan) ])
 
 type TestSpec =
    { file :: String