Merge pull request #797 from explorable-viz/benchmarking

Benchmarking

Benchmarks/benchmarks.csv

@@ -1,64 +1,67 @@
-Test-Name, Trace-Eval, Trace-Bwd, Trace-Fwd, Graph-Eval, Graph-Bwd, Graph-BwdDual, Graph-BwdAll, Graph-Fwd, Graph-FwdDual, Graph-FwdAsDeMorgan
-arithmetic,1.6,20.6,1.6,3.2,0.2,0.0,0.2,0.2,0.0,7.0
-array,3.0,15.4,4.0,6.8,0.2,0.2,0.2,0.0,0.0,3.0
-compose,0.4,19.4,0.4,2.6,0.0,0.0,0.0,0.0,0.0,3.6
-dicts,1.8,13.6,1.4,5.2,0.4,0.2,0.6,0.0,0.0,3.0
-div-mod-quot-rem,2.0,16.4,1.2,9.6,0.2,0.2,2.4,0.0,0.0,3.0
-factorial,6.2,22.4,5.6,9.2,0.0,0.0,23.2,0.0,0.0,109.6
-filter,2.4,16.8,4.0,6.6,0.0,0.4,0.0,0.0,0.0,2.2
-first-class-constr,2.0,16.0,1.8,7.2,0.0,0.0,0.4,0.0,0.0,2.8
-flatten,5.4,16.6,10.6,10.4,0.2,0.4,1.2,0.0,0.0,5.6
-foldr_sumSquares,2.6,18.8,2.4,9.2,0.0,0.0,0.0,0.0,0.0,2.6
-lexicalScoping,2.6,15.8,1.4,4.2,0.2,0.0,0.2,0.0,0.0,6.0
-length,1.4,13.8,1.2,2.8,0.2,0.0,0.2,0.0,0.2,3.2
-lookup,3.8,18.6,5.0,11.4,0.4,0.0,0.2,0.0,0.0,3.2
-map,2.8,17.6,2.6,7.6,0.0,0.0,1.0,0.0,0.0,2.4
-mergeSort,16.4,24.2,16.8,24.0,0.0,0.0,1.2,0.0,0.0,5.2
-normalise,2.2,20.2,1.6,10.0,0.4,0.0,1.8,0.0,0.2,5.6
-pattern-match,4.0,18.2,8.0,11.2,0.0,0.0,0.8,0.0,0.0,3.8
-range,17.4,30.4,24.0,23.0,0.0,0.0,1.0,0.0,0.0,5.8
-records,5.0,17.8,5.0,9.2,0.0,0.0,0.6,0.0,0.0,3.2
-reverse,1.2,17.6,1.4,5.2,0.0,0.0,0.4,0.0,0.0,3.0
-desugar/list-comp-1,17.2,26.8,16.2,27.6,0.0,0.0,1.4,0.0,0.0,5.8
-desugar/list-comp-2,62.4,71.0,50.4,94.6,0.2,0.0,8.4,0.0,0.0,18.4
-desugar/list-comp-3,14.2,24.0,13.0,26.4,0.2,0.2,0.4,0.0,0.0,5.0
-desugar/list-comp-4,6.0,15.0,4.6,9.8,0.2,0.0,0.2,0.2,0.0,3.0
-desugar/list-comp-5,3.4,14.0,2.8,8.2,0.0,0.0,0.8,0.0,0.2,2.2
-desugar/list-comp-6,0.0,13.4,3.2,1.2,0.0,0.0,0.2,0.0,0.0,2.4
-desugar/list-comp-7,5.2,14.0,4.0,8.2,0.0,0.0,0.2,0.0,0.2,1.8
-desugar/list-enum,2.6,16.0,2.4,7.4,0.0,0.2,4.4,0.0,0.0,5.8
-add,0.0,13.2,0.0,1.8,0.0,0.0,0.0,0.2,0.2,3.0
-array/lookup,17.8,22.8,19.0,35.4,0.0,0.2,0.0,0.0,0.0,11.4
-array/dims,1.6,15.2,1.8,2.6,0.4,0.0,0.2,0.0,0.2,3.4
-convolution/edgeDetect,1545.2,1143.8,1560.8,3243.8,0.6,0.6,10.6,6.6,311.4,2868.0
-convolution/emboss,1348.2,905.4,1313.0,2625.0,1.0,0.6,9.0,2.2,354.0,2566.0
-convolution/gaussian,1247.2,921.6,1325.8,2525.4,0.2,0.2,10.0,1.8,313.4,2486.0
-dict/create,0.6,14.8,0.0,3.0,0.4,0.4,0.6,0.2,0.0,5.0
-dict/difference,1.4,13.6,0.4,3.0,0.0,0.0,0.2,0.2,0.2,3.2
-dict/disjointUnion,0.2,12.4,0.4,2.2,0.2,0.2,0.0,0.0,0.0,3.0
-dict/foldl,6.4,24.2,7.4,11.8,0.0,0.0,0.0,0.0,0.0,3.0
-dict/intersectionWith,1.2,13.2,1.0,2.6,0.0,0.2,0.0,0.2,0.0,2.6
-dict/fromRecord,0.4,16.0,0.0,0.8,0.2,0.0,0.4,0.0,0.0,2.4
-dict/get,0.4,15.6,0.6,1.6,0.0,0.2,0.0,0.0,0.0,3.4
-dict/map,3.6,19.0,3.2,7.6,0.2,0.2,0.0,0.0,0.2,2.6
-divide,0.0,14.0,0.2,1.4,0.2,0.4,0.0,0.0,0.0,2.6
-filter,4.4,17.6,2.6,6.4,0.0,0.0,0.8,0.0,0.0,2.2
-intersperse,1.6,19.2,1.8,10.8,0.2,0.0,0.8,0.4,0.2,2.2
-intersperse,1.2,18.6,1.6,8.6,0.2,0.0,0.6,0.4,0.8,2.0
-length,1.2,17.6,2.4,4.4,0.2,2.0,0.4,0.0,0.6,1.8
-list-comp,25.0,31.8,23.8,41.8,0.0,0.0,0.6,1.4,2.2,5.4
-list-comp,23.4,27.8,23.4,37.2,0.2,0.0,0.6,1.8,1.4,6.0
-lookup,7.2,17.0,5.8,10.6,0.4,0.6,0.4,0.8,0.6,4.0
-map,1.2,16.8,2.2,3.4,0.0,0.2,0.2,0.0,0.2,3.2
-multiply,0.0,12.4,0.0,3.2,0.0,1.4,0.0,0.0,0.0,1.8
-nth,3.0,16.0,2.4,4.8,0.4,0.2,0.0,0.0,0.0,2.4
-section-5-example,30.4,36.4,26.6,58.4,0.0,0.6,0.4,1.4,1.4,6.2
-section-5-example,24.2,29.8,23.0,46.4,0.4,0.0,0.6,0.2,0.4,7.4
-section-5-example,25.4,28.6,27.0,51.2,0.0,0.2,0.8,0.4,0.6,7.6
-zeros,0.6,15.2,1.6,2.8,0.6,0.0,0.0,0.0,0.4,2.2
-zeros,0.8,16.0,3.0,4.2,0.2,0.0,0.0,0.2,0.0,3.4
-zipWith,4.0,15.8,4.4,7.8,0.0,0.2,0.4,0.2,0.4,2.2
-graphics/background,1.8,15.4,1.6,7.2,0.0,0.0,1.8,0.0,0.0,12.8
-graphics/grouped-bar-chart,145.2,126.0,148.8,255.2,0.8,0.2,775.0,0.0,0.4,2186.4
-graphics/line-chart,201.8,146.2,193.8,393.0,1.6,0.2,1728.0,0.0,0.2,2987.2
-graphics/stacked-bar-chart,81.6,67.0,77.2,159.0,0.2,0.6,208.2,0.0,0.4,142.0
+Test-Name,Graph-Bwd,Graph-BwdAll,Graph-BwdDual,Graph-Eval,Graph-Fwd,Graph-FwdAsDeMorgan,Graph-FwdDual,Trace-Bwd,Trace-Eval,Trace-Fwd
+arithmetic,1.0,1.0,0.0,14.0,0.0,5.0,0.0,12.0,13.0,2.0
+array,2.0,1.0,0.0,8.0,0.0,4.0,0.0,11.0,14.0,6.0
+compose,0.0,1.0,0.0,7.0,0.0,2.0,0.0,9.0,7.0,1.0
+dicts,1.0,2.0,1.0,9.0,2.0,3.0,0.0,12.0,7.0,2.0
+div-mod-quot-rem,0.0,4.0,0.0,17.0,1.0,5.0,0.0,8.0,13.0,1.0
+factorial,1.0,35.0,0.0,19.0,0.0,146.0,0.0,14.0,18.0,5.0
+filter,1.0,1.0,0.0,9.0,0.0,2.0,0.0,8.0,13.0,4.0
+first-class-constr,0.0,1.0,1.0,13.0,0.0,3.0,0.0,8.0,12.0,3.0
+flatten,2.0,2.0,0.0,17.0,0.0,3.0,0.0,10.0,16.0,7.0
+foldr_sumSquares,1.0,1.0,0.0,10.0,0.0,3.0,0.0,8.0,12.0,4.0
+lexicalScoping,1.0,1.0,0.0,2.0,0.0,3.0,0.0,23.0,17.0,1.0
+length,3.0,0.0,0.0,3.0,0.0,3.0,0.0,7.0,6.0,1.0
+lookup,2.0,1.0,0.0,26.0,0.0,4.0,0.0,20.0,40.0,6.0
+map,1.0,3.0,1.0,27.0,1.0,5.0,1.0,8.0,12.0,2.0
+mergeSort,1.0,1.0,0.0,43.0,0.0,6.0,0.0,23.0,32.0,22.0
+normalise,1.0,1.0,0.0,5.0,0.0,3.0,0.0,6.0,6.0,4.0
+pattern-match,1.0,2.0,0.0,17.0,0.0,5.0,0.0,8.0,13.0,5.0
+range,2.0,2.0,0.0,53.0,0.0,12.0,0.0,24.0,32.0,11.0
+records,1.0,1.0,0.0,22.0,0.0,4.0,0.0,11.0,14.0,10.0
+reverse,2.0,1.0,0.0,6.0,0.0,3.0,0.0,8.0,20.0,3.0
+dtw/next-indices,2.0,143.0,0.0,206.0,6.0,741.0,1.0,50.0,130.0,51.0
+dtw/cost-matrix,2.0,7.0,1.0,196.0,2.0,118.0,1.0,46.0,119.0,33.0
+dtw/matrix-update,0.0,1.0,0.0,7.0,1.0,3.0,0.0,9.0,8.0,2.0
+desugar/list-comp-1,1.0,2.0,0.0,51.0,0.0,7.0,0.0,23.0,37.0,15.0
+desugar/list-comp-2,1.0,10.0,0.0,208.0,1.0,28.0,0.0,38.0,111.0,44.0
+desugar/list-comp-3,1.0,2.0,0.0,47.0,0.0,7.0,0.0,14.0,32.0,10.0
+desugar/list-comp-4,1.0,1.0,0.0,20.0,0.0,3.0,0.0,9.0,16.0,3.0
+desugar/list-comp-5,0.0,1.0,0.0,15.0,0.0,3.0,0.0,8.0,17.0,3.0
+desugar/list-comp-6,1.0,0.0,0.0,1.0,0.0,9.0,0.0,11.0,11.0,1.0
+desugar/list-comp-7,1.0,1.0,0.0,16.0,0.0,3.0,0.0,11.0,19.0,3.0
+desugar/list-enum,1.0,6.0,0.0,13.0,0.0,7.0,0.0,8.0,15.0,2.0
+add,0.0,1.0,0.0,2.0,1.0,1.0,0.0,6.0,6.0,0.0
+array/lookup,0.0,1.0,0.0,69.0,0.0,16.0,0.0,15.0,44.0,18.0
+array/dims,0.0,1.0,0.0,7.0,0.0,3.0,0.0,9.0,10.0,3.0
+convolution/edgeDetect,31.0,44.0,1.0,6144.0,9.0,4437.0,341.0,1144.0,3552.0,1402.0
+convolution/emboss,24.0,35.0,0.0,5347.0,2.0,4085.0,334.0,890.0,3084.0,1182.0
+convolution/gaussian,26.0,36.0,1.0,4913.0,1.0,3813.0,360.0,828.0,2827.0,1306.0
+dict/create,2.0,5.0,0.0,5.0,1.0,6.0,0.0,11.0,30.0,0.0
+dict/difference,1.0,1.0,0.0,3.0,0.0,7.0,0.0,6.0,7.0,1.0
+dict/disjointUnion,1.0,1.0,0.0,2.0,0.0,3.0,0.0,10.0,6.0,0.0
+dict/foldl,2.0,2.0,0.0,46.0,0.0,13.0,0.0,15.0,18.0,5.0
+dict/intersectionWith,2.0,1.0,2.0,28.0,1.0,4.0,0.0,26.0,17.0,18.0
+dict/fromRecord,0.0,1.0,0.0,1.0,1.0,2.0,0.0,9.0,7.0,0.0
+dict/get,0.0,2.0,0.0,3.0,0.0,2.0,0.0,8.0,7.0,1.0
+dict/map,1.0,1.0,0.0,11.0,0.0,4.0,0.0,10.0,10.0,3.0
+divide,0.0,0.0,0.0,2.0,0.0,4.0,0.0,5.0,13.0,0.0
+filter,1.0,1.0,0.0,15.0,0.0,3.0,0.0,13.0,16.0,2.0
+intersperse,1.0,1.0,0.0,10.0,1.0,3.0,0.0,8.0,7.0,2.0
+intersperse,1.0,2.0,0.0,7.0,0.0,2.0,1.0,7.0,10.0,4.0
+length,1.0,1.0,0.0,8.0,0.0,2.0,0.0,6.0,15.0,1.0
+list-comp,1.0,2.0,1.0,94.0,2.0,13.0,3.0,28.0,67.0,29.0
+list-comp,1.0,1.0,0.0,77.0,1.0,7.0,2.0,24.0,65.0,24.0
+lookup,0.0,0.0,1.0,14.0,0.0,3.0,1.0,10.0,14.0,4.0
+map,0.0,0.0,0.0,8.0,0.0,3.0,0.0,10.0,8.0,1.0
+multiply,1.0,2.0,0.0,1.0,0.0,1.0,0.0,8.0,15.0,0.0
+nth,0.0,0.0,0.0,4.0,1.0,3.0,0.0,7.0,18.0,1.0
+section-5-example,1.0,3.0,1.0,87.0,1.0,7.0,4.0,25.0,71.0,30.0
+section-5-example,1.0,2.0,0.0,83.0,1.0,10.0,2.0,23.0,58.0,25.0
+section-5-example,2.0,2.0,0.0,111.0,1.0,9.0,3.0,21.0,62.0,28.0
+zeros,1.0,1.0,0.0,4.0,1.0,2.0,0.0,10.0,7.0,2.0
+zeros,0.0,0.0,0.0,3.0,0.0,2.0,0.0,5.0,8.0,5.0
+zipWith,1.0,2.0,0.0,13.0,0.0,2.0,1.0,9.0,14.0,6.0
+graphics/background,2.0,4.0,0.0,12.0,2.0,20.0,0.0,9.0,12.0,3.0
+graphics/grouped-bar-chart,6.0,853.0,1.0,526.0,250.0,2585.0,0.0,107.0,294.0,134.0
+graphics/line-chart,9.0,1953.0,1.0,799.0,856.0,3744.0,0.0,139.0,443.0,191.0
+graphics/stacked-bar-chart,4.0,228.0,0.0,288.0,143.0,189.0,0.0,64.0,173.0,70.0

test/Benchmark.purs

@@ -23,7 +23,7 @@ main = launchAff_ do
       iter = 1
       arr = concat ([ bench_misc, bench_desugaring, bench_bwd, bench_graphics ] <#> ((#) (iter × true)))
    outs <- sequence $ map (\(str × row) -> lift2 Tuple (pure str) row) arr
-   logShow $ BenchAcc outs
+   logShow (BenchAcc outs)
 
 bench_desugaring :: (Int × Boolean) -> Array (String × Aff BenchRow)
 bench_desugaring = many desugar_cases

test/Benchmark/Util.purs

@@ -2,95 +2,54 @@ module Benchmark.Util where
 
 import Prelude
 
-import Control.Monad.Writer (WriterT, runWriterT)
-import Data.Array (intersperse)
-import Data.Foldable (fold)
--- import Data.Lazy (Lazy)
--- import Data.Lazy (force, defer) as Lazy
+import Control.Monad.Writer.Class (class MonadWriter, tell)
+import Data.Array (intersperse, fromFoldable) as A
+import Data.Int (toNumber)
+import Data.List (fold)
+import Data.Map (Map, singleton, unionWith, fromFoldable, keys, values)
 import Effect.Class (class MonadEffect, liftEffect)
 import Test.Spec.Microtime (microtime)
 import Util (type (×), (×))
 
-data BenchRow = BenchRow TraceRow GraphRow
-
 newtype BenchAcc = BenchAcc (Array (String × BenchRow))
 
-type WithBenchAcc g a = WriterT BenchAcc g a
-
-runWithBenchAcc :: forall g a. Monad g => WithBenchAcc g a -> g (a × BenchAcc)
-runWithBenchAcc = runWriterT
-
-derive newtype instance Semigroup BenchAcc
-derive newtype instance Monoid BenchAcc
-
-type TraceRow =
-   { tEval :: Number
-   , tBwd :: Number
-   , tFwd :: Number
-   }
-
-type GraphRow =
-   { tEval :: Number
-   , tBwd :: Number
-   , tBwdDual :: Number
-   , tBwdAll :: Number
-   , tFwd :: Number
-   , tFwdDual :: Number
-   , tFwdAsDemorgan :: Number
-   }
-
 instance Show BenchAcc where
    show (BenchAcc rows) =
-      "Test-Name, Trace-Eval, Trace-Bwd, Trace-Fwd, Graph-Eval, Graph-Bwd, Graph-BwdDual, Graph-BwdAll, Graph-Fwd, Graph-FwdDual, Graph-FwdAsDeMorgan\n"
-         <> (fold $ intersperse "\n" $ rowShow <$> rows)
+      fold $ A.intersperse "\n" ([ showHeader ] <> (showRow <$> rows))
+      where
+      BenchRow empty_row = mempty
 
-rowShow :: String × BenchRow -> String
-rowShow (str × row) = str <> "," <> show row
+      showHeader :: String
+      showHeader =
+         fold $ A.intersperse "," ([ "Test-Name" ] <> A.fromFoldable (keys empty_row))
+
+      showRow :: String × BenchRow -> String
+      showRow (test_name × (BenchRow row)) =
+         fold $ A.intersperse "," ([ test_name ] <> (show <$> A.fromFoldable (values row)))
+
+newtype BenchRow = BenchRow (Map String Number)
 
 instance Semigroup BenchRow where
-   append (BenchRow trRow1 gRow1) (BenchRow trRow2 gRow2) =
-      BenchRow
-         { tEval: trRow1.tEval + trRow2.tEval
-         , tBwd: trRow1.tBwd + trRow2.tBwd
-         , tFwd: trRow1.tFwd + trRow2.tFwd
-         }
-         { tEval: gRow1.tEval + gRow2.tEval
-         , tBwd: gRow1.tBwd + gRow2.tBwd
-         , tBwdDual: gRow1.tBwdDual + gRow2.tBwdDual
-         , tBwdAll: gRow1.tBwdAll + gRow2.tBwdAll
-         , tFwd: gRow1.tFwd + gRow2.tFwd
-         , tFwdDual: gRow1.tFwdDual + gRow2.tFwdDual
-         , tFwdAsDemorgan: gRow1.tFwdAsDemorgan + gRow2.tFwdAsDemorgan
-         }
+   append (BenchRow row1) (BenchRow row2) = BenchRow (unionWith (+) row1 row2)
 
 instance Monoid BenchRow where
    mempty = BenchRow
-      { tEval: 0.0, tBwd: 0.0, tFwd: 0.0 }
-      { tEval: 0.0, tBwd: 0.0, tBwdDual: 0.0, tBwdAll: 0.0, tFwd: 0.0, tFwdDual: 0.0, tFwdAsDemorgan: 0.0 }
-
-instance Show BenchRow where
-   show (BenchRow trRow grRow) = fold $ intersperse "," $ (_ <#> show)
-      [ trRow.tEval
-      , trRow.tBwd
-      , trRow.tFwd
-      , grRow.tEval
-      , grRow.tBwd
-      , grRow.tBwdDual
-      , grRow.tBwdAll
-      , grRow.tFwd
-      , grRow.tFwdDual
-      , grRow.tFwdAsDemorgan
-      ]
+      (fromFoldable [ ("Trace-Eval" × 0.0), ("Trace-Bwd" × 0.0), ("Trace-Fwd" × 0.0), ("Graph-Eval" × 0.0), ("Graph-Bwd" × 0.0), ("Graph-Fwd" × 0.0), ("Graph-BwdDual" × 0.0), ("Graph-BwdAll" × 0.0), ("Graph-FwdDual" × 0.0), ("Graph-FwdAsDeMorgan" × 0.0) ])
 
 tdiff :: Number -> Number -> Number
 tdiff x y = sub y x
 
 preciseTime :: forall m. MonadEffect m => m Number
 preciseTime = liftEffect microtime
 
-bench :: forall m a. MonadEffect m => (Unit -> m a) -> m (a × Number)
-bench prog = do
+bench :: forall m a. MonadEffect m => MonadWriter BenchRow m => String -> (Unit -> m a) -> m a
+bench name prog = do
    t1 <- preciseTime
-   a <- prog unit
+   r <- prog unit
    t2 <- preciseTime
-   pure (a × tdiff t1 t2)
+   tell (BenchRow $ singleton name (tdiff t1 t2))
+   pure r
+
+divRow :: BenchRow -> Int -> BenchRow
+divRow (BenchRow row) n = BenchRow (map (_ `div` toNumber n) row)
+