diff --git a/Benchmarks/benchmarks.csv b/Benchmarks/benchmarks.csv index 4627af5b1..20cc0dc9d 100644 --- a/Benchmarks/benchmarks.csv +++ b/Benchmarks/benchmarks.csv @@ -1,64 +1,67 @@ -Test-Name, Trace-Eval, Trace-Bwd, Trace-Fwd, Graph-Eval, Graph-Bwd, Graph-BwdDual, Graph-BwdAll, Graph-Fwd, Graph-FwdDual, Graph-FwdAsDeMorgan -arithmetic,1.6,20.6,1.6,3.2,0.2,0.0,0.2,0.2,0.0,7.0 -array,3.0,15.4,4.0,6.8,0.2,0.2,0.2,0.0,0.0,3.0 -compose,0.4,19.4,0.4,2.6,0.0,0.0,0.0,0.0,0.0,3.6 -dicts,1.8,13.6,1.4,5.2,0.4,0.2,0.6,0.0,0.0,3.0 -div-mod-quot-rem,2.0,16.4,1.2,9.6,0.2,0.2,2.4,0.0,0.0,3.0 -factorial,6.2,22.4,5.6,9.2,0.0,0.0,23.2,0.0,0.0,109.6 -filter,2.4,16.8,4.0,6.6,0.0,0.4,0.0,0.0,0.0,2.2 -first-class-constr,2.0,16.0,1.8,7.2,0.0,0.0,0.4,0.0,0.0,2.8 -flatten,5.4,16.6,10.6,10.4,0.2,0.4,1.2,0.0,0.0,5.6 -foldr_sumSquares,2.6,18.8,2.4,9.2,0.0,0.0,0.0,0.0,0.0,2.6 -lexicalScoping,2.6,15.8,1.4,4.2,0.2,0.0,0.2,0.0,0.0,6.0 -length,1.4,13.8,1.2,2.8,0.2,0.0,0.2,0.0,0.2,3.2 -lookup,3.8,18.6,5.0,11.4,0.4,0.0,0.2,0.0,0.0,3.2 -map,2.8,17.6,2.6,7.6,0.0,0.0,1.0,0.0,0.0,2.4 -mergeSort,16.4,24.2,16.8,24.0,0.0,0.0,1.2,0.0,0.0,5.2 -normalise,2.2,20.2,1.6,10.0,0.4,0.0,1.8,0.0,0.2,5.6 -pattern-match,4.0,18.2,8.0,11.2,0.0,0.0,0.8,0.0,0.0,3.8 -range,17.4,30.4,24.0,23.0,0.0,0.0,1.0,0.0,0.0,5.8 -records,5.0,17.8,5.0,9.2,0.0,0.0,0.6,0.0,0.0,3.2 -reverse,1.2,17.6,1.4,5.2,0.0,0.0,0.4,0.0,0.0,3.0 -desugar/list-comp-1,17.2,26.8,16.2,27.6,0.0,0.0,1.4,0.0,0.0,5.8 -desugar/list-comp-2,62.4,71.0,50.4,94.6,0.2,0.0,8.4,0.0,0.0,18.4 -desugar/list-comp-3,14.2,24.0,13.0,26.4,0.2,0.2,0.4,0.0,0.0,5.0 -desugar/list-comp-4,6.0,15.0,4.6,9.8,0.2,0.0,0.2,0.2,0.0,3.0 -desugar/list-comp-5,3.4,14.0,2.8,8.2,0.0,0.0,0.8,0.0,0.2,2.2 -desugar/list-comp-6,0.0,13.4,3.2,1.2,0.0,0.0,0.2,0.0,0.0,2.4 -desugar/list-comp-7,5.2,14.0,4.0,8.2,0.0,0.0,0.2,0.0,0.2,1.8 -desugar/list-enum,2.6,16.0,2.4,7.4,0.0,0.2,4.4,0.0,0.0,5.8 -add,0.0,13.2,0.0,1.8,0.0,0.0,0.0,0.2,0.2,3.0 -array/lookup,17.8,22.8,19.0,35.4,0.0,0.2,0.0,0.0,0.0,11.4 -array/dims,1.6,15.2,1.8,2.6,0.4,0.0,0.2,0.0,0.2,3.4 -convolution/edgeDetect,1545.2,1143.8,1560.8,3243.8,0.6,0.6,10.6,6.6,311.4,2868.0 -convolution/emboss,1348.2,905.4,1313.0,2625.0,1.0,0.6,9.0,2.2,354.0,2566.0 -convolution/gaussian,1247.2,921.6,1325.8,2525.4,0.2,0.2,10.0,1.8,313.4,2486.0 -dict/create,0.6,14.8,0.0,3.0,0.4,0.4,0.6,0.2,0.0,5.0 -dict/difference,1.4,13.6,0.4,3.0,0.0,0.0,0.2,0.2,0.2,3.2 -dict/disjointUnion,0.2,12.4,0.4,2.2,0.2,0.2,0.0,0.0,0.0,3.0 -dict/foldl,6.4,24.2,7.4,11.8,0.0,0.0,0.0,0.0,0.0,3.0 -dict/intersectionWith,1.2,13.2,1.0,2.6,0.0,0.2,0.0,0.2,0.0,2.6 -dict/fromRecord,0.4,16.0,0.0,0.8,0.2,0.0,0.4,0.0,0.0,2.4 -dict/get,0.4,15.6,0.6,1.6,0.0,0.2,0.0,0.0,0.0,3.4 -dict/map,3.6,19.0,3.2,7.6,0.2,0.2,0.0,0.0,0.2,2.6 -divide,0.0,14.0,0.2,1.4,0.2,0.4,0.0,0.0,0.0,2.6 -filter,4.4,17.6,2.6,6.4,0.0,0.0,0.8,0.0,0.0,2.2 -intersperse,1.6,19.2,1.8,10.8,0.2,0.0,0.8,0.4,0.2,2.2 -intersperse,1.2,18.6,1.6,8.6,0.2,0.0,0.6,0.4,0.8,2.0 -length,1.2,17.6,2.4,4.4,0.2,2.0,0.4,0.0,0.6,1.8 -list-comp,25.0,31.8,23.8,41.8,0.0,0.0,0.6,1.4,2.2,5.4 -list-comp,23.4,27.8,23.4,37.2,0.2,0.0,0.6,1.8,1.4,6.0 -lookup,7.2,17.0,5.8,10.6,0.4,0.6,0.4,0.8,0.6,4.0 -map,1.2,16.8,2.2,3.4,0.0,0.2,0.2,0.0,0.2,3.2 -multiply,0.0,12.4,0.0,3.2,0.0,1.4,0.0,0.0,0.0,1.8 -nth,3.0,16.0,2.4,4.8,0.4,0.2,0.0,0.0,0.0,2.4 -section-5-example,30.4,36.4,26.6,58.4,0.0,0.6,0.4,1.4,1.4,6.2 -section-5-example,24.2,29.8,23.0,46.4,0.4,0.0,0.6,0.2,0.4,7.4 -section-5-example,25.4,28.6,27.0,51.2,0.0,0.2,0.8,0.4,0.6,7.6 -zeros,0.6,15.2,1.6,2.8,0.6,0.0,0.0,0.0,0.4,2.2 -zeros,0.8,16.0,3.0,4.2,0.2,0.0,0.0,0.2,0.0,3.4 -zipWith,4.0,15.8,4.4,7.8,0.0,0.2,0.4,0.2,0.4,2.2 -graphics/background,1.8,15.4,1.6,7.2,0.0,0.0,1.8,0.0,0.0,12.8 -graphics/grouped-bar-chart,145.2,126.0,148.8,255.2,0.8,0.2,775.0,0.0,0.4,2186.4 -graphics/line-chart,201.8,146.2,193.8,393.0,1.6,0.2,1728.0,0.0,0.2,2987.2 -graphics/stacked-bar-chart,81.6,67.0,77.2,159.0,0.2,0.6,208.2,0.0,0.4,142.0 \ No newline at end of file +Test-Name,Graph-Bwd,Graph-BwdAll,Graph-BwdDual,Graph-Eval,Graph-Fwd,Graph-FwdAsDeMorgan,Graph-FwdDual,Trace-Bwd,Trace-Eval,Trace-Fwd +arithmetic,1.0,1.0,0.0,14.0,0.0,5.0,0.0,12.0,13.0,2.0 +array,2.0,1.0,0.0,8.0,0.0,4.0,0.0,11.0,14.0,6.0 +compose,0.0,1.0,0.0,7.0,0.0,2.0,0.0,9.0,7.0,1.0 +dicts,1.0,2.0,1.0,9.0,2.0,3.0,0.0,12.0,7.0,2.0 +div-mod-quot-rem,0.0,4.0,0.0,17.0,1.0,5.0,0.0,8.0,13.0,1.0 +factorial,1.0,35.0,0.0,19.0,0.0,146.0,0.0,14.0,18.0,5.0 +filter,1.0,1.0,0.0,9.0,0.0,2.0,0.0,8.0,13.0,4.0 +first-class-constr,0.0,1.0,1.0,13.0,0.0,3.0,0.0,8.0,12.0,3.0 +flatten,2.0,2.0,0.0,17.0,0.0,3.0,0.0,10.0,16.0,7.0 +foldr_sumSquares,1.0,1.0,0.0,10.0,0.0,3.0,0.0,8.0,12.0,4.0 +lexicalScoping,1.0,1.0,0.0,2.0,0.0,3.0,0.0,23.0,17.0,1.0 +length,3.0,0.0,0.0,3.0,0.0,3.0,0.0,7.0,6.0,1.0 +lookup,2.0,1.0,0.0,26.0,0.0,4.0,0.0,20.0,40.0,6.0 +map,1.0,3.0,1.0,27.0,1.0,5.0,1.0,8.0,12.0,2.0 +mergeSort,1.0,1.0,0.0,43.0,0.0,6.0,0.0,23.0,32.0,22.0 +normalise,1.0,1.0,0.0,5.0,0.0,3.0,0.0,6.0,6.0,4.0 +pattern-match,1.0,2.0,0.0,17.0,0.0,5.0,0.0,8.0,13.0,5.0 +range,2.0,2.0,0.0,53.0,0.0,12.0,0.0,24.0,32.0,11.0 +records,1.0,1.0,0.0,22.0,0.0,4.0,0.0,11.0,14.0,10.0 +reverse,2.0,1.0,0.0,6.0,0.0,3.0,0.0,8.0,20.0,3.0 +dtw/next-indices,2.0,143.0,0.0,206.0,6.0,741.0,1.0,50.0,130.0,51.0 +dtw/cost-matrix,2.0,7.0,1.0,196.0,2.0,118.0,1.0,46.0,119.0,33.0 +dtw/matrix-update,0.0,1.0,0.0,7.0,1.0,3.0,0.0,9.0,8.0,2.0 +desugar/list-comp-1,1.0,2.0,0.0,51.0,0.0,7.0,0.0,23.0,37.0,15.0 +desugar/list-comp-2,1.0,10.0,0.0,208.0,1.0,28.0,0.0,38.0,111.0,44.0 +desugar/list-comp-3,1.0,2.0,0.0,47.0,0.0,7.0,0.0,14.0,32.0,10.0 +desugar/list-comp-4,1.0,1.0,0.0,20.0,0.0,3.0,0.0,9.0,16.0,3.0 +desugar/list-comp-5,0.0,1.0,0.0,15.0,0.0,3.0,0.0,8.0,17.0,3.0 +desugar/list-comp-6,1.0,0.0,0.0,1.0,0.0,9.0,0.0,11.0,11.0,1.0 +desugar/list-comp-7,1.0,1.0,0.0,16.0,0.0,3.0,0.0,11.0,19.0,3.0 +desugar/list-enum,1.0,6.0,0.0,13.0,0.0,7.0,0.0,8.0,15.0,2.0 +add,0.0,1.0,0.0,2.0,1.0,1.0,0.0,6.0,6.0,0.0 +array/lookup,0.0,1.0,0.0,69.0,0.0,16.0,0.0,15.0,44.0,18.0 +array/dims,0.0,1.0,0.0,7.0,0.0,3.0,0.0,9.0,10.0,3.0 +convolution/edgeDetect,31.0,44.0,1.0,6144.0,9.0,4437.0,341.0,1144.0,3552.0,1402.0 +convolution/emboss,24.0,35.0,0.0,5347.0,2.0,4085.0,334.0,890.0,3084.0,1182.0 +convolution/gaussian,26.0,36.0,1.0,4913.0,1.0,3813.0,360.0,828.0,2827.0,1306.0 +dict/create,2.0,5.0,0.0,5.0,1.0,6.0,0.0,11.0,30.0,0.0 +dict/difference,1.0,1.0,0.0,3.0,0.0,7.0,0.0,6.0,7.0,1.0 +dict/disjointUnion,1.0,1.0,0.0,2.0,0.0,3.0,0.0,10.0,6.0,0.0 +dict/foldl,2.0,2.0,0.0,46.0,0.0,13.0,0.0,15.0,18.0,5.0 +dict/intersectionWith,2.0,1.0,2.0,28.0,1.0,4.0,0.0,26.0,17.0,18.0 +dict/fromRecord,0.0,1.0,0.0,1.0,1.0,2.0,0.0,9.0,7.0,0.0 +dict/get,0.0,2.0,0.0,3.0,0.0,2.0,0.0,8.0,7.0,1.0 +dict/map,1.0,1.0,0.0,11.0,0.0,4.0,0.0,10.0,10.0,3.0 +divide,0.0,0.0,0.0,2.0,0.0,4.0,0.0,5.0,13.0,0.0 +filter,1.0,1.0,0.0,15.0,0.0,3.0,0.0,13.0,16.0,2.0 +intersperse,1.0,1.0,0.0,10.0,1.0,3.0,0.0,8.0,7.0,2.0 +intersperse,1.0,2.0,0.0,7.0,0.0,2.0,1.0,7.0,10.0,4.0 +length,1.0,1.0,0.0,8.0,0.0,2.0,0.0,6.0,15.0,1.0 +list-comp,1.0,2.0,1.0,94.0,2.0,13.0,3.0,28.0,67.0,29.0 +list-comp,1.0,1.0,0.0,77.0,1.0,7.0,2.0,24.0,65.0,24.0 +lookup,0.0,0.0,1.0,14.0,0.0,3.0,1.0,10.0,14.0,4.0 +map,0.0,0.0,0.0,8.0,0.0,3.0,0.0,10.0,8.0,1.0 +multiply,1.0,2.0,0.0,1.0,0.0,1.0,0.0,8.0,15.0,0.0 +nth,0.0,0.0,0.0,4.0,1.0,3.0,0.0,7.0,18.0,1.0 +section-5-example,1.0,3.0,1.0,87.0,1.0,7.0,4.0,25.0,71.0,30.0 +section-5-example,1.0,2.0,0.0,83.0,1.0,10.0,2.0,23.0,58.0,25.0 +section-5-example,2.0,2.0,0.0,111.0,1.0,9.0,3.0,21.0,62.0,28.0 +zeros,1.0,1.0,0.0,4.0,1.0,2.0,0.0,10.0,7.0,2.0 +zeros,0.0,0.0,0.0,3.0,0.0,2.0,0.0,5.0,8.0,5.0 +zipWith,1.0,2.0,0.0,13.0,0.0,2.0,1.0,9.0,14.0,6.0 +graphics/background,2.0,4.0,0.0,12.0,2.0,20.0,0.0,9.0,12.0,3.0 +graphics/grouped-bar-chart,6.0,853.0,1.0,526.0,250.0,2585.0,0.0,107.0,294.0,134.0 +graphics/line-chart,9.0,1953.0,1.0,799.0,856.0,3744.0,0.0,139.0,443.0,191.0 +graphics/stacked-bar-chart,4.0,228.0,0.0,288.0,143.0,189.0,0.0,64.0,173.0,70.0 \ No newline at end of file diff --git a/test/Benchmark.purs b/test/Benchmark.purs index 78ec0579a..e0e78f29d 100644 --- a/test/Benchmark.purs +++ b/test/Benchmark.purs @@ -23,7 +23,7 @@ main = launchAff_ do iter = 1 arr = concat ([ bench_misc, bench_desugaring, bench_bwd, bench_graphics ] <#> ((#) (iter × true))) outs <- sequence $ map (\(str × row) -> lift2 Tuple (pure str) row) arr - logShow $ BenchAcc outs + logShow (BenchAcc outs) bench_desugaring :: (Int × Boolean) -> Array (String × Aff BenchRow) bench_desugaring = many desugar_cases diff --git a/test/Benchmark/Util.purs b/test/Benchmark/Util.purs index 57281df46..a70ce6027 100644 --- a/test/Benchmark/Util.purs +++ b/test/Benchmark/Util.purs @@ -2,85 +2,39 @@ module Benchmark.Util where import Prelude -import Control.Monad.Writer (WriterT, runWriterT) -import Data.Array (intersperse) -import Data.Foldable (fold) --- import Data.Lazy (Lazy) --- import Data.Lazy (force, defer) as Lazy +import Control.Monad.Writer.Class (class MonadWriter, tell) +import Data.Array (intersperse, fromFoldable) as A +import Data.Int (toNumber) +import Data.List (fold) +import Data.Map (Map, singleton, unionWith, fromFoldable, keys, values) import Effect.Class (class MonadEffect, liftEffect) import Test.Spec.Microtime (microtime) import Util (type (×), (×)) -data BenchRow = BenchRow TraceRow GraphRow - newtype BenchAcc = BenchAcc (Array (String × BenchRow)) -type WithBenchAcc g a = WriterT BenchAcc g a - -runWithBenchAcc :: forall g a. Monad g => WithBenchAcc g a -> g (a × BenchAcc) -runWithBenchAcc = runWriterT - -derive newtype instance Semigroup BenchAcc -derive newtype instance Monoid BenchAcc - -type TraceRow = - { tEval :: Number - , tBwd :: Number - , tFwd :: Number - } - -type GraphRow = - { tEval :: Number - , tBwd :: Number - , tBwdDual :: Number - , tBwdAll :: Number - , tFwd :: Number - , tFwdDual :: Number - , tFwdAsDemorgan :: Number - } - instance Show BenchAcc where show (BenchAcc rows) = - "Test-Name, Trace-Eval, Trace-Bwd, Trace-Fwd, Graph-Eval, Graph-Bwd, Graph-BwdDual, Graph-BwdAll, Graph-Fwd, Graph-FwdDual, Graph-FwdAsDeMorgan\n" - <> (fold $ intersperse "\n" $ rowShow <$> rows) + fold $ A.intersperse "\n" ([ showHeader ] <> (showRow <$> rows)) + where + BenchRow empty_row = mempty -rowShow :: String × BenchRow -> String -rowShow (str × row) = str <> "," <> show row + showHeader :: String + showHeader = + fold $ A.intersperse "," ([ "Test-Name" ] <> A.fromFoldable (keys empty_row)) + + showRow :: String × BenchRow -> String + showRow (test_name × (BenchRow row)) = + fold $ A.intersperse "," ([ test_name ] <> (show <$> A.fromFoldable (values row))) + +newtype BenchRow = BenchRow (Map String Number) instance Semigroup BenchRow where - append (BenchRow trRow1 gRow1) (BenchRow trRow2 gRow2) = - BenchRow - { tEval: trRow1.tEval + trRow2.tEval - , tBwd: trRow1.tBwd + trRow2.tBwd - , tFwd: trRow1.tFwd + trRow2.tFwd - } - { tEval: gRow1.tEval + gRow2.tEval - , tBwd: gRow1.tBwd + gRow2.tBwd - , tBwdDual: gRow1.tBwdDual + gRow2.tBwdDual - , tBwdAll: gRow1.tBwdAll + gRow2.tBwdAll - , tFwd: gRow1.tFwd + gRow2.tFwd - , tFwdDual: gRow1.tFwdDual + gRow2.tFwdDual - , tFwdAsDemorgan: gRow1.tFwdAsDemorgan + gRow2.tFwdAsDemorgan - } + append (BenchRow row1) (BenchRow row2) = BenchRow (unionWith (+) row1 row2) instance Monoid BenchRow where mempty = BenchRow - { tEval: 0.0, tBwd: 0.0, tFwd: 0.0 } - { tEval: 0.0, tBwd: 0.0, tBwdDual: 0.0, tBwdAll: 0.0, tFwd: 0.0, tFwdDual: 0.0, tFwdAsDemorgan: 0.0 } - -instance Show BenchRow where - show (BenchRow trRow grRow) = fold $ intersperse "," $ (_ <#> show) - [ trRow.tEval - , trRow.tBwd - , trRow.tFwd - , grRow.tEval - , grRow.tBwd - , grRow.tBwdDual - , grRow.tBwdAll - , grRow.tFwd - , grRow.tFwdDual - , grRow.tFwdAsDemorgan - ] + (fromFoldable [ ("Trace-Eval" × 0.0), ("Trace-Bwd" × 0.0), ("Trace-Fwd" × 0.0), ("Graph-Eval" × 0.0), ("Graph-Bwd" × 0.0), ("Graph-Fwd" × 0.0), ("Graph-BwdDual" × 0.0), ("Graph-BwdAll" × 0.0), ("Graph-FwdDual" × 0.0), ("Graph-FwdAsDeMorgan" × 0.0) ]) tdiff :: Number -> Number -> Number tdiff x y = sub y x @@ -88,9 +42,14 @@ tdiff x y = sub y x preciseTime :: forall m. MonadEffect m => m Number preciseTime = liftEffect microtime -bench :: forall m a. MonadEffect m => (Unit -> m a) -> m (a × Number) -bench prog = do +bench :: forall m a. MonadEffect m => MonadWriter BenchRow m => String -> (Unit -> m a) -> m a +bench name prog = do t1 <- preciseTime - a <- prog unit + r <- prog unit t2 <- preciseTime - pure (a × tdiff t1 t2) \ No newline at end of file + tell (BenchRow $ singleton name (tdiff t1 t2)) + pure r + +divRow :: BenchRow -> Int -> BenchRow +divRow (BenchRow row) n = BenchRow (map (_ `div` toNumber n) row) + diff --git a/test/Util.purs b/test/Util.purs index 4509abfa1..9a032b5ba 100644 --- a/test/Util.purs +++ b/test/Util.purs @@ -4,18 +4,16 @@ import Prelude hiding (absurd) import App.Fig (LinkFigSpec) import App.Util (Selector) -import Benchmark.Util (BenchRow(..), GraphRow, TraceRow, bench) +import Benchmark.Util (BenchRow, bench, divRow) import Control.Monad.Error.Class (class MonadError, class MonadThrow) -import Data.Foldable (foldl) -import Data.Int (toNumber) +import Control.Monad.Writer.Class (class MonadWriter) +import Control.Monad.Writer.Trans (runWriterT) import Data.List (elem) -import Data.List.Lazy (List, length, replicateM) --- import Data.Lazy (defer) +import Data.List.Lazy (replicateM) import Data.Set (subset) import Data.String (null) import DataType (dataTypeFor, typeName) import Desug (desugGC) -import Effect.Aff (Aff) import Effect.Aff.Class (class MonadAff) import Effect.Class.Console (log) import Effect.Exception (Error) @@ -26,7 +24,6 @@ import GaloisConnection (GaloisConnection(..)) import Graph (Vertex, selectαs, select𝔹s, sinks, vertices) import Graph.GraphImpl (GraphImpl) import Graph.Slice (bwdSliceDual, fwdSliceDual, fwdSliceDeMorgan) as G -import Heterogeneous.Mapping (hmap) import Lattice (Raw, botOf, erase) import Module (File, initialConfig, open, parse) import Parse (program) @@ -42,21 +39,24 @@ type TestConfig = , bwd_expect :: String } +type AffError m a = MonadAff m => MonadError Error m => m a + logging :: Boolean logging = false -test ∷ File -> ProgCxt Unit -> TestConfig -> (Int × Boolean) -> Aff BenchRow +test ∷ forall m. File -> ProgCxt Unit -> TestConfig -> (Int × Boolean) -> AffError m BenchRow test file progCxt tconfig (n × is_bench) = do gconfig <- initialConfig progCxt s <- open file testPretty s - rows <- replicateM n $ do - trRow <- testTrace s gconfig.γ tconfig - grRow <- testGraph s gconfig tconfig is_bench - pure $ BenchRow trRow grRow - pure $ averageRows rows - -testPretty :: forall m a. MonadAff m => MonadError Error m => Ann a => SE.Expr a -> m Unit + _ × row_accum <- runWriterT + ( replicateM n $ do + testTrace s gconfig.γ tconfig + testGraph s gconfig tconfig is_bench + ) + pure $ row_accum `divRow` n + +testPretty :: forall m a. Ann a => SE.Expr a -> AffError m Unit testPretty s = do let src = prettyP s s' <- parse src program @@ -65,7 +65,7 @@ testPretty s = do log ("NEW\n" <> show (erase s')) fail "not equal" -testTrace :: forall m. MonadAff m => MonadError Error m => Raw SE.Expr -> Env Vertex -> TestConfig -> m TraceRow +testTrace :: forall m. MonadWriter BenchRow m => Raw SE.Expr -> Env Vertex -> TestConfig -> AffError m Unit testTrace s γ { δv, bwd_expect, fwd_expect } = do -- | Desugaring Galois connections for Unit and Boolean type selections GC desug <- desugGC s @@ -73,18 +73,18 @@ testTrace s γ { δv, bwd_expect, fwd_expect } = do -- | Eval let e = desug.fwd s - { gc: GC eval, v } × t_eval <- bench $ \_ -> + { gc: GC eval, v } <- bench "Trace-Eval" $ \_ -> traceGC (erase <$> γ) e -- | Backward - (γ𝔹 × e𝔹) × t_bwd <- bench $ \_ -> do + (γ𝔹 × e𝔹) <- bench "Trace-Bwd" $ \_ -> do let γ𝔹 × e𝔹 × _ = eval.bwd (δv (botOf v)) pure (γ𝔹 × e𝔹) let s𝔹 = desug𝔹.bwd e𝔹 -- | Forward (round-tripping) let e𝔹' = desug𝔹.fwd s𝔹 - v𝔹 × t_fwd <- bench $ \_ -> do + v𝔹 <- bench "Trace-Fwd" $ \_ -> do pure (eval.fwd (γ𝔹 × e𝔹' × top)) -- | Check backward selections @@ -95,9 +95,7 @@ testTrace s γ { δv, bwd_expect, fwd_expect } = do when logging $ log (prettyP v𝔹) checkPretty "Trace-based value" fwd_expect v𝔹 - pure { tEval: t_eval, tBwd: t_bwd, tFwd: t_fwd } - -testGraph :: forall m. MonadAff m => MonadError Error m => Raw SE.Expr -> GraphConfig GraphImpl -> TestConfig -> Boolean -> m GraphRow +testGraph :: forall m. MonadWriter BenchRow m => Raw SE.Expr -> GraphConfig GraphImpl -> TestConfig -> Boolean -> AffError m Unit testGraph s gconfig { δv, bwd_expect, fwd_expect } is_bench = do -- | Desugaring Galois connections for Unit and Boolean type selections GC desug <- desugGC s @@ -105,11 +103,11 @@ testGraph s gconfig { δv, bwd_expect, fwd_expect } is_bench = do -- | Eval let e = desug.fwd s - { gc: GC eval, eα, g, vα } × t_eval <- bench $ \_ -> + { gc: GC eval, eα, g, vα } <- bench "Graph-Eval" $ \_ -> graphGC gconfig e -- | Backward - (e𝔹 × αs_out × αs_in) × t_bwd <- bench $ \_ -> do + (e𝔹 × αs_out × αs_in) <- bench "Graph-Bwd" $ \_ -> do let αs_out = selectαs (δv (botOf vα)) vα αs_in = eval.bwd αs_out @@ -117,7 +115,7 @@ testGraph s gconfig { δv, bwd_expect, fwd_expect } is_bench = do let s𝔹 = desug𝔹.bwd e𝔹 -- | Forward (round-tripping) - (v𝔹 × αs_out') × t_fwd <- bench $ \_ -> do + (v𝔹 × αs_out') <- bench "Graph-Fwd" $ \_ -> do let αs_out' = eval.fwd αs_in pure (select𝔹s vα αs_out' × αs_out') @@ -131,27 +129,9 @@ testGraph s gconfig { δv, bwd_expect, fwd_expect } is_bench = do αs_out `shouldSatisfy "fwd ⚬ bwd round-tripping property"` (flip subset αs_out') - let - benchmarks = - { tEval: t_eval - , tBwd: t_bwd - , tFwd: t_fwd - , tBwdDual: 0.0 - , tBwdAll: 0.0 - , tFwdDual: 0.0 - , tFwdAsDemorgan: 0.0 - } - - if not is_bench then pure benchmarks - else do - -- | Forward (round-tripping) using De Morgan dual - v𝔹_demorgan × t_fwdAsDeMorgan <- bench $ \_ -> do - let - gfwd_demorgan = G.fwdSliceDeMorgan αs_in g - pure (select𝔹s vα (vertices gfwd_demorgan) <#> not) - + unless (not is_bench) do -- | De Morgan dual of backward - e𝔹_dual × t_bwdDual <- bench $ \_ -> do + e𝔹_dual <- bench "Graph-BwdDual" $ \_ -> do let αs_out_dual = selectαs (δv (botOf vα)) vα gbwd_dual = G.bwdSliceDual αs_out_dual g @@ -159,15 +139,21 @@ testGraph s gconfig { δv, bwd_expect, fwd_expect } is_bench = do pure (select𝔹s eα αs_in_dual) -- | Backward (all outputs selected) - e𝔹_all × t_bwdAll <- bench $ \_ -> do + e𝔹_all <- bench "Graph-BwdAll" $ \_ -> do pure (select𝔹s eα $ eval.bwd (vertices vα)) -- | De Morgan dual of forward - v𝔹_dual × t_fwdDual <- bench $ \_ -> do + v𝔹_dual <- bench "Graph-FwdDual" $ \_ -> do let gfwd_dual = G.fwdSliceDual αs_in g pure (select𝔹s vα (vertices gfwd_dual)) + -- | Forward (round-tripping) using De Morgan dual + v𝔹_demorgan <- bench "Graph-FwdAsDeMorgan" $ \_ -> do + let + gfwd_demorgan = G.fwdSliceDeMorgan αs_in g + pure (select𝔹s vα (vertices gfwd_demorgan) <#> not) + -- | To avoid unused variables when benchmarking when logging do log (prettyP v𝔹_demorgan) @@ -175,9 +161,6 @@ testGraph s gconfig { δv, bwd_expect, fwd_expect } is_bench = do log (prettyP e𝔹_all) log (prettyP v𝔹_dual) - pure $ benchmarks - { tBwdDual = t_bwdDual, tBwdAll = t_bwdAll, tFwdDual = t_fwdDual, tFwdAsDemorgan = t_fwdAsDeMorgan } - type TestSpec = { file :: String , fwd_expect :: String @@ -216,9 +199,3 @@ shouldSatisfy :: forall m t. MonadThrow Error m => Show t => String -> t -> (t - shouldSatisfy msg v pred = unless (pred v) $ fail (show v <> " doesn't satisfy predicate: " <> msg) - -averageRows :: List BenchRow -> BenchRow -averageRows rows = average $ foldl (<>) mempty rows - where - runs = toNumber $ length rows - average (BenchRow tr gr) = BenchRow (hmap (_ `div` runs) tr) (hmap (_ `div` runs) gr)