Merge pull request #762 from explorable-viz/benchmarking

Benchmarking

Benchmarks/benchmarks.csv

@@ -1,64 +1,64 @@
-Test-Name, Trace-Eval, Trace-Bwd, Trace-Fwd, Graph-Eval, Graph-Bwd, Graph-Fwd, Graph-FwdDeMorgan, Graph-BwdAll
-arithmetic,1.4,19.2,1.2,11.4,0.2,0.0,8.0,0.2
-array,2.6,18.6,2.4,14.8,0.2,0.0,9.2,0.2
-compose,0.8,15.8,0.8,10.8,0.0,0.2,2.0,0.2
-dicts,1.2,18.8,1.2,10.2,0.6,0.0,2.2,0.4
-div-mod-quot-rem,0.8,18.6,1.2,17.8,0.0,0.0,5.2,6.2
-factorial,7.2,18.4,4.2,25.8,0.0,0.2,112.0,29.0
-filter,4.8,18.8,5.8,13.0,0.2,0.0,4.0,0.4
-first-class-constr,9.0,17.6,7.4,16.8,0.2,0.0,5.8,0.8
-flatten,4.6,18.8,4.8,17.8,0.0,0.0,5.2,0.4
-foldr_sumSquares,2.6,17.8,2.2,10.2,0.0,0.0,2.8,0.0
-lexicalScoping,0.6,16.2,1.2,2.8,0.2,0.0,2.8,0.0
-length,1.4,15.4,0.6,4.0,0.0,0.2,1.8,0.2
-lookup,3.8,14.8,4.0,10.0,0.0,0.0,3.6,0.4
-map,3.0,19.0,2.6,11.0,0.2,0.2,3.6,0.6
-mergeSort,12.8,23.2,18.4,22.2,0.0,0.0,4.0,1.0
-normalise,0.8,18.4,0.8,5.6,0.0,0.0,3.4,0.2
-pattern-match,4.2,16.2,4.4,13.2,0.4,0.0,4.6,0.4
-range,12.8,24.0,13.2,39.6,0.0,0.0,9.4,2.2
-records,4.6,21.4,6.4,14.6,0.2,0.0,4.4,0.8
-reverse,1.4,19.4,2.8,7.4,0.0,0.0,4.8,0.2
-desugar/list-comp-1,19.2,29.4,17.0,55.0,0.4,0.0,13.0,1.2
-desugar/list-comp-2,53.8,52.6,49.6,180.4,0.0,0.0,27.8,11.0
-desugar/list-comp-3,15.4,23.6,15.0,44.4,0.0,0.0,9.2,0.4
-desugar/list-comp-4,5.8,17.6,9.2,17.4,0.2,0.0,3.4,0.8
-desugar/list-comp-5,4.4,16.4,5.0,12.4,0.2,0.0,2.8,0.2
-desugar/list-comp-6,0.2,15.2,2.8,4.2,0.2,0.0,4.6,0.0
-desugar/list-comp-7,7.0,16.4,4.6,13.6,0.0,0.0,5.0,0.0
-desugar/list-enum,3.8,17.2,4.6,12.2,0.0,0.0,10.2,4.4
-add,0.0,19.4,0.0,2.2,0.0,0.4,3.0,0.0
-array/lookup,20.8,23.8,19.4,61.2,0.0,0.2,16.0,0.2
-array/dims,2.4,18.4,1.6,5.2,0.4,1.0,2.8,0.2
-convolution/edgeDetect,1543.2,1075.6,1574.8,5635.2,0.4,6.8,3557.4,14.2
-convolution/emboss,1389.6,884.4,1323.4,4624.6,0.2,1.2,3251.8,11.6
-convolution/gaussian,1382.8,905.2,1372.6,4729.8,0.4,1.6,3298.0,14.4
-dict/create,1.0,23.8,0.4,4.8,0.4,0.4,8.4,0.8
-dict/difference,0.8,17.2,1.0,5.2,0.2,0.4,3.2,0.0
-dict/disjointUnion,0.8,14.4,0.0,2.8,0.0,0.2,3.6,0.4
-dict/foldl,6.8,20.0,5.6,16.2,0.4,0.2,2.8,0.0
-dict/intersectionWith,1.0,22.2,1.2,7.6,0.0,0.8,2.6,0.0
-dict/fromRecord,0.2,15.2,0.2,0.8,0.2,0.2,2.8,0.0
-dict/get,0.8,15.2,0.4,2.2,0.2,0.0,4.4,0.0
-dict/map,2.8,19.6,3.0,10.2,0.0,0.2,2.6,0.2
-divide,0.0,14.6,0.4,1.8,0.0,0.2,4.6,0.2
-filter,2.6,14.2,2.4,9.0,0.0,0.0,2.0,0.4
-intersperse,2.6,16.4,1.4,7.4,0.2,0.2,1.6,0.4
-intersperse,3.0,16.2,5.2,4.0,0.4,0.4,2.2,0.8
-length,1.0,14.6,1.0,6.2,0.0,0.8,2.4,0.4
-list-comp,31.8,42.8,28.8,87.8,0.4,2.4,7.0,0.8
-list-comp,24.4,31.4,26.0,79.2,0.6,1.0,7.0,0.6
-lookup,3.2,16.4,4.4,10.8,0.8,0.8,2.4,0.2
-map,5.4,21.2,3.8,11.4,0.6,0.8,5.2,0.4
-multiply,0.2,15.4,0.2,2.4,0.0,0.2,3.2,0.4
-nth,1.6,16.6,1.4,4.8,0.0,0.0,3.2,0.0
-section-5-example,26.0,39.0,23.4,91.4,0.6,1.4,9.2,0.8
-section-5-example,21.4,31.8,23.2,94.6,0.4,0.4,14.2,1.6
-section-5-example,23.8,28.4,20.8,88.2,0.2,0.8,10.0,0.4
-zeros,1.0,15.8,0.6,4.0,0.8,0.0,2.0,0.0
-zeros,0.4,14.2,0.8,2.2,0.0,0.4,4.2,0.0
-zipWith,2.8,14.6,2.4,15.0,0.2,0.2,4.2,0.4
-graphics/background,1.4,15.2,1.6,10.2,0.0,0.0,23.6,2.2
-graphics/grouped-bar-chart,158.6,114.8,135.6,481.4,0.4,0.2,2806.4,1138.0
-graphics/line-chart,210.0,147.4,191.2,820.6,1.4,0.2,4232.4,2574.6
-graphics/stacked-bar-chart,87.4,75.0,73.4,281.0,1.6,0.2,197.8,308.0
+Test-Name, Trace-Eval, Trace-Bwd, Trace-Fwd, Graph-Eval, Graph-Bwd, Graph-BwdDual, Graph-BwdAll, Graph-Fwd, Graph-FwdDual, Graph-FwdAsDeMorgan
+arithmetic,1.6,20.6,1.6,3.2,0.2,0.0,0.2,0.2,0.0,7.0
+array,3.0,15.4,4.0,6.8,0.2,0.2,0.2,0.0,0.0,3.0
+compose,0.4,19.4,0.4,2.6,0.0,0.0,0.0,0.0,0.0,3.6
+dicts,1.8,13.6,1.4,5.2,0.4,0.2,0.6,0.0,0.0,3.0
+div-mod-quot-rem,2.0,16.4,1.2,9.6,0.2,0.2,2.4,0.0,0.0,3.0
+factorial,6.2,22.4,5.6,9.2,0.0,0.0,23.2,0.0,0.0,109.6
+filter,2.4,16.8,4.0,6.6,0.0,0.4,0.0,0.0,0.0,2.2
+first-class-constr,2.0,16.0,1.8,7.2,0.0,0.0,0.4,0.0,0.0,2.8
+flatten,5.4,16.6,10.6,10.4,0.2,0.4,1.2,0.0,0.0,5.6
+foldr_sumSquares,2.6,18.8,2.4,9.2,0.0,0.0,0.0,0.0,0.0,2.6
+lexicalScoping,2.6,15.8,1.4,4.2,0.2,0.0,0.2,0.0,0.0,6.0
+length,1.4,13.8,1.2,2.8,0.2,0.0,0.2,0.0,0.2,3.2
+lookup,3.8,18.6,5.0,11.4,0.4,0.0,0.2,0.0,0.0,3.2
+map,2.8,17.6,2.6,7.6,0.0,0.0,1.0,0.0,0.0,2.4
+mergeSort,16.4,24.2,16.8,24.0,0.0,0.0,1.2,0.0,0.0,5.2
+normalise,2.2,20.2,1.6,10.0,0.4,0.0,1.8,0.0,0.2,5.6
+pattern-match,4.0,18.2,8.0,11.2,0.0,0.0,0.8,0.0,0.0,3.8
+range,17.4,30.4,24.0,23.0,0.0,0.0,1.0,0.0,0.0,5.8
+records,5.0,17.8,5.0,9.2,0.0,0.0,0.6,0.0,0.0,3.2
+reverse,1.2,17.6,1.4,5.2,0.0,0.0,0.4,0.0,0.0,3.0
+desugar/list-comp-1,17.2,26.8,16.2,27.6,0.0,0.0,1.4,0.0,0.0,5.8
+desugar/list-comp-2,62.4,71.0,50.4,94.6,0.2,0.0,8.4,0.0,0.0,18.4
+desugar/list-comp-3,14.2,24.0,13.0,26.4,0.2,0.2,0.4,0.0,0.0,5.0
+desugar/list-comp-4,6.0,15.0,4.6,9.8,0.2,0.0,0.2,0.2,0.0,3.0
+desugar/list-comp-5,3.4,14.0,2.8,8.2,0.0,0.0,0.8,0.0,0.2,2.2
+desugar/list-comp-6,0.0,13.4,3.2,1.2,0.0,0.0,0.2,0.0,0.0,2.4
+desugar/list-comp-7,5.2,14.0,4.0,8.2,0.0,0.0,0.2,0.0,0.2,1.8
+desugar/list-enum,2.6,16.0,2.4,7.4,0.0,0.2,4.4,0.0,0.0,5.8
+add,0.0,13.2,0.0,1.8,0.0,0.0,0.0,0.2,0.2,3.0
+array/lookup,17.8,22.8,19.0,35.4,0.0,0.2,0.0,0.0,0.0,11.4
+array/dims,1.6,15.2,1.8,2.6,0.4,0.0,0.2,0.0,0.2,3.4
+convolution/edgeDetect,1545.2,1143.8,1560.8,3243.8,0.6,0.6,10.6,6.6,311.4,2868.0
+convolution/emboss,1348.2,905.4,1313.0,2625.0,1.0,0.6,9.0,2.2,354.0,2566.0
+convolution/gaussian,1247.2,921.6,1325.8,2525.4,0.2,0.2,10.0,1.8,313.4,2486.0
+dict/create,0.6,14.8,0.0,3.0,0.4,0.4,0.6,0.2,0.0,5.0
+dict/difference,1.4,13.6,0.4,3.0,0.0,0.0,0.2,0.2,0.2,3.2
+dict/disjointUnion,0.2,12.4,0.4,2.2,0.2,0.2,0.0,0.0,0.0,3.0
+dict/foldl,6.4,24.2,7.4,11.8,0.0,0.0,0.0,0.0,0.0,3.0
+dict/intersectionWith,1.2,13.2,1.0,2.6,0.0,0.2,0.0,0.2,0.0,2.6
+dict/fromRecord,0.4,16.0,0.0,0.8,0.2,0.0,0.4,0.0,0.0,2.4
+dict/get,0.4,15.6,0.6,1.6,0.0,0.2,0.0,0.0,0.0,3.4
+dict/map,3.6,19.0,3.2,7.6,0.2,0.2,0.0,0.0,0.2,2.6
+divide,0.0,14.0,0.2,1.4,0.2,0.4,0.0,0.0,0.0,2.6
+filter,4.4,17.6,2.6,6.4,0.0,0.0,0.8,0.0,0.0,2.2
+intersperse,1.6,19.2,1.8,10.8,0.2,0.0,0.8,0.4,0.2,2.2
+intersperse,1.2,18.6,1.6,8.6,0.2,0.0,0.6,0.4,0.8,2.0
+length,1.2,17.6,2.4,4.4,0.2,2.0,0.4,0.0,0.6,1.8
+list-comp,25.0,31.8,23.8,41.8,0.0,0.0,0.6,1.4,2.2,5.4
+list-comp,23.4,27.8,23.4,37.2,0.2,0.0,0.6,1.8,1.4,6.0
+lookup,7.2,17.0,5.8,10.6,0.4,0.6,0.4,0.8,0.6,4.0
+map,1.2,16.8,2.2,3.4,0.0,0.2,0.2,0.0,0.2,3.2
+multiply,0.0,12.4,0.0,3.2,0.0,1.4,0.0,0.0,0.0,1.8
+nth,3.0,16.0,2.4,4.8,0.4,0.2,0.0,0.0,0.0,2.4
+section-5-example,30.4,36.4,26.6,58.4,0.0,0.6,0.4,1.4,1.4,6.2
+section-5-example,24.2,29.8,23.0,46.4,0.4,0.0,0.6,0.2,0.4,7.4
+section-5-example,25.4,28.6,27.0,51.2,0.0,0.2,0.8,0.4,0.6,7.6
+zeros,0.6,15.2,1.6,2.8,0.6,0.0,0.0,0.0,0.4,2.2
+zeros,0.8,16.0,3.0,4.2,0.2,0.0,0.0,0.2,0.0,3.4
+zipWith,4.0,15.8,4.4,7.8,0.0,0.2,0.4,0.2,0.4,2.2
+graphics/background,1.8,15.4,1.6,7.2,0.0,0.0,1.8,0.0,0.0,12.8
+graphics/grouped-bar-chart,145.2,126.0,148.8,255.2,0.8,0.2,775.0,0.0,0.4,2186.4
+graphics/line-chart,201.8,146.2,193.8,393.0,1.6,0.2,1728.0,0.0,0.2,2987.2
+graphics/stacked-bar-chart,81.6,67.0,77.2,159.0,0.2,0.6,208.2,0.0,0.4,142.0

plot_bench.py

@@ -12,11 +12,13 @@ def parse():
   print(df)
 
   # Reorder benchmark columns
-  column_order = ['Trace-Eval', 'Graph-Eval', 'Trace-Bwd', 'Graph-Bwd', 'Trace-Fwd', 'Graph-Fwd', 'Graph-FwdDeMorgan', 'Graph-BwdAll']
-  column_colors = ['#0d6b12', '#93dbb5', '#3e3875', '#b8bef5', '#910303', '#e84d4d', '#e8bceb', '#5073a1']
+  column_order = ['Trace-Eval', 'Trace-Bwd', 'Trace-Fwd', 'Graph-Eval', 'Graph-Bwd', 'Graph-BwdDual', 'Graph-BwdAll', 'Graph-Fwd', 'Graph-FwdDual', 'Graph-FwdAsDeMorgan']
+  # column_colors = ['#0d6b12', '#93dbb5', '#3e3875', '#b8bef5', '#910303', '#e84d4d', '#e8bceb', '#5073a1']
 
   # Plot as bar chart
-  df[column_order].plot(kind="bar", color=column_colors, ylabel="Milliseconds", rot=0)
+  df[column_order].plot(  kind="bar"
+                        # , color=column_colors
+                        , ylabel="Milliseconds", rot=0)
 
   # Inserting a coloured horizontal line just to make clearer which columns have zero values
   plt.ylim(bottom=-10)

src/Graph/Slice.purs

@@ -17,6 +17,7 @@ import Util (type (×), (×))
 
 type PendingVertices = Map Vertex (Set Vertex)
 
+-- | Backward slicing (◁_G)
 bwdSlice :: forall g. Graph g => Set Vertex -> g -> g
 bwdSlice αs0 g0 = fst $ runWithGraph $ tailRecM go (empty × L.fromFoldable αs0)
    where
@@ -27,10 +28,11 @@ bwdSlice αs0 g0 = fst $ runWithGraph $ tailRecM go (empty × L.fromFoldable αs
       extend α βs
       pure $ Loop ((visited # insert α) × (L.fromFoldable βs <> αs))
 
-fwdSliceDeMorgan :: forall g. Graph g => Set Vertex -> g -> g
-fwdSliceDeMorgan αs_0 g_0 =
-   bwdSlice (sinks g_0 `difference` αs_0) (op g_0)
+-- | De Morgan dual of backward slicing (◁_G)° ≡ Forward slicing on the opposite graph (▷_{G_op})
+bwdSliceDual :: forall g. Graph g => Set Vertex -> g -> g
+bwdSliceDual αs0 g0 = fwdSlice αs0 (op g0)
 
+-- | Forward slicing (▷_G)
 fwdSlice :: forall g. Graph g => Set Vertex -> g -> g
 fwdSlice αs0 g0 = fst $ runWithGraph $ tailRecM go (M.empty × inEdges g0 αs0)
    where
@@ -44,6 +46,15 @@ fwdSlice αs0 g0 = fst $ runWithGraph $ tailRecM go (M.empty × inEdges g0 αs0)
       else
          pure $ Loop (M.insert α βs h × es)
 
+-- | De Morgan dual of forward slicing (▷_G)° ≡ Backward slicing on the opposite graph (◁_{G_op})
+fwdSliceDual :: forall g. Graph g => Set Vertex -> g -> g
+fwdSliceDual αs0 g0 = bwdSlice αs0 (op g0)
+
+-- | Forward slicing (▷_G) ≡ De Morgan dual of backward slicing on the opposite graph (◁_{G_op})°
+fwdSliceAsDeMorgan :: forall g. Graph g => Set Vertex -> g -> g
+fwdSliceAsDeMorgan αs0 g0 =
+   bwdSlice (sinks g0 `difference` αs0) (op g0)
+
 vertices :: forall f. Apply f => Foldable f => f Vertex -> Set Vertex
 vertices vα = selectαs (const true <$> vα) vα
 

test/Benchmark/Util.purs

@@ -39,29 +39,54 @@ type TraceRow =
 type GraphRow =
    { tEval :: Number
    , tBwd :: Number
-   , tFwd :: Number
-   , tFwdDemorgan :: Number
+   , tBwdDual :: Number
    , tBwdAll :: Number
+   , tFwd :: Number
+   , tFwdDual :: Number
+   , tFwdAsDemorgan :: Number
    }
 
 instance Show BenchAcc where
    show (BenchAcc rows) =
-      "Test-Name, Trace-Eval, Trace-Bwd, Trace-Fwd, Graph-Eval, Graph-Bwd, Graph-Fwd, Graph-FwdDeMorgan, Graph-BwdAll\n"
+      "Test-Name, Trace-Eval, Trace-Bwd, Trace-Fwd, Graph-Eval, Graph-Bwd, Graph-BwdDual, Graph-BwdAll, Graph-Fwd, Graph-FwdDual, Graph-FwdAsDeMorgan\n"
          <> (fold $ intersperse "\n" (map rowShow rows))
 
 rowShow :: String × BenchRow -> String
 rowShow (str × row) = str <> "," <> show row
 
+zeroRow :: BenchRow
+zeroRow =
+   BenchRow { tEval: 0.0, tBwd: 0.0, tFwd: 0.0 }
+      { tEval: 0.0, tBwd: 0.0, tBwdDual: 0.0, tBwdAll: 0.0, tFwd: 0.0, tFwdDual: 0.0, tFwdAsDemorgan: 0.0 }
+
+sumRow :: BenchRow -> BenchRow -> BenchRow
+sumRow (BenchRow trRow1 gRow1) (BenchRow trRow2 gRow2) =
+   BenchRow
+      { tEval: trRow1.tEval + trRow2.tEval
+      , tBwd: trRow1.tBwd + trRow2.tBwd
+      , tFwd: trRow1.tFwd + trRow2.tFwd
+      }
+      { tEval: gRow1.tEval + gRow2.tEval
+      , tBwd: gRow1.tBwd + gRow2.tBwd
+      , tBwdDual: gRow1.tBwdDual + gRow2.tBwdDual
+      , tBwdAll: gRow1.tBwdAll + gRow2.tBwdAll
+      , tFwd: gRow1.tFwd + gRow2.tFwd
+      , tFwdDual: gRow1.tFwdDual + gRow2.tFwdDual
+      , tFwdAsDemorgan: gRow1.tFwdAsDemorgan + gRow2.tFwdAsDemorgan
+      }
+
 instance Show BenchRow where
    show (BenchRow trRow grRow) = fold $ intersperse ","
       [ show trRow.tEval
       , show trRow.tBwd
       , show trRow.tFwd
       , show grRow.tEval
       , show grRow.tBwd
-      , show grRow.tFwd
-      , show grRow.tFwdDemorgan
+      , show grRow.tBwdDual
       , show grRow.tBwdAll
+      , show grRow.tFwd
+      , show grRow.tFwdDual
+      , show grRow.tFwdAsDemorgan
       ]
 
 now :: forall m. MonadEffect m => m JSDate

test/Util.purs

@@ -4,7 +4,7 @@ import Prelude hiding (absurd)
 
 import App.Fig (LinkFigSpec)
 import App.Util (Selector)
-import Benchmark.Util (BenchRow(..), GraphRow, TraceRow, preciseTime, tdiff)
+import Benchmark.Util (BenchRow(..), GraphRow, TraceRow, zeroRow, sumRow, preciseTime, tdiff)
 import Control.Monad.Error.Class (class MonadThrow, liftEither)
 import Control.Monad.Except (runExceptT)
 import Control.Monad.Trans.Class (lift)
@@ -25,7 +25,7 @@ import EvalBwd (evalBwd)
 import EvalGraph (GraphConfig, evalWithConfig)
 import Graph (sinks, sources, vertices)
 import Graph.GraphImpl (GraphImpl)
-import Graph.Slice (bwdSlice, fwdSlice, fwdSliceDeMorgan) as G
+import Graph.Slice (bwdSlice, bwdSliceDual, fwdSlice, fwdSliceDual, fwdSliceAsDeMorgan) as G
 import Graph.Slice (selectαs, select𝔹s)
 import Heterogeneous.Mapping (hmap)
 import Lattice (bot, botOf, topOf, erase, Raw)
@@ -119,6 +119,15 @@ testGraph s gconf { δv, bwd_expect, fwd_expect } = do
    let
       s𝔹 = desugBwd e𝔹 (erase s)
 
+   -- | De Morgan dual of backward
+   tBwdDual1 <- preciseTime
+   let
+      αs_out_dual = selectαs (δv (botOf vα)) vα
+      gbwd_dual = G.bwdSliceDual αs_out_dual g
+      αs_in_dual = sinks gbwd_dual
+      e𝔹_dual = select𝔹s eα αs_in_dual
+   tBwdDual2 <- preciseTime
+
    -- | Backward (all outputs selected)
    tBwdAll1 <- preciseTime
    let
@@ -135,12 +144,19 @@ testGraph s gconf { δv, bwd_expect, fwd_expect } = do
       v𝔹 = select𝔹s vα (vertices gfwd)
    tFwd2 <- preciseTime
 
+   -- | De Morgan dual of forward
+   tFwdDual1 <- preciseTime
+   let
+      gfwd_dual = G.fwdSliceDual αs_in g
+      v𝔹_dual = select𝔹s vα (vertices gfwd_dual)
+   tFwdDual2 <- preciseTime
+
    -- | Forward (round-tripping) using De Morgan dual
-   tFwdDeMorgan1 <- preciseTime
+   tFwdAsDeMorgan1 <- preciseTime
    let
-      gfwd' = G.fwdSliceDeMorgan αs_in g
-      v𝔹' = select𝔹s vα (vertices gfwd') <#> not
-   tFwdDeMorgan2 <- preciseTime
+      gfwd_demorgan = G.fwdSliceAsDeMorgan αs_in g
+      v𝔹_demorgan = select𝔹s vα (vertices gfwd_demorgan) <#> not
+   tFwdAsDeMorgan2 <- preciseTime
 
    lift do
       -- | Check backward selections
@@ -149,15 +165,16 @@ testGraph s gconf { δv, bwd_expect, fwd_expect } = do
       -- | Check round-trip selections
       unless (isGraphical v𝔹) do
          checkPretty "Graph-based value" fwd_expect v𝔹
-         checkPretty "Graph-based value (De Morgan)" fwd_expect v𝔹'
+         checkPretty "Graph-based value (De Morgan)" fwd_expect v𝔹_demorgan
       sources gbwd `shouldSatisfy "fwd ⚬ bwd round-tripping property"`
          (flip subset (sources gfwd))
       -- | To avoid unused variables when benchmarking
       unless false do
-         log ("BwdAll selected nodes: " <> show αs_out_all)
+         log (prettyP e𝔹_dual)
          log (prettyP e𝔹_all)
+         log (prettyP v𝔹_dual)
 
-   pure { tEval: tdiff tEval1 tEval2, tBwd: tdiff tBwd1 tBwd2, tFwd: tdiff tFwd1 tFwd2, tFwdDemorgan: tdiff tFwdDeMorgan1 tFwdDeMorgan2, tBwdAll: tdiff tBwdAll1 tBwdAll2 }
+   pure { tEval: tdiff tEval1 tEval2, tBwd: tdiff tBwd1 tBwd2, tBwdDual: tdiff tBwdDual1 tBwdDual2, tBwdAll: tdiff tBwdAll1 tBwdAll2, tFwd: tdiff tFwd1 tFwd2, tFwdDual: tdiff tFwdDual1 tFwdDual2, tFwdAsDemorgan: tdiff tFwdAsDeMorgan1 tFwdAsDeMorgan2 }
 
 type TestSpec =
    { file :: String
@@ -203,22 +220,5 @@ averageRows rows = averagedTr
    where
    runs = toNumber $ length rows
 
-   zeroRow :: BenchRow
-   zeroRow = BenchRow { tEval: 0.0, tBwd: 0.0, tFwd: 0.0 } { tEval: 0.0, tBwd: 0.0, tFwd: 0.0, tFwdDemorgan: 0.0, tBwdAll: 0.0 }
-
-   sumRow :: BenchRow -> BenchRow -> BenchRow
-   sumRow (BenchRow trRow1 gRow1) (BenchRow trRow2 gRow2) =
-      BenchRow
-         { tEval: trRow1.tEval + trRow2.tEval
-         , tBwd: trRow1.tBwd + trRow2.tBwd
-         , tFwd: trRow1.tFwd + trRow2.tFwd
-         }
-         { tEval: gRow1.tEval + gRow2.tEval
-         , tBwd: gRow1.tBwd + gRow2.tBwd
-         , tFwd: gRow1.tFwd + gRow2.tFwd
-         , tFwdDemorgan: gRow1.tFwdDemorgan + gRow2.tFwdDemorgan
-         , tBwdAll: gRow1.tBwdAll + gRow2.tBwdAll
-         }
-
    summed = foldl sumRow zeroRow rows
    averagedTr = (\(BenchRow tr gr) -> BenchRow (hmap (\num -> num `div` runs) tr) (hmap (\num -> num `div` runs) gr)) $ summed