From 9a78339576a21bfaaed5c660fb33881a98f13aa9 Mon Sep 17 00:00:00 2001 From: min-nguyen Date: Sat, 23 Sep 2023 17:42:22 +0100 Subject: [PATCH] Add benchmarking for BwdAll --- Benchmarks/benchmarks.csv | 128 +++++++++++++++++++------------------- plot_bench.py | 6 +- test/Benchmark.purs | 2 +- test/Benchmark/Util.purs | 9 +-- test/Util.purs | 19 +++++- 5 files changed, 87 insertions(+), 77 deletions(-) diff --git a/Benchmarks/benchmarks.csv b/Benchmarks/benchmarks.csv index 8174fc38e..3a84664b9 100644 --- a/Benchmarks/benchmarks.csv +++ b/Benchmarks/benchmarks.csv @@ -1,64 +1,64 @@ -Test-Name, Trace-Eval, Trace-Bwd, Trace-Fwd, Graph-Eval, Graph-Bwd, Graph-Fwd, Graph-FwdDeMorgan -arithmetic,1.0200000286102295,9.889999961853027,0.5799999713897706,5.210000038146973,0.10000002384185791,0.06999998092651367,5.080000019073486 -array,3.039999985694885,8.610000061988831,1.1900000095367431,6.409999990463257,0.06000001430511474,0.010000014305114746,4.760000014305115 -compose,0.38000001907348635,8.089999985694885,0.6399999856948853,2.589999961853027,0.05000004768371582,0.04000000953674317,3.7799999713897705 -dicts,0.8599999904632568,7.600000047683716,1.9,5.339999985694885,0.2700000047683716,0.03000004291534424,3.0899999380111693 -div-mod-quot-rem,0.9299999713897705,7.419999980926514,1.590000033378601,9.280000066757202,0.13999996185302735,0.04000003337860107,3.4 -factorial,2.949999976158142,8.330000019073486,2.8500000476837157,12.27000002861023,0.009999990463256836,0.030000019073486327,72.85000004768372 -filter,2.1,8.330000042915344,2.6100000381469726,6.740000009536743,0.02999999523162842,0.010000014305114746,1.709999966621399 -first-class-constr,1.6199999809265138,10.180000019073486,2.2299999952316285,7.8599999904632565,0.04999997615814209,0.020000004768371583,3.1700000286102297 -flatten,3.1999999284744263,9.32000002861023,4.529999995231629,8.360000014305115,0.10999996662139892,0.009999990463256836,3.6299999475479128 -foldr_sumSquares,1.9799999952316285,7.110000014305115,2.3600000143051147,8.030000019073487,0.04999997615814209,0.020000004768371583,2.960000014305115 -lexicalScoping,2.4699999570846556,9.830000019073486,0.8599999666213989,2.9199999809265136,0.05,0.020000004768371583,4.640000033378601 -length,0.8100000143051147,8.690000009536742,0.7600000143051148,2.4799999713897707,0.07999999523162842,0.009999990463256836,5.6000000715255736 -lookup,2.15,9.119999980926513,3.2200000047683717,9.509999990463257,0.08000001907348633,0.020000004768371583,3.3199999809265135 -map,2.219999980926514,8.3,2.4800000190734863,7.989999985694885,0.09999997615814209,0.010000014305114746,3.7700000286102293 -mergeSort,7.790000057220459,7.2999999761581424,9.479999995231628,14.490000033378601,0.05,0.020000004768371583,2.9199999809265136 -normalise,0.5900000333786011,9.170000004768372,0.8999999284744262,5.3,0.039999985694885255,0.02999999523162842,3.4899999380111693 -pattern-match,3.259999966621399,6.95,3.0899999856948854,9.769999980926514,0.11000001430511475,0.05,4.35 -range,7.5499999761581424,9.990000033378601,7.9500000238418576,23.060000038146974,0.07999994754791259,0.029999971389770508,4.570000028610229 -records,2.999999976158142,8.659999966621399,3.8100000381469727,11.660000014305115,0.10999996662139892,0.04999997615814209,2.7399999856948853 -reverse,1.0900000095367433,7.769999957084655,2.400000047683716,4.259999942779541,0.0700000286102295,0.019999980926513672,4.62000002861023 -desugar/list-comp-1,10.319999957084656,13.090000033378601,11.769999980926514,29.70999994277954,0.03000004291534424,0.0,3.9400000095367433 -desugar/list-comp-2,30.269999957084657,31.59999997615814,28.689999985694886,77.7799999475479,0.08000006675720214,0.009999990463256836,10.7900000333786 -desugar/list-comp-3,6.620000004768372,11.600000047683716,7.150000023841858,25.980000019073486,0.04999997615814209,0.009999990463256836,3.9200000286102297 -desugar/list-comp-4,4.85,10.060000014305114,4.949999976158142,14.040000009536744,0.07999999523162842,0.039999985694885255,3.0900000095367433 -desugar/list-comp-5,3.349999952316284,8.210000014305114,4.3599999904632565,11.430000066757202,0.039999961853027344,0.009999990463256836,2.780000019073486 -desugar/list-comp-6,0.2800000190734863,9.179999947547913,0.1799999952316284,1.3299999952316284,0.05,0.0,4.580000042915344 -desugar/list-comp-7,3.0999999761581423,7.6800000190734865,3.7399999856948853,13.020000004768372,0.05999996662139893,0.009999990463256836,3.0000000476837156 -desugar/list-enum,2.519999957084656,7.169999980926514,3.1899999618530273,9.830000042915344,0.06999998092651367,0.019999980926513672,7.370000004768372 -add,0.15,8.890000009536744,0.209999942779541,1.8100000143051147,0.14999995231628419,0.25,3.299999976158142 -array/lookup,13.680000019073486,14.079999995231628,13.400000023841859,35.210000014305116,0.029999971389770508,0.0700000524520874,9.069999957084656 -array/dims,1.0599999904632569,7.769999980926514,1.109999990463257,3.0299999952316283,0.21000001430511475,0.2800000190734863,3.1799999713897704 -convolution/edgeDetect,775.6900000572205,497.75000002384184,680.8000000238419,2481.520000052452,0.47000000476837156,3.5200000047683715,1471.329999947548 -convolution/emboss,698.8,417.3700000047684,586.7700000524521,2058.8299999952314,0.2499999761581421,1.2200000047683717,1388.110000038147 -convolution/gaussian,676.3699999809265,422.510000038147,592.9799999713898,2059.100000023842,0.25999999046325684,1.2899999618530273,1393.2500000476837 -dict/create,0.13000001907348632,9.399999976158142,0.18000001907348634,1.0799999952316284,0.15999999046325683,0.36000001430511475,3.4899999380111693 -dict/difference,0.5099999904632568,10.02999997138977,0.4200000286102295,1.5300000190734864,0.14000003337860106,0.17000000476837157,3.2399999380111693 -dict/disjointUnion,0.24000000953674316,8.149999976158142,0.1499999761581421,1.3600000143051147,0.1799999713897705,0.08000001907348633,2.890000009536743 -dict/foldl,3.3299999952316286,9.970000004768371,4.640000033378601,12.160000014305115,0.1,0.12000002861022949,2.659999966621399 -dict/intersectionWith,1.0299999952316283,9.650000047683715,0.990000057220459,3.7899999618530273,0.15999999046325683,0.21000001430511475,2.3 -dict/fromRecord,0.07999999523162842,8.409999990463257,0.13000004291534423,1.099999976158142,0.3999999761581421,0.16000003814697267,2.7399999618530275 -dict/get,0.3799999952316284,8.170000004768372,0.35000004768371584,2.9400000095367433,0.05999999046325684,0.1099999189376831,2.4100000143051146 -dict/map,1.3100000381469727,7.880000019073487,1.1500000476837158,5.4700000286102295,0.11999998092651368,0.09000003337860107,2.1899999618530273 -divide,0.09999997615814209,6.970000052452088,0.07999999523162842,0.5400000095367432,0.04000000953674317,0.12000002861022949,1.9399999856948853 -filter,2.2799999713897705,7.589999985694885,1.7699999332427978,7.140000033378601,0.16999998092651367,0.19000000953674318,2.35 -intersperse,1.4799999952316285,7.5,1.6599999904632567,4.439999961853028,0.1299999713897705,0.18999998569488524,2.7899999618530273 -intersperse,1.05,9.230000019073486,1.5199999809265137,4.36000006198883,0.1299999713897705,0.3000000238418579,2.1600000143051146 -length,1.4600000143051148,8.439999985694886,1.7699999570846559,5.250000071525574,0.4099999666213989,0.5699999809265137,2.5799999952316286 -list-comp,16.65000002384186,17.7599999666214,14.830000042915344,35.58000001907349,0.18000001907348634,0.8300000190734863,3.5899999380111693 -list-comp,14.299999976158142,17.15,15.7900000333786,34.48999998569489,0.15000004768371583,0.8299999952316284,3.7200000047683717 -lookup,2.7700000047683715,7.219999957084656,3.1299999475479128,9.47000002861023,0.27999999523162844,0.6099999666213989,4.230000019073486 -map,0.849999976158142,7.470000004768371,1.2699999809265137,3.7600000143051147,0.2,0.30999996662139895,3.4200000524520875 -multiply,0.12999999523162842,6.139999985694885,0.13000001907348632,1.0200000047683715,0.13000001907348632,0.11000001430511475,3.3400000095367433 -nth,1.4699999570846558,8.850000047683716,1.6299999475479126,3.640000033378601,0.07999992370605469,0.14000003337860106,3.290000009536743 -section-5-example,17.5,20.840000057220458,16.309999918937685,42.45,0.1799999713897705,1.0800000190734864,4.519999933242798 -section-5-example,18.780000019073487,21.12999999523163,14.479999995231628,40.02999999523163,0.20000002384185792,0.4900000333786011,5.639999985694885 -section-5-example,15.420000028610229,20.690000009536742,14.87000002861023,39.93999998569488,0.16999995708465576,0.5099999666213989,5.100000047683716 -zeros,1.390000057220459,8.039999985694886,1.2299999952316285,3.0699999809265135,0.19000003337860108,0.2799999475479126,2.8500000476837157 -zeros,0.7300000429153443,8.910000014305115,0.7800000190734864,2.9200000047683714,0.10999999046325684,0.16000003814697267,2.5699999809265135 -zipWith,2.8099999904632567,7.6,3.0100000381469725,8.890000009536744,0.08999998569488525,0.33000004291534424,2.400000047683716 -graphics/background,1.0599999666213988,7.150000071525573,1.7200000047683717,5.539999985694886,0.07999997138977051,0.02000002861022949,12.439999961853028 -graphics/grouped-bar-chart,96.92999999523163,65.83000004291534,77.07000000476837,219.54000005722045,0.3499999284744263,0.09000005722045898,1209.0599999666215 -graphics/line-chart,130.67000000476838,89.93999998569488,106.73000001907349,339.16000003814696,0.5399999618530273,0.20000004768371582,1765.3199999809265 -graphics/stacked-bar-chart,53.93000001907349,34.70999999046326,40.12000000476837,121.52000000476838,0.23000004291534423,0.029999971389770508,80.32000000476837 \ No newline at end of file +Test-Name, Trace-Eval, Trace-Bwd, Trace-Fwd, Graph-Eval, Graph-Bwd, Graph-Fwd, Graph-FwdDeMorgan, Graph-BwdAll +arithmetic,1.4,19.2,1.2,11.4,0.2,0.0,8.0,0.2 +array,2.6,18.6,2.4,14.8,0.2,0.0,9.2,0.2 +compose,0.8,15.8,0.8,10.8,0.0,0.2,2.0,0.2 +dicts,1.2,18.8,1.2,10.2,0.6,0.0,2.2,0.4 +div-mod-quot-rem,0.8,18.6,1.2,17.8,0.0,0.0,5.2,6.2 +factorial,7.2,18.4,4.2,25.8,0.0,0.2,112.0,29.0 +filter,4.8,18.8,5.8,13.0,0.2,0.0,4.0,0.4 +first-class-constr,9.0,17.6,7.4,16.8,0.2,0.0,5.8,0.8 +flatten,4.6,18.8,4.8,17.8,0.0,0.0,5.2,0.4 +foldr_sumSquares,2.6,17.8,2.2,10.2,0.0,0.0,2.8,0.0 +lexicalScoping,0.6,16.2,1.2,2.8,0.2,0.0,2.8,0.0 +length,1.4,15.4,0.6,4.0,0.0,0.2,1.8,0.2 +lookup,3.8,14.8,4.0,10.0,0.0,0.0,3.6,0.4 +map,3.0,19.0,2.6,11.0,0.2,0.2,3.6,0.6 +mergeSort,12.8,23.2,18.4,22.2,0.0,0.0,4.0,1.0 +normalise,0.8,18.4,0.8,5.6,0.0,0.0,3.4,0.2 +pattern-match,4.2,16.2,4.4,13.2,0.4,0.0,4.6,0.4 +range,12.8,24.0,13.2,39.6,0.0,0.0,9.4,2.2 +records,4.6,21.4,6.4,14.6,0.2,0.0,4.4,0.8 +reverse,1.4,19.4,2.8,7.4,0.0,0.0,4.8,0.2 +desugar/list-comp-1,19.2,29.4,17.0,55.0,0.4,0.0,13.0,1.2 +desugar/list-comp-2,53.8,52.6,49.6,180.4,0.0,0.0,27.8,11.0 +desugar/list-comp-3,15.4,23.6,15.0,44.4,0.0,0.0,9.2,0.4 +desugar/list-comp-4,5.8,17.6,9.2,17.4,0.2,0.0,3.4,0.8 +desugar/list-comp-5,4.4,16.4,5.0,12.4,0.2,0.0,2.8,0.2 +desugar/list-comp-6,0.2,15.2,2.8,4.2,0.2,0.0,4.6,0.0 +desugar/list-comp-7,7.0,16.4,4.6,13.6,0.0,0.0,5.0,0.0 +desugar/list-enum,3.8,17.2,4.6,12.2,0.0,0.0,10.2,4.4 +add,0.0,19.4,0.0,2.2,0.0,0.4,3.0,0.0 +array/lookup,20.8,23.8,19.4,61.2,0.0,0.2,16.0,0.2 +array/dims,2.4,18.4,1.6,5.2,0.4,1.0,2.8,0.2 +convolution/edgeDetect,1543.2,1075.6,1574.8,5635.2,0.4,6.8,3557.4,14.2 +convolution/emboss,1389.6,884.4,1323.4,4624.6,0.2,1.2,3251.8,11.6 +convolution/gaussian,1382.8,905.2,1372.6,4729.8,0.4,1.6,3298.0,14.4 +dict/create,1.0,23.8,0.4,4.8,0.4,0.4,8.4,0.8 +dict/difference,0.8,17.2,1.0,5.2,0.2,0.4,3.2,0.0 +dict/disjointUnion,0.8,14.4,0.0,2.8,0.0,0.2,3.6,0.4 +dict/foldl,6.8,20.0,5.6,16.2,0.4,0.2,2.8,0.0 +dict/intersectionWith,1.0,22.2,1.2,7.6,0.0,0.8,2.6,0.0 +dict/fromRecord,0.2,15.2,0.2,0.8,0.2,0.2,2.8,0.0 +dict/get,0.8,15.2,0.4,2.2,0.2,0.0,4.4,0.0 +dict/map,2.8,19.6,3.0,10.2,0.0,0.2,2.6,0.2 +divide,0.0,14.6,0.4,1.8,0.0,0.2,4.6,0.2 +filter,2.6,14.2,2.4,9.0,0.0,0.0,2.0,0.4 +intersperse,2.6,16.4,1.4,7.4,0.2,0.2,1.6,0.4 +intersperse,3.0,16.2,5.2,4.0,0.4,0.4,2.2,0.8 +length,1.0,14.6,1.0,6.2,0.0,0.8,2.4,0.4 +list-comp,31.8,42.8,28.8,87.8,0.4,2.4,7.0,0.8 +list-comp,24.4,31.4,26.0,79.2,0.6,1.0,7.0,0.6 +lookup,3.2,16.4,4.4,10.8,0.8,0.8,2.4,0.2 +map,5.4,21.2,3.8,11.4,0.6,0.8,5.2,0.4 +multiply,0.2,15.4,0.2,2.4,0.0,0.2,3.2,0.4 +nth,1.6,16.6,1.4,4.8,0.0,0.0,3.2,0.0 +section-5-example,26.0,39.0,23.4,91.4,0.6,1.4,9.2,0.8 +section-5-example,21.4,31.8,23.2,94.6,0.4,0.4,14.2,1.6 +section-5-example,23.8,28.4,20.8,88.2,0.2,0.8,10.0,0.4 +zeros,1.0,15.8,0.6,4.0,0.8,0.0,2.0,0.0 +zeros,0.4,14.2,0.8,2.2,0.0,0.4,4.2,0.0 +zipWith,2.8,14.6,2.4,15.0,0.2,0.2,4.2,0.4 +graphics/background,1.4,15.2,1.6,10.2,0.0,0.0,23.6,2.2 +graphics/grouped-bar-chart,158.6,114.8,135.6,481.4,0.4,0.2,2806.4,1138.0 +graphics/line-chart,210.0,147.4,191.2,820.6,1.4,0.2,4232.4,2574.6 +graphics/stacked-bar-chart,87.4,75.0,73.4,281.0,1.6,0.2,197.8,308.0 \ No newline at end of file diff --git a/plot_bench.py b/plot_bench.py index 8df82c640..dcfbbb426 100644 --- a/plot_bench.py +++ b/plot_bench.py @@ -12,9 +12,9 @@ def parse(): print(df) # Reorder benchmark columns - column_order = ['Trace-Eval', 'Graph-Eval', 'Trace-Bwd', 'Graph-Bwd', 'Trace-Fwd', 'Graph-Fwd', 'Graph-FwdDeMorgan'] - column_colors = ['#0d6b12', '#93dbb5', '#3e3875', '#b8bef5', '#910303', '#e84d4d', '#e8bceb'] - + column_order = ['Trace-Eval', 'Graph-Eval', 'Trace-Bwd', 'Graph-Bwd', 'Trace-Fwd', 'Graph-Fwd', 'Graph-FwdDeMorgan', 'Graph-BwdAll'] + column_colors = ['#0d6b12', '#93dbb5', '#3e3875', '#b8bef5', '#910303', '#e84d4d', '#e8bceb', '#5073a1'] + # Plot as bar chart df[column_order].plot(kind="bar", color=column_colors, ylabel="Milliseconds", rot=0) diff --git a/test/Benchmark.purs b/test/Benchmark.purs index 40a8728c3..21ebfaf8c 100644 --- a/test/Benchmark.purs +++ b/test/Benchmark.purs @@ -20,7 +20,7 @@ import Util (type (×), (×)) main :: Effect Unit main = launchAff_ do let - iter = 10 + iter = 5 arr = concat ([ bench_misc, bench_desugaring, bench_bwd, bench_graphics ] <#> (#) iter) outs <- sequence $ map (\(str × row) -> lift2 Tuple (pure str) row) arr logShow $ BenchAcc outs diff --git a/test/Benchmark/Util.purs b/test/Benchmark/Util.purs index 5bd54933c..2a96d0a5d 100644 --- a/test/Benchmark/Util.purs +++ b/test/Benchmark/Util.purs @@ -41,11 +41,12 @@ type GraphRow = , tBwd :: Number , tFwd :: Number , tFwdDemorgan :: Number + , tBwdAll :: Number } instance Show BenchAcc where show (BenchAcc rows) = - "Test-Name, Trace-Eval, Trace-Bwd, Trace-Fwd, Graph-Eval, Graph-Bwd, Graph-Fwd, Graph-FwdDeMorgan\n" + "Test-Name, Trace-Eval, Trace-Bwd, Trace-Fwd, Graph-Eval, Graph-Bwd, Graph-Fwd, Graph-FwdDeMorgan, Graph-BwdAll\n" <> (fold $ intersperse "\n" (map rowShow rows)) rowShow :: String × BenchRow -> String @@ -60,16 +61,12 @@ instance Show BenchRow where , show grRow.tBwd , show grRow.tFwd , show grRow.tFwdDemorgan + , show grRow.tBwdAll ] now :: forall m. MonadEffect m => m JSDate now = liftEffect JSDate.now --- tdiff :: JSDate -> JSDate -> Number --- tdiff begin end = --- getTime end - getTime begin --- - tdiff :: Number -> Number -> Number tdiff x y = sub y x diff --git a/test/Util.purs b/test/Util.purs index a5583790a..92b51b284 100644 --- a/test/Util.purs +++ b/test/Util.purs @@ -28,7 +28,7 @@ import Graph.GraphImpl (GraphImpl) import Graph.Slice (bwdSlice, fwdSlice, fwdSliceDeMorgan) as G import Graph.Slice (selectαs, select𝔹s) import Heterogeneous.Mapping (hmap) -import Lattice (bot, botOf, erase, Raw) +import Lattice (bot, botOf, topOf, erase, Raw) import Module (parse) import Parse (program) import Pretty (class Pretty, prettyP) @@ -120,6 +120,15 @@ testGraph s gconf { δv, bwd_expect, fwd_expect } = do let s𝔹 = desugBwd e𝔹 (erase s) + -- | Backward (all outputs selected) + tBwdAll1 <- preciseTime + let + αs_out_all = selectαs (topOf vα) vα + gbwd_all = G.bwdSlice αs_out_all g + αs_in_all = sinks gbwd_all + e𝔹_all = select𝔹s eα αs_in_all + tBwdAll2 <- preciseTime + -- | Forward (round-tripping) tFwd1 <- preciseTime let @@ -144,8 +153,11 @@ testGraph s gconf { δv, bwd_expect, fwd_expect } = do checkPretty "Graph-based value (De Morgan)" fwd_expect v𝔹' sources gbwd `shouldSatisfy "fwd ⚬ bwd round-tripping property"` (flip subset (sources gfwd)) + unless false do -- to avoid unused variables + log ("BwdAll selected nodes: " <> show αs_out_all) + log (prettyP e𝔹_all) - pure { tEval: tdiff tEval1 tEval2, tBwd: tdiff tBwd1 tBwd2, tFwd: tdiff tFwd1 tFwd2, tFwdDemorgan: tdiff tFwdDeMorgan1 tFwdDeMorgan2 } + pure { tEval: tdiff tEval1 tEval2, tBwd: tdiff tBwd1 tBwd2, tFwd: tdiff tFwd1 tFwd2, tFwdDemorgan: tdiff tFwdDeMorgan1 tFwdDeMorgan2, tBwdAll: tdiff tBwdAll1 tBwdAll2 } type TestSpec = { file :: String @@ -192,7 +204,7 @@ averageRows rows = averagedTr runs = toNumber $ length rows zeroRow :: BenchRow - zeroRow = BenchRow { tEval: 0.0, tBwd: 0.0, tFwd: 0.0 } { tEval: 0.0, tBwd: 0.0, tFwd: 0.0, tFwdDemorgan: 0.0 } + zeroRow = BenchRow { tEval: 0.0, tBwd: 0.0, tFwd: 0.0 } { tEval: 0.0, tBwd: 0.0, tFwd: 0.0, tFwdDemorgan: 0.0, tBwdAll: 0.0 } sumRow :: BenchRow -> BenchRow -> BenchRow sumRow (BenchRow trRow1 gRow1) (BenchRow trRow2 gRow2) = @@ -205,6 +217,7 @@ averageRows rows = averagedTr , tBwd: gRow1.tBwd + gRow2.tBwd , tFwd: gRow1.tFwd + gRow2.tFwd , tFwdDemorgan: gRow1.tFwdDemorgan + gRow2.tFwdDemorgan + , tBwdAll: gRow1.tBwdAll + gRow2.tBwdAll } summed = foldl sumRow zeroRow rows