From 48a523f468aa2652a68d063928f9639ee13040a3 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Thu, 10 Aug 2023 15:48:02 +0200 Subject: [PATCH] Prototype setup and teardown method configuration --- .../Standard/Test/0.0.0-dev/src/Bench.enso | 127 +++++++++++++++--- test/Benchmarks/src/Vector/Distinct.enso | 24 ++-- 2 files changed, 124 insertions(+), 27 deletions(-) diff --git a/distribution/lib/Standard/Test/0.0.0-dev/src/Bench.enso b/distribution/lib/Standard/Test/0.0.0-dev/src/Bench.enso index 64ab4c01f540..725c741ca7ae 100644 --- a/distribution/lib/Standard/Test/0.0.0-dev/src/Bench.enso +++ b/distribution/lib/Standard/Test/0.0.0-dev/src/Bench.enso @@ -1,5 +1,7 @@ from Standard.Base import all import Standard.Base.Errors.Illegal_Argument.Illegal_Argument +import Standard.Base.Errors.Unimplemented.Unimplemented +import Standard.Base.Runtime.Debug ## Configuration for a measurement phase or a warmup phase of a benchmark. type Phase_Conf @@ -22,22 +24,101 @@ type Phase_Conf Phase_Conf.Secs secs -> secs.to_text + " seconds" Phase_Conf.Iters iters -> iters.to_text + " iterations" -## The benchmark options for a `Bench_Group`. These options roughly corresponds to the options - defined in the JMH benchmarking library. See the JMH documentation for more details: + ## Validates the config and throws a Panic if it is invalid. + validate : Nothing + validate self = case self of + Phase_Conf.Secs secs -> + if secs < 0 then Panic.throw (Illegal_Argument.Error "Seconds must be positive") + Phase_Conf.Iters iters -> + if iters < 0 then Panic.throw (Illegal_Argument.Error "Iterations must be positive") + + +type Default_Setup + setup : (Any|Nothing) + setup = Nothing + + +type Default_Teardown + teardown : (Any|Nothing) -> Nothing + teardown _ = Nothing + + +## The benchmark options for a `Bench_Group`. These options roughly corresponds + to the options defined in the JMH benchmarking library. See the JMH + documentation for more details: https://javadoc.io/doc/org.openjdk.jmh/jmh-core/latest/org/openjdk/jmh/annotations/package-summary.html + + ! Setup and teardown configuration + The `setup` and `teardown` are optional methods are called before starting + a single benchmark from `Bench_Spec` and after finishing a single benchmark + from `Bench_Spec` respectively. This frequency of calling `setup` and + `teardown` methods corresponds to `Level.Trial` in JMH - https://javadoc.io/doc/org.openjdk.jmh/jmh-core/latest/org/openjdk/jmh/annotations/Level.html#Trial + It is not possible to set a different frequency. + + To provide a custom implementation of the `setup` method, you need to pass + a type that contains a single static method called `setup` that takes no + arguments and returns any object that will be passed as input arguments to + the benchmark. + + To provide a custom implementation of the `teardown` method, you need to + pass a type that contains a single static method called `teardown` that + takes a single argument and returns `Nothing`. + + Setup can be specified without Teardown, but Teardown cannot be specified + without Setup. + + See the `Default_Setup` and `Default_Teardown` prototypes. + + Note that the providing setup or teardown behavior as static methods in + types allows for delaying the execution of the setup and teardown code until + the benchmark is actually run. type Bench_Options ## PRIVATE - Impl (warmup:Phase_Conf) (measure:Phase_Conf) + Impl (warmup:Phase_Conf) (measure:Phase_Conf) (setup:Any) (teardown:Any) - ## Sets the warmup phase. + ## Sets the length of the warmup phase. set_warmup : Phase_Conf -> Bench_Options - set_warmup self (warm:Phase_Conf) = Bench_Options.Impl warm self.measure + set_warmup self warm = Bench_Options.Impl warm self.measure self.setup self.teardown - ## Sets the measurement phase. + ## Sets the length the measurement phase. set_measure : Phase_Conf -> Bench_Options - set_measure self (meas:Phase_Conf) = Bench_Options.Impl self.warmup meas + set_measure self meas = Bench_Options.Impl self.warmup meas self.setup self.teardown + + ## Sets the setup hook that is called before every benchmark invocation. + + Arguments: + - setup: A type that has a `setup` static method that returns an object + that will be passed as input arguments to the benchmark. + set_setup : Any -> Bench_Options + set_setup self setup = Bench_Options.Impl self.warmup self.measure setup self.teardown + + ## Sets the teardown hook that is called after every benchmark invocation. + + Arguments: + - teardown: A type that has a `teardown` static method that takes a single + argument and returns `Nothing`. It will be called after every benchmark + invocation, with the argument being the object returned by the `setup` + method. + set_teardown : Any -> Bench_Options + set_teardown self teardown = Bench_Options.Impl self.warmup self.measure self.setup teardown + + to_text self = "[warmup={" + self.warmup.to_text + "}, measurement={" + self.measure.to_text + "}, setup=" + self.setup.to_text + ", teardown=" + self.teardown.to_text + "]" + + ## Validates the config and throws a Panic if it is invalid. + validate : Nothing + validate self = + self.warmup.validate + self.measure.validate + case (Meta.get_type_methods (Meta.type_of self.setup)).contains "setup" of + False -> Panic.throw (Illegal_Argument.Error "setup field is not a type with a method called `setup`") + True -> Nothing + case (Meta.get_type_methods (Meta.type_of self.teardown)).contains "teardown" of + False -> Panic.throw (Illegal_Argument.Error "teardown field is not a type with a method called `teardown`") + True -> Nothing + case (self.setup == Default_Setup && self.teardown != Default_Teardown) of + True -> Panic.throw (Illegal_Argument.Error "Teardown cannot be specified without setup") + False -> Nothing - to_text self = "[warmup=" + self.warmup.to_text + ", measurement=" + self.measure.to_text + "]" type Bench_Builder ## PRIVATE @@ -46,18 +127,27 @@ type Bench_Builder group : Text -> Bench_Options -> (Group_Builder -> Any) -> Any group self (name:Text) (configuration:Bench_Options) fn = validate_name name + configuration.validate b = Vector.new_builder fn (Group_Builder.Impl b) self.builder.append <| Bench.Group name configuration b.to_vector + type Group_Builder ## PRIVATE Impl builder - specify : Text -> Any -> Bench + ## Adds a benchmark specification to the group. + + Arguments: + - name: The name of the benchmark. Must be a valid Java identifier. + - benchmark: The benchmark function. Takes a single argument that is + produced by the `setup` method of the `Bench_Options` configuration. + The return value is ignored. + specify : Text -> (Any -> Nothing) -> Bench specify self (name:Text) ~benchmark = validate_name name - self.builder.append <| Bench.Spec name (_ -> benchmark) + self.builder.append <| Bench.Spec name benchmark type Bench @@ -72,7 +162,7 @@ type Bench Bench.All b.to_vector options : Bench_Options - options = Bench_Options.Impl (Phase_Conf.Secs 3) (Phase_Conf.Secs 3) + options = Bench_Options.Impl (Phase_Conf.Secs 3) (Phase_Conf.Secs 3) Default_Setup Default_Teardown fold : Any -> (Any -> Bench -> Bench -> Any) -> Any fold self value fn = case self of @@ -85,10 +175,12 @@ type Bench IO.println <| "Found " + count.to_text + " cases to execute" self.fold Nothing _-> g-> s-> - c = g.configuration + conf = g.configuration bench_name = g.name + "." + s.name - IO.println <| "Benchmarking '" + bench_name + "' configuration: " + c.to_text - Bench.measure bench_name c.warmup c.measure (s.code 0) + input_arg = conf.setup.setup + IO.println <| "Benchmarking '" + bench_name + "' configuration: " + conf.to_text + Bench.measure bench_name conf (s.code input_arg) + conf.teardown.teardown input_arg ## Measure the amount of time it takes to execute a given computation. @@ -117,7 +209,7 @@ type Bench example_measure = Bench.measure "foo" warmup_iters=2 measurement_iters=1 Examples.get_boolean measure : Text -> Phase_Conf -> Phase_Conf -> Boolean -> Any -> Nothing - measure (label:Text) (warmup_conf:Phase_Conf) (measure_conf:Phase_Conf) ~act = + measure (label:Text) (bench_opts:Bench_Options) ~act = dry_run = Environment.get "ENSO_BENCHMARK_TEST_DRY_RUN" "False" == "True" measure_start = System.nano_time @@ -155,9 +247,8 @@ type Bench empty_conf = Phase_Conf.Iters 0 if dry_run then run_phase "dummy" empty_conf else - warmup_durations = run_phase "Warmup" warmup_conf - measure_durations = run_phase "Measurement" measure_conf - # TODO: Print some stats at the end of the phase + warmup_durations = run_phase "Warmup" bench_opts.warmup + measure_durations = run_phase "Measurement" bench_opts.measure measure_end = System.nano_time measure_duration_ms = (measure_end - measure_start) / 1000000 fmt_duration_ms = measure_duration_ms.format "#.##" diff --git a/test/Benchmarks/src/Vector/Distinct.enso b/test/Benchmarks/src/Vector/Distinct.enso index 8a77c4aa5d2c..129489980727 100644 --- a/test/Benchmarks/src/Vector/Distinct.enso +++ b/test/Benchmarks/src/Vector/Distinct.enso @@ -4,24 +4,30 @@ from Standard.Test import Bench, Phase_Conf import project.Vector.Utils +type Setup + Value random_vec uniform_vec random_text_vec uniform_text_vec -options = Bench.options . set_warmup (Phase_Conf.Iters 2) . set_measure (Phase_Conf.Iters 2) + setup : Setup + setup = + random_vec = Utils.make_random_vec 100000 + uniform_vec = Vector.fill 100000 1 + random_text_vec = random_vec.map .to_text + uniform_text_vec = random_vec.map .to_text + Setup.Value random_vec uniform_vec random_text_vec uniform_text_vec -random_vec = Utils.make_random_vec 100000 -uniform_vec = Vector.fill 100000 1 -random_text_vec = random_vec.map .to_text -uniform_text_vec = random_vec.map .to_text + +options = Bench.options . set_warmup (Phase_Conf.Iters 1) . set_measure (Phase_Conf.Iters 1) . set_setup Setup collect_benches = Bench.build builder-> builder.group "Vector_Distinct" options group_builder-> group_builder.specify "Random_Integer_Vector_Distinct" <| - random_vec.distinct + arg -> arg.random_vec.distinct group_builder.specify "Uniform_Integer_Vector_Distinct" <| - uniform_vec.distinct + arg -> arg.uniform_vec.distinct group_builder.specify "Random_Text_Vector_Distinct" <| - random_text_vec.distinct + arg -> arg.random_text_vec.distinct group_builder.specify "Uniform_Text_Vector_Distinct" <| - uniform_text_vec.distinct + arg -> arg.uniform_text_vec.distinct main = collect_benches . run_main