First version of separating memory and runtime measurements

lib/benchee/benchmark/measurer.ex

@@ -0,0 +1,15 @@
+defmodule Benchee.Benchmark.Measurer do
+  @moduledoc """
+  A thing that measures something about a function execution - like time or
+  memory needed.
+  """
+
+  @doc """
+  Takes an anonymous 0 arity function to measure and returns the measurement
+  and the return value of the function in a tuple.
+
+  The returned measurement may be `nil` if the measurement failed for some 
+  reason - it will then be ignored and not counted.
+  """
+  @callback measure((() -> any)) :: {non_neg_integer | nil, any}
+end

lib/benchee/memory_measure.ex → lib/benchee/benchmark/measurer/memory.ex

@@ -1,19 +1,19 @@
-defmodule Benchee.MemoryMeasure do
+defmodule Benchee.Benchmark.Measurer.Memory do
   @moduledoc """
-  This exposes two functions, apply/1 and apply/3. Both execute a given function
-  and report on the memory used by monitoring the garbage collection process for
-  a single process.
+  Measure memory consumption of a function.
+
+  Returns `{nil, return_value}` in case the memory measurement went bad.
   """
-  import Kernel, except: [apply: 3, apply: 2]
 
-  @spec apply(fun) :: no_return() | tuple()
-  def apply(fun) do
+  @behaviour Benchee.Benchmark.Measurer
+
+  def measure(fun) do
     ref = make_ref()
     Process.flag(:trap_exit, true)
     start_runner(fun, ref)
 
     receive do
-      {^ref, memory_usage_info} -> memory_usage_info
+      {^ref, memory_usage_info} -> return_memory(memory_usage_info)
       :shutdown -> nil
     end
   end
@@ -35,6 +35,9 @@ defmodule Benchee.MemoryMeasure do
     end)
   end
 
+  defp return_memory({memory_usage, result}) when memory_usage < 0, do: {nil, result}
+  defp return_memory({memory_usage, result}), do: {memory_usage, result}
+
   defp measure_memory(fun, tracer) do
     word_size = :erlang.system_info(:wordsize)
     {:garbage_collection_info, info_before} = Process.info(self(), :garbage_collection_info)

lib/benchee/benchmark/measurer/time.ex

@@ -0,0 +1,11 @@
+defmodule Benchee.Benchmark.Measurer.Time do
+  @moduledoc """
+  Measure the time consumed by a executing function.
+  """
+
+  @behaviour Benchee.Benchmark.Measurer
+
+  def measure(function) do
+    :timer.tc(function)
+  end
+end

lib/benchee/benchmark/runner.ex

@@ -4,10 +4,11 @@ defmodule Benchee.Benchmark.Runner do
   run time and memory usage to each scenario.
   """
 
-  alias Benchee.{Benchmark, MemoryMeasure}
+  alias Benchee.Benchmark
   alias Benchee.Benchmark.{Scenario, ScenarioContext}
   alias Benchee.Utility.{RepeatN, Parallel}
   alias Benchee.Configuration
+  alias Benchee.Benchmark.Measurer
 
   @doc """
   Executes the benchmarks defined before by first running the defined functions
@@ -34,7 +35,7 @@ defmodule Benchee.Benchmark.Runner do
   defp pre_check(scenario, scenario_context = %ScenarioContext{config: %{pre_check: true}}) do
     scenario_input = run_before_scenario(scenario, scenario_context)
     scenario_context = %ScenarioContext{scenario_context | scenario_input: scenario_input}
-    _ = measure_iteration(scenario, scenario_context)
+    _ = measure_iteration(scenario, scenario_context, Measurer.Time)
     _ = run_after_scenario(scenario, scenario_context)
     nil
   end
@@ -58,7 +59,6 @@ defmodule Benchee.Benchmark.Runner do
   defp measure_scenario_parallel(config, scenario, scenario_context) do
     1..config.parallel
     |> Parallel.map(fn _ -> measure_scenario(scenario, scenario_context) end)
-    |> List.flatten()
   end
 
   defp add_measurements_to_scenario(measurements, scenario) do
@@ -72,9 +72,11 @@ defmodule Benchee.Benchmark.Runner do
     scenario_input = run_before_scenario(scenario, scenario_context)
     scenario_context = %ScenarioContext{scenario_context | scenario_input: scenario_input}
     _ = run_warmup(scenario, scenario_context)
-    measurements = run_benchmark(scenario, scenario_context)
+    runtimes = run_runtime_benchmark(scenario, scenario_context)
+    memory_usages = run_memory_benchmark(scenario, scenario_context)
     run_after_scenario(scenario, scenario_context)
-    measurements
+
+    {runtimes, memory_usages}
   end
 
   defp run_before_scenario(
@@ -103,7 +105,7 @@ defmodule Benchee.Benchmark.Runner do
     measure_runtimes(scenario, scenario_context, warmup, false)
   end
 
-  defp run_benchmark(
+  defp run_runtime_benchmark(
          scenario,
          scenario_context = %ScenarioContext{
            config: %Configuration{
@@ -115,6 +117,29 @@ defmodule Benchee.Benchmark.Runner do
     measure_runtimes(scenario, scenario_context, run_time, fast_warning)
   end
 
+  defp run_memory_benchmark(_, %ScenarioContext{config: %{memory_time: 0}}) do
+    []
+  end
+  defp run_memory_benchmark(
+         scenario,
+         scenario_context = %ScenarioContext{
+           config: %Configuration{
+             memory_time: memory_time
+           }
+         }
+       ) do
+
+    end_time = current_time() + memory_time
+
+    new_context = %ScenarioContext{
+      scenario_context
+      | current_time: current_time(),
+        end_time: end_time
+    }
+
+    do_benchmark(scenario, new_context, Measurer.Memory, [])
+  end
+
   defp run_after_scenario(
          %{
            after_scenario: local_after_scenario
@@ -135,7 +160,7 @@ defmodule Benchee.Benchmark.Runner do
     end_time = current_time() + run_time
     :erlang.garbage_collect()
 
-    {num_iterations, initial_run_time, initial_memory_usage} =
+    {num_iterations, initial_run_time} =
       determine_n_times(scenario, scenario_context, fast_warning)
 
     new_context = %ScenarioContext{
@@ -145,10 +170,7 @@ defmodule Benchee.Benchmark.Runner do
         num_iterations: num_iterations
     }
 
-    do_benchmark(
-      scenario,
-      new_context,
-      {[initial_run_time], updated_memory_usages(initial_memory_usage, [])})
+    do_benchmark(scenario, new_context, Measurer.Time, [initial_run_time])
   end
 
   defp current_time, do: :erlang.system_time(:micro_seconds)
@@ -166,12 +188,10 @@ defmodule Benchee.Benchmark.Runner do
          },
          fast_warning
        ) do
-    {run_time, memory_usage} = measure_iteration(scenario, scenario_context)
+    run_time = measure_iteration(scenario, scenario_context, Measurer.Time)
 
     if run_time >= @minimum_execution_time do
-      {adjusted_run_time, adjusted_memory_usage} =
-        adjust_for_iterations(run_time, memory_usage, num_iterations)
-      {num_iterations, adjusted_run_time, adjusted_memory_usage}
+      {num_iterations, adjust_for_iterations(run_time, num_iterations)}
     else
       if fast_warning, do: printer.fast_warning()
 
@@ -195,92 +215,77 @@ defmodule Benchee.Benchmark.Runner do
            current_time: current_time,
            end_time: end_time
          },
-         {run_times, memory_usages}
+         _measurer,
+         measurements
        )
        when current_time > end_time do
     # restore correct order - important for graphing
-    {Enum.reverse(run_times), Enum.reverse(memory_usages)}
+    Enum.reverse(measurements)
   end
 
-  defp do_benchmark(scenario, scenario_context, {run_times, memory_usages}) do
-    {run_time, memory_usage} = iteration_measurements(scenario, scenario_context)
+  defp do_benchmark(scenario, scenario_context, measurer, measurements) do
+    measurement = iteration_measurements(scenario, scenario_context, measurer)
     updated_context = %ScenarioContext{scenario_context | current_time: current_time()}
 
     do_benchmark(
       scenario,
       updated_context,
-      {[run_time | run_times], updated_memory_usages(memory_usage, memory_usages)}
+      measurer,
+      updated_measurements(measurement, measurements)
     )
   end
 
-  # We return nil if no memory measurement is performed so keep it empty
-  defp updated_memory_usages(nil, memory_usages), do: memory_usages
-  defp updated_memory_usages(memory_usage, memory_usages) when memory_usage < 0, do: memory_usages
-  defp updated_memory_usages(memory_usage, memory_usages), do: [memory_usage | memory_usages]
+  # We return `nil` if memory measurement failed so keep it empty
+  defp updated_measurements(nil, measurements), do: measurements
+  defp updated_measurements(measurement, measurements), do: [measurement | measurements]
 
   defp iteration_measurements(
          scenario,
          scenario_context = %ScenarioContext{
            num_iterations: num_iterations
-         }
+         },
+         measurer
        ) do
-    {run_time, memory_usage} = measure_iteration(scenario, scenario_context)
+    measurement = measure_iteration(scenario, scenario_context, measurer)
 
-    adjust_for_iterations(run_time, memory_usage, num_iterations)
+    adjust_for_iterations(measurement, num_iterations)
   end
 
-  defp adjust_for_iterations(run_time, nil, num_iterations) do
-    {run_time / num_iterations, nil}
-  end
-
-  defp adjust_for_iterations(run_time, memory_usage, num_iterations) do
-    {run_time / num_iterations, memory_usage / num_iterations}
-  end
+  defp adjust_for_iterations(measurement, 1), do: measurement
+  defp adjust_for_iterations(measurement, num_iterations), do: measurement / num_iterations
 
   defp measure_iteration(
          scenario = %Scenario{function: function},
          scenario_context = %ScenarioContext{
            num_iterations: 1,
-           config: %{measure_memory: measure_memory}
-         }
+         },
+         measurer
        ) do
     new_input = run_before_each(scenario, scenario_context)
     function = main_function(function, new_input)
 
-    {microseconds, memory_usage, return_value} = measure_time_and_memory(function, measure_memory)
+    {measurement, return_value} = measurer.measure(function)
 
     run_after_each(return_value, scenario, scenario_context)
-    {microseconds, memory_usage}
+    measurement
   end
 
   defp measure_iteration(
          scenario,
          scenario_context = %ScenarioContext{
-           num_iterations: iterations,
-           config: %{measure_memory: measure_memory}
-         }
+           num_iterations: iterations
+         },
+         measurer
        )
        when iterations > 1 do
     # When we have more than one iteration, then the repetition and calling
     # of hooks is already included in the function, for reference/reasoning see
     # `build_benchmarking_function/2`
     function = build_benchmarking_function(scenario, scenario_context)
 
-    {microseconds, memory_usage, _return_value} =
-      measure_time_and_memory(function, measure_memory)
-
-    {microseconds, memory_usage}
-  end
-
-  defp measure_time_and_memory(function, true) do
-    {microseconds, return_value} = :timer.tc(function)
-    {memory_usage, _} = MemoryMeasure.apply(function)
-    {microseconds, memory_usage, return_value}
-  end
+    {measurement, _return_value} = measurer.measure(function)
 
-  defp measure_time_and_memory(function, false) do
-    {microseconds, return_value} = :timer.tc(function)
-    {microseconds, nil, return_value}
+    measurement
   end
 
   @no_input Benchmark.no_input()

lib/benchee/configuration.ex

@@ -13,9 +13,9 @@ defmodule Benchee.Configuration do
   }
 
   defstruct parallel: 1,
-            measure_memory: false,
             time: 5,
             warmup: 2,
+            memory_time: 0,
             pre_check: false,
             formatters: [Console],
             print: %{
@@ -44,9 +44,9 @@ defmodule Benchee.Configuration do
 
   @type t :: %__MODULE__{
           parallel: integer,
-          measure_memory: boolean,
           time: number,
           warmup: number,
+          memory_time: number,
           pre_check: boolean,
           formatters: [(Suite.t() -> Suite.t())],
           print: map,
@@ -63,7 +63,7 @@ defmodule Benchee.Configuration do
         }
 
   @type user_configuration :: map | keyword
-  @time_keys [:time, :warmup]
+  @time_keys [:time, :warmup, :memory_time]
 
   @doc """
   Returns the initial benchmark configuration for Benchee, composed of defaults
@@ -327,10 +327,15 @@ defmodule Benchee.Configuration do
     end)
   end
 
-  defp update_measure_memory(config = %{measure_memory: measure_memory}) do
+  defp update_measure_memory(config = %{memory_time: memory_time}) do
     otp_version = List.to_integer(:erlang.system_info(:otp_release))
-    if measure_memory and otp_version <= 18, do: print_memory_measure_warning()
-    Map.put(config, :measure_memory, measure_memory and otp_version > 18)
+
+    if (memory_time > 0) and otp_version <= 18 do
+      print_memory_measure_warning()
+      Map.put(config, :memory_time, 0)
+    else
+      config
+    end
   end
 
   defp print_memory_measure_warning do