Add runtime estimator (#4718)

* Add runtime estimator

- Adds a utility to estimate runtime for QCS
- Estimates single circuit runs, sweeps, and batches.
- This is a rough approximation and only accurate to about 25% or so.

cirq-google/cirq_google/engine/__init__.py

@@ -49,6 +49,12 @@
     EngineProgram,
 )
 
+from cirq_google.engine.runtime_estimator import (
+    estimate_run_time,
+    estimate_run_batch_time,
+    estimate_run_sweep_time,
+)
+
 from cirq_google.engine.engine_sampler import (
     get_engine_sampler,
     QuantumEngineSampler,

cirq-google/cirq_google/engine/runtime_estimator.py

@@ -0,0 +1,201 @@
+# Copyright 2021 The Cirq Developers
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Utility functions to estimate runtime using Engine to execute circuits.
+
+Users can call estimate_run_time, estimate_run_sweep_time, or
+estimate_run_batch_time to retrieve approximations of runtime on QCS
+of various sizes and shapes of circuits.
+
+Times were extrapolated from actual runs on Sycamore processors duing
+November 2021.  These times should only be used as a rough guide.
+Your experience may vary based on many factors not captured here.
+
+Parameters were calculated using a variety of width/depth/sweeps from
+the rep rate calculator, see:
+
+https://github.com/quantumlib/ReCirq/blob/master/recirq/benchmarks/rep_rate/
+
+Model was then fitted by hand, correcting for anomalies and outliers
+when possible.
+
+"""
+from typing import List, Optional, Sequence
+import cirq
+
+# Estimated end-to-end latency of circuits through the system
+_BASE_LATENCY = 1.5
+
+
+def _initial_time(width, depth, sweeps):
+    """Estimates the initiation time of a circuit.
+
+    This estimate includes tasks like electronics setup, gate compiling,
+    and throughput of constant-time data.
+
+    This time depends on the size of the circuits being compiled
+    (width and depth) and also includes a factor for the number of
+    times the compilation is done (sweeps).  Since sweeps save some of
+    the work, this factor scales less than one.
+
+    Args:
+        width: number of qubits
+        depth: number of moments
+        sweeps: total number of parameter sweeps
+    """
+    return ((width / 8) * (depth / 125) + (width / 12)) * max(1, sweeps / 5)
+
+
+def _rep_time(width: int, depth: int, sweeps: int, reps: int) -> float:
+    """Estimated time of executing repetitions.
+
+    This includes all incremental costs of executing a repetition and of
+    sending data back and forth from the electronics.
+
+    This is based on an approximate rep rate for "fast" circuits at about
+    24k reps per second.  More qubits measured (width) primarily slows
+    this down, with an additional factor for very high depth circuits.
+
+    For multiple sweeps, there is some additional cost, since not all
+    sweeps can be batched together.  Sweeping in general is more efficient,
+    but it is not perfectly parallel.  Sweeping also seems to be more
+    sensitive to the number of qubits measured, for reasons not understood.
+
+    Args:
+        width: number of qubits
+        depth: number of moments
+        sweeps: total number of parameter sweeps
+        reps: number of repetitions per parameter sweep
+    """
+    total_reps = sweeps * reps
+    rep_rate = 24000 / (0.9 + width / 38) / (0.9 + depth / 5000)
+    if sweeps > 1:
+        rep_rate *= 0.72
+        rep_rate *= 1 - (width - 25) / 40
+    return total_reps / rep_rate
+
+
+def _estimate_run_time_seconds(
+    width: int, depth: int, sweeps: int, repetitions: int, latency: Optional[float] = _BASE_LATENCY
+) -> float:
+    """Returns an approximate number of seconds for execution of a single circuit.
+
+    This includes the total cost of set up (initial time), cost per repetition (rep_time),
+    and a base end-to-end latency cost of operation (configurable).
+
+
+    Args:
+        width: number of qubits
+        depth: number of moments
+        sweeps: total number of parameter sweeps
+        repetitions: number of repetitions per parameter sweep
+        latency: Optional latency to add (defaults to 1.5 seconds)
+    """
+    init_time = _initial_time(width, depth, sweeps)
+    rep_time = _rep_time(width, depth, sweeps, repetitions)
+    return rep_time + init_time + latency
+
+
+def estimate_run_time(
+    program: cirq.AbstractCircuit, repetitions: int, latency: Optional[float] = _BASE_LATENCY
+) -> float:
+    """Compute the estimated time for running a single circuit.
+
+    This should approximate, in seconds, the time for the execution of a batch of circuits
+    using Engine.run() on QCS at a time where there is no queue (such as a reserved slot).
+    This estimation should be considered a rough approximation.  Many factors can contribute to
+    the execution time of a circuit, and the run time can also vary as the service's code changes
+    frequently.
+
+    Args:
+        program: circuit to be executed
+        repetitions: number of repetitions to execute
+        latency: Optional latency to add (defaults to 1.5 seconds)
+    """
+    width = len(program.all_qubits())
+    depth = len(program)
+    return _estimate_run_time_seconds(width, depth, 1, repetitions, latency)
+
+
+def estimate_run_sweep_time(
+    program: cirq.AbstractCircuit,
+    params: cirq.Sweepable = None,
+    repetitions: int = 1000,
+    latency: Optional[float] = _BASE_LATENCY,
+) -> float:
+    """Compute the estimated time for running a parameter sweep across a single Circuit.
+
+    This should approximate, in seconds, the time for the execution of a batch of circuits
+    using Engine.run_sweep() on QCS at a time where there is no queue (such as a reserved slot).
+    This estimation should be considered a rough approximation.  Many factors can contribute to
+    the execution time of a circuit, and the run time can also vary as the service's code changes
+    frequently.
+
+    Args:
+        program: circuit to be executed
+        params: a parameter sweep of variable resolvers to use with the circuit
+        repetitions: number of repetitions to execute per parameter sweep
+        latency: Optional latency to add (defaults to 1.5 seconds)
+    """
+    width = len(program.all_qubits())
+    depth = len(program)
+    sweeps = len(list(cirq.to_resolvers(params)))
+    return _estimate_run_time_seconds(width, depth, sweeps, repetitions, latency)
+
+
+def estimate_run_batch_time(
+    programs: Sequence[cirq.AbstractCircuit],
+    params_list: List[cirq.Sweepable],
+    repetitions: int = 1000,
+    latency: float = _BASE_LATENCY,
+) -> float:
+    """Compute the estimated time for running a batch of programs.
+
+    This should approximate, in seconds, the time for the execution of a batch of circuits
+    using Engine.run_batch() on QCS at a time where there is no queue (such as a reserved slot).
+    This estimation should be considered a rough approximation.  Many factors can contribute to
+    the execution time of a circuit, and the run time can also vary as the service's code changes
+    frequently.
+
+    Args:
+        programs: a sequence of circuits to be executed
+        params_list: a parameter sweep for each circuit
+        repetitions: number of repetitions to execute per parameter sweep
+        latency: Optional latency to add (defaults to 1.5 seconds)
+    """
+    total_time = 0.0
+    current_width = None
+    total_depth = 0
+    total_sweeps = 0
+    num_circuits = 0
+    for idx, program in enumerate(programs):
+        width = len(program.all_qubits())
+        if width != current_width:
+            if num_circuits > 0:
+                total_time += _estimate_run_time_seconds(
+                    width, total_depth // num_circuits, total_sweeps, repetitions, 0.25
+                )
+            num_circuits = 0
+            total_depth = 0
+            total_sweeps = 0
+            current_width = width
+        total_depth += len(program)
+        num_circuits += 1
+        total_sweeps += len(list(cirq.to_resolvers(params_list[idx])))
+    if num_circuits > 0:
+        total_time += _estimate_run_time_seconds(
+            width, total_depth // num_circuits, total_sweeps, repetitions, 0.0
+        )
+
+    return total_time + latency

cirq-google/cirq_google/engine/runtime_estimator_test.py

@@ -0,0 +1,165 @@
+# Copyright 2021 The Cirq Developers
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import pytest
+import cirq
+import cirq_google.engine.runtime_estimator as runtime_estimator
+
+
+def _assert_about_equal(actual: float, expected: float):
+    """Assert that two times are within 25% of the expected time.
+
+    Used to test the estimator with noisy data from actual devices.
+    """
+    assert expected * 0.75 < actual < expected * 1.25
+
+
+@pytest.mark.parametrize("reps,expected", [(1000, 2.25), (16000, 2.9), (64000, 4.6), (128000, 7.4)])
+def test_estimate_run_time_vary_reps(reps, expected):
+    """Test various run times.
+    Values taken from Weber November 2021."""
+    qubits = cirq.GridQubit.rect(2, 5)
+    circuit = cirq.testing.random_circuit(qubits, n_moments=10, op_density=1.0)
+    runtime = runtime_estimator.estimate_run_time(circuit, repetitions=reps)
+    _assert_about_equal(runtime, expected)
+
+
+@pytest.mark.parametrize(
+    "depth, width, reps, expected",
+    [
+        (10, 80, 32000, 3.7),
+        (10, 160, 32000, 4.5),
+        (10, 320, 32000, 6.6),
+        (10, 10, 32000, 3.5),
+        (20, 10, 32000, 4.6),
+        (30, 10, 32000, 5.9),
+        (40, 10, 32000, 7.7),
+        (50, 10, 32000, 9.4),
+        (10, 10, 256000, 11.4),
+        (40, 40, 256000, 26.8),
+        (40, 160, 256000, 32.8),
+        (40, 80, 32000, 12.1),
+        (2, 40, 256000, 11.3),
+        (2, 160, 256000, 11.4),
+        (2, 640, 256000, 13.3),
+        (2, 1280, 256000, 16.5),
+        (2, 2560, 256000, 23.5),
+        (10, 160, 256000, 18.2),
+        (20, 160, 256000, 24.7),
+        (30, 160, 256000, 30.8),
+        (10, 1280, 256000, 38.6),
+        (10, 1280, 1000, 18.7),
+        (10, 1280, 256000, 38.6),
+    ],
+)
+def test_estimate_run_time(depth, width, reps, expected):
+    """Test various run times.
+    Values taken from Weber November 2021."""
+    qubits = cirq.GridQubit.rect(8, 8)
+    circuit = cirq.testing.random_circuit(qubits[:depth], n_moments=width, op_density=1.0)
+    runtime = runtime_estimator.estimate_run_time(circuit, repetitions=reps)
+    _assert_about_equal(runtime, expected)
+
+
+@pytest.mark.parametrize(
+    "depth, width, reps, sweeps, expected",
+    [
+        (10, 10, 1000, 1, 2.3),
+        (10, 10, 1000, 2, 2.8),
+        (10, 10, 1000, 4, 3.2),
+        (10, 10, 1000, 8, 4.1),
+        (10, 10, 1000, 16, 6.1),
+        (10, 10, 1000, 32, 10.2),
+        (10, 10, 1000, 64, 19.2),
+        (40, 10, 1000, 2, 6.0),
+        (40, 10, 1000, 4, 7.2),
+        (40, 10, 1000, 8, 10.9),
+        (40, 10, 1000, 16, 17.2),
+        (40, 10, 1000, 32, 32.2),
+        (40, 10, 1000, 64, 61.4),
+        (40, 10, 1000, 128, 107.5),
+        (40, 160, 32000, 32, 249.7),
+        (30, 40, 32000, 32, 171.0),
+        (40, 40, 32000, 32, 206.9),
+        (40, 80, 32000, 16, 90.4),
+        (40, 80, 32000, 8, 58.7),
+        (40, 80, 8000, 32, 80.1),
+        (20, 40, 32000, 32, 69.8),
+        (30, 40, 32000, 32, 170.9),
+        (40, 40, 32000, 32, 215.4),
+        (2, 40, 16000, 16, 10.5),
+        (2, 640, 16000, 16, 16.9),
+        (2, 1280, 16000, 16, 22.6),
+        (2, 2560, 16000, 16, 38.9),
+    ],
+)
+def test_estimate_run_sweep_time(depth, width, sweeps, reps, expected):
+    """Test various run times.
+    Values taken from Weber November 2021."""
+    qubits = cirq.GridQubit.rect(8, 8)
+    circuit = cirq.testing.random_circuit(qubits[:depth], n_moments=width, op_density=1.0)
+    params = cirq.Linspace('t', 0, 1, sweeps)
+    runtime = runtime_estimator.estimate_run_sweep_time(circuit, params, repetitions=reps)
+    _assert_about_equal(runtime, expected)
+
+
+def test_estimate_run_batch_time():
+    qubits = cirq.GridQubit.rect(4, 5)
+    circuit = cirq.testing.random_circuit(qubits[:19], n_moments=40, op_density=1.0)
+    circuit2 = cirq.testing.random_circuit(qubits[:19], n_moments=40, op_density=1.0)
+    circuit3 = cirq.testing.random_circuit(qubits, n_moments=40, op_density=1.0)
+    sweeps_10 = cirq.Linspace('t', 0, 1, 10)
+    sweeps_20 = cirq.Linspace('t', 0, 1, 20)
+    sweeps_30 = cirq.Linspace('t', 0, 1, 30)
+    sweeps_40 = cirq.Linspace('t', 0, 1, 40)
+
+    # 2 batches with same qubits is the same time as a combined sweep
+    sweep_runtime = runtime_estimator.estimate_run_sweep_time(circuit, sweeps_30, repetitions=1000)
+    batch_runtime = runtime_estimator.estimate_run_batch_time(
+        [circuit, circuit2], [sweeps_10, sweeps_20], repetitions=1000
+    )
+    assert sweep_runtime == batch_runtime
+
+    # 2 batches with same qubits and 1 batch with different qubits
+    # Should be equal to combining the first two batches
+    three_batches = runtime_estimator.estimate_run_batch_time(
+        [circuit, circuit2, circuit3], [sweeps_10, sweeps_20, sweeps_10], repetitions=1000
+    )
+    two_batches = runtime_estimator.estimate_run_batch_time(
+        [circuit, circuit3], [sweeps_30, sweeps_10], repetitions=1000
+    )
+    assert three_batches == two_batches
+    # The last batch cannot be combined since it has different qubits
+    sweep_runtime = runtime_estimator.estimate_run_sweep_time(circuit, sweeps_40, repetitions=1000)
+    assert three_batches > sweep_runtime
+
+
+def test_estimate_run_batch_time_average_depths():
+    qubits = cirq.GridQubit.rect(4, 5)
+    circuit_depth_20 = cirq.testing.random_circuit(qubits, n_moments=20, op_density=1.0)
+    circuit_depth_30 = cirq.testing.random_circuit(qubits, n_moments=30, op_density=1.0)
+    circuit_depth_40 = cirq.testing.random_circuit(qubits, n_moments=40, op_density=1.0)
+    sweeps_10 = cirq.Linspace('t', 0, 1, 10)
+    sweeps_20 = cirq.Linspace('t', 0, 1, 20)
+
+    depth_20_and_40 = runtime_estimator.estimate_run_batch_time(
+        [circuit_depth_20, circuit_depth_40], [sweeps_10, sweeps_10], repetitions=1000
+    )
+    depth_30 = runtime_estimator.estimate_run_sweep_time(
+        circuit_depth_30, sweeps_20, repetitions=1000
+    )
+    depth_40 = runtime_estimator.estimate_run_sweep_time(
+        circuit_depth_40, sweeps_20, repetitions=1000
+    )
+    assert depth_20_and_40 == depth_30
+    assert depth_20_and_40 < depth_40