Include calculated error in calibrator logs

Add calibrator run parameter log_errors that
allows to append errors calculated during calibration.

Calibration log table is sorted so that best
techniques are on the top of the table.

Fixes #2014

AUTHORS

@@ -52,3 +52,4 @@ Preksha Naik
 Abdulrahman Sahmoud
 Mohammad Zuhair Khan
 Brian Goldsmith
+Vladimir Kozhukalov

mitiq/calibration/calibrator.py

@@ -6,6 +6,7 @@
 import warnings
 from itertools import product
 from typing import (
+    Any,
     Callable,
     Dict,
     Iterator,
@@ -72,17 +73,28 @@ def add_result(
         self.noisy[strategy.id, problem.id] = noisy_val
         self.ideal[strategy.id, problem.id] = ideal_val
 
-    def _get_performance_symbol(
+    def _get_performance(
         self, strategy_id: int, problem_id: int
-    ) -> str:
-        """Returns ✔ the strategy performed better than no mitigation on this
-        problem,  and ✘ otherwise."""
+    ) -> Tuple[str, float, float]:
+        """Get performance symbol and errors.
+
+        Returns:
+            A performance tuple comprising:
+            - performance symbol either ✔ or ✘ depending
+              on whether mitigation technique works well or not.
+              It considered to work well if the mitigated error is less
+              than the noisy error.
+            - the absolute value of the noisy error
+            - the absolute value of the mitigated error
+        """
         mitigated = self.mitigated[strategy_id, problem_id]
         noisy = self.noisy[strategy_id, problem_id]
         ideal = self.ideal[strategy_id, problem_id]
-        mitigation_worked = abs(ideal - mitigated) < abs(ideal - noisy)
-        performance = "✔" if mitigation_worked else "✘"
-        return performance
+        mitigated_error = abs(ideal - mitigated)
+        noisy_error = abs(ideal - noisy)
+        mitigation_worked = mitigated_error < noisy_error
+        performance_symbol = "✔" if mitigation_worked else "✘"
+        return performance_symbol, noisy_error, mitigated_error
 
     def unique_techniques(self) -> Set[MitigationTechnique]:
         """Returns the unique mitigation techniques used across this
@@ -91,24 +103,25 @@ def unique_techniques(self) -> Set[MitigationTechnique]:
 
     def _technique_results(
         self, technique: MitigationTechnique
-    ) -> Iterator[Tuple[BenchmarkProblem, Strategy, str]]:
+    ) -> Iterator[Tuple[BenchmarkProblem, Strategy, str, float, float]]:
         """Yields the results from this collection of experiment results,
         limited to a specific technique."""
         for strategy, problem in product(self.strategies, self.problems):
             if strategy.technique is technique:
-                performance = self._get_performance_symbol(
-                    strategy.id, problem.id
-                )
-                yield problem, strategy, performance
+                performance_symbol, nerr, merr = \
+                    self._get_performance(
+                        strategy.id, problem.id
+                    )
+                yield problem, strategy, performance_symbol, nerr, merr
 
-    def log_technique(self, technique: MitigationTechnique) -> str:
+    def log_technique(self, technique: MitigationTechnique, log_errors: bool = False) -> str:
         """Creates a table displaying all results of a given mitigation
         technique."""
-        table = []
-        for problem, strategy, performance in self._technique_results(
+        table: List[Any] = []
+        for problem, strategy, performance_symbol, nerr, merr in self._technique_results(
             technique
         ):
-            row = [performance, problem.type, technique.name]
+            row = [performance_symbol, problem.type, technique.name]
             summary_dict = strategy.to_pretty_dict()
             if strategy.technique is MitigationTechnique.ZNE:
                 row.extend(
@@ -125,9 +138,17 @@ def log_technique(self, technique: MitigationTechnique) -> str:
                         summary_dict["representation_function"],
                     ]
                 )
-
+            row.extend([nerr, merr])
             table.append(row)
 
+        def _sort_best_perf(row):
+            return row[-1] - row[-2]
+
+        table.sort(key=_sort_best_perf)
+
+        if not log_errors:
+            table = [row[:-2] for row in table]
+
         if technique is MitigationTechnique.ZNE:
             headers = [
                 "performance",
@@ -146,15 +167,18 @@ def log_technique(self, technique: MitigationTechnique) -> str:
                 "noise representation",
             ]
 
+        if log_errors:
+            headers.extend(["noisy error", "mitigated error"])
+
         return tabulate(table, headers, tablefmt="simple_grid")
 
-    def log_results(self) -> None:
+    def log_results(self, log_errors: bool = False) -> None:
         """Log results from entire calibration run. Logging is performed on
         each mitigation technique individually to avoid confusion when many
         techniques are used."""
         for mitigation_technique in self.unique_techniques():
             print(f"{mitigation_technique.name} results:")
-            print(self.log_technique(mitigation_technique))
+            print(self.log_technique(mitigation_technique, log_errors))
             print()
 
     def is_missing_data(self) -> bool:
@@ -279,7 +303,7 @@ def get_cost(self) -> Dict[str, int]:
             "ideal_executions": ideal,
         }
 
-    def run(self, log: bool = False) -> None:
+    def run(self, log: bool = False, log_errors: bool = False) -> None:
         """Runs all the circuits required for calibration."""
         if not self.results.is_missing_data():
             self.results.reset_data()
@@ -311,7 +335,7 @@ def run(self, log: bool = False) -> None:
         self.results.ensure_full()
 
         if log:
-            self.results.log_results()
+            self.results.log_results(log_errors)
 
     def best_strategy(self) -> Strategy:
         """Finds the best strategy by using the parameters that had the