fixed type annotations

src/intelligence_layer/core/evaluation/evaluator.py

@@ -9,6 +9,7 @@
     Mapping,
     Optional,
     Sequence,
+    Tuple,
     TypeVar,
     cast,
     final,
@@ -432,14 +433,6 @@ def evaluate(
     ) -> None:
         ...
 
-    def evaluate_not_abstract(
-        self,
-        example: Example[Input, ExpectedOutput],
-        eval_id: str,
-        *example_output: SuccessfulExampleOutput[Output],
-    ) -> None:
-        self.evaluate(example, eval_id, example_output)
-
     @abstractmethod
     def aggregate(self, evaluations: Iterable[Evaluation]) -> AggregatedEvaluation:
         """`Evaluator`-specific method for aggregating individual `Evaluations` into report-like `Aggregated Evaluation`.
@@ -530,31 +523,38 @@ def load_overview(run_id: str) -> RunOverview:
             strict=True,
         )
 
-        evaluation_inputs = []
-
-        for example_outputs in examples_zipped:
-            if not any(
-                isinstance(output.output, FailedExampleRun)
-                for output in example_outputs
-            ):
-                example_id = example_outputs[0].example_id
-                assert all(
-                    example_output.example_id == example_id
-                    for example_output in example_outputs
-                )
-                example = self._dataset_repository.example(
-                    dataset_id,
-                    example_id,
-                    self.input_type(),
-                    self.expected_output_type(),
-                )
-                assert example is not None
+        def generate_evaluation_inputs() -> (
+            Iterable[
+                Tuple[
+                    Example[Input, ExpectedOutput],
+                    str,
+                    Sequence[SuccessfulExampleOutput[Output]],
+                ]
+            ]
+        ):
+            for example_outputs in examples_zipped:
+                if not any(
+                    isinstance(output.output, FailedExampleRun)
+                    for output in example_outputs
+                ):
+                    example_id = example_outputs[0].example_id
+                    assert all(
+                        example_output.example_id == example_id
+                        for example_output in example_outputs
+                    )
 
-                evaluation_inputs.append(
-                    (
+                    example = self._dataset_repository.example(
+                        dataset_id,
+                        example_id,
+                        self.input_type(),
+                        self.expected_output_type(),
+                    )
+                    assert example is not None
+
+                    yield (
                         example,
                         eval_id,
-                        *[
+                        [
                             SuccessfulExampleOutput(
                                 run_id=example_output.run_id,
                                 example_id=example_output.example_id,
@@ -564,11 +564,20 @@ def load_overview(run_id: str) -> RunOverview:
                             if not isinstance(example_output.output, FailedExampleRun)
                         ],
                     )
-                )
 
-        with ThreadPoolExecutor(max_workers=1) as executor:
+        def evaluate(
+            args: Tuple[
+                Example[Input, ExpectedOutput],
+                str,
+                Sequence[SuccessfulExampleOutput[Output]],
+            ]
+        ) -> None:
+            example, eval_id, example_outputs = args
+            self.evaluate(example, eval_id, *example_outputs)
+
+        with ThreadPoolExecutor(max_workers=10) as executor:
             tqdm(
-                executor.map(lambda args: self.evaluate(*args), evaluation_inputs),
+                executor.map(evaluate, generate_evaluation_inputs()),
                 desc="Evaluating",
             )