From 180626cac0783e093a20e655a0ddf24a7ecf51cb Mon Sep 17 00:00:00 2001 From: Benny Date: Tue, 9 Jul 2024 12:13:30 +0200 Subject: [PATCH 1/5] Recalculate bit if a config object changes --- octopoes/octopoes/core/service.py | 33 ++++++++++++++----- .../repositories/origin_repository.py | 5 +++ 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/octopoes/octopoes/core/service.py b/octopoes/octopoes/core/service.py index 67ea7956c97..28c18d414d3 100644 --- a/octopoes/octopoes/core/service.py +++ b/octopoes/octopoes/core/service.py @@ -2,6 +2,7 @@ from collections import Counter from collections.abc import Callable, ValuesView from datetime import datetime, timezone +from itertools import chain from logging import getLogger from typing import overload @@ -28,6 +29,7 @@ ) from octopoes.models.exception import ObjectNotFoundException from octopoes.models.explanation import InheritanceSection +from octopoes.models.ooi.config import Config from octopoes.models.origin import Origin, OriginParameter, OriginType from octopoes.models.pagination import Paginated from octopoes.models.path import ( @@ -400,16 +402,29 @@ def _on_update_ooi(self, event: OOIDBEvent) -> None: if event.new_data is None: raise ValueError("Update event new_data should not be None") - inference_origins = self.origin_repository.list_origins(event.valid_time, source=event.new_data.reference) - inference_params = self.origin_parameter_repository.list_by_reference( - event.new_data.reference, valid_time=event.valid_time - ) - for inference_param in inference_params: - inference_origins.append(self.origin_repository.get(inference_param.origin_id, event.valid_time)) + if isinstance(event.new_data, Config): + relevant_bits = [bit for bit in get_bit_definitions().values() if bit.config_ooi_relation_path is not None] + inference_origins = list( + { + id(x): x + for x in chain.from_iterable( + map(lambda x: self.origin_repository.list_origins(event.valid_time, method=x.id), relevant_bits) + ) + }.values() + ) + for inference_origin in inference_origins: + self._run_inference(inference_origin, event.valid_time) + else: + inference_origins = self.origin_repository.list_origins(event.valid_time, source=event.new_data.reference) + inference_params = self.origin_parameter_repository.list_by_reference( + event.new_data.reference, valid_time=event.valid_time + ) + for inference_param in inference_params: + inference_origins.append(self.origin_repository.get(inference_param.origin_id, event.valid_time)) - inference_origins = [o for o in inference_origins if o.origin_type == OriginType.INFERENCE] - for inference_origin in inference_origins: - self._run_inference(inference_origin, event.valid_time) + inference_origins = [o for o in inference_origins if o.origin_type == OriginType.INFERENCE] + for inference_origin in inference_origins: + self._run_inference(inference_origin, event.valid_time) def _on_delete_ooi(self, event: OOIDBEvent) -> None: if event.old_data is None: diff --git a/octopoes/octopoes/repositories/origin_repository.py b/octopoes/octopoes/repositories/origin_repository.py index c21dc4854cc..f2bab9a6ea8 100644 --- a/octopoes/octopoes/repositories/origin_repository.py +++ b/octopoes/octopoes/repositories/origin_repository.py @@ -37,6 +37,7 @@ def list_origins( task_id: UUID | None = None, source: Reference | None = None, result: Reference | None = None, + method: str | None = None, origin_type: OriginType | None = None, ) -> list[Origin]: raise NotImplementedError @@ -73,6 +74,7 @@ def list_origins( task_id: UUID | None = None, source: Reference | None = None, result: Reference | None = None, + method: str | None = None, origin_type: OriginType | None = None, ) -> list[Origin]: where_parameters = {"type": Origin.__name__} @@ -86,6 +88,9 @@ def list_origins( if result: where_parameters["result"] = str(result) + if method: + where_parameters["method"] = method + if origin_type: where_parameters["origin_type"] = origin_type.value From 7905e24f068b473098f36bbc6b2d7186b45d9c5b Mon Sep 17 00:00:00 2001 From: Benny Date: Tue, 9 Jul 2024 12:27:30 +0200 Subject: [PATCH 2/5] Use the right id --- octopoes/octopoes/core/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octopoes/octopoes/core/service.py b/octopoes/octopoes/core/service.py index 28c18d414d3..7c614a8d920 100644 --- a/octopoes/octopoes/core/service.py +++ b/octopoes/octopoes/core/service.py @@ -406,7 +406,7 @@ def _on_update_ooi(self, event: OOIDBEvent) -> None: relevant_bits = [bit for bit in get_bit_definitions().values() if bit.config_ooi_relation_path is not None] inference_origins = list( { - id(x): x + x.task_id: x for x in chain.from_iterable( map(lambda x: self.origin_repository.list_origins(event.valid_time, method=x.id), relevant_bits) ) From 84bfedd46a7a39d1ff1a00c5861ed48e954b84c1 Mon Sep 17 00:00:00 2001 From: Benny Date: Tue, 9 Jul 2024 13:09:31 +0200 Subject: [PATCH 3/5] Use model_dump for unhashable object --- octopoes/octopoes/core/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octopoes/octopoes/core/service.py b/octopoes/octopoes/core/service.py index 7c614a8d920..44df713f321 100644 --- a/octopoes/octopoes/core/service.py +++ b/octopoes/octopoes/core/service.py @@ -406,7 +406,7 @@ def _on_update_ooi(self, event: OOIDBEvent) -> None: relevant_bits = [bit for bit in get_bit_definitions().values() if bit.config_ooi_relation_path is not None] inference_origins = list( { - x.task_id: x + x.model_dump_json(): x for x in chain.from_iterable( map(lambda x: self.origin_repository.list_origins(event.valid_time, method=x.id), relevant_bits) ) From f1980e2550b1ead1fe9c4c1e0fdbe26045e20c9a Mon Sep 17 00:00:00 2001 From: Benny Date: Tue, 9 Jul 2024 13:24:23 +0200 Subject: [PATCH 4/5] Compress code --- octopoes/octopoes/core/service.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/octopoes/octopoes/core/service.py b/octopoes/octopoes/core/service.py index 44df713f321..910dc0eb262 100644 --- a/octopoes/octopoes/core/service.py +++ b/octopoes/octopoes/core/service.py @@ -412,8 +412,6 @@ def _on_update_ooi(self, event: OOIDBEvent) -> None: ) }.values() ) - for inference_origin in inference_origins: - self._run_inference(inference_origin, event.valid_time) else: inference_origins = self.origin_repository.list_origins(event.valid_time, source=event.new_data.reference) inference_params = self.origin_parameter_repository.list_by_reference( @@ -423,8 +421,8 @@ def _on_update_ooi(self, event: OOIDBEvent) -> None: inference_origins.append(self.origin_repository.get(inference_param.origin_id, event.valid_time)) inference_origins = [o for o in inference_origins if o.origin_type == OriginType.INFERENCE] - for inference_origin in inference_origins: - self._run_inference(inference_origin, event.valid_time) + for inference_origin in inference_origins: + self._run_inference(inference_origin, event.valid_time) def _on_delete_ooi(self, event: OOIDBEvent) -> None: if event.old_data is None: From 97b07513ad3f4046a4b9dfd2de97ca67e8e586c2 Mon Sep 17 00:00:00 2001 From: Benny Date: Tue, 9 Jul 2024 15:09:10 +0200 Subject: [PATCH 5/5] Lesten queries --- octopoes/octopoes/core/service.py | 14 ++++---------- .../octopoes/repositories/origin_repository.py | 6 +++--- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/octopoes/octopoes/core/service.py b/octopoes/octopoes/core/service.py index 910dc0eb262..1243ae629ae 100644 --- a/octopoes/octopoes/core/service.py +++ b/octopoes/octopoes/core/service.py @@ -2,7 +2,6 @@ from collections import Counter from collections.abc import Callable, ValuesView from datetime import datetime, timezone -from itertools import chain from logging import getLogger from typing import overload @@ -403,15 +402,10 @@ def _on_update_ooi(self, event: OOIDBEvent) -> None: raise ValueError("Update event new_data should not be None") if isinstance(event.new_data, Config): - relevant_bits = [bit for bit in get_bit_definitions().values() if bit.config_ooi_relation_path is not None] - inference_origins = list( - { - x.model_dump_json(): x - for x in chain.from_iterable( - map(lambda x: self.origin_repository.list_origins(event.valid_time, method=x.id), relevant_bits) - ) - }.values() - ) + relevant_bit_ids = [ + bit.id for bit in get_bit_definitions().values() if bit.config_ooi_relation_path is not None + ] + inference_origins = self.origin_repository.list_origins(event.valid_time, method=relevant_bit_ids) else: inference_origins = self.origin_repository.list_origins(event.valid_time, source=event.new_data.reference) inference_params = self.origin_parameter_repository.list_by_reference( diff --git a/octopoes/octopoes/repositories/origin_repository.py b/octopoes/octopoes/repositories/origin_repository.py index f2bab9a6ea8..1f9ca2e9d64 100644 --- a/octopoes/octopoes/repositories/origin_repository.py +++ b/octopoes/octopoes/repositories/origin_repository.py @@ -37,7 +37,7 @@ def list_origins( task_id: UUID | None = None, source: Reference | None = None, result: Reference | None = None, - method: str | None = None, + method: str | list[str] | None = None, origin_type: OriginType | None = None, ) -> list[Origin]: raise NotImplementedError @@ -74,10 +74,10 @@ def list_origins( task_id: UUID | None = None, source: Reference | None = None, result: Reference | None = None, - method: str | None = None, + method: str | list[str] | None = None, origin_type: OriginType | None = None, ) -> list[Origin]: - where_parameters = {"type": Origin.__name__} + where_parameters: dict[str, str | list[str]] = {"type": Origin.__name__} if task_id: where_parameters["task_id"] = str(task_id)