From 78a050ecc63c74f7d9354f5ad1656ad72e62435a Mon Sep 17 00:00:00 2001 From: guillaume-duong-bib Date: Mon, 13 May 2024 15:48:24 +0200 Subject: [PATCH 1/7] add facts deletion upon fact source deletion --- app/api/v2/handlers/fact_source_api.py | 6 ++++-- app/api/v2/managers/fact_source_manager.py | 6 ++++++ 2 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 app/api/v2/managers/fact_source_manager.py diff --git a/app/api/v2/handlers/fact_source_api.py b/app/api/v2/handlers/fact_source_api.py index 4ff87e791..4b94d9e42 100644 --- a/app/api/v2/handlers/fact_source_api.py +++ b/app/api/v2/handlers/fact_source_api.py @@ -2,7 +2,7 @@ from aiohttp import web from app.api.v2.handlers.base_object_api import BaseObjectApi -from app.api.v2.managers.base_api_manager import BaseApiManager +from app.api.v2.managers.fact_source_manager import FactSourceApiManager from app.api.v2.schemas.base_schemas import BaseGetAllQuerySchema, BaseGetOneQuerySchema from app.objects.c_source import Source, SourceSchema @@ -11,7 +11,7 @@ class FactSourceApi(BaseObjectApi): def __init__(self, services): super().__init__(description='Fact Source', obj_class=Source, schema=SourceSchema, ram_key='sources', id_property='id', auth_svc=services['auth_svc']) - self._api_manager = BaseApiManager(data_svc=services['data_svc'], file_svc=services['file_svc']) + self._api_manager = FactSourceApiManager(data_svc=services['data_svc'], file_svc=services['file_svc'], knowledge_svc=services['knowledge_svc']) def add_routes(self, app: web.Application): router = app.router @@ -102,4 +102,6 @@ async def create_or_update_source(self, request: web.Request): @aiohttp_apispec.response_schema(SourceSchema, description='Returns DELETE status.') async def delete_source(self, request: web.Request): await self.delete_on_disk_object(request) + knowledge_svc_handle = self._api_manager.knowledge_svc + await knowledge_svc_handle.delete_fact(criteria=dict(source=request.match_info.get('id'))) return web.HTTPNoContent() diff --git a/app/api/v2/managers/fact_source_manager.py b/app/api/v2/managers/fact_source_manager.py new file mode 100644 index 000000000..2a0afbb97 --- /dev/null +++ b/app/api/v2/managers/fact_source_manager.py @@ -0,0 +1,6 @@ +from app.api.v2.managers.base_api_manager import BaseApiManager + +class FactSourceApiManager(BaseApiManager): + def __init__(self, data_svc, file_svc, knowledge_svc): + super().__init__(data_svc=data_svc, file_svc=file_svc) + self.knowledge_svc = knowledge_svc From e85cc4f83463e95465dd7062d52cd1565c4bf286 Mon Sep 17 00:00:00 2001 From: guillaume-duong-bib Date: Mon, 13 May 2024 16:05:10 +0200 Subject: [PATCH 2/7] add facts deletion upon operation deletion --- app/api/v2/handlers/operation_api.py | 2 ++ app/api/v2/managers/operation_api_manager.py | 1 + 2 files changed, 3 insertions(+) diff --git a/app/api/v2/handlers/operation_api.py b/app/api/v2/handlers/operation_api.py index e8881d0f8..2d3e03747 100644 --- a/app/api/v2/handlers/operation_api.py +++ b/app/api/v2/handlers/operation_api.py @@ -132,6 +132,8 @@ async def update_operation(self, request: web.Request): description='There is an empty response from a successful delete request.') async def delete_operation(self, request: web.Request): await self.delete_object(request) + knowledge_svc_handle = self._api_manager.knowledge_svc + await knowledge_svc_handle.delete_fact(criteria=dict(source=request.match_info.get('id'))) return web.HTTPNoContent() @aiohttp_apispec.docs(tags=['operations'], diff --git a/app/api/v2/managers/operation_api_manager.py b/app/api/v2/managers/operation_api_manager.py index ada8be28f..e7409a62f 100644 --- a/app/api/v2/managers/operation_api_manager.py +++ b/app/api/v2/managers/operation_api_manager.py @@ -21,6 +21,7 @@ class OperationApiManager(BaseApiManager): def __init__(self, services): super().__init__(data_svc=services['data_svc'], file_svc=services['file_svc']) self.services = services + self.knowledge_svc = services['knowledge_svc'] async def get_operation_report(self, operation_id: str, access: dict, output: bool): operation = await self.get_operation_object(operation_id, access) From 8be9f0381ccde84da2e520b4c90be6bc1a964e84 Mon Sep 17 00:00:00 2001 From: guillaume-duong-bib Date: Mon, 13 May 2024 16:30:03 +0200 Subject: [PATCH 3/7] add relationships deletion upon fact sources deletion --- app/api/v2/handlers/fact_source_api.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/api/v2/handlers/fact_source_api.py b/app/api/v2/handlers/fact_source_api.py index 4b94d9e42..1112587e5 100644 --- a/app/api/v2/handlers/fact_source_api.py +++ b/app/api/v2/handlers/fact_source_api.py @@ -104,4 +104,5 @@ async def delete_source(self, request: web.Request): await self.delete_on_disk_object(request) knowledge_svc_handle = self._api_manager.knowledge_svc await knowledge_svc_handle.delete_fact(criteria=dict(source=request.match_info.get('id'))) + await knowledge_svc_handle.delete_relationship(criteria=dict(origin=request.match_info.get('id'))) return web.HTTPNoContent() From 735781851ddf9a93f8107f296f1d2472ccff1459 Mon Sep 17 00:00:00 2001 From: guillaume-duong-bib Date: Mon, 13 May 2024 16:31:05 +0200 Subject: [PATCH 4/7] add relationships deletion upon operation deletion --- app/api/v2/handlers/operation_api.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/api/v2/handlers/operation_api.py b/app/api/v2/handlers/operation_api.py index 2d3e03747..17bf13645 100644 --- a/app/api/v2/handlers/operation_api.py +++ b/app/api/v2/handlers/operation_api.py @@ -134,6 +134,7 @@ async def delete_operation(self, request: web.Request): await self.delete_object(request) knowledge_svc_handle = self._api_manager.knowledge_svc await knowledge_svc_handle.delete_fact(criteria=dict(source=request.match_info.get('id'))) + await knowledge_svc_handle.delete_relationship(criteria=dict(origin=request.match_info.get('id'))) return web.HTTPNoContent() @aiohttp_apispec.docs(tags=['operations'], From a253e6bb26fae9830b0c3df753153d19ea542714 Mon Sep 17 00:00:00 2001 From: guillaume-duong-bib Date: Fri, 14 Jun 2024 14:41:58 +0200 Subject: [PATCH 5/7] add get_facts_from_source --- app/service/data_svc.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/app/service/data_svc.py b/app/service/data_svc.py index 5f23a0d58..481c534ba 100644 --- a/app/service/data_svc.py +++ b/app/service/data_svc.py @@ -481,3 +481,12 @@ async def _verify_adversary_profiles(self): def _get_plugin_name(self, filename): plugin_path = pathlib.PurePath(filename).parts return plugin_path[1] if 'plugins' in plugin_path else '' + + async def get_facts_from_source(self, fact_source_id): + fact_sources = await self.locate('sources', match=dict(id=fact_source_id)) + if len(fact_sources) == 0: + return [] + elif len(fact_sources) > 1: + self.log.error('Found multiple fact sources with the same id') + return [] + return fact_sources[0].facts From 3ad88491bd9ec61e817e815f15c627002f2c0dc6 Mon Sep 17 00:00:00 2001 From: guillaume-duong-bib Date: Fri, 14 Jun 2024 14:43:49 +0200 Subject: [PATCH 6/7] retrieve seeded facts directly from source --- app/objects/c_operation.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/objects/c_operation.py b/app/objects/c_operation.py index d7b553773..7f961d8f1 100644 --- a/app/objects/c_operation.py +++ b/app/objects/c_operation.py @@ -187,9 +187,10 @@ def update_untrusted_agents(self, agent): async def all_facts(self): knowledge_svc_handle = BaseService.get_service('knowledge_svc') + data_svc_handle = BaseService.get_service('data_svc') seeded_facts = [] if self.source: - seeded_facts = await knowledge_svc_handle.get_facts(criteria=dict(source=self.source.id)) + seeded_facts = await data_svc_handle.get_facts_from_source(self.source.id) learned_facts = await knowledge_svc_handle.get_facts(criteria=dict(source=self.id)) learned_facts = [f for f in learned_facts if f.score > 0] return seeded_facts + learned_facts From ef24d733704849250652cc120b8181b15090b1af Mon Sep 17 00:00:00 2001 From: guillaume-duong-bib Date: Tue, 18 Jun 2024 10:36:34 +0200 Subject: [PATCH 7/7] update logging --- app/service/data_svc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/service/data_svc.py b/app/service/data_svc.py index 481c534ba..f768823bd 100644 --- a/app/service/data_svc.py +++ b/app/service/data_svc.py @@ -487,6 +487,6 @@ async def get_facts_from_source(self, fact_source_id): if len(fact_sources) == 0: return [] elif len(fact_sources) > 1: - self.log.error('Found multiple fact sources with the same id') + self.log.error('Found multiple fact sources with the same id', fact_source_id) return [] return fact_sources[0].facts