From 1afbe3dea51758a71936a55a51bdd80c7af0f1f1 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Tue, 28 Apr 2020 16:50:01 +0300 Subject: [PATCH 01/15] Improved ZT event aggregation performance --- .../monkey_island/cc/models/zero_trust/aggregate_finding.py | 4 ++-- monkey/monkey_island/cc/models/zero_trust/finding.py | 4 ---- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/monkey/monkey_island/cc/models/zero_trust/aggregate_finding.py b/monkey/monkey_island/cc/models/zero_trust/aggregate_finding.py index 38b5510478c..ff3c4e4d955 100644 --- a/monkey/monkey_island/cc/models/zero_trust/aggregate_finding.py +++ b/monkey/monkey_island/cc/models/zero_trust/aggregate_finding.py @@ -12,7 +12,7 @@ def create_or_add_to_existing(test, status, events): :raises: Assertion error if this is used when there's more then one finding which fits the query - this is not when this function should be used. """ - existing_findings = Finding.objects(test=test, status=status) + existing_findings = Finding.objects(test=test, status=status).exclude('events') assert (len(existing_findings) < 2), "More than one finding exists for {}:{}".format(test, status) if len(existing_findings) == 0: @@ -20,7 +20,7 @@ def create_or_add_to_existing(test, status, events): else: # Now we know for sure this is the only one orig_finding = existing_findings[0] - orig_finding.add_events(events) + orig_finding.update(push_all__events=events) orig_finding.save() diff --git a/monkey/monkey_island/cc/models/zero_trust/finding.py b/monkey/monkey_island/cc/models/zero_trust/finding.py index 90c9e1dc3cc..ae1114655b9 100644 --- a/monkey/monkey_island/cc/models/zero_trust/finding.py +++ b/monkey/monkey_island/cc/models/zero_trust/finding.py @@ -54,7 +54,3 @@ def save_finding(test, status, events): finding.save() return finding - - def add_events(self, events): - # type: (list) -> None - self.events.extend(events) From f73beac3a7ac3c85baf26237cf49f61587e7491e Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Thu, 30 Apr 2020 16:12:58 +0300 Subject: [PATCH 02/15] Implemented map/report generation tests which are based on telemetries rather than real exploitation --- envs/monkey_zoo/blackbox/test_blackbox.py | 17 ++++++--- .../map_generation_from_telemetries.py | 31 ++++++++++++++++ .../performance/performance_test_workflow.py | 2 ++ .../report_generation_from_telemetries.py | 35 +++++++++++++++++++ .../telemetry_performance_test_workflow.py | 21 +++++++++++ 5 files changed, 102 insertions(+), 4 deletions(-) create mode 100644 envs/monkey_zoo/blackbox/tests/performance/map_generation_from_telemetries.py create mode 100644 envs/monkey_zoo/blackbox/tests/performance/report_generation_from_telemetries.py create mode 100644 envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test_workflow.py diff --git a/envs/monkey_zoo/blackbox/test_blackbox.py b/envs/monkey_zoo/blackbox/test_blackbox.py index c0a7caf3d97..35db89eb498 100644 --- a/envs/monkey_zoo/blackbox/test_blackbox.py +++ b/envs/monkey_zoo/blackbox/test_blackbox.py @@ -10,7 +10,10 @@ from envs.monkey_zoo.blackbox.log_handlers.test_logs_handler import TestLogsHandler from envs.monkey_zoo.blackbox.tests.exploitation import ExploitationTest from envs.monkey_zoo.blackbox.tests.performance.map_generation import MapGenerationTest +from envs.monkey_zoo.blackbox.tests.performance.map_generation_from_telemetries import MapGenerationFromTelemetryTest from envs.monkey_zoo.blackbox.tests.performance.report_generation import ReportGenerationTest +from envs.monkey_zoo.blackbox.tests.performance.report_generation_from_telemetries import \ + ReportGenerationFromTelemetryTest from envs.monkey_zoo.blackbox.tests.performance.telemetry_performance_test import TelemetryPerformanceTest from envs.monkey_zoo.blackbox.utils import gcp_machine_handlers @@ -26,11 +29,11 @@ @pytest.fixture(autouse=True, scope='session') def GCPHandler(request): GCPHandler = gcp_machine_handlers.GCPHandler() - GCPHandler.start_machines(" ".join(GCP_TEST_MACHINE_LIST)) - wait_machine_bootup() + #GCPHandler.start_machines(" ".join(GCP_TEST_MACHINE_LIST)) + #wait_machine_bootup() def fin(): - GCPHandler.stop_machines(" ".join(GCP_TEST_MACHINE_LIST)) + #GCPHandler.stop_machines(" ".join(GCP_TEST_MACHINE_LIST)) pass request.addfinalizer(fin) @@ -49,7 +52,7 @@ def wait_machine_bootup(): @pytest.fixture(scope='class') def island_client(island): island_client_object = MonkeyIslandClient(island) - island_client_object.reset_env() + # island_client_object.reset_env() yield island_client_object @@ -147,5 +150,11 @@ def test_map_generation_performance(self, island_client): "PERFORMANCE.conf", timeout_in_seconds=10*60) + def test_report_generation_from_fake_telemetries(self, island_client): + ReportGenerationFromTelemetryTest(island_client).run() + + def test_map_generation_from_fake_telemetries(self, island_client): + MapGenerationFromTelemetryTest(island_client).run() + def test_telem_performance(self, island_client): TelemetryPerformanceTest(island_client).test_telemetry_performance() diff --git a/envs/monkey_zoo/blackbox/tests/performance/map_generation_from_telemetries.py b/envs/monkey_zoo/blackbox/tests/performance/map_generation_from_telemetries.py new file mode 100644 index 00000000000..c5344d8f7f3 --- /dev/null +++ b/envs/monkey_zoo/blackbox/tests/performance/map_generation_from_telemetries.py @@ -0,0 +1,31 @@ +from datetime import timedelta + +from envs.monkey_zoo.blackbox.tests.performance.performance_test import PerformanceTest +from envs.monkey_zoo.blackbox.tests.performance.performance_test_config import PerformanceTestConfig +from envs.monkey_zoo.blackbox.tests.performance.telemetry_performance_test_workflow import \ + TelemetryPerformanceTestWorkflow + +MAX_ALLOWED_SINGLE_PAGE_TIME = timedelta(seconds=2) +MAX_ALLOWED_TOTAL_TIME = timedelta(seconds=5) + +MAP_RESOURCES = [ + "api/netmap", +] + + +class MapGenerationFromTelemetryTest(PerformanceTest): + + TEST_NAME = "Map generation from fake telemetries test" + + def __init__(self, island_client, break_on_timeout=False): + self.island_client = island_client + performance_config = PerformanceTestConfig(max_allowed_single_page_time=MAX_ALLOWED_SINGLE_PAGE_TIME, + max_allowed_total_time=MAX_ALLOWED_TOTAL_TIME, + endpoints_to_test=MAP_RESOURCES, + break_on_timeout=break_on_timeout) + self.performance_test_workflow = TelemetryPerformanceTestWorkflow(MapGenerationFromTelemetryTest.TEST_NAME, + self.island_client, + performance_config) + + def run(self): + self.performance_test_workflow.run() diff --git a/envs/monkey_zoo/blackbox/tests/performance/performance_test_workflow.py b/envs/monkey_zoo/blackbox/tests/performance/performance_test_workflow.py index f6cd1dada29..cdb4f08acc2 100644 --- a/envs/monkey_zoo/blackbox/tests/performance/performance_test_workflow.py +++ b/envs/monkey_zoo/blackbox/tests/performance/performance_test_workflow.py @@ -25,6 +25,8 @@ def run(self): self.exploitation_test.wait_for_monkey_process_to_finish() performance_test = EndpointPerformanceTest(self.name, self.performance_config, self.island_client) try: + if not self.island_client.is_all_monkeys_dead(): + raise RuntimeError("Can't test report times since not all Monkeys have died.") assert performance_test.run() finally: self.exploitation_test.parse_logs() diff --git a/envs/monkey_zoo/blackbox/tests/performance/report_generation_from_telemetries.py b/envs/monkey_zoo/blackbox/tests/performance/report_generation_from_telemetries.py new file mode 100644 index 00000000000..a08bbda701c --- /dev/null +++ b/envs/monkey_zoo/blackbox/tests/performance/report_generation_from_telemetries.py @@ -0,0 +1,35 @@ +from datetime import timedelta + +from envs.monkey_zoo.blackbox.tests.performance.performance_test import PerformanceTest +from envs.monkey_zoo.blackbox.tests.performance.performance_test_config import PerformanceTestConfig +from envs.monkey_zoo.blackbox.tests.performance.telemetry_performance_test_workflow import \ + TelemetryPerformanceTestWorkflow + +MAX_ALLOWED_SINGLE_PAGE_TIME = timedelta(seconds=2) +MAX_ALLOWED_TOTAL_TIME = timedelta(seconds=5) + +REPORT_RESOURCES = [ + "api/report/security", + "api/attack/report", + "api/report/zero_trust/findings", + "api/report/zero_trust/principles", + "api/report/zero_trust/pillars" +] + + +class ReportGenerationFromTelemetryTest(PerformanceTest): + + TEST_NAME = "Map generation from fake telemetries test" + + def __init__(self, island_client, break_on_timeout=False): + self.island_client = island_client + performance_config = PerformanceTestConfig(max_allowed_single_page_time=MAX_ALLOWED_SINGLE_PAGE_TIME, + max_allowed_total_time=MAX_ALLOWED_TOTAL_TIME, + endpoints_to_test=REPORT_RESOURCES, + break_on_timeout=break_on_timeout) + self.performance_test_workflow = TelemetryPerformanceTestWorkflow(ReportGenerationFromTelemetryTest.TEST_NAME, + self.island_client, + performance_config) + + def run(self): + self.performance_test_workflow.run() diff --git a/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test_workflow.py b/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test_workflow.py new file mode 100644 index 00000000000..320973d9736 --- /dev/null +++ b/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test_workflow.py @@ -0,0 +1,21 @@ +from envs.monkey_zoo.blackbox.tests.basic_test import BasicTest +from envs.monkey_zoo.blackbox.tests.performance.endpoint_performance_test import EndpointPerformanceTest +from envs.monkey_zoo.blackbox.tests.performance.performance_test_config import PerformanceTestConfig +from envs.monkey_zoo.blackbox.tests.performance.telemetry_performance_test import TelemetryPerformanceTest + + +class TelemetryPerformanceTestWorkflow(BasicTest): + + def __init__(self, name, island_client, performance_config: PerformanceTestConfig): + self.name = name + self.island_client = island_client + self.performance_config = performance_config + + def run(self): + try: + # TelemetryPerformanceTest(island_client=self.island_client).test_telemetry_performance() + performance_test = EndpointPerformanceTest(self.name, self.performance_config, self.island_client) + assert performance_test.run() + finally: + pass + # self.island_client.reset_env() From 8603d1887958015d39daeb3d02bd9b0271773beb Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Thu, 30 Apr 2020 16:14:31 +0300 Subject: [PATCH 03/15] Added a profiling decorator, that can be used on methods to get their performance info --- .../cc/testing/profiler_decorator.py | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 monkey/monkey_island/cc/testing/profiler_decorator.py diff --git a/monkey/monkey_island/cc/testing/profiler_decorator.py b/monkey/monkey_island/cc/testing/profiler_decorator.py new file mode 100644 index 00000000000..403548adde6 --- /dev/null +++ b/monkey/monkey_island/cc/testing/profiler_decorator.py @@ -0,0 +1,25 @@ +from cProfile import Profile +import pstats + + +def profile(sort_args=['cumulative'], print_args=[100]): + profiler = Profile() + + def decorator(fn): + def inner(*args, **kwargs): + result = None + try: + result = profiler.runcall(fn, *args, **kwargs) + finally: + filename = _get_filename_for_function(fn) + with open(filename, 'w') as stream: + stats = pstats.Stats(profiler, stream=stream) + stats.strip_dirs().sort_stats(*sort_args).print_stats(*print_args) + return result + return inner + return decorator + + +def _get_filename_for_function(fn): + function_name = fn.__module__ + "." + fn.__name__ + return function_name.replace(".", "_") From 4dcae80a648bd8249e4f72822156b2600b9f6e64 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Thu, 30 Apr 2020 16:17:54 +0300 Subject: [PATCH 04/15] Improved ZT report generation performance. --- .../cc/services/reporting/zero_trust_service.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/monkey/monkey_island/cc/services/reporting/zero_trust_service.py b/monkey/monkey_island/cc/services/reporting/zero_trust_service.py index dd6fad1bc97..594aba70330 100644 --- a/monkey/monkey_island/cc/services/reporting/zero_trust_service.py +++ b/monkey/monkey_island/cc/services/reporting/zero_trust_service.py @@ -9,13 +9,13 @@ class ZeroTrustService(object): @staticmethod def get_pillars_grades(): pillars_grades = [] + all_findings = Finding.objects().exclude('events') for pillar in zero_trust_consts.PILLARS: - pillars_grades.append(ZeroTrustService.__get_pillar_grade(pillar)) + pillars_grades.append(ZeroTrustService.__get_pillar_grade(pillar, all_findings)) return pillars_grades @staticmethod - def __get_pillar_grade(pillar): - all_findings = Finding.objects() + def __get_pillar_grade(pillar, all_findings): pillar_grade = { "pillar": pillar, zero_trust_consts.STATUS_FAILED: 0, @@ -147,7 +147,8 @@ def get_pillars_to_statuses(): @staticmethod def __get_status_of_single_pillar(pillar): - grade = ZeroTrustService.__get_pillar_grade(pillar) + all_findings = Finding.objects().exclude('events') + grade = ZeroTrustService.__get_pillar_grade(pillar, all_findings) for status in zero_trust_consts.ORDERED_TEST_STATUSES: if grade[status] > 0: return status From 9be8d4af1bcd7bc3a047ffd6505c9a0fe2e7107b Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Thu, 30 Apr 2020 16:28:46 +0300 Subject: [PATCH 05/15] Fixed log paths for profiling decorator --- monkey/monkey_island/cc/testing/profiler_decorator.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/monkey/monkey_island/cc/testing/profiler_decorator.py b/monkey/monkey_island/cc/testing/profiler_decorator.py index 403548adde6..dd08eec2e2b 100644 --- a/monkey/monkey_island/cc/testing/profiler_decorator.py +++ b/monkey/monkey_island/cc/testing/profiler_decorator.py @@ -1,6 +1,9 @@ from cProfile import Profile +import os import pstats +PROFILER_LOG_DIR = "./profiler_logs/" + def profile(sort_args=['cumulative'], print_args=[100]): profiler = Profile() @@ -11,7 +14,11 @@ def inner(*args, **kwargs): try: result = profiler.runcall(fn, *args, **kwargs) finally: - filename = _get_filename_for_function(fn) + try: + os.mkdir(PROFILER_LOG_DIR) + except os.error: + pass + filename = PROFILER_LOG_DIR + _get_filename_for_function(fn) with open(filename, 'w') as stream: stats = pstats.Stats(profiler, stream=stream) stats.strip_dirs().sort_stats(*sort_args).print_stats(*print_args) From 7a13e71588be682c5147570572013dd88c58859d Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Thu, 30 Apr 2020 17:45:16 +0300 Subject: [PATCH 06/15] More simple ZT performance improvements and profiler decorator bugfix --- .../monkey_island/cc/services/reporting/zero_trust_service.py | 4 ++-- monkey/monkey_island/cc/testing/profiler_decorator.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/monkey/monkey_island/cc/services/reporting/zero_trust_service.py b/monkey/monkey_island/cc/services/reporting/zero_trust_service.py index 594aba70330..5a2045da53d 100644 --- a/monkey/monkey_island/cc/services/reporting/zero_trust_service.py +++ b/monkey/monkey_island/cc/services/reporting/zero_trust_service.py @@ -65,7 +65,7 @@ def __get_principle_status(principle_tests): worst_status = zero_trust_consts.STATUS_UNEXECUTED all_statuses = set() for test in principle_tests: - all_statuses |= set(Finding.objects(test=test).distinct("status")) + all_statuses |= set(Finding.objects(test=test).exclude('events').distinct("status")) for status in all_statuses: if zero_trust_consts.ORDERED_TEST_STATUSES.index(status) \ @@ -78,7 +78,7 @@ def __get_principle_status(principle_tests): def __get_tests_status(principle_tests): results = [] for test in principle_tests: - test_findings = Finding.objects(test=test) + test_findings = Finding.objects(test=test).exclude('events') results.append( { "test": zero_trust_consts.TESTS_MAP[test][zero_trust_consts.TEST_EXPLANATION_KEY], diff --git a/monkey/monkey_island/cc/testing/profiler_decorator.py b/monkey/monkey_island/cc/testing/profiler_decorator.py index dd08eec2e2b..997ef91aebc 100644 --- a/monkey/monkey_island/cc/testing/profiler_decorator.py +++ b/monkey/monkey_island/cc/testing/profiler_decorator.py @@ -6,12 +6,12 @@ def profile(sort_args=['cumulative'], print_args=[100]): - profiler = Profile() def decorator(fn): def inner(*args, **kwargs): result = None try: + profiler = Profile() result = profiler.runcall(fn, *args, **kwargs) finally: try: From 8a385eca9367dfc0b2c6aed6bdde527b2261fcb1 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Mon, 4 May 2020 16:47:17 +0300 Subject: [PATCH 07/15] Style fix for modal window and report tabs --- monkey/monkey_island/cc/ui/src/styles/report/ReportPage.scss | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monkey/monkey_island/cc/ui/src/styles/report/ReportPage.scss b/monkey/monkey_island/cc/ui/src/styles/report/ReportPage.scss index 4b1bd60c947..4429b4b6afb 100644 --- a/monkey/monkey_island/cc/ui/src/styles/report/ReportPage.scss +++ b/monkey/monkey_island/cc/ui/src/styles/report/ReportPage.scss @@ -2,7 +2,7 @@ margin-bottom: 2em !important; position: sticky; top: 0; - z-index: 1000000; + z-index: 1000; background-color: #ffffff; font-size: large; } From 4073e2f41f1be98dfc92b43e18905f1a2bbf05c8 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Wed, 6 May 2020 16:50:17 +0300 Subject: [PATCH 08/15] Fixed zero trust bug where all events had the same timestamp --- monkey/monkey_island/cc/models/zero_trust/event.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/monkey/monkey_island/cc/models/zero_trust/event.py b/monkey/monkey_island/cc/models/zero_trust/event.py index 89b581fa0e3..5ba909f2850 100644 --- a/monkey/monkey_island/cc/models/zero_trust/event.py +++ b/monkey/monkey_island/cc/models/zero_trust/event.py @@ -23,7 +23,9 @@ class Event(EmbeddedDocument): # LOGIC @staticmethod - def create_event(title, message, event_type, timestamp=datetime.now()): + def create_event(title, message, event_type, timestamp=None): + if not timestamp: + timestamp = datetime.now() event = Event( timestamp=timestamp, title=title, From 571682fff9cea192bd37cde04d63e3adb577cb43 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Wed, 6 May 2020 16:52:50 +0300 Subject: [PATCH 09/15] Refactored ZT events sending and display on report to improve performance and UX --- monkey/monkey_island/cc/app.py | 2 + .../cc/resources/reporting/report.py | 2 + .../cc/resources/zero_trust/finding_event.py | 14 +++++ .../reporting/test_zero_trust_service.py | 7 +++ .../services/reporting/zero_trust_service.py | 55 +++++++++++++++---- .../zerotrust/EventsButton.js | 13 ++++- .../zerotrust/EventsModal.js | 44 +++++++++++---- .../zerotrust/EventsTimeline.js | 2 +- .../zerotrust/FindingsTable.js | 6 +- .../zerotrust/SkippedEventsTimeline.js | 26 +++++++++ monkey/monkey_island/linux/install_mongo.sh | 3 - 11 files changed, 143 insertions(+), 31 deletions(-) create mode 100644 monkey/monkey_island/cc/resources/zero_trust/finding_event.py create mode 100644 monkey/monkey_island/cc/ui/src/components/report-components/zerotrust/SkippedEventsTimeline.js diff --git a/monkey/monkey_island/cc/app.py b/monkey/monkey_island/cc/app.py index 3a11349301b..13aac018aeb 100644 --- a/monkey/monkey_island/cc/app.py +++ b/monkey/monkey_island/cc/app.py @@ -32,6 +32,7 @@ from monkey_island.cc.resources.attack.attack_config import AttackConfiguration from monkey_island.cc.resources.attack.attack_report import AttackReport from monkey_island.cc.resources.bootloader import Bootloader +from monkey_island.cc.resources.zero_trust.finding_event import ZeroTrustFindingEvent from monkey_island.cc.services.database import Database from monkey_island.cc.services.remote_run_aws import RemoteRunAwsService from monkey_island.cc.services.representations import output_json @@ -107,6 +108,7 @@ def init_api_resources(api): Report, '/api/report/', '/api/report//') + api.add_resource(ZeroTrustFindingEvent, '/api/zero-trust/finding-event/') api.add_resource(TelemetryFeed, '/api/telemetry-feed', '/api/telemetry-feed/') api.add_resource(Log, '/api/log', '/api/log/') diff --git a/monkey/monkey_island/cc/resources/reporting/report.py b/monkey/monkey_island/cc/resources/reporting/report.py index 961e745a879..6770512e6df 100644 --- a/monkey/monkey_island/cc/resources/reporting/report.py +++ b/monkey/monkey_island/cc/resources/reporting/report.py @@ -6,6 +6,7 @@ from monkey_island.cc.auth import jwt_required from monkey_island.cc.services.reporting.report import ReportService from monkey_island.cc.services.reporting.zero_trust_service import ZeroTrustService +from monkey_island.cc.testing.profiler_decorator import profile ZERO_TRUST_REPORT_TYPE = "zero_trust" SECURITY_REPORT_TYPE = "security" @@ -21,6 +22,7 @@ class Report(flask_restful.Resource): @jwt_required() + @profile() def get(self, report_type=SECURITY_REPORT_TYPE, report_data=None): if report_type == SECURITY_REPORT_TYPE: return ReportService.get_report() diff --git a/monkey/monkey_island/cc/resources/zero_trust/finding_event.py b/monkey/monkey_island/cc/resources/zero_trust/finding_event.py new file mode 100644 index 00000000000..73cfa7f4c33 --- /dev/null +++ b/monkey/monkey_island/cc/resources/zero_trust/finding_event.py @@ -0,0 +1,14 @@ +import flask_restful +import json + +from monkey_island.cc.auth import jwt_required +from monkey_island.cc.services.reporting.zero_trust_service import ZeroTrustService +from monkey_island.cc.testing.profiler_decorator import profile + + +class ZeroTrustFindingEvent(flask_restful.Resource): + + @jwt_required() + @profile() + def get(self, finding_id: str): + return {'events_json': json.dumps(ZeroTrustService.get_events_by_finding(finding_id), default=str)} diff --git a/monkey/monkey_island/cc/services/reporting/test_zero_trust_service.py b/monkey/monkey_island/cc/services/reporting/test_zero_trust_service.py index 328be2e007d..403967d8f70 100644 --- a/monkey/monkey_island/cc/services/reporting/test_zero_trust_service.py +++ b/monkey/monkey_island/cc/services/reporting/test_zero_trust_service.py @@ -1,6 +1,7 @@ import common.data.zero_trust_consts as zero_trust_consts from monkey_island.cc.models.zero_trust.finding import Finding from monkey_island.cc.services.reporting.zero_trust_service import ZeroTrustService +import monkey_island.cc.services.reporting.zero_trust_service from monkey_island.cc.testing.IslandTestCase import IslandTestCase EXPECTED_DICT = { @@ -316,6 +317,12 @@ def test_get_pillars_to_statuses(self): self.assertEqual(ZeroTrustService.get_pillars_to_statuses(), expected) + def test_get_events_without_overlap(self): + monkey_island.cc.services.reporting.zero_trust_service.EVENT_FETCH_CNT = 5 + self.assertListEqual([], ZeroTrustService._ZeroTrustService__get_events_without_overlap(5, [1, 2, 3])) + self.assertListEqual([3], ZeroTrustService._ZeroTrustService__get_events_without_overlap(6, [1, 2, 3])) + self.assertListEqual([1, 2, 3, 4, 5], ZeroTrustService._ZeroTrustService__get_events_without_overlap(10, [1, 2, 3, 4, 5])) + def compare_lists_no_order(s, t): t = list(t) # make a mutable copy diff --git a/monkey/monkey_island/cc/services/reporting/zero_trust_service.py b/monkey/monkey_island/cc/services/reporting/zero_trust_service.py index 5a2045da53d..7ecac4e7fce 100644 --- a/monkey/monkey_island/cc/services/reporting/zero_trust_service.py +++ b/monkey/monkey_island/cc/services/reporting/zero_trust_service.py @@ -1,9 +1,14 @@ -import json +from typing import List import common.data.zero_trust_consts as zero_trust_consts +from bson.objectid import ObjectId from monkey_island.cc.models.zero_trust.finding import Finding +# How many events of a single finding to return to UI. +# 50 will return 50 latest and 50 oldest events from a finding +EVENT_FETCH_CNT = 50 + class ZeroTrustService(object): @staticmethod @@ -104,26 +109,44 @@ def __get_lcd_worst_status_for_test(all_findings_for_test): @staticmethod def get_all_findings(): - all_findings = Finding.objects() + pipeline = [{'$match': {}}, + {'$addFields': {'oldest_events': {'$slice': ['$events', EVENT_FETCH_CNT]}, + 'latest_events': {'$slice': ['$events', -1*EVENT_FETCH_CNT]}, + 'event_count': {'$size': '$events'}}}, + {'$unset': ['events']}] + all_findings = list(Finding.objects.aggregate(*pipeline)) + for finding in all_findings: + finding['latest_events'] = ZeroTrustService.__get_events_without_overlap(finding['event_count'], + finding['latest_events']) + enriched_findings = [ZeroTrustService.__get_enriched_finding(f) for f in all_findings] return enriched_findings + @staticmethod + def __get_events_without_overlap(event_count: int, events: List[object]) -> List[object]: + overlap_count = event_count - EVENT_FETCH_CNT + if overlap_count >= EVENT_FETCH_CNT: + return events + elif overlap_count <= 0: + return [] + else: + return events[ -overlap_count :] + @staticmethod def __get_enriched_finding(finding): - test_info = zero_trust_consts.TESTS_MAP[finding.test] + test_info = zero_trust_consts.TESTS_MAP[finding['test']] enriched_finding = { - "test": test_info[zero_trust_consts.FINDING_EXPLANATION_BY_STATUS_KEY][finding.status], - "test_key": finding.test, - "pillars": test_info[zero_trust_consts.PILLARS_KEY], - "status": finding.status, - "events": ZeroTrustService.__get_events_as_dict(finding.events) + 'finding_id': str(finding['_id']), + 'test': test_info[zero_trust_consts.FINDING_EXPLANATION_BY_STATUS_KEY][finding['status']], + 'test_key': finding['test'], + 'pillars': test_info[zero_trust_consts.PILLARS_KEY], + 'status': finding['status'], + 'latest_events': finding['latest_events'], + 'oldest_events': finding['oldest_events'], + 'event_count': finding['event_count'] } return enriched_finding - @staticmethod - def __get_events_as_dict(events): - return [json.loads(event.to_json()) for event in events] - @staticmethod def get_statuses_to_pillars(): results = { @@ -153,3 +176,11 @@ def __get_status_of_single_pillar(pillar): if grade[status] > 0: return status return zero_trust_consts.STATUS_UNEXECUTED + + @staticmethod + def get_events_by_finding(finding_id: str) -> List[object]: + pipeline = [{'$match': {'_id': ObjectId(finding_id)}}, + {'$unwind': '$events'}, + {'$project': {'events': '$events'}}, + {'$replaceRoot': {'newRoot': '$events'}}] + return list(Finding.objects.aggregate(*pipeline)) diff --git a/monkey/monkey_island/cc/ui/src/components/report-components/zerotrust/EventsButton.js b/monkey/monkey_island/cc/ui/src/components/report-components/zerotrust/EventsButton.js index 08c83babd08..94d92b27ac7 100644 --- a/monkey/monkey_island/cc/ui/src/components/report-components/zerotrust/EventsButton.js +++ b/monkey/monkey_island/cc/ui/src/components/report-components/zerotrust/EventsButton.js @@ -24,7 +24,12 @@ export default class EventsButton extends Component { render() { return -