From 3ec2a61d6967e2382f3ddd0ae9f29c803360ae5d Mon Sep 17 00:00:00 2001 From: lirshindalman Date: Thu, 18 Aug 2022 11:39:53 +0300 Subject: [PATCH 1/5] add env PRESENT_CACHED_RESULTS to gat report for save image sca --- .../integrations/docker_image_scanning.py | 4 +- checkov/sca_image/runner.py | 41 +++++++++++++++---- 2 files changed, 35 insertions(+), 10 deletions(-) diff --git a/checkov/common/bridgecrew/vulnerability_scanning/integrations/docker_image_scanning.py b/checkov/common/bridgecrew/vulnerability_scanning/integrations/docker_image_scanning.py index 7373a348999..7bc9bf69ff7 100644 --- a/checkov/common/bridgecrew/vulnerability_scanning/integrations/docker_image_scanning.py +++ b/checkov/common/bridgecrew/vulnerability_scanning/integrations/docker_image_scanning.py @@ -45,13 +45,14 @@ def report_results( docker_image_name=kwargs['docker_image_name'], ) - def create_report( # type:ignore[override] + def create_report( self, twistcli_scan_result: dict[str, Any], bc_platform_integration: BcPlatformIntegration, file_path: Path, file_content: str, docker_image_name: str, + related_resource_id: str, ) -> dict[str, Any]: if not bc_platform_integration.bc_source: logging.error("Source was not set") @@ -68,6 +69,7 @@ def create_report( # type:ignore[override] "sourceType": bc_platform_integration.bc_source.name, "vulnerabilities": self.get_vulnerabilities_for_report(results_dict), "packages": self.get_packages_for_report(results_dict), + "relatedResourceId": related_resource_id } if bc_platform_integration.cicd_details: payload["cicdDetails"] = bc_platform_integration.cicd_details diff --git a/checkov/sca_image/runner.py b/checkov/sca_image/runner.py index 0832c807f8e..bc1f9bf0ce2 100644 --- a/checkov/sca_image/runner.py +++ b/checkov/sca_image/runner.py @@ -26,6 +26,8 @@ from checkov.runner_filter import RunnerFilter from checkov.sca_package.runner import Runner as PackageRunner +PRESENT_CACHED_RESULTS = os.getenv('PRESENT_CACHED_RESULTS', False) + class Runner(PackageRunner): check_type = CheckType.SCA_IMAGE # noqa: CCE003 # a static attribute @@ -129,7 +131,7 @@ def run( runner_filter: RunnerFilter | None = None, collect_skip_comments: bool = True, **kwargs: str - ) -> Report: + ) -> Report | dict[str, Any]: runner_filter = runner_filter or RunnerFilter() if not runner_filter.show_progress_bar: self.pbar.turn_off_progress_bar() @@ -140,8 +142,10 @@ def run( dockerfile_path = kwargs['dockerfile_path'] image_id = kwargs['image_id'] return self.get_image_id_report(dockerfile_path, image_id, runner_filter) - - report = Report(self.check_type) + if not PRESENT_CACHED_RESULTS: + report = Report(self.check_type) + else: + report = [] if not files and not root_folder: logging.debug("No resources to scan.") return report @@ -152,7 +156,6 @@ def run( self.iterate_image_files(file, report, runner_filter) self.pbar.update() self.pbar.close() - if root_folder: for root, d_names, f_names in os.walk(root_folder): filter_ignored_paths(root, d_names, runner_filter.excluded_paths, included_paths=self.included_paths()) @@ -162,7 +165,7 @@ def run( self.iterate_image_files(abs_fname, report, runner_filter) return report - def iterate_image_files(self, abs_fname: str, report: Report, runner_filter: RunnerFilter) -> None: + def iterate_image_files(self, abs_fname: str, report: Report | List[Any], runner_filter: RunnerFilter) -> None: """ Get workflow file, and get the list of images from every relevant imagereferencer, and create a unified vulnrability report :param abs_fname: file path to inspect @@ -175,9 +178,13 @@ def iterate_image_files(self, abs_fname: str, report: Report, runner_filter: Run if image_referencer.is_workflow_file(abs_fname): images = image_referencer.get_images(file_path=abs_fname) for image in images: - image_report = self.get_image_report(dockerfile_path=abs_fname, image=image, - runner_filter=runner_filter) - merge_reports(report, image_report) + if not PRESENT_CACHED_RESULTS: + image_report = self.get_image_report(dockerfile_path=abs_fname, image=image, + runner_filter=runner_filter) + merge_reports(report, image_report) + else: + image_report = self.get_image_cached_results(dockerfile_path=abs_fname, image=image) + report.append(image_report) def get_report_from_scan_result(self, result: Dict[str, Any], dockerfile_path: str, rootless_file_path: str, image_details: ImageDetails | None, runner_filter: RunnerFilter) -> Report: @@ -185,6 +192,7 @@ def get_report_from_scan_result(self, result: Dict[str, Any], dockerfile_path: s vulnerabilities = result.get("vulnerabilities", []) packages = result.get("packages", []) license_statuses = get_license_statuses(packages) + parse_vulns_to_records( report=report, check_class=self._check_class, @@ -199,6 +207,22 @@ def get_report_from_scan_result(self, result: Dict[str, Any], dockerfile_path: s ) return report + def get_image_cached_results(self, dockerfile_path: str, image: Image) -> dict[str, Any]: + """ + :param dockerfile_path: path of a file that might contain a container image + :param image: Image object + :return: cached_results report + """ + cached_results: Dict[str, Any] = image_scanner.get_scan_results_from_cache(f"image:{image.name}") + payload: dict[str, Any] = docker_image_scanning_integration.create_report( + twistcli_scan_result=cached_results, + bc_platform_integration=bc_integration, + file_path=dockerfile_path, + file_content=image.file_path, + docker_image_name=image.name, + related_resource_id=image.related_resource_id) + return payload + def get_image_report(self, dockerfile_path: str, image: Image, runner_filter: RunnerFilter) -> Report: """ @@ -230,7 +254,6 @@ def get_image_report(self, dockerfile_path: str, image: Image, runner_filter: Ru rootless_file_path = dockerfile_path.replace(Path(dockerfile_path).anchor, "", 1) rootless_file_path_to_report = f"{rootless_file_path} ({image.name} lines:{image.start_line}-" \ f"{image.end_line} ({image_id}))" - return self.get_report_from_scan_result(result, dockerfile_path, rootless_file_path_to_report, image_details, runner_filter) From c8e7f5d990d245b1eeecca1f050aab12e86958c5 Mon Sep 17 00:00:00 2001 From: lirshindalman Date: Thu, 18 Aug 2022 11:58:15 +0300 Subject: [PATCH 2/5] Revert "add env PRESENT_CACHED_RESULTS to gat report for save image sca" This reverts commit 3ec2a61d6967e2382f3ddd0ae9f29c803360ae5d. --- .../integrations/docker_image_scanning.py | 4 +- checkov/sca_image/runner.py | 41 ++++--------------- 2 files changed, 10 insertions(+), 35 deletions(-) diff --git a/checkov/common/bridgecrew/vulnerability_scanning/integrations/docker_image_scanning.py b/checkov/common/bridgecrew/vulnerability_scanning/integrations/docker_image_scanning.py index 7bc9bf69ff7..7373a348999 100644 --- a/checkov/common/bridgecrew/vulnerability_scanning/integrations/docker_image_scanning.py +++ b/checkov/common/bridgecrew/vulnerability_scanning/integrations/docker_image_scanning.py @@ -45,14 +45,13 @@ def report_results( docker_image_name=kwargs['docker_image_name'], ) - def create_report( + def create_report( # type:ignore[override] self, twistcli_scan_result: dict[str, Any], bc_platform_integration: BcPlatformIntegration, file_path: Path, file_content: str, docker_image_name: str, - related_resource_id: str, ) -> dict[str, Any]: if not bc_platform_integration.bc_source: logging.error("Source was not set") @@ -69,7 +68,6 @@ def create_report( "sourceType": bc_platform_integration.bc_source.name, "vulnerabilities": self.get_vulnerabilities_for_report(results_dict), "packages": self.get_packages_for_report(results_dict), - "relatedResourceId": related_resource_id } if bc_platform_integration.cicd_details: payload["cicdDetails"] = bc_platform_integration.cicd_details diff --git a/checkov/sca_image/runner.py b/checkov/sca_image/runner.py index bc1f9bf0ce2..0832c807f8e 100644 --- a/checkov/sca_image/runner.py +++ b/checkov/sca_image/runner.py @@ -26,8 +26,6 @@ from checkov.runner_filter import RunnerFilter from checkov.sca_package.runner import Runner as PackageRunner -PRESENT_CACHED_RESULTS = os.getenv('PRESENT_CACHED_RESULTS', False) - class Runner(PackageRunner): check_type = CheckType.SCA_IMAGE # noqa: CCE003 # a static attribute @@ -131,7 +129,7 @@ def run( runner_filter: RunnerFilter | None = None, collect_skip_comments: bool = True, **kwargs: str - ) -> Report | dict[str, Any]: + ) -> Report: runner_filter = runner_filter or RunnerFilter() if not runner_filter.show_progress_bar: self.pbar.turn_off_progress_bar() @@ -142,10 +140,8 @@ def run( dockerfile_path = kwargs['dockerfile_path'] image_id = kwargs['image_id'] return self.get_image_id_report(dockerfile_path, image_id, runner_filter) - if not PRESENT_CACHED_RESULTS: - report = Report(self.check_type) - else: - report = [] + + report = Report(self.check_type) if not files and not root_folder: logging.debug("No resources to scan.") return report @@ -156,6 +152,7 @@ def run( self.iterate_image_files(file, report, runner_filter) self.pbar.update() self.pbar.close() + if root_folder: for root, d_names, f_names in os.walk(root_folder): filter_ignored_paths(root, d_names, runner_filter.excluded_paths, included_paths=self.included_paths()) @@ -165,7 +162,7 @@ def run( self.iterate_image_files(abs_fname, report, runner_filter) return report - def iterate_image_files(self, abs_fname: str, report: Report | List[Any], runner_filter: RunnerFilter) -> None: + def iterate_image_files(self, abs_fname: str, report: Report, runner_filter: RunnerFilter) -> None: """ Get workflow file, and get the list of images from every relevant imagereferencer, and create a unified vulnrability report :param abs_fname: file path to inspect @@ -178,13 +175,9 @@ def iterate_image_files(self, abs_fname: str, report: Report | List[Any], runner if image_referencer.is_workflow_file(abs_fname): images = image_referencer.get_images(file_path=abs_fname) for image in images: - if not PRESENT_CACHED_RESULTS: - image_report = self.get_image_report(dockerfile_path=abs_fname, image=image, - runner_filter=runner_filter) - merge_reports(report, image_report) - else: - image_report = self.get_image_cached_results(dockerfile_path=abs_fname, image=image) - report.append(image_report) + image_report = self.get_image_report(dockerfile_path=abs_fname, image=image, + runner_filter=runner_filter) + merge_reports(report, image_report) def get_report_from_scan_result(self, result: Dict[str, Any], dockerfile_path: str, rootless_file_path: str, image_details: ImageDetails | None, runner_filter: RunnerFilter) -> Report: @@ -192,7 +185,6 @@ def get_report_from_scan_result(self, result: Dict[str, Any], dockerfile_path: s vulnerabilities = result.get("vulnerabilities", []) packages = result.get("packages", []) license_statuses = get_license_statuses(packages) - parse_vulns_to_records( report=report, check_class=self._check_class, @@ -207,22 +199,6 @@ def get_report_from_scan_result(self, result: Dict[str, Any], dockerfile_path: s ) return report - def get_image_cached_results(self, dockerfile_path: str, image: Image) -> dict[str, Any]: - """ - :param dockerfile_path: path of a file that might contain a container image - :param image: Image object - :return: cached_results report - """ - cached_results: Dict[str, Any] = image_scanner.get_scan_results_from_cache(f"image:{image.name}") - payload: dict[str, Any] = docker_image_scanning_integration.create_report( - twistcli_scan_result=cached_results, - bc_platform_integration=bc_integration, - file_path=dockerfile_path, - file_content=image.file_path, - docker_image_name=image.name, - related_resource_id=image.related_resource_id) - return payload - def get_image_report(self, dockerfile_path: str, image: Image, runner_filter: RunnerFilter) -> Report: """ @@ -254,6 +230,7 @@ def get_image_report(self, dockerfile_path: str, image: Image, runner_filter: Ru rootless_file_path = dockerfile_path.replace(Path(dockerfile_path).anchor, "", 1) rootless_file_path_to_report = f"{rootless_file_path} ({image.name} lines:{image.start_line}-" \ f"{image.end_line} ({image_id}))" + return self.get_report_from_scan_result(result, dockerfile_path, rootless_file_path_to_report, image_details, runner_filter) From 06544da8c31195b6bea09a90440a390b6f15228b Mon Sep 17 00:00:00 2001 From: lirshindalman Date: Thu, 3 Nov 2022 16:22:42 +0200 Subject: [PATCH 3/5] add check_link to get_checks --- checkov/common/util/docs_generator.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/checkov/common/util/docs_generator.py b/checkov/common/util/docs_generator.py index 8ed50a90d66..40dca76faeb 100644 --- a/checkov/common/util/docs_generator.py +++ b/checkov/common/util/docs_generator.py @@ -3,6 +3,7 @@ from __future__ import annotations import re +import inspect from typing import List, Optional, Tuple, Union from tabulate import tabulate @@ -60,10 +61,15 @@ def print_checks(frameworks: Optional[List[str]] = None, use_bc_ids: bool = Fals print("\n\n---\n\n") +def get_check_link(absolute_path: str) -> str: + link_start = 'https://github.com/bridgecrewio/checkov/tree/master/checkov' + return f'{link_start}{absolute_path.split("/checkov")[1]}' + + def get_checks(frameworks: Optional[List[str]] = None, use_bc_ids: bool = False, - include_all_checkov_policies: bool = True, filtered_policy_ids: Optional[List[str]] = None) -> List[Tuple[str, str, int, int, str]]: + include_all_checkov_policies: bool = True, filtered_policy_ids: Optional[List[str]] = None) -> List[Tuple[str, str, int, int, str, str]]: framework_list = frameworks if frameworks else ["all"] - printable_checks_list: list[tuple[str, str, str, str, str]] = [] + printable_checks_list: list[tuple[str, str, str, str, str, str]] = [] filtered_policy_ids = filtered_policy_ids or [] runner_filter = RunnerFilter(include_all_checkov_policies=include_all_checkov_policies, filtered_policy_ids=filtered_policy_ids) @@ -73,8 +79,9 @@ def add_from_repository(registry: Union[BaseCheckRegistry, BaseGraphRegistry], c if isinstance(registry, BaseCheckRegistry): for entity, check in registry.all_checks(): if runner_filter.should_run_check(check, check.id, check.bc_id, check.severity): + check_link = get_check_link(inspect.getfile(check.__class__)) printable_checks_list.append( - (check.get_output_id(use_bc_ids), checked_type, entity, check.name, iac)) + (check.get_output_id(use_bc_ids), checked_type, entity, check.name, iac, check_link)) elif isinstance(registry, BaseGraphRegistry): for graph_check in registry.checks: if runner_filter.should_run_check(graph_check, graph_check.id, graph_check.bc_id, graph_check.severity): @@ -82,8 +89,9 @@ def add_from_repository(registry: Union[BaseCheckRegistry, BaseGraphRegistry], c # only for platform custom polices with resource_types == all graph_check.resource_types = ['all'] for rt in graph_check.resource_types: + check_link = get_check_link(inspect.getfile(graph_check.__class__)) printable_checks_list.append( - (graph_check.get_output_id(use_bc_ids), checked_type, rt, graph_check.name, iac)) + (graph_check.get_output_id(use_bc_ids), checked_type, rt, graph_check.name, iac, check_link)) if any(x in framework_list for x in ("all", "terraform")): add_from_repository(resource_registry, "resource", "Terraform") @@ -142,7 +150,8 @@ def add_from_repository(registry: Union[BaseCheckRegistry, BaseGraphRegistry], c if not filtered_policy_ids or check_id in filtered_policy_ids: if use_bc_ids: check_id = metadata_integration.get_bc_id(check_id) - printable_checks_list.append((check_id, check_type, "secrets", check_type, "secrets")) + check_link = get_check_link(inspect.getfile(metadata_integration.__class__)) + printable_checks_list.append((check_id, check_type, "secrets", check_type, "secrets", check_link)) return sorted(printable_checks_list, key=get_compare_key) # type:ignore[arg-type] From d755636cf1a1323002dd7e3df3dc7f3d569bc27a Mon Sep 17 00:00:00 2001 From: lirshindalman Date: Sun, 6 Nov 2022 12:07:20 +0200 Subject: [PATCH 4/5] CODE_LINK_BASE --- checkov/common/util/docs_generator.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/checkov/common/util/docs_generator.py b/checkov/common/util/docs_generator.py index 40dca76faeb..94275bca42f 100644 --- a/checkov/common/util/docs_generator.py +++ b/checkov/common/util/docs_generator.py @@ -36,7 +36,7 @@ from checkov.runner_filter import RunnerFilter ID_PARTS_PATTERN = re.compile(r'([^_]*)_([^_]*)_(\d+)') - +CODE_LINK_BASE = 'https://github.com/bridgecrewio/checkov/tree/master/checkov' def get_compare_key(c: list[str] | tuple[str, ...]) -> list[tuple[str, str, int, int, str]]: res = [] @@ -62,8 +62,7 @@ def print_checks(frameworks: Optional[List[str]] = None, use_bc_ids: bool = Fals def get_check_link(absolute_path: str) -> str: - link_start = 'https://github.com/bridgecrewio/checkov/tree/master/checkov' - return f'{link_start}{absolute_path.split("/checkov")[1]}' + return f'{CODE_LINK_BASE}{absolute_path.split("/checkov")[1]}' def get_checks(frameworks: Optional[List[str]] = None, use_bc_ids: bool = False, From 22b485bafacbdcd7b4817121dcf77e21404de771 Mon Sep 17 00:00:00 2001 From: lirshindalman Date: Sun, 6 Nov 2022 12:48:37 +0200 Subject: [PATCH 5/5] CODE_LINK_BASE --- checkov/common/util/docs_generator.py | 1 + 1 file changed, 1 insertion(+) diff --git a/checkov/common/util/docs_generator.py b/checkov/common/util/docs_generator.py index 94275bca42f..a845de1e000 100644 --- a/checkov/common/util/docs_generator.py +++ b/checkov/common/util/docs_generator.py @@ -38,6 +38,7 @@ ID_PARTS_PATTERN = re.compile(r'([^_]*)_([^_]*)_(\d+)') CODE_LINK_BASE = 'https://github.com/bridgecrewio/checkov/tree/master/checkov' + def get_compare_key(c: list[str] | tuple[str, ...]) -> list[tuple[str, str, int, int, str]]: res = [] for match in ID_PARTS_PATTERN.finditer(c[0]):