From 85dcd7e4ecda31ba37e973e2ebb30f16d13f0029 Mon Sep 17 00:00:00 2001 From: ajbara Date: Sun, 6 Nov 2022 14:59:07 +0200 Subject: [PATCH 01/14] add new sca package --- .../common/bridgecrew/platform_integration.py | 5 +- checkov/main.py | 7 +- checkov/sca_package_2/__init__.py | 0 checkov/sca_package_2/output.py | 364 ++++++++++++++++++ checkov/sca_package_2/runner.py | 159 ++++++++ checkov/sca_package_2/scanner.py | 172 +++++++++ 6 files changed, 704 insertions(+), 3 deletions(-) create mode 100644 checkov/sca_package_2/__init__.py create mode 100644 checkov/sca_package_2/output.py create mode 100644 checkov/sca_package_2/runner.py create mode 100644 checkov/sca_package_2/scanner.py diff --git a/checkov/common/bridgecrew/platform_integration.py b/checkov/common/bridgecrew/platform_integration.py index 4082f925c80..6a754d2d3eb 100644 --- a/checkov/common/bridgecrew/platform_integration.py +++ b/checkov/common/bridgecrew/platform_integration.py @@ -30,7 +30,7 @@ from checkov.common.bridgecrew.platform_key import read_key, persist_key, bridgecrew_file from checkov.common.bridgecrew.wrapper import reduce_scan_reports, persist_checks_results, \ enrich_and_persist_checks_metadata, checkov_results_prefix, persist_run_metadata, _put_json_object -from checkov.common.models.consts import SUPPORTED_FILE_EXTENSIONS, SUPPORTED_FILES +from checkov.common.models.consts import SUPPORTED_FILE_EXTENSIONS, SUPPORTED_FILES, SUPPORTED_PACKAGE_FILES from checkov.common.bridgecrew.check_type import CheckType from checkov.common.runners.base_runner import filter_ignored_paths from checkov.common.typing import _CicdDetails @@ -74,6 +74,7 @@ 'Content-Type': 'application/json;charset=UTF-8'}, get_user_agent_header()) CI_METADATA_EXTRACTOR = registry.get_extractor() +RUN_NEW_SCA_PACKAGE_SCAN = os.getenv('RUN_NEW_SCA_PACKAGE_SCAN', False) class BcPlatformIntegration: @@ -343,6 +344,8 @@ def persist_repository( for file_path in f_names: _, file_extension = os.path.splitext(file_path) if file_extension in SUPPORTED_FILE_EXTENSIONS or file_path in SUPPORTED_FILES: + if RUN_NEW_SCA_PACKAGE_SCAN and file_extension in SUPPORTED_PACKAGE_FILES: + continue full_file_path = os.path.join(root_path, file_path) relative_file_path = os.path.relpath(full_file_path, root_dir) files_to_persist.append(FileToPersist(full_file_path, relative_file_path)) diff --git a/checkov/main.py b/checkov/main.py index 527fc275355..ccc40f41334 100755 --- a/checkov/main.py +++ b/checkov/main.py @@ -55,6 +55,7 @@ from checkov.runner_filter import RunnerFilter from checkov.sca_image.runner import Runner as sca_image_runner from checkov.sca_package.runner import Runner as sca_package_runner +from checkov.sca_package_2.runner import Runner as sca_package_runner_2 from checkov.secrets.runner import Runner as secrets_runner from checkov.serverless.runner import Runner as sls_runner from checkov.terraform.plan_runner import Runner as tf_plan_runner @@ -75,6 +76,8 @@ logger = logging.getLogger(__name__) checkov_runners = [value for attr, value in CheckType.__dict__.items() if not attr.startswith("__")] +RUN_NEW_SCA_PACKAGE_SCAN = os.getenv('RUN_NEW_SCA_PACKAGE_SCAN', False) + DEFAULT_RUNNERS = ( tf_graph_runner(), cfn_runner(), @@ -93,7 +96,6 @@ bitbucket_configuration_runner(), bitbucket_pipelines_runner(), kustomize_runner(), - sca_package_runner(), github_actions_runner(), bicep_runner(), openapi_runner(), @@ -101,6 +103,7 @@ argo_workflows_runner(), circleci_pipelines_runner(), azure_pipelines_runner(), + sca_package_runner_2() if RUN_NEW_SCA_PACKAGE_SCAN else sca_package_runner() ) @@ -227,7 +230,7 @@ def run(banner: str = checkov_banner, argv: List[str] = sys.argv[1:]) -> Optiona source=source, source_version=source_version, repo_branch=config.branch, - prisma_api_url=config.prisma_api_url) + prisma_api_url=config.prisma_api_url) # TODO here we create the timestamp except MaxRetryError: return None except Exception: diff --git a/checkov/sca_package_2/__init__.py b/checkov/sca_package_2/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/checkov/sca_package_2/output.py b/checkov/sca_package_2/output.py new file mode 100644 index 00000000000..8466ce66aab --- /dev/null +++ b/checkov/sca_package_2/output.py @@ -0,0 +1,364 @@ +from __future__ import annotations + +import itertools +import logging +from collections import defaultdict +from dataclasses import dataclass +from typing import List, Union, Dict, Any + +from packaging import version as packaging_version +from prettytable import PrettyTable, SINGLE_BORDER + +from checkov.common.bridgecrew.severities import Severities, BcSeverities +from checkov.common.models.enums import CheckResult +from checkov.common.output.record import Record, DEFAULT_SEVERITY, SCA_PACKAGE_SCAN_CHECK_NAME, SCA_LICENSE_CHECK_NAME +from checkov.common.sca.commons import UNFIXABLE_VERSION +from checkov.common.typing import _LicenseStatus + + +@dataclass +class CveCount: + total: int = 0 + critical: int = 0 + high: int = 0 + medium: int = 0 + low: int = 0 + skipped: int = 0 + has_fix: int = 0 + to_fix: int = 0 + fixable: bool = True + + def output_row(self) -> List[str]: + return [ + f"Total CVEs: {self.total}", + f"critical: {self.critical}", + f"high: {self.high}", + f"medium: {self.medium}", + f"low: {self.low}", + f"skipped: {self.skipped}", + ] + + +def calculate_lowest_compliant_version( + fix_versions_lists: List[List[Union[packaging_version.Version, packaging_version.LegacyVersion]]] +) -> str: + """A best effort approach to find the lowest compliant version""" + + package_min_versions = set() + package_versions = set() + + for fix_versions in fix_versions_lists: + if fix_versions: + package_min_versions.add(min(fix_versions)) + package_versions.update(fix_versions) + if package_min_versions: + package_min_version = min(package_min_versions) + package_max_version = max(package_min_versions) + + if isinstance(package_min_version, packaging_version.LegacyVersion) or isinstance( + package_max_version, packaging_version.LegacyVersion + ): + return str(package_max_version) + elif package_min_version.major == package_max_version.major: + return str(package_max_version) + else: + lowest_version = max( + version + for version in package_versions + if isinstance(version, packaging_version.Version) and version.major == package_max_version.major + ) + return str(lowest_version) + + return UNFIXABLE_VERSION + + +def compare_cve_severity(cve: Dict[str, str]) -> int: + severity = (cve.get("severity") or DEFAULT_SEVERITY).upper() + return Severities[severity].level + + +def create_cli_output(fixable: bool = True, *cve_records: list[Record]) -> str: + cli_outputs = [] + group_by_file_path_package_map: dict[str, dict[str, list[Record]]] = defaultdict(dict) + + for record in itertools.chain(*cve_records): + if not record.vulnerability_details: + # this shouldn't happen + logging.error(f"'vulnerability_details' is not set for {record.check_id}") + continue + + group_by_file_path_package_map[record.file_path].setdefault( + record.vulnerability_details["package_name"], [] + ).append(record) + + for file_path, packages in group_by_file_path_package_map.items(): + cve_count = CveCount(fixable=fixable) + package_cves_details_map: dict[str, dict[str, Any]] = defaultdict(dict) + package_licenses_details_map = defaultdict(list) + should_print_licenses_table = False + for package_name, records in packages.items(): + package_version = None + fix_versions_lists = [] + + for record in records: + if not record.vulnerability_details: + # this shouldn't happen + logging.error(f"'vulnerability_details' is not set for {record.check_id}") + continue + + if record.check_name == SCA_PACKAGE_SCAN_CHECK_NAME: + cve_count.total += 1 + + if record.check_result["result"] == CheckResult.SKIPPED: + cve_count.skipped += 1 + continue + else: + cve_count.to_fix += 1 + + # best way to dynamically access a class instance attribute. + # (we can't just do cve_count.severity_str to access the correct severity) + severity_str = record.severity.name.lower() if record.severity else BcSeverities.NONE.lower() + setattr(cve_count, severity_str, getattr(cve_count, severity_str) + 1) + + if record.vulnerability_details["lowest_fixed_version"] != UNFIXABLE_VERSION: + cve_count.has_fix += 1 + + fix_versions_lists.append(record.vulnerability_details["fixed_versions"]) + if package_version is None: + package_version = record.vulnerability_details["package_version"] + + package_cves_details_map[package_name].setdefault("cves", []).append( + { + "id": record.vulnerability_details["id"], + "severity": severity_str, + "fixed_version": record.vulnerability_details["lowest_fixed_version"], + } + ) + elif record.check_name == SCA_LICENSE_CHECK_NAME: + if record.check_result["result"] == CheckResult.SKIPPED: + continue + should_print_licenses_table = True + package_licenses_details_map[package_name].append( + _LicenseStatus(package_name=package_name, + package_version=record.vulnerability_details["package_version"], + policy=record.vulnerability_details["policy"], + license=record.vulnerability_details["license"], + status=record.vulnerability_details["status"]) + ) + + if package_name in package_cves_details_map: + package_cves_details_map[package_name]["cves"].sort(key=compare_cve_severity, reverse=True) + package_cves_details_map[package_name]["current_version"] = package_version + package_cves_details_map[package_name]["compliant_version"] = calculate_lowest_compliant_version( + fix_versions_lists + ) + + if cve_count.total > 0: + cli_outputs.append( + create_cli_cves_table( + file_path=file_path, + cve_count=cve_count, + package_details_map=package_cves_details_map, + ) + ) + if should_print_licenses_table: + cli_outputs.append( + create_cli_license_violations_table( + file_path=file_path, + package_licenses_details_map=package_licenses_details_map + ) + ) + + return "\n".join(cli_outputs) + + +def create_cli_license_violations_table(file_path: str, package_licenses_details_map: Dict[str, List[_LicenseStatus]]) -> str: + package_table_lines: List[str] = [] + columns = 5 + table_width = 120.0 + column_width = int(table_width / columns) + package_table = PrettyTable(min_table_width=table_width, max_table_width=table_width) + package_table.set_style(SINGLE_BORDER) + package_table.field_names = [ + "Package name", + "Package version", + "Policy ID", + "License", + "Status", + ] + for package_idx, (package_name, license_statuses) in enumerate(package_licenses_details_map.items()): + if package_idx > 0: + del package_table_lines[-1] + package_table.header = False + package_table.clear_rows() + + for idx, license_status in enumerate(license_statuses): + col_package_name = "" + col_package_version = "" + if idx == 0: + col_package_name = package_name + col_package_version = license_status["package_version"] + + package_table.add_row( + [ + col_package_name, + col_package_version, + license_status["policy"], + license_status["license"], + license_status["status"], + ] + ) + + package_table.align = "l" + package_table.min_width = column_width + package_table.max_width = column_width + + for idx, line in enumerate(package_table.get_string().splitlines(keepends=True)): + if idx == 0 and package_idx != 0: + # hack to make multiple tables look like one + line = line.replace(package_table.top_left_junction_char, package_table.left_junction_char).replace( + package_table.top_right_junction_char, package_table.right_junction_char + ) + if package_idx > 0: + # hack to make multiple package tables look like one + line = line.replace(package_table.top_junction_char, package_table.junction_char) + + # hack for making the table's width as same as the cves-table's + package_table_lines.append(f"\t{line[:-2]}{line[-3]}{line[-2:]}") + + return ( + f"\t{file_path} - Licenses Statuses:\n" + f"{''.join(package_table_lines)}\n" + ) + + +def create_cli_cves_table(file_path: str, cve_count: CveCount, package_details_map: Dict[str, Dict[str, Any]]) -> str: + columns = 6 + table_width = 120 + column_width = int(120 / columns) + + cve_table_lines = create_cve_summary_table_part( + table_width=table_width, column_width=column_width, cve_count=cve_count + ) + + vulnerable_packages = True if package_details_map else False + fixable_table_lines = create_fixable_cve_summary_table_part( + table_width=table_width, column_count=columns, cve_count=cve_count, vulnerable_packages=vulnerable_packages + ) + + package_table_lines = create_package_overview_table_part( + table_width=table_width, column_width=column_width, package_details_map=package_details_map + ) + + return ( + f"\t{file_path} - CVEs Summary:\n" + f"{''.join(cve_table_lines)}\n" + f"{''.join(fixable_table_lines)}" + f"{''.join(package_table_lines)}\n" + ) + + +def create_cve_summary_table_part(table_width: int, column_width: int, cve_count: CveCount) -> List[str]: + cve_table = PrettyTable( + header=False, + padding_width=1, + min_table_width=table_width, + max_table_width=table_width, + ) + cve_table.set_style(SINGLE_BORDER) + cve_table.add_row(cve_count.output_row()) + cve_table.align = "l" + cve_table.min_width = column_width + cve_table.max_width = column_width + + cve_table_lines = [f"\t{line}" for line in cve_table.get_string().splitlines(keepends=True)] + # hack to make multiple tables look like one + cve_table_bottom_line = ( + cve_table_lines[-1] + .replace(cve_table.bottom_left_junction_char, cve_table.left_junction_char) + .replace(cve_table.bottom_right_junction_char, cve_table.right_junction_char) + ) + cve_table_lines[-1] = cve_table_bottom_line + + return cve_table_lines + + +def create_fixable_cve_summary_table_part( + table_width: int, column_count: int, cve_count: CveCount, vulnerable_packages: bool +) -> List[str]: + fixable_table = PrettyTable( + header=False, min_table_width=table_width + column_count * 2, max_table_width=table_width + column_count * 2 + ) + fixable_table.set_style(SINGLE_BORDER) + if cve_count.fixable: + fixable_table.add_row( + [f"To fix {cve_count.has_fix}/{cve_count.to_fix} CVEs, go to https://www.bridgecrew.cloud/"]) + fixable_table.align = "l" + + # hack to make multiple tables look like one + fixable_table_lines = [f"\t{line}" for line in fixable_table.get_string().splitlines(keepends=True)] + del fixable_table_lines[0] + # only remove the last line, if there are vulnerable packages + if vulnerable_packages: + del fixable_table_lines[-1] + + return fixable_table_lines + + +def create_package_overview_table_part( + table_width: int, column_width: int, package_details_map: Dict[str, Dict[str, Any]] +) -> List[str]: + package_table_lines: List[str] = [] + package_table = PrettyTable(min_table_width=table_width, max_table_width=table_width) + package_table.set_style(SINGLE_BORDER) + package_table.field_names = [ + "Package", + "CVE ID", + "Severity", + "Current version", + "Fixed version", + "Compliant version", + ] + for package_idx, (package_name, details) in enumerate(package_details_map.items()): + if package_idx > 0: + del package_table_lines[-1] + package_table.header = False + package_table.clear_rows() + + for cve_idx, cve in enumerate(details["cves"]): + col_package = "" + col_current_version = "" + col_compliant_version = "" + if cve_idx == 0: + col_package = package_name + col_current_version = details["current_version"] + col_compliant_version = details["compliant_version"] + + package_table.add_row( + [ + col_package, + cve["id"], + cve["severity"], + col_current_version, + cve["fixed_version"], + col_compliant_version, + ] + ) + + package_table.align = "l" + package_table.min_width = column_width + package_table.max_width = column_width + + for idx, line in enumerate(package_table.get_string().splitlines(keepends=True)): + if idx == 0: + # hack to make multiple tables look like one + line = line.replace(package_table.top_left_junction_char, package_table.left_junction_char).replace( + package_table.top_right_junction_char, package_table.right_junction_char + ) + if package_idx > 0: + # hack to make multiple package tables look like one + line = line.replace(package_table.top_junction_char, package_table.junction_char) + + package_table_lines.append(f"\t{line}") + + return package_table_lines diff --git a/checkov/sca_package_2/runner.py b/checkov/sca_package_2/runner.py new file mode 100644 index 00000000000..fe2e63c77a4 --- /dev/null +++ b/checkov/sca_package_2/runner.py @@ -0,0 +1,159 @@ +from __future__ import annotations + +import logging +import os +from pathlib import Path +from typing import Sequence, Any, List + +from checkov.common.sca.commons import should_run_scan +from checkov.common.sca.output import add_to_report_sca_data +from checkov.common.typing import _LicenseStatus +from checkov.common.bridgecrew.platform_integration import bc_integration, FileToPersist +from checkov.common.models.consts import SUPPORTED_PACKAGE_FILES +from checkov.common.output.report import Report +from checkov.common.bridgecrew.check_type import CheckType +from checkov.common.runners.base_runner import BaseRunner, ignored_directories +from checkov.runner_filter import RunnerFilter +from checkov.sca_package.scanner import Scanner + + +class Runner(BaseRunner[None]): + check_type = CheckType.SCA_PACKAGE # noqa: CCE003 # a static attribute + + def __init__(self, report_type: str = check_type) -> None: + super().__init__(file_names=SUPPORTED_PACKAGE_FILES) + self._check_class: str | None = None + self._code_repo_path: Path | None = None + self.report_type = report_type + + def prepare_and_scan( + self, + root_folder: str | Path | None, + files: list[str] | None = None, + runner_filter: RunnerFilter | None = None, + exclude_package_json: bool = True, + excluded_file_names: set[str] | None = None, + ) -> Sequence[dict[str, Any]] | None: + runner_filter = runner_filter or RunnerFilter() + excluded_file_names = excluded_file_names or set() + + # skip complete run, if flag '--check' was used without a CVE check ID or the license policies + if not should_run_scan(runner_filter.checks): + return None + + if not bc_integration.bc_api_key: + logging.info("The --bc-api-key flag needs to be set to run SCA package scanning") + return None + + logging.info("SCA package scanning searching for scannable files") + + self._code_repo_path = Path(root_folder) if root_folder else None + + excluded_paths = {*ignored_directories} + if runner_filter.excluded_paths: + excluded_paths.update(runner_filter.excluded_paths) + + input_paths = self.find_scannable_files( + root_path=self._code_repo_path, + files=files, + excluded_paths=excluded_paths, + exclude_package_json=exclude_package_json, + excluded_file_names=excluded_file_names + ) + if not input_paths: + # no packages found + return None + + logging.info(f"SCA package scanning will scan {len(input_paths)} files") + + scanner = Scanner(self.pbar, root_folder) + self._check_class = f"{scanner.__module__}.{scanner.__class__.__qualname__}" + scan_results = scanner.scan(input_paths) + + logging.info(f"SCA package scanning successfully scanned {len(scan_results)} files") + return scan_results + + def run( + self, + root_folder: str | Path | None, + external_checks_dir: list[str] | None = None, + files: list[str] | None = None, + runner_filter: RunnerFilter | None = None, + collect_skip_comments: bool = True, + ) -> Report | list[Report]: + runner_filter = runner_filter or RunnerFilter() + if not runner_filter.show_progress_bar: + self.pbar.turn_off_progress_bar() + + report = Report(self.check_type) + + scan_results = self.prepare_and_scan(root_folder, files, runner_filter) + if scan_results is None: + return report + + for result in scan_results: + if not result: + continue + package_file_path = Path(result["repository"]) + if self._code_repo_path: + try: + package_file_path = package_file_path.relative_to(self._code_repo_path) + except ValueError: + # Path.is_relative_to() was implemented in Python 3.9 + pass + + vulnerabilities = result.get("vulnerabilities") or [] + packages = result.get("packages") or [] + + license_statuses = [_LicenseStatus(package_name=elm["packageName"], package_version=elm["packageVersion"], + policy=elm["policy"], license=elm["license"], status=elm["status"]) + for elm in result.get("license_statuses") or []] + + rootless_file_path = str(package_file_path).replace(package_file_path.anchor, "", 1) + add_to_report_sca_data( + report=report, + check_class=self._check_class, + scanned_file_path=str(package_file_path), + rootless_file_path=rootless_file_path, + runner_filter=runner_filter, + vulnerabilities=vulnerabilities, + packages=packages, + license_statuses=license_statuses, + report_type=self.report_type, + ) + + return report + + def find_scannable_files( + self, + root_path: Path | None, + files: list[str] | None, + excluded_paths: set[str], + exclude_package_json: bool = True, + excluded_file_names: set[str] | None = None + ) -> set[Path]: + excluded_file_names = excluded_file_names or set() + input_paths: set[Path] = set() + package_files_to_persist: List[FileToPersist] = [] + if root_path: + input_paths = { + file_path + for file_path in root_path.glob("**/*") + if file_path.name in SUPPORTED_PACKAGE_FILES and not any( + p in file_path.parts for p in excluded_paths) and file_path.name not in excluded_file_names + } + + for file in files or []: + file_path = Path(file) + if not file_path.exists(): + logging.warning(f"File {file_path} doesn't exist") + continue + + # f_name = os.path.basename(file) + # _, file_extension = os.path.splitext(file) + # if file_extension in SUPPORTED_FILE_EXTENSIONS or f_name in SUPPORTED_FILES: + # files_to_persist.append(FileToPersist(file, os.path.relpath(file, root_dir))) + + input_paths.add(file_path) + + return input_paths diff --git a/checkov/sca_package_2/scanner.py b/checkov/sca_package_2/scanner.py new file mode 100644 index 00000000000..08e4ebfe55a --- /dev/null +++ b/checkov/sca_package_2/scanner.py @@ -0,0 +1,172 @@ +from __future__ import annotations + +import asyncio +import json +import logging +import os +import time +from collections.abc import Iterable, Sequence, Collection +from pathlib import Path +from typing import Dict, Any, List + +import requests + +from checkov.common.bridgecrew.platform_integration import bc_integration +from checkov.common.bridgecrew.platform_key import bridgecrew_dir +from checkov.common.bridgecrew.vulnerability_scanning.image_scanner import image_scanner, TWISTCLI_FILE_NAME +from checkov.common.bridgecrew.vulnerability_scanning.integrations.docker_image_scanning import \ + docker_image_scanning_integration +from checkov.common.util.file_utils import compress_file_gzip_base64, decompress_file_gzip_base64 +from checkov.common.util.http_utils import request_wrapper + +from checkov.common.util.tqdm_utils import ProgressBar + +SLEEP_DURATION = 2 +MAX_SLEEP_DURATION = 60 + + +class Scanner: + def __init__(self, pbar: ProgressBar | None = None, root_folder: str | Path | None = None) -> None: + self._base_url = bc_integration.api_url + if pbar: + self.pbar = pbar + else: + self.pbar = ProgressBar('') + self.pbar.turn_off_progress_bar() + self.root_folder = root_folder + + def scan(self, input_paths: Collection[Path]) -> Sequence[dict[str, Any]]: + self.pbar.initiate(len(input_paths)) + scan_results = asyncio.run( + self.run_scan_multi(input_paths=input_paths) + ) + self.pbar.close() + return scan_results + + async def run_scan_multi( + self, + input_paths: "Iterable[Path]", + ) -> "Sequence[Dict[str, Any]]": + + if os.getenv("PYCHARM_HOSTED") == "1": + # PYCHARM_HOSTED env variable equals 1 when running via Pycharm. + # it avoids us from crashing, which happens when using multiprocessing via Pycharm's debug-mode + logging.warning("Running the scans in sequence for avoiding crashing when running via Pycharm") + scan_results: list[dict[str, Any]] = [] + for input_path in input_paths: + scan_results.append(await self.run_scan(input_path)) + else: + scan_results = await asyncio.gather(*[self.run_scan(i) for i in input_paths]) + + if any(scan_result.get("packages") is None for scan_result in scan_results): + image_scanner.setup_twistcli() + + if os.getenv("PYCHARM_HOSTED") == "1": + # PYCHARM_HOSTED env variable equals 1 when running via Pycharm. + # it avoids us from crashing, which happens when using multiprocessing via Pycharm's debug-mode + logging.warning("Running the scans in sequence for avoiding crashing when running via Pycharm") + scan_results = [ + await self.execute_twistcli_scan(input_path) if scan_results[idx].get("packages") is None else + scan_results[idx] for idx, input_path in enumerate(input_paths) + ] + else: + input_paths_as_list: List[Path] = list(input_paths) # create a list from a set ("Iterable") + indices_to_fix: List[int] = [ + idx + for idx in range(len(input_paths_as_list)) + if scan_results[idx].get("packages") is None + ] + new_scan_results = await asyncio.gather(*[ + self.execute_twistcli_scan(input_paths_as_list[idx]) for idx in indices_to_fix + ]) + for idx in indices_to_fix: + res = new_scan_results.pop(0) + res['repository'] = str(input_paths_as_list[idx]) + scan_results[idx] = res + + return scan_results + + async def run_scan(self, input_path: Path) -> dict[str, Any]: + self.pbar.set_additional_data({'Current File Scanned': os.path.relpath(input_path, self.root_folder)}) + logging.info(f"Start to scan package file {input_path}") + + request_body = { + "compressedFileBody": compress_file_gzip_base64(str(input_path)), + "compressionMethod": "gzip", + "fileName": input_path.name + } + + response = request_wrapper( + "POST", f"{self._base_url}/api/v1/vulnerabilities/scan", + headers=bc_integration.get_default_headers("POST"), + json=request_body, + should_call_raise_for_status=True + ) + + response_json = response.json() + + if response_json["status"] == "already_exist": + logging.info(f"result for {input_path} exists in the cache") + return self.parse_api_result(input_path, response_json["outputData"]) + + return self.run_scan_busy_wait(input_path, response_json['id']) + + def run_scan_busy_wait(self, input_path: Path, scan_id: str) -> dict[str, Any]: + current_state = "Empty" + desired_state = "Result" + total_sleeping_time = 0 + response = requests.Response() + + while current_state != desired_state: + response = request_wrapper( + "GET", f"{self._base_url}/api/v1/vulnerabilities/scan-results/{scan_id}", + headers=bc_integration.get_default_headers("GET") + ) + response_json = response.json() + current_state = response_json["outputType"] + + if current_state == "Error": + logging.error(response_json["outputData"]) + return {} + + if total_sleeping_time > MAX_SLEEP_DURATION: + logging.info(f"Timeout, slept for {total_sleeping_time}") + return {} + + time.sleep(SLEEP_DURATION) + total_sleeping_time += SLEEP_DURATION + + return self.parse_api_result(input_path, response.json()["outputData"]) + + def parse_api_result(self, origin_file_path: Path, response: str) -> dict[str, Any]: + raw_result: dict[str, Any] = json.loads(decompress_file_gzip_base64(response)) + raw_result['repository'] = str(origin_file_path) + self.pbar.update() + return raw_result + + async def execute_twistcli_scan( + self, + input_path: Path, + ) -> Dict[str, Any]: + output_path = Path(f'results-{input_path.name}.json') + + command = f"{Path(bridgecrew_dir) / TWISTCLI_FILE_NAME} coderepo scan --address {docker_image_scanning_integration.get_proxy_address()} --token {docker_image_scanning_integration.get_bc_api_key()} --details --output-file \"{output_path}\" {input_path}" + process = await asyncio.create_subprocess_shell( + command, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE + ) + + stdout, stderr = await process.communicate() + + # log output for debugging + logging.debug(stdout.decode()) + + exit_code = await process.wait() + + if exit_code: + logging.error(stderr.decode()) + return {} + + # read the report file + scan_result: Dict[str, Any] = json.loads(output_path.read_text()) + output_path.unlink() + return scan_result From 86a47b5a20607ce87ccf83ad13b40b1da2448b60 Mon Sep 17 00:00:00 2001 From: ajbara Date: Sun, 6 Nov 2022 16:30:27 +0200 Subject: [PATCH 02/14] updated --- .../common/bridgecrew/platform_integration.py | 2 + checkov/main.py | 2 +- checkov/sca_package_2/runner.py | 47 +++--- checkov/sca_package_2/scanner.py | 156 +----------------- 4 files changed, 29 insertions(+), 178 deletions(-) diff --git a/checkov/common/bridgecrew/platform_integration.py b/checkov/common/bridgecrew/platform_integration.py index 6a754d2d3eb..1cbb8af1656 100644 --- a/checkov/common/bridgecrew/platform_integration.py +++ b/checkov/common/bridgecrew/platform_integration.py @@ -334,6 +334,8 @@ def persist_repository( f_name = os.path.basename(f) _, file_extension = os.path.splitext(f) if file_extension in SUPPORTED_FILE_EXTENSIONS or f_name in SUPPORTED_FILES: + if RUN_NEW_SCA_PACKAGE_SCAN and file_extension in SUPPORTED_PACKAGE_FILES: + continue files_to_persist.append(FileToPersist(f, os.path.relpath(f, root_dir))) else: for root_path, d_names, f_names in os.walk(root_dir): diff --git a/checkov/main.py b/checkov/main.py index ccc40f41334..18c4c44fb47 100755 --- a/checkov/main.py +++ b/checkov/main.py @@ -76,7 +76,7 @@ logger = logging.getLogger(__name__) checkov_runners = [value for attr, value in CheckType.__dict__.items() if not attr.startswith("__")] -RUN_NEW_SCA_PACKAGE_SCAN = os.getenv('RUN_NEW_SCA_PACKAGE_SCAN', False) +RUN_NEW_SCA_PACKAGE_SCAN = os.getenv('RUN_NEW_SCA_PACKAGE_SCAN', '').lower() == 'true' DEFAULT_RUNNERS = ( tf_graph_runner(), diff --git a/checkov/sca_package_2/runner.py b/checkov/sca_package_2/runner.py index fe2e63c77a4..4d716fd81e1 100644 --- a/checkov/sca_package_2/runner.py +++ b/checkov/sca_package_2/runner.py @@ -14,7 +14,7 @@ from checkov.common.bridgecrew.check_type import CheckType from checkov.common.runners.base_runner import BaseRunner, ignored_directories from checkov.runner_filter import RunnerFilter -from checkov.sca_package.scanner import Scanner +from checkov.sca_package_2.scanner import Scanner class Runner(BaseRunner[None]): @@ -31,7 +31,6 @@ def prepare_and_scan( root_folder: str | Path | None, files: list[str] | None = None, runner_filter: RunnerFilter | None = None, - exclude_package_json: bool = True, excluded_file_names: set[str] | None = None, ) -> Sequence[dict[str, Any]] | None: runner_filter = runner_filter or RunnerFilter() @@ -53,22 +52,19 @@ def prepare_and_scan( if runner_filter.excluded_paths: excluded_paths.update(runner_filter.excluded_paths) - input_paths = self.find_scannable_files( + if not self.upload_scannable_files( root_path=self._code_repo_path, files=files, excluded_paths=excluded_paths, - exclude_package_json=exclude_package_json, - excluded_file_names=excluded_file_names - ) - if not input_paths: + excluded_file_names=excluded_file_names, + root_folder=root_folder + ): # no packages found return None - logging.info(f"SCA package scanning will scan {len(input_paths)} files") - scanner = Scanner(self.pbar, root_folder) self._check_class = f"{scanner.__module__}.{scanner.__class__.__qualname__}" - scan_results = scanner.scan(input_paths) + scan_results = scanner.scan() logging.info(f"SCA package scanning successfully scanned {len(scan_results)} files") return scan_results @@ -124,36 +120,31 @@ def run( return report - def find_scannable_files( + def upload_scannable_files( self, root_path: Path | None, files: list[str] | None, excluded_paths: set[str], - exclude_package_json: bool = True, - excluded_file_names: set[str] | None = None - ) -> set[Path]: + excluded_file_names: set[str] | None = None, + root_folder: str = "" + ) -> bool: + """ upload scannable files to s3""" excluded_file_names = excluded_file_names or set() - input_paths: set[Path] = set() package_files_to_persist: List[FileToPersist] = [] if root_path: - input_paths = { - file_path - for file_path in root_path.glob("**/*") + for file_path in root_path.glob("**/*"): if file_path.name in SUPPORTED_PACKAGE_FILES and not any( - p in file_path.parts for p in excluded_paths) and file_path.name not in excluded_file_names - } + p in file_path.parts for p in excluded_paths) and file_path.name not in excluded_file_names: + file_path_str = str(file_path) + package_files_to_persist.append(FileToPersist(file_path_str, os.path.relpath(file_path_str, root_folder))) for file in files or []: file_path = Path(file) if not file_path.exists(): logging.warning(f"File {file_path} doesn't exist") continue + package_files_to_persist.append(FileToPersist(file, os.path.relpath(file, root_folder))) - # f_name = os.path.basename(file) - # _, file_extension = os.path.splitext(file) - # if file_extension in SUPPORTED_FILE_EXTENSIONS or f_name in SUPPORTED_FILES: - # files_to_persist.append(FileToPersist(file, os.path.relpath(file, root_dir))) - - input_paths.add(file_path) - - return input_paths + logging.info(f"{len(package_files_to_persist)} sca package files found.") + bc_integration.persist_files(package_files_to_persist) + return bool(package_files_to_persist) diff --git a/checkov/sca_package_2/scanner.py b/checkov/sca_package_2/scanner.py index 08e4ebfe55a..66ff0b3c611 100644 --- a/checkov/sca_package_2/scanner.py +++ b/checkov/sca_package_2/scanner.py @@ -1,23 +1,10 @@ from __future__ import annotations -import asyncio -import json -import logging -import os -import time -from collections.abc import Iterable, Sequence, Collection +from collections.abc import Sequence from pathlib import Path -from typing import Dict, Any, List - -import requests +from typing import Any from checkov.common.bridgecrew.platform_integration import bc_integration -from checkov.common.bridgecrew.platform_key import bridgecrew_dir -from checkov.common.bridgecrew.vulnerability_scanning.image_scanner import image_scanner, TWISTCLI_FILE_NAME -from checkov.common.bridgecrew.vulnerability_scanning.integrations.docker_image_scanning import \ - docker_image_scanning_integration -from checkov.common.util.file_utils import compress_file_gzip_base64, decompress_file_gzip_base64 -from checkov.common.util.http_utils import request_wrapper from checkov.common.util.tqdm_utils import ProgressBar @@ -35,138 +22,9 @@ def __init__(self, pbar: ProgressBar | None = None, root_folder: str | Path | No self.pbar.turn_off_progress_bar() self.root_folder = root_folder - def scan(self, input_paths: Collection[Path]) -> Sequence[dict[str, Any]]: - self.pbar.initiate(len(input_paths)) - scan_results = asyncio.run( - self.run_scan_multi(input_paths=input_paths) - ) - self.pbar.close() - return scan_results - - async def run_scan_multi( - self, - input_paths: "Iterable[Path]", - ) -> "Sequence[Dict[str, Any]]": - - if os.getenv("PYCHARM_HOSTED") == "1": - # PYCHARM_HOSTED env variable equals 1 when running via Pycharm. - # it avoids us from crashing, which happens when using multiprocessing via Pycharm's debug-mode - logging.warning("Running the scans in sequence for avoiding crashing when running via Pycharm") - scan_results: list[dict[str, Any]] = [] - for input_path in input_paths: - scan_results.append(await self.run_scan(input_path)) - else: - scan_results = await asyncio.gather(*[self.run_scan(i) for i in input_paths]) - - if any(scan_result.get("packages") is None for scan_result in scan_results): - image_scanner.setup_twistcli() - - if os.getenv("PYCHARM_HOSTED") == "1": - # PYCHARM_HOSTED env variable equals 1 when running via Pycharm. - # it avoids us from crashing, which happens when using multiprocessing via Pycharm's debug-mode - logging.warning("Running the scans in sequence for avoiding crashing when running via Pycharm") - scan_results = [ - await self.execute_twistcli_scan(input_path) if scan_results[idx].get("packages") is None else - scan_results[idx] for idx, input_path in enumerate(input_paths) - ] - else: - input_paths_as_list: List[Path] = list(input_paths) # create a list from a set ("Iterable") - indices_to_fix: List[int] = [ - idx - for idx in range(len(input_paths_as_list)) - if scan_results[idx].get("packages") is None - ] - new_scan_results = await asyncio.gather(*[ - self.execute_twistcli_scan(input_paths_as_list[idx]) for idx in indices_to_fix - ]) - for idx in indices_to_fix: - res = new_scan_results.pop(0) - res['repository'] = str(input_paths_as_list[idx]) - scan_results[idx] = res - - return scan_results - - async def run_scan(self, input_path: Path) -> dict[str, Any]: - self.pbar.set_additional_data({'Current File Scanned': os.path.relpath(input_path, self.root_folder)}) - logging.info(f"Start to scan package file {input_path}") - - request_body = { - "compressedFileBody": compress_file_gzip_base64(str(input_path)), - "compressionMethod": "gzip", - "fileName": input_path.name - } - - response = request_wrapper( - "POST", f"{self._base_url}/api/v1/vulnerabilities/scan", - headers=bc_integration.get_default_headers("POST"), - json=request_body, - should_call_raise_for_status=True - ) - - response_json = response.json() - - if response_json["status"] == "already_exist": - logging.info(f"result for {input_path} exists in the cache") - return self.parse_api_result(input_path, response_json["outputData"]) - - return self.run_scan_busy_wait(input_path, response_json['id']) - - def run_scan_busy_wait(self, input_path: Path, scan_id: str) -> dict[str, Any]: - current_state = "Empty" - desired_state = "Result" - total_sleeping_time = 0 - response = requests.Response() - - while current_state != desired_state: - response = request_wrapper( - "GET", f"{self._base_url}/api/v1/vulnerabilities/scan-results/{scan_id}", - headers=bc_integration.get_default_headers("GET") - ) - response_json = response.json() - current_state = response_json["outputType"] - - if current_state == "Error": - logging.error(response_json["outputData"]) - return {} - - if total_sleeping_time > MAX_SLEEP_DURATION: - logging.info(f"Timeout, slept for {total_sleeping_time}") - return {} - - time.sleep(SLEEP_DURATION) - total_sleeping_time += SLEEP_DURATION - - return self.parse_api_result(input_path, response.json()["outputData"]) - - def parse_api_result(self, origin_file_path: Path, response: str) -> dict[str, Any]: - raw_result: dict[str, Any] = json.loads(decompress_file_gzip_base64(response)) - raw_result['repository'] = str(origin_file_path) - self.pbar.update() - return raw_result - - async def execute_twistcli_scan( - self, - input_path: Path, - ) -> Dict[str, Any]: - output_path = Path(f'results-{input_path.name}.json') - - command = f"{Path(bridgecrew_dir) / TWISTCLI_FILE_NAME} coderepo scan --address {docker_image_scanning_integration.get_proxy_address()} --token {docker_image_scanning_integration.get_bc_api_key()} --details --output-file \"{output_path}\" {input_path}" - process = await asyncio.create_subprocess_shell( - command, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE - ) - - stdout, stderr = await process.communicate() - - # log output for debugging - logging.debug(stdout.decode()) - - exit_code = await process.wait() - - if exit_code: - logging.error(stderr.decode()) - return {} + def scan(self) -> Sequence[dict[str, Any]]: + """run SCA scan and poll the results are ready""" + pass - # read the report file - scan_result: Dict[str, Any] = json.loads(output_path.read_text()) - output_path.unlink() - return scan_result + def poll_scan_result(self) -> dict[str, Any]: + pass From 9dd100106ecea81f9d36cfcba89b018e0b1e4f31 Mon Sep 17 00:00:00 2001 From: ajbara Date: Sun, 6 Nov 2022 17:04:29 +0200 Subject: [PATCH 03/14] updated --- .../common/bridgecrew/platform_integration.py | 2 +- checkov/sca_package_2/output.py | 364 ------------------ 2 files changed, 1 insertion(+), 365 deletions(-) delete mode 100644 checkov/sca_package_2/output.py diff --git a/checkov/common/bridgecrew/platform_integration.py b/checkov/common/bridgecrew/platform_integration.py index 1cbb8af1656..cc82d989052 100644 --- a/checkov/common/bridgecrew/platform_integration.py +++ b/checkov/common/bridgecrew/platform_integration.py @@ -74,8 +74,8 @@ 'Content-Type': 'application/json;charset=UTF-8'}, get_user_agent_header()) CI_METADATA_EXTRACTOR = registry.get_extractor() -RUN_NEW_SCA_PACKAGE_SCAN = os.getenv('RUN_NEW_SCA_PACKAGE_SCAN', False) +RUN_NEW_SCA_PACKAGE_SCAN = os.getenv('RUN_NEW_SCA_PACKAGE_SCAN', '').lower() == 'true' class BcPlatformIntegration: def __init__(self) -> None: diff --git a/checkov/sca_package_2/output.py b/checkov/sca_package_2/output.py deleted file mode 100644 index 8466ce66aab..00000000000 --- a/checkov/sca_package_2/output.py +++ /dev/null @@ -1,364 +0,0 @@ -from __future__ import annotations - -import itertools -import logging -from collections import defaultdict -from dataclasses import dataclass -from typing import List, Union, Dict, Any - -from packaging import version as packaging_version -from prettytable import PrettyTable, SINGLE_BORDER - -from checkov.common.bridgecrew.severities import Severities, BcSeverities -from checkov.common.models.enums import CheckResult -from checkov.common.output.record import Record, DEFAULT_SEVERITY, SCA_PACKAGE_SCAN_CHECK_NAME, SCA_LICENSE_CHECK_NAME -from checkov.common.sca.commons import UNFIXABLE_VERSION -from checkov.common.typing import _LicenseStatus - - -@dataclass -class CveCount: - total: int = 0 - critical: int = 0 - high: int = 0 - medium: int = 0 - low: int = 0 - skipped: int = 0 - has_fix: int = 0 - to_fix: int = 0 - fixable: bool = True - - def output_row(self) -> List[str]: - return [ - f"Total CVEs: {self.total}", - f"critical: {self.critical}", - f"high: {self.high}", - f"medium: {self.medium}", - f"low: {self.low}", - f"skipped: {self.skipped}", - ] - - -def calculate_lowest_compliant_version( - fix_versions_lists: List[List[Union[packaging_version.Version, packaging_version.LegacyVersion]]] -) -> str: - """A best effort approach to find the lowest compliant version""" - - package_min_versions = set() - package_versions = set() - - for fix_versions in fix_versions_lists: - if fix_versions: - package_min_versions.add(min(fix_versions)) - package_versions.update(fix_versions) - if package_min_versions: - package_min_version = min(package_min_versions) - package_max_version = max(package_min_versions) - - if isinstance(package_min_version, packaging_version.LegacyVersion) or isinstance( - package_max_version, packaging_version.LegacyVersion - ): - return str(package_max_version) - elif package_min_version.major == package_max_version.major: - return str(package_max_version) - else: - lowest_version = max( - version - for version in package_versions - if isinstance(version, packaging_version.Version) and version.major == package_max_version.major - ) - return str(lowest_version) - - return UNFIXABLE_VERSION - - -def compare_cve_severity(cve: Dict[str, str]) -> int: - severity = (cve.get("severity") or DEFAULT_SEVERITY).upper() - return Severities[severity].level - - -def create_cli_output(fixable: bool = True, *cve_records: list[Record]) -> str: - cli_outputs = [] - group_by_file_path_package_map: dict[str, dict[str, list[Record]]] = defaultdict(dict) - - for record in itertools.chain(*cve_records): - if not record.vulnerability_details: - # this shouldn't happen - logging.error(f"'vulnerability_details' is not set for {record.check_id}") - continue - - group_by_file_path_package_map[record.file_path].setdefault( - record.vulnerability_details["package_name"], [] - ).append(record) - - for file_path, packages in group_by_file_path_package_map.items(): - cve_count = CveCount(fixable=fixable) - package_cves_details_map: dict[str, dict[str, Any]] = defaultdict(dict) - package_licenses_details_map = defaultdict(list) - should_print_licenses_table = False - for package_name, records in packages.items(): - package_version = None - fix_versions_lists = [] - - for record in records: - if not record.vulnerability_details: - # this shouldn't happen - logging.error(f"'vulnerability_details' is not set for {record.check_id}") - continue - - if record.check_name == SCA_PACKAGE_SCAN_CHECK_NAME: - cve_count.total += 1 - - if record.check_result["result"] == CheckResult.SKIPPED: - cve_count.skipped += 1 - continue - else: - cve_count.to_fix += 1 - - # best way to dynamically access a class instance attribute. - # (we can't just do cve_count.severity_str to access the correct severity) - severity_str = record.severity.name.lower() if record.severity else BcSeverities.NONE.lower() - setattr(cve_count, severity_str, getattr(cve_count, severity_str) + 1) - - if record.vulnerability_details["lowest_fixed_version"] != UNFIXABLE_VERSION: - cve_count.has_fix += 1 - - fix_versions_lists.append(record.vulnerability_details["fixed_versions"]) - if package_version is None: - package_version = record.vulnerability_details["package_version"] - - package_cves_details_map[package_name].setdefault("cves", []).append( - { - "id": record.vulnerability_details["id"], - "severity": severity_str, - "fixed_version": record.vulnerability_details["lowest_fixed_version"], - } - ) - elif record.check_name == SCA_LICENSE_CHECK_NAME: - if record.check_result["result"] == CheckResult.SKIPPED: - continue - should_print_licenses_table = True - package_licenses_details_map[package_name].append( - _LicenseStatus(package_name=package_name, - package_version=record.vulnerability_details["package_version"], - policy=record.vulnerability_details["policy"], - license=record.vulnerability_details["license"], - status=record.vulnerability_details["status"]) - ) - - if package_name in package_cves_details_map: - package_cves_details_map[package_name]["cves"].sort(key=compare_cve_severity, reverse=True) - package_cves_details_map[package_name]["current_version"] = package_version - package_cves_details_map[package_name]["compliant_version"] = calculate_lowest_compliant_version( - fix_versions_lists - ) - - if cve_count.total > 0: - cli_outputs.append( - create_cli_cves_table( - file_path=file_path, - cve_count=cve_count, - package_details_map=package_cves_details_map, - ) - ) - if should_print_licenses_table: - cli_outputs.append( - create_cli_license_violations_table( - file_path=file_path, - package_licenses_details_map=package_licenses_details_map - ) - ) - - return "\n".join(cli_outputs) - - -def create_cli_license_violations_table(file_path: str, package_licenses_details_map: Dict[str, List[_LicenseStatus]]) -> str: - package_table_lines: List[str] = [] - columns = 5 - table_width = 120.0 - column_width = int(table_width / columns) - package_table = PrettyTable(min_table_width=table_width, max_table_width=table_width) - package_table.set_style(SINGLE_BORDER) - package_table.field_names = [ - "Package name", - "Package version", - "Policy ID", - "License", - "Status", - ] - for package_idx, (package_name, license_statuses) in enumerate(package_licenses_details_map.items()): - if package_idx > 0: - del package_table_lines[-1] - package_table.header = False - package_table.clear_rows() - - for idx, license_status in enumerate(license_statuses): - col_package_name = "" - col_package_version = "" - if idx == 0: - col_package_name = package_name - col_package_version = license_status["package_version"] - - package_table.add_row( - [ - col_package_name, - col_package_version, - license_status["policy"], - license_status["license"], - license_status["status"], - ] - ) - - package_table.align = "l" - package_table.min_width = column_width - package_table.max_width = column_width - - for idx, line in enumerate(package_table.get_string().splitlines(keepends=True)): - if idx == 0 and package_idx != 0: - # hack to make multiple tables look like one - line = line.replace(package_table.top_left_junction_char, package_table.left_junction_char).replace( - package_table.top_right_junction_char, package_table.right_junction_char - ) - if package_idx > 0: - # hack to make multiple package tables look like one - line = line.replace(package_table.top_junction_char, package_table.junction_char) - - # hack for making the table's width as same as the cves-table's - package_table_lines.append(f"\t{line[:-2]}{line[-3]}{line[-2:]}") - - return ( - f"\t{file_path} - Licenses Statuses:\n" - f"{''.join(package_table_lines)}\n" - ) - - -def create_cli_cves_table(file_path: str, cve_count: CveCount, package_details_map: Dict[str, Dict[str, Any]]) -> str: - columns = 6 - table_width = 120 - column_width = int(120 / columns) - - cve_table_lines = create_cve_summary_table_part( - table_width=table_width, column_width=column_width, cve_count=cve_count - ) - - vulnerable_packages = True if package_details_map else False - fixable_table_lines = create_fixable_cve_summary_table_part( - table_width=table_width, column_count=columns, cve_count=cve_count, vulnerable_packages=vulnerable_packages - ) - - package_table_lines = create_package_overview_table_part( - table_width=table_width, column_width=column_width, package_details_map=package_details_map - ) - - return ( - f"\t{file_path} - CVEs Summary:\n" - f"{''.join(cve_table_lines)}\n" - f"{''.join(fixable_table_lines)}" - f"{''.join(package_table_lines)}\n" - ) - - -def create_cve_summary_table_part(table_width: int, column_width: int, cve_count: CveCount) -> List[str]: - cve_table = PrettyTable( - header=False, - padding_width=1, - min_table_width=table_width, - max_table_width=table_width, - ) - cve_table.set_style(SINGLE_BORDER) - cve_table.add_row(cve_count.output_row()) - cve_table.align = "l" - cve_table.min_width = column_width - cve_table.max_width = column_width - - cve_table_lines = [f"\t{line}" for line in cve_table.get_string().splitlines(keepends=True)] - # hack to make multiple tables look like one - cve_table_bottom_line = ( - cve_table_lines[-1] - .replace(cve_table.bottom_left_junction_char, cve_table.left_junction_char) - .replace(cve_table.bottom_right_junction_char, cve_table.right_junction_char) - ) - cve_table_lines[-1] = cve_table_bottom_line - - return cve_table_lines - - -def create_fixable_cve_summary_table_part( - table_width: int, column_count: int, cve_count: CveCount, vulnerable_packages: bool -) -> List[str]: - fixable_table = PrettyTable( - header=False, min_table_width=table_width + column_count * 2, max_table_width=table_width + column_count * 2 - ) - fixable_table.set_style(SINGLE_BORDER) - if cve_count.fixable: - fixable_table.add_row( - [f"To fix {cve_count.has_fix}/{cve_count.to_fix} CVEs, go to https://www.bridgecrew.cloud/"]) - fixable_table.align = "l" - - # hack to make multiple tables look like one - fixable_table_lines = [f"\t{line}" for line in fixable_table.get_string().splitlines(keepends=True)] - del fixable_table_lines[0] - # only remove the last line, if there are vulnerable packages - if vulnerable_packages: - del fixable_table_lines[-1] - - return fixable_table_lines - - -def create_package_overview_table_part( - table_width: int, column_width: int, package_details_map: Dict[str, Dict[str, Any]] -) -> List[str]: - package_table_lines: List[str] = [] - package_table = PrettyTable(min_table_width=table_width, max_table_width=table_width) - package_table.set_style(SINGLE_BORDER) - package_table.field_names = [ - "Package", - "CVE ID", - "Severity", - "Current version", - "Fixed version", - "Compliant version", - ] - for package_idx, (package_name, details) in enumerate(package_details_map.items()): - if package_idx > 0: - del package_table_lines[-1] - package_table.header = False - package_table.clear_rows() - - for cve_idx, cve in enumerate(details["cves"]): - col_package = "" - col_current_version = "" - col_compliant_version = "" - if cve_idx == 0: - col_package = package_name - col_current_version = details["current_version"] - col_compliant_version = details["compliant_version"] - - package_table.add_row( - [ - col_package, - cve["id"], - cve["severity"], - col_current_version, - cve["fixed_version"], - col_compliant_version, - ] - ) - - package_table.align = "l" - package_table.min_width = column_width - package_table.max_width = column_width - - for idx, line in enumerate(package_table.get_string().splitlines(keepends=True)): - if idx == 0: - # hack to make multiple tables look like one - line = line.replace(package_table.top_left_junction_char, package_table.left_junction_char).replace( - package_table.top_right_junction_char, package_table.right_junction_char - ) - if package_idx > 0: - # hack to make multiple package tables look like one - line = line.replace(package_table.top_junction_char, package_table.junction_char) - - package_table_lines.append(f"\t{line}") - - return package_table_lines From 02319d49fbba4a2ec632da2c305e1c7ca68a8958 Mon Sep 17 00:00:00 2001 From: ajbara Date: Sun, 6 Nov 2022 17:09:24 +0200 Subject: [PATCH 04/14] removed redundant lines --- checkov/main.py | 2 +- checkov/sca_package_2/scanner.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/checkov/main.py b/checkov/main.py index 18c4c44fb47..b8eafdddeff 100755 --- a/checkov/main.py +++ b/checkov/main.py @@ -230,7 +230,7 @@ def run(banner: str = checkov_banner, argv: List[str] = sys.argv[1:]) -> Optiona source=source, source_version=source_version, repo_branch=config.branch, - prisma_api_url=config.prisma_api_url) # TODO here we create the timestamp + prisma_api_url=config.prisma_api_url) except MaxRetryError: return None except Exception: diff --git a/checkov/sca_package_2/scanner.py b/checkov/sca_package_2/scanner.py index 66ff0b3c611..8b91f8f6ebc 100644 --- a/checkov/sca_package_2/scanner.py +++ b/checkov/sca_package_2/scanner.py @@ -23,7 +23,7 @@ def __init__(self, pbar: ProgressBar | None = None, root_folder: str | Path | No self.root_folder = root_folder def scan(self) -> Sequence[dict[str, Any]]: - """run SCA scan and poll the results are ready""" + """run SCA package scan and poll scan results""" pass def poll_scan_result(self) -> dict[str, Any]: From d0971c80e4d79393e1590532eb704cbbccb078ed Mon Sep 17 00:00:00 2001 From: ajbara Date: Sun, 6 Nov 2022 17:30:23 +0200 Subject: [PATCH 05/14] update class --- checkov/sca_package_2/scanner.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/checkov/sca_package_2/scanner.py b/checkov/sca_package_2/scanner.py index 8b91f8f6ebc..1702b657fda 100644 --- a/checkov/sca_package_2/scanner.py +++ b/checkov/sca_package_2/scanner.py @@ -1,6 +1,6 @@ from __future__ import annotations -from collections.abc import Sequence +from collections.abc import Sequence from pathlib import Path from typing import Any @@ -26,5 +26,9 @@ def scan(self) -> Sequence[dict[str, Any]]: """run SCA package scan and poll scan results""" pass + def run_scan(self) -> dict[str, Any]: + pass + def poll_scan_result(self) -> dict[str, Any]: pass + From b6892df5e2e35a06f4a17e32b6a27d575c4487d2 Mon Sep 17 00:00:00 2001 From: ajbara Date: Sun, 6 Nov 2022 18:05:36 +0200 Subject: [PATCH 06/14] mypy --- checkov/common/bridgecrew/platform_integration.py | 1 + checkov/sca_package_2/runner.py | 15 ++++++++------- checkov/sca_package_2/scanner.py | 1 - 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/checkov/common/bridgecrew/platform_integration.py b/checkov/common/bridgecrew/platform_integration.py index cc82d989052..d392919ef28 100644 --- a/checkov/common/bridgecrew/platform_integration.py +++ b/checkov/common/bridgecrew/platform_integration.py @@ -77,6 +77,7 @@ RUN_NEW_SCA_PACKAGE_SCAN = os.getenv('RUN_NEW_SCA_PACKAGE_SCAN', '').lower() == 'true' + class BcPlatformIntegration: def __init__(self) -> None: self.bc_api_key = read_key() diff --git a/checkov/sca_package_2/runner.py b/checkov/sca_package_2/runner.py index 4d716fd81e1..66cbddbac3c 100644 --- a/checkov/sca_package_2/runner.py +++ b/checkov/sca_package_2/runner.py @@ -53,11 +53,11 @@ def prepare_and_scan( excluded_paths.update(runner_filter.excluded_paths) if not self.upload_scannable_files( - root_path=self._code_repo_path, - files=files, - excluded_paths=excluded_paths, - excluded_file_names=excluded_file_names, - root_folder=root_folder + root_path=self._code_repo_path, + files=files, + excluded_paths=excluded_paths, + excluded_file_names=excluded_file_names, + root_folder=root_folder ): # no packages found return None @@ -126,7 +126,7 @@ def upload_scannable_files( files: list[str] | None, excluded_paths: set[str], excluded_file_names: set[str] | None = None, - root_folder: str = "" + root_folder: str | Path | None = "" ) -> bool: """ upload scannable files to s3""" excluded_file_names = excluded_file_names or set() @@ -136,7 +136,8 @@ def upload_scannable_files( if file_path.name in SUPPORTED_PACKAGE_FILES and not any( p in file_path.parts for p in excluded_paths) and file_path.name not in excluded_file_names: file_path_str = str(file_path) - package_files_to_persist.append(FileToPersist(file_path_str, os.path.relpath(file_path_str, root_folder))) + package_files_to_persist.append( + FileToPersist(file_path_str, os.path.relpath(file_path_str, root_folder))) for file in files or []: file_path = Path(file) diff --git a/checkov/sca_package_2/scanner.py b/checkov/sca_package_2/scanner.py index 1702b657fda..68c86f302af 100644 --- a/checkov/sca_package_2/scanner.py +++ b/checkov/sca_package_2/scanner.py @@ -31,4 +31,3 @@ def run_scan(self) -> dict[str, Any]: def poll_scan_result(self) -> dict[str, Any]: pass - From 220b20e024127c53b7e1b704b8f14755488ebc44 Mon Sep 17 00:00:00 2001 From: ajbara Date: Sun, 6 Nov 2022 18:08:31 +0200 Subject: [PATCH 07/14] flake8 --- checkov/sca_package_2/runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/checkov/sca_package_2/runner.py b/checkov/sca_package_2/runner.py index 66cbddbac3c..57ea86204b9 100644 --- a/checkov/sca_package_2/runner.py +++ b/checkov/sca_package_2/runner.py @@ -126,7 +126,7 @@ def upload_scannable_files( files: list[str] | None, excluded_paths: set[str], excluded_file_names: set[str] | None = None, - root_folder: str | Path | None = "" + root_folder: str | Path | None = "" ) -> bool: """ upload scannable files to s3""" excluded_file_names = excluded_file_names or set() From 4b63398bca62149259a6a5043432f1e298c9c99f Mon Sep 17 00:00:00 2001 From: ajbara Date: Mon, 7 Nov 2022 10:37:46 +0200 Subject: [PATCH 08/14] RUN_SCA_PACKAGE_SCAN_V2 --- checkov/common/bridgecrew/platform_integration.py | 6 +++--- checkov/main.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/checkov/common/bridgecrew/platform_integration.py b/checkov/common/bridgecrew/platform_integration.py index d392919ef28..d83bf6cf5e6 100644 --- a/checkov/common/bridgecrew/platform_integration.py +++ b/checkov/common/bridgecrew/platform_integration.py @@ -75,7 +75,7 @@ get_user_agent_header()) CI_METADATA_EXTRACTOR = registry.get_extractor() -RUN_NEW_SCA_PACKAGE_SCAN = os.getenv('RUN_NEW_SCA_PACKAGE_SCAN', '').lower() == 'true' +RUN_SCA_PACKAGE_SCAN_V2 = os.getenv('RUN_NEW_SCA_PACKAGE_SCAN', '').lower() == 'true' class BcPlatformIntegration: @@ -335,7 +335,7 @@ def persist_repository( f_name = os.path.basename(f) _, file_extension = os.path.splitext(f) if file_extension in SUPPORTED_FILE_EXTENSIONS or f_name in SUPPORTED_FILES: - if RUN_NEW_SCA_PACKAGE_SCAN and file_extension in SUPPORTED_PACKAGE_FILES: + if RUN_SCA_PACKAGE_SCAN_V2 and file_extension in SUPPORTED_PACKAGE_FILES: continue files_to_persist.append(FileToPersist(f, os.path.relpath(f, root_dir))) else: @@ -347,7 +347,7 @@ def persist_repository( for file_path in f_names: _, file_extension = os.path.splitext(file_path) if file_extension in SUPPORTED_FILE_EXTENSIONS or file_path in SUPPORTED_FILES: - if RUN_NEW_SCA_PACKAGE_SCAN and file_extension in SUPPORTED_PACKAGE_FILES: + if RUN_SCA_PACKAGE_SCAN_V2 and file_extension in SUPPORTED_PACKAGE_FILES: continue full_file_path = os.path.join(root_path, file_path) relative_file_path = os.path.relpath(full_file_path, root_dir) diff --git a/checkov/main.py b/checkov/main.py index b8eafdddeff..58f8a673c12 100755 --- a/checkov/main.py +++ b/checkov/main.py @@ -76,7 +76,7 @@ logger = logging.getLogger(__name__) checkov_runners = [value for attr, value in CheckType.__dict__.items() if not attr.startswith("__")] -RUN_NEW_SCA_PACKAGE_SCAN = os.getenv('RUN_NEW_SCA_PACKAGE_SCAN', '').lower() == 'true' +RUN_SCA_PACKAGE_SCAN_V2 = os.getenv('RUN_NEW_SCA_PACKAGE_SCAN', '').lower() == 'true' DEFAULT_RUNNERS = ( tf_graph_runner(), @@ -103,7 +103,7 @@ argo_workflows_runner(), circleci_pipelines_runner(), azure_pipelines_runner(), - sca_package_runner_2() if RUN_NEW_SCA_PACKAGE_SCAN else sca_package_runner() + sca_package_runner_2() if RUN_SCA_PACKAGE_SCAN_V2 else sca_package_runner() ) From d92d69ac1e26c797416b39eb1da13895024ec736 Mon Sep 17 00:00:00 2001 From: ajbara Date: Mon, 7 Nov 2022 10:53:18 +0200 Subject: [PATCH 09/14] RUN_SCA_PACKAGE_SCAN_V2 --- checkov/common/bridgecrew/platform_integration.py | 2 +- checkov/main.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/checkov/common/bridgecrew/platform_integration.py b/checkov/common/bridgecrew/platform_integration.py index d83bf6cf5e6..f0f310b9cba 100644 --- a/checkov/common/bridgecrew/platform_integration.py +++ b/checkov/common/bridgecrew/platform_integration.py @@ -75,7 +75,7 @@ get_user_agent_header()) CI_METADATA_EXTRACTOR = registry.get_extractor() -RUN_SCA_PACKAGE_SCAN_V2 = os.getenv('RUN_NEW_SCA_PACKAGE_SCAN', '').lower() == 'true' +RUN_SCA_PACKAGE_SCAN_V2 = os.getenv('RUN_SCA_PACKAGE_SCAN_V2', '').lower() == 'true' class BcPlatformIntegration: diff --git a/checkov/main.py b/checkov/main.py index 58f8a673c12..a108f0cbcb5 100755 --- a/checkov/main.py +++ b/checkov/main.py @@ -76,7 +76,7 @@ logger = logging.getLogger(__name__) checkov_runners = [value for attr, value in CheckType.__dict__.items() if not attr.startswith("__")] -RUN_SCA_PACKAGE_SCAN_V2 = os.getenv('RUN_NEW_SCA_PACKAGE_SCAN', '').lower() == 'true' +RUN_SCA_PACKAGE_SCAN_V2 = os.getenv('RUN_SCA_PACKAGE_SCAN_V2', '').lower() == 'true' DEFAULT_RUNNERS = ( tf_graph_runner(), From aea734ff0a7589c0a817b28edeb21409a88640ff Mon Sep 17 00:00:00 2001 From: ajbara Date: Mon, 7 Nov 2022 10:56:32 +0200 Subject: [PATCH 10/14] fixed --- checkov/sca_package_2/runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/checkov/sca_package_2/runner.py b/checkov/sca_package_2/runner.py index 57ea86204b9..520da343e9a 100644 --- a/checkov/sca_package_2/runner.py +++ b/checkov/sca_package_2/runner.py @@ -66,7 +66,7 @@ def prepare_and_scan( self._check_class = f"{scanner.__module__}.{scanner.__class__.__qualname__}" scan_results = scanner.scan() - logging.info(f"SCA package scanning successfully scanned {len(scan_results)} files") + # logging.info(f"SCA package scanning successfully scanned {len(scan_results)} files") return scan_results def run( From e300a146f735cd52cf700c22f3716f1f8f8e6675 Mon Sep 17 00:00:00 2001 From: ajbara Date: Mon, 7 Nov 2022 11:49:07 +0200 Subject: [PATCH 11/14] fix files config --- checkov/sca_package_2/runner.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/checkov/sca_package_2/runner.py b/checkov/sca_package_2/runner.py index 520da343e9a..874fa178bfb 100644 --- a/checkov/sca_package_2/runner.py +++ b/checkov/sca_package_2/runner.py @@ -57,7 +57,6 @@ def prepare_and_scan( files=files, excluded_paths=excluded_paths, excluded_file_names=excluded_file_names, - root_folder=root_folder ): # no packages found return None @@ -126,7 +125,6 @@ def upload_scannable_files( files: list[str] | None, excluded_paths: set[str], excluded_file_names: set[str] | None = None, - root_folder: str | Path | None = "" ) -> bool: """ upload scannable files to s3""" excluded_file_names = excluded_file_names or set() @@ -137,13 +135,14 @@ def upload_scannable_files( p in file_path.parts for p in excluded_paths) and file_path.name not in excluded_file_names: file_path_str = str(file_path) package_files_to_persist.append( - FileToPersist(file_path_str, os.path.relpath(file_path_str, root_folder))) + FileToPersist(file_path_str, os.path.relpath(file_path_str, root_path))) for file in files or []: file_path = Path(file) if not file_path.exists(): logging.warning(f"File {file_path} doesn't exist") continue + root_folder = os.path.split(os.path.commonprefix(files))[0] package_files_to_persist.append(FileToPersist(file, os.path.relpath(file, root_folder))) logging.info(f"{len(package_files_to_persist)} sca package files found.") From a3e5a05c65115e4be397698dccca6e571750d7bf Mon Sep 17 00:00:00 2001 From: ajbara Date: Mon, 7 Nov 2022 11:57:49 +0200 Subject: [PATCH 12/14] updated --- checkov/common/bridgecrew/platform_integration.py | 12 +++++------- checkov/common/util/consts.py | 3 +++ checkov/main.py | 5 ++--- checkov/sca_package_2/runner.py | 9 +++++---- 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/checkov/common/bridgecrew/platform_integration.py b/checkov/common/bridgecrew/platform_integration.py index f0f310b9cba..919311b29cf 100644 --- a/checkov/common/bridgecrew/platform_integration.py +++ b/checkov/common/bridgecrew/platform_integration.py @@ -34,7 +34,7 @@ from checkov.common.bridgecrew.check_type import CheckType from checkov.common.runners.base_runner import filter_ignored_paths from checkov.common.typing import _CicdDetails -from checkov.common.util.consts import PRISMA_PLATFORM, BRIDGECREW_PLATFORM +from checkov.common.util.consts import PRISMA_PLATFORM, BRIDGECREW_PLATFORM, CHECKOV_RUN_SCA_PACKAGE_SCAN_V2 from checkov.common.util.data_structures_utils import merge_dicts from checkov.common.util.http_utils import normalize_prisma_url, get_auth_header, get_default_get_headers, \ get_user_agent_header, get_default_post_headers, get_prisma_get_headers, get_prisma_auth_header, \ @@ -75,8 +75,6 @@ get_user_agent_header()) CI_METADATA_EXTRACTOR = registry.get_extractor() -RUN_SCA_PACKAGE_SCAN_V2 = os.getenv('RUN_SCA_PACKAGE_SCAN_V2', '').lower() == 'true' - class BcPlatformIntegration: def __init__(self) -> None: @@ -334,9 +332,9 @@ def persist_repository( for f in files: f_name = os.path.basename(f) _, file_extension = os.path.splitext(f) + if CHECKOV_RUN_SCA_PACKAGE_SCAN_V2 and file_extension in SUPPORTED_PACKAGE_FILES: + continue if file_extension in SUPPORTED_FILE_EXTENSIONS or f_name in SUPPORTED_FILES: - if RUN_SCA_PACKAGE_SCAN_V2 and file_extension in SUPPORTED_PACKAGE_FILES: - continue files_to_persist.append(FileToPersist(f, os.path.relpath(f, root_dir))) else: for root_path, d_names, f_names in os.walk(root_dir): @@ -346,9 +344,9 @@ def persist_repository( filter_ignored_paths(root_path, f_names, excluded_paths) for file_path in f_names: _, file_extension = os.path.splitext(file_path) + if CHECKOV_RUN_SCA_PACKAGE_SCAN_V2 and file_extension in SUPPORTED_PACKAGE_FILES: + continue if file_extension in SUPPORTED_FILE_EXTENSIONS or file_path in SUPPORTED_FILES: - if RUN_SCA_PACKAGE_SCAN_V2 and file_extension in SUPPORTED_PACKAGE_FILES: - continue full_file_path = os.path.join(root_path, file_path) relative_file_path = os.path.relpath(full_file_path, root_dir) files_to_persist.append(FileToPersist(full_file_path, relative_file_path)) diff --git a/checkov/common/util/consts.py b/checkov/common/util/consts.py index 751dc0f0c40..867509d7e25 100644 --- a/checkov/common/util/consts.py +++ b/checkov/common/util/consts.py @@ -24,3 +24,6 @@ BRIDGECREW_PLATFORM = 'Bridgecrew' MAX_IAC_FILE_SIZE = int(os.getenv('CHECKOV_MAX_IAC_FILE_SIZE', '50_000_000')) # 50 MB is default limit + + +CHECKOV_RUN_SCA_PACKAGE_SCAN_V2 = os.getenv('CHECKOV_RUN_SCA_PACKAGE_SCAN_V2', '').lower() == 'true' \ No newline at end of file diff --git a/checkov/main.py b/checkov/main.py index a108f0cbcb5..ce8758fecea 100755 --- a/checkov/main.py +++ b/checkov/main.py @@ -38,7 +38,7 @@ from checkov.common.util import prompt from checkov.common.util.banner import banner as checkov_banner from checkov.common.util.config_utils import get_default_config_paths -from checkov.common.util.consts import DEFAULT_EXTERNAL_MODULES_DIR +from checkov.common.util.consts import DEFAULT_EXTERNAL_MODULES_DIR, CHECKOV_RUN_SCA_PACKAGE_SCAN_V2 from checkov.common.util.docs_generator import print_checks from checkov.common.util.ext_argument_parser import ExtArgumentParser from checkov.common.util.runner_dependency_handler import RunnerDependencyHandler @@ -76,7 +76,6 @@ logger = logging.getLogger(__name__) checkov_runners = [value for attr, value in CheckType.__dict__.items() if not attr.startswith("__")] -RUN_SCA_PACKAGE_SCAN_V2 = os.getenv('RUN_SCA_PACKAGE_SCAN_V2', '').lower() == 'true' DEFAULT_RUNNERS = ( tf_graph_runner(), @@ -103,7 +102,7 @@ argo_workflows_runner(), circleci_pipelines_runner(), azure_pipelines_runner(), - sca_package_runner_2() if RUN_SCA_PACKAGE_SCAN_V2 else sca_package_runner() + sca_package_runner_2() if CHECKOV_RUN_SCA_PACKAGE_SCAN_V2 else sca_package_runner() ) diff --git a/checkov/sca_package_2/runner.py b/checkov/sca_package_2/runner.py index 874fa178bfb..deb5855947c 100644 --- a/checkov/sca_package_2/runner.py +++ b/checkov/sca_package_2/runner.py @@ -125,7 +125,7 @@ def upload_scannable_files( files: list[str] | None, excluded_paths: set[str], excluded_file_names: set[str] | None = None, - ) -> bool: + ) -> List[FileToPersist]: """ upload scannable files to s3""" excluded_file_names = excluded_file_names or set() package_files_to_persist: List[FileToPersist] = [] @@ -142,9 +142,10 @@ def upload_scannable_files( if not file_path.exists(): logging.warning(f"File {file_path} doesn't exist") continue - root_folder = os.path.split(os.path.commonprefix(files))[0] - package_files_to_persist.append(FileToPersist(file, os.path.relpath(file, root_folder))) + if file_path.name in SUPPORTED_PACKAGE_FILES: + root_folder = os.path.split(os.path.commonprefix(files))[0] + package_files_to_persist.append(FileToPersist(file, os.path.relpath(file, root_folder))) logging.info(f"{len(package_files_to_persist)} sca package files found.") bc_integration.persist_files(package_files_to_persist) - return bool(package_files_to_persist) + return package_files_to_persist From 677d5e3033887ae0b6880d3d297d0f50a4a09fb2 Mon Sep 17 00:00:00 2001 From: ajbara Date: Mon, 7 Nov 2022 12:16:21 +0200 Subject: [PATCH 13/14] mypy --- checkov/sca_package_2/runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/checkov/sca_package_2/runner.py b/checkov/sca_package_2/runner.py index deb5855947c..d93da474a78 100644 --- a/checkov/sca_package_2/runner.py +++ b/checkov/sca_package_2/runner.py @@ -143,7 +143,7 @@ def upload_scannable_files( logging.warning(f"File {file_path} doesn't exist") continue if file_path.name in SUPPORTED_PACKAGE_FILES: - root_folder = os.path.split(os.path.commonprefix(files))[0] + root_folder = os.path.split(os.path.commonprefix(files))[0] # type: ignore package_files_to_persist.append(FileToPersist(file, os.path.relpath(file, root_folder))) logging.info(f"{len(package_files_to_persist)} sca package files found.") From 24d545ef46b5ad722f21f46879a04f3ed1f543c7 Mon Sep 17 00:00:00 2001 From: ajbara Date: Mon, 7 Nov 2022 12:42:54 +0200 Subject: [PATCH 14/14] fixed root_folder --- checkov/sca_package_2/runner.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/checkov/sca_package_2/runner.py b/checkov/sca_package_2/runner.py index d93da474a78..8b0c2d917dd 100644 --- a/checkov/sca_package_2/runner.py +++ b/checkov/sca_package_2/runner.py @@ -137,14 +137,15 @@ def upload_scannable_files( package_files_to_persist.append( FileToPersist(file_path_str, os.path.relpath(file_path_str, root_path))) - for file in files or []: - file_path = Path(file) - if not file_path.exists(): - logging.warning(f"File {file_path} doesn't exist") - continue - if file_path.name in SUPPORTED_PACKAGE_FILES: - root_folder = os.path.split(os.path.commonprefix(files))[0] # type: ignore - package_files_to_persist.append(FileToPersist(file, os.path.relpath(file, root_folder))) + if files: + root_folder = os.path.split(os.path.commonprefix(files))[0] + for file in files: + file_path = Path(file) + if not file_path.exists(): + logging.warning(f"File {file_path} doesn't exist") + continue + if file_path.name in SUPPORTED_PACKAGE_FILES: + package_files_to_persist.append(FileToPersist(file, os.path.relpath(file, root_folder))) logging.info(f"{len(package_files_to_persist)} sca package files found.") bc_integration.persist_files(package_files_to_persist)