diff --git a/checkov/main.py b/checkov/main.py index 4f596461517..97b945a22d8 100755 --- a/checkov/main.py +++ b/checkov/main.py @@ -158,9 +158,10 @@ def run(banner: str = checkov_banner, argv: List[str] = sys.argv[1:]) -> Optiona runners=checkov_runners, excluded_paths=excluded_paths, all_external=config.run_all_external_checks, var_files=config.var_file, skip_cve_package=config.skip_cve_package, show_progress_bar=not config.quiet, - secrets_scan_file_type=config.secrets_scan_file_type, use_enforcement_rules=config.use_enforcement_rules, - run_image_referencer=run_image_referencer) + run_image_referencer=run_image_referencer, + enable_secret_scan_all_files=bool(convert_str_to_bool(config.enable_secret_scan_all_files)), + black_list_secret_scan=config.black_list_secret_scan) if outer_registry: runner_registry = outer_registry @@ -562,12 +563,17 @@ def add_parser_args(parser: ArgumentParser) -> None: default=[], env_var='CKV_SECRETS_SCAN_FILE_TYPE', action='append', - help='add scan secret for requested files. You can specify this argument multiple times to add ' - 'multiple file types. To scan all types (".tf", ".hcl", ".yml", ".yaml", ".json", ".template", ' - '".bicep", ".py", ".js", ".properties", ".pem", ".php", ".xml", ".ts", ".env", "Dockerfile", ' - '".java", ".rb", ".go", ".cs", ".txt") specify the argument with `--secrets-scan-file-type all`. ' - 'default scan will be for ".tf", ".hcl", ".yml", ".yaml", ".json", ".template", ".bicep" and ' - 'exclude "Pipfile.lock", "yarn.lock", "package-lock.json", "requirements.txt"') + help='not in use') + parser.add('--enable-secret-scan-all-files', + default=False, + env_var='CKV_SECRETS_SCAN_ENABLE_ALL', + action='store_true', + help='enable secret scan for all files') + parser.add('--black-list-secret-scan', + default=[], + env_var='CKV_SECRETS_SCAN_BLACK_LIST', + action='append', + help='black file list to filter out from the secret scanner') def get_external_checks_dir(config: Any) -> Any: diff --git a/checkov/runner_filter.py b/checkov/runner_filter.py index f0a8640d545..b9331398b00 100644 --- a/checkov/runner_filter.py +++ b/checkov/runner_filter.py @@ -39,8 +39,9 @@ def __init__( use_enforcement_rules: bool = False, filtered_policy_ids: Optional[List[str]] = None, show_progress_bar: Optional[bool] = True, - secrets_scan_file_type: Optional[List[str]] = None, - run_image_referencer: bool = False + run_image_referencer: bool = False, + enable_secret_scan_all_files: bool = False, + black_list_secret_scan: Optional[List[str]] = None ) -> None: checks = convert_csv_string_arg_to_list(checks) @@ -95,8 +96,9 @@ def __init__( self.var_files = var_files self.skip_cve_package = skip_cve_package self.filtered_policy_ids = filtered_policy_ids or [] - self.secrets_scan_file_type = secrets_scan_file_type self.run_image_referencer = run_image_referencer + self.enable_secret_scan_all_files = enable_secret_scan_all_files + self.black_list_secret_scan = black_list_secret_scan def apply_enforcement_rules(self, enforcement_rule_configs: Dict[str, CodeCategoryConfiguration]) -> None: self.enforcement_rule_configs = {} @@ -241,13 +243,14 @@ def from_dict(obj: Dict[str, Any]) -> RunnerFilter: show_progress_bar = obj.get('show_progress_bar') if show_progress_bar is None: show_progress_bar = True - secrets_scan_file_type = obj.get('secrets_scan_file_type') run_image_referencer = obj.get('run_image_referencer') if run_image_referencer is None: run_image_referencer = False + enable_secret_scan_all_files = bool(obj.get('enable_secret_scan_all_files')) + black_list_secret_scan = obj.get('black_list_secret_scan') runner_filter = RunnerFilter(framework, checks, skip_checks, include_all_checkov_policies, download_external_modules, external_modules_download_path, evaluate_variables, runners, skip_framework, excluded_paths, all_external, var_files, skip_cve_package, use_enforcement_rules, filtered_policy_ids, show_progress_bar, - secrets_scan_file_type, run_image_referencer) + run_image_referencer, enable_secret_scan_all_files, black_list_secret_scan) return runner_filter diff --git a/checkov/secrets/runner.py b/checkov/secrets/runner.py index 11acf781599..fe74141d23f 100644 --- a/checkov/secrets/runner.py +++ b/checkov/secrets/runner.py @@ -62,7 +62,6 @@ ENTROPY_KEYWORD_LIMIT = 3 PROHIBITED_FILES = ['Pipfile.lock', 'yarn.lock', 'package-lock.json', 'requirements.txt'] -ADDED_TO_SECRET_SCAN_FILES_TYPES = SOURCE_CODE_EXTENSION + SUPPORTED_FILE_EXTENSIONS MAX_FILE_SIZE = int(os.getenv('CHECKOV_MAX_FILE_SIZE', '5000000')) # 5 MB is default limit @@ -152,23 +151,19 @@ def run( files_to_scan = files or [] excluded_paths = (runner_filter.excluded_paths or []) + ignored_directories + [DEFAULT_EXTERNAL_MODULES_DIR] if root_folder: - secrets_scan_file_type = runner_filter.secrets_scan_file_type - if secrets_scan_file_type: - secrets_scan_file_type_lower = [file_type.lower() for file_type in secrets_scan_file_type] + enable_secret_scan_all_files = runner_filter.enable_secret_scan_all_files + black_list_secret_scan = runner_filter.black_list_secret_scan or [] + black_list_secret_scan_lower = [file_type.lower() for file_type in black_list_secret_scan] for root, d_names, f_names in os.walk(root_folder): filter_ignored_paths(root, d_names, excluded_paths) filter_ignored_paths(root, f_names, excluded_paths) for file in f_names: - if secrets_scan_file_type: - if 'all' in secrets_scan_file_type: - if is_docker_file(file) or f".{file.split('.')[-1]}" in ADDED_TO_SECRET_SCAN_FILES_TYPES: - files_to_scan.append(os.path.join(root, file)) - else: - if 'dockerfile' in secrets_scan_file_type_lower: - if is_docker_file(file): - files_to_scan.append(os.path.join(root, file)) - if f".{file.split('.')[-1]}" in secrets_scan_file_type: + if enable_secret_scan_all_files: + if is_docker_file(file): + if 'dockerfile' not in black_list_secret_scan_lower: files_to_scan.append(os.path.join(root, file)) + elif f".{file.split('.')[-1]}" not in black_list_secret_scan_lower: + files_to_scan.append(os.path.join(root, file)) elif file not in PROHIBITED_FILES and f".{file.split('.')[-1]}" in SUPPORTED_FILE_EXTENSIONS or is_docker_file(file): files_to_scan.append(os.path.join(root, file)) logging.info(f'Secrets scanning will scan {len(files_to_scan)} files') diff --git a/docs/2.Basics/CLI Command Reference.md b/docs/2.Basics/CLI Command Reference.md index 773e352c2d1..d17c76e32d5 100644 --- a/docs/2.Basics/CLI Command Reference.md +++ b/docs/2.Basics/CLI Command Reference.md @@ -57,4 +57,3 @@ nav_order: 2 | `--output-baseline-as-skipped` | Output checks that are skipped due to baseline file presence | | `--skip-cve-package SKIP_CVE_PACKAGE` | Filter scan to run on all packages but a specific package identifier (deny list), You can specify this argument multiple times to skip multiple packages | | `--policy-metadata-filter POLICY_METADATA_FILTER` | Comma separated key:value string to filter policies based on Prisma Cloud policy metadata. See https://prisma.pan.dev/api/cloud/cspm/policy#operation/get-policy-filters-and-options for information on allowed filters. Format: policy.label=test,cloud.type=aws | -| `--secrets-scan-file-type SECRETS_SCAN_FILE_TYPE` | Add scan secret for requested files. You can specify this argument multiple times to add multiple file types. To scan all types (".tf", ".yml", ".yaml", ".json, ".template", ".py", ".js", ".properties", ".pem, ".php", ".xml", ".ts", ".env", "Dockerfile", ".java, ".rb", ".go", ".cs", ".txt") specify the argument with `--secrets-scan-file-type all`. The default scan will be for ".tf", ".yml", ".yaml", ".json", ".template" and exclude "Pipfile.lock", "yarn.lock", "package-lock.json", "requirements.txt" [env var:CKV_SECRETS_SCAN_FILE_TYPE] | diff --git a/tests/secrets/test_runner.py b/tests/secrets/test_runner.py index 18f63a19c2e..7250f42f9c8 100644 --- a/tests/secrets/test_runner.py +++ b/tests/secrets/test_runner.py @@ -20,7 +20,7 @@ def test_runner_failing_check(self): valid_dir_path = current_dir + "/resources/cfn" runner = Runner() report = runner.run(root_folder=valid_dir_path, external_checks_dir=None, - runner_filter=RunnerFilter(framework='secrets')) + runner_filter=RunnerFilter(framework=['secrets'])) self.assertEqual(len(report.failed_checks), 2) self.assertEqual(report.parsing_errors, []) self.assertEqual(report.passed_checks, []) @@ -225,8 +225,11 @@ def test_runner_requested_file_type_only_ts(self): current_dir = os.path.dirname(os.path.realpath(__file__)) valid_dir_path = current_dir + "/resources" runner = Runner() - report = runner.run(root_folder=valid_dir_path, external_checks_dir=None, - runner_filter=RunnerFilter(framework='secrets', secrets_scan_file_type=['.ts'])) + report = runner.run(root_folder=valid_dir_path, + external_checks_dir=None, + runner_filter=RunnerFilter(framework=['secrets'], + black_list_secret_scan=['.py', 'Dockerfile', '.tf', '.yml'], + enable_secret_scan_all_files=True)) self.assertEqual(len(report.failed_checks), 2) def test_runner_requested_file_type_only_py(self): @@ -234,7 +237,8 @@ def test_runner_requested_file_type_only_py(self): valid_dir_path = current_dir + "/resources" runner = Runner() report = runner.run(root_folder=valid_dir_path, external_checks_dir=None, - runner_filter=RunnerFilter(framework='secrets', secrets_scan_file_type=['.py'])) + runner_filter=RunnerFilter(framework=['secrets'], black_list_secret_scan=['.ts', 'Dockerfile', '.tf', '.yml'], + enable_secret_scan_all_files=True)) self.assertEqual(len(report.failed_checks), 2) def test_runner_requested_file_type_only_yml(self): @@ -242,7 +246,8 @@ def test_runner_requested_file_type_only_yml(self): valid_dir_path = current_dir + "/resources" runner = Runner() report = runner.run(root_folder=valid_dir_path, external_checks_dir=None, - runner_filter=RunnerFilter(framework='secrets', secrets_scan_file_type=['.yml'])) + runner_filter=RunnerFilter(framework=['secrets'], black_list_secret_scan=['.py', 'Dockerfile', '.tf', '.ts'], + enable_secret_scan_all_files=True)) self.assertEqual(len(report.failed_checks), 2) def test_runner_requested_file_type_only_tf(self): @@ -250,16 +255,18 @@ def test_runner_requested_file_type_only_tf(self): valid_dir_path = current_dir + "/resources" runner = Runner() report = runner.run(root_folder=valid_dir_path, external_checks_dir=None, - runner_filter=RunnerFilter(framework='secrets', - secrets_scan_file_type=['.tf'])) + runner_filter=RunnerFilter(framework=['secrets'], + black_list_secret_scan=['.py', 'Dockerfile', '.ts', '.yml'], + enable_secret_scan_all_files=True)) self.assertEqual(len(report.failed_checks), 3) - def test_runner_requested_file_type_only_py_ts_yml(self): + def test_runner_requested_file_type_only_tf_yml(self): current_dir = os.path.dirname(os.path.realpath(__file__)) valid_dir_path = current_dir + "/resources" runner = Runner() report = runner.run(root_folder=valid_dir_path, external_checks_dir=None, - runner_filter=RunnerFilter(framework='secrets', secrets_scan_file_type=['.yml', '.tf'])) + runner_filter=RunnerFilter(framework=['secrets'], black_list_secret_scan=['.py', 'Dockerfile', '.ts'], + enable_secret_scan_all_files=True)) self.assertEqual(len(report.failed_checks), 5) def test_runner_requested_file_type_all(self): @@ -267,7 +274,7 @@ def test_runner_requested_file_type_all(self): valid_dir_path = current_dir + "/resources" runner = Runner() report = runner.run(root_folder=valid_dir_path, external_checks_dir=None, - runner_filter=RunnerFilter(framework='secrets', secrets_scan_file_type=['all'])) + runner_filter=RunnerFilter(framework=['secrets'], enable_secret_scan_all_files=True)) self.assertEqual(len(report.failed_checks), 11) def test_runner_requested_file_only_dockerfile(self): @@ -275,7 +282,8 @@ def test_runner_requested_file_only_dockerfile(self): valid_dir_path = current_dir + "/resources" runner = Runner() report = runner.run(root_folder=valid_dir_path, external_checks_dir=None, - runner_filter=RunnerFilter(framework='secrets', secrets_scan_file_type=['Dockerfile'])) + runner_filter=RunnerFilter(framework=['secrets'], black_list_secret_scan=['.py', '.tf', '.ts', '.yml'], + enable_secret_scan_all_files=True)) self.assertEqual(len(report.failed_checks), 2) def test_runner_no_requested_file(self): @@ -283,7 +291,7 @@ def test_runner_no_requested_file(self): valid_dir_path = current_dir + "/resources" runner = Runner() report = runner.run(root_folder=valid_dir_path, external_checks_dir=None, - runner_filter=RunnerFilter(framework='secrets', secrets_scan_file_type=[])) + runner_filter=RunnerFilter(framework=['secrets'])) self.assertEqual(len(report.failed_checks), 7)