platform(secrets): modify checkov secrets scanner to scan all files b…

…ased on ff (#3474)

* add env PRESENT_CACHED_RESULTS to gat report for save image sca

* Revert "add env PRESENT_CACHED_RESULTS to gat report for save image sca"

This reverts commit 3ec2a61.

* Modify_checkov_secrets_scanner_to_scan_all_files_based_on_FF

* fix ut

* add enable_secret_scan_all_files and blacklist instead of whitelist

* add enable_secret_scan_all_files and blacklist instead of whitelist

* merge master

* remove secrets_scan_file_type

* bool --enable-secret-scan-all-files

checkov/main.py

@@ -158,9 +158,10 @@ def run(banner: str = checkov_banner, argv: List[str] = sys.argv[1:]) -> Optiona
                                  runners=checkov_runners, excluded_paths=excluded_paths,
                                  all_external=config.run_all_external_checks, var_files=config.var_file,
                                  skip_cve_package=config.skip_cve_package, show_progress_bar=not config.quiet,
-                                 secrets_scan_file_type=config.secrets_scan_file_type,
                                  use_enforcement_rules=config.use_enforcement_rules,
-                                 run_image_referencer=run_image_referencer)
+                                 run_image_referencer=run_image_referencer,
+                                 enable_secret_scan_all_files=bool(convert_str_to_bool(config.enable_secret_scan_all_files)),
+                                 black_list_secret_scan=config.black_list_secret_scan)
 
     if outer_registry:
         runner_registry = outer_registry
@@ -562,12 +563,17 @@ def add_parser_args(parser: ArgumentParser) -> None:
                default=[],
                env_var='CKV_SECRETS_SCAN_FILE_TYPE',
                action='append',
-               help='add scan secret for requested files. You can specify this argument multiple times to add '
-                    'multiple file types. To scan all types (".tf", ".hcl", ".yml", ".yaml", ".json", ".template", '
-                    '".bicep", ".py", ".js", ".properties", ".pem", ".php", ".xml", ".ts", ".env", "Dockerfile", '
-                    '".java", ".rb", ".go", ".cs", ".txt") specify the argument with `--secrets-scan-file-type all`. '
-                    'default scan will be for ".tf", ".hcl", ".yml", ".yaml", ".json", ".template", ".bicep" and '
-                    'exclude "Pipfile.lock", "yarn.lock", "package-lock.json", "requirements.txt"')
+               help='not in use')
+    parser.add('--enable-secret-scan-all-files',
+               default=False,
+               env_var='CKV_SECRETS_SCAN_ENABLE_ALL',
+               action='store_true',
+               help='enable secret scan for all files')
+    parser.add('--black-list-secret-scan',
+               default=[],
+               env_var='CKV_SECRETS_SCAN_BLACK_LIST',
+               action='append',
+               help='black file list to filter out from the secret scanner')
 
 
 def get_external_checks_dir(config: Any) -> Any:

checkov/runner_filter.py

@@ -39,8 +39,9 @@ def __init__(
             use_enforcement_rules: bool = False,
             filtered_policy_ids: Optional[List[str]] = None,
             show_progress_bar: Optional[bool] = True,
-            secrets_scan_file_type: Optional[List[str]] = None,
-            run_image_referencer: bool = False
+            run_image_referencer: bool = False,
+            enable_secret_scan_all_files: bool = False,
+            black_list_secret_scan: Optional[List[str]] = None
     ) -> None:
 
         checks = convert_csv_string_arg_to_list(checks)
@@ -95,8 +96,9 @@ def __init__(
         self.var_files = var_files
         self.skip_cve_package = skip_cve_package
         self.filtered_policy_ids = filtered_policy_ids or []
-        self.secrets_scan_file_type = secrets_scan_file_type
         self.run_image_referencer = run_image_referencer
+        self.enable_secret_scan_all_files = enable_secret_scan_all_files
+        self.black_list_secret_scan = black_list_secret_scan
 
     def apply_enforcement_rules(self, enforcement_rule_configs: Dict[str, CodeCategoryConfiguration]) -> None:
         self.enforcement_rule_configs = {}
@@ -241,13 +243,14 @@ def from_dict(obj: Dict[str, Any]) -> RunnerFilter:
         show_progress_bar = obj.get('show_progress_bar')
         if show_progress_bar is None:
             show_progress_bar = True
-        secrets_scan_file_type = obj.get('secrets_scan_file_type')
         run_image_referencer = obj.get('run_image_referencer')
         if run_image_referencer is None:
             run_image_referencer = False
+        enable_secret_scan_all_files = bool(obj.get('enable_secret_scan_all_files'))
+        black_list_secret_scan = obj.get('black_list_secret_scan')
         runner_filter = RunnerFilter(framework, checks, skip_checks, include_all_checkov_policies,
                                      download_external_modules, external_modules_download_path, evaluate_variables,
                                      runners, skip_framework, excluded_paths, all_external, var_files,
                                      skip_cve_package, use_enforcement_rules, filtered_policy_ids, show_progress_bar,
-                                     secrets_scan_file_type, run_image_referencer)
+                                     run_image_referencer, enable_secret_scan_all_files, black_list_secret_scan)
         return runner_filter

checkov/secrets/runner.py

@@ -62,7 +62,6 @@
 
 ENTROPY_KEYWORD_LIMIT = 3
 PROHIBITED_FILES = ['Pipfile.lock', 'yarn.lock', 'package-lock.json', 'requirements.txt']
-ADDED_TO_SECRET_SCAN_FILES_TYPES = SOURCE_CODE_EXTENSION + SUPPORTED_FILE_EXTENSIONS
 
 MAX_FILE_SIZE = int(os.getenv('CHECKOV_MAX_FILE_SIZE', '5000000'))  # 5 MB is default limit
 
@@ -152,23 +151,19 @@ def run(
             files_to_scan = files or []
             excluded_paths = (runner_filter.excluded_paths or []) + ignored_directories + [DEFAULT_EXTERNAL_MODULES_DIR]
             if root_folder:
-                secrets_scan_file_type = runner_filter.secrets_scan_file_type
-                if secrets_scan_file_type:
-                    secrets_scan_file_type_lower = [file_type.lower() for file_type in secrets_scan_file_type]
+                enable_secret_scan_all_files = runner_filter.enable_secret_scan_all_files
+                black_list_secret_scan = runner_filter.black_list_secret_scan or []
+                black_list_secret_scan_lower = [file_type.lower() for file_type in black_list_secret_scan]
                 for root, d_names, f_names in os.walk(root_folder):
                     filter_ignored_paths(root, d_names, excluded_paths)
                     filter_ignored_paths(root, f_names, excluded_paths)
                     for file in f_names:
-                        if secrets_scan_file_type:
-                            if 'all' in secrets_scan_file_type:
-                                if is_docker_file(file) or f".{file.split('.')[-1]}" in ADDED_TO_SECRET_SCAN_FILES_TYPES:
-                                    files_to_scan.append(os.path.join(root, file))
-                            else:
-                                if 'dockerfile' in secrets_scan_file_type_lower:
-                                    if is_docker_file(file):
-                                        files_to_scan.append(os.path.join(root, file))
-                                if f".{file.split('.')[-1]}" in secrets_scan_file_type:
+                        if enable_secret_scan_all_files:
+                            if is_docker_file(file):
+                                if 'dockerfile' not in black_list_secret_scan_lower:
                                     files_to_scan.append(os.path.join(root, file))
+                            elif f".{file.split('.')[-1]}" not in black_list_secret_scan_lower:
+                                files_to_scan.append(os.path.join(root, file))
                         elif file not in PROHIBITED_FILES and f".{file.split('.')[-1]}" in SUPPORTED_FILE_EXTENSIONS or is_docker_file(file):
                             files_to_scan.append(os.path.join(root, file))
             logging.info(f'Secrets scanning will scan {len(files_to_scan)} files')

docs/2.Basics/CLI Command Reference.md

@@ -57,4 +57,3 @@ nav_order: 2
 | `--output-baseline-as-skipped` | Output checks that are skipped due to baseline file presence |
 | `--skip-cve-package SKIP_CVE_PACKAGE` | Filter scan to run on all packages but a specific package identifier (deny list), You can specify this argument multiple times to skip multiple packages |
 | `--policy-metadata-filter POLICY_METADATA_FILTER` | Comma separated key:value string to filter policies based on Prisma Cloud policy metadata. See https://prisma.pan.dev/api/cloud/cspm/policy#operation/get-policy-filters-and-options for information on allowed filters. Format: policy.label=test,cloud.type=aws |
-| `--secrets-scan-file-type SECRETS_SCAN_FILE_TYPE` | Add scan secret for requested files. You can specify this argument multiple times to add multiple file types. To scan all types (".tf", ".yml", ".yaml", ".json, ".template", ".py", ".js", ".properties", ".pem, ".php", ".xml", ".ts", ".env", "Dockerfile", ".java, ".rb", ".go", ".cs", ".txt") specify the argument with `--secrets-scan-file-type all`. The default scan will be for ".tf", ".yml", ".yaml", ".json", ".template" and exclude "Pipfile.lock", "yarn.lock", "package-lock.json", "requirements.txt" [env var:CKV_SECRETS_SCAN_FILE_TYPE] |

tests/secrets/test_runner.py

@@ -20,7 +20,7 @@ def test_runner_failing_check(self):
         valid_dir_path = current_dir + "/resources/cfn"
         runner = Runner()
         report = runner.run(root_folder=valid_dir_path, external_checks_dir=None,
-                            runner_filter=RunnerFilter(framework='secrets'))
+                            runner_filter=RunnerFilter(framework=['secrets']))
         self.assertEqual(len(report.failed_checks), 2)
         self.assertEqual(report.parsing_errors, [])
         self.assertEqual(report.passed_checks, [])
@@ -225,65 +225,73 @@ def test_runner_requested_file_type_only_ts(self):
         current_dir = os.path.dirname(os.path.realpath(__file__))
         valid_dir_path = current_dir + "/resources"
         runner = Runner()
-        report = runner.run(root_folder=valid_dir_path, external_checks_dir=None,
-                            runner_filter=RunnerFilter(framework='secrets', secrets_scan_file_type=['.ts']))
+        report = runner.run(root_folder=valid_dir_path,
+                            external_checks_dir=None,
+                            runner_filter=RunnerFilter(framework=['secrets'],
+                                                       black_list_secret_scan=['.py', 'Dockerfile', '.tf', '.yml'],
+                                                       enable_secret_scan_all_files=True))
         self.assertEqual(len(report.failed_checks), 2)
 
     def test_runner_requested_file_type_only_py(self):
         current_dir = os.path.dirname(os.path.realpath(__file__))
         valid_dir_path = current_dir + "/resources"
         runner = Runner()
         report = runner.run(root_folder=valid_dir_path, external_checks_dir=None,
-                            runner_filter=RunnerFilter(framework='secrets', secrets_scan_file_type=['.py']))
+                            runner_filter=RunnerFilter(framework=['secrets'], black_list_secret_scan=['.ts', 'Dockerfile', '.tf', '.yml'],
+                                                       enable_secret_scan_all_files=True))
         self.assertEqual(len(report.failed_checks), 2)
 
     def test_runner_requested_file_type_only_yml(self):
         current_dir = os.path.dirname(os.path.realpath(__file__))
         valid_dir_path = current_dir + "/resources"
         runner = Runner()
         report = runner.run(root_folder=valid_dir_path, external_checks_dir=None,
-                            runner_filter=RunnerFilter(framework='secrets', secrets_scan_file_type=['.yml']))
+                            runner_filter=RunnerFilter(framework=['secrets'], black_list_secret_scan=['.py', 'Dockerfile', '.tf', '.ts'],
+                                                       enable_secret_scan_all_files=True))
         self.assertEqual(len(report.failed_checks), 2)
 
     def test_runner_requested_file_type_only_tf(self):
         current_dir = os.path.dirname(os.path.realpath(__file__))
         valid_dir_path = current_dir + "/resources"
         runner = Runner()
         report = runner.run(root_folder=valid_dir_path, external_checks_dir=None,
-                            runner_filter=RunnerFilter(framework='secrets',
-                                                       secrets_scan_file_type=['.tf']))
+                            runner_filter=RunnerFilter(framework=['secrets'],
+                                                       black_list_secret_scan=['.py', 'Dockerfile', '.ts', '.yml'],
+                                                       enable_secret_scan_all_files=True))
         self.assertEqual(len(report.failed_checks), 3)
 
-    def test_runner_requested_file_type_only_py_ts_yml(self):
+    def test_runner_requested_file_type_only_tf_yml(self):
         current_dir = os.path.dirname(os.path.realpath(__file__))
         valid_dir_path = current_dir + "/resources"
         runner = Runner()
         report = runner.run(root_folder=valid_dir_path, external_checks_dir=None,
-                            runner_filter=RunnerFilter(framework='secrets', secrets_scan_file_type=['.yml', '.tf']))
+                            runner_filter=RunnerFilter(framework=['secrets'], black_list_secret_scan=['.py', 'Dockerfile', '.ts'],
+                                                       enable_secret_scan_all_files=True))
         self.assertEqual(len(report.failed_checks), 5)
 
     def test_runner_requested_file_type_all(self):
         current_dir = os.path.dirname(os.path.realpath(__file__))
         valid_dir_path = current_dir + "/resources"
         runner = Runner()
         report = runner.run(root_folder=valid_dir_path, external_checks_dir=None,
-                            runner_filter=RunnerFilter(framework='secrets', secrets_scan_file_type=['all']))
+                            runner_filter=RunnerFilter(framework=['secrets'], enable_secret_scan_all_files=True))
         self.assertEqual(len(report.failed_checks), 11)
 
     def test_runner_requested_file_only_dockerfile(self):
         current_dir = os.path.dirname(os.path.realpath(__file__))
         valid_dir_path = current_dir + "/resources"
         runner = Runner()
         report = runner.run(root_folder=valid_dir_path, external_checks_dir=None,
-                            runner_filter=RunnerFilter(framework='secrets', secrets_scan_file_type=['Dockerfile']))
+                            runner_filter=RunnerFilter(framework=['secrets'], black_list_secret_scan=['.py', '.tf', '.ts', '.yml'],
+                                                       enable_secret_scan_all_files=True))
         self.assertEqual(len(report.failed_checks), 2)
 
     def test_runner_no_requested_file(self):
         current_dir = os.path.dirname(os.path.realpath(__file__))
         valid_dir_path = current_dir + "/resources"
         runner = Runner()
         report = runner.run(root_folder=valid_dir_path, external_checks_dir=None,
-                            runner_filter=RunnerFilter(framework='secrets', secrets_scan_file_type=[]))
+                            runner_filter=RunnerFilter(framework=['secrets']))
         self.assertEqual(len(report.failed_checks), 7)