From 2b6a7144bf9eb0c327d3b36a7b5ea06b3056a70c Mon Sep 17 00:00:00 2001 From: Mike Urbanski Date: Tue, 4 Oct 2022 15:35:29 -0500 Subject: [PATCH 01/20] get attributes from the platform that should be applied to specific resources (not all) --- .../attribute_resource_types_integration.py | 79 +++++++++++++++++++ .../features/custom_policies_integration.py | 8 +- .../common/bridgecrew/platform_integration.py | 2 +- checkov/common/checks_infra/checks_parser.py | 26 +++--- .../common/graph/checks_infra/base_check.py | 1 + 5 files changed, 105 insertions(+), 11 deletions(-) create mode 100644 checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py diff --git a/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py b/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py new file mode 100644 index 00000000000..cf77fb96e74 --- /dev/null +++ b/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +import datetime +import logging +from typing import TYPE_CHECKING, Any, Dict, Optional, List + +from checkov.common.bridgecrew.integration_features.base_integration_feature import BaseIntegrationFeature +from checkov.common.bridgecrew.platform_integration import bc_integration + +if TYPE_CHECKING: + from checkov.common.bridgecrew.platform_integration import BcPlatformIntegration + + +ALL_TYPES = '__all__' + + +class AttributeResourceTypesIntegration(BaseIntegrationFeature): + def __init__(self, bc_integration: BcPlatformIntegration) -> None: + super().__init__(bc_integration=bc_integration, order=1) # must be after policy metadata + self.attribute_resources: Dict[str, Dict[str, List[str]]] = {} + + def is_valid(self) -> bool: + return ( + self.bc_integration.is_integration_configured() + and not self.bc_integration.skip_download + and not self.integration_feature_failures + ) + + def pre_scan(self) -> None: + try: + if not self.bc_integration.customer_run_config_response: + logging.debug('In the pre-scan for attribute resource types, but nothing was fetched from the platform') + self.integration_feature_failures = True + return + + logging.debug(f'Start time of processing API output: {datetime.datetime.now().timestamp()}') + + resourceDefinitions = self.bc_integration.customer_run_config_response.get('resourceDefinitions') + filterAttributes: Dict[str, List[str]] = resourceDefinitions.get('filterAttributes') + resourceTypes: Dict[str, Dict[str, Any]] = resourceDefinitions.get('resourceTypes') + + for attribute, providers in filterAttributes.items(): + self.attribute_resources[attribute] = {p: [] for p in providers} + self.attribute_resources[attribute][ALL_TYPES] = [] + + for resource, properties in resourceTypes.items(): + provider = properties['provider'].lower() + if provider == 'ali': + # 'alibabacloud' is the actual provider value in the custom policy, but the resource provider is just 'ali' + provider = 'alibabacloud' + for attribute in properties['arguments']: + if '.' in attribute: + attribute = attribute[:attribute.index('.')] + if attribute not in filterAttributes or provider not in filterAttributes[attribute]: + continue + self.attribute_resources[attribute][provider].append(resource) + self.attribute_resources[attribute][ALL_TYPES].append(resource) + + logging.debug(f'End time of processing API output: {datetime.datetime.now().timestamp()}') + + except Exception: + self.integration_feature_failures = True + logging.debug("Scanning without handling 'all' resource type policies.", exc_info=True) + + def get_attribute_resource_types(self, solver: Dict[str, Any], provider: Optional[str] = None) -> Optional[List[str]]: + attr = solver.get('attribute') + if not attr: + return None + if '.' in attr: + attr = attr[0:attr.index('.')] + + resource_types = self.attribute_resources.get(attr, None) + if not resource_types: + return None + + return resource_types.get(provider or '__all__') + + +integration = AttributeResourceTypesIntegration(bc_integration) diff --git a/checkov/common/bridgecrew/integration_features/features/custom_policies_integration.py b/checkov/common/bridgecrew/integration_features/features/custom_policies_integration.py index f03b3145679..e48372c6fd6 100644 --- a/checkov/common/bridgecrew/integration_features/features/custom_policies_integration.py +++ b/checkov/common/bridgecrew/integration_features/features/custom_policies_integration.py @@ -26,7 +26,6 @@ class CustomPoliciesIntegration(BaseIntegrationFeature): def __init__(self, bc_integration: BcPlatformIntegration) -> None: super().__init__(bc_integration=bc_integration, order=1) # must be after policy metadata self.platform_policy_parser = NXGraphCheckParser() - self.policies_url = f"{self.bc_integration.api_url}/api/v1/policies/table/data" self.bc_cloned_checks: dict[str, list[dict[str, Any]]] = defaultdict(list) def is_valid(self) -> bool: @@ -84,6 +83,13 @@ def _convert_raw_check(policy: dict[str, Any]) -> dict[str, Any]: 'category': policy['category'], 'frameworks': policy.get('frameworks', []) } + + provider = policy.get('provider') + if provider: + metadata['scope'] = { + 'provider': provider.lower() + } + check = { 'metadata': metadata, 'definition': json.loads(policy['code']) diff --git a/checkov/common/bridgecrew/platform_integration.py b/checkov/common/bridgecrew/platform_integration.py index 4082f925c80..e67bc848666 100644 --- a/checkov/common/bridgecrew/platform_integration.py +++ b/checkov/common/bridgecrew/platform_integration.py @@ -524,7 +524,7 @@ def get_platform_run_config(self) -> None: self.get_public_run_config() def get_run_config_url(self) -> str: - return f'{self.platform_run_config_url}?module={"bc" if self.is_bc_token(self.bc_api_key) else "pc"}' + return f'{self.platform_run_config_url}?module={"bc" if self.is_bc_token(self.bc_api_key) else "pc"}&includeResources=true' def get_customer_run_config(self) -> None: if self.skip_download is True: diff --git a/checkov/common/checks_infra/checks_parser.py b/checkov/common/checks_infra/checks_parser.py index 0dca7ddaf87..45548363b28 100644 --- a/checkov/common/checks_infra/checks_parser.py +++ b/checkov/common/checks_infra/checks_parser.py @@ -44,6 +44,7 @@ IntersectsAttributeSolver, NotIntersectsAttributeSolver ) +from checkov.common.bridgecrew.integration_features.features.attribute_resource_types_integration import integration as attribute_resource_type_integration from checkov.common.checks_infra.solvers.connections_solvers.connection_one_exists_solver import \ ConnectionOneExistsSolver from checkov.common.graph.checks_infra.base_check import BaseGraphCheck @@ -129,18 +130,24 @@ class NXGraphCheckParser(BaseGraphCheckParser): def parse_raw_check(self, raw_check: Dict[str, Dict[str, Any]], **kwargs: Any) -> BaseGraphCheck: policy_definition = raw_check.get("definition", {}) - check = self._parse_raw_check(policy_definition, kwargs.get("resources_types")) - check.id = raw_check.get("metadata", {}).get("id", "") - check.name = raw_check.get("metadata", {}).get("name", "") - check.category = raw_check.get("metadata", {}).get("category", "") - check.frameworks = raw_check.get("metadata", {}).get("frameworks", []) - check.guideline = raw_check.get("metadata", {}).get("guideline") + + metadata = raw_check.get("metadata", {}) + provider = metadata.get("scope", {}).get("provider") + + check = self._parse_raw_check(policy_definition, kwargs.get("resources_types"), provider) + + check.id = metadata.get("id", "") + check.name = metadata.get("name", "") + check.category = metadata.get("category", "") + check.frameworks = metadata.get("frameworks", []) + check.provider = provider + solver = self.get_check_solver(check) check.set_solver(solver) return check - def _parse_raw_check(self, raw_check: Dict[str, Any], resources_types: Optional[List[str]]) -> BaseGraphCheck: + def _parse_raw_check(self, raw_check: Dict[str, Any], resources_types: Optional[List[str]], provider: Optional[str]) -> BaseGraphCheck: check = BaseGraphCheck() complex_operator = get_complex_operator(raw_check) if complex_operator: @@ -154,7 +161,7 @@ def _parse_raw_check(self, raw_check: Dict[str, Any], resources_types: Optional[ sub_solvers = [sub_solvers] for sub_solver in sub_solvers: - check.sub_checks.append(self._parse_raw_check(sub_solver, resources_types)) + check.sub_checks.append(self._parse_raw_check(sub_solver, resources_types, provider)) resources_types_of_sub_solvers = [ force_list(q.resource_types) for q in check.sub_checks if q is not None and q.resource_types is not None ] @@ -169,7 +176,8 @@ def _parse_raw_check(self, raw_check: Dict[str, Any], resources_types: Optional[ or (isinstance(resource_type, str) and resource_type.lower() == "all") or (isinstance(resource_type, list) and resource_type[0].lower() == "all") ): - check.resource_types = resources_types or [] + resource_types_for_attribute = attribute_resource_type_integration.get_attribute_resource_types(raw_check, provider) + check.resource_types = resource_types_for_attribute or resources_types or [] else: check.resource_types = resource_type diff --git a/checkov/common/graph/checks_infra/base_check.py b/checkov/common/graph/checks_infra/base_check.py index 9e1cf52dd4c..c4d50b9c5f1 100644 --- a/checkov/common/graph/checks_infra/base_check.py +++ b/checkov/common/graph/checks_infra/base_check.py @@ -32,6 +32,7 @@ def __init__(self) -> None: self.bc_category: Optional[str] = None self.frameworks: list[str] = [] self.is_jsonpath_check: bool = False + self.provider: Optional[str] = None def set_solver(self, solver: BaseSolver) -> None: self.solver = solver From 2ff1a0ed4dcd0c112022f8ce25f69e741248b8e5 Mon Sep 17 00:00:00 2001 From: Mike Urbanski Date: Wed, 5 Oct 2022 13:00:53 -0500 Subject: [PATCH 02/20] use a set for the taggable resource types --- .../attribute_resource_types_integration.py | 58 +++++++++++-------- .../features/custom_policies_integration.py | 2 +- checkov/common/checks_infra/checks_parser.py | 15 ++++- checkov/common/util/type_forcers.py | 6 ++ 4 files changed, 52 insertions(+), 29 deletions(-) diff --git a/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py b/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py index cf77fb96e74..86c80117f08 100644 --- a/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py +++ b/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py @@ -2,7 +2,7 @@ import datetime import logging -from typing import TYPE_CHECKING, Any, Dict, Optional, List +from typing import TYPE_CHECKING, Any, Dict, Optional, List, Set from checkov.common.bridgecrew.integration_features.base_integration_feature import BaseIntegrationFeature from checkov.common.bridgecrew.platform_integration import bc_integration @@ -17,7 +17,7 @@ class AttributeResourceTypesIntegration(BaseIntegrationFeature): def __init__(self, bc_integration: BcPlatformIntegration) -> None: super().__init__(bc_integration=bc_integration, order=1) # must be after policy metadata - self.attribute_resources: Dict[str, Dict[str, List[str]]] = {} + self.attribute_resources: Dict[str, Dict[str, Set[str]]] = {} def is_valid(self) -> bool: return ( @@ -33,30 +33,13 @@ def pre_scan(self) -> None: self.integration_feature_failures = True return - logging.debug(f'Start time of processing API output: {datetime.datetime.now().timestamp()}') - - resourceDefinitions = self.bc_integration.customer_run_config_response.get('resourceDefinitions') - filterAttributes: Dict[str, List[str]] = resourceDefinitions.get('filterAttributes') - resourceTypes: Dict[str, Dict[str, Any]] = resourceDefinitions.get('resourceTypes') - - for attribute, providers in filterAttributes.items(): - self.attribute_resources[attribute] = {p: [] for p in providers} - self.attribute_resources[attribute][ALL_TYPES] = [] - - for resource, properties in resourceTypes.items(): - provider = properties['provider'].lower() - if provider == 'ali': - # 'alibabacloud' is the actual provider value in the custom policy, but the resource provider is just 'ali' - provider = 'alibabacloud' - for attribute in properties['arguments']: - if '.' in attribute: - attribute = attribute[:attribute.index('.')] - if attribute not in filterAttributes or provider not in filterAttributes[attribute]: - continue - self.attribute_resources[attribute][provider].append(resource) - self.attribute_resources[attribute][ALL_TYPES].append(resource) + if 'resourceDefinitions' not in self.bc_integration.customer_run_config_response: + # TODO remove - this makes it easier to make sure that platform scans will also work + logging.debug('resourceDefinitions is not in the run config response - might not be deployed to the platform yet') + return - logging.debug(f'End time of processing API output: {datetime.datetime.now().timestamp()}') + resource_definitions = self.bc_integration.customer_run_config_response.get('resourceDefinitions') + self._build_attribute_resource_map(resource_definitions) except Exception: self.integration_feature_failures = True @@ -75,5 +58,30 @@ def get_attribute_resource_types(self, solver: Dict[str, Any], provider: Optiona return resource_types.get(provider or '__all__') + def _build_attribute_resource_map(self, resource_definitions): + logging.debug(f'Start time of processing API output: {datetime.datetime.now().timestamp()}') + + filter_attributes: Dict[str, List[str]] = resource_definitions.get('filterAttributes') + resource_types: Dict[str, Dict[str, Any]] = resource_definitions.get('resourceTypes') + + for attribute, providers in filter_attributes.items(): + self.attribute_resources[attribute] = {p: set() for p in providers} + self.attribute_resources[attribute][ALL_TYPES] = set() + + for resource, properties in resource_types.items(): + provider = properties['provider'].lower() + if provider == 'ali': + # 'alibabacloud' is the actual provider value in the custom policy, but the resource provider is just 'ali' + provider = 'alibabacloud' + for attribute in properties['arguments']: + if '.' in attribute: + attribute = attribute[:attribute.index('.')] + if attribute not in filter_attributes or provider not in filter_attributes[attribute]: + continue + self.attribute_resources[attribute][provider].add(resource) + self.attribute_resources[attribute][ALL_TYPES].add(resource) + + logging.debug(f'End time of processing API output: {datetime.datetime.now().timestamp()}') + integration = AttributeResourceTypesIntegration(bc_integration) diff --git a/checkov/common/bridgecrew/integration_features/features/custom_policies_integration.py b/checkov/common/bridgecrew/integration_features/features/custom_policies_integration.py index e48372c6fd6..a2f69fc54cf 100644 --- a/checkov/common/bridgecrew/integration_features/features/custom_policies_integration.py +++ b/checkov/common/bridgecrew/integration_features/features/custom_policies_integration.py @@ -24,7 +24,7 @@ class CustomPoliciesIntegration(BaseIntegrationFeature): def __init__(self, bc_integration: BcPlatformIntegration) -> None: - super().__init__(bc_integration=bc_integration, order=1) # must be after policy metadata + super().__init__(bc_integration=bc_integration, order=2) # must be after resource types self.platform_policy_parser = NXGraphCheckParser() self.bc_cloned_checks: dict[str, list[dict[str, Any]]] = defaultdict(list) diff --git a/checkov/common/checks_infra/checks_parser.py b/checkov/common/checks_infra/checks_parser.py index 45548363b28..82ec6039991 100644 --- a/checkov/common/checks_infra/checks_parser.py +++ b/checkov/common/checks_infra/checks_parser.py @@ -51,7 +51,7 @@ from checkov.common.graph.checks_infra.base_parser import BaseGraphCheckParser from checkov.common.graph.checks_infra.enums import SolverType from checkov.common.graph.checks_infra.solvers.base_solver import BaseSolver -from checkov.common.util.type_forcers import force_list +from checkov.common.util.type_forcers import force_list, force_list_or_set if TYPE_CHECKING: from checkov.common.checks_infra.solvers.attribute_solvers.base_attribute_solver import BaseAttributeSolver @@ -160,12 +160,21 @@ def _parse_raw_check(self, raw_check: Dict[str, Any], resources_types: Optional[ if isinstance(sub_solvers, dict): sub_solvers = [sub_solvers] + # we handle resource types as a list (in the case where the types are enumerated) or as a set (when we + # replace a value of 'all' from the platform). sets are convenient because these lists are large, so + # faster lookups are nice, and because then we don't need to worry about duplicate (e.g., CFN Tags.Key and Tags.Value) + for sub_solver in sub_solvers: check.sub_checks.append(self._parse_raw_check(sub_solver, resources_types, provider)) resources_types_of_sub_solvers = [ - force_list(q.resource_types) for q in check.sub_checks if q is not None and q.resource_types is not None + force_list_or_set(q.resource_types) for q in check.sub_checks if q is not None and q.resource_types is not None ] - check.resource_types = list(set(sum(resources_types_of_sub_solvers, []))) + if resources_types_of_sub_solvers and isinstance(resources_types_of_sub_solvers[0], list): + check.resource_types = list(set(sum(resources_types_of_sub_solvers, []))) + elif resources_types_of_sub_solvers and isinstance(resources_types_of_sub_solvers[0], set): + check.resource_types = list(set().union(*resources_types_of_sub_solvers)) + else: + check.resource_types = [] if any(q.type in [SolverType.CONNECTION, SolverType.COMPLEX_CONNECTION] for q in check.sub_checks): check.type = SolverType.COMPLEX_CONNECTION diff --git a/checkov/common/util/type_forcers.py b/checkov/common/util/type_forcers.py index ca44e2cbdf3..d57a7572217 100644 --- a/checkov/common/util/type_forcers.py +++ b/checkov/common/util/type_forcers.py @@ -26,6 +26,12 @@ def force_list(var: T | list[T]) -> list[T]: return var +def force_list_or_set(var: T | list[T] | set[T]) -> list[T] | set[T]: + if isinstance(var, list) or isinstance(var, set): + return var + return [var] + + def force_int(var: Any) -> int | None: try: if not isinstance(var, int): From c6272b17b64b57bbb05027b173d91656820302e0 Mon Sep 17 00:00:00 2001 From: Mike Urbanski Date: Wed, 5 Oct 2022 13:01:04 -0500 Subject: [PATCH 03/20] add test of full policy scan --- .../integration_features/resources/main.tf | 8 + ...est_attribute_resource_type_integration.py | 316 ++++++++++++++++++ 2 files changed, 324 insertions(+) create mode 100644 tests/common/integration_features/test_attribute_resource_type_integration.py diff --git a/tests/common/integration_features/resources/main.tf b/tests/common/integration_features/resources/main.tf index 5a9126a80df..5f8a58e52b2 100644 --- a/tests/common/integration_features/resources/main.tf +++ b/tests/common/integration_features/resources/main.tf @@ -1,3 +1,11 @@ resource "aws_subnet" "s" { map_public_ip_on_launch = true } + +resource "aws_s3_bucket" "b" { + +} + +resource "aws_security_group_rule" "r" { + +} diff --git a/tests/common/integration_features/test_attribute_resource_type_integration.py b/tests/common/integration_features/test_attribute_resource_type_integration.py new file mode 100644 index 00000000000..4e2be883330 --- /dev/null +++ b/tests/common/integration_features/test_attribute_resource_type_integration.py @@ -0,0 +1,316 @@ +import json +import os +import unittest + +from checkov.common.bridgecrew.integration_features.features.attribute_resource_types_integration import \ + AttributeResourceTypesIntegration, integration as attribute_resource_types_integration +from checkov.common.bridgecrew.integration_features.features.custom_policies_integration import \ + CustomPoliciesIntegration +from checkov.common.bridgecrew.platform_integration import BcPlatformIntegration, bc_integration +from checkov.common.checks_infra.checks_parser import NXGraphCheckParser +from checkov.common.checks_infra.registry import Registry, get_graph_checks_registry +from checkov.common.models.enums import CheckResult +from checkov.common.output.record import Record +from checkov.common.output.report import Report +from checkov.terraform.runner import Runner as TerraformRunner +from checkov.cloudformation.runner import Runner as CFNRunner +from checkov.runner_filter import RunnerFilter +from pathlib import Path + + +class TestAttributeResourceTypeIntegration(unittest.TestCase): + def tearDown(self) -> None: + get_graph_checks_registry("cloudformation").checks = [] + get_graph_checks_registry("terraform").checks = [] + + def test_integration_valid(self): + instance = BcPlatformIntegration() + instance.skip_download = False + instance.platform_integration_configured = True + + attr_res_integration = AttributeResourceTypesIntegration(instance) + + self.assertTrue(attr_res_integration.is_valid()) + + instance.skip_download = True + self.assertFalse(attr_res_integration.is_valid()) + + instance.platform_integration_configured = False + self.assertFalse(attr_res_integration.is_valid()) + + instance.skip_download = False + self.assertFalse(attr_res_integration.is_valid()) + + attr_res_integration.integration_feature_failures = True + self.assertFalse(attr_res_integration.is_valid()) + + def test_get_attribute_resource_types(self): + attr_res_integration = AttributeResourceTypesIntegration(None) + attr_res_integration.attribute_resources = mock_attribute_resource_definitions() + # resource + self.assertIsNone(attr_res_integration.get_attribute_resource_types({})) + self.assertIsNone(attr_res_integration.get_attribute_resource_types({'attribute': 'abc'})) + self.assertIsNotNone(attr_res_integration.get_attribute_resource_types({'attribute': 'labels'})) + self.assertIsNotNone(attr_res_integration.get_attribute_resource_types({'attribute': 'tags'})) + self.assertIsNotNone(attr_res_integration.get_attribute_resource_types({'attribute': 'Tags'})) + self.assertIsNotNone(attr_res_integration.get_attribute_resource_types({'attribute': 'tags.owner'})) + self.assertIsNotNone(attr_res_integration.get_attribute_resource_types({'attribute': 'labels.owner'})) + self.assertIsNotNone(attr_res_integration.get_attribute_resource_types({'attribute': 'Tags.Key'})) + self.assertIsNotNone(attr_res_integration.get_attribute_resource_types({'attribute': 'labels'}, provider='gcp')) + self.assertIsNotNone(attr_res_integration.get_attribute_resource_types({'attribute': 'tags'}, provider='aws')) + self.assertIsNotNone( + attr_res_integration.get_attribute_resource_types({'attribute': 'tags'}, provider='alibabacloud')) + self.assertIsNotNone(attr_res_integration.get_attribute_resource_types({'attribute': 'Tags'}, provider='aws')) + self.assertIsNone(attr_res_integration.get_attribute_resource_types({'attribute': 'labels'}, provider='aws')) + self.assertIsNone(attr_res_integration.get_attribute_resource_types({'attribute': 'tags'}, provider='gcp')) + self.assertIsNone(attr_res_integration.get_attribute_resource_types({'attribute': 'Tags'}, provider='gcp')) + self.assertTrue( + any(r.startswith('aws') for r in attr_res_integration.get_attribute_resource_types({'attribute': 'tags'}))) + self.assertTrue(any(r.startswith('aws') for r in + attr_res_integration.get_attribute_resource_types({'attribute': 'tags'}, provider='aws'))) + + # alicloud is the resource type prefix, but alibabacloud is the name of the custom policy provider, so this is intentional + self.assertFalse(any(r.startswith('alicloud') for r in + attr_res_integration.get_attribute_resource_types({'attribute': 'tags'}, provider='aws'))) + self.assertTrue(any( + r.startswith('alicloud') for r in attr_res_integration.get_attribute_resource_types({'attribute': 'tags'}))) + self.assertTrue(any(r.startswith('alicloud') for r in + attr_res_integration.get_attribute_resource_types({'attribute': 'tags'}, + provider='alibabacloud'))) + self.assertFalse(any(r.startswith('aws') for r in + attr_res_integration.get_attribute_resource_types({'attribute': 'tags'}, + provider='alibabacloud'))) + + def test_build_resource_definitions(self): + attr_res_integration = AttributeResourceTypesIntegration(None) + attr_res_integration._build_attribute_resource_map(mock_resource_definition_response()) + expected = { + 'tags': { + 'aws': {'aws_s3_bucket'}, + 'azure': {'azurerm_storage_account'}, + 'alibabacloud': {'alicloud_adb_db_cluster'}, + '__all__': {'alicloud_adb_db_cluster', 'aws_s3_bucket', 'azurerm_storage_account'} + }, + 'Tags': { + 'aws': {'AWS::S3::Bucket'}, + '__all__': {'AWS::S3::Bucket'} + }, + 'labels': { + 'gcp': {'google_bigquery_dataset'}, + '__all__': {'google_bigquery_dataset'} + } + } + self.assertEqual(attr_res_integration.attribute_resources, expected) + + def test_scan_with_attribute(self): + temp_integration = BcPlatformIntegration() + attribute_resource_types_integration.bc_integration = temp_integration + temp_integration.platform_integration_configured = True + temp_integration.customer_run_config_response = mock_customer_run_config() + + attribute_resource_types_integration.pre_scan() + + # for this test, we simulate some of the check registry manipulation; otherwise the singleton + # instance will be modified and break other tests. + + parser = NXGraphCheckParser() + + registry = Registry(parser=NXGraphCheckParser(), checks_dir=str( + Path(__file__).parent.parent.parent.parent / "checkov" / "terraform" / "checks" / "graph_checks")) + checks = [parser.parse_raw_check(CustomPoliciesIntegration._convert_raw_check(p)) for p in temp_integration.customer_run_config_response['customPolicies']] + registry.checks = checks # simulate that the policy downloader will do + + tf_runner = TerraformRunner(external_registries=[registry]) + current_dir = os.path.dirname(os.path.realpath(__file__)) + + test_files_dir = current_dir + "/resources" + + report = tf_runner.run(root_folder=test_files_dir, runner_filter=RunnerFilter(checks=['policy_id_1'])) + + # we only expect the aws_s3_bucket to fail, because the mock response does not include aws_subnet as a taggable resource + self.assertEqual(len(report.failed_checks), 1) + self.assertEqual(len(report.passed_checks), 0) + self.assertEqual(report.failed_checks[0].check_id, 'policy_id_1') + self.assertEqual(report.failed_checks[0].resource, 'aws_s3_bucket.b') + + attribute_resource_types_integration.bc_integration = bc_integration + + +def mock_attribute_resource_definitions(): + return { + 'tags': { + 'aws': ['aws_s3_bucket'], + 'alibabacloud': ['alicloud_adb_db_cluster'], + '__all__': ['aws_s3_bucket', 'alicloud_adb_db_cluster'] + }, + 'Tags': { + 'aws': ['AWS::S3::Bucket'], + '__all__': ['AWS::S3::Bucket'] + }, + 'labels': { + 'gcp': ['google_bigquery_dataset'], + '__all__': ['google_bigquery_dataset'] + } + } + + +def mock_resource_definition_response(): + return { + "resourceTypes": { + "aws": { + "provider": "AWS", + "text": "AWS Terraform Provider", + "arguments": [] + }, + "aws_s3_bucket": { + "provider": "AWS", + "text": "S3 Bucket", + "arguments": [ + "acceleration_status", + "acl", + "tags", + "tags_all" + ] + }, + "aws_security_group_rule": { + "provider": "AWS", + "text": "Security Group Rule", + "arguments": [ + "cidr_blocks", + "description", + "from_port", + "to_port" + ] + }, + "azurerm_storage_account": { + "provider": "Azure", + "text": "Storage Account", + "arguments": [ + "access_tier", + "account_kind", + "account_replication_type", + "account_tier", + "allow_blob_public_access", + "tags" + ] + }, + "azurerm_lb_rule": { + "provider": "Azure", + "text": "ALB Rule", + "arguments": [ + "backend_address_pool_ids", + "backend_port", + "disable_outbound_snat" + ] + }, + "alicloud_adb_db_cluster": { + "provider": "ALI", + "text": "Adb DB Cluster", + "arguments": [ + "auto_renew_period", + "compute_resource", + "connection_string", + "tags" + ] + }, + "alicloud_alb_rule": { + "provider": "ALI", + "text": "ALB Rule", + "arguments": [ + "dry_run", + "id", + "listener_id", + "priority" + ] + }, + "google_bigquery_dataset": { + "provider": "GCP", + "text": "Bigquery Dataset", + "arguments": [ + "access.dataset.dataset.dataset_id", + "access.dataset.dataset.project_id", + "access.dataset.target_types", + "access.domain", + "labels" + ], + "prismaResourceTypeId": 20002 + }, + "google_app_engine_firewall_rule": { + "provider": "GCP", + "text": "App Engine Firewall Rule", + "arguments": [ + "action", + "description", + "id", + "priority", + "project" + ], + "prismaResourceTypeId": 20020 + }, + "AWS::S3::Bucket": { + "provider": "AWS", + "text": "S3 Bucket", + "arguments": [ + "AccelerateConfiguration.AccelerationStatus", + "AccessControl", + "Tags.Key", + "Tags.Value", + "VersioningConfiguration.Status" + ] + }, + "AWS::Macie::Session": { + "provider": "AWS", + "text": "Macie Session", + "arguments": [ + "FindingPublishingFrequency", + "Status" + ] + } + }, + "filterAttributes": { + "tags": [ + "aws", + "azure", + 'alibabacloud' + ], + "labels": [ + "gcp" + ], + "Tags": [ + "aws" + ] + } + } + + +def mock_customer_run_config(): + return { + "customPolicies": [ + { + "id": "policy_id_1", + "title": "yaml1", + "severity": "MEDIUM", + "category": "General", + "guideline": "yaml1", + "code": json.dumps({ + "or": [ + { + "value": "xyz", + "operator": "contains", + "attribute": "tags", + "cond_type": "attribute", + "resource_types": [ + "all" + ] + } + ] + }), + "benchmarks": {}, + } + ], + "resourceDefinitions": mock_resource_definition_response() + } + + +if __name__ == '__main__': + unittest.main() From 802f0df18151bc5de94e8d60cc4f484466a1a431 Mon Sep 17 00:00:00 2001 From: Mike Urbanski Date: Thu, 6 Oct 2022 14:04:25 -0500 Subject: [PATCH 04/20] fix tests --- checkov/common/checks_infra/checks_parser.py | 1 + .../test_attribute_resource_type_integration.py | 8 ++++---- tests/common/test_platform_integration.py | 4 ++-- tests/terraform/runner/test_plan_runner.py | 1 + tests/terraform/runner/test_runner.py | 1 + 5 files changed, 9 insertions(+), 6 deletions(-) diff --git a/checkov/common/checks_infra/checks_parser.py b/checkov/common/checks_infra/checks_parser.py index 82ec6039991..3df93ffe3f4 100644 --- a/checkov/common/checks_infra/checks_parser.py +++ b/checkov/common/checks_infra/checks_parser.py @@ -140,6 +140,7 @@ def parse_raw_check(self, raw_check: Dict[str, Dict[str, Any]], **kwargs: Any) - check.name = metadata.get("name", "") check.category = metadata.get("category", "") check.frameworks = metadata.get("frameworks", []) + check.guideline = metadata.get("guideline") check.provider = provider solver = self.get_check_solver(check) diff --git a/tests/common/integration_features/test_attribute_resource_type_integration.py b/tests/common/integration_features/test_attribute_resource_type_integration.py index 4e2be883330..d9e81fdf60a 100644 --- a/tests/common/integration_features/test_attribute_resource_type_integration.py +++ b/tests/common/integration_features/test_attribute_resource_type_integration.py @@ -20,7 +20,6 @@ class TestAttributeResourceTypeIntegration(unittest.TestCase): def tearDown(self) -> None: - get_graph_checks_registry("cloudformation").checks = [] get_graph_checks_registry("terraform").checks = [] def test_integration_valid(self): @@ -45,7 +44,7 @@ def test_integration_valid(self): self.assertFalse(attr_res_integration.is_valid()) def test_get_attribute_resource_types(self): - attr_res_integration = AttributeResourceTypesIntegration(None) + attr_res_integration = AttributeResourceTypesIntegration(bc_integration) attr_res_integration.attribute_resources = mock_attribute_resource_definitions() # resource self.assertIsNone(attr_res_integration.get_attribute_resource_types({})) @@ -82,7 +81,7 @@ def test_get_attribute_resource_types(self): provider='alibabacloud'))) def test_build_resource_definitions(self): - attr_res_integration = AttributeResourceTypesIntegration(None) + attr_res_integration = AttributeResourceTypesIntegration(bc_integration) attr_res_integration._build_attribute_resource_map(mock_resource_definition_response()) expected = { 'tags': { @@ -126,7 +125,7 @@ def test_scan_with_attribute(self): test_files_dir = current_dir + "/resources" report = tf_runner.run(root_folder=test_files_dir, runner_filter=RunnerFilter(checks=['policy_id_1'])) - + # we only expect the aws_s3_bucket to fail, because the mock response does not include aws_subnet as a taggable resource self.assertEqual(len(report.failed_checks), 1) self.assertEqual(len(report.passed_checks), 0) @@ -134,6 +133,7 @@ def test_scan_with_attribute(self): self.assertEqual(report.failed_checks[0].resource, 'aws_s3_bucket.b') attribute_resource_types_integration.bc_integration = bc_integration + attribute_resource_types_integration.attribute_resources = {} def mock_attribute_resource_definitions(): diff --git a/tests/common/test_platform_integration.py b/tests/common/test_platform_integration.py index 85658fed147..e1bfd1ad285 100644 --- a/tests/common/test_platform_integration.py +++ b/tests/common/test_platform_integration.py @@ -137,9 +137,9 @@ def test_should_upload(self): def test_run_config_url(self): instance = BcPlatformIntegration() instance.bc_api_key = '00000000-0000-0000-0000-000000000000' - self.assertTrue(instance.get_run_config_url().endswith('/runConfiguration?module=bc')) + self.assertTrue(instance.get_run_config_url().endswith('/runConfiguration?module=bc&includeResources=true')) instance.bc_api_key = '00000000-0000-0000-0000-000000000000::1234==' - self.assertTrue(instance.get_run_config_url().endswith('/runConfiguration?module=pc')) + self.assertTrue(instance.get_run_config_url().endswith('/runConfiguration?module=pc&includeResources=true')) def test_is_valid_policy_filter(self): instance = BcPlatformIntegration() diff --git a/tests/terraform/runner/test_plan_runner.py b/tests/terraform/runner/test_plan_runner.py index 9ac10634018..fdee73175a5 100644 --- a/tests/terraform/runner/test_plan_runner.py +++ b/tests/terraform/runner/test_plan_runner.py @@ -6,6 +6,7 @@ from typing import Dict, Any # do not remove - prevents circular import +import checkov.common.bridgecrew.integration_features.features from checkov.common.bridgecrew.check_type import CheckType from checkov.common.bridgecrew.severities import BcSeverities, Severities from checkov.common.models.enums import CheckCategories, CheckResult diff --git a/tests/terraform/runner/test_runner.py b/tests/terraform/runner/test_runner.py index 06e6a0984c9..2bbb8a8299d 100644 --- a/tests/terraform/runner/test_runner.py +++ b/tests/terraform/runner/test_runner.py @@ -9,6 +9,7 @@ # do not remove; prevents circular import error from typing import Dict, Any from unittest import mock +import checkov.common.bridgecrew.integration_features.features from checkov.common.bridgecrew.check_type import CheckType from checkov.common.bridgecrew.severities import Severities, BcSeverities From 4d08e1b5a9e25cf1b05f38c9193d74c94e677599 Mon Sep 17 00:00:00 2001 From: Mike Urbanski Date: Thu, 6 Oct 2022 15:26:25 -0500 Subject: [PATCH 05/20] remove local resource types --- .../features/custom_policies_integration.py | 3 +- checkov/common/checks_infra/checks_parser.py | 2 +- checkov/common/checks_infra/registry.py | 10 +- .../common/checks_infra/resources_types.py | 1291 ----------------- 4 files changed, 3 insertions(+), 1303 deletions(-) delete mode 100644 checkov/common/checks_infra/resources_types.py diff --git a/checkov/common/bridgecrew/integration_features/features/custom_policies_integration.py b/checkov/common/bridgecrew/integration_features/features/custom_policies_integration.py index a2f69fc54cf..3d2c16d24df 100644 --- a/checkov/common/bridgecrew/integration_features/features/custom_policies_integration.py +++ b/checkov/common/bridgecrew/integration_features/features/custom_policies_integration.py @@ -52,8 +52,7 @@ def pre_scan(self) -> None: policy['severity'] = Severities[policy['severity']] self.bc_cloned_checks[source_incident_id].append(policy) continue - resource_types = Registry._get_resource_types(converted_check['metadata']) - check = self.platform_policy_parser.parse_raw_check(converted_check, resources_types=resource_types) + check = self.platform_policy_parser.parse_raw_check(converted_check) check.severity = Severities[policy['severity']] check.bc_id = check.id if check.frameworks: diff --git a/checkov/common/checks_infra/checks_parser.py b/checkov/common/checks_infra/checks_parser.py index 3df93ffe3f4..8a51e83ef59 100644 --- a/checkov/common/checks_infra/checks_parser.py +++ b/checkov/common/checks_infra/checks_parser.py @@ -134,7 +134,7 @@ def parse_raw_check(self, raw_check: Dict[str, Dict[str, Any]], **kwargs: Any) - metadata = raw_check.get("metadata", {}) provider = metadata.get("scope", {}).get("provider") - check = self._parse_raw_check(policy_definition, kwargs.get("resources_types"), provider) + check = self._parse_raw_check(policy_definition, [], provider) check.id = metadata.get("id", "") check.name = metadata.get("name", "") diff --git a/checkov/common/checks_infra/registry.py b/checkov/common/checks_infra/registry.py index d1b5752e2b8..f83112eccba 100644 --- a/checkov/common/checks_infra/registry.py +++ b/checkov/common/checks_infra/registry.py @@ -12,7 +12,6 @@ from checkov.common.graph.checks_infra.base_parser import BaseGraphCheckParser from checkov.common.graph.checks_infra.registry import BaseRegistry from checkov.runner_filter import RunnerFilter -from checkov.common.checks_infra.resources_types import resources_types if TYPE_CHECKING: from checkov.common.graph.checks_infra.base_check import BaseGraphCheck @@ -48,9 +47,7 @@ def _load_checks_from_dir(self, directory: str, external_check: bool) -> None: if not isinstance(check_json, dict): self.logger.error(f"Loaded data from JSON is not Dict. Skipping. Data: {check_json}.") continue - check = self.parser.parse_raw_check( - check_json, resources_types=self._get_resource_types(check_json) - ) + check = self.parser.parse_raw_check(check_json) if not any(c for c in self.checks if check.id == c.id): if external_check: # Note the external check; used in the should_run_check logic @@ -60,11 +57,6 @@ def _load_checks_from_dir(self, directory: str, external_check: bool) -> None: def load_external_checks(self, dir: str) -> None: self._load_checks_from_dir(dir, True) - @staticmethod - def _get_resource_types(check_json: dict[str, dict[str, Any]]) -> list[str] | None: - provider = check_json.get("scope", {}).get("provider", "").lower() - return resources_types.get(provider) - _registry_instances: dict[str, Registry] = {} diff --git a/checkov/common/checks_infra/resources_types.py b/checkov/common/checks_infra/resources_types.py deleted file mode 100644 index d0fc4d94034..00000000000 --- a/checkov/common/checks_infra/resources_types.py +++ /dev/null @@ -1,1291 +0,0 @@ -resources_types = { - "aws": [ - "aws", - "aws_root", - "aws_root_access_key", - "aws_acm_certificate", - "aws_acm_certificate_validation", - "aws_acmpca_certificate_authority", - "aws_api_gateway_account", - "aws_api_gateway_api_key", - "aws_api_gateway_authorizer", - "aws_api_gateway_base_path_mapping", - "aws_api_gateway_client_certificate", - "aws_api_gateway_deployment", - "aws_api_gateway_documentation_part", - "aws_api_gateway_documentation_version", - "aws_api_gateway_domain_name", - "aws_api_gateway_gateway_response", - "aws_api_gateway_integration", - "aws_api_gateway_integration_response", - "aws_api_gateway_method", - "aws_api_gateway_method_response", - "aws_api_gateway_method_settings", - "aws_api_gateway_model", - "aws_api_gateway_request_validator", - "aws_api_gateway_resource", - "aws_api_gateway_rest_api", - "aws_api_gateway_stage", - "aws_api_gateway_usage_plan", - "aws_api_gateway_usage_plan_key", - "aws_api_gateway_vpc_link", - "aws_apigatewayv2_api", - "aws_apigatewayv2_api_mapping", - "aws_apigatewayv2_authorizer", - "aws_apigatewayv2_deployment", - "aws_apigatewayv2_domain_name", - "aws_apigatewayv2_integration", - "aws_apigatewayv2_integration_response", - "aws_apigatewayv2_model", - "aws_apigatewayv2_route", - "aws_apigatewayv2_route_response", - "aws_apigatewayv2_stage", - "aws_apigatewayv2_vpc_link", - "aws_accessanalyzer_analyzer", - "aws_appmesh_mesh", - "aws_appmesh_route", - "aws_appmesh_virtual_node", - "aws_appmesh_virtual_router", - "aws_appmesh_virtual_service", - "aws_appsync_api_key", - "aws_appsync_datasource", - "aws_appsync_function", - "aws_appsync_graphql_api", - "aws_appsync_resolver", - "aws_appautoscaling_policy", - "aws_appautoscaling_scheduled_action", - "aws_appautoscaling_target", - "aws_athena_database", - "aws_athena_named_query", - "aws_athena_workgroup", - "aws_autoscaling_attachment", - "aws_autoscaling_group", - "aws_autoscaling_lifecycle_hook", - "aws_autoscaling_notification", - "aws_autoscaling_policy", - "aws_autoscaling_schedule", - "aws_backup_plan", - "aws_backup_selection", - "aws_backup_vault", - "aws_batch_compute_environment", - "aws_batch_job_definition", - "aws_batch_job_queue", - "aws_budgets_budget", - "aws_cloud9_environment_ec2", - "aws_cloudformation_stack", - "aws_cloudformation_stack_set", - "aws_cloudformation_stack_set_instance", - "aws_cloudfront_distribution", - "aws_cloudfront_origin_access_identity", - "aws_cloudfront_public_key", - "aws_cloudhsm_v2_cluster", - "aws_cloudhsm_v2_hsm", - "aws_cloudtrail", - "aws_cloudwatch_dashboard", - "aws_cloudwatch_event_permission", - "aws_cloudwatch_event_rule", - "aws_cloudwatch_event_target", - "aws_cloudwatch_log_destination", - "aws_cloudwatch_log_destination_policy", - "aws_cloudwatch_log_group", - "aws_cloudwatch_log_metric_filter", - "aws_cloudwatch_log_resource_policy", - "aws_cloudwatch_log_stream", - "aws_cloudwatch_log_subscription_filter", - "aws_cloudwatch_metric_alarm", - "aws_codebuild_project", - "aws_codebuild_source_credential", - "aws_codebuild_webhook", - "aws_codecommit_repository", - "aws_codecommit_trigger", - "aws_codedeploy_app", - "aws_codedeploy_deployment_config", - "aws_codedeploy_deployment_group", - "aws_codepipeline", - "aws_codepipeline_webhook", - "aws_codestarnotifications_notification_rule", - "aws_cognito_identity_pool", - "aws_cognito_identity_pool_roles_attachment", - "aws_cognito_identity_provider", - "aws_cognito_resource_server", - "aws_cognito_user_group", - "aws_cognito_user_pool", - "aws_cognito_user_pool_client", - "aws_cognito_user_pool_domain", - "aws_config_aggregate_authorization", - "aws_config_config_rule", - "aws_config_configuration_aggregator", - "aws_config_configuration_recorder", - "aws_config_configuration_recorder_status", - "aws_config_delivery_channel", - "aws_config_organization_custom_rule", - "aws_config_organization_managed_rule", - "aws_cur_report_definition", - "aws_dlm_lifecycle_policy", - "aws_datapipeline_pipeline", - "aws_datasync_agent", - "aws_datasync_location_efs", - "aws_datasync_location_nfs", - "aws_datasync_location_s3", - "aws_datasync_location_smb", - "aws_datasync_task", - "aws_dms_certificate", - "aws_dms_endpoint", - "aws_dms_event_subscription", - "aws_dms_replication_instance", - "aws_dms_replication_subnet_group", - "aws_dms_replication_task", - "aws_devicefarm_project", - "aws_dx_bgp_peer", - "aws_dx_connection", - "aws_dx_connection_association", - "aws_dx_gateway", - "aws_dx_gateway_association", - "aws_dx_gateway_association_proposal", - "aws_dx_hosted_private_virtual_interface", - "aws_dx_hosted_private_virtual_interface_accepter", - "aws_dx_hosted_public_virtual_interface", - "aws_dx_hosted_public_virtual_interface_accepter", - "aws_dx_hosted_transit_virtual_interface", - "aws_dx_hosted_transit_virtual_interface_accepter", - "aws_dx_lag", - "aws_dx_private_virtual_interface", - "aws_dx_public_virtual_interface", - "aws_dx_transit_virtual_interface", - "aws_directory_service_conditional_forwarder", - "aws_directory_service_directory", - "aws_directory_service_log_subscription", - "aws_docdb_cluster", - "aws_docdb_cluster_instance", - "aws_docdb_cluster_parameter_group", - "aws_docdb_cluster_snapshot", - "aws_docdb_subnet_group", - "aws_dynamodb_global_table", - "aws_dynamodb_table", - "aws_dynamodb_table_item", - "aws_dax_cluster", - "aws_dax_parameter_group", - "aws_dax_subnet_group", - "aws_ami", - "aws_ami_copy", - "aws_ami_from_instance", - "aws_ami_launch_permission", - "aws_ebs_default_kms_key", - "aws_ebs_encryption_by_default", - "aws_ebs_snapshot", - "aws_ebs_snapshot_copy", - "aws_ebs_volume", - "aws_ec2_availability_zone_group", - "aws_ec2_capacity_reservation", - "aws_ec2_client_vpn_authorization_rule", - "aws_ec2_client_vpn_endpoint", - "aws_ec2_client_vpn_network_association", - "aws_ec2_client_vpn_route", - "aws_ec2_fleet", - "aws_ec2_local_gateway_route", - "aws_ec2_local_gateway_route_table_vpc_association", - "aws_ec2_tag", - "aws_ec2_traffic_mirror_filter", - "aws_ec2_traffic_mirror_filter_rule", - "aws_ec2_traffic_mirror_session", - "aws_ec2_traffic_mirror_target", - "aws_ec2_transit_gateway", - "aws_ec2_transit_gateway_peering_attachment", - "aws_ec2_transit_gateway_peering_attachment_accepter", - "aws_ec2_transit_gateway_route", - "aws_ec2_transit_gateway_route_table", - "aws_ec2_transit_gateway_route_table_association", - "aws_ec2_transit_gateway_route_table_propagation", - "aws_ec2_transit_gateway_vpc_attachment", - "aws_ec2_transit_gateway_vpc_attachment_accepter", - "aws_eip", - "aws_eip_association", - "aws_instance", - "aws_key_pair", - "aws_launch_configuration", - "aws_launch_template", - "aws_placement_group", - "aws_snapshot_create_volume_permission", - "aws_spot_datafeed_subscription", - "aws_spot_fleet_request", - "aws_spot_instance_request", - "aws_volume_attachment", - "aws_ecr_lifecycle_policy", - "aws_ecr_repository", - "aws_ecr_repository_policy", - "aws_ecs_capacity_provider", - "aws_ecs_cluster", - "aws_ecs_service", - "aws_ecs_task_definition", - "aws_efs_access_point", - "aws_efs_file_system", - "aws_efs_file_system_policy", - "aws_efs_mount_target", - "aws_eks_cluster", - "aws_eks_fargate_profile", - "aws_eks_node_group", - "aws_elasticache_cluster", - "aws_elasticache_parameter_group", - "aws_elasticache_replication_group", - "aws_elasticache_security_group", - "aws_elasticache_subnet_group", - "aws_elastic_beanstalk_application", - "aws_elastic_beanstalk_application_version", - "aws_elastic_beanstalk_configuration_template", - "aws_elastic_beanstalk_environment", - "aws_app_cookie_stickiness_policy", - "aws_elb", - "aws_elb_attachment", - "aws_lb_cookie_stickiness_policy", - "aws_lb_ssl_negotiation_policy", - "aws_load_balancer_backend_server_policy", - "aws_load_balancer_listener_policy", - "aws_load_balancer_policy", - "aws_proxy_protocol_policy", - "aws_lb", - "aws_lb_listener", - "aws_lb_listener_certificate", - "aws_lb_listener_rule", - "aws_lb_target_group", - "aws_lb_target_group_attachment", - "aws_emr_cluster", - "aws_emr_instance_group", - "aws_emr_security_configuration", - "aws_elastictranscoder_pipeline", - "aws_elastictranscoder_preset", - "aws_elasticsearch_domain", - "aws_elasticsearch_domain_policy", - "aws_fsx_lustre_file_system", - "aws_fsx_windows_file_system", - "aws_fms_admin_account", - "aws_gamelift_alias", - "aws_gamelift_build", - "aws_gamelift_fleet", - "aws_gamelift_game_session_queue", - "aws_glacier_vault", - "aws_glacier_vault_lock", - "aws_globalaccelerator_accelerator", - "aws_globalaccelerator_endpoint_group", - "aws_globalaccelerator_listener", - "aws_glue_catalog_database", - "aws_glue_catalog_table", - "aws_glue_classifier", - "aws_glue_connection", - "aws_glue_crawler", - "aws_glue_job", - "aws_glue_security_configuration", - "aws_glue_trigger", - "aws_glue_workflow", - "aws_guardduty_detector", - "aws_guardduty_invite_accepter", - "aws_guardduty_ipset", - "aws_guardduty_member", - "aws_guardduty_organization_admin_account", - "aws_guardduty_organization_configuration", - "aws_guardduty_threatintelset", - "aws_iam_access_key", - "aws_iam_account_alias", - "aws_iam_account_password_policy", - "aws_iam_group", - "aws_iam_group_membership", - "aws_iam_group_policy", - "aws_iam_group_policy_attachment", - "aws_iam_instance_profile", - "aws_iam_openid_connect_provider", - "aws_iam_policy", - "aws_iam_policy_attachment", - "aws_iam_policy_document", - "aws_iam_role", - "aws_iam_role_policy", - "aws_iam_role_policy_attachment", - "aws_iam_saml_provider", - "aws_iam_server_certificate", - "aws_iam_service_linked_role", - "aws_iam_user", - "aws_iam_user_group_membership", - "aws_iam_user_login_profile", - "aws_iam_user_policy", - "aws_iam_user_policy_attachment", - "aws_iam_user_ssh_key", - "aws_inspector_assessment_target", - "aws_inspector_assessment_template", - "aws_inspector_resource_group", - "aws_iot_certificate", - "aws_iot_policy", - "aws_iot_policy_attachment", - "aws_iot_role_alias", - "aws_iot_thing", - "aws_iot_thing_principal_attachment", - "aws_iot_thing_type", - "aws_iot_topic_rule", - "aws_kms_alias", - "aws_kms_ciphertext", - "aws_kms_external_key", - "aws_kms_grant", - "aws_kms_key", - "aws_kinesis_analytics_application", - "aws_kinesis_stream", - "aws_kinesis_firehose_delivery_stream", - "aws_kinesis_video_stream", - "aws_lambda_alias", - "aws_lambda_event_source_mapping", - "aws_lambda_function", - "aws_lambda_function_event_invoke_config", - "aws_lambda_layer_version", - "aws_lambda_permission", - "aws_lambda_provisioned_concurrency_config", - "aws_licensemanager_association", - "aws_licensemanager_license_configuration", - "aws_lightsail_domain", - "aws_lightsail_instance", - "aws_lightsail_key_pair", - "aws_lightsail_static_ip", - "aws_lightsail_static_ip_attachment", - "aws_mq_broker", - "aws_mq_configuration", - "aws_macie_member_account_association", - "aws_macie_s3_bucket_association", - "aws_msk_cluster", - "aws_msk_configuration", - "aws_media_convert_queue", - "aws_media_package_channel", - "aws_media_store_container", - "aws_media_store_container_policy", - "aws_neptune_cluster", - "aws_neptune_cluster_instance", - "aws_neptune_cluster_parameter_group", - "aws_neptune_cluster_snapshot", - "aws_neptune_event_subscription", - "aws_neptune_parameter_group", - "aws_neptune_subnet_group", - "aws_opsworks_application", - "aws_opsworks_custom_layer", - "aws_opsworks_ganglia_layer", - "aws_opsworks_haproxy_layer", - "aws_opsworks_instance", - "aws_opsworks_java_app_layer", - "aws_opsworks_memcached_layer", - "aws_opsworks_mysql_layer", - "aws_opsworks_nodejs_app_layer", - "aws_opsworks_permission", - "aws_opsworks_php_app_layer", - "aws_opsworks_rails_app_layer", - "aws_opsworks_rds_db_instance", - "aws_opsworks_stack", - "aws_opsworks_static_web_layer", - "aws_opsworks_user_profile", - "aws_organizations_account", - "aws_organizations_organization", - "aws_organizations_organizational_unit", - "aws_organizations_policy", - "aws_organizations_policy_attachment", - "aws_pinpoint_adm_channel", - "aws_pinpoint_apns_channel", - "aws_pinpoint_apns_sandbox_channel", - "aws_pinpoint_apns_voip_channel", - "aws_pinpoint_apns_voip_sandbox_channel", - "aws_pinpoint_app", - "aws_pinpoint_baidu_channel", - "aws_pinpoint_email_channel", - "aws_pinpoint_event_stream", - "aws_pinpoint_gcm_channel", - "aws_pinpoint_sms_channel", - "aws_qldb_ledger", - "aws_quicksight_group", - "aws_quicksight_user", - "aws_ram_principal_association", - "aws_ram_resource_association", - "aws_ram_resource_share", - "aws_ram_resource_share_accepter", - "aws_db_cluster_snapshot", - "aws_db_event_subscription", - "aws_db_instance", - "aws_db_instance_role_association", - "aws_db_option_group", - "aws_db_parameter_group", - "aws_db_security_group", - "aws_db_snapshot", - "aws_db_subnet_group", - "aws_rds_cluster", - "aws_rds_cluster_endpoint", - "aws_rds_cluster_instance", - "aws_rds_cluster_parameter_group", - "aws_rds_global_cluster", - "aws_redshift_cluster", - "aws_redshift_event_subscription", - "aws_redshift_parameter_group", - "aws_redshift_security_group", - "aws_redshift_snapshot_copy_grant", - "aws_redshift_snapshot_schedule", - "aws_redshift_snapshot_schedule_association", - "aws_redshift_subnet_group", - "aws_resourcegroups_group", - "aws_route53_delegation_set", - "aws_route53_health_check", - "aws_route53_query_log", - "aws_route53_record", - "aws_route53_zone", - "aws_route53_zone_association", - "aws_route53_resolver_endpoint", - "aws_route53_resolver_rule", - "aws_route53_resolver_rule_association", - "aws_s3_access_point", - "aws_s3_account_public_access_block", - "aws_s3_bucket", - "aws_s3_bucket_analytics_configuration", - "aws_s3_bucket_inventory", - "aws_s3_bucket_metric", - "aws_s3_bucket_notification", - "aws_s3_bucket_object", - "aws_s3_bucket_policy", - "aws_s3_bucket_public_access_block", - "aws_ses_active_receipt_rule_set", - "aws_ses_configuration_set", - "aws_ses_domain_dkim", - "aws_ses_domain_identity", - "aws_ses_domain_identity_verification", - "aws_ses_domain_mail_from", - "aws_ses_email_identity", - "aws_ses_event_destination", - "aws_ses_identity_notification_topic", - "aws_ses_identity_policy", - "aws_ses_receipt_filter", - "aws_ses_receipt_rule", - "aws_ses_receipt_rule_set", - "aws_ses_template", - "aws_sns_platform_application", - "aws_sns_sms_preferences", - "aws_sns_topic", - "aws_sns_topic_policy", - "aws_sns_topic_subscription", - "aws_sqs_queue", - "aws_sqs_queue_policy", - "aws_ssm_activation", - "aws_ssm_association", - "aws_ssm_document", - "aws_ssm_maintenance_window", - "aws_ssm_maintenance_window_target", - "aws_ssm_maintenance_window_task", - "aws_ssm_parameter", - "aws_ssm_patch_baseline", - "aws_ssm_patch_group", - "aws_ssm_resource_data_sync", - "aws_swf_domain", - "aws_sagemaker_endpoint", - "aws_sagemaker_endpoint_configuration", - "aws_sagemaker_model", - "aws_sagemaker_notebook_instance", - "aws_sagemaker_notebook_instance_lifecycle_configuration", - "aws_secretsmanager_secret", - "aws_secretsmanager_secret_rotation", - "aws_secretsmanager_secret_version", - "aws_securityhub_account", - "aws_securityhub_member", - "aws_securityhub_product_subscription", - "aws_securityhub_standards_subscription", - "aws_servicecatalog_portfolio", - "aws_service_discovery_http_namespace", - "aws_service_discovery_private_dns_namespace", - "aws_service_discovery_public_dns_namespace", - "aws_service_discovery_service", - "aws_servicequotas_service_quota", - "aws_shield_protection", - "aws_simpledb_domain", - "aws_sfn_activity", - "aws_sfn_state_machine", - "aws_storagegateway_cache", - "aws_storagegateway_cached_iscsi_volume", - "aws_storagegateway_gateway", - "aws_storagegateway_nfs_file_share", - "aws_storagegateway_smb_file_share", - "aws_storagegateway_upload_buffer", - "aws_storagegateway_working_storage", - "aws_transfer_server", - "aws_transfer_ssh_key", - "aws_transfer_user", - "aws_customer_gateway", - "aws_default_network_acl", - "aws_default_route_table", - "aws_default_security_group", - "aws_default_subnet", - "aws_default_vpc", - "aws_default_vpc_dhcp_options", - "aws_egress_only_internet_gateway", - "aws_flow_log", - "aws_internet_gateway", - "aws_main_route_table_association", - "aws_nat_gateway", - "aws_network_acl", - "aws_network_acl_rule", - "aws_network_interface", - "aws_network_interface_attachment", - "aws_network_interface_sg_attachment", - "aws_route", - "aws_route_table", - "aws_route_table_association", - "aws_security_group", - "aws_security_group_rule", - "aws_subnet", - "aws_vpc", - "aws_vpc_dhcp_options", - "aws_vpc_dhcp_options_association", - "aws_vpc_endpoint", - "aws_vpc_endpoint_connection_notification", - "aws_vpc_endpoint_route_table_association", - "aws_vpc_endpoint_service", - "aws_vpc_endpoint_service_allowed_principal", - "aws_vpc_endpoint_subnet_association", - "aws_vpc_ipv4_cidr_block_association", - "aws_vpc_peering_connection", - "aws_vpc_peering_connection_accepter", - "aws_vpc_peering_connection_options", - "aws_vpn_connection", - "aws_vpn_connection_route", - "aws_vpn_gateway", - "aws_vpn_gateway_attachment", - "aws_vpn_gateway_route_propagation", - "aws_waf_byte_match_set", - "aws_waf_geo_match_set", - "aws_waf_ipset", - "aws_waf_rate_based_rule", - "aws_waf_regex_match_set", - "aws_waf_regex_pattern_set", - "aws_waf_rule", - "aws_waf_rule_group", - "aws_waf_size_constraint_set", - "aws_waf_sql_injection_match_set", - "aws_waf_web_acl", - "aws_waf_xss_match_set", - "aws_wafregional_byte_match_set", - "aws_wafregional_geo_match_set", - "aws_wafregional_ipset", - "aws_wafregional_rate_based_rule", - "aws_wafregional_regex_match_set", - "aws_wafregional_regex_pattern_set", - "aws_wafregional_rule", - "aws_wafregional_rule_group", - "aws_wafregional_size_constraint_set", - "aws_wafregional_sql_injection_match_set", - "aws_wafregional_web_acl", - "aws_wafregional_web_acl_association", - "aws_wafregional_xss_match_set", - "aws_wafv2_ip_set", - "aws_wafv2_regex_pattern_set", - "aws_wafv2_rule_group", - "aws_wafv2_web_acl", - "aws_wafv2_web_acl_association", - "aws_wafv2_web_acl_logging_configuration", - "aws_worklink_fleet", - "aws_worklink_website_certificate_authority_association", - "aws_workspaces_directory", - "aws_workspaces_ip_group", - "aws_workspaces_workspace", - "aws_xray_sampling_rule", - "aws_route53_vpc_association_authorization" - ], - "gcp": [ - "google_access_context_manager_access_level", - "google_access_context_manager_access_policy", - "google_access_context_manager_service_perimeter", - "google_access_context_manager_service_perimeter_resource", - "google_app_engine_application", - "google_app_engine_application_url_dispatch_rules", - "google_app_engine_domain_mapping", - "google_app_engine_firewall_rule", - "google_app_engine_flexible_app_version", - "google_app_engine_service_split_traffic", - "google_app_engine_standard_app_version", - "google_bigquery_dataset", - "google_bigquery_dataset_access", - "google_bigquery_dataset_iam", - "google_bigquery_job", - "google_bigquery_table", - "google_bigquery_data_transfer_config", - "google_binary_authorization_attestor", - "google_binary_authorization_attestor_iam", - "google_binary_authorization_policy", - "google_logging_billing_account_bucket_config", - "google_logging_billing_account_exclusion", - "google_logging_billing_account_sink", - "google_logging_folder_bucket_config", - "google_logging_folder_exclusion", - "google_logging_folder_sink", - "google_logging_metric", - "google_logging_organization_bucket_config", - "google_logging_organization_exclusion", - "google_logging_organization_sink", - "google_logging_project_bucket_config", - "google_logging_project_exclusion", - "google_logging_project_sink", - "google_monitoring_alert_policy", - "google_monitoring_dashboard", - "google_monitoring_group", - "google_monitoring_metric_descriptor", - "google_monitoring_notification_channel", - "google_monitoring_service", - "google_monitoring_slo", - "google_monitoring_uptime_check_config", - "google_cloud_asset_folder_feed", - "google_cloud_asset_organization_feed", - "google_cloud_asset_project_feed", - "google_bigtable_app_profile", - "google_bigtable_gc_policy", - "google_bigtable_instance", - "google_bigtable_instance_iam", - "google_bigtable_table", - "google_cloudbuild_trigger", - "google_composer_environment", - "google_dns_managed_zone", - "google_dns_policy", - "google_dns_record_set", - "google_deployment_manager_deployment", - "google_endpoints_service", - "google_endpoints_service_iam", - "google_cloudfunctions_cloud_function_iam", - "google_cloudfunctions_function", - "google_healthcare_dataset", - "google_healthcare_dataset_iam", - "google_healthcare_dicom_store", - "google_healthcare_dicom_store_iam", - "google_healthcare_fhir_store", - "google_healthcare_fhir_store_iam", - "google_healthcare_hl7_v2_store", - "google_healthcare_hl7_v2_store_iam", - "google_cloudiot_device", - "google_cloudiot_device_registry", - "google_kms_crypto_key_iam", - "google_kms_key_ring_iam", - "google_kms_crypto_key", - "google_kms_key_ring", - "google_kms_key_ring_import_job", - "google_kms_secret_ciphertext", - "google_billing_account_iam_binding", - "google_billing_account_iam_member", - "google_billing_account_iam_policy", - "google_folder", - "google_folder_iam_audit_config", - "google_folder_iam_binding", - "google_folder_iam_member", - "google_folder_iam_policy", - "google_folder_organization_policy", - "google_organization_iam_audit_config", - "google_organization_iam_binding", - "google_organization_iam_custom_role", - "google_organization_iam_member", - "google_organization_iam_policy", - "google_organization_policy", - "google_project", - "google_project_iam_binding", - "google_project_iam_member", - "google_project_iam_custom_role", - "google_project_organization_policy", - "google_project_service", - "google_service_account", - "google_service_account_iam", - "google_service_account_key", - "google_usage_export_bucket", - "google_pubsub_subscription", - "google_pubsub_subscription_iam", - "google_pubsub_topic", - "google_pubsub_topic_iam", - "google_cloud_run_domain_mapping", - "google_cloud_run_service", - "google_cloud_run_service_iam", - "google_sql_database", - "google_sql_database_instance", - "google_sql_source_representation_instance", - "google_sql_ssl_cert", - "google_sql_user", - "google_cloud_scheduler_job", - "google_sourcerepo_repository", - "google_sourcerepo_repository_iam", - "google_spanner_database", - "google_spanner_database_iam", - "google_spanner_instance", - "google_spanner_instance_iam", - "google_storage_bucket", - "google_storage_bucket_access_control", - "google_storage_bucket_acl", - "google_storage_bucket_iam_binding", - "google_storage_bucket_iam_member", - "google_storage_bucket_object", - "google_storage_default_object_access_control", - "google_storage_default_object_acl", - "google_storage_hmac_key", - "google_storage_notification", - "google_storage_object_access_control", - "google_storage_object_acl", - "google_tpu_node", - "google_cloud_tasks_queue", - "google_compute_address", - "google_compute_attached_disk", - "google_compute_autoscaler", - "google_compute_backend_bucket", - "google_compute_backend_bucket_signed_url_key", - "google_compute_backend_service", - "google_compute_backend_service_signed_url_key", - "google_compute_disk", - "google_compute_disk_resource_policy_attachment", - "google_compute_firewall", - "google_compute_forwarding_rule", - "google_compute_global_address", - "google_compute_global_forwarding_rule", - "google_compute_global_network_endpoint", - "google_compute_global_network_endpoint_group", - "google_compute_health_check", - "google_compute_http_health_check", - "google_compute_https_health_check", - "google_compute_image", - "google_compute_instance", - "google_compute_instance_from_template", - "google_compute_instance_group", - "google_compute_instance_group_manager", - "google_compute_instance_group_named_port", - "google_compute_instance_iam", - "google_compute_instance_template", - "google_compute_interconnect_attachment", - "google_compute_network", - "google_compute_network_endpoint", - "google_compute_network_endpoint_group", - "google_compute_network_peering", - "google_compute_network_peering_routes_config", - "google_compute_node_group", - "google_compute_node_template", - "google_compute_project_default_network_tier", - "google_compute_project_metadata", - "google_compute_project_metadata_item", - "google_compute_region_autoscaler", - "google_compute_region_backend_service", - "google_compute_region_disk", - "google_compute_region_disk_resource_policy_attachment", - "google_compute_region_health_check", - "google_compute_region_instance_group_manager", - "google_compute_region_ssl_certificate", - "google_compute_region_target_http_proxy", - "google_compute_region_target_https_proxy", - "google_compute_region_url_map", - "google_compute_reservation", - "google_compute_resource_policy", - "google_compute_route", - "google_compute_router", - "google_compute_router_bgp_peer", - "google_compute_router_interface", - "google_compute_router_nat", - "google_compute_security_policy", - "google_compute_shared_vpc_host_project", - "google_compute_shared_vpc_service_project", - "google_compute_snapshot", - "google_compute_ssl_certificate", - "google_compute_ssl_policy", - "google_compute_subnetwork", - "google_compute_subnetwork_iam", - "google_compute_target_http_proxy", - "google_compute_target_https_proxy", - "google_compute_target_instance", - "google_compute_target_pool", - "google_compute_target_ssl_proxy", - "google_compute_target_tcp_proxy", - "google_compute_url_map", - "google_compute_vpn_gateway", - "google_compute_vpn_tunnel", - "google_container_analysis_note", - "google_container_analysis_occurrence", - "google_container_registry", - "google_data_catalog_entry", - "google_data_catalog_entry_group", - "google_data_catalog_entry_group_iam", - "google_data_catalog_tag", - "google_data_catalog_tag_template", - "google_dataflow_job", - "google_dataproc_autoscaling_policy", - "google_dataproc_cluster", - "google_dataproc_cluster_iam", - "google_dataproc_job", - "google_dataproc_job_iam", - "google_datastore_index", - "google_dialogflow_agent", - "google_dialogflow_entity_type", - "google_dialogflow_intent", - "google_filestore_instance", - "google_firestore_index", - "google_identity_platform_default_supported_idp_config", - "google_identity_platform_inbound_saml_config", - "google_identity_platform_oauth_idp_config", - "google_identity_platform_tenant", - "google_identity_platform_tenant_default_supported_idp_config", - "google_identity_platform_tenant_inbound_saml_config", - "google_identity_platform_tenant_oauth_idp_config", - "google_iap_app_engine_service_iam", - "google_iap_app_engine_version_iam", - "google_iap_brand", - "google_iap_client", - "google_iap_tunnel_instance_iam", - "google_iap_web_backend_service_iam", - "google_iap_web_iam", - "google_iap_web_type_app_engine_iam", - "google_iap_web_type_compute_iam", - "google_container_cluster", - "google_container_node_pool", - "google_ml_engine_model", - "google_redis_instance", - "google_network_management_connectivity_test_resource", - "google_os_config_patch_deployment", - "google_os_login_ssh_public_key", - "google_resource_manager_lien", - "google_runtimeconfig_config", - "google_runtimeconfig_config_iam", - "google_runtimeconfig_variable", - "google_secret_manager_secret", - "google_secret_manager_secret_iam", - "google_secret_manager_secret_version", - "google_scc_source", - "google_vpc_access_connector", - "google_service_networking_connection", - "google_storage_transfer_job", - "google_project_iam", - "google_storage_bucket_iam", - "google_dataflow_flex_template_job", - "google_active_directory_domain" - ], - "azure": [ - "azurerm_api_management", - "azurerm_api_management_api", - "azurerm_api_management_api_operation", - "azurerm_api_management_api_operation_policy", - "azurerm_api_management_api_policy", - "azurerm_api_management_api_schema", - "azurerm_api_management_api_version_set", - "azurerm_api_management_authorization_server", - "azurerm_api_management_backend", - "azurerm_api_management_certificate", - "azurerm_api_management_diagnostic", - "azurerm_api_management_group", - "azurerm_api_management_group_user", - "azurerm_api_management_identity_provider_aad", - "azurerm_api_management_identity_provider_facebook", - "azurerm_api_management_identity_provider_google", - "azurerm_api_management_identity_provider_microsoft", - "azurerm_api_management_identity_provider_twitter", - "azurerm_api_management_logger", - "azurerm_api_management_named_value", - "azurerm_api_management_openid_connect_provider", - "azurerm_api_management_product", - "azurerm_api_management_product_api", - "azurerm_api_management_product_group", - "azurerm_api_management_product_policy", - "azurerm_api_management_property", - "azurerm_api_management_subscription", - "azurerm_api_management_user", - "azurerm_analysis_services_server", - "azurerm_app_configuration", - "azurerm_app_service", - "azurerm_app_service_active_slot", - "azurerm_app_service_certificate", - "azurerm_app_service_certificate_order", - "azurerm_app_service_custom_hostname_binding", - "azurerm_app_service_environment", - "azurerm_app_service_hybrid_connection", - "azurerm_app_service_plan", - "azurerm_app_service_slot", - "azurerm_app_service_slot_virtual_network_swift_connection", - "azurerm_app_service_source_control_token", - "azurerm_app_service_virtual_network_swift_connection", - "azurerm_function_app", - "azurerm_function_app_slot", - "azurerm_application_insights", - "azurerm_application_insights_analytics_item", - "azurerm_application_insights_api_key", - "azurerm_application_insights_web_test", - "azurerm_role_assignment", - "azurerm_role_definition", - "azurerm_user_assigned_identity", - "azurerm_automation_account", - "azurerm_automation_certificate", - "azurerm_automation_connection", - "azurerm_automation_connection_certificate", - "azurerm_automation_connection_classic_certificate", - "azurerm_automation_connection_service_principal", - "azurerm_automation_credential", - "azurerm_automation_dsc_configuration", - "azurerm_automation_dsc_nodeconfiguration", - "azurerm_automation_job_schedule", - "azurerm_automation_module", - "azurerm_automation_runbook", - "azurerm_automation_schedule", - "azurerm_automation_variable_bool", - "azurerm_automation_variable_datetime", - "azurerm_automation_variable_int", - "azurerm_automation_variable_string", - "azurerm_resource_group", - "azurerm_batch_account", - "azurerm_batch_application", - "azurerm_batch_certificate", - "azurerm_batch_pool", - "azurerm_blueprint_assignment", - "azurerm_bot_channel_directline", - "azurerm_bot_channel_email", - "azurerm_bot_channel_ms_teams", - "azurerm_bot_channel_slack", - "azurerm_bot_channels_registration", - "azurerm_bot_connection", - "azurerm_bot_web_app", - "azurerm_cdn_endpoint", - "azurerm_cdn_profile", - "azurerm_cognitive_account", - "azurerm_availability_set", - "azurerm_dedicated_host", - "azurerm_dedicated_host_group", - "azurerm_disk_encryption_set", - "azurerm_image", - "azurerm_linux_virtual_machine", - "azurerm_linux_virtual_machine_scale_set", - "azurerm_managed_disk", - "azurerm_marketplace_agreement", - "azurerm_orchestrated_virtual_machine_scale_set", - "azurerm_proximity_placement_group", - "azurerm_shared_image", - "azurerm_shared_image_gallery", - "azurerm_shared_image_version", - "azurerm_snapshot", - "azurerm_virtual_machine", - "azurerm_virtual_machine_data_disk_attachment", - "azurerm_virtual_machine_extension", - "azurerm_virtual_machine_scale_set", - "azurerm_virtual_machine_scale_set_extension", - "azurerm_windows_virtual_machine", - "azurerm_windows_virtual_machine_scale_set", - "azurerm_container_group", - "azurerm_container_registry", - "azurerm_container_registry_webhook", - "azurerm_kubernetes_cluster", - "azurerm_kubernetes_cluster_node_pool", - "azurerm_cosmosdb_account", - "azurerm_cosmosdb_cassandra_keyspace", - "azurerm_cosmosdb_gremlin_database", - "azurerm_cosmosdb_gremlin_graph", - "azurerm_cosmosdb_mongo_collection", - "azurerm_cosmosdb_mongo_database", - "azurerm_cosmosdb_sql_container", - "azurerm_cosmosdb_sql_database", - "azurerm_cosmosdb_table", - "azurerm_cost_management_export_resource_group", - "azurerm_custom_provider", - "azurerm_dns_a_record", - "azurerm_dns_aaaa_record", - "azurerm_dns_caa_record", - "azurerm_dns_cname_record", - "azurerm_dns_mx_record", - "azurerm_dns_ns_record", - "azurerm_dns_ptr_record", - "azurerm_dns_srv_record", - "azurerm_dns_txt_record", - "azurerm_dns_zone", - "azurerm_kusto_attached_database_configuration", - "azurerm_kusto_cluster", - "azurerm_kusto_cluster_customer_managed_key", - "azurerm_kusto_cluster_principal_assignment", - "azurerm_kusto_database", - "azurerm_kusto_database_principal", - "azurerm_kusto_database_principal_assignment", - "azurerm_kusto_eventhub_data_connection", - "azurerm_data_factory", - "azurerm_data_factory_dataset_azure_blob", - "azurerm_data_factory_dataset_cosmosdb_sqlapi", - "azurerm_data_factory_dataset_delimited_text", - "azurerm_data_factory_dataset_http", - "azurerm_data_factory_dataset_json", - "azurerm_data_factory_dataset_mysql", - "azurerm_data_factory_dataset_postgresql", - "azurerm_data_factory_dataset_sql_server_table", - "azurerm_data_factory_integration_runtime_managed", - "azurerm_data_factory_integration_runtime_self_hosted", - "azurerm_data_factory_linked_service_azure_blob_storage", - "azurerm_data_factory_linked_service_azure_file_storage", - "azurerm_data_factory_linked_service_azure_function", - "azurerm_data_factory_linked_service_cosmosdb", - "azurerm_data_factory_linked_service_data_lake_storage_gen2", - "azurerm_data_factory_linked_service_key_vault", - "azurerm_data_factory_linked_service_mysql", - "azurerm_data_factory_linked_service_postgresql", - "azurerm_data_factory_linked_service_sftp", - "azurerm_data_factory_linked_service_sql_server", - "azurerm_data_factory_linked_service_web", - "azurerm_data_factory_pipeline", - "azurerm_data_factory_trigger_schedule", - "azurerm_data_lake_analytics_account", - "azurerm_data_lake_analytics_firewall_rule", - "azurerm_data_lake_store", - "azurerm_data_lake_store_file", - "azurerm_data_lake_store_firewall_rule", - "azurerm_data_share", - "azurerm_data_share_account", - "azurerm_data_share_dataset_blob_storage", - "azurerm_data_share_dataset_data_lake_gen1", - "azurerm_mariadb_configuration", - "azurerm_mariadb_database", - "azurerm_mariadb_firewall_rule", - "azurerm_mariadb_server", - "azurerm_mariadb_virtual_network_rule", - "azurerm_mssql_database", - "azurerm_mssql_database_vulnerability_assessment_rule_baseline", - "azurerm_mssql_elasticpool", - "azurerm_mssql_server", - "azurerm_mssql_server_security_alert_policy", - "azurerm_mssql_server_vulnerability_assessment", - "azurerm_mssql_virtual_machine", - "azurerm_mysql_active_directory_administrator", - "azurerm_mysql_configuration", - "azurerm_mysql_database", - "azurerm_mysql_firewall_rule", - "azurerm_mysql_server", - "azurerm_mysql_virtual_network_rule", - "azurerm_postgresql_active_directory_administrator", - "azurerm_postgresql_configuration", - "azurerm_postgresql_database", - "azurerm_postgresql_firewall_rule", - "azurerm_postgresql_server", - "azurerm_postgresql_virtual_network_rule", - "azurerm_sql_active_directory_administrator", - "azurerm_sql_database", - "azurerm_sql_elasticpool", - "azurerm_sql_failover_group", - "azurerm_sql_firewall_rule", - "azurerm_sql_server", - "azurerm_sql_virtual_network_rule", - "azurerm_database_migration_project", - "azurerm_database_migration_service", - "azurerm_databricks_workspace", - "azurerm_dev_test_global_vm_shutdown_schedule", - "azurerm_dev_test_lab", - "azurerm_dev_test_linux_virtual_machine", - "azurerm_dev_test_policy", - "azurerm_dev_test_schedule", - "azurerm_dev_test_virtual_network", - "azurerm_dev_test_windows_virtual_machine", - "azurerm_devspace_controller", - "azurerm_hdinsight_hadoop_cluster", - "azurerm_hdinsight_hbase_cluster", - "azurerm_hdinsight_interactive_query_cluster", - "azurerm_hdinsight_kafka_cluster", - "azurerm_hdinsight_ml_services_cluster", - "azurerm_hdinsight_rserver_cluster", - "azurerm_hdinsight_spark_cluster", - "azurerm_hdinsight_storm_cluster", - "azurerm_dedicated_hardware_security_module", - "azurerm_healthcare_service", - "azurerm_iotcentral_application", - "azurerm_iothub", - "azurerm_iothub_consumer_group", - "azurerm_iothub_dps", - "azurerm_iothub_dps_certificate", - "azurerm_iothub_dps_shared_access_policy", - "azurerm_iothub_shared_access_policy", - "azurerm_key_vault", - "azurerm_key_vault_access_policy", - "azurerm_key_vault_certificate", - "azurerm_key_vault_certificate_issuer", - "azurerm_key_vault_key", - "azurerm_key_vault_secret", - "azurerm_lb", - "azurerm_lb_backend_address_pool", - "azurerm_lb_nat_pool", - "azurerm_lb_nat_rule", - "azurerm_lb_outbound_rule", - "azurerm_lb_probe", - "azurerm_lb_rule", - "azurerm_log_analytics_datasource_windows_event", - "azurerm_log_analytics_datasource_windows_performance_counter", - "azurerm_log_analytics_linked_service", - "azurerm_log_analytics_solution", - "azurerm_log_analytics_workspace", - "azurerm_logic_app_action_custom", - "azurerm_logic_app_action_http", - "azurerm_logic_app_integration_account", - "azurerm_logic_app_trigger_custom", - "azurerm_logic_app_trigger_http_request", - "azurerm_logic_app_trigger_recurrence", - "azurerm_logic_app_workflow", - "azurerm_machine_learning_workspace", - "azurerm_maintenance_assignment_dedicated_host", - "azurerm_maintenance_assignment_virtual_machine", - "azurerm_maintenance_configuration", - "azurerm_managed_application", - "azurerm_managed_application_definition", - "azurerm_management_group", - "azurerm_management_lock", - "azurerm_maps_account", - "azurerm_media_services_account", - "azurerm_eventgrid_domain", - "azurerm_eventgrid_domain_topic", - "azurerm_eventgrid_event_subscription", - "azurerm_eventgrid_topic", - "azurerm_eventhub", - "azurerm_eventhub_authorization_rule", - "azurerm_eventhub_cluster", - "azurerm_eventhub_consumer_group", - "azurerm_eventhub_namespace", - "azurerm_eventhub_namespace_authorization_rule", - "azurerm_eventhub_namespace_disaster_recovery_config", - "azurerm_iothub_endpoint_eventhub", - "azurerm_iothub_endpoint_servicebus_queue", - "azurerm_iothub_endpoint_servicebus_topic", - "azurerm_iothub_endpoint_storage_container", - "azurerm_iothub_fallback_route", - "azurerm_iothub_route", - "azurerm_notification_hub", - "azurerm_notification_hub_authorization_rule", - "azurerm_notification_hub_namespace", - "azurerm_relay_hybrid_connection", - "azurerm_relay_namespace", - "azurerm_servicebus_namespace", - "azurerm_servicebus_namespace_authorization_rule", - "azurerm_servicebus_namespace_network_rule_set", - "azurerm_servicebus_queue", - "azurerm_servicebus_queue_authorization_rule", - "azurerm_servicebus_subscription", - "azurerm_servicebus_subscription_rule", - "azurerm_servicebus_topic", - "azurerm_servicebus_topic_authorization_rule", - "azurerm_signalr_service", - "azurerm_spatial_anchors_account", - "azurerm_monitor_action_group", - "azurerm_monitor_action_rule_action_group", - "azurerm_monitor_action_rule_suppression", - "azurerm_monitor_activity_log_alert", - "azurerm_monitor_autoscale_setting", - "azurerm_monitor_diagnostic_setting", - "azurerm_monitor_log_profile", - "azurerm_monitor_metric_alert", - "azurerm_monitor_scheduled_query_rules_alert", - "azurerm_monitor_scheduled_query_rules_log", - "azurerm_netapp_account", - "azurerm_netapp_pool", - "azurerm_netapp_snapshot", - "azurerm_netapp_volume", - "azurerm_application_gateway", - "azurerm_application_security_group", - "azurerm_bastion_host", - "azurerm_express_route_circuit", - "azurerm_express_route_circuit_authorization", - "azurerm_express_route_circuit_peering", - "azurerm_express_route_gateway", - "azurerm_firewall", - "azurerm_firewall_application_rule_collection", - "azurerm_firewall_nat_rule_collection", - "azurerm_firewall_network_rule_collection", - "azurerm_frontdoor", - "azurerm_frontdoor_custom_https_configuration", - "azurerm_frontdoor_firewall_policy", - "azurerm_local_network_gateway", - "azurerm_nat_gateway", - "azurerm_nat_gateway_public_ip_association", - "azurerm_network_ddos_protection_plan", - "azurerm_network_interface", - "azurerm_network_interface_application_gateway_backend_address_pool_association", - "azurerm_network_interface_application_security_group_association", - "azurerm_network_interface_backend_address_pool_association", - "azurerm_network_interface_nat_rule_association", - "azurerm_network_interface_security_group_association", - "azurerm_network_packet_capture", - "azurerm_network_profile", - "azurerm_network_security_group", - "azurerm_network_security_rule", - "azurerm_network_watcher", - "azurerm_network_watcher_flow_log", - "azurerm_packet_capture", - "azurerm_point_to_site_vpn_gateway", - "azurerm_private_endpoint", - "azurerm_private_link_service", - "azurerm_public_ip", - "azurerm_public_ip_prefix", - "azurerm_route", - "azurerm_route_filter", - "azurerm_route_table", - "azurerm_subnet", - "azurerm_subnet_nat_gateway_association", - "azurerm_subnet_network_security_group_association", - "azurerm_subnet_route_table_association", - "azurerm_traffic_manager_endpoint", - "azurerm_traffic_manager_profile", - "azurerm_virtual_hub", - "azurerm_virtual_hub_connection", - "azurerm_virtual_network", - "azurerm_virtual_network_gateway", - "azurerm_virtual_network_gateway_connection", - "azurerm_virtual_network_peering", - "azurerm_virtual_wan", - "azurerm_vpn_gateway", - "azurerm_vpn_server_configuration", - "azurerm_web_application_firewall_policy", - "azurerm_policy_assignment", - "azurerm_policy_definition", - "azurerm_policy_remediation", - "azurerm_policy_set_definition", - "azurerm_dashboard", - "azurerm_powerbi_embedded", - "azurerm_private_dns_a_record", - "azurerm_private_dns_aaaa_record", - "azurerm_private_dns_cname_record", - "azurerm_private_dns_mx_record", - "azurerm_private_dns_ptr_record", - "azurerm_private_dns_srv_record", - "azurerm_private_dns_txt_record", - "azurerm_private_dns_zone", - "azurerm_private_dns_zone_virtual_network_link", - "azurerm_backup_container_storage_account", - "azurerm_backup_policy_file_share", - "azurerm_backup_policy_vm", - "azurerm_backup_protected_file_share", - "azurerm_backup_protected_vm", - "azurerm_recovery_services_vault", - "azurerm_site_recovery_fabric", - "azurerm_site_recovery_network_mapping", - "azurerm_site_recovery_protection_container", - "azurerm_site_recovery_protection_container_mapping", - "azurerm_site_recovery_replicated_vm", - "azurerm_site_recovery_replication_policy", - "azurerm_redis_cache", - "azurerm_redis_firewall_rule", - "azurerm_search_service", - "azurerm_advanced_threat_protection", - "azurerm_security_center_contact", - "azurerm_security_center_subscription_pricing", - "azurerm_security_center_workspace", - "azurerm_sentinel_alert_rule_ms_security_incident", - "azurerm_sentinel_alert_rule_scheduled", - "azurerm_service_fabric_cluster", - "azurerm_spring_cloud_app", - "azurerm_spring_cloud_service", - "azurerm_hpc_cache", - "azurerm_hpc_cache_blob_target", - "azurerm_hpc_cache_nfs_target", - "azurerm_storage_account", - "azurerm_storage_account_customer_managed_key", - "azurerm_storage_account_network_rules", - "azurerm_storage_blob", - "azurerm_storage_container", - "azurerm_storage_data_lake_gen2_filesystem", - "azurerm_storage_management_policy", - "azurerm_storage_queue", - "azurerm_storage_share", - "azurerm_storage_share_directory", - "azurerm_storage_table", - "azurerm_storage_table_entity", - "azurerm_stream_analytics_function_javascript_udf", - "azurerm_stream_analytics_job", - "azurerm_stream_analytics_output_blob", - "azurerm_stream_analytics_output_eventhub", - "azurerm_stream_analytics_output_mssql", - "azurerm_stream_analytics_output_servicebus_queue", - "azurerm_stream_analytics_output_servicebus_topic", - "azurerm_stream_analytics_reference_input_blob", - "azurerm_stream_analytics_stream_input_blob", - "azurerm_stream_analytics_stream_input_eventhub", - "azurerm_stream_analytics_stream_input_iothub", - "azurerm_synapse_workspace", - "azurerm_template_deployment", - "azurerm_iot_time_series_insights_access_policy", - "azurerm_iot_time_series_insights_reference_data_set", - "azurerm_iot_time_series_insights_standard_environment" - ] -} \ No newline at end of file From 9a5b06d07a09a10f722d951c031a7614eedf9291 Mon Sep 17 00:00:00 2001 From: Mike Urbanski Date: Thu, 6 Oct 2022 17:12:38 -0500 Subject: [PATCH 06/20] cleanup comments and code --- .../attribute_resource_types_integration.py | 4 --- checkov/common/checks_infra/checks_parser.py | 25 ++++++++++--------- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py b/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py index 86c80117f08..96a74e484f6 100644 --- a/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py +++ b/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py @@ -59,8 +59,6 @@ def get_attribute_resource_types(self, solver: Dict[str, Any], provider: Optiona return resource_types.get(provider or '__all__') def _build_attribute_resource_map(self, resource_definitions): - logging.debug(f'Start time of processing API output: {datetime.datetime.now().timestamp()}') - filter_attributes: Dict[str, List[str]] = resource_definitions.get('filterAttributes') resource_types: Dict[str, Dict[str, Any]] = resource_definitions.get('resourceTypes') @@ -81,7 +79,5 @@ def _build_attribute_resource_map(self, resource_definitions): self.attribute_resources[attribute][provider].add(resource) self.attribute_resources[attribute][ALL_TYPES].add(resource) - logging.debug(f'End time of processing API output: {datetime.datetime.now().timestamp()}') - integration = AttributeResourceTypesIntegration(bc_integration) diff --git a/checkov/common/checks_infra/checks_parser.py b/checkov/common/checks_infra/checks_parser.py index 8a51e83ef59..c10c33a60bf 100644 --- a/checkov/common/checks_infra/checks_parser.py +++ b/checkov/common/checks_infra/checks_parser.py @@ -134,7 +134,7 @@ def parse_raw_check(self, raw_check: Dict[str, Dict[str, Any]], **kwargs: Any) - metadata = raw_check.get("metadata", {}) provider = metadata.get("scope", {}).get("provider") - check = self._parse_raw_check(policy_definition, [], provider) + check = self._parse_raw_check(policy_definition, provider) check.id = metadata.get("id", "") check.name = metadata.get("name", "") @@ -148,7 +148,7 @@ def parse_raw_check(self, raw_check: Dict[str, Dict[str, Any]], **kwargs: Any) - return check - def _parse_raw_check(self, raw_check: Dict[str, Any], resources_types: Optional[List[str]], provider: Optional[str]) -> BaseGraphCheck: + def _parse_raw_check(self, raw_check: Dict[str, Any], provider: Optional[str]) -> BaseGraphCheck: check = BaseGraphCheck() complex_operator = get_complex_operator(raw_check) if complex_operator: @@ -161,18 +161,16 @@ def _parse_raw_check(self, raw_check: Dict[str, Any], resources_types: Optional[ if isinstance(sub_solvers, dict): sub_solvers = [sub_solvers] - # we handle resource types as a list (in the case where the types are enumerated) or as a set (when we - # replace a value of 'all' from the platform). sets are convenient because these lists are large, so - # faster lookups are nice, and because then we don't need to worry about duplicate (e.g., CFN Tags.Key and Tags.Value) - for sub_solver in sub_solvers: - check.sub_checks.append(self._parse_raw_check(sub_solver, resources_types, provider)) + check.sub_checks.append(self._parse_raw_check(sub_solver, provider)) + + # conditions with enumerated resource types will have them as a list. conditions where `all` is replaced with the + # actual list of resource for the attribute (e.g. tags) will have them as a set, because that logic works best with sets + # here, they will end up as a list in the policy resource types resources_types_of_sub_solvers = [ force_list_or_set(q.resource_types) for q in check.sub_checks if q is not None and q.resource_types is not None ] - if resources_types_of_sub_solvers and isinstance(resources_types_of_sub_solvers[0], list): - check.resource_types = list(set(sum(resources_types_of_sub_solvers, []))) - elif resources_types_of_sub_solvers and isinstance(resources_types_of_sub_solvers[0], set): + if resources_types_of_sub_solvers: check.resource_types = list(set().union(*resources_types_of_sub_solvers)) else: check.resource_types = [] @@ -187,13 +185,16 @@ def _parse_raw_check(self, raw_check: Dict[str, Any], resources_types: Optional[ or (isinstance(resource_type, list) and resource_type[0].lower() == "all") ): resource_types_for_attribute = attribute_resource_type_integration.get_attribute_resource_types(raw_check, provider) - check.resource_types = resource_types_for_attribute or resources_types or [] + check.resource_types = resource_types_for_attribute or [] else: check.resource_types = resource_type connected_resources_type = raw_check.get("connected_resource_types", []) + + # TODO this code has a capital 'All', so I am pretty sure this rarely gets used. need to validate the use case + # and make it work with the resource types from the platform if needed if connected_resources_type == ["All"] or connected_resources_type == "all": - check.connected_resources_types = resources_types or [] + check.connected_resources_types = [] else: check.connected_resources_types = connected_resources_type From 996011ae38b83fa28b1b8a3876c4919cdea902a4 Mon Sep 17 00:00:00 2001 From: Mike Urbanski Date: Thu, 6 Oct 2022 17:15:56 -0500 Subject: [PATCH 07/20] remove unused imports --- .../features/attribute_resource_types_integration.py | 1 - .../features/custom_policies_integration.py | 2 +- checkov/common/checks_infra/checks_parser.py | 2 +- checkov/common/checks_infra/registry.py | 2 +- 4 files changed, 3 insertions(+), 4 deletions(-) diff --git a/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py b/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py index 96a74e484f6..8d985b188ef 100644 --- a/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py +++ b/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py @@ -1,6 +1,5 @@ from __future__ import annotations -import datetime import logging from typing import TYPE_CHECKING, Any, Dict, Optional, List, Set diff --git a/checkov/common/bridgecrew/integration_features/features/custom_policies_integration.py b/checkov/common/bridgecrew/integration_features/features/custom_policies_integration.py index 3d2c16d24df..2d71ac0897c 100644 --- a/checkov/common/bridgecrew/integration_features/features/custom_policies_integration.py +++ b/checkov/common/bridgecrew/integration_features/features/custom_policies_integration.py @@ -11,7 +11,7 @@ from checkov.common.bridgecrew.platform_integration import bc_integration from checkov.common.bridgecrew.severities import Severities from checkov.common.checks_infra.checks_parser import NXGraphCheckParser -from checkov.common.checks_infra.registry import Registry, get_graph_checks_registry +from checkov.common.checks_infra.registry import get_graph_checks_registry if TYPE_CHECKING: from checkov.common.bridgecrew.platform_integration import BcPlatformIntegration diff --git a/checkov/common/checks_infra/checks_parser.py b/checkov/common/checks_infra/checks_parser.py index c10c33a60bf..c9edf10d7ef 100644 --- a/checkov/common/checks_infra/checks_parser.py +++ b/checkov/common/checks_infra/checks_parser.py @@ -51,7 +51,7 @@ from checkov.common.graph.checks_infra.base_parser import BaseGraphCheckParser from checkov.common.graph.checks_infra.enums import SolverType from checkov.common.graph.checks_infra.solvers.base_solver import BaseSolver -from checkov.common.util.type_forcers import force_list, force_list_or_set +from checkov.common.util.type_forcers import force_list_or_set if TYPE_CHECKING: from checkov.common.checks_infra.solvers.attribute_solvers.base_attribute_solver import BaseAttributeSolver diff --git a/checkov/common/checks_infra/registry.py b/checkov/common/checks_infra/registry.py index f83112eccba..a121488cf5a 100644 --- a/checkov/common/checks_infra/registry.py +++ b/checkov/common/checks_infra/registry.py @@ -4,7 +4,7 @@ import logging import os from pathlib import Path -from typing import Any, TYPE_CHECKING +from typing import TYPE_CHECKING import yaml From a61fd9cacdd0cc0feb1281806af1b39145f16804 Mon Sep 17 00:00:00 2001 From: Mike Urbanski Date: Thu, 6 Oct 2022 17:25:48 -0500 Subject: [PATCH 08/20] fix mypy? --- .../attribute_resource_types_integration.py | 4 ++-- checkov/common/util/type_forcers.py | 17 ++++++++++++++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py b/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py index 8d985b188ef..e9571e3082f 100644 --- a/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py +++ b/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py @@ -44,7 +44,7 @@ def pre_scan(self) -> None: self.integration_feature_failures = True logging.debug("Scanning without handling 'all' resource type policies.", exc_info=True) - def get_attribute_resource_types(self, solver: Dict[str, Any], provider: Optional[str] = None) -> Optional[List[str]]: + def get_attribute_resource_types(self, solver: Dict[str, Any], provider: Optional[str] = None) -> Optional[Set[str]]: attr = solver.get('attribute') if not attr: return None @@ -57,7 +57,7 @@ def get_attribute_resource_types(self, solver: Dict[str, Any], provider: Optiona return resource_types.get(provider or '__all__') - def _build_attribute_resource_map(self, resource_definitions): + def _build_attribute_resource_map(self, resource_definitions) -> None: filter_attributes: Dict[str, List[str]] = resource_definitions.get('filterAttributes') resource_types: Dict[str, Dict[str, Any]] = resource_definitions.get('resourceTypes') diff --git a/checkov/common/util/type_forcers.py b/checkov/common/util/type_forcers.py index d57a7572217..a5fcd296d66 100644 --- a/checkov/common/util/type_forcers.py +++ b/checkov/common/util/type_forcers.py @@ -26,8 +26,23 @@ def force_list(var: T | list[T]) -> list[T]: return var +@overload +def force_list_or_set(var: T) -> list[T]: + ... + + +@overload +def force_list_or_set(var: list[T]) -> list[T]: + ... + + +@overload +def force_list_or_set(var: set[T]) -> set[T]: + ... + + def force_list_or_set(var: T | list[T] | set[T]) -> list[T] | set[T]: - if isinstance(var, list) or isinstance(var, set): + if isinstance(var, (list, set)): return var return [var] From b3859e17c34e413b73341397a2c9a2656e4ea4e6 Mon Sep 17 00:00:00 2001 From: Mike Urbanski Date: Thu, 6 Oct 2022 17:38:34 -0500 Subject: [PATCH 09/20] fix mypy! --- .../features/attribute_resource_types_integration.py | 9 +++++---- checkov/common/typing.py | 11 +++++++++++ checkov/common/util/type_forcers.py | 6 +++--- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py b/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py index e9571e3082f..a6c309e9a67 100644 --- a/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py +++ b/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py @@ -1,10 +1,11 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, Dict, Optional, List, Set +from typing import TYPE_CHECKING, Any, Dict, Optional, List, Set, Union from checkov.common.bridgecrew.integration_features.base_integration_feature import BaseIntegrationFeature from checkov.common.bridgecrew.platform_integration import bc_integration +from checkov.common.typing import _ResourceDefinitions, _ResourceTypes if TYPE_CHECKING: from checkov.common.bridgecrew.platform_integration import BcPlatformIntegration @@ -57,9 +58,9 @@ def get_attribute_resource_types(self, solver: Dict[str, Any], provider: Optiona return resource_types.get(provider or '__all__') - def _build_attribute_resource_map(self, resource_definitions) -> None: - filter_attributes: Dict[str, List[str]] = resource_definitions.get('filterAttributes') - resource_types: Dict[str, Dict[str, Any]] = resource_definitions.get('resourceTypes') + def _build_attribute_resource_map(self, resource_definitions: _ResourceDefinitions) -> None: + filter_attributes: Dict[str, List[str]] = resource_definitions['filterAttributes'] + resource_types: Dict[str, _ResourceTypes] = resource_definitions['resourceTypes'] for attribute, providers in filter_attributes.items(): self.attribute_resources[attribute] = {p: set() for p in providers} diff --git a/checkov/common/typing.py b/checkov/common/typing.py index 82591066523..a82ff2244d2 100644 --- a/checkov/common/typing.py +++ b/checkov/common/typing.py @@ -70,3 +70,14 @@ class _LicenseStatus(TypedDict): policy: str license: str status: str + + +class _ResourceDefinitions(TypedDict): + filterAttributes: dict[str, list[str]] + resourceTypes: dict[str, _ResourceTypes] + + +class _ResourceTypes(TypedDict): + provider: str + text: str + argument: list[str] \ No newline at end of file diff --git a/checkov/common/util/type_forcers.py b/checkov/common/util/type_forcers.py index a5fcd296d66..1d180ed8d03 100644 --- a/checkov/common/util/type_forcers.py +++ b/checkov/common/util/type_forcers.py @@ -27,17 +27,17 @@ def force_list(var: T | list[T]) -> list[T]: @overload -def force_list_or_set(var: T) -> list[T]: +def force_list_or_set(var: list[T]) -> list[T] | set[T]: ... @overload -def force_list_or_set(var: list[T]) -> list[T]: +def force_list_or_set(var: set[T]) -> list[T] | set[T]: ... @overload -def force_list_or_set(var: set[T]) -> set[T]: +def force_list_or_set(var: T) -> list[T] | set[T]: ... From fb324b5cabeeb2838dc3405bba438ccb355de188 Mon Sep 17 00:00:00 2001 From: Mike Urbanski Date: Thu, 6 Oct 2022 17:45:11 -0500 Subject: [PATCH 10/20] make resource types list or set --- .../features/attribute_resource_types_integration.py | 2 +- checkov/common/graph/checks_infra/base_check.py | 5 ++--- checkov/common/typing.py | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py b/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py index a6c309e9a67..3f61b55d933 100644 --- a/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py +++ b/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py @@ -45,7 +45,7 @@ def pre_scan(self) -> None: self.integration_feature_failures = True logging.debug("Scanning without handling 'all' resource type policies.", exc_info=True) - def get_attribute_resource_types(self, solver: Dict[str, Any], provider: Optional[str] = None) -> Optional[Set[str]]: + def get_attribute_resource_types(self, solver: Dict[str, Any], provider: Optional[str] = None) -> Optional[List[str]]: attr = solver.get('attribute') if not attr: return None diff --git a/checkov/common/graph/checks_infra/base_check.py b/checkov/common/graph/checks_infra/base_check.py index c4d50b9c5f1..032414ebcf1 100644 --- a/checkov/common/graph/checks_infra/base_check.py +++ b/checkov/common/graph/checks_infra/base_check.py @@ -1,8 +1,7 @@ from __future__ import annotations import itertools -from typing import Optional, Tuple, List, Dict, Any, TYPE_CHECKING - +from typing import Optional, Tuple, List, Dict, Any, TYPE_CHECKING, Set from checkov.common.graph.checks_infra.enums import SolverType from checkov.common.graph.checks_infra.solvers.base_solver import BaseSolver @@ -18,7 +17,7 @@ def __init__(self) -> None: self.bc_id = None self.name = "" self.category = "" - self.resource_types: List[str] = [] + self.resource_types: Set[str] | List[str] = [] # when `all` is replaced by a long list of resources, a set is better self.connected_resources_types: List[str] = [] self.operator = "" self.attribute: Optional[str] = None diff --git a/checkov/common/typing.py b/checkov/common/typing.py index a82ff2244d2..58a2e51687e 100644 --- a/checkov/common/typing.py +++ b/checkov/common/typing.py @@ -80,4 +80,4 @@ class _ResourceDefinitions(TypedDict): class _ResourceTypes(TypedDict): provider: str text: str - argument: list[str] \ No newline at end of file + arguments: list[str] From 82eb8cfbd6bd194cc71ea6640776fa4f5c94be78 Mon Sep 17 00:00:00 2001 From: Mike Urbanski Date: Thu, 6 Oct 2022 17:45:41 -0500 Subject: [PATCH 11/20] make resource types list or set --- .../features/attribute_resource_types_integration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py b/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py index 3f61b55d933..8c3bec25784 100644 --- a/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py +++ b/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, Dict, Optional, List, Set, Union +from typing import TYPE_CHECKING, Any, Dict, Optional, List, Set from checkov.common.bridgecrew.integration_features.base_integration_feature import BaseIntegrationFeature from checkov.common.bridgecrew.platform_integration import bc_integration From 7e1052a67f9608eb2cfdd7757aca228eb59634ca Mon Sep 17 00:00:00 2001 From: Mike Urbanski Date: Thu, 6 Oct 2022 17:57:34 -0500 Subject: [PATCH 12/20] change resource types to set --- .../attribute_resource_types_integration.py | 2 +- checkov/common/checks_infra/checks_parser.py | 6 +++--- .../attribute_solvers/base_attribute_solver.py | 4 ++-- checkov/common/graph/checks_infra/base_check.py | 2 +- .../graph/checks_infra/solvers/base_solver.py | 4 ++-- checkov/common/util/type_forcers.py | 14 ++++++++------ 6 files changed, 17 insertions(+), 15 deletions(-) diff --git a/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py b/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py index 8c3bec25784..c920a606ed4 100644 --- a/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py +++ b/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py @@ -45,7 +45,7 @@ def pre_scan(self) -> None: self.integration_feature_failures = True logging.debug("Scanning without handling 'all' resource type policies.", exc_info=True) - def get_attribute_resource_types(self, solver: Dict[str, Any], provider: Optional[str] = None) -> Optional[List[str]]: + def get_attribute_resource_types(self, solver: Dict[str, Any], provider: Optional[str] = None) -> Optional[Set[str]]: attr = solver.get('attribute') if not attr: return None diff --git a/checkov/common/checks_infra/checks_parser.py b/checkov/common/checks_infra/checks_parser.py index c9edf10d7ef..8ad6254b9fb 100644 --- a/checkov/common/checks_infra/checks_parser.py +++ b/checkov/common/checks_infra/checks_parser.py @@ -51,7 +51,7 @@ from checkov.common.graph.checks_infra.base_parser import BaseGraphCheckParser from checkov.common.graph.checks_infra.enums import SolverType from checkov.common.graph.checks_infra.solvers.base_solver import BaseSolver -from checkov.common.util.type_forcers import force_list_or_set +from checkov.common.util.type_forcers import force_set if TYPE_CHECKING: from checkov.common.checks_infra.solvers.attribute_solvers.base_attribute_solver import BaseAttributeSolver @@ -168,12 +168,12 @@ def _parse_raw_check(self, raw_check: Dict[str, Any], provider: Optional[str]) - # actual list of resource for the attribute (e.g. tags) will have them as a set, because that logic works best with sets # here, they will end up as a list in the policy resource types resources_types_of_sub_solvers = [ - force_list_or_set(q.resource_types) for q in check.sub_checks if q is not None and q.resource_types is not None + force_set(q.resource_types) for q in check.sub_checks if q is not None and q.resource_types is not None ] if resources_types_of_sub_solvers: check.resource_types = list(set().union(*resources_types_of_sub_solvers)) else: - check.resource_types = [] + check.resource_types = {} if any(q.type in [SolverType.CONNECTION, SolverType.COMPLEX_CONNECTION] for q in check.sub_checks): check.type = SolverType.COMPLEX_CONNECTION diff --git a/checkov/common/checks_infra/solvers/attribute_solvers/base_attribute_solver.py b/checkov/common/checks_infra/solvers/attribute_solvers/base_attribute_solver.py index 00506368ed3..448777f8e93 100644 --- a/checkov/common/checks_infra/solvers/attribute_solvers/base_attribute_solver.py +++ b/checkov/common/checks_infra/solvers/attribute_solvers/base_attribute_solver.py @@ -2,7 +2,7 @@ import concurrent.futures import re -from typing import List, Tuple, Dict, Any, Optional, Pattern, TYPE_CHECKING +from typing import List, Tuple, Dict, Any, Optional, Pattern, TYPE_CHECKING, Set from jsonpath_ng.ext import parse @@ -31,7 +31,7 @@ class BaseAttributeSolver(BaseSolver): operator = "" # noqa: CCE003 # a static attribute - def __init__(self, resource_types: List[str], attribute: Optional[str], value: Any, is_jsonpath_check: bool = False + def __init__(self, resource_types: Set[str], attribute: Optional[str], value: Any, is_jsonpath_check: bool = False ) -> None: super().__init__(SolverType.ATTRIBUTE) self.resource_types = resource_types diff --git a/checkov/common/graph/checks_infra/base_check.py b/checkov/common/graph/checks_infra/base_check.py index 032414ebcf1..18fd2b207cc 100644 --- a/checkov/common/graph/checks_infra/base_check.py +++ b/checkov/common/graph/checks_infra/base_check.py @@ -17,7 +17,7 @@ def __init__(self) -> None: self.bc_id = None self.name = "" self.category = "" - self.resource_types: Set[str] | List[str] = [] # when `all` is replaced by a long list of resources, a set is better + self.resource_types: Set[str] = set() # when `all` is replaced by a long list of resources, a set is better self.connected_resources_types: List[str] = [] self.operator = "" self.attribute: Optional[str] = None diff --git a/checkov/common/graph/checks_infra/solvers/base_solver.py b/checkov/common/graph/checks_infra/solvers/base_solver.py index e1028bba027..00d4c98bff5 100644 --- a/checkov/common/graph/checks_infra/solvers/base_solver.py +++ b/checkov/common/graph/checks_infra/solvers/base_solver.py @@ -1,7 +1,7 @@ from __future__ import annotations from abc import abstractmethod -from typing import Tuple, List, Dict, Any, TYPE_CHECKING +from typing import Tuple, List, Dict, Any, TYPE_CHECKING, Set from checkov.common.graph.checks_infra.enums import SolverType @@ -28,5 +28,5 @@ def run(self, graph_connector: DiGraph) -> Tuple[List[Dict[str, Any]], List[Dict raise NotImplementedError() @staticmethod - def resource_type_pred(v: Dict[str, Any], resource_types: List[str]) -> bool: + def resource_type_pred(v: Dict[str, Any], resource_types: Set[str]) -> bool: return not resource_types or ("resource_type" in v and v["resource_type"] in resource_types) diff --git a/checkov/common/util/type_forcers.py b/checkov/common/util/type_forcers.py index 1d180ed8d03..100596193bf 100644 --- a/checkov/common/util/type_forcers.py +++ b/checkov/common/util/type_forcers.py @@ -27,24 +27,26 @@ def force_list(var: T | list[T]) -> list[T]: @overload -def force_list_or_set(var: list[T]) -> list[T] | set[T]: +def force_set(var: set[T]) -> set[T]: ... @overload -def force_list_or_set(var: set[T]) -> list[T] | set[T]: +def force_set(var: list[T]) -> set[T]: ... @overload -def force_list_or_set(var: T) -> list[T] | set[T]: +def force_set(var: T) -> set[T]: ... -def force_list_or_set(var: T | list[T] | set[T]) -> list[T] | set[T]: - if isinstance(var, (list, set)): +def force_set(var: T | list[T] | set[T]) -> set[T]: + if isinstance(var, set): return var - return [var] + if isinstance(var, list): + return set(var) + return {var} def force_int(var: Any) -> int | None: From 4673e9fe61e1f772c0a307af19002dc4c71ff51e Mon Sep 17 00:00:00 2001 From: Mike Urbanski Date: Fri, 7 Oct 2022 09:20:22 -0500 Subject: [PATCH 13/20] resource types as list --- .../attribute_resource_types_integration.py | 23 ++++++++++++++----- checkov/common/checks_infra/checks_parser.py | 9 +++----- .../base_attribute_solver.py | 4 ++-- .../common/graph/checks_infra/base_check.py | 4 ++-- .../graph/checks_infra/solvers/base_solver.py | 4 ++-- checkov/common/util/type_forcers.py | 23 ------------------- ...est_attribute_resource_type_integration.py | 10 +++++++- 7 files changed, 35 insertions(+), 42 deletions(-) diff --git a/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py b/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py index c920a606ed4..7ebc271f663 100644 --- a/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py +++ b/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py @@ -17,7 +17,7 @@ class AttributeResourceTypesIntegration(BaseIntegrationFeature): def __init__(self, bc_integration: BcPlatformIntegration) -> None: super().__init__(bc_integration=bc_integration, order=1) # must be after policy metadata - self.attribute_resources: Dict[str, Dict[str, Set[str]]] = {} + self.attribute_resources: Dict[str, Dict[str, List[str]]] = {} def is_valid(self) -> bool: return ( @@ -45,7 +45,7 @@ def pre_scan(self) -> None: self.integration_feature_failures = True logging.debug("Scanning without handling 'all' resource type policies.", exc_info=True) - def get_attribute_resource_types(self, solver: Dict[str, Any], provider: Optional[str] = None) -> Optional[Set[str]]: + def get_attribute_resource_types(self, solver: Dict[str, Any], provider: Optional[str] = None) -> Optional[List[str]]: attr = solver.get('attribute') if not attr: return None @@ -62,9 +62,11 @@ def _build_attribute_resource_map(self, resource_definitions: _ResourceDefinitio filter_attributes: Dict[str, List[str]] = resource_definitions['filterAttributes'] resource_types: Dict[str, _ResourceTypes] = resource_definitions['resourceTypes'] + attribute_resources: Dict[str, Dict[str, Set[str]]] = {} + for attribute, providers in filter_attributes.items(): - self.attribute_resources[attribute] = {p: set() for p in providers} - self.attribute_resources[attribute][ALL_TYPES] = set() + attribute_resources[attribute] = {p: set() for p in providers} + attribute_resources[attribute][ALL_TYPES] = set() for resource, properties in resource_types.items(): provider = properties['provider'].lower() @@ -76,8 +78,17 @@ def _build_attribute_resource_map(self, resource_definitions: _ResourceDefinitio attribute = attribute[:attribute.index('.')] if attribute not in filter_attributes or provider not in filter_attributes[attribute]: continue - self.attribute_resources[attribute][provider].add(resource) - self.attribute_resources[attribute][ALL_TYPES].add(resource) + attribute_resources[attribute][provider].add(resource) + attribute_resources[attribute][ALL_TYPES].add(resource) + + # convert to list + self.attribute_resources = { + attribute: { + provider: list(resources) for provider, resources in provider_map.items() + } for attribute, provider_map in attribute_resources.items() + } + + print(1) integration = AttributeResourceTypesIntegration(bc_integration) diff --git a/checkov/common/checks_infra/checks_parser.py b/checkov/common/checks_infra/checks_parser.py index 8ad6254b9fb..8cb97f20df1 100644 --- a/checkov/common/checks_infra/checks_parser.py +++ b/checkov/common/checks_infra/checks_parser.py @@ -51,7 +51,7 @@ from checkov.common.graph.checks_infra.base_parser import BaseGraphCheckParser from checkov.common.graph.checks_infra.enums import SolverType from checkov.common.graph.checks_infra.solvers.base_solver import BaseSolver -from checkov.common.util.type_forcers import force_set +from checkov.common.util.type_forcers import force_list if TYPE_CHECKING: from checkov.common.checks_infra.solvers.attribute_solvers.base_attribute_solver import BaseAttributeSolver @@ -168,12 +168,9 @@ def _parse_raw_check(self, raw_check: Dict[str, Any], provider: Optional[str]) - # actual list of resource for the attribute (e.g. tags) will have them as a set, because that logic works best with sets # here, they will end up as a list in the policy resource types resources_types_of_sub_solvers = [ - force_set(q.resource_types) for q in check.sub_checks if q is not None and q.resource_types is not None + force_list(q.resource_types) for q in check.sub_checks if q is not None and q.resource_types is not None ] - if resources_types_of_sub_solvers: - check.resource_types = list(set().union(*resources_types_of_sub_solvers)) - else: - check.resource_types = {} + check.resource_types = list(set(sum(resources_types_of_sub_solvers, []))) if any(q.type in [SolverType.CONNECTION, SolverType.COMPLEX_CONNECTION] for q in check.sub_checks): check.type = SolverType.COMPLEX_CONNECTION diff --git a/checkov/common/checks_infra/solvers/attribute_solvers/base_attribute_solver.py b/checkov/common/checks_infra/solvers/attribute_solvers/base_attribute_solver.py index 448777f8e93..00506368ed3 100644 --- a/checkov/common/checks_infra/solvers/attribute_solvers/base_attribute_solver.py +++ b/checkov/common/checks_infra/solvers/attribute_solvers/base_attribute_solver.py @@ -2,7 +2,7 @@ import concurrent.futures import re -from typing import List, Tuple, Dict, Any, Optional, Pattern, TYPE_CHECKING, Set +from typing import List, Tuple, Dict, Any, Optional, Pattern, TYPE_CHECKING from jsonpath_ng.ext import parse @@ -31,7 +31,7 @@ class BaseAttributeSolver(BaseSolver): operator = "" # noqa: CCE003 # a static attribute - def __init__(self, resource_types: Set[str], attribute: Optional[str], value: Any, is_jsonpath_check: bool = False + def __init__(self, resource_types: List[str], attribute: Optional[str], value: Any, is_jsonpath_check: bool = False ) -> None: super().__init__(SolverType.ATTRIBUTE) self.resource_types = resource_types diff --git a/checkov/common/graph/checks_infra/base_check.py b/checkov/common/graph/checks_infra/base_check.py index 18fd2b207cc..926ef1e6bcc 100644 --- a/checkov/common/graph/checks_infra/base_check.py +++ b/checkov/common/graph/checks_infra/base_check.py @@ -1,7 +1,7 @@ from __future__ import annotations import itertools -from typing import Optional, Tuple, List, Dict, Any, TYPE_CHECKING, Set +from typing import Optional, Tuple, List, Dict, Any, TYPE_CHECKING from checkov.common.graph.checks_infra.enums import SolverType from checkov.common.graph.checks_infra.solvers.base_solver import BaseSolver @@ -17,7 +17,7 @@ def __init__(self) -> None: self.bc_id = None self.name = "" self.category = "" - self.resource_types: Set[str] = set() # when `all` is replaced by a long list of resources, a set is better + self.resource_types: List[str] = [] self.connected_resources_types: List[str] = [] self.operator = "" self.attribute: Optional[str] = None diff --git a/checkov/common/graph/checks_infra/solvers/base_solver.py b/checkov/common/graph/checks_infra/solvers/base_solver.py index 00d4c98bff5..e1028bba027 100644 --- a/checkov/common/graph/checks_infra/solvers/base_solver.py +++ b/checkov/common/graph/checks_infra/solvers/base_solver.py @@ -1,7 +1,7 @@ from __future__ import annotations from abc import abstractmethod -from typing import Tuple, List, Dict, Any, TYPE_CHECKING, Set +from typing import Tuple, List, Dict, Any, TYPE_CHECKING from checkov.common.graph.checks_infra.enums import SolverType @@ -28,5 +28,5 @@ def run(self, graph_connector: DiGraph) -> Tuple[List[Dict[str, Any]], List[Dict raise NotImplementedError() @staticmethod - def resource_type_pred(v: Dict[str, Any], resource_types: Set[str]) -> bool: + def resource_type_pred(v: Dict[str, Any], resource_types: List[str]) -> bool: return not resource_types or ("resource_type" in v and v["resource_type"] in resource_types) diff --git a/checkov/common/util/type_forcers.py b/checkov/common/util/type_forcers.py index 100596193bf..ca44e2cbdf3 100644 --- a/checkov/common/util/type_forcers.py +++ b/checkov/common/util/type_forcers.py @@ -26,29 +26,6 @@ def force_list(var: T | list[T]) -> list[T]: return var -@overload -def force_set(var: set[T]) -> set[T]: - ... - - -@overload -def force_set(var: list[T]) -> set[T]: - ... - - -@overload -def force_set(var: T) -> set[T]: - ... - - -def force_set(var: T | list[T] | set[T]) -> set[T]: - if isinstance(var, set): - return var - if isinstance(var, list): - return set(var) - return {var} - - def force_int(var: Any) -> int | None: try: if not isinstance(var, int): diff --git a/tests/common/integration_features/test_attribute_resource_type_integration.py b/tests/common/integration_features/test_attribute_resource_type_integration.py index d9e81fdf60a..88a575f0817 100644 --- a/tests/common/integration_features/test_attribute_resource_type_integration.py +++ b/tests/common/integration_features/test_attribute_resource_type_integration.py @@ -83,6 +83,14 @@ def test_get_attribute_resource_types(self): def test_build_resource_definitions(self): attr_res_integration = AttributeResourceTypesIntegration(bc_integration) attr_res_integration._build_attribute_resource_map(mock_resource_definition_response()) + + # do equality check as set + attribute_resources = { + attribute: { + provider: set(resources) for provider, resources in provider_map.items() + } for attribute, provider_map in attr_res_integration.attribute_resources.items() + } + expected = { 'tags': { 'aws': {'aws_s3_bucket'}, @@ -99,7 +107,7 @@ def test_build_resource_definitions(self): '__all__': {'google_bigquery_dataset'} } } - self.assertEqual(attr_res_integration.attribute_resources, expected) + self.assertEqual(attribute_resources, expected) def test_scan_with_attribute(self): temp_integration = BcPlatformIntegration() From 8bf6a33f91eac01abd4cd999562e5ccbd1d6bc42 Mon Sep 17 00:00:00 2001 From: Mike Urbanski Date: Mon, 24 Oct 2022 14:26:54 -0500 Subject: [PATCH 14/20] handle custom policies injected by platform --- .../features/attribute_resource_types_integration.py | 2 -- checkov/common/checks_infra/checks_parser.py | 10 +++++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py b/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py index 7ebc271f663..282953c3b91 100644 --- a/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py +++ b/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py @@ -88,7 +88,5 @@ def _build_attribute_resource_map(self, resource_definitions: _ResourceDefinitio } for attribute, provider_map in attribute_resources.items() } - print(1) - integration = AttributeResourceTypesIntegration(bc_integration) diff --git a/checkov/common/checks_infra/checks_parser.py b/checkov/common/checks_infra/checks_parser.py index 8cb97f20df1..438c681b67f 100644 --- a/checkov/common/checks_infra/checks_parser.py +++ b/checkov/common/checks_infra/checks_parser.py @@ -132,7 +132,15 @@ def parse_raw_check(self, raw_check: Dict[str, Dict[str, Any]], **kwargs: Any) - policy_definition = raw_check.get("definition", {}) metadata = raw_check.get("metadata", {}) - provider = metadata.get("scope", {}).get("provider") + + # the first approach comes from the custom policy integration + provider = metadata.get("scope", {}).get("provider")\ + + # but the platform injects check metadata in a different way + if not provider and "scope" in raw_check: + raw_provider = raw_check["scope"].get("provider") # will be a None, an empty list, or a list with the provider + if raw_provider: + provider = raw_provider[0].lower() check = self._parse_raw_check(policy_definition, provider) From ecf22a7b134ed7d12f6d2ea107a122d0fb5c8f31 Mon Sep 17 00:00:00 2001 From: Mike Urbanski Date: Mon, 24 Oct 2022 16:57:31 -0500 Subject: [PATCH 15/20] save provider to resource mapping --- .../features/attribute_resource_types_integration.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py b/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py index 282953c3b91..84d00f3f418 100644 --- a/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py +++ b/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py @@ -18,6 +18,7 @@ class AttributeResourceTypesIntegration(BaseIntegrationFeature): def __init__(self, bc_integration: BcPlatformIntegration) -> None: super().__init__(bc_integration=bc_integration, order=1) # must be after policy metadata self.attribute_resources: Dict[str, Dict[str, List[str]]] = {} + self.provider_resources: Dict[str: List[str]] = {} def is_valid(self) -> bool: return ( @@ -73,6 +74,12 @@ def _build_attribute_resource_map(self, resource_definitions: _ResourceDefinitio if provider == 'ali': # 'alibabacloud' is the actual provider value in the custom policy, but the resource provider is just 'ali' provider = 'alibabacloud' + + if provider in self.provider_resources: + self.provider_resources[provider].append(resource) + else: + self.provider_resources[provider] = [resource] + for attribute in properties['arguments']: if '.' in attribute: attribute = attribute[:attribute.index('.')] From ea9d9e4a1fe09d7cba9bdc181ea87ee0851bceac Mon Sep 17 00:00:00 2001 From: Mike Urbanski Date: Tue, 25 Oct 2022 10:26:03 -0500 Subject: [PATCH 16/20] add abstract method shells --- .../features/attribute_resource_types_integration.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py b/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py index 84d00f3f418..ec5af8843cd 100644 --- a/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py +++ b/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py @@ -5,6 +5,7 @@ from checkov.common.bridgecrew.integration_features.base_integration_feature import BaseIntegrationFeature from checkov.common.bridgecrew.platform_integration import bc_integration +from checkov.common.output.report import Report from checkov.common.typing import _ResourceDefinitions, _ResourceTypes if TYPE_CHECKING: @@ -27,6 +28,14 @@ def is_valid(self) -> bool: and not self.integration_feature_failures ) + def pre_runner(self) -> None: + # not used + pass + + def post_runner(self, scan_reports: Report) -> None: + # not used + pass + def pre_scan(self) -> None: try: if not self.bc_integration.customer_run_config_response: From d56f1d81cf3b8e41f846c0375bb95421fe62dc83 Mon Sep 17 00:00:00 2001 From: Mike Urbanski Date: Tue, 25 Oct 2022 10:56:27 -0500 Subject: [PATCH 17/20] mypy --- .../features/attribute_resource_types_integration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py b/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py index ec5af8843cd..a54db4a50a9 100644 --- a/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py +++ b/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py @@ -19,7 +19,7 @@ class AttributeResourceTypesIntegration(BaseIntegrationFeature): def __init__(self, bc_integration: BcPlatformIntegration) -> None: super().__init__(bc_integration=bc_integration, order=1) # must be after policy metadata self.attribute_resources: Dict[str, Dict[str, List[str]]] = {} - self.provider_resources: Dict[str: List[str]] = {} + self.provider_resources: Dict[str, List[str]] = {} def is_valid(self) -> bool: return ( From 225d84e1ecffb319c33977f9786308eecfc2136d Mon Sep 17 00:00:00 2001 From: Mike Urbanski Date: Thu, 27 Oct 2022 08:39:26 -0500 Subject: [PATCH 18/20] add unique orders for integrations --- .../features/attribute_resource_types_integration.py | 2 +- .../features/custom_policies_integration.py | 2 +- .../integration_features/features/repo_config_integration.py | 2 +- .../integration_features/features/suppressions_integration.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py b/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py index a54db4a50a9..d0b25ff5212 100644 --- a/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py +++ b/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py @@ -17,7 +17,7 @@ class AttributeResourceTypesIntegration(BaseIntegrationFeature): def __init__(self, bc_integration: BcPlatformIntegration) -> None: - super().__init__(bc_integration=bc_integration, order=1) # must be after policy metadata + super().__init__(bc_integration=bc_integration, order=3) # must be after policy metadata self.attribute_resources: Dict[str, Dict[str, List[str]]] = {} self.provider_resources: Dict[str, List[str]] = {} diff --git a/checkov/common/bridgecrew/integration_features/features/custom_policies_integration.py b/checkov/common/bridgecrew/integration_features/features/custom_policies_integration.py index 40af7470ae0..ebf4c341099 100644 --- a/checkov/common/bridgecrew/integration_features/features/custom_policies_integration.py +++ b/checkov/common/bridgecrew/integration_features/features/custom_policies_integration.py @@ -24,7 +24,7 @@ class CustomPoliciesIntegration(BaseIntegrationFeature): def __init__(self, bc_integration: BcPlatformIntegration) -> None: - super().__init__(bc_integration=bc_integration, order=1) # must be after policy metadata and before suppression integration + super().__init__(bc_integration=bc_integration, order=2) # must be after policy metadata and before suppression integration self.platform_policy_parser = NXGraphCheckParser() self.bc_cloned_checks: dict[str, list[dict[str, Any]]] = defaultdict(list) diff --git a/checkov/common/bridgecrew/integration_features/features/repo_config_integration.py b/checkov/common/bridgecrew/integration_features/features/repo_config_integration.py index 977e39c2da1..62794ff9ad1 100644 --- a/checkov/common/bridgecrew/integration_features/features/repo_config_integration.py +++ b/checkov/common/bridgecrew/integration_features/features/repo_config_integration.py @@ -16,7 +16,7 @@ class RepoConfigIntegration(BaseIntegrationFeature): def __init__(self, bc_integration: BcPlatformIntegration) -> None: - super().__init__(bc_integration=bc_integration, order=0) + super().__init__(bc_integration=bc_integration, order=1) self.skip_paths: set[str] = set() self.enforcement_rule: dict[str, Any] = {} self.code_category_configs: dict[str, CodeCategoryConfiguration] = {} diff --git a/checkov/common/bridgecrew/integration_features/features/suppressions_integration.py b/checkov/common/bridgecrew/integration_features/features/suppressions_integration.py index 0654bda79e0..b748117fe7c 100644 --- a/checkov/common/bridgecrew/integration_features/features/suppressions_integration.py +++ b/checkov/common/bridgecrew/integration_features/features/suppressions_integration.py @@ -22,7 +22,7 @@ class SuppressionsIntegration(BaseIntegrationFeature): def __init__(self, bc_integration: BcPlatformIntegration) -> None: - super().__init__(bc_integration=bc_integration, order=2) # must be after the custom policies integration + super().__init__(bc_integration=bc_integration, order=4) # must be after the custom policies integration self.suppressions: dict[str, list[dict[str, Any]]] = {} self.suppressions_url = f"{self.bc_integration.api_url}/api/v1/suppressions" From b78869a295e8e5508aaa8b044c5e02bd25a4cfd0 Mon Sep 17 00:00:00 2001 From: Mike Urbanski Date: Thu, 27 Oct 2022 08:48:37 -0500 Subject: [PATCH 19/20] resolve review comments --- .../attribute_resource_types_integration.py | 51 ++++++++++++++++++- checkov/common/checks_infra/checks_parser.py | 2 +- .../base_attribute_solver.py | 3 +- 3 files changed, 53 insertions(+), 3 deletions(-) diff --git a/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py b/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py index d0b25ff5212..a16f56097b2 100644 --- a/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py +++ b/checkov/common/bridgecrew/integration_features/features/attribute_resource_types_integration.py @@ -66,9 +66,58 @@ def get_attribute_resource_types(self, solver: Dict[str, Any], provider: Optiona if not resource_types: return None - return resource_types.get(provider or '__all__') + return resource_types.get(provider or ALL_TYPES) def _build_attribute_resource_map(self, resource_definitions: _ResourceDefinitions) -> None: + """ + Builds two internal maps to be referenced during policy evaluation. + + 1. self.attribute_resources - a mapping of attributes to providers to resource types in that provider + that have the attribute. + + Example: + { + tags: { + aws: [ + aws_s3_bucket, + aws_instance, + ... + ], + azure: [ + azurerm_storage_account, + ... + ], + __all__: [ + aws_s3_bucket, + aws_instance, + ... + azurerm_storage_account, + ... + ] + }, + labels: + gcp: [ + google_sql_database_instance, + ... + ], + __all__: [...] + }, + freeform_tags: { + oci: + ...etc + } + + Later, whenever we see a policy condition with "all" resource types and one of these attributes, we can + replace the resource list with the list from the given provider, or __all__ if we do not know the provider + + 2. self.provider_resources - A mapping of providers to all resource types for that provider (irrespective of attributes) + + :param resource_definitions: returned from the platform, contains a map of resource types to their metadata + (provider and attributes), and a map of attribute names to their providers that we should substitute whenever + we see "all" resource types in a yaml policy + :return: + """ + filter_attributes: Dict[str, List[str]] = resource_definitions['filterAttributes'] resource_types: Dict[str, _ResourceTypes] = resource_definitions['resourceTypes'] diff --git a/checkov/common/checks_infra/checks_parser.py b/checkov/common/checks_infra/checks_parser.py index 289476dbeeb..101e7d7c07f 100644 --- a/checkov/common/checks_infra/checks_parser.py +++ b/checkov/common/checks_infra/checks_parser.py @@ -138,7 +138,7 @@ def parse_raw_check(self, raw_check: Dict[str, Dict[str, Any]], **kwargs: Any) - metadata = raw_check.get("metadata", {}) # the first approach comes from the custom policy integration - provider = metadata.get("scope", {}).get("provider")\ + provider = metadata.get("scope", {}).get("provider") # but the platform injects check metadata in a different way if not provider and "scope" in raw_check: diff --git a/checkov/common/checks_infra/solvers/attribute_solvers/base_attribute_solver.py b/checkov/common/checks_infra/solvers/attribute_solvers/base_attribute_solver.py index a04cf4c55cc..e721f9b0b61 100644 --- a/checkov/common/checks_infra/solvers/attribute_solvers/base_attribute_solver.py +++ b/checkov/common/checks_infra/solvers/attribute_solvers/base_attribute_solver.py @@ -1,6 +1,7 @@ from __future__ import annotations import concurrent.futures +import logging import re from typing import List, Tuple, Dict, Any, Optional, Pattern, TYPE_CHECKING @@ -88,7 +89,7 @@ def get_attribute_matches(self, vertex: Dict[str, Any]) -> List[str]: try: parsed_attr = parse(self.attribute) except Exception as e: - print(e) + logging.debug('Got an error parsing a jsonpath expression', exc_info=True) raise e self.parsed_attributes[self.attribute] = parsed_attr for match in parsed_attr.find(vertex): From feffc99544204d34d4ca658a13316fa2d5a26b72 Mon Sep 17 00:00:00 2001 From: Mike Urbanski Date: Thu, 27 Oct 2022 09:41:18 -0500 Subject: [PATCH 20/20] fix test --- tests/terraform/runner/test_runner.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/terraform/runner/test_runner.py b/tests/terraform/runner/test_runner.py index 10b61201c62..20a991ebf15 100644 --- a/tests/terraform/runner/test_runner.py +++ b/tests/terraform/runner/test_runner.py @@ -197,8 +197,11 @@ def test_runner_yaml_module_check(self): extra_checks_dir_path = current_dir / "extra_yaml_checks" runner = Runner() + # not needed for this test + runner_filter = RunnerFilter(checks=['CUSTOM_GRAPH_AWS_2']) + # when - report = runner.run(root_folder=str(tf_dir_path), external_checks_dir=[str(extra_checks_dir_path)]) + report = runner.run(root_folder=str(tf_dir_path), external_checks_dir=[str(extra_checks_dir_path)], runner_filter=runner_filter) # then summary = report.get_summary()