Skip to content

Commit

Permalink
feat(terraform): Mask secret values in Terraform plan file reports by…
Browse files Browse the repository at this point in the history 
… resource (#3868)

* add test

* update set with list[str]

* typing

* linting

* add secret based test

* aws provider multi secret test

* Update checkov/common/util/secrets.py

Co-authored-by: Anton Grübel <[email protected]>

* code improvements

* code improvements

* code improvements

* add resources to censor

* dict to Dict

* typing

Co-authored-by: Anton Grübel <[email protected]>
  • Loading branch information
@arielkru @gruebel
arielkru and gruebel authored Nov 14, 2022
1 parent be9ba97 commit 3ca2d60
Show file tree
Hide file tree
Showing 5 changed files with 165 additions and 18 deletions.
6 changes: 5 additions & 1 deletion checkov/common/typing.py
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from __future__ import annotations

from typing import TYPE_CHECKING, Any, Callable
from typing import TYPE_CHECKING, Any, Callable, Dict
from typing_extensions import TypeAlias, TypedDict

if TYPE_CHECKING:
Expand All @@ -13,6 +13,10 @@
[str, "BaseCheck", "_SkippedCheck", "dict[str, Any]", str, str, "dict[str, Any]"], None
]

_Resource: TypeAlias = str
_Attribute: TypeAlias = str
ResourceAttributesToOmit: TypeAlias = Dict[_Resource, _Attribute]


class _CheckResult(TypedDict, total=False):
result: "CheckResult" | tuple["CheckResult", dict[str, Any]]
Expand Down
36 changes: 22 additions & 14 deletions checkov/common/util/secrets.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@

if TYPE_CHECKING:
from checkov.common.checks.base_check import BaseCheck
from checkov.common.typing import _CheckResult
from checkov.common.typing import _CheckResult, ResourceAttributesToOmit
from pycep.typing import ParameterAttributes, ResourceAttributes


Expand Down Expand Up @@ -109,7 +109,7 @@ def string_has_secrets(s: str, *categories: str) -> bool:
return False


def omit_multiple_secret_values_from_line(secrets: list[str], line_text: str) -> str:
def omit_multiple_secret_values_from_line(secrets: set[str], line_text: str) -> str:
censored_line = line_text
for secret in secrets:
censored_line = omit_secret_value_from_line(secret, censored_line)
Expand All @@ -133,20 +133,28 @@ def omit_secret_value_from_line(secret: str, line_text: str) -> str:

def omit_secret_value_from_checks(check: BaseCheck, check_result: dict[str, CheckResult] | _CheckResult,
entity_code_lines: list[tuple[int, str]],
entity_config: dict[str, Any] | ParameterAttributes | ResourceAttributes) -> \
entity_config: dict[str, Any] | ParameterAttributes | ResourceAttributes,
resource_attributes_to_omit: ResourceAttributesToOmit | None = None) -> \
list[tuple[int, str]]:
secrets = set() # a set, to efficiently avoid duplicates in case the same secret is found in the following conditions
censored_code_lines = []

if CheckCategories.SECRETS in check.categories and check_result.get('result') == CheckResult.FAILED:
censored_code_lines = []
secrets = [str(secret) for key, secret in entity_config.items() if key.startswith(f'{check.id}_secret')]
if not secrets:
logging.debug(f"Secret was not saved in {check.id}, can't omit")
return entity_code_lines

for idx, line in entity_code_lines:
censored_line = omit_multiple_secret_values_from_line(secrets, line)
censored_code_lines.append((idx, censored_line))
else:
censored_code_lines = entity_code_lines
secrets.update([str(secret) for key, secret in entity_config.items() if key.startswith(f'{check.id}_secret')])

if resource_attributes_to_omit and check.entity_type in resource_attributes_to_omit and \
resource_attributes_to_omit.get(check.entity_type) in entity_config:
secret = entity_config.get(resource_attributes_to_omit.get(check.entity_type, ''), [])
if isinstance(secret, list) and secret:
secrets.add(secret[0])

if not secrets:
logging.debug(f"Secret was not saved in {check.id}, can't omit")
return entity_code_lines

for idx, line in entity_code_lines:
censored_line = omit_multiple_secret_values_from_line(secrets, line)
censored_code_lines.append((idx, censored_line))

return censored_code_lines

Expand Down
18 changes: 16 additions & 2 deletions checkov/terraform/plan_runner.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
from checkov.common.checks_infra.registry import get_graph_checks_registry
from checkov.common.graph.graph_builder.graph_components.attribute_names import CustomAttributes
from checkov.common.output.record import Record
from checkov.common.util.secrets import omit_secret_value_from_checks

from checkov.common.bridgecrew.check_type import CheckType
from checkov.common.output.report import Report
Expand All @@ -31,6 +32,14 @@
"CKV_GCP_82",
}

RESOURCE_ATTRIBUTES_TO_OMIT = {
'azurerm_key_vault_secret': 'value',
'aws_secretsmanager_secret_version': 'secret_string',
'google_kms_secret_ciphertext': 'plaintext',
'aws_ssm_parameter': 'value',
'aws_db_instance': 'password'
}


class Runner(TerraformRunner):
check_type = CheckType.TERRAFORM_PLAN # noqa: CCE003 # a static attribute
Expand Down Expand Up @@ -115,16 +124,21 @@ def run_block(self, entities,
entity_lines_range = [entity_context.get('start_line'), entity_context.get('end_line')]
entity_code_lines = entity_context.get('code_lines')
entity_address = entity_context.get('address')
_, _, entity_config = registry.extract_entity_details(entity)

results = registry.scan(scanned_file, entity, [], runner_filter, report_type=CheckType.TERRAFORM_PLAN)
for check, check_result in results.items():
if check.id in TF_LIFECYCLE_CHECK_IDS:
# can't be evaluated in TF plan
continue

censored_code_lines = omit_secret_value_from_checks(check=check,
check_result=check_result,
entity_code_lines=entity_code_lines,
entity_config=entity_config,
resource_attributes_to_omit=RESOURCE_ATTRIBUTES_TO_OMIT)
record = Record(check_id=check.id, bc_check_id=check.bc_id, check_name=check.name,
check_result=check_result,
code_block=entity_code_lines, file_path=scanned_file,
code_block=censored_code_lines, file_path=scanned_file,
file_line_range=entity_lines_range,
resource=entity_id, resource_address=entity_address, evaluations=None,
check_class=check.__class__.__module__, file_abs_path=full_file_path,
Expand Down
97 changes: 96 additions & 1 deletion tests/common/utils/conftest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,4 +55,99 @@ def scan_result_success_response() -> Dict[str, Any]:
"p8l4+R7ZCLcic6WOF3Wq5X+G2TnZ263529x9Lw54L51uGzaNZDrhKoupKXEtWsR1UOrfufqwSxflFS3KLjnd5ueu"
"anz3q7neGie2cS8HcBin/BL8U8U/AL0XOSX+jt75P82r6+RIV6DoZDXW14oKMNz5rR2TA6fr6j3WG52dFumrjvsG"
"sp7dAH12j5wbWz+sG1vfOD6+m3b/8HQd/FwVgXAAA=",
'compressionMethod': 'gzip'}
'compressionMethod': 'gzip'}


@pytest.fixture
def aws_provider_config_with_secrets():
return {
'__end_line__': 12,
'__start_line__': 7,
'access_key': ['AKIAIOSFODNN7EXAMPLE'],
'alias': ['plain_text_access_keys_provider'],
'region': ['us-west-1'],
'secret_key': ['wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'],
'CKV_AWS_41_secret_access_key': 'AKIAIOSFODNN7EXAMPLE',
'CKV_AWS_41_secret_secret_key': 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'
}


@pytest.fixture
def aws_provider_lines_with_secrets():
return [(7, 'provider "aws" {\n'),
(8, ' alias = "plain_text_access_keys_provider"\n'),
(9, ' region = "us-west-1"\n'),
(10, ' access_key = "AKIAIOSFODNN7EXAMPLE"\n'),
(11, ' secret_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"\n'),
(12, '}\n')]


@pytest.fixture
def aws_provider_lines_without_secrets():
return [(7, 'provider "aws" {\n'),
(8, ' alias = "plain_text_access_keys_provider"\n'),
(9, ' region = "us-west-1"\n'),
(10, ' access_key = "AKIAI***************"\n'),
(11, ' secret_key = "wJalrXUtnF******************************"\n'),
(12, '}\n')]


@pytest.fixture
def tfplan_resource_config_with_secrets():
return {
'content_type': [''],
'expiration_date': [None],
'id': ['https://test-123-abcdse-02.vault.azure.net/secrets/test-123-abcdse-02-primary-key/352d0b63ac873c528170cb366b570da5'],
'key_vault_id': ['/subscriptions/resourceGroups/'],
'name': ['test-123-abcdse-02-primary-key'],
'not_before_date': [None],
'resource_id': ['/subscriptions/resourceGroups/'],
'resource_versionless_id': ['/subscriptions/resourceGroups/'],
'tags': [{'__startline__': 45, '__endline__': 45, 'start_line': 44, 'end_line': 44}],
'timeouts': [None],
'value': ['IClnjeTb8fgd14LyV9m1qG0xvFfUyQY3qHq/slUIrk5='],
'version': ['123d0b12ab123c123456ab123e120bc1'],
'versionless_id': ['https://test-123-abcdse-02.vault.azure.net/secrets/test-123-abcdse-02'],
'__startline__': [35],
'__endline__': [50],
'start_line': [34],
'end_line': [49],
'__address__': 'module.test.azurerm_key_vault_secret.te_primary_key["test-123-abcdse-02"]'}


@pytest.fixture
def tfplan_resource_lines_with_secrets():
return [(35, ' {\n'),
(36, ' "content_type": "",\n'),
(37, ' "expiration_date": null,\n'),
(38, ' "id": "https://test-123-abcdse-02.vault.azure.net/secrets/test-123-abcdse-02-primary-key/352d0b63ac873c528170cb366b570da5",\n'),
(39, ' "key_vault_id": "abcd/subscriptions/123/resourceGroups/abcd",\n'),
(40, ' "name": "test-123-abcdse-02-primary-key",\n'),
(41, ' "not_before_date": null,\n'),
(42, ' "resource_id": "abcd/subscriptions/123/resourceGroups/abcd",\n'),
(43, ' "resource_versionless_id": "abcd/subscriptions/123/resourceGroups/abcd",\n'),
(44, ' "tags":\n'),
(45, ' {},\n'),
(46, ' "timeouts": null,\n'),
(47, ' "value": "IClnjeTb8fgd14LyV9m1qG0xvFfUyQY3qHq/slUIrk5=",\n'),
(48, ' "version": "123d0b12ab123c123456ab123e120bc1",\n'),
(49, ' "versionless_id": "https://test-123-abcdse-02.vault.azure.net/secrets/test-123-abcdse-02"\n')]


@pytest.fixture
def tfplan_resource_lines_without_secrets():
return [(35, ' {\n'),
(36, ' "content_type": "",\n'),
(37, ' "expiration_date": null,\n'),
(38, ' "id": "https://test-123-abcdse-02.vault.azure.net/secrets/test-123-abcdse-02-primary-key/352d0b63ac873c528170cb366b570da5",\n'),
(39, ' "key_vault_id": "abcd/subscriptions/123/resourceGroups/abcd",\n'),
(40, ' "name": "test-123-abcdse-02-primary-key",\n'),
(41, ' "not_before_date": null,\n'),
(42, ' "resource_id": "abcd/subscriptions/123/resourceGroups/abcd",\n'),
(43, ' "resource_versionless_id": "abcd/subscriptions/123/resourceGroups/abcd",\n'),
(44, ' "tags":\n'),
(45, ' {},\n'),
(46, ' "timeouts": null,\n'),
(47, ' "value": "IClnjeTb8fg*********************************",\n'),
(48, ' "version": "123d0b12ab123c123456ab123e120bc1",\n'),
(49, ' "versionless_id": "https://test-123-abcdse-02.vault.azure.net/secrets/test-123-abcdse-02"\n')]
26 changes: 26 additions & 0 deletions tests/common/utils/test_secrets_utils.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
from checkov.common.util.secrets import omit_secret_value_from_checks
from checkov.common.models.enums import CheckResult
from checkov.terraform.checks.resource.azure.SecretExpirationDate import SecretExpirationDate
from checkov.terraform.checks.provider.aws.credentials import AWSCredentials


def test_omit_secret_value_from_checks_by_attribute(tfplan_resource_lines_with_secrets, tfplan_resource_config_with_secrets,
tfplan_resource_lines_without_secrets):
check = SecretExpirationDate()
check.entity_type = 'azurerm_key_vault_secret'
check_result = {'result': CheckResult.FAILED}
resource_attributes_to_omit = {'azurerm_key_vault_secret': 'value'}

assert omit_secret_value_from_checks(check, check_result, tfplan_resource_lines_with_secrets,
tfplan_resource_config_with_secrets, resource_attributes_to_omit
) == tfplan_resource_lines_without_secrets


def test_omit_secret_value_from_checks_by_secret(aws_provider_lines_with_secrets, aws_provider_config_with_secrets,
aws_provider_lines_without_secrets):
check = AWSCredentials()
check_result = {'result': CheckResult.FAILED}

assert omit_secret_value_from_checks(check, check_result, aws_provider_lines_with_secrets,
aws_provider_config_with_secrets
) == aws_provider_lines_without_secrets

0 comments on commit 3ca2d60

Please sign in to comment.