diff --git a/.isort.cfg b/.isort.cfg index 89d17c8..2e76cfb 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,3 +1,4 @@ [settings] profile=black src_paths=cdpctl,tests +skip=scripts/update_issue_templates.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4fd28b3..4dd673a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,8 @@ repos: rev: 20.8b1 hooks: - id: black - args: [--safe, --quiet, --target-version, py36] + args: [--safe, --quiet, --target-version, py36, --exclude, scripts/update_issue_templates.py] + exclude: issues.py - repo: https://github.com/asottile/blacken-docs rev: v1.8.0 hooks: @@ -20,6 +21,7 @@ repos: - id: check-json exclude: ^.vscode/ - id: check-yaml + args: [--allow-multiple-documents] - id: requirements-txt-fixer - id: check-byte-order-marker - id: check-case-conflict @@ -32,10 +34,11 @@ repos: rev: v2.7.0 hooks: - id: validate_manifest - - repo: https://github.com/pre-commit/mirrors-autopep8 - rev: v1.5.4 - hooks: - - id: autopep8 + # - repo: https://github.com/pre-commit/mirrors-autopep8 + # rev: v1.5.4 + # hooks: + # - id: autopep8 + # args: ["--exclude scripts/update_issue_templates.py"] - repo: https://github.com/PyCQA/isort rev: 5.5.0 hooks: diff --git a/.vscode/launch.json b/.vscode/launch.json index 665680f..e6d219b 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -20,8 +20,24 @@ "validate", "infra", "--config_file", - "config.yaml" - ] + "config.yml" + ], + "justMyCode": false + }, + { + "name": "cdpctl validate infra -f json", + "type": "python", + "request": "launch", + "module": "cdpctl", + "args": [ + "validate", + "infra", + "--config_file", + "config.yml", + "-f", + "json" + ], + "justMyCode": false }, { "name": "cdpctl provision infra", @@ -32,7 +48,7 @@ "provision", "infra", "--config_file", - "config.yaml" + "config.yml" ] }, { @@ -44,6 +60,14 @@ "config", "skeleton" ] + }, + { + "name": "generate issue templates", + "type": "python", + "request": "launch", + "program": "scripts/update_issue_templates.py", + "console": "integratedTerminal" } + ] } diff --git a/.yamllint b/.yamllint index c2f877a..16e7d17 100644 --- a/.yamllint +++ b/.yamllint @@ -34,6 +34,8 @@ rules: document-start: level: error present: false + ignore: | + **/issue_templates.yml empty-lines: level: error max: 1 diff --git a/cdpctl/cli.py b/cdpctl/cli.py index df1b3bb..7a1c14f 100644 --- a/cdpctl/cli.py +++ b/cdpctl/cli.py @@ -50,15 +50,19 @@ from cdpctl.command.config import render_skeleton from cdpctl.command.validate import run_validation +SUPPORTED_OUTPUT_TYPES = ["text", "json"] + @click.group(invoke_without_command=True) +@click.option("-q", "--quiet", is_flag=True, default=False) @click.option("--debug/--no-debug", default=False) @click.option("-v", "--version", is_flag=True, default=False) @click.pass_context -def _cli(ctx, debug=False, version=False) -> None: +def _cli(ctx, debug=False, version=False, quiet=False) -> None: """Run the cli.""" ctx.ensure_object(dict) ctx.obj["DEBUG"] = debug + ctx.obj["QUIET"] = quiet if version: print_version() if ctx.invoked_subcommand is None: @@ -77,9 +81,31 @@ def _cli(ctx, debug=False, version=False) -> None: help="The config file to use. Defaults to config.yml.", type=click.Path(exists=False), ) -def validate(ctx, target: str, config_file) -> None: # pylint: disable=unused-argument +@click.option( + "-o", + "--output_file", + default="-", + help="The file to output the results to. Defaults to stdout.", + type=click.Path(exists=False), +) +@click.option( + "-f", + "--output_format", + default="text", + help="The format to output the results as.", + type=click.Choice(SUPPORTED_OUTPUT_TYPES, case_sensitive=False), +) +def validate( + ctx, target: str, config_file, output_file, output_format +) -> None: # pylint: disable=unused-argument """Run validation checks on provided section.""" - run_validation(target=target, config_file=config_file, debug=ctx.obj["DEBUG"]) + run_validation( + target=target, + config_file=config_file, + debug=ctx.obj["DEBUG"], + output_format=output_format, + output_file=output_file, + ) @click.group() diff --git a/cdpctl/command/validate.py b/cdpctl/command/validate.py index 636e375..5fa7fd7 100644 --- a/cdpctl/command/validate.py +++ b/cdpctl/command/validate.py @@ -50,11 +50,18 @@ import cdpctl.validation as validation from cdpctl import SUPPORTED_PLATFORMS from cdpctl.utils import load_config -from cdpctl.validation import UnrecoverableValidationError, conftest +from cdpctl.validation import UnrecoverableValidationError, conftest, get_issues from cdpctl.validation.aws_utils import validate_aws_config +from cdpctl.validation.renderer import get_renderer -def run_validation(target: str, config_file: str, debug: bool = False) -> None: +def run_validation( + target: str, + config_file: str, + debug: bool = False, + output_format: str = "text", + output_file: str = "-", +) -> None: """Run the validate command.""" click.echo( f"Targeting {click.style(target, fg='blue')} section with config file " @@ -113,3 +120,11 @@ def run_validation(target: str, config_file: str, debug: bool = False) -> None: options.append("-s") pytest.main(options) + + renderer = get_renderer(output_format=output_format) + renderer.render(get_issues(), output_file) + if output_file != "-": + click.echo( + message=f"Results written to file {click.format_filename(output_file)}.", + err=True, + ) diff --git a/cdpctl/validation/__init__.py b/cdpctl/validation/__init__.py index 067c2ab..dcf1feb 100644 --- a/cdpctl/validation/__init__.py +++ b/cdpctl/validation/__init__.py @@ -40,9 +40,22 @@ # Source File Name: __init__.py ### """Shared validation functions.""" -from typing import Any, Dict +import os +from collections import namedtuple +from enum import Enum +from typing import Any, Dict, List import pytest +import yaml +from marshmallow import Schema, fields, post_load + +from cdpctl.validation.issues import ( + CONFIG_OPTION_DATA_NOT_DEFINED, + CONFIG_OPTION_KEY_NOT_DEFINED, + CONFIG_OPTION_PARENT_PATH_NOT_DEFINED, +) + +ISSUE_TEMPLATES_FILE = "issue_templates.yml" class ValidationError(Exception): @@ -57,14 +70,210 @@ class UnrecoverableValidationError(ValidationError): pass +class IssueTemplate: + """Issue Templates.""" + + def __init__( + self, + template_id: str, + summary: str, + docs_link: str = None, + render_type: str = "inline", + ) -> None: + """Initialize the IssueTemplate.""" + self.id = template_id + self.summary = summary + self.docs_link = docs_link + self.render_type = render_type + + +class IssueTemplateSchema(Schema): + """Schema for Issue Templates.""" + + template_id = fields.Str(required=True, data_key="id") + summary = fields.Str(required=True) + docs_link = fields.Str(required=False) + render_type = fields.Str(default="list", required=False) + + @post_load + def make_issue_template( + self, data, **kwargs + ): # pylint: disable=unused-argument,no-self-use + """Make the IssueTemplate based on the Schema.""" + return IssueTemplate(**data) + + +class Issue: + """Issue representaiton.""" + + def __init__( + self, + template: IssueTemplate, + subjects: List[str] = None, + resources: List[str] = None, + ) -> None: + """Initialize the Issue.""" + self.validation: str = None + self._template: str = template + self._subjects = subjects + self._resources = resources + + @property + def message(self): + """Get the message.""" + return self._template.summary.format(*self._subjects) + + @property + def resources(self) -> List[str]: + """Get the resources.""" + if not self._resources: + return [] + return self._resources + + @property + def docs_link(self): + """Get the document link.""" + return self._template.docs_link + + @property + def render_type(self): + """Get the render type.""" + return self._template.render_type + + +class IssueType(Enum): + """Issue Types enum.""" + + PROBLEM = "problem" + WARNING = "warning" + + +def load_issue_templates(path): + """Get the IssueTemplates found in a path.""" + templates = [] + schema = IssueTemplateSchema(many=True) + with open(os.path.join(path)) as input_file: + temp_gen = yaml.load_all(input_file, Loader=yaml.FullLoader) + templates = schema.load(temp_gen) + return templates + + +def load_all_issue_templates(): + """Get all of the IssueTemplates found.""" + issue_templates = {} + for root, _, files in os.walk(os.path.dirname(__file__)): + if ISSUE_TEMPLATES_FILE in files: + loading_templates = load_issue_templates( + os.path.join(root, ISSUE_TEMPLATES_FILE) + ) + for template in loading_templates: + issue_templates[template.id] = template + return issue_templates + + +# List of issues found during the validation run +_issues: Dict[str, Dict[str, List[Issue]]] = {} + + +def get_issues() -> Dict[str, Dict[str, List[Issue]]]: + """Get the Issues found during the validation run.""" + return _issues + + +_issue_templates: Dict[str, IssueTemplate] = load_all_issue_templates() + + +ValidationContext = namedtuple("CurrentValidationContext", "validation_name") + + +class Context: + """Basic Validation Context.""" + + def __init__(self) -> None: + """Initialize the Context.""" + self.validation_name = None + self.function = None + self.nodeid = None + + def clear(self) -> None: + """Clear all the context values.""" + self.validation_name = None + self.function = None + self.nodeid = None + + +current_context: Context = Context() + + +def validator(func): + """Wrap a validator function to handle errors better.""" + + def wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except Exception as e: + if isinstance(e, UnrecoverableValidationError): + raise e + raise UnrecoverableValidationError("Unhandled exception:", e) from e + + return wrapper + + +def _add_issue(issue_type: IssueType, issue: Issue): + """Add an Issue to the selected type for the current validation.""" + context = current_context + if context.validation_name not in get_issues(): + get_issues()[context.validation_name] = { + IssueType.PROBLEM.value: [], + IssueType.WARNING.value: [], + } + get_issues()[context.validation_name][issue_type.value].append(issue) + + +def fail( + template: str, subjects: List[str] = None, resources: List[str] = None +) -> None: + """Fail the validation.""" + if template not in _issue_templates: + raise UnrecoverableValidationError(f"Unable to find issue template {template}") + + subjects = [subjects] if isinstance(subjects, str) else subjects + resources = [resources] if isinstance(resources, str) else resources + + _add_issue( + IssueType.PROBLEM, + Issue( + template=_issue_templates[template], subjects=subjects, resources=resources + ), + ) + raise pytest.fail("", False) + + +def warn( + template: str, subjects: List[str] = None, resources: List[str] = None +) -> None: + """Issue a warning for the validation.""" + if template not in _issue_templates: + raise UnrecoverableValidationError(f"Unable to find issue template {template}") + + subjects = [subjects] if isinstance(subjects, str) else subjects + resources = [resources] if isinstance(resources, str) else resources + _add_issue( + IssueType.WARNING, + Issue( + template=_issue_templates[template], subjects=subjects, resources=resources + ), + ) + + def get_config_value( config: Dict[str, Any], key: str, key_value_expected: bool = True, path_delimiter: str = ":", - key_missing_message: str = "Unable to find key in config {0}", - data_expected_error_message: str = "No entry was provided for config option {0}", - parent_key_missing_message: str = "Unable to find key path {0}", + key_not_found_issue: str = CONFIG_OPTION_KEY_NOT_DEFINED, + data_expected_issue: str = CONFIG_OPTION_DATA_NOT_DEFINED, + parent_key_missing_issue: str = CONFIG_OPTION_PARENT_PATH_NOT_DEFINED, ) -> Any: """Get the value of a config key or have the proper error handling.""" paths = key.split(path_delimiter) @@ -81,25 +290,11 @@ def get_config_value( data = data[paths[i]] except KeyError: if key_found: - pytest.fail(key_missing_message.format(path_found), False) + fail(template=key_not_found_issue, subjects=path_found) else: - pytest.fail(parent_key_missing_message.format(path_found), False) + fail(template=parent_key_missing_issue, subjects=path_found) if key_value_expected and data is None: - pytest.fail(data_expected_error_message.format(key), False) + fail(template=data_expected_issue, subjects=key) return data - - -def validator(func): - """Wrap a validator function to handle errors better.""" - - def wrapper(*args, **kwargs): - try: - return func(*args, **kwargs) - except Exception as e: - if isinstance(e, UnrecoverableValidationError): - raise e - raise UnrecoverableValidationError("Unhandled exception:", e) from e - - return wrapper diff --git a/cdpctl/validation/aws_utils.py b/cdpctl/validation/aws_utils.py index 16be62e..41c0d56 100644 --- a/cdpctl/validation/aws_utils.py +++ b/cdpctl/validation/aws_utils.py @@ -44,11 +44,19 @@ from typing import Dict, List, Optional import boto3 -import pytest from boto3_type_annotations.iam import Client as IAMClient from botocore.exceptions import ClientError, ProfileNotFound -from cdpctl.validation import UnrecoverableValidationError, get_config_value +from cdpctl.validation import UnrecoverableValidationError, fail, get_config_value +from cdpctl.validation.issues import ( + AWS_INSTANCE_PROFILE_NOT_FOUND, + AWS_MISSING_ACTIONS, + AWS_PROFILE_CONFIG_NOT_DEFINED, + AWS_PROFILE_NOT_DEFINED, + AWS_REGION_CONFIG_NOT_DEFINED, + AWS_REGION_NOT_DEFINED, + AWS_ROLE_MISSING, +) def get_client(client_type: str, config): @@ -63,32 +71,25 @@ def get_client(client_type: str, config): config, "infra:aws:profile", key_value_expected=False, - key_missing_message=( - "No profile has been defined for config option: env:aws:profile" - ), - data_expected_error_message=( - "No profile was provided for config option: env:aws:profile" - ), + key_not_found_issue=AWS_PROFILE_CONFIG_NOT_DEFINED, + data_expected_issue=AWS_PROFILE_NOT_DEFINED, ) region_name: Optional[str] = get_config_value( config, "infra:aws:region", key_value_expected=False, - key_missing_message=( - "No region has been defined for config option: env:aws:region" - ), - data_expected_error_message=( - "No region was provided for config option: env:aws:region" - ), + key_not_found_issue=AWS_REGION_CONFIG_NOT_DEFINED, + data_expected_issue=AWS_REGION_NOT_DEFINED, ) - if profile_name: - session = boto3.session.Session(profile_name=profile_name) - return session.client(client_type) if region_name: + if profile_name: + session = boto3.session.Session(profile_name=profile_name) + return session.client(client_type, region_name=region_name) return boto3.client(client_type, region_name=region_name) - - raise Exception(f"Unable to create AWS client for type {client_type}") + raise UnrecoverableValidationError( + "No AWS region name has been defined for the config option infra:aws:region." + ) def parse_arn(arn: str) -> Dict[str, str]: @@ -139,7 +140,8 @@ def simulate_policy( policy_source_arn: str, resource_arns: List[str], needed_actions: List[str], - missing_actions_message: str = "The following actions are required:\n{0}", + subjects: List[str] = None, + missing_actions_issue: str = AWS_MISSING_ACTIONS, ) -> None: """ Simulate a list of actions against a resource. @@ -148,7 +150,6 @@ def simulate_policy( in missing_actions_message. The first string-formatted argument is the list of actions that were missing from the required list. """ - response = iam_client.simulate_principal_policy( PolicySourceArn=policy_source_arn, ActionNames=needed_actions, @@ -162,9 +163,8 @@ def simulate_policy( ] if len(missing_actions) > 0: - pytest.fail( - missing_actions_message.format(set(missing_actions)), - False, + fail( + template=missing_actions_issue, subjects=subjects, resources=missing_actions ) @@ -173,10 +173,7 @@ def get_instance_profile(iam_client: IAMClient, name: str) -> Dict: try: instance_profile = iam_client.get_instance_profile(InstanceProfileName=name) except iam_client.exceptions.NoSuchEntityException: - pytest.fail( - f"instance profile {name} was not found", - False, - ) + fail(AWS_INSTANCE_PROFILE_NOT_FOUND, name) except iam_client.exceptions.ServiceFailureException as e: raise Exception( "Unable to retrieve role information due to a service failure" @@ -188,7 +185,7 @@ def get_instance_profile(iam_client: IAMClient, name: str) -> Dict: def get_role( iam_client: IAMClient, role_name: str, - role_missing_message: str = "Unable to find role ({0})", + missing_issue: str = AWS_ROLE_MISSING, service_failure_message: str = "Unable to retrieve role information due to a \ service failure", ) -> Dict: @@ -198,11 +195,11 @@ def get_role( try: role = iam_client.get_role(RoleName=role_name) except iam_client.exceptions.NoSuchEntityException: - pytest.fail(role_missing_message.format(role_name), False) + fail(template=missing_issue, resources=[role_name]) except iam_client.exceptions.ServiceFailureException as e: - raise Exception(service_failure_message) from e + raise UnrecoverableValidationError(service_failure_message) from e # handling stub client error except ClientError: - pytest.fail(role_missing_message.format(role_name), False) + fail(missing_issue, role_name) return role diff --git a/cdpctl/validation/conftest.py b/cdpctl/validation/conftest.py index 888d267..960deb2 100644 --- a/cdpctl/validation/conftest.py +++ b/cdpctl/validation/conftest.py @@ -55,7 +55,7 @@ from cdpctl.utils import load_config -from . import UnrecoverableValidationError, get_config_value +from . import UnrecoverableValidationError, current_context, get_config_value this = sys.modules[__name__] this.config_file = "config.yaml" @@ -84,19 +84,23 @@ def pytest_runtest_makereport(item: Item, call: CallInfo[None]) -> TestReport: node = item.obj if call.when == "setup": # Validation is starting + current_context.clear() suf = node.__doc__.strip() if node.__doc__ else node.__name__ if result.failed: - click.echo(f"Unable to setup validation '{suf}'") + click.echo(f"Unable to setup validation '{suf}'", err=True) if result.passed: - click.echo(suf, nl=False) + current_context.validation_name = suf + current_context.function = item.name + current_context.nodeid = item.nodeid + click.echo(suf, nl=False, err=True) elif call.when == "call": # Validation was called if result.passed: # All good - click.echo(f" {emoji.emojize(':check_mark:')}") + click.echo(f" {emoji.emojize(':check_mark:')}", err=True) elif result.failed: # Catch any issues - click.echo(f" {emoji.emojize(':cross_mark:')}") - item.session.issues[item] = str(result.longrepr.chain[-1][0]) + click.echo(f" {emoji.emojize(':cross_mark:')}", err=True) + # item.session.issues[item] = str(result.longrepr.chain[-1][0]) elif call.when == "teardown": this.run_validations += 1 sys.stdout.flush() @@ -109,16 +113,17 @@ def pytest_exception_interact( ) -> None: """Catch exceptions and fail out on Unrecoverable ones.""" if isinstance(call.excinfo.value, UnrecoverableValidationError): - click.echo("") + click.echo("", err=True) suf = node.obj.__doc__.strip() if node.obj.__doc__ else node.obj.__name__ - click.secho("\n--- An Error Occured ---", fg="red") + click.secho("\n--- An Error Occured ---", fg="red", err=True) click.echo( f'An Error occured while running the "{suf}" validation.\n' + "It has the following information:\n", + err=True, ) - click.echo(f"{str(call.excinfo.value)}\n") - click.echo(f"({node.nodeid})") - click.secho("-------------", fg="red") + click.echo(f"{str(call.excinfo.value)}\n", err=True) + click.echo(f"({node.nodeid})", err=True) + click.secho("-------------", fg="red", err=True) pytest.exit(1) @@ -128,17 +133,7 @@ def pytest_sessionfinish( ) -> None: """Finish the validation session.""" if session.exitstatus != ExitCode.INTERRUPTED: - # time.sleep(0.1) # letting progress bar finish click.echo("") - if session.issues: - click.secho("\n--- Issues Found ---", fg="yellow") - # pylint: disable=unused-variable - for item, result in session.issues.items(): - click.echo(f"- {result}") - click.echo("\n") - click.secho("-------------------", fg="yellow") - else: - click.secho("No Issues Found", fg="green") def pytest_report_teststatus( diff --git a/cdpctl/validation/infra/__init__.py b/cdpctl/validation/infra/__init__.py index 79815ab..e08ab24 100644 --- a/cdpctl/validation/infra/__init__.py +++ b/cdpctl/validation/infra/__init__.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 ### # CLOUDERA CDP Control (cdpctl) # @@ -39,5 +38,5 @@ # # Source File Name: __init__.py ### -"""This is empty to fix package for VSCode.""" +"""Base infa package.""" sections = ["infra"] diff --git a/cdpctl/validation/infra/issue_templates.yml b/cdpctl/validation/infra/issue_templates.yml new file mode 100644 index 0000000..254d0d1 --- /dev/null +++ b/cdpctl/validation/infra/issue_templates.yml @@ -0,0 +1,137 @@ +--- +id: AWS_CROSS_ACCOUNT_ROLE_MISSING +summary: "Unable to find the Cross-account access IAM role {0}" +docs_link: https://docs.cloudera.com/cdp/latest/requirements-aws/topics/mc-aws-req-credential.html +--- +id: AWS_ACCOUNT_ID_NOT_IN_CROSS_ACCOUNT_ROLE +summary: Account ID {0} not in Cross-account access IAM role {1} +docs_link: https://docs.cloudera.com/cdp/latest/requirements-aws/topics/mc-create-credentialrole.html +--- +id: AWS_EXTERNAL_ID_NOT_IN_CROSS_ACCOUNT_ROLE +summary: External ID {0} not in Cross-account access IAM role {1} +docs_link: https://docs.cloudera.com/cdp/latest/requirements-aws/topics/mc-create-credentialrole.html +--- +id: AWS_IDBROKER_INSTANCE_PROLFILE_NEEDS_ROLE +summary: "IDBroker instance profile {0} should contain an IDBroker role." +--- +id: AWS_IDBROKER_ROLE_NEED_EC2_TRUST_POLICY +summary: "The IDBroker role {0} should contain a trust policy for EC2" +--- +id: AWS_ROLE_FOR_DL_BUCKET_MISSING_ACTIONS +summary: "The role ({0}) requires the following actions for the Datalake S3 bucket ({1})." +render_type: list +--- +id: AWS_ROLE_REQUIRES_ACTIONS_FOR_ALL_S3_RESOURCES +summary: "The role ({0}) requires the following actions for all S3 resources" +render_type: list +--- +id: AWS_ROLE_FOR_DATA_BUCKET_MISSING_ACTIONS +summary: "The role ({0}) requires the following actions for the S3 data location ({1})." +render_type: list +--- +id: AWS_IDBROKER_ROLE_REQUIRES_ACTIONS_FOR_ALL_RESOURCES +summary: "The role ({0}) requires the following actions for resource wildcard (*)." +render_type: list +--- +id: AWS_ROLE_REQUIRES_ACTIONS_FOR_ALL_EC2_RESOURCES +summary: "The role ({0}) requires the following actions for all EC2 resources ([*])" +render_type: list +--- +id: AWS_ROLE_REQUIRES_ACTIONS_FOR_ALL_RESOURCES +summary: "The role ({0}) requires the following actions for all resources ([*])" +render_type: list +--- +id: AWS_ROLE_REQUIRES_ACTIONS_FOR_SERVICE_ROLL_RESOURCES +summary: "The role ({0}) requires the following actions for all resources (\"arn:aws:iam::*:role/aws-service-role/*\")" +render_type: list +--- +id: AWS_ROLE_REQUIRES_ACTIONS_FOR_LOG_PATH +summary: "The role ({0}) requires the following actions for the log storage path ({1}):" +render_type: list +--- +id: AWS_ROLE_REQUIRES_ACTIONS_FOR_LOG_BUCKET +summary: "The role ({0}) requires the following actions for the log storage bucket ({1}):" +render_type: list +--- +id: AWS_LOGGER_INSTANCE_PROFILE_SHOULD_CONTAIN_LOGGER_ROLE +summary: "The logger instance profile {0} set in config {1} should contain a logger role." +--- +id: AWS_LOGGER_ROLE_SHOULD_HAVE_EC2_TRUST +summary: "The logger role {0} should contain a trust policy for EC2" +--- +id: AWS_S3_BUCKET_INVALID +summary: "The s3a url {0} is invalid." +--- +id: AWS_S3_BUCKET_DOES_NOT_EXIST +summary: "S3 bucket {0} does not exist." +--- +id: AWS_S3_BUCKET_FORBIDDEN_ACCESS +summary: "S3 bucket {0} has forbidden access." +--- +id: AWS_NON_CCM_DEFAULT_SG_NEEDS_TO_ALLOW_CDP_CIDRS +summary: "The default security group {0} is missing TCP port 9443 for the following Cloudera CDP CIDRs when not using CCM (env:tunnel = false)." +render_type: list +--- +id: AWS_NON_CCM_GATEWAY_SG_MISSING_CIDRS +summary: "When not using CCM (tunnel = false), the gateway security group {0} is missing the following access:" +render_type: list +--- +id: AWS_DEFAULT_SG_NEEDS_ALLOW_ACCESS_INTERNAL_TO_VPC +summary: "Your Default Security Group {0} should allow access to all TCP and UDP ports (0-65535) internal to the VPC" +--- +id: AWS_GATEWAY_SG_NEEDS_ALLOW_ACCESS_INTERNAL_TO_VPC +summary: "Your Gateway Security Group {0} should allow access to all TCP and UDP ports (0-65535) internal to the VPC" +--- +id: AWS_SSH_KEY_ID_DOES_NOT_EXIST +summary: "SSH key id ({0}) does not exist." +--- +id: AWS_SSH_IS_INVALID +summary: "SSH Key ID ({0}) is not valid." +--- +id: AWS_NOT_ENOUGH_SUBNETS_PROVIDED +summary: "Not enough {0} Subnets provided, at least 3 subnets required." +--- +id: AWS_INVALID_SUBNET_ID +summary: The {0} Subnet ID {1} is invalid. +--- +id: AWS_REQUIRED_DATA_MISSING +summary: "Missing required data: {0}" +--- +id: AWS_INVALID_DATA +summary: "Validation Error - invalid data : {0}" +--- +id: AWS_SUBNETS_DO_NOT_EXIST +summary: "The following {0} Subnets do not exist." +render_type: list +--- +id: AWS_NOT_ENOUGH_AZ_FOR_SUBNETS +summary: "Not enough availability zones, {0} subnets should spread across at least 2 availability zones." +--- +id: AWS_SUBNETS_WITHOUT_INTERNET_GATEWAY +summary: "These {0} Subnets do not have an internet gateway(s)" +docs_link: https://docs.cloudera.com/cdp/latest/requirements-aws/topics/mc-aws-req-vpc.html +render_type: list +--- +id: AWS_SUBNETS_OR_VPC_WITHOUT_INTERNET_GATEWAY +summary: "These {0} Subnets or the VPC {1} do not have an internet gateway(s)" +docs_link: https://docs.cloudera.com/cdp/latest/requirements-aws/topics/mc-aws-req-vpc.html +render_type: list +--- +id: AWS_SUBNETS_WITHOUT_VALID_RANGE +summary: "These {0} Subnets do not have the valid required ranges" +docs_link: https://docs.cloudera.com/cdp/latest/requirements-aws/topics/mc-aws-req-vpc.html +render_type: list +--- +id: AWS_SUBNETS_MISSING_K8S_LB_TAG +summary: "These {0} Subnets do not have the nessesary 'kubernetes.io/role/elb' tag." +render_type: list +--- +id: AWS_SUBNETS_NOT_PART_OF_VPC +summary: "The following subnets are not associated with the provided VPC {0}:" +render_type: list +--- +id: AWS_DNS_SUPPORT_NOT_ENABLED_FOR_VPC +summary: "DNS support is not enabled for VPC id {0}" +--- +id: AWS_VPC_NOT_FOUND_IN_ACCOUNT +summary: "VPC ID {0} set in infra:aws:vpc:existing:vpc_id was not found in the AWS account." diff --git a/cdpctl/validation/infra/issues.py b/cdpctl/validation/infra/issues.py new file mode 100644 index 0000000..944f2c7 --- /dev/null +++ b/cdpctl/validation/infra/issues.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +### +# CLOUDERA CDP Control (cdpctl) +# +# (C) Cloudera, Inc. 2021-2021 +# All rights reserved. +# +# Applicable Open Source License: GNU AFFERO GENERAL PUBLIC LICENSE +# +# NOTE: Cloudera open source products are modular software products +# made up of hundreds of individual components, each of which was +# individually copyrighted. Each Cloudera open source product is a +# collective work under U.S. Copyright Law. Your license to use the +# collective work is as provided in your written agreement with +# Cloudera. Used apart from the collective work, this file is +# licensed for your use pursuant to the open source license +# identified above. +# +# This code is provided to you pursuant a written agreement with +# (i) Cloudera, Inc. or (ii) a third-party authorized to distribute +# this code. If you do not have a written agreement with Cloudera nor +# with an authorized and properly licensed third party, you do not +# have any rights to access nor to use this code. +# +# Absent a written agreement with Cloudera, Inc. (“Cloudera”) to the +# contrary, A) CLOUDERA PROVIDES THIS CODE TO YOU WITHOUT WARRANTIES OF ANY +# KIND; (B) CLOUDERA DISCLAIMS ANY AND ALL EXPRESS AND IMPLIED +# WARRANTIES WITH RESPECT TO THIS CODE, INCLUDING BUT NOT LIMITED TO +# IMPLIED WARRANTIES OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY AND +# FITNESS FOR A PARTICULAR PURPOSE; (C) CLOUDERA IS NOT LIABLE TO YOU, +# AND WILL NOT DEFEND, INDEMNIFY, NOR HOLD YOU HARMLESS FOR ANY CLAIMS +# ARISING FROM OR RELATED TO THE CODE; AND (D)WITH RESPECT TO YOUR EXERCISE +# OF ANY RIGHTS GRANTED TO YOU FOR THE CODE, CLOUDERA IS NOT LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, PUNITIVE OR +# CONSEQUENTIAL DAMAGES INCLUDING, BUT NOT LIMITED TO, DAMAGES +# RELATED TO LOST REVENUE, LOST PROFITS, LOSS OF INCOME, LOSS OF +# BUSINESS ADVANTAGE OR UNAVAILABILITY, OR LOSS OR CORRUPTION OF +# DATA. +# +# Source File Name: issues.py +### +# flake8: noqa +# pylint: skip-file + +# THIS FILE IS GENERATED. DO NOT UPDATE BY HAND. +# Use the update_issue_templates.py script +"""Issue Templates.""" + +AWS_CROSS_ACCOUNT_ROLE_MISSING = "AWS_CROSS_ACCOUNT_ROLE_MISSING" + +AWS_ACCOUNT_ID_NOT_IN_CROSS_ACCOUNT_ROLE = "AWS_ACCOUNT_ID_NOT_IN_CROSS_ACCOUNT_ROLE" + +AWS_EXTERNAL_ID_NOT_IN_CROSS_ACCOUNT_ROLE = "AWS_EXTERNAL_ID_NOT_IN_CROSS_ACCOUNT_ROLE" + +AWS_IDBROKER_INSTANCE_PROLFILE_NEEDS_ROLE = "AWS_IDBROKER_INSTANCE_PROLFILE_NEEDS_ROLE" + +AWS_IDBROKER_ROLE_NEED_EC2_TRUST_POLICY = "AWS_IDBROKER_ROLE_NEED_EC2_TRUST_POLICY" + +AWS_ROLE_FOR_DL_BUCKET_MISSING_ACTIONS = "AWS_ROLE_FOR_DL_BUCKET_MISSING_ACTIONS" + +AWS_ROLE_REQUIRES_ACTIONS_FOR_ALL_S3_RESOURCES = "AWS_ROLE_REQUIRES_ACTIONS_FOR_ALL_S3_RESOURCES" + +AWS_ROLE_FOR_DATA_BUCKET_MISSING_ACTIONS = "AWS_ROLE_FOR_DATA_BUCKET_MISSING_ACTIONS" + +AWS_IDBROKER_ROLE_REQUIRES_ACTIONS_FOR_ALL_RESOURCES = "AWS_IDBROKER_ROLE_REQUIRES_ACTIONS_FOR_ALL_RESOURCES" + +AWS_ROLE_REQUIRES_ACTIONS_FOR_ALL_EC2_RESOURCES = "AWS_ROLE_REQUIRES_ACTIONS_FOR_ALL_EC2_RESOURCES" + +AWS_ROLE_REQUIRES_ACTIONS_FOR_ALL_RESOURCES = "AWS_ROLE_REQUIRES_ACTIONS_FOR_ALL_RESOURCES" + +AWS_ROLE_REQUIRES_ACTIONS_FOR_SERVICE_ROLL_RESOURCES = "AWS_ROLE_REQUIRES_ACTIONS_FOR_SERVICE_ROLL_RESOURCES" + +AWS_ROLE_REQUIRES_ACTIONS_FOR_LOG_PATH = "AWS_ROLE_REQUIRES_ACTIONS_FOR_LOG_PATH" + +AWS_ROLE_REQUIRES_ACTIONS_FOR_LOG_BUCKET = "AWS_ROLE_REQUIRES_ACTIONS_FOR_LOG_BUCKET" + +AWS_LOGGER_INSTANCE_PROFILE_SHOULD_CONTAIN_LOGGER_ROLE = "AWS_LOGGER_INSTANCE_PROFILE_SHOULD_CONTAIN_LOGGER_ROLE" + +AWS_LOGGER_ROLE_SHOULD_HAVE_EC2_TRUST = "AWS_LOGGER_ROLE_SHOULD_HAVE_EC2_TRUST" + +AWS_S3_BUCKET_INVALID = "AWS_S3_BUCKET_INVALID" + +AWS_S3_BUCKET_DOES_NOT_EXIST = "AWS_S3_BUCKET_DOES_NOT_EXIST" + +AWS_S3_BUCKET_FORBIDDEN_ACCESS = "AWS_S3_BUCKET_FORBIDDEN_ACCESS" + +AWS_NON_CCM_DEFAULT_SG_NEEDS_TO_ALLOW_CDP_CIDRS = "AWS_NON_CCM_DEFAULT_SG_NEEDS_TO_ALLOW_CDP_CIDRS" + +AWS_NON_CCM_GATEWAY_SG_MISSING_CIDRS = "AWS_NON_CCM_GATEWAY_SG_MISSING_CIDRS" + +AWS_DEFAULT_SG_NEEDS_ALLOW_ACCESS_INTERNAL_TO_VPC = "AWS_DEFAULT_SG_NEEDS_ALLOW_ACCESS_INTERNAL_TO_VPC" + +AWS_GATEWAY_SG_NEEDS_ALLOW_ACCESS_INTERNAL_TO_VPC = "AWS_GATEWAY_SG_NEEDS_ALLOW_ACCESS_INTERNAL_TO_VPC" + +AWS_SSH_KEY_ID_DOES_NOT_EXIST = "AWS_SSH_KEY_ID_DOES_NOT_EXIST" + +AWS_SSH_IS_INVALID = "AWS_SSH_IS_INVALID" + +AWS_NOT_ENOUGH_SUBNETS_PROVIDED = "AWS_NOT_ENOUGH_SUBNETS_PROVIDED" + +AWS_INVALID_SUBNET_ID = "AWS_INVALID_SUBNET_ID" + +AWS_REQUIRED_DATA_MISSING = "AWS_REQUIRED_DATA_MISSING" + +AWS_INVALID_DATA = "AWS_INVALID_DATA" + +AWS_SUBNETS_DO_NOT_EXIST = "AWS_SUBNETS_DO_NOT_EXIST" + +AWS_NOT_ENOUGH_AZ_FOR_SUBNETS = "AWS_NOT_ENOUGH_AZ_FOR_SUBNETS" + +AWS_SUBNETS_WITHOUT_INTERNET_GATEWAY = "AWS_SUBNETS_WITHOUT_INTERNET_GATEWAY" + +AWS_SUBNETS_OR_VPC_WITHOUT_INTERNET_GATEWAY = "AWS_SUBNETS_OR_VPC_WITHOUT_INTERNET_GATEWAY" + +AWS_SUBNETS_WITHOUT_VALID_RANGE = "AWS_SUBNETS_WITHOUT_VALID_RANGE" + +AWS_SUBNETS_MISSING_K8S_LB_TAG = "AWS_SUBNETS_MISSING_K8S_LB_TAG" + +AWS_SUBNETS_NOT_PART_OF_VPC = "AWS_SUBNETS_NOT_PART_OF_VPC" + +AWS_DNS_SUPPORT_NOT_ENABLED_FOR_VPC = "AWS_DNS_SUPPORT_NOT_ENABLED_FOR_VPC" + +AWS_VPC_NOT_FOUND_IN_ACCOUNT = "AWS_VPC_NOT_FOUND_IN_ACCOUNT" diff --git a/cdpctl/validation/infra/validate_aws_cross_account_role.py b/cdpctl/validation/infra/validate_aws_cross_account_role.py index ebc3541..1e341bb 100644 --- a/cdpctl/validation/infra/validate_aws_cross_account_role.py +++ b/cdpctl/validation/infra/validate_aws_cross_account_role.py @@ -45,8 +45,17 @@ import pytest from boto3_type_annotations.iam import Client as IAMClient -from cdpctl.validation import get_config_value +from cdpctl.validation import fail, get_config_value from cdpctl.validation.aws_utils import get_client, get_role, simulate_policy +from cdpctl.validation.infra.issues import ( + AWS_ACCOUNT_ID_NOT_IN_CROSS_ACCOUNT_ROLE, + AWS_CROSS_ACCOUNT_ROLE_MISSING, + AWS_EXTERNAL_ID_NOT_IN_CROSS_ACCOUNT_ROLE, + AWS_REQUIRED_DATA_MISSING, + AWS_ROLE_REQUIRES_ACTIONS_FOR_ALL_EC2_RESOURCES, + AWS_ROLE_REQUIRES_ACTIONS_FOR_ALL_RESOURCES, + AWS_ROLE_REQUIRES_ACTIONS_FOR_SERVICE_ROLL_RESOURCES, +) cross_account_role_data = {} @@ -67,11 +76,8 @@ def aws_cross_account_role_exists_validation( cross_account_role: str = get_config_value( config, "env:aws:role:name:cross_account", - key_missing_message="No role defined for config option: {0}", - data_expected_error_message="No role was provided for config option: {0}", ) - - role = get_role(iam_client, cross_account_role) + role = get_role(iam_client, cross_account_role, AWS_CROSS_ACCOUNT_ROLE_MISSING) role_arn = role["Role"]["Arn"] cross_account_role_data["role_arn"] = role_arn cross_account_role_data["cross_account_role"] = cross_account_role @@ -88,8 +94,6 @@ def aws_cross_account_role_account_id_validation( account_id: str = get_config_value( config, "env:cdp:cross_account:account_id", - key_missing_message="No account id defined for config option: {0}", - data_expected_error_message="No account id was provided for config option: {0}", ) found_account_id = False for s in cross_account_role_data["role"]["Role"]["AssumeRolePolicyDocument"][ @@ -105,7 +109,10 @@ def aws_cross_account_role_account_id_validation( found_account_id = True if not found_account_id: - pytest.fail("Account id not in cross account role", False) + fail( + template=AWS_ACCOUNT_ID_NOT_IN_CROSS_ACCOUNT_ROLE, + subjects=[account_id, cross_account_role_data["cross_account_role"]], + ) @pytest.mark.aws @@ -118,8 +125,6 @@ def aws_cross_account_role_external_id_validation( external_id: str = get_config_value( config, "env:cdp:cross_account:external_id", - key_missing_message="No external id defined for config option: {0}", - data_expected_error_message="No external was provided for config option: {0}", ) for s in cross_account_role_data["role"]["Role"]["AssumeRolePolicyDocument"][ "Statement" @@ -127,7 +132,13 @@ def aws_cross_account_role_external_id_validation( if "Condition" in s.keys(): if s["Condition"]["StringEquals"]["sts:ExternalId"]: if external_id not in s["Condition"]["StringEquals"]["sts:ExternalId"]: - pytest.fail("External id not in cross account role", False) + fail( + template=AWS_EXTERNAL_ID_NOT_IN_CROSS_ACCOUNT_ROLE, + subjects=[ + external_id, + cross_account_role_data["cross_account_role"], + ], + ) @pytest.mark.aws @@ -140,17 +151,15 @@ def aws_cross_account_role_ec2_needed_actions_validation( """Cross Account role has the needed actions for EC2.""" # noqa: D401 try: simulate_policy( - iam_client, - cross_account_role_data["role_arn"], - ["*"], - ec2_needed_actions, - f"""The role ({0}) requires the following actions for all ec2 - resources ([*]) : \n {{}}""".format( - cross_account_role_data["cross_account_role"] - ), + iam_client=iam_client, + policy_source_arn=cross_account_role_data["role_arn"], + resource_arns=["*"], + needed_actions=ec2_needed_actions, + subjects=cross_account_role_data["cross_account_role"], + missing_actions_issue=AWS_ROLE_REQUIRES_ACTIONS_FOR_ALL_EC2_RESOURCES, ) except KeyError as e: - pytest.fail(f"Validation error - missing required data : {e.args[0]}", False) + fail(AWS_REQUIRED_DATA_MISSING, e.args[0]) @pytest.mark.aws @@ -163,17 +172,15 @@ def aws_cross_account_role_autoscaling_resources_needed_actions_validation( """Cross Account role has the needed actions for autoscaling resources.""" # noqa: D401,E501 try: simulate_policy( - iam_client, - cross_account_role_data["role_arn"], - ["*"], - autoscaling_resources_needed_actions, - f"""The role ({0}) requires the following actions for all resources - ([*]) : \n {{}}""".format( - cross_account_role_data["cross_account_role"] - ), + iam_client=iam_client, + policy_source_arn=cross_account_role_data["role_arn"], + resource_arns=["*"], + needed_actions=autoscaling_resources_needed_actions, + subjects=cross_account_role_data["cross_account_role"], + missing_actions_issue=AWS_ROLE_REQUIRES_ACTIONS_FOR_ALL_RESOURCES, ) except KeyError as e: - pytest.fail(f"Validation error - missing required data : {e.args[0]}", False) + fail(AWS_REQUIRED_DATA_MISSING, e.args[0]) @pytest.mark.aws @@ -186,17 +193,15 @@ def aws_cross_account_role_cloud_formation_needed_actions_validation( """Cross Account role has the needed actions for CloudFormation.""" # noqa: D401,E501 try: simulate_policy( - iam_client, - cross_account_role_data["role_arn"], - ["*"], - cloud_formation_needed_actions, - f"""The role ({0}) requires the following actions for all resources - ([*]) : \n {{}}""".format( - cross_account_role_data["cross_account_role"] - ), + iam_client=iam_client, + policy_source_arn=cross_account_role_data["role_arn"], + resource_arns=["*"], + needed_actions=cloud_formation_needed_actions, + subjects=cross_account_role_data["cross_account_role"], + missing_actions_issue=AWS_ROLE_REQUIRES_ACTIONS_FOR_ALL_RESOURCES, ) except KeyError as e: - pytest.fail(f"Validation error - missing required data : {e.args[0]}", False) + fail(AWS_REQUIRED_DATA_MISSING, e.args[0]) @pytest.mark.aws @@ -209,17 +214,15 @@ def aws_cross_account_role_cdp_environment_resources_needed_actions_validation( """Cross Account role has the needed actions for CDP environment resources.""" # noqa: D401,E501 try: simulate_policy( - iam_client, - cross_account_role_data["role_arn"], - ["*"], - cdp_environment_resources_needed_actions, - f"""The role ({0}) requires the following actions for all resources - ([*]) : \n {{}}""".format( - cross_account_role_data["cross_account_role"] - ), + iam_client=iam_client, + policy_source_arn=cross_account_role_data["role_arn"], + resource_arns=["*"], + needed_actions=cdp_environment_resources_needed_actions, + subjects=cross_account_role_data["cross_account_role"], + missing_actions_issue=AWS_ROLE_REQUIRES_ACTIONS_FOR_ALL_RESOURCES, ) except KeyError as e: - pytest.fail(f"Validation error - missing required data : {e.args[0]}", False) + fail(AWS_REQUIRED_DATA_MISSING, e.args[0]) @pytest.mark.aws @@ -232,17 +235,15 @@ def aws_cross_account_role_pass_role_needed_actions_validation( """Cross Account role has the needed actions for the pass role.""" # noqa: D401,E501 try: simulate_policy( - iam_client, - cross_account_role_data["role_arn"], - ["*"], - pass_role_needed_actions, - f"""The role ({0}) requires the following actions for all resources - ([*]) : \n {{}}""".format( - cross_account_role_data["cross_account_role"] - ), + iam_client=iam_client, + policy_source_arn=cross_account_role_data["role_arn"], + resource_arns=["*"], + needed_actions=pass_role_needed_actions, + subjects=cross_account_role_data["cross_account_role"], + missing_actions_issue=AWS_ROLE_REQUIRES_ACTIONS_FOR_ALL_RESOURCES, ) except KeyError as e: - pytest.fail(f"Validation error - missing required data : {e.args[0]}", False) + fail(AWS_REQUIRED_DATA_MISSING, e.args[0]) @pytest.mark.aws @@ -255,14 +256,12 @@ def aws_cross_account_identity_management_needed_actions_validation( """Cross Account role has the needed actions for identity management.""" # noqa: D401,E501 try: simulate_policy( - iam_client, - cross_account_role_data["role_arn"], - ["arn:aws:iam::*:role/aws-service-role/*"], - identity_management_needed_actions, - f"""The role ({0}) requires the following actions for all resources - ("arn:aws:iam::*:role/aws-service-role/*") : \n {{}}""".format( - cross_account_role_data["cross_account_role"] - ), + iam_client=iam_client, + policy_source_arn=cross_account_role_data["role_arn"], + resource_arns=["arn:aws:iam::*:role/aws-service-role/*"], + needed_actions=identity_management_needed_actions, + subjects=cross_account_role_data["cross_account_role"], + missing_actions_issue=AWS_ROLE_REQUIRES_ACTIONS_FOR_SERVICE_ROLL_RESOURCES, ) except KeyError as e: - pytest.fail(f"Validation error - missing required data : {e.args[0]}", False) + fail(AWS_REQUIRED_DATA_MISSING, e.args[0]) diff --git a/cdpctl/validation/infra/validate_aws_datalake_admin_role.py b/cdpctl/validation/infra/validate_aws_datalake_admin_role.py index 9216abb..18bb862 100644 --- a/cdpctl/validation/infra/validate_aws_datalake_admin_role.py +++ b/cdpctl/validation/infra/validate_aws_datalake_admin_role.py @@ -53,6 +53,11 @@ parse_arn, simulate_policy, ) +from cdpctl.validation.infra.issues import ( + AWS_ROLE_FOR_DATA_BUCKET_MISSING_ACTIONS, + AWS_ROLE_FOR_DL_BUCKET_MISSING_ACTIONS, + AWS_ROLE_REQUIRES_ACTIONS_FOR_ALL_S3_RESOURCES, +) @pytest.fixture(scope="module", name="bucket_access_policy_actions") @@ -179,15 +184,11 @@ def aws_datalake_admin_role_has_bucket_access_policy( datalake_admin_role_name: str = get_config_value( config, "env:aws:role:name:datalake_admin", - key_missing_message="No role defined for config option: {0}", - data_expected_error_message="No role was provided for config option: {0}", ) data_location: str = get_config_value( config, "infra:aws:vpc:existing:storage:data", - key_missing_message="No s3a url defined for config option: {0}", - data_expected_error_message="No s3a url was provided for config option: {0}", ) data_location_arn = convert_s3a_to_arn(data_location) @@ -198,12 +199,12 @@ def aws_datalake_admin_role_has_bucket_access_policy( datalake_admin_role_arn = datalake_admin_role["Role"]["Arn"] simulate_policy( - iam_client, - datalake_admin_role_arn, - [bucket_arn, f"{bucket_arn}/*"], - bucket_access_policy_actions, - missing_actions_message=f"The role ({datalake_admin_role_name}) requires the following actions for the \ - datalake S3 bucket ({bucket_name}):\n{{0}}", + iam_client=iam_client, + policy_source_arn=datalake_admin_role_arn, + resource_arns=[bucket_arn, f"{bucket_arn}/*"], + needed_actions=bucket_access_policy_actions, + subjects=[datalake_admin_role_name, bucket_name], + missing_actions_issue=AWS_ROLE_FOR_DL_BUCKET_MISSING_ACTIONS, ) @@ -237,20 +238,18 @@ def aws_datalake_admin_role_has_bucket_access_policy_all_resources( datalake_admin_role_name: str = get_config_value( config, "env:aws:role:name:datalake_admin", - key_missing_message="No role defined for config option: {0}", - data_expected_error_message="No role was provided for config option: {0}", ) datalake_admin_role = get_role(iam_client, datalake_admin_role_name) datalake_admin_role_arn = datalake_admin_role["Role"]["Arn"] simulate_policy( - iam_client, - datalake_admin_role_arn, - ["*"], - bucket_access_policy_all_resources_actions, - missing_actions_message=f"The role ({datalake_admin_role_name}) requires the following actions for all \ - S3 resources:\n{{0}}", + iam_client=iam_client, + policy_source_arn=datalake_admin_role_arn, + resource_arns=["*"], + needed_actions=bucket_access_policy_all_resources_actions, + subjects=[datalake_admin_role_name], + missing_actions_issue=AWS_ROLE_REQUIRES_ACTIONS_FOR_ALL_S3_RESOURCES, ) @@ -286,15 +285,11 @@ def aws_datalake_admin_role_has_s3_policy( datalake_admin_role_name: str = get_config_value( config, "env:aws:role:name:datalake_admin", - key_missing_message="No role defined for config option: {0}", - data_expected_error_message="No role was provided for config option: {0}", ) data_location: str = get_config_value( config, "infra:aws:vpc:existing:storage:data", - key_missing_message="No s3a url defined for config option: {0}", - data_expected_error_message="No s3a url was provided for config option: {0}", ) data_location_arn = convert_s3a_to_arn(data_location) @@ -303,10 +298,10 @@ def aws_datalake_admin_role_has_s3_policy( datalake_admin_role_arn = datalake_admin_role["Role"]["Arn"] simulate_policy( - iam_client, - datalake_admin_role_arn, - [data_location_arn, f"{data_location_arn}/*"], - datalake_admin_s3_policy_actions, - missing_actions_message=f"The role ({datalake_admin_role_name}) requires the following actions for the \ - S3 data location ({data_location}):\n{{0}}", + iam_client=iam_client, + policy_source_arn=datalake_admin_role_arn, + resource_arns=[data_location_arn, f"{data_location_arn}/*"], + needed_actions=datalake_admin_s3_policy_actions, + subjects=[datalake_admin_role_name, data_location], + missing_actions_issue=AWS_ROLE_FOR_DATA_BUCKET_MISSING_ACTIONS, ) diff --git a/cdpctl/validation/infra/validate_aws_idbroker_role.py b/cdpctl/validation/infra/validate_aws_idbroker_role.py index 07e4d2e..b9a0033 100644 --- a/cdpctl/validation/infra/validate_aws_idbroker_role.py +++ b/cdpctl/validation/infra/validate_aws_idbroker_role.py @@ -46,7 +46,7 @@ import pytest from boto3_type_annotations.iam import Client as IAMClient -from cdpctl.validation import get_config_value, validator +from cdpctl.validation import fail, get_config_value, validator from cdpctl.validation.aws_utils import ( convert_s3a_to_arn, get_client, @@ -55,6 +55,13 @@ parse_arn, simulate_policy, ) +from cdpctl.validation.infra.issues import ( + AWS_IDBROKER_INSTANCE_PROLFILE_NEEDS_ROLE, + AWS_IDBROKER_ROLE_NEED_EC2_TRUST_POLICY, + AWS_IDBROKER_ROLE_REQUIRES_ACTIONS_FOR_ALL_RESOURCES, + AWS_ROLE_REQUIRES_ACTIONS_FOR_LOG_BUCKET, + AWS_ROLE_REQUIRES_ACTIONS_FOR_LOG_PATH, +) @pytest.fixture(autouse=True, name="iam_client") @@ -70,12 +77,6 @@ def get_idbroker_instance_profile( idbroker_instance_profile: str = get_config_value( config, "env:aws:instance_profile:name:idbroker", - key_missing_message=( - "No idbroker instance profile defined for config option: {}" - ), - data_expected_error_message=( - "No idbroker instance profile provided for config option: {}" - ), ) return get_instance_profile(iam_client, idbroker_instance_profile) @@ -98,12 +99,8 @@ def _aws_idbroker_instance_profile_exists_with_role_validation( profile = get_idbroker_instance_profile(config=config, iam_client=iam_client) if not profile["InstanceProfile"]["Roles"]: - pytest.fail( - """IdBroker instance profile {} set in config env:aws:instance_profile:name:idbroker - should contain a idbroker role.""".format( - profile["InstanceProfile"]["Arn"] - ), - False, + fail( + AWS_IDBROKER_INSTANCE_PROLFILE_NEEDS_ROLE, profile["InstanceProfile"]["Arn"] ) @@ -150,12 +147,7 @@ def _aws_idbroker_role_has_ec2_trust_policy_validation( continue if not found_ec2_trust: - pytest.fail( - """The idbroker role {} should contain a trust policy for ec2""".format( - role - ), - False, - ) + fail(AWS_IDBROKER_ROLE_NEED_EC2_TRUST_POLICY, role) @pytest.mark.aws @@ -185,8 +177,6 @@ def _aws_idbroker_role_has_assumerole_policy_validation( ranger_audit_role_name: str = get_config_value( config, "env:aws:role:name:ranger_audit", - key_missing_message="No role defined for config option: {0}", - data_expected_error_message="No role was provided for config option: {0}", ) ranger_audit_role = get_role(iam_client, ranger_audit_role_name) ranger_audit_role_arn = ranger_audit_role["Role"]["Arn"] @@ -194,20 +184,18 @@ def _aws_idbroker_role_has_assumerole_policy_validation( datalake_admin_role_name: str = get_config_value( config, "env:aws:role:name:datalake_admin", - key_missing_message="No role defined for config option: {0}", - data_expected_error_message="No role was provided for config option: {0}", ) datalake_admin_role = get_role(iam_client, datalake_admin_role_name) datalake_admin_role_arn = datalake_admin_role["Role"]["Arn"] simulate_policy( - iam_client, - idbroker_role_arn, - [ranger_audit_role_arn, datalake_admin_role_arn], - ["sts:AssumeRole"], - f"The role ({idbroker_role_arn}) requires the following " - f"actions for resource wildcard (*) :\n{{}}", + iam_client=iam_client, + policy_source_arn=idbroker_role_arn, + resource_arns=[ranger_audit_role_arn, datalake_admin_role_arn], + needed_actions=["sts:AssumeRole"], + subjects=idbroker_role_arn, + missing_actions_issue=AWS_IDBROKER_ROLE_REQUIRES_ACTIONS_FOR_ALL_RESOURCES, ) @@ -241,18 +229,16 @@ def _aws_idbroker_role_has_necessary_s3_actions_validation( log_location: str = get_config_value( config, "infra:aws:vpc:existing:storage:logs", - key_missing_message="No log storage path defined for config option: {}", - data_expected_error_message="No log storage path provided for config option: {}", # noqa: E501 ) log_location_arn = convert_s3a_to_arn(log_location) simulate_policy( - iam_client, - role_arn, - [f"{log_location_arn}/*"], - logs_needed_actions, - f"""The role ({role_arn}) requires the following actions - for the log storage path ({log_location}):\n{{}}""", + iam_client=iam_client, + policy_source_arn=role_arn, + resource_arns=[f"{log_location_arn}/*"], + needed_actions=logs_needed_actions, + subjects=[role_arn, log_location], + missing_actions_issue=AWS_ROLE_REQUIRES_ACTIONS_FOR_LOG_PATH, ) @@ -288,8 +274,6 @@ def _aws_idbroker_role_has_necessary_s3_bucket_actions_validation( log_location: str = get_config_value( config, "infra:aws:vpc:existing:storage:logs", - key_missing_message="No log storage path defined for config option: {}", - data_expected_error_message="No log storage path provided for config option: {}", # noqa: E501 ) log_location_arn = convert_s3a_to_arn(log_location) @@ -297,10 +281,10 @@ def _aws_idbroker_role_has_necessary_s3_bucket_actions_validation( log_bucket_arn = convert_s3a_to_arn(f"s3a://{log_bucket_name}") simulate_policy( - iam_client, - role_arn, - [log_bucket_arn], - log_bucket_needed_actions, - f"""The role ({role_arn}) requires the following actions - for the log storage bucket ({log_bucket_arn}):\n{{}}""", + iam_client=iam_client, + policy_source_arn=role_arn, + resource_arns=[log_bucket_arn], + needed_actions=log_bucket_needed_actions, + subjects=[role_arn, log_bucket_arn], + missing_actions_issue=AWS_ROLE_REQUIRES_ACTIONS_FOR_LOG_BUCKET, ) diff --git a/cdpctl/validation/infra/validate_aws_logger_role.py b/cdpctl/validation/infra/validate_aws_logger_role.py index 9f7ea8f..6419fff 100644 --- a/cdpctl/validation/infra/validate_aws_logger_role.py +++ b/cdpctl/validation/infra/validate_aws_logger_role.py @@ -46,7 +46,7 @@ import pytest from boto3_type_annotations.iam import Client as IAMClient -from cdpctl.validation import get_config_value, validator +from cdpctl.validation import fail, get_config_value, validator from cdpctl.validation.aws_utils import ( convert_s3a_to_arn, get_client, @@ -54,6 +54,12 @@ parse_arn, simulate_policy, ) +from cdpctl.validation.infra.issues import ( + AWS_LOGGER_INSTANCE_PROFILE_SHOULD_CONTAIN_LOGGER_ROLE, + AWS_LOGGER_ROLE_SHOULD_HAVE_EC2_TRUST, + AWS_ROLE_REQUIRES_ACTIONS_FOR_LOG_BUCKET, + AWS_ROLE_REQUIRES_ACTIONS_FOR_LOG_PATH, +) @pytest.fixture(autouse=True, name="iam_client") @@ -69,10 +75,6 @@ def get_logger_instance_profile( logger_instance_profile: str = get_config_value( config, "env:aws:instance_profile:name:log", - key_missing_message=("No logger instance profile defined for config option {}"), - data_expected_error_message=( - "No logger instance profile provided for config option {}" - ), ) return get_instance_profile(iam_client, logger_instance_profile) @@ -95,12 +97,9 @@ def _aws_logger_instance_profile_exists_with_role_validation( profile = get_logger_instance_profile(config=config, iam_client=iam_client) if not profile["InstanceProfile"]["Roles"]: - pytest.fail( - """The logger instance profile {} set in config env:aws:instance_profile:name:log - should contain a logger role.""".format( - profile["InstanceProfile"]["Arn"] - ), - False, + fail( + template=AWS_LOGGER_INSTANCE_PROFILE_SHOULD_CONTAIN_LOGGER_ROLE, + subjects=profile["InstanceProfile"]["Arn"], ) @@ -147,10 +146,7 @@ def _aws_logger_role_has_ec2_trust_policy_validation( continue if not found_ec2_trust: - pytest.fail( - f"""The logger role {role} should contain a trust policy for ec2""", - False, - ) + fail(AWS_LOGGER_ROLE_SHOULD_HAVE_EC2_TRUST, subjects=[role]) @pytest.mark.aws @@ -183,18 +179,16 @@ def _aws_logger_role_has_necessary_s3_actions_validation( log_location: str = get_config_value( config, "infra:aws:vpc:existing:storage:logs", - key_missing_message="No log storage path defined for config option: {}", - data_expected_error_message="No log storage path provided for config option: {}", # noqa: E501 ) log_location_arn = convert_s3a_to_arn(log_location) simulate_policy( - iam_client, - role_arn, - [f"{log_location_arn}/*"], - logs_needed_actions, - f"""The role ({role_arn}) requires the following actions - for the log storage path ({log_location}):\n{{}}""", + iam_client=iam_client, + policy_source_arn=role_arn, + resource_arns=[f"{log_location_arn}/*"], + needed_actions=logs_needed_actions, + subjects=[role_arn, log_location], + missing_actions_issue=AWS_ROLE_REQUIRES_ACTIONS_FOR_LOG_PATH, ) @@ -230,8 +224,6 @@ def _aws_logger_role_has_necessary_s3_bucket_actions_validation( log_location: str = get_config_value( config, "infra:aws:vpc:existing:storage:logs", - key_missing_message="No log storage path defined for config option: {}", - data_expected_error_message="No log storage path provided for config option: {}", # noqa: E501 ) log_location_arn = convert_s3a_to_arn(log_location) @@ -239,10 +231,10 @@ def _aws_logger_role_has_necessary_s3_bucket_actions_validation( log_bucket_arn = convert_s3a_to_arn(f"s3a://{log_bucket_name}") simulate_policy( - iam_client, - role_arn, - [log_bucket_arn], - log_bucket_needed_actions, - f"""The role ({role_arn}) requires the following actions - for the log storage bucket ({log_bucket_arn}):\n{{}}""", + iam_client=iam_client, + policy_source_arn=role_arn, + resource_arns=[log_bucket_arn], + needed_actions=log_bucket_needed_actions, + subjects=[role_arn, log_bucket_arn], + missing_actions_issue=AWS_ROLE_REQUIRES_ACTIONS_FOR_LOG_BUCKET, ) diff --git a/cdpctl/validation/infra/validate_aws_ranger_audit_role.py b/cdpctl/validation/infra/validate_aws_ranger_audit_role.py index 9e9543f..a41abdc 100644 --- a/cdpctl/validation/infra/validate_aws_ranger_audit_role.py +++ b/cdpctl/validation/infra/validate_aws_ranger_audit_role.py @@ -45,7 +45,7 @@ import pytest from boto3_type_annotations.iam import Client as IAMClient -from cdpctl.validation import get_config_value +from cdpctl.validation import fail, get_config_value from cdpctl.validation.aws_utils import ( convert_s3a_to_arn, get_client, @@ -53,6 +53,12 @@ parse_arn, simulate_policy, ) +from cdpctl.validation.infra.issues import ( + AWS_REQUIRED_DATA_MISSING, + AWS_ROLE_FOR_DATA_BUCKET_MISSING_ACTIONS, + AWS_ROLE_FOR_DL_BUCKET_MISSING_ACTIONS, + AWS_ROLE_REQUIRES_ACTIONS_FOR_ALL_S3_RESOURCES, +) ranger_audit_data = {} @@ -72,8 +78,6 @@ def aws_ranger_audit_role_exists_validation( ranger_audit_role: str = get_config_value( config, "env:aws:role:name:ranger_audit", - key_missing_message="No role defined for config option: {0}", - data_expected_error_message="No role was provided for config option: {0}", ) # ranger audit role arn @@ -92,8 +96,6 @@ def aws_ranger_audit_role_data_location_exist_validation( data_location: str = get_config_value( config, "infra:aws:vpc:existing:storage:data", - key_missing_message="No s3a url defined for config option: {0}", - data_expected_error_message="No s3a url was provided for config option: {0}", ) # data access s3 bucket arn data_location_arn = convert_s3a_to_arn(data_location) @@ -111,9 +113,6 @@ def aws_ranger_audit_role_audit_location_exist_validation( ranger_audit_location: str = get_config_value( config, "infra:aws:vpc:existing:storage:ranger_audit", - key_missing_message="No ranger audit s3a url defined for config option: {0}", - data_expected_error_message="No ranger audit s3a url was provided for config " - "option: {0}", ) # ranger audit s3 bucket arn @@ -136,16 +135,18 @@ def aws_ranger_audit_location_needed_actions_validation( try: # aws-cdp-ranger-audit-s3-policy simulate_policy( - iam_client, - ranger_audit_data["role_arn"], - [ranger_audit_data["ranger_audit_location_arn"] + "/*"], - ranger_audit_location_needed_actions, - f"""The role ({ranger_audit_data["ranger_audit_role"]}) requires the - following actions for the datalake S3 bucket - ({ranger_audit_data["ranger_audit_location_arn"] + "/*"}) : \n {{}}""", + iam_client=iam_client, + policy_source_arn=ranger_audit_data["role_arn"], + resource_arns=[ranger_audit_data["ranger_audit_location_arn"] + "/*"], + needed_actions=ranger_audit_location_needed_actions, + subjects=[ + ranger_audit_data["ranger_audit_role"], + ranger_audit_data["ranger_audit_location_arn"] + "/*", + ], + missing_actions_issue=AWS_ROLE_FOR_DL_BUCKET_MISSING_ACTIONS, ) except KeyError as e: - pytest.fail(f"Validation error - missing required data : {e.args[0]}", False) + fail(AWS_REQUIRED_DATA_MISSING, e.args[0]) @pytest.mark.aws @@ -162,16 +163,18 @@ def aws_ranger_audit_s3_bucket_needed_actions_validation( """Ranger audit role has the needed actions for audit s3.""" # noqa: D401 try: simulate_policy( - iam_client, - ranger_audit_data["role_arn"], - [ranger_audit_data["ranger_audit_bucket_arn"]], - ranger_audit_bucket_needed_actions, - f"""The role ({ranger_audit_data["ranger_audit_role"]}) requires the - following actions for the datalake S3 bucket - ({ranger_audit_data["ranger_audit_bucket_arn"]}) : \n {{}}""", + iam_client=iam_client, + policy_source_arn=ranger_audit_data["role_arn"], + resource_arns=[ranger_audit_data["ranger_audit_bucket_arn"]], + needed_actions=ranger_audit_bucket_needed_actions, + subjects=[ + ranger_audit_data["ranger_audit_role"], + ranger_audit_data["ranger_audit_bucket_arn"], + ], + missing_actions_issue=AWS_ROLE_FOR_DL_BUCKET_MISSING_ACTIONS, ) except KeyError as e: - pytest.fail(f"Validation error - missing required data : {e.args[0]}", False) + fail(AWS_REQUIRED_DATA_MISSING, e.args[0]) @pytest.mark.aws @@ -184,15 +187,15 @@ def aws_ranger_audit_cdp_s3_needed_actions_validation( try: # aws-cdp-bucket-access-policy simulate_policy( - iam_client, - ranger_audit_data["role_arn"], - ["*"], - s3_needed_actions_to_all, - f"""The role ({ranger_audit_data["ranger_audit_role"]}) requires the - following actions for all S3 resources ([*]) : \n {{}}""", + iam_client=iam_client, + policy_source_arn=ranger_audit_data["role_arn"], + resource_arns=["*"], + needed_actions=s3_needed_actions_to_all, + subjects=[ranger_audit_data["ranger_audit_role"]], + missing_actions_issue=AWS_ROLE_REQUIRES_ACTIONS_FOR_ALL_S3_RESOURCES, ) except KeyError as e: - pytest.fail(f"Validation error - missing required data : {e.args[0]}", False) + fail(AWS_REQUIRED_DATA_MISSING, e.args[0]) @pytest.mark.aws @@ -204,16 +207,18 @@ def aws_ranger_audit_data_location_needed_actions_validation( """Ranger audit role has needed actions for the data location.""" # noqa: D401 try: simulate_policy( - iam_client, - ranger_audit_data["role_arn"], - [ + iam_client=iam_client, + policy_source_arn=ranger_audit_data["role_arn"], + resource_arns=[ ranger_audit_data["data_location_arn"], ranger_audit_data["data_location_arn"] + "/*", ], - data_location_needed_actions, - f"The role ({ranger_audit_data['ranger_audit_role']}) " - "requires the following actions for the S3 data location " - f"({ranger_audit_data['data_location_arn']}/*): \n {{}}", + needed_actions=data_location_needed_actions, + subjects=[ + ranger_audit_data["ranger_audit_role"], + ranger_audit_data["data_location_arn"] + "/*", + ], + missing_actions_issue=AWS_ROLE_FOR_DATA_BUCKET_MISSING_ACTIONS, ) except KeyError as e: - pytest.fail(f"Validation error - missing required data : {e.args[0]}", False) + fail(AWS_REQUIRED_DATA_MISSING, e.args[0]) diff --git a/cdpctl/validation/infra/validate_aws_s3_locations.py b/cdpctl/validation/infra/validate_aws_s3_locations.py index f4b4577..da537e1 100644 --- a/cdpctl/validation/infra/validate_aws_s3_locations.py +++ b/cdpctl/validation/infra/validate_aws_s3_locations.py @@ -46,13 +46,18 @@ import pytest from boto3_type_annotations.s3 import Client as S3Client -from cdpctl.validation import get_config_value, validator +from cdpctl.validation import fail, get_config_value, validator from cdpctl.validation.aws_utils import ( convert_s3a_to_arn, get_client, is_valid_s3a_url, parse_arn, ) +from cdpctl.validation.infra.issues import ( + AWS_S3_BUCKET_DOES_NOT_EXIST, + AWS_S3_BUCKET_FORBIDDEN_ACCESS, + AWS_S3_BUCKET_INVALID, +) @pytest.fixture(autouse=True, name="s3_client") @@ -77,8 +82,6 @@ def aws_s3_data_bucket_exists(config: Dict[str, Any], s3_client: S3Client) -> No data_bucket_url: str = get_config_value( config, "infra:aws:vpc:existing:storage:data", - key_missing_message="No s3a url defined for config option: {0}", - data_expected_error_message="No s3a url was provided for config option: {0}", ) aws_s3_bucket_exists(data_bucket_url, s3_client) @@ -99,8 +102,6 @@ def aws_s3_logs_bucket_exists(config: Dict[str, Any], s3_client: S3Client) -> No logs_bucket_url: str = get_config_value( config, "infra:aws:vpc:existing:storage:logs", - key_missing_message="No s3a url defined for config option: {0}", - data_expected_error_message="No s3a url was provided for config option: {0}", ) aws_s3_bucket_exists(logs_bucket_url, s3_client) @@ -122,8 +123,6 @@ def aws_s3_backup_bucket_exists(config: Dict[str, Any], s3_client: S3Client) -> backup_bucket_url: str = get_config_value( config, "infra:aws:vpc:existing:storage:backup", - key_missing_message="No s3a url defined for config option: {0}", - data_expected_error_message="No s3a url was provided for config option: {0}", ) # TODO: Handle a specific parameter for backup S3 location once it exists aws_s3_bucket_exists(backup_bucket_url, s3_client) @@ -132,7 +131,7 @@ def aws_s3_backup_bucket_exists(config: Dict[str, Any], s3_client: S3Client) -> def aws_s3_bucket_exists(bucket_url: str, s3_client: S3Client) -> None: """Check to see if the s3 bucket exists.""" if not is_valid_s3a_url(bucket_url): - pytest.fail(f"The s3a url {bucket_url} is invalid.", False) + fail(AWS_S3_BUCKET_INVALID, bucket_url) # get bucket name bucket_name = parse_arn(convert_s3a_to_arn(bucket_url))["resource_type"] @@ -145,6 +144,6 @@ def aws_s3_bucket_exists(bucket_url: str, s3_client: S3Client) -> None: # If it was a 404 error, then the bucket does not exist. error_code = int(e.response["Error"]["Code"]) if error_code == 403: - pytest.fail(f"S3 bucket {bucket_name} has forbidden access.", False) + fail(AWS_S3_BUCKET_FORBIDDEN_ACCESS, bucket_name) elif error_code == 404: - pytest.fail(f"S3 bucket {bucket_name} does not exist.", False) + fail(AWS_S3_BUCKET_DOES_NOT_EXIST, bucket_name) diff --git a/cdpctl/validation/infra/validate_aws_security_groups.py b/cdpctl/validation/infra/validate_aws_security_groups.py index cb50999..7c6b225 100644 --- a/cdpctl/validation/infra/validate_aws_security_groups.py +++ b/cdpctl/validation/infra/validate_aws_security_groups.py @@ -40,13 +40,20 @@ # Source File Name: validate_aws_security_groups.py ### """Validation of AWS security groups.""" -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List import pytest from boto3_type_annotations.ec2 import Client as EC2Client -from cdpctl.validation import get_config_value, validator +from cdpctl.validation import fail, get_config_value, validator from cdpctl.validation.aws_utils import get_client +from cdpctl.validation.infra.issues import ( + AWS_DEFAULT_SG_NEEDS_ALLOW_ACCESS_INTERNAL_TO_VPC, + AWS_GATEWAY_SG_NEEDS_ALLOW_ACCESS_INTERNAL_TO_VPC, + AWS_NON_CCM_DEFAULT_SG_NEEDS_TO_ALLOW_CDP_CIDRS, + AWS_NON_CCM_GATEWAY_SG_MISSING_CIDRS, + AWS_VPC_NOT_FOUND_IN_ACCOUNT, +) @pytest.fixture(autouse=True, name="ec2_client") @@ -55,22 +62,6 @@ def iam_client_fixture(config: Dict[str, Any]) -> EC2Client: return get_client("ec2", config) -def tunnel_enabled(config: Dict[str, Any]): - """Return the tunnel enabled condition, which is sourced from config.yaml.""" - tunnel_val: Optional[str] = get_config_value( - config, - "env:tunnel", - key_value_expected=False, - key_missing_message="", - data_expected_error_message="", - ) - - if tunnel_val: - return tunnel_val - - return False - - @pytest.mark.aws @pytest.mark.infra @pytest.mark.config_value(path="env:tunnel", value=False) @@ -90,14 +81,10 @@ def _aws_default_security_groups_contains_cdp_cidr_validation( config: Dict[str, Any], ec2_client: EC2Client, cdp_cidrs: List[str] ) -> None: """Default security groups contain CDP CIDRs if CCM is not enabled.""" # noqa: D401,E501 - if tunnel_enabled(config): - return default_security_groups_id: str = get_config_value( config, "infra:aws:vpc:existing:security_groups:default_id", - key_missing_message="No default security groups id defined for config option {}", # noqa: E501 - data_expected_error_message="No default security groups id provided for config option: {}", # noqa: E501 ) security_groups = ec2_client.describe_security_groups( @@ -126,10 +113,9 @@ def _aws_default_security_groups_contains_cdp_cidr_validation( missing_cdp_cidr_9443.append(cdp_cidr) if len(missing_cdp_cidr_9443) > 0: - pytest.fail( - "For non-CCM setup (tunnel = false/unset): " - f" TCP Port 9443 needs to allow CDP CIDRs {missing_cdp_cidr_9443}", - False, + fail( + AWS_NON_CCM_DEFAULT_SG_NEEDS_TO_ALLOW_CDP_CIDRS, + resources=missing_cdp_cidr_9443, ) @@ -152,16 +138,10 @@ def _aws_gateway_security_groups_contains_cdp_cidr_validation( config: Dict[str, Any], ec2_client: EC2Client, cdp_cidrs: List[str] ) -> None: """Validate that gateway security groups contain CDP CIDRs if CCM not used.""" - if tunnel_enabled(config): - return gateway_security_groups_id: str = get_config_value( config, "infra:aws:vpc:existing:security_groups:knox_id", - key_missing_message="No gateway security groups id defined for " - "config option: {}", - data_expected_error_message="No gateway security groups id provided for " - "config option: {}", ) security_groups = ec2_client.describe_security_groups( @@ -198,18 +178,22 @@ def _aws_gateway_security_groups_contains_cdp_cidr_validation( if not found_cidr_9443: missing_cdp_cidr_9443.append(cdp_cidr) - missing_cidrs = "" + missing_cidrs = [] if len(missing_cdp_cidr_443) > 0: - missing_cidrs = f" TCP Port 443 needs to allow CDP CIDRs {missing_cdp_cidr_443}" + missing_cidrs = ( + "Access to TCP port 443 needs to be allowed for " + f"Cloudera CDP CIDR {missing_cdp_cidr_443}" + ) if len(missing_cdp_cidr_9443) > 0: missing_cidrs = ( - missing_cidrs - + f" TCP Port 9443 needs to allow CDP CIDRs {missing_cdp_cidr_9443}" + missing_cidrs + "Access to TCP port 9443 needs to be allowed " + f"for Cloudera CDP CIDR {missing_cdp_cidr_9443}" ) if missing_cidrs: - pytest.fail( - "For non-CCM setup (tunnel = false/unset): " + missing_cidrs, - False, + fail( + AWS_NON_CCM_GATEWAY_SG_MISSING_CIDRS, + subjects=gateway_security_groups_id, + resources=missing_cidrs, ) @@ -220,17 +204,12 @@ def security_groups_contains_vpc_cidr( vpc_id: str = get_config_value( config, "infra:aws:vpc:existing:vpc_id", - key_missing_message="No vpc id defined for " "config option: {}", - data_expected_error_message="No vpc id provided for " "config option: {}", ) vpcs = ec2_client.describe_vpcs(VpcIds=[vpc_id]) if len(vpcs["Vpcs"]) == 0: - pytest.fail( - f"vpc id {vpc_id} set in infra:aws:vpc:existing:vpc_id " - "was not found on the AWS account." - ) + fail(AWS_VPC_NOT_FOUND_IN_ACCOUNT, subjects=[vpc_id]) vpc_cidr = vpcs["Vpcs"][0]["CidrBlock"] @@ -279,19 +258,14 @@ def _aws_default_security_groups_contains_vpc_cidr_validation( default_security_groups_id: str = get_config_value( config, "infra:aws:vpc:existing:security_groups:default_id", - key_missing_message="No default security groups id defined for " - "config option: {}", - data_expected_error_message="No default security groups id provided for " - "config option: {}", ) if not security_groups_contains_vpc_cidr( config, ec2_client, default_security_groups_id ): - pytest.fail( - "Your VPC CIDR should be allowed for port " - " 0-65535 in default security group.", - False, + fail( + AWS_DEFAULT_SG_NEEDS_ALLOW_ACCESS_INTERNAL_TO_VPC, + subjects=default_security_groups_id, ) @@ -312,17 +286,12 @@ def _aws_gateway_security_groups_contains_vpc_cidr_validation( gateway_security_groups_id: str = get_config_value( config, "infra:aws:vpc:existing:security_groups:knox_id", - key_missing_message="No gateway security groups id defined for " - "config option: {}", - data_expected_error_message="No gateway security groups id provided for " - "config option: {}", ) if not security_groups_contains_vpc_cidr( config, ec2_client, gateway_security_groups_id ): - pytest.fail( - "Your VPC CIDR should be allowed for port " - "0-65535 in gateway security group.", - False, + fail( + AWS_GATEWAY_SG_NEEDS_ALLOW_ACCESS_INTERNAL_TO_VPC, + subjects=gateway_security_groups_id, ) diff --git a/cdpctl/validation/infra/validate_aws_ssh_key.py b/cdpctl/validation/infra/validate_aws_ssh_key.py index 90716ff..fb7d4e3 100644 --- a/cdpctl/validation/infra/validate_aws_ssh_key.py +++ b/cdpctl/validation/infra/validate_aws_ssh_key.py @@ -46,8 +46,13 @@ import pytest from boto3_type_annotations.iam import Client as EC2Client -from cdpctl.validation import get_config_value +from cdpctl.validation import fail, get_config_value from cdpctl.validation.aws_utils import get_client +from cdpctl.validation.infra.issues import ( + AWS_REQUIRED_DATA_MISSING, + AWS_SSH_IS_INVALID, + AWS_SSH_KEY_ID_DOES_NOT_EXIST, +) subnets_data = {} @@ -69,14 +74,12 @@ def aws_ssh_key_validation( ssh_key_id: List[str] = get_config_value( config, "globals:ssh:public_key_id", - key_missing_message="No ssh key is defined for config option: {0}", - data_expected_error_message="No ssh key were provided for config option: {0}", # noqa: E501 ) key_pairs = ec2_client.describe_key_pairs(KeyPairIds=[ssh_key_id])["KeyPairs"] if not key_pairs: - pytest.fail(f"SSH key id ({ssh_key_id}) do not exist.", False) + fail(AWS_SSH_KEY_ID_DOES_NOT_EXIST, ssh_key_id) except KeyError as e: - pytest.fail(f"Validation error - missing required data : {e.args[0]}", False) - except ec2_client.exceptions.ClientError as ce: - pytest.fail(f"Validation error - invalid ssh key id : {ce.args[0]}", False) + fail(AWS_REQUIRED_DATA_MISSING, e.args[0]) + except ec2_client.exceptions.ClientError: + fail(AWS_SSH_IS_INVALID, ssh_key_id) diff --git a/cdpctl/validation/infra/validate_aws_subnets.py b/cdpctl/validation/infra/validate_aws_subnets.py index 34de1ee..4c82db1 100644 --- a/cdpctl/validation/infra/validate_aws_subnets.py +++ b/cdpctl/validation/infra/validate_aws_subnets.py @@ -46,8 +46,22 @@ import pytest from boto3_type_annotations.iam import Client as EC2Client -from cdpctl.validation import get_config_value +from cdpctl.validation import fail, get_config_value from cdpctl.validation.aws_utils import get_client +from cdpctl.validation.infra.issues import ( + AWS_DNS_SUPPORT_NOT_ENABLED_FOR_VPC, + AWS_INVALID_DATA, + AWS_INVALID_SUBNET_ID, + AWS_NOT_ENOUGH_AZ_FOR_SUBNETS, + AWS_NOT_ENOUGH_SUBNETS_PROVIDED, + AWS_REQUIRED_DATA_MISSING, + AWS_SUBNETS_DO_NOT_EXIST, + AWS_SUBNETS_MISSING_K8S_LB_TAG, + AWS_SUBNETS_NOT_PART_OF_VPC, + AWS_SUBNETS_OR_VPC_WITHOUT_INTERNET_GATEWAY, + AWS_SUBNETS_WITHOUT_INTERNET_GATEWAY, + AWS_SUBNETS_WITHOUT_VALID_RANGE, +) subnets_data = {} @@ -68,21 +82,13 @@ def aws_public_subnets_validation( public_subnets: List[str] = get_config_value( config, "infra:aws:vpc:existing:public_subnet_ids", - key_missing_message="No public subnets defined for config option: {0}", - data_expected_error_message="No public subnets were provided for config option: {0}", # noqa: E501 ) - if not isinstance(public_subnets, List): - pytest.fail( - """Invalid syntax, config data expected in following format - public_subnet_ids: - - subnetId-1 - - subnetId-2 - - subnetId-3""", - False, - ) + public_subnets = ( + [public_subnets] if isinstance(public_subnets, str) else public_subnets + ) if not len(public_subnets) > 2: - pytest.fail("Not enough subnets provided, at least 3 subnets required.", False) + fail(AWS_NOT_ENOUGH_SUBNETS_PROVIDED, subjects=["Public"]) try: # query subnets @@ -94,43 +100,13 @@ def aws_public_subnets_validation( if subnet["SubnetId"] == pu_id: missing_subnets.remove(pu_id) if len(missing_subnets) > 0: - pytest.fail( - f"Subnets ({missing_subnets}) do not exist.", - False, - ) + fail(AWS_SUBNETS_DO_NOT_EXIST, subjects="Public", resources=missing_subnets) subnets_data["public_subnets"] = subnets["Subnets"] subnets_data["public_subnets_ids"] = public_subnets except KeyError as e: - pytest.fail(f"Validation error - missing required data : {e.args[0]}", False) + fail(AWS_REQUIRED_DATA_MISSING, e.args[0]) except ec2_client.exceptions.ClientError as ce: - pytest.fail(f"Validation error - invalid subnetId : {ce.args[0]}", False) - - -@pytest.mark.aws -@pytest.mark.infra -@pytest.mark.dependency(depends=["aws_public_subnets_validation"]) -@pytest.mark.skip(reason="Suffixes are not part of config at this time.") -def aws_public_subnets_suffix_validation(config: Dict[str, Any]) -> None: - """Public subnets have the defined suffix.""" # noqa: D401,E501 - public_subnets_suffix: List[str] = get_config_value( - config, - "infra:aws:vpc:existing:public_subnets_suffix", - key_missing_message="No public subnets suffix defined for config option: {0}", - data_expected_error_message="No public subnets suffix was provided for config option: {0}", # noqa: E501 - ) - try: - subnets_wo_valid_suffix = [] - for subnet_id in subnets_data["public_subnets_ids"]: - if not subnet_id.endswith(public_subnets_suffix): - subnets_wo_valid_suffix.append(subnet_id) - - if len(subnets_wo_valid_suffix) > 0: - pytest.fail( - f"Subnets ({subnets_wo_valid_suffix}) without valid suffix ({public_subnets_suffix}).", # noqa: E501 - False, - ) - except KeyError as e: - pytest.fail(f"Validation error - missing required data : {e.args[0]}", False) + fail(AWS_INVALID_SUBNET_ID, subjects=["Public", ce.args[0]]) @pytest.mark.aws @@ -146,13 +122,9 @@ def aws_public_subnets_availablity_zone_validation() -> None: # minimum two availability zones validations if len(azs) <= 1: - pytest.fail( - """Not enough availability zones, subnets should be in - atleast 2 availability zones.""", - False, - ) + fail(AWS_NOT_ENOUGH_AZ_FOR_SUBNETS, subjects=["Public"]) except KeyError as e: - pytest.fail(f"Validation error - missing required data : {e.args[0]}", False) + fail(AWS_REQUIRED_DATA_MISSING, e.args[0]) @pytest.mark.aws @@ -179,8 +151,6 @@ def aws_public_subnets_route_validation( vpc_id: List[str] = get_config_value( config, "infra:aws:vpc:existing:vpc_id", - key_missing_message="No VPC id defined for config option: {0}", - data_expected_error_message="No VPC id was provided for config option: {0}", ) Filters = [ { @@ -204,23 +174,23 @@ def aws_public_subnets_route_validation( if "GatewayId" in route: gateway_ids.append(route["GatewayId"]) else: - pytest.fail( - f"""Provided subnets {subnets_data["public_subnets_ids"]} and - or VPC {vpc_id} do not have internet gateway(s).""", - False, + fail( + AWS_SUBNETS_OR_VPC_WITHOUT_INTERNET_GATEWAY, + subjects=["Public", vpc_id], + resources=subnets_data["public_subnets_ids"], ) if not set(igw_ids) & set(gateway_ids): - pytest.fail( - f"""Provided subnets {subnets_data["public_subnets_ids"]} do not - have internet gateway(s).""", - False, + fail( + AWS_SUBNETS_WITHOUT_INTERNET_GATEWAY, + subjects=["Public"], + resources=subnets_data["public_subnets_ids"], ) except KeyError as e: - pytest.fail(f"Validation error - missing required data : {e.args[0]}", False) + fail(AWS_REQUIRED_DATA_MISSING, e.args[0]) except ec2_client.exceptions.ClientError as ce: - pytest.fail(f"Validation error - invalid data : {ce.args[0]}", False) + fail(AWS_INVALID_DATA, ce.args[0]) @pytest.mark.aws @@ -236,12 +206,13 @@ def aws_public_subnets_range_validation() -> None: subnets_wo_valid_range.append(subnet["SubnetId"]) if len(subnets_wo_valid_range) > 0: - pytest.fail( - f"Public subnets ({subnets_wo_valid_range}) without valid required range.", # noqa: E501 - False, + fail( + AWS_SUBNETS_WITHOUT_VALID_RANGE, + subjects=["Public"], + resources=subnets_wo_valid_range, ) except KeyError as e: - pytest.fail(f"Validation error - missing required data : {e.args[0]}", False) + fail(AWS_REQUIRED_DATA_MISSING, e.args[0]) @pytest.mark.aws @@ -262,13 +233,13 @@ def aws_public_subnets_tags_validation() -> None: subnet_missing_tags = [k for k, v in subnets_w_valid_tag.items() if not v] if len(subnet_missing_tags) > 0: - pytest.fail( - f"""Public subnet(s) {subnet_missing_tags} missing tag - 'kubernetes.io/role/elb'.""", - False, + fail( + AWS_SUBNETS_MISSING_K8S_LB_TAG, + subjects=["Public"], + resources=subnet_missing_tags, ) except KeyError as e: - pytest.fail(f"Validation error - missing required data : {e.args[0]}", False) + fail(AWS_REQUIRED_DATA_MISSING, e.args[0]) @pytest.mark.aws @@ -281,21 +252,12 @@ def aws_private_subnets_validation( private_subnets: List[str] = get_config_value( config, "infra:aws:vpc:existing:private_subnet_ids", - key_missing_message="No private subnets defined for config option: {0}", - data_expected_error_message="No private subnets were provided for config " - "option: {0}", ) - if not isinstance(private_subnets, List): - pytest.fail( - """Invalid syntax, config data expected in following format - private_subnet_ids: - - subnetId-1 - - subnetId-2 - - subnetId-3""", - False, - ) + private_subnets = ( + [private_subnets] if isinstance(private_subnets, str) else private_subnets + ) if not len(private_subnets) > 2: - pytest.fail("Not enough subnets provided, at least 3 subnets required.", False) + fail(AWS_NOT_ENOUGH_SUBNETS_PROVIDED, subjects=["Private"]) try: # query subnets @@ -307,43 +269,15 @@ def aws_private_subnets_validation( if subnet["SubnetId"] == pvt_id: missing_subnets.remove(pvt_id) if len(missing_subnets) > 0: - pytest.fail( - f"Subnets ({missing_subnets}) do not exist.", - False, + fail( + AWS_SUBNETS_DO_NOT_EXIST, subjects="Private", resources=missing_subnets ) subnets_data["private_subnets"] = subnets["Subnets"] subnets_data["private_subnets_ids"] = private_subnets except KeyError as e: - pytest.fail(f"Validation error - missing required data : {e.args[0]}", False) + fail(AWS_REQUIRED_DATA_MISSING, e.args[0]) except ec2_client.exceptions.ClientError as ce: - pytest.fail(f"Validation error - invalid subnetId : {ce.args[0]}", False) - - -@pytest.mark.aws -@pytest.mark.infra -@pytest.mark.dependency(depends=["aws_private_subnets_validation"]) -@pytest.mark.skip(reason="Suffixes are not part of config at this time.") -def aws_private_subnets_suffix_validation(config: Dict[str, Any]) -> None: - """Private subnets have the defined suffix.""" # noqa: D401,E501 - private_subnets_suffix: List[str] = get_config_value( - config, - "infra:aws:vpc:existing:private_subnets_suffix", - key_missing_message="No private subnets suffix defined for config option: {0}", - data_expected_error_message="No private subnets suffix was provided for config option: {0}", # noqa: E501 - ) - try: - subnets_wo_valid_suffix = [] - for subnet_id in subnets_data["private_subnets_ids"]: - if not subnet_id.endswith(private_subnets_suffix): - subnets_wo_valid_suffix.append(subnet_id) - - if len(subnets_wo_valid_suffix) > 0: - pytest.fail( - f"Subnets ({subnets_wo_valid_suffix}) without valid suffix ({private_subnets_suffix}).", # noqa: E501 - False, - ) - except KeyError as e: - pytest.fail(f"Validation error - missing required data : {e.args[0]}", False) + fail(AWS_INVALID_SUBNET_ID, subjects=["Private", ce.args[0]]) @pytest.mark.aws @@ -359,13 +293,10 @@ def aws_private_subnets_availablity_zone_validation() -> None: # minimum two availability zones validations if len(azs) <= 1: - pytest.fail( - """Not enough availability zones, subnets should be in - at least 2 availability zones.""", - False, - ) + fail(AWS_NOT_ENOUGH_AZ_FOR_SUBNETS, subjects=["Private"]) + except KeyError as e: - pytest.fail(f"Validation error - missing required data : {e.args[0]}", False) + fail(AWS_REQUIRED_DATA_MISSING, e.args[0]) @pytest.mark.aws @@ -392,8 +323,6 @@ def aws_private_subnets_route_validation( vpc_id: List[str] = get_config_value( config, "infra:aws:vpc:existing:vpc_id", - key_missing_message="No vpc id defined for config option: {0}", - data_expected_error_message="No vpc id was provided for config option: {0}", ) Filters = [ { @@ -415,22 +344,22 @@ def aws_private_subnets_route_validation( if "NatGatewayId" in route: gateway_ids.append(route["NatGatewayId"]) else: - pytest.fail( - f"""Provided Subnets {subnets_data["private_subnets_ids"]} and - or VPC {vpc_id} do not have NAT gateway(s).""", - False, + fail( + AWS_SUBNETS_OR_VPC_WITHOUT_INTERNET_GATEWAY, + subjects=["Private", vpc_id], + resources=subnets_data["public_subnets_ids"], ) if not set(igw_ids) & set(gateway_ids): - pytest.fail( - f"""Provided subnets {subnets_data["private_subnets_ids"]} do not - have NAT gateway(s).""", - False, + fail( + AWS_SUBNETS_WITHOUT_INTERNET_GATEWAY, + subjects=["Private"], + resources=subnets_data["public_subnets_ids"], ) except KeyError as e: - pytest.fail(f"Validation error - missing required data : {e.args[0]}", False) + fail(AWS_REQUIRED_DATA_MISSING, e.args[0]) except ec2_client.exceptions.ClientError as ce: - pytest.fail(f"Validation error - invalid data : {ce.args[0]}", False) + fail(AWS_INVALID_DATA, ce.args[0]) @pytest.mark.aws @@ -446,12 +375,13 @@ def aws_private_subnets_range_validation() -> None: subnets_wo_valid_range.append(subnet["SubnetId"]) if len(subnets_wo_valid_range) > 0: - pytest.fail( - f"Private subnets ({subnets_wo_valid_range}) without valid required range.", # noqa: E501 - False, + fail( + AWS_SUBNETS_WITHOUT_VALID_RANGE, + subjects=["Private"], + resources=subnets_wo_valid_range, ) except KeyError as e: - pytest.fail(f"Validation error - missing required data : {e.args[0]}", False) + fail(AWS_REQUIRED_DATA_MISSING, e.args[0]) @pytest.mark.aws @@ -475,13 +405,13 @@ def aws_private_subnets_tags_validation() -> None: subnet_missing_tags = [k for k, v in subnets_w_valid_tag.items() if not v] if len(subnet_missing_tags) > 0: - pytest.fail( - f"""Private subnet(s) {subnet_missing_tags} missing tag - 'kubernetes.io/role/internal-elb'.""", - False, + fail( + AWS_SUBNETS_MISSING_K8S_LB_TAG, + subjects=["Private"], + resources=subnet_missing_tags, ) except KeyError as e: - pytest.fail(f"Validation error - missing required data : {e.args[0]}", False) + fail(AWS_REQUIRED_DATA_MISSING, e.args[0]) @pytest.mark.aws @@ -497,8 +427,6 @@ def aws_vpc_subnets_validation( vpc_id: List[str] = get_config_value( config, "infra:aws:vpc:existing:vpc_id", - key_missing_message="No vpc id defined for config option: {0}", - data_expected_error_message="No vpc id was provided for config option: {0}", ) try: vpc_d = ec2_client.describe_vpcs(VpcIds=[vpc_id]) @@ -515,9 +443,10 @@ def aws_vpc_subnets_validation( missing_subnets.append(public_id) if len(missing_subnets) > 0: - pytest.fail( - f"""Subnets {missing_subnets} are not associated to provided vpc.""", - False, + fail( + AWS_SUBNETS_NOT_PART_OF_VPC, + subjects=[vpc_id], + resources=missing_subnets, ) # DNS names and DNS resolution enabled @@ -529,8 +458,8 @@ def aws_vpc_subnets_validation( )["EnableDnsHostnames"]["Value"] if not (enable_dns_hostnames and enable_dns_support): - pytest.fail("DNS support not enabled for provided vpc.", False) + fail(AWS_DNS_SUPPORT_NOT_ENABLED_FOR_VPC, subjects=[vpc_id]) except KeyError as e: - pytest.fail(f"Validation error - missing required data : {e.args[0]}", False) + fail(AWS_REQUIRED_DATA_MISSING, e.args[0]) except ec2_client.exceptions.ClientError as ce: - pytest.fail(f"Validation error - invalid data : {ce.args[0]}", False) + fail(AWS_INVALID_DATA, ce.args[0]) diff --git a/cdpctl/validation/issue_templates.yml b/cdpctl/validation/issue_templates.yml new file mode 100644 index 0000000..4a42ad7 --- /dev/null +++ b/cdpctl/validation/issue_templates.yml @@ -0,0 +1,33 @@ +--- +id: CONFIG_OPTION_KEY_NOT_DEFINED +summary: "The config option {0} is missing" +--- +id: CONFIG_OPTION_DATA_NOT_DEFINED +summary: No entry was provided for config option {0} +--- +id: CONFIG_OPTION_PARENT_PATH_NOT_DEFINED +summary: Unable to find key path +render_type: list +--- +id: AWS_ROLE_MISSING +summary: Unable to find the following IAM role. +docs_link: https://docs.cloudera.com/cdp/latest/requirements-aws/topics/mc-aws-req-credential.html +render_type: list +--- +id: AWS_PROFILE_CONFIG_NOT_DEFINED +summary: No profile config option defined {0} +--- +id: AWS_PROFILE_NOT_DEFINED +summary: No profile was defined for for config option {0} +--- +id: AWS_REGION_CONFIG_NOT_DEFINED +summary: No region config option defined {0} +--- +id: AWS_REGION_NOT_DEFINED +summary: No region was defined for for config option {0} +--- +id: AWS_MISSING_ACTIONS +summary: "The following IAM actions are required:" +--- +id: AWS_INSTANCE_PROFILE_NOT_FOUND +summary: The IAM Instance Profile {0} was not found diff --git a/cdpctl/validation/issues.py b/cdpctl/validation/issues.py new file mode 100644 index 0000000..f3589ef --- /dev/null +++ b/cdpctl/validation/issues.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +### +# CLOUDERA CDP Control (cdpctl) +# +# (C) Cloudera, Inc. 2021-2021 +# All rights reserved. +# +# Applicable Open Source License: GNU AFFERO GENERAL PUBLIC LICENSE +# +# NOTE: Cloudera open source products are modular software products +# made up of hundreds of individual components, each of which was +# individually copyrighted. Each Cloudera open source product is a +# collective work under U.S. Copyright Law. Your license to use the +# collective work is as provided in your written agreement with +# Cloudera. Used apart from the collective work, this file is +# licensed for your use pursuant to the open source license +# identified above. +# +# This code is provided to you pursuant a written agreement with +# (i) Cloudera, Inc. or (ii) a third-party authorized to distribute +# this code. If you do not have a written agreement with Cloudera nor +# with an authorized and properly licensed third party, you do not +# have any rights to access nor to use this code. +# +# Absent a written agreement with Cloudera, Inc. (“Cloudera”) to the +# contrary, A) CLOUDERA PROVIDES THIS CODE TO YOU WITHOUT WARRANTIES OF ANY +# KIND; (B) CLOUDERA DISCLAIMS ANY AND ALL EXPRESS AND IMPLIED +# WARRANTIES WITH RESPECT TO THIS CODE, INCLUDING BUT NOT LIMITED TO +# IMPLIED WARRANTIES OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY AND +# FITNESS FOR A PARTICULAR PURPOSE; (C) CLOUDERA IS NOT LIABLE TO YOU, +# AND WILL NOT DEFEND, INDEMNIFY, NOR HOLD YOU HARMLESS FOR ANY CLAIMS +# ARISING FROM OR RELATED TO THE CODE; AND (D)WITH RESPECT TO YOUR EXERCISE +# OF ANY RIGHTS GRANTED TO YOU FOR THE CODE, CLOUDERA IS NOT LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, PUNITIVE OR +# CONSEQUENTIAL DAMAGES INCLUDING, BUT NOT LIMITED TO, DAMAGES +# RELATED TO LOST REVENUE, LOST PROFITS, LOSS OF INCOME, LOSS OF +# BUSINESS ADVANTAGE OR UNAVAILABILITY, OR LOSS OR CORRUPTION OF +# DATA. +# +# Source File Name: issues.py +### +# flake8: noqa +# pylint: skip-file + +# THIS FILE IS GENERATED. DO NOT UPDATE BY HAND. +# Use the update_issue_templates.py script +"""Issue Templates.""" + +CONFIG_OPTION_KEY_NOT_DEFINED = "CONFIG_OPTION_KEY_NOT_DEFINED" + +CONFIG_OPTION_DATA_NOT_DEFINED = "CONFIG_OPTION_DATA_NOT_DEFINED" + +CONFIG_OPTION_PARENT_PATH_NOT_DEFINED = "CONFIG_OPTION_PARENT_PATH_NOT_DEFINED" + +AWS_ROLE_MISSING = "AWS_ROLE_MISSING" + +AWS_PROFILE_CONFIG_NOT_DEFINED = "AWS_PROFILE_CONFIG_NOT_DEFINED" + +AWS_PROFILE_NOT_DEFINED = "AWS_PROFILE_NOT_DEFINED" + +AWS_REGION_CONFIG_NOT_DEFINED = "AWS_REGION_CONFIG_NOT_DEFINED" + +AWS_REGION_NOT_DEFINED = "AWS_REGION_NOT_DEFINED" + +AWS_MISSING_ACTIONS = "AWS_MISSING_ACTIONS" + +AWS_INSTANCE_PROFILE_NOT_FOUND = "AWS_INSTANCE_PROFILE_NOT_FOUND" diff --git a/cdpctl/validation/renderer/__init__.py b/cdpctl/validation/renderer/__init__.py new file mode 100644 index 0000000..1166b3d --- /dev/null +++ b/cdpctl/validation/renderer/__init__.py @@ -0,0 +1,113 @@ +### +# CLOUDERA CDP Control (cdpctl) +# +# (C) Cloudera, Inc. 2021-2021 +# All rights reserved. +# +# Applicable Open Source License: GNU AFFERO GENERAL PUBLIC LICENSE +# +# NOTE: Cloudera open source products are modular software products +# made up of hundreds of individual components, each of which was +# individually copyrighted. Each Cloudera open source product is a +# collective work under U.S. Copyright Law. Your license to use the +# collective work is as provided in your written agreement with +# Cloudera. Used apart from the collective work, this file is +# licensed for your use pursuant to the open source license +# identified above. +# +# This code is provided to you pursuant a written agreement with +# (i) Cloudera, Inc. or (ii) a third-party authorized to distribute +# this code. If you do not have a written agreement with Cloudera nor +# with an authorized and properly licensed third party, you do not +# have any rights to access nor to use this code. +# +# Absent a written agreement with Cloudera, Inc. (“Cloudera”) to the +# contrary, A) CLOUDERA PROVIDES THIS CODE TO YOU WITHOUT WARRANTIES OF ANY +# KIND; (B) CLOUDERA DISCLAIMS ANY AND ALL EXPRESS AND IMPLIED +# WARRANTIES WITH RESPECT TO THIS CODE, INCLUDING BUT NOT LIMITED TO +# IMPLIED WARRANTIES OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY AND +# FITNESS FOR A PARTICULAR PURPOSE; (C) CLOUDERA IS NOT LIABLE TO YOU, +# AND WILL NOT DEFEND, INDEMNIFY, NOR HOLD YOU HARMLESS FOR ANY CLAIMS +# ARISING FROM OR RELATED TO THE CODE; AND (D)WITH RESPECT TO YOUR EXERCISE +# OF ANY RIGHTS GRANTED TO YOU FOR THE CODE, CLOUDERA IS NOT LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, PUNITIVE OR +# CONSEQUENTIAL DAMAGES INCLUDING, BUT NOT LIMITED TO, DAMAGES +# RELATED TO LOST REVENUE, LOST PROFITS, LOSS OF INCOME, LOSS OF +# BUSINESS ADVANTAGE OR UNAVAILABILITY, OR LOSS OR CORRUPTION OF +# DATA. +# +# Source File Name: __init__.py +### +"""Base Renderer Module.""" +import json + +from jinja2 import Environment, PackageLoader, select_autoescape + +from cdpctl.utils import smart_open +from cdpctl.validation import UnrecoverableValidationError + + +class ValidationRenderer: + """Base renderer class.""" + + def render(self, issues, output_file): + """Render the issues found.""" + pass + + +class TextValidationRenderer(ValidationRenderer): + """Text renderer class.""" + + def render(self, issues, output_file): + """Render the issues found as a text format.""" + env = Environment( + loader=PackageLoader( + package_name="cdpctl.validation.renderer", package_path="templates" + ), + autoescape=select_autoescape(), + ) + template = env.get_template("text.j2") + with smart_open(output_file) as f: + f.write(template.render(issues=issues)) + + +class JsonValidationRenderer(ValidationRenderer): + """Json renderer class.""" + + def render(self, issues, output_file): + """Render the issues found as a json format.""" + + json_issues = [] + for key, value in issues.items(): + json_rep = {} + json_rep["validation"] = key + json_rep["problems"] = [] + json_rep["warnings"] = [] + for problem in value["problem"]: + json_rep["problems"].append( + {"message": problem.message, "resources": problem.resources} + ) + for warning in value["warning"]: + json_rep["warnings"].append( + {"message": warning.message, "resources": warning.resources} + ) + json_issues.append(json_rep) + + with smart_open(output_file) as f: + f.write( + json.dumps( + json_issues, + indent=4, + ) + ) + + +def get_renderer(output_format: str) -> ValidationRenderer: + """Get the correct renderer for the output format.""" + if output_format == "text": + return TextValidationRenderer() + if output_format == "json": + return JsonValidationRenderer() + raise UnrecoverableValidationError( + f"Unknown validation output format: {output_format}." + ) diff --git a/cdpctl/validation/renderer/templates/text.j2 b/cdpctl/validation/renderer/templates/text.j2 new file mode 100644 index 0000000..ca06704 --- /dev/null +++ b/cdpctl/validation/renderer/templates/text.j2 @@ -0,0 +1,37 @@ +{%- if issues|length %} +--- Issues Found --- +{% for validation_name, issue_types in issues.items() %} +{% for n in range(validation_name|length+13)-%}-{%endfor%} +Validation: {{ validation_name }} +{% if issue_types["problem"]|length -%} +Problems Found: +{% for problem in issue_types["problem"] -%} +* {{problem.message}} + {%- if problem.render_type == "list" -%} + {%- for resource in problem.resources %} + - {{ resource }} + {%- endfor %} + {%- endif -%} +{%- if problem.docs_link %} + Please see the following documentation for more information: + {{problem.docs_link}} +{%- endif -%} +{% endfor %} +{%- endif %} +{% if issue_types["warning"]|length %} +Warnings Found: +{% for warning in issue_types["warning"] -%} +* {{warning.message}} + {%- if warning.render_type == "list" -%} + {%- for resource in warning.resources %} + - {{ resource }} + {%- endfor %} + {%- endif %} +{% if warning.docs_link %} + Please see the following documentation for more information: + {{warning.docs_link}} +{%- endif %} +{% endfor %} +{%- endif %} +{% endfor %} +{% endif -%} diff --git a/requirements.txt b/requirements.txt index f3718f5..511e60f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,7 @@ boto3_type_annotations==0.3.1 click==8.0.1 emoji==1.2.0 Jinja2==2.11.3 +marshmallow==3.13.0 progressbar2==3.53.1 pytest==6.2.4 pytest-dependency==0.5.1 diff --git a/scripts/templates/issues.py.j2 b/scripts/templates/issues.py.j2 new file mode 100644 index 0000000..7b0686a --- /dev/null +++ b/scripts/templates/issues.py.j2 @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +### +# CLOUDERA CDP Control (cdpctl) +# +# (C) Cloudera, Inc. 2021-2021 +# All rights reserved. +# +# Applicable Open Source License: GNU AFFERO GENERAL PUBLIC LICENSE +# +# NOTE: Cloudera open source products are modular software products +# made up of hundreds of individual components, each of which was +# individually copyrighted. Each Cloudera open source product is a +# collective work under U.S. Copyright Law. Your license to use the +# collective work is as provided in your written agreement with +# Cloudera. Used apart from the collective work, this file is +# licensed for your use pursuant to the open source license +# identified above. +# +# This code is provided to you pursuant a written agreement with +# (i) Cloudera, Inc. or (ii) a third-party authorized to distribute +# this code. If you do not have a written agreement with Cloudera nor +# with an authorized and properly licensed third party, you do not +# have any rights to access nor to use this code. +# +# Absent a written agreement with Cloudera, Inc. (“Cloudera”) to the +# contrary, A) CLOUDERA PROVIDES THIS CODE TO YOU WITHOUT WARRANTIES OF ANY +# KIND; (B) CLOUDERA DISCLAIMS ANY AND ALL EXPRESS AND IMPLIED +# WARRANTIES WITH RESPECT TO THIS CODE, INCLUDING BUT NOT LIMITED TO +# IMPLIED WARRANTIES OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY AND +# FITNESS FOR A PARTICULAR PURPOSE; (C) CLOUDERA IS NOT LIABLE TO YOU, +# AND WILL NOT DEFEND, INDEMNIFY, NOR HOLD YOU HARMLESS FOR ANY CLAIMS +# ARISING FROM OR RELATED TO THE CODE; AND (D)WITH RESPECT TO YOUR EXERCISE +# OF ANY RIGHTS GRANTED TO YOU FOR THE CODE, CLOUDERA IS NOT LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, PUNITIVE OR +# CONSEQUENTIAL DAMAGES INCLUDING, BUT NOT LIMITED TO, DAMAGES +# RELATED TO LOST REVENUE, LOST PROFITS, LOSS OF INCOME, LOSS OF +# BUSINESS ADVANTAGE OR UNAVAILABILITY, OR LOSS OR CORRUPTION OF +# DATA. +# +# Source File Name: issues.py +### +# flake8: noqa +# pylint: skip-file + +# THIS FILE IS GENERATED. DO NOT UPDATE BY HAND. +# Use the update_issue_templates.py script +"""Issue Templates.""" +{% for issue_template in issue_templates %} +{{ issue_template.id }} = "{{ issue_template.id }}" +{% endfor %} diff --git a/scripts/update_issue_templates.py b/scripts/update_issue_templates.py new file mode 100644 index 0000000..11665f4 --- /dev/null +++ b/scripts/update_issue_templates.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +### +# CLOUDERA CDP Control (cdpctl) +# +# (C) Cloudera, Inc. 2021-2021 +# All rights reserved. +# +# Applicable Open Source License: GNU AFFERO GENERAL PUBLIC LICENSE +# +# NOTE: Cloudera open source products are modular software products +# made up of hundreds of individual components, each of which was +# individually copyrighted. Each Cloudera open source product is a +# collective work under U.S. Copyright Law. Your license to use the +# collective work is as provided in your written agreement with +# Cloudera. Used apart from the collective work, this file is +# licensed for your use pursuant to the open source license +# identified above. +# +# This code is provided to you pursuant a written agreement with +# (i) Cloudera, Inc. or (ii) a third-party authorized to distribute +# this code. If you do not have a written agreement with Cloudera nor +# with an authorized and properly licensed third party, you do not +# have any rights to access nor to use this code. +# +# Absent a written agreement with Cloudera, Inc. (“Cloudera”) to the +# contrary, A) CLOUDERA PROVIDES THIS CODE TO YOU WITHOUT WARRANTIES OF ANY +# KIND; (B) CLOUDERA DISCLAIMS ANY AND ALL EXPRESS AND IMPLIED +# WARRANTIES WITH RESPECT TO THIS CODE, INCLUDING BUT NOT LIMITED TO +# IMPLIED WARRANTIES OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY AND +# FITNESS FOR A PARTICULAR PURPOSE; (C) CLOUDERA IS NOT LIABLE TO YOU, +# AND WILL NOT DEFEND, INDEMNIFY, NOR HOLD YOU HARMLESS FOR ANY CLAIMS +# ARISING FROM OR RELATED TO THE CODE; AND (D)WITH RESPECT TO YOUR EXERCISE +# OF ANY RIGHTS GRANTED TO YOU FOR THE CODE, CLOUDERA IS NOT LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, PUNITIVE OR +# CONSEQUENTIAL DAMAGES INCLUDING, BUT NOT LIMITED TO, DAMAGES +# RELATED TO LOST REVENUE, LOST PROFITS, LOSS OF INCOME, LOSS OF +# BUSINESS ADVANTAGE OR UNAVAILABILITY, OR LOSS OR CORRUPTION OF +# DATA. +# +# Source File Name: update_issue_templates.py +### +# flake8: noqa +# pylint: skip-file +"""Update the issue templates.""" +import os +import sys + +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) + +from cdpctl.validation import ( + ISSUE_TEMPLATES_FILE, + load_issue_templates, +) +from jinja2 import ( + Environment, + FileSystemLoader, + select_autoescape, +) + + +VALIDATIONS_ROOT = "cdpctl/validation" +ISSUE_OUTPUT_FILE = "issues.py" + + +def render_template_for_dir(path): + issue_templates = load_issue_templates(os.path.join(path, ISSUE_TEMPLATES_FILE)) + template.stream(issue_templates=issue_templates).dump( + os.path.join(path, ISSUE_OUTPUT_FILE) + ) + + +basedir = os.path.abspath( + os.path.join(os.path.basename(__file__), "..", VALIDATIONS_ROOT) +) + +env = Environment( + loader=FileSystemLoader(os.path.join(os.path.dirname(__file__), "templates")), + autoescape=select_autoescape(), +) +template = env.get_template("issues.py.j2") + + +for root, dirs, files in os.walk(basedir): + if ISSUE_TEMPLATES_FILE in files: + print(f"Processing issue templates in dir {root}") + render_template_for_dir(root) diff --git a/tests/validation/infra/test_validate_aws_subnets.py b/tests/validation/infra/test_validate_aws_subnets.py index 8a35688..163bd4b 100644 --- a/tests/validation/infra/test_validate_aws_subnets.py +++ b/tests/validation/infra/test_validate_aws_subnets.py @@ -52,13 +52,11 @@ aws_private_subnets_availablity_zone_validation, aws_private_subnets_range_validation, aws_private_subnets_route_validation, - aws_private_subnets_suffix_validation, aws_private_subnets_tags_validation, aws_private_subnets_validation, aws_public_subnets_availablity_zone_validation, aws_public_subnets_range_validation, aws_public_subnets_route_validation, - aws_public_subnets_suffix_validation, aws_public_subnets_tags_validation, aws_public_subnets_validation, aws_vpc_subnets_validation, @@ -211,45 +209,6 @@ def test_aws_public_subnets_validation_failure( func(config, ec2_client) -def test_aws_public_subnets_suffix_validation_success( - ec2_client: EC2Client, -) -> None: - """Unit test public subnets suffix success.""" - config = get_config( - public_subnet_ids_val=public_subnet_ids, public_suffix_val="cdp" - ) - stubber = Stubber(ec2_client) - stubber.add_response( - "describe_subnets", - sample_public_subnets_response, - expected_params={"SubnetIds": public_subnet_ids}, - ) - with stubber: - func = expect_validation_success(aws_public_subnets_validation) - func(config, ec2_client) - with stubber: - func = expect_validation_success(aws_public_subnets_suffix_validation) - func(config) - - -def test_aws_public_subnets_suffix_validation_failure( - ec2_client: EC2Client, -) -> None: - """Unit test public subnets suffix failure.""" - config = get_config( - public_subnet_ids_val=public_subnet_ids, public_suffix_val="fail" - ) - stubber = Stubber(ec2_client) - stubber.add_response( - "describe_subnets", - sample_public_subnets_response, - expected_params={"SubnetIds": public_subnet_ids}, - ) - with stubber: - func = expect_validation_failure(aws_public_subnets_suffix_validation) - func(config) - - def test_aws_public_subnets_availablity_zone_validation_success( ec2_client: EC2Client, ) -> None: @@ -627,44 +586,6 @@ def test_aws_private_subnets_validation_failure(ec2_client: EC2Client) -> None: func(config, ec2_client) -def test_aws_private_subnets_suffix_validation_success(ec2_client: EC2Client) -> None: - """Unit test private subnets suffix success.""" - config = get_config( - private_subnet_ids_val=private_subnet_ids, private_suffix_val="cdp" - ) - stubber = Stubber(ec2_client) - stubber.add_response( - "describe_subnets", - sample_private_subnets_response, - expected_params={"SubnetIds": private_subnet_ids}, - ) - with stubber: - func = expect_validation_success(aws_private_subnets_validation) - func(config, ec2_client) - with stubber: - func = expect_validation_success(aws_private_subnets_suffix_validation) - func(config) - - -def test_aws_private_subnets_suffix_validation_failure(ec2_client: EC2Client) -> None: - """Unit test private subnets suffix failure.""" - config = get_config( - private_subnet_ids_val=private_subnet_ids, private_suffix_val="fail" - ) - stubber = Stubber(ec2_client) - stubber.add_response( - "describe_subnets", - sample_private_subnets_response, - expected_params={"SubnetIds": private_subnet_ids}, - ) - with stubber: - func = expect_validation_success(aws_private_subnets_validation) - func(config, ec2_client) - with stubber: - func = expect_validation_failure(aws_private_subnets_suffix_validation) - func(config) - - def test_aws_private_subnets_availablity_zone_validation_success( ec2_client: EC2Client, ) -> None: