From 03d038152715653ea4f605a778abd7957ef99979 Mon Sep 17 00:00:00 2001 From: Anurag Patro Date: Fri, 27 Aug 2021 22:54:37 +0530 Subject: [PATCH] validation - s3 bucket in same region as env Signed-off-by: Anurag Patro --- cdpctl/validation/infra/issue_templates.yml | 4 ++ cdpctl/validation/infra/issues.py | 2 + .../infra/validate_aws_s3_locations.py | 41 ++++++++++++----- .../infra/test_validate_aws_s3_locations.py | 46 ++++++++++--------- 4 files changed, 60 insertions(+), 33 deletions(-) diff --git a/cdpctl/validation/infra/issue_templates.yml b/cdpctl/validation/infra/issue_templates.yml index fc1fb7a..3966069 100644 --- a/cdpctl/validation/infra/issue_templates.yml +++ b/cdpctl/validation/infra/issue_templates.yml @@ -169,3 +169,7 @@ docs_link: https://docs.cloudera.com/cdp/latest/requirements-aws/topics/mc-aws-r id: AWS_SUBNETS_WITH_PUBLIC_IPS_ENABLED summary: "These {0} Subnets must have 'Auto-assign Public IPs' disabled for a fully-private network configuration." render_type: list +--- +id: AWS_S3_BUCKET_NOT_IN_SAME_REGION_AS_ENVIRONMENT +summary: "S3 bucket {0} does not exist in the same region as the environment." +docs_link: https://docs.cloudera.com/cdp/latest/requirements-aws/topics/mc-aws-req-s3.html diff --git a/cdpctl/validation/infra/issues.py b/cdpctl/validation/infra/issues.py index bb22d12..78c4819 100644 --- a/cdpctl/validation/infra/issues.py +++ b/cdpctl/validation/infra/issues.py @@ -123,3 +123,5 @@ AWS_VPC_NOT_FOUND_IN_ACCOUNT = "AWS_VPC_NOT_FOUND_IN_ACCOUNT" AWS_SUBNETS_WITH_PUBLIC_IPS_ENABLED = "AWS_SUBNETS_WITH_PUBLIC_IPS_ENABLED" + +AWS_S3_BUCKET_NOT_IN_SAME_REGION_AS_ENVIRONMENT = "AWS_S3_BUCKET_NOT_IN_SAME_REGION_AS_ENVIRONMENT" diff --git a/cdpctl/validation/infra/validate_aws_s3_locations.py b/cdpctl/validation/infra/validate_aws_s3_locations.py index da537e1..f3516ff 100644 --- a/cdpctl/validation/infra/validate_aws_s3_locations.py +++ b/cdpctl/validation/infra/validate_aws_s3_locations.py @@ -57,6 +57,7 @@ AWS_S3_BUCKET_DOES_NOT_EXIST, AWS_S3_BUCKET_FORBIDDEN_ACCESS, AWS_S3_BUCKET_INVALID, + AWS_S3_BUCKET_NOT_IN_SAME_REGION_AS_ENVIRONMENT, ) @@ -83,7 +84,11 @@ def aws_s3_data_bucket_exists(config: Dict[str, Any], s3_client: S3Client) -> No config, "infra:aws:vpc:existing:storage:data", ) - aws_s3_bucket_exists(data_bucket_url, s3_client) + region: str = get_config_value( + config, + "infra:aws:region", + ) + aws_s3_bucket_exists(region, data_bucket_url, s3_client) @pytest.mark.aws @@ -103,7 +108,11 @@ def aws_s3_logs_bucket_exists(config: Dict[str, Any], s3_client: S3Client) -> No config, "infra:aws:vpc:existing:storage:logs", ) - aws_s3_bucket_exists(logs_bucket_url, s3_client) + region: str = get_config_value( + config, + "infra:aws:region", + ) + aws_s3_bucket_exists(region, logs_bucket_url, s3_client) @pytest.mark.aws @@ -124,11 +133,15 @@ def aws_s3_backup_bucket_exists(config: Dict[str, Any], s3_client: S3Client) -> config, "infra:aws:vpc:existing:storage:backup", ) + region: str = get_config_value( + config, + "infra:aws:region", + ) # TODO: Handle a specific parameter for backup S3 location once it exists - aws_s3_bucket_exists(backup_bucket_url, s3_client) + aws_s3_bucket_exists(region, backup_bucket_url, s3_client) -def aws_s3_bucket_exists(bucket_url: str, s3_client: S3Client) -> None: +def aws_s3_bucket_exists(region: str, bucket_url: str, s3_client: S3Client) -> None: """Check to see if the s3 bucket exists.""" if not is_valid_s3a_url(bucket_url): fail(AWS_S3_BUCKET_INVALID, bucket_url) @@ -136,14 +149,18 @@ def aws_s3_bucket_exists(bucket_url: str, s3_client: S3Client) -> None: # get bucket name bucket_name = parse_arn(convert_s3a_to_arn(bucket_url))["resource_type"] - # check if s3 bucket exists + # check if bucket exists in same region try: - s3_client.head_bucket(Bucket=bucket_name) + if ( + s3_client.get_bucket_location(Bucket=bucket_name)["LocationConstraint"] + != region + ): + fail(AWS_S3_BUCKET_NOT_IN_SAME_REGION_AS_ENVIRONMENT, bucket_name) except botocore.exceptions.ClientError as e: - # If a client error is thrown, then check that it was a 404 error. - # If it was a 404 error, then the bucket does not exist. - error_code = int(e.response["Error"]["Code"]) - if error_code == 403: - fail(AWS_S3_BUCKET_FORBIDDEN_ACCESS, bucket_name) - elif error_code == 404: + # if a client error is thrown, check if it is a NoSuchBucket error + # if NoSuchBucket error thrown, the bucket does not exist + error_code = e.response["Error"]["Code"] + if error_code == "NoSuchBucket": fail(AWS_S3_BUCKET_DOES_NOT_EXIST, bucket_name) + elif int(error_code) == 403: + fail(AWS_S3_BUCKET_FORBIDDEN_ACCESS, bucket_name) diff --git a/tests/validation/infra/test_validate_aws_s3_locations.py b/tests/validation/infra/test_validate_aws_s3_locations.py index ab37704..b97fc4a 100644 --- a/tests/validation/infra/test_validate_aws_s3_locations.py +++ b/tests/validation/infra/test_validate_aws_s3_locations.py @@ -67,13 +67,16 @@ def test_aws_s3_data_bucket_exists(s3_client: S3Client) -> None: """Test the check of an existing s3 bucket.""" config: Dict[str, Any] = { "infra": { - "aws": {"vpc": {"existing": {"storage": {"data": "s3a://mybucket/data"}}}} + "aws": { + "region": "us-west-2", + "vpc": {"existing": {"storage": {"data": "s3a://mybucket/data"}}}, + } } } stubber = Stubber(s3_client) stubber.add_response( - "head_bucket", - {"ResponseMetadata": {}}, + "get_bucket_location", + {"LocationConstraint": "us-west-2"}, expected_params={"Bucket": "mybucket"}, ) stubber.activate() @@ -91,10 +94,9 @@ def test_aws_s3_data_bucket_missing(s3_client: S3Client) -> None: } stubber = Stubber(s3_client) stubber.add_client_error( - "head_bucket", - service_error_code="404", - service_message="Bucket Not Found", - http_status_code=404, + "get_bucket_location", + service_error_code="NoSuchBucket", + service_message="The specified bucket does not exist", expected_params={"Bucket": "mybucket"}, ) stubber.activate() @@ -123,13 +125,16 @@ def test_aws_s3_logs_bucket_exists(s3_client: S3Client) -> None: """Test the check of an existing s3 bucket.""" config: Dict[str, Any] = { "infra": { - "aws": {"vpc": {"existing": {"storage": {"logs": "s3a://mybucket/logs"}}}} + "aws": { + "region": "us-west-2", + "vpc": {"existing": {"storage": {"logs": "s3a://mybucket/logs"}}}, + } } } stubber = Stubber(s3_client) stubber.add_response( - "head_bucket", - {"ResponseMetadata": {}}, + "get_bucket_location", + {"LocationConstraint": "us-west-2"}, expected_params={"Bucket": "mybucket"}, ) stubber.activate() @@ -147,10 +152,9 @@ def test_aws_s3_logs_bucket_missing(s3_client: S3Client) -> None: } stubber = Stubber(s3_client) stubber.add_client_error( - "head_bucket", - service_error_code="404", - service_message="Bucket Not Found", - http_status_code=404, + "get_bucket_location", + service_error_code="NoSuchBucket", + service_message="The specified bucket does not exist", expected_params={"Bucket": "mybucket"}, ) stubber.activate() @@ -180,14 +184,15 @@ def test_aws_s3_backup_bucket_exists(s3_client: S3Client) -> None: config: Dict[str, Any] = { "infra": { "aws": { - "vpc": {"existing": {"storage": {"backup": "s3a://mybucket/backup"}}} + "region": "us-west-2", + "vpc": {"existing": {"storage": {"backup": "s3a://mybucket/backup"}}}, } } } stubber = Stubber(s3_client) stubber.add_response( - "head_bucket", - {"ResponseMetadata": {}}, + "get_bucket_location", + {"LocationConstraint": "us-west-2"}, expected_params={"Bucket": "mybucket"}, ) stubber.activate() @@ -207,10 +212,9 @@ def test_aws_s3_backup_bucket_missing(s3_client: S3Client) -> None: } stubber = Stubber(s3_client) stubber.add_client_error( - "head_bucket", - service_error_code="404", - service_message="Bucket Not Found", - http_status_code=404, + "get_bucket_location", + service_error_code="NoSuchBucket", + service_message="The specified bucket does not exist", expected_params={"Bucket": "mybucket"}, ) stubber.activate()