Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Validation: S3 Bucket must exist in the same region as the environment #36

Merged
merged 1 commit into from
Aug 27, 2021
Merged

Validation: S3 Bucket must exist in the same region as the environment #36

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions cdpctl/validation/infra/issue_templates.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -169,3 +169,7 @@ docs_link: https://docs.cloudera.com/cdp/latest/requirements-aws/topics/mc-aws-r
id: AWS_SUBNETS_WITH_PUBLIC_IPS_ENABLED
summary: "These {0} Subnets must have 'Auto-assign Public IPs' disabled for a fully-private network configuration."
render_type: list
---
id: AWS_S3_BUCKET_NOT_IN_SAME_REGION_AS_ENVIRONMENT
summary: "S3 bucket {0} does not exist in the same region as the environment."
docs_link: https://docs.cloudera.com/cdp/latest/requirements-aws/topics/mc-aws-req-s3.html
2 changes: 2 additions & 0 deletions cdpctl/validation/infra/issues.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,3 +123,5 @@
AWS_VPC_NOT_FOUND_IN_ACCOUNT = "AWS_VPC_NOT_FOUND_IN_ACCOUNT"

AWS_SUBNETS_WITH_PUBLIC_IPS_ENABLED = "AWS_SUBNETS_WITH_PUBLIC_IPS_ENABLED"

AWS_S3_BUCKET_NOT_IN_SAME_REGION_AS_ENVIRONMENT = "AWS_S3_BUCKET_NOT_IN_SAME_REGION_AS_ENVIRONMENT"
41 changes: 29 additions & 12 deletions cdpctl/validation/infra/validate_aws_s3_locations.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@
AWS_S3_BUCKET_DOES_NOT_EXIST,
AWS_S3_BUCKET_FORBIDDEN_ACCESS,
AWS_S3_BUCKET_INVALID,
AWS_S3_BUCKET_NOT_IN_SAME_REGION_AS_ENVIRONMENT,
)


Expand All @@ -83,7 +84,11 @@ def aws_s3_data_bucket_exists(config: Dict[str, Any], s3_client: S3Client) -> No
config,
"infra:aws:vpc:existing:storage:data",
)
aws_s3_bucket_exists(data_bucket_url, s3_client)
region: str = get_config_value(
config,
"infra:aws:region",
)
aws_s3_bucket_exists(region, data_bucket_url, s3_client)


@pytest.mark.aws
Expand All @@ -103,7 +108,11 @@ def aws_s3_logs_bucket_exists(config: Dict[str, Any], s3_client: S3Client) -> No
config,
"infra:aws:vpc:existing:storage:logs",
)
aws_s3_bucket_exists(logs_bucket_url, s3_client)
region: str = get_config_value(
config,
"infra:aws:region",
)
aws_s3_bucket_exists(region, logs_bucket_url, s3_client)


@pytest.mark.aws
Expand All @@ -124,26 +133,34 @@ def aws_s3_backup_bucket_exists(config: Dict[str, Any], s3_client: S3Client) ->
config,
"infra:aws:vpc:existing:storage:backup",
)
region: str = get_config_value(
config,
"infra:aws:region",
)
# TODO: Handle a specific parameter for backup S3 location once it exists
aws_s3_bucket_exists(backup_bucket_url, s3_client)
aws_s3_bucket_exists(region, backup_bucket_url, s3_client)


def aws_s3_bucket_exists(bucket_url: str, s3_client: S3Client) -> None:
def aws_s3_bucket_exists(region: str, bucket_url: str, s3_client: S3Client) -> None:
"""Check to see if the s3 bucket exists."""
if not is_valid_s3a_url(bucket_url):
fail(AWS_S3_BUCKET_INVALID, bucket_url)

# get bucket name
bucket_name = parse_arn(convert_s3a_to_arn(bucket_url))["resource_type"]

# check if s3 bucket exists
# check if bucket exists in same region
try:
s3_client.head_bucket(Bucket=bucket_name)
if (
s3_client.get_bucket_location(Bucket=bucket_name)["LocationConstraint"]
!= region
):
fail(AWS_S3_BUCKET_NOT_IN_SAME_REGION_AS_ENVIRONMENT, bucket_name)
except botocore.exceptions.ClientError as e:
# If a client error is thrown, then check that it was a 404 error.
# If it was a 404 error, then the bucket does not exist.
error_code = int(e.response["Error"]["Code"])
if error_code == 403:
fail(AWS_S3_BUCKET_FORBIDDEN_ACCESS, bucket_name)
elif error_code == 404:
# if a client error is thrown, check if it is a NoSuchBucket error
# if NoSuchBucket error thrown, the bucket does not exist
error_code = e.response["Error"]["Code"]
if error_code == "NoSuchBucket":
fail(AWS_S3_BUCKET_DOES_NOT_EXIST, bucket_name)
elif int(error_code) == 403:
fail(AWS_S3_BUCKET_FORBIDDEN_ACCESS, bucket_name)
46 changes: 25 additions & 21 deletions tests/validation/infra/test_validate_aws_s3_locations.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,13 +67,16 @@ def test_aws_s3_data_bucket_exists(s3_client: S3Client) -> None:
"""Test the check of an existing s3 bucket."""
config: Dict[str, Any] = {
"infra": {
"aws": {"vpc": {"existing": {"storage": {"data": "s3a://mybucket/data"}}}}
"aws": {
"region": "us-west-2",
"vpc": {"existing": {"storage": {"data": "s3a://mybucket/data"}}},
}
}
}
stubber = Stubber(s3_client)
stubber.add_response(
"head_bucket",
{"ResponseMetadata": {}},
"get_bucket_location",
{"LocationConstraint": "us-west-2"},
expected_params={"Bucket": "mybucket"},
)
stubber.activate()
Expand All @@ -91,10 +94,9 @@ def test_aws_s3_data_bucket_missing(s3_client: S3Client) -> None:
}
stubber = Stubber(s3_client)
stubber.add_client_error(
"head_bucket",
service_error_code="404",
service_message="Bucket Not Found",
http_status_code=404,
"get_bucket_location",
service_error_code="NoSuchBucket",
service_message="The specified bucket does not exist",
expected_params={"Bucket": "mybucket"},
)
stubber.activate()
Expand Down Expand Up @@ -123,13 +125,16 @@ def test_aws_s3_logs_bucket_exists(s3_client: S3Client) -> None:
"""Test the check of an existing s3 bucket."""
config: Dict[str, Any] = {
"infra": {
"aws": {"vpc": {"existing": {"storage": {"logs": "s3a://mybucket/logs"}}}}
"aws": {
"region": "us-west-2",
"vpc": {"existing": {"storage": {"logs": "s3a://mybucket/logs"}}},
}
}
}
stubber = Stubber(s3_client)
stubber.add_response(
"head_bucket",
{"ResponseMetadata": {}},
"get_bucket_location",
{"LocationConstraint": "us-west-2"},
expected_params={"Bucket": "mybucket"},
)
stubber.activate()
Expand All @@ -147,10 +152,9 @@ def test_aws_s3_logs_bucket_missing(s3_client: S3Client) -> None:
}
stubber = Stubber(s3_client)
stubber.add_client_error(
"head_bucket",
service_error_code="404",
service_message="Bucket Not Found",
http_status_code=404,
"get_bucket_location",
service_error_code="NoSuchBucket",
service_message="The specified bucket does not exist",
expected_params={"Bucket": "mybucket"},
)
stubber.activate()
Expand Down Expand Up @@ -180,14 +184,15 @@ def test_aws_s3_backup_bucket_exists(s3_client: S3Client) -> None:
config: Dict[str, Any] = {
"infra": {
"aws": {
"vpc": {"existing": {"storage": {"backup": "s3a://mybucket/backup"}}}
"region": "us-west-2",
"vpc": {"existing": {"storage": {"backup": "s3a://mybucket/backup"}}},
}
}
}
stubber = Stubber(s3_client)
stubber.add_response(
"head_bucket",
{"ResponseMetadata": {}},
"get_bucket_location",
{"LocationConstraint": "us-west-2"},
expected_params={"Bucket": "mybucket"},
)
stubber.activate()
Expand All @@ -207,10 +212,9 @@ def test_aws_s3_backup_bucket_missing(s3_client: S3Client) -> None:
}
stubber = Stubber(s3_client)
stubber.add_client_error(
"head_bucket",
service_error_code="404",
service_message="Bucket Not Found",
http_status_code=404,
"get_bucket_location",
service_error_code="NoSuchBucket",
service_message="The specified bucket does not exist",
expected_params={"Bucket": "mybucket"},
)
stubber.activate()
Expand Down