From 693e75cad048b5d21be68ca2b1e8053dc76a090f Mon Sep 17 00:00:00 2001 From: rpoluri <38321430+rpoluri@users.noreply.github.com> Date: Wed, 2 Sep 2020 13:27:48 -0500 Subject: [PATCH] Feature/hms iam wildcard (#174) * use wildcard to configure metastore iam roles * fix * arn fix * fix master user secret count * fix templates * fix allow-grant path * k8s-secret fix * fix init container commands * update changelog * remove mysql_commands template variable * remove init container image and use hms docker for init container also * fix Co-authored-by: Raj Poluri --- CHANGELOG.md | 5 + VARIABLES.md | 2 - db.tf | 8 +- iam-policy-s3-buckets.tf | 158 +--------------------------- k8s-readonly.tf | 18 ++-- k8s-readwrite.tf | 22 ++-- k8s-secrets.tf | 6 +- templates.tf | 21 ++-- templates/apiary-hms-readonly.json | 6 +- templates/apiary-hms-readwrite.json | 6 +- variables.tf | 13 --- 11 files changed, 47 insertions(+), 218 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d36164..17ff26f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## [6.5.1] - 2020-09-02 +### Changed +- [Issue 165](https://github.com/ExpediaGroup/apiary-data-lake/issues/173) Configure metastore IAM roles using apiary bucket prefix. +- Fix init container deployment. + ## [6.5.0] - 2020-08-31 ### Changed - [Issue 165](https://github.com/ExpediaGroup/apiary-data-lake/issues/165) Use init containers instead of `mysql` commands to initialize mysql users. diff --git a/VARIABLES.md b/VARIABLES.md index 6186402..ad985dc 100644 --- a/VARIABLES.md +++ b/VARIABLES.md @@ -52,8 +52,6 @@ | hms_rw_heapsize | Heapsize for the read/write Hive Metastore. Valid values: https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-cpu-memory-error.html | string | - | yes | | iam_name_root | Name to identify Hive Metastore IAM roles. | string | `hms` | no | | ingress_cidr | Generally allowed ingress CIDR list. | list | - | yes | -| init_container_image | Docker image for running HMS init container. Required if `external_database_host` is unset. | string | `` | no | -| init_container_version | Docker image version for running HMS init container. Required if `external_database_host` is unset. | string | `` | no | | instance_name | Apiary instance name to identify resources in multi-instance deployments. | string | `` | no | | k8s_docker_registry_secret| Docker Registry authentication K8s secret name. | string | `` | no | | kiam_arn | Kiam server IAM role ARN. | string | `` | no | diff --git a/db.tf b/db.tf index 8ffe91d..72664e7 100644 --- a/db.tf +++ b/db.tf @@ -92,21 +92,21 @@ resource "aws_rds_cluster_instance" "apiary_cluster_instance" { # In order to avoid resource collision when deleting & immediately recreating SecretsManager secrets in AWS, we set a random suffix on the name of the secret. # This allows us to avoid the issue of AWS's imposed 7 day recovery window. resource "random_string" "secret_name_suffix" { - count = "${var.external_database_host == "" ? var.db_instance_count : 0}" + count = var.external_database_host == "" ? 1 : 0 length = 8 special = false } resource "aws_secretsmanager_secret" "apiary_mysql_master_credentials" { - count = "${var.external_database_host == "" ? var.db_instance_count : 0}" + count = var.external_database_host == "" ? 1 : 0 name = "${local.instance_alias}_db_master_user_${random_string.secret_name_suffix[0].result}" tags = var.apiary_tags recovery_window_in_days = 0 } resource "aws_secretsmanager_secret_version" "apiary_mysql_master_credentials" { - count = "${var.external_database_host == "" ? var.db_instance_count : 0}" - secret_id = aws_secretsmanager_secret.apiary_mysql_master_credentials[0].id + count = var.external_database_host == "" ? 1 : 0 + secret_id = aws_secretsmanager_secret.apiary_mysql_master_credentials[0].id secret_string = jsonencode( map( "username", var.db_master_username, diff --git a/iam-policy-s3-buckets.tf b/iam-policy-s3-buckets.tf index 7e919fd..4b18e35 100644 --- a/iam-policy-s3-buckets.tf +++ b/iam-policy-s3-buckets.tf @@ -30,8 +30,8 @@ resource "aws_iam_role_policy" "s3_data_for_hms_readwrite" { "s3:PutObjectVersionTagging" ], "Resource": [ - "${join("\",\"", formatlist("arn:aws:s3:::%s", local.schemas_info[*]["data_bucket"]))}", - "${join("\",\"", formatlist("arn:aws:s3:::%s/*", local.schemas_info[*]["data_bucket"]))}" + "arn:aws:s3:::${local.apiary_bucket_prefix}-*", + "arn:aws:s3:::${local.apiary_bucket_prefix}-*/*" ] } ] @@ -55,8 +55,8 @@ resource "aws_iam_role_policy" "s3_data_for_hms_readonly" { "s3:List*" ], "Resource": [ - "${join("\",\"", formatlist("arn:aws:s3:::%s", local.schemas_info[*]["data_bucket"]))}", - "${join("\",\"", formatlist("arn:aws:s3:::%s/*", local.schemas_info[*]["data_bucket"]))}" + "arn:aws:s3:::${local.apiary_bucket_prefix}-*", + "arn:aws:s3:::${local.apiary_bucket_prefix}-*/*" ] } ] @@ -123,153 +123,3 @@ resource "aws_iam_role_policy" "external_s3_data_for_hms_readonly" { } EOF } - -resource "aws_iam_role_policy" "s3_inventory_for_hms_readwrite" { - count = var.s3_enable_inventory ? 1 : 0 - name = "s3-inventory" - role = "${aws_iam_role.apiary_hms_readwrite.id}" - - policy = <