Feature/hms iam wildcard (#174)

* use wildcard to configure metastore iam roles * fix * arn fix * fix master user secret count * fix templates * fix allow-grant path * k8s-secret fix * fix init container commands * update changelog * remove mysql_commands template variable * remove init container image and use hms docker for init container also * fix Co-authored-by: Raj Poluri <[email protected]>
ExpediaGroup · Sep 2, 2020 · 693e75c · 693e75c
1 parent 01aaf9e
commit 693e75c
Show file tree

Hide file tree

Showing 11 changed files with 47 additions and 218 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -3,6 +3,11 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [6.5.1] - 2020-09-02
+### Changed
+- [Issue 165](https://github.com/ExpediaGroup/apiary-data-lake/issues/173) Configure metastore IAM roles using apiary bucket prefix.
+- Fix init container deployment.
+
 ## [6.5.0] - 2020-08-31
 ### Changed
 - [Issue 165](https://github.com/ExpediaGroup/apiary-data-lake/issues/165) Use init containers instead of `mysql` commands to initialize mysql users.

diff --git a/VARIABLES.md b/VARIABLES.md
@@ -52,8 +52,6 @@
 | hms_rw_heapsize | Heapsize for the read/write Hive Metastore. Valid values: https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-cpu-memory-error.html | string | - | yes |
 | iam_name_root | Name to identify Hive Metastore IAM roles. | string | `hms` | no |
 | ingress_cidr | Generally allowed ingress CIDR list. | list | - | yes |
-| init_container_image | Docker image for running HMS init container. Required if `external_database_host` is unset. | string | `` | no |
-| init_container_version | Docker image version for running HMS init container. Required if `external_database_host` is unset. | string | `` | no |
 | instance_name | Apiary instance name to identify resources in multi-instance deployments. | string | `` | no |
 | k8s_docker_registry_secret| Docker Registry authentication K8s secret name. | string | `` | no |
 | kiam_arn | Kiam server IAM role ARN. | string | `` | no |

diff --git a/db.tf b/db.tf
@@ -92,21 +92,21 @@ resource "aws_rds_cluster_instance" "apiary_cluster_instance" {
 # In order to avoid resource collision when deleting & immediately recreating SecretsManager secrets in AWS, we set a random suffix on the name of the secret.
 # This allows us to avoid the issue of AWS's imposed 7 day recovery window.
 resource "random_string" "secret_name_suffix" {
-  count   = "${var.external_database_host == "" ? var.db_instance_count : 0}"
+  count   = var.external_database_host == "" ? 1 : 0
   length  = 8
   special = false
 }
 
 resource "aws_secretsmanager_secret" "apiary_mysql_master_credentials" {
-  count                   = "${var.external_database_host == "" ? var.db_instance_count : 0}"
+  count                   = var.external_database_host == "" ? 1 : 0
   name                    = "${local.instance_alias}_db_master_user_${random_string.secret_name_suffix[0].result}"
   tags                    = var.apiary_tags
   recovery_window_in_days = 0
 }
 
 resource "aws_secretsmanager_secret_version" "apiary_mysql_master_credentials" {
-  count         = "${var.external_database_host == "" ? var.db_instance_count : 0}"
-  secret_id     = aws_secretsmanager_secret.apiary_mysql_master_credentials[0].id
+  count     = var.external_database_host == "" ? 1 : 0
+  secret_id = aws_secretsmanager_secret.apiary_mysql_master_credentials[0].id
   secret_string = jsonencode(
     map(
       "username", var.db_master_username,

diff --git a/iam-policy-s3-buckets.tf b/iam-policy-s3-buckets.tf
@@ -30,8 +30,8 @@ resource "aws_iam_role_policy" "s3_data_for_hms_readwrite" {
                               "s3:PutObjectVersionTagging"
                             ],
                   "Resource": [
-                                "${join("\",\"", formatlist("arn:aws:s3:::%s", local.schemas_info[*]["data_bucket"]))}",
-                                "${join("\",\"", formatlist("arn:aws:s3:::%s/*", local.schemas_info[*]["data_bucket"]))}"
+                                "arn:aws:s3:::${local.apiary_bucket_prefix}-*",
+                                "arn:aws:s3:::${local.apiary_bucket_prefix}-*/*"
                               ]
                 }
               ]
@@ -55,8 +55,8 @@ resource "aws_iam_role_policy" "s3_data_for_hms_readonly" {
                               "s3:List*"
                             ],
                   "Resource": [
-                                "${join("\",\"", formatlist("arn:aws:s3:::%s", local.schemas_info[*]["data_bucket"]))}",
-                                "${join("\",\"", formatlist("arn:aws:s3:::%s/*", local.schemas_info[*]["data_bucket"]))}"
+                                "arn:aws:s3:::${local.apiary_bucket_prefix}-*",
+                                "arn:aws:s3:::${local.apiary_bucket_prefix}-*/*"
                               ]
                 }
               ]
@@ -123,153 +123,3 @@ resource "aws_iam_role_policy" "external_s3_data_for_hms_readonly" {
 }
 EOF
 }
-
-resource "aws_iam_role_policy" "s3_inventory_for_hms_readwrite" {
-  count = var.s3_enable_inventory ? 1 : 0
-  name  = "s3-inventory"
-  role  = "${aws_iam_role.apiary_hms_readwrite.id}"
-
-  policy = <<EOF
-{
- "Version": "2012-10-17",
- "Statement": [
-                {
-                  "Effect": "Allow",
-                  "Action": [
-                              "s3:Get*",
-                              "s3:List*"
-                            ],
-                  "Resource": [
-                                "${format("arn:aws:s3:::%s", local.s3_inventory_bucket)}",
-                                "${format("arn:aws:s3:::%s/*", local.s3_inventory_bucket)}"
-                              ]
-                }
-              ]
-}
-EOF
-}
-
-resource "aws_iam_role_policy" "s3_inventory_for_hms_readonly" {
-  count = var.s3_enable_inventory ? 1 : 0
-  name  = "s3-inventory"
-  role  = "${aws_iam_role.apiary_hms_readonly.id}"
-
-  policy = <<EOF
-{
- "Version": "2012-10-17",
- "Statement": [
-                {
-                  "Effect": "Allow",
-                  "Action": [
-                              "s3:Get*",
-                              "s3:List*"
-                            ],
-                  "Resource": [
-                                "${format("arn:aws:s3:::%s", local.s3_inventory_bucket)}",
-                                "${format("arn:aws:s3:::%s/*", local.s3_inventory_bucket)}"
-                              ]
-                }
-              ]
-}
-EOF
-}
-
-resource "aws_iam_role_policy" "s3_access_logs_for_hms_readwrite" {
-  count = local.enable_apiary_s3_log_management ? 1 : 0
-  name  = "s3-access-logs"
-  role  = "${aws_iam_role.apiary_hms_readwrite.id}"
-
-  policy = <<EOF
-{
- "Version": "2012-10-17",
- "Statement": [
-                {
-                  "Effect": "Allow",
-                  "Action": [
-                              "s3:Get*",
-                              "s3:List*"
-                            ],
-                  "Resource": [
-                                "${format("arn:aws:s3:::%s", local.apiary_s3_hive_logs_bucket)}",
-                                "${format("arn:aws:s3:::%s/*", local.apiary_s3_hive_logs_bucket)}"
-                              ]
-                }
-              ]
-}
-EOF
-}
-
-resource "aws_iam_role_policy" "s3_access_logs_for_hms_readonly" {
-  count = local.enable_apiary_s3_log_management ? 1 : 0
-  name  = "s3-access-logs"
-  role  = "${aws_iam_role.apiary_hms_readonly.id}"
-
-  policy = <<EOF
-{
- "Version": "2012-10-17",
- "Statement": [
-                {
-                  "Effect": "Allow",
-                  "Action": [
-                              "s3:Get*",
-                              "s3:List*"
-                            ],
-                  "Resource": [
-                                "${format("arn:aws:s3:::%s", local.apiary_s3_hive_logs_bucket)}",
-                                "${format("arn:aws:s3:::%s/*", local.apiary_s3_hive_logs_bucket)}"
-                              ]
-                }
-              ]
-}
-EOF
-}
-
-resource "aws_iam_role_policy" "system_for_hms_readwrite" {
-
-  name  = "system"
-  role  = "${aws_iam_role.apiary_hms_readwrite.id}"
-
-  policy = <<EOF
-{
- "Version": "2012-10-17",
- "Statement": [
-                {
-                  "Effect": "Allow",
-                  "Action": [
-                              "s3:Get*",
-                              "s3:List*"
-                            ],
-                  "Resource": [
-                                "${format("arn:aws:s3:::%s", local.apiary_system_bucket)}",
-                                "${format("arn:aws:s3:::%s/*", local.apiary_system_bucket)}"
-                              ]
-                }
-              ]
-}
-EOF
-}
-
-resource "aws_iam_role_policy" "system_for_hms_readonly" {
-
-  name  = "system"
-  role  = "${aws_iam_role.apiary_hms_readonly.id}"
-
-  policy = <<EOF
-{
- "Version": "2012-10-17",
- "Statement": [
-                {
-                  "Effect": "Allow",
-                  "Action": [
-                              "s3:Get*",
-                              "s3:List*"
-                            ],
-                  "Resource": [
-                                "${format("arn:aws:s3:::%s", local.apiary_system_bucket)}",
-                                "${format("arn:aws:s3:::%s/*", local.apiary_system_bucket)}"
-                              ]
-                }
-              ]
-}
-EOF
-}
diff --git a/k8s-readonly.tf b/k8s-readonly.tf
@@ -39,25 +39,25 @@ resource "kubernetes_deployment" "apiary_hms_readonly" {
       spec {
         dynamic "init_container" {
           for_each = var.external_database_host == "" ? ["enabled"] : []
-          
+
           content {
-            image = "${var.init_container_image}:${var.init_container_version}"
+            image = "${var.hms_docker_image}:${var.hms_docker_version}"
             name  = "${local.hms_alias}-sql-init-readonly"
-            
-            command = ["sh allow-grant.sh"]
+
+            command = ["sh", "/allow-grant.sh"]
 
             env {
-              name = "MYSQL_HOST"
+              name  = "MYSQL_HOST"
               value = var.external_database_host == "" ? join("", aws_rds_cluster.apiary_cluster.*.endpoint) : var.external_database_host
             }
 
             env {
-              name = "MYSQL_DB"
+              name  = "MYSQL_DB"
               value = var.apiary_database_name
             }
 
             env {
-              name = "MYSQL_PERMISSIONS"
+              name  = "MYSQL_PERMISSIONS"
               value = "SELECT"
             }
 
@@ -66,7 +66,7 @@ resource "kubernetes_deployment" "apiary_hms_readonly" {
               value_from {
                 secret_key_ref {
                   name = kubernetes_secret.hms_secrets[0].metadata[0].name
-                  key = "master_creds"
+                  key  = "master_creds"
                 }
               }
             }
@@ -76,7 +76,7 @@ resource "kubernetes_deployment" "apiary_hms_readonly" {
               value_from {
                 secret_key_ref {
                   name = kubernetes_secret.hms_secrets[0].metadata[0].name
-                  key = "ro_creds"
+                  key  = "ro_creds"
                 }
               }
             }

diff --git a/k8s-readwrite.tf b/k8s-readwrite.tf
@@ -37,26 +37,26 @@ resource "kubernetes_deployment" "apiary_hms_readwrite" {
       }
 
       spec {
-         dynamic "init_container" {
-           for_each = var.external_database_host == "" ? ["enabled"] : []
-           content {
-            image = "${var.init_container_image}:${var.init_container_version}"
+        dynamic "init_container" {
+          for_each = var.external_database_host == "" ? ["enabled"] : []
+          content {
+            image = "${var.hms_docker_image}:${var.hms_docker_version}"
             name  = "${local.hms_alias}-sql-init-readwrite"
-            
-            command = ["sh allow-grant.sh"]
+
+            command = ["sh", "/allow-grant.sh"]
 
             env {
-              name = "MYSQL_HOST"
+              name  = "MYSQL_HOST"
               value = var.external_database_host == "" ? join("", aws_rds_cluster.apiary_cluster.*.endpoint) : var.external_database_host
             }
 
             env {
-              name = "MYSQL_DB"
+              name  = "MYSQL_DB"
               value = var.apiary_database_name
             }
 
             env {
-              name = "MYSQL_PERMISSIONS"
+              name  = "MYSQL_PERMISSIONS"
               value = "ALL"
             }
 
@@ -65,7 +65,7 @@ resource "kubernetes_deployment" "apiary_hms_readwrite" {
               value_from {
                 secret_key_ref {
                   name = kubernetes_secret.hms_secrets[0].metadata[0].name
-                  key = "master_creds"
+                  key  = "master_creds"
                 }
               }
             }
@@ -75,7 +75,7 @@ resource "kubernetes_deployment" "apiary_hms_readwrite" {
               value_from {
                 secret_key_ref {
                   name = kubernetes_secret.hms_secrets[0].metadata[0].name
-                  key = "rw_creds"
+                  key  = "rw_creds"
                 }
               }
             }

diff --git a/k8s-secrets.tf b/k8s-secrets.tf
@@ -7,9 +7,7 @@ resource "kubernetes_secret" "hms_secrets" {
 
   data = {
     master_creds = aws_secretsmanager_secret_version.apiary_mysql_master_credentials[0].secret_string
-    ro_creds = data.aws_secretsmanager_secret_version.db_ro_user.secret_string
-    rw_creds = data.aws_secretsmanager_secret_version.db_rw_user.secret_string
+    ro_creds     = data.aws_secretsmanager_secret_version.db_ro_user.secret_string
+    rw_creds     = data.aws_secretsmanager_secret_version.db_rw_user.secret_string
   }
-
-  type = "kubernetes.io/basic-auth"
 }
diff --git a/templates.tf b/templates.tf
@@ -52,16 +52,13 @@ data "template_file" "hms_readwrite" {
 
     s3_enable_inventory = var.s3_enable_inventory ? "1" : ""
     # If user sets "apiary_log_bucket", then they are doing their own access logs mgmt, and not using Apiary's log mgmt.
-    s3_enable_logs      = local.enable_apiary_s3_log_management ? "1" : ""
+    s3_enable_logs = local.enable_apiary_s3_log_management ? "1" : ""
 
     # Template vars for init container
     init_container_enabled = var.external_database_host == "" ? true : false
-    init_container_image = "${var.init_container_image}"
-    init_container_version = "${var.init_container_version}"
-    mysql_permissions = "ALL"
-    mysql_master_cred_arn = aws_secretsmanager_secret.apiary_mysql_master_credentials[0].arn
-    mysql_user_cred_arn = data.aws_secretsmanager_secret.db_rw_user.arn
-    mysql_commands = "sh allow-grant.sh"
+    mysql_permissions      = "ALL"
+    mysql_master_cred_arn  = aws_secretsmanager_secret.apiary_mysql_master_credentials[0].arn
+    mysql_user_cred_arn    = data.aws_secretsmanager_secret.db_rw_user.arn
   }
 }
 
@@ -101,11 +98,9 @@ data "template_file" "hms_readonly" {
 
     # Template vars for init container
     init_container_enabled = var.external_database_host == "" ? true : false
-    init_container_image = "${var.init_container_image}"
-    mysql_permissions = "SELECT"
-    mysql_write_db = "${var.external_database_host == "" ? join("", aws_rds_cluster.apiary_cluster.*.endpoint) : var.external_database_host}"
-    mysql_master_cred_arn = aws_secretsmanager_secret.apiary_mysql_master_credentials[0].arn
-    mysql_user_cred_arn = data.aws_secretsmanager_secret.db_ro_user.arn
-    mysql_commands = "sh allow-grant.sh"
+    mysql_permissions      = "SELECT"
+    mysql_write_db         = "${var.external_database_host == "" ? join("", aws_rds_cluster.apiary_cluster.*.endpoint) : var.external_database_host}"
+    mysql_master_cred_arn  = aws_secretsmanager_secret.apiary_mysql_master_credentials[0].arn
+    mysql_user_cred_arn    = data.aws_secretsmanager_secret.db_ro_user.arn
   }
 }
diff --git a/templates/apiary-hms-readonly.json b/templates/apiary-hms-readonly.json
@@ -3,7 +3,7 @@
   {
     "name": "mysql-setup",
     "essential": false,
-    "image": "${var.init_container_image}:${var.init_container_version}",
+    "image": "${hms_docker_image}:${hms_docker_version}",
     ${docker_auth}
     "logConfiguration": {
        "logDriver": "awslogs",
@@ -37,9 +37,7 @@
          "name": "MYSQL_USER_CREDS"
        }
     ],
-    "entryPoint": [ "/bin/sh", "-c" ],
-    "workingDirectory": "/init",
-    "command": ["${mysql_commands}"]
+    "command": ["sh", "/allow-grant.sh"]
   },
 %{ endif } 
   {