Add hive metastore type change validation setting (#195)

* Add hive metastore type change validation setting * Add Changelog * Change default value Co-authored-by: rpoluri <[email protected]>
ExpediaGroup · Jul 8, 2021 · 7e97d5b · 7e97d5b
1 parent 1b5debe
commit 7e97d5b
Show file tree

Hide file tree

Showing 5 changed files with 34 additions and 15 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -3,7 +3,11 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
-## [6.9.1] - 2021-06-17
+## [6.9.2] - 2021-07-08
+### Added
+- Added disallow_incompatible_col_type_changes variable to disable hive validation when schema changes. This variable will help Apache Iceberg to make schema-evolution.
+
+## [6.9.1] - 2021-07-08
 ### Added
 - Add support for cross account access to system schema.
 

diff --git a/k8s-readwrite.tf b/k8s-readwrite.tf
@@ -205,6 +205,10 @@ resource "kubernetes_deployment" "apiary_hms_readwrite" {
             name  = "APIARY_SYSTEM_SCHEMA"
             value = var.system_schema_name
           }
+          env {
+            name  = "DISALLOW_INCOMPATIBLE_COL_TYPE_CHANGES"
+            value = var.disallow_incompatible_col_type_changes
+          }
 
           resources {
             limits {

diff --git a/templates.tf b/templates.tf
@@ -29,20 +29,21 @@ data "template_file" "hms_readwrite" {
     enable_gluesync            = var.enable_gluesync ? "1" : ""
     gluedb_prefix              = "${local.gluedb_prefix}"
 
-    ranger_service_name           = "${local.instance_alias}-metastore"
-    ranger_policy_manager_url     = "${var.ranger_policy_manager_url}"
-    ranger_audit_solr_url         = "${var.ranger_audit_solr_url}"
-    atlas_kafka_bootstrap_servers = "${var.atlas_kafka_bootstrap_servers}"
-    atlas_cluster_name            = "${local.final_atlas_cluster_name}"
-    ranger_audit_db_url           = "${var.ranger_audit_db_url}"
-    ranger_audit_secret_arn       = "${var.ranger_audit_db_url == "" ? "" : join("", data.aws_secretsmanager_secret.ranger_audit.*.arn)}"
-    ldap_url                      = "${var.ldap_url}"
-    ldap_ca_cert                  = "${var.ldap_ca_cert}"
-    ldap_base                     = "${var.ldap_base}"
-    ldap_secret_arn               = "${var.ldap_url == "" ? "" : join("", data.aws_secretsmanager_secret.ldap_user.*.arn)}"
-    kafka_bootstrap_servers       = var.kafka_bootstrap_servers
-    kafka_topic_name              = var.kafka_topic_name
-    system_schema_name            = var.system_schema_name
+    ranger_service_name                    = "${local.instance_alias}-metastore"
+    ranger_policy_manager_url              = "${var.ranger_policy_manager_url}"
+    ranger_audit_solr_url                  = "${var.ranger_audit_solr_url}"
+    atlas_kafka_bootstrap_servers          = "${var.atlas_kafka_bootstrap_servers}"
+    atlas_cluster_name                     = "${local.final_atlas_cluster_name}"
+    ranger_audit_db_url                    = "${var.ranger_audit_db_url}"
+    ranger_audit_secret_arn                = "${var.ranger_audit_db_url == "" ? "" : join("", data.aws_secretsmanager_secret.ranger_audit.*.arn)}"
+    ldap_url                               = "${var.ldap_url}"
+    ldap_ca_cert                           = "${var.ldap_ca_cert}"
+    ldap_base                              = "${var.ldap_base}"
+    ldap_secret_arn                        = "${var.ldap_url == "" ? "" : join("", data.aws_secretsmanager_secret.ldap_user.*.arn)}"
+    kafka_bootstrap_servers                = var.kafka_bootstrap_servers
+    kafka_topic_name                       = var.kafka_topic_name
+    system_schema_name                     = var.system_schema_name
+    disallow_incompatible_col_type_changes = var.disallow_incompatible_col_type_changes
 
     #to instruct docker to turn off upgrading hive db schema when using external database
     external_database = "${var.external_database_host == "" ? "" : "1"}"

diff --git a/templates/apiary-hms-readwrite.json b/templates/apiary-hms-readwrite.json
@@ -206,6 +206,10 @@
      {
        "name": "APIARY_SYSTEM_SCHEMA",
        "value": "${system_schema_name}"
+     },
+     {
+      "name": "DISALLOW_INCOMPATIBLE_COL_TYPE_CHANGES",
+      "value": "${disallow_incompatible_col_type_changes}"
      }
     ]
   }

diff --git a/variables.tf b/variables.tf
@@ -511,3 +511,9 @@ variable "rds_max_allowed_packet" {
   type        = number
   default     = 134217728
 }
+
+variable "disallow_incompatible_col_type_changes" {
+  description = "Hive metastore setting to disallow validation when incompatible schema type changes. Default is false."
+  type        = bool
+  default     = true
+}