From 7e97d5bb2422123e153cb72eb48b2055193d1e7a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Javier=20S=C3=A1nchez=20Beltr=C3=A1n?=
 <36443689+javsanbel2@users.noreply.github.com>
Date: Thu, 8 Jul 2021 15:19:17 +0100
Subject: [PATCH] Add hive metastore type change validation setting (#195)

* Add hive metastore type change validation setting

* Add Changelog

* Change default value

Co-authored-by: rpoluri <38321430+rpoluri@users.noreply.github.com>
---
 CHANGELOG.md                        |  6 +++++-
 k8s-readwrite.tf                    |  4 ++++
 templates.tf                        | 29 +++++++++++++++--------------
 templates/apiary-hms-readwrite.json |  4 ++++
 variables.tf                        |  6 ++++++
 5 files changed, 34 insertions(+), 15 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5e14ff5..662d56e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,7 +3,11 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
-## [6.9.1] - 2021-06-17
+## [6.9.2] - 2021-07-08
+### Added
+- Added disallow_incompatible_col_type_changes variable to disable hive validation when schema changes. This variable will help Apache Iceberg to make schema-evolution.
+
+## [6.9.1] - 2021-07-08
 ### Added
 - Add support for cross account access to system schema.
 
diff --git a/k8s-readwrite.tf b/k8s-readwrite.tf
index 63b36eb..013cfd0 100644
--- a/k8s-readwrite.tf
+++ b/k8s-readwrite.tf
@@ -205,6 +205,10 @@ resource "kubernetes_deployment" "apiary_hms_readwrite" {
             name  = "APIARY_SYSTEM_SCHEMA"
             value = var.system_schema_name
           }
+          env {
+            name  = "DISALLOW_INCOMPATIBLE_COL_TYPE_CHANGES"
+            value = var.disallow_incompatible_col_type_changes
+          }
 
           resources {
             limits {
diff --git a/templates.tf b/templates.tf
index cf847a6..0d4aa2e 100644
--- a/templates.tf
+++ b/templates.tf
@@ -29,20 +29,21 @@ data "template_file" "hms_readwrite" {
     enable_gluesync            = var.enable_gluesync ? "1" : ""
     gluedb_prefix              = "${local.gluedb_prefix}"
 
-    ranger_service_name           = "${local.instance_alias}-metastore"
-    ranger_policy_manager_url     = "${var.ranger_policy_manager_url}"
-    ranger_audit_solr_url         = "${var.ranger_audit_solr_url}"
-    atlas_kafka_bootstrap_servers = "${var.atlas_kafka_bootstrap_servers}"
-    atlas_cluster_name            = "${local.final_atlas_cluster_name}"
-    ranger_audit_db_url           = "${var.ranger_audit_db_url}"
-    ranger_audit_secret_arn       = "${var.ranger_audit_db_url == "" ? "" : join("", data.aws_secretsmanager_secret.ranger_audit.*.arn)}"
-    ldap_url                      = "${var.ldap_url}"
-    ldap_ca_cert                  = "${var.ldap_ca_cert}"
-    ldap_base                     = "${var.ldap_base}"
-    ldap_secret_arn               = "${var.ldap_url == "" ? "" : join("", data.aws_secretsmanager_secret.ldap_user.*.arn)}"
-    kafka_bootstrap_servers       = var.kafka_bootstrap_servers
-    kafka_topic_name              = var.kafka_topic_name
-    system_schema_name            = var.system_schema_name
+    ranger_service_name                    = "${local.instance_alias}-metastore"
+    ranger_policy_manager_url              = "${var.ranger_policy_manager_url}"
+    ranger_audit_solr_url                  = "${var.ranger_audit_solr_url}"
+    atlas_kafka_bootstrap_servers          = "${var.atlas_kafka_bootstrap_servers}"
+    atlas_cluster_name                     = "${local.final_atlas_cluster_name}"
+    ranger_audit_db_url                    = "${var.ranger_audit_db_url}"
+    ranger_audit_secret_arn                = "${var.ranger_audit_db_url == "" ? "" : join("", data.aws_secretsmanager_secret.ranger_audit.*.arn)}"
+    ldap_url                               = "${var.ldap_url}"
+    ldap_ca_cert                           = "${var.ldap_ca_cert}"
+    ldap_base                              = "${var.ldap_base}"
+    ldap_secret_arn                        = "${var.ldap_url == "" ? "" : join("", data.aws_secretsmanager_secret.ldap_user.*.arn)}"
+    kafka_bootstrap_servers                = var.kafka_bootstrap_servers
+    kafka_topic_name                       = var.kafka_topic_name
+    system_schema_name                     = var.system_schema_name
+    disallow_incompatible_col_type_changes = var.disallow_incompatible_col_type_changes
 
     #to instruct docker to turn off upgrading hive db schema when using external database
     external_database = "${var.external_database_host == "" ? "" : "1"}"
diff --git a/templates/apiary-hms-readwrite.json b/templates/apiary-hms-readwrite.json
index 7e424d0..f2707ae 100644
--- a/templates/apiary-hms-readwrite.json
+++ b/templates/apiary-hms-readwrite.json
@@ -206,6 +206,10 @@
      {
        "name": "APIARY_SYSTEM_SCHEMA",
        "value": "${system_schema_name}"
+     },
+     {
+      "name": "DISALLOW_INCOMPATIBLE_COL_TYPE_CHANGES",
+      "value": "${disallow_incompatible_col_type_changes}"
      }
     ]
   }
diff --git a/variables.tf b/variables.tf
index 3a4ced8..0015f51 100644
--- a/variables.tf
+++ b/variables.tf
@@ -511,3 +511,9 @@ variable "rds_max_allowed_packet" {
   type        = number
   default     = 134217728
 }
+
+variable "disallow_incompatible_col_type_changes" {
+  description = "Hive metastore setting to disallow validation when incompatible schema type changes. Default is false."
+  type        = bool
+  default     = true
+}