From ecbaebd329b22a56868510fd63433a5f579bdafb Mon Sep 17 00:00:00 2001 From: felipemiquelim Date: Tue, 9 Mar 2021 16:54:51 -0300 Subject: [PATCH 1/2] Adding feat to allow columns types to change when dropping column --- hive_metastore_client/hive_metastore_client.py | 8 ++++++++ .../test_hive_metastore_client.py | 10 +++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/hive_metastore_client/hive_metastore_client.py b/hive_metastore_client/hive_metastore_client.py index c590a37..af08679 100644 --- a/hive_metastore_client/hive_metastore_client.py +++ b/hive_metastore_client/hive_metastore_client.py @@ -23,6 +23,10 @@ class HiveMetastoreClient(ThriftClient): """User main interface with the metastore server methods.""" + COL_TYPE_INCOMPATIBILITY_DISALLOW_CONFIG = ( + "hive.metastore.disallow.incompatible.col.type.changes" + ) + def __init__(self, host: str, port: int = 9083) -> None: """ Instantiates the client object for given host and port. @@ -113,6 +117,10 @@ def drop_columns_from_table( cols.append(col) table.sd.cols = cols + # Hive Metastore enforces that the schema prior and after an ALTER TABLE should be the same, + # however when dropping a column the schema will definitely change + self.setMetaConf(self.COL_TYPE_INCOMPATIBILITY_DISALLOW_CONFIG, "false") + # call alter table to drop columns removed from list of table columns self.alter_table(dbname=db_name, tbl_name=table_name, new_tbl=table) diff --git a/tests/unit/hive_metastore_client/test_hive_metastore_client.py b/tests/unit/hive_metastore_client/test_hive_metastore_client.py index b29ff12..34d9724 100644 --- a/tests/unit/hive_metastore_client/test_hive_metastore_client.py +++ b/tests/unit/hive_metastore_client/test_hive_metastore_client.py @@ -106,8 +106,13 @@ def test_add_columns_to_table( @mock.patch.object(HiveMetastoreClient, "get_table") @mock.patch.object(HiveMetastoreClient, "alter_table") + @mock.patch.object(HiveMetastoreClient, "setMetaConf") def test_drop_columns_from_table( - self, mocked_alter_table, mocked_get_table, hive_metastore_client + self, + mocked_set_meta_conf, + mocked_alter_table, + mocked_get_table, + hive_metastore_client, ): # arrange db_name = "db_name" @@ -131,6 +136,9 @@ def test_drop_columns_from_table( ) # assert + mocked_set_meta_conf.assert_called_once_with( + hive_metastore_client.COL_TYPE_INCOMPATIBILITY_DISALLOW_CONFIG, "false" + ) mocked_get_table.assert_called_once_with(dbname=db_name, tbl_name=table_name) mocked_alter_table.assert_called_once_with( dbname=db_name, tbl_name=table_name, new_tbl=expected_mocked_table From 777ea0aca09bce9ba28b24b10b1d2161404eb84a Mon Sep 17 00:00:00 2001 From: felipemiquelim Date: Tue, 9 Mar 2021 17:48:42 -0300 Subject: [PATCH 2/2] Lint --- hive_metastore_client/hive_metastore_client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hive_metastore_client/hive_metastore_client.py b/hive_metastore_client/hive_metastore_client.py index af08679..b1aa5bd 100644 --- a/hive_metastore_client/hive_metastore_client.py +++ b/hive_metastore_client/hive_metastore_client.py @@ -117,7 +117,8 @@ def drop_columns_from_table( cols.append(col) table.sd.cols = cols - # Hive Metastore enforces that the schema prior and after an ALTER TABLE should be the same, + # Hive Metastore enforces that the schema prior and after + # an ALTER TABLE should be the same, # however when dropping a column the schema will definitely change self.setMetaConf(self.COL_TYPE_INCOMPATIBILITY_DISALLOW_CONFIG, "false")