From e837e85a85fc128bbe2f6eb22544ca16eb67b3cb Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 19 Jun 2024 18:00:26 +0800 Subject: [PATCH 1/6] Add user authentication for multiple HMS --- catalogs/catalog-hive/build.gradle.kts | 2 +- .../test/HiveUserAuthenticationIT.java | 4 ++-- .../test/MultipleHMSUserAuthenticationIT.java | 18 ++++++++++++++++++ 3 files changed, 21 insertions(+), 3 deletions(-) create mode 100644 catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/integration/test/MultipleHMSUserAuthenticationIT.java diff --git a/catalogs/catalog-hive/build.gradle.kts b/catalogs/catalog-hive/build.gradle.kts index 65a36cd6b37..f7f9287c5d7 100644 --- a/catalogs/catalog-hive/build.gradle.kts +++ b/catalogs/catalog-hive/build.gradle.kts @@ -165,7 +165,7 @@ tasks.test { doFirst { environment("GRAVITINO_CI_HIVE_DOCKER_IMAGE", "datastrato/gravitino-ci-hive:0.1.12") - environment("GRAVITINO_CI_KERBEROS_HIVE_DOCKER_IMAGE", "datastrato/gravitino-ci-kerberos-hive:0.1.2") + environment("GRAVITINO_CI_KERBEROS_HIVE_DOCKER_IMAGE", "datastrato/gravitino-ci-kerberos-hive:0.1.4") } val init = project.extra.get("initIntegrationTest") as (Test) -> Unit diff --git a/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/integration/test/HiveUserAuthenticationIT.java b/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/integration/test/HiveUserAuthenticationIT.java index 5d2ee86a5af..c4ccf9e4dde 100644 --- a/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/integration/test/HiveUserAuthenticationIT.java +++ b/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/integration/test/HiveUserAuthenticationIT.java @@ -67,11 +67,11 @@ public class HiveUserAuthenticationIT extends AbstractIT { private static String TMP_DIR; - private static String HIVE_METASTORE_URI; + protected static String HIVE_METASTORE_URI; private static GravitinoAdminClient adminClient; - private static HiveContainer kerberosHiveContainer; + protected static HiveContainer kerberosHiveContainer; private static final String METALAKE_NAME = GravitinoITUtils.genRandomName("test_metalake"); private static final String CATALOG_NAME = GravitinoITUtils.genRandomName("test_catalog"); diff --git a/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/integration/test/MultipleHMSUserAuthenticationIT.java b/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/integration/test/MultipleHMSUserAuthenticationIT.java new file mode 100644 index 00000000000..e27e7712e88 --- /dev/null +++ b/catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/integration/test/MultipleHMSUserAuthenticationIT.java @@ -0,0 +1,18 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.catalog.hive.integration.test; + +import com.datastrato.gravitino.integration.test.container.HiveContainer; +import org.junit.jupiter.api.BeforeAll; + +public class MultipleHMSUserAuthenticationIT extends HiveUserAuthenticationIT { + @BeforeAll + static void setHiveURI() { + String ip = kerberosHiveContainer.getContainerIpAddress(); + int port = HiveContainer.HIVE_METASTORE_PORT; + // Multiple HMS URIs, I put the new one first to test the new HMS URI first. + HIVE_METASTORE_URI = String.format("thrift://%s:1%d,thrift://%s:%d", ip, port, ip, port); + } +} From 81e87f4451b1d5d11ed00a30b3ca0ef6d83cfbe0 Mon Sep 17 00:00:00 2001 From: yuqi Date: Mon, 22 Jul 2024 20:32:06 +0800 Subject: [PATCH 2/6] Fix --- .../test/MultipleHMSUserAuthenticationIT.java | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/MultipleHMSUserAuthenticationIT.java b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/MultipleHMSUserAuthenticationIT.java index a2a31f4f0d4..1dbeec2f6e3 100644 --- a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/MultipleHMSUserAuthenticationIT.java +++ b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/MultipleHMSUserAuthenticationIT.java @@ -1,7 +1,22 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ + package org.apache.gravitino.catalog.hive.integration.test; import org.apache.gravitino.integration.test.container.HiveContainer; From 34ce9911a3b8be2481ba6b3a84c435a5dc3d22db Mon Sep 17 00:00:00 2001 From: yuqi Date: Mon, 22 Jul 2024 21:03:14 +0800 Subject: [PATCH 3/6] Fix --- .../hive/integration/test/MultipleHMSUserAuthenticationIT.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/MultipleHMSUserAuthenticationIT.java b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/MultipleHMSUserAuthenticationIT.java index a2a31f4f0d4..20167432deb 100644 --- a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/MultipleHMSUserAuthenticationIT.java +++ b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/MultipleHMSUserAuthenticationIT.java @@ -6,7 +6,9 @@ import org.apache.gravitino.integration.test.container.HiveContainer; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +@Tag("gravitino-docker-test") public class MultipleHMSUserAuthenticationIT extends HiveUserAuthenticationIT { @BeforeAll static void setHiveURI() { From 3e7ebc1a48d3bce0277ae0c8256761683dccf9f9 Mon Sep 17 00:00:00 2001 From: yuqi Date: Tue, 23 Jul 2024 11:19:22 +0800 Subject: [PATCH 4/6] Fix --- .../test/HiveUserAuthenticationIT.java | 18 ++++++++++-------- .../test/MultipleHMSUserAuthenticationIT.java | 13 +++++++++++++ 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/HiveUserAuthenticationIT.java b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/HiveUserAuthenticationIT.java index 029655d7f5e..50299e5d102 100644 --- a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/HiveUserAuthenticationIT.java +++ b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/HiveUserAuthenticationIT.java @@ -87,10 +87,10 @@ public class HiveUserAuthenticationIT extends AbstractIT { protected static HiveContainer kerberosHiveContainer; - private static final String METALAKE_NAME = GravitinoITUtils.genRandomName("test_metalake"); - private static final String CATALOG_NAME = GravitinoITUtils.genRandomName("test_catalog"); - private static final String SCHEMA_NAME = GravitinoITUtils.genRandomName("test_schema"); - private static final String TABLE_NAME = GravitinoITUtils.genRandomName("test_table"); + static String METALAKE_NAME = GravitinoITUtils.genRandomName("test_metalake"); + static String CATALOG_NAME = GravitinoITUtils.genRandomName("test_catalog"); + static String SCHEMA_NAME = GravitinoITUtils.genRandomName("test_schema"); + static String TABLE_NAME = GravitinoITUtils.genRandomName("test_table"); private static final String HIVE_COL_NAME1 = "col1"; private static final String HIVE_COL_NAME2 = "col2"; @@ -205,10 +205,6 @@ public void testUserAuthentication() { .withKeyTabFile(new File(TMP_DIR + GRAVITINO_CLIENT_KEYTAB)) .build(); adminClient = GravitinoAdminClient.builder(serverUri).withKerberosAuth(provider).build(); - - GravitinoMetalake[] metalakes = adminClient.listMetalakes(); - Assertions.assertEquals(0, metalakes.length); - GravitinoMetalake gravitinoMetalake = adminClient.createMetalake(METALAKE_NAME, null, ImmutableMap.of()); @@ -277,6 +273,12 @@ public void testUserAuthentication() { Assertions.assertTrue(gravitinoMetalake.dropCatalog(CATALOG_NAME)); } + @AfterAll + static void restoreFilePermission() { + kerberosHiveContainer.executeInContainer( + "hadoop", "fs", "-chmod", "-R", "755", "/user/hive/warehouse"); + } + private static Column[] createColumns() { Column col1 = Column.of(HIVE_COL_NAME1, Types.ByteType.get(), "col_1_comment"); Column col2 = Column.of(HIVE_COL_NAME2, Types.DateType.get(), "col_2_comment"); diff --git a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/MultipleHMSUserAuthenticationIT.java b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/MultipleHMSUserAuthenticationIT.java index 1dbeec2f6e3..4d2b61a4576 100644 --- a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/MultipleHMSUserAuthenticationIT.java +++ b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/MultipleHMSUserAuthenticationIT.java @@ -20,7 +20,9 @@ package org.apache.gravitino.catalog.hive.integration.test; import org.apache.gravitino.integration.test.container.HiveContainer; +import org.apache.gravitino.integration.test.util.GravitinoITUtils; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; public class MultipleHMSUserAuthenticationIT extends HiveUserAuthenticationIT { @BeforeAll @@ -29,5 +31,16 @@ static void setHiveURI() { int port = HiveContainer.HIVE_METASTORE_PORT; // Multiple HMS URIs, I put the new one first to test the new HMS URI first. HIVE_METASTORE_URI = String.format("thrift://%s:1%d,thrift://%s:%d", ip, port, ip, port); + + + } + + @Test + public void testUserAuthentication() { + METALAKE_NAME = GravitinoITUtils.genRandomName("test_metalake"); + CATALOG_NAME = GravitinoITUtils.genRandomName("test_catalog"); + SCHEMA_NAME = GravitinoITUtils.genRandomName("test_schema"); + TABLE_NAME = GravitinoITUtils.genRandomName("test_table"); + super.testUserAuthentication(); } } From fcba74d504cfaa442ac923d8921db9aaac1d4635 Mon Sep 17 00:00:00 2001 From: yuqi Date: Tue, 23 Jul 2024 11:22:12 +0800 Subject: [PATCH 5/6] Fix --- .../hive/integration/test/MultipleHMSUserAuthenticationIT.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/MultipleHMSUserAuthenticationIT.java b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/MultipleHMSUserAuthenticationIT.java index 725ce22586b..96885aaa721 100644 --- a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/MultipleHMSUserAuthenticationIT.java +++ b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/MultipleHMSUserAuthenticationIT.java @@ -33,8 +33,6 @@ static void setHiveURI() { int port = HiveContainer.HIVE_METASTORE_PORT; // Multiple HMS URIs, I put the new one first to test the new HMS URI first. HIVE_METASTORE_URI = String.format("thrift://%s:1%d,thrift://%s:%d", ip, port, ip, port); - - } @Test From a6de37a877edfa846605e8150a168bd37b977299 Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 31 Jul 2024 19:17:00 +0800 Subject: [PATCH 6/6] Fix comments. --- .../test/MultipleHMSUserAuthenticationIT.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/MultipleHMSUserAuthenticationIT.java b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/MultipleHMSUserAuthenticationIT.java index 96885aaa721..4a6b54816da 100644 --- a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/MultipleHMSUserAuthenticationIT.java +++ b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/MultipleHMSUserAuthenticationIT.java @@ -30,9 +30,14 @@ public class MultipleHMSUserAuthenticationIT extends HiveUserAuthenticationIT { @BeforeAll static void setHiveURI() { String ip = kerberosHiveContainer.getContainerIpAddress(); - int port = HiveContainer.HIVE_METASTORE_PORT; + int oldHMSPort = HiveContainer.HIVE_METASTORE_PORT; + + // About the value of `newAddedHMSPort`, please see the value `hive.metastore.port` in the + // dev/docker/kerberos-hive/hive-site1.xml + int newAddedHMSPort = 19083; // Multiple HMS URIs, I put the new one first to test the new HMS URI first. - HIVE_METASTORE_URI = String.format("thrift://%s:1%d,thrift://%s:%d", ip, port, ip, port); + HIVE_METASTORE_URI = + String.format("thrift://%s:%d,thrift://%s:%d", ip, newAddedHMSPort, ip, oldHMSPort); } @Test