From 7cd40fb35604e94bf712ab51616a6ad5c7abfa0a Mon Sep 17 00:00:00 2001 From: Jerry Shao Date: Tue, 19 Dec 2023 10:35:22 +0800 Subject: [PATCH] Make Gravitino build and run against JDK8, 11, 17 --- build.gradle.kts | 46 +++++++++++++++++-- .../catalog/hive/dyn/DynConstructors.java | 4 +- .../gravitino/catalog/hive/dyn/DynFields.java | 4 +- .../catalog/hive/dyn/DynMethods.java | 4 +- .../gravitino/EntitySerDeFactory.java | 2 +- .../gravitino/EntityStoreFactory.java | 2 +- .../aux/AuxiliaryServiceManager.java | 2 +- .../gravitino/catalog/CatalogManager.java | 2 +- .../gravitino/proto/ProtoEntitySerDe.java | 2 +- .../gravitino/storage/kv/KvEntityStore.java | 3 +- docs/trino-connector/index.md | 2 +- gradle.properties | 4 ++ .../gravitino/server/web/JettyServer.java | 4 +- 13 files changed, 63 insertions(+), 18 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index ed1b59048d2..ab9c74558a3 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -32,7 +32,7 @@ plugins { alias(libs.plugins.spotless) } else { throw GradleException( - "Gravitino Gradle current doesn't support " + + "Gravitino Gradle toolchain current doesn't support " + "Java version: ${JavaVersion.current()}. Please use JDK8 to 17." ) } @@ -44,6 +44,42 @@ plugins { id("org.cyclonedx.bom") version "1.5.0" // Newer version fail due to our setup } +if (extra["jdkVersion"] !in listOf("8", "11", "17")) { + throw GradleException( + "Gravitino current doesn't support building with " + + "Java version: ${extra["jdkVersion"]}. Please use JDK8, 11 or 17." + ) +} + +project.extra["extraJvmArgs"] = if (extra["jdkVersion"] in listOf("8", "11")) { + listOf() +} else { + listOf( + "-XX:+IgnoreUnrecognizedVMOptions", + "--add-opens", "java.base/java.io=ALL-UNNAMED", + "--add-opens", "java.base/java.lang.invoke=ALL-UNNAMED", + "--add-opens", "java.base/java.lang.reflect=ALL-UNNAMED", + "--add-opens", "java.base/java.lang=ALL-UNNAMED", + "--add-opens", "java.base/java.math=ALL-UNNAMED", + "--add-opens", "java.base/java.net=ALL-UNNAMED", + "--add-opens", "java.base/java.nio=ALL-UNNAMED", + "--add-opens", "java.base/java.text=ALL-UNNAMED", + "--add-opens", "java.base/java.time=ALL-UNNAMED", + "--add-opens", "java.base/java.util.concurrent.atomic=ALL-UNNAMED", + "--add-opens", "java.base/java.util.concurrent=ALL-UNNAMED", + "--add-opens", "java.base/java.util.regex=ALL-UNNAMED", + "--add-opens", "java.base/java.util=ALL-UNNAMED", + "--add-opens", "java.base/jdk.internal.ref=ALL-UNNAMED", + "--add-opens", "java.base/jdk.internal.reflect=ALL-UNNAMED", + "--add-opens", "java.sql/java.sql=ALL-UNNAMED", + "--add-opens", "java.base/sun.util.calendar=ALL-UNNAMED", + "--add-opens", "java.base/sun.nio.ch=ALL-UNNAMED", + "--add-opens", "java.base/sun.nio.cs=ALL-UNNAMED", + "--add-opens", "java.base/sun.security.action=ALL-UNNAMED", + "--add-opens", "java.base/sun.util.calendar=ALL-UNNAMED" + ) +} + licenseReport { renderers = arrayOf(InventoryHtmlReportRenderer("report.html", "Backend")) filters = arrayOf(LicenseBundleNormalizer()) @@ -129,7 +165,9 @@ subprojects { if (project.name == "trino-connector") { languageVersion.set(JavaLanguageVersion.of(17)) } else { - languageVersion.set(JavaLanguageVersion.of(8)) + languageVersion.set(JavaLanguageVersion.of(extra["jdkVersion"].toString().toInt())) + sourceCompatibility = JavaVersion.VERSION_1_8 + targetCompatibility = JavaVersion.VERSION_1_8 } } } @@ -208,7 +246,7 @@ subprojects { sign(publishing.publications) } - tasks.configureEach { + tasks.withType { testLogging { exceptionFormat = TestExceptionFormat.FULL showExceptions = true @@ -224,6 +262,8 @@ subprojects { } else { useJUnitPlatform() } + + jvmArgs(project.property("extraJvmArgs") as List<*>) finalizedBy(tasks.getByName("jacocoTestReport")) } } diff --git a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/dyn/DynConstructors.java b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/dyn/DynConstructors.java index 9a254951a4e..593df47eb86 100644 --- a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/dyn/DynConstructors.java +++ b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/dyn/DynConstructors.java @@ -23,7 +23,6 @@ import com.google.common.collect.Maps; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; -import java.security.AccessController; import java.security.PrivilegedAction; import java.util.Arrays; import java.util.Map; @@ -245,6 +244,7 @@ public Builder hiddenImpl(String className, Class... types) { return this; } + @SuppressWarnings("removal") public Builder hiddenImpl(Class targetClass, Class... types) { // don't do any work if an implementation has been found if (ctor != null) { @@ -253,7 +253,7 @@ public Builder hiddenImpl(Class targetClass, Class... types) { try { Constructor hidden = targetClass.getDeclaredConstructor(types); - AccessController.doPrivileged(new MakeAccessible(hidden)); + java.security.AccessController.doPrivileged(new MakeAccessible(hidden)); ctor = new Ctor(hidden, targetClass); } catch (SecurityException e) { // unusable diff --git a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/dyn/DynFields.java b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/dyn/DynFields.java index 3c8f61cba8a..0d7f67d98dc 100644 --- a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/dyn/DynFields.java +++ b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/dyn/DynFields.java @@ -25,7 +25,6 @@ import com.google.common.collect.Sets; import java.lang.reflect.Field; import java.lang.reflect.Modifier; -import java.security.AccessController; import java.security.PrivilegedAction; import java.util.Set; @@ -302,6 +301,7 @@ public Builder hiddenImpl(String className, String fieldName) { * @see Class#forName(String) * @see Class#getField(String) */ + @SuppressWarnings("removal") public Builder hiddenImpl(Class targetClass, String fieldName) { // don't do any work if an implementation has been found if (field != null || targetClass == null) { @@ -310,7 +310,7 @@ public Builder hiddenImpl(Class targetClass, String fieldName) { try { Field hidden = targetClass.getDeclaredField(fieldName); - AccessController.doPrivileged(new MakeFieldAccessible(hidden)); + java.security.AccessController.doPrivileged(new MakeFieldAccessible(hidden)); this.field = new UnboundField(hidden, fieldName); } catch (SecurityException | NoSuchFieldException e) { // unusable diff --git a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/dyn/DynMethods.java b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/dyn/DynMethods.java index 7d07585ba34..cc27dbb002c 100644 --- a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/dyn/DynMethods.java +++ b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/dyn/DynMethods.java @@ -23,7 +23,6 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.lang.reflect.Modifier; -import java.security.AccessController; import java.security.PrivilegedAction; import java.util.Arrays; @@ -393,6 +392,7 @@ public Builder hiddenImpl(String className, Class... argClasses) { * @see Class#forName(String) * @see Class#getMethod(String, Class[]) */ + @SuppressWarnings("removal") public Builder hiddenImpl(Class targetClass, String methodName, Class... argClasses) { // don't do any work if an implementation has been found if (method != null) { @@ -401,7 +401,7 @@ public Builder hiddenImpl(Class targetClass, String methodName, Class... a try { Method hidden = targetClass.getDeclaredMethod(methodName, argClasses); - AccessController.doPrivileged(new MakeAccessible(hidden)); + java.security.AccessController.doPrivileged(new MakeAccessible(hidden)); this.method = new UnboundMethod(hidden, name); } catch (SecurityException | NoSuchMethodException e) { // unusable or not the right implementation diff --git a/core/src/main/java/com/datastrato/gravitino/EntitySerDeFactory.java b/core/src/main/java/com/datastrato/gravitino/EntitySerDeFactory.java index a5472f2e261..fc59fe77357 100644 --- a/core/src/main/java/com/datastrato/gravitino/EntitySerDeFactory.java +++ b/core/src/main/java/com/datastrato/gravitino/EntitySerDeFactory.java @@ -48,7 +48,7 @@ public static EntitySerDe createEntitySerDe(String name) { String className = ENTITY_SERDES.getOrDefault(name, name); try { - return (EntitySerDe) Class.forName(className).newInstance(); + return (EntitySerDe) Class.forName(className).getDeclaredConstructor().newInstance(); } catch (Exception e) { LOG.error("Failed to create EntitySerDe by name {}.", name, e); throw new RuntimeException("Failed to create EntitySerDe: " + name, e); diff --git a/core/src/main/java/com/datastrato/gravitino/EntityStoreFactory.java b/core/src/main/java/com/datastrato/gravitino/EntityStoreFactory.java index 2bac6badeed..8f1a0367821 100644 --- a/core/src/main/java/com/datastrato/gravitino/EntityStoreFactory.java +++ b/core/src/main/java/com/datastrato/gravitino/EntityStoreFactory.java @@ -36,7 +36,7 @@ public static EntityStore createEntityStore(Config config) { String className = ENTITY_STORES.getOrDefault(name, name); try { - return (EntityStore) Class.forName(className).newInstance(); + return (EntityStore) Class.forName(className).getDeclaredConstructor().newInstance(); } catch (Exception e) { LOG.error("Failed to create and initialize EntityStore by name {}.", name, e); throw new RuntimeException("Failed to create and initialize EntityStore: " + name, e); diff --git a/core/src/main/java/com/datastrato/gravitino/aux/AuxiliaryServiceManager.java b/core/src/main/java/com/datastrato/gravitino/aux/AuxiliaryServiceManager.java index cc1ce3d1167..05fb03f3c28 100644 --- a/core/src/main/java/com/datastrato/gravitino/aux/AuxiliaryServiceManager.java +++ b/core/src/main/java/com/datastrato/gravitino/aux/AuxiliaryServiceManager.java @@ -71,7 +71,7 @@ public GravitinoAuxiliaryService loadAuxService( try { Class providerClz = lookupAuxService(auxServiceName, cl); - return providerClz.newInstance(); + return providerClz.getDeclaredConstructor().newInstance(); } catch (Exception e) { throw new RuntimeException(e); } diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java b/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java index 4f41a0e7c32..9fb32d2c9b3 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java @@ -494,7 +494,7 @@ private BaseCatalog createCatalogInstance(IsolatedClassLoader classLoader, St try { Class providerClz = lookupCatalogProvider(provider, cl); - return (BaseCatalog) providerClz.newInstance(); + return (BaseCatalog) providerClz.getDeclaredConstructor().newInstance(); } catch (Exception e) { LOG.error("Failed to load catalog with provider: {}", provider, e); throw new RuntimeException(e); diff --git a/core/src/main/java/com/datastrato/gravitino/proto/ProtoEntitySerDe.java b/core/src/main/java/com/datastrato/gravitino/proto/ProtoEntitySerDe.java index dd93680d976..0026e6df918 100644 --- a/core/src/main/java/com/datastrato/gravitino/proto/ProtoEntitySerDe.java +++ b/core/src/main/java/com/datastrato/gravitino/proto/ProtoEntitySerDe.java @@ -93,7 +93,7 @@ private ProtoSerDe getProtoSerde( Class> serdeClazz = (Class>) loadClass(ENTITY_TO_SERDE.get(k.getCanonicalName()), classLoader); - return serdeClazz.newInstance(); + return serdeClazz.getDeclaredConstructor().newInstance(); } catch (Exception e) { throw new RuntimeException( "Failed to instantiate serde class " + k.getCanonicalName(), e); diff --git a/core/src/main/java/com/datastrato/gravitino/storage/kv/KvEntityStore.java b/core/src/main/java/com/datastrato/gravitino/storage/kv/KvEntityStore.java index 8db173f834e..d985e7c2f9c 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/kv/KvEntityStore.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/kv/KvEntityStore.java @@ -421,7 +421,8 @@ private static KvBackend createKvEntityBackend(Config config) { } try { - KvBackend kvBackend = (KvBackend) Class.forName(className).newInstance(); + KvBackend kvBackend = + (KvBackend) Class.forName(className).getDeclaredConstructor().newInstance(); kvBackend.initialize(config); return kvBackend; } catch (Exception e) { diff --git a/docs/trino-connector/index.md b/docs/trino-connector/index.md index 3a1bd676904..4c1d2e9f5be 100644 --- a/docs/trino-connector/index.md +++ b/docs/trino-connector/index.md @@ -17,4 +17,4 @@ Gravitino connector index: - [Iceberg](catalog-iceberg.md) - [MySQL](catalog-mysql.md) - [PostgreSQL](catalog-postgresql.md) - - [Supported SQL](sql-support.md) \ No newline at end of file + - [Supported SQL](sql-support.md) diff --git a/gradle.properties b/gradle.properties index 819f90e6cc7..0dbdc6de0a2 100644 --- a/gradle.properties +++ b/gradle.properties @@ -14,3 +14,7 @@ version = 0.4.0-SNAPSHOT #sonatype credentials SONATYPE_USER = admin SONATYPE_PASSWORD = password + +#jdkVersion is used to specify the version of JDK to build and test against Gravitino, current +# supported version is 8, 11, and 17. +jdkVersion = 8 diff --git a/server-common/src/main/java/com/datastrato/gravitino/server/web/JettyServer.java b/server-common/src/main/java/com/datastrato/gravitino/server/web/JettyServer.java index cd6592d2ee8..f111af3a0a3 100644 --- a/server-common/src/main/java/com/datastrato/gravitino/server/web/JettyServer.java +++ b/server-common/src/main/java/com/datastrato/gravitino/server/web/JettyServer.java @@ -16,7 +16,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.security.AccessController; import java.security.PrivilegedAction; import java.util.EnumSet; import java.util.Optional; @@ -393,6 +392,7 @@ private ServerConnector createServerConnector( return new ServerConnector(server, null, serverExecutor, null, -1, -1, connectionFactories); } + @SuppressWarnings("removal") private ThreadPool createThreadPool(int minThreads, int maxThreads, int threadPoolWorkQueueSize) { ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); @@ -404,7 +404,7 @@ private ThreadPool createThreadPool(int minThreads, int maxThreads, int threadPo @Override public Thread newThread(Runnable runnable) { - return AccessController.doPrivileged( + return java.security.AccessController.doPrivileged( new PrivilegedAction() { @Override public Thread run() {