From 2477685aa93c277eb632106581e4f993ed85ff61 Mon Sep 17 00:00:00 2001 From: Jerry Shao Date: Tue, 19 Dec 2023 17:36:33 +0800 Subject: [PATCH] [#1179] feat(build): Make Gravitino build and run against JDK8, 11, 17 (#1199) ### What changes were proposed in this pull request? This PR proposes to add multiple JDK version support for building and running Gravitino server, including JDK8, 11, and 17. Users could specify the version they want to build in `gradle.properties`, like: ``` jdkVersion = 8 or jdkVersion = 11 or jdkVersion = 17 ``` Or users could specify the version by `./gradlew build -PjdkVersion=8`. Our generated jars are JDK 8 version compatible, so it can run against Java runtime 8, 11, and 17. ### Why are the changes needed? To make Gravitino build, test and run against multiple JDK versions. Fix: #1179 ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? This was compiled, tested, run on JDK8, 11, and 17 locally. --- build.gradle.kts | 46 +++++++++++++++++-- .../catalog/hive/dyn/DynConstructors.java | 4 +- .../gravitino/catalog/hive/dyn/DynFields.java | 4 +- .../catalog/hive/dyn/DynMethods.java | 4 +- .../gravitino/EntitySerDeFactory.java | 2 +- .../gravitino/EntityStoreFactory.java | 2 +- .../aux/AuxiliaryServiceManager.java | 2 +- .../gravitino/catalog/CatalogManager.java | 2 +- .../gravitino/proto/ProtoEntitySerDe.java | 2 +- .../gravitino/storage/kv/KvEntityStore.java | 3 +- docs/trino-connector/index.md | 2 +- gradle.properties | 4 ++ .../gravitino/server/web/JettyServer.java | 4 +- 13 files changed, 63 insertions(+), 18 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index ed1b59048d2..ab9c74558a3 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -32,7 +32,7 @@ plugins { alias(libs.plugins.spotless) } else { throw GradleException( - "Gravitino Gradle current doesn't support " + + "Gravitino Gradle toolchain current doesn't support " + "Java version: ${JavaVersion.current()}. Please use JDK8 to 17." ) } @@ -44,6 +44,42 @@ plugins { id("org.cyclonedx.bom") version "1.5.0" // Newer version fail due to our setup } +if (extra["jdkVersion"] !in listOf("8", "11", "17")) { + throw GradleException( + "Gravitino current doesn't support building with " + + "Java version: ${extra["jdkVersion"]}. Please use JDK8, 11 or 17." + ) +} + +project.extra["extraJvmArgs"] = if (extra["jdkVersion"] in listOf("8", "11")) { + listOf() +} else { + listOf( + "-XX:+IgnoreUnrecognizedVMOptions", + "--add-opens", "java.base/java.io=ALL-UNNAMED", + "--add-opens", "java.base/java.lang.invoke=ALL-UNNAMED", + "--add-opens", "java.base/java.lang.reflect=ALL-UNNAMED", + "--add-opens", "java.base/java.lang=ALL-UNNAMED", + "--add-opens", "java.base/java.math=ALL-UNNAMED", + "--add-opens", "java.base/java.net=ALL-UNNAMED", + "--add-opens", "java.base/java.nio=ALL-UNNAMED", + "--add-opens", "java.base/java.text=ALL-UNNAMED", + "--add-opens", "java.base/java.time=ALL-UNNAMED", + "--add-opens", "java.base/java.util.concurrent.atomic=ALL-UNNAMED", + "--add-opens", "java.base/java.util.concurrent=ALL-UNNAMED", + "--add-opens", "java.base/java.util.regex=ALL-UNNAMED", + "--add-opens", "java.base/java.util=ALL-UNNAMED", + "--add-opens", "java.base/jdk.internal.ref=ALL-UNNAMED", + "--add-opens", "java.base/jdk.internal.reflect=ALL-UNNAMED", + "--add-opens", "java.sql/java.sql=ALL-UNNAMED", + "--add-opens", "java.base/sun.util.calendar=ALL-UNNAMED", + "--add-opens", "java.base/sun.nio.ch=ALL-UNNAMED", + "--add-opens", "java.base/sun.nio.cs=ALL-UNNAMED", + "--add-opens", "java.base/sun.security.action=ALL-UNNAMED", + "--add-opens", "java.base/sun.util.calendar=ALL-UNNAMED" + ) +} + licenseReport { renderers = arrayOf(InventoryHtmlReportRenderer("report.html", "Backend")) filters = arrayOf(LicenseBundleNormalizer()) @@ -129,7 +165,9 @@ subprojects { if (project.name == "trino-connector") { languageVersion.set(JavaLanguageVersion.of(17)) } else { - languageVersion.set(JavaLanguageVersion.of(8)) + languageVersion.set(JavaLanguageVersion.of(extra["jdkVersion"].toString().toInt())) + sourceCompatibility = JavaVersion.VERSION_1_8 + targetCompatibility = JavaVersion.VERSION_1_8 } } } @@ -208,7 +246,7 @@ subprojects { sign(publishing.publications) } - tasks.configureEach { + tasks.withType { testLogging { exceptionFormat = TestExceptionFormat.FULL showExceptions = true @@ -224,6 +262,8 @@ subprojects { } else { useJUnitPlatform() } + + jvmArgs(project.property("extraJvmArgs") as List<*>) finalizedBy(tasks.getByName("jacocoTestReport")) } } diff --git a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/dyn/DynConstructors.java b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/dyn/DynConstructors.java index 9a254951a4e..593df47eb86 100644 --- a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/dyn/DynConstructors.java +++ b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/dyn/DynConstructors.java @@ -23,7 +23,6 @@ import com.google.common.collect.Maps; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; -import java.security.AccessController; import java.security.PrivilegedAction; import java.util.Arrays; import java.util.Map; @@ -245,6 +244,7 @@ public Builder hiddenImpl(String className, Class... types) { return this; } + @SuppressWarnings("removal") public Builder hiddenImpl(Class targetClass, Class... types) { // don't do any work if an implementation has been found if (ctor != null) { @@ -253,7 +253,7 @@ public Builder hiddenImpl(Class targetClass, Class... types) { try { Constructor hidden = targetClass.getDeclaredConstructor(types); - AccessController.doPrivileged(new MakeAccessible(hidden)); + java.security.AccessController.doPrivileged(new MakeAccessible(hidden)); ctor = new Ctor(hidden, targetClass); } catch (SecurityException e) { // unusable diff --git a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/dyn/DynFields.java b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/dyn/DynFields.java index 3c8f61cba8a..0d7f67d98dc 100644 --- a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/dyn/DynFields.java +++ b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/dyn/DynFields.java @@ -25,7 +25,6 @@ import com.google.common.collect.Sets; import java.lang.reflect.Field; import java.lang.reflect.Modifier; -import java.security.AccessController; import java.security.PrivilegedAction; import java.util.Set; @@ -302,6 +301,7 @@ public Builder hiddenImpl(String className, String fieldName) { * @see Class#forName(String) * @see Class#getField(String) */ + @SuppressWarnings("removal") public Builder hiddenImpl(Class targetClass, String fieldName) { // don't do any work if an implementation has been found if (field != null || targetClass == null) { @@ -310,7 +310,7 @@ public Builder hiddenImpl(Class targetClass, String fieldName) { try { Field hidden = targetClass.getDeclaredField(fieldName); - AccessController.doPrivileged(new MakeFieldAccessible(hidden)); + java.security.AccessController.doPrivileged(new MakeFieldAccessible(hidden)); this.field = new UnboundField(hidden, fieldName); } catch (SecurityException | NoSuchFieldException e) { // unusable diff --git a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/dyn/DynMethods.java b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/dyn/DynMethods.java index 7d07585ba34..cc27dbb002c 100644 --- a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/dyn/DynMethods.java +++ b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/dyn/DynMethods.java @@ -23,7 +23,6 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.lang.reflect.Modifier; -import java.security.AccessController; import java.security.PrivilegedAction; import java.util.Arrays; @@ -393,6 +392,7 @@ public Builder hiddenImpl(String className, Class... argClasses) { * @see Class#forName(String) * @see Class#getMethod(String, Class[]) */ + @SuppressWarnings("removal") public Builder hiddenImpl(Class targetClass, String methodName, Class... argClasses) { // don't do any work if an implementation has been found if (method != null) { @@ -401,7 +401,7 @@ public Builder hiddenImpl(Class targetClass, String methodName, Class... a try { Method hidden = targetClass.getDeclaredMethod(methodName, argClasses); - AccessController.doPrivileged(new MakeAccessible(hidden)); + java.security.AccessController.doPrivileged(new MakeAccessible(hidden)); this.method = new UnboundMethod(hidden, name); } catch (SecurityException | NoSuchMethodException e) { // unusable or not the right implementation diff --git a/core/src/main/java/com/datastrato/gravitino/EntitySerDeFactory.java b/core/src/main/java/com/datastrato/gravitino/EntitySerDeFactory.java index a5472f2e261..fc59fe77357 100644 --- a/core/src/main/java/com/datastrato/gravitino/EntitySerDeFactory.java +++ b/core/src/main/java/com/datastrato/gravitino/EntitySerDeFactory.java @@ -48,7 +48,7 @@ public static EntitySerDe createEntitySerDe(String name) { String className = ENTITY_SERDES.getOrDefault(name, name); try { - return (EntitySerDe) Class.forName(className).newInstance(); + return (EntitySerDe) Class.forName(className).getDeclaredConstructor().newInstance(); } catch (Exception e) { LOG.error("Failed to create EntitySerDe by name {}.", name, e); throw new RuntimeException("Failed to create EntitySerDe: " + name, e); diff --git a/core/src/main/java/com/datastrato/gravitino/EntityStoreFactory.java b/core/src/main/java/com/datastrato/gravitino/EntityStoreFactory.java index 2bac6badeed..8f1a0367821 100644 --- a/core/src/main/java/com/datastrato/gravitino/EntityStoreFactory.java +++ b/core/src/main/java/com/datastrato/gravitino/EntityStoreFactory.java @@ -36,7 +36,7 @@ public static EntityStore createEntityStore(Config config) { String className = ENTITY_STORES.getOrDefault(name, name); try { - return (EntityStore) Class.forName(className).newInstance(); + return (EntityStore) Class.forName(className).getDeclaredConstructor().newInstance(); } catch (Exception e) { LOG.error("Failed to create and initialize EntityStore by name {}.", name, e); throw new RuntimeException("Failed to create and initialize EntityStore: " + name, e); diff --git a/core/src/main/java/com/datastrato/gravitino/aux/AuxiliaryServiceManager.java b/core/src/main/java/com/datastrato/gravitino/aux/AuxiliaryServiceManager.java index cc1ce3d1167..05fb03f3c28 100644 --- a/core/src/main/java/com/datastrato/gravitino/aux/AuxiliaryServiceManager.java +++ b/core/src/main/java/com/datastrato/gravitino/aux/AuxiliaryServiceManager.java @@ -71,7 +71,7 @@ public GravitinoAuxiliaryService loadAuxService( try { Class providerClz = lookupAuxService(auxServiceName, cl); - return providerClz.newInstance(); + return providerClz.getDeclaredConstructor().newInstance(); } catch (Exception e) { throw new RuntimeException(e); } diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java b/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java index 4f41a0e7c32..9fb32d2c9b3 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java @@ -494,7 +494,7 @@ private BaseCatalog createCatalogInstance(IsolatedClassLoader classLoader, St try { Class providerClz = lookupCatalogProvider(provider, cl); - return (BaseCatalog) providerClz.newInstance(); + return (BaseCatalog) providerClz.getDeclaredConstructor().newInstance(); } catch (Exception e) { LOG.error("Failed to load catalog with provider: {}", provider, e); throw new RuntimeException(e); diff --git a/core/src/main/java/com/datastrato/gravitino/proto/ProtoEntitySerDe.java b/core/src/main/java/com/datastrato/gravitino/proto/ProtoEntitySerDe.java index dd93680d976..0026e6df918 100644 --- a/core/src/main/java/com/datastrato/gravitino/proto/ProtoEntitySerDe.java +++ b/core/src/main/java/com/datastrato/gravitino/proto/ProtoEntitySerDe.java @@ -93,7 +93,7 @@ private ProtoSerDe getProtoSerde( Class> serdeClazz = (Class>) loadClass(ENTITY_TO_SERDE.get(k.getCanonicalName()), classLoader); - return serdeClazz.newInstance(); + return serdeClazz.getDeclaredConstructor().newInstance(); } catch (Exception e) { throw new RuntimeException( "Failed to instantiate serde class " + k.getCanonicalName(), e); diff --git a/core/src/main/java/com/datastrato/gravitino/storage/kv/KvEntityStore.java b/core/src/main/java/com/datastrato/gravitino/storage/kv/KvEntityStore.java index 8db173f834e..d985e7c2f9c 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/kv/KvEntityStore.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/kv/KvEntityStore.java @@ -421,7 +421,8 @@ private static KvBackend createKvEntityBackend(Config config) { } try { - KvBackend kvBackend = (KvBackend) Class.forName(className).newInstance(); + KvBackend kvBackend = + (KvBackend) Class.forName(className).getDeclaredConstructor().newInstance(); kvBackend.initialize(config); return kvBackend; } catch (Exception e) { diff --git a/docs/trino-connector/index.md b/docs/trino-connector/index.md index 3a1bd676904..4c1d2e9f5be 100644 --- a/docs/trino-connector/index.md +++ b/docs/trino-connector/index.md @@ -17,4 +17,4 @@ Gravitino connector index: - [Iceberg](catalog-iceberg.md) - [MySQL](catalog-mysql.md) - [PostgreSQL](catalog-postgresql.md) - - [Supported SQL](sql-support.md) \ No newline at end of file + - [Supported SQL](sql-support.md) diff --git a/gradle.properties b/gradle.properties index 819f90e6cc7..0dbdc6de0a2 100644 --- a/gradle.properties +++ b/gradle.properties @@ -14,3 +14,7 @@ version = 0.4.0-SNAPSHOT #sonatype credentials SONATYPE_USER = admin SONATYPE_PASSWORD = password + +#jdkVersion is used to specify the version of JDK to build and test against Gravitino, current +# supported version is 8, 11, and 17. +jdkVersion = 8 diff --git a/server-common/src/main/java/com/datastrato/gravitino/server/web/JettyServer.java b/server-common/src/main/java/com/datastrato/gravitino/server/web/JettyServer.java index cd6592d2ee8..f111af3a0a3 100644 --- a/server-common/src/main/java/com/datastrato/gravitino/server/web/JettyServer.java +++ b/server-common/src/main/java/com/datastrato/gravitino/server/web/JettyServer.java @@ -16,7 +16,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.security.AccessController; import java.security.PrivilegedAction; import java.util.EnumSet; import java.util.Optional; @@ -393,6 +392,7 @@ private ServerConnector createServerConnector( return new ServerConnector(server, null, serverExecutor, null, -1, -1, connectionFactories); } + @SuppressWarnings("removal") private ThreadPool createThreadPool(int minThreads, int maxThreads, int threadPoolWorkQueueSize) { ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); @@ -404,7 +404,7 @@ private ThreadPool createThreadPool(int minThreads, int maxThreads, int threadPo @Override public Thread newThread(Runnable runnable) { - return AccessController.doPrivileged( + return java.security.AccessController.doPrivileged( new PrivilegedAction() { @Override public Thread run() {