From 16824ca5fdd1e662d50be3d492c2d6431da9fda9 Mon Sep 17 00:00:00 2001 From: Laurent Goujon Date: Tue, 29 Oct 2024 10:53:40 -0500 Subject: [PATCH] GH-44564: [Java][FlightSQL] Fix native libraries relocation Prefix used by native libraries shipped in the JDBC driver do not match the prefix used by Netty `NativeLibraryLoader` class, preventing them to be detected and loaded. Change the prefix of the libraries and add a integration test to verify the libraries are loaded Also exclude several group of data which are not properly relocated and may cause conflict with existing classpath. --- java/flight/flight-sql-jdbc-driver/pom.xml | 13 ++- .../driver/jdbc/ITDriverJarValidation.java | 96 ++++++++++++++----- 2 files changed, 79 insertions(+), 30 deletions(-) diff --git a/java/flight/flight-sql-jdbc-driver/pom.xml b/java/flight/flight-sql-jdbc-driver/pom.xml index 148319e5d9d64..a739874d3924e 100644 --- a/java/flight/flight-sql-jdbc-driver/pom.xml +++ b/java/flight/flight-sql-jdbc-driver/pom.xml @@ -127,14 +127,14 @@ under the License. mozilla. org.apache.arrow.driver.jdbc.shaded.mozilla. - + META-INF.native.libnetty_ - META-INF.native.liboaadj_netty_ + META-INF.native.liborg_apache_arrow_driver_jdbc_shaded_netty_ META-INF.native.netty_ - META-INF.native.oaadj_netty_ + META-INF.native.org_apache_arrow_driver_jdbc_shaded_netty_ @@ -159,8 +159,11 @@ under the License. **/*.SF **/*.RSA **/*.DSA - META-INF/native/libio_grpc_netty* - META-INF/native/io_grpc_netty_shaded* + + META-INF/native-image/ + META-INF/proguard/ + + META-INF/versions/ **/*.proto **/module-info.class diff --git a/java/flight/flight-sql-jdbc-driver/src/test/java/org/apache/arrow/driver/jdbc/ITDriverJarValidation.java b/java/flight/flight-sql-jdbc-driver/src/test/java/org/apache/arrow/driver/jdbc/ITDriverJarValidation.java index b45845485a041..a0e108d6a067b 100644 --- a/java/flight/flight-sql-jdbc-driver/src/test/java/org/apache/arrow/driver/jdbc/ITDriverJarValidation.java +++ b/java/flight/flight-sql-jdbc-driver/src/test/java/org/apache/arrow/driver/jdbc/ITDriverJarValidation.java @@ -18,20 +18,26 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.fail; import com.google.common.collect.ImmutableSet; import java.io.File; import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.net.JarURLConnection; +import java.net.URISyntaxException; import java.net.URL; -import java.util.Enumeration; +import java.net.URLClassLoader; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.function.Predicate; import java.util.jar.JarEntry; import java.util.jar.JarFile; +import java.util.stream.Stream; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.api.function.Executable; /** * Check the content of the JDBC driver jar @@ -48,52 +54,92 @@ public class ITDriverJarValidation { /** List of allowed prefixes a jar entry may match. */ public static final Set ALLOWED_PREFIXES = - ImmutableSet.of("org/apache/arrow/driver/jdbc/", "META-INF/"); + ImmutableSet.of( + "org/apache/arrow/driver/jdbc/", // Driver code + "META-INF/maven/", // Maven metadata (useful for security scanner + "META-INF/services/", // ServiceLoader implementations + "META-INF/license/", + "META-INF/licenses/", + // Prefixes for native libraries + "META-INF/native/liborg_apache_arrow_driver_jdbc_shaded_", + "META-INF/native/org_apache_arrow_driver_jdbc_shaded_"); /** List of allowed files a jar entry may match. */ public static final Set ALLOWED_FILES = - ImmutableSet.of("arrow-git.properties", "properties/flight.properties"); + ImmutableSet.of( + "arrow-git.properties", + "properties/flight.properties", + "META-INF/io.netty.versions.properties", + "META-INF/MANIFEST.MF", + "META-INF/DEPENDENCIES", + "META-INF/FastDoubleParser-LICENSE", + "META-INF/FastDoubleParser-NOTICE", + "META-INF/LICENSE", + "META-INF/LICENSE.txt", + "META-INF/NOTICE", + "META-INF/NOTICE.txt", + "META-INF/thirdparty-LICENSE", + "META-INF/bigint-LICENSE"); // This method is designed to work with Maven failsafe plugin and expects the // JDBC driver jar to be present in the test classpath (instead of the individual classes) - private static JarFile getJdbcJarFile() throws IOException { + private static File getJdbcJarFile() throws IOException { // Check if an override has been set if (JDBC_DRIVER_PATH_OVERRIDE != null) { - return new JarFile(new File(JDBC_DRIVER_PATH_OVERRIDE)); + return new File(JDBC_DRIVER_PATH_OVERRIDE); } - // Check classpath to find the driver jar + // Check classpath to find the driver jar (without loading the class) URL driverClassURL = ITDriverJarValidation.class .getClassLoader() .getResource("org/apache/arrow/driver/jdbc/ArrowFlightJdbcDriver.class"); - assertNotNull(driverClassURL, "Driver jar was not detected in the classpath"); + assertNotNull(driverClassURL, "Driver class was not detected in the classpath"); assertEquals( - "jar", driverClassURL.getProtocol(), "Driver jar was not detected in the classpath"); + "jar", driverClassURL.getProtocol(), "Driver class was not found inside a jar file"); + // Return the enclosing jar file JarURLConnection connection = (JarURLConnection) driverClassURL.openConnection(); - return connection.getJarFile(); + try { + return new File(connection.getJarFileURL().toURI()); + } catch (URISyntaxException e) { + throw new IOException(e); + } } + /** Validate the content of the jar to enforce all 3rd party dependencies have been shaded. */ @Test @Timeout(value = 2, unit = TimeUnit.MINUTES) public void validateShadedJar() throws IOException { - // Validate the content of the jar to enforce all 3rd party dependencies have - // been shaded - try (JarFile jar = getJdbcJarFile()) { - for (Enumeration entries = jar.entries(); entries.hasMoreElements(); ) { - final JarEntry entry = entries.nextElement(); - if (entry.isDirectory()) { - // Directories are ignored - continue; - } - - try { - checkEntryAllowed(entry.getName()); - } catch (AssertionError e) { - fail(e.getMessage()); - } + + try (JarFile jar = new JarFile(getJdbcJarFile())) { + Stream executables = + jar.stream() + .filter(Predicate.not(JarEntry::isDirectory)) + .map( + entry -> { + return () -> checkEntryAllowed(entry.getName()); + }); + + Assertions.assertAll(executables); + } + } + + /** Check that relocated netty code can also load matching native library. */ + @Test + @Timeout(value = 2, unit = TimeUnit.MINUTES) + public void checkNettyOpenSslNativeLoader() throws Throwable { + try (URLClassLoader driverClassLoader = + new URLClassLoader(new URL[] {getJdbcJarFile().toURI().toURL()}, null)) { + Class openSslClass = + driverClassLoader.loadClass( + "org.apache.arrow.driver.jdbc.shaded.io.netty.handler.ssl.OpenSsl"); + Method method = openSslClass.getDeclaredMethod("ensureAvailability"); + try { + method.invoke(null); + } catch (InvocationTargetException e) { + throw e.getCause(); } } }