Skip to content

Commit

Permalink
apacheGH-44564: [Java][FlightSQL] Fix native libraries relocation
Browse files Browse the repository at this point in the history 
Prefix used by native libraries shipped in the JDBC driver do not match
the prefix used by Netty `NativeLibraryLoader` class, preventing them to
be detected and loaded.

Change the prefix of the libraries and add a integration test to verify
the libraries are loaded

Also exclude several group of data which are not properly relocated and
may cause conflict with existing classpath.
  • Loading branch information
@laurentgo
laurentgo committed Oct 29, 2024
1 parent 2bbd67d commit 16824ca
Show file tree
Hide file tree
Showing 2 changed files with 79 additions and 30 deletions.
13 changes: 8 additions & 5 deletions java/flight/flight-sql-jdbc-driver/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,14 +127,14 @@ under the License.
<pattern>mozilla.</pattern>
<shadedPattern>org.apache.arrow.driver.jdbc.shaded.mozilla.</shadedPattern>
</relocation>
<!-- Entries to relocate netty native libraries -->
<!-- Entries to relocate netty native libraries. Prefix has to match relocation prefix (dots replaced with underscore) -->
<relocation>
<pattern>META-INF.native.libnetty_</pattern>
<shadedPattern>META-INF.native.liboaadj_netty_</shadedPattern>
<shadedPattern>META-INF.native.liborg_apache_arrow_driver_jdbc_shaded_netty_</shadedPattern>
</relocation>
<relocation>
<pattern>META-INF.native.netty_</pattern>
<shadedPattern>META-INF.native.oaadj_netty_</shadedPattern>
<shadedPattern>META-INF.native.org_apache_arrow_driver_jdbc_shaded_netty_</shadedPattern>
</relocation>
</relocations>
<transformers>
Expand All @@ -159,8 +159,11 @@ under the License.
<exclude>**/*.SF</exclude>
<exclude>**/*.RSA</exclude>
<exclude>**/*.DSA</exclude>
<exclude>META-INF/native/libio_grpc_netty*</exclude>
<exclude>META-INF/native/io_grpc_netty_shaded*</exclude>
<!-- Requires some resource transformer -->
<exclude>META-INF/native-image/</exclude>
<exclude>META-INF/proguard/</exclude>
<!-- Requires MSHADE-406 -->
<exclude>META-INF/versions/</exclude>
<exclude>**/*.proto</exclude>
<exclude>**/module-info.class</exclude>
</excludes>
Expand Down
96 changes: 71 additions & 25 deletions ...ght-sql-jdbc-driver/src/test/java/org/apache/arrow/driver/jdbc/ITDriverJarValidation.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,20 +18,26 @@

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.fail;

import com.google.common.collect.ImmutableSet;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.net.JarURLConnection;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.Enumeration;
import java.net.URLClassLoader;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.function.Predicate;
import java.util.jar.JarEntry;
import java.util.jar.JarFile;
import java.util.stream.Stream;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.Timeout;
import org.junit.jupiter.api.function.Executable;

/**
* Check the content of the JDBC driver jar
Expand All @@ -48,52 +54,92 @@ public class ITDriverJarValidation {

/** List of allowed prefixes a jar entry may match. */
public static final Set<String> ALLOWED_PREFIXES =
ImmutableSet.of("org/apache/arrow/driver/jdbc/", "META-INF/");
ImmutableSet.of(
"org/apache/arrow/driver/jdbc/", // Driver code
"META-INF/maven/", // Maven metadata (useful for security scanner
"META-INF/services/", // ServiceLoader implementations
"META-INF/license/",
"META-INF/licenses/",
// Prefixes for native libraries
"META-INF/native/liborg_apache_arrow_driver_jdbc_shaded_",
"META-INF/native/org_apache_arrow_driver_jdbc_shaded_");

/** List of allowed files a jar entry may match. */
public static final Set<String> ALLOWED_FILES =
ImmutableSet.of("arrow-git.properties", "properties/flight.properties");
ImmutableSet.of(
"arrow-git.properties",
"properties/flight.properties",
"META-INF/io.netty.versions.properties",
"META-INF/MANIFEST.MF",
"META-INF/DEPENDENCIES",
"META-INF/FastDoubleParser-LICENSE",
"META-INF/FastDoubleParser-NOTICE",
"META-INF/LICENSE",
"META-INF/LICENSE.txt",
"META-INF/NOTICE",
"META-INF/NOTICE.txt",
"META-INF/thirdparty-LICENSE",
"META-INF/bigint-LICENSE");

// This method is designed to work with Maven failsafe plugin and expects the
// JDBC driver jar to be present in the test classpath (instead of the individual classes)
private static JarFile getJdbcJarFile() throws IOException {
private static File getJdbcJarFile() throws IOException {
// Check if an override has been set
if (JDBC_DRIVER_PATH_OVERRIDE != null) {
return new JarFile(new File(JDBC_DRIVER_PATH_OVERRIDE));
return new File(JDBC_DRIVER_PATH_OVERRIDE);
}

// Check classpath to find the driver jar
// Check classpath to find the driver jar (without loading the class)
URL driverClassURL =
ITDriverJarValidation.class
.getClassLoader()
.getResource("org/apache/arrow/driver/jdbc/ArrowFlightJdbcDriver.class");

assertNotNull(driverClassURL, "Driver jar was not detected in the classpath");
assertNotNull(driverClassURL, "Driver class was not detected in the classpath");
assertEquals(
"jar", driverClassURL.getProtocol(), "Driver jar was not detected in the classpath");
"jar", driverClassURL.getProtocol(), "Driver class was not found inside a jar file");

// Return the enclosing jar file
JarURLConnection connection = (JarURLConnection) driverClassURL.openConnection();
return connection.getJarFile();
try {
return new File(connection.getJarFileURL().toURI());
} catch (URISyntaxException e) {
throw new IOException(e);
}
}

/** Validate the content of the jar to enforce all 3rd party dependencies have been shaded. */
@Test
@Timeout(value = 2, unit = TimeUnit.MINUTES)
public void validateShadedJar() throws IOException {
// Validate the content of the jar to enforce all 3rd party dependencies have
// been shaded
try (JarFile jar = getJdbcJarFile()) {
for (Enumeration<JarEntry> entries = jar.entries(); entries.hasMoreElements(); ) {
final JarEntry entry = entries.nextElement();
if (entry.isDirectory()) {
// Directories are ignored
continue;
}

try {
checkEntryAllowed(entry.getName());
} catch (AssertionError e) {
fail(e.getMessage());
}

try (JarFile jar = new JarFile(getJdbcJarFile())) {
Stream<Executable> executables =
jar.stream()
.filter(Predicate.not(JarEntry::isDirectory))
.map(
entry -> {
return () -> checkEntryAllowed(entry.getName());
});

Assertions.assertAll(executables);
}
}

/** Check that relocated netty code can also load matching native library. */
@Test
@Timeout(value = 2, unit = TimeUnit.MINUTES)
public void checkNettyOpenSslNativeLoader() throws Throwable {
try (URLClassLoader driverClassLoader =
new URLClassLoader(new URL[] {getJdbcJarFile().toURI().toURL()}, null)) {
Class<?> openSslClass =
driverClassLoader.loadClass(
"org.apache.arrow.driver.jdbc.shaded.io.netty.handler.ssl.OpenSsl");
Method method = openSslClass.getDeclaredMethod("ensureAvailability");
try {
method.invoke(null);
} catch (InvocationTargetException e) {
throw e.getCause();
}
}
}
Expand Down

0 comments on commit 16824ca

Please sign in to comment.