Skip to content

Commit

Permalink
GH-44564: [Java][FlightSQL] Fix native libraries relocation (#44565)
Browse files Browse the repository at this point in the history 
### Rationale for this change

Prefix used by native libraries shipped in the JDBC driver do not match the prefix used by Netty `NativeLibraryLoader` class, preventing them to be detected and loaded.

### What changes are included in this PR?

Change the prefix of the libraries and add a integration test to verify the libraries are loaded

Also exclude several group of data which are not properly relocated and may cause conflict with existing classpath.

### Are these changes tested?

Yes, new test added for checking the native library loader

### Are there any user-facing changes?

No

* GitHub Issue: #44564

Authored-by: Laurent Goujon <[email protected]>
Signed-off-by: David Li <[email protected]>
  • Loading branch information
@laurentgo
laurentgo authored Oct 30, 2024
1 parent 5e60823 commit 3217fc2
Show file tree
Hide file tree
Showing 2 changed files with 79 additions and 30 deletions.
13 changes: 8 additions & 5 deletions java/flight/flight-sql-jdbc-driver/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,14 +127,14 @@ under the License.
<pattern>mozilla.</pattern>
<shadedPattern>org.apache.arrow.driver.jdbc.shaded.mozilla.</shadedPattern>
</relocation>
<!-- Entries to relocate netty native libraries -->
<!-- Entries to relocate netty native libraries. Prefix has to match relocation prefix (dots replaced with underscore) -->
<relocation>
<pattern>META-INF.native.libnetty_</pattern>
<shadedPattern>META-INF.native.liboaadj_netty_</shadedPattern>
<shadedPattern>META-INF.native.liborg_apache_arrow_driver_jdbc_shaded_netty_</shadedPattern>
</relocation>
<relocation>
<pattern>META-INF.native.netty_</pattern>
<shadedPattern>META-INF.native.oaadj_netty_</shadedPattern>
<shadedPattern>META-INF.native.org_apache_arrow_driver_jdbc_shaded_netty_</shadedPattern>
</relocation>
</relocations>
<transformers>
Expand All @@ -159,8 +159,11 @@ under the License.
<exclude>**/*.SF</exclude>
<exclude>**/*.RSA</exclude>
<exclude>**/*.DSA</exclude>
<exclude>META-INF/native/libio_grpc_netty*</exclude>
<exclude>META-INF/native/io_grpc_netty_shaded*</exclude>
<!-- Requires some resource transformer -->
<exclude>META-INF/native-image/</exclude>
<exclude>META-INF/proguard/</exclude>
<!-- Requires MSHADE-406 -->
<exclude>META-INF/versions/</exclude>
<exclude>**/*.proto</exclude>
<exclude>**/module-info.class</exclude>
</excludes>
Expand Down
96 changes: 71 additions & 25 deletions ...ght-sql-jdbc-driver/src/test/java/org/apache/arrow/driver/jdbc/ITDriverJarValidation.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,20 +18,26 @@

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.fail;

import com.google.common.collect.ImmutableSet;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.net.JarURLConnection;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.Enumeration;
import java.net.URLClassLoader;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.function.Predicate;
import java.util.jar.JarEntry;
import java.util.jar.JarFile;
import java.util.stream.Stream;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.Timeout;
import org.junit.jupiter.api.function.Executable;

/**
* Check the content of the JDBC driver jar
Expand All @@ -48,52 +54,92 @@ public class ITDriverJarValidation {

/** List of allowed prefixes a jar entry may match. */
public static final Set<String> ALLOWED_PREFIXES =
ImmutableSet.of("org/apache/arrow/driver/jdbc/", "META-INF/");
ImmutableSet.of(
"org/apache/arrow/driver/jdbc/", // Driver code
"META-INF/maven/", // Maven metadata (useful for security scanner
"META-INF/services/", // ServiceLoader implementations
"META-INF/license/",
"META-INF/licenses/",
// Prefixes for native libraries
"META-INF/native/liborg_apache_arrow_driver_jdbc_shaded_",
"META-INF/native/org_apache_arrow_driver_jdbc_shaded_");

/** List of allowed files a jar entry may match. */
public static final Set<String> ALLOWED_FILES =
ImmutableSet.of("arrow-git.properties", "properties/flight.properties");
ImmutableSet.of(
"arrow-git.properties",
"properties/flight.properties",
"META-INF/io.netty.versions.properties",
"META-INF/MANIFEST.MF",
"META-INF/DEPENDENCIES",
"META-INF/FastDoubleParser-LICENSE",
"META-INF/FastDoubleParser-NOTICE",
"META-INF/LICENSE",
"META-INF/LICENSE.txt",
"META-INF/NOTICE",
"META-INF/NOTICE.txt",
"META-INF/thirdparty-LICENSE",
"META-INF/bigint-LICENSE");

// This method is designed to work with Maven failsafe plugin and expects the
// JDBC driver jar to be present in the test classpath (instead of the individual classes)
private static JarFile getJdbcJarFile() throws IOException {
private static File getJdbcJarFile() throws IOException {
// Check if an override has been set
if (JDBC_DRIVER_PATH_OVERRIDE != null) {
return new JarFile(new File(JDBC_DRIVER_PATH_OVERRIDE));
return new File(JDBC_DRIVER_PATH_OVERRIDE);
}

// Check classpath to find the driver jar
// Check classpath to find the driver jar (without loading the class)
URL driverClassURL =
ITDriverJarValidation.class
.getClassLoader()
.getResource("org/apache/arrow/driver/jdbc/ArrowFlightJdbcDriver.class");

assertNotNull(driverClassURL, "Driver jar was not detected in the classpath");
assertNotNull(driverClassURL, "Driver class was not detected in the classpath");
assertEquals(
"jar", driverClassURL.getProtocol(), "Driver jar was not detected in the classpath");
"jar", driverClassURL.getProtocol(), "Driver class was not found inside a jar file");

// Return the enclosing jar file
JarURLConnection connection = (JarURLConnection) driverClassURL.openConnection();
return connection.getJarFile();
try {
return new File(connection.getJarFileURL().toURI());
} catch (URISyntaxException e) {
throw new IOException(e);
}
}

/** Validate the content of the jar to enforce all 3rd party dependencies have been shaded. */
@Test
@Timeout(value = 2, unit = TimeUnit.MINUTES)
public void validateShadedJar() throws IOException {
// Validate the content of the jar to enforce all 3rd party dependencies have
// been shaded
try (JarFile jar = getJdbcJarFile()) {
for (Enumeration<JarEntry> entries = jar.entries(); entries.hasMoreElements(); ) {
final JarEntry entry = entries.nextElement();
if (entry.isDirectory()) {
// Directories are ignored
continue;
}

try {
checkEntryAllowed(entry.getName());
} catch (AssertionError e) {
fail(e.getMessage());
}

try (JarFile jar = new JarFile(getJdbcJarFile())) {
Stream<Executable> executables =
jar.stream()
.filter(Predicate.not(JarEntry::isDirectory))
.map(
entry -> {
return () -> checkEntryAllowed(entry.getName());
});

Assertions.assertAll(executables);
}
}

/** Check that relocated netty code can also load matching native library. */
@Test
@Timeout(value = 2, unit = TimeUnit.MINUTES)
public void checkNettyOpenSslNativeLoader() throws Throwable {
try (URLClassLoader driverClassLoader =
new URLClassLoader(new URL[] {getJdbcJarFile().toURI().toURL()}, null)) {
Class<?> openSslClass =
driverClassLoader.loadClass(
"org.apache.arrow.driver.jdbc.shaded.io.netty.handler.ssl.OpenSsl");
Method method = openSslClass.getDeclaredMethod("ensureAvailability");
try {
method.invoke(null);
} catch (InvocationTargetException e) {
throw e.getCause();
}
}
}
Expand Down

0 comments on commit 3217fc2

Please sign in to comment.