Skip to content

Commit

Permalink
Merge pull request #34722 from cescoffier/snappy-conditional-loading-jvm
Browse files Browse the repository at this point in the history
Make Snappy optional in JVM mode
  • Loading branch information
cescoffier authored Jul 19, 2023
2 parents b9449e5 + 2a482ef commit e0bcf74
Show file tree
Hide file tree
Showing 4 changed files with 87 additions and 65 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,6 @@
import org.eclipse.microprofile.config.ConfigProvider;
import org.jboss.jandex.ClassInfo;
import org.jboss.jandex.DotName;
import org.xerial.snappy.OSInfo;

import io.quarkus.arc.deployment.AdditionalBeanBuildItem;
import io.quarkus.arc.deployment.UnremovableBeanBuildItem;
Expand Down Expand Up @@ -95,6 +94,7 @@
import io.quarkus.kafka.client.runtime.KafkaBindingConverter;
import io.quarkus.kafka.client.runtime.KafkaRecorder;
import io.quarkus.kafka.client.runtime.KafkaRuntimeConfigProducer;
import io.quarkus.kafka.client.runtime.SnappyRecorder;
import io.quarkus.kafka.client.runtime.ui.KafkaTopicClient;
import io.quarkus.kafka.client.runtime.ui.KafkaUiRecorder;
import io.quarkus.kafka.client.runtime.ui.KafkaUiUtils;
Expand Down Expand Up @@ -292,7 +292,7 @@ public void build(

}

@BuildStep(onlyIf = { IsSnappy.class, NativeOrNativeSourcesBuild.class })
@BuildStep(onlyIf = { HasSnappy.class, NativeOrNativeSourcesBuild.class })
public void handleSnappyInNative(NativeImageRunnerBuildItem nativeImageRunner,
BuildProducer<ReflectiveClassBuildItem> reflectiveClass,
BuildProducer<NativeImageResourceBuildItem> nativeLibs) {
Expand All @@ -307,19 +307,17 @@ public void handleSnappyInNative(NativeImageRunnerBuildItem nativeImageRunner,
String path = root + dir + "/" + snappyNativeLibraryName;
nativeLibs.produce(new NativeImageResourceBuildItem(path));
} else { // otherwise the native lib of the platform this build runs on
String dir = OSInfo.getNativeLibFolderPathForCurrentOS();
String dir = SnappyUtils.getNativeLibFolderPathForCurrentOS();
String snappyNativeLibraryName = System.mapLibraryName("snappyjava");
String path = root + dir + "/" + snappyNativeLibraryName;
nativeLibs.produce(new NativeImageResourceBuildItem(path));
}
}

@BuildStep
@BuildStep(onlyIf = HasSnappy.class)
@Record(ExecutionTime.RUNTIME_INIT)
void loadSnappyIfEnabled(KafkaRecorder recorder, KafkaBuildTimeConfig config) {
if (config.snappyEnabled) {
recorder.loadSnappy();
}
void loadSnappyIfEnabled(SnappyRecorder recorder, KafkaBuildTimeConfig config) {
recorder.loadSnappy();
}

@Consume(RuntimeConfigSetupCompleteBuildItem.class)
Expand Down Expand Up @@ -585,17 +583,17 @@ public DevConsoleWebjarBuildItem setupWebJar(LaunchModeBuildItem launchModeBuild
.build();
}

public static final class IsSnappy implements BooleanSupplier {
public static final class HasSnappy implements BooleanSupplier {

private final KafkaBuildTimeConfig config;

public IsSnappy(KafkaBuildTimeConfig config) {
public HasSnappy(KafkaBuildTimeConfig config) {
this.config = config;
}

@Override
public boolean getAsBoolean() {
return config.snappyEnabled;
return QuarkusClassLoader.isClassPresentAtRuntime("org.xerial.snappy.OSInfo") && config.snappyEnabled;
}
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
package io.quarkus.kafka.client.deployment;

import org.xerial.snappy.OSInfo;

/**
* This class should only be used if Snappy is available on the classpath.
*/
public class SnappyUtils {

private SnappyUtils() {
// Avoid direct instantiation
}

public static String getNativeLibFolderPathForCurrentOS() {
return OSInfo.getNativeLibFolderPathForCurrentOS();
}
}
Original file line number Diff line number Diff line change
@@ -1,69 +1,15 @@
package io.quarkus.kafka.client.runtime;

import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.net.URL;
import java.util.Optional;

import org.eclipse.microprofile.config.Config;
import org.eclipse.microprofile.config.ConfigProvider;
import org.xerial.snappy.OSInfo;
import org.xerial.snappy.SnappyError;
import org.xerial.snappy.SnappyErrorCode;
import org.xerial.snappy.SnappyLoader;

import io.quarkus.runtime.annotations.Recorder;

@Recorder
public class KafkaRecorder {

public void loadSnappy() {
// Resolve the library file name with a suffix (e.g., dll, .so, etc.)
String snappyNativeLibraryName = System.mapLibraryName("snappyjava");
String snappyNativeLibraryPath = "/org/xerial/snappy/native/" + OSInfo.getNativeLibFolderPathForCurrentOS();
boolean hasNativeLib = hasResource(snappyNativeLibraryPath + "/" + snappyNativeLibraryName);

if (!hasNativeLib) {
String errorMessage = String.format("no native library is found for os.name=%s and os.arch=%s", OSInfo.getOSName(),
OSInfo.getArchName());
throw new SnappyError(SnappyErrorCode.FAILED_TO_LOAD_NATIVE_LIBRARY, errorMessage);
}

File out = extractLibraryFile(
SnappyLoader.class.getResource(snappyNativeLibraryPath + "/" + snappyNativeLibraryName),
snappyNativeLibraryName);

System.load(out.getAbsolutePath());
}

private static boolean hasResource(String path) {
return SnappyLoader.class.getResource(path) != null;
}

private static File extractLibraryFile(URL library, String name) {
String tmp = System.getProperty("java.io.tmpdir");
File extractedLibFile = new File(tmp, name);

try (BufferedInputStream inputStream = new BufferedInputStream(library.openStream());
FileOutputStream fileOS = new FileOutputStream(extractedLibFile)) {
byte[] data = new byte[8192];
int byteContent;
while ((byteContent = inputStream.read(data, 0, 8192)) != -1) {
fileOS.write(data, 0, byteContent);
}
} catch (IOException e) {
throw new UncheckedIOException(
"Unable to extract native library " + name + " to " + extractedLibFile.getAbsolutePath(), e);
}

extractedLibFile.deleteOnExit();

return extractedLibFile;
}

public void checkBoostrapServers() {
Config config = ConfigProvider.getConfig();
Boolean serviceBindingEnabled = config.getValue("quarkus.kubernetes-service-binding.enabled", Boolean.class);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
package io.quarkus.kafka.client.runtime;

import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.net.URL;

import org.xerial.snappy.OSInfo;
import org.xerial.snappy.SnappyLoader;

import io.quarkus.runtime.annotations.Recorder;

@Recorder
public class SnappyRecorder {

public void loadSnappy() {
// Resolve the library file name with a suffix (e.g., dll, .so, etc.)
String snappyNativeLibraryName = System.mapLibraryName("snappyjava");
String snappyNativeLibraryPath = "/org/xerial/snappy/native/" + OSInfo.getNativeLibFolderPathForCurrentOS();
boolean hasNativeLib = hasResource(snappyNativeLibraryPath + "/" + snappyNativeLibraryName);

if (!hasNativeLib) {
String errorMessage = String.format("no native library is found for os.name=%s and os.arch=%s", OSInfo.getOSName(),
OSInfo.getArchName());
throw new RuntimeException(errorMessage);
}

File out = extractLibraryFile(
SnappyLoader.class.getResource(snappyNativeLibraryPath + "/" + snappyNativeLibraryName),
snappyNativeLibraryName);

System.load(out.getAbsolutePath());
}

private static boolean hasResource(String path) {
return SnappyLoader.class.getResource(path) != null;
}

private static File extractLibraryFile(URL library, String name) {
String tmp = System.getProperty("java.io.tmpdir");
File extractedLibFile = new File(tmp, name);

try (BufferedInputStream inputStream = new BufferedInputStream(library.openStream());
FileOutputStream fileOS = new FileOutputStream(extractedLibFile)) {
byte[] data = new byte[8192];
int byteContent;
while ((byteContent = inputStream.read(data, 0, 8192)) != -1) {
fileOS.write(data, 0, byteContent);
}
} catch (IOException e) {
throw new UncheckedIOException(
"Unable to extract native library " + name + " to " + extractedLibFile.getAbsolutePath(), e);
}

extractedLibFile.deleteOnExit();

return extractedLibFile;
}
}

0 comments on commit e0bcf74

Please sign in to comment.