From 6d2fbd18a5ff3e91257c1505df2b76294089783d Mon Sep 17 00:00:00 2001 From: Yuan Date: Fri, 16 Dec 2022 08:49:36 +0800 Subject: [PATCH] fix jni unload (#1180) fix the seg fault on spark-shell exit Signed-off-by: Yuan Zhou --- .../com/intel/oap/vectorized/JniUtils.java | 84 +++++++++++++++++++ .../spark/util/GazelleShutdownManager.scala | 25 ++++++ 2 files changed, 109 insertions(+) create mode 100644 native-sql-engine/core/src/main/scala/org/apache/spark/util/GazelleShutdownManager.scala diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/vectorized/JniUtils.java b/native-sql-engine/core/src/main/java/com/intel/oap/vectorized/JniUtils.java index 4579be179..0ca1e0dab 100644 --- a/native-sql-engine/core/src/main/java/com/intel/oap/vectorized/JniUtils.java +++ b/native-sql-engine/core/src/main/java/com/intel/oap/vectorized/JniUtils.java @@ -17,11 +17,19 @@ package com.intel.oap.vectorized; +import org.apache.spark.util.GazelleShutdownManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import scala.Function0; +import scala.runtime.BoxedUnit; + import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; +import java.lang.reflect.Field; +import java.lang.reflect.Method; import java.net.JarURLConnection; import java.net.URL; import java.net.URLConnection; @@ -30,9 +38,15 @@ import java.nio.file.Paths; import java.nio.file.StandardCopyOption; import java.util.ArrayList; +import java.util.Collections; import java.util.Enumeration; +import java.util.HashSet; +import java.util.Iterator; import java.util.List; +import java.util.LinkedHashSet; import java.util.Objects; +import java.util.Vector; +import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import java.util.jar.JarEntry; import java.util.jar.JarFile; @@ -53,6 +67,23 @@ public class JniUtils { private static List codegenJarsLoadedCache = new ArrayList<>(); private static volatile JniUtils INSTANCE; private static String tmp_dir; + private static final Logger LOG = + LoggerFactory.getLogger(JniUtils.class); + + public static Set LOADED_LIBRARY_PATHS = new HashSet<>(); + public static Set REQUIRE_UNLOAD_LIBRARY_PATHS = new LinkedHashSet<>(); + + static { + GazelleShutdownManager.registerUnloadLibShutdownHook(new Function0() { + @Override + public BoxedUnit apply() { + List loaded = new ArrayList<>(REQUIRE_UNLOAD_LIBRARY_PATHS); + Collections.reverse(loaded); // use reversed order to unload + loaded.forEach(JniUtils::unloadFromPath); + return BoxedUnit.UNIT; + } + }); + } public static JniUtils getInstance() throws IOException { String tmp_dir = System.getProperty("java.io.tmpdir"); @@ -118,6 +149,19 @@ public void setJars(List list_jars) throws IOException, IllegalAccessExc } } + private static synchronized void loadFromPath0(String libPath, boolean requireUnload) { + if (LOADED_LIBRARY_PATHS.contains(libPath)) { + LOG.debug("Library in path {} has already been loaded, skipping", libPath); + } else { + System.load(libPath); + LOADED_LIBRARY_PATHS.add(libPath); + LOG.info("Library {} has been loaded using path-loading method", libPath); + } + if (requireUnload) { + REQUIRE_UNLOAD_LIBRARY_PATHS.add(libPath); + } + } + static void loadLibraryFromJar(String tmp_dir) throws IOException, IllegalAccessException { synchronized (JniUtils.class) { if (tmp_dir == null) { @@ -127,15 +171,18 @@ static void loadLibraryFromJar(String tmp_dir) throws IOException, IllegalAccess Path arrowMiddleLink = createSoftLink(arrowlibraryFile, ARROW_PARENT_LIBRARY_NAME); Path arrowShortLink = createSoftLink(new File(arrowMiddleLink.toString()), ARROW_PARENT_LIBRARY_SHORT); System.load(arrowShortLink.toAbsolutePath().toString()); + loadFromPath0(arrowShortLink.toAbsolutePath().toString(), true); final File gandivalibraryFile = moveFileFromJarToTemp(tmp_dir, GANDIVA_LIBRARY_NAME); Path gandivaMiddleLink = createSoftLink(gandivalibraryFile, GANDIVA_PARENT_LIBRARY_NAME); Path gandivaShortLink = createSoftLink(new File(gandivaMiddleLink.toString()), GANDIVA_PARENT_LIBRARY_SHORT); System.load(gandivaShortLink.toAbsolutePath().toString()); + loadFromPath0(gandivaShortLink.toAbsolutePath().toString(), true); final String libraryToLoad = System.mapLibraryName(LIBRARY_NAME); final File libraryFile = moveFileFromJarToTemp(tmp_dir, libraryToLoad); System.load(libraryFile.getAbsolutePath()); + loadFromPath0(libraryFile.getAbsolutePath(), true); } } @@ -282,4 +329,41 @@ public static void copyResourcesToDirectory(URLConnection urlConnection, } } } + + public static synchronized void unloadFromPath(String libPath) { + if (!LOADED_LIBRARY_PATHS.remove(libPath)) { + throw new IllegalStateException("Library not exist: " + libPath); + } + + REQUIRE_UNLOAD_LIBRARY_PATHS.remove(libPath); + + try { + while (Files.isSymbolicLink(Paths.get(libPath))) { + libPath = Files.readSymbolicLink(Paths.get(libPath)).toString(); + } + + ClassLoader classLoader = JniUtils.class.getClassLoader(); + Field field = ClassLoader.class.getDeclaredField("nativeLibraries"); + field.setAccessible(true); + Vector libs = (Vector) field.get(classLoader); + Iterator it = libs.iterator(); + while (it.hasNext()) { + Object object = it.next(); + Field[] fs = object.getClass().getDeclaredFields(); + for (int k = 0; k < fs.length; k++) { + if (fs[k].getName().equals("name")) { + fs[k].setAccessible(true); + String verbosePath = fs[k].get(object).toString(); + if (verbosePath.endsWith(libPath)) { + Method finalize = object.getClass().getDeclaredMethod("finalize"); + finalize.setAccessible(true); + finalize.invoke(object); + } + } + } + } + } catch (Throwable th) { + LOG.error("Unload native library error: ", th); + } + } } diff --git a/native-sql-engine/core/src/main/scala/org/apache/spark/util/GazelleShutdownManager.scala b/native-sql-engine/core/src/main/scala/org/apache/spark/util/GazelleShutdownManager.scala new file mode 100644 index 000000000..6e07d8230 --- /dev/null +++ b/native-sql-engine/core/src/main/scala/org/apache/spark/util/GazelleShutdownManager.scala @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.util + +object GazelleShutdownManager { + + def registerUnloadLibShutdownHook(hook: () => Unit): AnyRef = { + ShutdownHookManager.addShutdownHook(hook) + } +} +