From 16b562fcdfe3810145460e1f4c73ab7352ba49db Mon Sep 17 00:00:00 2001 From: weiting-chen Date: Thu, 5 Aug 2021 16:33:44 +0800 Subject: [PATCH 1/2] [NSE-254]Fix redundant arrow library issue. --- .../com/intel/oap/vectorized/JniUtils.java | 20 +++++- native-sql-engine/cpp/src/CMakeLists.txt | 62 +++++++++---------- pom.xml | 5 ++ 3 files changed, 54 insertions(+), 33 deletions(-) diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/vectorized/JniUtils.java b/native-sql-engine/core/src/main/java/com/intel/oap/vectorized/JniUtils.java index 65f3f887f..9f55054ce 100644 --- a/native-sql-engine/core/src/main/java/com/intel/oap/vectorized/JniUtils.java +++ b/native-sql-engine/core/src/main/java/com/intel/oap/vectorized/JniUtils.java @@ -39,8 +39,10 @@ /** Helper class for JNI related operations. */ public class JniUtils { private static final String LIBRARY_NAME = "spark_columnar_jni"; - private static final String ARROW_LIBRARY_NAME = "libarrow.so.400"; - private static final String GANDIVA_LIBRARY_NAME = "libgandiva.so.400"; + private static final String ARROW_LIBRARY_NAME = "libarrow.so.400.0.0"; + private static final String ARROW_PARENT_LIBRARY_NAME = "libarrow.so.400"; + private static final String GANDIVA_LIBRARY_NAME = "libgandiva.so.400.0.0"; + private static final String GANDIVA_PARENT_LIBRARY_NAME = "libgandiva.so.400"; private static boolean isLoaded = false; private static boolean isCodegenDependencyLoaded = false; private static List codegenJarsLoadedCache = new ArrayList<>(); @@ -113,9 +115,23 @@ static void loadLibraryFromJar(String tmp_dir) throws IOException, IllegalAccess tmp_dir = System.getProperty("java.io.tmpdir"); } final File arrowlibraryFile = moveFileFromJarToTemp(tmp_dir, ARROW_LIBRARY_NAME); + Path arrow_target = Paths.get(arrowlibraryFile.getPath()); + Path arrow_link = Paths.get(tmp_dir, ARROW_PARENT_LIBRARY_NAME); + if (Files.exists(arrow_link)) { + Files.delete(arrow_link); + } + Path symLink = Files.createSymbolicLink(arrow_link, arrow_target); System.load(arrowlibraryFile.getAbsolutePath()); + final File gandivalibraryFile = moveFileFromJarToTemp(tmp_dir, GANDIVA_LIBRARY_NAME); + Path gandiva_target = Paths.get(gandivalibraryFile.getPath()); + Path gandiva_link = Paths.get(tmp_dir, GANDIVA_PARENT_LIBRARY_NAME); + if (Files.exists(gandiva_link)) { + Files.delete(gandiva_link); + } + Files.createSymbolicLink(gandiva_link, gandiva_target); System.load(gandivalibraryFile.getAbsolutePath()); + final String libraryToLoad = System.mapLibraryName(LIBRARY_NAME); final File libraryFile = moveFileFromJarToTemp(tmp_dir, libraryToLoad); System.load(libraryFile.getAbsolutePath()); diff --git a/native-sql-engine/cpp/src/CMakeLists.txt b/native-sql-engine/cpp/src/CMakeLists.txt index ef6ebb1d5..998729caf 100644 --- a/native-sql-engine/cpp/src/CMakeLists.txt +++ b/native-sql-engine/cpp/src/CMakeLists.txt @@ -17,7 +17,7 @@ set(ARROW_ROOT "/usr/local" CACHE PATH "Arrow Root dir") set(ARROW_BFS_INSTALL_DIR "/usr/local" CACHE PATH "Arrow Build from Source dir") set(ARROW_LIB_NAME arrow) set(GANDIVA_LIB_NAME gandiva) -set(ARROW_SHARED_LIBRARY_SUFFIX ".so") +set(ARROW_SHARED_LIBRARY_SUFFIX ".so.400") option(BUILD_ARROW "Build Arrow from Source" ON) option(STATIC_ARROW "Build Arrow with Static Libraries" OFF) @@ -214,22 +214,22 @@ macro(build_arrow STATIC_ARROW) else() # Copy Arrow Shared Library to releases directory for package jar - ExternalProject_Add_Step(arrow_ep copy_arrow_binary + #ExternalProject_Add_Step(arrow_ep copy_arrow_binary + # COMMAND cp -a ${ARROW_PREFIX}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}${ARROW_LIB_NAME}${ARROW_SHARED_LIBRARY_SUFFIX} ${root_directory}/releases/ + # COMMENT "Copy libarrow.so to releases/" + # DEPENDEES mkdir download update patch configure build install java_install + # WORKING_DIRECTORY "${ARROW_PREFIX}/" + #) + + #ExternalProject_Add_Step(arrow_ep copy_arrow_binary_400 + # COMMAND cp -a ${ARROW_PREFIX}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}${ARROW_LIB_NAME}${ARROW_SHARED_LIBRARY_SUFFIX}.400 ${root_directory}/releases/ + # COMMENT "Copy libarrow.so.400 to releases/" + # DEPENDEES mkdir download update patch configure build install java_install + # WORKING_DIRECTORY "${ARROW_PREFIX}/" + #) + + ExternalProject_Add_Step(arrow_ep copy_arrow_binary_400_0_0 COMMAND cp -a ${ARROW_PREFIX}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}${ARROW_LIB_NAME}${ARROW_SHARED_LIBRARY_SUFFIX} ${root_directory}/releases/ - COMMENT "Copy libarrow.so to releases/" - DEPENDEES mkdir download update patch configure build install java_install - WORKING_DIRECTORY "${ARROW_PREFIX}/" - ) - - ExternalProject_Add_Step(arrow_ep copy_arrow_binary_300 - COMMAND cp -a ${ARROW_PREFIX}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}${ARROW_LIB_NAME}${ARROW_SHARED_LIBRARY_SUFFIX}.400 ${root_directory}/releases/ - COMMENT "Copy libarrow.so.400 to releases/" - DEPENDEES mkdir download update patch configure build install java_install - WORKING_DIRECTORY "${ARROW_PREFIX}/" - ) - - ExternalProject_Add_Step(arrow_ep copy_arrow_binary_300_0_0 - COMMAND cp -a ${ARROW_PREFIX}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}${ARROW_LIB_NAME}${ARROW_SHARED_LIBRARY_SUFFIX}.400.0.0 ${root_directory}/releases/ COMMENT "Copy libarrow.so.400.0.0 to releases/" DEPENDEES mkdir download update patch configure build install java_install WORKING_DIRECTORY "${ARROW_PREFIX}/" @@ -237,22 +237,22 @@ macro(build_arrow STATIC_ARROW) # Copy Gandiva Shared Library to releases directory for package jar - ExternalProject_Add_Step(arrow_ep copy_gandiva_binary + #ExternalProject_Add_Step(arrow_ep copy_gandiva_binary + # COMMAND cp -a ${ARROW_PREFIX}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}${GANDIVA_LIB_NAME}${ARROW_SHARED_LIBRARY_SUFFIX} ${root_directory}/releases/ + # COMMENT "Copy libgandiva.so to releases/" + # DEPENDEES mkdir download update patch configure build install java_install + # WORKING_DIRECTORY "${ARROW_PREFIX}/" + #) + + #ExternalProject_Add_Step(arrow_ep copy_gandiva_binary_400 + # COMMAND cp -a ${ARROW_PREFIX}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}${GANDIVA_LIB_NAME}${ARROW_SHARED_LIBRARY_SUFFIX}.400 ${root_directory}/releases/ + # COMMENT "Copy libgandiva.so.400 to releases/" + # DEPENDEES mkdir download update patch configure build install java_install + # WORKING_DIRECTORY "${ARROW_PREFIX}/" + #) + + ExternalProject_Add_Step(arrow_ep copy_gandiva_binary_400_0_0 COMMAND cp -a ${ARROW_PREFIX}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}${GANDIVA_LIB_NAME}${ARROW_SHARED_LIBRARY_SUFFIX} ${root_directory}/releases/ - COMMENT "Copy libgandiva.so to releases/" - DEPENDEES mkdir download update patch configure build install java_install - WORKING_DIRECTORY "${ARROW_PREFIX}/" - ) - - ExternalProject_Add_Step(arrow_ep copy_gandiva_binary_300 - COMMAND cp -a ${ARROW_PREFIX}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}${GANDIVA_LIB_NAME}${ARROW_SHARED_LIBRARY_SUFFIX}.400 ${root_directory}/releases/ - COMMENT "Copy libgandiva.so.400 to releases/" - DEPENDEES mkdir download update patch configure build install java_install - WORKING_DIRECTORY "${ARROW_PREFIX}/" - ) - - ExternalProject_Add_Step(arrow_ep copy_gandiva_binary_300_0_0 - COMMAND cp -a ${ARROW_PREFIX}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}${GANDIVA_LIB_NAME}${ARROW_SHARED_LIBRARY_SUFFIX}.400.0.0 ${root_directory}/releases/ COMMENT "Copy libgandiva.so.400.0.0 to releases/" DEPENDEES mkdir download update patch configure build install java_install WORKING_DIRECTORY "${ARROW_PREFIX}/" diff --git a/pom.xml b/pom.xml index a67263266..1fefadcdd 100644 --- a/pom.xml +++ b/pom.xml @@ -266,6 +266,11 @@ + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + org.apache.maven.plugins maven-enforcer-plugin From c6f7f735496e5ae646d6eeda99c109b17a6c8dd3 Mon Sep 17 00:00:00 2001 From: weiting-chen Date: Thu, 5 Aug 2021 21:44:28 +0800 Subject: [PATCH 2/2] remote the unneccessary marked code --- native-sql-engine/cpp/src/CMakeLists.txt | 31 ------------------------ 1 file changed, 31 deletions(-) diff --git a/native-sql-engine/cpp/src/CMakeLists.txt b/native-sql-engine/cpp/src/CMakeLists.txt index 998729caf..29c170cf0 100644 --- a/native-sql-engine/cpp/src/CMakeLists.txt +++ b/native-sql-engine/cpp/src/CMakeLists.txt @@ -213,21 +213,6 @@ macro(build_arrow STATIC_ARROW) else() - # Copy Arrow Shared Library to releases directory for package jar - #ExternalProject_Add_Step(arrow_ep copy_arrow_binary - # COMMAND cp -a ${ARROW_PREFIX}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}${ARROW_LIB_NAME}${ARROW_SHARED_LIBRARY_SUFFIX} ${root_directory}/releases/ - # COMMENT "Copy libarrow.so to releases/" - # DEPENDEES mkdir download update patch configure build install java_install - # WORKING_DIRECTORY "${ARROW_PREFIX}/" - #) - - #ExternalProject_Add_Step(arrow_ep copy_arrow_binary_400 - # COMMAND cp -a ${ARROW_PREFIX}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}${ARROW_LIB_NAME}${ARROW_SHARED_LIBRARY_SUFFIX}.400 ${root_directory}/releases/ - # COMMENT "Copy libarrow.so.400 to releases/" - # DEPENDEES mkdir download update patch configure build install java_install - # WORKING_DIRECTORY "${ARROW_PREFIX}/" - #) - ExternalProject_Add_Step(arrow_ep copy_arrow_binary_400_0_0 COMMAND cp -a ${ARROW_PREFIX}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}${ARROW_LIB_NAME}${ARROW_SHARED_LIBRARY_SUFFIX} ${root_directory}/releases/ COMMENT "Copy libarrow.so.400.0.0 to releases/" @@ -235,22 +220,6 @@ macro(build_arrow STATIC_ARROW) WORKING_DIRECTORY "${ARROW_PREFIX}/" ) - - # Copy Gandiva Shared Library to releases directory for package jar - #ExternalProject_Add_Step(arrow_ep copy_gandiva_binary - # COMMAND cp -a ${ARROW_PREFIX}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}${GANDIVA_LIB_NAME}${ARROW_SHARED_LIBRARY_SUFFIX} ${root_directory}/releases/ - # COMMENT "Copy libgandiva.so to releases/" - # DEPENDEES mkdir download update patch configure build install java_install - # WORKING_DIRECTORY "${ARROW_PREFIX}/" - #) - - #ExternalProject_Add_Step(arrow_ep copy_gandiva_binary_400 - # COMMAND cp -a ${ARROW_PREFIX}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}${GANDIVA_LIB_NAME}${ARROW_SHARED_LIBRARY_SUFFIX}.400 ${root_directory}/releases/ - # COMMENT "Copy libgandiva.so.400 to releases/" - # DEPENDEES mkdir download update patch configure build install java_install - # WORKING_DIRECTORY "${ARROW_PREFIX}/" - #) - ExternalProject_Add_Step(arrow_ep copy_gandiva_binary_400_0_0 COMMAND cp -a ${ARROW_PREFIX}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}${GANDIVA_LIB_NAME}${ARROW_SHARED_LIBRARY_SUFFIX} ${root_directory}/releases/ COMMENT "Copy libgandiva.so.400.0.0 to releases/"