Skip to content
This repository has been archived by the owner on Sep 18, 2023. It is now read-only.

[NSE-254]Fix redundant arrow library issue. #452

Merged
merged 2 commits into from
Aug 6, 2021
+26 −36
Merged

[NSE-254]Fix redundant arrow library issue. #452

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
16b562f
[NSE-254]Fix redundant arrow library issue.
weiting-chen Aug 5, 2021
c6f7f73
remote the unneccessary marked code
weiting-chen Aug 5, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 18 additions & 2 deletions native-sql-engine/core/src/main/java/com/intel/oap/vectorized/JniUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,10 @@
/** Helper class for JNI related operations. */
public class JniUtils {
private static final String LIBRARY_NAME = "spark_columnar_jni";
private static final String ARROW_LIBRARY_NAME = "libarrow.so.400";
private static final String GANDIVA_LIBRARY_NAME = "libgandiva.so.400";
private static final String ARROW_LIBRARY_NAME = "libarrow.so.400.0.0";
private static final String ARROW_PARENT_LIBRARY_NAME = "libarrow.so.400";
private static final String GANDIVA_LIBRARY_NAME = "libgandiva.so.400.0.0";
private static final String GANDIVA_PARENT_LIBRARY_NAME = "libgandiva.so.400";
private static boolean isLoaded = false;
private static boolean isCodegenDependencyLoaded = false;
private static List<String> codegenJarsLoadedCache = new ArrayList<>();
Expand Down Expand Up @@ -113,9 +115,23 @@ static void loadLibraryFromJar(String tmp_dir) throws IOException, IllegalAccess
tmp_dir = System.getProperty("java.io.tmpdir");
}
final File arrowlibraryFile = moveFileFromJarToTemp(tmp_dir, ARROW_LIBRARY_NAME);
Path arrow_target = Paths.get(arrowlibraryFile.getPath());
Path arrow_link = Paths.get(tmp_dir, ARROW_PARENT_LIBRARY_NAME);
if (Files.exists(arrow_link)) {
Files.delete(arrow_link);
}
Path symLink = Files.createSymbolicLink(arrow_link, arrow_target);
System.load(arrowlibraryFile.getAbsolutePath());

final File gandivalibraryFile = moveFileFromJarToTemp(tmp_dir, GANDIVA_LIBRARY_NAME);
Path gandiva_target = Paths.get(gandivalibraryFile.getPath());
Path gandiva_link = Paths.get(tmp_dir, GANDIVA_PARENT_LIBRARY_NAME);
if (Files.exists(gandiva_link)) {
Files.delete(gandiva_link);
}
Files.createSymbolicLink(gandiva_link, gandiva_target);
System.load(gandivalibraryFile.getAbsolutePath());

final String libraryToLoad = System.mapLibraryName(LIBRARY_NAME);
final File libraryFile = moveFileFromJarToTemp(tmp_dir, libraryToLoad);
System.load(libraryFile.getAbsolutePath());
Expand Down
37 changes: 3 additions & 34 deletions native-sql-engine/cpp/src/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ set(ARROW_ROOT "/usr/local" CACHE PATH "Arrow Root dir")
set(ARROW_BFS_INSTALL_DIR "/usr/local" CACHE PATH "Arrow Build from Source dir")
set(ARROW_LIB_NAME arrow)
set(GANDIVA_LIB_NAME gandiva)
set(ARROW_SHARED_LIBRARY_SUFFIX ".so")
set(ARROW_SHARED_LIBRARY_SUFFIX ".so.400")

option(BUILD_ARROW "Build Arrow from Source" ON)
option(STATIC_ARROW "Build Arrow with Static Libraries" OFF)
Expand Down Expand Up @@ -213,46 +213,15 @@ macro(build_arrow STATIC_ARROW)

else()

# Copy Arrow Shared Library to releases directory for package jar
ExternalProject_Add_Step(arrow_ep copy_arrow_binary
ExternalProject_Add_Step(arrow_ep copy_arrow_binary_400_0_0
COMMAND cp -a ${ARROW_PREFIX}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}${ARROW_LIB_NAME}${ARROW_SHARED_LIBRARY_SUFFIX} ${root_directory}/releases/
COMMENT "Copy libarrow.so to releases/"
DEPENDEES mkdir download update patch configure build install java_install
WORKING_DIRECTORY "${ARROW_PREFIX}/"
)

ExternalProject_Add_Step(arrow_ep copy_arrow_binary_300
COMMAND cp -a ${ARROW_PREFIX}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}${ARROW_LIB_NAME}${ARROW_SHARED_LIBRARY_SUFFIX}.400 ${root_directory}/releases/
COMMENT "Copy libarrow.so.400 to releases/"
DEPENDEES mkdir download update patch configure build install java_install
WORKING_DIRECTORY "${ARROW_PREFIX}/"
)

ExternalProject_Add_Step(arrow_ep copy_arrow_binary_300_0_0
COMMAND cp -a ${ARROW_PREFIX}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}${ARROW_LIB_NAME}${ARROW_SHARED_LIBRARY_SUFFIX}.400.0.0 ${root_directory}/releases/
COMMENT "Copy libarrow.so.400.0.0 to releases/"
DEPENDEES mkdir download update patch configure build install java_install
WORKING_DIRECTORY "${ARROW_PREFIX}/"
)


# Copy Gandiva Shared Library to releases directory for package jar
ExternalProject_Add_Step(arrow_ep copy_gandiva_binary
ExternalProject_Add_Step(arrow_ep copy_gandiva_binary_400_0_0
COMMAND cp -a ${ARROW_PREFIX}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}${GANDIVA_LIB_NAME}${ARROW_SHARED_LIBRARY_SUFFIX} ${root_directory}/releases/
COMMENT "Copy libgandiva.so to releases/"
DEPENDEES mkdir download update patch configure build install java_install
WORKING_DIRECTORY "${ARROW_PREFIX}/"
)

ExternalProject_Add_Step(arrow_ep copy_gandiva_binary_300
COMMAND cp -a ${ARROW_PREFIX}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}${GANDIVA_LIB_NAME}${ARROW_SHARED_LIBRARY_SUFFIX}.400 ${root_directory}/releases/
COMMENT "Copy libgandiva.so.400 to releases/"
DEPENDEES mkdir download update patch configure build install java_install
WORKING_DIRECTORY "${ARROW_PREFIX}/"
)

ExternalProject_Add_Step(arrow_ep copy_gandiva_binary_300_0_0
COMMAND cp -a ${ARROW_PREFIX}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}${GANDIVA_LIB_NAME}${ARROW_SHARED_LIBRARY_SUFFIX}.400.0.0 ${root_directory}/releases/
COMMENT "Copy libgandiva.so.400.0.0 to releases/"
DEPENDEES mkdir download update patch configure build install java_install
WORKING_DIRECTORY "${ARROW_PREFIX}/"
Expand Down
5 changes: 5 additions & 0 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -266,6 +266,11 @@

<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>3.2.0</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-enforcer-plugin</artifactId>
Expand Down