[NSE-176]Arrow install order issue

.github/workflows/unittests.yml

@@ -96,8 +96,9 @@ jobs:
           mvn clean install -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -P arrow-jni -am -Darrow.cpp.build.dir=/tmp/arrow/cpp/build/release/ -DskipTests -Dcheckstyle.skip
       - name: Run unit tests
         run: |
+          mvn clean install -N
           cd arrow-data-source
-          mvn clean install -DskipTests
+          mvn clean install -DskipTests -Dbuild_arrow=OFF
           cd ..
           mvn clean package -am -pl native-sql-engine/core -DskipTests -Dbuild_arrow=OFF
           cd native-sql-engine/core/

arrow-data-source/common/pom.xml

@@ -7,13 +7,47 @@
         <groupId>com.intel.oap</groupId>
         <version>1.1.0</version>
     </parent>
+
     <modelVersion>4.0.0</modelVersion>
-
     <artifactId>spark-arrow-datasource-common</artifactId>
     <build>
         <sourceDirectory>${project.basedir}/src/main/scala</sourceDirectory>
         <testSourceDirectory>${project.basedir}/src/test/scala</testSourceDirectory>
         <plugins>
+            <plugin>
+                <artifactId>exec-maven-plugin</artifactId>
+                <groupId>org.codehaus.mojo</groupId>
+                <version>1.6.0</version>
+                <executions>
+                    <execution>
+                    <id>Build arrow</id>
+                    <phase>generate-resources</phase>
+                    <goals>
+                        <goal>exec</goal>
+                    </goals>
+                    <configuration>
+                        <executable>bash</executable>
+                        <arguments>
+                            <argument>${script.dir}/build_arrow.sh</argument>
+                            <argument>--tests=${datasource.cpp_tests}</argument>
+                            <argument>--build_arrow=${datasource.build_arrow}</argument>
+                            <argument>--static_arrow=${datasource.static_arrow}</argument>
+                            <argument>--arrow_root=${datasource.arrow_root}</argument>
+                        </arguments>
+                    </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <artifactId>maven-clean-plugin</artifactId>
+                <configuration>
+                    <filesets>
+                        <fileset>
+                            <directory>${script.dir}/build</directory>
+                        </fileset>
+                    </filesets>
+                </configuration>
+            </plugin>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-source-plugin</artifactId>

arrow-data-source/common/script/build_arrow.sh

@@ -0,0 +1,105 @@
+#!/bin/bash
+
+set -eu
+
+NPROC=$(nproc)
+
+TESTS=OFF
+BUILD_ARROW=OFF
+STATIC_ARROW=OFF
+ARROW_ROOT=/usr/local
+
+for arg in "$@"
+do
+    case $arg in
+        -t=*|--tests=*)
+        TESTS=("${arg#*=}")
+        shift # Remove argument name from processing
+        ;;
+        -a=*|--build_arrow=*)
+        BUILD_ARROW=("${arg#*=}")
+        shift # Remove argument name from processing
+        ;;
+        -s=*|--static_arrow=*)
+        STATIC_ARROW=("${arg#*=}")
+        shift # Remove argument name from processing
+        ;;
+        -ar=*|--arrow_root=*)
+        ARROW_ROOT=("${arg#*=}")
+        shift # Remove argument name from processing
+        ;;
+        *)
+        OTHER_ARGUMENTS+=("$1")
+        shift # Remove generic argument from processing
+        ;;
+    esac
+done
+
+echo "CMAKE Arguments:"
+echo "TESTS=${TESTS}"
+echo "BUILD_ARROW=${BUILD_ARROW}"
+echo "STATIC_ARROW=${STATIC_ARROW}"
+echo "ARROW_ROOT=${ARROW_ROOT}"
+
+CURRENT_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd)
+echo $CURRENT_DIR
+
+cd ${CURRENT_DIR}
+if [ -d build ]; then
+    rm -r build
+fi
+
+if [ $BUILD_ARROW == "ON" ]; then
+echo "Building Arrow from Source ..."
+mkdir build
+cd build
+ARROW_PREFIX="${CURRENT_DIR}/build" # Use build directory as ARROW_PREFIX
+ARROW_SOURCE_DIR="${ARROW_PREFIX}/arrow_ep"
+ARROW_INSTALL_DIR="${ARROW_PREFIX}/arrow_install"
+
+echo "ARROW_PREFIX=${ARROW_PREFIX}"
+echo "ARROW_SOURCE_DIR=${ARROW_SOURCE_DIR}"
+echo "ARROW_INSTALL_DIR=${ARROW_INSTALL_DIR}"
+mkdir -p $ARROW_SOURCE_DIR
+mkdir -p $ARROW_INSTALL_DIR
+git clone https://github.com/oap-project/arrow.git  --branch arrow-3.0.0-oap $ARROW_SOURCE_DIR
+pushd $ARROW_SOURCE_DIR
+
+cmake ./cpp \
+	-DARROW_BUILD_STATIC=OFF -DARROW_BUILD_SHARED=ON -DARROW_COMPUTE=ON \
+        -DARROW_GANDIVA_JAVA=ON \
+        -DARROW_GANDIVA=ON \
+        -DARROW_PARQUET=ON \
+        -DARROW_HDFS=ON \
+        -DARROW_BOOST_USE_SHARED=OFF \
+        -DARROW_JNI=ON \
+        -DARROW_DATASET=ON \
+        -DARROW_WITH_PROTOBUF=ON \
+        -DARROW_WITH_SNAPPY=ON \
+        -DARROW_WITH_LZ4=ON \
+        -DARROW_WITH_ZSTD=OFF \
+        -DARROW_WITH_BROTLI=OFF \
+        -DARROW_WITH_ZLIB=OFF \
+        -DARROW_WITH_FASTPFOR=ON \
+        -DARROW_FILESYSTEM=ON \
+        -DARROW_JSON=ON \
+        -DARROW_CSV=ON \
+        -DARROW_FLIGHT=OFF \
+        -DARROW_JEMALLOC=ON \
+        -DARROW_SIMD_LEVEL=AVX2 \
+        -DARROW_RUNTIME_SIMD_LEVEL=MAX \
+        -DARROW_DEPENDENCY_SOURCE=BUNDLED \
+        -DCMAKE_INSTALL_PREFIX=${ARROW_INSTALL_DIR} \
+        -DCMAKE_INSTALL_LIBDIR=lib
+
+make -j$NPROC
+make install
+
+cd java        
+mvn clean install -P arrow-jni -am -Darrow.cpp.build.dir=${ARROW_INSTALL_DIR}/lib -DskipTests -Dcheckstyle.skip
+popd
+echo "Finish to build Arrow from Source !!!"
+else
+echo "Use ARROW_ROOT as Arrow Library Path"
+echo "ARROW_ROOT=${ARROW_ROOT}"
+fi

arrow-data-source/pom.xml

@@ -1,7 +1,11 @@
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
-  <modelVersion>4.0.0</modelVersion>
-
+  <parent>
+    <groupId>com.intel.oap</groupId>
+    <artifactId>native-sql-engine-parent</artifactId>
+    <version>1.1.0</version>
+  </parent>  
 
+  <modelVersion>4.0.0</modelVersion>
   <groupId>com.intel.oap</groupId>
   <artifactId>spark-arrow-datasource</artifactId>
   <name>OAP Project Spark Arrow Datasource</name>
@@ -20,6 +24,11 @@
     <arrow.version>3.0.0</arrow.version>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+    <script.dir>${arrow.script.dir}</script.dir>
+    <datasource.cpp_tests>${cpp_tests}</datasource.cpp_tests>
+    <datasource.build_arrow>${build_arrow}</datasource.build_arrow>
+    <datasource.static_arrow>${static_arrow}</datasource.static_arrow>
+    <datasource.arrow_root>${arrow_root}</datasource.arrow_root>
   </properties>
 
   <repositories>

native-sql-engine/core/pom.xml

@@ -14,7 +14,12 @@
   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
   xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>
-
+  <parent>
+    <groupId>com.intel.oap</groupId>
+    <artifactId>native-sql-engine-parent</artifactId>
+    <version>1.1.0</version>
+    <relativePath>../../pom.xml</relativePath>
+  </parent>
 
   <groupId>com.intel.oap</groupId>
   <artifactId>spark-columnar-core</artifactId>
@@ -36,11 +41,12 @@
     <hive.parquet.group>com.twitter</hive.parquet.group>
     <parquet.deps.scope>provided</parquet.deps.scope>
     <jars.target.dir>${project.build.directory}/scala-${scala.binary.version}/jars</jars.target.dir>
-    <cpp_tests>OFF</cpp_tests>
-    <build_arrow>ON</build_arrow>
-    <static_arrow>OFF</static_arrow>
-    <build_protobuf>ON</build_protobuf>
-    <arrow_root>/usr/local</arrow_root>
+    <nativesql.cpp_tests>${cpp_tests}</nativesql.cpp_tests>
+    <nativesql.build_arrow>OFF</nativesql.build_arrow>
+    <nativesql.static_arrow>${static_arrow}</nativesql.static_arrow>
+    <nativesql.arrow.bfs.install.dir>${project.basedir}/../../arrow-data-source/common/script/build/arrow_install</nativesql.arrow.bfs.install.dir>
+    <nativesql.arrow_root>${arrow_root}</nativesql.arrow_root>
+    <nativesql.build_protobuf>${build_protobuf}</nativesql.build_protobuf>
   </properties>
   <dependencies>
     <!-- Prevent our dummy JAR from being included in Spark distributions or uploaded to YARN -->
@@ -304,11 +310,12 @@
                       <executable>bash</executable>
                       <arguments>
                           <argument>${cpp.dir}/compile.sh</argument>
-                          <argument>${cpp_tests}</argument>
-                          <argument>${build_arrow}</argument>
-                          <argument>${static_arrow}</argument>
-                          <argument>${build_protobuf}</argument>
-                          <argument>${arrow_root}</argument>
+                          <argument>${nativesql.cpp_tests}</argument>
+                          <argument>${nativesql.build_arrow}</argument>
+                          <argument>${nativesql.static_arrow}</argument>
+                          <argument>${nativesql.build_protobuf}</argument>
+                          <argument>${nativesql.arrow_root}</argument>
+                          <argument>${nativesql.arrow.bfs.install.dir}</argument>
                       </arguments>
                   </configuration>
               </execution>

native-sql-engine/cpp/compile.sh

@@ -7,13 +7,15 @@ BUILD_ARROW=${2:-ON}
 STATIC_ARROW=${3:-OFF}
 BUILD_PROTOBUF=${4:-ON}
 ARROW_ROOT=${5:-/usr/local}
+ARROW_BFS_INSTALL_DIR=${6}
 
 echo "CMAKE Arguments:"
 echo "TESTS=${TESTS}"
 echo "BUILD_ARROW=${BUILD_ARROW}"
 echo "STATIC_ARROW=${STATIC_ARROW}"
 echo "BUILD_PROTOBUF=${BUILD_PROTOBUF}"
 echo "ARROW_ROOT=${ARROW_ROOT}"
+echo "ARROW_BUILD_FROM_SOURCE_INSTALL_DIR=${ARROW_BFS_INSTALL_DIR}"
 
 CURRENT_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd)
 echo $CURRENT_DIR
@@ -24,7 +26,7 @@ if [ -d build ]; then
 fi
 mkdir build
 cd build
-cmake .. -DTESTS=${TESTS} -DBUILD_ARROW=${BUILD_ARROW} -DSTATIC_ARROW=${STATIC_ARROW} -DBUILD_PROTOBUF=${BUILD_PROTOBUF} -DARROW_ROOT=${ARROW_ROOT}
+cmake .. -DTESTS=${TESTS} -DBUILD_ARROW=${BUILD_ARROW} -DSTATIC_ARROW=${STATIC_ARROW} -DBUILD_PROTOBUF=${BUILD_PROTOBUF} -DARROW_ROOT=${ARROW_ROOT} -DARROW_BFS_INSTALL_DIR=${ARROW_BFS_INSTALL_DIR}
 make
 
 set +eu