build(deps): use cmake ExternalProject to manage dependencies (#438)

.travis.yml

@@ -15,9 +15,9 @@ before_install:
 
 before_script:
     - cd thirdparty
-    - wget https://github.com/XiaoMi/pegasus-common/releases/download/deps/pegasus-thirdparty-prebuild.zip
-    - unzip pegasus-thirdparty-prebuild.zip
-    - rm -f pegasus-thirdparty-prebuild.zip
+    - wget https://github.com/XiaoMi/pegasus-common/releases/download/deps/pegasus-thirdparty-prebuild-bionic.zip
+    - unzip pegasus-thirdparty-prebuild-bionic.zip
+    - rm -f pegasus-thirdparty-prebuild-bionic.zip
     - cd ..
     - ulimit -c unlimited -S
 

bin/dsn.cmake

@@ -296,7 +296,7 @@ function(dsn_setup_thirdparty_libs)
     set(DEFAULT_THIRDPARTY_LIBS ${THRIFT_LIB} fmt::fmt CACHE STRING "default thirdparty libs" FORCE)
 
     # rocksdb
-    file(GLOB ROCKSDB_DEPENDS_MODULE_PATH ${DSN_PROJECT_DIR}/thirdparty/src/*/cmake/modules)
+    file(GLOB ROCKSDB_DEPENDS_MODULE_PATH ${DSN_PROJECT_DIR}/thirdparty/build/Source/rocksdb/cmake/modules)
     if(NOT ROCKSDB_DEPENDS_MODULE_PATH)
         message(WARNING "Cannot find RocksDB depends cmake modules path, might not find snappy, zstd, lz4")
     endif()

run.sh

@@ -47,7 +47,6 @@ function usage_build()
     echo "                         e.g., \"gcc,g++\" or \"clang-3.9,clang++-3.9\""
     echo "                         default is \"gcc,g++\""
     echo "   -j|--jobs <num>       the number of jobs to run simultaneously, default 8"
-    echo "   -b|--boost_dir <dir>  specify customized boost directory, use system boost if not set"
     echo "   --enable_gcov         generate gcov code coverage report, default no"
     echo "   -v|--verbose          build in verbose mode, default no"
     echo "   --notest              build without building unit tests, default no"
@@ -75,7 +74,6 @@ function run_build()
     CLEAR=NO
     CLEAR_THIRDPARTY=NO
     JOB_NUM=8
-    BOOST_DIR=""
     ENABLE_GCOV=NO
     RUN_VERBOSE=NO
     NO_TEST=NO
@@ -116,10 +114,6 @@ function run_build()
                 JOB_NUM="$2"
                 shift
                 ;;
-            -b|--boost_dir)
-                BOOST_DIR="$2"
-                shift
-                ;;
             --enable_gcov)
                 ENABLE_GCOV=YES
                 BUILD_TYPE="debug"
@@ -175,25 +169,21 @@ function run_build()
     fi
 
     if [[ ${SKIP_THIRDPARTY} == "YES" ]]; then
-        echo "Skip building thirdparty..."
+        echo "Skip building third-parties..."
     else
-        # build thirdparty first
         cd thirdparty
         if [[ "$CLEAR_THIRDPARTY" == "YES" ]]; then
-            echo "Clear thirdparty..."
-            rm -rf src build output &>/dev/null
-            CLEAR=YES
+            echo "Clear third-parties..."
+            rm -rf build
+            rm -rf output
         fi
-        echo "Start building thirdparty..."
-        ./download-thirdparty.sh
+        echo "Start building third-parties..."
+        mkdir -p build
+        pushd build
+        cmake .. -DCMAKE_C_COMPILER=$C_COMPILER -DCMAKE_CXX_COMPILER=$CXX_COMPILER -DCMAKE_BUILD_TYPE=Release
+        make -j$JOB_NUM
         exit_if_fail $?
-        if [[ "x"$BOOST_DIR != "x" ]]; then
-            ./build-thirdparty.sh -b $BOOST_DIR
-            exit_if_fail $?
-        else
-            ./build-thirdparty.sh
-            exit_if_fail $?
-        fi
+        popd
         cd ..
     fi
 
@@ -212,7 +202,7 @@ function run_build()
     fi
     C_COMPILER="$C_COMPILER" CXX_COMPILER="$CXX_COMPILER" BUILD_TYPE="$BUILD_TYPE" \
         ONLY_BUILD="$ONLY_BUILD" CLEAR="$CLEAR" JOB_NUM="$JOB_NUM" \
-        BOOST_DIR="$BOOST_DIR" ENABLE_GCOV="$ENABLE_GCOV" SANITIZER="$SANITIZER" \
+        ENABLE_GCOV="$ENABLE_GCOV" SANITIZER="$SANITIZER" \
         RUN_VERBOSE="$RUN_VERBOSE" TEST_MODULE="$TEST_MODULE" NO_TEST="$NO_TEST" \
         DISABLE_GPERF="$DISABLE_GPERF" $scripts_dir/build.sh
 }

scripts/linux/build.sh

@@ -10,15 +10,13 @@
 #    ONLY_BUILD     YES|NO
 #    RUN_VERBOSE    YES|NO
 #    ENABLE_GCOV    YES|NO
-#    BOOST_DIR      <dir>|""
 #    TEST_MODULE    "<module1> <module2> ..."
 #
 # CMake options:
 #    -DCMAKE_C_COMPILER=gcc|clang
 #    -DCMAKE_CXX_COMPILER=g++|clang++
 #    [-DCMAKE_BUILD_TYPE=Debug]
 #    [-DENABLE_GCOV=TRUE]
-#    [-DBoost_NO_BOOST_CMAKE=ON -DBOOST_ROOT=$BOOST_DIR -DBoost_NO_SYSTEM_PATHS=ON]
 
 ROOT=`pwd`
 REPORT_DIR=$ROOT/test_reports
@@ -97,23 +95,7 @@ else
     echo "Build without sanitizer"
 fi
 
-# You can specify customized boost by defining BOOST_DIR.
-# Install boost like this:
-#   wget http://downloads.sourceforge.net/project/boost/boost/1.54.0/boost_1_54_0.zip?r=&ts=1442891144&use_mirror=jaist
-#   unzip -q boost_1_54_0.zip
-#   cd boost_1_54_0
-#   ./bootstrap.sh --with-libraries=system,filesystem --with-toolset=gcc
-#   ./b2 toolset=gcc cxxflags="-std=c++11 -fPIC" -j8 -d0
-#   ./b2 install --prefix=$DSN_ROOT -d0
-# And set BOOST_DIR as:
-#   export BOOST_DIR=/path/to/boost_1_54_0/output
-if [ -n "$BOOST_DIR" ]
-then
-    echo "Use customized boost: $BOOST_DIR"
-    CMAKE_OPTIONS="$CMAKE_OPTIONS -DBoost_NO_BOOST_CMAKE=ON -DBOOST_ROOT=$BOOST_DIR -DBoost_NO_SYSTEM_PATHS=ON"
-else
-    echo "Use system boost"
-fi
+CMAKE_OPTIONS="$CMAKE_OPTIONS -DBoost_NO_BOOST_CMAKE=ON -DBOOST_ROOT=${ROOT}/thirdparty/output -DBoost_NO_SYSTEM_PATHS=ON"
 
 echo "CMAKE_OPTIONS=$CMAKE_OPTIONS"
 echo "MAKE_OPTIONS=$MAKE_OPTIONS"

src/block_service/fds/CMakeLists.txt

@@ -15,6 +15,4 @@ set(MY_PROJ_LIBS "")
 set(MY_BINPLACES "")
 
 dsn_add_static_library()
-
-find_path(FDS_INCLUDE_DIR NAMES constants.h PATHS ${DSN_THIRDPARTY_ROOT}/include PATH_SUFFIXES fds) # TODO(huangwei5): move to dsn_setup_thirdparty_libs(), if not found, return fatal
-target_include_directories(dsn.block_service.fds PRIVATE "${FDS_INCLUDE_DIR}")
+target_include_directories(dsn.block_service.fds PRIVATE ${DSN_THIRDPARTY_ROOT}/include/fds)