Refactor building (#1332)

This PR is trying to improve the building experience, by:
 1. Providing `third-party/install-cmake.sh` to download and install a prebuilt CMake
 2. Providing `third-party/build-gcc.sh` to automatically download and build GCC 7.5.0
 3. Providing `third-party/install-gcc.sh` to download and install a prebuilt GCC 7.5.0 according to the current distro and libc
 4. Providing `third-party/build-third-party.sh` to build and optionally package our third party dependencies
 4. Providing `third-party/build-all-third-party.sh` to invoke `build-third-party.sh` using several preset versions of GCC
 5. Providing `third-party/install-third-party.sh` to download and install a prebuilt third party according to the current version of glibc, GCC(along with its ABI version)
 6. Invoke `install-third-party.sh` automatically during the configure stage(cmake)

Besides, all installing scripts support an `--prefix` option to allow one customizing the install location. For the third party installation, works are done to make *Bison* and *krb5* relocatable.

The fresh new building shceme has been tested under various environments:
 1. CentOS 6/7/8
 2. Debian 7/8/9
 3. Ubuntu 16.04/18.04/19.04
 4. GCC 7/8/9, and devtoolset-7 of CentOS 6/7

Since most of these testing are from inside a Docker image, so the Linux Kernel used is nearly the newest one(5.0). There might be other unfound issues.

To try out this PR, run
```sh
$ git clone --single-branch --depth=1 --branch refactor-building https://github.com/dutor/nebula.git nebula-refactor-building
$ cd nebula-refactor-building
$ mkdir build && cd build
$ ../third-party/install-cmake.sh             # Can be skipped if you have CMake 3.5.0+
$ source cmake-3.15.5/bin/enable-cmake.sh     # Only necessary if you invoked install-cmake.sh
$ sudo ../third-party/install-gcc.sh          # Can be skipped if you have GCC 7.1.0+
$ source /opt/vesoft/toolset/gcc/7.5.0/enable # Only necessary if you invoked install-gcc.sh
$ cmake ..
$ make
```

BTW. This PR also fixes #1449 
BTW. This PR reduces the build size a lot, as mentioned in #1401

CMakeLists.txt

@@ -18,10 +18,17 @@
 #   ENABLE_NATIVE                  -- Build native client
 #   ENABLE_TESTING                 -- Build unit test
 #
-cmake_minimum_required(VERSION 3.0.0)
+cmake_minimum_required(VERSION 3.5.0)
 
 project("Nebula Graph" C CXX)
 
+# Set the project home dir
+set(NEBULA_HOME ${CMAKE_CURRENT_SOURCE_DIR})
+# To include customized FindXXX.cmake modules
+set(CMAKE_MODULE_PATH "${NEBULA_HOME}/cmake" ${CMAKE_MODULE_PATH})
+include(LinkerConfig)
+set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libstdc++ -no-pie")
+
 set(CMAKE_SKIP_RPATH TRUE)
 
 option(ENABLE_JEMALLOC "Whether to link jemalloc to all executables" ON)
@@ -53,8 +60,7 @@ endif()
 
 set(CMAKE_CXX_STANDARD 14)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
-
-set(CMAKE_EXE_LINKER_FLAGS "-static-libstdc++ -no-pie")
+set(CMAKE_CXX_EXTENSIONS OFF)
 
 # To detect if ccache available
 find_program(ccache_program_found "ccache")
@@ -84,15 +90,31 @@ if(NOT ${CMAKE_BUILD_TYPE} STREQUAL "Debug")
     add_definitions(-D_FORTIFY_SOURCE=2)
 endif()
 
+if(CMAKE_BUILD_TYPE STREQUAL "Debug" OR CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo")
+    set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--compress-debug-sections=zlib")
+endif()
+
+if(CMAKE_BUILD_TYPE STREQUAL "Debug")
+    if (NOT ${NEBULA_USE_LINKER} STREQUAL "gold")
+        # gold linker is buggy for `--gc-sections', disabled for now
+        add_compile_options(-ffunction-sections)
+        add_compile_options(-fdata-sections)
+        set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections")
+    endif()
+endif()
+
 message(STATUS "CMAKE_BUILD_TYPE = ${CMAKE_BUILD_TYPE} "
                "(Options are: Debug, Release, RelWithDebInfo, MinSizeRel)")
 
-set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "_build")
-set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "_build")
-set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "_build")
+# By default, all dynamic and static libraries will be placed at ${CMAKE_BINARY_DIR}/lib,
+# while all executables at ${CMAKE_BINARY_DIR}/bin.
+# But for the sake of cleanliness, all executables ending with `_test' will be placed
+# at ${CMAKE_BINARY_DIR}/bin/test, while those ending with `_bm' at ${CMAKE_BINARY_DIR}/bin/bench.
+# Please see `nebula_add_executable' for this rule.
+set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib")
+set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib")
+set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin")
 
-# Set the project home dir
-set(NEBULA_HOME ${CMAKE_CURRENT_SOURCE_DIR})
 add_definitions(-DNEBULA_HOME=${NEBULA_HOME})
 
 # Let s2 use glog
@@ -121,12 +143,22 @@ if (GIT_INFO_SHA)
     add_definitions(-DGIT_INFO_SHA=${GIT_INFO_SHA})
 endif()
 
-# To include customized FindXXX.cmake modules
-set(CMAKE_MODULE_PATH "${NEBULA_HOME}/cmake" ${CMAKE_MODULE_PATH})
-
-# When NEBULA_THIRDPARTY_ROOT is null, set default value as /opt/nebula/third-party
+# The precedence to decide NEBULA_THIRDPARTY_ROOT is:
+#   1. The path defined with CMake argument, i.e -DNEBULA_THIRDPARTY_ROOT=path
+#   2. ${CMAKE_BINARY_DIR}/third-party/install, if exists
+#   3. The path specified with environment variable NEBULA_THIRDPARTY_ROOT=path
+#   4. /opt/vesoft/third-party, if exists
+#   5. At last, one copy will be downloaded and installed to ${CMAKE_BINARY_DIR}/third-party/install
 if("${NEBULA_THIRDPARTY_ROOT}" STREQUAL "")
-    SET(NEBULA_THIRDPARTY_ROOT "/opt/nebula/third-party")
+    if(EXISTS ${CMAKE_BINARY_DIR}/third-party/install)
+        SET(NEBULA_THIRDPARTY_ROOT ${CMAKE_BINARY_DIR}/third-party/install)
+    elseif(NOT $ENV{NEBULA_THIRDPARTY_ROOT} STREQUAL "")
+        SET(NEBULA_THIRDPARTY_ROOT $ENV{NEBULA_THIRDPARTY_ROOT})
+    elseif(EXISTS /opt/vesoft/third-party)
+        SET(NEBULA_THIRDPARTY_ROOT "/opt/vesoft/third-party")
+    else()
+        include(InstallThirdParty)
+    endif()
 endif()
 
 if ("${NEBULA_CLANG_USED_GCC_TOOLCHAIN}" STREQUAL "")
@@ -190,6 +222,7 @@ find_package(Krb5 REQUIRED gssapi)
 find_package(GPERF 2.8 REQUIRED)
 find_package(Libunwind REQUIRED)
 find_package(BISON 3.0.5 REQUIRED)
+include(MakeBisonRelocatable)
 find_package(FLEX REQUIRED)
 find_package(Readline REQUIRED)
 find_package(NCURSES REQUIRED)
@@ -274,6 +307,18 @@ macro(nebula_add_executable)
         ${nebula_exec_NAME}
         ${nebula_exec_LIBRARIES}
     )
+
+    if(${nebula_exec_NAME} MATCHES "_test$")
+        set_target_properties(
+            ${nebula_exec_NAME}
+            PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin/test
+        )
+    elseif(${nebula_exec_NAME} MATCHES "_bm$")
+        set_target_properties(
+            ${nebula_exec_NAME}
+            PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin/bench
+        )
+    endif()
 endmacro()
 
 macro(nebula_add_test)
@@ -364,7 +409,7 @@ set(THRIFT_LIBRARIES
 set(ROCKSDB_LIBRARIES ${Rocksdb_LIBRARY})
 
 # All compression libraries
-set(COMPRESSION_LIBRARIES bz2 snappy zstd z)
+set(COMPRESSION_LIBRARIES bz2 snappy zstd z lz4)
 if (LIBLZMA_FOUND)
     include_directories(SYSTEM ${LIBLZMA_INCLUDE_DIRS})
     list(APPEND COMPRESSION_LIBRARIES ${LIBLZMA_LIBRARIES})
@@ -404,14 +449,15 @@ macro(nebula_link_libraries target)
         boost_system
         event
         double-conversion
-        resolv
         s2
         ${OPENSSL_SSL_LIBRARY}
         ${OPENSSL_CRYPTO_LIBRARY}
         ${KRB5_LIBRARIES}
         ${COMPRESSION_LIBRARIES}
         ${JEMALLOC_LIB}
         ${LIBUNWIND_LIBRARIES}
+        keyutils
+        resolv
         dl
         ${GETTIME_LIB}
         -pthread

cmake/BuildThirdParty.cmake

@@ -0,0 +1,7 @@
+execute_process(
+    COMMAND
+        ${CMAKE_SOURCE_DIR}/third-party/build-third-party.sh
+    WORKING_DIRECTORY
+        ${CMAKE_BINARY_DIR}
+)
+set(NEBULA_THIRDPARTY_ROOT ${CMAKE_BINARY_DIR}/third-party/install)

cmake/FindDoubleConversion.cmake

@@ -13,7 +13,7 @@
 #  DoubleConversion_INCLUDE_DIR      The double-conversion includes directories.
 #  DoubleConversion_LIBRARY          The double-conversion library.
 
-find_path(DoubleConversion_INCLUDE_DIR NAMES double-conversion.h)
+find_path(DoubleConversion_INCLUDE_DIR NAMES double-conversion/double-conversion.h)
 find_library(DoubleConversion_LIBRARY NAMES libdouble-conversion.a)
 
 if(DoubleConversion_INCLUDE_DIR AND DoubleConversion_LIBRARY)

cmake/InstallThirdParty.cmake

@@ -0,0 +1,12 @@
+set(third_party_install_prefix ${CMAKE_BINARY_DIR}/third-party/install)
+message(STATUS "Downloading prebuilt third party automatically...")
+execute_process(
+    COMMAND
+        env CXX=${CMAKE_CXX_COMPILER} ${CMAKE_SOURCE_DIR}/third-party/install-third-party.sh --prefix=${third_party_install_prefix}
+    WORKING_DIRECTORY
+        ${CMAKE_BINARY_DIR}
+)
+if(EXISTS ${third_party_install_prefix})
+    set(NEBULA_THIRDPARTY_ROOT ${third_party_install_prefix})
+endif()
+unset(third_party_install_prefix)

cmake/LinkerConfig.cmake

@@ -0,0 +1,22 @@
+execute_process(
+    COMMAND
+        ld --version
+    COMMAND
+        head -1
+    COMMAND
+        cut -d " " -f2
+    OUTPUT_VARIABLE default_linker_type
+    OUTPUT_STRIP_TRAILING_WHITESPACE
+)
+
+if (${default_linker_type} STREQUAL "ld")
+    set(default_linker_type "bfd")
+endif()
+
+if (NOT DEFINED NEBULA_USE_LINKER)
+    set(NEBULA_USE_LINKER ${default_linker_type})
+endif()
+
+message(STATUS "NEBULA_USE_LINKER: ${NEBULA_USE_LINKER}")
+set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=${NEBULA_USE_LINKER}")
+set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=${NEBULA_USE_LINKER}")

cmake/MakeBisonRelocatable.cmake

@@ -0,0 +1,17 @@
+set(BISON_EXECUTE_ENV "")
+execute_process(
+    COMMAND ${BISON_EXECUTABLE} --print-datadir
+    OUTPUT_VARIABLE bison_encoded_data_dir
+    OUTPUT_STRIP_TRAILING_WHITESPACE
+)
+if(NOT EXISTS ${bison_encoded_data_dir})
+    get_filename_component(bison_prefix ${BISON_EXECUTABLE} DIRECTORY)
+    get_filename_component(bison_prefix ${bison_prefix} DIRECTORY)
+    if(EXISTS ${bison_prefix}/share/bison)
+        set(BISON_EXECUTE_ENV "BISON_PKGDATADIR=${bison_prefix}/share/bison")
+    endif()
+endif()
+if(NOT ${BISON_EXECUTE_ENV} STREQUAL "")
+    set(BISON_EXECUTABLE ${CMAKE_COMMAND} -E env ${BISON_EXECUTE_ENV} ${BISON_EXECUTABLE})
+endif()
+

src/graph/GroupByExecutor.cpp

@@ -390,7 +390,7 @@ Status GroupByExecutor::generateOutputSchema() {
 StatusOr<std::unique_ptr<InterimResult>> GroupByExecutor::setupInterimResult() {
     auto result = std::make_unique<InterimResult>(getResultColumnNames());
     if (rows_.empty() || resultSchema_ == nullptr) {
-        return result;
+        return std::move(result);
     }
     // Generate results
     std::unique_ptr<RowSetWriter> rsWriter = std::make_unique<RowSetWriter>(resultSchema_);
@@ -399,7 +399,7 @@ StatusOr<std::unique_ptr<InterimResult>> GroupByExecutor::setupInterimResult() {
     if (rsWriter != nullptr) {
         result->setInterim(std::move(rsWriter));
     }
-    return result;
+    return std::move(result);
 }
 
 

src/graph/LimitExecutor.cpp

@@ -82,7 +82,7 @@ void LimitExecutor::feedResult(std::unique_ptr<InterimResult> result) {
 StatusOr<std::unique_ptr<InterimResult>> LimitExecutor::setupInterimResult() {
     auto result = std::make_unique<InterimResult>(std::move(colNames_));
     if (rows_.empty()) {
-        return result;
+        return std::move(result);
     }
 
     auto rsWriter = std::make_unique<RowSetWriter>(inputs_->schema());
@@ -121,7 +121,7 @@ StatusOr<std::unique_ptr<InterimResult>> LimitExecutor::setupInterimResult() {
     if (rsWriter != nullptr) {
         result->setInterim(std::move(rsWriter));
     }
-    return result;
+    return std::move(result);
 }
 
 

src/graph/OrderByExecutor.cpp

@@ -163,7 +163,7 @@ Status OrderByExecutor::beforeExecute() {
 StatusOr<std::unique_ptr<InterimResult>> OrderByExecutor::setupInterimResult() {
     auto result = std::make_unique<InterimResult>(std::move(colNames_));
     if (rows_.empty()) {
-        return result;
+        return std::move(result);
     }
 
     auto schema = inputs_->schema();
@@ -201,7 +201,7 @@ StatusOr<std::unique_ptr<InterimResult>> OrderByExecutor::setupInterimResult() {
     }
 
     result->setInterim(std::move(rsWriter));
-    return result;
+    return std::move(result);
 }
 
 void OrderByExecutor::setupResponse(cpp2::ExecutionResponse &resp) {

src/parser/test/CMakeLists.txt

@@ -33,7 +33,7 @@ nebula_add_test(
 )
 
 nebula_add_executable(
-    NAME parser_benchmark
+    NAME parser_bm
     SOURCES ParserBenchmark.cpp
     OBJECTS ${PARSER_TEST_LIBS}
     LIBRARIES follybenchmark boost_regex ${THRIFT_LIBRARIES} wangle