From 3cb2757f0bd9c303b19fe33e769db4a084c854bc Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Wed, 20 Nov 2019 16:30:54 +0800 Subject: [PATCH 01/80] Refactor building Fixed for CentOS 6 Support to custom download_dir Added s2geometry Detect version of cmake Added build-cmake.sh Download third-party from oss Fixed issue due to wget compatibility Clean build-third-party.sh Fixed building scripts Fix building Fix build-cmake.sh Fix checksum --- CMakeLists.txt | 12 +- cmake/FindDoubleConversion.cmake | 2 +- src/graph/GroupByExecutor.cpp | 2 +- third-party/CMakeLists.txt | 215 +++++++++++ third-party/build-cmake.sh | 57 +++ third-party/build-third-party.sh | 104 ++++++ third-party/cmake/FindAutoconf.cmake | 14 + third-party/cmake/FindAutoconfArchive.cmake | 3 + third-party/cmake/FindAutomake.cmake | 14 + third-party/cmake/FindLibtool.cmake | 14 + third-party/externals/autoconf-archive.cmake | 40 ++ third-party/externals/autoconf.cmake | 35 ++ third-party/externals/automake.cmake | 35 ++ third-party/externals/bison.cmake | 37 ++ third-party/externals/boost.cmake | 54 +++ third-party/externals/bzip2.cmake | 34 ++ third-party/externals/capstone.cmake | 56 +++ third-party/externals/double-conversion.cmake | 36 ++ third-party/externals/fatal.cmake | 36 ++ third-party/externals/fbthrift.cmake | 42 +++ third-party/externals/flex.cmake | 37 ++ third-party/externals/folly.cmake | 42 +++ third-party/externals/gettext.cmake | 37 ++ third-party/externals/gflags.cmake | 35 ++ third-party/externals/glog.cmake | 46 +++ third-party/externals/googletest.cmake | 34 ++ third-party/externals/gperf.cmake | 38 ++ third-party/externals/jemalloc.cmake | 37 ++ third-party/externals/keyutils.cmake | 38 ++ third-party/externals/krb5.cmake | 57 +++ third-party/externals/libaio.cmake | 40 ++ third-party/externals/libevent.cmake | 40 ++ third-party/externals/libtool.cmake | 35 ++ third-party/externals/libunwind.cmake | 38 ++ third-party/externals/lzma.cmake | 38 ++ third-party/externals/mstch.cmake | 36 ++ third-party/externals/openssl.cmake | 37 ++ third-party/externals/pkg.m4 | 343 ++++++++++++++++++ third-party/externals/pkgconf.cmake | 20 + third-party/externals/proxygen.cmake | 59 +++ third-party/externals/rocksdb.cmake | 50 +++ third-party/externals/s2geometry.cmake | 42 +++ third-party/externals/snappy.cmake | 39 ++ third-party/externals/wangle.cmake | 51 +++ third-party/externals/zlib.cmake | 41 +++ third-party/externals/zstd.cmake | 39 ++ third-party/patches/fbthrift-2018-08-20.patch | 192 ++++++++++ third-party/patches/proxygen-2018-08-20.patch | 13 + third-party/patches/rocksdb-5.15.10.patch | 5 + 49 files changed, 2396 insertions(+), 5 deletions(-) create mode 100644 third-party/CMakeLists.txt create mode 100755 third-party/build-cmake.sh create mode 100755 third-party/build-third-party.sh create mode 100644 third-party/cmake/FindAutoconf.cmake create mode 100644 third-party/cmake/FindAutoconfArchive.cmake create mode 100644 third-party/cmake/FindAutomake.cmake create mode 100644 third-party/cmake/FindLibtool.cmake create mode 100644 third-party/externals/autoconf-archive.cmake create mode 100644 third-party/externals/autoconf.cmake create mode 100644 third-party/externals/automake.cmake create mode 100644 third-party/externals/bison.cmake create mode 100644 third-party/externals/boost.cmake create mode 100644 third-party/externals/bzip2.cmake create mode 100644 third-party/externals/capstone.cmake create mode 100644 third-party/externals/double-conversion.cmake create mode 100644 third-party/externals/fatal.cmake create mode 100644 third-party/externals/fbthrift.cmake create mode 100644 third-party/externals/flex.cmake create mode 100644 third-party/externals/folly.cmake create mode 100644 third-party/externals/gettext.cmake create mode 100644 third-party/externals/gflags.cmake create mode 100644 third-party/externals/glog.cmake create mode 100644 third-party/externals/googletest.cmake create mode 100644 third-party/externals/gperf.cmake create mode 100644 third-party/externals/jemalloc.cmake create mode 100644 third-party/externals/keyutils.cmake create mode 100644 third-party/externals/krb5.cmake create mode 100644 third-party/externals/libaio.cmake create mode 100644 third-party/externals/libevent.cmake create mode 100644 third-party/externals/libtool.cmake create mode 100644 third-party/externals/libunwind.cmake create mode 100644 third-party/externals/lzma.cmake create mode 100644 third-party/externals/mstch.cmake create mode 100644 third-party/externals/openssl.cmake create mode 100644 third-party/externals/pkg.m4 create mode 100644 third-party/externals/pkgconf.cmake create mode 100644 third-party/externals/proxygen.cmake create mode 100644 third-party/externals/rocksdb.cmake create mode 100644 third-party/externals/s2geometry.cmake create mode 100644 third-party/externals/snappy.cmake create mode 100644 third-party/externals/wangle.cmake create mode 100644 third-party/externals/zlib.cmake create mode 100644 third-party/externals/zstd.cmake create mode 100644 third-party/patches/fbthrift-2018-08-20.patch create mode 100644 third-party/patches/proxygen-2018-08-20.patch create mode 100644 third-party/patches/rocksdb-5.15.10.patch diff --git a/CMakeLists.txt b/CMakeLists.txt index 127442c4227..23b6ff6bbe3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,7 +18,7 @@ # ENABLE_NATIVE -- Build native client # ENABLE_TESTING -- Build unit test # -cmake_minimum_required(VERSION 3.0.0) +cmake_minimum_required(VERSION 3.8.0) project("Nebula Graph" C CXX) @@ -52,6 +52,7 @@ endif() set(CMAKE_CXX_STANDARD 14) set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) set(CMAKE_EXE_LINKER_FLAGS "-static-libstdc++ -no-pie") @@ -125,7 +126,11 @@ set(CMAKE_MODULE_PATH "${NEBULA_HOME}/cmake" ${CMAKE_MODULE_PATH}) # When NEBULA_THIRDPARTY_ROOT is null, set default value as /opt/nebula/third-party if("${NEBULA_THIRDPARTY_ROOT}" STREQUAL "") - SET(NEBULA_THIRDPARTY_ROOT "/opt/nebula/third-party") + if (EXISTS ${CMAKE_BINARY_DIR}/third-party/install) + SET(NEBULA_THIRDPARTY_ROOT ${CMAKE_BINARY_DIR}/third-party/install) + else() + SET(NEBULA_THIRDPARTY_ROOT "/opt/nebula/third-party") + endif() endif() # third-party @@ -380,7 +385,6 @@ macro(nebula_link_libraries target) boost_system event double-conversion - resolv s2 ${OPENSSL_SSL_LIBRARY} ${OPENSSL_CRYPTO_LIBRARY} @@ -388,6 +392,8 @@ macro(nebula_link_libraries target) ${COMPRESSION_LIBRARIES} ${JEMALLOC_LIB} ${LIBUNWIND_LIBRARIES} + keyutils + resolv dl ${GETTIME_LIB} -pthread diff --git a/cmake/FindDoubleConversion.cmake b/cmake/FindDoubleConversion.cmake index 3580d889ad0..cdfc87fc302 100644 --- a/cmake/FindDoubleConversion.cmake +++ b/cmake/FindDoubleConversion.cmake @@ -13,7 +13,7 @@ # DoubleConversion_INCLUDE_DIR The double-conversion includes directories. # DoubleConversion_LIBRARY The double-conversion library. -find_path(DoubleConversion_INCLUDE_DIR NAMES double-conversion.h) +find_path(DoubleConversion_INCLUDE_DIR NAMES double-conversion/double-conversion.h) find_library(DoubleConversion_LIBRARY NAMES libdouble-conversion.a) if(DoubleConversion_INCLUDE_DIR AND DoubleConversion_LIBRARY) diff --git a/src/graph/GroupByExecutor.cpp b/src/graph/GroupByExecutor.cpp index 60b13da514f..8676bc0db7a 100644 --- a/src/graph/GroupByExecutor.cpp +++ b/src/graph/GroupByExecutor.cpp @@ -403,7 +403,7 @@ StatusOr> GroupByExecutor::setupInterimResult() { if (rsWriter != nullptr) { result->setInterim(std::move(rsWriter)); } - return result; + return std::move(result); } diff --git a/third-party/CMakeLists.txt b/third-party/CMakeLists.txt new file mode 100644 index 00000000000..eed6d621436 --- /dev/null +++ b/third-party/CMakeLists.txt @@ -0,0 +1,215 @@ +cmake_minimum_required(VERSION 3.8.0) + +project("Nebula Third Party" C CXX) + +set(CXX_STANDARD 14) +set(CXX_STANDARD_REQUIRED ON) + +# Required CMake modules +include(ExternalProject) + +# Get number of physical CPU cores and megabytes of available memory +cmake_host_system_information(RESULT num_cores QUERY NUMBER_OF_PHYSICAL_CORES) +cmake_host_system_information(RESULT available_memory_mb QUERY AVAILABLE_PHYSICAL_MEMORY) + +execute_process( + COMMAND ldd --version + COMMAND head -1 + COMMAND cut -d ")" -f 2 + COMMAND cut -d " " -f 2 + OUTPUT_VARIABLE GLIBC_VERSION + OUTPUT_STRIP_TRAILING_WHITESPACE +) + +# Guess the number of building jobs based on the available memories +set(jobs_by_cpu ${num_cores}) +math(EXPR jobs_by_mem "${available_memory_mb} / 1024 / 2") +if (jobs_by_mem EQUAL 0) + set(jobs_by_mem 1) +endif() + +# Set the number of building jobs to min(jobs_by_mem, jobs_by_cpu), +# iff BUILDING_JOBS_NUM has not been set or set to 0. +if (NOT BUILDING_JOBS_NUM OR BUILDING_JOBS_NUM EQUAL 0) + set(BUILDING_JOBS_NUM ${jobs_by_cpu}) + if (BUILDING_JOBS_NUM GREATER jobs_by_mem) + set(BUILDING_JOBS_NUM ${jobs_by_mem}) + endif() +endif() + +message(STATUS "Number of online physcial CPU cores: ${num_cores}") +message(STATUS "Available physical memory: ${available_memory_mb} MB") +message(STATUS "Building third party with ${BUILDING_JOBS_NUM} jobs") +message(STATUS "Glibc version: ${GLIBC_VERSION}") + +if (NOT DOWNLOAD_DIR) + set(DOWNLOAD_DIR ${CMAKE_CURRENT_BINARY_DIR}/downloads) +endif() +set(BUILD_INFO_DIR ${CMAKE_CURRENT_BINARY_DIR}/build-info) +set(BUILDING_PATH "${CMAKE_INSTALL_PREFIX}/bin:${CMAKE_INSTALL_PREFIX}/sbin:$ENV{PATH}") +set(ACLOCAL_PATH "${CMAKE_INSTALL_PREFIX}/share/aclocal:${CMAKE_INSTALL_PREFIX}/share/aclocal-1.15") + +set(extra_link_libs "-static-libstdc++ -static-libgcc") + +if(GLIBC_VERSION VERSION_LESS 2.17) + set(extra_link_libs "${extra_link_libs} -lrt") +endif() + +set(common_cmake_args + -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX} + -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} + -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} + "-DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} -fPIC" + "-DCMAKE_C_FLAGS=${CMAKE_C_FLAGS} -fPIC" + "-DCMAKE_EXE_LINKER_FLAGS=${extra_link_libs}" + "-DCMAKE_SHARED_LINKER_FLAGS=${extra_link_libs}" + -DCMAKE_INCLUDE_PATH=${CMAKE_INSTALL_PREFIX}/include + -DCMAKE_LIBRARY_PATH=${CMAKE_INSTALL_PREFIX}/lib +) + +set(common_configure_args + --prefix=${CMAKE_INSTALL_PREFIX} + --quiet +) + +set(common_configure_envs + "env" + "CC=${CMAKE_C_COMPILER}" + "CXX=${CMAKE_CXX_COMPILER}" + "CFLAGS=${CMAKE_C_FLAGS} -fPIC -O2 -D_DEFAULT_SOURCE -D_GNU_SOURCE" + "CXXFLAGS=${CMAKE_CXX_FLAGS} -fPIC -O2 -D_DEFAULT_SOURCE -D_GNU_SOURCE" + "CPPFLAGS=-isystem ${CMAKE_INSTALL_PREFIX}/include" + "LDFLAGS=-L${CMAKE_INSTALL_PREFIX}/lib -L${CMAKE_INSTALL_PREFIX}/lib64" + "PATH=${BUILDING_PATH}" + "ACLOCAL_PATH=${ACLOCAL_PATH}" +) + +set(ALL_TARGETS + libunwind + openssl + libevent + keyutils + krb5 + jemalloc + bzip2 + zstd + zlib + snappy + lzma + libaio + mstch + fatal + double-conversion + gflags + googletest + glog + boost + rocksdb + folly + wangle + proxygen + fbthrift + capstone + s2geometry +) + +list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/cmake) +list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/../cmake) + +find_package(Libtool 2.4.2 QUIET) +if (NOT Libtool_FOUND) + list(APPEND ALL_TARGETS libtool) +endif() + +find_package(Autoconf 2.69 QUIET) +if (NOT Autoconf_FOUND) + list(APPEND ALL_TARGETS autoconf) +endif() + +find_package(AutoconfArchive QUIET) +if (NOT AutoconfArchive_FOUND) + list(APPEND ALL_TARGETS autoconf-archive) +endif() + +find_package(Automake 1.13.4 QUIET) +if (NOT Automake_FOUND) + list(APPEND ALL_TARGETS automake) +endif() + +find_package(BISON 3.0.5 QUIET) +if (NOT BISON_FOUND) + list(APPEND ALL_TARGETS bison) +endif() + +find_package(FLEX QUIET) +if (NOT FLEX_FOUND) + list(APPEND ALL_TARGETS flex) +endif() + +find_package(GPERF 2.8 QUIET REQUIRED) +if (NOT GPERF_FOUND) + list(APPEND ALL_TARGETS gperf) +endif() + +find_package(Gettext QUIET) +if (NOT Gettext_FOUND) + list(APPEND ALL_TARGETS gettext) +endif() + +foreach(target ${ALL_TARGETS}) + include(externals/${target}.cmake) + list(APPEND CLEAN_TARGETS ${target}-clean) +endforeach() + +macro(maybe_add_dependencies depender) + if (TARGET ${depender}) + foreach (dependee ${ARGN}) + if (TARGET ${dependee}) + add_dependencies(${depender} ${dependee}) + endif() + endforeach() + endif() +endmacro() + +maybe_add_dependencies(autoconf-archive autoconf) +maybe_add_dependencies(automake autoconf autoconf-archive) +maybe_add_dependencies(libtool automake) + +maybe_add_dependencies(glog libtool) +maybe_add_dependencies(bison libtool) +maybe_add_dependencies(flex libtool) +maybe_add_dependencies(zlib libtool) +maybe_add_dependencies(gettext libtool) +maybe_add_dependencies(lzma gettext) +maybe_add_dependencies(libevent libtool) +maybe_add_dependencies(gperf libtool) + +maybe_add_dependencies(boost zlib bzip2 lzma) + +maybe_add_dependencies(glog gflags) + +maybe_add_dependencies(s2geometry googletest glog openssl) + +maybe_add_dependencies(krb5 keyutils openssl gettext) +maybe_add_dependencies(folly glog boost double-conversion openssl libevent lzma zstd snappy libunwind libaio) +maybe_add_dependencies(wangle folly) +maybe_add_dependencies(fbthrift folly krb5 bison flex mstch zlib zstd wangle fatal) +maybe_add_dependencies(proxygen wangle libunwind gperf) +maybe_add_dependencies(rocksdb snappy zlib zstd bzip2 lzma libunwind) + +add_custom_command( + TARGET zstd POST_BUILD + COMMAND + rm -rf ${CMAKE_INSTALL_PREFIX}/lib/libzstd.so* +) + +add_custom_command( + TARGET rocksdb POST_BUILD + COMMAND + rm -rf ${CMAKE_INSTALL_PREFIX}/lib64/librocksdb.so* +) + +add_custom_target( + clean-all + DEPENDS ${CLEAN_TARGETS} +) diff --git a/third-party/build-cmake.sh b/third-party/build-cmake.sh new file mode 100755 index 00000000000..52a4af06604 --- /dev/null +++ b/third-party/build-cmake.sh @@ -0,0 +1,57 @@ +#! /usr/bin/env bash + +# Copyright (c) 2019 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. + +# Usage: build-cmake.sh [prefix] + +# Always use bash +shell=$(basename $(readlink /proc/$$/exe)) +if [ ! x$shell = x"bash" ] +then + bash $0 $@ + exit $? +fi + +archive=cmake-3.15.5-Linux-x86_64.sh +url=https://github.com/Kitware/CMake/releases/download/v3.15.5/cmake-3.15.5-Linux-x86_64.sh +prefix=`pwd`/cmake-3.15.5 + +if [[ -n $1 ]] +then + prefix=$1 +fi + +if [[ -f $archive ]] +then + checksum=$(md5sum $archive | cut -d ' ' -f 1) +fi + +if [[ ! $checksum = f73d4daf34cb5e5119ee95c53696f322 ]] +then + hash wget &> /dev/null && download_cmd="wget -c" + hash axel &> /dev/null && download_cmd="axel -a -n 16" + if [[ -z $download_cmd ]] + then + echo "Neither 'wget' nor 'axel' available for downloading" 1>&2; + exit 1; + fi + + echo "Downloading...please install 'axel' and retry if too slow." + if ! bash -c "$download_cmd $url" + then + echo "Download failed." + exit 1 + fi +fi + +mkdir -p $prefix +bash $archive --prefix=$prefix &> /dev/null < /dev/null || { echo "No cmake found." 1>&2; exit 1; } + local cmake_version=$(cmake --version | head -1 | cut -d ' ' -f 3) + least_cmake_version=3.8.0 + if [[ $(version_cmp $cmake_version $least_cmake_version) -lt 0 ]] + then + echo "cmake $least_cmake_version or later required" 1>&2 + exit 1 + fi +} + +check_cmake + + +# Exit on any failure here after +set -e +set -o pipefail + +# Directories setup +cur_dir=`pwd` +source_dir=$(readlink -f $(dirname $0)/..)/third-party +build_root=$cur_dir/third-party +build_dir=$build_root/build +install_dir=$build_root/install +download_dir=$build_root/downloads +source_tar_name=nebula-third-party-src-1.0.tgz +source_url=https://nebula-graph.oss-accelerate.aliyuncs.com/third-party/${source_tar_name} + +# Allow to customize compilers +[[ -n ${CC} ]] && C_COMPILER_ARG="-DCMAKE_C_COMPILER=${CC}" +[[ -n ${CXX} ]] && CXX_COMPILER_ARG="-DCMAKE_CXX_COMPILER=${CXX}" + +# Download source archives if necessary +mkdir -p $build_root +cd $build_root + +if [[ -f $source_tar_name ]] +then + checksum=$(md5sum $source_tar_name | cut -d ' ' -f 1) +fi + +if [[ ! $checksum = aa3356c67ea34d38ed28f762dccf1559 ]] +then + hash wget &> /dev/null && download_cmd="wget -c" + hash axel &> /dev/null && download_cmd="axel -a -n 16" + if [[ -z $download_cmd ]] + then + echo "Neither 'wget' nor 'axel' available for downloading" 1>&2; + elif ! bash -c "$download_cmd $source_url" + then + # Resort to the builtin download method of cmake on failure + echo "Download from $source_url failed." 1>&2 + else + tar --skip-old-files -xzf $source_tar_name + fi +else + tar --skip-old-files -xzf $source_tar_name +fi + +# Build and install +mkdir -p $build_dir $install_dir +cd $build_dir + +cmake -DDOWNLOAD_DIR=$download_dir \ + -DCMAKE_INSTALL_PREFIX=$install_dir \ + ${C_COMPILER_ARG} ${CXX_COMPILER_ARG} \ + $source_dir + +make diff --git a/third-party/cmake/FindAutoconf.cmake b/third-party/cmake/FindAutoconf.cmake new file mode 100644 index 00000000000..e6f50c12bc0 --- /dev/null +++ b/third-party/cmake/FindAutoconf.cmake @@ -0,0 +1,14 @@ +find_program(Autoconf_EXECUTABLE NAMES libtool DOC "Path to the autconf executable") +if (Autoconf_EXECUTABLE) + execute_process( + OUTPUT_VARIABLE Autoconf_VERSION + OUTPUT_STRIP_TRAILING_WHITESPACE + COMMAND "autoconf" "--version" + COMMAND "head" "-1" + COMMAND "cut" "-d" " " "-f4" + ) +endif() + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(Autoconf REQUIRED_VARS Autoconf_EXECUTABLE + VERSION_VAR Autoconf_VERSION) diff --git a/third-party/cmake/FindAutoconfArchive.cmake b/third-party/cmake/FindAutoconfArchive.cmake new file mode 100644 index 00000000000..c49166c21e3 --- /dev/null +++ b/third-party/cmake/FindAutoconfArchive.cmake @@ -0,0 +1,3 @@ +find_path(AutoconfArchive_DIR NAMES aclocal/ax_prefix_config_h.m4 PATHS /usr/share) +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(AutoconfArchive REQUIRED_VARS AutoconfArchive_DIR) diff --git a/third-party/cmake/FindAutomake.cmake b/third-party/cmake/FindAutomake.cmake new file mode 100644 index 00000000000..48d70aeaefe --- /dev/null +++ b/third-party/cmake/FindAutomake.cmake @@ -0,0 +1,14 @@ +find_program(Automake_EXECUTABLE NAMES libtool DOC "Path to the automake executable") +if (Automake_EXECUTABLE) + execute_process( + OUTPUT_VARIABLE Automake_VERSION + OUTPUT_STRIP_TRAILING_WHITESPACE + COMMAND "automake" "--version" + COMMAND "head" "-1" + COMMAND "cut" "-d" " " "-f4" + ) +endif() + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(Automake REQUIRED_VARS Automake_EXECUTABLE + VERSION_VAR Automake_VERSION) diff --git a/third-party/cmake/FindLibtool.cmake b/third-party/cmake/FindLibtool.cmake new file mode 100644 index 00000000000..a7e194db047 --- /dev/null +++ b/third-party/cmake/FindLibtool.cmake @@ -0,0 +1,14 @@ +find_program(Libtool_EXECUTABLE NAMES libtool DOC "Path to the libtool executable") +if (Libtool_EXECUTABLE) + execute_process( + OUTPUT_VARIABLE Libtool_VERSION + OUTPUT_STRIP_TRAILING_WHITESPACE + COMMAND "libtool" "--version" + COMMAND "head" "-1" + COMMAND "cut" "-d" " " "-f4" + ) +endif() + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(Libtool REQUIRED_VARS Libtool_EXECUTABLE + VERSION_VAR Libtool_VERSION) diff --git a/third-party/externals/autoconf-archive.cmake b/third-party/externals/autoconf-archive.cmake new file mode 100644 index 00000000000..45f317f5f89 --- /dev/null +++ b/third-party/externals/autoconf-archive.cmake @@ -0,0 +1,40 @@ +# Copyright (c) 2019 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. + +set(name autoconf-archive) +set(source_dir ${CMAKE_CURRENT_BINARY_DIR}/${name}/source) +ExternalProject_Add( + ${name} + URL https://ftp.gnu.org/gnu/autoconf-archive/autoconf-archive-2019.01.06.tar.xz + URL_HASH MD5=d46413c8b00a125b1529bae385bbec55 + PREFIX ${CMAKE_CURRENT_BINARY_DIR}/${name} + TMP_DIR ${BUILD_INFO_DIR} + STAMP_DIR ${BUILD_INFO_DIR} + DOWNLOAD_DIR ${DOWNLOAD_DIR} + SOURCE_DIR ${source_dir} + CONFIGURE_COMMAND + ${common_configure_envs} + "LIBS=${LIBS}" + ./configure ${common_configure_args} + BUILD_IN_SOURCE 1 + BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} + INSTALL_COMMAND make -s -j${BUILDING_JOBS_NUM} install +) + +ExternalProject_Add_Step(${name} clean + EXCLUDE_FROM_MAIN TRUE + ALWAYS TRUE + DEPENDEES configure + COMMAND make clean -j + COMMAND rm -f ${BUILD_INFO_DIR}/${name}-build + WORKING_DIRECTORY ${source_dir} +) + +ExternalProject_Add_StepTargets(${name} clean) + +add_custom_command( + TARGET ${name} POST_BUILD + COMMAND install -m 0644 ${CMAKE_SOURCE_DIR}/externals/pkg.m4 ${CMAKE_INSTALL_PREFIX}/share/aclocal +) diff --git a/third-party/externals/autoconf.cmake b/third-party/externals/autoconf.cmake new file mode 100644 index 00000000000..c52fd882def --- /dev/null +++ b/third-party/externals/autoconf.cmake @@ -0,0 +1,35 @@ +# Copyright (c) 2019 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. + +set(name autoconf) +set(source_dir ${CMAKE_CURRENT_BINARY_DIR}/${name}/source) +ExternalProject_Add( + ${name} + URL https://ftp.gnu.org/gnu/autoconf/autoconf-2.69.tar.gz + URL_HASH MD5=82d05e03b93e45f5a39b828dc9c6c29b + PREFIX ${CMAKE_CURRENT_BINARY_DIR}/${name} + TMP_DIR ${BUILD_INFO_DIR} + STAMP_DIR ${BUILD_INFO_DIR} + DOWNLOAD_DIR ${DOWNLOAD_DIR} + SOURCE_DIR ${source_dir} + CONFIGURE_COMMAND + ${common_configure_envs} + "LIBS=${LIBS}" + ./configure ${common_configure_args} + BUILD_IN_SOURCE 1 + BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} + INSTALL_COMMAND make -s -j${BUILDING_JOBS_NUM} install +) + +ExternalProject_Add_Step(${name} clean + EXCLUDE_FROM_MAIN TRUE + ALWAYS TRUE + DEPENDEES configure + COMMAND make clean -j + COMMAND rm -f ${BUILD_INFO_DIR}/${name}-build + WORKING_DIRECTORY ${source_dir} +) + +ExternalProject_Add_StepTargets(${name} clean) diff --git a/third-party/externals/automake.cmake b/third-party/externals/automake.cmake new file mode 100644 index 00000000000..25ab6ebd55e --- /dev/null +++ b/third-party/externals/automake.cmake @@ -0,0 +1,35 @@ +# Copyright (c) 2019 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. + +set(name automake) +set(source_dir ${CMAKE_CURRENT_BINARY_DIR}/${name}/source) +ExternalProject_Add( + ${name} + URL https://ftp.gnu.org/gnu/automake/automake-1.15.1.tar.xz + URL_HASH MD5=24cd3501b6ad8cd4d7e2546f07e8b4d4 + PREFIX ${CMAKE_CURRENT_BINARY_DIR}/${name} + TMP_DIR ${BUILD_INFO_DIR} + STAMP_DIR ${BUILD_INFO_DIR} + DOWNLOAD_DIR ${DOWNLOAD_DIR} + SOURCE_DIR ${source_dir} + CONFIGURE_COMMAND + ${common_configure_envs} + "LIBS=${LIBS}" + ./configure ${common_configure_args} + BUILD_IN_SOURCE 1 + BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} + INSTALL_COMMAND make -s -j${BUILDING_JOBS_NUM} install +) + +ExternalProject_Add_Step(${name} clean + EXCLUDE_FROM_MAIN TRUE + ALWAYS TRUE + DEPENDEES configure + COMMAND make clean -j + COMMAND rm -f ${BUILD_INFO_DIR}/${name}-build + WORKING_DIRECTORY ${source_dir} +) + +ExternalProject_Add_StepTargets(${name} clean) diff --git a/third-party/externals/bison.cmake b/third-party/externals/bison.cmake new file mode 100644 index 00000000000..7eb8d16dae9 --- /dev/null +++ b/third-party/externals/bison.cmake @@ -0,0 +1,37 @@ +# Copyright (c) 2019 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. + +set(name bison) +set(source_dir ${CMAKE_CURRENT_BINARY_DIR}/${name}/source) +ExternalProject_Add( + ${name} + URL http://ftp.gnu.org/gnu/bison//bison-3.0.5.tar.gz + URL_HASH MD5=41ad57813157b61bfa47e33067a9d6f0 + PREFIX ${CMAKE_CURRENT_BINARY_DIR}/${name} + TMP_DIR ${BUILD_INFO_DIR} + STAMP_DIR ${BUILD_INFO_DIR} + DOWNLOAD_DIR ${DOWNLOAD_DIR} + SOURCE_DIR ${source_dir} + CONFIGURE_COMMAND + ${common_configure_envs} + ./configure ${common_configure_args} + --quiet + BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} + BUILD_IN_SOURCE 1 + INSTALL_COMMAND make -s install -j${BUILDING_JOBS_NUM} + LOG_BUILD 1 + LOG_INSTALL 1 +) + +ExternalProject_Add_Step(${name} clean + EXCLUDE_FROM_MAIN TRUE + ALWAYS TRUE + DEPENDEES configure + COMMAND make clean -j + COMMAND rm -f ${BUILD_INFO_DIR}/${name}-build + WORKING_DIRECTORY ${source_dir} +) + +ExternalProject_Add_StepTargets(${name} clean) diff --git a/third-party/externals/boost.cmake b/third-party/externals/boost.cmake new file mode 100644 index 00000000000..1050ed10726 --- /dev/null +++ b/third-party/externals/boost.cmake @@ -0,0 +1,54 @@ +# Copyright (c) 2019 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. + +set(name boost) +set(source_dir ${CMAKE_CURRENT_BINARY_DIR}/${name}/source) +get_filename_component(compiler_path ${CMAKE_CXX_COMPILER} DIRECTORY) +ExternalProject_Add( + ${name} + URL https://dl.bintray.com/boostorg/release/1.67.0/source/boost_1_67_0.tar.gz + URL_HASH MD5=4850fceb3f2222ee011d4f3ea304d2cb + DOWNLOAD_NAME boost-1.67.0.tar.gz + PREFIX ${CMAKE_CURRENT_BINARY_DIR}/${name} + TMP_DIR ${BUILD_INFO_DIR} + STAMP_DIR ${BUILD_INFO_DIR} + DOWNLOAD_DIR ${DOWNLOAD_DIR} + SOURCE_DIR ${source_dir} + CONFIGURE_COMMAND "" + CONFIGURE_COMMAND + env PATH=${compiler_path}:${BUILDING_PATH} + ./bootstrap.sh + --without-icu + --without-libraries=python,test,stacktrace,mpi,log,graph,graph_parallel + --prefix=${CMAKE_INSTALL_PREFIX} + BUILD_COMMAND + env PATH=${compiler_path}:${BUILDING_PATH} + ./b2 install + -d0 + -j${BUILDING_JOBS_NUM} + --prefix=${CMAKE_INSTALL_PREFIX} + --disable-icu + include=${CMAKE_INSTALL_PREFIX}/include + linkflags=-L${CMAKE_INSTALL_PREFIX}/lib + cxxflags="-fPIC" + runtime-link=static + link=static + variant=release + BUILD_IN_SOURCE 1 + INSTALL_COMMAND "" + LOG_BUILD 0 + LOG_INSTALL 0 +) + +ExternalProject_Add_Step(${name} clean + EXCLUDE_FROM_MAIN TRUE + ALWAYS TRUE + DEPENDEES configure + COMMAND ./b2 clean + COMMAND rm -f ${BUILD_INFO_DIR}/${name}-build + WORKING_DIRECTORY ${source_dir} +) + +ExternalProject_Add_StepTargets(${name} clean) diff --git a/third-party/externals/bzip2.cmake b/third-party/externals/bzip2.cmake new file mode 100644 index 00000000000..d13efa2ccc0 --- /dev/null +++ b/third-party/externals/bzip2.cmake @@ -0,0 +1,34 @@ +# Copyright (c) 2019 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. + +set(name bzip2) +set(source_dir ${CMAKE_CURRENT_BINARY_DIR}/${name}/source) +ExternalProject_Add( + ${name} + URL https://nchc.dl.sourceforge.net/project/bzip2/bzip2-1.0.6.tar.gz + URL_HASH MD5=00b516f4704d4a7cb50a1d97e6e8e15b + PREFIX ${CMAKE_CURRENT_BINARY_DIR}/${name} + TMP_DIR ${BUILD_INFO_DIR} + STAMP_DIR ${BUILD_INFO_DIR} + DOWNLOAD_DIR ${DOWNLOAD_DIR} + SOURCE_DIR ${source_dir} + CONFIGURE_COMMAND "" + BUILD_COMMAND make CFLAGS=-fPIC -s -j${BUILDING_JOBS_NUM} + BUILD_IN_SOURCE 1 + INSTALL_COMMAND make -s install -j${BUILDING_JOBS_NUM} PREFIX=${CMAKE_INSTALL_PREFIX} + LOG_BUILD 1 + LOG_INSTALL 1 +) + +ExternalProject_Add_Step(${name} clean + EXCLUDE_FROM_MAIN TRUE + ALWAYS TRUE + DEPENDEES configure + COMMAND make clean -j + COMMAND rm -f ${BUILD_INFO_DIR}/${name}-build + WORKING_DIRECTORY ${source_dir} +) + +ExternalProject_Add_StepTargets(${name} clean) diff --git a/third-party/externals/capstone.cmake b/third-party/externals/capstone.cmake new file mode 100644 index 00000000000..b14610a5f23 --- /dev/null +++ b/third-party/externals/capstone.cmake @@ -0,0 +1,56 @@ +# Copyright (c) 2019 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. + +set(name capstone) +set(source_dir ${CMAKE_CURRENT_BINARY_DIR}/${name}/source) +ExternalProject_Add( + ${name} + URL https://github.com/aquynh/capstone/archive/4.0.1.tar.gz + URL_HASH MD5=1b0a9a0d50d9515dcf7684ce0a2270a4 + DOWNLOAD_NAME capstone-4.0.1.tar.gz + PREFIX ${CMAKE_CURRENT_BINARY_DIR}/${name} + TMP_DIR ${BUILD_INFO_DIR} + STAMP_DIR ${BUILD_INFO_DIR} + DOWNLOAD_DIR ${DOWNLOAD_DIR} + SOURCE_DIR ${source_dir} + CMAKE_ARGS + ${common_cmake_args} + -DCAPSTONE_X86_SUPPORT=ON + -DCAPSTONE_ARM_SUPPORT=OFF + -DCAPSTONE_ARM64_SUPPORT=OFF + -DCAPSTONE_M680X_SUPPORT=OFF + -DCAPSTONE_M68K_SUPPORT=OFF + -DCAPSTONE_MIPS_SUPPORT=OFF + -DCAPSTONE_MOS65XX_SUPPORT=OFF + -DCAPSTONE_PPC_SUPPORT=OFF + -DCAPSTONE_SPARC_SUPPORT=OFF + -DCAPSTONE_SYSZ_SUPPORT=OFF + -DCAPSTONE_XCORE_SUPPORT=OFF + -DCAPSTONE_TMS320C64X_SUPPORT=OFF + -DCAPSTONE_M680X_SUPPORT=OFF + -DCAPSTONE_EVM_SUPPORT=OFF + -DCAPSTONE_BUILD_DIET=OFF + -DCAPSTONE_X86_REDUCE=OFF + -DCAPSTONE_BUILD_TESTS=OFF + -DCAPSTONE_BUILD_STATIC=ON + -DCAPSTONE_BUILD_SHARED=OFF + -DCMAKE_BUILD_TYPE=Release + BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} + BUILD_IN_SOURCE 1 + INSTALL_COMMAND make -s install -j${BUILDING_JOBS_NUM} + LOG_BUILD 1 + LOG_INSTALL 1 +) + +ExternalProject_Add_Step(${name} clean + EXCLUDE_FROM_MAIN TRUE + ALWAYS TRUE + DEPENDEES configure + COMMAND make clean -j + COMMAND rm -f ${BUILD_INFO_DIR}/${name}-build + WORKING_DIRECTORY ${source_dir} +) + +ExternalProject_Add_StepTargets(${name} clean) diff --git a/third-party/externals/double-conversion.cmake b/third-party/externals/double-conversion.cmake new file mode 100644 index 00000000000..5cbb5e6a76c --- /dev/null +++ b/third-party/externals/double-conversion.cmake @@ -0,0 +1,36 @@ +# Copyright (c) 2019 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. + +set(name double-conversion) +set(source_dir ${CMAKE_CURRENT_BINARY_DIR}/${name}/source) +ExternalProject_Add( + ${name} + URL https://github.com/google/double-conversion/archive/v1.1.6.tar.gz + URL_HASH MD5=94f9abc9b1367083cf3e4569886b4170 + DOWNLOAD_NAME double-conversion-1.1.6.tar.gz + PREFIX ${CMAKE_CURRENT_BINARY_DIR}/${name} + TMP_DIR ${BUILD_INFO_DIR} + STAMP_DIR ${BUILD_INFO_DIR} + DOWNLOAD_DIR ${DOWNLOAD_DIR} + SOURCE_DIR ${source_dir} + CMAKE_ARGS + ${common_cmake_args} + BUILD_IN_SOURCE 1 + BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} + INSTALL_COMMAND make -s install -j${BUILDING_JOBS_NUM} + LOG_BUILD 1 + LOG_INSTALL 1 +) + +ExternalProject_Add_Step(${name} clean + EXCLUDE_FROM_MAIN TRUE + ALWAYS TRUE + DEPENDEES configure + COMMAND make clean -j + COMMAND rm -f ${BUILD_INFO_DIR}/${name}-build + WORKING_DIRECTORY ${source_dir} +) + +ExternalProject_Add_StepTargets(${name} clean) diff --git a/third-party/externals/fatal.cmake b/third-party/externals/fatal.cmake new file mode 100644 index 00000000000..da33cbb2989 --- /dev/null +++ b/third-party/externals/fatal.cmake @@ -0,0 +1,36 @@ +# Copyright (c) 2019 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. + +set(name fatal) +set(source_dir ${CMAKE_CURRENT_BINARY_DIR}/${name}/source) +ExternalProject_Add( + ${name} + URL https://github.com/facebook/fatal/archive/v2018.08.20.00.tar.gz + URL_HASH MD5=b0887650f53ba8a73924351024f761ad + DOWNLOAD_NAME fatal-2018-08-20.tar.gz + PREFIX ${CMAKE_CURRENT_BINARY_DIR}/${name} + TMP_DIR ${BUILD_INFO_DIR} + STAMP_DIR ${BUILD_INFO_DIR} + DOWNLOAD_DIR ${DOWNLOAD_DIR} + SOURCE_DIR ${source_dir} + CONFIGURE_COMMAND "" + BUILD_COMMAND install -d ${CMAKE_INSTALL_PREFIX}/include + BUILD_IN_SOURCE 1 + INSTALL_COMMAND cp -r fatal ${CMAKE_INSTALL_PREFIX}/include + LOG_CONFIGURE TRUE + LOG_BUILD TRUE + LOG_INSTALL TRUE + LOG_MERGED_STDOUTERR TRUE +) + +ExternalProject_Add_Step(${name} clean + EXCLUDE_FROM_MAIN TRUE + ALWAYS TRUE + DEPENDEES configure + COMMAND rm -f ${BUILD_INFO_DIR}/${name}-build + WORKING_DIRECTORY ${source_dir} +) + +ExternalProject_Add_StepTargets(${name} clean) diff --git a/third-party/externals/fbthrift.cmake b/third-party/externals/fbthrift.cmake new file mode 100644 index 00000000000..5aad9c848a2 --- /dev/null +++ b/third-party/externals/fbthrift.cmake @@ -0,0 +1,42 @@ +# Copyright (c) 2019 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. + +set(name fbthrift) +set(source_dir ${CMAKE_CURRENT_BINARY_DIR}/${name}/source) +ExternalProject_Add( + ${name} + URL https://github.com/facebook/fbthrift/archive/v2018.08.20.00.tar.gz + URL_HASH MD5=346627716bae0a4015f67ab33f255173 + DOWNLOAD_NAME fbthrift-2018-08-20.tar.gz + PREFIX ${CMAKE_CURRENT_BINARY_DIR}/${name} + TMP_DIR ${BUILD_INFO_DIR} + STAMP_DIR ${BUILD_INFO_DIR} + DOWNLOAD_DIR ${DOWNLOAD_DIR} + SOURCE_DIR ${source_dir} + PATCH_COMMAND patch -p1 < ${CMAKE_SOURCE_DIR}/patches/fbthrift-2018-08-20.patch + CMAKE_COMMAND env PATH=${CMAKE_INSTALL_PREFIX}/bin:$ENV{PATH} ${CMAKE_COMMAND} + CMAKE_ARGS + ${common_cmake_args} + "-DCMAKE_EXE_LINKER_FLAGS=-static-libstdc++ -static-libgcc" + -D_OPENSSL_LIBDIR=${CMAKE_INSTALL_PREFIX}/lib64 + BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} + BUILD_IN_SOURCE 1 + INSTALL_COMMAND make -s -j${BUILDING_JOBS_NUM} install + LOG_CONFIGURE TRUE + LOG_BUILD TRUE + LOG_INSTALL TRUE + LOG_MERGED_STDOUTERR TRUE +) + +ExternalProject_Add_Step(${name} clean + EXCLUDE_FROM_MAIN TRUE + ALWAYS TRUE + DEPENDEES configure + COMMAND make clean -j + COMMAND rm -f ${BUILD_INFO_DIR}/${name}-build + WORKING_DIRECTORY ${source_dir} +) + +ExternalProject_Add_StepTargets(${name} clean) diff --git a/third-party/externals/flex.cmake b/third-party/externals/flex.cmake new file mode 100644 index 00000000000..831fbfdc173 --- /dev/null +++ b/third-party/externals/flex.cmake @@ -0,0 +1,37 @@ +# Copyright (c) 2019 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. + +set(name flex) +set(source_dir ${CMAKE_CURRENT_BINARY_DIR}/${name}/source) +ExternalProject_Add( + ${name} + URL https://github.com/westes/flex/releases/download/v2.6.4/flex-2.6.4.tar.gz + URL_HASH MD5=2882e3179748cc9f9c23ec593d6adc8d + PREFIX ${CMAKE_CURRENT_BINARY_DIR}/${name} + TMP_DIR ${BUILD_INFO_DIR} + STAMP_DIR ${BUILD_INFO_DIR} + DOWNLOAD_DIR ${DOWNLOAD_DIR} + SOURCE_DIR ${source_dir} + CONFIGURE_COMMAND + ${common_configure_envs} + ./configure ${common_configure_args} + --enable-static --disable-shared + BUILD_COMMAND make -s + BUILD_IN_SOURCE 1 + INSTALL_COMMAND make -s install + LOG_BUILD 1 + LOG_INSTALL 1 +) + +ExternalProject_Add_Step(${name} clean + EXCLUDE_FROM_MAIN TRUE + ALWAYS TRUE + DEPENDEES configure + COMMAND make clean -j + COMMAND rm -f ${BUILD_INFO_DIR}/${name}-build + WORKING_DIRECTORY ${source_dir} +) + +ExternalProject_Add_StepTargets(${name} clean) diff --git a/third-party/externals/folly.cmake b/third-party/externals/folly.cmake new file mode 100644 index 00000000000..038b07eec77 --- /dev/null +++ b/third-party/externals/folly.cmake @@ -0,0 +1,42 @@ +# Copyright (c) 2019 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. + +set(name folly) +set(source_dir ${CMAKE_CURRENT_BINARY_DIR}/${name}/source) +ExternalProject_Add( + ${name} + URL https://github.com/facebook/folly/archive/v2018.08.20.00.tar.gz + URL_HASH MD5=1260231dd088526297ec52e3e12bf0ee + DOWNLOAD_NAME folly-2018-08-20.tar.gz + PREFIX ${CMAKE_CURRENT_BINARY_DIR}/${name} + TMP_DIR ${BUILD_INFO_DIR} + STAMP_DIR ${BUILD_INFO_DIR} + DOWNLOAD_DIR ${DOWNLOAD_DIR} + SOURCE_DIR ${source_dir} + CMAKE_ARGS + ${common_cmake_args} + -DCMAKE_BUILD_TYPE=Release + "-DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} -fPIC -DFOLLY_HAVE_CLOCK_GETTIME -D__USE_POSIX199309" + -DFOLLY_CXX_FLAGS=-Wno-error + + BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} + BUILD_IN_SOURCE 1 + INSTALL_COMMAND make -s -j${BUILDING_JOBS_NUM} install + LOG_CONFIGURE TRUE + LOG_BUILD TRUE + LOG_INSTALL TRUE + LOG_MERGED_STDOUTERR TRUE +) + +ExternalProject_Add_Step(${name} clean + EXCLUDE_FROM_MAIN TRUE + ALWAYS TRUE + DEPENDEES configure + COMMAND make clean -j + COMMAND rm -f ${BUILD_INFO_DIR}/${name}-build + WORKING_DIRECTORY ${source_dir} +) + +ExternalProject_Add_StepTargets(${name} clean) diff --git a/third-party/externals/gettext.cmake b/third-party/externals/gettext.cmake new file mode 100644 index 00000000000..d2e9aba9c44 --- /dev/null +++ b/third-party/externals/gettext.cmake @@ -0,0 +1,37 @@ +# Copyright (c) 2019 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. + +set(name gettext) +set(source_dir ${CMAKE_CURRENT_BINARY_DIR}/${name}/source) +ExternalProject_Add( + ${name} + URL http://ftp.gnu.org/gnu/gettext/gettext-0.19.8.1.tar.gz + URL_HASH MD5=97e034cf8ce5ba73a28ff6c3c0638092 + PREFIX ${CMAKE_CURRENT_BINARY_DIR}/${name} + TMP_DIR ${BUILD_INFO_DIR} + STAMP_DIR ${BUILD_INFO_DIR} + DOWNLOAD_DIR ${DOWNLOAD_DIR} + SOURCE_DIR ${source_dir} + CONFIGURE_COMMAND + ${common_configure_envs} + ./configure ${common_configure_args} + --disable-shared --enable-static + BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} + BUILD_IN_SOURCE 1 + INSTALL_COMMAND make -s install -j${BUILDING_JOBS_NUM} + LOG_BUILD 1 + LOG_INSTALL 1 +) + +ExternalProject_Add_Step(${name} clean + EXCLUDE_FROM_MAIN TRUE + ALWAYS TRUE + DEPENDEES configure + COMMAND make clean -j + COMMAND rm -f ${BUILD_INFO_DIR}/${name}-build + WORKING_DIRECTORY ${source_dir} +) + +ExternalProject_Add_StepTargets(${name} clean) diff --git a/third-party/externals/gflags.cmake b/third-party/externals/gflags.cmake new file mode 100644 index 00000000000..787def77eed --- /dev/null +++ b/third-party/externals/gflags.cmake @@ -0,0 +1,35 @@ +# Copyright (c) 2019 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. + +set(name gflags) +set(source_dir ${CMAKE_CURRENT_BINARY_DIR}/${name}/source) +ExternalProject_Add( + ${name} + URL https://github.com/gflags/gflags/archive/v2.2.1.tar.gz + URL_HASH MD5=b98e772b4490c84fc5a87681973f75d1 + DOWNLOAD_NAME gflags-2.2.1.tar.gz + PREFIX ${CMAKE_CURRENT_BINARY_DIR}/${name} + TMP_DIR ${BUILD_INFO_DIR} + STAMP_DIR ${BUILD_INFO_DIR} + DOWNLOAD_DIR ${DOWNLOAD_DIR} + SOURCE_DIR ${source_dir} + CMAKE_ARGS + ${common_cmake_args} + -DCMAKE_BUILD_TYPE=Release + BUILD_IN_SOURCE 1 + LOG_BUILD 1 + LOG_INSTALL 1 +) + +ExternalProject_Add_Step(${name} clean + EXCLUDE_FROM_MAIN TRUE + ALWAYS TRUE + DEPENDEES configure + COMMAND make clean -j + COMMAND rm -f ${BUILD_INFO_DIR}/${name}-build + WORKING_DIRECTORY ${source_dir} +) + +ExternalProject_Add_StepTargets(${name} clean) diff --git a/third-party/externals/glog.cmake b/third-party/externals/glog.cmake new file mode 100644 index 00000000000..633f07b128c --- /dev/null +++ b/third-party/externals/glog.cmake @@ -0,0 +1,46 @@ +# Copyright (c) 2019 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. + +set(name glog) +set(source_dir ${CMAKE_CURRENT_BINARY_DIR}/${name}/source) +ExternalProject_Add( + ${name} + URL https://github.com/google/glog/archive/v0.3.5.tar.gz + URL_HASH MD5=5df6d78b81e51b90ac0ecd7ed932b0d4 + DOWNLOAD_NAME glog-0.3.5.tar.gz + PREFIX ${CMAKE_CURRENT_BINARY_DIR}/${name} + TMP_DIR ${BUILD_INFO_DIR} + STAMP_DIR ${BUILD_INFO_DIR} + DOWNLOAD_DIR ${DOWNLOAD_DIR} + SOURCE_DIR ${source_dir} + CONFIGURE_COMMAND + ${common_configure_envs} + ./configure ${common_configure_args} + --disable-shared + --enable-static + BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} + BUILD_IN_SOURCE 1 + INSTALL_COMMAND make -s -j${BUILDING_JOBS_NUM} install + LOG_BUILD 1 + LOG_INSTALL 1 +) + +ExternalProject_Add_Step(glog pre-configure + DEPENDEES download update patch + DEPENDERS configure + COMMAND env PATH=${BUILDING_PATH} ACLOCAL_PATH=${ACLOCAL_PATH} autoreconf -if + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/glog/source +) + +ExternalProject_Add_Step(${name} clean + EXCLUDE_FROM_MAIN TRUE + ALWAYS TRUE + DEPENDEES configure + COMMAND make clean -j + COMMAND rm -f ${BUILD_INFO_DIR}/${name}-build + WORKING_DIRECTORY ${source_dir} +) + +ExternalProject_Add_StepTargets(${name} clean) diff --git a/third-party/externals/googletest.cmake b/third-party/externals/googletest.cmake new file mode 100644 index 00000000000..8f5bf4307fb --- /dev/null +++ b/third-party/externals/googletest.cmake @@ -0,0 +1,34 @@ +# Copyright (c) 2019 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. + +set(name googletest) +set(source_dir ${CMAKE_CURRENT_BINARY_DIR}/${name}/source) +ExternalProject_Add( + ${name} + URL https://github.com/google/googletest/archive/release-1.8.0.tar.gz + URL_HASH MD5=16877098823401d1bf2ed7891d7dce36 + DOWNLOAD_NAME googletest-1.8.0.tar.gz + PREFIX ${CMAKE_CURRENT_BINARY_DIR}/${name} + TMP_DIR ${BUILD_INFO_DIR} + STAMP_DIR ${BUILD_INFO_DIR} + DOWNLOAD_DIR ${DOWNLOAD_DIR} + SOURCE_DIR ${source_dir} + BUILD_IN_SOURCE 1 + CMAKE_ARGS + ${common_cmake_args} + LOG_BUILD 1 + LOG_INSTALL 1 +) + +ExternalProject_Add_Step(${name} clean + EXCLUDE_FROM_MAIN TRUE + ALWAYS TRUE + DEPENDEES configure + COMMAND make clean -j + COMMAND rm -f ${BUILD_INFO_DIR}/${name}-build + WORKING_DIRECTORY ${source_dir} +) + +ExternalProject_Add_StepTargets(${name} clean) diff --git a/third-party/externals/gperf.cmake b/third-party/externals/gperf.cmake new file mode 100644 index 00000000000..6e0eb0229eb --- /dev/null +++ b/third-party/externals/gperf.cmake @@ -0,0 +1,38 @@ +# Copyright (c) 2019 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. + +set(name gperf) +set(source_dir ${CMAKE_CURRENT_BINARY_DIR}/${name}/source) +ExternalProject_Add( + ${name} + URL http://ftpmirror.gnu.org/gperf/gperf-3.1.tar.gz + URL_HASH MD5=9e251c0a618ad0824b51117d5d9db87e + PREFIX ${CMAKE_CURRENT_BINARY_DIR}/${name} + TMP_DIR ${BUILD_INFO_DIR} + STAMP_DIR ${BUILD_INFO_DIR} + DOWNLOAD_DIR ${DOWNLOAD_DIR} + SOURCE_DIR ${source_dir} + CONFIGURE_COMMAND + ${common_configure_envs} + "LIBS=-static-libstdc++ -static-libgcc" + ./configure ${common_configure_args} + --disable-shared --enable-static + BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} + BUILD_IN_SOURCE 1 + INSTALL_COMMAND make -s install -j${BUILDING_JOBS_NUM} + LOG_BUILD 1 + LOG_INSTALL 1 +) + +ExternalProject_Add_Step(${name} clean + EXCLUDE_FROM_MAIN TRUE + ALWAYS TRUE + DEPENDEES configure + COMMAND make clean -j + COMMAND rm -f ${BUILD_INFO_DIR}/${name}-build + WORKING_DIRECTORY ${source_dir} +) + +ExternalProject_Add_StepTargets(${name} clean) diff --git a/third-party/externals/jemalloc.cmake b/third-party/externals/jemalloc.cmake new file mode 100644 index 00000000000..5f1471f92e3 --- /dev/null +++ b/third-party/externals/jemalloc.cmake @@ -0,0 +1,37 @@ +# Copyright (c) 2019 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. + +set(name jemalloc) +set(source_dir ${CMAKE_CURRENT_BINARY_DIR}/${name}/source) +ExternalProject_Add( + ${name} + URL https://github.com/jemalloc/jemalloc/releases/download/5.1.0/jemalloc-5.1.0.tar.bz2 + URL_HASH MD5=1f47a5aff2d323c317dfa4cf23be1ce4 + PREFIX ${CMAKE_CURRENT_BINARY_DIR}/${name} + TMP_DIR ${BUILD_INFO_DIR} + STAMP_DIR ${BUILD_INFO_DIR} + DOWNLOAD_DIR ${DOWNLOAD_DIR} + SOURCE_DIR ${source_dir} + CONFIGURE_COMMAND + ${common_configure_envs} + ./configure ${common_configure_args} + --disable-stats --enable-prof + BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} + BUILD_IN_SOURCE 1 + INSTALL_COMMAND make -s install_bin install_include install_lib_static -j${BUILDING_JOBS_NUM} + LOG_BUILD 1 + LOG_INSTALL 1 +) + +ExternalProject_Add_Step(${name} clean + EXCLUDE_FROM_MAIN TRUE + ALWAYS TRUE + DEPENDEES configure + COMMAND make clean -j + COMMAND rm -f ${BUILD_INFO_DIR}/${name}-build + WORKING_DIRECTORY ${source_dir} +) + +ExternalProject_Add_StepTargets(${name} clean) diff --git a/third-party/externals/keyutils.cmake b/third-party/externals/keyutils.cmake new file mode 100644 index 00000000000..2cc8679eb52 --- /dev/null +++ b/third-party/externals/keyutils.cmake @@ -0,0 +1,38 @@ +# Copyright (c) 2019 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. + +set(name keyutils) +set(source_dir ${CMAKE_CURRENT_BINARY_DIR}/${name}/source) +ExternalProject_Add( + ${name} + URL https://people.redhat.com/dhowells/keyutils/keyutils-1.6.tar.bz2 + URL_HASH MD5=191987b0ab46bb5b50efd70a6e6ce808 + PREFIX ${CMAKE_CURRENT_BINARY_DIR}/${name} + TMP_DIR ${BUILD_INFO_DIR} + STAMP_DIR ${BUILD_INFO_DIR} + DOWNLOAD_DIR ${DOWNLOAD_DIR} + SOURCE_DIR ${source_dir} + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + BUILD_IN_SOURCE 1 + INSTALL_COMMAND make BINDIR=/bin SBINDIR=/sbin + SHAREDIR=/share MANDIR=/man + INCLUDEDIR=/include LIBDIR=/lib + CFLAGS=-fPIC NO_SOLIB=1 DESTDIR=${CMAKE_INSTALL_PREFIX} + install + LOG_BUILD 1 + LOG_INSTALL 1 +) + +ExternalProject_Add_Step(${name} clean + EXCLUDE_FROM_MAIN TRUE + ALWAYS TRUE + DEPENDEES configure + COMMAND make clean -j + COMMAND rm -f ${BUILD_INFO_DIR}/${name}-build + WORKING_DIRECTORY ${source_dir} +) + +ExternalProject_Add_StepTargets(${name} clean) diff --git a/third-party/externals/krb5.cmake b/third-party/externals/krb5.cmake new file mode 100644 index 00000000000..af54f249593 --- /dev/null +++ b/third-party/externals/krb5.cmake @@ -0,0 +1,57 @@ +# Copyright (c) 2019 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. + +set(name krb5) +set(source_dir ${CMAKE_CURRENT_BINARY_DIR}/${name}/source) +ExternalProject_Add( + ${name} + URL https://kerberos.org/dist/krb5/1.16/krb5-1.16.1.tar.gz + URL_HASH MD5=848e9b80d6aaaa798e3f3df24b83c407 +#URL https://kerberos.org/dist/krb5/1.16/krb5-1.16.3.tar.gz +#URL_HASH MD5=65f5f695bd78ba6a64ac786f571047f4 + PREFIX ${CMAKE_CURRENT_BINARY_DIR}/${name} + TMP_DIR ${BUILD_INFO_DIR} + STAMP_DIR ${BUILD_INFO_DIR} + DOWNLOAD_DIR ${DOWNLOAD_DIR} + SOURCE_DIR ${source_dir} + CONFIGURE_COMMAND "" + BUILD_COMMAND + env PATH=${BUILDING_PATH} + make -s -j${BUILDING_JOBS_NUM} -C src + BUILD_IN_SOURCE 1 + INSTALL_COMMAND + env PATH=${BUILDING_PATH} + make -s install -j${BUILDING_JOBS_NUM} -C src + LOG_BUILD 1 + LOG_INSTALL 1 +) + +ExternalProject_Add_Step(krb5 mannual-configure + DEPENDEES download update patch configure + DEPENDERS build install + COMMAND + ${common_configure_envs} + ./configure + ${common_configure_args} + --enable-static + --disable-shared + --disable-rpath + --disable-aesni + --disable-thread-support + WORKING_DIRECTORY ${source_dir}/src +) + +ExternalProject_Add_Step(${name} clean + EXCLUDE_FROM_MAIN TRUE + ALWAYS TRUE + DEPENDEES mannual-configure + COMMAND + env PATH=${BUILDING_PATH} + make clean -j -C src + COMMAND rm -f ${BUILD_INFO_DIR}/${name}-build + WORKING_DIRECTORY ${source_dir} +) + +ExternalProject_Add_StepTargets(${name} clean) diff --git a/third-party/externals/libaio.cmake b/third-party/externals/libaio.cmake new file mode 100644 index 00000000000..b5dfdd42756 --- /dev/null +++ b/third-party/externals/libaio.cmake @@ -0,0 +1,40 @@ +# Copyright (c) 2019 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. + +set(name libaio) +set(source_dir ${CMAKE_CURRENT_BINARY_DIR}/${name}/source) +ExternalProject_Add( + ${name} + URL https://github.com/crossbuild/libaio/archive/libaio-0.3.110-1.tar.gz + URL_HASH MD5=266b58badf6d010eab433abc8713d959 + PREFIX ${CMAKE_CURRENT_BINARY_DIR}/${name} + TMP_DIR ${BUILD_INFO_DIR} + STAMP_DIR ${BUILD_INFO_DIR} + DOWNLOAD_DIR ${DOWNLOAD_DIR} + SOURCE_DIR ${source_dir} + CONFIGURE_COMMAND "" + BUILD_COMMAND env CFLAGS=-fPIC make -s -j${BUILDING_JOBS_NUM} + BUILD_IN_SOURCE 1 + INSTALL_COMMAND make prefix=${CMAKE_INSTALL_PREFIX} -s install -j${BUILDING_JOBS_NUM} + LOG_BUILD 1 + LOG_INSTALL 1 +) + +ExternalProject_Add_Step(${name} clean + EXCLUDE_FROM_MAIN TRUE + ALWAYS TRUE + DEPENDEES configure + COMMAND make clean -j + COMMAND rm -f ${BUILD_INFO_DIR}/${name}-build + WORKING_DIRECTORY ${source_dir} +) + +ExternalProject_Add_StepTargets(${name} clean) + +add_custom_command( + TARGET libaio POST_BUILD + COMMAND + rm -f ${CMAKE_INSTALL_PREFIX}/lib/libaio.so* +) diff --git a/third-party/externals/libevent.cmake b/third-party/externals/libevent.cmake new file mode 100644 index 00000000000..10b6dc8c137 --- /dev/null +++ b/third-party/externals/libevent.cmake @@ -0,0 +1,40 @@ +# Copyright (c) 2019 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. + +set(name libevent) +set(source_dir ${CMAKE_CURRENT_BINARY_DIR}/${name}/source) +ExternalProject_Add( + ${name} + URL https://github.com/libevent/libevent/releases/download/release-2.1.11-stable/libevent-2.1.11-stable.tar.gz + URL_HASH MD5=7f35cfe69b82d879111ec0d7b7b1c531 + PREFIX ${CMAKE_CURRENT_BINARY_DIR}/${name} + TMP_DIR ${BUILD_INFO_DIR} + STAMP_DIR ${BUILD_INFO_DIR} + DOWNLOAD_DIR ${DOWNLOAD_DIR} + SOURCE_DIR ${source_dir} + CONFIGURE_COMMAND + ${common_configure_envs} + ./configure ${common_configure_args} + --disable-shared + --enable-static + --disable-samples + --disable-libevent-regress + BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} + BUILD_IN_SOURCE 1 + INSTALL_COMMAND make -s install -j${BUILDING_JOBS_NUM} + LOG_BUILD 1 + LOG_INSTALL 1 +) + +ExternalProject_Add_Step(${name} clean + EXCLUDE_FROM_MAIN TRUE + ALWAYS TRUE + DEPENDEES configure + COMMAND make clean -j + COMMAND rm -f ${BUILD_INFO_DIR}/${name}-build + WORKING_DIRECTORY ${source_dir} +) + +ExternalProject_Add_StepTargets(${name} clean) diff --git a/third-party/externals/libtool.cmake b/third-party/externals/libtool.cmake new file mode 100644 index 00000000000..fb33e1096e6 --- /dev/null +++ b/third-party/externals/libtool.cmake @@ -0,0 +1,35 @@ +# Copyright (c) 2019 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. + +set(name libtool) +set(source_dir ${CMAKE_CURRENT_BINARY_DIR}/${name}/source) +ExternalProject_Add( + ${name} + URL https://ftp.gnu.org/gnu/libtool/libtool-2.4.6.tar.xz + URL_HASH MD5=1bfb9b923f2c1339b4d2ce1807064aa5 + PREFIX ${CMAKE_CURRENT_BINARY_DIR}/${name} + TMP_DIR ${BUILD_INFO_DIR} + STAMP_DIR ${BUILD_INFO_DIR} + DOWNLOAD_DIR ${DOWNLOAD_DIR} + SOURCE_DIR ${source_dir} + CONFIGURE_COMMAND + ${common_configure_envs} + "LIBS=${LIBS}" + ./configure ${common_configure_args} + BUILD_IN_SOURCE 1 + BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} + INSTALL_COMMAND make -s -j${BUILDING_JOBS_NUM} install +) + +ExternalProject_Add_Step(${name} clean + EXCLUDE_FROM_MAIN TRUE + ALWAYS TRUE + DEPENDEES configure + COMMAND make clean -j + COMMAND rm -f ${BUILD_INFO_DIR}/${name}-build + WORKING_DIRECTORY ${source_dir} +) + +ExternalProject_Add_StepTargets(${name} clean) diff --git a/third-party/externals/libunwind.cmake b/third-party/externals/libunwind.cmake new file mode 100644 index 00000000000..137b227951a --- /dev/null +++ b/third-party/externals/libunwind.cmake @@ -0,0 +1,38 @@ +# Copyright (c) 2019 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. + +set(name libunwind) +set(source_dir ${CMAKE_CURRENT_BINARY_DIR}/${name}/source) +ExternalProject_Add( + ${name} + URL https://github.com/libunwind/libunwind/releases/download/v1.2.1/libunwind-1.2.1.tar.gz + URL_HASH MD5=06ba9e60d92fd6f55cd9dadb084df19e + PREFIX ${CMAKE_CURRENT_BINARY_DIR}/${name} + TMP_DIR ${BUILD_INFO_DIR} + STAMP_DIR ${BUILD_INFO_DIR} + DOWNLOAD_DIR ${DOWNLOAD_DIR} + SOURCE_DIR ${source_dir} + CONFIGURE_COMMAND + ${common_configure_envs} + ./configure ${common_configure_args} + --disable-minidebuginfo + --disable-shared --enable-static + BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} + BUILD_IN_SOURCE 1 + INSTALL_COMMAND make -s install -j${BUILDING_JOBS_NUM} + LOG_BUILD 1 + LOG_INSTALL 1 +) + +ExternalProject_Add_Step(${name} clean + EXCLUDE_FROM_MAIN TRUE + ALWAYS TRUE + DEPENDEES configure + COMMAND make clean -j + COMMAND rm -f ${BUILD_INFO_DIR}/${name}-build + WORKING_DIRECTORY ${source_dir} +) + +ExternalProject_Add_StepTargets(${name} clean) diff --git a/third-party/externals/lzma.cmake b/third-party/externals/lzma.cmake new file mode 100644 index 00000000000..2f54e73871f --- /dev/null +++ b/third-party/externals/lzma.cmake @@ -0,0 +1,38 @@ +# Copyright (c) 2019 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. + +set(name lzma) +set(source_dir ${CMAKE_CURRENT_BINARY_DIR}/${name}/source) +ExternalProject_Add( + ${name} + URL https://tukaani.org/xz/xz-5.2.4.tar.xz + URL_HASH MD5=003e4d0b1b1899fc6e3000b24feddf7c + DOWNLOAD_NAME lzma-5.2.4.tar.xz + PREFIX ${CMAKE_CURRENT_BINARY_DIR}/${name} + TMP_DIR ${BUILD_INFO_DIR} + STAMP_DIR ${BUILD_INFO_DIR} + DOWNLOAD_DIR ${DOWNLOAD_DIR} + SOURCE_DIR ${source_dir} + CONFIGURE_COMMAND + ${common_configure_envs} + ./configure ${common_configure_args} + --disable-shared --enable-static + BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} + BUILD_IN_SOURCE 1 + INSTALL_COMMAND make -s install -j${BUILDING_JOBS_NUM} PREFIX=${CMAKE_INSTALL_PREFIX} + LOG_BUILD 1 + LOG_INSTALL 1 +) + +ExternalProject_Add_Step(${name} clean + EXCLUDE_FROM_MAIN TRUE + ALWAYS TRUE + DEPENDEES configure + COMMAND make clean -j + COMMAND rm -f ${BUILD_INFO_DIR}/${name}-build + WORKING_DIRECTORY ${source_dir} +) + +ExternalProject_Add_StepTargets(${name} clean) diff --git a/third-party/externals/mstch.cmake b/third-party/externals/mstch.cmake new file mode 100644 index 00000000000..fc14e3c427e --- /dev/null +++ b/third-party/externals/mstch.cmake @@ -0,0 +1,36 @@ +# Copyright (c) 2019 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. + +set(name mstch) +set(source_dir ${CMAKE_CURRENT_BINARY_DIR}/${name}/source) +ExternalProject_Add( + ${name} + URL https://github.com/no1msd/mstch/archive/1.0.2.tar.gz + URL_HASH MD5=306e7fead7480884f698ab47a6082e18 + DOWNLOAD_NAME mstch-1.0.2.tar.gz + PREFIX ${CMAKE_CURRENT_BINARY_DIR}/${name} + TMP_DIR ${BUILD_INFO_DIR} + STAMP_DIR ${BUILD_INFO_DIR} + DOWNLOAD_DIR ${DOWNLOAD_DIR} + SOURCE_DIR ${source_dir} + CMAKE_ARGS + ${common_cmake_args} + BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} + BUILD_IN_SOURCE 1 + INSTALL_COMMAND make -s install -j${BUILDING_JOBS_NUM} + LOG_BUILD 1 + LOG_INSTALL 1 +) + +ExternalProject_Add_Step(${name} clean + EXCLUDE_FROM_MAIN TRUE + ALWAYS TRUE + DEPENDEES configure + COMMAND make clean -j + COMMAND rm -f ${BUILD_INFO_DIR}/${name}-build + WORKING_DIRECTORY ${source_dir} +) + +ExternalProject_Add_StepTargets(${name} clean) diff --git a/third-party/externals/openssl.cmake b/third-party/externals/openssl.cmake new file mode 100644 index 00000000000..472f71f95a1 --- /dev/null +++ b/third-party/externals/openssl.cmake @@ -0,0 +1,37 @@ +# Copyright (c) 2019 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. + +set(name openssl) +set(source_dir ${CMAKE_CURRENT_BINARY_DIR}/${name}/source) +ExternalProject_Add( + ${name} + URL https://github.com/openssl/openssl/archive/OpenSSL_1_1_1c.tar.gz + URL_HASH MD5=e54191af2dbef5f172ca5b7ceea08307 + DOWNLOAD_NAME openssl-1.1.1c.tar.gz + PREFIX ${CMAKE_CURRENT_BINARY_DIR}/${name} + TMP_DIR ${BUILD_INFO_DIR} + STAMP_DIR ${BUILD_INFO_DIR} + DOWNLOAD_DIR ${DOWNLOAD_DIR} + SOURCE_DIR ${source_dir} + CONFIGURE_COMMAND + ${common_configure_envs} + ./config no-shared threads --prefix=${CMAKE_INSTALL_PREFIX} + BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} + BUILD_IN_SOURCE 1 + INSTALL_COMMAND make -s install_sw -j${BUILDING_JOBS_NUM} + LOG_BUILD 1 + LOG_INSTALL 1 +) + +ExternalProject_Add_Step(${name} clean + EXCLUDE_FROM_MAIN TRUE + ALWAYS TRUE + DEPENDEES configure + COMMAND make clean -j + COMMAND rm -f ${BUILD_INFO_DIR}/${name}-build + WORKING_DIRECTORY ${source_dir} +) + +ExternalProject_Add_StepTargets(${name} clean) diff --git a/third-party/externals/pkg.m4 b/third-party/externals/pkg.m4 new file mode 100644 index 00000000000..d8549a47896 --- /dev/null +++ b/third-party/externals/pkg.m4 @@ -0,0 +1,343 @@ +# pkg.m4 - Macros to locate and utilise pkg-config. -*- Autoconf -*- +# serial 11 (pkg-config-0.29.1) + +dnl Copyright © 2004 Scott James Remnant . +dnl Copyright © 2012-2015 Dan Nicholson +dnl +dnl This program is free software; you can redistribute it and/or modify +dnl it under the terms of the GNU General Public License as published by +dnl the Free Software Foundation; either version 2 of the License, or +dnl (at your option) any later version. +dnl +dnl This program is distributed in the hope that it will be useful, but +dnl WITHOUT ANY WARRANTY; without even the implied warranty of +dnl MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +dnl General Public License for more details. +dnl +dnl You should have received a copy of the GNU General Public License +dnl along with this program; if not, write to the Free Software +dnl Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA +dnl 02111-1307, USA. +dnl +dnl As a special exception to the GNU General Public License, if you +dnl distribute this file as part of a program that contains a +dnl configuration script generated by Autoconf, you may include it under +dnl the same distribution terms that you use for the rest of that +dnl program. + +dnl PKG_PREREQ(MIN-VERSION) +dnl ----------------------- +dnl Since: 0.29 +dnl +dnl Verify that the version of the pkg-config macros are at least +dnl MIN-VERSION. Unlike PKG_PROG_PKG_CONFIG, which checks the user's +dnl installed version of pkg-config, this checks the developer's version +dnl of pkg.m4 when generating configure. +dnl +dnl To ensure that this macro is defined, also add: +dnl m4_ifndef([PKG_PREREQ], +dnl [m4_fatal([must install pkg-config 0.29 or later before running autoconf/autogen])]) +dnl +dnl See the "Since" comment for each macro you use to see what version +dnl of the macros you require. +m4_defun([PKG_PREREQ], +[m4_define([PKG_MACROS_VERSION], [0.29.1]) +m4_if(m4_version_compare(PKG_MACROS_VERSION, [$1]), -1, + [m4_fatal([pkg.m4 version $1 or higher is required but ]PKG_MACROS_VERSION[ found])]) +])dnl PKG_PREREQ + +dnl PKG_PROG_PKG_CONFIG([MIN-VERSION]) +dnl ---------------------------------- +dnl Since: 0.16 +dnl +dnl Search for the pkg-config tool and set the PKG_CONFIG variable to +dnl first found in the path. Checks that the version of pkg-config found +dnl is at least MIN-VERSION. If MIN-VERSION is not specified, 0.9.0 is +dnl used since that's the first version where most current features of +dnl pkg-config existed. +AC_DEFUN([PKG_PROG_PKG_CONFIG], +[m4_pattern_forbid([^_?PKG_[A-Z_]+$]) +m4_pattern_allow([^PKG_CONFIG(_(PATH|LIBDIR|SYSROOT_DIR|ALLOW_SYSTEM_(CFLAGS|LIBS)))?$]) +m4_pattern_allow([^PKG_CONFIG_(DISABLE_UNINSTALLED|TOP_BUILD_DIR|DEBUG_SPEW)$]) +AC_ARG_VAR([PKG_CONFIG], [path to pkg-config utility]) +AC_ARG_VAR([PKG_CONFIG_PATH], [directories to add to pkg-config's search path]) +AC_ARG_VAR([PKG_CONFIG_LIBDIR], [path overriding pkg-config's built-in search path]) + +if test "x$ac_cv_env_PKG_CONFIG_set" != "xset"; then + AC_PATH_TOOL([PKG_CONFIG], [pkg-config]) +fi +if test -n "$PKG_CONFIG"; then + _pkg_min_version=m4_default([$1], [0.9.0]) + AC_MSG_CHECKING([pkg-config is at least version $_pkg_min_version]) + if $PKG_CONFIG --atleast-pkgconfig-version $_pkg_min_version; then + AC_MSG_RESULT([yes]) + else + AC_MSG_RESULT([no]) + PKG_CONFIG="" + fi +fi[]dnl +])dnl PKG_PROG_PKG_CONFIG + +dnl PKG_CHECK_EXISTS(MODULES, [ACTION-IF-FOUND], [ACTION-IF-NOT-FOUND]) +dnl ------------------------------------------------------------------- +dnl Since: 0.18 +dnl +dnl Check to see whether a particular set of modules exists. Similar to +dnl PKG_CHECK_MODULES(), but does not set variables or print errors. +dnl +dnl Please remember that m4 expands AC_REQUIRE([PKG_PROG_PKG_CONFIG]) +dnl only at the first occurence in configure.ac, so if the first place +dnl it's called might be skipped (such as if it is within an "if", you +dnl have to call PKG_CHECK_EXISTS manually +AC_DEFUN([PKG_CHECK_EXISTS], +[AC_REQUIRE([PKG_PROG_PKG_CONFIG])dnl +if test -n "$PKG_CONFIG" && \ + AC_RUN_LOG([$PKG_CONFIG --exists --print-errors "$1"]); then + m4_default([$2], [:]) +m4_ifvaln([$3], [else + $3])dnl +fi]) + +dnl _PKG_CONFIG([VARIABLE], [COMMAND], [MODULES]) +dnl --------------------------------------------- +dnl Internal wrapper calling pkg-config via PKG_CONFIG and setting +dnl pkg_failed based on the result. +m4_define([_PKG_CONFIG], +[if test -n "$$1"; then + pkg_cv_[]$1="$$1" + elif test -n "$PKG_CONFIG"; then + PKG_CHECK_EXISTS([$3], + [pkg_cv_[]$1=`$PKG_CONFIG --[]$2 "$3" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes ], + [pkg_failed=yes]) + else + pkg_failed=untried +fi[]dnl +])dnl _PKG_CONFIG + +dnl _PKG_SHORT_ERRORS_SUPPORTED +dnl --------------------------- +dnl Internal check to see if pkg-config supports short errors. +AC_DEFUN([_PKG_SHORT_ERRORS_SUPPORTED], +[AC_REQUIRE([PKG_PROG_PKG_CONFIG]) +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi[]dnl +])dnl _PKG_SHORT_ERRORS_SUPPORTED + + +dnl PKG_CHECK_MODULES(VARIABLE-PREFIX, MODULES, [ACTION-IF-FOUND], +dnl [ACTION-IF-NOT-FOUND]) +dnl -------------------------------------------------------------- +dnl Since: 0.4.0 +dnl +dnl Note that if there is a possibility the first call to +dnl PKG_CHECK_MODULES might not happen, you should be sure to include an +dnl explicit call to PKG_PROG_PKG_CONFIG in your configure.ac +AC_DEFUN([PKG_CHECK_MODULES], +[AC_REQUIRE([PKG_PROG_PKG_CONFIG])dnl +AC_ARG_VAR([$1][_CFLAGS], [C compiler flags for $1, overriding pkg-config])dnl +AC_ARG_VAR([$1][_LIBS], [linker flags for $1, overriding pkg-config])dnl + +pkg_failed=no +AC_MSG_CHECKING([for $1]) + +_PKG_CONFIG([$1][_CFLAGS], [cflags], [$2]) +_PKG_CONFIG([$1][_LIBS], [libs], [$2]) + +m4_define([_PKG_TEXT], [Alternatively, you may set the environment variables $1[]_CFLAGS +and $1[]_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details.]) + +if test $pkg_failed = yes; then + AC_MSG_RESULT([no]) + _PKG_SHORT_ERRORS_SUPPORTED + if test $_pkg_short_errors_supported = yes; then + $1[]_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "$2" 2>&1` + else + $1[]_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "$2" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$$1[]_PKG_ERRORS" >&AS_MESSAGE_LOG_FD + + m4_default([$4], [AC_MSG_ERROR( +[Package requirements ($2) were not met: + +$$1_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +_PKG_TEXT])[]dnl + ]) +elif test $pkg_failed = untried; then + AC_MSG_RESULT([no]) + m4_default([$4], [AC_MSG_FAILURE( +[The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +_PKG_TEXT + +To get pkg-config, see .])[]dnl + ]) +else + $1[]_CFLAGS=$pkg_cv_[]$1[]_CFLAGS + $1[]_LIBS=$pkg_cv_[]$1[]_LIBS + AC_MSG_RESULT([yes]) + $3 +fi[]dnl +])dnl PKG_CHECK_MODULES + + +dnl PKG_CHECK_MODULES_STATIC(VARIABLE-PREFIX, MODULES, [ACTION-IF-FOUND], +dnl [ACTION-IF-NOT-FOUND]) +dnl --------------------------------------------------------------------- +dnl Since: 0.29 +dnl +dnl Checks for existence of MODULES and gathers its build flags with +dnl static libraries enabled. Sets VARIABLE-PREFIX_CFLAGS from --cflags +dnl and VARIABLE-PREFIX_LIBS from --libs. +dnl +dnl Note that if there is a possibility the first call to +dnl PKG_CHECK_MODULES_STATIC might not happen, you should be sure to +dnl include an explicit call to PKG_PROG_PKG_CONFIG in your +dnl configure.ac. +AC_DEFUN([PKG_CHECK_MODULES_STATIC], +[AC_REQUIRE([PKG_PROG_PKG_CONFIG])dnl +_save_PKG_CONFIG=$PKG_CONFIG +PKG_CONFIG="$PKG_CONFIG --static" +PKG_CHECK_MODULES($@) +PKG_CONFIG=$_save_PKG_CONFIG[]dnl +])dnl PKG_CHECK_MODULES_STATIC + + +dnl PKG_INSTALLDIR([DIRECTORY]) +dnl ------------------------- +dnl Since: 0.27 +dnl +dnl Substitutes the variable pkgconfigdir as the location where a module +dnl should install pkg-config .pc files. By default the directory is +dnl $libdir/pkgconfig, but the default can be changed by passing +dnl DIRECTORY. The user can override through the --with-pkgconfigdir +dnl parameter. +AC_DEFUN([PKG_INSTALLDIR], +[m4_pushdef([pkg_default], [m4_default([$1], ['${libdir}/pkgconfig'])]) +m4_pushdef([pkg_description], + [pkg-config installation directory @<:@]pkg_default[@:>@]) +AC_ARG_WITH([pkgconfigdir], + [AS_HELP_STRING([--with-pkgconfigdir], pkg_description)],, + [with_pkgconfigdir=]pkg_default) +AC_SUBST([pkgconfigdir], [$with_pkgconfigdir]) +m4_popdef([pkg_default]) +m4_popdef([pkg_description]) +])dnl PKG_INSTALLDIR + + +dnl PKG_NOARCH_INSTALLDIR([DIRECTORY]) +dnl -------------------------------- +dnl Since: 0.27 +dnl +dnl Substitutes the variable noarch_pkgconfigdir as the location where a +dnl module should install arch-independent pkg-config .pc files. By +dnl default the directory is $datadir/pkgconfig, but the default can be +dnl changed by passing DIRECTORY. The user can override through the +dnl --with-noarch-pkgconfigdir parameter. +AC_DEFUN([PKG_NOARCH_INSTALLDIR], +[m4_pushdef([pkg_default], [m4_default([$1], ['${datadir}/pkgconfig'])]) +m4_pushdef([pkg_description], + [pkg-config arch-independent installation directory @<:@]pkg_default[@:>@]) +AC_ARG_WITH([noarch-pkgconfigdir], + [AS_HELP_STRING([--with-noarch-pkgconfigdir], pkg_description)],, + [with_noarch_pkgconfigdir=]pkg_default) +AC_SUBST([noarch_pkgconfigdir], [$with_noarch_pkgconfigdir]) +m4_popdef([pkg_default]) +m4_popdef([pkg_description]) +])dnl PKG_NOARCH_INSTALLDIR + + +dnl PKG_CHECK_VAR(VARIABLE, MODULE, CONFIG-VARIABLE, +dnl [ACTION-IF-FOUND], [ACTION-IF-NOT-FOUND]) +dnl ------------------------------------------- +dnl Since: 0.28 +dnl +dnl Retrieves the value of the pkg-config variable for the given module. +AC_DEFUN([PKG_CHECK_VAR], +[AC_REQUIRE([PKG_PROG_PKG_CONFIG])dnl +AC_ARG_VAR([$1], [value of $3 for $2, overriding pkg-config])dnl + +_PKG_CONFIG([$1], [variable="][$3]["], [$2]) +AS_VAR_COPY([$1], [pkg_cv_][$1]) + +AS_VAR_IF([$1], [""], [$5], [$4])dnl +])dnl PKG_CHECK_VAR + +dnl PKG_WITH_MODULES(VARIABLE-PREFIX, MODULES, +dnl [ACTION-IF-FOUND],[ACTION-IF-NOT-FOUND], +dnl [DESCRIPTION], [DEFAULT]) +dnl ------------------------------------------ +dnl +dnl Prepare a "--with-" configure option using the lowercase +dnl [VARIABLE-PREFIX] name, merging the behaviour of AC_ARG_WITH and +dnl PKG_CHECK_MODULES in a single macro. +AC_DEFUN([PKG_WITH_MODULES], +[ +m4_pushdef([with_arg], m4_tolower([$1])) + +m4_pushdef([description], + [m4_default([$5], [build with ]with_arg[ support])]) + +m4_pushdef([def_arg], [m4_default([$6], [auto])]) +m4_pushdef([def_action_if_found], [AS_TR_SH([with_]with_arg)=yes]) +m4_pushdef([def_action_if_not_found], [AS_TR_SH([with_]with_arg)=no]) + +m4_case(def_arg, + [yes],[m4_pushdef([with_without], [--without-]with_arg)], + [m4_pushdef([with_without],[--with-]with_arg)]) + +AC_ARG_WITH(with_arg, + AS_HELP_STRING(with_without, description[ @<:@default=]def_arg[@:>@]),, + [AS_TR_SH([with_]with_arg)=def_arg]) + +AS_CASE([$AS_TR_SH([with_]with_arg)], + [yes],[PKG_CHECK_MODULES([$1],[$2],$3,$4)], + [auto],[PKG_CHECK_MODULES([$1],[$2], + [m4_n([def_action_if_found]) $3], + [m4_n([def_action_if_not_found]) $4])]) + +m4_popdef([with_arg]) +m4_popdef([description]) +m4_popdef([def_arg]) + +])dnl PKG_WITH_MODULES + +dnl PKG_HAVE_WITH_MODULES(VARIABLE-PREFIX, MODULES, +dnl [DESCRIPTION], [DEFAULT]) +dnl ----------------------------------------------- +dnl +dnl Convenience macro to trigger AM_CONDITIONAL after PKG_WITH_MODULES +dnl check._[VARIABLE-PREFIX] is exported as make variable. +AC_DEFUN([PKG_HAVE_WITH_MODULES], +[ +PKG_WITH_MODULES([$1],[$2],,,[$3],[$4]) + +AM_CONDITIONAL([HAVE_][$1], + [test "$AS_TR_SH([with_]m4_tolower([$1]))" = "yes"]) +])dnl PKG_HAVE_WITH_MODULES + +dnl PKG_HAVE_DEFINE_WITH_MODULES(VARIABLE-PREFIX, MODULES, +dnl [DESCRIPTION], [DEFAULT]) +dnl ------------------------------------------------------ +dnl +dnl Convenience macro to run AM_CONDITIONAL and AC_DEFINE after +dnl PKG_WITH_MODULES check. HAVE_[VARIABLE-PREFIX] is exported as make +dnl and preprocessor variable. +AC_DEFUN([PKG_HAVE_DEFINE_WITH_MODULES], +[ +PKG_HAVE_WITH_MODULES([$1],[$2],[$3],[$4]) + +AS_IF([test "$AS_TR_SH([with_]m4_tolower([$1]))" = "yes"], + [AC_DEFINE([HAVE_][$1], 1, [Enable ]m4_tolower([$1])[ support])]) +])dnl PKG_HAVE_DEFINE_WITH_MODULES diff --git a/third-party/externals/pkgconf.cmake b/third-party/externals/pkgconf.cmake new file mode 100644 index 00000000000..a4e8cb4aa96 --- /dev/null +++ b/third-party/externals/pkgconf.cmake @@ -0,0 +1,20 @@ +# Copyright (c) 2019 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. + +ExternalProject_Add( + pkgconf + URL https://github.com/pkgconf/pkgconf/archive/pkgconf-1.6.1.tar.gz + URL_HASH MD5=ba6bda0fca2010a05de3ccda3fcd5b50 + PREFIX ${CMAKE_CURRENT_BINARY_DIR}/pkgconf + TMP_DIR ${BUILD_INFO_DIR} + STAMP_DIR ${BUILD_INFO_DIR} + DOWNLOAD_DIR ${DOWNLOAD_DIR} + SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/pkgconf/source + CMAKE_ARGS ${common_cmake_args} + BUILD_IN_SOURCE 1 + BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} + INSTALL_COMMAND make -s -j${BUILDING_JOBS_NUM} install +) + diff --git a/third-party/externals/proxygen.cmake b/third-party/externals/proxygen.cmake new file mode 100644 index 00000000000..95682f9a9cf --- /dev/null +++ b/third-party/externals/proxygen.cmake @@ -0,0 +1,59 @@ +# Copyright (c) 2019 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. + +set(name proxygen) +set(source_dir ${CMAKE_CURRENT_BINARY_DIR}/${name}/source) + +set(ProxygenLibs "-lssl -lcrypto -ldl -lglog -lunwind ${extra_link_libs}") +if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") + # clang requires explicitly linking to libatomic + set(ProxygenLibs "${ProxygenLibs} -latomic") +endif() + +ExternalProject_Add( + ${name} + URL https://github.com/facebook/proxygen/archive/v2018.08.20.00.tar.gz + URL_HASH MD5=cc71ffdf502355b05451bcd81478f3d7 + DOWNLOAD_NAME proxygen-2018-08-20.tar.gz + PREFIX ${CMAKE_CURRENT_BINARY_DIR}/${name} + TMP_DIR ${BUILD_INFO_DIR} + STAMP_DIR ${BUILD_INFO_DIR} + DOWNLOAD_DIR ${DOWNLOAD_DIR} + SOURCE_DIR ${source_dir} + PATCH_COMMAND patch -p0 < ${CMAKE_SOURCE_DIR}/patches/proxygen-2018-08-20.patch + CONFIGURE_COMMAND "" + BUILD_COMMAND env PATH=${BUILDING_PATH} make -s -j${BUILDING_JOBS_NUM} -C proxygen + BUILD_IN_SOURCE 1 + INSTALL_COMMAND env PATH=${BUILDING_PATH} make -s -j${BUILDING_JOBS_NUM} install -C proxygen + LOG_CONFIGURE TRUE + LOG_BUILD TRUE + LOG_INSTALL TRUE + LOG_MERGED_STDOUTERR TRUE +) + +ExternalProject_Add_Step(proxygen mannual-configure + DEPENDEES download update patch configure + DEPENDERS build install + COMMAND env PATH=${BUILDING_PATH} ACLOCAL_PATH=${ACLOCAL_PATH} autoreconf -if + COMMAND + ${common_configure_envs} + "LIBS=${ProxygenLibs}" + ./configure + ${common_configure_args} + --disable-shared + --enable-static + WORKING_DIRECTORY ${source_dir}/proxygen +) + +ExternalProject_Add_Step(${name} clean + EXCLUDE_FROM_MAIN TRUE + ALWAYS TRUE + DEPENDEES mannual-configure + COMMAND make clean -j + COMMAND rm -f ${BUILD_INFO_DIR}/${name}-build + WORKING_DIRECTORY ${source_dir}/proxygen +) + +ExternalProject_Add_StepTargets(${name} clean) diff --git a/third-party/externals/rocksdb.cmake b/third-party/externals/rocksdb.cmake new file mode 100644 index 00000000000..c99be0ed599 --- /dev/null +++ b/third-party/externals/rocksdb.cmake @@ -0,0 +1,50 @@ +# Copyright (c) 2019 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. + +set(name rocksdb) +set(source_dir ${CMAKE_CURRENT_BINARY_DIR}/${name}/source) +set(ROCKSDB_CXX_FLAGS "-Wno-pessimizing-move -Wno-redundant-move -Wno-deprecated-copy -Wno-error=shadow -Wno-error=sign-compare") +ExternalProject_Add( + ${name} + URL https://github.com/facebook/rocksdb/archive/v5.15.10.tar.gz + URL_HASH MD5=5b1c1fa7ff4756218514205238d8900d + DOWNLOAD_NAME rocksdb-5.15.10.tar.gz + PREFIX ${CMAKE_CURRENT_BINARY_DIR}/${name} + TMP_DIR ${BUILD_INFO_DIR} + STAMP_DIR ${BUILD_INFO_DIR} + DOWNLOAD_DIR ${DOWNLOAD_DIR} + SOURCE_DIR ${source_dir} + UPDATE_COMMAND "" + CMAKE_ARGS + ${common_cmake_args} + -DPORTABLE=ON + -DWITH_SNAPPY=ON + -DWITH_ZSTD=ON + -DWITH_ZLIB=ON + -DWITH_JEMALLOC=OFF + -DWITH_GFLAGS=OFF + -DWITH_TESTS=OFF + -DWITH_TOOLS=OFF + -DFAIL_ON_WARNINGS=OFF + -DCMAKE_BUILD_TYPE=Release +#-DCMAKE_CXX_FLAGS:STRING=${ROCKSDB_CXX_FLAGS} + PATCH_COMMAND patch CMakeLists.txt ${CMAKE_SOURCE_DIR}/patches/rocksdb-5.15.10.patch + BUILD_IN_SOURCE 1 + BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} VERBOSE=1 + INSTALL_COMMAND make -s install -j${BUILDING_JOBS_NUM} + LOG_BUILD 1 + LOG_INSTALL 1 +) + +ExternalProject_Add_Step(${name} clean + EXCLUDE_FROM_MAIN TRUE + ALWAYS TRUE + DEPENDEES configure + COMMAND make clean -j + COMMAND rm -f ${BUILD_INFO_DIR}/${name}-build + WORKING_DIRECTORY ${source_dir} +) + +ExternalProject_Add_StepTargets(${name} clean) diff --git a/third-party/externals/s2geometry.cmake b/third-party/externals/s2geometry.cmake new file mode 100644 index 00000000000..2ce58b5c6fe --- /dev/null +++ b/third-party/externals/s2geometry.cmake @@ -0,0 +1,42 @@ +# Copyright (c) 2019 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. + +set(name s2geometry) +set(source_dir ${CMAKE_CURRENT_BINARY_DIR}/${name}/source) +ExternalProject_Add( + ${name} + URL https://github.com/google/s2geometry/archive/v0.9.0.tar.gz + URL_HASH MD5=293552c7646193b8b4a01556808fe155 + DOWNLOAD_NAME ${name}-0.9.0.tar.gz + PREFIX ${CMAKE_CURRENT_BINARY_DIR}/${name} + TMP_DIR ${BUILD_INFO_DIR} + STAMP_DIR ${BUILD_INFO_DIR} + DOWNLOAD_DIR ${DOWNLOAD_DIR} + SOURCE_DIR ${source_dir} + CMAKE_ARGS + ${common_cmake_args} + -DCMAKE_BUILD_TYPE=Release + -DBUILD_EXAMPLES=OFF + -DBUILD_SHARED_LIBS=OFF + + BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} + BUILD_IN_SOURCE 1 + INSTALL_COMMAND make -s -j${BUILDING_JOBS_NUM} install + LOG_CONFIGURE TRUE + LOG_BUILD TRUE + LOG_INSTALL TRUE + LOG_MERGED_STDOUTERR TRUE +) + +ExternalProject_Add_Step(${name} clean + EXCLUDE_FROM_MAIN TRUE + ALWAYS TRUE + DEPENDEES configure + COMMAND make clean -j + COMMAND rm -f ${BUILD_INFO_DIR}/${name}-build + WORKING_DIRECTORY ${source_dir} +) + +ExternalProject_Add_StepTargets(${name} clean) diff --git a/third-party/externals/snappy.cmake b/third-party/externals/snappy.cmake new file mode 100644 index 00000000000..12609b47248 --- /dev/null +++ b/third-party/externals/snappy.cmake @@ -0,0 +1,39 @@ +# Copyright (c) 2019 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. + +set(name snappy) +set(source_dir ${CMAKE_CURRENT_BINARY_DIR}/${name}/source) +ExternalProject_Add( + ${name} + URL https://github.com/google/snappy/archive/1.1.7.tar.gz + URL_HASH MD5=ee9086291c9ae8deb4dac5e0b85bf54a + DOWNLOAD_NAME snappy-1.1.7.tar.gz + PREFIX ${CMAKE_CURRENT_BINARY_DIR}/${name} + TMP_DIR ${BUILD_INFO_DIR} + STAMP_DIR ${BUILD_INFO_DIR} + DOWNLOAD_DIR ${DOWNLOAD_DIR} + SOURCE_DIR ${source_dir} + UPDATE_COMMAND "" + CMAKE_ARGS + ${common_cmake_args} + -DCMAKE_BUILD_TYPE=Release + -DSNAPPY_BUILD_TESTS=OFF + BUILD_IN_SOURCE 1 + BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} + INSTALL_COMMAND make -s install -j${BUILDING_JOBS_NUM} + LOG_BUILD 1 + LOG_INSTALL 1 +) + +ExternalProject_Add_Step(${name} clean + EXCLUDE_FROM_MAIN TRUE + ALWAYS TRUE + DEPENDEES configure + COMMAND make clean -j + COMMAND rm -f ${BUILD_INFO_DIR}/${name}-build + WORKING_DIRECTORY ${source_dir} +) + +ExternalProject_Add_StepTargets(${name} clean) diff --git a/third-party/externals/wangle.cmake b/third-party/externals/wangle.cmake new file mode 100644 index 00000000000..949741b72aa --- /dev/null +++ b/third-party/externals/wangle.cmake @@ -0,0 +1,51 @@ +# Copyright (c) 2019 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. + +set(name wangle) +set(source_dir ${CMAKE_CURRENT_BINARY_DIR}/${name}/source) +ExternalProject_Add( + ${name} + URL https://github.com/facebook/wangle/archive/v2018.08.20.00.tar.gz + URL_HASH MD5=b20856081c1d21c1a033f9ca161398c5 + DOWNLOAD_NAME wangle-2018-08-20.tar.gz + PREFIX ${CMAKE_CURRENT_BINARY_DIR}/${name} + TMP_DIR ${BUILD_INFO_DIR} + STAMP_DIR ${BUILD_INFO_DIR} + DOWNLOAD_DIR ${DOWNLOAD_DIR} + SOURCE_DIR ${source_dir} + CONFIGURE_COMMAND "" + BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} -C wangle + BUILD_IN_SOURCE 1 + INSTALL_COMMAND make -s -j${BUILDING_JOBS_NUM} install -C wangle + LOG_CONFIGURE TRUE + LOG_BUILD TRUE + LOG_INSTALL TRUE + LOG_MERGED_STDOUTERR TRUE +) + +ExternalProject_Add_Step(${name} mannual-configure + DEPENDEES download update patch + DEPENDERS build + COMMAND ${CMAKE_COMMAND} + ${common_cmake_args} + -DBoost_NO_SYSTEM_PATHS=OFF + -DBUILD_TESTS=OFF + -DCMAKE_EXE_LINKER_FLAGS=-latomic + -DCMAKE_SHARED_LINKER_FLAGS=-latomic + -D_OPENSSL_LIBDIR=${CMAKE_INSTALL_PREFIX}/lib64 + . + WORKING_DIRECTORY /wangle +) + +ExternalProject_Add_Step(${name} clean + EXCLUDE_FROM_MAIN TRUE + ALWAYS TRUE + DEPENDEES configure + COMMAND make clean -j + COMMAND rm -f ${BUILD_INFO_DIR}/${name}-build + WORKING_DIRECTORY /wangle +) + +ExternalProject_Add_StepTargets(${name} clean) diff --git a/third-party/externals/zlib.cmake b/third-party/externals/zlib.cmake new file mode 100644 index 00000000000..125ca8199d2 --- /dev/null +++ b/third-party/externals/zlib.cmake @@ -0,0 +1,41 @@ +# Copyright (c) 2019 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. + +set(name zlib) +set(source_dir ${CMAKE_CURRENT_BINARY_DIR}/${name}/source) +ExternalProject_Add( + ${name} + URL https://github.com/madler/zlib/archive/v1.2.11.tar.gz + URL_HASH MD5=0095d2d2d1f3442ce1318336637b695f + DOWNLOAD_NAME zlib-1.2.11.tar.gz + PREFIX ${CMAKE_CURRENT_BINARY_DIR}/${name} + TMP_DIR ${BUILD_INFO_DIR} + STAMP_DIR ${BUILD_INFO_DIR} + DOWNLOAD_DIR ${DOWNLOAD_DIR} + SOURCE_DIR ${source_dir} + CONFIGURE_COMMAND + "env" + "CC=${CMAKE_C_COMPILER}" + "CFLAGS=${CMAKE_C_FLAGS} -fPIC -O2" + "CPPFLAGS=-isystem ${CMAKE_INSTALL_PREFIX}/include" + "PATH=${BUILDING_PATH}" + ./configure --prefix=${CMAKE_INSTALL_PREFIX} --static + BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} + BUILD_IN_SOURCE 1 + INSTALL_COMMAND make -s install -j${BUILDING_JOBS_NUM} PREFIX=${CMAKE_INSTALL_PREFIX} + LOG_BUILD 1 + LOG_INSTALL 1 +) + +ExternalProject_Add_Step(${name} clean + EXCLUDE_FROM_MAIN TRUE + ALWAYS TRUE + DEPENDEES configure + COMMAND make clean -j + COMMAND rm -f ${BUILD_INFO_DIR}/${name}-build + WORKING_DIRECTORY ${source_dir} +) + +ExternalProject_Add_StepTargets(${name} clean) diff --git a/third-party/externals/zstd.cmake b/third-party/externals/zstd.cmake new file mode 100644 index 00000000000..eee90cc40d3 --- /dev/null +++ b/third-party/externals/zstd.cmake @@ -0,0 +1,39 @@ +# Copyright (c) 2019 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. + +# TODO Upgrade to take advantage of optimization +set(name zstd) +set(source_dir ${CMAKE_CURRENT_BINARY_DIR}/${name}/source) +set(MakeEnvs "env" "CFLAGS=-fPIC") +ExternalProject_Add( + ${name} + URL https://github.com/facebook/zstd/archive/v1.3.4.tar.gz + URL_HASH MD5=10bf0353e3dedd8bae34a188c25d4261 + DOWNLOAD_NAME zstd-1.3.4.tar.gz + PREFIX ${CMAKE_CURRENT_BINARY_DIR}/${name} + TMP_DIR ${BUILD_INFO_DIR} + STAMP_DIR ${BUILD_INFO_DIR} + DOWNLOAD_DIR ${DOWNLOAD_DIR} + SOURCE_DIR ${source_dir} + CONFIGURE_COMMAND "" + BUILD_COMMAND + "${MakeEnvs}" + make -e -s -j${BUILDING_JOBS_NUM} + BUILD_IN_SOURCE 1 + INSTALL_COMMAND make -s install -j${BUILDING_JOBS_NUM} PREFIX=${CMAKE_INSTALL_PREFIX} + LOG_BUILD 1 + LOG_INSTALL 1 +) + +ExternalProject_Add_Step(${name} clean + EXCLUDE_FROM_MAIN TRUE + ALWAYS TRUE + DEPENDEES configure + COMMAND make clean -j + COMMAND rm -f ${BUILD_INFO_DIR}/${name}-build + WORKING_DIRECTORY ${source_dir} +) + +ExternalProject_Add_StepTargets(${name} clean) diff --git a/third-party/patches/fbthrift-2018-08-20.patch b/third-party/patches/fbthrift-2018-08-20.patch new file mode 100644 index 00000000000..a908b75e15f --- /dev/null +++ b/third-party/patches/fbthrift-2018-08-20.patch @@ -0,0 +1,192 @@ +diff --git a/thrift/compiler/parse/thriftl.ll b/thrift/compiler/parse/thriftl.ll +index 703ea6e19..b33be69d1 100644 +--- a/thrift/compiler/parse/thriftl.ll ++++ b/thrift/compiler/parse/thriftl.ll +@@ -312,27 +312,23 @@ st_identifier ([a-zA-Z-][\.a-zA-Z_0-9-]*) + } + + {identifier} { +- const char *val = strdup(yytext); +- return apache::thrift::yy::parser::make_tok_identifier(val); ++ return apache::thrift::yy::parser::make_tok_identifier(std::string{yytext}); + } + + {st_identifier} { +- const char *val = strdup(yytext); +- return apache::thrift::yy::parser::make_tok_st_identifier(val); ++ return apache::thrift::yy::parser::make_tok_st_identifier(std::string{yytext}); + } + + {dliteral} { +- char *val = strdup(yytext+1); +- val[strlen(val)-1] = '\0'; +- const char *const_val = val; +- return apache::thrift::yy::parser::make_tok_literal(const_val); ++ std::string val{yytext + 1}; ++ val = val.substr(0, val.length() - 1); ++ return apache::thrift::yy::parser::make_tok_literal(std::move(val)); + } + + {sliteral} { +- char *val = strdup(yytext+1); +- val[strlen(val)-1] = '\0'; +- const char *const_val = val; +- return apache::thrift::yy::parser::make_tok_literal(const_val); ++ std::string val{yytext + 1}; ++ val = val.substr(0, val.length() - 1); ++ return apache::thrift::yy::parser::make_tok_literal(std::move(val)); + } + + {doctext} { +diff --git a/thrift/compiler/parse/thrifty.yy b/thrift/compiler/parse/thrifty.yy +index d4b65bbe6..988956569 100644 +--- a/thrift/compiler/parse/thrifty.yy ++++ b/thrift/compiler/parse/thrifty.yy +@@ -67,7 +67,7 @@ int32_t y_enum_val = -1; + int g_arglist = 0; + const int struct_is_struct = 0; + const int struct_is_union = 1; +-char* y_enum_name = nullptr; ++const char* y_enum_name = nullptr; + + // Define an enum class for all types that have lineno embedded. + enum class LineType { +@@ -125,10 +125,10 @@ class parsing_driver; + /** + * Strings identifier + */ +-%token tok_identifier +-%token tok_literal +-%token tok_doctext +-%token tok_st_identifier ++%token tok_identifier ++%token tok_literal ++%token tok_doctext ++%token tok_st_identifier + + /** + * Constant values +@@ -242,7 +242,7 @@ class parsing_driver; + %type TypeAnnotations + %type TypeAnnotationList + %type TypeAnnotation +-%type TypeAnnotationValue ++%type TypeAnnotationValue + %type FunctionAnnotations + + %type Field +@@ -287,7 +287,7 @@ class parsing_driver; + %type Oneway + + %type CaptureDocText +-%type IntOrLiteral ++%type IntOrLiteral + + %% + +@@ -602,7 +602,7 @@ Enum: + tok_identifier + { + assert(y_enum_name == nullptr); +- y_enum_name = $3; ++ y_enum_name = $3.c_str(); + } + "{" EnumDefList "}" TypeAnnotations + { +@@ -666,7 +666,7 @@ EnumValue: + { + driver.debug("EnumValue -> tok_identifier = tok_int_constant"); + if ($3 < 0 && !driver.params.allow_neg_enum_vals) { +- driver.warning(1, "Negative value supplied for enum %s.", $1); ++ driver.warning(1, "Negative value supplied for enum %s.", $1.c_str()); + } + if ($3 < INT32_MIN || $3 > INT32_MAX) { + // Note: this used to be just a warning. However, since thrift always +@@ -674,7 +674,7 @@ EnumValue: + // I doubt this will affect many people, but users who run into this + // will have to update their thrift files to manually specify the + // truncated i32 value that thrift has always been using anyway. +- driver.failure("64-bit value supplied for enum %s will be truncated.", $1); ++ driver.failure("64-bit value supplied for enum %s will be truncated.", $1.c_str()); + } + y_enum_val = $3; + $$ = new t_enum_value($1, y_enum_val); +@@ -685,7 +685,7 @@ EnumValue: + { + driver.debug("EnumValue -> tok_identifier"); + if (y_enum_val == INT32_MAX) { +- driver.failure("enum value overflow at enum %s", $1); ++ driver.failure("enum value overflow at enum %s", $1.c_str()); + } + $$ = new t_enum_value($1); + +@@ -754,7 +754,7 @@ ConstValue: + $$ = new t_const_value(*const_value); + } else { + if (driver.mode == apache::thrift::parsing_mode::PROGRAM) { +- driver.warning(1, "Constant strings should be quoted: %s", $1); ++ driver.warning(1, "Constant strings should be quoted: %s", $1.c_str()); + } + $$ = new t_const_value($1); + } +@@ -878,13 +878,13 @@ Xception: + + if (!$$->has_field_named(v.c_str())) { + driver.failure("member specified as exception 'message' should be a valid" +- " struct member, '%s' in '%s' is not", v.c_str(), $3); ++ " struct member, '%s' in '%s' is not", v.c_str(), $3.c_str()); + } + + auto field = $$->get_field_named(v.c_str()); + if (!field->get_type()->is_string()) { + driver.failure("member specified as exception 'message' should be of type " +- "STRING, '%s' in '%s' is not", v.c_str(), $3); ++ "STRING, '%s' in '%s' is not", v.c_str(), $3.c_str()); + } + } + +@@ -929,7 +929,7 @@ Extends: + $$ = driver.scope_cache->get_service(driver.program->get_name() + "." + $2); + } + if ($$ == NULL) { +- driver.yyerror("Service \"%s\" has not been defined.", $2); ++ driver.yyerror("Service \"%s\" has not been defined.", $2.c_str()); + driver.end_parsing(); + } + } +@@ -1092,7 +1092,7 @@ Field: + { + driver.debug("tok_int_constant : Field -> FieldType tok_identifier"); + if ($2.auto_assigned) { +- driver.warning(1, "No field key specified for %s, resulting protocol may have conflicts or not be backwards compatible!", $5); ++ driver.warning(1, "No field key specified for %s, resulting protocol may have conflicts or not be backwards compatible!", $5.c_str()); + if (driver.params.strict >= 192) { + driver.yyerror("Implicit field keys are deprecated and not allowed with -strict"); + driver.end_parsing(); +@@ -1477,7 +1477,7 @@ TypeAnnotationValue: + | + { + driver.debug("TypeAnnotationValue ->"); +- $$ = strdup("1"); ++ $$ = "1"; + } + + FunctionAnnotations: +@@ -1519,7 +1519,7 @@ IntOrLiteral: + char buf[21]; // max len of int64_t as string + null terminator + driver.debug("IntOrLiteral -> tok_bool_constant"); + sprintf(buf, "%" PRIi64, $1); +- $$ = strdup(buf); ++ $$ = buf; + } + | + tok_int_constant +@@ -1527,7 +1527,7 @@ IntOrLiteral: + char buf[21]; // max len of int64_t as string + null terminator + driver.debug("IntOrLiteral -> tok_int_constant"); + sprintf(buf, "%" PRIi64, $1); +- $$ = strdup(buf); ++ $$ = buf; + } + + %% diff --git a/third-party/patches/proxygen-2018-08-20.patch b/third-party/patches/proxygen-2018-08-20.patch new file mode 100644 index 00000000000..077f78c06dd --- /dev/null +++ b/third-party/patches/proxygen-2018-08-20.patch @@ -0,0 +1,13 @@ +--- proxygen/configure.ac 2018-08-20 05:26:51.000000000 +0800 ++++ proxygen/configure.ac.new 2019-11-09 21:20:51.597440341 +0800 +@@ -205,8 +205,8 @@ + AC_MSG_ERROR([Please install gperf first.]) + fi + +-LIBS="$LIBS $BOOST_LDFLAGS -lpthread -pthread -lfolly -lglog" +-LIBS="$LIBS -ldouble-conversion -lboost_system -lboost_thread" ++LIBS="$LIBS $BOOST_LDFLAGS -lpthread -pthread" ++LIBS="$LIBS -ldouble-conversion -lboost_context -lboost_system -lboost_thread" + + AM_CONDITIONAL([HAVE_STD_THREAD], [test "$ac_cv_header_features" = "yes"]) + AM_CONDITIONAL([HAVE_X86_64], [test "$build_cpu" = "x86_64"]) diff --git a/third-party/patches/rocksdb-5.15.10.patch b/third-party/patches/rocksdb-5.15.10.patch new file mode 100644 index 00000000000..3ff0fd79656 --- /dev/null +++ b/third-party/patches/rocksdb-5.15.10.patch @@ -0,0 +1,5 @@ +448,449d447 +< add_subdirectory(third-party/gtest-1.7.0/fused-src/gtest) +< +830a829 +> add_subdirectory(third-party/gtest-1.7.0/fused-src/gtest) From 809df1c4d69b7bb44f985f8d530d7eb1dccc5141 Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Wed, 27 Nov 2019 18:45:32 +0800 Subject: [PATCH 02/80] Fix krb5 to depend on bison --- CMakeLists.txt | 2 +- third-party/CMakeLists.txt | 4 ++-- third-party/build-cmake.sh | 9 ++++----- third-party/build-third-party.sh | 31 ++++++++++++++++++++++++++++--- 4 files changed, 35 insertions(+), 11 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 23b6ff6bbe3..eded95aef02 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,7 +18,7 @@ # ENABLE_NATIVE -- Build native client # ENABLE_TESTING -- Build unit test # -cmake_minimum_required(VERSION 3.8.0) +cmake_minimum_required(VERSION 3.5.0) project("Nebula Graph" C CXX) diff --git a/third-party/CMakeLists.txt b/third-party/CMakeLists.txt index eed6d621436..0904feceddc 100644 --- a/third-party/CMakeLists.txt +++ b/third-party/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.8.0) +cmake_minimum_required(VERSION 3.5.0) project("Nebula Third Party" C CXX) @@ -190,7 +190,7 @@ maybe_add_dependencies(glog gflags) maybe_add_dependencies(s2geometry googletest glog openssl) -maybe_add_dependencies(krb5 keyutils openssl gettext) +maybe_add_dependencies(krb5 keyutils openssl gettext bison) maybe_add_dependencies(folly glog boost double-conversion openssl libevent lzma zstd snappy libunwind libaio) maybe_add_dependencies(wangle folly) maybe_add_dependencies(fbthrift folly krb5 bison flex mstch zlib zstd wangle fatal) diff --git a/third-party/build-cmake.sh b/third-party/build-cmake.sh index 52a4af06604..79a2e1231ed 100755 --- a/third-party/build-cmake.sh +++ b/third-party/build-cmake.sh @@ -16,7 +16,7 @@ then fi archive=cmake-3.15.5-Linux-x86_64.sh -url=https://github.com/Kitware/CMake/releases/download/v3.15.5/cmake-3.15.5-Linux-x86_64.sh +url=https://cmake.org/files/v3.15/$archive prefix=`pwd`/cmake-3.15.5 if [[ -n $1 ]] @@ -29,7 +29,7 @@ then checksum=$(md5sum $archive | cut -d ' ' -f 1) fi -if [[ ! $checksum = f73d4daf34cb5e5119ee95c53696f322 ]] +if [[ ! $checksum = 35d56e9c27b4fd2819a11c29320c655a ]] then hash wget &> /dev/null && download_cmd="wget -c" hash axel &> /dev/null && download_cmd="axel -a -n 16" @@ -39,7 +39,7 @@ then exit 1; fi - echo "Downloading...please install 'axel' and retry if too slow." + echo "Downloading with $download_cmd..." if ! bash -c "$download_cmd $url" then echo "Download failed." @@ -53,5 +53,4 @@ yes no EOF -echo "cmake has been installed to prefix=$prefix" -echo "You could invoke via \`PATH=\$prefix/bin:\$PATH cmake'" +echo "CMake has been installed to prefix=$prefix" diff --git a/third-party/build-third-party.sh b/third-party/build-third-party.sh index f3d11aa5cd7..0915c17ca79 100755 --- a/third-party/build-third-party.sh +++ b/third-party/build-third-party.sh @@ -8,6 +8,8 @@ # Use CC and CXX environment variables to use custom compilers. +start_time=$(date +%s) + # Always use bash shell=$(basename $(readlink /proc/$$/exe)) if [ ! x$shell = x"bash" ] @@ -16,7 +18,7 @@ then exit $? fi -# cmake checking +# CMake and GCC version checking function version_cmp { mapfile -t left < <( echo $1 | tr . '\n' ) mapfile -t right < <( echo $2 | tr . '\n') @@ -36,7 +38,7 @@ function version_cmp { function check_cmake { hash cmake &> /dev/null || { echo "No cmake found." 1>&2; exit 1; } local cmake_version=$(cmake --version | head -1 | cut -d ' ' -f 3) - least_cmake_version=3.8.0 + local least_cmake_version=3.8.0 if [[ $(version_cmp $cmake_version $least_cmake_version) -lt 0 ]] then echo "cmake $least_cmake_version or later required" 1>&2 @@ -44,8 +46,23 @@ function check_cmake { fi } -check_cmake +function check_cxx { + # TODO To consider clang++ + local cxx_cmd + hash g++ &> /dev/null && cxx_cmd=g++ + [[ -n $CXX ]] && cxx_cmd=$CXX + [[ -z $cxx_cmd ]] && { echo "No C++ compiler found" 1>&2; exit 1; } + cxx_version=$($cxx_cmd -dumpfullversion) + local least_cxx_version=9.1.1 + if [[ $(version_cmp $cxx_version $least_cxx_version) -lt 0 ]] + then + echo "g++ $least_cxx_version or higher required, but you have $cxx_version" 1>&2 + exit 1 + fi +} +check_cmake +check_cxx # Exit on any failure here after set -e @@ -86,7 +103,10 @@ then # Resort to the builtin download method of cmake on failure echo "Download from $source_url failed." 1>&2 else + echo "Source of third party was downdloaded to $build_root" + echo -n "Extracting into $download_dir..." tar --skip-old-files -xzf $source_tar_name + echo "done" fi else tar --skip-old-files -xzf $source_tar_name @@ -102,3 +122,8 @@ cmake -DDOWNLOAD_DIR=$download_dir \ $source_dir make +end_time=$(date +%s) + +echo +echo "Third parties have been successfully installed to $install_dir" +echo "$((end_time - start_time)) seconds been taken." From f1e87e9b3a7b984c056c872b2dc93234644f2ed9 Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Thu, 28 Nov 2019 10:45:24 +0800 Subject: [PATCH 03/80] Fixed required GCC version --- third-party/CMakeLists.txt | 2 +- third-party/build-third-party.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/third-party/CMakeLists.txt b/third-party/CMakeLists.txt index 0904feceddc..8d9b30827ec 100644 --- a/third-party/CMakeLists.txt +++ b/third-party/CMakeLists.txt @@ -146,7 +146,7 @@ if (NOT FLEX_FOUND) list(APPEND ALL_TARGETS flex) endif() -find_package(GPERF 2.8 QUIET REQUIRED) +find_package(GPERF 2.8 QUIET) if (NOT GPERF_FOUND) list(APPEND ALL_TARGETS gperf) endif() diff --git a/third-party/build-third-party.sh b/third-party/build-third-party.sh index 0915c17ca79..dcbb9a88589 100755 --- a/third-party/build-third-party.sh +++ b/third-party/build-third-party.sh @@ -53,7 +53,7 @@ function check_cxx { [[ -n $CXX ]] && cxx_cmd=$CXX [[ -z $cxx_cmd ]] && { echo "No C++ compiler found" 1>&2; exit 1; } cxx_version=$($cxx_cmd -dumpfullversion) - local least_cxx_version=9.1.1 + local least_cxx_version=7.0.0 if [[ $(version_cmp $cxx_version $least_cxx_version) -lt 0 ]] then echo "g++ $least_cxx_version or higher required, but you have $cxx_version" 1>&2 From 161191540c55abad249065e865d98609b7602822 Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Thu, 28 Nov 2019 15:06:36 +0800 Subject: [PATCH 04/80] Fix building on ubuntu 16.04 --- third-party/CMakeLists.txt | 2 +- third-party/cmake/FindAutoconf.cmake | 2 +- third-party/cmake/FindAutomake.cmake | 2 +- third-party/cmake/FindLibtool.cmake | 4 ++-- third-party/externals/boost.cmake | 11 +++++++++-- 5 files changed, 14 insertions(+), 7 deletions(-) diff --git a/third-party/CMakeLists.txt b/third-party/CMakeLists.txt index 8d9b30827ec..d354345e173 100644 --- a/third-party/CMakeLists.txt +++ b/third-party/CMakeLists.txt @@ -141,7 +141,7 @@ if (NOT BISON_FOUND) list(APPEND ALL_TARGETS bison) endif() -find_package(FLEX QUIET) +find_package(FLEX 2.6.0 QUIET) if (NOT FLEX_FOUND) list(APPEND ALL_TARGETS flex) endif() diff --git a/third-party/cmake/FindAutoconf.cmake b/third-party/cmake/FindAutoconf.cmake index e6f50c12bc0..0068e69582c 100644 --- a/third-party/cmake/FindAutoconf.cmake +++ b/third-party/cmake/FindAutoconf.cmake @@ -1,4 +1,4 @@ -find_program(Autoconf_EXECUTABLE NAMES libtool DOC "Path to the autconf executable") +find_program(Autoconf_EXECUTABLE NAMES autoconf DOC "Path to the autconf executable") if (Autoconf_EXECUTABLE) execute_process( OUTPUT_VARIABLE Autoconf_VERSION diff --git a/third-party/cmake/FindAutomake.cmake b/third-party/cmake/FindAutomake.cmake index 48d70aeaefe..97225d1bd00 100644 --- a/third-party/cmake/FindAutomake.cmake +++ b/third-party/cmake/FindAutomake.cmake @@ -1,4 +1,4 @@ -find_program(Automake_EXECUTABLE NAMES libtool DOC "Path to the automake executable") +find_program(Automake_EXECUTABLE NAMES automake DOC "Path to the automake executable") if (Automake_EXECUTABLE) execute_process( OUTPUT_VARIABLE Automake_VERSION diff --git a/third-party/cmake/FindLibtool.cmake b/third-party/cmake/FindLibtool.cmake index a7e194db047..17f54eb9ff0 100644 --- a/third-party/cmake/FindLibtool.cmake +++ b/third-party/cmake/FindLibtool.cmake @@ -1,9 +1,9 @@ -find_program(Libtool_EXECUTABLE NAMES libtool DOC "Path to the libtool executable") +find_program(Libtool_EXECUTABLE NAMES libtool libtoolize DOC "Path to the libtool executable") if (Libtool_EXECUTABLE) execute_process( OUTPUT_VARIABLE Libtool_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE - COMMAND "libtool" "--version" + COMMAND ${Libtool_EXECUTABLE} "--version" COMMAND "head" "-1" COMMAND "cut" "-d" " " "-f4" ) diff --git a/third-party/externals/boost.cmake b/third-party/externals/boost.cmake index 1050ed10726..d197f535870 100644 --- a/third-party/externals/boost.cmake +++ b/third-party/externals/boost.cmake @@ -18,13 +18,11 @@ ExternalProject_Add( SOURCE_DIR ${source_dir} CONFIGURE_COMMAND "" CONFIGURE_COMMAND - env PATH=${compiler_path}:${BUILDING_PATH} ./bootstrap.sh --without-icu --without-libraries=python,test,stacktrace,mpi,log,graph,graph_parallel --prefix=${CMAKE_INSTALL_PREFIX} BUILD_COMMAND - env PATH=${compiler_path}:${BUILDING_PATH} ./b2 install -d0 -j${BUILDING_JOBS_NUM} @@ -42,6 +40,15 @@ ExternalProject_Add( LOG_INSTALL 0 ) +ExternalProject_Add_Step(${name} setup-compiler + DEPENDEES configure + DEPENDERS build + COMMAND + echo "using gcc : : ${CMAKE_CXX_COMPILER} $" + > ${source_dir}/tools/build/src/user-config.jam + WORKING_DIRECTORY ${source_dir} +) + ExternalProject_Add_Step(${name} clean EXCLUDE_FROM_MAIN TRUE ALWAYS TRUE From 34ae1f27a9befb86a0a1b7f84c7b8e97827fbbeb Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Thu, 28 Nov 2019 16:42:34 +0800 Subject: [PATCH 05/80] Fixed cmake version requirement --- third-party/build-third-party.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/third-party/build-third-party.sh b/third-party/build-third-party.sh index dcbb9a88589..00a3323ca40 100755 --- a/third-party/build-third-party.sh +++ b/third-party/build-third-party.sh @@ -38,7 +38,7 @@ function version_cmp { function check_cmake { hash cmake &> /dev/null || { echo "No cmake found." 1>&2; exit 1; } local cmake_version=$(cmake --version | head -1 | cut -d ' ' -f 3) - local least_cmake_version=3.8.0 + local least_cmake_version=3.5.0 if [[ $(version_cmp $cmake_version $least_cmake_version) -lt 0 ]] then echo "cmake $least_cmake_version or later required" 1>&2 From c3bfefc986707d2c4a1ff2c9fbf816f4eaa38cc8 Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Thu, 28 Nov 2019 17:11:45 +0800 Subject: [PATCH 06/80] Fixed mstch to depend on boost --- third-party/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/third-party/CMakeLists.txt b/third-party/CMakeLists.txt index d354345e173..cedd4f013ca 100644 --- a/third-party/CMakeLists.txt +++ b/third-party/CMakeLists.txt @@ -185,6 +185,7 @@ maybe_add_dependencies(libevent libtool) maybe_add_dependencies(gperf libtool) maybe_add_dependencies(boost zlib bzip2 lzma) +maybe_add_dependencies(mstch boost) maybe_add_dependencies(glog gflags) From 3d69e96f01602647edce911022b717bd4e8864b8 Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Thu, 28 Nov 2019 19:49:30 +0800 Subject: [PATCH 07/80] Added lz4 --- CMakeLists.txt | 2 +- third-party/CMakeLists.txt | 12 +++++++++- third-party/build-third-party.sh | 2 +- third-party/externals/lz4.cmake | 40 ++++++++++++++++++++++++++++++++ 4 files changed, 53 insertions(+), 3 deletions(-) create mode 100644 third-party/externals/lz4.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index eded95aef02..cd7c24f912d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -345,7 +345,7 @@ set(THRIFT_LIBRARIES set(ROCKSDB_LIBRARIES ${Rocksdb_LIBRARY}) # All compression libraries -set(COMPRESSION_LIBRARIES bz2 snappy zstd z) +set(COMPRESSION_LIBRARIES bz2 snappy zstd z lz4) if (LIBLZMA_FOUND) include_directories(SYSTEM ${LIBLZMA_INCLUDE_DIRS}) list(APPEND COMPRESSION_LIBRARIES ${LIBLZMA_LIBRARIES}) diff --git a/third-party/CMakeLists.txt b/third-party/CMakeLists.txt index cedd4f013ca..91c3c0d8864 100644 --- a/third-party/CMakeLists.txt +++ b/third-party/CMakeLists.txt @@ -96,6 +96,7 @@ set(ALL_TARGETS zlib snappy lzma + lz4 libaio mstch fatal @@ -192,7 +193,7 @@ maybe_add_dependencies(glog gflags) maybe_add_dependencies(s2geometry googletest glog openssl) maybe_add_dependencies(krb5 keyutils openssl gettext bison) -maybe_add_dependencies(folly glog boost double-conversion openssl libevent lzma zstd snappy libunwind libaio) +maybe_add_dependencies(folly glog boost double-conversion openssl libevent lzma zstd snappy lz4 libunwind libaio) maybe_add_dependencies(wangle folly) maybe_add_dependencies(fbthrift folly krb5 bison flex mstch zlib zstd wangle fatal) maybe_add_dependencies(proxygen wangle libunwind gperf) @@ -214,3 +215,12 @@ add_custom_target( clean-all DEPENDS ${CLEAN_TARGETS} ) + +add_custom_target( + pack-downloads + COMMAND + tar czvf nebula-third-party-src-1.0.tgz downloads + COMMAND + md5sum nebula-third-party-src-1.0.tgz + WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/.. +) diff --git a/third-party/build-third-party.sh b/third-party/build-third-party.sh index 00a3323ca40..c4be74bef38 100755 --- a/third-party/build-third-party.sh +++ b/third-party/build-third-party.sh @@ -91,7 +91,7 @@ then checksum=$(md5sum $source_tar_name | cut -d ' ' -f 1) fi -if [[ ! $checksum = aa3356c67ea34d38ed28f762dccf1559 ]] +if [[ ! $checksum = 375f349b7b5ae1212bd4195bfc30f43a ]] then hash wget &> /dev/null && download_cmd="wget -c" hash axel &> /dev/null && download_cmd="axel -a -n 16" diff --git a/third-party/externals/lz4.cmake b/third-party/externals/lz4.cmake new file mode 100644 index 00000000000..c80cf63282a --- /dev/null +++ b/third-party/externals/lz4.cmake @@ -0,0 +1,40 @@ +# Copyright (c) 2019 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. + +set(name lz4) +set(source_dir ${CMAKE_CURRENT_BINARY_DIR}/${name}/source) +ExternalProject_Add( + ${name} + URL https://github.com/lz4/lz4/archive/v1.9.2.tar.gz + URL_HASH MD5=3898c56c82fb3d9455aefd48db48eaad + DOWNLOAD_NAME lz4-1.9.2.tar.gz + PREFIX ${CMAKE_CURRENT_BINARY_DIR}/${name} + TMP_DIR ${BUILD_INFO_DIR} + STAMP_DIR ${BUILD_INFO_DIR} + DOWNLOAD_DIR ${DOWNLOAD_DIR} + SOURCE_DIR ${source_dir} + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + BUILD_IN_SOURCE 1 + INSTALL_COMMAND + make install -s + MOREFLAGS=-fPIC + BUILD_SHARED=no + -j${BUILDING_JOBS_NUM} + PREFIX=${CMAKE_INSTALL_PREFIX} + LOG_BUILD 1 + LOG_INSTALL 1 +) + +ExternalProject_Add_Step(${name} clean + EXCLUDE_FROM_MAIN TRUE + ALWAYS TRUE + DEPENDEES build + COMMAND make clean -j + COMMAND rm -f ${BUILD_INFO_DIR}/${name}-build + WORKING_DIRECTORY ${source_dir} +) + +ExternalProject_Add_StepTargets(${name} clean) From 4f89b5ea3ec665017f623e518f19609750118ddf Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Sat, 30 Nov 2019 14:50:01 +0800 Subject: [PATCH 08/80] Added build-gcc.sh --- third-party/build-gcc.sh | 158 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 158 insertions(+) create mode 100755 third-party/build-gcc.sh diff --git a/third-party/build-gcc.sh b/third-party/build-gcc.sh new file mode 100755 index 00000000000..d4f327c636b --- /dev/null +++ b/third-party/build-gcc.sh @@ -0,0 +1,158 @@ +#! /usr/bin/env bash + +# Copyright (c) 2019 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. + +# Usage: build-gcc.sh [prefix] + +# Always use bash +shell=$(basename $(readlink /proc/$$/exe)) +if [ ! x$shell = x"bash" ] +then + bash $0 $@ + exit $? +fi + +url_base=http://ftpmirror.gnu.org + +gcc_version=7.5.0 +gcc_tarball=gcc-$gcc_version.tar.xz +gcc_url=$url_base/gcc/gcc-$gcc_version/$gcc_tarball + +gmp_version=5.1.3 +gmp_tarball=gmp-$gmp_version.tar.xz +gmp_url=$url_base/gmp/$gmp_tarball + +mpfr_version=3.1.4 +mpfr_tarball=mpfr-$mpfr_version.tar.xz +mpfr_url=$url_base/mpfr/$mpfr_tarball + +mpc_version=1.0.3 +mpc_tarball=mpc-$mpc_version.tar.gz +mpc_url=$url_base/mpc/$mpc_tarball + +gcc_checksum=79cb8a65d44dfc8a2402b46395535c9a +gmp_checksum=e5fe367801ff067b923d1e6a126448aa +mpfr_checksum=064b2c18185038e404a401b830d59be8 +mpc_checksum=d6a1d5f8ddea3abd2cc3e98f58352d26 + +cur_dir=$PWD +build_dir=$PWD/gcc-build +tarballs_dir=$build_dir/downloads +source_dir=$build_dir/source +prefix=$1 +install_dir=${prefix:-$PWD/gcc-install}/gcc/$gcc_version + +function get_checksum { + md5sum $1 | cut -d ' ' -f 1 +} + +# args: +function fetch_tarball { + local checksum + [[ -f $2 ]] && checksum=$(get_checksum $2) + [[ -n $checksum ]] && [[ $checksum = $4 ]] && return 0 + echo "Downloading $2..." + if ! bash -c "$1 $3" + then + echo "Download $2 Failed" + exit 1 + fi +} + +function fetch_tarballs { + hash wget &> /dev/null && download_cmd="wget -c" + hash axel &> /dev/null && download_cmd="axel -a -n 8" + if [[ -z $download_cmd ]] + then + echo "Neither 'wget' nor 'axel' available for downloading" 1>&2; + exit 1; + fi + + mkdir -p $tarballs_dir && cd $tarballs_dir + + fetch_tarball "$download_cmd" $gcc_tarball $gcc_url $gcc_checksum + fetch_tarball "$download_cmd" $gmp_tarball $gmp_url $gmp_checksum + fetch_tarball "$download_cmd" $mpfr_tarball $mpfr_url $mpfr_checksum + fetch_tarball "$download_cmd" $mpc_tarball $mpc_url $mpc_checksum + cd $OLDPWD +} + +function unpack_tarballs { + mkdir -p $source_dir + cd $tarballs_dir + set -e + + echo "Unpacking $gcc_tarball..." + tar --skip-old-files -xf $gcc_tarball -C $source_dir + + echo "Unpacking $gmp_tarball..." + tar --skip-old-files -xf $gmp_tarball -C $source_dir + + echo "Unpacking $mpfr_tarball..." + tar --skip-old-files -xf $mpfr_tarball -C $source_dir + + echo "Unpacking $mpc_tarball..." + tar --skip-old-files -xf $mpc_tarball -C $source_dir + + set +e + cd $OLDPWD +} + +function setup_deps { + cd $source_dir/gcc-$gcc_version + ln -sf ../gmp-$gmp_version gmp + ln -sf ../mpfr-$mpfr_version mpfr + ln -sf ../mpc-$mpc_version mpc + + cd $source_dir/gmp-$gmp_version + cp -f configfsf.guess config.guess + cp -f configfsf.sub config.sub + + cd $OLDPWD +} + +function configure_gcc { + cd $source_dir/gcc-$gcc_version + ./configure --prefix=$install_dir \ + --enable-shared \ + --enable-threads=posix \ + --enable-__cxa_atexit \ + --enable-clocale=gnu \ + --enable-languages=c,c++ \ + --enable-lto \ + --enable-bootstrap \ + --disable-nls \ + --disable-multilib \ + --disable-install-libiberty \ + --disable-werror \ + --with-system-zlib + [[ $? -eq 0 ]] || exit 1 + cd $OLDPWD +} + +function build_gcc { + cd $source_dir/gcc-$gcc_version + make -j 20 || exit 1 + cd $OLDPWD +} + +function install_gcc { + cd $source_dir/gcc-$gcc_version + make -j install-strip || exit 1 + cd $OLDPWD +} + +start_time=$(date +%s) +fetch_tarballs +unpack_tarballs +setup_deps +configure_gcc +build_gcc +install_gcc +end_time=$(date +%s) + +echo "GCC-$gcc_version has been installed to prefix=$install_dir" +echo "$((end_time - start_time)) seconds been taken" From 11ec02a15befbaeb88fc65dd2e59710e14246f5b Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Sat, 30 Nov 2019 21:12:32 +0800 Subject: [PATCH 09/80] Added setup/resort-env.sh for GCC and CMake --- third-party/build-cmake.sh | 15 ++++++++++++++- third-party/build-gcc.sh | 19 +++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/third-party/build-cmake.sh b/third-party/build-cmake.sh index 79a2e1231ed..be0a4b48cce 100755 --- a/third-party/build-cmake.sh +++ b/third-party/build-cmake.sh @@ -32,7 +32,7 @@ fi if [[ ! $checksum = 35d56e9c27b4fd2819a11c29320c655a ]] then hash wget &> /dev/null && download_cmd="wget -c" - hash axel &> /dev/null && download_cmd="axel -a -n 16" + hash axel &> /dev/null && download_cmd="axel -a -n 8" if [[ -z $download_cmd ]] then echo "Neither 'wget' nor 'axel' available for downloading" 1>&2; @@ -47,10 +47,23 @@ then fi fi +set -e mkdir -p $prefix bash $archive --prefix=$prefix &> /dev/null < $prefix/bin/setup-env.sh < $prefix/bin/restore-env.sh < $install_dir/bin/setup-env.sh < $install_dir/bin/restore-env.sh < Date: Sat, 30 Nov 2019 21:18:07 +0800 Subject: [PATCH 10/80] Added hints for setup/restore-env.sh --- third-party/build-cmake.sh | 2 ++ third-party/build-gcc.sh | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/third-party/build-cmake.sh b/third-party/build-cmake.sh index be0a4b48cce..c44ef9af2b0 100755 --- a/third-party/build-cmake.sh +++ b/third-party/build-cmake.sh @@ -67,3 +67,5 @@ hash -r EOF echo "CMake has been installed to prefix=$prefix" +echo "Run 'source $prefix/bin/setup-env.sh' to make it ready to use." +echo "Run 'source $prefix/bin/restore-env.sh' to disable it." diff --git a/third-party/build-gcc.sh b/third-party/build-gcc.sh index f74a4c46b59..1725f3c5641 100755 --- a/third-party/build-gcc.sh +++ b/third-party/build-gcc.sh @@ -174,4 +174,6 @@ hash -r EOF echo "GCC-$gcc_version has been installed to prefix=$install_dir" -echo "$((end_time - start_time)) seconds been taken" +echo "$((end_time - start_time)) seconds been taken." +echo "Run 'source $install_dir/bin/setup-env.sh' to make it ready to use." +echo "Run 'source $install_dir/bin/restore-env.sh' to disable it." From cd4ccc4a5bf2485f21c06e9c55d8cecb945ea041 Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Sat, 30 Nov 2019 21:26:51 +0800 Subject: [PATCH 11/80] Fix to use readlink instead of realpath --- third-party/build-cmake.sh | 4 ++-- third-party/build-gcc.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/third-party/build-cmake.sh b/third-party/build-cmake.sh index c44ef9af2b0..0d94f052839 100755 --- a/third-party/build-cmake.sh +++ b/third-party/build-cmake.sh @@ -55,13 +55,13 @@ no EOF cat > $prefix/bin/setup-env.sh < $prefix/bin/restore-env.sh < $install_dir/bin/setup-env.sh < $install_dir/bin/restore-env.sh < Date: Sat, 30 Nov 2019 21:58:51 +0800 Subject: [PATCH 12/80] Renamed setup/restore-env to enable/disable --- third-party/build-cmake.sh | 8 ++++---- third-party/build-gcc.sh | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/third-party/build-cmake.sh b/third-party/build-cmake.sh index 0d94f052839..488ebe6fc95 100755 --- a/third-party/build-cmake.sh +++ b/third-party/build-cmake.sh @@ -54,18 +54,18 @@ yes no EOF -cat > $prefix/bin/setup-env.sh < $prefix/bin/enable-cmake.sh < $prefix/bin/restore-env.sh < $prefix/bin/disable-cmake.sh < $install_dir/bin/setup-env.sh < $install_dir/bin/enable-gcc.sh < $install_dir/bin/restore-env.sh < $install_dir/bin/disable-gcc.sh < Date: Sat, 30 Nov 2019 23:55:13 +0800 Subject: [PATCH 13/80] Added binutils --- third-party/build-gcc.sh | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/third-party/build-gcc.sh b/third-party/build-gcc.sh index e52d4765471..018540a3e38 100755 --- a/third-party/build-gcc.sh +++ b/third-party/build-gcc.sh @@ -33,10 +33,15 @@ mpc_version=1.0.3 mpc_tarball=mpc-$mpc_version.tar.gz mpc_url=$url_base/mpc/$mpc_tarball +bu_version=2.28.1 +bu_tarball=binutils-$bu_version.tar.xz +bu_url=$url_base/binutils/$bu_tarball + gcc_checksum=79cb8a65d44dfc8a2402b46395535c9a gmp_checksum=e5fe367801ff067b923d1e6a126448aa mpfr_checksum=064b2c18185038e404a401b830d59be8 mpc_checksum=d6a1d5f8ddea3abd2cc3e98f58352d26 +bu_checksum=a3bf359889e4b299fce1f4cb919dc7b6 cur_dir=$PWD build_dir=$PWD/gcc-build @@ -77,6 +82,7 @@ function fetch_tarballs { fetch_tarball "$download_cmd" $gmp_tarball $gmp_url $gmp_checksum fetch_tarball "$download_cmd" $mpfr_tarball $mpfr_url $mpfr_checksum fetch_tarball "$download_cmd" $mpc_tarball $mpc_url $mpc_checksum + fetch_tarball "$download_cmd" $bu_tarball $bu_url $bu_checksum cd $OLDPWD } @@ -97,6 +103,9 @@ function unpack_tarballs { echo "Unpacking $mpc_tarball..." tar --skip-old-files -xf $mpc_tarball -C $source_dir + echo "Unpacking $bu_tarball..." + tar --skip-old-files -xf $bu_tarball -C $source_dir + set +e cd $OLDPWD } @@ -146,6 +155,27 @@ function install_gcc { cd $OLDPWD } +function build_binutils { + cd $source_dir/binutils-$bu_version + ./configure --prefix=$install_dir --enable-gold + [[ $? -eq 0 ]] || exit 1 + make -j8 + [[ $? -eq 0 ]] || exit 1 + cd $OLDPWD +} + +function install_binutils { + cd $source_dir/binutils-$bu_version + make -j8 install-gas + make -j8 install-ld + [[ $? -eq 0 ]] || exit 1 + gcc_triple=$($source_dir/gcc-$gcc_version/config.guess) + cd $install_dir + cp -v bin/as libexec/gcc/$gcc_triple/$gcc_version/ + cp -v bin/ld* libexec/gcc/$gcc_triple/$gcc_version/ + cd $OLDPWD +} + start_time=$(date +%s) fetch_tarballs unpack_tarballs @@ -153,6 +183,8 @@ setup_deps configure_gcc build_gcc install_gcc +build_binutils +install_binutils end_time=$(date +%s) cat > $install_dir/bin/enable-gcc.sh < Date: Sun, 1 Dec 2019 01:04:56 +0800 Subject: [PATCH 14/80] Fixed symlink creation in lz4 --- third-party/externals/lz4.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/third-party/externals/lz4.cmake b/third-party/externals/lz4.cmake index c80cf63282a..f1d05dac237 100644 --- a/third-party/externals/lz4.cmake +++ b/third-party/externals/lz4.cmake @@ -21,6 +21,7 @@ ExternalProject_Add( INSTALL_COMMAND make install -s MOREFLAGS=-fPIC + "LN_S=ln -sf" BUILD_SHARED=no -j${BUILDING_JOBS_NUM} PREFIX=${CMAKE_INSTALL_PREFIX} From e22b3e1162cebfdc98ef14a6c412f7cf6c848a2d Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Sun, 1 Dec 2019 02:33:51 +0800 Subject: [PATCH 15/80] Stripped binutils --- third-party/build-gcc.sh | 9 +++++++-- third-party/build-third-party.sh | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/third-party/build-gcc.sh b/third-party/build-gcc.sh index 018540a3e38..68f40603716 100755 --- a/third-party/build-gcc.sh +++ b/third-party/build-gcc.sh @@ -166,8 +166,8 @@ function build_binutils { function install_binutils { cd $source_dir/binutils-$bu_version - make -j8 install-gas - make -j8 install-ld + make -j8 install-strip-gas + make -j8 install-strip-ld [[ $? -eq 0 ]] || exit 1 gcc_triple=$($source_dir/gcc-$gcc_version/config.guess) cd $install_dir @@ -176,6 +176,10 @@ function install_binutils { cd $OLDPWD } +function finalize { + find $install_dir -name '*.la' | xargs rm -f +} + start_time=$(date +%s) fetch_tarballs unpack_tarballs @@ -185,6 +189,7 @@ build_gcc install_gcc build_binutils install_binutils +finalize end_time=$(date +%s) cat > $install_dir/bin/enable-gcc.sh < /dev/null && download_cmd="wget -c" - hash axel &> /dev/null && download_cmd="axel -a -n 16" + hash axel &> /dev/null && download_cmd="axel -a -n 8" if [[ -z $download_cmd ]] then echo "Neither 'wget' nor 'axel' available for downloading" 1>&2; From 980952f2040ce00c083e65fd3153f06fdd16da5f Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Sun, 1 Dec 2019 13:34:49 +0800 Subject: [PATCH 16/80] Add make-package --- third-party/build-gcc.sh | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/third-party/build-gcc.sh b/third-party/build-gcc.sh index 68f40603716..78ada7468f2 100755 --- a/third-party/build-gcc.sh +++ b/third-party/build-gcc.sh @@ -166,8 +166,7 @@ function build_binutils { function install_binutils { cd $source_dir/binutils-$bu_version - make -j8 install-strip-gas - make -j8 install-strip-ld + make -j8 install-strip [[ $? -eq 0 ]] || exit 1 gcc_triple=$($source_dir/gcc-$gcc_version/config.guess) cd $install_dir @@ -180,6 +179,30 @@ function finalize { find $install_dir -name '*.la' | xargs rm -f } +function make_package { +glibc_version=$(ldd --version | head -1 | cut -d ' ' -f4) +exec_file=$build_dir/gcc-$gcc_version-linux-glibc-$glibc_version-x86_64.sh +cat > $exec_file < /dev/null || { echo "xz: Command not found"; exit 1; } + +mkdir -p \$prefix +[[ -w \$prefix ]] || { echo "\$prefix: No permission to write"; exit 1; } + +archive_offset=\$(awk '/^__start_of_archive__$/{print NR+1; exit 0;}' \$0) +tail -n+\$archive_offset \$0 | tar --numeric-owner -xJf - -C \$prefix + +exit 0 + +__start_of_archive__ +EOF +cd $install_dir/../.. +tar -cJvf - * >> $exec_file +cd $OLDPWD +} + start_time=$(date +%s) fetch_tarballs unpack_tarballs @@ -190,6 +213,7 @@ install_gcc build_binutils install_binutils finalize +make_package end_time=$(date +%s) cat > $install_dir/bin/enable-gcc.sh < Date: Sun, 1 Dec 2019 14:37:15 +0800 Subject: [PATCH 17/80] Add --prefix to build-gcc --- third-party/build-gcc.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/third-party/build-gcc.sh b/third-party/build-gcc.sh index 78ada7468f2..f4bdfc01d3b 100755 --- a/third-party/build-gcc.sh +++ b/third-party/build-gcc.sh @@ -181,10 +181,13 @@ function finalize { function make_package { glibc_version=$(ldd --version | head -1 | cut -d ' ' -f4) -exec_file=$build_dir/gcc-$gcc_version-linux-glibc-$glibc_version-x86_64.sh +exec_file=$build_dir/gcc-$gcc_version-linux-x86_64-glibc-$glibc_version.sh +echo "Creating self-extracting package $exec_file" cat > $exec_file < /dev/null || { echo "xz: Command not found"; exit 1; } @@ -200,6 +203,7 @@ __start_of_archive__ EOF cd $install_dir/../.. tar -cJvf - * >> $exec_file +chmod 0755 $exec_file cd $OLDPWD } From d1e8736ccd3098ce6cf2585b06917455174bbb47 Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Sun, 1 Dec 2019 14:50:27 +0800 Subject: [PATCH 18/80] Fix prefix --- third-party/build-gcc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/third-party/build-gcc.sh b/third-party/build-gcc.sh index f4bdfc01d3b..f635631a365 100755 --- a/third-party/build-gcc.sh +++ b/third-party/build-gcc.sh @@ -186,7 +186,7 @@ echo "Creating self-extracting package $exec_file" cat > $exec_file < /dev/null || { echo "xz: Command not found"; exit 1; } From fb4a4f8e41e3d10c610ab61568834ff7ec8bedf2 Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Sun, 1 Dec 2019 16:02:15 +0800 Subject: [PATCH 19/80] Fixed indentation --- third-party/build-gcc.sh | 48 ++++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/third-party/build-gcc.sh b/third-party/build-gcc.sh index f635631a365..19b0ef8967c 100755 --- a/third-party/build-gcc.sh +++ b/third-party/build-gcc.sh @@ -48,7 +48,7 @@ build_dir=$PWD/gcc-build tarballs_dir=$build_dir/downloads source_dir=$build_dir/source prefix=$1 -install_dir=${prefix:-$PWD/gcc-install}/gcc/$gcc_version +install_dir=${prefix:-$PWD/$build_dir/install}/gcc/$gcc_version function get_checksum { md5sum $1 | cut -d ' ' -f 1 @@ -124,21 +124,21 @@ function setup_deps { } function configure_gcc { - cd $source_dir/gcc-$gcc_version - ./configure --prefix=$install_dir \ - --enable-shared \ - --with-libc-version=2.18 \ - --enable-threads=posix \ - --enable-__cxa_atexit \ - --enable-clocale=gnu \ - --enable-languages=c,c++ \ - --enable-lto \ - --enable-bootstrap \ - --disable-nls \ - --disable-multilib \ - --disable-install-libiberty \ - --disable-werror \ - --with-system-zlib + cd $source_dir/gcc-$gcc_version + ./configure --prefix=$install_dir \ + --enable-shared \ + --with-glibc-version=2.12 \ + --enable-threads=posix \ + --enable-__cxa_atexit \ + --enable-clocale=gnu \ + --enable-languages=c,c++ \ + --enable-lto \ + --enable-bootstrap \ + --disable-nls \ + --disable-multilib \ + --disable-install-libiberty \ + --disable-werror \ + --with-system-zlib [[ $? -eq 0 ]] || exit 1 cd $OLDPWD } @@ -180,10 +180,10 @@ function finalize { } function make_package { -glibc_version=$(ldd --version | head -1 | cut -d ' ' -f4) -exec_file=$build_dir/gcc-$gcc_version-linux-x86_64-glibc-$glibc_version.sh -echo "Creating self-extracting package $exec_file" -cat > $exec_file < $exec_file <> $exec_file -chmod 0755 $exec_file -cd $OLDPWD + cd $install_dir/../.. + tar -cJf - * >> $exec_file + chmod 0755 $exec_file + cd $OLDPWD } start_time=$(date +%s) From 66c7373d3701a19d9f7a92caf999fbfdffc6a2c2 Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Sun, 1 Dec 2019 17:42:55 +0800 Subject: [PATCH 20/80] Fix make-package --- third-party/build-gcc.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/third-party/build-gcc.sh b/third-party/build-gcc.sh index 19b0ef8967c..3ab0f7ca769 100755 --- a/third-party/build-gcc.sh +++ b/third-party/build-gcc.sh @@ -184,6 +184,7 @@ function make_package { exec_file=$build_dir/gcc-$gcc_version-linux-x86_64-glibc-$glibc_version.sh echo "Creating self-extracting package $exec_file" cat > $exec_file < $install_dir/bin/enable-gcc.sh < Date: Mon, 2 Dec 2019 20:21:54 +0800 Subject: [PATCH 21/80] Build binutils in source --- third-party/build-gcc.sh | 110 ++++++++++++++++++++++++++------------- 1 file changed, 74 insertions(+), 36 deletions(-) diff --git a/third-party/build-gcc.sh b/third-party/build-gcc.sh index 3ab0f7ca769..f56342a9ceb 100755 --- a/third-party/build-gcc.sh +++ b/third-party/build-gcc.sh @@ -47,8 +47,9 @@ cur_dir=$PWD build_dir=$PWD/gcc-build tarballs_dir=$build_dir/downloads source_dir=$build_dir/source +object_dir=$build_dir/build prefix=$1 -install_dir=${prefix:-$PWD/$build_dir/install}/gcc/$gcc_version +install_dir=${prefix:-$build_dir/install}/gcc/$gcc_version function get_checksum { md5sum $1 | cut -d ' ' -f 1 @@ -91,20 +92,35 @@ function unpack_tarballs { cd $tarballs_dir set -e - echo "Unpacking $gcc_tarball..." - tar --skip-old-files -xf $gcc_tarball -C $source_dir + if [[ ! -d $source_dir/gcc-$gcc_version ]] + then + echo "Unpacking $gcc_tarball..." + tar --skip-old-files -xf $gcc_tarball -C $source_dir + fi - echo "Unpacking $gmp_tarball..." - tar --skip-old-files -xf $gmp_tarball -C $source_dir + if [[ ! -d $source_dir/gmp-$gmp_version ]] + then + echo "Unpacking $gmp_tarball..." + tar --skip-old-files -xf $gmp_tarball -C $source_dir + fi - echo "Unpacking $mpfr_tarball..." - tar --skip-old-files -xf $mpfr_tarball -C $source_dir + if [[ ! -d $source_dir/mpfr-$mpfr_version ]] + then + echo "Unpacking $mpfr_tarball..." + tar --skip-old-files -xf $mpfr_tarball -C $source_dir + fi - echo "Unpacking $mpc_tarball..." - tar --skip-old-files -xf $mpc_tarball -C $source_dir + if [[ ! -d $source_dir/mpc-$mpc_version ]] + then + echo "Unpacking $mpc_tarball..." + tar --skip-old-files -xf $mpc_tarball -C $source_dir + fi - echo "Unpacking $bu_tarball..." - tar --skip-old-files -xf $bu_tarball -C $source_dir + if [[ ! -d $source_dir/binutils-$bu_version ]] + then + echo "Unpacking $bu_tarball..." + tar --skip-old-files -xf $bu_tarball -C $source_dir + fi set +e cd $OLDPWD @@ -116,6 +132,24 @@ function setup_deps { ln -sf ../mpfr-$mpfr_version mpfr ln -sf ../mpc-$mpc_version mpc + #[[ ! -e config.guess.orig ]] && cp -vp config.guess config.guess.orig + #cat > config.guess < $exec_file < $install_dir/bin/enable-gcc.sh < Date: Mon, 2 Dec 2019 22:39:05 +0800 Subject: [PATCH 22/80] Added gdb --- third-party/build-gcc.sh | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/third-party/build-gcc.sh b/third-party/build-gcc.sh index f56342a9ceb..73a93c1194f 100755 --- a/third-party/build-gcc.sh +++ b/third-party/build-gcc.sh @@ -37,11 +37,16 @@ bu_version=2.28.1 bu_tarball=binutils-$bu_version.tar.xz bu_url=$url_base/binutils/$bu_tarball +gdb_version=8.3 +gdb_tarball=gdb-$gdb_version.tar.xz +gdb_url=$url_base/gdb/$gdb_tarball + gcc_checksum=79cb8a65d44dfc8a2402b46395535c9a gmp_checksum=e5fe367801ff067b923d1e6a126448aa mpfr_checksum=064b2c18185038e404a401b830d59be8 mpc_checksum=d6a1d5f8ddea3abd2cc3e98f58352d26 bu_checksum=a3bf359889e4b299fce1f4cb919dc7b6 +gdb_checksum=bbd95b2f9b34621ad7a19a3965476314 cur_dir=$PWD build_dir=$PWD/gcc-build @@ -84,6 +89,8 @@ function fetch_tarballs { fetch_tarball "$download_cmd" $mpfr_tarball $mpfr_url $mpfr_checksum fetch_tarball "$download_cmd" $mpc_tarball $mpc_url $mpc_checksum fetch_tarball "$download_cmd" $bu_tarball $bu_url $bu_checksum + fetch_tarball "$download_cmd" $gdb_tarball $gdb_url $gdb_checksum + cd $OLDPWD } @@ -122,6 +129,12 @@ function unpack_tarballs { tar --skip-old-files -xf $bu_tarball -C $source_dir fi + if [[ ! -d $source_dir/gdb-$gdb_version ]] + then + echo "Unpacking $gdb_tarball..." + tar --skip-old-files -xf $gdb_tarball -C $source_dir + fi + set +e cd $OLDPWD } @@ -173,6 +186,7 @@ function configure_gcc { --disable-multilib \ --disable-install-libiberty \ --disable-werror \ + --with-system-readline \ --with-system-zlib #--host=x86_64-vesoft-linux \ #--build=x86_64-vesoft-linux \ @@ -192,24 +206,13 @@ function install_gcc { cd $OLDPWD } -function build_binutils { - cd $object_dir - $source_dir/binutils-$bu_version/configure --prefix=$install_dir --enable-gold +function build_gdb { + cd $source_dir/gdb-$gdb_version + ./configure --prefix=$install_dir --with-system-readline [[ $? -eq 0 ]] || exit 1 make -j8 [[ $? -eq 0 ]] || exit 1 - cd $OLDPWD -} - -function install_binutils { - cd $object_dir - make -j8 install-strip - [[ $? -eq 0 ]] || exit 1 - cd $OLDPWD - - cd $install_dir - cp -v bin/as libexec/gcc/x86_64-vesoft-linux/$gcc_version/ - cp -v bin/ld* libexec/gcc/x86_64-vesoft-linux/$gcc_version/ + make install cd $OLDPWD } @@ -253,8 +256,7 @@ setup_deps configure_gcc build_gcc install_gcc -#build_binutils -#install_binutils +build_gdb finalize cat > $install_dir/bin/enable-gcc.sh < Date: Tue, 3 Dec 2019 01:40:38 +0800 Subject: [PATCH 23/80] Upgrade toolchain --- third-party/build-gcc.sh | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/third-party/build-gcc.sh b/third-party/build-gcc.sh index 73a93c1194f..f359d4fa28e 100755 --- a/third-party/build-gcc.sh +++ b/third-party/build-gcc.sh @@ -15,25 +15,26 @@ then exit $? fi -url_base=http://ftpmirror.gnu.org +#url_base=http://ftpmirror.gnu.org +url_base=http://mirrors.ustc.edu.cn/gnu -gcc_version=7.5.0 +gcc_version=9.1.0 gcc_tarball=gcc-$gcc_version.tar.xz gcc_url=$url_base/gcc/gcc-$gcc_version/$gcc_tarball -gmp_version=5.1.3 +gmp_version=6.1.2 gmp_tarball=gmp-$gmp_version.tar.xz gmp_url=$url_base/gmp/$gmp_tarball -mpfr_version=3.1.4 +mpfr_version=4.0.2 mpfr_tarball=mpfr-$mpfr_version.tar.xz mpfr_url=$url_base/mpfr/$mpfr_tarball -mpc_version=1.0.3 +mpc_version=1.1.0 mpc_tarball=mpc-$mpc_version.tar.gz mpc_url=$url_base/mpc/$mpc_tarball -bu_version=2.28.1 +bu_version=2.32 bu_tarball=binutils-$bu_version.tar.xz bu_url=$url_base/binutils/$bu_tarball @@ -41,11 +42,11 @@ gdb_version=8.3 gdb_tarball=gdb-$gdb_version.tar.xz gdb_url=$url_base/gdb/$gdb_tarball -gcc_checksum=79cb8a65d44dfc8a2402b46395535c9a -gmp_checksum=e5fe367801ff067b923d1e6a126448aa -mpfr_checksum=064b2c18185038e404a401b830d59be8 -mpc_checksum=d6a1d5f8ddea3abd2cc3e98f58352d26 -bu_checksum=a3bf359889e4b299fce1f4cb919dc7b6 +gcc_checksum=6069ae3737cf02bf2cb44a391ef0e937 +gmp_checksum=f58fa8001d60c4c77595fbbb62b63c1d +mpfr_checksum=320fbc4463d4c8cb1e566929d8adc4f8 +mpc_checksum=4125404e41e482ec68282a2e687f6c73 +bu_checksum=0d174cdaf85721c5723bf52355be41e6 gdb_checksum=bbd95b2f9b34621ad7a19a3965476314 cur_dir=$PWD @@ -144,6 +145,7 @@ function setup_deps { ln -sf ../gmp-$gmp_version gmp ln -sf ../mpfr-$mpfr_version mpfr ln -sf ../mpc-$mpc_version mpc + ln -sf ../gdb-$gdb_version/gdb gdb #[[ ! -e config.guess.orig ]] && cp -vp config.guess config.guess.orig #cat > config.guess < $install_dir/bin/enable-gcc.sh < Date: Tue, 3 Dec 2019 10:17:56 +0800 Subject: [PATCH 24/80] Remove gdb due to the need of a C++11 compiler --- third-party/build-gcc.sh | 30 ++---------------------------- 1 file changed, 2 insertions(+), 28 deletions(-) diff --git a/third-party/build-gcc.sh b/third-party/build-gcc.sh index f359d4fa28e..10d00900967 100755 --- a/third-party/build-gcc.sh +++ b/third-party/build-gcc.sh @@ -38,16 +38,11 @@ bu_version=2.32 bu_tarball=binutils-$bu_version.tar.xz bu_url=$url_base/binutils/$bu_tarball -gdb_version=8.3 -gdb_tarball=gdb-$gdb_version.tar.xz -gdb_url=$url_base/gdb/$gdb_tarball - gcc_checksum=6069ae3737cf02bf2cb44a391ef0e937 gmp_checksum=f58fa8001d60c4c77595fbbb62b63c1d mpfr_checksum=320fbc4463d4c8cb1e566929d8adc4f8 mpc_checksum=4125404e41e482ec68282a2e687f6c73 bu_checksum=0d174cdaf85721c5723bf52355be41e6 -gdb_checksum=bbd95b2f9b34621ad7a19a3965476314 cur_dir=$PWD build_dir=$PWD/gcc-build @@ -90,7 +85,6 @@ function fetch_tarballs { fetch_tarball "$download_cmd" $mpfr_tarball $mpfr_url $mpfr_checksum fetch_tarball "$download_cmd" $mpc_tarball $mpc_url $mpc_checksum fetch_tarball "$download_cmd" $bu_tarball $bu_url $bu_checksum - fetch_tarball "$download_cmd" $gdb_tarball $gdb_url $gdb_checksum cd $OLDPWD } @@ -130,12 +124,6 @@ function unpack_tarballs { tar --skip-old-files -xf $bu_tarball -C $source_dir fi - if [[ ! -d $source_dir/gdb-$gdb_version ]] - then - echo "Unpacking $gdb_tarball..." - tar --skip-old-files -xf $gdb_tarball -C $source_dir - fi - set +e cd $OLDPWD } @@ -145,7 +133,6 @@ function setup_deps { ln -sf ../gmp-$gmp_version gmp ln -sf ../mpfr-$mpfr_version mpfr ln -sf ../mpc-$mpc_version mpc - ln -sf ../gdb-$gdb_version/gdb gdb #[[ ! -e config.guess.orig ]] && cp -vp config.guess config.guess.orig #cat > config.guess < $exec_file < $install_dir/bin/enable-gcc.sh < Date: Tue, 3 Dec 2019 11:05:44 +0800 Subject: [PATCH 25/80] Downgrade --- third-party/build-gcc.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/third-party/build-gcc.sh b/third-party/build-gcc.sh index 10d00900967..3f442b9601b 100755 --- a/third-party/build-gcc.sh +++ b/third-party/build-gcc.sh @@ -18,23 +18,23 @@ fi #url_base=http://ftpmirror.gnu.org url_base=http://mirrors.ustc.edu.cn/gnu -gcc_version=9.1.0 +lcc_version=7.5.0 gcc_tarball=gcc-$gcc_version.tar.xz gcc_url=$url_base/gcc/gcc-$gcc_version/$gcc_tarball -gmp_version=6.1.2 +gmp_version=5.1.3 gmp_tarball=gmp-$gmp_version.tar.xz gmp_url=$url_base/gmp/$gmp_tarball -mpfr_version=4.0.2 +mpfr_version=3.1.4 mpfr_tarball=mpfr-$mpfr_version.tar.xz mpfr_url=$url_base/mpfr/$mpfr_tarball -mpc_version=1.1.0 +mpc_version=1.0.3 mpc_tarball=mpc-$mpc_version.tar.gz mpc_url=$url_base/mpc/$mpc_tarball -bu_version=2.32 +bu_version=2.28.1 bu_tarball=binutils-$bu_version.tar.xz bu_url=$url_base/binutils/$bu_tarball @@ -202,7 +202,7 @@ function finalize { function make_package { glibc_version=$(ldd --version | head -1 | cut -d ' ' -f4) - exec_file=$build_dir/vesoft-gcc-linux-x86_64-glibc-$glibc_version.sh + exec_file=$build_dir/vesoft-gcc-$gcc_version-linux-x86_64-glibc-$glibc_version.sh echo "Creating self-extracting package $exec_file" cat > $exec_file < Date: Tue, 3 Dec 2019 11:14:35 +0800 Subject: [PATCH 26/80] fix typo --- third-party/build-gcc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/third-party/build-gcc.sh b/third-party/build-gcc.sh index 3f442b9601b..a8e1b66997b 100755 --- a/third-party/build-gcc.sh +++ b/third-party/build-gcc.sh @@ -18,7 +18,7 @@ fi #url_base=http://ftpmirror.gnu.org url_base=http://mirrors.ustc.edu.cn/gnu -lcc_version=7.5.0 +gcc_version=7.5.0 gcc_tarball=gcc-$gcc_version.tar.xz gcc_url=$url_base/gcc/gcc-$gcc_version/$gcc_tarball From 8fa93eae8279902be08519360e97debce55be453 Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Tue, 3 Dec 2019 12:07:48 +0800 Subject: [PATCH 27/80] Fix checksum --- third-party/build-gcc.sh | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/third-party/build-gcc.sh b/third-party/build-gcc.sh index a8e1b66997b..d6873e1a8f1 100755 --- a/third-party/build-gcc.sh +++ b/third-party/build-gcc.sh @@ -38,11 +38,11 @@ bu_version=2.28.1 bu_tarball=binutils-$bu_version.tar.xz bu_url=$url_base/binutils/$bu_tarball -gcc_checksum=6069ae3737cf02bf2cb44a391ef0e937 -gmp_checksum=f58fa8001d60c4c77595fbbb62b63c1d -mpfr_checksum=320fbc4463d4c8cb1e566929d8adc4f8 -mpc_checksum=4125404e41e482ec68282a2e687f6c73 -bu_checksum=0d174cdaf85721c5723bf52355be41e6 +gcc_checksum=79cb8a65d44dfc8a2402b46395535c9a +gmp_checksum=e5fe367801ff067b923d1e6a126448aa +mpfr_checksum=064b2c18185038e404a401b830d59be8 +mpc_checksum=d6a1d5f8ddea3abd2cc3e98f58352d26 +bu_checksum=a3bf359889e4b299fce1f4cb919dc7b6 cur_dir=$PWD build_dir=$PWD/gcc-build @@ -71,10 +71,9 @@ function fetch_tarball { function fetch_tarballs { hash wget &> /dev/null && download_cmd="wget -c" - hash axel &> /dev/null && download_cmd="axel -a -n 8" if [[ -z $download_cmd ]] then - echo "Neither 'wget' nor 'axel' available for downloading" 1>&2; + echo "'wget' not found for downloading" 1>&2; exit 1; fi @@ -175,7 +174,6 @@ function configure_gcc { --disable-multilib \ --disable-install-libiberty \ --disable-werror \ - --with-system-readline \ --with-system-zlib #--host=x86_64-vesoft-linux \ #--build=x86_64-vesoft-linux \ @@ -185,14 +183,14 @@ function configure_gcc { function build_gcc { cd $object_dir - make MAKEINFO=true -j 20 |& tee build.log + make -j 20 |& tee build.log [[ $? -ne 0 ]] && exit 1 cd $OLDPWD } function install_gcc { cd $object_dir - make MAKEINFO=true install-strip + make install-strip cd $OLDPWD } From 91c9702e5e274698074bbd5c4a169fd55f316703 Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Tue, 3 Dec 2019 17:23:07 +0800 Subject: [PATCH 28/80] Fixed building under GCC 4.4.7 --- third-party/build-gcc.sh | 102 +++++++++++++++++++++---------- third-party/build-third-party.sh | 2 +- 2 files changed, 71 insertions(+), 33 deletions(-) diff --git a/third-party/build-gcc.sh b/third-party/build-gcc.sh index d6873e1a8f1..08b11f4cbaa 100755 --- a/third-party/build-gcc.sh +++ b/third-party/build-gcc.sh @@ -48,9 +48,11 @@ cur_dir=$PWD build_dir=$PWD/gcc-build tarballs_dir=$build_dir/downloads source_dir=$build_dir/source -object_dir=$build_dir/build +gcc_object_dir=$build_dir/gcc-build +bu_object_dir=$build_dir/binutils-build prefix=$1 install_dir=${prefix:-$build_dir/install}/gcc/$gcc_version +triplet=x86_64-vesoft-linux function get_checksum { md5sum $1 | cut -d ' ' -f 1 @@ -141,14 +143,6 @@ function setup_deps { #EOF #chmod +x config.guess - ln -sf ../binutils-$bu_version/binutils binutils - ln -sf ../binutils-$bu_version/gas gas - ln -sf ../binutils-$bu_version/gold gold - ln -sf ../binutils-$bu_version/gprof gprof - ln -sf ../binutils-$bu_version/ld ld - ln -sf ../binutils-$bu_version/bfd bfd - ln -sf ../binutils-$bu_version/opcodes opcodes - cd $OLDPWD cd $source_dir/gmp-$gmp_version @@ -159,38 +153,76 @@ function setup_deps { } function configure_gcc { - mkdir -p $object_dir - cd $object_dir - $source_dir/gcc-$gcc_version/configure --prefix=$install_dir \ - --with-pkgversion="Nebula Graph Build" \ - --enable-shared \ - --enable-threads=posix \ - --enable-__cxa_atexit \ - --enable-clocale=gnu \ - --enable-languages=c,c++ \ - --enable-lto \ - --disable-bootstrap \ - --disable-nls \ - --disable-multilib \ - --disable-install-libiberty \ - --disable-werror \ - --with-system-zlib - #--host=x86_64-vesoft-linux \ - #--build=x86_64-vesoft-linux \ + mkdir -p $gcc_object_dir + cd $gcc_object_dir + $source_dir/gcc-$gcc_version/configure \ + --prefix=$install_dir \ + --with-pkgversion="Nebula Graph Build" \ + --enable-shared \ + --enable-threads=posix \ + --enable-__cxa_atexit \ + --enable-clocale=gnu \ + --enable-languages=c,c++ \ + --enable-lto \ + --enable-bootstrap \ + --disable-nls \ + --disable-multilib \ + --disable-install-libiberty \ + --with-system-zlib \ + --build=$triplet \ + --host=$triplet \ + --host=$triplet \ + --disable-werror [[ $? -eq 0 ]] || exit 1 cd $OLDPWD } function build_gcc { - cd $object_dir - make -j 20 |& tee build.log + cd $gcc_object_dir + make -s -j 20 |& tee build.log [[ $? -ne 0 ]] && exit 1 cd $OLDPWD } function install_gcc { - cd $object_dir - make install-strip + cd $gcc_object_dir + make -s -j8 install-strip + [[ $? -ne 0 ]] && exit 1 + cd $OLDPWD +} + +function configure_binutils { + mkdir -p $bu_object_dir + cd $bu_object_dir + $source_dir/binutils-$bu_version/configure \ + --prefix=$install_dir \ + --with-pkgversion="Nebula Graph Build" \ + --disable-shared \ + --disable-nls \ + --enable-gold \ + --enable-ld=default \ + --with-system-zlib \ + --build=$triplet \ + --host=$triplet \ + --host=$triplet \ + --disable-werror + + cd $OLDPWD +} + +function build_binutils { + cd $bu_object_dir + make -s -j 20 || exit 1 + cd $OLDPWD +} + +function install_binutils { + cd $bu_object_dir + make -s install-strip || exit 1 + cd $OLDPWD + cd $install_dir + cp -vp bin/as libexec/gcc/$triplet/$gcc_version + cp -vp bin/ld* libexec/gcc/$triplet/$gcc_version cd $OLDPWD } @@ -199,7 +231,7 @@ function finalize { } function make_package { - glibc_version=$(ldd --version | head -1 | cut -d ' ' -f4) + glibc_version=$(ldd --version | head -1 | cut -d ' ' -f4 | cut -d '-' -f1) exec_file=$build_dir/vesoft-gcc-$gcc_version-linux-x86_64-glibc-$glibc_version.sh echo "Creating self-extracting package $exec_file" cat > $exec_file < $install_dir/bin/enable-gcc.sh < /dev/null && cxx_cmd=g++ [[ -n $CXX ]] && cxx_cmd=$CXX [[ -z $cxx_cmd ]] && { echo "No C++ compiler found" 1>&2; exit 1; } - cxx_version=$($cxx_cmd -dumpfullversion) + cxx_version=$($cxx_cmd -dumpfullversion -dumpversion 2>/dev/null) local least_cxx_version=7.0.0 if [[ $(version_cmp $cxx_version $least_cxx_version) -lt 0 ]] then From 685d2445f48c40f929644f796492823bdcffe69b Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Tue, 3 Dec 2019 18:02:00 +0800 Subject: [PATCH 29/80] Added some comments --- third-party/build-gcc.sh | 39 +++++++++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/third-party/build-gcc.sh b/third-party/build-gcc.sh index 08b11f4cbaa..5c9583e0924 100755 --- a/third-party/build-gcc.sh +++ b/third-party/build-gcc.sh @@ -15,6 +15,7 @@ then exit $? fi +# GCC, binutils and support libraries #url_base=http://ftpmirror.gnu.org url_base=http://mirrors.ustc.edu.cn/gnu @@ -44,16 +45,18 @@ mpfr_checksum=064b2c18185038e404a401b830d59be8 mpc_checksum=d6a1d5f8ddea3abd2cc3e98f58352d26 bu_checksum=a3bf359889e4b299fce1f4cb919dc7b6 +# Building directories setup cur_dir=$PWD -build_dir=$PWD/gcc-build -tarballs_dir=$build_dir/downloads -source_dir=$build_dir/source -gcc_object_dir=$build_dir/gcc-build -bu_object_dir=$build_dir/binutils-build +root_dir=$PWD/gcc-build +tarballs_dir=$root_dir/downloads +source_dir=$root_dir/source +gcc_object_dir=$root_dir/gcc-build +bu_object_dir=$root_dir/binutils-build prefix=$1 -install_dir=${prefix:-$build_dir/install}/gcc/$gcc_version +install_dir=${prefix:-$root_dir/install}/gcc/$gcc_version triplet=x86_64-vesoft-linux +# Download source tarballs function get_checksum { md5sum $1 | cut -d ' ' -f 1 } @@ -90,6 +93,7 @@ function fetch_tarballs { cd $OLDPWD } +# Unpack source tarballs function unpack_tarballs { mkdir -p $source_dir cd $tarballs_dir @@ -129,6 +133,7 @@ function unpack_tarballs { cd $OLDPWD } +# Necessary dependency setup function setup_deps { cd $source_dir/gcc-$gcc_version ln -sf ../gmp-$gmp_version gmp @@ -145,6 +150,7 @@ function setup_deps { cd $OLDPWD + # Tweak GMP to not build with native code cd $source_dir/gmp-$gmp_version cp -f configfsf.guess config.guess cp -f configfsf.sub config.sub @@ -152,6 +158,7 @@ function setup_deps { cd $OLDPWD } +# Configure GCC function configure_gcc { mkdir -p $gcc_object_dir cd $gcc_object_dir @@ -171,12 +178,13 @@ function configure_gcc { --with-system-zlib \ --build=$triplet \ --host=$triplet \ - --host=$triplet \ + --target=$triplet \ --disable-werror [[ $? -eq 0 ]] || exit 1 cd $OLDPWD } +# Start building GCC function build_gcc { cd $gcc_object_dir make -s -j 20 |& tee build.log @@ -184,6 +192,7 @@ function build_gcc { cd $OLDPWD } +# Install GCC function install_gcc { cd $gcc_object_dir make -s -j8 install-strip @@ -191,6 +200,7 @@ function install_gcc { cd $OLDPWD } +# Configure binutils function configure_binutils { mkdir -p $bu_object_dir cd $bu_object_dir @@ -204,36 +214,43 @@ function configure_binutils { --with-system-zlib \ --build=$triplet \ --host=$triplet \ - --host=$triplet \ + --target=$triplet \ --disable-werror cd $OLDPWD } +# Build binutils function build_binutils { cd $bu_object_dir make -s -j 20 || exit 1 cd $OLDPWD } +# Install binutils function install_binutils { cd $bu_object_dir make -s install-strip || exit 1 cd $OLDPWD cd $install_dir + # Place a copy of assembler and linker to libexec cp -vp bin/as libexec/gcc/$triplet/$gcc_version cp -vp bin/ld* libexec/gcc/$triplet/$gcc_version cd $OLDPWD } +# Finalize the building function finalize { + # Remove all of the annoying libtool files, + # so that the installation would be copied around find $install_dir -name '*.la' | xargs rm -f } +# Build a self-extractable package function make_package { glibc_version=$(ldd --version | head -1 | cut -d ' ' -f4 | cut -d '-' -f1) - exec_file=$build_dir/vesoft-gcc-$gcc_version-linux-x86_64-glibc-$glibc_version.sh - echo "Creating self-extracting package $exec_file" + exec_file=$root_dir/vesoft-gcc-$gcc_version-linux-x86_64-glibc-$glibc_version.sh + echo "Creating self-extractable package $exec_file" cat > $exec_file < $install_dir/bin/disable-gcc.sh < Date: Wed, 4 Dec 2019 11:03:38 +0800 Subject: [PATCH 30/80] Add distro name --- third-party/build-cmake.sh | 2 +- third-party/build-gcc.sh | 7 ++++--- third-party/build-third-party.sh | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/third-party/build-cmake.sh b/third-party/build-cmake.sh index 488ebe6fc95..f969975146b 100755 --- a/third-party/build-cmake.sh +++ b/third-party/build-cmake.sh @@ -35,7 +35,7 @@ then hash axel &> /dev/null && download_cmd="axel -a -n 8" if [[ -z $download_cmd ]] then - echo "Neither 'wget' nor 'axel' available for downloading" 1>&2; + echo "'wget' not found for downloading" 1>&2; exit 1; fi diff --git a/third-party/build-gcc.sh b/third-party/build-gcc.sh index 5c9583e0924..8504f1b3147 100755 --- a/third-party/build-gcc.sh +++ b/third-party/build-gcc.sh @@ -55,6 +55,7 @@ bu_object_dir=$root_dir/binutils-build prefix=$1 install_dir=${prefix:-$root_dir/install}/gcc/$gcc_version triplet=x86_64-vesoft-linux +distro=$(lsb_release -si) # Download source tarballs function get_checksum { @@ -249,7 +250,7 @@ function finalize { # Build a self-extractable package function make_package { glibc_version=$(ldd --version | head -1 | cut -d ' ' -f4 | cut -d '-' -f1) - exec_file=$root_dir/vesoft-gcc-$gcc_version-linux-x86_64-glibc-$glibc_version.sh + exec_file=$root_dir/vesoft-gcc-$gcc_version-$distro-x86_64-glibc-$glibc_version.sh echo "Creating self-extractable package $exec_file" cat > $exec_file < $install_dir/bin/enable-gcc.sh < $install_dir/enable-gcc.sh < $install_dir/bin/disable-gcc.sh < $install_dir/disable-gcc.sh <&2 + echo "cmake $least_cmake_version or higher required" 1>&2 exit 1 fi } From 0ce40b99a581c3051842179a7de0a180149b5d18 Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Wed, 4 Dec 2019 11:28:14 +0800 Subject: [PATCH 31/80] Remove --skip-old-files due to lack support by older tar command --- third-party/build-gcc.sh | 10 +++++----- third-party/build-third-party.sh | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/third-party/build-gcc.sh b/third-party/build-gcc.sh index 8504f1b3147..91a7c4193af 100755 --- a/third-party/build-gcc.sh +++ b/third-party/build-gcc.sh @@ -103,31 +103,31 @@ function unpack_tarballs { if [[ ! -d $source_dir/gcc-$gcc_version ]] then echo "Unpacking $gcc_tarball..." - tar --skip-old-files -xf $gcc_tarball -C $source_dir + tar -xf $gcc_tarball -C $source_dir fi if [[ ! -d $source_dir/gmp-$gmp_version ]] then echo "Unpacking $gmp_tarball..." - tar --skip-old-files -xf $gmp_tarball -C $source_dir + tar -xf $gmp_tarball -C $source_dir fi if [[ ! -d $source_dir/mpfr-$mpfr_version ]] then echo "Unpacking $mpfr_tarball..." - tar --skip-old-files -xf $mpfr_tarball -C $source_dir + tar -xf $mpfr_tarball -C $source_dir fi if [[ ! -d $source_dir/mpc-$mpc_version ]] then echo "Unpacking $mpc_tarball..." - tar --skip-old-files -xf $mpc_tarball -C $source_dir + tar -xf $mpc_tarball -C $source_dir fi if [[ ! -d $source_dir/binutils-$bu_version ]] then echo "Unpacking $bu_tarball..." - tar --skip-old-files -xf $bu_tarball -C $source_dir + tar -xf $bu_tarball -C $source_dir fi set +e diff --git a/third-party/build-third-party.sh b/third-party/build-third-party.sh index 88fd0801651..4a9d3070277 100755 --- a/third-party/build-third-party.sh +++ b/third-party/build-third-party.sh @@ -105,11 +105,11 @@ then else echo "Source of third party was downdloaded to $build_root" echo -n "Extracting into $download_dir..." - tar --skip-old-files -xzf $source_tar_name + tar -xzf $source_tar_name echo "done" fi else - tar --skip-old-files -xzf $source_tar_name + tar -xzf $source_tar_name fi # Build and install From 8471e19603c36791441fced325a56f78dfca20f6 Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Wed, 4 Dec 2019 11:49:50 +0800 Subject: [PATCH 32/80] Added guess-building-jobs-num.sh --- third-party/build-gcc.sh | 11 ++++++++--- third-party/guess-building-jobs-num.sh | 10 ++++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) create mode 100644 third-party/guess-building-jobs-num.sh diff --git a/third-party/build-gcc.sh b/third-party/build-gcc.sh index 91a7c4193af..9f700d645b8 100755 --- a/third-party/build-gcc.sh +++ b/third-party/build-gcc.sh @@ -15,6 +15,8 @@ then exit $? fi +this_dir=$(dirname $(readlink -f $0)) + # GCC, binutils and support libraries #url_base=http://ftpmirror.gnu.org url_base=http://mirrors.ustc.edu.cn/gnu @@ -57,6 +59,9 @@ install_dir=${prefix:-$root_dir/install}/gcc/$gcc_version triplet=x86_64-vesoft-linux distro=$(lsb_release -si) +# Guess number of building jobs +source $this_dir/guess-building-jobs-num.sh + # Download source tarballs function get_checksum { md5sum $1 | cut -d ' ' -f 1 @@ -188,7 +193,7 @@ function configure_gcc { # Start building GCC function build_gcc { cd $gcc_object_dir - make -s -j 20 |& tee build.log + make -s -j $building_jobs_num |& tee build.log [[ $? -ne 0 ]] && exit 1 cd $OLDPWD } @@ -196,7 +201,7 @@ function build_gcc { # Install GCC function install_gcc { cd $gcc_object_dir - make -s -j8 install-strip + make -s -j $building_jobs_num install-strip [[ $? -ne 0 ]] && exit 1 cd $OLDPWD } @@ -224,7 +229,7 @@ function configure_binutils { # Build binutils function build_binutils { cd $bu_object_dir - make -s -j 20 || exit 1 + make -s -j $building_jobs_num || exit 1 cd $OLDPWD } diff --git a/third-party/guess-building-jobs-num.sh b/third-party/guess-building-jobs-num.sh new file mode 100644 index 00000000000..4ecbbd0cfce --- /dev/null +++ b/third-party/guess-building-jobs-num.sh @@ -0,0 +1,10 @@ +available_mem_mb=$(awk '/^MemFree|^Buffers|^Cached/{s+=$2;}END{print s/1024;}' /proc/meminfo 2>/dev/null) +physical_cores=$(lscpu | awk 'BEGIN{s=1;}/Core\(s\) per socket|Socket\(s\)/{s*=$NF;}END{print s;}' 2>/dev/null) +building_jobs_num=0 + +[[ -n $available_mem_mb ]] && jobs_by_mem=$((available_mem_mb / 1024 / 3)) +[[ -n $physical_cores ]] && jobs_by_cpu=$physical_cores + +[[ -n $jobs_by_mem ]] && building_jobs_num=$jobs_by_mem +[[ -n $jobs_by_cpu ]] && [[ $jobs_by_cpu -lt $jobs_by_mem ]] && building_jobs_num=$jobs_by_cpu +[[ $building_jobs_num -eq 0 ]] && building_jobs_num=1 From 7cd974c07bb36bb2184c5170ba287f1d63cb8747 Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Wed, 4 Dec 2019 14:01:09 +0800 Subject: [PATCH 33/80] Added compiler testing script --- third-party/cxx-compiler-usability-test.sh | 90 ++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100755 third-party/cxx-compiler-usability-test.sh diff --git a/third-party/cxx-compiler-usability-test.sh b/third-party/cxx-compiler-usability-test.sh new file mode 100755 index 00000000000..1fee20b3061 --- /dev/null +++ b/third-party/cxx-compiler-usability-test.sh @@ -0,0 +1,90 @@ +#! /usr/bin/env bash + +# Copyright (c) 2019 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. + +cxx_cmd=${CXX:-g++} +link_flags="-std=c++14 -static-libstdc++ -static-libgcc" + +# Compile source from stdin and run. +# input envs: +# $output, which is the expected output. +# $exit_code, which is the exit status of the compiled program. +function compile-and-run { + local tmpdir=$(mktemp -q -d /tmp/nebula-compiler-test.XXXX 2>/dev/null) + [[ -z $tmpdir ]] && { + echo "Failed" + echo "Create directory /tmp/nebula-compiler-test.XXXX failed." + exit 1; + } + + local exe=$tmpdir/a.out + local logfile=$tmpdir/testing.log + $cxx_cmd $link_flags $extra_flags -g0 -x c++ - -o $exe &> $logfile + + [[ $? -eq 0 ]] || { + echo "Failed" + echo "See $logfile for details." 1>&2; + exit 1; + } + + true_output=$($exe 2>&1) + true_exit_code=$? + # Convert non-zero to 1 + [[ $true_exit_code -ne 0 ]] && true_exit_code=1 + + if [[ -n "$output" ]] && [[ ! "$true_output" = "$output" ]] + then + echo "expected: $output" >> $logfile + echo "actual: $true_output" >> $logfile + echo "Failed" + echo "See $logfile for details." 1>&2; + exit 1; + fi + + if [[ -n "$exit_code" ]] && [[ $true_exit_code -ne $exit_code ]] + then + echo "expected: $exit_code" >> $logfile + echo "actual: $true_exit_code" >> $logfile + echo "Failed" + echo "See $logfile for details." 1>&2; + exit 1; + fi + + rm -rf $tmpdir +} + +# make_unique, regex, (templated)lambda, move capture +echo -n "Performing regular C++14 tests..." +output="Nebula" exit_code=0 compile-and-run < +#include +#include +#include +int main() { + auto hello = std::make_unique("Hello"); + auto _666_ = [hello = std::move(hello)] (auto bravo) { + return *hello + " " + *bravo; + } (std::make_unique("Nebula Graph")); + std::smatch sm; + if (std::regex_match(_666_, sm, + std::regex(R"(^(Hello) (Nebula) (Graph)$)"))) { + std::cout << sm[2].str() << std::endl; + } + return 0; +} +EOF +echo "OK" + +echo -n "Performing LeakSanitizer tests..." +exit_code=1 extra_flags="-fsanitize=address -g" compile-and-run < Date: Wed, 4 Dec 2019 16:59:09 +0800 Subject: [PATCH 34/80] Added testing after build and install --- third-party/build-gcc.sh | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/third-party/build-gcc.sh b/third-party/build-gcc.sh index 9f700d645b8..7e3c5cd7a00 100755 --- a/third-party/build-gcc.sh +++ b/third-party/build-gcc.sh @@ -252,6 +252,13 @@ function finalize { find $install_dir -name '*.la' | xargs rm -f } +# Test usability of GCC +function usability_test { + CXX=$install_dir/bin/g++ $this_dir/cxx-compiler-usability-test.sh + [[ $? -eq 0 ]] || exit 1 + cp -p $this_dir/cxx-compiler-usability-test.sh $install_dir +} + # Build a self-extractable package function make_package { glibc_version=$(ldd --version | head -1 | cut -d ' ' -f4 | cut -d '-' -f1) @@ -262,7 +269,7 @@ function make_package { set -e [[ \$# -ne 0 ]] && prefix=\$(echo "\$@" | sed 's;.*--prefix=(\S*).*;\1;' -r) -prefix=\${prefix:-/usr/local} +prefix=\${prefix:-/opt/nebula/toolchain} hash xz &> /dev/null || { echo "xz: Command not found"; exit 1; } @@ -272,6 +279,12 @@ mkdir -p \$prefix archive_offset=\$(awk '/^__start_of_archive__$/{print NR+1; exit 0;}' \$0) tail -n+\$archive_offset \$0 | tar --numeric-owner -xJf - -C \$prefix +echo "GCC-$gcc_version has been installed to \$prefix/gcc/$gcc_version" +echo "Performing usability tests" +\$prefix/gcc/$gcc_version/cxx-compiler-usability-test.sh +echo "All tests passed, now you are ready to go." +echo "Run '\$prefix/gcc/$gcc_version/enable-gcc.sh' to enable it." + exit 0 __start_of_archive__ @@ -283,6 +296,7 @@ EOF } start_time=$(date +%s) + fetch_tarballs unpack_tarballs setup_deps @@ -297,13 +311,15 @@ install_binutils finalize +usability_test + cat > $install_dir/enable-gcc.sh < $install_dir/disable-gcc.sh < Date: Wed, 4 Dec 2019 17:05:52 +0800 Subject: [PATCH 35/80] Change enable-gcc.sh to enable --- third-party/build-gcc.sh | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/third-party/build-gcc.sh b/third-party/build-gcc.sh index 7e3c5cd7a00..7ff9b884f4c 100755 --- a/third-party/build-gcc.sh +++ b/third-party/build-gcc.sh @@ -282,8 +282,7 @@ tail -n+\$archive_offset \$0 | tar --numeric-owner -xJf - -C \$prefix echo "GCC-$gcc_version has been installed to \$prefix/gcc/$gcc_version" echo "Performing usability tests" \$prefix/gcc/$gcc_version/cxx-compiler-usability-test.sh -echo "All tests passed, now you are ready to go." -echo "Run '\$prefix/gcc/$gcc_version/enable-gcc.sh' to enable it." +echo "Run '\$prefix/gcc/$gcc_version/enable' to start using." exit 0 @@ -313,7 +312,7 @@ finalize usability_test -cat > $install_dir/enable-gcc.sh < $install_dir/enable < $install_dir/disable-gcc.sh < $install_dir/disable < Date: Wed, 4 Dec 2019 17:38:50 +0800 Subject: [PATCH 36/80] Fix guessing building jobs num --- third-party/guess-building-jobs-num.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/third-party/guess-building-jobs-num.sh b/third-party/guess-building-jobs-num.sh index 4ecbbd0cfce..8eb8832cd90 100644 --- a/third-party/guess-building-jobs-num.sh +++ b/third-party/guess-building-jobs-num.sh @@ -1,8 +1,8 @@ -available_mem_mb=$(awk '/^MemFree|^Buffers|^Cached/{s+=$2;}END{print s/1024;}' /proc/meminfo 2>/dev/null) +available_mem_mb=$(awk '/^MemFree|^Buffers|^Cached/{s+=$2;}END{print int(s/1024);}' /proc/meminfo 2>/dev/null) physical_cores=$(lscpu | awk 'BEGIN{s=1;}/Core\(s\) per socket|Socket\(s\)/{s*=$NF;}END{print s;}' 2>/dev/null) building_jobs_num=0 -[[ -n $available_mem_mb ]] && jobs_by_mem=$((available_mem_mb / 1024 / 3)) +[[ -n $available_mem_mb ]] && jobs_by_mem=$((available_mem_mb / 1024 / 2)) [[ -n $physical_cores ]] && jobs_by_cpu=$physical_cores [[ -n $jobs_by_mem ]] && building_jobs_num=$jobs_by_mem From b290cadc3b30380e847f7c5cc7856b26d961c641 Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Wed, 4 Dec 2019 22:34:57 +0800 Subject: [PATCH 37/80] Make build log less verbose --- third-party/build-gcc.sh | 34 ++++++++++++++++++++------------ third-party/build-third-party.sh | 15 ++++++++++---- 2 files changed, 32 insertions(+), 17 deletions(-) diff --git a/third-party/build-gcc.sh b/third-party/build-gcc.sh index 7ff9b884f4c..1ec7d7c7d91 100755 --- a/third-party/build-gcc.sh +++ b/third-party/build-gcc.sh @@ -56,6 +56,7 @@ gcc_object_dir=$root_dir/gcc-build bu_object_dir=$root_dir/binutils-build prefix=$1 install_dir=${prefix:-$root_dir/install}/gcc/$gcc_version +logfile=$root_dir/build.log triplet=x86_64-vesoft-linux distro=$(lsb_release -si) @@ -269,7 +270,7 @@ function make_package { set -e [[ \$# -ne 0 ]] && prefix=\$(echo "\$@" | sed 's;.*--prefix=(\S*).*;\1;' -r) -prefix=\${prefix:-/opt/nebula/toolchain} +prefix=\${prefix:-/opt/nebula/toolset} hash xz &> /dev/null || { echo "xz: Command not found"; exit 1; } @@ -281,8 +282,8 @@ tail -n+\$archive_offset \$0 | tar --numeric-owner -xJf - -C \$prefix echo "GCC-$gcc_version has been installed to \$prefix/gcc/$gcc_version" echo "Performing usability tests" -\$prefix/gcc/$gcc_version/cxx-compiler-usability-test.sh -echo "Run '\$prefix/gcc/$gcc_version/enable' to start using." +CXX=\$prefix/gcc/$gcc_version/bin/g++ \$prefix/gcc/$gcc_version/cxx-compiler-usability-test.sh +echo "Run 'source \$prefix/gcc/$gcc_version/enable' to start using." exit 0 @@ -296,19 +297,26 @@ EOF start_time=$(date +%s) -fetch_tarballs -unpack_tarballs -setup_deps +echo "Starting build, on any failure, see $logfile" -configure_gcc -build_gcc -install_gcc +{ + fetch_tarballs + unpack_tarballs + setup_deps +} |& tee $logfile -configure_binutils -build_binutils -install_binutils +{ + configure_gcc + build_gcc + install_gcc -finalize + configure_binutils + build_binutils + install_binutils + + finalize +} |& tee $logfile \ + | grep --line-buffered '^Making\|^Configuring\|^Comaparing\|^Comparison' usability_test diff --git a/third-party/build-third-party.sh b/third-party/build-third-party.sh index 4a9d3070277..5cf4b83375c 100755 --- a/third-party/build-third-party.sh +++ b/third-party/build-third-party.sh @@ -66,7 +66,6 @@ check_cxx # Exit on any failure here after set -e -set -o pipefail # Directories setup cur_dir=`pwd` @@ -77,6 +76,7 @@ install_dir=$build_root/install download_dir=$build_root/downloads source_tar_name=nebula-third-party-src-1.0.tgz source_url=https://nebula-graph.oss-accelerate.aliyuncs.com/third-party/${source_tar_name} +logfile=$build_root/build.log # Allow to customize compilers [[ -n ${CC} ]] && C_COMPILER_ARG="-DCMAKE_C_COMPILER=${CC}" @@ -116,12 +116,19 @@ fi mkdir -p $build_dir $install_dir cd $build_dir +echo "Starting build, on any failure, please see $logfile" + cmake -DDOWNLOAD_DIR=$download_dir \ -DCMAKE_INSTALL_PREFIX=$install_dir \ ${C_COMPILER_ARG} ${CXX_COMPILER_ARG} \ - $source_dir - -make + $source_dir |& tee $logfile + +make |& \ + tee $logfile | \ + grep --line-buffered '^Scanning\|^\[.*Built' | \ + awk '/Scanning dependencies of target/ {printf "...... Build target %s\n", $NF; } \ + /Built/ {print $0;}' +#make |& tee -a $logfile | grep '^Scanning\|^\[.*Built' | awk '{print $0;}' end_time=$(date +%s) echo From 6c47611c9b7911c0724b43f18eece77f00f4c0c3 Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Wed, 4 Dec 2019 23:10:40 +0800 Subject: [PATCH 38/80] Disable CMakel log --- third-party/CMakeLists.txt | 1 - third-party/build-gcc.sh | 3 ++- third-party/externals/bison.cmake | 2 -- third-party/externals/boost.cmake | 2 -- third-party/externals/bzip2.cmake | 2 -- third-party/externals/capstone.cmake | 2 -- third-party/externals/double-conversion.cmake | 2 -- third-party/externals/fatal.cmake | 4 ---- third-party/externals/fbthrift.cmake | 4 ---- third-party/externals/flex.cmake | 2 -- third-party/externals/folly.cmake | 4 ---- third-party/externals/gettext.cmake | 2 -- third-party/externals/gflags.cmake | 2 -- third-party/externals/glog.cmake | 2 -- third-party/externals/googletest.cmake | 2 -- third-party/externals/gperf.cmake | 2 -- third-party/externals/jemalloc.cmake | 2 -- third-party/externals/keyutils.cmake | 2 -- third-party/externals/krb5.cmake | 2 -- third-party/externals/libaio.cmake | 2 -- third-party/externals/libevent.cmake | 2 -- third-party/externals/libunwind.cmake | 2 -- third-party/externals/lz4.cmake | 2 -- third-party/externals/lzma.cmake | 2 -- third-party/externals/mstch.cmake | 2 -- third-party/externals/openssl.cmake | 2 -- third-party/externals/proxygen.cmake | 4 ---- third-party/externals/s2geometry.cmake | 4 ---- third-party/externals/snappy.cmake | 2 -- third-party/externals/wangle.cmake | 4 ---- third-party/externals/zlib.cmake | 2 -- third-party/externals/zstd.cmake | 2 -- 32 files changed, 2 insertions(+), 74 deletions(-) diff --git a/third-party/CMakeLists.txt b/third-party/CMakeLists.txt index 91c3c0d8864..88eaffab91e 100644 --- a/third-party/CMakeLists.txt +++ b/third-party/CMakeLists.txt @@ -69,7 +69,6 @@ set(common_cmake_args set(common_configure_args --prefix=${CMAKE_INSTALL_PREFIX} - --quiet ) set(common_configure_envs diff --git a/third-party/build-gcc.sh b/third-party/build-gcc.sh index 1ec7d7c7d91..83103c9b04a 100755 --- a/third-party/build-gcc.sh +++ b/third-party/build-gcc.sh @@ -62,6 +62,7 @@ distro=$(lsb_release -si) # Guess number of building jobs source $this_dir/guess-building-jobs-num.sh +echo "Building GCC-$gcc_version with $building_jobs_num jobs" # Download source tarballs function get_checksum { @@ -343,7 +344,7 @@ EOF make_package end_time=$(date +%s) -echo "GCC-$gcc_version has been installed to prefix=$install_dir" +echo "GCC-$gcc_version has been installed to $install_dir" echo "$((end_time - start_time)) seconds been taken." echo "Run 'source $install_dir/enable' to start using." echo "Run 'source $install_dir/disable' to disable it." diff --git a/third-party/externals/bison.cmake b/third-party/externals/bison.cmake index 7eb8d16dae9..10891404a83 100644 --- a/third-party/externals/bison.cmake +++ b/third-party/externals/bison.cmake @@ -21,8 +21,6 @@ ExternalProject_Add( BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} BUILD_IN_SOURCE 1 INSTALL_COMMAND make -s install -j${BUILDING_JOBS_NUM} - LOG_BUILD 1 - LOG_INSTALL 1 ) ExternalProject_Add_Step(${name} clean diff --git a/third-party/externals/boost.cmake b/third-party/externals/boost.cmake index d197f535870..0c8143024c2 100644 --- a/third-party/externals/boost.cmake +++ b/third-party/externals/boost.cmake @@ -36,8 +36,6 @@ ExternalProject_Add( variant=release BUILD_IN_SOURCE 1 INSTALL_COMMAND "" - LOG_BUILD 0 - LOG_INSTALL 0 ) ExternalProject_Add_Step(${name} setup-compiler diff --git a/third-party/externals/bzip2.cmake b/third-party/externals/bzip2.cmake index d13efa2ccc0..57c6ba544eb 100644 --- a/third-party/externals/bzip2.cmake +++ b/third-party/externals/bzip2.cmake @@ -18,8 +18,6 @@ ExternalProject_Add( BUILD_COMMAND make CFLAGS=-fPIC -s -j${BUILDING_JOBS_NUM} BUILD_IN_SOURCE 1 INSTALL_COMMAND make -s install -j${BUILDING_JOBS_NUM} PREFIX=${CMAKE_INSTALL_PREFIX} - LOG_BUILD 1 - LOG_INSTALL 1 ) ExternalProject_Add_Step(${name} clean diff --git a/third-party/externals/capstone.cmake b/third-party/externals/capstone.cmake index b14610a5f23..552083ab0c5 100644 --- a/third-party/externals/capstone.cmake +++ b/third-party/externals/capstone.cmake @@ -40,8 +40,6 @@ ExternalProject_Add( BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} BUILD_IN_SOURCE 1 INSTALL_COMMAND make -s install -j${BUILDING_JOBS_NUM} - LOG_BUILD 1 - LOG_INSTALL 1 ) ExternalProject_Add_Step(${name} clean diff --git a/third-party/externals/double-conversion.cmake b/third-party/externals/double-conversion.cmake index 5cbb5e6a76c..15c743c3274 100644 --- a/third-party/externals/double-conversion.cmake +++ b/third-party/externals/double-conversion.cmake @@ -20,8 +20,6 @@ ExternalProject_Add( BUILD_IN_SOURCE 1 BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} INSTALL_COMMAND make -s install -j${BUILDING_JOBS_NUM} - LOG_BUILD 1 - LOG_INSTALL 1 ) ExternalProject_Add_Step(${name} clean diff --git a/third-party/externals/fatal.cmake b/third-party/externals/fatal.cmake index da33cbb2989..3460ade5539 100644 --- a/third-party/externals/fatal.cmake +++ b/third-party/externals/fatal.cmake @@ -19,10 +19,6 @@ ExternalProject_Add( BUILD_COMMAND install -d ${CMAKE_INSTALL_PREFIX}/include BUILD_IN_SOURCE 1 INSTALL_COMMAND cp -r fatal ${CMAKE_INSTALL_PREFIX}/include - LOG_CONFIGURE TRUE - LOG_BUILD TRUE - LOG_INSTALL TRUE - LOG_MERGED_STDOUTERR TRUE ) ExternalProject_Add_Step(${name} clean diff --git a/third-party/externals/fbthrift.cmake b/third-party/externals/fbthrift.cmake index 5aad9c848a2..0293f80c38d 100644 --- a/third-party/externals/fbthrift.cmake +++ b/third-party/externals/fbthrift.cmake @@ -24,10 +24,6 @@ ExternalProject_Add( BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} BUILD_IN_SOURCE 1 INSTALL_COMMAND make -s -j${BUILDING_JOBS_NUM} install - LOG_CONFIGURE TRUE - LOG_BUILD TRUE - LOG_INSTALL TRUE - LOG_MERGED_STDOUTERR TRUE ) ExternalProject_Add_Step(${name} clean diff --git a/third-party/externals/flex.cmake b/third-party/externals/flex.cmake index 831fbfdc173..e705f4b54ed 100644 --- a/third-party/externals/flex.cmake +++ b/third-party/externals/flex.cmake @@ -21,8 +21,6 @@ ExternalProject_Add( BUILD_COMMAND make -s BUILD_IN_SOURCE 1 INSTALL_COMMAND make -s install - LOG_BUILD 1 - LOG_INSTALL 1 ) ExternalProject_Add_Step(${name} clean diff --git a/third-party/externals/folly.cmake b/third-party/externals/folly.cmake index 038b07eec77..df160a4b5fa 100644 --- a/third-party/externals/folly.cmake +++ b/third-party/externals/folly.cmake @@ -24,10 +24,6 @@ ExternalProject_Add( BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} BUILD_IN_SOURCE 1 INSTALL_COMMAND make -s -j${BUILDING_JOBS_NUM} install - LOG_CONFIGURE TRUE - LOG_BUILD TRUE - LOG_INSTALL TRUE - LOG_MERGED_STDOUTERR TRUE ) ExternalProject_Add_Step(${name} clean diff --git a/third-party/externals/gettext.cmake b/third-party/externals/gettext.cmake index d2e9aba9c44..a2a3fed6344 100644 --- a/third-party/externals/gettext.cmake +++ b/third-party/externals/gettext.cmake @@ -21,8 +21,6 @@ ExternalProject_Add( BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} BUILD_IN_SOURCE 1 INSTALL_COMMAND make -s install -j${BUILDING_JOBS_NUM} - LOG_BUILD 1 - LOG_INSTALL 1 ) ExternalProject_Add_Step(${name} clean diff --git a/third-party/externals/gflags.cmake b/third-party/externals/gflags.cmake index 787def77eed..fed034f73a1 100644 --- a/third-party/externals/gflags.cmake +++ b/third-party/externals/gflags.cmake @@ -19,8 +19,6 @@ ExternalProject_Add( ${common_cmake_args} -DCMAKE_BUILD_TYPE=Release BUILD_IN_SOURCE 1 - LOG_BUILD 1 - LOG_INSTALL 1 ) ExternalProject_Add_Step(${name} clean diff --git a/third-party/externals/glog.cmake b/third-party/externals/glog.cmake index 633f07b128c..f123686bb36 100644 --- a/third-party/externals/glog.cmake +++ b/third-party/externals/glog.cmake @@ -23,8 +23,6 @@ ExternalProject_Add( BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} BUILD_IN_SOURCE 1 INSTALL_COMMAND make -s -j${BUILDING_JOBS_NUM} install - LOG_BUILD 1 - LOG_INSTALL 1 ) ExternalProject_Add_Step(glog pre-configure diff --git a/third-party/externals/googletest.cmake b/third-party/externals/googletest.cmake index 8f5bf4307fb..cc3dd63d124 100644 --- a/third-party/externals/googletest.cmake +++ b/third-party/externals/googletest.cmake @@ -18,8 +18,6 @@ ExternalProject_Add( BUILD_IN_SOURCE 1 CMAKE_ARGS ${common_cmake_args} - LOG_BUILD 1 - LOG_INSTALL 1 ) ExternalProject_Add_Step(${name} clean diff --git a/third-party/externals/gperf.cmake b/third-party/externals/gperf.cmake index 6e0eb0229eb..58206919376 100644 --- a/third-party/externals/gperf.cmake +++ b/third-party/externals/gperf.cmake @@ -22,8 +22,6 @@ ExternalProject_Add( BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} BUILD_IN_SOURCE 1 INSTALL_COMMAND make -s install -j${BUILDING_JOBS_NUM} - LOG_BUILD 1 - LOG_INSTALL 1 ) ExternalProject_Add_Step(${name} clean diff --git a/third-party/externals/jemalloc.cmake b/third-party/externals/jemalloc.cmake index 5f1471f92e3..6f44f061ad0 100644 --- a/third-party/externals/jemalloc.cmake +++ b/third-party/externals/jemalloc.cmake @@ -21,8 +21,6 @@ ExternalProject_Add( BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} BUILD_IN_SOURCE 1 INSTALL_COMMAND make -s install_bin install_include install_lib_static -j${BUILDING_JOBS_NUM} - LOG_BUILD 1 - LOG_INSTALL 1 ) ExternalProject_Add_Step(${name} clean diff --git a/third-party/externals/keyutils.cmake b/third-party/externals/keyutils.cmake index 2cc8679eb52..80eb008d4be 100644 --- a/third-party/externals/keyutils.cmake +++ b/third-party/externals/keyutils.cmake @@ -22,8 +22,6 @@ ExternalProject_Add( INCLUDEDIR=/include LIBDIR=/lib CFLAGS=-fPIC NO_SOLIB=1 DESTDIR=${CMAKE_INSTALL_PREFIX} install - LOG_BUILD 1 - LOG_INSTALL 1 ) ExternalProject_Add_Step(${name} clean diff --git a/third-party/externals/krb5.cmake b/third-party/externals/krb5.cmake index af54f249593..2ba0d457def 100644 --- a/third-party/externals/krb5.cmake +++ b/third-party/externals/krb5.cmake @@ -24,8 +24,6 @@ ExternalProject_Add( INSTALL_COMMAND env PATH=${BUILDING_PATH} make -s install -j${BUILDING_JOBS_NUM} -C src - LOG_BUILD 1 - LOG_INSTALL 1 ) ExternalProject_Add_Step(krb5 mannual-configure diff --git a/third-party/externals/libaio.cmake b/third-party/externals/libaio.cmake index b5dfdd42756..d9efad43393 100644 --- a/third-party/externals/libaio.cmake +++ b/third-party/externals/libaio.cmake @@ -18,8 +18,6 @@ ExternalProject_Add( BUILD_COMMAND env CFLAGS=-fPIC make -s -j${BUILDING_JOBS_NUM} BUILD_IN_SOURCE 1 INSTALL_COMMAND make prefix=${CMAKE_INSTALL_PREFIX} -s install -j${BUILDING_JOBS_NUM} - LOG_BUILD 1 - LOG_INSTALL 1 ) ExternalProject_Add_Step(${name} clean diff --git a/third-party/externals/libevent.cmake b/third-party/externals/libevent.cmake index 10b6dc8c137..19d98d13297 100644 --- a/third-party/externals/libevent.cmake +++ b/third-party/externals/libevent.cmake @@ -24,8 +24,6 @@ ExternalProject_Add( BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} BUILD_IN_SOURCE 1 INSTALL_COMMAND make -s install -j${BUILDING_JOBS_NUM} - LOG_BUILD 1 - LOG_INSTALL 1 ) ExternalProject_Add_Step(${name} clean diff --git a/third-party/externals/libunwind.cmake b/third-party/externals/libunwind.cmake index 137b227951a..985887980b8 100644 --- a/third-party/externals/libunwind.cmake +++ b/third-party/externals/libunwind.cmake @@ -22,8 +22,6 @@ ExternalProject_Add( BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} BUILD_IN_SOURCE 1 INSTALL_COMMAND make -s install -j${BUILDING_JOBS_NUM} - LOG_BUILD 1 - LOG_INSTALL 1 ) ExternalProject_Add_Step(${name} clean diff --git a/third-party/externals/lz4.cmake b/third-party/externals/lz4.cmake index f1d05dac237..a243284295a 100644 --- a/third-party/externals/lz4.cmake +++ b/third-party/externals/lz4.cmake @@ -25,8 +25,6 @@ ExternalProject_Add( BUILD_SHARED=no -j${BUILDING_JOBS_NUM} PREFIX=${CMAKE_INSTALL_PREFIX} - LOG_BUILD 1 - LOG_INSTALL 1 ) ExternalProject_Add_Step(${name} clean diff --git a/third-party/externals/lzma.cmake b/third-party/externals/lzma.cmake index 2f54e73871f..2c6ef10692a 100644 --- a/third-party/externals/lzma.cmake +++ b/third-party/externals/lzma.cmake @@ -22,8 +22,6 @@ ExternalProject_Add( BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} BUILD_IN_SOURCE 1 INSTALL_COMMAND make -s install -j${BUILDING_JOBS_NUM} PREFIX=${CMAKE_INSTALL_PREFIX} - LOG_BUILD 1 - LOG_INSTALL 1 ) ExternalProject_Add_Step(${name} clean diff --git a/third-party/externals/mstch.cmake b/third-party/externals/mstch.cmake index fc14e3c427e..d4dfab07691 100644 --- a/third-party/externals/mstch.cmake +++ b/third-party/externals/mstch.cmake @@ -20,8 +20,6 @@ ExternalProject_Add( BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} BUILD_IN_SOURCE 1 INSTALL_COMMAND make -s install -j${BUILDING_JOBS_NUM} - LOG_BUILD 1 - LOG_INSTALL 1 ) ExternalProject_Add_Step(${name} clean diff --git a/third-party/externals/openssl.cmake b/third-party/externals/openssl.cmake index 472f71f95a1..a609827ac2c 100644 --- a/third-party/externals/openssl.cmake +++ b/third-party/externals/openssl.cmake @@ -21,8 +21,6 @@ ExternalProject_Add( BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} BUILD_IN_SOURCE 1 INSTALL_COMMAND make -s install_sw -j${BUILDING_JOBS_NUM} - LOG_BUILD 1 - LOG_INSTALL 1 ) ExternalProject_Add_Step(${name} clean diff --git a/third-party/externals/proxygen.cmake b/third-party/externals/proxygen.cmake index 95682f9a9cf..80889a08a7d 100644 --- a/third-party/externals/proxygen.cmake +++ b/third-party/externals/proxygen.cmake @@ -27,10 +27,6 @@ ExternalProject_Add( BUILD_COMMAND env PATH=${BUILDING_PATH} make -s -j${BUILDING_JOBS_NUM} -C proxygen BUILD_IN_SOURCE 1 INSTALL_COMMAND env PATH=${BUILDING_PATH} make -s -j${BUILDING_JOBS_NUM} install -C proxygen - LOG_CONFIGURE TRUE - LOG_BUILD TRUE - LOG_INSTALL TRUE - LOG_MERGED_STDOUTERR TRUE ) ExternalProject_Add_Step(proxygen mannual-configure diff --git a/third-party/externals/s2geometry.cmake b/third-party/externals/s2geometry.cmake index 2ce58b5c6fe..021b148f88a 100644 --- a/third-party/externals/s2geometry.cmake +++ b/third-party/externals/s2geometry.cmake @@ -24,10 +24,6 @@ ExternalProject_Add( BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} BUILD_IN_SOURCE 1 INSTALL_COMMAND make -s -j${BUILDING_JOBS_NUM} install - LOG_CONFIGURE TRUE - LOG_BUILD TRUE - LOG_INSTALL TRUE - LOG_MERGED_STDOUTERR TRUE ) ExternalProject_Add_Step(${name} clean diff --git a/third-party/externals/snappy.cmake b/third-party/externals/snappy.cmake index 12609b47248..5b067409abf 100644 --- a/third-party/externals/snappy.cmake +++ b/third-party/externals/snappy.cmake @@ -23,8 +23,6 @@ ExternalProject_Add( BUILD_IN_SOURCE 1 BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} INSTALL_COMMAND make -s install -j${BUILDING_JOBS_NUM} - LOG_BUILD 1 - LOG_INSTALL 1 ) ExternalProject_Add_Step(${name} clean diff --git a/third-party/externals/wangle.cmake b/third-party/externals/wangle.cmake index 949741b72aa..20d03b6b029 100644 --- a/third-party/externals/wangle.cmake +++ b/third-party/externals/wangle.cmake @@ -19,10 +19,6 @@ ExternalProject_Add( BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} -C wangle BUILD_IN_SOURCE 1 INSTALL_COMMAND make -s -j${BUILDING_JOBS_NUM} install -C wangle - LOG_CONFIGURE TRUE - LOG_BUILD TRUE - LOG_INSTALL TRUE - LOG_MERGED_STDOUTERR TRUE ) ExternalProject_Add_Step(${name} mannual-configure diff --git a/third-party/externals/zlib.cmake b/third-party/externals/zlib.cmake index 125ca8199d2..a6619b12a8a 100644 --- a/third-party/externals/zlib.cmake +++ b/third-party/externals/zlib.cmake @@ -25,8 +25,6 @@ ExternalProject_Add( BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} BUILD_IN_SOURCE 1 INSTALL_COMMAND make -s install -j${BUILDING_JOBS_NUM} PREFIX=${CMAKE_INSTALL_PREFIX} - LOG_BUILD 1 - LOG_INSTALL 1 ) ExternalProject_Add_Step(${name} clean diff --git a/third-party/externals/zstd.cmake b/third-party/externals/zstd.cmake index eee90cc40d3..5f9b7f21191 100644 --- a/third-party/externals/zstd.cmake +++ b/third-party/externals/zstd.cmake @@ -23,8 +23,6 @@ ExternalProject_Add( make -e -s -j${BUILDING_JOBS_NUM} BUILD_IN_SOURCE 1 INSTALL_COMMAND make -s install -j${BUILDING_JOBS_NUM} PREFIX=${CMAKE_INSTALL_PREFIX} - LOG_BUILD 1 - LOG_INSTALL 1 ) ExternalProject_Add_Step(${name} clean From 669392f5ea275c7e882296e1d40a2e419a110a9d Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Thu, 5 Dec 2019 13:31:43 +0800 Subject: [PATCH 39/80] Fixed exit on error --- third-party/build-gcc.sh | 33 +++++++++---------- third-party/build-third-party.sh | 8 ++--- third-party/externals/autoconf-archive.cmake | 3 ++ third-party/externals/autoconf.cmake | 3 ++ third-party/externals/automake.cmake | 3 ++ third-party/externals/bison.cmake | 3 ++ third-party/externals/boost.cmake | 5 ++- third-party/externals/bzip2.cmake | 3 ++ third-party/externals/capstone.cmake | 3 ++ third-party/externals/double-conversion.cmake | 3 ++ third-party/externals/fatal.cmake | 3 ++ third-party/externals/fbthrift.cmake | 3 ++ third-party/externals/flex.cmake | 3 ++ third-party/externals/folly.cmake | 3 ++ third-party/externals/gettext.cmake | 3 ++ third-party/externals/glog.cmake | 3 ++ third-party/externals/googletest.cmake | 3 ++ third-party/externals/gperf.cmake | 3 ++ third-party/externals/jemalloc.cmake | 3 ++ third-party/externals/keyutils.cmake | 3 ++ third-party/externals/krb5.cmake | 3 ++ third-party/externals/libaio.cmake | 3 ++ third-party/externals/libevent.cmake | 3 ++ third-party/externals/libtool.cmake | 3 ++ third-party/externals/libunwind.cmake | 3 ++ third-party/externals/lz4.cmake | 3 ++ third-party/externals/lzma.cmake | 3 ++ third-party/externals/mstch.cmake | 3 ++ third-party/externals/openssl.cmake | 3 ++ third-party/externals/pkgconf.cmake | 3 ++ third-party/externals/proxygen.cmake | 3 ++ third-party/externals/rocksdb.cmake | 5 +-- third-party/externals/s2geometry.cmake | 3 ++ third-party/externals/snappy.cmake | 3 ++ third-party/externals/wangle.cmake | 3 ++ third-party/externals/zlib.cmake | 3 ++ third-party/externals/zstd.cmake | 3 ++ 37 files changed, 125 insertions(+), 25 deletions(-) diff --git a/third-party/build-gcc.sh b/third-party/build-gcc.sh index 83103c9b04a..876c700641a 100755 --- a/third-party/build-gcc.sh +++ b/third-party/build-gcc.sh @@ -105,39 +105,37 @@ function fetch_tarballs { function unpack_tarballs { mkdir -p $source_dir cd $tarballs_dir - set -e if [[ ! -d $source_dir/gcc-$gcc_version ]] then echo "Unpacking $gcc_tarball..." - tar -xf $gcc_tarball -C $source_dir + tar -xf $gcc_tarball -C $source_dir || exit 1 fi if [[ ! -d $source_dir/gmp-$gmp_version ]] then echo "Unpacking $gmp_tarball..." - tar -xf $gmp_tarball -C $source_dir + tar -xf $gmp_tarball -C $source_dir || exit 1 fi if [[ ! -d $source_dir/mpfr-$mpfr_version ]] then echo "Unpacking $mpfr_tarball..." - tar -xf $mpfr_tarball -C $source_dir + tar -xf $mpfr_tarball -C $source_dir || exit 1 fi if [[ ! -d $source_dir/mpc-$mpc_version ]] then echo "Unpacking $mpc_tarball..." - tar -xf $mpc_tarball -C $source_dir + tar -xf $mpc_tarball -C $source_dir || exit 1 fi if [[ ! -d $source_dir/binutils-$bu_version ]] then echo "Unpacking $bu_tarball..." - tar -xf $bu_tarball -C $source_dir + tar -xf $bu_tarball -C $source_dir || exit 1 fi - set +e cd $OLDPWD } @@ -188,15 +186,15 @@ function configure_gcc { --host=$triplet \ --target=$triplet \ --disable-werror - [[ $? -eq 0 ]] || exit 1 + [[ $? -ne 0 ]] && { echo "Failed to configure GCC" 1>&2; exit 1; } cd $OLDPWD } # Start building GCC function build_gcc { cd $gcc_object_dir - make -s -j $building_jobs_num |& tee build.log - [[ $? -ne 0 ]] && exit 1 + make -s -j $building_jobs_num + [[ $? -ne 0 ]] && { echo "Failed to build GCC" 1>&2; exit 1; } cd $OLDPWD } @@ -204,7 +202,7 @@ function build_gcc { function install_gcc { cd $gcc_object_dir make -s -j $building_jobs_num install-strip - [[ $? -ne 0 ]] && exit 1 + [[ $? -ne 0 ]] && { echo "Failed to install GCC" 1>&2; exit 1; } cd $OLDPWD } @@ -224,21 +222,21 @@ function configure_binutils { --host=$triplet \ --target=$triplet \ --disable-werror - + [[ $? -ne 0 ]] && { echo "Failed to configure binutils" 1>&2; exit 1; } cd $OLDPWD } # Build binutils function build_binutils { cd $bu_object_dir - make -s -j $building_jobs_num || exit 1 + make -s -j $building_jobs_num || { echo "Failed to build binutils" 1>&2; exit 1; } cd $OLDPWD } # Install binutils function install_binutils { cd $bu_object_dir - make -s install-strip || exit 1 + make -s install-strip || { echo "Failed to install binutils" 1>&2; exit 1; } cd $OLDPWD cd $install_dir # Place a copy of assembler and linker to libexec @@ -297,7 +295,8 @@ EOF } start_time=$(date +%s) - +set -e +set -o pipefail echo "Starting build, on any failure, see $logfile" { @@ -316,8 +315,8 @@ echo "Starting build, on any failure, see $logfile" install_binutils finalize -} |& tee $logfile \ - | grep --line-buffered '^Making\|^Configuring\|^Comaparing\|^Comparison' +} |& tee -a $logfile \ + | grep --line-buffered '^Making\|^Configuring\|^Comaparing\|^Comparison\|^Failed to' usability_test diff --git a/third-party/build-third-party.sh b/third-party/build-third-party.sh index 5cf4b83375c..66a2fdf8263 100755 --- a/third-party/build-third-party.sh +++ b/third-party/build-third-party.sh @@ -66,6 +66,7 @@ check_cxx # Exit on any failure here after set -e +set -o pipefail # Directories setup cur_dir=`pwd` @@ -124,11 +125,8 @@ cmake -DDOWNLOAD_DIR=$download_dir \ $source_dir |& tee $logfile make |& \ - tee $logfile | \ - grep --line-buffered '^Scanning\|^\[.*Built' | \ - awk '/Scanning dependencies of target/ {printf "...... Build target %s\n", $NF; } \ - /Built/ {print $0;}' -#make |& tee -a $logfile | grep '^Scanning\|^\[.*Built' | awk '{print $0;}' + tee -a $logfile | \ + grep --line-buffered 'Creating\|^Scanning\|Performing\|Completed\|CMakeFiles.*Error' end_time=$(date +%s) echo diff --git a/third-party/externals/autoconf-archive.cmake b/third-party/externals/autoconf-archive.cmake index 45f317f5f89..7f076cd47d6 100644 --- a/third-party/externals/autoconf-archive.cmake +++ b/third-party/externals/autoconf-archive.cmake @@ -21,6 +21,9 @@ ExternalProject_Add( BUILD_IN_SOURCE 1 BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} INSTALL_COMMAND make -s -j${BUILDING_JOBS_NUM} install + LOG_CONFIGURE TRUE + LOG_BUILD TRUE + LOG_INSTALL TRUE ) ExternalProject_Add_Step(${name} clean diff --git a/third-party/externals/autoconf.cmake b/third-party/externals/autoconf.cmake index c52fd882def..feafa58b44c 100644 --- a/third-party/externals/autoconf.cmake +++ b/third-party/externals/autoconf.cmake @@ -21,6 +21,9 @@ ExternalProject_Add( BUILD_IN_SOURCE 1 BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} INSTALL_COMMAND make -s -j${BUILDING_JOBS_NUM} install + LOG_CONFIGURE TRUE + LOG_BUILD TRUE + LOG_INSTALL TRUE ) ExternalProject_Add_Step(${name} clean diff --git a/third-party/externals/automake.cmake b/third-party/externals/automake.cmake index 25ab6ebd55e..bfb5cbaf53b 100644 --- a/third-party/externals/automake.cmake +++ b/third-party/externals/automake.cmake @@ -21,6 +21,9 @@ ExternalProject_Add( BUILD_IN_SOURCE 1 BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} INSTALL_COMMAND make -s -j${BUILDING_JOBS_NUM} install + LOG_CONFIGURE TRUE + LOG_BUILD TRUE + LOG_INSTALL TRUE ) ExternalProject_Add_Step(${name} clean diff --git a/third-party/externals/bison.cmake b/third-party/externals/bison.cmake index 10891404a83..031e66d330f 100644 --- a/third-party/externals/bison.cmake +++ b/third-party/externals/bison.cmake @@ -21,6 +21,9 @@ ExternalProject_Add( BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} BUILD_IN_SOURCE 1 INSTALL_COMMAND make -s install -j${BUILDING_JOBS_NUM} + LOG_CONFIGURE TRUE + LOG_BUILD TRUE + LOG_INSTALL TRUE ) ExternalProject_Add_Step(${name} clean diff --git a/third-party/externals/boost.cmake b/third-party/externals/boost.cmake index 0c8143024c2..9738c567cee 100644 --- a/third-party/externals/boost.cmake +++ b/third-party/externals/boost.cmake @@ -30,12 +30,15 @@ ExternalProject_Add( --disable-icu include=${CMAKE_INSTALL_PREFIX}/include linkflags=-L${CMAKE_INSTALL_PREFIX}/lib - cxxflags="-fPIC" + "cxxflags=-fPIC" runtime-link=static link=static variant=release BUILD_IN_SOURCE 1 INSTALL_COMMAND "" + LOG_CONFIGURE TRUE + LOG_BUILD TRUE + LOG_INSTALL TRUE ) ExternalProject_Add_Step(${name} setup-compiler diff --git a/third-party/externals/bzip2.cmake b/third-party/externals/bzip2.cmake index 57c6ba544eb..4d6cad5bfc8 100644 --- a/third-party/externals/bzip2.cmake +++ b/third-party/externals/bzip2.cmake @@ -18,6 +18,9 @@ ExternalProject_Add( BUILD_COMMAND make CFLAGS=-fPIC -s -j${BUILDING_JOBS_NUM} BUILD_IN_SOURCE 1 INSTALL_COMMAND make -s install -j${BUILDING_JOBS_NUM} PREFIX=${CMAKE_INSTALL_PREFIX} + LOG_CONFIGURE TRUE + LOG_BUILD TRUE + LOG_INSTALL TRUE ) ExternalProject_Add_Step(${name} clean diff --git a/third-party/externals/capstone.cmake b/third-party/externals/capstone.cmake index 552083ab0c5..552fcd2f23c 100644 --- a/third-party/externals/capstone.cmake +++ b/third-party/externals/capstone.cmake @@ -40,6 +40,9 @@ ExternalProject_Add( BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} BUILD_IN_SOURCE 1 INSTALL_COMMAND make -s install -j${BUILDING_JOBS_NUM} + LOG_CONFIGURE TRUE + LOG_BUILD TRUE + LOG_INSTALL TRUE ) ExternalProject_Add_Step(${name} clean diff --git a/third-party/externals/double-conversion.cmake b/third-party/externals/double-conversion.cmake index 15c743c3274..1c9ec4ddb17 100644 --- a/third-party/externals/double-conversion.cmake +++ b/third-party/externals/double-conversion.cmake @@ -20,6 +20,9 @@ ExternalProject_Add( BUILD_IN_SOURCE 1 BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} INSTALL_COMMAND make -s install -j${BUILDING_JOBS_NUM} + LOG_CONFIGURE TRUE + LOG_BUILD TRUE + LOG_INSTALL TRUE ) ExternalProject_Add_Step(${name} clean diff --git a/third-party/externals/fatal.cmake b/third-party/externals/fatal.cmake index 3460ade5539..496ea8a04b7 100644 --- a/third-party/externals/fatal.cmake +++ b/third-party/externals/fatal.cmake @@ -19,6 +19,9 @@ ExternalProject_Add( BUILD_COMMAND install -d ${CMAKE_INSTALL_PREFIX}/include BUILD_IN_SOURCE 1 INSTALL_COMMAND cp -r fatal ${CMAKE_INSTALL_PREFIX}/include + LOG_CONFIGURE TRUE + LOG_BUILD TRUE + LOG_INSTALL TRUE ) ExternalProject_Add_Step(${name} clean diff --git a/third-party/externals/fbthrift.cmake b/third-party/externals/fbthrift.cmake index 0293f80c38d..0cc26c57185 100644 --- a/third-party/externals/fbthrift.cmake +++ b/third-party/externals/fbthrift.cmake @@ -24,6 +24,9 @@ ExternalProject_Add( BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} BUILD_IN_SOURCE 1 INSTALL_COMMAND make -s -j${BUILDING_JOBS_NUM} install + LOG_CONFIGURE TRUE + LOG_BUILD TRUE + LOG_INSTALL TRUE ) ExternalProject_Add_Step(${name} clean diff --git a/third-party/externals/flex.cmake b/third-party/externals/flex.cmake index e705f4b54ed..a01ab707f5c 100644 --- a/third-party/externals/flex.cmake +++ b/third-party/externals/flex.cmake @@ -21,6 +21,9 @@ ExternalProject_Add( BUILD_COMMAND make -s BUILD_IN_SOURCE 1 INSTALL_COMMAND make -s install + LOG_CONFIGURE TRUE + LOG_BUILD TRUE + LOG_INSTALL TRUE ) ExternalProject_Add_Step(${name} clean diff --git a/third-party/externals/folly.cmake b/third-party/externals/folly.cmake index df160a4b5fa..f49756d3d52 100644 --- a/third-party/externals/folly.cmake +++ b/third-party/externals/folly.cmake @@ -24,6 +24,9 @@ ExternalProject_Add( BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} BUILD_IN_SOURCE 1 INSTALL_COMMAND make -s -j${BUILDING_JOBS_NUM} install + LOG_CONFIGURE TRUE + LOG_BUILD TRUE + LOG_INSTALL TRUE ) ExternalProject_Add_Step(${name} clean diff --git a/third-party/externals/gettext.cmake b/third-party/externals/gettext.cmake index a2a3fed6344..e87c95ef835 100644 --- a/third-party/externals/gettext.cmake +++ b/third-party/externals/gettext.cmake @@ -21,6 +21,9 @@ ExternalProject_Add( BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} BUILD_IN_SOURCE 1 INSTALL_COMMAND make -s install -j${BUILDING_JOBS_NUM} + LOG_CONFIGURE TRUE + LOG_BUILD TRUE + LOG_INSTALL TRUE ) ExternalProject_Add_Step(${name} clean diff --git a/third-party/externals/glog.cmake b/third-party/externals/glog.cmake index f123686bb36..b86120b801c 100644 --- a/third-party/externals/glog.cmake +++ b/third-party/externals/glog.cmake @@ -23,6 +23,9 @@ ExternalProject_Add( BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} BUILD_IN_SOURCE 1 INSTALL_COMMAND make -s -j${BUILDING_JOBS_NUM} install + LOG_CONFIGURE TRUE + LOG_BUILD TRUE + LOG_INSTALL TRUE ) ExternalProject_Add_Step(glog pre-configure diff --git a/third-party/externals/googletest.cmake b/third-party/externals/googletest.cmake index cc3dd63d124..e68a22d94d4 100644 --- a/third-party/externals/googletest.cmake +++ b/third-party/externals/googletest.cmake @@ -18,6 +18,9 @@ ExternalProject_Add( BUILD_IN_SOURCE 1 CMAKE_ARGS ${common_cmake_args} + LOG_CONFIGURE TRUE + LOG_BUILD TRUE + LOG_INSTALL TRUE ) ExternalProject_Add_Step(${name} clean diff --git a/third-party/externals/gperf.cmake b/third-party/externals/gperf.cmake index 58206919376..b07984c80de 100644 --- a/third-party/externals/gperf.cmake +++ b/third-party/externals/gperf.cmake @@ -22,6 +22,9 @@ ExternalProject_Add( BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} BUILD_IN_SOURCE 1 INSTALL_COMMAND make -s install -j${BUILDING_JOBS_NUM} + LOG_CONFIGURE TRUE + LOG_BUILD TRUE + LOG_INSTALL TRUE ) ExternalProject_Add_Step(${name} clean diff --git a/third-party/externals/jemalloc.cmake b/third-party/externals/jemalloc.cmake index 6f44f061ad0..8b3c4b7e509 100644 --- a/third-party/externals/jemalloc.cmake +++ b/third-party/externals/jemalloc.cmake @@ -21,6 +21,9 @@ ExternalProject_Add( BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} BUILD_IN_SOURCE 1 INSTALL_COMMAND make -s install_bin install_include install_lib_static -j${BUILDING_JOBS_NUM} + LOG_CONFIGURE TRUE + LOG_BUILD TRUE + LOG_INSTALL TRUE ) ExternalProject_Add_Step(${name} clean diff --git a/third-party/externals/keyutils.cmake b/third-party/externals/keyutils.cmake index 80eb008d4be..932f4ab15be 100644 --- a/third-party/externals/keyutils.cmake +++ b/third-party/externals/keyutils.cmake @@ -22,6 +22,9 @@ ExternalProject_Add( INCLUDEDIR=/include LIBDIR=/lib CFLAGS=-fPIC NO_SOLIB=1 DESTDIR=${CMAKE_INSTALL_PREFIX} install + LOG_CONFIGURE TRUE + LOG_BUILD TRUE + LOG_INSTALL TRUE ) ExternalProject_Add_Step(${name} clean diff --git a/third-party/externals/krb5.cmake b/third-party/externals/krb5.cmake index 2ba0d457def..4012716fb91 100644 --- a/third-party/externals/krb5.cmake +++ b/third-party/externals/krb5.cmake @@ -24,6 +24,9 @@ ExternalProject_Add( INSTALL_COMMAND env PATH=${BUILDING_PATH} make -s install -j${BUILDING_JOBS_NUM} -C src + LOG_CONFIGURE TRUE + LOG_BUILD TRUE + LOG_INSTALL TRUE ) ExternalProject_Add_Step(krb5 mannual-configure diff --git a/third-party/externals/libaio.cmake b/third-party/externals/libaio.cmake index d9efad43393..6018b1e9b2d 100644 --- a/third-party/externals/libaio.cmake +++ b/third-party/externals/libaio.cmake @@ -18,6 +18,9 @@ ExternalProject_Add( BUILD_COMMAND env CFLAGS=-fPIC make -s -j${BUILDING_JOBS_NUM} BUILD_IN_SOURCE 1 INSTALL_COMMAND make prefix=${CMAKE_INSTALL_PREFIX} -s install -j${BUILDING_JOBS_NUM} + LOG_CONFIGURE TRUE + LOG_BUILD TRUE + LOG_INSTALL TRUE ) ExternalProject_Add_Step(${name} clean diff --git a/third-party/externals/libevent.cmake b/third-party/externals/libevent.cmake index 19d98d13297..b786a49a714 100644 --- a/third-party/externals/libevent.cmake +++ b/third-party/externals/libevent.cmake @@ -24,6 +24,9 @@ ExternalProject_Add( BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} BUILD_IN_SOURCE 1 INSTALL_COMMAND make -s install -j${BUILDING_JOBS_NUM} + LOG_CONFIGURE TRUE + LOG_BUILD TRUE + LOG_INSTALL TRUE ) ExternalProject_Add_Step(${name} clean diff --git a/third-party/externals/libtool.cmake b/third-party/externals/libtool.cmake index fb33e1096e6..520c0b3b2fd 100644 --- a/third-party/externals/libtool.cmake +++ b/third-party/externals/libtool.cmake @@ -21,6 +21,9 @@ ExternalProject_Add( BUILD_IN_SOURCE 1 BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} INSTALL_COMMAND make -s -j${BUILDING_JOBS_NUM} install + LOG_CONFIGURE TRUE + LOG_BUILD TRUE + LOG_INSTALL TRUE ) ExternalProject_Add_Step(${name} clean diff --git a/third-party/externals/libunwind.cmake b/third-party/externals/libunwind.cmake index 985887980b8..54bbc6fd7ae 100644 --- a/third-party/externals/libunwind.cmake +++ b/third-party/externals/libunwind.cmake @@ -22,6 +22,9 @@ ExternalProject_Add( BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} BUILD_IN_SOURCE 1 INSTALL_COMMAND make -s install -j${BUILDING_JOBS_NUM} + LOG_CONFIGURE TRUE + LOG_BUILD TRUE + LOG_INSTALL TRUE ) ExternalProject_Add_Step(${name} clean diff --git a/third-party/externals/lz4.cmake b/third-party/externals/lz4.cmake index a243284295a..34bf5941004 100644 --- a/third-party/externals/lz4.cmake +++ b/third-party/externals/lz4.cmake @@ -25,6 +25,9 @@ ExternalProject_Add( BUILD_SHARED=no -j${BUILDING_JOBS_NUM} PREFIX=${CMAKE_INSTALL_PREFIX} + LOG_CONFIGURE TRUE + LOG_BUILD TRUE + LOG_INSTALL TRUE ) ExternalProject_Add_Step(${name} clean diff --git a/third-party/externals/lzma.cmake b/third-party/externals/lzma.cmake index 2c6ef10692a..45b1866a74f 100644 --- a/third-party/externals/lzma.cmake +++ b/third-party/externals/lzma.cmake @@ -22,6 +22,9 @@ ExternalProject_Add( BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} BUILD_IN_SOURCE 1 INSTALL_COMMAND make -s install -j${BUILDING_JOBS_NUM} PREFIX=${CMAKE_INSTALL_PREFIX} + LOG_CONFIGURE TRUE + LOG_BUILD TRUE + LOG_INSTALL TRUE ) ExternalProject_Add_Step(${name} clean diff --git a/third-party/externals/mstch.cmake b/third-party/externals/mstch.cmake index d4dfab07691..f49983b21b6 100644 --- a/third-party/externals/mstch.cmake +++ b/third-party/externals/mstch.cmake @@ -20,6 +20,9 @@ ExternalProject_Add( BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} BUILD_IN_SOURCE 1 INSTALL_COMMAND make -s install -j${BUILDING_JOBS_NUM} + LOG_CONFIGURE TRUE + LOG_BUILD TRUE + LOG_INSTALL TRUE ) ExternalProject_Add_Step(${name} clean diff --git a/third-party/externals/openssl.cmake b/third-party/externals/openssl.cmake index a609827ac2c..38889c75df9 100644 --- a/third-party/externals/openssl.cmake +++ b/third-party/externals/openssl.cmake @@ -21,6 +21,9 @@ ExternalProject_Add( BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} BUILD_IN_SOURCE 1 INSTALL_COMMAND make -s install_sw -j${BUILDING_JOBS_NUM} + LOG_CONFIGURE TRUE + LOG_BUILD TRUE + LOG_INSTALL TRUE ) ExternalProject_Add_Step(${name} clean diff --git a/third-party/externals/pkgconf.cmake b/third-party/externals/pkgconf.cmake index a4e8cb4aa96..ffd67abe406 100644 --- a/third-party/externals/pkgconf.cmake +++ b/third-party/externals/pkgconf.cmake @@ -16,5 +16,8 @@ ExternalProject_Add( BUILD_IN_SOURCE 1 BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} INSTALL_COMMAND make -s -j${BUILDING_JOBS_NUM} install + LOG_CONFIGURE TRUE + LOG_BUILD TRUE + LOG_INSTALL TRUE ) diff --git a/third-party/externals/proxygen.cmake b/third-party/externals/proxygen.cmake index 80889a08a7d..57cc05b3c2a 100644 --- a/third-party/externals/proxygen.cmake +++ b/third-party/externals/proxygen.cmake @@ -27,6 +27,9 @@ ExternalProject_Add( BUILD_COMMAND env PATH=${BUILDING_PATH} make -s -j${BUILDING_JOBS_NUM} -C proxygen BUILD_IN_SOURCE 1 INSTALL_COMMAND env PATH=${BUILDING_PATH} make -s -j${BUILDING_JOBS_NUM} install -C proxygen + LOG_CONFIGURE TRUE + LOG_BUILD TRUE + LOG_INSTALL TRUE ) ExternalProject_Add_Step(proxygen mannual-configure diff --git a/third-party/externals/rocksdb.cmake b/third-party/externals/rocksdb.cmake index c99be0ed599..81cf394105e 100644 --- a/third-party/externals/rocksdb.cmake +++ b/third-party/externals/rocksdb.cmake @@ -34,8 +34,9 @@ ExternalProject_Add( BUILD_IN_SOURCE 1 BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} VERBOSE=1 INSTALL_COMMAND make -s install -j${BUILDING_JOBS_NUM} - LOG_BUILD 1 - LOG_INSTALL 1 + LOG_CONFIGURE TRUE + LOG_BUILD TRUE + LOG_INSTALL TRUE ) ExternalProject_Add_Step(${name} clean diff --git a/third-party/externals/s2geometry.cmake b/third-party/externals/s2geometry.cmake index 021b148f88a..ef4aa099720 100644 --- a/third-party/externals/s2geometry.cmake +++ b/third-party/externals/s2geometry.cmake @@ -24,6 +24,9 @@ ExternalProject_Add( BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} BUILD_IN_SOURCE 1 INSTALL_COMMAND make -s -j${BUILDING_JOBS_NUM} install + LOG_CONFIGURE TRUE + LOG_BUILD TRUE + LOG_INSTALL TRUE ) ExternalProject_Add_Step(${name} clean diff --git a/third-party/externals/snappy.cmake b/third-party/externals/snappy.cmake index 5b067409abf..be2a4b6d3e5 100644 --- a/third-party/externals/snappy.cmake +++ b/third-party/externals/snappy.cmake @@ -23,6 +23,9 @@ ExternalProject_Add( BUILD_IN_SOURCE 1 BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} INSTALL_COMMAND make -s install -j${BUILDING_JOBS_NUM} + LOG_CONFIGURE TRUE + LOG_BUILD TRUE + LOG_INSTALL TRUE ) ExternalProject_Add_Step(${name} clean diff --git a/third-party/externals/wangle.cmake b/third-party/externals/wangle.cmake index 20d03b6b029..a9d73891b75 100644 --- a/third-party/externals/wangle.cmake +++ b/third-party/externals/wangle.cmake @@ -19,6 +19,9 @@ ExternalProject_Add( BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} -C wangle BUILD_IN_SOURCE 1 INSTALL_COMMAND make -s -j${BUILDING_JOBS_NUM} install -C wangle + LOG_CONFIGURE TRUE + LOG_BUILD TRUE + LOG_INSTALL TRUE ) ExternalProject_Add_Step(${name} mannual-configure diff --git a/third-party/externals/zlib.cmake b/third-party/externals/zlib.cmake index a6619b12a8a..b202bb14734 100644 --- a/third-party/externals/zlib.cmake +++ b/third-party/externals/zlib.cmake @@ -25,6 +25,9 @@ ExternalProject_Add( BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} BUILD_IN_SOURCE 1 INSTALL_COMMAND make -s install -j${BUILDING_JOBS_NUM} PREFIX=${CMAKE_INSTALL_PREFIX} + LOG_CONFIGURE TRUE + LOG_BUILD TRUE + LOG_INSTALL TRUE ) ExternalProject_Add_Step(${name} clean diff --git a/third-party/externals/zstd.cmake b/third-party/externals/zstd.cmake index 5f9b7f21191..1784c6b37a3 100644 --- a/third-party/externals/zstd.cmake +++ b/third-party/externals/zstd.cmake @@ -23,6 +23,9 @@ ExternalProject_Add( make -e -s -j${BUILDING_JOBS_NUM} BUILD_IN_SOURCE 1 INSTALL_COMMAND make -s install -j${BUILDING_JOBS_NUM} PREFIX=${CMAKE_INSTALL_PREFIX} + LOG_CONFIGURE TRUE + LOG_BUILD TRUE + LOG_INSTALL TRUE ) ExternalProject_Add_Step(${name} clean From 50e4538f1e6daf43bfe858be43274ac080f6401a Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Thu, 5 Dec 2019 15:55:56 +0800 Subject: [PATCH 40/80] Added install-gcc.sh --- third-party/build-gcc.sh | 2 + .../{build-cmake.sh => install-cmake.sh} | 0 third-party/install-gcc.sh | 111 ++++++++++++++++++ 3 files changed, 113 insertions(+) rename third-party/{build-cmake.sh => install-cmake.sh} (100%) create mode 100755 third-party/install-gcc.sh diff --git a/third-party/build-gcc.sh b/third-party/build-gcc.sh index 876c700641a..ed2763dad86 100755 --- a/third-party/build-gcc.sh +++ b/third-party/build-gcc.sh @@ -261,7 +261,9 @@ function usability_test { # Build a self-extractable package function make_package { + set +e glibc_version=$(ldd --version | head -1 | cut -d ' ' -f4 | cut -d '-' -f1) + set -e exec_file=$root_dir/vesoft-gcc-$gcc_version-$distro-x86_64-glibc-$glibc_version.sh echo "Creating self-extractable package $exec_file" cat > $exec_file </dev/null || { + echo "'wget' not fould, please install it first" 1>&2 + exit 1 +} + +# Guess the root distro +[[ "$this_distro" = CentOS ]] && selected_distro=CentOS +[[ "$this_distro" = RedHat ]] && selected_distro=CentOS +[[ "$this_distro" = Fedora ]] && selected_distro=CentOS +[[ "$this_distro" = Debian ]] && selected_distro=Debian +[[ "$this_distro" = Ubuntu ]] && selected_distro=Debian +[[ "$this_distro" = LinuxMint ]] && selected_distro=Debian + +# backoff distro +[[ -n $this_distro ]] || selected_distro=Debian + +function version_cmp { + mapfile -t left < <( echo $1 | tr . '\n' ) + mapfile -t right < <( echo $2 | tr . '\n') + local i + for i in ${!left[@]} + do + local lv=${left[$i]} + local rv=${right[$i]} + [[ -z $rv ]] && { echo $lv; return; } + [[ $lv -ne $rv ]] && { echo $((lv - rv)); return; } + done + ((i++)) + rv=${right[$i]} + [[ ${#right[@]} -gt ${#left[@]} ]] && { echo $((0-rv)); return; } +} + +# Find the maximum version not greater than candidates +function select_libc { + local this_version=$1 + shift 1 + local candidates="$@" + for v in $candidates + do + if [[ $(version_cmp $v $this_version) -le 0 ]] + then + echo $v + break + fi + done +} + +case $selected_distro in + CentOS) + selected_libc=$(select_libc $this_libc "${CentOS_libc_preset[@]}") + ;; + Debian) + selected_libc=$(select_libc $this_libc "${Debian_libc_preset[@]}") + ;; +esac + +[[ -z $selected_libc ]] && { + echo "No suitable GCC found to download for your environment: $this_distro, glibc-$this_libc" 1>&2 + echo "Please invoke $this_dir/build-gcc.sh to build one manually" 1>&2 + exit 1 +} + +selected_archive=vesoft-gcc-$version-$selected_distro-x86_64-glibc-$selected_libc.sh + +url=$url_base/$selected_archive +wget -c $url +[[ $? -ne 0 ]] && { + echo "Downloading $selected_archive failed" 1>&2 + exit 1 +} + +bash $selected_archive $@ + +rm -rf $selected_archive From 8864383411fbcd418969a7fdcbb28e8b1abe065c Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Thu, 5 Dec 2019 16:44:21 +0800 Subject: [PATCH 41/80] Address darionyaphet's comment --- third-party/cmake/FindLibtool.cmake | 2 +- third-party/install-gcc.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/third-party/cmake/FindLibtool.cmake b/third-party/cmake/FindLibtool.cmake index 17f54eb9ff0..61d0eb1f27d 100644 --- a/third-party/cmake/FindLibtool.cmake +++ b/third-party/cmake/FindLibtool.cmake @@ -11,4 +11,4 @@ endif() include(FindPackageHandleStandardArgs) find_package_handle_standard_args(Libtool REQUIRED_VARS Libtool_EXECUTABLE - VERSION_VAR Libtool_VERSION) + VERSION_VAR Libtool_VERSION) diff --git a/third-party/install-gcc.sh b/third-party/install-gcc.sh index 312ef346763..1db3ddddb78 100755 --- a/third-party/install-gcc.sh +++ b/third-party/install-gcc.sh @@ -5,7 +5,7 @@ # This source code is licensed under Apache 2.0 License, # attached with Common Clause Condition 1.0, found in the LICENSES directory. -# Usage: install-gcc.sh --prefix=/opt/nebula/toolchain +# Usage: install-gcc.sh --prefix=/opt/nebula/toolset # Always use bash shell=$(basename $(readlink /proc/$$/exe)) @@ -67,7 +67,7 @@ function version_cmp { [[ ${#right[@]} -gt ${#left[@]} ]] && { echo $((0-rv)); return; } } -# Find the maximum version not greater than candidates +# Find the maximum version not greater than the system one function select_libc { local this_version=$1 shift 1 From a7843cb09428497fbfd0a73e3f8e9311b9c5f380 Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Thu, 5 Dec 2019 17:15:59 +0800 Subject: [PATCH 42/80] Update cn docs on build --- .../1.build/1.build-source-code.md | 260 ++++++++++-------- 1 file changed, 143 insertions(+), 117 deletions(-) diff --git a/docs/manual-CN/3.build-develop-and-administration/1.build/1.build-source-code.md b/docs/manual-CN/3.build-develop-and-administration/1.build/1.build-source-code.md index 49863b34707..975af05ac86 100644 --- a/docs/manual-CN/3.build-develop-and-administration/1.build/1.build-source-code.md +++ b/docs/manual-CN/3.build-develop-and-administration/1.build/1.build-source-code.md @@ -1,156 +1,182 @@ -# 使用源码编译 +# 源码编译安装 -## 编译器要求 +# 简介 +从源代码编译安装 Nebula Graph，大体分为四步： -Nebula 基于 C++ 14 开发，因此它需要一个支持 C++ 14 的编译器。 - -### 支持系统版本 - -- Fedora 29, 30 -- CentOS 6.5, CentOS 7.0 ~ 7.6 -- Ubuntu 16.04, 18.04 - -### 需要的硬盘和内存空间 - -当编译类型为 **Debug** 的时候，最好预留 **50G** 的 SSD 硬盘空间。 - -并提供至少 **8GB** 的内存。 - -### 本地构建 - -#### 步骤 1: 克隆代码 +1. 克隆代码 +1. 安装依赖 +1. 安装 GCC 和 CMake +1. 编译安装 Nebula Graph +其中，如果您的系统中已经安装了 GCC 7.0+ 和 CMake 3.5+，第三步可以略过。 +# 第一步：克隆代码 ```bash -bash> git clone https://github.com/vesoft-inc/nebula.git +$ git clone https://github.com/vesoft-inc/nebula.git && cd nebula +nebula$ mkdir build && cd build ``` - -#### 步骤 2 : 安装依赖 - -- 环境能直接下载云存储 OSS 包的用户 +# 第二步：安装依赖 +## Redhat/CentOS/Fedora 安装命令 ```bash -bash> cd nebula && ./build_dep.sh +$ sudo yum install -y make \ + m4 \ + wget \ + unzip \ + xz \ + patch \ + python \ + redhat-lsb-core \ + perl-Data-Dumper \ + perl-Thread-Queue \ + readline-devel \ + ncurses-devel \ + zlib-devel \ + gcc \ + gcc-c++ \ + cmake \ + libtool \ + autoconf \ + automake \ + bison \ + flex \ + gperf \ + gettext +# Red Hat 7+ 可额外安装 autoconf-archive +$ sudo yum install -y autoconf-archive ``` -- 环境不能直接下载云存储 OSS 包的用户 - -1. 从本地源下载依赖和进行配置 - +## Debian/Ubuntu/Mint 安装命令 ```bash -bash> cd nebula && ./build_dep.sh N +$ sudo apt-get update +$ sudo apt-get install -y make \ + m4 \ + wget \ + unzip \ + xz-utils \ + patch \ + python \ + lsb-core \ + libz-dev \ + build-essential \ + libreadline-dev \ + ncurses-dev \ + build-essential \ + cmake \ + libtool \ + automake \ + autoconf \ + autoconf-archive \ + autotools-dev \ + bison \ + flex \ + gperf \ + gettext ``` -2. 从下面链接中下载对应版本的压缩包 - -- [Fedora 29/30](https://nebula-graph.oss-accelerate.aliyuncs.com/third-party/fedora29.tar.gz) -- [CentOS 7.0~7.6](https://nebula-graph.oss-accelerate.aliyuncs.com/third-party/centos7.5.tar.gz) -- [CentOS 6.5](https://nebula-graph.oss-accelerate.aliyuncs.com/third-party/centos6.5.tar.gz) -- [Ubuntu 1604](https://nebula-graph.oss-accelerate.aliyuncs.com/third-party/ubuntu16.tar.gz) -- [Ubuntu 1804](https://nebula-graph.oss-accelerate.aliyuncs.com/third-party/ubuntu18.tar.gz) - -3. 安装下载好的压缩包 - +# 第三步：安装 CMake 和 GCC +_**若您的系统为 Ubuntu 18.04+，CentOS 8+，Red Hat 8+，或其他提供 GCC 7+ 和 CMake 3.5+ 的 Linux 发行版，可从官方源安装，此步可跳过。**_可使用下面命令查看 GCC 与 CMake 的版本号： ```bash -tar xf ${package_name}.tar.gz -cd ${package_name} && ./install.sh +$ gcc --version +$ cmake --version ``` -#### 步骤 3: 应用 **~/.bashrc** 修改 - +## 安装 CMake +以下命令，假设当前目录为 nebula/build，如果您的情况有所不同，请自行调整相应路径。 ```bash -bash> source ~/.bashrc +# 脚本自动下载安装 CMake +nebula/build$ ../third-party/install-cmake.sh +# 安装完成，将 cmake 命令加入 PATH（假设 CMake 版本为 3.15.5） +nebula/build$ source cmake-3.15.5/bin/enable-cmake.sh ``` -#### 步骤 4: 构建 Debug 版本 - +## 安装 GCC +### 包管理器安装 +#### CentOS 6 / CentOS 7 ```bash -bash> mkdir build && cd build -bash> cmake .. -bash> make -bash> sudo make install +# 安装 SCL，即 Software Collection +$ sudo yum install centos-release-scl +# 安装 devtoolset +$ sudo yum install devtoolset-7 +# 设置 PATH 等相关变量，启用 devtoolset +$ scl enable devtoolset-7 bash +# 重设命令搜索缓存 +$ hash -r +# 确认 GCC 版本 +$ g++ --version +g++ (GCC) 7.3.1 20180303 (Red Hat 7.3.1-5) ``` -#### 构建 Release 版本，重新生成 Makefile - +#### Red Hat 6 / Red Hat 7 ```bash -bash> cmake -DCMAKE_BUILD_TYPE=Release .. -bash> make -bash> sudo make install +# 安装 SCL，即 Software Collection + +# Red Hat 6 +$ sudo yum-config-manager --enable rhel-server-rhscl-6-rpms +# Red Hat 7 +$ sudo yum-config-manager --enable rhel-server-rhscl-7-rpms + +# 安装 devtoolset +$ sudo yum install devtoolset-7 +# 设置 PATH 等相关变量，启用 devtoolset +$ scl enable devtoolset-7 bash +# 重设命令搜索缓存 +$ hash -r +# 确认 GCC 版本 +$ g++ --version +g++ (GCC) 7.3.1 20180303 (Red Hat 7.3.1-5) ``` -#### **构建完成** - -- 如果没有任何错误信息 - - ```text - [100%] Built target .... - ``` - - **编译成功！** - -- 在安装目录 **/usr/local/nebula** 下有如下四个子目录 **etc/**, **bin/**, **scripts/** **share/** - - ```bash - [root@centos 7.5 nebula]# ls /usr/local/nebula/ - bin etc scripts share - ``` - - **现在可以开始运行 Nebula** 。 - - - -#### 启动 nebula 服务 - -经过上述的安装后，便可以在容器内部启动 nebula 的服务，nebula 默认的安装目录为 `/usr/local/nebula` - +### 源码安装 ```bash -docker> cd /usr/local/nebula +nebula/build$ ../third-party/build-gcc.sh ``` -重命名 nebula 服务的配置文件 - +# 第四步：编译 Nebula Graph ```bash -docker> cp etc/nebula-graphd.conf.default etc/nebula-graphd.conf -docker> cp etc/nebula-metad.conf.default etc/nebula-metad.conf -docker> cp etc/nebula-storaged.conf.default etc/nebula-storaged.conf +# 注意 build-third-party.sh 和 cmake 命令需要在同一目录下运行 +nebula/build$ ../third-party/build-third-party.sh +nebula/build$ cmake -DENABLE_TESTING=OFF \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=$HOME/nebula-install \ + .. +nebula/build$ make -j && make install +nebula/build$ cd $HOME/nebula-install +nebula-install$ ls +bin/ share/ etc/ scripts/ ``` -启动服务 - -```bash -docker> ./scripts/nebula.service start all -docker> ./bin/nebula -u user -p password --port 3699 --addr="127.0.0.1" -nebula> SHOW HOSTS; -``` ### 常见问题和解决方案 From 8e5d1c7b4ddb6e72721ae2855f193712076341d8 Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Thu, 5 Dec 2019 17:23:21 +0800 Subject: [PATCH 43/80] Update cn docs on build --- .../1.build/1.build-source-code.md | 59 ++----------------- 1 file changed, 6 insertions(+), 53 deletions(-) diff --git a/docs/manual-CN/3.build-develop-and-administration/1.build/1.build-source-code.md b/docs/manual-CN/3.build-develop-and-administration/1.build/1.build-source-code.md index 975af05ac86..98d77ebf778 100644 --- a/docs/manual-CN/3.build-develop-and-administration/1.build/1.build-source-code.md +++ b/docs/manual-CN/3.build-develop-and-administration/1.build/1.build-source-code.md @@ -4,9 +4,9 @@ 从源代码编译安装 Nebula Graph，大体分为四步： 1. 克隆代码 -1. 安装依赖 -1. 安装 GCC 和 CMake -1. 编译安装 Nebula Graph +2. 安装依赖 +3. 安装 GCC 和 CMake +4. 编译安装 Nebula Graph 其中，如果您的系统中已经安装了 GCC 7.0+ 和 CMake 3.5+，第三步可以略过。 # 第一步：克隆代码 @@ -74,7 +74,7 @@ $ sudo apt-get install -y make \ ``` # 第三步：安装 CMake 和 GCC -_**若您的系统为 Ubuntu 18.04+，CentOS 8+，Red Hat 8+，或其他提供 GCC 7+ 和 CMake 3.5+ 的 Linux 发行版，可从官方源安装，此步可跳过。**_可使用下面命令查看 GCC 与 CMake 的版本号： +**若您的系统为 Ubuntu 18.04+，CentOS 8+，Red Hat 8+，或其他提供 GCC 7+ 和 CMake 3.5+ 的 Linux 发行版，可从官方源安装，此步可跳过。**可使用下面命令查看 GCC 与 CMake 的版本号： ```bash $ gcc --version $ cmake --version @@ -136,7 +136,7 @@ $ export CC=gcc-7 $ export CXX=g++-7 ``` -### 使用我们预先编译好的 +### 使用我们预先编译好的GCC 若系统是 Debian 或其他未提及的 Linux 发行版，您可以执行我们提供的脚本下载安装 GCC： ```bash # 自动下载安装 @@ -178,23 +178,7 @@ bin/ share/ etc/ scripts/ ``` -### 常见问题和解决方案 - -- **错误信息**: `/usr/bin/ld: cannot find Scrt1.o: No such file or directory` - - **解决方案**: - - **步骤 1**: 在 **~/.bashrc** 末添加如下行 - - ```bash - export LIBRARY_PATH=/usr/lib/x86_64-linux-gnu:$LIBRARY_PATH - ``` - - **步骤 2**: 应用 **~/.bashrc** 修改 - - ```bash - bash> source ~/.bashrc - ``` +# 常见问题和解决方案 - **错误信息**: `[ERROR] No compiler is provided in this environment. Perhaps you are running on a JRE rather than a JDK?` @@ -233,34 +217,3 @@ bin/ share/ etc/ scripts/ export CLASSPATH=$JAVA_HOME/lib:$JRE_HOME/lib:$CLASSPATH export PATH=$JAVA_HOME/bin:$JRE_HOME/bin:$PATH ``` - -- **错误信息**: `internal error` - - **解决方案**: - - 你需要自己编译第三方库，第三方库的安装路径为 **/opt/nebula/third-party** - - 步骤 1: 下载第三方库的源码包 - - ```bash - bash> wget https://nebula-graph.oss-accelerate.aliyuncs.com/third-party/nebula-3rdparty.tar.gz - ``` - - 步骤 2: 解压并编译third-party - - ```bash - bash> tar xf nebula-3rdparty.tar.gz && cd nebula-3rdparty - bash> ./install_deps.sh - bash> cmake ./ - bash> sudo make - ``` - - 步骤 3: 配置项目的环境变量和其他依赖 - - ```bash - bash> cd nebula && ./build_dep.sh N - bash> source ~/.bashrc - bash> mkdir build && cd build - bash> cmake .. - bash> make - ``` From 33658f4663aced2968b83a777ca1f71ebe8be49c Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Thu, 5 Dec 2019 20:19:55 +0800 Subject: [PATCH 44/80] Added prebuilt GCC for glibc 2.17, 2.19 --- third-party/build-gcc.sh | 1 + third-party/install-gcc.sh | 11 ++++++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/third-party/build-gcc.sh b/third-party/build-gcc.sh index ed2763dad86..fc0f4a0a02e 100755 --- a/third-party/build-gcc.sh +++ b/third-party/build-gcc.sh @@ -300,6 +300,7 @@ start_time=$(date +%s) set -e set -o pipefail echo "Starting build, on any failure, see $logfile" +mkdir -p $root_dir { fetch_tarballs diff --git a/third-party/install-gcc.sh b/third-party/install-gcc.sh index 1db3ddddb78..847fd7db68c 100755 --- a/third-party/install-gcc.sh +++ b/third-party/install-gcc.sh @@ -26,8 +26,8 @@ this_dir=$(dirname $(readlink -f $0)) # We consider two derivatives: Red Hat and Debian # Place preset libc versions of each from newer to older -CentOS_libc_preset=( 2.12 ) -Debian_libc_preset=( 2.13 ) +CentOS_libc_preset=( 2.17 2.12 ) +Debian_libc_preset=( 2.19 2.13 ) selected_distro= selected_libc= @@ -40,6 +40,11 @@ hash wget &>/dev/null || { exit 1 } +download_cmd="wget -c" +wget --help | grep -q '\--show-progress' && \ + download_cmd="$download_cmd -q --show-progress" || \ + download_cmd="$download_cmd --progress=bar:force:noscroll" + # Guess the root distro [[ "$this_distro" = CentOS ]] && selected_distro=CentOS [[ "$this_distro" = RedHat ]] && selected_distro=CentOS @@ -100,7 +105,7 @@ esac selected_archive=vesoft-gcc-$version-$selected_distro-x86_64-glibc-$selected_libc.sh url=$url_base/$selected_archive -wget -c $url +$download_cmd $url [[ $? -ne 0 ]] && { echo "Downloading $selected_archive failed" 1>&2 exit 1 From be22919224a6e4d68194ab97486338bc4c4261e0 Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Fri, 13 Dec 2019 15:52:31 +0800 Subject: [PATCH 45/80] Changed prefix --- third-party/build-gcc.sh | 45 +++++++++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/third-party/build-gcc.sh b/third-party/build-gcc.sh index fc0f4a0a02e..1bee204be93 100755 --- a/third-party/build-gcc.sh +++ b/third-party/build-gcc.sh @@ -49,14 +49,14 @@ bu_checksum=a3bf359889e4b299fce1f4cb919dc7b6 # Building directories setup cur_dir=$PWD -root_dir=$PWD/gcc-build +root_dir=$PWD/toolset-build tarballs_dir=$root_dir/downloads source_dir=$root_dir/source gcc_object_dir=$root_dir/gcc-build bu_object_dir=$root_dir/binutils-build prefix=$1 -install_dir=${prefix:-$root_dir/install}/gcc/$gcc_version -logfile=$root_dir/build.log +install_dir=${prefix:-$root_dir/install}/nebula/toolset/gcc/$gcc_version +logfile=$root_dir/gcc-build.log triplet=x86_64-vesoft-linux distro=$(lsb_release -si) @@ -83,12 +83,19 @@ function fetch_tarball { } function fetch_tarballs { - hash wget &> /dev/null && download_cmd="wget -c" + hash wget &> /dev/null && download_cmd="wget -c --progress=bar:force:noscroll" if [[ -z $download_cmd ]] then echo "'wget' not found for downloading" 1>&2; exit 1; fi + set +e + set +o pipefail + wget --help | grep -q '\--show-progress' && \ + download_cmd="$download_cmd -q --show-progress" + set -e + set -o pipefail + echo "Download command: '$download_cmd'" mkdir -p $tarballs_dir && cd $tarballs_dir @@ -193,7 +200,7 @@ function configure_gcc { # Start building GCC function build_gcc { cd $gcc_object_dir - make -s -j $building_jobs_num + make -s -j $building_jobs_num bootstrap-lean [[ $? -ne 0 ]] && { echo "Failed to build GCC" 1>&2; exit 1; } cd $OLDPWD } @@ -206,6 +213,14 @@ function install_gcc { cd $OLDPWD } +# Clean GCC +function clean_gcc { + cd $gcc_object_dir + make -s -j $building_jobs_num clean + [[ $? -ne 0 ]] && { echo "Failed to clean GCC" 1>&2; exit 1; } + cd $OLDPWD +} + # Configure binutils function configure_binutils { mkdir -p $bu_object_dir @@ -245,10 +260,17 @@ function install_binutils { cd $OLDPWD } +# Clean binutils +function install_binutils { + cd $bu_object_dir + make -s clean || { echo "Failed to clean binutils" 1>&2; exit 1; } + cd $OLDPWD +} + # Finalize the building function finalize { # Remove all of the annoying libtool files, - # so that the installation would be copied around + # so that the installation could be copied around find $install_dir -name '*.la' | xargs rm -f } @@ -271,7 +293,7 @@ function make_package { set -e [[ \$# -ne 0 ]] && prefix=\$(echo "\$@" | sed 's;.*--prefix=(\S*).*;\1;' -r) -prefix=\${prefix:-/opt/nebula/toolset} +prefix=\${prefix:-/opt}/nebula/toolset hash xz &> /dev/null || { echo "xz: Command not found"; exit 1; } @@ -285,13 +307,14 @@ echo "GCC-$gcc_version has been installed to \$prefix/gcc/$gcc_version" echo "Performing usability tests" CXX=\$prefix/gcc/$gcc_version/bin/g++ \$prefix/gcc/$gcc_version/cxx-compiler-usability-test.sh echo "Run 'source \$prefix/gcc/$gcc_version/enable' to start using." +echo "Run 'source \$prefix/gcc/$gcc_version/disable' to stop using." exit 0 __start_of_archive__ EOF cd $install_dir/../.. - tar -cJf - * >> $exec_file + tar -cJf - gcc/$gcc_version >> $exec_file chmod 0755 $exec_file cd $OLDPWD } @@ -312,14 +335,16 @@ mkdir -p $root_dir configure_gcc build_gcc install_gcc + clean_gcc configure_binutils build_binutils install_binutils + clean_binutils finalize } |& tee -a $logfile \ - | grep --line-buffered '^Making\|^Configuring\|^Comaparing\|^Comparison\|^Failed to' + | grep --line-buffered '^Making\|^Configuring\|^Comparing\|^Comparison\|^Failed to' usability_test @@ -332,7 +357,7 @@ export CC=\$this_path/bin/gcc export CXX=\$this_path/bin/g++ hash -r echo "Only PATH was setup so as not to pollute your library path" -echo "You could run 'export LD_LIBRARY_PATH=\$this_path/lib64:\\\$LD_LIBRARY_PATH'" +echo "You could run 'export LD_LIBRARY_PATH=\$this_path/lib64:\\\$LD_LIBRARY_PATH' if needed" EOF cat > $install_dir/disable < Date: Fri, 13 Dec 2019 16:25:15 +0800 Subject: [PATCH 46/80] Ignore error when cleaning --- third-party/build-gcc.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/third-party/build-gcc.sh b/third-party/build-gcc.sh index 1bee204be93..690e68b7885 100755 --- a/third-party/build-gcc.sh +++ b/third-party/build-gcc.sh @@ -216,8 +216,7 @@ function install_gcc { # Clean GCC function clean_gcc { cd $gcc_object_dir - make -s -j $building_jobs_num clean - [[ $? -ne 0 ]] && { echo "Failed to clean GCC" 1>&2; exit 1; } + make -s -j $building_jobs_num clean || true cd $OLDPWD } @@ -261,9 +260,9 @@ function install_binutils { } # Clean binutils -function install_binutils { +function clean_binutils { cd $bu_object_dir - make -s clean || { echo "Failed to clean binutils" 1>&2; exit 1; } + make -s clean || true cd $OLDPWD } @@ -325,8 +324,8 @@ set -o pipefail echo "Starting build, on any failure, see $logfile" mkdir -p $root_dir +fetch_tarballs { - fetch_tarballs unpack_tarballs setup_deps } |& tee $logfile From 6934c2c91499abe98245e42ba5d14e5f24ccfa85 Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Tue, 17 Dec 2019 14:43:57 +0800 Subject: [PATCH 47/80] Fixed gcc/disable --- src/graph/GroupByExecutor.cpp | 2 +- third-party/build-gcc.sh | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/graph/GroupByExecutor.cpp b/src/graph/GroupByExecutor.cpp index 0da05be198c..af0b2ed2117 100644 --- a/src/graph/GroupByExecutor.cpp +++ b/src/graph/GroupByExecutor.cpp @@ -398,7 +398,7 @@ Status GroupByExecutor::generateOutputSchema() { StatusOr> GroupByExecutor::setupInterimResult() { auto result = std::make_unique(getResultColumnNames()); if (rows_.empty() || resultSchema_ == nullptr) { - return result; + return std::move(result); } // Generate results std::unique_ptr rsWriter = std::make_unique(resultSchema_); diff --git a/third-party/build-gcc.sh b/third-party/build-gcc.sh index 690e68b7885..161a5376566 100755 --- a/third-party/build-gcc.sh +++ b/third-party/build-gcc.sh @@ -364,6 +364,8 @@ this_path=\$(dirname \$(readlink -f \$BASH_SOURCE)) export PATH=\$(echo \$PATH | sed "s#\$this_path/bin:##") export CC=\$OLD_CC export CXX=\$OLD_CXX +[ -z \$CC ] && unset CC +[ -z \$CXX ] && unset CXX hash -r EOF From 3ed629c3596952b2c583a3b4ddda5985f05b2fff Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Tue, 17 Dec 2019 21:01:51 +0800 Subject: [PATCH 48/80] Reduce build size --- CMakeLists.txt | 21 +++++++++++++++++++-- third-party/CMakeLists.txt | 9 +++++++++ third-party/build-third-party.sh | 5 +++-- third-party/externals/folly.cmake | 2 +- 4 files changed, 32 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index cd7c24f912d..31c45ba8b63 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -31,6 +31,7 @@ option(ENABLE_CCACHE "Whether to use ccache to speed up compiling" ON) option(ENABLE_ASAN "Whether to turn AddressSanitizer ON or OFF" OFF) option(ENABLE_TESTING "Whether to turn unit test ON or OFF" ON) option(ENABLE_UBSAN "Whether to turn Undefined Behavior Sanitizer ON or OFF" OFF) +set(NEBULA_DEBUG_INFO_LEVEL 1 CACHE STRING "Level of debug info, used only in Debug or RelWithDebInfo") message(STATUS "ENABLE_ASAN: ${ENABLE_ASAN}") message(STATUS "ENABLE_TESTING: ${ENABLE_TESTING}") @@ -84,8 +85,22 @@ if(NOT ${CMAKE_BUILD_TYPE} STREQUAL "Debug") add_definitions(-D_FORTIFY_SOURCE=2) endif() +if(CMAKE_BUILD_TYPE STREQUAL "Debug" OR CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo") + add_compile_options(-g${NEBULA_DEBUG_INFO_LEVEL}) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--compress-debug-sections=zlib") +endif() + +if(CMAKE_BUILD_TYPE STREQUAL "Debug") + # TODO(dutor) To check if the linker supports this. + add_compile_options(-ffunction-sections) + add_compile_options(-fdata-sections) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections") +endif() + message(STATUS "CMAKE_BUILD_TYPE = ${CMAKE_BUILD_TYPE} " "(Options are: Debug, Release, RelWithDebInfo, MinSizeRel)") +message(STATUS "NEBULA_DEBUG_INFO_LEVEL = ${NEBULA_DEBUG_INFO_LEVEL} " + "(Options are: 0, 1, 2, 3)") set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "_build") set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "_build") @@ -145,6 +160,8 @@ if(NOT ${NEBULA_THIRDPARTY_ROOT} STREQUAL "") ${NEBULA_THIRDPARTY_ROOT}/lib ${NEBULA_THIRDPARTY_ROOT}/lib64 ) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-rpath=${NEBULA_THIRDPARTY_ROOT}/lib") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-rpath=${NEBULA_THIRDPARTY_ROOT}/lib64") endif() if(NOT ${NEBULA_OTHER_ROOT} STREQUAL "") @@ -226,7 +243,7 @@ if(ENABLE_ASAN) MESSAGE(FATAL_ERROR "The compiler does not support address sanitizer") endif() add_compile_options(-fsanitize=address) - add_compile_options(-g) + add_compile_options(-g${NEBULA_DEBUG_INFO_LEVEL}) add_compile_options(-fno-omit-frame-pointer) set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address") endif() @@ -238,7 +255,7 @@ if(ENABLE_UBSAN) endif() add_compile_options(-fsanitize=undefined -fno-sanitize=alignment) if(NOT ENABLE_ASAN) - add_compile_options(-g) + add_compile_options(-g${NEBULA_DEBUG_INFO_LEVEL}) add_compile_options(-fno-omit-frame-pointer) endif() set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=undefined") diff --git a/third-party/CMakeLists.txt b/third-party/CMakeLists.txt index 88eaffab91e..70cea605d3e 100644 --- a/third-party/CMakeLists.txt +++ b/third-party/CMakeLists.txt @@ -210,6 +210,15 @@ add_custom_command( rm -rf ${CMAKE_INSTALL_PREFIX}/lib64/librocksdb.so* ) +add_custom_target( + strip-archives ALL + COMMAND + strip --strip-unneeded ${CMAKE_INSTALL_PREFIX}/lib/*.a + COMMAND + strip --strip-unneeded ${CMAKE_INSTALL_PREFIX}/lib64/*.a + DEPENDS ${ALL_TARGETS} +) + add_custom_target( clean-all DEPENDS ${CLEAN_TARGETS} diff --git a/third-party/build-third-party.sh b/third-party/build-third-party.sh index 66a2fdf8263..495960a8159 100755 --- a/third-party/build-third-party.sh +++ b/third-party/build-third-party.sh @@ -95,10 +95,9 @@ fi if [[ ! $checksum = 375f349b7b5ae1212bd4195bfc30f43a ]] then hash wget &> /dev/null && download_cmd="wget -c" - hash axel &> /dev/null && download_cmd="axel -a -n 8" if [[ -z $download_cmd ]] then - echo "Neither 'wget' nor 'axel' available for downloading" 1>&2; + echo "'wget' not found for downloading" 1>&2; elif ! bash -c "$download_cmd $source_url" then # Resort to the builtin download method of cmake on failure @@ -129,6 +128,8 @@ make |& \ grep --line-buffered 'Creating\|^Scanning\|Performing\|Completed\|CMakeFiles.*Error' end_time=$(date +%s) +cd $OLDPWD && rm -rf $build_dir + echo echo "Third parties have been successfully installed to $install_dir" echo "$((end_time - start_time)) seconds been taken." diff --git a/third-party/externals/folly.cmake b/third-party/externals/folly.cmake index f49756d3d52..c67f8919361 100644 --- a/third-party/externals/folly.cmake +++ b/third-party/externals/folly.cmake @@ -23,7 +23,7 @@ ExternalProject_Add( BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} BUILD_IN_SOURCE 1 - INSTALL_COMMAND make -s -j${BUILDING_JOBS_NUM} install + INSTALL_COMMAND make -s -j${BUILDING_JOBS_NUM} install/strip LOG_CONFIGURE TRUE LOG_BUILD TRUE LOG_INSTALL TRUE From 8ac52bd2dc86e49e9320064e40ab605c6fe69a25 Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Wed, 18 Dec 2019 13:26:16 +0800 Subject: [PATCH 49/80] Fixed possible depending on system libverto --- third-party/externals/krb5.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/third-party/externals/krb5.cmake b/third-party/externals/krb5.cmake index 4012716fb91..e4f9941b1c5 100644 --- a/third-party/externals/krb5.cmake +++ b/third-party/externals/krb5.cmake @@ -41,6 +41,7 @@ ExternalProject_Add_Step(krb5 mannual-configure --disable-rpath --disable-aesni --disable-thread-support + --without-system-verto WORKING_DIRECTORY ${source_dir}/src ) From 528b1648acde6631b55946c2f0e6734288c4fd53 Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Wed, 18 Dec 2019 13:34:27 +0800 Subject: [PATCH 50/80] Removed rpath --- CMakeLists.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 61be4459d18..b6a590a896a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -160,8 +160,6 @@ if(NOT ${NEBULA_THIRDPARTY_ROOT} STREQUAL "") ${NEBULA_THIRDPARTY_ROOT}/lib ${NEBULA_THIRDPARTY_ROOT}/lib64 ) - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-rpath=${NEBULA_THIRDPARTY_ROOT}/lib") - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-rpath=${NEBULA_THIRDPARTY_ROOT}/lib64") endif() if(NOT ${NEBULA_OTHER_ROOT} STREQUAL "") From 3c1205731e567f0ea8beb9f42082af7897782a74 Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Wed, 18 Dec 2019 14:05:03 +0800 Subject: [PATCH 51/80] Print on failure --- third-party/build-gcc.sh | 3 ++- third-party/build-third-party.sh | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/third-party/build-gcc.sh b/third-party/build-gcc.sh index 161a5376566..f45b987a8c0 100755 --- a/third-party/build-gcc.sh +++ b/third-party/build-gcc.sh @@ -321,7 +321,8 @@ EOF start_time=$(date +%s) set -e set -o pipefail -echo "Starting build, on any failure, see $logfile" +echo "Starting build" +trap '[[ $? -ne 0 ]] && echo "Building failed, see $logfile for more details." 1>&2' EXIT mkdir -p $root_dir fetch_tarballs diff --git a/third-party/build-third-party.sh b/third-party/build-third-party.sh index 495960a8159..531557718ae 100755 --- a/third-party/build-third-party.sh +++ b/third-party/build-third-party.sh @@ -79,6 +79,8 @@ source_tar_name=nebula-third-party-src-1.0.tgz source_url=https://nebula-graph.oss-accelerate.aliyuncs.com/third-party/${source_tar_name} logfile=$build_root/build.log +trap '[[ $? -ne 0 ]] && echo "Building failed, see $logfile for more details." 1>&2' EXIT + # Allow to customize compilers [[ -n ${CC} ]] && C_COMPILER_ARG="-DCMAKE_C_COMPILER=${CC}" [[ -n ${CXX} ]] && CXX_COMPILER_ARG="-DCMAKE_CXX_COMPILER=${CXX}" @@ -116,7 +118,7 @@ fi mkdir -p $build_dir $install_dir cd $build_dir -echo "Starting build, on any failure, please see $logfile" +echo "Starting build" cmake -DDOWNLOAD_DIR=$download_dir \ -DCMAKE_INSTALL_PREFIX=$install_dir \ @@ -125,7 +127,7 @@ cmake -DDOWNLOAD_DIR=$download_dir \ make |& \ tee -a $logfile | \ - grep --line-buffered 'Creating\|^Scanning\|Performing\|Completed\|CMakeFiles.*Error' + grep --line-buffered 'Creating\|^Scanning\|Performing\|Completed\|CMakeFiles.*Error\|strip-archives' end_time=$(date +%s) cd $OLDPWD && rm -rf $build_dir From 2aeca6ee29387142cc209150d56f0f21054bba78 Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Wed, 18 Dec 2019 14:17:02 +0800 Subject: [PATCH 52/80] Don't fail when grep not matched --- third-party/build-third-party.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/third-party/build-third-party.sh b/third-party/build-third-party.sh index 531557718ae..207b94b0a95 100755 --- a/third-party/build-third-party.sh +++ b/third-party/build-third-party.sh @@ -127,7 +127,7 @@ cmake -DDOWNLOAD_DIR=$download_dir \ make |& \ tee -a $logfile | \ - grep --line-buffered 'Creating\|^Scanning\|Performing\|Completed\|CMakeFiles.*Error\|strip-archives' + { grep --line-buffered 'Creating\|^Scanning\|Performing\|Completed\|CMakeFiles.*Error' || true; } end_time=$(date +%s) cd $OLDPWD && rm -rf $build_dir From a0d86c93286e2acf9112fb0f266d033287326c68 Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Wed, 18 Dec 2019 20:26:15 +0800 Subject: [PATCH 53/80] Build third party automatically --- CMakeLists.txt | 6 ++++-- cmake/BuildThirdParty.cmake | 7 +++++++ 2 files changed, 11 insertions(+), 2 deletions(-) create mode 100644 cmake/BuildThirdParty.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index b6a590a896a..4e602477ced 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -141,10 +141,12 @@ set(CMAKE_MODULE_PATH "${NEBULA_HOME}/cmake" ${CMAKE_MODULE_PATH}) # When NEBULA_THIRDPARTY_ROOT is null, set default value as /opt/nebula/third-party if("${NEBULA_THIRDPARTY_ROOT}" STREQUAL "") - if (EXISTS ${CMAKE_BINARY_DIR}/third-party/install) + if(EXISTS ${CMAKE_BINARY_DIR}/third-party/install) SET(NEBULA_THIRDPARTY_ROOT ${CMAKE_BINARY_DIR}/third-party/install) + elseif(EXISTS /opt/nebula/third-party) + set(NEBULA_THIRDPARTY_ROOT "/opt/nebula/third-party") else() - SET(NEBULA_THIRDPARTY_ROOT "/opt/nebula/third-party") + include(BuildThirdParty) endif() endif() diff --git a/cmake/BuildThirdParty.cmake b/cmake/BuildThirdParty.cmake new file mode 100644 index 00000000000..8aa5a0203da --- /dev/null +++ b/cmake/BuildThirdParty.cmake @@ -0,0 +1,7 @@ +execute_process( + COMMAND + ${CMAKE_SOURCE_DIR}/third-party/build-third-party.sh + WORKING_DIRECTORY + ${CMAKE_BINARY_DIR} +) +set(NEBULA_THIRDPARTY_ROOT ${CMAKE_BINARY_DIR}/third-party/install) From 7a02fc85b5165b21d45c421bf199ed1228cc7d4b Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Thu, 19 Dec 2019 10:32:15 +0800 Subject: [PATCH 54/80] Revert the last commit --- CMakeLists.txt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4e602477ced..0d162b9759b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -143,10 +143,8 @@ set(CMAKE_MODULE_PATH "${NEBULA_HOME}/cmake" ${CMAKE_MODULE_PATH}) if("${NEBULA_THIRDPARTY_ROOT}" STREQUAL "") if(EXISTS ${CMAKE_BINARY_DIR}/third-party/install) SET(NEBULA_THIRDPARTY_ROOT ${CMAKE_BINARY_DIR}/third-party/install) - elseif(EXISTS /opt/nebula/third-party) - set(NEBULA_THIRDPARTY_ROOT "/opt/nebula/third-party") else() - include(BuildThirdParty) + SET(NEBULA_THIRDPARTY_ROOT "/opt/nebula/third-party") endif() endif() From 29f96513aced2dca66dde41f722011fa058f8b04 Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Fri, 20 Dec 2019 15:53:49 +0800 Subject: [PATCH 55/80] Remove docs(will update in another PR) --- .../1.build/1.build-source-code.md | 313 +++++++++--------- 1 file changed, 162 insertions(+), 151 deletions(-) diff --git a/docs/manual-CN/3.build-develop-and-administration/1.build/1.build-source-code.md b/docs/manual-CN/3.build-develop-and-administration/1.build/1.build-source-code.md index d1d5e0279b1..0fffa45ac14 100644 --- a/docs/manual-CN/3.build-develop-and-administration/1.build/1.build-source-code.md +++ b/docs/manual-CN/3.build-develop-and-administration/1.build/1.build-source-code.md @@ -1,188 +1,157 @@ -# 源码编译安装 +# 使用源码编译 -# 简介 -从源代码编译安装 Nebula Graph，大体分为四步： +## 编译器要求 -1. 克隆代码 -2. 安装依赖 -3. 安装 GCC 和 CMake -4. 编译安装 Nebula Graph +**Nebula Graph** 基于 C++ 14 开发，因此它需要一个支持 C++ 14 的编译器。 + +### 支持系统版本 + +- Fedora 29, 30 +- CentOS 6.5, CentOS 7.0 ~ 7.6 +- Ubuntu 16.04, 18.04 + +### 需要的硬盘和内存空间 + +当编译类型为 **Debug** 的时候，最好预留 **50G** 的 SSD 硬盘空间。 + +并提供至少 **8GB** 的内存。 + +### 本地构建 + +#### 步骤 1: 克隆代码 + +```bash +bash> git clone https://github.com/vesoft-inc/nebula.git +``` + +#### 步骤 2 : 安装依赖 + +- 环境能直接下载云存储 OSS 包的用户 -其中，如果您的系统中已经安装了 GCC 7.0+ 和 CMake 3.5+，第三步可以略过。 -# 第一步：克隆代码 ```bash -$ git clone https://github.com/vesoft-inc/nebula.git && cd nebula -nebula$ mkdir build && cd build +bash> cd nebula && ./build_dep.sh ``` -# 第二步：安装依赖 -## Redhat/CentOS/Fedora 安装命令 + +- 环境不能直接下载云存储 OSS 包的用户 + +1. 从本地源下载依赖和进行配置 ```bash -$ sudo yum install -y make \ - m4 \ - wget \ - unzip \ - xz \ - patch \ - python \ - redhat-lsb-core \ - perl-Data-Dumper \ - perl-Thread-Queue \ - readline-devel \ - ncurses-devel \ - zlib-devel \ - gcc \ - gcc-c++ \ - cmake \ - libtool \ - autoconf \ - automake \ - bison \ - flex \ - gperf \ - gettext -# Red Hat 7+ 可额外安装 autoconf-archive -$ sudo yum install -y autoconf-archive +bash> cd nebula && ./build_dep.sh N ``` -## Debian/Ubuntu/Mint 安装命令 +2. 从下面链接中下载对应版本的压缩包 + +- [Fedora 29/30](https://nebula-graph.oss-accelerate.aliyuncs.com/third-party/fedora29.tar.gz) +- [CentOS 7.0~7.6](https://nebula-graph.oss-accelerate.aliyuncs.com/third-party/centos7.5.tar.gz) +- [CentOS 6.5](https://nebula-graph.oss-accelerate.aliyuncs.com/third-party/centos6.5.tar.gz) +- [Ubuntu 1604](https://nebula-graph.oss-accelerate.aliyuncs.com/third-party/ubuntu16.tar.gz) +- [Ubuntu 1804](https://nebula-graph.oss-accelerate.aliyuncs.com/third-party/ubuntu18.tar.gz) + +3. 安装下载好的压缩包 + ```bash -$ sudo apt-get update -$ sudo apt-get install -y make \ - m4 \ - wget \ - unzip \ - xz-utils \ - patch \ - python \ - lsb-core \ - libz-dev \ - build-essential \ - libreadline-dev \ - ncurses-dev \ - build-essential \ - cmake \ - libtool \ - automake \ - autoconf \ - autoconf-archive \ - autotools-dev \ - bison \ - flex \ - gperf \ - gettext +tar xf ${package_name}.tar.gz +cd ${package_name} && ./install.sh ``` -# 第三步：安装 CMake 和 GCC -**若您的系统为 Ubuntu 18.04+，CentOS 8+，Red Hat 8+，或其他提供 GCC 7+ 和 CMake 3.5+ 的 Linux 发行版，可从官方源安装，此步可跳过。**可使用下面命令查看 GCC 与 CMake 的版本号： +#### 步骤 3: 应用 **~/.bashrc** 修改 + ```bash -$ gcc --version -$ cmake --version +bash> source ~/.bashrc ``` -## 安装 CMake -以下命令，假设当前目录为 nebula/build，如果您的情况有所不同，请自行调整相应路径。 +#### 步骤 4A: 构建 Debug 版本 + ```bash -# 脚本自动下载安装 CMake -nebula/build$ ../third-party/install-cmake.sh -# 安装完成，将 cmake 命令加入 PATH（假设 CMake 版本为 3.15.5） -nebula/build$ source cmake-3.15.5/bin/enable-cmake.sh +bash> mkdir build && cd build +bash> cmake .. +bash> make -j$(nproc) +bash> sudo make install -j$(nproc) ``` -## 安装 GCC -### 包管理器安装 -#### CentOS 6 / CentOS 7 +或构建 Release 版本。 + +#### 步骤 4B: 构建 Release 版本，重新生成 Makefile + ```bash -# 安装 SCL，即 Software Collection -$ sudo yum install centos-release-scl -# 安装 devtoolset -$ sudo yum install devtoolset-7 -# 设置 PATH 等相关变量，启用 devtoolset -$ scl enable devtoolset-7 bash -# 重设命令搜索缓存 -$ hash -r -# 确认 GCC 版本 -$ g++ --version -g++ (GCC) 7.3.1 20180303 (Red Hat 7.3.1-5) +bash> cmake -DCMAKE_BUILD_TYPE=Release .. +bash> make -j$(nproc) +bash> sudo make install -j$(nproc) ``` -#### Red Hat 6 / Red Hat 7 +#### 变量说明： +- CMAKE\_INSTALL\_PREFIX: 指定安装目录，在 cmake 的过程指定要安装到的目录，执行 `make install` 的时候就会安装到指定目录 + +例如要安装到 /home/test 目录下面，使用如下 + ```bash -# 安装 SCL，即 Software Collection - -# Red Hat 6 -$ sudo yum-config-manager --enable rhel-server-rhscl-6-rpms -# Red Hat 7 -$ sudo yum-config-manager --enable rhel-server-rhscl-7-rpms - -# 安装 devtoolset -$ sudo yum install devtoolset-7 -# 设置 PATH 等相关变量，启用 devtoolset -$ scl enable devtoolset-7 bash -# 重设命令搜索缓存 -$ hash -r -# 确认 GCC 版本 -$ g++ --version -g++ (GCC) 7.3.1 20180303 (Red Hat 7.3.1-5) +bash> cd nebula/build +bash> cmake -DCMAKE_INSTALL_PREFIX=/home/test .. +bash> make -j$(nproc) && make install -j$(nproc) ``` -#### Ubuntu 14.04 / Ubuntu 16.04 +- NEBULA\_BUILD\_VERSION: 给二进制文件设置版本号 + +例如设置版本号为`1.0.0` + ```bash -$ sudo apt-get install -y software-properties-common -$ sudo add-apt-repository ppa:ubuntu-toolchain-r/test -$ sudo apt-get update -$ sudo apt-get install -y gcc-7 g++-7 -$ export CC=gcc-7 -$ export CXX=g++-7 +bash> cd nebula/build +bash> cmake -DNEBULA_BUILD_VERSION="1.0.0" .. +bash> make -j$(nproc) && make install -j$(nproc) ``` -### 使用我们预先编译好的GCC -若系统是 Debian 或其他未提及的 Linux 发行版，您可以执行我们提供的脚本下载安装 GCC： +查看编译好的二进制文件的版本号, 例如查看安装在 /usr/local/nebula/bin/ 下面的 nebula 二进制文件的版本号 + ```bash -# 自动下载安装 -nebula/build$ ../third-party/install-gcc.sh --prefix=/opt/nebula/toolset -GCC-7.5.0 has been installed to /opt/nebula/toolset/gcc/7.5.0 -Performing usability tests -Performing regular C++14 tests...OK -Performing LeakSanitizer tests...OK -Run 'source /opt/nebula/toolset/gcc/7.5.0/enable' to start using. - -# 修改环境变量（方便起见，您可以手动加入 $HOME/.bashrc） -nebula/build$ source /opt/nebula/toolset/gcc/7.5.0/enable - -# 确认新的编译器已经生效 -nebula/build$ which g++ -/opt/nebula/toolset/gcc/7.5.0/bin/g++ -nebula/build$ $CXX -v -g++ (Nebula Graph Build) 7.5.0 -... +bash> /usr/local/nebula/bin/nebula --version ``` -### 源码安装 +- ENABLE\_TESTING: 是否编译单元测试代码，默认为 ON 。想要关闭，可以在 cmake 的时候，改为 OFF 。 + +使用如下 + ```bash -nebula/build$ ../third-party/build-gcc.sh +bash> cd nebula/build +bash> cmake -DENABLE_TESTING=OFF .. +bash> make -j$(nproc) && make install -j$(nproc) ``` -# 第四步：编译 Nebula Graph +- ENABLE\_ASAN: 编译的时候添加 AddressSanitizer，默认为 OFF 。想要开启，可以在 cmake 的时候，改为 ON 。 + +使用如下 + ```bash -# 注意 build-third-party.sh 和 cmake 命令需要在同一目录下运行 -nebula/build$ ../third-party/build-third-party.sh -nebula/build$ cmake -DENABLE_TESTING=OFF \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX=$HOME/nebula-install \ - .. -nebula/build$ make -j && make install -nebula/build$ cd $HOME/nebula-install -nebula-install$ ls -bin/ share/ etc/ scripts/ +bash> cd nebula/build +bash> cmake -DENABLE_ASAN=ON .. +bash> make -j$(nproc) && make install -j$(nproc) ``` -# 试用 Nebula Graph +#### **构建完成** + +- 如果没有任何错误信息 + + ```text + [100%] Built target .... + ``` + + **编译成功！** + +- 在安装目录 **/usr/local/nebula** 下有如下四个子目录 **etc/**, **bin/**, **scripts/** **share/** + + ```bash + [root@centos 7.5 nebula]# ls /usr/local/nebula/ + bin etc scripts share + ``` + + **现在可以开始运行 Nebula Graph** 。 + +#### 启动 Nebula Graph 服务 -经过上述的安装后，便可以启动 **Nebula Graph** 的服务。 +经过上述的安装后，便可以启动 **Nebula Graph** 的服务，**Nebula Graph** 默认的安装目录为 `/usr/local/nebula` ```bash -bash> cd $HOME/nebula-install +bash> cd /usr/local/nebula ``` 重命名 **Nebula Graph** 服务的配置文件 @@ -201,7 +170,23 @@ bash> ./bin/nebula -u user -p password --port 3699 --addr "127.0.0.1" nebula> SHOW HOSTS; ``` -# 常见问题和解决方案 +### 常见问题和解决方案 + +- **错误信息**: `/usr/bin/ld: cannot find Scrt1.o: No such file or directory` + + **解决方案**: + + **步骤 1**: 在 **~/.bashrc** 末添加如下行 + + ```bash + export LIBRARY_PATH=/usr/lib/x86_64-linux-gnu:$LIBRARY_PATH + ``` + + **步骤 2**: 应用 **~/.bashrc** 修改 + + ```bash + bash> source ~/.bashrc + ``` - **错误信息**: `[ERROR] No compiler is provided in this environment. Perhaps you are running on a JRE rather than a JDK?` @@ -227,9 +212,35 @@ nebula> SHOW HOSTS; bash> sudo update-alternatives --config java ``` + 并选择 `java-1.8.0-openjdk/java-8-openjdk` + +- **错误信息**: `internal error` + + **解决方案**: + + 你需要自己编译第三方库，第三方库的安装路径为 **/opt/nebula/third-party** + + 步骤 1: 下载第三方库的源码包 + + ```bash + bash> wget https://nebula-graph.oss-accelerate.aliyuncs.com/third-party/nebula-3rdparty.tar.gz + ``` + + 步骤 2: 解压并编译third-party + + ```bash + bash> tar xf nebula-3rdparty.tar.gz && cd nebula-3rdparty + bash> ./install_deps.sh + bash> cmake ./ + bash> sudo make + ``` + + 步骤 3: 配置项目的环境变量和其他依赖 + ```bash - export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64 - export JRE_HOME=$JAVA_HOME/jre - export CLASSPATH=$JAVA_HOME/lib:$JRE_HOME/lib:$CLASSPATH - export PATH=$JAVA_HOME/bin:$JRE_HOME/bin:$PATH + bash> cd nebula && ./build_dep.sh N + bash> source ~/.bashrc + bash> mkdir build && cd build + bash> cmake .. + bash> make ``` From b44f36cb948c660a5337470dcc4249906158c9b3 Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Fri, 20 Dec 2019 21:42:34 +0800 Subject: [PATCH 56/80] Build third party package --- CMakeLists.txt | 2 + third-party/build-third-party.sh | 45 +++++++++++++++++++++- third-party/cxx-compiler-abi-version.sh | 26 +++++++++++++ third-party/cxx-compiler-libcxx-version.sh | 13 +++++++ 4 files changed, 85 insertions(+), 1 deletion(-) create mode 100755 third-party/cxx-compiler-abi-version.sh create mode 100755 third-party/cxx-compiler-libcxx-version.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 0d162b9759b..f147fb15d77 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -143,6 +143,8 @@ set(CMAKE_MODULE_PATH "${NEBULA_HOME}/cmake" ${CMAKE_MODULE_PATH}) if("${NEBULA_THIRDPARTY_ROOT}" STREQUAL "") if(EXISTS ${CMAKE_BINARY_DIR}/third-party/install) SET(NEBULA_THIRDPARTY_ROOT ${CMAKE_BINARY_DIR}/third-party/install) + elseif(NOT $ENV{NEBULA_THIRDPARTY_ROOT} STREQUAL "") + SET(NEBULA_THIRDPARTY_ROOT $ENV{NEBULA_THIRDPARTY_ROOT}) else() SET(NEBULA_THIRDPARTY_ROOT "/opt/nebula/third-party") endif() diff --git a/third-party/build-third-party.sh b/third-party/build-third-party.sh index 207b94b0a95..e6544cf3605 100755 --- a/third-party/build-third-party.sh +++ b/third-party/build-third-party.sh @@ -18,6 +18,8 @@ then exit $? fi +this_dir=$(dirname $(readlink -f $0)) + # CMake and GCC version checking function version_cmp { mapfile -t left < <( echo $1 | tr . '\n' ) @@ -79,6 +81,7 @@ source_tar_name=nebula-third-party-src-1.0.tgz source_url=https://nebula-graph.oss-accelerate.aliyuncs.com/third-party/${source_tar_name} logfile=$build_root/build.log + trap '[[ $? -ne 0 ]] && echo "Building failed, see $logfile for more details." 1>&2' EXIT # Allow to customize compilers @@ -125,13 +128,53 @@ cmake -DDOWNLOAD_DIR=$download_dir \ ${C_COMPILER_ARG} ${CXX_COMPILER_ARG} \ $source_dir |& tee $logfile -make |& \ +make -j2 |& \ tee -a $logfile | \ { grep --line-buffered 'Creating\|^Scanning\|Performing\|Completed\|CMakeFiles.*Error' || true; } end_time=$(date +%s) cd $OLDPWD && rm -rf $build_dir +find $install_dir -name '*.la' | xargs rm -f + +function make_package { + libcxx_version=$($this_dir/cxx-compiler-libcxx-version.sh) + abi_version=$($this_dir/cxx-compiler-abi-version.sh) + set +e + libc_version=$(ldd --version | head -1 | cut -d ' ' -f4 | cut -d '-' -f1) + set -e + exec_file=$build_root/vesoft-third-party-x86_64-glibc-$libc_version-glibcxx-$libcxx_version-abi-$abi_version.sh + + echo "Creating self-extractable package $exec_file" + cat > $exec_file < /dev/null || { echo "xz: Command not found"; exit 1; } + +[[ \$# -ne 0 ]] && prefix=\$(echo "\$@" | sed 's;.*--prefix=(\S*).*;\1;p' -rn) +prefix=\${prefix:-\$PWD/third-party/install} +mkdir -p \$prefix + +[[ -w \$prefix ]] || { echo "\$prefix: No permission to write"; exit 1; } + +archive_offset=\$(awk '/^__start_of_archive__$/{print NR+1; exit 0;}' \$0) +tail -n+\$archive_offset \$0 | tar --no-same-owner --numeric-owner -xJf - -C \$prefix + +echo "Nebula Third Party has been installed to \$prefix" + +exit 0 + +__start_of_archive__ +EOF + cd $install_dir + tar -cJf - * >> $exec_file + chmod 0755 $exec_file + cd $OLDPWD +} + +[[ $build_package -ne 0 ]] && make_package + echo echo "Third parties have been successfully installed to $install_dir" echo "$((end_time - start_time)) seconds been taken." diff --git a/third-party/cxx-compiler-abi-version.sh b/third-party/cxx-compiler-abi-version.sh new file mode 100755 index 00000000000..490e67f2e8d --- /dev/null +++ b/third-party/cxx-compiler-abi-version.sh @@ -0,0 +1,26 @@ +#! /usr/bin/env bash + +# Copyright (c) 2019 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. + +set -e +abi="unknown" +cxx_cmd=${CXX:-g++} +link_flags="-std=c++14 -static-libstdc++ -static-libgcc" +tmpdir=$(mktemp -q -d /tmp/nebula-compiler-test.XXXX 2>/dev/null) +object=$tmpdir/a.out.o + +$cxx_cmd $link_flags $extra_flags -g -x c++ - -c -o $object > /dev/null < +void foobar(std::string) { +} +EOF + +string=$(nm -C $object | sed -nr 's/.*foobar.*(std.*string).*/\1/p') + +[[ $string = 'std::__cxx11::basic_string' ]] && abi=11 +[[ $string = 'std::string' ]] && abi=98 + +echo $abi diff --git a/third-party/cxx-compiler-libcxx-version.sh b/third-party/cxx-compiler-libcxx-version.sh new file mode 100755 index 00000000000..8576a3d1fb2 --- /dev/null +++ b/third-party/cxx-compiler-libcxx-version.sh @@ -0,0 +1,13 @@ +#! /usr/bin/env bash + +# Copyright (c) 2019 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. + +set -e +cxx_cmd=${CXX:-g++} + +libcxx=$($cxx_cmd --print-file-name=libstdc++.so) + +strings $libcxx | egrep '^GLIBCXX_[.0-9]+$' | sed -r 's/.*_([.0-9]+)$/\1/' | sort -t'.' -ug -k1,1 -k2,2 -k3,3 | tail -1 From 89ecc9d9441350558192ce64e539306056a32d6e Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Sat, 21 Dec 2019 01:34:58 +0800 Subject: [PATCH 57/80] Fix unknown automake building issue due to pipefail and multi-jobs --- third-party/build-third-party.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/third-party/build-third-party.sh b/third-party/build-third-party.sh index e6544cf3605..04e20520f91 100755 --- a/third-party/build-third-party.sh +++ b/third-party/build-third-party.sh @@ -128,7 +128,7 @@ cmake -DDOWNLOAD_DIR=$download_dir \ ${C_COMPILER_ARG} ${CXX_COMPILER_ARG} \ $source_dir |& tee $logfile -make -j2 |& \ +make |& \ tee -a $logfile | \ { grep --line-buffered 'Creating\|^Scanning\|Performing\|Completed\|CMakeFiles.*Error' || true; } end_time=$(date +%s) @@ -143,7 +143,7 @@ function make_package { set +e libc_version=$(ldd --version | head -1 | cut -d ' ' -f4 | cut -d '-' -f1) set -e - exec_file=$build_root/vesoft-third-party-x86_64-glibc-$libc_version-glibcxx-$libcxx_version-abi-$abi_version.sh + exec_file=$build_root/vesoft-third-party-x86_64-libc-$libc_version-libcxx-$libcxx_version-abi-$abi_version.sh echo "Creating self-extractable package $exec_file" cat > $exec_file < Date: Sun, 22 Dec 2019 13:38:06 +0800 Subject: [PATCH 58/80] Changed default installation prefix to /opt/vesoft --- CMakeLists.txt | 2 +- third-party/CMakeLists.txt | 6 +----- third-party/build-gcc.sh | 4 ++-- third-party/build-third-party.sh | 5 +++-- 4 files changed, 7 insertions(+), 10 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f147fb15d77..6a40ce9089e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -146,7 +146,7 @@ if("${NEBULA_THIRDPARTY_ROOT}" STREQUAL "") elseif(NOT $ENV{NEBULA_THIRDPARTY_ROOT} STREQUAL "") SET(NEBULA_THIRDPARTY_ROOT $ENV{NEBULA_THIRDPARTY_ROOT}) else() - SET(NEBULA_THIRDPARTY_ROOT "/opt/nebula/third-party") + SET(NEBULA_THIRDPARTY_ROOT "/opt/vesoft/third-party") endif() endif() diff --git a/third-party/CMakeLists.txt b/third-party/CMakeLists.txt index 70cea605d3e..68de6a33ff6 100644 --- a/third-party/CMakeLists.txt +++ b/third-party/CMakeLists.txt @@ -98,6 +98,7 @@ set(ALL_TARGETS lz4 libaio mstch + gperf fatal double-conversion gflags @@ -146,11 +147,6 @@ if (NOT FLEX_FOUND) list(APPEND ALL_TARGETS flex) endif() -find_package(GPERF 2.8 QUIET) -if (NOT GPERF_FOUND) - list(APPEND ALL_TARGETS gperf) -endif() - find_package(Gettext QUIET) if (NOT Gettext_FOUND) list(APPEND ALL_TARGETS gettext) diff --git a/third-party/build-gcc.sh b/third-party/build-gcc.sh index f45b987a8c0..b0214a64a12 100755 --- a/third-party/build-gcc.sh +++ b/third-party/build-gcc.sh @@ -55,7 +55,7 @@ source_dir=$root_dir/source gcc_object_dir=$root_dir/gcc-build bu_object_dir=$root_dir/binutils-build prefix=$1 -install_dir=${prefix:-$root_dir/install}/nebula/toolset/gcc/$gcc_version +install_dir=${prefix:-$root_dir/install}/vesoft/toolset/gcc/$gcc_version logfile=$root_dir/gcc-build.log triplet=x86_64-vesoft-linux distro=$(lsb_release -si) @@ -292,7 +292,7 @@ function make_package { set -e [[ \$# -ne 0 ]] && prefix=\$(echo "\$@" | sed 's;.*--prefix=(\S*).*;\1;' -r) -prefix=\${prefix:-/opt}/nebula/toolset +prefix=\${prefix:-/opt}/vesoft/toolset hash xz &> /dev/null || { echo "xz: Command not found"; exit 1; } diff --git a/third-party/build-third-party.sh b/third-party/build-third-party.sh index 04e20520f91..bc860fd9dc1 100755 --- a/third-party/build-third-party.sh +++ b/third-party/build-third-party.sh @@ -75,7 +75,8 @@ cur_dir=`pwd` source_dir=$(readlink -f $(dirname $0)/..)/third-party build_root=$cur_dir/third-party build_dir=$build_root/build -install_dir=$build_root/install +prefix=$1 +install_dir=${prefix:-$build_root/install} download_dir=$build_root/downloads source_tar_name=nebula-third-party-src-1.0.tgz source_url=https://nebula-graph.oss-accelerate.aliyuncs.com/third-party/${source_tar_name} @@ -153,7 +154,7 @@ set -e hash xz &> /dev/null || { echo "xz: Command not found"; exit 1; } [[ \$# -ne 0 ]] && prefix=\$(echo "\$@" | sed 's;.*--prefix=(\S*).*;\1;p' -rn) -prefix=\${prefix:-\$PWD/third-party/install} +prefix=\${prefix:-/opt/vesoft/third-party} mkdir -p \$prefix [[ -w \$prefix ]] || { echo "\$prefix: No permission to write"; exit 1; } From a8fe38c74089cf8ff92d1b1984bb0fc2a98afd47 Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Sun, 22 Dec 2019 14:04:34 +0800 Subject: [PATCH 59/80] Fixed static-libstdc++ --- third-party/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/third-party/CMakeLists.txt b/third-party/CMakeLists.txt index 68de6a33ff6..18746d01411 100644 --- a/third-party/CMakeLists.txt +++ b/third-party/CMakeLists.txt @@ -78,7 +78,7 @@ set(common_configure_envs "CFLAGS=${CMAKE_C_FLAGS} -fPIC -O2 -D_DEFAULT_SOURCE -D_GNU_SOURCE" "CXXFLAGS=${CMAKE_CXX_FLAGS} -fPIC -O2 -D_DEFAULT_SOURCE -D_GNU_SOURCE" "CPPFLAGS=-isystem ${CMAKE_INSTALL_PREFIX}/include" - "LDFLAGS=-L${CMAKE_INSTALL_PREFIX}/lib -L${CMAKE_INSTALL_PREFIX}/lib64" + "LDFLAGS=-L${CMAKE_INSTALL_PREFIX}/lib -L${CMAKE_INSTALL_PREFIX}/lib64 ${extra_link_libs}" "PATH=${BUILDING_PATH}" "ACLOCAL_PATH=${ACLOCAL_PATH}" ) From 76406ee9d77a73402aa11cdfb7d3b7c3f86e6504 Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Sun, 22 Dec 2019 19:20:20 +0800 Subject: [PATCH 60/80] Supported build with cxx11 abi --- third-party/CMakeLists.txt | 17 +++-- third-party/install-third-party.sh | 102 +++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+), 7 deletions(-) create mode 100755 third-party/install-third-party.sh diff --git a/third-party/CMakeLists.txt b/third-party/CMakeLists.txt index 18746d01411..52edd9ad916 100644 --- a/third-party/CMakeLists.txt +++ b/third-party/CMakeLists.txt @@ -50,6 +50,9 @@ set(BUILDING_PATH "${CMAKE_INSTALL_PREFIX}/bin:${CMAKE_INSTALL_PREFIX}/sbin:$ENV set(ACLOCAL_PATH "${CMAKE_INSTALL_PREFIX}/share/aclocal:${CMAKE_INSTALL_PREFIX}/share/aclocal-1.15") set(extra_link_libs "-static-libstdc++ -static-libgcc") +if (USE_CXX11_ABI) + set(extra_cpp_flags "-D_GLIBCXX_USE_CXX11_ABI=1") +endif() if(GLIBC_VERSION VERSION_LESS 2.17) set(extra_link_libs "${extra_link_libs} -lrt") @@ -57,10 +60,10 @@ endif() set(common_cmake_args -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX} - -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} - -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} - "-DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} -fPIC" - "-DCMAKE_C_FLAGS=${CMAKE_C_FLAGS} -fPIC" + "-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}" + "-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}" + "-DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} -fPIC ${extra_cpp_flags}" + "-DCMAKE_C_FLAGS=${CMAKE_C_FLAGS} -fPIC ${extra_cpp_flags}" "-DCMAKE_EXE_LINKER_FLAGS=${extra_link_libs}" "-DCMAKE_SHARED_LINKER_FLAGS=${extra_link_libs}" -DCMAKE_INCLUDE_PATH=${CMAKE_INSTALL_PREFIX}/include @@ -75,9 +78,9 @@ set(common_configure_envs "env" "CC=${CMAKE_C_COMPILER}" "CXX=${CMAKE_CXX_COMPILER}" - "CFLAGS=${CMAKE_C_FLAGS} -fPIC -O2 -D_DEFAULT_SOURCE -D_GNU_SOURCE" - "CXXFLAGS=${CMAKE_CXX_FLAGS} -fPIC -O2 -D_DEFAULT_SOURCE -D_GNU_SOURCE" - "CPPFLAGS=-isystem ${CMAKE_INSTALL_PREFIX}/include" + "CFLAGS=${CMAKE_C_FLAGS} -fPIC -O2 -D_DEFAULT_SOURCE -D_GNU_SOURCE ${extra_cpp_flags}" + "CXXFLAGS=${CMAKE_CXX_FLAGS} -fPIC -O2 -D_DEFAULT_SOURCE -D_GNU_SOURCE ${extra_cpp_flags}" + "CPPFLAGS=-isystem ${CMAKE_INSTALL_PREFIX}/include ${extra_cpp_flags}" "LDFLAGS=-L${CMAKE_INSTALL_PREFIX}/lib -L${CMAKE_INSTALL_PREFIX}/lib64 ${extra_link_libs}" "PATH=${BUILDING_PATH}" "ACLOCAL_PATH=${ACLOCAL_PATH}" diff --git a/third-party/install-third-party.sh b/third-party/install-third-party.sh new file mode 100755 index 00000000000..cf60a1530c5 --- /dev/null +++ b/third-party/install-third-party.sh @@ -0,0 +1,102 @@ +#! /usr/bin/env bash + +# Copyright (c) 2019 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. + +# Usage: install-third-party.sh --prefix=/opt/vesoft/third-party + +# Always use bash +shell=$(basename $(readlink /proc/$$/exe)) +if [ ! x$shell = x"bash" ] +then + bash $0 $@ + exit $? +fi + +[[ $(uname) = Linux ]] || { + echo "Only Linux is supported" + exit 1 +} + +version=7.5.0 +url_base=https://nebula-graph.oss-accelerate.aliyuncs.com/third-party +this_dir=$(dirname $(readlink -f $0)) + +# We consider two derivatives: Red Hat and Debian +# Place preset libc versions of each from newer to older +libc_preset=( 2.17 2.12 ) +libcxx_preset=( 3.4.25 3.4.23 ) + +selected_libc= +selected_libcxx= +selected_archive= +this_libc=$(ldd --version | head -1 | cut -d ')' -f 2 | cut -d ' ' -f 2) +this_libcxx=$($this_dir/cxx-compiler-libcxx-version.sh) +this_abi=$($this_dir/cxx-compiler-abi-version.sh) + +hash wget &>/dev/null || { + echo "'wget' not fould, please install it first" 1>&2 + exit 1 +} + +download_cmd="wget -c" +wget --help | grep -q '\--show-progress' && \ + download_cmd="$download_cmd -q --show-progress" || \ + download_cmd="$download_cmd --progress=bar:force:noscroll" + +function version_cmp { + mapfile -t left < <( echo $1 | tr . '\n' ) + mapfile -t right < <( echo $2 | tr . '\n') + local i + for i in ${!left[@]} + do + local lv=${left[$i]} + local rv=${right[$i]} + [[ -z $rv ]] && { echo $lv; return; } + [[ $lv -ne $rv ]] && { echo $((lv - rv)); return; } + done + ((i++)) + rv=${right[$i]} + [[ ${#right[@]} -gt ${#left[@]} ]] && { echo $((0-rv)); return; } +} + +# Find the maximum version not greater than the system one +function select_by_version { + local this_version=$1 + shift 1 + local candidates="$@" + for v in $candidates + do + if [[ $(version_cmp $v $this_version) -le 0 ]] + then + echo $v + break + fi + done +} + +selected_libc=$(select_by_version $this_libc "${libc_preset[@]}") +selected_libcxx=$(select_by_version $this_libcxx "${libcxx_preset[@]}") + +[[ -z $selected_libc ]] && { + echo "No prebuilt third-party found to download for your environment: libc-$this_libc, libcxx-$this_libcxx, ABI $this_abi_version" 1>&2 + echo "Please invoke $this_dir/build-third-party.sh to build manually" 1>&2 + exit 1 +} + +selected_archive=vesoft-third-party-x86_64-libc-$selected_libc-libcxx-$selected_libcxx-abi-$this_abi.sh + +echo $selected_archive + +url=$url_base/$selected_archive +$download_cmd $url +[[ $? -ne 0 ]] && { + echo "Downloading $selected_archive failed" 1>&2 + exit 1 +} + +bash $selected_archive $@ + +rm -rf $selected_archive From 8560842978e79973aaa5ba9381e21a8951c727f0 Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Sun, 22 Dec 2019 19:33:11 +0800 Subject: [PATCH 61/80] Support for toggle cxx abi --- third-party/CMakeLists.txt | 8 +++++--- third-party/build-third-party.sh | 8 +++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/third-party/CMakeLists.txt b/third-party/CMakeLists.txt index 52edd9ad916..93c52545cef 100644 --- a/third-party/CMakeLists.txt +++ b/third-party/CMakeLists.txt @@ -50,7 +50,9 @@ set(BUILDING_PATH "${CMAKE_INSTALL_PREFIX}/bin:${CMAKE_INSTALL_PREFIX}/sbin:$ENV set(ACLOCAL_PATH "${CMAKE_INSTALL_PREFIX}/share/aclocal:${CMAKE_INSTALL_PREFIX}/share/aclocal-1.15") set(extra_link_libs "-static-libstdc++ -static-libgcc") -if (USE_CXX11_ABI) +if (DISABLE_CXX11_ABI) + set(extra_cpp_flags "-D_GLIBCXX_USE_CXX11_ABI=0") +else() set(extra_cpp_flags "-D_GLIBCXX_USE_CXX11_ABI=1") endif() @@ -63,7 +65,7 @@ set(common_cmake_args "-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}" "-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}" "-DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} -fPIC ${extra_cpp_flags}" - "-DCMAKE_C_FLAGS=${CMAKE_C_FLAGS} -fPIC ${extra_cpp_flags}" + "-DCMAKE_C_FLAGS=${CMAKE_C_FLAGS} -fPIC" "-DCMAKE_EXE_LINKER_FLAGS=${extra_link_libs}" "-DCMAKE_SHARED_LINKER_FLAGS=${extra_link_libs}" -DCMAKE_INCLUDE_PATH=${CMAKE_INSTALL_PREFIX}/include @@ -79,7 +81,7 @@ set(common_configure_envs "CC=${CMAKE_C_COMPILER}" "CXX=${CMAKE_CXX_COMPILER}" "CFLAGS=${CMAKE_C_FLAGS} -fPIC -O2 -D_DEFAULT_SOURCE -D_GNU_SOURCE ${extra_cpp_flags}" - "CXXFLAGS=${CMAKE_CXX_FLAGS} -fPIC -O2 -D_DEFAULT_SOURCE -D_GNU_SOURCE ${extra_cpp_flags}" + "CXXFLAGS=${CMAKE_CXX_FLAGS} -fPIC -O2 -D_DEFAULT_SOURCE -D_GNU_SOURCE" "CPPFLAGS=-isystem ${CMAKE_INSTALL_PREFIX}/include ${extra_cpp_flags}" "LDFLAGS=-L${CMAKE_INSTALL_PREFIX}/lib -L${CMAKE_INSTALL_PREFIX}/lib64 ${extra_link_libs}" "PATH=${BUILDING_PATH}" diff --git a/third-party/build-third-party.sh b/third-party/build-third-party.sh index bc860fd9dc1..0aea084f286 100755 --- a/third-party/build-third-party.sh +++ b/third-party/build-third-party.sh @@ -88,6 +88,7 @@ trap '[[ $? -ne 0 ]] && echo "Building failed, see $logfile for more details." 1 # Allow to customize compilers [[ -n ${CC} ]] && C_COMPILER_ARG="-DCMAKE_C_COMPILER=${CC}" [[ -n ${CXX} ]] && CXX_COMPILER_ARG="-DCMAKE_CXX_COMPILER=${CXX}" +[[ ${disable_cxx11_abi} -ne 0 ]] && DISABLE_CXX11_ABI="-DDISABLE_CXX11_ABI=1" # Download source archives if necessary mkdir -p $build_root @@ -124,9 +125,10 @@ cd $build_dir echo "Starting build" -cmake -DDOWNLOAD_DIR=$download_dir \ - -DCMAKE_INSTALL_PREFIX=$install_dir \ - ${C_COMPILER_ARG} ${CXX_COMPILER_ARG} \ +cmake -DDOWNLOAD_DIR=$download_dir \ + -DCMAKE_INSTALL_PREFIX=$install_dir \ + ${C_COMPILER_ARG} ${CXX_COMPILER_ARG} \ + ${diable_cxx11_abi} \ $source_dir |& tee $logfile make |& \ From 85bce0026176d4d32382ead36405e8d554397847 Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Sun, 22 Dec 2019 21:49:03 +0800 Subject: [PATCH 62/80] Fix DISABLE_CXX11_ABI --- third-party/CMakeLists.txt | 4 ++-- third-party/build-third-party.sh | 2 +- third-party/externals/boost.cmake | 2 +- third-party/externals/folly.cmake | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/third-party/CMakeLists.txt b/third-party/CMakeLists.txt index 93c52545cef..b7812608308 100644 --- a/third-party/CMakeLists.txt +++ b/third-party/CMakeLists.txt @@ -50,7 +50,7 @@ set(BUILDING_PATH "${CMAKE_INSTALL_PREFIX}/bin:${CMAKE_INSTALL_PREFIX}/sbin:$ENV set(ACLOCAL_PATH "${CMAKE_INSTALL_PREFIX}/share/aclocal:${CMAKE_INSTALL_PREFIX}/share/aclocal-1.15") set(extra_link_libs "-static-libstdc++ -static-libgcc") -if (DISABLE_CXX11_ABI) +if (${DISABLE_CXX11_ABI}) set(extra_cpp_flags "-D_GLIBCXX_USE_CXX11_ABI=0") else() set(extra_cpp_flags "-D_GLIBCXX_USE_CXX11_ABI=1") @@ -81,7 +81,7 @@ set(common_configure_envs "CC=${CMAKE_C_COMPILER}" "CXX=${CMAKE_CXX_COMPILER}" "CFLAGS=${CMAKE_C_FLAGS} -fPIC -O2 -D_DEFAULT_SOURCE -D_GNU_SOURCE ${extra_cpp_flags}" - "CXXFLAGS=${CMAKE_CXX_FLAGS} -fPIC -O2 -D_DEFAULT_SOURCE -D_GNU_SOURCE" + "CXXFLAGS=${CMAKE_CXX_FLAGS} -fPIC -O2 -D_DEFAULT_SOURCE -D_GNU_SOURCE ${extra_cpp_flags}" "CPPFLAGS=-isystem ${CMAKE_INSTALL_PREFIX}/include ${extra_cpp_flags}" "LDFLAGS=-L${CMAKE_INSTALL_PREFIX}/lib -L${CMAKE_INSTALL_PREFIX}/lib64 ${extra_link_libs}" "PATH=${BUILDING_PATH}" diff --git a/third-party/build-third-party.sh b/third-party/build-third-party.sh index 0aea084f286..9e3794f9397 100755 --- a/third-party/build-third-party.sh +++ b/third-party/build-third-party.sh @@ -128,7 +128,7 @@ echo "Starting build" cmake -DDOWNLOAD_DIR=$download_dir \ -DCMAKE_INSTALL_PREFIX=$install_dir \ ${C_COMPILER_ARG} ${CXX_COMPILER_ARG} \ - ${diable_cxx11_abi} \ + ${DISABLE_CXX11_ABI} \ $source_dir |& tee $logfile make |& \ diff --git a/third-party/externals/boost.cmake b/third-party/externals/boost.cmake index 9738c567cee..c5d6b52d022 100644 --- a/third-party/externals/boost.cmake +++ b/third-party/externals/boost.cmake @@ -30,7 +30,7 @@ ExternalProject_Add( --disable-icu include=${CMAKE_INSTALL_PREFIX}/include linkflags=-L${CMAKE_INSTALL_PREFIX}/lib - "cxxflags=-fPIC" + "cxxflags=-fPIC ${extra_cpp_flags}" runtime-link=static link=static variant=release diff --git a/third-party/externals/folly.cmake b/third-party/externals/folly.cmake index c67f8919361..1a8f74111dc 100644 --- a/third-party/externals/folly.cmake +++ b/third-party/externals/folly.cmake @@ -18,7 +18,7 @@ ExternalProject_Add( CMAKE_ARGS ${common_cmake_args} -DCMAKE_BUILD_TYPE=Release - "-DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} -fPIC -DFOLLY_HAVE_CLOCK_GETTIME -D__USE_POSIX199309" + "-DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} -fPIC -DFOLLY_HAVE_CLOCK_GETTIME -D__USE_POSIX199309 ${extra_cpp_flags}" -DFOLLY_CXX_FLAGS=-Wno-error BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} From bc8751f9f6ae8224646e1f79b79c5706a0c2b680 Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Mon, 23 Dec 2019 11:42:27 +0800 Subject: [PATCH 63/80] Fixed for RedHat devtoolset --- third-party/install-third-party.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/third-party/install-third-party.sh b/third-party/install-third-party.sh index cf60a1530c5..d4443e6b405 100755 --- a/third-party/install-third-party.sh +++ b/third-party/install-third-party.sh @@ -27,7 +27,7 @@ this_dir=$(dirname $(readlink -f $0)) # We consider two derivatives: Red Hat and Debian # Place preset libc versions of each from newer to older libc_preset=( 2.17 2.12 ) -libcxx_preset=( 3.4.25 3.4.23 ) +libcxx_preset=( 3.4.26 3.4.25 3.4.23 ) selected_libc= selected_libcxx= @@ -36,6 +36,10 @@ this_libc=$(ldd --version | head -1 | cut -d ')' -f 2 | cut -d ' ' -f 2) this_libcxx=$($this_dir/cxx-compiler-libcxx-version.sh) this_abi=$($this_dir/cxx-compiler-abi-version.sh) +# There is no reliable way to detect version of libcxx of RedHat devtoolset. +# So we back off to use the oldest usable version +[[ -z $this_libcxx ]] && this_libcxx=3.4.23 + hash wget &>/dev/null || { echo "'wget' not fould, please install it first" 1>&2 exit 1 From b0126dad3e96e7f4821076eba0be2b72814cc984 Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Mon, 23 Dec 2019 15:09:48 +0800 Subject: [PATCH 64/80] Make Bison and krb5 relocatable --- cmake/FindBISON.cmake | 305 ++++++++++++++++++++++++ third-party/build-third-party.sh | 7 + third-party/cxx-compiler-abi-version.sh | 1 + 3 files changed, 313 insertions(+) create mode 100644 cmake/FindBISON.cmake diff --git a/cmake/FindBISON.cmake b/cmake/FindBISON.cmake new file mode 100644 index 00000000000..73cbc708cee --- /dev/null +++ b/cmake/FindBISON.cmake @@ -0,0 +1,305 @@ +# Distributed under the OSI-approved BSD 3-Clause License. See accompanying +# file Copyright.txt or https://cmake.org/licensing for details. + +#[=======================================================================[.rst: +FindBISON +--------- + +Find ``bison`` executable and provide a macro to generate custom build rules. + +The module defines the following variables: + +``BISON_EXECUTABLE`` + path to the ``bison`` program + +``BISON_VERSION`` + version of ``bison`` + +``BISON_FOUND`` + true if the program was found + +The minimum required version of ``bison`` can be specified using the +standard CMake syntax, e.g. ``find_package(BISON 2.1.3)``. + +If ``bison`` is found, the module defines the macro:: + + BISON_TARGET( + [COMPILE_FLAGS ] + [DEFINES_FILE ] + [VERBOSE []] + [REPORT_FILE ] + ) + +which will create a custom rule to generate a parser. ```` is +the path to a yacc file. ```` is the name of the source file +generated by bison. A header file is also be generated, and contains +the token list. + +The options are: + +``COMPILE_FLAGS `` + Specify flags to be added to the ``bison`` command line. + +``DEFINES_FILE `` + Specify a non-default header ```` to be generated by ``bison``. + +``VERBOSE []`` + Tell ``bison`` to write a report file of the grammar and parser. + If ```` is given, it specifies path the report file is copied to. + ``[]`` is left for backward compatibility of this module. + Use ``VERBOSE REPORT_FILE ``. + +``REPORT_FILE `` + Specify a non-default report ````, if generated. + +The macro defines the following variables: + +``BISON__DEFINED`` + true is the macro ran successfully + +``BISON__INPUT`` + The input source file, an alias for + +``BISON__OUTPUT_SOURCE`` + The source file generated by bison + +``BISON__OUTPUT_HEADER`` + The header file generated by bison + +``BISON__OUTPUTS`` + All files generated by bison including the source, the header and the report + +``BISON__COMPILE_FLAGS`` + Options used in the ``bison`` command line + +Example usage: + +.. code-block:: cmake + + find_package(BISON) + BISON_TARGET(MyParser parser.y ${CMAKE_CURRENT_BINARY_DIR}/parser.cpp + DEFINES_FILE ${CMAKE_CURRENT_BINARY_DIR}/parser.h) + add_executable(Foo main.cpp ${BISON_MyParser_OUTPUTS}) +#]=======================================================================] + +find_program(BISON_EXECUTABLE NAMES bison win_bison DOC "path to the bison executable") +mark_as_advanced(BISON_EXECUTABLE) + +if(BISON_EXECUTABLE) + # the bison commands should be executed with the C locale, otherwise + # the message (which are parsed) may be translated + set(_Bison_SAVED_LC_ALL "$ENV{LC_ALL}") + set(ENV{LC_ALL} C) + + execute_process(COMMAND ${BISON_EXECUTABLE} --version + OUTPUT_VARIABLE BISON_version_output + ERROR_VARIABLE BISON_version_error + RESULT_VARIABLE BISON_version_result + OUTPUT_STRIP_TRAILING_WHITESPACE) + + set(ENV{LC_ALL} ${_Bison_SAVED_LC_ALL}) + + if(NOT ${BISON_version_result} EQUAL 0) + message(SEND_ERROR "Command \"${BISON_EXECUTABLE} --version\" failed with output:\n${BISON_version_error}") + else() + # Bison++ + if("${BISON_version_output}" MATCHES "^bison\\+\\+ Version ([^,]+)") + set(BISON_VERSION "${CMAKE_MATCH_1}") + # GNU Bison + elseif("${BISON_version_output}" MATCHES "^bison \\(GNU Bison\\) ([^\n]+)\n") + set(BISON_VERSION "${CMAKE_MATCH_1}") + elseif("${BISON_version_output}" MATCHES "^GNU Bison (version )?([^\n]+)") + set(BISON_VERSION "${CMAKE_MATCH_2}") + endif() + endif() + + # internal macro + # sets BISON_TARGET_cmdopt + macro(BISON_TARGET_option_extraopts Options) + set(BISON_TARGET_cmdopt "") + set(BISON_TARGET_extraopts "${Options}") + separate_arguments(BISON_TARGET_extraopts) + list(APPEND BISON_TARGET_cmdopt ${BISON_TARGET_extraopts}) + endmacro() + + # internal macro + # sets BISON_TARGET_output_header and BISON_TARGET_cmdopt + macro(BISON_TARGET_option_defines BisonOutput Header) + if("${Header}" STREQUAL "") + # default header path generated by bison (see option -d) + string(REGEX REPLACE "^(.*)(\\.[^.]*)$" "\\2" _fileext "${BisonOutput}") + string(REPLACE "c" "h" _fileext ${_fileext}) + string(REGEX REPLACE "^(.*)(\\.[^.]*)$" "\\1${_fileext}" + BISON_TARGET_output_header "${BisonOutput}") + list(APPEND BISON_TARGET_cmdopt "-d") + else() + set(BISON_TARGET_output_header "${Header}") + list(APPEND BISON_TARGET_cmdopt "--defines=${BISON_TARGET_output_header}") + endif() + endmacro() + + # internal macro + # sets BISON_TARGET_verbose_file and BISON_TARGET_cmdopt + macro(BISON_TARGET_option_report_file BisonOutput ReportFile) + if("${ReportFile}" STREQUAL "") + get_filename_component(BISON_TARGET_output_path "${BisonOutput}" PATH) + get_filename_component(BISON_TARGET_output_name "${BisonOutput}" NAME_WE) + set(BISON_TARGET_verbose_file + "${BISON_TARGET_output_path}/${BISON_TARGET_output_name}.output") + else() + set(BISON_TARGET_verbose_file "${ReportFile}") + list(APPEND BISON_TARGET_cmdopt "--report-file=${BISON_TARGET_verbose_file}") + endif() + if(NOT IS_ABSOLUTE "${BISON_TARGET_verbose_file}") + cmake_policy(GET CMP0088 _BISON_CMP0088 + PARENT_SCOPE # undocumented, do not use outside of CMake + ) + if("x${_BISON_CMP0088}x" STREQUAL "xNEWx") + set(BISON_TARGET_verbose_file "${CMAKE_CURRENT_BINARY_DIR}/${BISON_TARGET_verbose_file}") + else() + set(BISON_TARGET_verbose_file "${CMAKE_CURRENT_SOURCE_DIR}/${BISON_TARGET_verbose_file}") + endif() + unset(_BISON_CMP0088) + endif() + endmacro() + + # internal macro + # adds a custom command and sets + # BISON_TARGET_cmdopt, BISON_TARGET_extraoutputs + macro(BISON_TARGET_option_verbose Name BisonOutput filename) + cmake_policy(GET CMP0088 _BISON_CMP0088 + PARENT_SCOPE # undocumented, do not use outside of CMake + ) + set(_BISON_WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) + if("x${_BISON_CMP0088}x" STREQUAL "xNEWx") + set(_BISON_WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) + endif() + unset(_BISON_CMP0088) + + list(APPEND BISON_TARGET_cmdopt "--verbose") + list(APPEND BISON_TARGET_outputs + "${BISON_TARGET_verbose_file}") + if (NOT "${filename}" STREQUAL "") + if(IS_ABSOLUTE "${filename}") + set(BISON_TARGET_verbose_extra_file "${filename}") + else() + set(BISON_TARGET_verbose_extra_file "${_BISON_WORKING_DIRECTORY}/${filename}") + endif() + + add_custom_command(OUTPUT ${BISON_TARGET_verbose_extra_file} + COMMAND ${CMAKE_COMMAND} -E copy + "${BISON_TARGET_verbose_file}" + "${filename}" + VERBATIM + DEPENDS + "${BISON_TARGET_verbose_file}" + COMMENT "[BISON][${Name}] Copying bison verbose table to ${filename}" + WORKING_DIRECTORY ${_BISON_WORKING_DIRECTORY}) + list(APPEND BISON_TARGET_extraoutputs + "${BISON_TARGET_verbose_extra_file}") + unset(BISON_TARGET_verbose_extra_file) + unset(_BISON_WORKING_DIRECTORY) + endif() + endmacro() + + #============================================================ + # BISON_TARGET (public macro) + #============================================================ + # + macro(BISON_TARGET Name BisonInput BisonOutput) + set(BISON_TARGET_outputs "${BisonOutput}") + set(BISON_TARGET_extraoutputs "") + + # Parsing parameters + set(BISON_TARGET_PARAM_OPTIONS + ) + set(BISON_TARGET_PARAM_ONE_VALUE_KEYWORDS + COMPILE_FLAGS + DEFINES_FILE + REPORT_FILE + ) + set(BISON_TARGET_PARAM_MULTI_VALUE_KEYWORDS + VERBOSE + ) + cmake_parse_arguments( + BISON_TARGET_ARG + "${BISON_TARGET_PARAM_OPTIONS}" + "${BISON_TARGET_PARAM_ONE_VALUE_KEYWORDS}" + "${BISON_TARGET_PARAM_MULTI_VALUE_KEYWORDS}" + ${ARGN} + ) + + if(NOT "${BISON_TARGET_ARG_UNPARSED_ARGUMENTS}" STREQUAL "") + message(SEND_ERROR "Usage") + elseif("${BISON_TARGET_ARG_VERBOSE}" MATCHES ";") + # [VERBOSE [] hack: is non-multi value by usage + message(SEND_ERROR "Usage") + else() + + BISON_TARGET_option_extraopts("${BISON_TARGET_ARG_COMPILE_FLAGS}") + BISON_TARGET_option_defines("${BisonOutput}" "${BISON_TARGET_ARG_DEFINES_FILE}") + BISON_TARGET_option_report_file("${BisonOutput}" "${BISON_TARGET_ARG_REPORT_FILE}") + if(NOT "${BISON_TARGET_ARG_VERBOSE}" STREQUAL "") + BISON_TARGET_option_verbose(${Name} ${BisonOutput} "${BISON_TARGET_ARG_VERBOSE}") + else() + # [VERBOSE []] is used with no argument or is not used + set(BISON_TARGET_args "${ARGN}") + list(FIND BISON_TARGET_args "VERBOSE" BISON_TARGET_args_indexof_verbose) + if(${BISON_TARGET_args_indexof_verbose} GREATER -1) + # VERBOSE is used without + BISON_TARGET_option_verbose(${Name} ${BisonOutput} "") + endif() + endif() + + list(APPEND BISON_TARGET_outputs "${BISON_TARGET_output_header}") + + cmake_policy(GET CMP0088 _BISON_CMP0088 + PARENT_SCOPE # undocumented, do not use outside of CMake + ) + set(_BISON_WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) + if("x${_BISON_CMP0088}x" STREQUAL "xNEWx") + set(_BISON_WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) + endif() + unset(_BISON_CMP0088) + + # Customized by Nebula Graph + set(BISON_EXECUTE_ENV "") + execute_process( + COMMAND ${BISON_EXECUTABLE} --print-datadir + OUTPUT_VARIABLE bison_encoded_data_dir + OUTPUT_STRIP_TRAILING_WHITESPACE) + if(NOT EXISTS ${bison_encoded_data_dir}) + get_filename_component(bison_prefix ${BISON_EXECUTABLE} DIRECTORY) + get_filename_component(bison_prefix ${bison_prefix} DIRECTORY) + if(EXISTS ${bison_prefix}/share/bison) + set(BISON_EXECUTE_ENV "BISON_PKGDATADIR=${bison_prefix}/share/bison") + endif() + endif() + add_custom_command(OUTPUT ${BISON_TARGET_outputs} + COMMAND ${CMAKE_COMMAND} -E env ${BISON_EXECUTE_ENV} ${BISON_EXECUTABLE} ${BISON_TARGET_cmdopt} -o ${BisonOutput} ${BisonInput} + VERBATIM + DEPENDS ${BisonInput} + COMMENT "[BISON][${Name}] Building parser with bison ${BISON_VERSION}" + WORKING_DIRECTORY ${_BISON_WORKING_DIRECTORY}) + # End customized by Nebula Graph + + unset(_BISON_WORKING_DIRECTORY) + + # define target variables + set(BISON_${Name}_DEFINED TRUE) + set(BISON_${Name}_INPUT ${BisonInput}) + set(BISON_${Name}_OUTPUTS ${BISON_TARGET_outputs} ${BISON_TARGET_extraoutputs}) + set(BISON_${Name}_COMPILE_FLAGS ${BISON_TARGET_cmdopt}) + set(BISON_${Name}_OUTPUT_SOURCE "${BisonOutput}") + set(BISON_${Name}_OUTPUT_HEADER "${BISON_TARGET_output_header}") + + endif() + endmacro() + # + #============================================================ + +endif() + +include(FindPackageHandleStandardArgs) +FIND_PACKAGE_HANDLE_STANDARD_ARGS(BISON REQUIRED_VARS BISON_EXECUTABLE + VERSION_VAR BISON_VERSION) diff --git a/third-party/build-third-party.sh b/third-party/build-third-party.sh index 9e3794f9397..a9d2f6ef693 100755 --- a/third-party/build-third-party.sh +++ b/third-party/build-third-party.sh @@ -89,6 +89,7 @@ trap '[[ $? -ne 0 ]] && echo "Building failed, see $logfile for more details." 1 [[ -n ${CC} ]] && C_COMPILER_ARG="-DCMAKE_C_COMPILER=${CC}" [[ -n ${CXX} ]] && CXX_COMPILER_ARG="-DCMAKE_CXX_COMPILER=${CXX}" [[ ${disable_cxx11_abi} -ne 0 ]] && DISABLE_CXX11_ABI="-DDISABLE_CXX11_ABI=1" +export disable_cxx11_abi # Download source archives if necessary mkdir -p $build_root @@ -138,8 +139,14 @@ end_time=$(date +%s) cd $OLDPWD && rm -rf $build_dir +# Remove all libtool files find $install_dir -name '*.la' | xargs rm -f +# Make krb5 relocatable +sed -i 's/^prefix=.*$/prefix=$(dirname $(dirname $(readlink -f $0)))/' $install_dir/bin/krb5-config +sed -i 's#^LDFLAGS=.*$#LDFLAGS="-L$prefix/lib -L$prefix/lib64"#' $install_dir/bin/krb5-config +sed -i -r 's#^DEFCKTNAME=.*(/var.*keytab).*#DEFCKTNAME="FILE:$prefix\1"#' $install_dir/bin/krb5-config + function make_package { libcxx_version=$($this_dir/cxx-compiler-libcxx-version.sh) abi_version=$($this_dir/cxx-compiler-abi-version.sh) diff --git a/third-party/cxx-compiler-abi-version.sh b/third-party/cxx-compiler-abi-version.sh index 490e67f2e8d..150643856d9 100755 --- a/third-party/cxx-compiler-abi-version.sh +++ b/third-party/cxx-compiler-abi-version.sh @@ -9,6 +9,7 @@ set -e abi="unknown" cxx_cmd=${CXX:-g++} link_flags="-std=c++14 -static-libstdc++ -static-libgcc" +[[ $disable_cxx11_abi -ne 0 ]] && extra_flags="$extra_flags -D_GLIBCXX_USE_CXX11_ABI=0" tmpdir=$(mktemp -q -d /tmp/nebula-compiler-test.XXXX 2>/dev/null) object=$tmpdir/a.out.o From e3e193e85cc569967eff11067c8219e18d72051b Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Mon, 23 Dec 2019 17:26:37 +0800 Subject: [PATCH 65/80] Install third-party automatically --- CMakeLists.txt | 4 +++- cmake/InstallThirdParty.cmake | 12 ++++++++++++ third-party/CMakeLists.txt | 4 ++-- third-party/install-third-party.sh | 13 ++++++++----- 4 files changed, 25 insertions(+), 8 deletions(-) create mode 100644 cmake/InstallThirdParty.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 6a40ce9089e..9ea925d47e3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -145,8 +145,10 @@ if("${NEBULA_THIRDPARTY_ROOT}" STREQUAL "") SET(NEBULA_THIRDPARTY_ROOT ${CMAKE_BINARY_DIR}/third-party/install) elseif(NOT $ENV{NEBULA_THIRDPARTY_ROOT} STREQUAL "") SET(NEBULA_THIRDPARTY_ROOT $ENV{NEBULA_THIRDPARTY_ROOT}) - else() + elseif(EXISTS /opt/vesoft/third-party) SET(NEBULA_THIRDPARTY_ROOT "/opt/vesoft/third-party") + else() + include(InstallThirdParty) endif() endif() diff --git a/cmake/InstallThirdParty.cmake b/cmake/InstallThirdParty.cmake new file mode 100644 index 00000000000..73a28714af1 --- /dev/null +++ b/cmake/InstallThirdParty.cmake @@ -0,0 +1,12 @@ +set(third_party_install_prefix ${CMAKE_BINARY_DIR}/third-party/install) +message(STATUS "Downloading prebuilt third party automatically...") +execute_process( + COMMAND + ${CMAKE_SOURCE_DIR}/third-party/install-third-party.sh --prefix=${third_party_install_prefix} + WORKING_DIRECTORY + ${CMAKE_BINARY_DIR} +) +if(EXISTS ${third_party_install_prefix}) + set(NEBULA_THIRDPARTY_ROOT ${third_party_install_prefix}) +endif() +unset(third_party_install_prefix) diff --git a/third-party/CMakeLists.txt b/third-party/CMakeLists.txt index b7812608308..1168509da4b 100644 --- a/third-party/CMakeLists.txt +++ b/third-party/CMakeLists.txt @@ -209,14 +209,14 @@ add_custom_command( TARGET rocksdb POST_BUILD COMMAND rm -rf ${CMAKE_INSTALL_PREFIX}/lib64/librocksdb.so* + COMMAND + rm -rf ${CMAKE_INSTALL_PREFIX}/lib/librocksdb.so* ) add_custom_target( strip-archives ALL COMMAND strip --strip-unneeded ${CMAKE_INSTALL_PREFIX}/lib/*.a - COMMAND - strip --strip-unneeded ${CMAKE_INSTALL_PREFIX}/lib64/*.a DEPENDS ${ALL_TARGETS} ) diff --git a/third-party/install-third-party.sh b/third-party/install-third-party.sh index d4443e6b405..b83ba627f31 100755 --- a/third-party/install-third-party.sh +++ b/third-party/install-third-party.sh @@ -46,9 +46,14 @@ hash wget &>/dev/null || { } download_cmd="wget -c" -wget --help | grep -q '\--show-progress' && \ - download_cmd="$download_cmd -q --show-progress" || \ - download_cmd="$download_cmd --progress=bar:force:noscroll" +if [[ -t 1 ]] +then + wget --help | grep -q '\--show-progress' && \ + download_cmd="$download_cmd -q --show-progress" || \ + download_cmd="$download_cmd --progress=bar:force:noscroll" +else + download_cmd="$download_cmd -q" +fi function version_cmp { mapfile -t left < <( echo $1 | tr . '\n' ) @@ -92,8 +97,6 @@ selected_libcxx=$(select_by_version $this_libcxx "${libcxx_preset[@]}") selected_archive=vesoft-third-party-x86_64-libc-$selected_libc-libcxx-$selected_libcxx-abi-$this_abi.sh -echo $selected_archive - url=$url_base/$selected_archive $download_cmd $url [[ $? -ne 0 ]] && { From 1821ce00c7abb3ad003cee9d5c608c425f7b3bcb Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Mon, 23 Dec 2019 17:31:42 +0800 Subject: [PATCH 66/80] Show selected version while downloading --- third-party/install-third-party.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/third-party/install-third-party.sh b/third-party/install-third-party.sh index b83ba627f31..63e492d0d6b 100755 --- a/third-party/install-third-party.sh +++ b/third-party/install-third-party.sh @@ -98,6 +98,7 @@ selected_libcxx=$(select_by_version $this_libcxx "${libcxx_preset[@]}") selected_archive=vesoft-third-party-x86_64-libc-$selected_libc-libcxx-$selected_libcxx-abi-$this_abi.sh url=$url_base/$selected_archive +echo "Downloading $selected_archive..." $download_cmd $url [[ $? -ne 0 ]] && { echo "Downloading $selected_archive failed" 1>&2 From ed33656fb8017514a2c7080c7d41964732a4fa16 Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Mon, 23 Dec 2019 18:23:54 +0800 Subject: [PATCH 67/80] Fixed relocatable for Bison in old CMake --- CMakeLists.txt | 1 + cmake/FindBISON.cmake | 305 ------------------------------- cmake/MakeBisonRelocatable.cmake | 17 ++ 3 files changed, 18 insertions(+), 305 deletions(-) delete mode 100644 cmake/FindBISON.cmake create mode 100644 cmake/MakeBisonRelocatable.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 9ea925d47e3..10d56066240 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -209,6 +209,7 @@ find_package(Krb5 REQUIRED gssapi) find_package(GPERF 2.8 REQUIRED) find_package(Libunwind REQUIRED) find_package(BISON 3.0.5 REQUIRED) +include(MakeBisonRelocatable) find_package(FLEX REQUIRED) find_package(Readline REQUIRED) find_package(NCURSES REQUIRED) diff --git a/cmake/FindBISON.cmake b/cmake/FindBISON.cmake deleted file mode 100644 index 73cbc708cee..00000000000 --- a/cmake/FindBISON.cmake +++ /dev/null @@ -1,305 +0,0 @@ -# Distributed under the OSI-approved BSD 3-Clause License. See accompanying -# file Copyright.txt or https://cmake.org/licensing for details. - -#[=======================================================================[.rst: -FindBISON ---------- - -Find ``bison`` executable and provide a macro to generate custom build rules. - -The module defines the following variables: - -``BISON_EXECUTABLE`` - path to the ``bison`` program - -``BISON_VERSION`` - version of ``bison`` - -``BISON_FOUND`` - true if the program was found - -The minimum required version of ``bison`` can be specified using the -standard CMake syntax, e.g. ``find_package(BISON 2.1.3)``. - -If ``bison`` is found, the module defines the macro:: - - BISON_TARGET( - [COMPILE_FLAGS ] - [DEFINES_FILE ] - [VERBOSE []] - [REPORT_FILE ] - ) - -which will create a custom rule to generate a parser. ```` is -the path to a yacc file. ```` is the name of the source file -generated by bison. A header file is also be generated, and contains -the token list. - -The options are: - -``COMPILE_FLAGS `` - Specify flags to be added to the ``bison`` command line. - -``DEFINES_FILE `` - Specify a non-default header ```` to be generated by ``bison``. - -``VERBOSE []`` - Tell ``bison`` to write a report file of the grammar and parser. - If ```` is given, it specifies path the report file is copied to. - ``[]`` is left for backward compatibility of this module. - Use ``VERBOSE REPORT_FILE ``. - -``REPORT_FILE `` - Specify a non-default report ````, if generated. - -The macro defines the following variables: - -``BISON__DEFINED`` - true is the macro ran successfully - -``BISON__INPUT`` - The input source file, an alias for - -``BISON__OUTPUT_SOURCE`` - The source file generated by bison - -``BISON__OUTPUT_HEADER`` - The header file generated by bison - -``BISON__OUTPUTS`` - All files generated by bison including the source, the header and the report - -``BISON__COMPILE_FLAGS`` - Options used in the ``bison`` command line - -Example usage: - -.. code-block:: cmake - - find_package(BISON) - BISON_TARGET(MyParser parser.y ${CMAKE_CURRENT_BINARY_DIR}/parser.cpp - DEFINES_FILE ${CMAKE_CURRENT_BINARY_DIR}/parser.h) - add_executable(Foo main.cpp ${BISON_MyParser_OUTPUTS}) -#]=======================================================================] - -find_program(BISON_EXECUTABLE NAMES bison win_bison DOC "path to the bison executable") -mark_as_advanced(BISON_EXECUTABLE) - -if(BISON_EXECUTABLE) - # the bison commands should be executed with the C locale, otherwise - # the message (which are parsed) may be translated - set(_Bison_SAVED_LC_ALL "$ENV{LC_ALL}") - set(ENV{LC_ALL} C) - - execute_process(COMMAND ${BISON_EXECUTABLE} --version - OUTPUT_VARIABLE BISON_version_output - ERROR_VARIABLE BISON_version_error - RESULT_VARIABLE BISON_version_result - OUTPUT_STRIP_TRAILING_WHITESPACE) - - set(ENV{LC_ALL} ${_Bison_SAVED_LC_ALL}) - - if(NOT ${BISON_version_result} EQUAL 0) - message(SEND_ERROR "Command \"${BISON_EXECUTABLE} --version\" failed with output:\n${BISON_version_error}") - else() - # Bison++ - if("${BISON_version_output}" MATCHES "^bison\\+\\+ Version ([^,]+)") - set(BISON_VERSION "${CMAKE_MATCH_1}") - # GNU Bison - elseif("${BISON_version_output}" MATCHES "^bison \\(GNU Bison\\) ([^\n]+)\n") - set(BISON_VERSION "${CMAKE_MATCH_1}") - elseif("${BISON_version_output}" MATCHES "^GNU Bison (version )?([^\n]+)") - set(BISON_VERSION "${CMAKE_MATCH_2}") - endif() - endif() - - # internal macro - # sets BISON_TARGET_cmdopt - macro(BISON_TARGET_option_extraopts Options) - set(BISON_TARGET_cmdopt "") - set(BISON_TARGET_extraopts "${Options}") - separate_arguments(BISON_TARGET_extraopts) - list(APPEND BISON_TARGET_cmdopt ${BISON_TARGET_extraopts}) - endmacro() - - # internal macro - # sets BISON_TARGET_output_header and BISON_TARGET_cmdopt - macro(BISON_TARGET_option_defines BisonOutput Header) - if("${Header}" STREQUAL "") - # default header path generated by bison (see option -d) - string(REGEX REPLACE "^(.*)(\\.[^.]*)$" "\\2" _fileext "${BisonOutput}") - string(REPLACE "c" "h" _fileext ${_fileext}) - string(REGEX REPLACE "^(.*)(\\.[^.]*)$" "\\1${_fileext}" - BISON_TARGET_output_header "${BisonOutput}") - list(APPEND BISON_TARGET_cmdopt "-d") - else() - set(BISON_TARGET_output_header "${Header}") - list(APPEND BISON_TARGET_cmdopt "--defines=${BISON_TARGET_output_header}") - endif() - endmacro() - - # internal macro - # sets BISON_TARGET_verbose_file and BISON_TARGET_cmdopt - macro(BISON_TARGET_option_report_file BisonOutput ReportFile) - if("${ReportFile}" STREQUAL "") - get_filename_component(BISON_TARGET_output_path "${BisonOutput}" PATH) - get_filename_component(BISON_TARGET_output_name "${BisonOutput}" NAME_WE) - set(BISON_TARGET_verbose_file - "${BISON_TARGET_output_path}/${BISON_TARGET_output_name}.output") - else() - set(BISON_TARGET_verbose_file "${ReportFile}") - list(APPEND BISON_TARGET_cmdopt "--report-file=${BISON_TARGET_verbose_file}") - endif() - if(NOT IS_ABSOLUTE "${BISON_TARGET_verbose_file}") - cmake_policy(GET CMP0088 _BISON_CMP0088 - PARENT_SCOPE # undocumented, do not use outside of CMake - ) - if("x${_BISON_CMP0088}x" STREQUAL "xNEWx") - set(BISON_TARGET_verbose_file "${CMAKE_CURRENT_BINARY_DIR}/${BISON_TARGET_verbose_file}") - else() - set(BISON_TARGET_verbose_file "${CMAKE_CURRENT_SOURCE_DIR}/${BISON_TARGET_verbose_file}") - endif() - unset(_BISON_CMP0088) - endif() - endmacro() - - # internal macro - # adds a custom command and sets - # BISON_TARGET_cmdopt, BISON_TARGET_extraoutputs - macro(BISON_TARGET_option_verbose Name BisonOutput filename) - cmake_policy(GET CMP0088 _BISON_CMP0088 - PARENT_SCOPE # undocumented, do not use outside of CMake - ) - set(_BISON_WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) - if("x${_BISON_CMP0088}x" STREQUAL "xNEWx") - set(_BISON_WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) - endif() - unset(_BISON_CMP0088) - - list(APPEND BISON_TARGET_cmdopt "--verbose") - list(APPEND BISON_TARGET_outputs - "${BISON_TARGET_verbose_file}") - if (NOT "${filename}" STREQUAL "") - if(IS_ABSOLUTE "${filename}") - set(BISON_TARGET_verbose_extra_file "${filename}") - else() - set(BISON_TARGET_verbose_extra_file "${_BISON_WORKING_DIRECTORY}/${filename}") - endif() - - add_custom_command(OUTPUT ${BISON_TARGET_verbose_extra_file} - COMMAND ${CMAKE_COMMAND} -E copy - "${BISON_TARGET_verbose_file}" - "${filename}" - VERBATIM - DEPENDS - "${BISON_TARGET_verbose_file}" - COMMENT "[BISON][${Name}] Copying bison verbose table to ${filename}" - WORKING_DIRECTORY ${_BISON_WORKING_DIRECTORY}) - list(APPEND BISON_TARGET_extraoutputs - "${BISON_TARGET_verbose_extra_file}") - unset(BISON_TARGET_verbose_extra_file) - unset(_BISON_WORKING_DIRECTORY) - endif() - endmacro() - - #============================================================ - # BISON_TARGET (public macro) - #============================================================ - # - macro(BISON_TARGET Name BisonInput BisonOutput) - set(BISON_TARGET_outputs "${BisonOutput}") - set(BISON_TARGET_extraoutputs "") - - # Parsing parameters - set(BISON_TARGET_PARAM_OPTIONS - ) - set(BISON_TARGET_PARAM_ONE_VALUE_KEYWORDS - COMPILE_FLAGS - DEFINES_FILE - REPORT_FILE - ) - set(BISON_TARGET_PARAM_MULTI_VALUE_KEYWORDS - VERBOSE - ) - cmake_parse_arguments( - BISON_TARGET_ARG - "${BISON_TARGET_PARAM_OPTIONS}" - "${BISON_TARGET_PARAM_ONE_VALUE_KEYWORDS}" - "${BISON_TARGET_PARAM_MULTI_VALUE_KEYWORDS}" - ${ARGN} - ) - - if(NOT "${BISON_TARGET_ARG_UNPARSED_ARGUMENTS}" STREQUAL "") - message(SEND_ERROR "Usage") - elseif("${BISON_TARGET_ARG_VERBOSE}" MATCHES ";") - # [VERBOSE [] hack: is non-multi value by usage - message(SEND_ERROR "Usage") - else() - - BISON_TARGET_option_extraopts("${BISON_TARGET_ARG_COMPILE_FLAGS}") - BISON_TARGET_option_defines("${BisonOutput}" "${BISON_TARGET_ARG_DEFINES_FILE}") - BISON_TARGET_option_report_file("${BisonOutput}" "${BISON_TARGET_ARG_REPORT_FILE}") - if(NOT "${BISON_TARGET_ARG_VERBOSE}" STREQUAL "") - BISON_TARGET_option_verbose(${Name} ${BisonOutput} "${BISON_TARGET_ARG_VERBOSE}") - else() - # [VERBOSE []] is used with no argument or is not used - set(BISON_TARGET_args "${ARGN}") - list(FIND BISON_TARGET_args "VERBOSE" BISON_TARGET_args_indexof_verbose) - if(${BISON_TARGET_args_indexof_verbose} GREATER -1) - # VERBOSE is used without - BISON_TARGET_option_verbose(${Name} ${BisonOutput} "") - endif() - endif() - - list(APPEND BISON_TARGET_outputs "${BISON_TARGET_output_header}") - - cmake_policy(GET CMP0088 _BISON_CMP0088 - PARENT_SCOPE # undocumented, do not use outside of CMake - ) - set(_BISON_WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) - if("x${_BISON_CMP0088}x" STREQUAL "xNEWx") - set(_BISON_WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) - endif() - unset(_BISON_CMP0088) - - # Customized by Nebula Graph - set(BISON_EXECUTE_ENV "") - execute_process( - COMMAND ${BISON_EXECUTABLE} --print-datadir - OUTPUT_VARIABLE bison_encoded_data_dir - OUTPUT_STRIP_TRAILING_WHITESPACE) - if(NOT EXISTS ${bison_encoded_data_dir}) - get_filename_component(bison_prefix ${BISON_EXECUTABLE} DIRECTORY) - get_filename_component(bison_prefix ${bison_prefix} DIRECTORY) - if(EXISTS ${bison_prefix}/share/bison) - set(BISON_EXECUTE_ENV "BISON_PKGDATADIR=${bison_prefix}/share/bison") - endif() - endif() - add_custom_command(OUTPUT ${BISON_TARGET_outputs} - COMMAND ${CMAKE_COMMAND} -E env ${BISON_EXECUTE_ENV} ${BISON_EXECUTABLE} ${BISON_TARGET_cmdopt} -o ${BisonOutput} ${BisonInput} - VERBATIM - DEPENDS ${BisonInput} - COMMENT "[BISON][${Name}] Building parser with bison ${BISON_VERSION}" - WORKING_DIRECTORY ${_BISON_WORKING_DIRECTORY}) - # End customized by Nebula Graph - - unset(_BISON_WORKING_DIRECTORY) - - # define target variables - set(BISON_${Name}_DEFINED TRUE) - set(BISON_${Name}_INPUT ${BisonInput}) - set(BISON_${Name}_OUTPUTS ${BISON_TARGET_outputs} ${BISON_TARGET_extraoutputs}) - set(BISON_${Name}_COMPILE_FLAGS ${BISON_TARGET_cmdopt}) - set(BISON_${Name}_OUTPUT_SOURCE "${BisonOutput}") - set(BISON_${Name}_OUTPUT_HEADER "${BISON_TARGET_output_header}") - - endif() - endmacro() - # - #============================================================ - -endif() - -include(FindPackageHandleStandardArgs) -FIND_PACKAGE_HANDLE_STANDARD_ARGS(BISON REQUIRED_VARS BISON_EXECUTABLE - VERSION_VAR BISON_VERSION) diff --git a/cmake/MakeBisonRelocatable.cmake b/cmake/MakeBisonRelocatable.cmake new file mode 100644 index 00000000000..683c9365e13 --- /dev/null +++ b/cmake/MakeBisonRelocatable.cmake @@ -0,0 +1,17 @@ +set(BISON_EXECUTE_ENV "") +execute_process( + COMMAND ${BISON_EXECUTABLE} --print-datadir + OUTPUT_VARIABLE bison_encoded_data_dir + OUTPUT_STRIP_TRAILING_WHITESPACE +) +if(NOT EXISTS ${bison_encoded_data_dir}) + get_filename_component(bison_prefix ${BISON_EXECUTABLE} DIRECTORY) + get_filename_component(bison_prefix ${bison_prefix} DIRECTORY) + if(EXISTS ${bison_prefix}/share/bison) + set(BISON_EXECUTE_ENV "BISON_PKGDATADIR=${bison_prefix}/share/bison") + endif() +endif() +if(NOT ${BISON_EXECUTE_ENV} STREQUAL "") + set(BISON_EXECUTABLE ${CMAKE_COMMAND} -E env ${BISON_EXECUTE_ENV} ${BISON_EXECUTABLE}) +endif() + From 50acf8ed4e8e8199e5d208e9243be4c36c8462f4 Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Tue, 24 Dec 2019 10:31:22 +0800 Subject: [PATCH 68/80] Changed presets of libc and libcxx --- third-party/install-third-party.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/third-party/install-third-party.sh b/third-party/install-third-party.sh index 63e492d0d6b..0150fba31d9 100755 --- a/third-party/install-third-party.sh +++ b/third-party/install-third-party.sh @@ -20,14 +20,13 @@ fi exit 1 } -version=7.5.0 url_base=https://nebula-graph.oss-accelerate.aliyuncs.com/third-party this_dir=$(dirname $(readlink -f $0)) # We consider two derivatives: Red Hat and Debian # Place preset libc versions of each from newer to older -libc_preset=( 2.17 2.12 ) -libcxx_preset=( 3.4.26 3.4.25 3.4.23 ) +libc_preset=( 2.23 2.17 2.12 ) +libcxx_preset=( 3.4.27 3.4.26 3.4.25 3.4.24 3.4.23 ) selected_libc= selected_libcxx= From 9b4dc27a0d20936675eedcccf173e9e3416b7f3d Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Tue, 24 Dec 2019 10:58:36 +0800 Subject: [PATCH 69/80] Add build-all-third-party.sh to build third-party with multiple versions of GCC --- third-party/build-all-third-party.sh | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100755 third-party/build-all-third-party.sh diff --git a/third-party/build-all-third-party.sh b/third-party/build-all-third-party.sh new file mode 100755 index 00000000000..c9d2069ade5 --- /dev/null +++ b/third-party/build-all-third-party.sh @@ -0,0 +1,21 @@ +#! /usr/bin/env bash +# +# Copyright (c) 2019 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. + +# Please note that this utility must be invoked with root privilege. + +[[ $# -ne 1 ]] && { echo "$0 " 1>&2; exit 1; } + +for v in 7.1.0 7.5.0 8.3.0 9.1.0 9.2.0 +do + echo $v; + source /opt/vesoft/toolset/gcc/$v/enable; + rm -rf /opt/vesoft/third-party + build_package=1 disable_cxx11_abi=0 $1 /opt/vesoft/third-party + rm -rf /opt/vesoft/third-party + build_package=1 disable_cxx11_abi=1 $1 /opt/vesoft/third-party + source /opt/vesoft/toolset/gcc/$v/disable; +done From 9a2351e86bca58787798536b49873181d68171b9 Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Tue, 24 Dec 2019 11:19:11 +0800 Subject: [PATCH 70/80] Unconditionally build flex and bison while building package --- third-party/CMakeLists.txt | 16 ++++++++++++---- third-party/build-all-third-party.sh | 6 +++--- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/third-party/CMakeLists.txt b/third-party/CMakeLists.txt index 1168509da4b..493dc11da19 100644 --- a/third-party/CMakeLists.txt +++ b/third-party/CMakeLists.txt @@ -142,14 +142,22 @@ if (NOT Automake_FOUND) list(APPEND ALL_TARGETS automake) endif() -find_package(BISON 3.0.5 QUIET) -if (NOT BISON_FOUND) +if ($ENV{build_package} EQUAL 1) list(APPEND ALL_TARGETS bison) +else() + find_package(BISON 3.0.5 QUIET) + if (NOT BISON_FOUND) + list(APPEND ALL_TARGETS bison) + endif() endif() -find_package(FLEX 2.6.0 QUIET) -if (NOT FLEX_FOUND) +if ($ENV{build_package} EQUAL 1) list(APPEND ALL_TARGETS flex) +else() + find_package(FLEX 2.6.0 QUIET) + if (NOT FLEX_FOUND) + list(APPEND ALL_TARGETS flex) + endif() endif() find_package(Gettext QUIET) diff --git a/third-party/build-all-third-party.sh b/third-party/build-all-third-party.sh index c9d2069ade5..46a9b84d94c 100755 --- a/third-party/build-all-third-party.sh +++ b/third-party/build-all-third-party.sh @@ -7,15 +7,15 @@ # Please note that this utility must be invoked with root privilege. -[[ $# -ne 1 ]] && { echo "$0 " 1>&2; exit 1; } +this_dir=$(dirname $(readlink -f $0)) for v in 7.1.0 7.5.0 8.3.0 9.1.0 9.2.0 do echo $v; source /opt/vesoft/toolset/gcc/$v/enable; rm -rf /opt/vesoft/third-party - build_package=1 disable_cxx11_abi=0 $1 /opt/vesoft/third-party + build_package=1 disable_cxx11_abi=0 $this_dir/build-third-party.sh /opt/vesoft/third-party rm -rf /opt/vesoft/third-party - build_package=1 disable_cxx11_abi=1 $1 /opt/vesoft/third-party + build_package=1 disable_cxx11_abi=1 $this_dir/build-third-party.sh /opt/vesoft/third-party source /opt/vesoft/toolset/gcc/$v/disable; done From 1a84d71423d424eac675137ab690212f1dbdefa4 Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Tue, 24 Dec 2019 11:45:07 +0800 Subject: [PATCH 71/80] Fixed ENV testing --- third-party/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/third-party/CMakeLists.txt b/third-party/CMakeLists.txt index 493dc11da19..641efa180fc 100644 --- a/third-party/CMakeLists.txt +++ b/third-party/CMakeLists.txt @@ -142,7 +142,7 @@ if (NOT Automake_FOUND) list(APPEND ALL_TARGETS automake) endif() -if ($ENV{build_package} EQUAL 1) +if ("$ENV{build_package}" STREQUAL "1") list(APPEND ALL_TARGETS bison) else() find_package(BISON 3.0.5 QUIET) @@ -151,7 +151,7 @@ else() endif() endif() -if ($ENV{build_package} EQUAL 1) +if ("$ENV{build_package}" STREQUAL "1") list(APPEND ALL_TARGETS flex) else() find_package(FLEX 2.6.0 QUIET) From 1a9d06a6680da2a4bae9cc956db048da1669e837 Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Tue, 24 Dec 2019 14:45:08 +0800 Subject: [PATCH 72/80] Added prebuilt third party for libc-2.27 --- third-party/install-third-party.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/third-party/install-third-party.sh b/third-party/install-third-party.sh index 0150fba31d9..a0348c7b67e 100755 --- a/third-party/install-third-party.sh +++ b/third-party/install-third-party.sh @@ -25,7 +25,7 @@ this_dir=$(dirname $(readlink -f $0)) # We consider two derivatives: Red Hat and Debian # Place preset libc versions of each from newer to older -libc_preset=( 2.23 2.17 2.12 ) +libc_preset=( 2.27 2.23 2.17 2.12 ) libcxx_preset=( 3.4.27 3.4.26 3.4.25 3.4.24 3.4.23 ) selected_libc= From 6452f82f2875f2bf9177a0c027252487df7900e6 Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Wed, 25 Dec 2019 14:21:04 +0800 Subject: [PATCH 73/80] Deal with gold linker --- CMakeLists.txt | 53 ++++++++++++++++++++++++++++------------ cmake/LinkerConfig.cmake | 22 +++++++++++++++++ 2 files changed, 60 insertions(+), 15 deletions(-) create mode 100644 cmake/LinkerConfig.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 10d56066240..972330052c7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -21,6 +21,13 @@ cmake_minimum_required(VERSION 3.5.0) project("Nebula Graph" C CXX) +# Set the project home dir +set(NEBULA_HOME ${CMAKE_CURRENT_SOURCE_DIR}) +# To include customized FindXXX.cmake modules +set(CMAKE_MODULE_PATH "${NEBULA_HOME}/cmake" ${CMAKE_MODULE_PATH}) +include(LinkerConfig) +set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libstdc++ -no-pie") + set(CMAKE_SKIP_RPATH TRUE) option(ENABLE_JEMALLOC "Whether to link jemalloc to all executables" ON) @@ -55,8 +62,6 @@ set(CMAKE_CXX_STANDARD 14) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) -set(CMAKE_EXE_LINKER_FLAGS "-static-libstdc++ -no-pie") - # To detect if ccache available find_program(ccache_program_found "ccache") if (ENABLE_CCACHE AND ccache_program_found) @@ -91,10 +96,12 @@ if(CMAKE_BUILD_TYPE STREQUAL "Debug" OR CMAKE_BUILD_TYPE STREQUAL "RelWithDebInf endif() if(CMAKE_BUILD_TYPE STREQUAL "Debug") - # TODO(dutor) To check if the linker supports this. - add_compile_options(-ffunction-sections) - add_compile_options(-fdata-sections) - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections") + if (NOT ${NEBULA_USE_LINKER} STREQUAL "gold") + # gold linker is buggy for `--gc-sections', disabled for now + add_compile_options(-ffunction-sections) + add_compile_options(-fdata-sections) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections") + endif() endif() message(STATUS "CMAKE_BUILD_TYPE = ${CMAKE_BUILD_TYPE} " @@ -102,12 +109,14 @@ message(STATUS "CMAKE_BUILD_TYPE = ${CMAKE_BUILD_TYPE} " message(STATUS "NEBULA_DEBUG_INFO_LEVEL = ${NEBULA_DEBUG_INFO_LEVEL} " "(Options are: 0, 1, 2, 3)") -set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "_build") -set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "_build") -set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "_build") +# By default, all dynamic and static libraries will be placed at ${CMAKE_BINARY_DIR}/lib, +# while all executables at ${CMAKE_BINARY_DIR}/bin. +# But for the sake of cleanliness, all executables ending with `_test' will be placed +# at ${CMAKE_BINARY_DIR}/bin/test, while those ending with `_bm' at ${CMAKE_BINARY_DIR}/bin/bench. +set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib") +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib") +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin") -# Set the project home dir -set(NEBULA_HOME ${CMAKE_CURRENT_SOURCE_DIR}) add_definitions(-DNEBULA_HOME=${NEBULA_HOME}) # Let s2 use glog @@ -136,10 +145,12 @@ if (GIT_INFO_SHA) add_definitions(-DGIT_INFO_SHA=${GIT_INFO_SHA}) endif() -# To include customized FindXXX.cmake modules -set(CMAKE_MODULE_PATH "${NEBULA_HOME}/cmake" ${CMAKE_MODULE_PATH}) - -# When NEBULA_THIRDPARTY_ROOT is null, set default value as /opt/nebula/third-party +# The precedence to decide NEBULA_THIRDPARTY_ROOT is: +# 1. The path defined with CMake argument, i.e -DNEBULA_THIRDPARTY_ROOT=path +# 2. ${CMAKE_BINARY_DIR}/third-party/install, if exists +# 3. The path specified with environment variable NEBULA_THIRDPARTY_ROOT=path +# 4. /opt/vesoft/third-party, if exists +# 5. At last, one copy will be downloaded and installed to ${CMAKE_BINARY_DIR}/third-party/install if("${NEBULA_THIRDPARTY_ROOT}" STREQUAL "") if(EXISTS ${CMAKE_BINARY_DIR}/third-party/install) SET(NEBULA_THIRDPARTY_ROOT ${CMAKE_BINARY_DIR}/third-party/install) @@ -288,6 +299,13 @@ macro(nebula_add_executable) ${nebula_exec_NAME} ${nebula_exec_LIBRARIES} ) + + if(${nebula_exec_NAME} MATCHES "_bm$") + set_target_properties( + ${nebula_exec_NAME} + PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin/bench + ) + endif() endmacro() macro(nebula_add_test) @@ -306,6 +324,11 @@ macro(nebula_add_test) LIBRARIES ${nebula_test_LIBRARIES} ) + set_target_properties( + ${nebula_test_NAME} + PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin/test + ) + if (${nebula_test_FUZZER}) #Currently only Clang supports fuzz test if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") diff --git a/cmake/LinkerConfig.cmake b/cmake/LinkerConfig.cmake new file mode 100644 index 00000000000..5b24ce49644 --- /dev/null +++ b/cmake/LinkerConfig.cmake @@ -0,0 +1,22 @@ +execute_process( + COMMAND + ld --version + COMMAND + head -1 + COMMAND + cut -d " " -f2 + OUTPUT_VARIABLE default_linker_type + OUTPUT_STRIP_TRAILING_WHITESPACE +) + +if (${default_linker_type} STREQUAL "ld") + set(default_linker_type "bfd") +endif() + +if (NOT DEFINED NEBULA_USE_LINKER) + set(NEBULA_USE_LINKER ${default_linker_type}) +endif() + +message(STATUS "NEBULA_USE_LINKER: ${NEBULA_USE_LINKER}") +set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=${NEBULA_USE_LINKER}") +set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=${NEBULA_USE_LINKER}") From 10d6a73eb31e01b7afa28a637adf6aaaa3a15437 Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Wed, 25 Dec 2019 14:25:45 +0800 Subject: [PATCH 74/80] Renamed parser_benchmark to parser_bm --- src/parser/test/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/parser/test/CMakeLists.txt b/src/parser/test/CMakeLists.txt index b3e0f5b71f2..4cf438859ff 100644 --- a/src/parser/test/CMakeLists.txt +++ b/src/parser/test/CMakeLists.txt @@ -33,7 +33,7 @@ nebula_add_test( ) nebula_add_executable( - NAME parser_benchmark + NAME parser_bm SOURCES ParserBenchmark.cpp OBJECTS ${PARSER_TEST_LIBS} LIBRARIES follybenchmark boost_regex ${THRIFT_LIBRARIES} wangle From 8c5d21aa5274fc8e29c9abc5abe35b2f94416ba6 Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Wed, 25 Dec 2019 14:37:42 +0800 Subject: [PATCH 75/80] Moved setting on test binaries to nebula_add_executable --- CMakeLists.txt | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e4a5685e7b9..e71a18538b8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -114,6 +114,7 @@ message(STATUS "NEBULA_DEBUG_INFO_LEVEL = ${NEBULA_DEBUG_INFO_LEVEL} " # while all executables at ${CMAKE_BINARY_DIR}/bin. # But for the sake of cleanliness, all executables ending with `_test' will be placed # at ${CMAKE_BINARY_DIR}/bin/test, while those ending with `_bm' at ${CMAKE_BINARY_DIR}/bin/bench. +# Please see `nebula_add_executable' for this rule. set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib") set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib") set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin") @@ -311,6 +312,13 @@ macro(nebula_add_executable) ${nebula_exec_LIBRARIES} ) + if(${nebula_exec_NAME} MATCHES "_test") + set_target_properties( + ${nebula_exec_NAME} + PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin/test + ) + endif() + if(${nebula_exec_NAME} MATCHES "_bm$") set_target_properties( ${nebula_exec_NAME} @@ -335,11 +343,6 @@ macro(nebula_add_test) LIBRARIES ${nebula_test_LIBRARIES} ) - set_target_properties( - ${nebula_test_NAME} - PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin/test - ) - if (${nebula_test_FUZZER}) #Currently only Clang supports fuzz test if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") From 7d699d54330fc7563af9101634e83fab2ad2ca85 Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Wed, 25 Dec 2019 18:33:26 +0800 Subject: [PATCH 76/80] Remove support for customizing level of debug info --- CMakeLists.txt | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e71a18538b8..f31eabaecda 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -38,7 +38,6 @@ option(ENABLE_ASAN "Whether to turn AddressSanitizer ON or OFF" OFF) option(ENABLE_TESTING "Whether to turn unit test ON or OFF" ON) option(ENABLE_UBSAN "Whether to turn Undefined Behavior Sanitizer ON or OFF" OFF) option(ENABLE_FUZZ_TEST "Whether to turn Fuzz Test ON or OFF" OFF) -set(NEBULA_DEBUG_INFO_LEVEL 1 CACHE STRING "Level of debug info, used only in Debug or RelWithDebInfo") message(STATUS "ENABLE_ASAN: ${ENABLE_ASAN}") message(STATUS "ENABLE_TESTING: ${ENABLE_TESTING}") @@ -92,7 +91,6 @@ if(NOT ${CMAKE_BUILD_TYPE} STREQUAL "Debug") endif() if(CMAKE_BUILD_TYPE STREQUAL "Debug" OR CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo") - add_compile_options(-g${NEBULA_DEBUG_INFO_LEVEL}) set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--compress-debug-sections=zlib") endif() @@ -107,8 +105,6 @@ endif() message(STATUS "CMAKE_BUILD_TYPE = ${CMAKE_BUILD_TYPE} " "(Options are: Debug, Release, RelWithDebInfo, MinSizeRel)") -message(STATUS "NEBULA_DEBUG_INFO_LEVEL = ${NEBULA_DEBUG_INFO_LEVEL} " - "(Options are: 0, 1, 2, 3)") # By default, all dynamic and static libraries will be placed at ${CMAKE_BINARY_DIR}/lib, # while all executables at ${CMAKE_BINARY_DIR}/bin. @@ -269,7 +265,7 @@ if(ENABLE_ASAN) MESSAGE(FATAL_ERROR "The compiler does not support address sanitizer") endif() add_compile_options(-fsanitize=address) - add_compile_options(-g${NEBULA_DEBUG_INFO_LEVEL}) + add_compile_options(-g) add_compile_options(-fno-omit-frame-pointer) set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address") endif() @@ -281,7 +277,7 @@ if(ENABLE_UBSAN) endif() add_compile_options(-fsanitize=undefined -fno-sanitize=alignment) if(NOT ENABLE_ASAN) - add_compile_options(-g${NEBULA_DEBUG_INFO_LEVEL}) + add_compile_options(-g) add_compile_options(-fno-omit-frame-pointer) endif() set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=undefined") From 37dc887b4fbdf7ccac49f8a9c9164aeabd67e725 Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Thu, 26 Dec 2019 15:35:39 +0800 Subject: [PATCH 77/80] Fixed for ubuntu 19.04 whose glibcxx version is messed --- cmake/InstallThirdParty.cmake | 2 +- third-party/build-third-party.sh | 5 +++-- third-party/install-third-party.sh | 17 +++++++---------- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/cmake/InstallThirdParty.cmake b/cmake/InstallThirdParty.cmake index 73a28714af1..ee9fd3f397b 100644 --- a/cmake/InstallThirdParty.cmake +++ b/cmake/InstallThirdParty.cmake @@ -2,7 +2,7 @@ set(third_party_install_prefix ${CMAKE_BINARY_DIR}/third-party/install) message(STATUS "Downloading prebuilt third party automatically...") execute_process( COMMAND - ${CMAKE_SOURCE_DIR}/third-party/install-third-party.sh --prefix=${third_party_install_prefix} + env CXX=${CMAKE_CXX_COMPILER} ${CMAKE_SOURCE_DIR}/third-party/install-third-party.sh --prefix=${third_party_install_prefix} WORKING_DIRECTORY ${CMAKE_BINARY_DIR} ) diff --git a/third-party/build-third-party.sh b/third-party/build-third-party.sh index a9d2f6ef693..85452728d9f 100755 --- a/third-party/build-third-party.sh +++ b/third-party/build-third-party.sh @@ -148,12 +148,13 @@ sed -i 's#^LDFLAGS=.*$#LDFLAGS="-L$prefix/lib -L$prefix/lib64"#' $install_dir/bi sed -i -r 's#^DEFCKTNAME=.*(/var.*keytab).*#DEFCKTNAME="FILE:$prefix\1"#' $install_dir/bin/krb5-config function make_package { - libcxx_version=$($this_dir/cxx-compiler-libcxx-version.sh) + cxx_cmd=${CXX:-g++} + gcc_version=$($cxx_cmd -dumpfullversion -dumpversion) abi_version=$($this_dir/cxx-compiler-abi-version.sh) set +e libc_version=$(ldd --version | head -1 | cut -d ' ' -f4 | cut -d '-' -f1) set -e - exec_file=$build_root/vesoft-third-party-x86_64-libc-$libc_version-libcxx-$libcxx_version-abi-$abi_version.sh + exec_file=$build_root/vesoft-third-party-x86_64-libc-$libc_version-gcc-$gcc_version-abi-$abi_version.sh echo "Creating self-extractable package $exec_file" cat > $exec_file </dev/null || { echo "'wget' not fould, please install it first" 1>&2 exit 1 @@ -86,15 +83,15 @@ function select_by_version { } selected_libc=$(select_by_version $this_libc "${libc_preset[@]}") -selected_libcxx=$(select_by_version $this_libcxx "${libcxx_preset[@]}") +selected_gcc=$(select_by_version $this_gcc "${gcc_preset[@]}") [[ -z $selected_libc ]] && { - echo "No prebuilt third-party found to download for your environment: libc-$this_libc, libcxx-$this_libcxx, ABI $this_abi_version" 1>&2 + echo "No prebuilt third-party found to download for your environment: libc-$this_libc, GCC-$this_gcc, ABI $this_abi_version" 1>&2 echo "Please invoke $this_dir/build-third-party.sh to build manually" 1>&2 exit 1 } -selected_archive=vesoft-third-party-x86_64-libc-$selected_libc-libcxx-$selected_libcxx-abi-$this_abi.sh +selected_archive=vesoft-third-party-x86_64-libc-$selected_libc-gcc-$selected_gcc-abi-$this_abi.sh url=$url_base/$selected_archive echo "Downloading $selected_archive..." From 78e77261b8d2140b69cbdd3fa91f93da39eecc5b Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Fri, 27 Dec 2019 15:11:19 +0800 Subject: [PATCH 78/80] Address @laura-ding's comments --- third-party/build-all-third-party.sh | 2 +- third-party/build-third-party.sh | 2 +- third-party/install-gcc.sh | 20 ++++++++++---------- third-party/install-third-party.sh | 24 ++++++++++++------------ 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/third-party/build-all-third-party.sh b/third-party/build-all-third-party.sh index 46a9b84d94c..2b50fbf5e43 100755 --- a/third-party/build-all-third-party.sh +++ b/third-party/build-all-third-party.sh @@ -11,7 +11,7 @@ this_dir=$(dirname $(readlink -f $0)) for v in 7.1.0 7.5.0 8.3.0 9.1.0 9.2.0 do - echo $v; + echo $v source /opt/vesoft/toolset/gcc/$v/enable; rm -rf /opt/vesoft/third-party build_package=1 disable_cxx11_abi=0 $this_dir/build-third-party.sh /opt/vesoft/third-party diff --git a/third-party/build-third-party.sh b/third-party/build-third-party.sh index 85452728d9f..d46878ad975 100755 --- a/third-party/build-third-party.sh +++ b/third-party/build-third-party.sh @@ -114,7 +114,7 @@ then echo "Source of third party was downdloaded to $build_root" echo -n "Extracting into $download_dir..." tar -xzf $source_tar_name - echo "done" + echo "done" fi else tar -xzf $source_tar_name diff --git a/third-party/install-gcc.sh b/third-party/install-gcc.sh index 847fd7db68c..5d148fd9b47 100755 --- a/third-party/install-gcc.sh +++ b/third-party/install-gcc.sh @@ -25,15 +25,15 @@ url_base=https://nebula-graph.oss-accelerate.aliyuncs.com/toolset this_dir=$(dirname $(readlink -f $0)) # We consider two derivatives: Red Hat and Debian -# Place preset libc versions of each from newer to older -CentOS_libc_preset=( 2.17 2.12 ) -Debian_libc_preset=( 2.19 2.13 ) +# Place preset libc versions of each from higher to lower +CentOS_libc_preset_version=( 2.17 2.12 ) +Debian_libc_preset_version=( 2.19 2.13 ) selected_distro= -selected_libc= +selected_libc_version= selected_archive= this_distro=$(lsb_release -si) -this_libc=$(ldd --version | head -1 | cut -d ')' -f 2 | cut -d ' ' -f 2) +this_libc_version=$(ldd --version | head -1 | cut -d ')' -f 2 | cut -d ' ' -f 2) hash wget &>/dev/null || { echo "'wget' not fould, please install it first" 1>&2 @@ -89,20 +89,20 @@ function select_libc { case $selected_distro in CentOS) - selected_libc=$(select_libc $this_libc "${CentOS_libc_preset[@]}") + selected_libc_version=$(select_libc $this_libc_version "${CentOS_libc_preset_version[@]}") ;; Debian) - selected_libc=$(select_libc $this_libc "${Debian_libc_preset[@]}") + selected_libc_version=$(select_libc $this_libc_version "${Debian_libc_preset_version[@]}") ;; esac -[[ -z $selected_libc ]] && { - echo "No suitable GCC found to download for your environment: $this_distro, glibc-$this_libc" 1>&2 +[[ -z $selected_libc_version ]] && { + echo "No suitable GCC found to download for your environment: $this_distro, glibc-$this_libc_version" 1>&2 echo "Please invoke $this_dir/build-gcc.sh to build one manually" 1>&2 exit 1 } -selected_archive=vesoft-gcc-$version-$selected_distro-x86_64-glibc-$selected_libc.sh +selected_archive=vesoft-gcc-$version-$selected_distro-x86_64-glibc-$selected_libc_version.sh url=$url_base/$selected_archive $download_cmd $url diff --git a/third-party/install-third-party.sh b/third-party/install-third-party.sh index 3943f10c9a2..3110f40a8d3 100755 --- a/third-party/install-third-party.sh +++ b/third-party/install-third-party.sh @@ -26,15 +26,15 @@ cxx_cmd=${CXX:-g++} # We consider two derivatives: Red Hat and Debian # Place preset libc versions of each from newer to older -libc_preset=( 2.27 2.23 2.17 2.12 ) -gcc_preset=( 9.2.0 9.1.0 8.3.0 7.5.0 7.1.0 ) +libc_preset_versions=( 2.27 2.23 2.17 2.12 ) +gcc_preset_versions=( 9.2.0 9.1.0 8.3.0 7.5.0 7.1.0 ) -selected_libc= -selected_gcc= +selected_libc_version= +selected_gcc_version= selected_archive= -this_libc=$(ldd --version | head -1 | cut -d ')' -f 2 | cut -d ' ' -f 2) -this_gcc=$($cxx_cmd -dumpfullversion -dumpversion) -this_abi=$($this_dir/cxx-compiler-abi-version.sh) +this_libc_version=$(ldd --version | head -1 | cut -d ')' -f 2 | cut -d ' ' -f 2) +this_gcc_version=$($cxx_cmd -dumpfullversion -dumpversion) +this_abi_version=$($this_dir/cxx-compiler-abi-version.sh) hash wget &>/dev/null || { echo "'wget' not fould, please install it first" 1>&2 @@ -82,16 +82,16 @@ function select_by_version { done } -selected_libc=$(select_by_version $this_libc "${libc_preset[@]}") -selected_gcc=$(select_by_version $this_gcc "${gcc_preset[@]}") +selected_libc_version=$(select_by_version $this_libc_version "${libc_preset_versions[@]}") +selected_gcc_version=$(select_by_version $this_gcc_version "${gcc_preset_versions[@]}") -[[ -z $selected_libc ]] && { - echo "No prebuilt third-party found to download for your environment: libc-$this_libc, GCC-$this_gcc, ABI $this_abi_version" 1>&2 +[[ -z $selected_libc_version ]] && { + echo "No prebuilt third-party found to download for your environment: libc-$this_libc_version, GCC-$this_gcc_version, ABI $this_abi_version" 1>&2 echo "Please invoke $this_dir/build-third-party.sh to build manually" 1>&2 exit 1 } -selected_archive=vesoft-third-party-x86_64-libc-$selected_libc-gcc-$selected_gcc-abi-$this_abi.sh +selected_archive=vesoft-third-party-x86_64-libc-$selected_libc_version-gcc-$selected_gcc_version-abi-$this_abi_version.sh url=$url_base/$selected_archive echo "Downloading $selected_archive..." From c7d41da52a45be0c1d862d6a0026c595fad715be Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Fri, 27 Dec 2019 15:13:10 +0800 Subject: [PATCH 79/80] Address @laura-ding's comments --- CMakeLists.txt | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f31eabaecda..db97c3c0e48 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -308,14 +308,12 @@ macro(nebula_add_executable) ${nebula_exec_LIBRARIES} ) - if(${nebula_exec_NAME} MATCHES "_test") + if(${nebula_exec_NAME} MATCHES "_test$") set_target_properties( ${nebula_exec_NAME} PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin/test ) - endif() - - if(${nebula_exec_NAME} MATCHES "_bm$") + elseif(${nebula_exec_NAME} MATCHES "_bm$") set_target_properties( ${nebula_exec_NAME} PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin/bench From f0101582aefd1f6cdd9802ca7b5b14ede2081709 Mon Sep 17 00:00:00 2001 From: dutor <440396+dutor@users.noreply.github.com> Date: Tue, 31 Dec 2019 11:11:53 +0800 Subject: [PATCH 80/80] Fixed for GCC 7 --- src/graph/LimitExecutor.cpp | 4 ++-- src/graph/OrderByExecutor.cpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/graph/LimitExecutor.cpp b/src/graph/LimitExecutor.cpp index 02af1f76816..03eea371f3d 100644 --- a/src/graph/LimitExecutor.cpp +++ b/src/graph/LimitExecutor.cpp @@ -82,7 +82,7 @@ void LimitExecutor::feedResult(std::unique_ptr result) { StatusOr> LimitExecutor::setupInterimResult() { auto result = std::make_unique(std::move(colNames_)); if (rows_.empty()) { - return result; + return std::move(result); } auto rsWriter = std::make_unique(inputs_->schema()); @@ -121,7 +121,7 @@ StatusOr> LimitExecutor::setupInterimResult() { if (rsWriter != nullptr) { result->setInterim(std::move(rsWriter)); } - return result; + return std::move(result); } diff --git a/src/graph/OrderByExecutor.cpp b/src/graph/OrderByExecutor.cpp index ce77bb524ea..3db736e4ff6 100644 --- a/src/graph/OrderByExecutor.cpp +++ b/src/graph/OrderByExecutor.cpp @@ -163,7 +163,7 @@ Status OrderByExecutor::beforeExecute() { StatusOr> OrderByExecutor::setupInterimResult() { auto result = std::make_unique(std::move(colNames_)); if (rows_.empty()) { - return result; + return std::move(result); } auto schema = inputs_->schema(); @@ -201,7 +201,7 @@ StatusOr> OrderByExecutor::setupInterimResult() { } result->setInterim(std::move(rsWriter)); - return result; + return std::move(result); } void OrderByExecutor::setupResponse(cpp2::ExecutionResponse &resp) {