diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5a6aeb5..6308589 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -41,20 +41,11 @@ jobs: #- ubuntu1604 #- ubuntu1804 - ubuntu2004 - #- ubuntu2204 - compiler: - - 9.3.0 - - '' + - ubuntu2204 + - ubuntu2404 arch: - x64 - arm64 - exclude: - - os: centos7 - compiler: '' - #- os: centos8 - # compiler: '' - #- os: ubuntu2204 - # compiler: '9.3.0' container: image: vesoft/third-party-build:${{ matrix.os }} steps: @@ -63,11 +54,11 @@ jobs: run: find . -mindepth 1 -delete - uses: actions/checkout@v4 - name: Set up environment - if: matrix.compiler != '' + if: matrix.os = 'centos7' run: | export URL=https://raw.githubusercontent.com/vesoft-inc/nebula-gears/master/install bash <(curl -Ls $URL) - install-gcc --version=${{ matrix.compiler }} + install-gcc --version=9.5.0 - name: Pre-Build run: | mkdir -p _build @@ -76,8 +67,8 @@ jobs: cat .env - name: Build run: | - if [[ -n "${{ matrix.compiler }}" ]]; then - export TOOLSET_GCC_DIR=/opt/vesoft/toolset/gcc/${{ matrix.compiler }} + if [[ "${{ matrix.os }}" == "centos7" }}" ]]; then + export TOOLSET_GCC_DIR=/opt/vesoft/toolset/gcc/9.5.0 export CC=${TOOLSET_GCC_DIR}/bin/gcc export CXX=${TOOLSET_GCC_DIR}/bin/g++ export PATH=${TOOLSET_GCC_DIR}/bin:${PATH} diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index d2fdf74..c583f57 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -27,6 +27,7 @@ jobs: - centos-8 - ubuntu-2004 - ubuntu-2204 + - ubuntu-2404 steps: - name: Setup environment variables id: vars diff --git a/docker/buildx/ubuntu-2404.Dockerfile b/docker/buildx/ubuntu-2404.Dockerfile new file mode 100644 index 0000000..37091a4 --- /dev/null +++ b/docker/buildx/ubuntu-2404.Dockerfile @@ -0,0 +1,64 @@ +FROM ubuntu:24.04 +SHELL ["/bin/bash", "-c"] +ARG DEBIAN_FRONTEND=noninteractive +ARG GOLANG_VERSION=1.21.6 +RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && echo 'Asia/Shanghai' >/etc/timezone +RUN apt update \ + && apt install -y make \ + git \ + m4 \ + curl \ + wget \ + unzip \ + xz-utils \ + patch \ + python3 \ + python3-dev \ + lsb-release \ + zlib1g-dev \ + build-essential \ + libreadline-dev \ + libncurses-dev \ + cmake \ + libtool \ + automake \ + autoconf \ + autoconf-archive \ + autotools-dev \ + bison \ + flex \ + gperf \ + gettext \ + ninja-build \ + groff-base \ + texinfo \ + && apt clean \ + && rm -rf /var/lib/apt/lists/* + +# Install golang +RUN ARCH="$(uname -m)"; \ + case "${ARCH}" in \ + x86_64) GOARCH='amd64';; \ + aarch64) GOARCH='arm64';; \ + *) echo "Unsupported architecture: ${ARCH}" && exit 1;; \ + esac; \ + curl -L https://go.dev/dl/go${GOLANG_VERSION}.linux-${GOARCH}.tar.gz -o go.tar.gz && \ + tar -C /usr/local -xzf go.tar.gz && \ + rm go.tar.gz +ENV PATH="/usr/local/go/bin:${PATH}" + +# Install ossutil +RUN curl https://gosspublic.alicdn.com/ossutil/install.sh | bash + +# Install MinIO Client +RUN if [ "$(uname -m)" = "aarch64" ]; then \ + curl -O https://dl.min.io/client/mc/release/linux-arm64/mc; \ + else \ + curl -O https://dl.min.io/client/mc/release/linux-amd64/mc; \ + fi \ + && chmod +x mc \ + && mv mc /usr/local/bin + +ENV PACKAGE_DIR=/usr/src/third-party +RUN mkdir -p ${PACKAGE_DIR} +WORKDIR ${PACKAGE_DIR} diff --git a/install-third-party.sh b/install-third-party.sh index 8cb1ab2..15466f3 100755 --- a/install-third-party.sh +++ b/install-third-party.sh @@ -26,8 +26,8 @@ source $this_dir/.env # We consider two derivatives: Red Hat and Debian # Place preset libc versions of each from newer to older -libc_preset_versions=( 2.35 2.34 2.31 2.28 2.27 2.23 2.17 ) -gcc_preset_versions=( 10.1.0 9.3.0 9.2.0 9.1.0 8.3.0 7.5.0 7.1.0 ) ++libc_preset_versions=( 2.35 2.34 2.31 2.28 2.17 ) ++gcc_preset_versions=( 13.2.0 11.4.0 9.5.0 9.4.0 9.3.0 8.5.0 ) selected_libc_version= selected_gcc_version= diff --git a/project/CMakeLists.txt b/project/CMakeLists.txt index 4e3ae6b..467996e 100644 --- a/project/CMakeLists.txt +++ b/project/CMakeLists.txt @@ -246,6 +246,7 @@ set(ALL_TARGETS hnswlib openblas annoy + libxml2 ) list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/cmake) @@ -334,7 +335,7 @@ maybe_add_dependencies(cachelib fbthrift sparsemap fizz googletest) # Here we build gtest after rocksdb intentionally maybe_add_dependencies(googletest rocksdb gflags) -maybe_add_dependencies(arrow openssl xsimd llvm boost jemalloc zstd utf8proc protobuf re2) +maybe_add_dependencies(arrow openssl xsimd llvm boost jemalloc zstd utf8proc protobuf re2 libxml2) maybe_add_dependencies(protobuf zlib abseil) maybe_add_dependencies(re2 abseil) maybe_add_dependencies(grpc protobuf re2 c-ares openssl) diff --git a/project/externals/arrow.cmake b/project/externals/arrow.cmake index 5bb17b3..40bf617 100644 --- a/project/externals/arrow.cmake +++ b/project/externals/arrow.cmake @@ -2,16 +2,18 @@ # # This source code is licensed under Apache 2.0 License. # This source code is licensed under Apache 2.0 License. +set(name arrow) +set(source_dir ${CMAKE_CURRENT_BINARY_DIR}/${name}/source) + if(DISTRO_NAME STREQUAL "CentOS Linux" AND DISTRO_VERSION_ID STREQUAL "7") set(USE_LLVM_CXX ON) else() set(USE_LLVM_CXX OFF) endif() -set(name arrow) -set(source_dir ${CMAKE_CURRENT_BINARY_DIR}/${name}/source) set(ARROW_CMAKE_ARGS - -DProtobuf_SOURCE=BUNDLED + ${common_cmake_args} + -DProtobuf_SOURCE=SYSTEM -Dre2_SOURCE=SYSTEM -DBoost_ROOT=${CMAKE_INSTALL_PREFIX} -DGTest_ROOT=${CMAKE_INSTALL_PREFIX} @@ -36,6 +38,9 @@ set(ARROW_CMAKE_ARGS -DARROW_BUILD_SHARED=ON -DARROW_TEST_MEMCHECK=OFF -DARROW_PYTHON=OFF + -DARROW_PARQUET=ON + -DARROW_ORC=ON + -DARROW_DATASET=ON -DUSE_LLVM_CXX=${USE_LLVM_CXX} -DCMAKE_SHARED_LINKER_FLAGS=-Wl,-rpath=\$ORIGIN:\$ORIGIN/../3rd -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX} @@ -43,14 +48,14 @@ set(ARROW_CMAKE_ARGS ExternalProject_Add( ${name} - URL https://github.com/apache/arrow/archive/refs/tags/apache-arrow-11.0.0.tar.gz - URL_HASH MD5=c31335b6f5422a352e2aef5a7a469c91 - DOWNLOAD_NAME apache-arrow-11.0.0.tar.gz + URL https://github.com/apache/arrow/archive/refs/tags/apache-arrow-18.0.0.tar.gz + URL_HASH MD5=4a6dfd10649ab03caf71d740edff4889 + DOWNLOAD_NAME apache-arrow-18.0.0.tar.gz PREFIX ${CMAKE_CURRENT_BINARY_DIR}/${name} TMP_DIR ${BUILD_INFO_DIR} STAMP_DIR ${BUILD_INFO_DIR} DOWNLOAD_DIR ${DOWNLOAD_DIR} - PATCH_COMMAND patch -p1 < ${CMAKE_SOURCE_DIR}/patches/${name}-11.0.0.patch + PATCH_COMMAND patch -p1 < ${CMAKE_SOURCE_DIR}/patches/${name}-18.0.0.patch CONFIGURE_COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" ${ARROW_CMAKE_ARGS} ./cpp BUILD_COMMAND "${MakeEnvs}" diff --git a/project/externals/boost.cmake b/project/externals/boost.cmake index 4644e4e..af20a03 100644 --- a/project/externals/boost.cmake +++ b/project/externals/boost.cmake @@ -66,7 +66,7 @@ ExternalProject_Add_Step(${name} setup-compiler ExternalProject_Add_Step(${name} trim DEPENDEES install COMMAND - rm -rf ${CMAKE_INSTALL_PREFIX}/include/boost/{wave,log,atomic,test,fusion,geometry,gil,phoenix,spirit,beast,asio,compute,polygon,proto,units,metaparse,qvm,vmd,xpressive} + bash -c "rm -rf ${CMAKE_INSTALL_PREFIX}/include/boost/{wave,log,atomic,test,fusion,geometry,gil,phoenix,spirit,beast,asio,compute,polygon,proto,units,metaparse,qvm,vmd,xpressive}" WORKING_DIRECTORY ${source_dir} ) diff --git a/project/externals/faiss.cmake b/project/externals/faiss.cmake index 99b98c5..8ee9c3a 100644 --- a/project/externals/faiss.cmake +++ b/project/externals/faiss.cmake @@ -44,4 +44,3 @@ ExternalProject_Add_Step(${name} clean ) ExternalProject_Add_StepTargets(${name} clean) - diff --git a/project/externals/libxml2.cmake b/project/externals/libxml2.cmake new file mode 100644 index 0000000..ab68525 --- /dev/null +++ b/project/externals/libxml2.cmake @@ -0,0 +1,43 @@ +# Copyright (c) 2024 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License. + +set(name libxml2) +set(source_dir ${CMAKE_CURRENT_BINARY_DIR}/${name}/source) + +ExternalProject_Add( + ${name} + URL https://github.com/GNOME/libxml2/archive/refs/tags/v2.13.4.tar.gz + URL_HASH MD5=69d7c63c7fe5858ba7a462f303939c4a + DOWNLOAD_NAME libxml2-v2.13.4.tar.gz + PREFIX ${CMAKE_CURRENT_BINARY_DIR}/${name} + TMP_DIR ${BUILD_INFO_DIR} + STAMP_DIR ${BUILD_INFO_DIR} + DOWNLOAD_DIR ${DOWNLOAD_DIR} + SOURCE_DIR ${source_dir} + CMAKE_ARGS + ${common_cmake_args} + -DCMAKE_BUILD_TYPE=Release + -DBUILD_SHARED_LIBS=ON + -DLIBXML2_WITH_ICONV=OFF + -DLIBXML2_WITH_LZMA=OFF + -DLIBXML2_WITH_PYTHON=OFF + -DLIBXML2_WITH_ZLIB=OFF + BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM} + BUILD_IN_SOURCE 1 + INSTALL_COMMAND make -s install -j${BUILDING_JOBS_NUM} + LOG_CONFIGURE TRUE + LOG_BUILD TRUE + LOG_INSTALL TRUE +) + +ExternalProject_Add_Step(${name} clean + EXCLUDE_FROM_MAIN TRUE + ALWAYS TRUE + DEPENDEES configure + COMMAND make clean -j + COMMAND rm -f ${BUILD_INFO_DIR}/${name}-build + WORKING_DIRECTORY ${source_dir} +) + +ExternalProject_Add_StepTargets(${name} clean) diff --git a/project/patches/arrow-11.0.0.patch b/project/patches/arrow-11.0.0.patch deleted file mode 100644 index 8fbfcee..0000000 --- a/project/patches/arrow-11.0.0.patch +++ /dev/null @@ -1,32 +0,0 @@ -diff --git a/cpp/src/gandiva/precompiled/CMakeLists.txt b/cpp/src/gandiva/precompiled/CMakeLists.txt -index d7c7ef157..2c92e28d2 100644 ---- a/cpp/src/gandiva/precompiled/CMakeLists.txt -+++ b/cpp/src/gandiva/precompiled/CMakeLists.txt -@@ -49,6 +49,19 @@ else() - set(PLATFORM_CLANG_OPTIONS -std=c++17) - endif() - -+if (USE_LLVM_CXX) -+ message(STATUS "USE LLVM CXX") -+ file(GLOB_RECURSE LLVM_CXX_INCLUDE_DIR -+ "${LLVM_INCLUDE_DIRS}/*-gnu/c++/v1/__config_site") -+ -+ list(GET LLVM_CXX_INCLUDE_DIR 0 LLVM_CXX_INCLUDE_DIR) -+ get_filename_component(LLVM_CXX_INCLUDE_DIR ${LLVM_CXX_INCLUDE_DIR} DIRECTORY) -+ set(LLVM_CXX_INCLUDE -I${LLVM_INCLUDE_DIRS}/c++/v1 -I${LLVM_CXX_INCLUDE_DIR}) -+else() -+ message(STATUS "NOT USE LLVM CXX") -+ set(LLVM_CXX_INCLUDE "") -+endif() -+ - # Create bitcode for each of the source files. - foreach(SRC_FILE ${PRECOMPILED_SRCS}) - get_filename_component(SRC_BASE ${SRC_FILE} NAME_WE) -@@ -79,6 +87,7 @@ foreach(SRC_FILE ${PRECOMPILED_SRCS}) - -o - ${BC_FILE} - ${ARROW_GANDIVA_PC_CXX_FLAGS} -+ ${LLVM_CXX_INCLUDE} - -I${CMAKE_SOURCE_DIR}/src - -I${ARROW_BINARY_DIR}/src) - diff --git a/project/patches/arrow-18.0.0.patch b/project/patches/arrow-18.0.0.patch new file mode 100644 index 0000000..dc7eff1 --- /dev/null +++ b/project/patches/arrow-18.0.0.patch @@ -0,0 +1,30 @@ +--- a/cpp/cmake_modules/GandivaAddBitcode.cmake 2024-10-16 09:20:06.000000000 +0000 ++++ b/cpp/cmake_modules/GandivaAddBitcode.cmake 2024-11-05 10:07:31.339538160 +0000 +@@ -30,6 +30,16 @@ + -fms-compatibility-version=${FMS_COMPATIBILITY}) + endif() + ++ if (USE_LLVM_CXX) ++ message(STATUS "USE LLVM CXX") ++ file(GLOB_RECURSE LLVM_CXX_INCLUDE_DIR ++ "${LLVM_INCLUDE_DIRS}/*-gnu/c++/v1/__config_site") ++ ++ list(GET LLVM_CXX_INCLUDE_DIR 0 LLVM_CXX_INCLUDE_DIR) ++ get_filename_component(LLVM_CXX_INCLUDE_DIR ${LLVM_CXX_INCLUDE_DIR} DIRECTORY) ++ list(APPEND CLANG_OPTIONS -I${LLVM_INCLUDE_DIRS}/c++/v1 -I${LLVM_CXX_INCLUDE_DIR}) ++ endif() ++ + get_filename_component(SOURCE_BASE ${SOURCE} NAME_WE) + get_filename_component(ABSOLUTE_SOURCE ${SOURCE} ABSOLUTE) + set(BC_FILE ${CMAKE_CURRENT_BINARY_DIR}/${SOURCE_BASE}.bc) +--- a/cpp/src/arrow/testing/process.cc 2024-11-05 08:13:47.487969501 +0000 ++++ b/cpp/src/arrow/testing/process.cc 2024-11-05 09:27:42.138928785 +0000 +@@ -18,7 +18,7 @@ + #include "arrow/testing/process.h" + #include "arrow/result.h" + +-#define BOOST_PROCESS_AVAILABLE ++#undef BOOST_PROCESS_AVAILABLE + #ifdef __EMSCRIPTEN__ + # undef BOOST_PROCESS_AVAILABLE + #endif