Add Ubuntu 24.04 support & upgrade arrow 18

.github/workflows/build.yml

@@ -41,20 +41,11 @@ jobs:
           #- ubuntu1604
           #- ubuntu1804
           - ubuntu2004
-          #- ubuntu2204
-        compiler:
-          - 9.3.0
-          - ''
+          - ubuntu2204
+          - ubuntu2404
         arch:
           - x64
           - arm64
-        exclude:
-          - os: centos7
-            compiler: ''
-          #- os: centos8
-          #  compiler: ''
-          #- os: ubuntu2204
-          #  compiler: '9.3.0'
     container:
       image: vesoft/third-party-build:${{ matrix.os }}
     steps:
@@ -63,11 +54,11 @@ jobs:
         run: find . -mindepth 1 -delete
     - uses: actions/checkout@v4
     - name: Set up environment
-      if: matrix.compiler != ''
+      if: matrix.os = 'centos7'
       run: |
         export URL=https://raw.githubusercontent.com/vesoft-inc/nebula-gears/master/install
         bash <(curl -Ls $URL)
-        install-gcc --version=${{ matrix.compiler }}
+        install-gcc --version=9.5.0
     - name: Pre-Build
       run: |
         mkdir -p _build
@@ -76,8 +67,8 @@ jobs:
         cat .env
     - name: Build
       run: |
-        if [[ -n "${{ matrix.compiler }}" ]]; then
-          export TOOLSET_GCC_DIR=/opt/vesoft/toolset/gcc/${{ matrix.compiler }}
+        if [[ "${{ matrix.os }}" == "centos7" }}" ]]; then
+          export TOOLSET_GCC_DIR=/opt/vesoft/toolset/gcc/9.5.0
           export CC=${TOOLSET_GCC_DIR}/bin/gcc
           export CXX=${TOOLSET_GCC_DIR}/bin/g++
           export PATH=${TOOLSET_GCC_DIR}/bin:${PATH}

.github/workflows/docker.yml

@@ -27,6 +27,7 @@ jobs:
           - centos-8
           - ubuntu-2004
           - ubuntu-2204
+          - ubuntu-2404
     steps:
     - name: Setup environment variables
       id: vars

docker/buildx/ubuntu-2404.Dockerfile

@@ -0,0 +1,64 @@
+FROM ubuntu:24.04
+SHELL ["/bin/bash", "-c"]
+ARG DEBIAN_FRONTEND=noninteractive
+ARG GOLANG_VERSION=1.21.6
+RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && echo 'Asia/Shanghai' >/etc/timezone
+RUN apt update \
+     && apt install -y make \
+                       git \
+                       m4 \
+                       curl \
+                       wget \
+                       unzip \
+                       xz-utils \
+                       patch \
+                       python3 \
+                       python3-dev \
+                       lsb-release \
+                       zlib1g-dev \
+                       build-essential \
+                       libreadline-dev \
+                       libncurses-dev \
+                       cmake \
+                       libtool \
+                       automake \
+                       autoconf \
+                       autoconf-archive \
+                       autotools-dev \
+                       bison \
+                       flex \
+                       gperf \
+                       gettext \
+                       ninja-build \
+                       groff-base \
+                       texinfo \
+    && apt clean \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install golang
+RUN ARCH="$(uname -m)"; \
+    case "${ARCH}" in \
+        x86_64) GOARCH='amd64';; \
+        aarch64) GOARCH='arm64';; \
+        *) echo "Unsupported architecture: ${ARCH}" && exit 1;; \
+    esac; \
+    curl -L https://go.dev/dl/go${GOLANG_VERSION}.linux-${GOARCH}.tar.gz -o go.tar.gz && \
+    tar -C /usr/local -xzf go.tar.gz && \
+    rm go.tar.gz
+ENV PATH="/usr/local/go/bin:${PATH}"
+
+# Install ossutil
+RUN curl https://gosspublic.alicdn.com/ossutil/install.sh | bash
+
+# Install MinIO Client
+RUN if [ "$(uname -m)" = "aarch64" ]; then \
+        curl -O https://dl.min.io/client/mc/release/linux-arm64/mc; \
+    else \
+        curl -O https://dl.min.io/client/mc/release/linux-amd64/mc; \
+    fi \
+    && chmod +x mc \
+    && mv mc /usr/local/bin
+
+ENV PACKAGE_DIR=/usr/src/third-party
+RUN mkdir -p ${PACKAGE_DIR}
+WORKDIR ${PACKAGE_DIR}

install-third-party.sh

@@ -26,8 +26,8 @@ source $this_dir/.env
 
 # We consider two derivatives: Red Hat and Debian
 # Place preset libc versions of each from newer to older
-libc_preset_versions=( 2.35 2.34 2.31 2.28 2.27 2.23 2.17 )
-gcc_preset_versions=( 10.1.0 9.3.0 9.2.0 9.1.0 8.3.0 7.5.0 7.1.0 )
++libc_preset_versions=( 2.35 2.34 2.31 2.28 2.17 )
++gcc_preset_versions=( 13.2.0 11.4.0 9.5.0 9.4.0 9.3.0 8.5.0 )
 
 selected_libc_version=
 selected_gcc_version=

project/CMakeLists.txt

@@ -246,6 +246,7 @@ set(ALL_TARGETS
     hnswlib
     openblas
     annoy
+    libxml2
 )
 
 list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/cmake)
@@ -334,7 +335,7 @@ maybe_add_dependencies(cachelib fbthrift sparsemap fizz googletest)
 # Here we build gtest after rocksdb intentionally
 maybe_add_dependencies(googletest rocksdb gflags)
 
-maybe_add_dependencies(arrow openssl xsimd llvm boost jemalloc zstd utf8proc protobuf re2)
+maybe_add_dependencies(arrow openssl xsimd llvm boost jemalloc zstd utf8proc protobuf re2 libxml2)
 maybe_add_dependencies(protobuf zlib abseil)
 maybe_add_dependencies(re2 abseil)
 maybe_add_dependencies(grpc protobuf re2 c-ares openssl)

project/externals/arrow.cmake

@@ -2,16 +2,18 @@
 #
 # This source code is licensed under Apache 2.0 License.
 # This source code is licensed under Apache 2.0 License.
+set(name arrow)
+set(source_dir ${CMAKE_CURRENT_BINARY_DIR}/${name}/source)
+
 if(DISTRO_NAME STREQUAL "CentOS Linux" AND DISTRO_VERSION_ID STREQUAL "7")
     set(USE_LLVM_CXX ON)
 else()
     set(USE_LLVM_CXX OFF)
 endif()
 
-set(name arrow)
-set(source_dir ${CMAKE_CURRENT_BINARY_DIR}/${name}/source)
 set(ARROW_CMAKE_ARGS
-        -DProtobuf_SOURCE=BUNDLED
+        ${common_cmake_args}
+        -DProtobuf_SOURCE=SYSTEM
         -Dre2_SOURCE=SYSTEM
         -DBoost_ROOT=${CMAKE_INSTALL_PREFIX}
         -DGTest_ROOT=${CMAKE_INSTALL_PREFIX}
@@ -36,21 +38,24 @@ set(ARROW_CMAKE_ARGS
         -DARROW_BUILD_SHARED=ON
         -DARROW_TEST_MEMCHECK=OFF
         -DARROW_PYTHON=OFF
+        -DARROW_PARQUET=ON
+        -DARROW_ORC=ON
+        -DARROW_DATASET=ON
         -DUSE_LLVM_CXX=${USE_LLVM_CXX}
         -DCMAKE_SHARED_LINKER_FLAGS=-Wl,-rpath=\$ORIGIN:\$ORIGIN/../3rd
         -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}
         )
 
 ExternalProject_Add(
         ${name}
-        URL https://github.com/apache/arrow/archive/refs/tags/apache-arrow-11.0.0.tar.gz
-        URL_HASH MD5=c31335b6f5422a352e2aef5a7a469c91
-        DOWNLOAD_NAME apache-arrow-11.0.0.tar.gz
+        URL https://github.com/apache/arrow/archive/refs/tags/apache-arrow-18.0.0.tar.gz
+        URL_HASH MD5=4a6dfd10649ab03caf71d740edff4889
+        DOWNLOAD_NAME apache-arrow-18.0.0.tar.gz
         PREFIX ${CMAKE_CURRENT_BINARY_DIR}/${name}
         TMP_DIR ${BUILD_INFO_DIR}
         STAMP_DIR ${BUILD_INFO_DIR}
         DOWNLOAD_DIR ${DOWNLOAD_DIR}
-        PATCH_COMMAND patch -p1 < ${CMAKE_SOURCE_DIR}/patches/${name}-11.0.0.patch
+        PATCH_COMMAND patch -p1 < ${CMAKE_SOURCE_DIR}/patches/${name}-18.0.0.patch
         CONFIGURE_COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" ${ARROW_CMAKE_ARGS} ./cpp
         BUILD_COMMAND
         "${MakeEnvs}"

project/externals/boost.cmake

@@ -66,7 +66,7 @@ ExternalProject_Add_Step(${name} setup-compiler
 ExternalProject_Add_Step(${name} trim
     DEPENDEES install
     COMMAND
-        rm -rf ${CMAKE_INSTALL_PREFIX}/include/boost/{wave,log,atomic,test,fusion,geometry,gil,phoenix,spirit,beast,asio,compute,polygon,proto,units,metaparse,qvm,vmd,xpressive}
+        bash -c "rm -rf ${CMAKE_INSTALL_PREFIX}/include/boost/{wave,log,atomic,test,fusion,geometry,gil,phoenix,spirit,beast,asio,compute,polygon,proto,units,metaparse,qvm,vmd,xpressive}"
     WORKING_DIRECTORY ${source_dir}
 )
 

project/externals/faiss.cmake

@@ -44,4 +44,3 @@ ExternalProject_Add_Step(${name} clean
 )
 
 ExternalProject_Add_StepTargets(${name} clean)
-

project/externals/libxml2.cmake

@@ -0,0 +1,43 @@
+# Copyright (c) 2024 vesoft inc. All rights reserved.
+#
+# This source code is licensed under Apache 2.0 License.
+
+set(name libxml2)
+set(source_dir ${CMAKE_CURRENT_BINARY_DIR}/${name}/source)
+
+ExternalProject_Add(
+    ${name}
+    URL https://github.com/GNOME/libxml2/archive/refs/tags/v2.13.4.tar.gz
+    URL_HASH MD5=69d7c63c7fe5858ba7a462f303939c4a
+    DOWNLOAD_NAME libxml2-v2.13.4.tar.gz
+    PREFIX ${CMAKE_CURRENT_BINARY_DIR}/${name}
+    TMP_DIR ${BUILD_INFO_DIR}
+    STAMP_DIR ${BUILD_INFO_DIR}
+    DOWNLOAD_DIR ${DOWNLOAD_DIR}
+    SOURCE_DIR ${source_dir}
+    CMAKE_ARGS
+        ${common_cmake_args}
+        -DCMAKE_BUILD_TYPE=Release
+        -DBUILD_SHARED_LIBS=ON
+        -DLIBXML2_WITH_ICONV=OFF
+        -DLIBXML2_WITH_LZMA=OFF
+        -DLIBXML2_WITH_PYTHON=OFF
+        -DLIBXML2_WITH_ZLIB=OFF
+    BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM}
+    BUILD_IN_SOURCE 1
+    INSTALL_COMMAND make -s install -j${BUILDING_JOBS_NUM}
+    LOG_CONFIGURE TRUE
+    LOG_BUILD TRUE
+    LOG_INSTALL TRUE
+)
+
+ExternalProject_Add_Step(${name} clean
+    EXCLUDE_FROM_MAIN TRUE
+    ALWAYS TRUE
+    DEPENDEES configure
+    COMMAND make clean -j
+    COMMAND rm -f ${BUILD_INFO_DIR}/${name}-build
+    WORKING_DIRECTORY ${source_dir}
+)
+
+ExternalProject_Add_StepTargets(${name} clean)

project/patches/arrow-18.0.0.patch

@@ -0,0 +1,30 @@
+--- a/cpp/cmake_modules/GandivaAddBitcode.cmake	2024-10-16 09:20:06.000000000 +0000
++++ b/cpp/cmake_modules/GandivaAddBitcode.cmake	2024-11-05 10:07:31.339538160 +0000
+@@ -30,6 +30,16 @@
+          -fms-compatibility-version=${FMS_COMPATIBILITY})
+   endif()
+
++  if (USE_LLVM_CXX)
++    message(STATUS "USE LLVM CXX")
++    file(GLOB_RECURSE LLVM_CXX_INCLUDE_DIR
++      "${LLVM_INCLUDE_DIRS}/*-gnu/c++/v1/__config_site")
++
++    list(GET LLVM_CXX_INCLUDE_DIR 0 LLVM_CXX_INCLUDE_DIR)
++    get_filename_component(LLVM_CXX_INCLUDE_DIR ${LLVM_CXX_INCLUDE_DIR} DIRECTORY)
++    list(APPEND CLANG_OPTIONS -I${LLVM_INCLUDE_DIRS}/c++/v1 -I${LLVM_CXX_INCLUDE_DIR})
++  endif()
++
+   get_filename_component(SOURCE_BASE ${SOURCE} NAME_WE)
+   get_filename_component(ABSOLUTE_SOURCE ${SOURCE} ABSOLUTE)
+   set(BC_FILE ${CMAKE_CURRENT_BINARY_DIR}/${SOURCE_BASE}.bc)
+--- a/cpp/src/arrow/testing/process.cc	2024-11-05 08:13:47.487969501 +0000
++++ b/cpp/src/arrow/testing/process.cc	2024-11-05 09:27:42.138928785 +0000
+@@ -18,7 +18,7 @@
+ #include "arrow/testing/process.h"
+ #include "arrow/result.h"
+
+-#define BOOST_PROCESS_AVAILABLE
++#undef BOOST_PROCESS_AVAILABLE
+ #ifdef __EMSCRIPTEN__
+ #  undef BOOST_PROCESS_AVAILABLE
+ #endif