Skip to content

Commit

Permalink
Add Ubuntu 24.04 support & upgrade arrow 18 (#151)
Browse files Browse the repository at this point in the history
* add ubuntu 24.04 support;upgrade arrow18

* add libxml2

* Update build.yml

* Update docker.yml

* Update build.yml

* Update docker.yml

* Update libxml2.cmake

* Update boost.cmake

* Update boost.cmake

* Update install-third-party.sh

* Update build.yml

* Update install-third-party.sh

* Delete project/patches/arrow-11.0.0.patch

* Update libxml2.cmake

* Update arrow.cmake

* Update libxml2.cmake

* Update libxml2.cmake

* Update build.yml
  • Loading branch information
Shinji-IkariG authored Nov 19, 2024
1 parent 6e04c3f commit 0d43d91
Show file tree
Hide file tree
Showing 11 changed files with 161 additions and 59 deletions.
21 changes: 6 additions & 15 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -41,20 +41,11 @@ jobs:
#- ubuntu1604
#- ubuntu1804
- ubuntu2004
#- ubuntu2204
compiler:
- 9.3.0
- ''
- ubuntu2204
- ubuntu2404
arch:
- x64
- arm64
exclude:
- os: centos7
compiler: ''
#- os: centos8
# compiler: ''
#- os: ubuntu2204
# compiler: '9.3.0'
container:
image: vesoft/third-party-build:${{ matrix.os }}
steps:
Expand All @@ -63,11 +54,11 @@ jobs:
run: find . -mindepth 1 -delete
- uses: actions/checkout@v4
- name: Set up environment
if: matrix.compiler != ''
if: matrix.os = 'centos7'
run: |
export URL=https://raw.githubusercontent.com/vesoft-inc/nebula-gears/master/install
bash <(curl -Ls $URL)
install-gcc --version=${{ matrix.compiler }}
install-gcc --version=9.5.0
- name: Pre-Build
run: |
mkdir -p _build
Expand All @@ -76,8 +67,8 @@ jobs:
cat .env
- name: Build
run: |
if [[ -n "${{ matrix.compiler }}" ]]; then
export TOOLSET_GCC_DIR=/opt/vesoft/toolset/gcc/${{ matrix.compiler }}
if [[ "${{ matrix.os }}" == "centos7" }}" ]]; then
export TOOLSET_GCC_DIR=/opt/vesoft/toolset/gcc/9.5.0
export CC=${TOOLSET_GCC_DIR}/bin/gcc
export CXX=${TOOLSET_GCC_DIR}/bin/g++
export PATH=${TOOLSET_GCC_DIR}/bin:${PATH}
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/docker.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ jobs:
- centos-8
- ubuntu-2004
- ubuntu-2204
- ubuntu-2404
steps:
- name: Setup environment variables
id: vars
Expand Down
64 changes: 64 additions & 0 deletions docker/buildx/ubuntu-2404.Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
FROM ubuntu:24.04
SHELL ["/bin/bash", "-c"]
ARG DEBIAN_FRONTEND=noninteractive
ARG GOLANG_VERSION=1.21.6
RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && echo 'Asia/Shanghai' >/etc/timezone
RUN apt update \
&& apt install -y make \
git \
m4 \
curl \
wget \
unzip \
xz-utils \
patch \
python3 \
python3-dev \
lsb-release \
zlib1g-dev \
build-essential \
libreadline-dev \
libncurses-dev \
cmake \
libtool \
automake \
autoconf \
autoconf-archive \
autotools-dev \
bison \
flex \
gperf \
gettext \
ninja-build \
groff-base \
texinfo \
&& apt clean \
&& rm -rf /var/lib/apt/lists/*

# Install golang
RUN ARCH="$(uname -m)"; \
case "${ARCH}" in \
x86_64) GOARCH='amd64';; \
aarch64) GOARCH='arm64';; \
*) echo "Unsupported architecture: ${ARCH}" && exit 1;; \
esac; \
curl -L https://go.dev/dl/go${GOLANG_VERSION}.linux-${GOARCH}.tar.gz -o go.tar.gz && \
tar -C /usr/local -xzf go.tar.gz && \
rm go.tar.gz
ENV PATH="/usr/local/go/bin:${PATH}"

# Install ossutil
RUN curl https://gosspublic.alicdn.com/ossutil/install.sh | bash

# Install MinIO Client
RUN if [ "$(uname -m)" = "aarch64" ]; then \
curl -O https://dl.min.io/client/mc/release/linux-arm64/mc; \
else \
curl -O https://dl.min.io/client/mc/release/linux-amd64/mc; \
fi \
&& chmod +x mc \
&& mv mc /usr/local/bin

ENV PACKAGE_DIR=/usr/src/third-party
RUN mkdir -p ${PACKAGE_DIR}
WORKDIR ${PACKAGE_DIR}
4 changes: 2 additions & 2 deletions install-third-party.sh
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,8 @@ source $this_dir/.env

# We consider two derivatives: Red Hat and Debian
# Place preset libc versions of each from newer to older
libc_preset_versions=( 2.35 2.34 2.31 2.28 2.27 2.23 2.17 )
gcc_preset_versions=( 10.1.0 9.3.0 9.2.0 9.1.0 8.3.0 7.5.0 7.1.0 )
+libc_preset_versions=( 2.35 2.34 2.31 2.28 2.17 )
+gcc_preset_versions=( 13.2.0 11.4.0 9.5.0 9.4.0 9.3.0 8.5.0 )

selected_libc_version=
selected_gcc_version=
Expand Down
3 changes: 2 additions & 1 deletion project/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -246,6 +246,7 @@ set(ALL_TARGETS
hnswlib
openblas
annoy
libxml2
)

list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/cmake)
Expand Down Expand Up @@ -334,7 +335,7 @@ maybe_add_dependencies(cachelib fbthrift sparsemap fizz googletest)
# Here we build gtest after rocksdb intentionally
maybe_add_dependencies(googletest rocksdb gflags)

maybe_add_dependencies(arrow openssl xsimd llvm boost jemalloc zstd utf8proc protobuf re2)
maybe_add_dependencies(arrow openssl xsimd llvm boost jemalloc zstd utf8proc protobuf re2 libxml2)
maybe_add_dependencies(protobuf zlib abseil)
maybe_add_dependencies(re2 abseil)
maybe_add_dependencies(grpc protobuf re2 c-ares openssl)
Expand Down
19 changes: 12 additions & 7 deletions project/externals/arrow.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,18 @@
#
# This source code is licensed under Apache 2.0 License.
# This source code is licensed under Apache 2.0 License.
set(name arrow)
set(source_dir ${CMAKE_CURRENT_BINARY_DIR}/${name}/source)

if(DISTRO_NAME STREQUAL "CentOS Linux" AND DISTRO_VERSION_ID STREQUAL "7")
set(USE_LLVM_CXX ON)
else()
set(USE_LLVM_CXX OFF)
endif()

set(name arrow)
set(source_dir ${CMAKE_CURRENT_BINARY_DIR}/${name}/source)
set(ARROW_CMAKE_ARGS
-DProtobuf_SOURCE=BUNDLED
${common_cmake_args}
-DProtobuf_SOURCE=SYSTEM
-Dre2_SOURCE=SYSTEM
-DBoost_ROOT=${CMAKE_INSTALL_PREFIX}
-DGTest_ROOT=${CMAKE_INSTALL_PREFIX}
Expand All @@ -36,21 +38,24 @@ set(ARROW_CMAKE_ARGS
-DARROW_BUILD_SHARED=ON
-DARROW_TEST_MEMCHECK=OFF
-DARROW_PYTHON=OFF
-DARROW_PARQUET=ON
-DARROW_ORC=ON
-DARROW_DATASET=ON
-DUSE_LLVM_CXX=${USE_LLVM_CXX}
-DCMAKE_SHARED_LINKER_FLAGS=-Wl,-rpath=\$ORIGIN:\$ORIGIN/../3rd
-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}
)

ExternalProject_Add(
${name}
URL https://github.com/apache/arrow/archive/refs/tags/apache-arrow-11.0.0.tar.gz
URL_HASH MD5=c31335b6f5422a352e2aef5a7a469c91
DOWNLOAD_NAME apache-arrow-11.0.0.tar.gz
URL https://github.com/apache/arrow/archive/refs/tags/apache-arrow-18.0.0.tar.gz
URL_HASH MD5=4a6dfd10649ab03caf71d740edff4889
DOWNLOAD_NAME apache-arrow-18.0.0.tar.gz
PREFIX ${CMAKE_CURRENT_BINARY_DIR}/${name}
TMP_DIR ${BUILD_INFO_DIR}
STAMP_DIR ${BUILD_INFO_DIR}
DOWNLOAD_DIR ${DOWNLOAD_DIR}
PATCH_COMMAND patch -p1 < ${CMAKE_SOURCE_DIR}/patches/${name}-11.0.0.patch
PATCH_COMMAND patch -p1 < ${CMAKE_SOURCE_DIR}/patches/${name}-18.0.0.patch
CONFIGURE_COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" ${ARROW_CMAKE_ARGS} ./cpp
BUILD_COMMAND
"${MakeEnvs}"
Expand Down
2 changes: 1 addition & 1 deletion project/externals/boost.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ ExternalProject_Add_Step(${name} setup-compiler
ExternalProject_Add_Step(${name} trim
DEPENDEES install
COMMAND
rm -rf ${CMAKE_INSTALL_PREFIX}/include/boost/{wave,log,atomic,test,fusion,geometry,gil,phoenix,spirit,beast,asio,compute,polygon,proto,units,metaparse,qvm,vmd,xpressive}
bash -c "rm -rf ${CMAKE_INSTALL_PREFIX}/include/boost/{wave,log,atomic,test,fusion,geometry,gil,phoenix,spirit,beast,asio,compute,polygon,proto,units,metaparse,qvm,vmd,xpressive}"
WORKING_DIRECTORY ${source_dir}
)

Expand Down
1 change: 0 additions & 1 deletion project/externals/faiss.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -44,4 +44,3 @@ ExternalProject_Add_Step(${name} clean
)

ExternalProject_Add_StepTargets(${name} clean)

43 changes: 43 additions & 0 deletions project/externals/libxml2.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
# Copyright (c) 2024 vesoft inc. All rights reserved.
#
# This source code is licensed under Apache 2.0 License.

set(name libxml2)
set(source_dir ${CMAKE_CURRENT_BINARY_DIR}/${name}/source)

ExternalProject_Add(
${name}
URL https://github.com/GNOME/libxml2/archive/refs/tags/v2.13.4.tar.gz
URL_HASH MD5=69d7c63c7fe5858ba7a462f303939c4a
DOWNLOAD_NAME libxml2-v2.13.4.tar.gz
PREFIX ${CMAKE_CURRENT_BINARY_DIR}/${name}
TMP_DIR ${BUILD_INFO_DIR}
STAMP_DIR ${BUILD_INFO_DIR}
DOWNLOAD_DIR ${DOWNLOAD_DIR}
SOURCE_DIR ${source_dir}
CMAKE_ARGS
${common_cmake_args}
-DCMAKE_BUILD_TYPE=Release
-DBUILD_SHARED_LIBS=ON
-DLIBXML2_WITH_ICONV=OFF
-DLIBXML2_WITH_LZMA=OFF
-DLIBXML2_WITH_PYTHON=OFF
-DLIBXML2_WITH_ZLIB=OFF
BUILD_COMMAND make -s -j${BUILDING_JOBS_NUM}
BUILD_IN_SOURCE 1
INSTALL_COMMAND make -s install -j${BUILDING_JOBS_NUM}
LOG_CONFIGURE TRUE
LOG_BUILD TRUE
LOG_INSTALL TRUE
)

ExternalProject_Add_Step(${name} clean
EXCLUDE_FROM_MAIN TRUE
ALWAYS TRUE
DEPENDEES configure
COMMAND make clean -j
COMMAND rm -f ${BUILD_INFO_DIR}/${name}-build
WORKING_DIRECTORY ${source_dir}
)

ExternalProject_Add_StepTargets(${name} clean)
32 changes: 0 additions & 32 deletions project/patches/arrow-11.0.0.patch

This file was deleted.

30 changes: 30 additions & 0 deletions project/patches/arrow-18.0.0.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
--- a/cpp/cmake_modules/GandivaAddBitcode.cmake 2024-10-16 09:20:06.000000000 +0000
+++ b/cpp/cmake_modules/GandivaAddBitcode.cmake 2024-11-05 10:07:31.339538160 +0000
@@ -30,6 +30,16 @@
-fms-compatibility-version=${FMS_COMPATIBILITY})
endif()

+ if (USE_LLVM_CXX)
+ message(STATUS "USE LLVM CXX")
+ file(GLOB_RECURSE LLVM_CXX_INCLUDE_DIR
+ "${LLVM_INCLUDE_DIRS}/*-gnu/c++/v1/__config_site")
+
+ list(GET LLVM_CXX_INCLUDE_DIR 0 LLVM_CXX_INCLUDE_DIR)
+ get_filename_component(LLVM_CXX_INCLUDE_DIR ${LLVM_CXX_INCLUDE_DIR} DIRECTORY)
+ list(APPEND CLANG_OPTIONS -I${LLVM_INCLUDE_DIRS}/c++/v1 -I${LLVM_CXX_INCLUDE_DIR})
+ endif()
+
get_filename_component(SOURCE_BASE ${SOURCE} NAME_WE)
get_filename_component(ABSOLUTE_SOURCE ${SOURCE} ABSOLUTE)
set(BC_FILE ${CMAKE_CURRENT_BINARY_DIR}/${SOURCE_BASE}.bc)
--- a/cpp/src/arrow/testing/process.cc 2024-11-05 08:13:47.487969501 +0000
+++ b/cpp/src/arrow/testing/process.cc 2024-11-05 09:27:42.138928785 +0000
@@ -18,7 +18,7 @@
#include "arrow/testing/process.h"
#include "arrow/result.h"

-#define BOOST_PROCESS_AVAILABLE
+#undef BOOST_PROCESS_AVAILABLE
#ifdef __EMSCRIPTEN__
# undef BOOST_PROCESS_AVAILABLE
#endif

0 comments on commit 0d43d91

Please sign in to comment.