Merge branch '3rd/xsimd-13' of github.com:mapleFU/arrow into 3rd/xsim…

…d-13
mapleFU · May 22, 2024 · 2833bd8 · 2833bd8
2 parents 362872a + 66a9a37
commit 2833bd8
Show file tree

Hide file tree

Showing 171 changed files with 2,615 additions and 1,238 deletions.
diff --git a/.asf.yaml b/.asf.yaml
@@ -21,11 +21,12 @@ github:
   collaborators:
     - anjakefala
     - benibus
-    - danepitkin
     - davisusanibar
     - jbonofre
     - js8544
     - vibhatha
+    - zanmato1984
+    - ZhangHuiGui
 
 notifications:
   commits:      [email protected]

diff --git a/.env b/.env
@@ -86,9 +86,6 @@ ARROW_R_DEV=TRUE
 R_PRUNE_DEPS=FALSE
 TZ=UTC
 
-# Any non-empty string will install devtoolset-${DEVTOOLSET_VERSION}
-DEVTOOLSET_VERSION=
-
 # Used through docker-compose.yml and serves as the default version for the
 # ci/scripts/install_vcpkg.sh script. Prefer to use short SHAs to keep the
 # docker tags more readable.

diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -32,12 +32,12 @@ env:
 jobs:
 
   complete:
-    name: AMD64 Ubuntu 22.04 Complete Documentation
+    name: AMD64 Debian 12 Complete Documentation
     runs-on: ubuntu-latest
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
     timeout-minutes: 150
     env:
-      UBUNTU: "22.04"
+      JDK: 17
     steps:
       - name: Checkout Arrow
         uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
@@ -50,8 +50,8 @@ jobs:
         uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0
         with:
           path: .docker
-          key: ubuntu-docs-${{ hashFiles('cpp/**') }}
-          restore-keys: ubuntu-docs-
+          key: debian-docs-${{ hashFiles('cpp/**') }}
+          restore-keys: debian-docs-
       - name: Setup Python
         uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
         with:
@@ -62,7 +62,8 @@ jobs:
         env:
           ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
           ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
-        run: archery docker run ubuntu-docs
+          JDK: 17
+        run: archery docker run debian-docs
       - name: Docker Push
         if: >-
           success() &&
@@ -73,4 +74,4 @@ jobs:
           ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
           ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
         continue-on-error: true
-        run: archery docker push ubuntu-docs
+        run: archery docker push debian-docs
diff --git a/.github/workflows/docs_light.yml b/.github/workflows/docs_light.yml
@@ -31,7 +31,7 @@ on:
 
 permissions:
   contents: read
-  
+
 env:
   ARCHERY_DEBUG: 1
   ARCHERY_USE_DOCKER_CLI: 1

diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
@@ -75,6 +75,11 @@ jobs:
         with:
           repository: apache/arrow-rs
           path: rust
+      - name: Checkout Arrow nanoarrow
+        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
+        with:
+          repository: apache/arrow-nanoarrow
+          path: nanoarrow
       - name: Free up disk space
         run: |
           ci/scripts/util_free_space.sh
@@ -97,6 +102,7 @@ jobs:
         run: >
           archery docker run \
             -e ARCHERY_DEFAULT_BRANCH=${{ github.event.repository.default_branch }} \
+            -e ARCHERY_INTEGRATION_WITH_NANOARROW=1 \
             -e ARCHERY_INTEGRATION_WITH_RUST=1 \
             conda-integration
       - name: Docker Push

diff --git a/.github/workflows/matlab.yml b/.github/workflows/matlab.yml
@@ -98,9 +98,16 @@ jobs:
           select-by-folder: matlab/test
           strict: true
   macos:
-    name: AMD64 macOS 12 MATLAB
-    runs-on: macos-12
+    name: ${{ matrix.architecture }} macOS ${{ matrix.macos-version }} MATLAB
+    runs-on: macos-${{ matrix.macos-version }}
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
+    strategy:
+      matrix:
+        include:
+            - architecture: AMD64
+              macos-version: "12"
+            - architecture: ARM64
+              macos-version: "14"
     steps:
       - name: Check out repository
         uses: actions/checkout@v4

diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml
@@ -182,6 +182,19 @@ jobs:
           python -m pip install \
             -r python/requirements-build.txt \
             -r python/requirements-test.txt
+      - name: Setup ccache
+        shell: bash
+        run: ci/scripts/ccache_setup.sh
+      - name: ccache info
+        id: ccache-info
+        shell: bash
+        run: echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT
+      - name: Cache ccache
+        uses: actions/cache@v4
+        with:
+          path: ${{ steps.ccache-info.outputs.cache-dir }}
+          key: python-ccache-macos-${{ matrix.macos-version }}-${{ hashFiles('cpp/**', 'python/**') }}
+          restore-keys: python-ccache-macos-${{ matrix.macos-version }}-
       - name: Build
         shell: bash
         run: |

diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml
@@ -192,12 +192,11 @@ jobs:
       fail-fast: false
       matrix:
         config:
-          - { org: "rhub", image: "ubuntu-gcc12", tag: "latest", devtoolset: "" }
+          - { org: "rhub", image: "ubuntu-gcc12", tag: "latest" }
     env:
       R_ORG: ${{ matrix.config.org }}
       R_IMAGE: ${{ matrix.config.image }}
       R_TAG: ${{ matrix.config.tag }}
-      DEVTOOLSET_VERSION: ${{ matrix.config.devtoolset }}
     steps:
       - name: Checkout Arrow
         uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0

diff --git a/c_glib/meson.build b/c_glib/meson.build
@@ -26,8 +26,6 @@ project('arrow-glib', 'c', 'cpp',
         # Debian:
         #   https://packages.debian.org/search?keywords=meson
         #
-        #   * bullseye: 0.56.2
-        #   * bullseye-backports:1.0.0
         #   * bookworm: 1.0.0
         #
         # Ubuntu:

diff --git a/ci/conda_env_sphinx.txt b/ci/conda_env_sphinx.txt
@@ -28,6 +28,7 @@ sphinx-design
 sphinx-copybutton
 sphinx-lint
 sphinxcontrib-jquery
+sphinxcontrib-mermaid
 sphinx==6.2
 # Requirement for doctest-cython
 # Needs upper pin of 0.3.0, see:

diff --git a/ci/docker/debian-12-cpp.dockerfile b/ci/docker/debian-12-cpp.dockerfile
@@ -119,7 +119,6 @@ ENV ARROW_ACERO=ON \
     ARROW_GANDIVA=ON \
     ARROW_GCS=ON \
     ARROW_HOME=/usr/local \
-    ARROW_NO_DEPRECATED_API=ON \
     ARROW_ORC=ON \
     ARROW_PARQUET=ON \
     ARROW_S3=ON \

diff --git a/ci/docker/linux-apt-docs.dockerfile b/ci/docker/linux-apt-docs.dockerfile
@@ -21,18 +21,34 @@ FROM ${base}
 ARG r=4.4
 ARG jdk=8
 
-# See R install instructions at https://cloud.r-project.org/bin/linux/ubuntu/
+ENV PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium
+
+# See R install instructions at https://cloud.r-project.org/bin/linux/
 RUN apt-get update -y && \
     apt-get install -y \
-        dirmngr \
         apt-transport-https \
-        software-properties-common && \
-    wget -qO- https://cloud.r-project.org/bin/linux/ubuntu/marutter_pubkey.asc | \
-        tee -a /etc/apt/trusted.gpg.d/cran_ubuntu_key.asc && \
-    add-apt-repository 'deb https://cloud.r-project.org/bin/linux/ubuntu '$(lsb_release -cs)'-cran40/' && \
+        dirmngr \
+        gpg \
+        lsb-release && \
+    gpg --keyserver keyserver.ubuntu.com \
+        --recv-key 95C0FAF38DB3CCAD0C080A7BDC78B2DDEABC47B7 && \
+    gpg --export 95C0FAF38DB3CCAD0C080A7BDC78B2DDEABC47B7 | \
+        gpg --no-default-keyring \
+            --keyring /usr/share/keyrings/cran.gpg \
+            --import - && \
+    echo "deb [signed-by=/usr/share/keyrings/cran.gpg] https://cloud.r-project.org/bin/linux/$(lsb_release -is | tr 'A-Z' 'a-z') $(lsb_release -cs)-cran40/" | \
+        tee /etc/apt/sources.list.d/cran.list && \
+    if [ -f /etc/apt/sources.list.d/debian.sources ]; then \
+        sed -i \
+            -e 's/main$/main contrib non-free non-free-firmware/g' \
+            /etc/apt/sources.list.d/debian.sources; \
+    fi && \
+    apt-get update -y && \
     apt-get install -y --no-install-recommends \
         autoconf-archive \
         automake \
+        chromium \
+        chromium-sandbox \
         curl \
         doxygen \
         gi-docgen \
@@ -48,16 +64,21 @@ RUN apt-get update -y && \
         libxml2-dev \
         meson \
         ninja-build \
+        nodejs \
+        npm \
         nvidia-cuda-toolkit \
         openjdk-${jdk}-jdk-headless \
         pandoc \
         r-recommended=${r}* \
         r-base=${r}* \
         rsync \
         ruby-dev \
+        sudo \
         wget && \
     apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
+    rm -rf /var/lib/apt/lists/* && \
+    PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
+       npm install -g yarn @mermaid-js/mermaid-cli
 
 ENV JAVA_HOME=/usr/lib/jvm/java-${jdk}-openjdk-amd64
 
@@ -68,20 +89,6 @@ RUN /arrow/ci/scripts/util_download_apache.sh \
 ENV PATH=/opt/apache-maven-${maven}/bin:$PATH
 RUN mvn -version
 
-ARG node=16
-RUN apt-get purge -y npm && \
-    apt-get autoremove -y --purge && \
-    wget -q -O - https://deb.nodesource.com/setup_${node}.x | bash - && \
-    apt-get install -y nodejs && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/* && \
-    npm install -g yarn
-
-COPY docs/requirements.txt /arrow/docs/
-RUN python3 -m venv ${ARROW_PYTHON_VENV} && \
-    . ${ARROW_PYTHON_VENV}/bin/activate && \
-    pip install -r arrow/docs/requirements.txt
-
 COPY c_glib/Gemfile /arrow/c_glib/
 RUN gem install --no-document bundler && \
     bundle install --gemfile /arrow/c_glib/Gemfile
@@ -98,6 +105,17 @@ COPY r/DESCRIPTION /arrow/r/
 RUN /arrow/ci/scripts/r_deps.sh /arrow && \
     R -e "install.packages('pkgdown')"
 
+RUN useradd --user-group --create-home --groups audio,video arrow
+RUN echo "arrow ALL=(ALL:ALL) NOPASSWD:ALL" | \
+        EDITOR=tee visudo -f /etc/sudoers.d/arrow
+USER arrow
+
+COPY docs/requirements.txt /arrow/docs/
+RUN sudo chown -R arrow: ${ARROW_PYTHON_VENV} && \
+    python3 -m venv ${ARROW_PYTHON_VENV} && \
+    . ${ARROW_PYTHON_VENV}/bin/activate && \
+    pip install -r arrow/docs/requirements.txt
+
 ENV ARROW_ACERO=ON \
     ARROW_AZURE=OFF \
     ARROW_BUILD_STATIC=OFF \

diff --git a/ci/docker/linux-apt-r.dockerfile b/ci/docker/linux-apt-r.dockerfile
@@ -113,7 +113,6 @@ ENV \
     ARROW_GANDIVA=OFF \
     ARROW_HDFS=OFF \
     ARROW_JSON=ON \
-    ARROW_NO_DEPRECATED_API=ON \
     ARROW_ORC=OFF \
     ARROW_PARQUET=ON \
     ARROW_S3=ON \

diff --git a/ci/docker/linux-r.dockerfile b/ci/docker/linux-r.dockerfile
@@ -27,9 +27,6 @@ ENV R_BIN=${r_bin}
 ARG r_dev=FALSE
 ENV ARROW_R_DEV=${r_dev}
 
-ARG devtoolset_version=
-ENV DEVTOOLSET_VERSION=${devtoolset_version}
-
 ARG r_prune_deps=FALSE
 ENV R_PRUNE_DEPS=${r_prune_deps}
 

diff --git a/ci/docker/ubuntu-20.04-cpp-minimal.dockerfile b/ci/docker/ubuntu-20.04-cpp-minimal.dockerfile
@@ -85,7 +85,6 @@ ENV ARROW_ACERO=ON \
     ARROW_HDFS=ON \
     ARROW_HOME=/usr/local \
     ARROW_INSTALL_NAME_RPATH=OFF \
-    ARROW_NO_DEPRECATED_API=ON \
     ARROW_ORC=ON \
     ARROW_PARQUET=ON \
     ARROW_S3=ON \

diff --git a/ci/docker/ubuntu-20.04-cpp.dockerfile b/ci/docker/ubuntu-20.04-cpp.dockerfile
@@ -158,7 +158,6 @@ ENV absl_SOURCE=BUNDLED \
     ARROW_HDFS=ON \
     ARROW_HOME=/usr/local \
     ARROW_INSTALL_NAME_RPATH=OFF \
-    ARROW_NO_DEPRECATED_API=ON \
     ARROW_ORC=ON \
     ARROW_PARQUET=ON \
     ARROW_S3=ON \

diff --git a/ci/docker/ubuntu-22.04-cpp-minimal.dockerfile b/ci/docker/ubuntu-22.04-cpp-minimal.dockerfile
@@ -85,7 +85,6 @@ ENV ARROW_ACERO=ON \
     ARROW_HDFS=ON \
     ARROW_HOME=/usr/local \
     ARROW_INSTALL_NAME_RPATH=OFF \
-    ARROW_NO_DEPRECATED_API=ON \
     ARROW_ORC=ON \
     ARROW_PARQUET=ON \
     ARROW_S3=ON \

diff --git a/ci/docker/ubuntu-22.04-cpp.dockerfile b/ci/docker/ubuntu-22.04-cpp.dockerfile
@@ -196,7 +196,6 @@ ENV absl_SOURCE=BUNDLED \
     ARROW_HDFS=ON \
     ARROW_HOME=/usr/local \
     ARROW_INSTALL_NAME_RPATH=OFF \
-    ARROW_NO_DEPRECATED_API=ON \
     ARROW_ORC=ON \
     ARROW_PARQUET=ON \
     ARROW_S3=ON \

diff --git a/ci/docker/ubuntu-24.04-cpp.dockerfile b/ci/docker/ubuntu-24.04-cpp.dockerfile
@@ -178,7 +178,6 @@ ENV ARROW_ACERO=ON \
     ARROW_HDFS=ON \
     ARROW_HOME=/usr/local \
     ARROW_INSTALL_NAME_RPATH=OFF \
-    ARROW_NO_DEPRECATED_API=ON \
     ARROW_ORC=ON \
     ARROW_PARQUET=ON \
     ARROW_S3=ON \

diff --git a/ci/scripts/c_glib_build.sh b/ci/scripts/c_glib_build.sh
@@ -30,9 +30,6 @@ with_doc=$([ "${BUILD_DOCS_C_GLIB}" == "ON" ] && echo "true" || echo "false")
 
 export PKG_CONFIG_PATH=${ARROW_HOME}/lib/pkgconfig
 
-export CFLAGS="-DARROW_NO_DEPRECATED_API"
-export CXXFLAGS="-DARROW_NO_DEPRECATED_API"
-
 mkdir -p ${build_dir}
 
 # Build with Meson

diff --git a/ci/scripts/cpp_build.sh b/ci/scripts/cpp_build.sh
@@ -152,7 +152,6 @@ else
     -DARROW_JSON=${ARROW_JSON:-ON} \
     -DARROW_LARGE_MEMORY_TESTS=${ARROW_LARGE_MEMORY_TESTS:-OFF} \
     -DARROW_MIMALLOC=${ARROW_MIMALLOC:-OFF} \
-    -DARROW_NO_DEPRECATED_API=${ARROW_NO_DEPRECATED_API:-OFF} \
     -DARROW_ORC=${ARROW_ORC:-OFF} \
     -DARROW_PARQUET=${ARROW_PARQUET:-OFF} \
     -DARROW_RUNTIME_SIMD_LEVEL=${ARROW_RUNTIME_SIMD_LEVEL:-MAX} \
@@ -229,12 +228,17 @@ find . -name "*.o" -delete
 popd
 
 if [ -x "$(command -v ldconfig)" ]; then
-  ldconfig ${ARROW_HOME}/${CMAKE_INSTALL_LIBDIR:-lib}
+  if [ -x "$(command -v sudo)" ]; then
+    SUDO=sudo
+  else
+    SUDO=
+  fi
+  ${SUDO} ldconfig ${ARROW_HOME}/${CMAKE_INSTALL_LIBDIR:-lib}
 fi
 
 if [ "${ARROW_USE_CCACHE}" == "ON" ]; then
-    echo -e "===\n=== ccache statistics after build\n==="
-    ccache -sv 2>/dev/null || ccache -s
+  echo -e "===\n=== ccache statistics after build\n==="
+  ccache -sv 2>/dev/null || ccache -s
 fi
 
 if command -v sccache &> /dev/null; then
@@ -244,6 +248,6 @@ fi
 
 if [ "${BUILD_DOCS_CPP}" == "ON" ]; then
   pushd ${source_dir}/apidoc
-  doxygen
+  OUTPUT_DIRECTORY=${build_dir}/apidoc doxygen
   popd
 fi
diff --git a/ci/scripts/integration_arrow.sh b/ci/scripts/integration_arrow.sh
@@ -40,6 +40,8 @@ if [ "${ARROW_INTEGRATION_JAVA}" == "ON" ]; then
     pip install jpype1
 fi
 
+export ARROW_BUILD_ROOT=${build_dir}
+
 # Get more detailed context on crashes
 export PYTHONFAULTHANDLER=1
 

diff --git a/ci/scripts/integration_arrow_build.sh b/ci/scripts/integration_arrow_build.sh
@@ -30,6 +30,8 @@ build_dir=${2}
 
 ${arrow_dir}/ci/scripts/rust_build.sh ${arrow_dir} ${build_dir}
 
+${arrow_dir}/ci/scripts/nanoarrow_build.sh ${arrow_dir} ${build_dir}
+
 if [ "${ARROW_INTEGRATION_CPP}" == "ON" ]; then
     ${arrow_dir}/ci/scripts/cpp_build.sh ${arrow_dir} ${build_dir}
 fi