Merge branch 'apache:main' into apacheGH-40968

bookingnizer · Apr 9, 2024 · e3410c6 · e3410c6
2 parents 9b9e3c7 + 433ceef
commit e3410c6
Show file tree

Hide file tree

Showing 151 changed files with 2,655 additions and 654 deletions.
diff --git a/.github/workflows/archery.yml b/.github/workflows/archery.yml
@@ -32,7 +32,9 @@ on:
       - 'docker-compose.yml'
 
 env:
+  ARCHERY_DEBUG: 1
   ARCHERY_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
+  ARCHERY_USE_DOCKER_CLI: 1
 
 concurrency:
   group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
@@ -59,7 +61,7 @@ jobs:
       - name: Setup Python
         uses: actions/[email protected]
         with:
-          python-version: '3.12'
+          python-version: '3.9'
       - name: Install pygit2 binary wheel
         run: pip install pygit2 --only-binary pygit2
       - name: Install Archery, Crossbow- and Test Dependencies

diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml
@@ -53,6 +53,7 @@ permissions:
   contents: read
 
 env:
+  ARCHERY_DEBUG: 1
   ARROW_ENABLE_TIMING_TESTS: OFF
   DOCKER_VOLUME_PREFIX: ".docker/"
 
@@ -94,6 +95,7 @@ jobs:
             cat <<JSON >> "$GITHUB_OUTPUT"
           {
             "arch": "arm64v8",
+            "archery-use-docker-cli": "0",
             "clang-tools": "10",
             "image": "ubuntu-cpp",
             "llvm": "10",
@@ -118,6 +120,9 @@ jobs:
         include: ${{ fromJson(needs.docker-targets.outputs.targets) }}
     env:
       ARCH: ${{ matrix.arch }}
+      # By default, use Docker CLI because docker-compose v1 is obsolete,
+      # except where the Docker client version is too old.
+      ARCHERY_USE_DOCKER_CLI: ${{ matrix.archery-use-docker-cli || '1' }}
       ARROW_SIMD_LEVEL: ${{ matrix.simd-level }}
       CLANG_TOOLS: ${{ matrix.clang-tools }}
       LLVM: ${{ matrix.llvm }}

diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml
@@ -29,6 +29,10 @@ concurrency:
 permissions:
   contents: read
 
+env:
+  ARCHERY_DEBUG: 1
+  ARCHERY_USE_DOCKER_CLI: 1
+
 jobs:
 
   lint:

diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -24,6 +24,8 @@ permissions:
   contents: read
 
 env:
+  ARCHERY_DEBUG: 1
+  ARCHERY_USE_DOCKER_CLI: 1
   ARROW_ENABLE_TIMING_TESTS: OFF
   DOCKER_VOLUME_PREFIX: ".docker/"
 

diff --git a/.github/workflows/docs_light.yml b/.github/workflows/docs_light.yml
@@ -33,6 +33,8 @@ permissions:
   contents: read
 
 env:
+  ARCHERY_DEBUG: 1
+  ARCHERY_USE_DOCKER_CLI: 1
   ARROW_ENABLE_TIMING_TESTS: OFF
   DOCKER_VOLUME_PREFIX: ".docker/"
 

diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml
@@ -41,6 +41,10 @@ concurrency:
 permissions:
   contents: read
 
+env:
+  ARCHERY_DEBUG: 1
+  ARCHERY_USE_DOCKER_CLI: 1
+
 jobs:
 
   docker-targets:
@@ -75,12 +79,14 @@ jobs:
           {
             "arch-label": "ARM64",
             "arch": "arm64v8",
+            "archery-use-docker-cli": "0",
             "go": "1.21",
             "runs-on": ["self-hosted", "arm", "linux"]
           },
           {
             "arch-label": "ARM64",
             "arch": "arm64v8",
+            "archery-use-docker-cli": "0",
             "go": "1.22",
             "runs-on": ["self-hosted", "arm", "linux"]
           }
@@ -101,6 +107,9 @@ jobs:
         include: ${{ fromJson(needs.docker-targets.outputs.targets) }}
     env:
       ARCH: ${{ matrix.arch }}
+      # By default, use Docker CLI because docker-compose v1 is obsolete,
+      # except where the Docker client version is too old.
+      ARCHERY_USE_DOCKER_CLI: ${{ matrix.archery-use-docker-cli || '1' }}
       GO: ${{ matrix.go }}
     steps:
       - name: Checkout Arrow

diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
@@ -51,6 +51,8 @@ permissions:
   contents: read
 
 env:
+  ARCHERY_DEBUG: 1
+  ARCHERY_USE_DOCKER_CLI: 1
   DOCKER_VOLUME_PREFIX: ".docker/"
 
 jobs:

diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml
@@ -45,6 +45,8 @@ permissions:
   contents: read
 
 env:
+  ARCHERY_DEBUG: 1
+  ARCHERY_USE_DOCKER_CLI: 1
   DOCKER_VOLUME_PREFIX: ".docker/"
 
 jobs:
@@ -56,8 +58,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        jdk: [8, 11, 17, 21]
-        maven: [3.9.5]
+        jdk: [8, 11, 17, 21, 22]
+        maven: [3.9.6]
         image: [java]
     env:
       JDK: ${{ matrix.jdk }}

diff --git a/.github/workflows/java_jni.yml b/.github/workflows/java_jni.yml
@@ -45,6 +45,8 @@ permissions:
   contents: read
 
 env:
+  ARCHERY_DEBUG: 1
+  ARCHERY_USE_DOCKER_CLI: 1
   DOCKER_VOLUME_PREFIX: ".docker/"
 
 jobs:

diff --git a/.github/workflows/js.yml b/.github/workflows/js.yml
@@ -38,6 +38,10 @@ concurrency:
 permissions:
   contents: read
 
+env:
+  ARCHERY_DEBUG: 1
+  ARCHERY_USE_DOCKER_CLI: 1
+
 jobs:
 
   docker:

diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml
@@ -41,6 +41,8 @@ permissions:
   contents: read
 
 env:
+  ARCHERY_DEBUG: 1
+  ARCHERY_USE_DOCKER_CLI: 1
   DOCKER_VOLUME_PREFIX: ".docker/"
 
 jobs:

diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml
@@ -51,6 +51,8 @@ permissions:
   contents: read
 
 env:
+  ARCHERY_DEBUG: 1
+  ARCHERY_USE_DOCKER_CLI: 1
   DOCKER_VOLUME_PREFIX: ".docker/"
 
 jobs:

diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml
@@ -53,6 +53,8 @@ permissions:
   contents: read
 
 env:
+  ARCHERY_DEBUG: 1
+  ARCHERY_USE_DOCKER_CLI: 1
   DOCKER_VOLUME_PREFIX: ".docker/"
 
 jobs:

diff --git a/.github/workflows/swift.yml b/.github/workflows/swift.yml
@@ -41,6 +41,8 @@ permissions:
   contents: read
 
 env:
+  ARCHERY_DEBUG: 1
+  ARCHERY_USE_DOCKER_CLI: 1
   DOCKER_VOLUME_PREFIX: ".docker/"
 
 jobs:

diff --git a/c_glib/example/vala/meson.build b/c_glib/example/vala/meson.build
@@ -18,11 +18,15 @@
 # under the License.
 
 if generate_vapi
+  c_flags = [
+    '-Wno-unused-but-set-variable',
+  ]
+  c_flags = meson.get_compiler('c').get_supported_arguments(c_flags)
   vala_example_executable_kwargs = {
     'c_args': [
       '-I' + project_build_root,
       '-I' + project_source_root,
-    ],
+    ] + c_flags,
     'dependencies': [
       arrow_glib_vapi,
       dependency('gio-2.0'),

diff --git a/c_glib/example/vala/read-file.vala b/c_glib/example/vala/read-file.vala
@@ -119,8 +119,8 @@ void print_array(GArrow.Array array) {
 
 void print_record_batch(GArrow.RecordBatch record_batch) {
     var n_columns = record_batch.get_n_columns();
-    for (var nth_column = 0; nth_column < n_columns; nth_column++) {
-        stdout.printf("columns[%" + int64.FORMAT + "](%s): ",
+    for (int nth_column = 0; nth_column < n_columns; nth_column++) {
+        stdout.printf("columns[%d](%s): ",
                       nth_column,
                       record_batch.get_column_name(nth_column));
         var array = record_batch.get_column_data(nth_column);

diff --git a/c_glib/example/vala/read-stream.vala b/c_glib/example/vala/read-stream.vala
@@ -119,8 +119,8 @@ void print_array(GArrow.Array array) {
 
 void print_record_batch(GArrow.RecordBatch record_batch) {
     var n_columns = record_batch.get_n_columns();
-    for (var nth_column = 0; nth_column < n_columns; nth_column++) {
-        stdout.printf("columns[%" + int64.FORMAT + "](%s): ",
+    for (int nth_column = 0; nth_column < n_columns; nth_column++) {
+        stdout.printf("columns[%d](%s): ",
                       nth_column,
                       record_batch.get_column_name(nth_column));
         var array = record_batch.get_column_data(nth_column);

diff --git a/c_glib/test/test-half-float-scalar.rb b/c_glib/test/test-half-float-scalar.rb
@@ -41,7 +41,7 @@ def test_equal
   end
 
   def test_to_s
-    assert_equal("[\n  #{@half_float}\n]", @scalar.to_s)
+    assert_equal("1.0009765625", @scalar.to_s)
   end
 
   def test_value

diff --git a/ci/conda_env_sphinx.txt b/ci/conda_env_sphinx.txt
@@ -29,5 +29,8 @@ sphinx-copybutton
 sphinxcontrib-jquery
 sphinx==6.2
 # Requirement for doctest-cython
-pytest-cython
+# Needs upper pin of 0.3.0, see:
+# https://github.com/lgpage/pytest-cython/issues/67
+# With 0.3.* bug fix release, the pin can be removed
+pytest-cython==0.2.2
 pandas
diff --git a/ci/docker/ubuntu-20.04-cpp.dockerfile b/ci/docker/ubuntu-20.04-cpp.dockerfile
@@ -101,6 +101,7 @@ RUN apt-get update -y -q && \
         libutf8proc-dev \
         libxml2-dev \
         libzstd-dev \
+        lld \
         make \
         ninja-build \
         nlohmann-json3-dev \
@@ -164,6 +165,7 @@ ENV absl_SOURCE=BUNDLED \
     ARROW_SUBSTRAIT=ON \
     ARROW_USE_ASAN=OFF \
     ARROW_USE_CCACHE=ON \
+    ARROW_USE_LLD=ON \
     ARROW_USE_UBSAN=OFF \
     ARROW_WITH_BROTLI=ON \
     ARROW_WITH_BZ2=ON \

diff --git a/ci/docker/ubuntu-22.04-cpp.dockerfile b/ci/docker/ubuntu-22.04-cpp.dockerfile
@@ -65,6 +65,7 @@ RUN latest_system_llvm=14 && \
 RUN apt-get update -y -q && \
     apt-get install -y -q --no-install-recommends \
         autoconf \
+        bzip2 \
         ca-certificates \
         ccache \
         cmake \
@@ -115,10 +116,20 @@ RUN apt-get update -y -q && \
         rapidjson-dev \
         rsync \
         tzdata \
-        wget && \
+        wget \
+        xz-utils && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists*
 
+# install emscripten using EMSDK
+ARG emscripten_version="3.1.45"
+RUN cd ~ && git clone https://github.com/emscripten-core/emsdk.git && \
+    cd emsdk && \
+    ./emsdk install ${emscripten_version} && \
+    ./emsdk activate ${emscripten_version} && \
+    echo "Installed emsdk to:" ~/emsdk
+
+
 ARG gcc_version=""
 RUN if [ "${gcc_version}" = "" ]; then \
       apt-get update -y -q && \
@@ -151,6 +162,9 @@ RUN if [ "${gcc_version}" = "" ]; then \
       update-alternatives --set c++ /usr/bin/g++; \
     fi
 
+# make sure zlib is cached in the EMSDK folder
+RUN source ~/emsdk/emsdk_env.sh && embuilder --pic build zlib
+
 COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
 RUN /arrow/ci/scripts/install_minio.sh latest /usr/local