Merge branch 'main' into minor-enhance-ipc-write

.github/workflows/cpp.yml

@@ -21,6 +21,7 @@ on:
   push:
     paths:
       - '.github/workflows/cpp.yml'
+      - 'ci/conda_env_*'
       - 'ci/docker/**'
       - 'ci/scripts/cpp_*'
       - 'ci/scripts/install_azurite.sh'
@@ -35,6 +36,7 @@ on:
   pull_request:
     paths:
       - '.github/workflows/cpp.yml'
+      - 'ci/conda_env_*'
       - 'ci/docker/**'
       - 'ci/scripts/cpp_*'
       - 'ci/scripts/install_azurite.sh'

.github/workflows/csharp.yml

@@ -94,8 +94,8 @@ jobs:
         run: ci/scripts/csharp_test.sh $(pwd)
 
   macos:
-    name: ARM64 macOS 14 C# ${{ matrix.dotnet }}
-    runs-on: macos-latest
+    name: AMD64 macOS 13 C# ${{ matrix.dotnet }}
+    runs-on: macos-13 # Pending https://github.com/pythonnet/pythonnet/issues/2396
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
     timeout-minutes: 15
     strategy:

.github/workflows/dev.yml

@@ -120,7 +120,7 @@ jobs:
         shell: bash
         run: |
           gem install test-unit
-          pip install "cython>=0.29.31" setuptools six pytest jira
+          pip install "cython>=0.29.31" setuptools six pytest jira setuptools-scm
       - name: Run Release Test
         env:
           ARROW_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/issue_bot.yml

@@ -21,7 +21,6 @@ on:
   issues:
     types:
       - opened
-      - edited
 
 permissions:
   contents: read

.github/workflows/js.yml

@@ -106,10 +106,10 @@ jobs:
           node-version: ${{ matrix.node }}
       - name: Build
         shell: bash
-        run: ci/scripts/js_build.sh $(pwd)
+        run: ci/scripts/js_build.sh $(pwd) build
       - name: Test
         shell: bash
-        run: ci/scripts/js_test.sh $(pwd)
+        run: ci/scripts/js_test.sh $(pwd) build
 
   windows:
     name: AMD64 Windows NodeJS ${{ matrix.node }}
@@ -136,7 +136,7 @@ jobs:
           node-version: ${{ matrix.node }}
       - name: Build
         shell: bash
-        run: ci/scripts/js_build.sh $(pwd)
+        run: ci/scripts/js_build.sh $(pwd) build
       - name: Test
         shell: bash
-        run: ci/scripts/js_test.sh $(pwd)
+        run: ci/scripts/js_test.sh $(pwd) build

.github/workflows/ruby.yml

@@ -197,9 +197,7 @@ jobs:
         mingw-n-bits:
           - 64
         ruby-version:
-          # TODO: Use the latest Ruby again when we fix GH-39130.
-          # - ruby
-          - "3.1"
+          - ruby
     env:
       ARROW_BUILD_STATIC: OFF
       ARROW_BUILD_TESTS: OFF
@@ -313,15 +311,17 @@ jobs:
     strategy:
       fail-fast: false
     env:
+      ARROW_ACERO: ON
       ARROW_BOOST_USE_SHARED: OFF
       ARROW_BUILD_BENCHMARKS: OFF
       ARROW_BUILD_SHARED: ON
       ARROW_BUILD_STATIC: OFF
       ARROW_BUILD_TESTS: OFF
-      ARROW_ACERO: ON
       ARROW_DATASET: ON
-      ARROW_FLIGHT: OFF
-      ARROW_FLIGHT_SQL: OFF
+      ARROW_DEPENDENCY_SOURCE: VCPKG
+      ARROW_DEPENDENCY_USE_SHARED: OFF
+      ARROW_FLIGHT: ON
+      ARROW_FLIGHT_SQL: ON
       ARROW_GANDIVA: OFF
       ARROW_HDFS: OFF
       ARROW_HOME: "${{ github.workspace }}/dist"
@@ -337,13 +337,16 @@ jobs:
       ARROW_WITH_LZ4: OFF
       ARROW_WITH_OPENTELEMETRY: OFF
       ARROW_WITH_SNAPPY: ON
-      ARROW_WITH_ZLIB: OFF
+      ARROW_WITH_ZLIB: ON
       ARROW_WITH_ZSTD: ON
-      BOOST_SOURCE: BUNDLED
       CMAKE_CXX_STANDARD: "17"
       CMAKE_GENERATOR: Ninja
       CMAKE_INSTALL_PREFIX: "${{ github.workspace }}/dist"
       CMAKE_UNITY_BUILD: ON
+      VCPKG_BINARY_SOURCES: 'clear;nuget,GitHub,readwrite'
+      VCPKG_ROOT: "${{ github.workspace }}/vcpkg"
+    permissions:
+      packages: write
     steps:
       - name: Disable Crash Dialogs
         run: |
@@ -361,7 +364,7 @@ jobs:
       - name: Install vcpkg
         shell: bash
         run: |
-          ci/scripts/install_vcpkg.sh ./vcpkg
+          ci/scripts/install_vcpkg.sh "${VCPKG_ROOT}"
       - name: Install meson
         run: |
           python -m pip install meson
@@ -387,6 +390,22 @@ jobs:
         env:
           # We can invalidate the current cache by updating this.
           CACHE_VERSION: "2024-05-09"
+      - name: Setup NuGet credentials for vcpkg caching
+        shell: bash
+        run: |
+          $(vcpkg/vcpkg.exe fetch nuget | tail -n 1) \
+            sources add \
+            -source "https://nuget.pkg.github.com/$GITHUB_REPOSITORY_OWNER/index.json" \
+            -storepasswordincleartext \
+            -name "GitHub" \
+            -username "$GITHUB_REPOSITORY_OWNER" \
+            -password "${{ secrets.GITHUB_TOKEN }}"
+          $(vcpkg/vcpkg.exe fetch nuget | tail -n 1) \
+            setapikey "${{ secrets.GITHUB_TOKEN }}" \
+            -source "https://nuget.pkg.github.com/$GITHUB_REPOSITORY_OWNER/index.json"
+      - name: Build C++ vcpkg dependencies
+        run: |
+          vcpkg\vcpkg.exe install --triplet x64-windows --x-manifest-root cpp --x-install-root build\cpp\vcpkg_installed
       - name: Build C++
         shell: cmd
         run: |
@@ -396,4 +415,4 @@ jobs:
         shell: cmd
         run: |
           call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
-          bash -c "VCPKG_ROOT=\"$(pwd)/vcpkg\" ci/scripts/c_glib_build.sh $(pwd) $(pwd)/build"
+          bash -c "ci/scripts/c_glib_build.sh $(pwd) $(pwd)/build"

.gitignore

@@ -102,4 +102,9 @@ __debug_bin
 .envrc
 
 # Develocity
-.mvn/.develocity/
+java/.mvn/.gradle-enterprise/
+java/.mvn/.develocity/
+
+# rat
+filtered_rat.txt
+rat.txt

.golangci.yaml

@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+linters:
+  # Disable all linters.
+  # Default: false
+  disable-all: true
+  # Enable specific linter
+  # https://golangci-lint.run/usage/linters/#enabled-by-default
+  enable:
+    - gofmt
+    - goimports
+
+issues:
+  fix: true

.pre-commit-config.yaml

@@ -141,3 +141,17 @@ repos:
           '--disable',
           'dangling-hyphen,line-too-long',
         ]
+  - repo: https://github.com/golangci/golangci-lint
+    rev: v1.59.0
+    hooks:
+      # no built-in support for multiple go.mod
+      # https://github.com/golangci/golangci-lint/issues/828
+      - id: golangci-lint-full
+        name: golangci-lint-full-arrow
+        entry: bash -c 'cd go/arrow && golangci-lint run'
+      - id: golangci-lint-full
+        name: golangci-lint-full-parquet
+        entry: bash -c 'cd go/parquet && golangci-lint run'
+      - id: golangci-lint-full
+        name: golangci-lint-full-internal
+        entry: bash -c 'cd go/internal && golangci-lint run'

ci/appveyor-cpp-setup.bat

@@ -66,6 +66,9 @@ set CONDA_PACKAGES=%CONDA_PACKAGES% --file=ci\conda_env_cpp.txt
 @rem Force conda to use conda-forge
 conda config --add channels conda-forge
 conda config --remove channels defaults
+@rem Ensure using the latest information. If there are invalid caches,
+@rem mamba may use invalid download URL.
+mamba clean --all -y
 @rem Arrow conda environment
 mamba create -n arrow -y -c conda-forge ^
   --file=ci\conda_env_python.txt ^

ci/conda_env_cpp.txt

@@ -21,7 +21,7 @@ azure-identity-cpp>=1.6.0
 azure-storage-blobs-cpp>=12.10.0
 azure-storage-common-cpp>=12.5.0
 azure-storage-files-datalake-cpp>=12.9.0
-benchmark>=1.6.0
+benchmark>=1.6.0,!=1.8.4
 boost-cpp>=1.68.0
 brotli
 bzip2

ci/conda_env_python.txt

@@ -26,5 +26,5 @@ numpy>=1.16.6
 pytest
 pytest-faulthandler
 s3fs>=2023.10.0
-setuptools
-setuptools_scm
+setuptools>=64
+setuptools_scm>=8

ci/scripts/cpp_build.sh

@@ -137,6 +137,7 @@ else
     -DARROW_C_FLAGS_RELWITHDEBINFO="${ARROW_C_FLAGS_RELWITHDEBINFO:-}" \
     -DARROW_DATASET=${ARROW_DATASET:-OFF} \
     -DARROW_DEPENDENCY_SOURCE=${ARROW_DEPENDENCY_SOURCE:-AUTO} \
+    -DARROW_DEPENDENCY_USE_SHARED=${ARROW_DEPENDENCY_USE_SHARED:-ON} \
     -DARROW_ENABLE_THREADING=${ARROW_ENABLE_THREADING:-ON} \
     -DARROW_ENABLE_TIMING_TESTS=${ARROW_ENABLE_TIMING_TESTS:-ON} \
     -DARROW_EXTRA_ERROR_CONTEXT=${ARROW_EXTRA_ERROR_CONTEXT:-OFF} \

ci/scripts/java_full_build.sh

@@ -49,21 +49,13 @@ fi
 # build the entire project
 mvn clean \
     install \
-    assembly:single \
-    source:jar \
-    javadoc:jar \
     -Papache-release \
     -Parrow-c-data \
     -Parrow-jni \
     -Darrow.cpp.build.dir=$dist_dir \
-    -Darrow.c.jni.dist.dir=$dist_dir \
-    -DdescriptorId=source-release
+    -Darrow.c.jni.dist.dir=$dist_dir
 
 # copy all jar, zip and pom files to the distribution folder
-find . \
-     "(" -name "*-javadoc.jar" -o -name "*-sources.jar" ")" \
-     -exec echo {} ";" \
-     -exec cp {} $dist_dir ";"
 find ~/.m2/repository/org/apache/arrow \
      "(" \
      -name "*.jar" -o \

cpp/cmake_modules/Findutf8proc.cmake

@@ -19,7 +19,7 @@ if(utf8proc_FOUND)
   return()
 endif()
 
-if(ARROW_PACKAGE_KIND STREQUAL "vcpkg")
+if(ARROW_PACKAGE_KIND STREQUAL "vcpkg" OR VCPKG_TOOLCHAIN)
   set(find_package_args "")
   if(utf8proc_FIND_VERSION)
     list(APPEND find_package_args ${utf8proc_FIND_VERSION})

cpp/cmake_modules/ThirdpartyToolchain.cmake

@@ -2819,11 +2819,13 @@ macro(build_utf8proc)
 endmacro()
 
 if(ARROW_WITH_UTF8PROC)
-  resolve_dependency(utf8proc
-                     PC_PACKAGE_NAMES
-                     libutf8proc
-                     REQUIRED_VERSION
-                     "2.2.0")
+  set(utf8proc_resolve_dependency_args utf8proc PC_PACKAGE_NAMES libutf8proc)
+  if(NOT VCPKG_TOOLCHAIN)
+    # utf8proc in vcpkg doesn't provide version information:
+    # https://github.com/microsoft/vcpkg/issues/39176
+    list(APPEND utf8proc_resolve_dependency_args REQUIRED_VERSION "2.2.0")
+  endif()
+  resolve_dependency(${utf8proc_resolve_dependency_args})
 endif()
 
 macro(build_cares)

cpp/src/arrow/CMakeLists.txt

@@ -200,22 +200,29 @@ function(arrow_add_object_library PREFIX)
   set(SOURCES ${ARGN})
   string(TOLOWER "${PREFIX}" prefix)
   if(WIN32)
-    add_library(${prefix}_shared OBJECT ${SOURCES})
-    add_library(${prefix}_static OBJECT ${SOURCES})
-    set_target_properties(${prefix}_shared PROPERTIES POSITION_INDEPENDENT_CODE ON)
-    set_target_properties(${prefix}_static PROPERTIES POSITION_INDEPENDENT_CODE ON)
-    target_compile_definitions(${prefix}_shared PRIVATE ARROW_EXPORTING)
-    target_compile_definitions(${prefix}_static PRIVATE ARROW_STATIC)
-    target_compile_features(${prefix}_shared PRIVATE cxx_std_17)
-    target_compile_features(${prefix}_static PRIVATE cxx_std_17)
-    set(${PREFIX}_TARGET_SHARED
-        ${prefix}_shared
-        PARENT_SCOPE)
-    set(${PREFIX}_TARGET_STATIC
-        ${prefix}_static
-        PARENT_SCOPE)
+    set(targets)
+    if(ARROW_BUILD_SHARED)
+      add_library(${prefix}_shared OBJECT ${SOURCES})
+      set_target_properties(${prefix}_shared PROPERTIES POSITION_INDEPENDENT_CODE ON)
+      target_compile_definitions(${prefix}_shared PRIVATE ARROW_EXPORTING)
+      target_compile_features(${prefix}_shared PRIVATE cxx_std_17)
+      set(${PREFIX}_TARGET_SHARED
+          ${prefix}_shared
+          PARENT_SCOPE)
+      list(APPEND targets ${prefix}_shared)
+    endif()
+    if(ARROW_BUILD_STATIC)
+      add_library(${prefix}_static OBJECT ${SOURCES})
+      set_target_properties(${prefix}_static PROPERTIES POSITION_INDEPENDENT_CODE ON)
+      target_compile_definitions(${prefix}_static PRIVATE ARROW_STATIC)
+      target_compile_features(${prefix}_static PRIVATE cxx_std_17)
+      set(${PREFIX}_TARGET_STATIC
+          ${prefix}_static
+          PARENT_SCOPE)
+      list(APPEND targets ${prefix}_static)
+    endif()
     set(${PREFIX}_TARGETS
-        ${prefix}_shared ${prefix}_static
+        ${targets}
         PARENT_SCOPE)
   else()
     add_library(${prefix} OBJECT ${SOURCES})

cpp/src/arrow/array/array_list_test.cc

@@ -1368,6 +1368,23 @@ TEST_F(TestMapArray, FromArrays) {
   ASSERT_EQ(keys_with_null->length(), tmp_items->length());
   ASSERT_RAISES(Invalid,
                 MapArray::FromArrays(offsets1, keys_with_null, tmp_items, pool_));
+
+  // With null_bitmap
+  ASSERT_OK_AND_ASSIGN(auto map7, MapArray::FromArrays(offsets1, keys, items, pool_,
+                                                       offsets3->data()->buffers[0]));
+  ASSERT_OK(map7->Validate());
+  MapArray expected7(map_type, length, offsets1->data()->buffers[1], keys, items,
+                     offsets3->data()->buffers[0], 1);
+  AssertArraysEqual(expected7, *map7);
+
+  // Null bitmap and offset with null
+  ASSERT_RAISES(Invalid, MapArray::FromArrays(offsets3, keys, items, pool_,
+                                              offsets3->data()->buffers[0]));
+
+  // Null bitmap and offset with offset
+  ASSERT_RAISES(NotImplemented,
+                MapArray::FromArrays(offsets1->Slice(2), keys, items, pool_,
+                                     offsets3->data()->buffers[0]));
 }
 
 TEST_F(TestMapArray, FromArraysEquality) {