Merge branch 'main' into fixing-buffered-read

.asf.yaml

@@ -16,7 +16,7 @@
 # under the License.
 
 github:
-  description: "Apache Arrow is a multi-language toolbox for accelerated data interchange and in-memory processing"
+  description: "Apache Arrow is the universal columnar format and multi-language toolbox for fast data interchange and in-memory analytics"
   homepage: https://arrow.apache.org/
   collaborators:
     - anjakefala

.github/workflows/archery.yml

@@ -19,6 +19,11 @@ name: Archery & Crossbow
 
 on:
   push:
+    branches:
+      - '**'
+      - '!dependabot/**'
+    tags:
+      - '**'
     paths:
       - '.dockerignore'
       - '.github/workflows/archery.yml'

.github/workflows/cpp.yml

@@ -19,6 +19,11 @@ name: C++
 
 on:
   push:
+    branches:
+      - '**'
+      - '!dependabot/**'
+    tags:
+      - '**'
     paths:
       - '.dockerignore'
       - '.github/workflows/cpp.yml'
@@ -141,7 +146,15 @@ jobs:
           path: .docker
           key: ${{ matrix.image }}-${{ hashFiles('cpp/**') }}
           restore-keys: ${{ matrix.image }}-
-      - name: Setup Python
+      - name: Setup Python on hosted runner
+        if: |
+          matrix.runs-on == 'ubuntu-latest'
+        uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
+        with:
+          python-version: 3
+      - name: Setup Python on self-hosted runner
+        if: |
+          contains(matrix.runs-on, 'self-hosted')
         run: |
           sudo apt update
           sudo apt install -y --no-install-recommends python3 python3-dev python3-pip

.github/workflows/csharp.yml

@@ -19,6 +19,11 @@ name: C#
 
 on:
   push:
+    branches:
+      - '**'
+      - '!dependabot/**'
+    tags:
+      - '**'
     paths:
       - '.github/workflows/csharp.yml'
       - 'ci/scripts/csharp_*'
@@ -39,7 +44,7 @@ permissions:
 jobs:
 
   ubuntu:
-    name: AMD64 Ubuntu 18.04 C# ${{ matrix.dotnet }}
+    name: AMD64 Ubuntu 24.04 C# ${{ matrix.dotnet }}
     runs-on: ubuntu-latest
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
     timeout-minutes: 15
@@ -52,6 +57,10 @@ jobs:
         uses: actions/[email protected]
         with:
           dotnet-version: ${{ matrix.dotnet }}
+      - name: Setup Python
+        uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
+        with:
+          python-version: 3
       - name: Checkout Arrow
         uses: actions/checkout@v4
         with:

.github/workflows/dev.yml

@@ -18,8 +18,13 @@
 name: Dev
 
 on:
-  # always trigger
+  # always trigger except Dependabot "push"
   push:
+    branches:
+      - '**'
+      - '!dependabot/**'
+    tags:
+      - '**'
   pull_request:
 
 concurrency:

.github/workflows/integration.yml

@@ -19,6 +19,11 @@ name: Integration
 
 on:
   push:
+    branches:
+      - '**'
+      - '!dependabot/**'
+    tags:
+      - '**'
     paths:
       - '.dockerignore'
       - '.github/workflows/integration.yml'

.github/workflows/java.yml

@@ -19,6 +19,11 @@ name: Java
 
 on:
   push:
+    branches:
+      - '**'
+      - '!dependabot/**'
+    tags:
+      - '**'
     paths:
       - '.dockerignore'
       - '.github/workflows/java.yml'

.github/workflows/java_jni.yml

@@ -19,6 +19,11 @@ name: Java JNI
 
 on:
   push:
+    branches:
+      - '**'
+      - '!dependabot/**'
+    tags:
+      - '**'
     paths:
       - '.dockerignore'
       - '.github/workflows/java_jni.yml'

.github/workflows/js.yml

@@ -19,6 +19,11 @@ name: NodeJS
 
 on:
   push:
+    branches:
+      - '**'
+      - '!dependabot/**'
+    tags:
+      - '**'
     paths:
       - '.dockerignore'
       - '.github/workflows/js.yml'

.github/workflows/matlab.yml

@@ -19,6 +19,11 @@ name: MATLAB
 
 on:
   push:
+    branches:
+      - '**'
+      - '!dependabot/**'
+    tags:
+      - '**'
     paths:
       - '.github/workflows/matlab.yml'
       - 'ci/scripts/matlab*.sh'

.github/workflows/pr_review_trigger.yml

@@ -29,7 +29,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: "Upload PR review Payload"
-        uses: actions/upload-artifact@604373da6381bf24206979c74d06a550515601b9 # v4.4.1
+        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
         with:
           path: "${{ github.event_path }}"
           name: "pr_review_payload"

.github/workflows/python.yml

@@ -19,6 +19,11 @@ name: Python
 
 on:
   push:
+    branches:
+      - '**'
+      - '!dependabot/**'
+    tags:
+      - '**'
     paths:
       - '.dockerignore'
       - '.github/workflows/python.yml'

.github/workflows/r.yml

@@ -19,6 +19,11 @@ name: R
 
 on:
   push:
+    branches:
+      - '**'
+      - '!dependabot/**'
+    tags:
+      - '**'
     paths:
       - '.dockerignore'
       - ".github/workflows/r.yml"
@@ -59,7 +64,9 @@ env:
 jobs:
   ubuntu-minimum-cpp-version:
     name: Check minimum supported Arrow C++ Version (${{ matrix.cpp_version }})
-    runs-on: ubuntu-latest
+    # We don't provide Apache Arrow C++ 15.0.2 deb packages for Ubuntu 24.04.
+    # So we use ubuntu-22.04 here.
+    runs-on: ubuntu-22.04
     strategy:
       matrix:
         include:
@@ -170,7 +177,7 @@ jobs:
         if: always()
       - name: Save the test output
         if: always()
-        uses: actions/upload-artifact@604373da6381bf24206979c74d06a550515601b9 # v4.4.1
+        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
         with:
           name: test-output-${{ matrix.ubuntu }}-${{ matrix.r }}
           path: r/check/arrow.Rcheck/tests/testthat.Rout*
@@ -230,7 +237,7 @@ jobs:
         if: always()
       - name: Save the test output
         if: always()
-        uses: actions/upload-artifact@604373da6381bf24206979c74d06a550515601b9 # v4.4.1
+        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
         with:
           name: test-output-bundled
           path: r/check/arrow.Rcheck/tests/testthat.Rout*
@@ -292,7 +299,7 @@ jobs:
         # So that they're unique when multiple are downloaded in the next step
         shell: bash
         run: mv libarrow.zip libarrow-rtools${{ matrix.config.rtools }}-${{ matrix.config.arch }}.zip
-      - uses: actions/upload-artifact@604373da6381bf24206979c74d06a550515601b9 # v4.4.1
+      - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
         with:
           name: libarrow-rtools${{ matrix.config.rtools }}-${{ matrix.config.arch }}.zip
           path: libarrow-rtools${{ matrix.config.rtools }}-${{ matrix.config.arch }}.zip

.github/workflows/ruby.yml

@@ -19,6 +19,11 @@ name: C GLib & Ruby
 
 on:
   push:
+    branches:
+      - '**'
+      - '!dependabot/**'
+    tags:
+      - '**'
     paths:
       - '.dockerignore'
       - '.github/workflows/ruby.yml'

.github/workflows/swift.yml

@@ -19,6 +19,11 @@ name: Swift
 
 on:
   push:
+    branches:
+      - '**'
+      - '!dependabot/**'
+    tags:
+      - '**'
     paths:
       - '.dockerignore'
       - '.github/workflows/swift.yml'
@@ -58,6 +63,10 @@ jobs:
         with:
           fetch-depth: 0
           submodules: recursive
+      - name: Setup Python on hosted runner
+        uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
+        with:
+          python-version: 3
       - name: Setup Archery
         run: pip install -e dev/archery[docker]
       - name: Execute Docker Build

README.md

@@ -25,8 +25,9 @@
 
 ## Powering In-Memory Analytics
 
-Apache Arrow is a development platform for in-memory analytics. It contains a
-set of technologies that enable big data systems to process and move data fast.
+Apache Arrow is a universal columnar format and multi-language toolbox for fast
+data interchange and in-memory analytics. It contains a set of technologies that
+enable data systems to efficiently store, process, and move data.
 
 Major components of the project include:
 

c_glib/arrow-glib/basic-data-type.cpp

@@ -1660,9 +1660,9 @@ enum {
   PROP_STORAGE_DATA_TYPE = 1
 };
 
-G_DEFINE_TYPE_WITH_PRIVATE(GArrowExtensionDataType,
-                           garrow_extension_data_type,
-                           GARROW_TYPE_DATA_TYPE)
+G_DEFINE_ABSTRACT_TYPE_WITH_PRIVATE(GArrowExtensionDataType,
+                                    garrow_extension_data_type,
+                                    GARROW_TYPE_DATA_TYPE)
 
 #define GARROW_EXTENSION_DATA_TYPE_GET_PRIVATE(obj)                                      \
   static_cast<GArrowExtensionDataTypePrivate *>(                                         \

c_glib/meson.build

@@ -103,23 +103,23 @@ else
 endif
 
 if arrow_cpp_build_lib_dir == ''
-  arrow = dependency('arrow')
+  arrow = dependency('arrow', version: ['>=' + version])
   # They are just for checking required modules are enabled. They are built into
   # libarrow.so. So we don't need additional build flags for them.
-  dependency('arrow-compute')
-  dependency('arrow-csv')
-  dependency('arrow-filesystem')
-  dependency('arrow-json')
+  dependency('arrow-compute', version: ['>=' + version])
+  dependency('arrow-csv', version: ['>=' + version])
+  dependency('arrow-filesystem', version: ['>=' + version])
+  dependency('arrow-json', version: ['>=' + version])
 
-  have_arrow_orc = dependency('arrow-orc', required: false).found()
-  arrow_cuda = dependency('arrow-cuda', required: false)
+  have_arrow_orc = dependency('arrow-orc', required: false, version: ['>=' + version]).found()
+  arrow_cuda = dependency('arrow-cuda', required: false, version: ['>=' + version])
   # we do not support compiling glib without acero engine
-  arrow_acero = dependency('arrow-acero', required: true)
-  arrow_dataset = dependency('arrow-dataset', required: false)
-  arrow_flight = dependency('arrow-flight', required: false)
-  arrow_flight_sql = dependency('arrow-flight-sql', required: false)
-  gandiva = dependency('gandiva', required: false)
-  parquet = dependency('parquet', required: false)
+  arrow_acero = dependency('arrow-acero', required: true, version: ['>=' + version])
+  arrow_dataset = dependency('arrow-dataset', required: false, version: ['>=' + version])
+  arrow_flight = dependency('arrow-flight', required: false, version: ['>=' + version])
+  arrow_flight_sql = dependency('arrow-flight-sql', required: false, version: ['>=' + version])
+  gandiva = dependency('gandiva', required: false, version: ['>=' + version])
+  parquet = dependency('parquet', required: false, version: ['>=' + version])
 else
   base_include_directories += [
     include_directories(join_paths(arrow_cpp_build_dir, 'src')),

c_glib/test/test-decimal128-data-type.rb

@@ -47,9 +47,7 @@ def test_decimal_data_type_new
   end
 
   def test_invalid_precision
-    message =
-      "[decimal128-data-type][new]: Invalid: Decimal precision out of range [1, 38]: 39"
-    assert_raise(Arrow::Error::Invalid.new(message)) do
+    assert_raise(Arrow::Error::Invalid) do
       Arrow::Decimal128DataType.new(39, 1)
     end
   end

c_glib/test/test-decimal256-data-type.rb

@@ -47,9 +47,7 @@ def test_decimal_data_type_new
   end
 
   def test_invalid_precision
-    message =
-      "[decimal256-data-type][new]: Invalid: Decimal precision out of range [1, 76]: 77"
-    assert_raise(Arrow::Error::Invalid.new(message)) do
+    assert_raise(Arrow::Error::Invalid) do
       Arrow::Decimal256DataType.new(77, 1)
     end
   end

c_glib/test/test-extension-data-type.rb

@@ -102,4 +102,10 @@ def test_wrap_chunked_array
                    extension_chunked_array.chunks.collect(&:class),
                  ])
   end
+
+  def test_abstract_class
+    assert_raise(TypeError) do
+      Arrow::ExtensionDataType.new
+    end
+  end
 end

ci/appveyor-cpp-build.bat

@@ -139,7 +139,7 @@ set PARQUET_HOME=%CONDA_PREFIX%\Library
 
 @rem Download IANA Timezone Database to a non-standard location to
 @rem test the configurability of the timezone database path
-curl https://data.iana.org/time-zones/releases/tzdata2021e.tar.gz --output tzdata.tar.gz || exit /B
+curl https://data.iana.org/time-zones/releases/tzdata2024b.tar.gz --output tzdata.tar.gz || exit /B
 mkdir %USERPROFILE%\Downloads\test\tzdata
 tar --extract --file tzdata.tar.gz --directory %USERPROFILE%\Downloads\test\tzdata
 curl https://raw.githubusercontent.com/unicode-org/cldr/master/common/supplemental/windowsZones.xml ^

ci/scripts/python_wheel_manylinux_build.sh

@@ -55,7 +55,6 @@ echo "=== (${PYTHON_VERSION}) Building Arrow C++ libraries ==="
 : ${ARROW_GANDIVA:=OFF}
 : ${ARROW_GCS:=ON}
 : ${ARROW_HDFS:=ON}
-: ${ARROW_JEMALLOC:=ON}
 : ${ARROW_MIMALLOC:=ON}
 : ${ARROW_ORC:=ON}
 : ${ARROW_PARQUET:=ON}
@@ -81,6 +80,9 @@ if [[ "$(uname -m)" == arm* ]] || [[ "$(uname -m)" == aarch* ]]; then
     # 4k and 64k page arm64 systems. For more context see
     # https://github.com/apache/arrow/issues/10929
     export ARROW_EXTRA_CMAKE_FLAGS="-DARROW_JEMALLOC_LG_PAGE=16"
+    : ${ARROW_JEMALLOC:=OFF}
+else
+    : ${ARROW_JEMALLOC:=ON}
 fi
 
 mkdir /tmp/arrow-build

cpp/README.md

@@ -31,4 +31,4 @@ to install pre-compiled binary versions of the library.
 
 Please refer to our latest [C++ Development Documentation][1].
 
-[1]: https://github.com/apache/arrow/blob/main/docs/source/developers/cpp
+[1]: https://arrow.apache.org/docs/dev/developers/cpp/