Merge branch 'reproduce-payload-overflow' into more-overflow-safe-swi…

…ss-table-temp
zanmato1984 · Jan 24, 2025 · 01e1cc2 · 01e1cc2
2 parents 53c8064 + d4c2af3
commit 01e1cc2
Show file tree

Hide file tree

Showing 203 changed files with 3,884 additions and 1,898 deletions.
diff --git a/.asf.yaml b/.asf.yaml
@@ -23,7 +23,6 @@ github:
     - benibus
     - jbonofre
     - js8544
-    - laurentgo
     - vibhatha
     - ZhangHuiGui
 

diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml
@@ -69,65 +69,37 @@ env:
   DOCKER_VOLUME_PREFIX: ".docker/"
 
 jobs:
-  docker-targets:
-    name: Docker targets
-    runs-on: ubuntu-latest
-    outputs:
-      targets: ${{ steps.detect-targets.outputs.targets }}
-    steps:
-      - name: Detect targets
-        id: detect-targets
-        run: |
-          echo "targets<<JSON" >> "$GITHUB_OUTPUT"
-          echo "[" >> "$GITHUB_OUTPUT"
-          cat <<JSON >> "$GITHUB_OUTPUT"
-          {
-            "arch": "amd64",
-            "clang-tools": "14",
-            "image": "conda-cpp",
-            "llvm": "14",
-            "runs-on": "ubuntu-latest",
-            "simd-level": "AVX2",
-            "title": "AMD64 Conda C++ AVX2",
-            "ubuntu": "22.04"
-          },
-          {
-            "arch": "amd64",
-            "clang-tools": "14",
-            "image": "ubuntu-cpp-sanitizer",
-            "llvm": "14",
-            "runs-on": "ubuntu-latest",
-            "title": "AMD64 Ubuntu 22.04 C++ ASAN UBSAN",
-            "ubuntu": "22.04"
-          }
-          JSON
-          if [ "$GITHUB_REPOSITORY_OWNER" = "apache" ]; then
-            echo "," >> "$GITHUB_OUTPUT"
-            cat <<JSON >> "$GITHUB_OUTPUT"
-          {
-            "arch": "arm64v8",
-            "clang-tools": "10",
-            "image": "ubuntu-cpp",
-            "llvm": "10",
-            "runs-on": ["self-hosted", "arm", "linux"],
-            "title": "ARM64 Ubuntu 20.04 C++",
-            "ubuntu": "20.04"
-          }
-          JSON
-          fi
-          echo "]" >> "$GITHUB_OUTPUT"
-          echo "JSON" >> "$GITHUB_OUTPUT"
-
   docker:
     name: ${{ matrix.title }}
-    needs: docker-targets
     runs-on: ${{ matrix.runs-on }}
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
     timeout-minutes: 75
     strategy:
       fail-fast: false
       matrix:
-        include: ${{ fromJson(needs.docker-targets.outputs.targets) }}
+        include:
+          - arch: amd64
+            clang-tools: 14
+            image: conda-cpp
+            llvm: 14
+            runs-on: ubuntu-latest
+            simd-level: AVX2
+            title: AMD64 Conda C++ AVX2
+            ubuntu: 22.04
+          - arch: amd64
+            clang-tools: 14
+            image: ubuntu-cpp-sanitizer
+            llvm: 14
+            runs-on: ubuntu-latest
+            title: AMD64 Ubuntu 22.04 C++ ASAN UBSAN
+            ubuntu: 22.04
+          - arch: arm64v8
+            clang-tools: 10
+            image: ubuntu-cpp
+            llvm: 10
+            runs-on: ubuntu-24.04-arm
+            title: ARM64 Ubuntu 20.04 C++
+            ubuntu: 20.04
     env:
       ARCH: ${{ matrix.arch }}
       ARROW_SIMD_LEVEL: ${{ matrix.simd-level }}

diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml
@@ -124,7 +124,7 @@ jobs:
         shell: bash
         run: |
           gem install test-unit
-          pip install "cython>=0.29.31" setuptools pytest requests setuptools-scm
+          pip install "cython>=3" setuptools pytest requests setuptools-scm
       - name: Run Release Test
         env:
           ARROW_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}

diff --git a/.github/workflows/pr_review_trigger.yml b/.github/workflows/pr_review_trigger.yml
@@ -29,7 +29,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: "Upload PR review Payload"
-        uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
+        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
         with:
           path: "${{ github.event_path }}"
           name: "pr_review_payload"
diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml
@@ -177,7 +177,7 @@ jobs:
         if: always()
       - name: Save the test output
         if: always()
-        uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
+        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
         with:
           name: test-output-${{ matrix.ubuntu }}-${{ matrix.r }}
           path: r/check/arrow.Rcheck/tests/testthat.Rout*
@@ -237,7 +237,7 @@ jobs:
         if: always()
       - name: Save the test output
         if: always()
-        uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
+        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
         with:
           name: test-output-bundled
           path: r/check/arrow.Rcheck/tests/testthat.Rout*
@@ -299,7 +299,7 @@ jobs:
         # So that they're unique when multiple are downloaded in the next step
         shell: bash
         run: mv libarrow.zip libarrow-rtools${{ matrix.config.rtools }}-${{ matrix.config.arch }}.zip
-      - uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
+      - uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
         with:
           name: libarrow-rtools${{ matrix.config.rtools }}-${{ matrix.config.arch }}.zip
           path: libarrow-rtools${{ matrix.config.rtools }}-${{ matrix.config.arch }}.zip

diff --git a/c_glib/meson.build b/c_glib/meson.build
@@ -35,7 +35,7 @@ project('arrow-glib', 'c', 'cpp',
         #   * 22.04: 0.61.2
         meson_version: '>=0.53.2')
 
-version = '19.0.0-SNAPSHOT'
+version = '20.0.0-SNAPSHOT'
 if version.endswith('-SNAPSHOT')
   version_numbers = version.split('-')[0].split('.')
   version_tag = version.split('-')[1]

diff --git a/c_glib/parquet-glib/arrow-file-writer.cpp b/c_glib/parquet-glib/arrow-file-writer.cpp
@@ -574,7 +574,6 @@ gparquet_arrow_file_writer_write_table(GParquetArrowFileWriter *writer,
 /**
  * gparquet_arrow_file_writer_new_row_group:
  * @writer: A #GParquetArrowFileWriter.
- * @chunk_size: The max number of rows in a row group.
  * @error: (nullable): Return location for a #GError or %NULL.
  *
  * Start a new row group.
@@ -584,13 +583,11 @@ gparquet_arrow_file_writer_write_table(GParquetArrowFileWriter *writer,
  * Since: 18.0.0
  */
 gboolean
-gparquet_arrow_file_writer_new_row_group(GParquetArrowFileWriter *writer,
-                                         gsize chunk_size,
-                                         GError **error)
+gparquet_arrow_file_writer_new_row_group(GParquetArrowFileWriter *writer, GError **error)
 {
   auto parquet_arrow_file_writer = gparquet_arrow_file_writer_get_raw(writer);
   return garrow::check(error,
-                       parquet_arrow_file_writer->NewRowGroup(chunk_size),
+                       parquet_arrow_file_writer->NewRowGroup(),
                        "[parquet][arrow][file-writer][new-row-group]");
 }
 

diff --git a/c_glib/parquet-glib/arrow-file-writer.h b/c_glib/parquet-glib/arrow-file-writer.h
@@ -135,9 +135,7 @@ gparquet_arrow_file_writer_write_table(GParquetArrowFileWriter *writer,
 
 GPARQUET_AVAILABLE_IN_18_0
 gboolean
-gparquet_arrow_file_writer_new_row_group(GParquetArrowFileWriter *writer,
-                                         gsize chunk_size,
-                                         GError **error);
+gparquet_arrow_file_writer_new_row_group(GParquetArrowFileWriter *writer, GError **error);
 
 GPARQUET_AVAILABLE_IN_18_0
 gboolean

diff --git a/c_glib/test/parquet/test-arrow-file-writer.rb b/c_glib/test/parquet/test-arrow-file-writer.rb
@@ -89,10 +89,10 @@ def test_write_table
   def test_write_chunked_array
     schema = build_schema("enabled" => :boolean)
     writer = Parquet::ArrowFileWriter.new(schema, @file.path)
-    writer.new_row_group(2)
+    writer.new_row_group
     chunked_array = Arrow::ChunkedArray.new([build_boolean_array([true, nil])])
     writer.write_chunked_array(chunked_array)
-    writer.new_row_group(1)
+    writer.new_row_group
     chunked_array = Arrow::ChunkedArray.new([build_boolean_array([false])])
     writer.write_chunked_array(chunked_array)
     writer.close

diff --git a/c_glib/tool/generate-version-header.py b/c_glib/tool/generate-version-header.py
@@ -140,6 +140,7 @@ def generate_availability_macros(library: str) -> str:
 
 
 ALL_VERSIONS = [
+        (20, 0),
         (19, 0),
         (18, 0),
         (17, 0),

diff --git a/c_glib/vcpkg.json b/c_glib/vcpkg.json
@@ -1,6 +1,6 @@
 {
   "name": "arrow-glib",
-  "version-string": "19.0.0-SNAPSHOT",
+  "version-string": "20.0.0-SNAPSHOT",
   "dependencies": [
     "glib",
     "gobject-introspection",

diff --git a/ci/appveyor-cpp-setup.bat b/ci/appveyor-cpp-setup.bat
@@ -70,7 +70,6 @@ conda create -n arrow ^
   "ninja" ^
   "nomkl" ^
   "pandas" ^
-  "fsspec" ^
   "python=%PYTHON%" ^
   || exit /B
 conda list -n arrow
@@ -86,7 +85,7 @@ set CXX=cl.exe
 @rem Download Minio somewhere on PATH, for unit tests
 @rem
 if "%ARROW_S3%" == "ON" (
-  appveyor DownloadFile https://dl.min.io/server/minio/release/windows-amd64/archive/minio.RELEASE.2024-09-13T20-26-02Z -FileName C:\Windows\Minio.exe || exit /B
+  appveyor DownloadFile https://dl.min.io/server/minio/release/windows-amd64/archive/minio.RELEASE.2025-01-20T14-49-07Z -FileName C:\Windows\Minio.exe || exit /B
 )
 
 @rem

diff --git a/ci/conda_env_python.txt b/ci/conda_env_python.txt
@@ -15,10 +15,12 @@
 # specific language governing permissions and limitations
 # under the License.
 
-# don't add pandas here, because it is not a mandatory test dependency
-boto3  # not a direct dependency of s3fs, but needed for our s3fs fixture
+# Don't add pandas here, because it is not a mandatory test dependency
+
+# Not a direct dependency of s3fs, but needed for our s3fs fixture
+boto3
 cffi
-cython>=0.29.31
+cython>=3
 cloudpickle
 fsspec
 hypothesis

diff --git a/ci/conda_env_sphinx.txt b/ci/conda_env_sphinx.txt
@@ -30,9 +30,5 @@ sphinx-lint
 sphinxcontrib-jquery
 sphinxcontrib-mermaid
 sphinx==6.2
-# Requirement for doctest-cython
-# Needs upper pin of 0.3.0, see:
-# https://github.com/lgpage/pytest-cython/issues/67
-# With 0.3.* bug fix release, the pin can be removed
-pytest-cython==0.2.2
+pytest-cython
 pandas
diff --git a/ci/docker/conda-python-cython2.dockerfile b/ci/docker/conda-python-cython2.dockerfile
diff --git a/ci/docker/conda.dockerfile b/ci/docker/conda.dockerfile
@@ -21,9 +21,15 @@ FROM ${arch}/ubuntu:22.04
 # install build essentials
 RUN export DEBIAN_FRONTEND=noninteractive && \
     apt-get update -y -q && \
-    apt-get install -y -q curl wget tzdata libc6-dbg gdb \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/*
+    apt-get install -y -q \
+    curl \
+    gdb \
+    libc6-dbg \
+    patch \
+    tzdata \
+    wget && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
 
 # install conda and mamba via miniforge
 COPY ci/scripts/install_conda.sh /arrow/ci/scripts/

diff --git a/ci/docker/debian-12-cpp.dockerfile b/ci/docker/debian-12-cpp.dockerfile
@@ -84,6 +84,7 @@ RUN apt-get update -y -q && \
         ninja-build \
         nlohmann-json3-dev \
         npm \
+        patch \
         pkg-config \
         protobuf-compiler-grpc \
         python3-dev \

diff --git a/ci/docker/ubuntu-20.04-cpp.dockerfile b/ci/docker/ubuntu-20.04-cpp.dockerfile
@@ -106,6 +106,7 @@ RUN apt-get update -y -q && \
         ninja-build \
         nlohmann-json3-dev \
         npm \
+        patch \
         pkg-config \
         protobuf-compiler \
         python3-dev \

diff --git a/ci/docker/ubuntu-22.04-cpp.dockerfile b/ci/docker/ubuntu-22.04-cpp.dockerfile
@@ -111,6 +111,7 @@ RUN apt-get update -y -q && \
         ninja-build \
         nlohmann-json3-dev \
         npm \
+        patch \
         pkg-config \
         protobuf-compiler \
         protobuf-compiler-grpc \

diff --git a/ci/docker/ubuntu-24.04-cpp.dockerfile b/ci/docker/ubuntu-24.04-cpp.dockerfile
@@ -111,6 +111,7 @@ RUN apt-get update -y -q && \
         ninja-build \
         nlohmann-json3-dev \
         npm \
+        patch \
         pkg-config \
         protobuf-compiler \
         protobuf-compiler-grpc \

diff --git a/ci/scripts/PKGBUILD b/ci/scripts/PKGBUILD
@@ -18,7 +18,7 @@
 _realname=arrow
 pkgbase=mingw-w64-${_realname}
 pkgname="${MINGW_PACKAGE_PREFIX}-${_realname}"
-pkgver=18.1.0.9000
+pkgver=19.0.0.9000
 pkgrel=8000
 pkgdesc="Apache Arrow is a cross-language development platform for in-memory data (mingw-w64)"
 arch=("any")

diff --git a/ci/scripts/install_minio.sh b/ci/scripts/install_minio.sh
@@ -63,7 +63,7 @@ if [ "${version}" != "latest" ]; then
 fi
 
 # Use specific versions for minio server and client to avoid CI failures on new releases.
-minio_version="minio.RELEASE.2024-09-13T20-26-02Z"
+minio_version="minio.RELEASE.2025-01-20T14-49-07Z"
 mc_version="mc.RELEASE.2024-09-16T17-43-14Z"
 
 download()

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
@@ -79,7 +79,7 @@ if(POLICY CMP0170)
   cmake_policy(SET CMP0170 NEW)
 endif()
 
-set(ARROW_VERSION "19.0.0-SNAPSHOT")
+set(ARROW_VERSION "20.0.0-SNAPSHOT")
 
 string(REGEX MATCH "^[0-9]+\\.[0-9]+\\.[0-9]+" ARROW_BASE_VERSION "${ARROW_VERSION}")