Merge branch 'apache:main' into delete-file-branch

apache · Feb 7, 2024 · 168356e · 168356e
2 parents 3cce85c + a6e577d
commit 168356e
Show file tree

Hide file tree

Showing 83 changed files with 837 additions and 304 deletions.
diff --git a/.env b/.env
@@ -92,13 +92,13 @@ DEVTOOLSET_VERSION=
 # Used through docker-compose.yml and serves as the default version for the
 # ci/scripts/install_vcpkg.sh script. Prefer to use short SHAs to keep the
 # docker tags more readable.
-VCPKG="501db0f17ef6df184fcdbfbe0f87cde2313b6ab1"    # 2023.04.15 Release
+VCPKG="a42af01b72c28a8e1d7b48107b33e4f286a55ef6"    # 2023.11.20 Release
 
 # This must be updated when we update
-# ci/docker/python-wheel-windows-vs2017.dockerfile.
+# ci/docker/python-wheel-windows-vs2019.dockerfile.
 # This is a workaround for our CI problem that "archery docker build" doesn't
 # use pulled built images in dev/tasks/python-wheels/github.windows.yml.
-PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2023-08-02
+PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2024-02-05
 
 # Use conanio/${CONAN} for "docker-compose run --rm conan". See
 # https://github.com/conan-io/conan-docker-tools#readme for available

diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml
@@ -57,37 +57,65 @@ env:
   DOCKER_VOLUME_PREFIX: ".docker/"
 
 jobs:
+  docker-targets:
+    name: Docker targets
+    runs-on: ubuntu-latest
+    outputs:
+      targets: ${{ steps.detect-targets.outputs.targets }}
+    steps:
+      - name: Detect targets
+        id: detect-targets
+        run: |
+          echo "targets<<JSON" >> "$GITHUB_OUTPUT"
+          echo "[" >> "$GITHUB_OUTPUT"
+          cat <<JSON >> "$GITHUB_OUTPUT"
+          {
+            "arch": "amd64",
+            "clang-tools": "14",
+            "image": "conda-cpp",
+            "llvm": "14",
+            "runs-on": "ubuntu-latest",
+            "simd-level": "AVX2",
+            "title": "AMD64 Conda C++ AVX2",
+            "ubuntu": "22.04"
+          },
+          {
+            "arch": "amd64",
+            "clang-tools": "14",
+            "image": "ubuntu-cpp-sanitizer",
+            "llvm": "14",
+            "runs-on": "ubuntu-latest",
+            "title": "AMD64 Ubuntu 22.04 C++ ASAN UBSAN",
+            "ubuntu": "22.04"
+          }
+          JSON
+          if [ "$GITHUB_REPOSITORY_OWNER" = "apache" ]; then
+            echo "," >> "$GITHUB_OUTPUT"
+            cat <<JSON >> "$GITHUB_OUTPUT"
+          {
+            "arch": "arm64v8",
+            "clang-tools": "10",
+            "image": "ubuntu-cpp",
+            "llvm": "10",
+            "runs-on": ["self-hosted", "arm", "linux"],
+            "title": "ARM64 Ubuntu 20.04 C++",
+            "ubuntu": "20.04"
+          }
+          JSON
+          fi
+          echo "]" >> "$GITHUB_OUTPUT"
+          echo "JSON" >> "$GITHUB_OUTPUT"
+
   docker:
     name: ${{ matrix.title }}
+    needs: docker-targets
     runs-on: ${{ matrix.runs-on }}
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
     timeout-minutes: 75
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - arch: amd64
-            clang-tools: "14"
-            image: conda-cpp
-            llvm: "14"
-            runs-on: ubuntu-latest
-            simd-level: AVX2
-            title: AMD64 Conda C++ AVX2
-            ubuntu: "22.04"
-          - arch: amd64
-            clang-tools: "14"
-            image: ubuntu-cpp-sanitizer
-            llvm: "14"
-            runs-on: ubuntu-latest
-            title: AMD64 Ubuntu 22.04 C++ ASAN UBSAN
-            ubuntu: "22.04"
-          - arch: arm64v8
-            clang-tools: "10"
-            image: ubuntu-cpp
-            llvm: "10"
-            runs-on: ["self-hosted", "arm", "linux"]
-            title: ARM64 Ubuntu 20.04 C++
-            ubuntu: "20.04"
+        include: ${{ fromJson(needs.docker-targets.outputs.targets) }}
     env:
       ARCH: ${{ matrix.arch }}
       ARROW_SIMD_LEVEL: ${{ matrix.simd-level }}

diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml
@@ -43,31 +43,62 @@ permissions:
 
 jobs:
 
+  docker-targets:
+    name: Docker targets
+    runs-on: ubuntu-latest
+    if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
+    outputs:
+      targets: ${{ steps.detect-targets.outputs.targets }}
+    steps:
+      - name: Detect targets
+        id: detect-targets
+        run: |
+          echo "targets<<JSON" >> "$GITHUB_OUTPUT"
+          echo "[" >> "$GITHUB_OUTPUT"
+          cat <<JSON >> "$GITHUB_OUTPUT"
+          {
+            "arch-label": "AMD64",
+            "arch": "amd64",
+            "go": "1.19",
+            "runs-on": "ubuntu-latest"
+          },
+          {
+            "arch-label": "AMD64",
+            "arch": "amd64",
+            "go": "1.20",
+            "runs-on": "ubuntu-latest"
+          }
+          JSON
+          if [ "$GITHUB_REPOSITORY_OWNER" = "apache" ]; then
+            echo "," >> "$GITHUB_OUTPUT"
+            cat <<JSON >> "$GITHUB_OUTPUT"
+          {
+            "arch-label": "ARM64",
+            "arch": "arm64v8",
+            "go": "1.19",
+            "runs-on": ["self-hosted", "arm", "linux"]
+          },
+          {
+            "arch-label": "ARM64",
+            "arch": "arm64v8",
+            "go": "1.20",
+            "runs-on": ["self-hosted", "arm", "linux"]
+          }
+          JSON
+          fi
+          echo "]" >> "$GITHUB_OUTPUT"
+          echo "JSON" >> "$GITHUB_OUTPUT"
+
   docker:
     name: ${{ matrix.arch-label }} Debian 11 Go ${{ matrix.go }}
+    needs: docker-targets
     runs-on: ${{ matrix.runs-on }}
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
     timeout-minutes: 60
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - arch-label: AMD64
-            arch: amd64
-            go: 1.19
-            runs-on: ubuntu-latest
-          - arch-label: AMD64
-            arch: amd64
-            go: '1.20'
-            runs-on: ubuntu-latest
-          - arch-label: ARM64
-            arch: arm64v8
-            go: 1.19
-            runs-on: ["self-hosted", "arm", "linux"]
-          - arch-label: ARM64
-            arch: arm64v8
-            go: '1.20'
-            runs-on: ["self-hosted", "arm", "linux"]
+        include: ${{ fromJson(needs.docker-targets.outputs.targets) }}
     env:
       ARCH: ${{ matrix.arch }}
       GO: ${{ matrix.go }}

diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml
@@ -21,24 +21,26 @@ on:
   push:
     paths:
       - ".github/workflows/r.yml"
-      - "ci/scripts/r_*.sh"
-      - "ci/scripts/cpp_*.sh"
-      - "ci/scripts/PKGBUILD"
-      - "ci/etc/rprofile"
       - "ci/docker/**"
+      - "ci/etc/rprofile"
+      - "ci/scripts/PKGBUILD"
+      - "ci/scripts/cpp_*.sh"
+      - "ci/scripts/install_minio.sh"
+      - "ci/scripts/r_*.sh"
       - "cpp/**"
-      - 'docker-compose.yml'
+      - "docker-compose.yml"
       - "r/**"
   pull_request:
     paths:
       - ".github/workflows/r.yml"
-      - "ci/scripts/r_*.sh"
-      - "ci/scripts/cpp_*.sh"
-      - "ci/scripts/PKGBUILD"
-      - "ci/etc/rprofile"
       - "ci/docker/**"
+      - "ci/etc/rprofile"
+      - "ci/scripts/PKGBUILD"
+      - "ci/scripts/cpp_*.sh"
+      - "ci/scripts/install_minio.sh"
+      - "ci/scripts/r_*.sh"
       - "cpp/**"
-      - 'docker-compose.yml'
+      - "docker-compose.yml"
       - "r/**"
 
 concurrency:
@@ -256,6 +258,16 @@ jobs:
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
+      # This must be done before r-lib/actions/setup-r because curl in
+      # Rtools doesn't work on non Rtools' MSYS2 environment. If we
+      # use "shell: bash" after r-lib/actions/setup-r, bash in Rtools
+      # is used on non Rtools' MSYS2 environment.
+      - name: Install MinIO
+        shell: bash
+        run: |
+          mkdir -p "$HOME/.local/bin"
+          ci/scripts/install_minio.sh latest "$HOME/.local"
+          echo "$HOME/.local/bin" >> $GITHUB_PATH
       - run: mkdir r/windows
       - name: Download artifacts
         uses: actions/download-artifact@v3
@@ -282,15 +294,6 @@ jobs:
           working-directory: 'r'
           extra-packages: |
             any::rcmdcheck
-      - name: Install MinIO
-        shell: bash
-        run: |
-          mkdir -p "$HOME/.local/bin"
-          curl \
-            --output "$HOME/.local/bin/minio.exe" \
-            https://dl.min.io/server/minio/release/windows-amd64/archive/minio.RELEASE.2022-05-26T05-48-41Z
-          chmod +x "$HOME/.local/bin/minio.exe"
-          echo "$HOME/.local/bin" >> $GITHUB_PATH
       # TODO(ARROW-17149): figure out why the GCS tests are hanging on Windows
       # - name: Install Google Cloud Storage Testbench
       #   shell: bash

diff --git a/c_glib/README.md b/c_glib/README.md
@@ -101,7 +101,7 @@ $ sudo meson install -C c_glib.build
 You need to install Arrow C++ before you install Arrow GLib. See Arrow
 C++ document about how to install Arrow C++.
 
-You need [GTK-Doc](https://www.gtk.org/gtk-doc/) and
+You need [GTK-Doc](https://gitlab.gnome.org/GNOME/gtk-doc) and
 [GObject Introspection](https://wiki.gnome.org/Projects/GObjectIntrospection)
 to build Arrow GLib. You can install them by the followings:
 

diff --git a/ci/docker/python-wheel-manylinux.dockerfile b/ci/docker/python-wheel-manylinux.dockerfile
@@ -62,15 +62,16 @@ COPY ci/vcpkg/*.patch \
 COPY ci/scripts/install_vcpkg.sh \
      arrow/ci/scripts/
 ENV VCPKG_ROOT=/opt/vcpkg
-RUN arrow/ci/scripts/install_vcpkg.sh ${VCPKG_ROOT} ${vcpkg}
-ENV PATH="${PATH}:${VCPKG_ROOT}"
-
 ARG build_type=release
 ENV CMAKE_BUILD_TYPE=${build_type} \
     VCPKG_FORCE_SYSTEM_BINARIES=1 \
     VCPKG_OVERLAY_TRIPLETS=/arrow/ci/vcpkg \
     VCPKG_DEFAULT_TRIPLET=${arch_short}-linux-static-${build_type} \
     VCPKG_FEATURE_FLAGS="manifests"
+
+RUN arrow/ci/scripts/install_vcpkg.sh ${VCPKG_ROOT} ${vcpkg}
+ENV PATH="${PATH}:${VCPKG_ROOT}"
+
 COPY ci/vcpkg/vcpkg.json arrow/ci/vcpkg/
 # cannot use the S3 feature here because while aws-sdk-cpp=1.9.160 contains
 # ssl related fixes as well as we can patch the vcpkg portfile to support

diff --git a/...thon-wheel-windows-test-vs2017.dockerfile → ...thon-wheel-windows-test-vs2019.dockerfile b/...thon-wheel-windows-test-vs2017.dockerfile → ...thon-wheel-windows-test-vs2019.dockerfile
@@ -19,8 +19,8 @@
 # when you update this file.
 
 # based on mcr.microsoft.com/windows/servercore:ltsc2019
-# contains choco and vs2017 preinstalled
-FROM abrarov/msvc-2017:2.11.0
+# contains choco and vs2019 preinstalled
+FROM abrarov/msvc-2019:2.11.0
 
 # Add unix tools to path
 RUN setx path "%path%;C:\Program Files\Git\usr\bin"

diff --git a/...er/python-wheel-windows-vs2017.dockerfile → ...er/python-wheel-windows-vs2019.dockerfile b/...er/python-wheel-windows-vs2017.dockerfile → ...er/python-wheel-windows-vs2019.dockerfile
@@ -19,8 +19,8 @@
 # when you update this file.
 
 # based on mcr.microsoft.com/windows/servercore:ltsc2019
-# contains choco and vs2017 preinstalled
-FROM abrarov/msvc-2017:2.11.0
+# contains choco and vs2019 preinstalled
+FROM abrarov/msvc-2019:2.11.0
 
 # Install CMake and Ninja
 ARG cmake=3.21.4