crossbow.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.


# NOTE: must set "Crossbow" as name to have the badge links working in the
# github comment reports!
name: Crossbow
on:
  push:
    branches:
      - "*-github-*"


jobs:

  build-cpp-ubuntu:
    name: Build C++ libraries Ubuntu ${{ matrix.platform.arch }}
    runs-on: ${{ matrix.platform.runs_on }}
    env:
      # architecture name used for archery build
      ARCH: ${{ matrix.platform.archery_arch }}
      ARCH_ALIAS: ${{ matrix.platform.archery_arch_alias }}
      ARCH_SHORT: ${{ matrix.platform.archery_arch_short }}
    strategy:
      fail-fast: false
      matrix:
        platform:
          - runs_on: ["ubuntu-latest"]
            arch: "x86_64"
            archery_arch: "amd64"
            archery_arch_alias: "x86_64"
            archery_arch_short: "amd64"
          - runs_on: ["self-hosted", "Linux", "arm64"]
            arch: "aarch_64"
            archery_arch: "arm64v8"
            archery_arch_alias: "aarch64"
            archery_arch_short: "arm64"
    steps:
      - name: Checkout Arrow
        uses: actions/checkout@v4
        with:
          fetch-depth: 1
          path: arrow
          repository: lidavidm/arrow
          ref: 1070ea5b0c149a5eb271c2ab47ec3a0f82a70c77
          submodules: recursive

      - name: Free up disk space
        if: runner.os == 'Linux' && runner.arch == 'X64'
        shell: bash
        run: |
          arrow/ci/scripts/util_free_space.sh

      - name: Set up Python by actions/setup-python
        if: runner.arch == 'X64'
        uses: actions/setup-python@v4
        with:
          cache: 'pip'
          python-version: 3.12
      - name: Set up Python by apt
        if: runner.os == 'Linux' && runner.arch != 'X64'
        run: |
          sudo apt update
          sudo apt-get install -y python3-pip
          pip install -U pip
          echo "$HOME/.local/bin" >>"$GITHUB_PATH"
      - name: Install Archery
        shell: bash
        run: pip install -e arrow/dev/archery[all]

      - name: Build C++ libraries
        env:
          AWS_ACCESS_KEY_ID: "${{ secrets.AWS_ACCESS_KEY_ID }}"
          AWS_SECRET_ACCESS_KEY: "${{ secrets.AWS_SECRET_ACCESS_KEY }}"
          SCCACHE_BUCKET: "${{ secrets.SCCACHE_BUCKET }}"
          SCCACHE_REGION: "${{ secrets.SCCACHE_REGION }}"
          SCCACHE_S3_KEY_PREFIX: "sccache"

        run: |
          archery docker run \
            -e ARROW_JAVA_BUILD=OFF \
            -e ARROW_JAVA_TEST=OFF \
            java-jni-manylinux-2014
      - name: Compress into single artifact to keep directory structure
        run: tar -cvzf arrow-shared-libs-linux-${{ matrix.platform.arch }}.tar.gz arrow/java-dist/
      - name: Upload artifacts
        uses: actions/upload-artifact@v2
        with:
          name: ubuntu-shared-lib-${{ matrix.platform.arch }}
          path: arrow-shared-libs-linux-${{ matrix.platform.arch }}.tar.gz

  build-cpp-macos:
    name: Build C++ libraries macOS ${{ matrix.platform.arch }}
    runs-on: ${{ matrix.platform.runs_on }}
    strategy:
      fail-fast: false
      matrix:
        platform:
          - { runs_on: ["macos-latest"], arch: "x86_64"}
          - { runs_on: ["macos-14"], arch: "aarch_64" }
    env:
      MACOSX_DEPLOYMENT_TARGET: "10.15"
    steps:
      - name: Checkout Arrow
        uses: actions/checkout@v4
        with:
          fetch-depth: 1
          path: arrow
          repository: lidavidm/arrow
          ref: 1070ea5b0c149a5eb271c2ab47ec3a0f82a70c77
          submodules: recursive

      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          cache: 'pip'
          python-version: 3.12
      - name: Install Archery
        shell: bash
        run: pip install -e arrow/dev/archery[all]
      - name: Install dependencies
        run: |
          # We want to use llvm@14 to avoid shared z3
          # dependency. llvm@14 doesn't depend on z3 and llvm depends
          # on z3. And Homebrew's z3 provides only shared library. It
          # doesn't provides static z3 because z3's CMake doesn't accept
          # building both shared and static libraries at once.
          # See also: Z3_BUILD_LIBZ3_SHARED in
          # https://github.com/Z3Prover/z3/blob/master/README-CMake.md
          #
          # If llvm is installed, Apache Arrow C++ uses llvm rather than
          # llvm@14 because llvm is newer than llvm@14.
          brew uninstall llvm || :

          brew bundle --file=arrow/cpp/Brewfile
          # We want to link aws-sdk-cpp statically but Homebrew's
          # aws-sdk-cpp provides only shared library. If we have
          # Homebrew's aws-sdk-cpp, our build mix Homebrew's
          # aws-sdk-cpp and bundled aws-sdk-cpp. We uninstall Homebrew's
          # aws-sdk-cpp to ensure using only bundled aws-sdk-cpp.
          brew uninstall aws-sdk-cpp
          # We want to use bundled RE2 for static linking. If
          # Homebrew's RE2 is installed, its header file may be used.
          # We uninstall Homebrew's RE2 to ensure using bundled RE2.
          brew uninstall grpc || : # gRPC depends on RE2
          brew uninstall grpc@1.54 || : # gRPC 1.54 may be installed too
          brew uninstall re2
          # We want to use bundled Protobuf for static linking. If
          # Homebrew's Protobuf is installed, its library file may be
          # used on test  We uninstall Homebrew's Protobuf to ensure using
          # bundled Protobuf.
          brew uninstall protobuf

          brew bundle --file=arrow/java/Brewfile
      - name: Build C++ libraries
        env:
          AWS_ACCESS_KEY_ID: "${{ secrets.AWS_ACCESS_KEY_ID }}"
          AWS_SECRET_ACCESS_KEY: "${{ secrets.AWS_SECRET_ACCESS_KEY }}"
          SCCACHE_BUCKET: "${{ secrets.SCCACHE_BUCKET }}"
          SCCACHE_REGION: "${{ secrets.SCCACHE_REGION }}"
          SCCACHE_S3_KEY_PREFIX: "sccache"

        run: |
          set -e
          # make brew Java available to CMake
          export JAVA_HOME=$(brew --prefix openjdk@11)/libexec/openjdk.jdk/Contents/Home
          arrow/ci/scripts/java_jni_macos_build.sh \
            $GITHUB_WORKSPACE/arrow \
            $GITHUB_WORKSPACE/arrow/cpp-build \
            $GITHUB_WORKSPACE/arrow/java-dist
      - name: Compress into single artifact to keep directory structure
        run: tar -cvzf arrow-shared-libs-macos-${{ matrix.platform.arch }}.tar.gz arrow/java-dist/
      - name: Upload artifacts
        uses: actions/upload-artifact@v2
        with:
          name: macos-shared-lib-${{ matrix.platform.arch }}
          path: arrow-shared-libs-macos-${{ matrix.platform.arch }}.tar.gz

  build-cpp-windows:
    name: Build C++ libraries Windows
    runs-on: windows-2019
    steps:
      - name: Checkout Arrow
        uses: actions/checkout@v4
        with:
          fetch-depth: 1
          path: arrow
          repository: lidavidm/arrow
          ref: 1070ea5b0c149a5eb271c2ab47ec3a0f82a70c77
          submodules: recursive

      - name: Set up Java
        uses: actions/setup-java@v3
        with:
          java-version: '11'
          distribution: 'temurin'
      - name: Download Timezone Database
        shell: bash
        run: arrow/ci/scripts/download_tz_database.sh
      - name: Install sccache
        shell: bash
        run: arrow/ci/scripts/install_sccache.sh pc-windows-msvc $(pwd)/sccache
      - name: Build C++ libraries
        shell: cmd
        env:
          AWS_ACCESS_KEY_ID: "${{ secrets.AWS_ACCESS_KEY_ID }}"
          AWS_SECRET_ACCESS_KEY: "${{ secrets.AWS_SECRET_ACCESS_KEY }}"
          SCCACHE_BUCKET: "${{ secrets.SCCACHE_BUCKET }}"
          SCCACHE_REGION: "${{ secrets.SCCACHE_REGION }}"
          SCCACHE_S3_KEY_PREFIX: "sccache"

        run: |
          call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
          REM For ORC
          set TZDIR=/c/msys64/usr/share/zoneinfo
          bash -c "arrow/ci/scripts/java_jni_windows_build.sh $(pwd)/arrow $(pwd)/arrow/cpp-build $(pwd)/arrow/java-dist"
      - name: Compress into single artifact to keep directory structure
        shell: bash
        run: tar -cvzf arrow-shared-libs-windows.tar.gz arrow/java-dist/
      - name: Upload artifacts
        uses: actions/upload-artifact@v2
        with:
          name: windows-shared-lib
          path: arrow-shared-libs-windows.tar.gz

  package-jars:
    name: Build jar files
    runs-on: macos-latest
    needs:
      - build-cpp-ubuntu
      - build-cpp-macos
      - build-cpp-windows
    steps:
      - name: Checkout Arrow
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          path: arrow
          repository: lidavidm/arrow
          ref: 1070ea5b0c149a5eb271c2ab47ec3a0f82a70c77
          submodules: recursive

      - name: Download Libraries
        uses: actions/download-artifact@v3
        with:
          path: artifacts
      - name: Decompress artifacts
        run: |
          mv artifacts/*/*.tar.gz .
          tar -xvzf arrow-shared-libs-linux-x86_64.tar.gz
          tar -xvzf arrow-shared-libs-linux-aarch_64.tar.gz
          tar -xvzf arrow-shared-libs-macos-x86_64.tar.gz
          tar -xvzf arrow-shared-libs-macos-aarch_64.tar.gz
          tar -xvzf arrow-shared-libs-windows.tar.gz
      - name: Test that shared libraries exist
        run: |
          set -x

          test -f arrow/java-dist/arrow_cdata_jni/x86_64/libarrow_cdata_jni.so
          test -f arrow/java-dist/arrow_dataset_jni/x86_64/libarrow_dataset_jni.so
          test -f arrow/java-dist/arrow_orc_jni/x86_64/libarrow_orc_jni.so
          test -f arrow/java-dist/gandiva_jni/x86_64/libgandiva_jni.so

          test -f arrow/java-dist/arrow_cdata_jni/aarch_64/libarrow_cdata_jni.so
          test -f arrow/java-dist/arrow_dataset_jni/aarch_64/libarrow_dataset_jni.so
          test -f arrow/java-dist/arrow_orc_jni/aarch_64/libarrow_orc_jni.so
          test -f arrow/java-dist/gandiva_jni/aarch_64/libgandiva_jni.so

          test -f arrow/java-dist/arrow_cdata_jni/x86_64/libarrow_cdata_jni.dylib
          test -f arrow/java-dist/arrow_dataset_jni/x86_64/libarrow_dataset_jni.dylib
          test -f arrow/java-dist/arrow_orc_jni/x86_64/libarrow_orc_jni.dylib
          test -f arrow/java-dist/gandiva_jni/x86_64/libgandiva_jni.dylib

          test -f arrow/java-dist/arrow_cdata_jni/aarch_64/libarrow_cdata_jni.dylib
          test -f arrow/java-dist/arrow_dataset_jni/aarch_64/libarrow_dataset_jni.dylib
          test -f arrow/java-dist/arrow_orc_jni/aarch_64/libarrow_orc_jni.dylib
          test -f arrow/java-dist/gandiva_jni/aarch_64/libgandiva_jni.dylib

          test -f arrow/java-dist/arrow_cdata_jni/x86_64/arrow_cdata_jni.dll
          test -f arrow/java-dist/arrow_dataset_jni/x86_64/arrow_dataset_jni.dll
          test -f arrow/java-dist/arrow_orc_jni/x86_64/arrow_orc_jni.dll
      - name: Build bundled jar
        run: |
          set -e
          pushd arrow/java
          mvn versions:set -DnewVersion=16.0.0-SNAPSHOT
          mvn versions:set -DnewVersion=16.0.0-SNAPSHOT -f bom
          mvn versions:set -DnewVersion=16.0.0-SNAPSHOT -f maven
          popd
          arrow/ci/scripts/java_full_build.sh \
            $GITHUB_WORKSPACE/arrow \
            $GITHUB_WORKSPACE/arrow/java-dist
      - name: Set up Python by actions/setup-python
        if: runner.arch == 'X64'
        uses: actions/setup-python@v4
        with:
          python-version: 3.12
      - name: Set up Python by apt
        if: runner.os == 'Linux' && runner.arch != 'X64'
        run: |
          sudo apt update
          sudo apt install -y \
            libgit2-dev \
            libpython3-dev \
            python3-pip
          sudo python3 -m pip install --upgrade pip
      - name: Checkout Crossbow
        uses: actions/checkout@v4
        with:
          path: crossbow
          ref: actions-f3f687f53c
      - name: Setup Crossbow
        shell: bash
        run: |
          python3 -m pip install -e arrow/dev/archery[crossbow]
          echo "$HOME/.local/bin" >> $GITHUB_PATH
      - name: Upload artifacts
        shell: bash
        run: |
          archery crossbow \
            --queue-path $(pwd)/crossbow \
            --queue-remote https://github.com/ursacomputing/crossbow \
            upload-artifacts \
            --sha actions-f3f687f53c-github-java-jars \
            --tag actions-f3f687f53c-github-java-jars \
            "arrow/java-dist/*.jar" \
            "arrow/java-dist/*.json" \
            "arrow/java-dist/*.pom" \
            "arrow/java-dist/*.xml" \
            "arrow/java-dist/*.zip" 
        env:
          CROSSBOW_GITHUB_TOKEN: ${{ secrets.CROSSBOW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}
      - name: Verify uploaded artifacts
        shell: bash
        run: |
          archery crossbow \
            --queue-path $(pwd)/crossbow \
            --queue-remote https://github.com/ursacomputing/crossbow \
            status \
            --task-filter 'java-jars' \
            --no-fetch \
            --validate \
            actions-f3f687f53c
        env:
          CROSSBOW_GITHUB_TOKEN: ${{ secrets.CROSSBOW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}