cpp.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

name: C++

on:
  push:
    paths:
      - '.github/workflows/cpp.yml'
      - 'ci/docker/**'
      - 'ci/scripts/cpp_*'
      - 'ci/scripts/install_gcs_testbench.sh'
      - 'ci/scripts/install_minio.sh'
      - 'ci/scripts/msys2_*'
      - 'ci/scripts/util_*'
      - 'cpp/**'
      - 'format/Flight.proto'
  pull_request:
    paths:
      - '.github/workflows/cpp.yml'
      - 'ci/docker/**'
      - 'ci/scripts/cpp_*'
      - 'ci/scripts/install_gcs_testbench.sh'
      - 'ci/scripts/install_minio.sh'
      - 'ci/scripts/msys2_*'
      - 'ci/scripts/util_*'
      - 'cpp/**'
      - 'format/Flight.proto'

concurrency:
  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
  cancel-in-progress: true

env:
  ARROW_ENABLE_TIMING_TESTS: OFF
  DOCKER_VOLUME_PREFIX: ".docker/"
  ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
  ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}

jobs:
  docker:
    name: ${{ matrix.title }}
    runs-on: ubuntu-latest
    if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
    timeout-minutes: 60
    strategy:
      fail-fast: false
      matrix:
        image:
          - conda-cpp
          - ubuntu-cpp-sanitizer
        include:
          - image: conda-cpp
            title: AMD64 Conda C++
          - image: ubuntu-cpp-sanitizer
            title: AMD64 Ubuntu 20.04 C++ ASAN UBSAN
    steps:
      - name: Checkout Arrow
        uses: actions/checkout@v3
        with:
          fetch-depth: 0
          submodules: recursive
      - name: Cache Docker Volumes
        uses: actions/cache@v2
        with:
          path: .docker
          key: ${{ matrix.image }}-${{ hashFiles('cpp/**') }}
          restore-keys: ${{ matrix.image }}-
      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: 3.8
      - name: Setup Archery
        run: pip install -e dev/archery[docker]
      - name: Execute Docker Build
        run: |
          sudo sysctl -w kernel.core_pattern="core.%e.%p"
          ulimit -c unlimited
          archery docker run ${{ matrix.image }}
      - name: Docker Push
        if: success() && github.event_name == 'push' && github.repository == 'apache/arrow'
        continue-on-error: true
        run: archery docker push ${{ matrix.image }}

  build-example:
    name: C++ Minimal Build Example
    runs-on: ubuntu-latest
    if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
    timeout-minutes: 45
    strategy:
      fail-fast: false
    steps:
      - name: Checkout Arrow
        uses: actions/checkout@v3
        with:
          fetch-depth: 0
          submodules: recursive
      - name: Check CMake presets
        run: |
          cd cpp
          cmake --list-presets
      - name: Run minimal example
        run: |
          cd cpp/examples/minimal_build
          docker-compose run --rm minimal

  macos:
    name: AMD64 macOS 11 C++
    runs-on: macos-latest
    if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
    timeout-minutes: 60
    strategy:
      fail-fast: false
    env:
      ARROW_BUILD_TESTS: ON
      ARROW_DATASET: ON
      ARROW_FLIGHT: ON
      ARROW_GANDIVA: ON
      ARROW_GCS: ON
      ARROW_HDFS: ON
      ARROW_HOME: /usr/local
      ARROW_JEMALLOC: ON
      # TODO(kszucs): link error in the tests
      ARROW_ORC: OFF
      ARROW_PARQUET: ON
      ARROW_PLASMA: ON
      ARROW_S3: ON
      ARROW_WITH_BROTLI: ON
      ARROW_WITH_BZ2: ON
      ARROW_WITH_LZ4: ON
      ARROW_WITH_OPENTELEMETRY: ON
      ARROW_WITH_SNAPPY: ON
      ARROW_WITH_ZLIB: ON
      ARROW_WITH_ZSTD: ON
      # System Abseil installed by Homebrew uses C++ 17
      CMAKE_CXX_STANDARD: 17
    steps:
      - name: Checkout Arrow
        uses: actions/checkout@v3
        with:
          fetch-depth: 0
          submodules: recursive
      - name: Install Dependencies
        run: |
          rm -f /usr/local/bin/2to3
          brew update --preinstall
          brew install --overwrite git
          brew bundle --file=cpp/Brewfile
      - name: Install MinIO
        run: |
          $(brew --prefix bash)/bin/bash \
            ci/scripts/install_minio.sh latest /usr/local
      - name: Install Google Cloud Storage Testbench
        run: ci/scripts/install_gcs_testbench.sh default
      - name: Setup ccache
        run: |
          ci/scripts/ccache_setup.sh
      - name: ccache info
        id: ccache-info
        run: |
          echo "::set-output name=cache-dir::$(ccache --get-config cache_dir)"
      - name: Cache ccache
        uses: actions/cache@v2
        with:
          path: ${{ steps.ccache-info.outputs.cache-dir }}
          key: cpp-ccache-macos-${{ hashFiles('cpp/**') }}
          restore-keys: cpp-ccache-macos-
      - name: Build
        run: ci/scripts/cpp_build.sh $(pwd) $(pwd)/build
      - name: Test
        shell: bash
        run: |
          sudo sysctl -w kern.coredump=1
          sudo sysctl -w kern.corefile=core.%N.%P
          ulimit -c unlimited  # must enable within the same shell
          ci/scripts/cpp_test.sh $(pwd) $(pwd)/build

  windows:
    name: AMD64 ${{ matrix.name }} C++17
    runs-on: ${{ matrix.os }}
    if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
    timeout-minutes: 60
    strategy:
      fail-fast: false
      matrix:
        os:
          - windows-2019
        include:
          - os: windows-2019
            name: Windows 2019
    env:
      ARROW_BOOST_USE_SHARED: OFF
      ARROW_BUILD_BENCHMARKS: ON
      ARROW_BUILD_SHARED: ON
      ARROW_BUILD_STATIC: OFF
      ARROW_BUILD_TESTS: ON
      ARROW_DATASET: ON
      ARROW_FLIGHT: OFF
      ARROW_HDFS: ON
      ARROW_HOME: /usr
      ARROW_JEMALLOC: OFF
      ARROW_MIMALLOC: ON
      ARROW_PARQUET: ON
      ARROW_USE_GLOG: OFF
      ARROW_VERBOSE_THIRDPARTY_BUILD: OFF
      ARROW_WITH_BROTLI: OFF
      ARROW_WITH_BZ2: OFF
      ARROW_WITH_LZ4: OFF
      ARROW_WITH_OPENTELEMETRY: OFF
      ARROW_WITH_SNAPPY: ON
      ARROW_WITH_ZLIB: ON
      ARROW_WITH_ZSTD: ON
      BOOST_SOURCE: BUNDLED
      CMAKE_CXX_STANDARD: "17"
      CMAKE_GENERATOR: Ninja
      CMAKE_INSTALL_LIBDIR: bin
      CMAKE_INSTALL_PREFIX: /usr
      CMAKE_UNITY_BUILD: ON
      OPENSSL_ROOT_DIR: >-
        C:\Program Files\OpenSSL-Win64
      NPROC: 3
    steps:
      - name: Disable Crash Dialogs
        run: |
          reg add `
            "HKCU\SOFTWARE\Microsoft\Windows\Windows Error Reporting" `
            /v DontShowUI `
            /t REG_DWORD `
            /d 1 `
            /f
      - name: Installed Packages
        run: choco list -l
      - name: Install Dependencies
        run: choco install -y --no-progress openssl
      - name: Checkout Arrow
        uses: actions/checkout@v3
        with:
          fetch-depth: 0
          submodules: recursive
      - name: Download Timezone Database
        shell: bash
        run: ci/scripts/download_tz_database.sh
      - name: Install ccache
        shell: bash
        run: |
          ci/scripts/install_ccache.sh 4.6.2 /usr
      - name: Setup ccache
        shell: bash
        run: |
          ci/scripts/ccache_setup.sh
      - name: ccache info
        id: ccache-info
        shell: bash
        run: |
          echo "::set-output name=cache-dir::$(ccache --get-config cache_dir)"
      - name: Cache ccache
        uses: actions/cache@v2
        with:
          path: ${{ steps.ccache-info.outputs.cache-dir }}
          key: cpp-ccache-windows-${{ hashFiles('cpp/**') }}
          restore-keys: cpp-ccache-windows-
      - name: Build
        shell: cmd
        run: |
          call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
          bash -c "ci/scripts/cpp_build.sh $(pwd) $(pwd)/build"
      - name: Test
        shell: bash
        run: ci/scripts/cpp_test.sh $(pwd) $(pwd)/build

  windows-mingw:
    name: AMD64 Windows MinGW ${{ matrix.mingw-n-bits }} C++
    runs-on: windows-2019
    if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
    # Build may take 1h+ without cache and installing Google Cloud
    # Storage Testbench may take 20m+ without cache.
    timeout-minutes: 120
    strategy:
      fail-fast: false
      matrix:
        mingw-n-bits:
          - 32
          - 64
    env:
      ARROW_BUILD_SHARED: ON
      ARROW_BUILD_STATIC: OFF
      ARROW_BUILD_TESTS: ON
      ARROW_BUILD_TYPE: release
      ARROW_DATASET: ON
      ARROW_FLIGHT: ON
      ARROW_FLIGHT_SQL: ON
      ARROW_GANDIVA: ON
      ARROW_GCS: ON
      ARROW_HDFS: OFF
      ARROW_HOME: /mingw${{ matrix.mingw-n-bits }}
      ARROW_JEMALLOC: OFF
      ARROW_PARQUET: ON
      ARROW_PYTHON: ON
      ARROW_S3: ON
      ARROW_USE_GLOG: OFF
      ARROW_VERBOSE_THIRDPARTY_BUILD: OFF
      ARROW_WITH_BROTLI: ON
      ARROW_WITH_BZ2: ON
      ARROW_WITH_LZ4: ON
      ARROW_WITH_OPENTELEMETRY: OFF
      ARROW_WITH_SNAPPY: ON
      ARROW_WITH_ZLIB: ON
      ARROW_WITH_ZSTD: ON
      # Don't use preinstalled Boost by empty BOOST_ROOT and
      # -DBoost_NO_BOOST_CMAKE=ON
      BOOST_ROOT: ""
      CMAKE_ARGS: >-
        -DARROW_PACKAGE_PREFIX=/mingw${{ matrix.mingw-n-bits }}
        -DBoost_NO_BOOST_CMAKE=ON
      # We can't use unity build because we don't have enough memory on
      # GitHub Actions.
      # CMAKE_UNITY_BUILD: ON
    steps:
      - name: Disable Crash Dialogs
        run: |
          reg add `
            "HKCU\SOFTWARE\Microsoft\Windows\Windows Error Reporting" `
            /v DontShowUI `
            /t REG_DWORD `
            /d 1 `
            /f
      - name: Checkout Arrow
        uses: actions/checkout@v3
        with:
          fetch-depth: 0
          submodules: recursive
      - uses: msys2/setup-msys2@v2
        with:
          msystem: MINGW${{ matrix.mingw-n-bits }}
          update: true
      - name: Setup MSYS2
        shell: msys2 {0}
        run: ci/scripts/msys2_setup.sh cpp
      - name: Cache ccache
        uses: actions/cache@v2
        with:
          path: ccache
          key: cpp-ccache-mingw${{ matrix.mingw-n-bits }}-${{ hashFiles('cpp/**') }}
          restore-keys: cpp-ccache-mingw${{ matrix.mingw-n-bits }}-
      - name: Build
        shell: msys2 {0}
        run: |
          export CMAKE_BUILD_PARALLEL_LEVEL=$NUMBER_OF_PROCESSORS
          ci/scripts/cpp_build.sh "$(pwd)" "$(pwd)/build"
      - name: Download Timezone Database
        shell: bash
        run: ci/scripts/download_tz_database.sh
      - name: Download MinIO
        shell: msys2 {0}
        run: |
          mkdir -p /usr/local/bin
          wget \
            --output-document /usr/local/bin/minio.exe \
            https://dl.min.io/server/minio/release/windows-amd64/archive/minio.RELEASE.2022-05-26T05-48-41Z
          chmod +x /usr/local/bin/minio.exe
      - name: Cache Python wheels
        uses: actions/cache@v2
        with:
          path: "${{ env.PIP_CACHE_DIR }}"
          key: cpp-wheels-mingw${{ matrix.mingw-n-bits }}-${{ hashFiles('ci/scripts/install_gcs_testbench.sh') }}
          restore-keys: cpp-wheels-mingw${{ matrix.mingw-n-bits }}-
      - name: Install Google Cloud Storage Testbench
        shell: msys2 {0}
        run: |
          ci/scripts/install_gcs_testbench.sh default
      - name: Test
        shell: msys2 {0}
        run: |
          python_version=$(python3 -c "import sys; print('.'.join(map(str, sys.version_info[0:2])))")
          export PYTHONHOME="$(cygpath --windows ${MINGW_PREFIX})\lib\python${python_version}"
          PYTHONPATH="${PYTHONHOME}"
          PYTHONPATH="${PYTHONPATH};${PYTHONHOME}\lib-dynload"
          PYTHONPATH="${PYTHONPATH};${PYTHONHOME}\site-packages"
          export PYTHONPATH
          ci/scripts/cpp_test.sh "$(pwd)" "$(pwd)/build"