diff --git a/.github/workflows/archery.yml b/.github/workflows/archery.yml deleted file mode 100644 index fa84f4dc3b08b..0000000000000 --- a/.github/workflows/archery.yml +++ /dev/null @@ -1,70 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Archery & Crossbow - -on: - push: - paths: - - '.github/workflows/archery.yml' - - 'dev/archery/**' - - 'dev/tasks/**' - - 'docker-compose.yml' - pull_request: - paths: - - '.github/workflows/archery.yml' - - 'dev/archery/**' - - 'dev/tasks/**' - - 'docker-compose.yml' - -concurrency: - group: ${{ github.repository }}-${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: true - -jobs: - - test: - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - name: Archery Unittests and Crossbow Check Config - runs-on: ubuntu-latest - timeout-minutes: 15 - steps: - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Git Fixup - shell: bash - run: git branch master origin/master || true - - name: Free Up Disk Space - run: ci/scripts/util_cleanup.sh - - name: Setup Python - uses: actions/setup-python@v1 - with: - python-version: '3.6' - - name: Install pygit2 binary wheel - run: pip install pygit2 --only-binary pygit2 - - name: Install Archery, Crossbow- and Test Dependencies - run: pip install pytest responses -e dev/archery[all] - - name: Archery Unittests - working-directory: dev/archery - run: pytest -v archery - - name: Archery Docker Validation - run: archery docker check-config - - name: Crossbow Check Config - working-directory: dev/tasks - run: archery crossbow check-config diff --git a/.github/workflows/comment_bot.yml b/.github/workflows/comment_bot.yml deleted file mode 100644 index b78ae80fb9774..0000000000000 --- a/.github/workflows/comment_bot.yml +++ /dev/null @@ -1,170 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Comment Bot - -on: - # TODO(kszucs): support pull_request_review_comment - issue_comment: - types: - - created - - edited - -jobs: - crossbow: - name: Listen! - if: startsWith(github.event.comment.body, '@github-actions crossbow') - runs-on: ubuntu-latest - steps: - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - path: arrow - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - name: Install Archery and Crossbow dependencies - run: pip install -e arrow/dev/archery[bot] - - name: Handle Github comment event - env: - ARROW_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - CROSSBOW_GITHUB_TOKEN: ${{ secrets.CROSSBOW_GITHUB_TOKEN }} - run: | - archery trigger-bot \ - --event-name ${{ github.event_name }} \ - --event-payload ${{ github.event_path }} - - autotune: - name: "Fix all the things" - if: startsWith(github.event.comment.body, '@github-actions autotune') - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: r-lib/actions/pr-fetch@master - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: See what is different - run: | - set -ex - git remote add upstream https://github.com/apache/arrow - git fetch upstream - changed() { - git diff --name-only HEAD..upstream/master | grep -e "$1" >/dev/null 2>&1 - } - if changed '^r/.*\.R$'; then - echo "R_DOCS=true" >> $GITHUB_ENV - echo "R_CODE=true" >> $GITHUB_ENV - fi - if changed 'cmake' || changed 'CMake'; then - echo "CMAKE_FORMAT=true" >> $GITHUB_ENV - fi - if changed '^cpp/src'; then - echo "CLANG_FORMAT_CPP=true" >> $GITHUB_ENV - fi - if changed '^r/src'; then - echo "CLANG_FORMAT_R=true" >> $GITHUB_ENV - fi - - name: Ensure clang-format has the appropriate versoin - if: env.CMAKE_FORMAT == 'true' || - env.CLANG_FORMAT_CPP == 'true' || - env.CLANG_FORMAT_R == 'true' || - endsWith(github.event.comment.body, 'everything') - run: | - set -e - . .env # To get the clang version we use - sudo apt update - sudo apt install -y clang-format-${CLANG_TOOLS} - - name: Run cmake_format - if: env.CMAKE_FORMAT == 'true' || endsWith(github.event.comment.body, 'everything') - run: | - set -ex - export PATH=/home/runner/.local/bin:$PATH - python3 -m pip install --upgrade pip setuptools wheel - python3 -m pip install -e dev/archery[lint] - archery lint --cmake-format --fix - - name: Run clang-format on cpp - if: env.CLANG_FORMAT_CPP == 'true' || endsWith(github.event.comment.body, 'everything') - run: | - . .env # To get the clang version we use - cpp/build-support/run_clang_format.py \ - --clang_format_binary=clang-format-${CLANG_TOOLS} \ - --exclude_glob=cpp/build-support/lint_exclusions.txt \ - --source_dir=cpp/src --quiet --fix - - name: Run clang-format on r - if: env.CLANG_FORMAT_R == 'true' || endsWith(github.event.comment.body, 'everything') - run: | - . .env # To get the clang version we use - cpp/build-support/run_clang_format.py \ - --clang_format_binary=clang-format-${CLANG_TOOLS} \ - --exclude_glob=cpp/build-support/lint_exclusions.txt \ - --source_dir=r/src --quiet --fix - - uses: r-lib/actions/setup-r@v1 - if: env.R_DOCS == 'true' || env.R_CODE == 'true' || endsWith(github.event.comment.body, 'everything') - - name: Update R docs - if: env.R_DOCS == 'true' || endsWith(github.event.comment.body, 'everything') - shell: Rscript {0} - run: | - source("ci/etc/rprofile") - install.packages(c("remotes", "roxygen2")) - # We currently need dev roxygen2 (> 7.1.1) until they release - remotes::install_github("r-lib/roxygen2") - remotes::install_deps("r") - roxygen2::roxygenize("r") - - name: Style R code - if: env.R_CODE == 'true' || endsWith(github.event.comment.body, 'everything') - shell: Rscript {0} - run: | - changed_files <- system("git diff --name-only HEAD..upstream/master 2>&1", intern = TRUE) - # only grab the .R files under r/ - changed_files <- grep('^r/.*\\.R$', changed_files, value = TRUE) - # remove latin1 which is unstylable due to encoding and codegen.R which is unique - changed_files <- changed_files[!changed_files %in% file.path("r", source("r/.styler_excludes.R")$value)] - source("ci/etc/rprofile") - install.packages(c("remotes", "styler")) - remotes::install_deps("r") - styler::style_file(changed_files) - - name: Commit results - run: | - git config user.name "$(git log -1 --pretty=format:%an)" - git config user.email "$(git log -1 --pretty=format:%ae)" - git commit -a -m 'Autoformat/render all the things [automated commit]' || echo "No changes to commit" - - uses: r-lib/actions/pr-push@master - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - - rebase: - name: "Rebase" - if: startsWith(github.event.comment.body, '@github-actions rebase') - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: r-lib/actions/pr-fetch@master - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: Rebase on ${{ github.repository }} master - run: | - set -ex - git config user.name "$(git log -1 --pretty=format:%an)" - git config user.email "$(git log -1 --pretty=format:%ae)" - git remote add upstream https://github.com/${{ github.repository }} - git fetch --unshallow upstream master - git rebase upstream/master - - uses: r-lib/actions/pr-push@master - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - args: "--force" diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml deleted file mode 100644 index 0f19f7351c325..0000000000000 --- a/.github/workflows/cpp.yml +++ /dev/null @@ -1,329 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: C++ - -on: - push: - paths: - - '.github/workflows/cpp.yml' - - 'ci/docker/**' - - 'ci/scripts/cpp_*' - - 'ci/scripts/msys2_*' - - 'ci/scripts/util_*' - - 'cpp/**' - - 'format/Flight.proto' - pull_request: - paths: - - '.github/workflows/cpp.yml' - - 'ci/docker/**' - - 'ci/scripts/cpp_*' - - 'ci/scripts/msys2_*' - - 'ci/scripts/util_*' - - 'cpp/**' - - 'format/Flight.proto' - -concurrency: - group: ${{ github.repository }}-${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: true - -env: - ARROW_ENABLE_TIMING_TESTS: OFF - DOCKER_VOLUME_PREFIX: ".docker/" - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - -jobs: - - docker: - name: ${{ matrix.title }} - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 45 - strategy: - fail-fast: false - matrix: - image: - - conda-cpp - - ubuntu-cpp-sanitizer - include: - - image: conda-cpp - title: AMD64 Conda C++ - - image: ubuntu-cpp-sanitizer - title: AMD64 Ubuntu 20.04 C++ ASAN UBSAN - steps: - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - run: ci/scripts/util_checkout.sh - - name: Free Up Disk Space - run: ci/scripts/util_cleanup.sh - - name: Cache Docker Volumes - uses: actions/cache@v2 - with: - path: .docker - key: ${{ matrix.image }}-${{ hashFiles('cpp/**') }} - restore-keys: ${{ matrix.image }}- - - name: Setup Python - uses: actions/setup-python@v1 - with: - python-version: 3.8 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - run: | - sudo sysctl -w kernel.core_pattern="core.%e.%p" - ulimit -c unlimited - archery docker run ${{ matrix.image }} - - name: Docker Push - if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' - continue-on-error: true - run: archery docker push ${{ matrix.image }} - - build-example: - name: C++ Minimal Build Example - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 45 - strategy: - fail-fast: false - steps: - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Run - run: | - cd cpp/examples/minimal_build - docker-compose run --rm minimal - - macos: - name: AMD64 MacOS 10.15 C++ - runs-on: macos-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 45 - strategy: - fail-fast: false - env: - ARROW_BUILD_TESTS: ON - ARROW_DATASET: ON - ARROW_FLIGHT: ON - ARROW_GANDIVA: ON - ARROW_HDFS: ON - ARROW_HOME: /usr/local - ARROW_JEMALLOC: ON - # TODO(kszucs): link error in the tests - ARROW_ORC: OFF - ARROW_PARQUET: ON - ARROW_PLASMA: ON - ARROW_S3: ON - ARROW_WITH_BROTLI: ON - ARROW_WITH_BZ2: ON - ARROW_WITH_LZ4: ON - ARROW_WITH_SNAPPY: ON - ARROW_WITH_ZLIB: ON - ARROW_WITH_ZSTD: ON - steps: - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh - - name: Install Dependencies - shell: bash - run: | - rm -f /usr/local/bin/2to3 - brew update --preinstall - brew bundle --file=cpp/Brewfile - - name: Build - shell: bash - run: ci/scripts/cpp_build.sh $(pwd) $(pwd)/build - - name: Test - shell: bash - run: | - sudo sysctl -w kern.coredump=1 - sudo sysctl -w kern.corefile=core.%N.%P - ulimit -c unlimited # must enable within the same shell - ci/scripts/cpp_test.sh $(pwd) $(pwd)/build - - windows: - name: AMD64 ${{ matrix.name }} C++ - runs-on: ${{ matrix.os }} - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 45 - strategy: - fail-fast: false - matrix: - os: - - windows-latest - include: - - os: windows-latest - name: Windows 2019 - generator: Visual Studio 16 2019 - env: - ARROW_BOOST_USE_SHARED: OFF - ARROW_BUILD_BENCHMARKS: ON - ARROW_BUILD_SHARED: ON - ARROW_BUILD_STATIC: OFF - ARROW_BUILD_TESTS: ON - ARROW_DATASET: ON - ARROW_FLIGHT: OFF - ARROW_HDFS: ON - ARROW_HOME: /usr - ARROW_JEMALLOC: OFF - ARROW_MIMALLOC: ON - ARROW_PARQUET: ON - ARROW_USE_GLOG: OFF - ARROW_VERBOSE_THIRDPARTY_BUILD: OFF - ARROW_WITH_BROTLI: OFF - ARROW_WITH_BZ2: OFF - ARROW_WITH_LZ4: OFF - ARROW_WITH_SNAPPY: ON - ARROW_WITH_ZLIB: ON - ARROW_WITH_ZSTD: ON - BOOST_SOURCE: BUNDLED - CMAKE_ARGS: '-A x64 -DOPENSSL_ROOT_DIR=C:\Program Files\OpenSSL-Win64' - CMAKE_GENERATOR: ${{ matrix.generator }} - CMAKE_INSTALL_LIBDIR: bin - CMAKE_INSTALL_PREFIX: /usr - CMAKE_UNITY_BUILD: ON - NPROC: 2 - steps: - - name: Disable Crash Dialogs - run: | - reg add ` - "HKCU\SOFTWARE\Microsoft\Windows\Windows Error Reporting" ` - /v DontShowUI ` - /t REG_DWORD ` - /d 1 ` - /f - - name: Installed Packages - run: choco list -l - - name: Install Dependencies - run: choco install -y --no-progress openssl - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh - - name: Build - shell: bash - run: | - ci/scripts/cpp_build.sh $(pwd) $(pwd)/build - - name: Test - shell: bash - run: ci/scripts/cpp_test.sh $(pwd) $(pwd)/build - - windows-mingw: - name: AMD64 Windows MinGW ${{ matrix.mingw-n-bits }} C++ - runs-on: windows-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - mingw-n-bits: - - 32 - - 64 - env: - ARROW_BUILD_SHARED: ON - ARROW_BUILD_STATIC: OFF - ARROW_BUILD_TESTS: ON - ARROW_BUILD_TYPE: release - ARROW_DATASET: ON - ARROW_FLIGHT: ON - ARROW_GANDIVA: ON - ARROW_HDFS: OFF - ARROW_HOME: /mingw${{ matrix.mingw-n-bits }} - ARROW_JEMALLOC: OFF - ARROW_PARQUET: ON - ARROW_PYTHON: ON - ARROW_S3: ON - ARROW_USE_GLOG: OFF - ARROW_VERBOSE_THIRDPARTY_BUILD: OFF - ARROW_WITH_BROTLI: ON - ARROW_WITH_BZ2: ON - ARROW_WITH_LZ4: ON - ARROW_WITH_SNAPPY: ON - ARROW_WITH_ZLIB: ON - ARROW_WITH_ZSTD: ON - # Don't use preinstalled Boost by empty BOOST_ROOT and - # -DBoost_NO_BOOST_CMAKE=ON - BOOST_ROOT: "" - CMAKE_ARGS: >- - -DARROW_PACKAGE_PREFIX=/mingw${{ matrix.mingw-n-bits }} - -DBoost_NO_BOOST_CMAKE=ON - CMAKE_UNITY_BUILD: ON - steps: - - name: Disable Crash Dialogs - run: | - reg add ` - "HKCU\SOFTWARE\Microsoft\Windows\Windows Error Reporting" ` - /v DontShowUI ` - /t REG_DWORD ` - /d 1 ` - /f - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh - - uses: msys2/setup-msys2@v2 - with: - msystem: MINGW${{ matrix.mingw-n-bits }} - update: true - - name: Setup MSYS2 - shell: msys2 {0} - run: | - ci/scripts/msys2_setup.sh cpp - - name: Cache ccache - uses: actions/cache@v2 - with: - path: ccache - key: cpp-ccache-mingw${{ matrix.mingw-n-bits }}-${{ hashFiles('cpp/**') }} - restore-keys: cpp-ccache-mingw${{ matrix.mingw-n-bits }}- - - name: Build - shell: msys2 {0} - run: | - export CMAKE_BUILD_PARALLEL_LEVEL=$NUMBER_OF_PROCESSORS - ci/scripts/cpp_build.sh "$(pwd)" "$(pwd)/build" - - name: Download MinIO - shell: msys2 {0} - run: | - mkdir -p /usr/local/bin - wget \ - --output-document /usr/local/bin/minio.exe \ - https://dl.min.io/server/minio/release/windows-amd64/minio.exe - chmod +x /usr/local/bin/minio.exe - - name: Test - shell: msys2 {0} - run: | - python_version=$(python -c "import sys; print('.'.join(map(str, sys.version_info[0:2])))") - export PYTHONHOME="$(cygpath --windows ${MINGW_PREFIX})\lib\python${python_version}" - PYTHONPATH="${PYTHONHOME}" - PYTHONPATH="${PYTHONPATH};${PYTHONHOME}\lib-dynload" - PYTHONPATH="${PYTHONPATH};${PYTHONHOME}\site-packages" - export PYTHONPATH - ci/scripts/cpp_test.sh "$(pwd)" "$(pwd)/build" diff --git a/.github/workflows/cpp_cron.yml b/.github/workflows/cpp_cron.yml deleted file mode 100644 index c031e5961cb34..0000000000000 --- a/.github/workflows/cpp_cron.yml +++ /dev/null @@ -1,85 +0,0 @@ - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: C++ Cron - -on: - push: - paths: - - '.github/workflows/cpp_cron.yml' - pull_request: - paths: - - '.github/workflows/cpp_cron.yml' - schedule: - - cron: | - 0 */12 * * * - -env: - ARROW_ENABLE_TIMING_TESTS: OFF - DOCKER_VOLUME_PREFIX: ".docker/" - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - -jobs: - - oss-fuzz: - name: OSS-Fuzz build check - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') && github.repository == 'apache/arrow' }} - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - ubuntu: [18.04] - steps: - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh - - name: Free Up Disk Space - shell: bash - run: ci/scripts/util_cleanup.sh - - name: Checkout OSS-Fuzz - uses: actions/checkout@v1 - with: - path: oss-fuzz - repository: google/oss-fuzz - ref: master - - name: Install dependencies - working-directory: ../oss-fuzz - run: | - python3 -m pip install setuptools - python3 -m pip install -r infra/ci/requirements.txt - - name: Build image - shell: bash - working-directory: ../oss-fuzz - run: | - python3 infra/helper.py build_image --pull arrow - - name: Build fuzzers - shell: bash - working-directory: ../oss-fuzz - run: | - python3 infra/helper.py build_fuzzers arrow `pwd`/../arrow - - name: Check build - shell: bash - working-directory: ../oss-fuzz - run: | - python3 infra/helper.py check_build arrow diff --git a/.github/workflows/csharp.yml b/.github/workflows/csharp.yml deleted file mode 100644 index b339b8f46555d..0000000000000 --- a/.github/workflows/csharp.yml +++ /dev/null @@ -1,128 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: C# - -on: - push: - paths: - - '.github/workflows/csharp.yml' - - 'ci/scripts/csharp_*' - - 'csharp/**' - pull_request: - paths: - - '.github/workflows/csharp.yml' - - 'ci/scripts/csharp_*' - - 'csharp/**' - -concurrency: - group: ${{ github.repository }}-${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: true - -jobs: - - ubuntu: - name: AMD64 Ubuntu 18.04 C# ${{ matrix.dotnet }} - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - dotnet: ['3.1.x'] - steps: - - name: Install C# - uses: actions/setup-dotnet@v1 - with: - dotnet-version: ${{ matrix.dotnet }} - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh - - name: Install Source Link - shell: bash - run: dotnet tool install --global sourcelink - - name: Build - shell: bash - run: ci/scripts/csharp_build.sh $(pwd) - - name: Test - shell: bash - run: ci/scripts/csharp_test.sh $(pwd) - - windows: - name: AMD64 Windows 2019 18.04 C# ${{ matrix.dotnet }} - runs-on: windows-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - dotnet: ['3.1.x'] - steps: - - name: Install C# - uses: actions/setup-dotnet@v1 - with: - dotnet-version: ${{ matrix.dotnet }} - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh - - name: Install Source Link - run: dotnet tool install --global sourcelink - - name: Build - shell: bash - run: ci/scripts/csharp_build.sh $(pwd) - - name: Test - shell: bash - run: ci/scripts/csharp_test.sh $(pwd) - - macos: - name: AMD64 MacOS 10.15 C# ${{ matrix.dotnet }} - runs-on: macos-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - dotnet: ['3.1.x'] - steps: - - name: Install C# - uses: actions/setup-dotnet@v1 - with: - dotnet-version: ${{ matrix.dotnet }} - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Install Source Link - shell: bash - run: dotnet tool install --global sourcelink - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh - - name: Build - shell: bash - run: ci/scripts/csharp_build.sh $(pwd) - - name: Test - shell: bash - run: ci/scripts/csharp_test.sh $(pwd) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml deleted file mode 100644 index 9ef46c31fa34e..0000000000000 --- a/.github/workflows/dev.yml +++ /dev/null @@ -1,100 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Dev - -on: - # always trigger - push: - pull_request: - -concurrency: - group: ${{ github.repository }}-${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: true - -env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - -jobs: - - lint: - name: Lint C++, Python, R, Docker, RAT - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - steps: - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - run: ci/scripts/util_checkout.sh - - name: Free Up Disk Space - run: ci/scripts/util_cleanup.sh - - name: Setup Python - uses: actions/setup-python@v1 - with: - python-version: 3.8 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - run: | - sudo sysctl -w kernel.core_pattern="core.%e.%p" - ulimit -c unlimited - archery docker run -e GITHUB_ACTIONS=true ubuntu-lint - - name: Docker Push - if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' - continue-on-error: true - run: archery docker push ubuntu-lint - - release: - name: Source Release and Merge Script - runs-on: ubuntu-20.04 - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - env: - GIT_AUTHOR_NAME: Github Actions - GIT_AUTHOR_EMAIL: github@actions - GIT_COMMITTER_NAME: Github Actions - GIT_COMMITTER_EMAIL: github@actions - steps: - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh - - name: Install Python - uses: actions/setup-python@v1 - with: - python-version: '3.6' - - name: Install Ruby - uses: ruby/setup-ruby@v1 - with: - ruby-version: '2.6' - - name: Install Dependencies - shell: bash - run: | - pip install cython setuptools six pytest jira - - name: Run Release Test - shell: bash - run: | - ci/scripts/release_test.sh $(pwd) - - name: Run Merge Script Test - shell: bash - run: | - pytest -v dev/test_merge_arrow_pr.py diff --git a/.github/workflows/dev_pr.yml b/.github/workflows/dev_pr.yml deleted file mode 100644 index 86a5c98f8a606..0000000000000 --- a/.github/workflows/dev_pr.yml +++ /dev/null @@ -1,99 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Dev PR - -on: - # TODO: Enable this when eps1lon/actions-label-merge-conflict is available. - # push: - # branches: - # - master - pull_request_target: - types: - - opened - - edited - - synchronize - -# NOTE: not using the "cancel-in-progress" feature here as the group key -# does not have enough information for linking it to a particular PR - -jobs: - process: - name: Process - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - - name: Comment JIRA link - if: | - github.event_name == 'pull_request_target' && - (github.event.action == 'opened' || - github.event.action == 'edited') - uses: actions/github-script@v3 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const script = require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/dev_pr/link.js`); - script({github, context}); - - - name: Check title - if: | - github.event_name == 'pull_request_target' && - (github.event.action == 'opened' || - github.event.action == 'edited') - uses: actions/github-script@v3 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const script = require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/dev_pr/title_check.js`); - script({github, context}); - - - name: Check Jira Issue - if: | - github.event_name == 'pull_request_target' && - (github.event.action == 'opened' || - github.event.action == 'edited') - uses: actions/github-script@v3 - with: - debug: true - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const script = require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/dev_pr/jira_check.js`); - script({github, context}); - - - name: Assign GitHub labels - if: | - github.event_name == 'pull_request_target' && - (github.event.action == 'opened' || - github.event.action == 'synchronize') - uses: actions/labeler@2.2.0 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - configuration-path: .github/workflows/dev_pr/labeler.yml - sync-labels: true - - # TODO: Enable this when eps1lon/actions-label-merge-conflict is available. - # - name: Checks if PR needs rebase - # if: | - # github.event_name == 'push' || - # (github.event_name == 'pull_request_target' && - # (github.event.action == 'opened' || - # github.event.action == 'synchronize')) - # uses: eps1lon/actions-label-merge-conflict@releases/2.x - # with: - # dirtyLabel: "needs-rebase" - # repoToken: "${{ secrets.GITHUB_TOKEN }}" diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml deleted file mode 100644 index 5d8034ff09f1f..0000000000000 --- a/.github/workflows/go.yml +++ /dev/null @@ -1,287 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Go - -on: - push: - paths: - - '.github/workflows/go.yml' - - 'ci/docker/*_go.dockerfile' - - 'ci/scripts/go_*' - - 'go/**' - pull_request: - paths: - - '.github/workflows/go.yml' - - 'ci/docker/*_go.dockerfile' - - 'ci/docker/**' - - 'ci/scripts/go_*' - - 'go/**' - -concurrency: - group: ${{ github.repository }}-${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: true - -env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - -jobs: - - docker: - name: AMD64 Debian 11 Go ${{ matrix.go }} - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - go: [1.15] - env: - GO: ${{ matrix.go }} - steps: - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - run: ci/scripts/util_checkout.sh - - name: Free Up Disk Space - run: ci/scripts/util_cleanup.sh - - name: Setup Python - uses: actions/setup-python@v1 - with: - python-version: 3.8 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - run: archery docker run debian-go - - name: Docker Push - if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' - continue-on-error: true - run: archery docker push debian-go - - docker_cgo: - name: AMD64 Debian 11 GO ${{ matrix.go }} - CGO - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - go: [1.15] - env: - GO: ${{ matrix.go }} - steps: - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - run: ci/scripts/util_checkout.sh - - name: Free Up Disk Space - run: ci/scripts/util_cleanup.sh - - name: Setup Python - uses: actions/setup-python@v1 - with: - python-version: 3.8 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - run: archery docker run debian-go-cgo - - name: Docker Push - if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' - continue-on-error: true - run: archery docker push debian-go-cgo - - - docker_cgo_python: - name: AMD64 Debian 11 GO ${{ matrix.go }} - CGO Python - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - go: [1.15] - env: - GO: ${{ matrix.go }} - steps: - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - run: ci/scripts/util_checkout.sh - - name: Free Up Disk Space - run: ci/scripts/util_cleanup.sh - - name: Setup Python - uses: actions/setup-python@v1 - with: - python-version: 3.8 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - run: archery docker run debian-go-cgo-python - - name: Docker Push - if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' - continue-on-error: true - run: archery docker push debian-go-cgo-python - - windows: - name: AMD64 Windows 2019 Go ${{ matrix.go }} - runs-on: windows-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - go: [1.15] - steps: - - name: Install go - uses: actions/setup-go@v1 - with: - go-version: ${{ matrix.go }} - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh - - name: Build - shell: bash - run: ci/scripts/go_build.sh . - - name: Test - shell: bash - run: ci/scripts/go_test.sh . - - macos: - name: AMD64 MacOS 10.15 Go ${{ matrix.go }} - runs-on: macos-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - go: [1.15] - steps: - - name: Install go - uses: actions/setup-go@v1 - with: - go-version: ${{ matrix.go }} - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh - - name: Build - shell: bash - run: ci/scripts/go_build.sh . - - name: Test - shell: bash - run: ci/scripts/go_test.sh . - - macos-cgo: - name: AMD64 MacOS 10.15 Go ${{ matrix.go }} - CGO - runs-on: macos-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - go: [1.15] - env: - ARROW_GO_TESTCGO: "1" - steps: - - name: Install go - uses: actions/setup-go@v1 - with: - go-version: ${{ matrix.go }} - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh - - name: Brew Install Arrow - shell: bash - run: brew install apache-arrow - - name: Build - shell: bash - run: ci/scripts/go_build.sh . - - name: Test - shell: bash - run: ci/scripts/go_test.sh . - - windows-mingw: - name: AMD64 Windows MinGW ${{ matrix.mingw-n-bits }} CGO - runs-on: windows-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - go: [1.15] - mingw-n-bits: - #- 32 runtime handling for CGO needs 64-bit currently - - 64 - env: - ARROW_GO_TESTCGO: "1" - steps: - - name: Disable Crash Dialogs - run: | - reg add ` - "HKCU\SOFTWARE\Microsoft\Windows\Windows Error Reporting" ` - /v DontShowUI ` - /t REG_DWORD ` - /d 1 ` - /f - - name: Install go - uses: actions/setup-go@v1 - with: - go-version: ${{ matrix.go }} - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh - - uses: msys2/setup-msys2@v2 - with: - msystem: MINGW${{ matrix.mingw-n-bits }} - update: true - - name: Setup MSYS2 - shell: msys2 {0} - run: | - ci/scripts/msys2_setup.sh cgo - - name: Update CGO Env vars - shell: msys2 {0} - run: | - echo "CGO_CPPFLAGS=-I$(cygpath --windows ${MINGW_PREFIX}/include)" >> $GITHUB_ENV - echo "CGO_LDFLAGS=-g -O2 -L$(cygpath --windows ${MINGW_PREFIX}/lib) -L$(cygpath --windows ${MINGW_PREFIX}/bin)" >> $GITHUB_ENV - echo "$(cygpath --windows ${MINGW_PREFIX}/bin)" >> $GITHUB_PATH - - name: Build - shell: bash - run: ci/scripts/go_build.sh . - - name: Test - shell: bash - run: ci/scripts/go_test.sh . diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml deleted file mode 100644 index 7a4deb8e3ea31..0000000000000 --- a/.github/workflows/integration.yml +++ /dev/null @@ -1,91 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Integration - -on: - push: - paths: - - '.github/workflows/integration.yml' - - 'ci/**' - - 'dev/archery/**' - - 'go/**' - - 'integration/**' - - 'js/**' - - 'cpp/**' - - 'java/**' - - 'format/**' - pull_request: - paths: - - '.github/workflows/integration.yml' - - 'ci/**' - - 'dev/archery/**' - - 'go/**' - - 'integration/**' - - 'js/**' - - 'cpp/**' - - 'java/**' - - 'format/**' - -concurrency: - group: ${{ github.repository }}-${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: true - -env: - DOCKER_VOLUME_PREFIX: ".docker/" - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - -jobs: - - docker: - name: AMD64 Conda Integration Test - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 60 - steps: - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - run: ci/scripts/util_checkout.sh - - name: Checkout Arrow Rust - uses: actions/checkout@v2 - with: - repository: apache/arrow-rs - path: rust - - name: Free Up Disk Space - run: ci/scripts/util_cleanup.sh - - name: Cache Docker Volumes - uses: actions/cache@v2 - with: - path: .docker - key: conda-${{ hashFiles('cpp/**') }} - restore-keys: conda- - - name: Setup Python - uses: actions/setup-python@v1 - with: - python-version: 3.8 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - run: archery docker run -e ARCHERY_INTEGRATION_WITH_RUST=1 conda-integration - - name: Docker Push - if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' - continue-on-error: true - run: archery docker push conda-integration diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml deleted file mode 100644 index 72f4df7e36e38..0000000000000 --- a/.github/workflows/java.yml +++ /dev/null @@ -1,118 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Java - -on: - push: - paths: - - '.github/workflows/java.yml' - - 'ci/docker/*java*' - - 'ci/scripts/java*.sh' - - 'ci/scripts/util_*.sh' - - 'format/Flight.proto' - - 'java/**' - pull_request: - paths: - - '.github/workflows/java.yml' - - 'ci/docker/*java*' - - 'ci/scripts/java*.sh' - - 'ci/scripts/util_*.sh' - - 'format/Flight.proto' - - 'java/**' - -concurrency: - group: ${{ github.repository }}-${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: true - -env: - DOCKER_VOLUME_PREFIX: ".docker/" - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - -jobs: - - debian: - name: AMD64 Debian 9 Java JDK ${{ matrix.jdk }} Maven ${{ matrix.maven }} - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - jdk: [11] - maven: [3.6.2] - env: - JDK: ${{ matrix.jdk }} - MAVEN: ${{ matrix.maven }} - steps: - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh - - name: Free Up Disk Space - shell: bash - run: ci/scripts/util_cleanup.sh - - name: Cache Docker Volumes - uses: actions/cache@v2 - with: - path: .docker - key: maven-${{ hashFiles('java/**') }} - restore-keys: maven- - - name: Setup Python - uses: actions/setup-python@v1 - with: - python-version: 3.8 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - run: archery docker run debian-java - - name: Docker Push - if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' - continue-on-error: true - run: archery docker push debian-java - - macos: - name: AMD64 MacOS 10.15 Java JDK ${{ matrix.jdk }} - runs-on: macos-latest - if: github.event_name == 'push' - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - jdk: [11] - steps: - - name: Set up Java - uses: actions/setup-java@v1 - with: - java-version: ${{ matrix.jdk }} - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh - - name: Build - shell: bash - run: ci/scripts/java_build.sh $(pwd) $(pwd)/build - - name: Test - shell: bash - run: ci/scripts/java_test.sh $(pwd) $(pwd)/build diff --git a/.github/workflows/java_jni.yml b/.github/workflows/java_jni.yml deleted file mode 100644 index 48351f3c22ad3..0000000000000 --- a/.github/workflows/java_jni.yml +++ /dev/null @@ -1,80 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Java JNI - -on: - push: - paths: - - '.github/workflows/java_jni.yml' - - 'ci/docker/**' - - 'ci/scripts/cpp_build.sh' - - 'ci/scripts/java_*' - - 'cpp/**' - - 'java/**' - pull_request: - paths: - - '.github/workflows/java_jni.yml' - - 'ci/docker/**' - - 'ci/scripts/cpp_build.sh' - - 'ci/scripts/java_*' - - 'cpp/**' - - 'java/**' - -concurrency: - group: ${{ github.repository }}-${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: true - -env: - DOCKER_VOLUME_PREFIX: ".docker/" - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - -jobs: - - docker: - name: AMD64 Debian 9 Java JNI (Gandiva, Plasma, ORC, Dataset) - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 90 - steps: - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - run: ci/scripts/util_checkout.sh - - name: Free Up Disk Space - run: ci/scripts/util_cleanup.sh - - name: Cache Docker Volumes - uses: actions/cache@v2 - with: - path: .docker - key: maven-${{ hashFiles('java/**') }} - restore-keys: maven- - - name: Setup Python - uses: actions/setup-python@v1 - with: - python-version: 3.8 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - run: archery docker run debian-java-jni - - name: Docker Push - if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' - continue-on-error: true - run: archery docker push debian-java-jni diff --git a/.github/workflows/js.yml b/.github/workflows/js.yml deleted file mode 100644 index cbd6ce0acef8f..0000000000000 --- a/.github/workflows/js.yml +++ /dev/null @@ -1,124 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: NodeJS - -on: - push: - paths: - - '.github/workflows/js.yml' - - 'ci/docker/*js.dockerfile' - - 'ci/scripts/js_*' - - 'js/**' - pull_request: - paths: - - '.github/workflows/js.yml' - - 'ci/docker/*js.dockerfile' - - 'ci/scripts/js_*' - - 'js/**' - -concurrency: - group: ${{ github.repository }}-${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: true - -env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - -jobs: - - docker: - name: AMD64 Debian 11 NodeJS 14 - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 60 - steps: - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - run: ci/scripts/util_checkout.sh - - name: Free Up Disk Space - run: ci/scripts/util_cleanup.sh - - name: Setup Python - uses: actions/setup-python@v1 - with: - python-version: 3.8 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - run: | - sudo sysctl -w kernel.core_pattern="core.%e.%p" - ulimit -c unlimited - archery docker run debian-js - - name: Docker Push - if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' - continue-on-error: true - run: archery docker push debian-js - - macos: - name: AMD64 MacOS 10.15 NodeJS ${{ matrix.node }} - runs-on: macos-latest - if: github.event_name == 'push' - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - node: [14] - steps: - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh - - name: Install NodeJS - uses: actions/setup-node@v1 - with: - node-version: ${{ matrix.node }} - - name: Build - shell: bash - run: ci/scripts/js_build.sh $(pwd) - - name: Test - shell: bash - run: ci/scripts/js_test.sh $(pwd) - - windows: - name: AMD64 Windows 2019 NodeJS ${{ matrix.node }} - runs-on: windows-latest - if: github.event_name == 'push' - strategy: - fail-fast: false - matrix: - node: [14] - steps: - - name: Checkout Arrow - uses: actions/checkout@v1 - with: - submodules: true - - name: Install NodeJS - uses: actions/setup-node@v1 - with: - node-version: ${{ matrix.node }} - - name: Build - shell: bash - run: ci/scripts/js_build.sh $(pwd) - - name: Test - shell: bash - run: ci/scripts/js_test.sh $(pwd) diff --git a/.github/workflows/julia.yml b/.github/workflows/julia.yml deleted file mode 100644 index 226ec3e6ad042..0000000000000 --- a/.github/workflows/julia.yml +++ /dev/null @@ -1,58 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Julia -on: - push: - paths: - - '.github/workflows/julia.yml' - - 'julia/**' - pull_request: - paths: - - '.github/workflows/julia.yml' - - 'julia/**' - -concurrency: - group: ${{ github.repository }}-${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: true - -jobs: - test: - name: AMD64 ${{ matrix.os }} Julia ${{ matrix.version }} - env: - JULIA_NUM_THREADS: 2 - runs-on: ${{ matrix.os }} - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - version: - - '1.3' - - '1' # automatically expands to the latest stable 1.x release of Julia - - 'nightly' - os: - - ubuntu-latest - - windows-latest - steps: - - uses: actions/checkout@v2 - - uses: julia-actions/setup-julia@v1 - with: - version: ${{ matrix.version }} - arch: x64 - - uses: julia-actions/julia-runtest@v1 - with: - project: julia/Arrow diff --git a/.github/workflows/matlab.yml b/.github/workflows/matlab.yml index 1ac3df6020ba6..d66e0761f2a75 100644 --- a/.github/workflows/matlab.yml +++ b/.github/workflows/matlab.yml @@ -32,13 +32,13 @@ on: - 'cpp/src/arrow/**' concurrency: - group: ${{ github.repository }}-${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: true + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: matlab: - name: AMD64 Ubuntu 20.04 MATLAB + name: AMD64 Ubuntu 20.04 MATLAB runs-on: ubuntu-latest steps: - name: Check out repository @@ -46,22 +46,4 @@ jobs: with: fetch-depth: 0 - name: Fetch Submodules and Tags - run: ci/scripts/util_checkout.sh - - name: Install ninja-build - run: sudo apt-get install ninja-build - - name: Install MATLAB - uses: matlab-actions/setup-matlab@v0 - - name: Build MATLAB Interface - run: ci/scripts/matlab_build.sh $(pwd) - - name: Run MATLAB Tests - env: - # libarrow.so requires a more recent version of libstdc++.so - # than is bundled with MATLAB under /sys/os/glnxa64. - # Therefore, if a MEX function that depends on libarrow.so - # is executed within the MATLAB address space, runtime linking - # errors will occur. To work around this issue, we can explicitly - # force MATLAB to use the system libstdc++.so via LD_PRELOAD. - LD_PRELOAD: /usr/lib/x86_64-linux-gnu/libstdc++.so.6 - uses: matlab-actions/run-tests@v1 - with: - select-by-folder: matlab/test + run: sleep 240 diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml deleted file mode 100644 index cce66fe71ba79..0000000000000 --- a/.github/workflows/python.yml +++ /dev/null @@ -1,161 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Python - -on: - push: - paths: - - '.github/workflows/python.yml' - - 'ci/**' - - 'cpp/**' - - 'python/**' - pull_request: - paths: - - '.github/workflows/python.yml' - - 'ci/**' - - 'cpp/**' - - 'python/**' - -concurrency: - group: ${{ github.repository }}-${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: true - -env: - DOCKER_VOLUME_PREFIX: ".docker/" - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - -jobs: - - docker: - name: ${{ matrix.title }} - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - name: - - conda-python-3.8-nopandas - - conda-python-3.6-pandas-0.23 - - conda-python-3.7-pandas-latest - include: - - name: conda-python-3.8-nopandas - cache: conda-python-3.8 - image: conda-python - title: AMD64 Conda Python 3.8 Without Pandas - python: 3.8 - - name: conda-python-3.6-pandas-0.23 - cache: conda-python-3.6 - image: conda-python-pandas - title: AMD64 Conda Python 3.6 Pandas 0.23 - python: 3.6 - pandas: 0.23 - numpy: 1.16 - - name: conda-python-3.7-pandas-latest - cache: conda-python-3.7 - image: conda-python-pandas - title: AMD64 Conda Python 3.7 Pandas latest - python: 3.7 - pandas: latest - env: - PYTHON: ${{ matrix.python || 3.7 }} - UBUNTU: ${{ matrix.ubuntu || 18.04 }} - PANDAS: ${{ matrix.pandas || 'latest' }} - NUMPY: ${{ matrix.numpy || 'latest' }} - steps: - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - run: ci/scripts/util_checkout.sh - - name: Free Up Disk Space - run: ci/scripts/util_cleanup.sh - - name: Cache Docker Volumes - uses: actions/cache@v2 - with: - path: .docker - key: ${{ matrix.cache }}-${{ hashFiles('cpp/**') }} - restore-keys: ${{ matrix.cache }}- - - name: Setup Python - uses: actions/setup-python@v1 - with: - python-version: 3.8 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - run: | - sudo sysctl -w kernel.core_pattern="core.%e.%p" - ulimit -c unlimited - archery docker run ${{ matrix.image }} - - name: Docker Push - if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' - continue-on-error: true - run: archery docker push ${{ matrix.image }} - - macos: - name: AMD64 MacOS 10.15 Python 3 - runs-on: macos-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 60 - env: - ARROW_HOME: /usr/local - ARROW_DATASET: ON - ARROW_GANDIVA: ON - ARROW_JEMALLOC: ON - ARROW_ORC: ON - ARROW_PARQUET: ON - ARROW_PLASMA: ON - ARROW_PYTHON: ON - ARROW_WITH_ZLIB: ON - ARROW_WITH_LZ4: ON - ARROW_WITH_BZ2: ON - ARROW_WITH_ZSTD: ON - ARROW_WITH_SNAPPY: ON - ARROW_WITH_BROTLI: ON - ARROW_BUILD_TESTS: OFF - CMAKE_ARGS: "-DPython3_EXECUTABLE=/usr/local/bin/python3" - PYARROW_TEST_LARGE_MEMORY: ON - steps: - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh - - name: Install Dependencies - shell: bash - run: | - rm -f /usr/local/bin/2to3 - brew update --preinstall - brew bundle --file=cpp/Brewfile - brew install coreutils - python3 -mpip install \ - -r python/requirements-build.txt \ - -r python/requirements-test.txt - - name: Build - shell: bash - run: | - export PYTHON=python3 - ci/scripts/cpp_build.sh $(pwd) $(pwd)/build - ci/scripts/python_build.sh $(pwd) $(pwd)/build - - name: Test - shell: bash - run: ci/scripts/python_test.sh $(pwd) $(pwd)/build diff --git a/.github/workflows/r-without-arrow.yml b/.github/workflows/r-without-arrow.yml deleted file mode 100644 index a517cfcd18744..0000000000000 --- a/.github/workflows/r-without-arrow.yml +++ /dev/null @@ -1,92 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: R without Arrow - -on: - push: - paths: - - ".github/workflows/r-without-arrow.yml" - - "r/src/**" - pull_request: - paths: - - ".github/workflows/r-without-arrow.yml" - - "r/src/**" - -concurrency: - group: ${{ github.repository }}-${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: true - -env: - DOCKER_VOLUME_PREFIX: ".docker/" - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - -jobs: - bundled: - name: "R package without arrow" - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 60 - strategy: - fail-fast: false - env: - R_ORG: rhub - R_IMAGE: ubuntu-gcc-release - R_TAG: latest - steps: - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - run: ci/scripts/util_checkout.sh - - name: Free Up Disk Space - run: ci/scripts/util_cleanup.sh - - name: Cache Docker Volumes - uses: actions/cache@v2 - with: - path: .docker - key: ubuntu-gcc-release-r-${{ hashFiles('cpp/**') }} - restore-keys: ubuntu-gcc-release-r- - - name: Setup Python - uses: actions/setup-python@v1 - with: - python-version: 3.8 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - run: | - sudo sysctl -w kernel.core_pattern="core.%e.%p" - ulimit -c unlimited - archery docker run -e LIBARROW_DOWNLOAD=FALSE -e LIBARROW_BUILD=FALSE -e TEST_R_WITHOUT_LIBARROW=TRUE -e NOT_CRAN=FALSE r - - name: Dump install logs - run: cat r/check/arrow.Rcheck/00install.out - if: always() - - name: Dump test logs - run: cat r/check/arrow.Rcheck/tests/testthat.Rout* - if: always() - - name: Save the test output - if: always() - uses: actions/upload-artifact@v2 - with: - name: test-output - path: r/check/arrow.Rcheck/tests/testthat.Rout* - - name: Docker Push - if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' - continue-on-error: true - run: archery docker push r diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml deleted file mode 100644 index 900acdac794fb..0000000000000 --- a/.github/workflows/r.yml +++ /dev/null @@ -1,371 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: R - -on: - push: - paths: - - ".github/workflows/r.yml" - - "ci/scripts/r_*.sh" - - "ci/scripts/cpp_*.sh" - - "ci/scripts/PKGBUILD" - - "ci/etc/rprofile" - - "ci/docker/**" - - "cpp/**" - - "r/**" - pull_request: - paths: - - ".github/workflows/r.yml" - - "ci/scripts/r_*.sh" - - "ci/scripts/cpp_*.sh" - - "ci/scripts/PKGBUILD" - - "ci/etc/rprofile" - - "ci/docker/**" - - "cpp/**" - - "r/**" - -concurrency: - group: ${{ github.repository }}-${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: true - -env: - DOCKER_VOLUME_PREFIX: ".docker/" - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - -jobs: - ubuntu: - name: AMD64 Ubuntu ${{ matrix.ubuntu }} R ${{ matrix.r }} - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 75 - strategy: - fail-fast: false - matrix: - r: ["4.1"] - ubuntu: [20.04] - env: - R: ${{ matrix.r }} - UBUNTU: ${{ matrix.ubuntu }} - steps: - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - run: ci/scripts/util_checkout.sh - - name: Free Up Disk Space - run: ci/scripts/util_cleanup.sh - - name: Cache Docker Volumes - uses: actions/cache@v2 - with: - path: .docker - key: ubuntu-${{ matrix.ubuntu }}-r-${{ matrix.r }}-${{ hashFiles('cpp/**') }} - restore-keys: ubuntu-${{ matrix.ubuntu }}-r-${{ matrix.r }}- - - name: Check pkgdown reference sections - run: ci/scripts/r_pkgdown_check.sh - - name: Setup Python - uses: actions/setup-python@v1 - with: - python-version: 3.8 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - run: | - sudo sysctl -w kernel.core_pattern="core.%e.%p" - ulimit -c unlimited - # Setting a non-default and non-probable Marquesas French Polynesia time - # it has both with a .45 offset and very very few people who live there. - archery docker run -e TZ=MART ubuntu-r - - name: Dump install logs - run: cat r/check/arrow.Rcheck/00install.out - if: always() - - name: Dump test logs - run: cat r/check/arrow.Rcheck/tests/testthat.Rout* - if: always() - - name: Save the test output - if: always() - uses: actions/upload-artifact@v2 - with: - name: test-output - path: r/check/arrow.Rcheck/tests/testthat.Rout* - - name: Docker Push - if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' - continue-on-error: true - run: archery docker push ubuntu-r - - bundled: - name: "${{ matrix.config.org }}/${{ matrix.config.image }}:${{ matrix.config.tag }}" - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - config: - - { org: "rstudio", image: "r-base", tag: "4.0-centos7" } - - { org: "rhub", image: "debian-gcc-devel", tag: "latest" } - env: - R_ORG: ${{ matrix.config.org }} - R_IMAGE: ${{ matrix.config.image }} - R_TAG: ${{ matrix.config.tag }} - steps: - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - run: ci/scripts/util_checkout.sh - - name: Free Up Disk Space - run: ci/scripts/util_cleanup.sh - - name: Cache Docker Volumes - uses: actions/cache@v2 - with: - path: .docker - key: ${{ matrix.config.image }}-r-${{ hashFiles('cpp/**') }} - restore-keys: ${{ matrix.config.image }}-r- - - name: Setup Python - uses: actions/setup-python@v1 - with: - python-version: 3.8 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - run: | - sudo sysctl -w kernel.core_pattern="core.%e.%p" - ulimit -c unlimited - # Don't set a TZ here to test that case. These builds will have the following warning in them: - # System has not been booted with systemd as init system (PID 1). Can't operate. - # Failed to connect to bus: Host is down - archery docker run -e TZ="" r - - name: Dump install logs - run: cat r/check/arrow.Rcheck/00install.out - if: always() - - name: Dump test logs - run: cat r/check/arrow.Rcheck/tests/testthat.Rout* - if: always() - - name: Save the test output - if: always() - uses: actions/upload-artifact@v2 - with: - name: test-output - path: r/check/arrow.Rcheck/tests/testthat.Rout* - - name: Docker Push - if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' - continue-on-error: true - run: archery docker push r - - windows-cpp: - name: AMD64 Windows C++ RTools ${{ matrix.config.rtools }} ${{ matrix.config.arch }} - runs-on: windows-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - config: - - { rtools: 35, arch: 'mingw32' } - - { rtools: 35, arch: 'mingw64' } - - { rtools: 40, arch: 'mingw32' } - - { rtools: 40, arch: 'mingw64' } - - { rtools: 40, arch: 'ucrt64' } - steps: - - run: git config --global core.autocrlf false - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Setup ccache - shell: bash - run: | - ci/scripts/ccache_setup.sh - echo "CCACHE_DIR=$(cygpath --absolute --windows ccache)" >> $GITHUB_ENV - # We must enable actions/cache before r-lib/actions/setup-r to ensure - # using system tar instead of tar provided by Rtools. - # We can use tar provided by Rtools when we drop support for Rtools 3.5. - # Because Rtools 4.0 or later has zstd. actions/cache requires zstd - # when tar is GNU tar. - - name: Cache ccache - uses: actions/cache@v2 - with: - path: ccache - key: r-${{ matrix.config.rtools }}-ccache-mingw-${{ hashFiles('cpp/**') }} - restore-keys: r-${{ matrix.config.rtools }}-ccache-mingw- - # We use the makepkg-mingw setup that is included in rtools40 even when - # we use the rtools35 compilers, so we always install R 4.0/Rtools40 - - uses: r-lib/actions/setup-r@master - with: - r-version: "4.1" - rtools-version: 40 - Ncpus: 2 - - uses: r-lib/actions/setup-r@master - if: ${{ matrix.config.rtools == 35 }} - with: - rtools-version: 35 - r-version: "3.6" - Ncpus: 2 - - name: Build Arrow C++ - shell: bash - env: - RTOOLS_VERSION: ${{ matrix.config.rtools }} - MINGW_ARCH: ${{ matrix.config.arch }} - run: ci/scripts/r_windows_build.sh - - name: Rename libarrow.zip - # So that they're unique when multiple are downloaded in the next step - shell: bash - run: mv libarrow.zip libarrow-rtools${{ matrix.config.rtools }}-${{ matrix.config.arch }}.zip - - uses: actions/upload-artifact@v1 - with: - name: libarrow-rtools${{ matrix.config.rtools }}-${{ matrix.config.arch }}.zip - path: libarrow-rtools${{ matrix.config.rtools }}-${{ matrix.config.arch }}.zip - # We can remove this when we drop support for Rtools 3.5. - - name: Ensure using system tar in actions/cache - run: | - Write-Output "${Env:windir}\System32" | ` - Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - - windows-r: - needs: [windows-cpp] - name: AMD64 Windows R RTools ${{ matrix.rtools }} - runs-on: windows-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - rtools: [35, 40] - env: - ARROW_R_CXXFLAGS: "-Werror" - _R_CHECK_TESTS_NLINES_: 0 - steps: - - run: git config --global core.autocrlf false - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - run: mkdir r/windows - - name: Download artifacts - if: ${{ matrix.rtools == 35 }} - uses: actions/download-artifact@v2 - with: - name: libarrow-rtools35-mingw32.zip - path: r/windows - - name: Download artifacts - if: ${{ matrix.rtools == 35 }} - uses: actions/download-artifact@v2 - with: - name: libarrow-rtools35-mingw64.zip - path: r/windows - - name: Download artifacts - if: ${{ matrix.rtools == 40 }} - uses: actions/download-artifact@v2 - with: - name: libarrow-rtools40-mingw32.zip - path: r/windows - - name: Download artifacts - if: ${{ matrix.rtools == 40 }} - uses: actions/download-artifact@v2 - with: - name: libarrow-rtools40-mingw64.zip - path: r/windows - - name: Download artifacts - if: ${{ matrix.rtools == 40 }} - uses: actions/download-artifact@v2 - with: - name: libarrow-rtools40-ucrt64.zip - path: r/windows - - name: Unzip and rezip libarrows - shell: bash - run: | - cd r/windows - ls *.zip | xargs -n 1 unzip -uo - rm -rf *.zip - - name: Setup ccache - shell: bash - run: | - ci/scripts/ccache_setup.sh - echo "CCACHE_DIR=$(cygpath --absolute --windows ccache)" >> $GITHUB_ENV - # We must enable actions/cache before r-lib/actions/setup-r to ensure - # using system tar instead of tar provided by Rtools. - # We can use tar provided by Rtools when we drop support for Rtools 3.5. - # Because Rtools 4.0 or later has zstd. actions/cache requires zstd - # when tar is GNU tar. - - name: Cache ccache - uses: actions/cache@v2 - with: - path: ccache - key: r-${{ matrix.rtools }}-ccache-mingw-${{ hashFiles('cpp/**') }} - restore-keys: r-${{ matrix.rtools }}-ccache-mingw- - - uses: r-lib/actions/setup-r@master - if: ${{ matrix.rtools == 40 }} - with: - r-version: "4.1" - rtools-version: 40 - Ncpus: 2 - - uses: r-lib/actions/setup-r@master - if: ${{ matrix.rtools == 35 }} - with: - rtools-version: 35 - r-version: "3.6" - Ncpus: 2 - - name: Make R tests verbose - # If you get a segfault/mysterious test Execution halted, - # make this `true` to see where it dies. - if: false - shell: cmd - run: | - cd r/tests - sed -i.bak -E -e 's/"arrow"/"arrow", reporter = "location"/' testthat.R - rm -f testthat.R.bak - - name: Install R package dependencies - shell: Rscript {0} - run: | - options(pkgType="win.binary") - install.packages(c("remotes", "rcmdcheck")) - remotes::install_deps("r", dependencies = TRUE) - - name: Check - shell: Rscript {0} - run: | - # Because we do R CMD build and r/windows is in .Rbuildignore, - # assemble the libarrow.zip file and pass it as an env var - setwd("r/windows") - zip("libarrow.zip", ".") - setwd("..") - - Sys.setenv( - RWINLIB_LOCAL = file.path(Sys.getenv("GITHUB_WORKSPACE"), "r", "windows", "libarrow.zip"), - MAKEFLAGS = paste0("-j", parallel::detectCores()), - ARROW_R_DEV = TRUE, - "_R_CHECK_FORCE_SUGGESTS_" = FALSE - ) - rcmdcheck::rcmdcheck(".", - build_args = '--no-build-vignettes', - args = c('--no-manual', '--as-cran', '--ignore-vignettes', '--run-donttest'), - error_on = 'warning', - check_dir = 'check', - timeout = 3600 - ) - - name: Dump install logs - shell: cmd - run: cat r/check/arrow.Rcheck/00install.out - if: always() - # We can remove this when we drop support for Rtools 3.5. - - name: Ensure using system tar in actions/cache - run: | - Write-Output "${Env:windir}\System32" | ` - Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml deleted file mode 100644 index 067b40aefe92f..0000000000000 --- a/.github/workflows/ruby.yml +++ /dev/null @@ -1,299 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: C GLib & Ruby - -on: - push: - paths: - - '.github/workflows/ruby.yml' - - 'ci/docker/**' - - 'ci/scripts/c_glib_*' - - 'ci/scripts/cpp_*' - - 'ci/scripts/msys2_*' - - 'ci/scripts/ruby_*' - - 'ci/scripts/util_*' - - 'c_glib/**' - - 'cpp/**' - - 'ruby/**' - pull_request: - paths: - - '.github/workflows/ruby.yml' - - 'ci/docker/**' - - 'ci/scripts/c_glib_*' - - 'ci/scripts/cpp_*' - - 'ci/scripts/msys2_*' - - 'ci/scripts/ruby_*' - - 'ci/scripts/util_*' - - 'c_glib/**' - - 'cpp/**' - - 'ruby/**' - -concurrency: - group: ${{ github.repository }}-${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: true - -env: - DOCKER_VOLUME_PREFIX: ".docker/" - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - -jobs: - - ubuntu: - name: AMD64 Ubuntu ${{ matrix.ubuntu }} GLib & Ruby - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 40 - strategy: - fail-fast: false - matrix: - ubuntu: - - 20.04 - env: - UBUNTU: ${{ matrix.ubuntu }} - steps: - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh - - name: Free Up Disk Space - shell: bash - run: ci/scripts/util_cleanup.sh - - name: Cache Docker Volumes - uses: actions/cache@v2 - with: - path: .docker - key: ubuntu-${{ matrix.ubuntu }}-ruby-${{ hashFiles('cpp/**') }} - restore-keys: ubuntu-${{ matrix.ubuntu }}-ruby- - - name: Setup Python - uses: actions/setup-python@v1 - with: - python-version: 3.8 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - run: | - sudo sysctl -w kernel.core_pattern="core.%e.%p" - ulimit -c unlimited - archery docker run \ - -e ARROW_FLIGHT=ON \ - -e Protobuf_SOURCE=BUNDLED \ - -e gRPC_SOURCE=BUNDLED \ - ubuntu-ruby - - name: Docker Push - if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' - continue-on-error: true - shell: bash - run: archery docker push ubuntu-ruby - - macos: - name: AMD64 MacOS 10.15 GLib & Ruby - runs-on: macos-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 40 - strategy: - fail-fast: false - env: - ARROW_BUILD_TESTS: OFF - ARROW_FLIGHT: ON - ARROW_GANDIVA: ON - ARROW_GLIB_DEVELOPMENT_MODE: true - ARROW_GLIB_GTK_DOC: true - ARROW_HOME: /usr/local - ARROW_JEMALLOC: OFF - ARROW_ORC: OFF - ARROW_PARQUET: ON - ARROW_WITH_BROTLI: ON - ARROW_WITH_LZ4: ON - ARROW_WITH_SNAPPY: ON - ARROW_WITH_ZLIB: ON - ARROW_WITH_ZSTD: ON - XML_CATALOG_FILES: /usr/local/etc/xml/catalog - steps: - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh - - name: Install Homebrew Dependencies - shell: bash - run: | - rm -f /usr/local/bin/2to3 - brew update --preinstall - brew bundle --file=cpp/Brewfile - brew bundle --file=c_glib/Brewfile - - name: Install Ruby Dependencies - run: | - export MAKEFLAGS="-j$(sysctl -n hw.ncpu)" - bundle install --gemfile c_glib/Gemfile - bundle install --gemfile ruby/Gemfile - for ruby_package_gemfile in ruby/*/Gemfile; do \ - bundle install --gemfile ${ruby_package_gemfile} - done - - name: Setup ccache - run: | - ci/scripts/ccache_setup.sh - - name: ccache info - id: ccache-info - run: | - echo "::set-output name=cache-dir::$(ccache --get-config cache_dir)" - - name: Cache ccache - uses: actions/cache@v2 - with: - path: ${{ steps.ccache-info.outputs.cache-dir }} - key: ruby-ccache-macos-${{ hashFiles('cpp/**') }} - restore-keys: ruby-ccache-macos- - - name: Build C++ - run: | - ci/scripts/cpp_build.sh $(pwd) $(pwd)/build - - name: Build GLib - run: | - ci/scripts/c_glib_build.sh $(pwd) $(pwd)/build - - name: Test GLib - shell: bash - run: ci/scripts/c_glib_test.sh $(pwd) $(pwd)/build - - name: Test Ruby - shell: bash - run: ci/scripts/ruby_test.sh $(pwd) $(pwd)/build - - windows: - name: AMD64 Windows MinGW ${{ matrix.mingw-n-bits }} GLib & Ruby - runs-on: windows-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 40 - strategy: - fail-fast: false - matrix: - mingw-n-bits: - - 64 - ruby-version: - - "3.0" - env: - ARROW_BUILD_SHARED: ON - ARROW_BUILD_STATIC: OFF - ARROW_BUILD_TESTS: OFF - ARROW_BUILD_TYPE: release - ARROW_FLIGHT: ON - ARROW_GANDIVA: ON - ARROW_HDFS: OFF - ARROW_HOME: /mingw${{ matrix.mingw-n-bits }} - ARROW_JEMALLOC: OFF - ARROW_PARQUET: ON - ARROW_PYTHON: OFF - ARROW_S3: ON - ARROW_USE_GLOG: OFF - ARROW_WITH_BROTLI: ON - ARROW_WITH_BZ2: ON - ARROW_WITH_LZ4: ON - ARROW_WITH_SNAPPY: ON - ARROW_WITH_ZLIB: ON - ARROW_WITH_ZSTD: ON - # Don't use preinstalled Boost by empty BOOST_ROOT and - # -DBoost_NO_BOOST_CMAKE=ON - BOOST_ROOT: "" - CMAKE_ARGS: >- - -DARROW_PACKAGE_PREFIX=/mingw${{ matrix.mingw-n-bits }} - -DBoost_NO_BOOST_CMAKE=ON - CMAKE_UNITY_BUILD: ON - steps: - - name: Disable Crash Dialogs - run: | - reg add ` - "HKCU\SOFTWARE\Microsoft\Windows\Windows Error Reporting" ` - /v DontShowUI ` - /t REG_DWORD ` - /d 1 ` - /f - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh - - name: Setup Ruby - uses: ruby/setup-ruby@v1 - with: - ruby-version: ${{ matrix.ruby-version }} - - name: Upgrade MSYS2 - run: | - ridk exec bash ci\scripts\msys2_system_upgrade.sh - taskkill /F /FI "MODULES eq msys-2.0.dll" - - name: Clean MSYS2 - run: | - ridk exec bash ci\scripts\msys2_system_clean.sh - - name: Setup MSYS2 - run: | - ridk exec bash ci\scripts\msys2_setup.sh ruby - - name: Cache ccache - uses: actions/cache@v2 - with: - path: ccache - key: ruby-ccache-mingw${{ matrix.mingw-n-bits }}-${{ hashFiles('cpp/**') }} - restore-keys: ruby-ccache-mingw${{ matrix.mingw-n-bits }}- - - name: Build C++ - run: | - $Env:CMAKE_BUILD_PARALLEL_LEVEL = $Env:NUMBER_OF_PROCESSORS - $source_dir = "$(ridk exec cygpath --unix "$(Get-Location)")" - $build_dir = "$(ridk exec cygpath --unix "$(Get-Location)\build")" - $ErrorActionPreference = "Continue" - ridk exec bash ci\scripts\cpp_build.sh "${source_dir}" "${build_dir}" - - name: Build GLib - run: | - $Env:CMAKE_BUILD_PARALLEL_LEVEL = $Env:NUMBER_OF_PROCESSORS - $source_dir = "$(ridk exec cygpath --unix "$(Get-Location)")" - $build_dir = "$(ridk exec cygpath --unix "$(Get-Location)\build")" - $ErrorActionPreference = "Continue" - ridk exec bash ci\scripts\c_glib_build.sh "${source_dir}" "${build_dir}" - - name: RubyGems info - id: rubygems-info - run: | - Write-Output "::set-output name=gem-dir::$(ridk exec gem env gemdir)" - - name: Cache RubyGems - uses: actions/cache@v2 - with: - path: ${{ steps.rubygems-info.outputs.gem-dir }} - key: ruby-rubygems-mingw${{ matrix.mingw-n-bits }}-${{ hashFiles('**/Gemfile', 'ruby/*/*.gemspec') }} - restore-keys: ruby-rubygems-mingw${{ matrix.mingw-n-bits }}- - - name: Install test dependencies - run: | - bundle install --gemfile c_glib\Gemfile - bundle install --gemfile ruby\Gemfile - Get-ChildItem ruby\*\Gemfile | ` - ForEach-Object {bundle install --gemfile $_} - - name: Test GLib - run: | - $source_dir = "$(ridk exec cygpath --unix "$(Get-Location)")" - $build_dir = "$(ridk exec cygpath --unix "$(Get-Location)\build")" - $ErrorActionPreference = "Continue" - ridk exec bash ci\scripts\c_glib_test.sh "${source_dir}" "${build_dir}" - - name: Test Ruby - run: | - $Env:PKG_CONFIG_PATH = ` - "$(ridk exec cygpath --absolute --windows "${Env:ARROW_HOME}/lib/pkgconfig")" - $Env:GI_TYPELIB_PATH = ` - "$(ridk exec cygpath --absolute --windows "${Env:ARROW_HOME}/lib/girepository-1.0")" - $Env:RUBYOPTS = "-rdevkit" - $Env:MAKE = "ridk exec make" - $ErrorActionPreference = "Continue" - rake -f ruby\Rakefile diff --git a/cpp/src/arrow/python/python_to_arrow.cc b/cpp/src/arrow/python/python_to_arrow.cc index 10250d165747f..ffbbed151402c 100644 --- a/cpp/src/arrow/python/python_to_arrow.cc +++ b/cpp/src/arrow/python/python_to_arrow.cc @@ -25,7 +25,6 @@ #include #include #include -#include #include "arrow/array.h" #include "arrow/array/builder_binary.h" diff --git a/matlab/src/feather_reader.h b/matlab/src/feather_reader.h index 197e470bf6ed3..08ced72cd4f03 100644 --- a/matlab/src/feather_reader.h +++ b/matlab/src/feather_reader.h @@ -20,6 +20,20 @@ #include #include +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 + #include #include #include