diff --git a/.github/workflows/comment_bot.yml b/.github/workflows/comment_bot.yml deleted file mode 100644 index 4a5461752b337..0000000000000 --- a/.github/workflows/comment_bot.yml +++ /dev/null @@ -1,176 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Comment Bot - -on: - # TODO(kszucs): support pull_request_review_comment - issue_comment: - types: - - created - - edited - -permissions: - contents: read - pull-requests: write - -jobs: - crossbow: - name: Listen! - if: startsWith(github.event.comment.body, '@github-actions crossbow') - runs-on: ubuntu-latest - steps: - - name: Checkout Arrow - uses: actions/checkout@v3 - with: - path: arrow - # fetch the tags for version number generation - fetch-depth: 0 - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: 3.8 - - name: Install Archery and Crossbow dependencies - run: pip install -e arrow/dev/archery[bot] - - name: Handle Github comment event - env: - ARROW_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - CROSSBOW_GITHUB_TOKEN: ${{ secrets.CROSSBOW_GITHUB_TOKEN }} - run: | - archery trigger-bot \ - --event-name ${{ github.event_name }} \ - --event-payload ${{ github.event_path }} - - autotune: - name: "Fix all the things" - if: startsWith(github.event.comment.body, '@github-actions autotune') - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: r-lib/actions/pr-fetch@v2 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: See what is different - run: | - set -ex - DEFAULT_BRANCH=${{ github.event.repository.default_branch }} - git remote add upstream https://github.com/apache/arrow - git fetch upstream - - changed() { - git diff --name-only upstream/$DEFAULT_BRANCH... | grep -e "$1" >/dev/null 2>&1 - } - if changed '^r/.*\.R$'; then - echo "R_DOCS=true" >> $GITHUB_ENV - echo "R_CODE=true" >> $GITHUB_ENV - fi - if changed 'cmake' || changed 'CMake'; then - echo "CMAKE_FORMAT=true" >> $GITHUB_ENV - fi - if changed '^cpp/src'; then - echo "CLANG_FORMAT_CPP=true" >> $GITHUB_ENV - fi - if changed '^r/src'; then - echo "CLANG_FORMAT_R=true" >> $GITHUB_ENV - fi - - name: Ensure clang-format has the appropriate version - if: env.CMAKE_FORMAT == 'true' || - env.CLANG_FORMAT_CPP == 'true' || - env.CLANG_FORMAT_R == 'true' || - endsWith(github.event.comment.body, 'everything') - run: | - set -e - . .env # To get the clang version we use - sudo apt update - sudo apt install -y clang-format-${CLANG_TOOLS} - - name: Run cmake_format - if: env.CMAKE_FORMAT == 'true' || endsWith(github.event.comment.body, 'everything') - run: | - set -ex - export PATH=/home/runner/.local/bin:$PATH - python3 -m pip install --upgrade pip setuptools wheel - python3 -m pip install -e dev/archery[lint] - archery lint --cmake-format --fix - - name: Run clang-format on cpp - if: env.CLANG_FORMAT_CPP == 'true' || endsWith(github.event.comment.body, 'everything') - run: | - . .env # To get the clang version we use - cpp/build-support/run_clang_format.py \ - --clang_format_binary=clang-format-${CLANG_TOOLS} \ - --exclude_glob=cpp/build-support/lint_exclusions.txt \ - --source_dir=cpp/src --quiet --fix - - name: Run clang-format on r - if: env.CLANG_FORMAT_R == 'true' || endsWith(github.event.comment.body, 'everything') - run: | - . .env # To get the clang version we use - cpp/build-support/run_clang_format.py \ - --clang_format_binary=clang-format-${CLANG_TOOLS} \ - --exclude_glob=cpp/build-support/lint_exclusions.txt \ - --source_dir=r/src --quiet --fix - - uses: r-lib/actions/setup-r@v2 - if: env.R_DOCS == 'true' || env.R_CODE == 'true' || endsWith(github.event.comment.body, 'everything') - - name: Update R docs - if: env.R_DOCS == 'true' || endsWith(github.event.comment.body, 'everything') - shell: Rscript {0} - run: | - source("ci/etc/rprofile") - install.packages(c("remotes", "roxygen2")) - remotes::install_deps("r") - roxygen2::roxygenize("r") - - name: Style R code - if: env.R_CODE == 'true' || endsWith(github.event.comment.body, 'everything') - shell: Rscript {0} - run: | - changed_files <- system("git diff --name-only upstream/${{ github.event.repository.default_branch }}... 2>&1", intern = TRUE) - # only grab the .R files under r/ - changed_files <- grep('^r/.*\\.R$', changed_files, value = TRUE) - # remove codegen.R and other possible exclusions - changed_files <- changed_files[!changed_files %in% file.path("r", source("r/.styler_excludes.R")$value)] - source("ci/etc/rprofile") - install.packages(c("remotes", "styler")) - remotes::install_deps("r") - styler::style_file(changed_files) - - name: Commit results - run: | - git config user.name "$(git log -1 --pretty=format:%an)" - git config user.email "$(git log -1 --pretty=format:%ae)" - git commit -a -m 'Autoformat/render all the things [automated commit]' || echo "No changes to commit" - - uses: r-lib/actions/pr-push@v2 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - - rebase: - name: "Rebase" - if: startsWith(github.event.comment.body, '@github-actions rebase') - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: r-lib/actions/pr-fetch@v2 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: Rebase on ${{ github.repository }} default branch - run: | - set -ex - git config user.name "$(git log -1 --pretty=format:%an)" - git config user.email "$(git log -1 --pretty=format:%ae)" - git remote add upstream https://github.com/${{ github.repository }} - git fetch --unshallow upstream ${{ github.event.repository.default_branch }} - git rebase upstream/${{ github.event.repository.default_branch }} - - uses: r-lib/actions/pr-push@v2 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - args: "--force" diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml index de76a3daa98a6..ced0c504f637c 100644 --- a/.github/workflows/cpp.yml +++ b/.github/workflows/cpp.yml @@ -191,204 +191,3 @@ jobs: sudo sysctl -w kern.corefile=core.%N.%P ulimit -c unlimited # must enable within the same shell ci/scripts/cpp_test.sh $(pwd) $(pwd)/build - - windows: - name: AMD64 ${{ matrix.name }} C++17 - runs-on: ${{ matrix.os }} - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - os: - - windows-2019 - include: - - os: windows-2019 - name: Windows 2019 - env: - ARROW_BOOST_USE_SHARED: OFF - ARROW_BUILD_BENCHMARKS: ON - ARROW_BUILD_SHARED: ON - ARROW_BUILD_STATIC: OFF - ARROW_BUILD_TESTS: ON - ARROW_DATASET: ON - ARROW_FLIGHT: OFF - ARROW_HDFS: ON - ARROW_HOME: /usr - ARROW_JEMALLOC: OFF - ARROW_MIMALLOC: ON - ARROW_ORC: ON - ARROW_PARQUET: ON - ARROW_USE_GLOG: OFF - ARROW_VERBOSE_THIRDPARTY_BUILD: OFF - ARROW_WITH_BROTLI: OFF - ARROW_WITH_BZ2: OFF - ARROW_WITH_LZ4: OFF - ARROW_WITH_OPENTELEMETRY: OFF - ARROW_WITH_SNAPPY: ON - ARROW_WITH_ZLIB: ON - ARROW_WITH_ZSTD: ON - BOOST_SOURCE: BUNDLED - CMAKE_CXX_STANDARD: "17" - CMAKE_GENERATOR: Ninja - CMAKE_INSTALL_LIBDIR: bin - CMAKE_INSTALL_PREFIX: /usr - CMAKE_UNITY_BUILD: ON - OPENSSL_ROOT_DIR: >- - C:\Program Files\OpenSSL-Win64 - NPROC: 3 - steps: - - name: Disable Crash Dialogs - run: | - reg add ` - "HKCU\SOFTWARE\Microsoft\Windows\Windows Error Reporting" ` - /v DontShowUI ` - /t REG_DWORD ` - /d 1 ` - /f - - name: Installed Packages - run: choco list -l - - name: Install Dependencies - run: choco install -y --no-progress openssl - - name: Checkout Arrow - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: recursive - - name: Download Timezone Database - shell: bash - run: ci/scripts/download_tz_database.sh - - name: Install ccache - shell: bash - run: | - ci/scripts/install_ccache.sh 4.6.3 /usr - - name: Setup ccache - shell: bash - run: | - ci/scripts/ccache_setup.sh - - name: ccache info - id: ccache-info - shell: bash - run: | - echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT - - name: Cache ccache - uses: actions/cache@v3 - with: - path: ${{ steps.ccache-info.outputs.cache-dir }} - key: cpp-ccache-windows-${{ env.CACHE_VERSION }}-${{ hashFiles('cpp/**') }} - restore-keys: cpp-ccache-windows-${{ env.CACHE_VERSION }}- - env: - # We can invalidate the current cache by updating this. - CACHE_VERSION: "2022-09-13" - - name: Build - shell: cmd - run: | - call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 - bash -c "ci/scripts/cpp_build.sh $(pwd) $(pwd)/build" - - name: Test - shell: bash - run: | - # For ORC - export TZDIR=/c/msys64/usr/share/zoneinfo - ci/scripts/cpp_test.sh $(pwd) $(pwd)/build - - windows-mingw: - name: AMD64 Windows MinGW ${{ matrix.mingw-n-bits }} C++ - runs-on: windows-2019 - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - # Build may take 1h+ without cache and installing Google Cloud - # Storage Testbench may take 20m+ without cache. - timeout-minutes: 120 - strategy: - fail-fast: false - matrix: - mingw-n-bits: - - 32 - - 64 - env: - ARROW_BUILD_SHARED: ON - ARROW_BUILD_STATIC: OFF - ARROW_BUILD_TESTS: ON - ARROW_BUILD_TYPE: release - ARROW_DATASET: ON - ARROW_FLIGHT: ON - ARROW_FLIGHT_SQL: ON - ARROW_GANDIVA: ON - ARROW_GCS: ON - ARROW_HDFS: OFF - ARROW_HOME: /mingw${{ matrix.mingw-n-bits }} - ARROW_JEMALLOC: OFF - ARROW_PARQUET: ON - ARROW_PYTHON: ON - ARROW_S3: ON - ARROW_USE_GLOG: OFF - ARROW_VERBOSE_THIRDPARTY_BUILD: OFF - ARROW_WITH_BROTLI: ON - ARROW_WITH_BZ2: ON - ARROW_WITH_LZ4: ON - ARROW_WITH_OPENTELEMETRY: OFF - ARROW_WITH_SNAPPY: ON - ARROW_WITH_ZLIB: ON - ARROW_WITH_ZSTD: ON - # Don't use preinstalled Boost by empty BOOST_ROOT and - # -DBoost_NO_BOOST_CMAKE=ON - BOOST_ROOT: "" - CMAKE_ARGS: >- - -DARROW_PACKAGE_PREFIX=/mingw${{ matrix.mingw-n-bits }} - -DBoost_NO_BOOST_CMAKE=ON - # We can't use unity build because we don't have enough memory on - # GitHub Actions. - # CMAKE_UNITY_BUILD: ON - steps: - - name: Disable Crash Dialogs - run: | - reg add ` - "HKCU\SOFTWARE\Microsoft\Windows\Windows Error Reporting" ` - /v DontShowUI ` - /t REG_DWORD ` - /d 1 ` - /f - - name: Checkout Arrow - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: recursive - - uses: msys2/setup-msys2@v2 - with: - msystem: MINGW${{ matrix.mingw-n-bits }} - update: true - - name: Setup MSYS2 - shell: msys2 {0} - run: ci/scripts/msys2_setup.sh cpp - - name: Cache ccache - uses: actions/cache@v3 - with: - path: ccache - key: cpp-ccache-mingw${{ matrix.mingw-n-bits }}-${{ hashFiles('cpp/**') }} - restore-keys: cpp-ccache-mingw${{ matrix.mingw-n-bits }}- - - name: Build - shell: msys2 {0} - run: | - export CMAKE_BUILD_PARALLEL_LEVEL=$NUMBER_OF_PROCESSORS - ci/scripts/cpp_build.sh "$(pwd)" "$(pwd)/build" - - name: Download Timezone Database - shell: bash - run: ci/scripts/download_tz_database.sh - - name: Download MinIO - shell: msys2 {0} - run: | - mkdir -p /usr/local/bin - wget \ - --output-document /usr/local/bin/minio.exe \ - https://dl.min.io/server/minio/release/windows-amd64/archive/minio.RELEASE.2022-05-26T05-48-41Z - chmod +x /usr/local/bin/minio.exe - - name: Install Google Cloud Storage Testbench - shell: bash - run: | - ci/scripts/install_gcs_testbench.sh default - echo "PYTHON_BIN_DIR=$(cygpath --windows $(dirname $(which python3.exe)))" >> $GITHUB_ENV - - name: Test - shell: msys2 {0} - run: | - PATH="$(cygpath --unix ${PYTHON_BIN_DIR}):${PATH}" - ci/scripts/cpp_test.sh "$(pwd)" "$(pwd)/build" diff --git a/.github/workflows/csharp.yml b/.github/workflows/csharp.yml deleted file mode 100644 index 5968dded43c9e..0000000000000 --- a/.github/workflows/csharp.yml +++ /dev/null @@ -1,122 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: C# - -on: - push: - paths: - - '.github/workflows/csharp.yml' - - 'ci/scripts/csharp_*' - - 'csharp/**' - pull_request: - paths: - - '.github/workflows/csharp.yml' - - 'ci/scripts/csharp_*' - - 'csharp/**' - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -permissions: - contents: read - -jobs: - - ubuntu: - name: AMD64 Ubuntu 18.04 C# ${{ matrix.dotnet }} - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - dotnet: ['6.0.x'] - steps: - - name: Install C# - uses: actions/setup-dotnet@v2 - with: - dotnet-version: ${{ matrix.dotnet }} - - name: Checkout Arrow - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Install Source Link - shell: bash - run: dotnet tool install --global sourcelink - - name: Build - shell: bash - run: ci/scripts/csharp_build.sh $(pwd) - - name: Test - shell: bash - run: ci/scripts/csharp_test.sh $(pwd) - - windows: - name: AMD64 Windows 2019 18.04 C# ${{ matrix.dotnet }} - runs-on: windows-2019 - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - dotnet: ['6.0.x'] - steps: - - name: Install C# - uses: actions/setup-dotnet@v2 - with: - dotnet-version: ${{ matrix.dotnet }} - - name: Checkout Arrow - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Install Source Link - run: dotnet tool install --global sourcelink - - name: Build - shell: bash - run: ci/scripts/csharp_build.sh $(pwd) - - name: Test - shell: bash - run: ci/scripts/csharp_test.sh $(pwd) - - macos: - name: AMD64 macOS 11 C# ${{ matrix.dotnet }} - runs-on: macos-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - dotnet: ['6.0.x'] - steps: - - name: Install C# - uses: actions/setup-dotnet@v2 - with: - dotnet-version: ${{ matrix.dotnet }} - - name: Checkout Arrow - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Install Source Link - shell: bash - run: dotnet tool install --global sourcelink - - name: Build - shell: bash - run: ci/scripts/csharp_build.sh $(pwd) - - name: Test - shell: bash - run: ci/scripts/csharp_test.sh $(pwd) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml deleted file mode 100644 index 27968ad28c886..0000000000000 --- a/.github/workflows/docs.yml +++ /dev/null @@ -1,67 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Docs - -on: - push: - -permissions: - contents: read - -env: - ARROW_ENABLE_TIMING_TESTS: OFF - DOCKER_VOLUME_PREFIX: ".docker/" - -jobs: - - complete: - name: AMD64 Ubuntu 20.04 Complete Documentation - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 150 - env: - UBUNTU: "20.04" - steps: - - name: Checkout Arrow - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Cache Docker Volumes - uses: actions/cache@v3 - with: - path: .docker - key: ubuntu-docs-${{ hashFiles('cpp/**') }} - restore-keys: ubuntu-docs- - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: 3.8 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - run: archery docker run ubuntu-docs - - name: Docker Push - if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - continue-on-error: true - run: archery docker push ubuntu-docs diff --git a/.github/workflows/docs_light.yml b/.github/workflows/docs_light.yml deleted file mode 100644 index ed8cd12ca361e..0000000000000 --- a/.github/workflows/docs_light.yml +++ /dev/null @@ -1,68 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Docs - -on: - pull_request: - paths: - - 'docs/**' - - '.github/workflows/docs_light.yml' - - 'ci/docker/conda.dockerfile' - - 'ci/docker/conda-cpp.dockerfile' - - 'ci/docker/conda-python.dockerfile' - - 'ci/scripts/cpp_build.sh' - - 'ci/scripts/python_build.sh' - -permissions: - contents: read - -env: - ARROW_ENABLE_TIMING_TESTS: OFF - DOCKER_VOLUME_PREFIX: ".docker/" - -jobs: - - light: - name: AMD64 Conda Python 3.9 Sphinx Documentation - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 45 - env: - PYTHON: "3.9" - steps: - - name: Checkout Arrow - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Cache Docker Volumes - uses: actions/cache@v3 - with: - path: .docker - key: conda-docs-${{ hashFiles('cpp/**') }} - restore-keys: conda-docs- - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: 3.8 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - run: archery docker run conda-python-docs diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml deleted file mode 100644 index 8d028b3e45e52..0000000000000 --- a/.github/workflows/go.yml +++ /dev/null @@ -1,335 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Go - -on: - push: - paths: - - '.github/workflows/go.yml' - - 'ci/docker/*_go.dockerfile' - - 'ci/scripts/go_*' - - 'go/**' - pull_request: - paths: - - '.github/workflows/go.yml' - - 'ci/docker/*_go.dockerfile' - - 'ci/docker/**' - - 'ci/scripts/go_*' - - 'go/**' - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -permissions: - contents: read - -jobs: - - docker: - name: AMD64 Debian 11 Go ${{ matrix.go }} - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - go: [1.17, 1.18] - include: - - go: 1.17 - staticcheck: v0.2.2 - - go: 1.18 - staticcheck: latest - env: - GO: ${{ matrix.go }} - STATICCHECK: ${{ matrix.staticcheck }} - steps: - - name: Checkout Arrow - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: recursive - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: 3.8 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - run: archery docker run debian-go - - name: Docker Push - if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - continue-on-error: true - run: archery docker push debian-go - - docker_cgo: - name: AMD64 Debian 11 GO ${{ matrix.go }} - CGO - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - go: [1.17, 1.18] - include: - - go: 1.17 - staticcheck: v0.2.2 - - go: 1.18 - staticcheck: latest - env: - GO: ${{ matrix.go }} - STATICCHECK: ${{ matrix.staticcheck }} - steps: - - name: Checkout Arrow - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: recursive - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: 3.8 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - run: archery docker run debian-go-cgo - - name: Docker Push - if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - continue-on-error: true - run: archery docker push debian-go-cgo - - - docker_cgo_python: - name: AMD64 Debian 11 GO ${{ matrix.go }} - CGO Python - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - go: [1.17, 1.18] - include: - - go: 1.17 - staticcheck: v0.2.2 - - go: 1.18 - staticcheck: latest - env: - GO: ${{ matrix.go }} - STATICCHECK: ${{ matrix.staticcheck }} - steps: - - name: Checkout Arrow - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: 3.8 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - run: archery docker run debian-go-cgo-python - - name: Docker Push - if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - continue-on-error: true - run: archery docker push debian-go-cgo-python - - windows: - name: AMD64 Windows 2019 Go ${{ matrix.go }} - runs-on: windows-2019 - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - go: [1.17, 1.18] - include: - - go: 1.17 - staticcheck: v0.2.2 - - go: 1.18 - staticcheck: latest - steps: - - name: Checkout Arrow - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: recursive - - name: Install go - uses: actions/setup-go@v3 - with: - go-version: ${{ matrix.go }} - cache: true - cache-dependency-path: go/go.sum - - name: Install staticcheck - run: go install honnef.co/go/tools/cmd/staticcheck@${{ matrix.staticcheck }} - - name: Build - shell: bash - run: ci/scripts/go_build.sh $(pwd) - - name: Test - shell: bash - run: ci/scripts/go_test.sh $(pwd) - - macos: - name: AMD64 macOS 11 Go ${{ matrix.go }} - runs-on: macos-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - go: [1.17, 1.18] - include: - - go: 1.17 - staticcheck: v0.2.2 - - go: 1.18 - staticcheck: latest - steps: - - name: Checkout Arrow - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: recursive - - name: Install go - uses: actions/setup-go@v3 - with: - go-version: ${{ matrix.go }} - cache: true - cache-dependency-path: go/go.sum - - name: Install staticcheck - run: go install honnef.co/go/tools/cmd/staticcheck@${{ matrix.staticcheck }} - - name: Build - shell: bash - run: ci/scripts/go_build.sh $(pwd) - - name: Test - shell: bash - run: ci/scripts/go_test.sh $(pwd) - - macos-cgo: - name: AMD64 macOS 11 Go ${{ matrix.go }} - CGO - runs-on: macos-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - go: [1.17, 1.18] - include: - - go: 1.17 - staticcheck: v0.2.2 - - go: 1.18 - staticcheck: latest - env: - ARROW_GO_TESTCGO: "1" - steps: - - name: Checkout Arrow - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: recursive - - name: Install go - uses: actions/setup-go@v3 - with: - go-version: ${{ matrix.go }} - cache: true - cache-dependency-path: go/go.sum - - name: Brew Install Arrow - shell: bash - run: brew install apache-arrow - - name: Install staticcheck - run: go install honnef.co/go/tools/cmd/staticcheck@${{ matrix.staticcheck }} - - name: Build - shell: bash - run: ci/scripts/go_build.sh $(pwd) - - name: Test - shell: bash - run: ci/scripts/go_test.sh $(pwd) - - windows-mingw: - name: AMD64 Windows MinGW ${{ matrix.mingw-n-bits }} CGO - runs-on: windows-2019 - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - mingw-n-bits: - #- 32 runtime handling for CGO needs 64-bit currently - - 64 - env: - ARROW_GO_TESTCGO: "1" - MINGW_LINT: "1" - steps: - - name: Disable Crash Dialogs - run: | - reg add ` - "HKCU\SOFTWARE\Microsoft\Windows\Windows Error Reporting" ` - /v DontShowUI ` - /t REG_DWORD ` - /d 1 ` - /f - - name: Checkout Arrow - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: recursive - - uses: msys2/setup-msys2@v2 - with: - msystem: MINGW${{ matrix.mingw-n-bits }} - update: true - - name: Setup MSYS2 - shell: msys2 {0} - run: | - ci/scripts/msys2_setup.sh cgo - - name: Update CGO Env vars - shell: msys2 {0} - run: | - echo "CGO_CPPFLAGS=-I$(cygpath --windows ${MINGW_PREFIX}/include)" >> $GITHUB_ENV - echo "CGO_LDFLAGS=-g -O2 -L$(cygpath --windows ${MINGW_PREFIX}/lib) -L$(cygpath --windows ${MINGW_PREFIX}/bin)" >> $GITHUB_ENV - echo "MINGW_PREFIX=$(cygpath --windows ${MINGW_PREFIX})" >> $GITHUB_ENV - - name: Install go - uses: actions/setup-go@v3 - with: - go-version: '1.18' - cache: true - cache-dependency-path: go/go.sum - - name: Install staticcheck - run: go install honnef.co/go/tools/cmd/staticcheck@latest - - name: Build - shell: bash - run: ci/scripts/go_build.sh $(pwd) - - name: Test - shell: bash - run: ci/scripts/go_test.sh $(pwd) diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml deleted file mode 100644 index 86b5799a0139e..0000000000000 --- a/.github/workflows/java.yml +++ /dev/null @@ -1,162 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Java - -on: - push: - paths: - - '.github/workflows/java.yml' - - 'ci/docker/*java*' - - 'ci/scripts/java*.sh' - - 'ci/scripts/util_*.sh' - - 'format/Flight.proto' - - 'java/**' - pull_request: - paths: - - '.github/workflows/java.yml' - - 'ci/docker/*java*' - - 'ci/scripts/java*.sh' - - 'ci/scripts/util_*.sh' - - 'format/Flight.proto' - - 'java/**' - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -permissions: - contents: read - -env: - DOCKER_VOLUME_PREFIX: ".docker/" - -jobs: - - debian: - name: ${{ matrix.title }} - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - jdk: [8, 11, 17, 18] - include: - - jdk: 8 - title: AMD64 Debian 9 Java JDK 8 Maven 3.5.4 - maven: 3.5.4 - image: debian-java - - jdk: 11 - title: AMD64 Debian 9 Java JDK 11 Maven 3.6.2 - maven: 3.6.2 - image: debian-java - - jdk: 17 - title: AMD64 Oracle Linux Server 8.5 Java JDK 17 Maven 3.8.5 - maven: 3.8.5 - image: oracle-java - - jdk: 18 - title: AMD64 Oracle Linux Server 8.6 Java JDK 18 Maven 3.8.5 - maven: 3.8.5 - image: oracle-java - env: - JDK: ${{ matrix.jdk }} - MAVEN: ${{ matrix.maven }} - steps: - - name: Checkout Arrow - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: recursive - - name: Cache Docker Volumes - uses: actions/cache@v3 - with: - path: .docker - key: maven-${{ hashFiles('java/**') }} - restore-keys: maven- - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: 3.8 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - run: archery docker run ${{ matrix.image }} - - name: Docker Push - if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - continue-on-error: true - run: archery docker push ${{ matrix.image }} - - macos: - name: AMD64 macOS 11 Java JDK ${{ matrix.jdk }} - runs-on: macos-latest - if: github.event_name == 'push' - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - jdk: [11] - steps: - - name: Set up Java - uses: actions/setup-java@v3 - with: - distribution: 'zulu' - java-version: ${{ matrix.jdk }} - - name: Checkout Arrow - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: recursive - - name: Build - shell: bash - run: ci/scripts/java_build.sh $(pwd) $(pwd)/build - - name: Test - shell: bash - run: ci/scripts/java_test.sh $(pwd) $(pwd)/build - - windows: - name: AMD64 Windows Server 2022 Java JDK ${{ matrix.jdk }} - runs-on: windows-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - jdk: [11] - steps: - - name: Set up Java - uses: actions/setup-java@v3 - with: - java-version: ${{ matrix.jdk }} - distribution: 'temurin' - - name: Checkout Arrow - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: recursive - - name: Build - shell: bash - run: ci/scripts/java_build.sh $(pwd) $(pwd)/build - - name: Test - shell: bash - run: ci/scripts/java_test.sh $(pwd) $(pwd)/build diff --git a/.github/workflows/java_jni.yml b/.github/workflows/java_jni.yml deleted file mode 100644 index e78a7605eb0f7..0000000000000 --- a/.github/workflows/java_jni.yml +++ /dev/null @@ -1,120 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Java JNI - -on: - push: - paths: - - '.github/workflows/java_jni.yml' - - 'ci/docker/**' - - 'ci/scripts/cpp_build.sh' - - 'ci/scripts/java_*' - - 'cpp/**' - - 'java/**' - pull_request: - paths: - - '.github/workflows/java_jni.yml' - - 'ci/docker/**' - - 'ci/scripts/cpp_build.sh' - - 'ci/scripts/java_*' - - 'cpp/**' - - 'java/**' - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -permissions: - contents: read - -env: - DOCKER_VOLUME_PREFIX: ".docker/" - -jobs: - - docker: - name: AMD64 manylinux2014 Java JNI - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 90 - steps: - - name: Checkout Arrow - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: recursive - - name: Cache Docker Volumes - uses: actions/cache@v3 - with: - path: .docker - key: java-jni-manylinux-2014-${{ hashFiles('cpp/**', 'java/**') }} - restore-keys: java-jni-manylinux-2014- - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: 3.8 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - run: archery docker run java-jni-manylinux-2014 - - name: Docker Push - if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - continue-on-error: true - run: archery docker push java-jni-manylinux-2014 - - docker_integration_python: - name: AMD64 Conda Java C Data Interface Integration - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 90 - steps: - - name: Checkout Arrow - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: recursive - - name: Cache Docker Volumes - uses: actions/cache@v3 - with: - path: .docker - key: maven-${{ hashFiles('java/**') }} - restore-keys: maven- - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: 3.8 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - run: archery docker run conda-python-java-integration - - name: Docker Push - if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - continue-on-error: true - run: archery docker push conda-python-java-integration diff --git a/.github/workflows/java_nightly.yml b/.github/workflows/java_nightly.yml deleted file mode 100644 index b47306eec8b97..0000000000000 --- a/.github/workflows/java_nightly.yml +++ /dev/null @@ -1,144 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Upload Java Nightly builds - -on: - workflow_dispatch: - inputs: - prefix: - description: Job prefix to use. - required: false - default: '' - keep: - description: Number of versions to keep. - required: false - default: 14 - schedule: - - cron: '0 14 * * *' - -permissions: - contents: read - -jobs: - upload: - if: github.repository == 'apache/arrow' - env: - PREFIX: ${{ github.event.inputs.prefix || ''}} - CROSSBOW_GITHUB_TOKEN: ${{ github.token }} - runs-on: ubuntu-latest - steps: - - name: Checkout Arrow - uses: actions/checkout@v3 - with: - fetch-depth: 1 - path: arrow - repository: apache/arrow - ref: master - submodules: recursive - - name: Checkout Crossbow - uses: actions/checkout@v3 - with: - fetch-depth: 0 - path: crossbow - repository: ursacomputing/crossbow - ref: master - - name: Set up Python - uses: actions/setup-python@v4 - with: - cache: 'pip' - python-version: 3.8 - - name: Install Archery - shell: bash - run: pip install -e arrow/dev/archery[all] - - run: mkdir -p binaries - - name: Download Artifacts - run: | - if [ -z $PREFIX ]; then - PREFIX=nightly-packaging-$(date +%Y-%m-%d)-0 - fi - echo $PREFIX - archery crossbow download-artifacts -f java-jars -t binaries $PREFIX - - name: Cache Repo - uses: actions/cache@v3 - with: - path: repo - key: java-nightly-${{ github.run_id }} - restore-keys: java-nightly - - name: Sync from Remote - uses: ./arrow/.github/actions/sync-nightlies - with: - switches: -avzh --update --delete --progress - local_path: repo - remote_path: ${{ secrets.NIGHTLIES_RSYNC_PATH }}/arrow/java - remote_host: ${{ secrets.NIGHTLIES_RSYNC_HOST }} - remote_port: ${{ secrets.NIGHTLIES_RSYNC_PORT }} - remote_user: ${{ secrets.NIGHTLIES_RSYNC_USER }} - remote_key: ${{ secrets.NIGHTLIES_RSYNC_KEY }} - remote_host_key: ${{ secrets.NIGHTLIES_RSYNC_HOST_KEY }} - - shell: bash - name: Show local repo sync from remote - run: | - for i in `ls -t repo/org/apache/arrow`; do - echo "- $i: $(find repo/org/apache/arrow/$i -mindepth 1 -maxdepth 1 -type d \ - | wc -l \ - | xargs) versions available" - done - - shell: bash - name: Build Repository - run: | - DATE=$(date +%Y-%m-%d) - if [ -z $PREFIX ]; then - PREFIX=nightly-packaging-${DATE}-0 - fi - PATTERN_TO_GET_LIB_AND_VERSION='([a-z].+)-([0-9]+.[0-9]+.[0-9]+-SNAPSHOT)' - mkdir -p repo/org/apache/arrow/ - for LIBRARY in $(ls binaries/$PREFIX/java-jars | grep -E '.jar|.pom' | grep SNAPSHOT); do - [[ $LIBRARY =~ $PATTERN_TO_GET_LIB_AND_VERSION ]] - mkdir -p repo/org/apache/arrow/${BASH_REMATCH[1]}/${BASH_REMATCH[2]} - mkdir -p repo/org/apache/arrow/${BASH_REMATCH[1]}/${DATE} - # Copy twice to maintain a latest snapshot and some earlier versions - cp binaries/$PREFIX/java-jars/$LIBRARY repo/org/apache/arrow/${BASH_REMATCH[1]}/${BASH_REMATCH[2]} - cp binaries/$PREFIX/java-jars/$LIBRARY repo/org/apache/arrow/${BASH_REMATCH[1]}/${DATE} - echo "Artifacts $LIBRARY configured" - done - - name: Prune Repository - shell: bash - env: - KEEP: ${{ github.event.inputs.keep || 14 }} - run: | - for i in `ls -t repo/org/apache/arrow`; do - find repo/org/apache/arrow/$i -mindepth 1 -maxdepth 1 -type d -print0 \ - | xargs -0 ls -t -d \ - | tail -n +$((KEEP + 1)) \ - | xargs rm -rf - done - - name: Show repo contents - run: tree repo - - name: Sync to Remote - if: ${{ github.repository == 'apache/arrow' }} - uses: ./arrow/.github/actions/sync-nightlies - with: - upload: true - switches: -avzh --update --delete --progress - local_path: repo - remote_path: ${{ secrets.NIGHTLIES_RSYNC_PATH }}/arrow/java - remote_host: ${{ secrets.NIGHTLIES_RSYNC_HOST }} - remote_port: ${{ secrets.NIGHTLIES_RSYNC_PORT }} - remote_user: ${{ secrets.NIGHTLIES_RSYNC_USER }} - remote_key: ${{ secrets.NIGHTLIES_RSYNC_KEY }} - remote_host_key: ${{ secrets.NIGHTLIES_RSYNC_HOST_KEY }} diff --git a/.github/workflows/js.yml b/.github/workflows/js.yml deleted file mode 100644 index 239de36eee88f..0000000000000 --- a/.github/workflows/js.yml +++ /dev/null @@ -1,122 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: NodeJS - -on: - push: - paths: - - '.github/workflows/js.yml' - - 'ci/docker/*js.dockerfile' - - 'ci/scripts/js_*' - - 'js/**' - pull_request: - paths: - - '.github/workflows/js.yml' - - 'ci/docker/*js.dockerfile' - - 'ci/scripts/js_*' - - 'js/**' - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -permissions: - contents: read - -jobs: - - docker: - name: AMD64 Debian 11 NodeJS 16 - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 60 - steps: - - name: Checkout Arrow - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: 3.8 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - run: | - sudo sysctl -w kernel.core_pattern="core.%e.%p" - ulimit -c unlimited - archery docker run debian-js - - name: Docker Push - if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - continue-on-error: true - run: archery docker push debian-js - - macos: - name: AMD64 macOS 11 NodeJS ${{ matrix.node }} - runs-on: macos-latest - if: github.event_name == 'push' - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - node: [16] - steps: - - name: Checkout Arrow - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Install NodeJS - uses: actions/setup-node@v3 - with: - node-version: ${{ matrix.node }} - - name: Build - shell: bash - run: ci/scripts/js_build.sh $(pwd) - - name: Test - shell: bash - run: ci/scripts/js_test.sh $(pwd) - - windows: - name: AMD64 Windows NodeJS ${{ matrix.node }} - runs-on: windows-latest - if: github.event_name == 'push' - strategy: - fail-fast: false - matrix: - node: [16] - steps: - - name: Checkout Arrow - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Install NodeJS - uses: actions/setup-node@v3 - with: - node-version: ${{ matrix.node }} - - name: Build - shell: bash - run: ci/scripts/js_build.sh $(pwd) - - name: Test - shell: bash - run: ci/scripts/js_test.sh $(pwd) diff --git a/.github/workflows/matlab.yml b/.github/workflows/matlab.yml deleted file mode 100644 index 541ffcea831e3..0000000000000 --- a/.github/workflows/matlab.yml +++ /dev/null @@ -1,96 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: MATLAB - -on: - push: - paths: - - '.github/workflows/matlab.yml' - - 'ci/scripts/matlab*.sh' - - 'matlab/**' - - 'cpp/src/arrow/**' - pull_request: - paths: - - '.github/workflows/matlab.yml' - - 'ci/scripts/matlab*.sh' - - 'matlab/**' - - 'cpp/src/arrow/**' - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -permissions: - contents: read - -jobs: - - ubuntu: - name: AMD64 Ubuntu 20.04 MATLAB - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - steps: - - name: Check out repository - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Install ninja-build - run: sudo apt-get install ninja-build - - name: Install MATLAB - uses: matlab-actions/setup-matlab@v1 - - name: Build MATLAB Interface - run: ci/scripts/matlab_build.sh $(pwd) - - name: Run MATLAB Tests - env: - # libarrow.so requires a more recent version of libstdc++.so - # than is bundled with MATLAB under /sys/os/glnxa64. - # Therefore, if a MEX function that depends on libarrow.so - # is executed within the MATLAB address space, runtime linking - # errors will occur. To work around this issue, we can explicitly - # force MATLAB to use the system libstdc++.so via LD_PRELOAD. - LD_PRELOAD: /usr/lib/x86_64-linux-gnu/libstdc++.so.6 - - # Add the installation directory to the MATLAB Search Path by - # setting the MATLABPATH environment variable. - MATLABPATH: matlab/install/arrow_matlab - uses: matlab-actions/run-tests@v1 - with: - select-by-folder: matlab/test - macos: - name: AMD64 macOS 11 MATLAB - runs-on: macos-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - steps: - - name: Check out repository - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Install ninja-build - run: brew install ninja - - name: Install MATLAB - uses: matlab-actions/setup-matlab@v1 - - name: Build MATLAB Interface - run: ci/scripts/matlab_build.sh $(pwd) - - name: Run MATLAB Tests - env: - # Add the installation directory to the MATLAB Search Path by - # setting the MATLABPATH environment variable. - MATLABPATH: matlab/install/arrow_matlab - uses: matlab-actions/run-tests@v1 - with: - select-by-folder: matlab/test diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml deleted file mode 100644 index 7275baca2bf83..0000000000000 --- a/.github/workflows/r.yml +++ /dev/null @@ -1,335 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: R - -on: - push: - paths: - - ".github/workflows/r.yml" - - "ci/scripts/r_*.sh" - - "ci/scripts/cpp_*.sh" - - "ci/scripts/PKGBUILD" - - "ci/etc/rprofile" - - "ci/docker/**" - - "cpp/**" - - "r/**" - pull_request: - paths: - - ".github/workflows/r.yml" - - "ci/scripts/r_*.sh" - - "ci/scripts/cpp_*.sh" - - "ci/scripts/PKGBUILD" - - "ci/etc/rprofile" - - "ci/docker/**" - - "cpp/**" - - "r/**" - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -permissions: - contents: read - -env: - DOCKER_VOLUME_PREFIX: ".docker/" - -jobs: - ubuntu: - name: AMD64 Ubuntu ${{ matrix.ubuntu }} R ${{ matrix.r }} Force-Tests ${{ matrix.force-tests }} - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 75 - strategy: - fail-fast: false - matrix: - r: ["4.2"] - ubuntu: [20.04] - force-tests: ["true"] - env: - R: ${{ matrix.r }} - UBUNTU: ${{ matrix.ubuntu }} - steps: - - name: Checkout Arrow - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Cache Docker Volumes - uses: actions/cache@v3 - with: - path: .docker - # As this key is identical on both matrix builds only one will be able to successfully cache, - # this is fine as there are no differences in the build - key: ubuntu-${{ matrix.ubuntu }}-r-${{ matrix.r }}-${{ hashFiles('cpp/src/**/*.cc','cpp/src/**/*.h)') }}-${{ github.run_id }} - restore-keys: | - ubuntu-${{ matrix.ubuntu }}-r-${{ matrix.r }}-${{ hashFiles('cpp/src/**/*.cc','cpp/src/**/*.h)') }}- - ubuntu-${{ matrix.ubuntu }}-r-${{ matrix.r }}- - - name: Check pkgdown reference sections - run: ci/scripts/r_pkgdown_check.sh - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: 3.8 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - run: | - sudo sysctl -w kernel.core_pattern="core.%e.%p" - ulimit -c unlimited - # Setting a non-default and non-probable Marquesas French Polynesia time - # it has both with a .45 offset and very very few people who live there. - archery docker run -e TZ=MART -e ARROW_R_FORCE_TESTS=${{ matrix.force-tests }} ubuntu-r - - name: Dump install logs - run: cat r/check/arrow.Rcheck/00install.out - if: always() - - name: Dump test logs - run: cat r/check/arrow.Rcheck/tests/testthat.Rout* - if: always() - - name: Save the test output - if: always() - uses: actions/upload-artifact@v3 - with: - name: test-output - path: r/check/arrow.Rcheck/tests/testthat.Rout* - - name: Docker Push - if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - continue-on-error: true - run: archery docker push ubuntu-r - - bundled: - name: "${{ matrix.config.org }}/${{ matrix.config.image }}:${{ matrix.config.tag }}" - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - config: - - { org: "rhub", image: "debian-gcc-devel", tag: "latest", devtoolset: "" } - env: - R_ORG: ${{ matrix.config.org }} - R_IMAGE: ${{ matrix.config.image }} - R_TAG: ${{ matrix.config.tag }} - DEVTOOLSET_VERSION: ${{ matrix.config.devtoolset }} - steps: - - name: Checkout Arrow - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: 3.8 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - run: | - sudo sysctl -w kernel.core_pattern="core.%e.%p" - ulimit -c unlimited - # Don't set a TZ here to test that case. These builds will have the following warning in them: - # System has not been booted with systemd as init system (PID 1). Can't operate. - # Failed to connect to bus: Host is down - archery docker run -e TZ="" r - - name: Dump install logs - run: cat r/check/arrow.Rcheck/00install.out - if: always() - - name: Dump test logs - run: cat r/check/arrow.Rcheck/tests/testthat.Rout* - if: always() - - name: Save the test output - if: always() - uses: actions/upload-artifact@v3 - with: - name: test-output - path: r/check/arrow.Rcheck/tests/testthat.Rout* - - name: Docker Push - if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - continue-on-error: true - run: archery docker push r - - windows-cpp: - name: AMD64 Windows C++ RTools ${{ matrix.config.rtools }} ${{ matrix.config.arch }} - runs-on: windows-2019 - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 90 - strategy: - fail-fast: false - matrix: - config: - - { rtools: 40, arch: 'ucrt64' } - steps: - - run: git config --global core.autocrlf false - - name: Checkout Arrow - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Setup ccache - shell: bash - run: | - ci/scripts/ccache_setup.sh - echo "CCACHE_DIR=$(cygpath --absolute --windows ccache)" >> $GITHUB_ENV - - name: Cache ccache - uses: actions/cache@v3 - with: - path: ccache - key: r-${{ matrix.config.rtools }}-ccache-mingw-${{ matrix.config.arch }}-${{ hashFiles('cpp/src/**/*.cc','cpp/src/**/*.h)') }}-${{ github.run_id }} - restore-keys: | - r-${{ matrix.config.rtools }}-ccache-mingw-${{ matrix.config.arch }}-${{ hashFiles('cpp/src/**/*.cc','cpp/src/**/*.h)') }}- - r-${{ matrix.config.rtools }}-ccache-mingw-${{ matrix.config.arch }}- - - uses: r-lib/actions/setup-r@v2 - with: - r-version: "4.1" - rtools-version: 40 - Ncpus: 2 - - name: Build Arrow C++ - shell: bash - env: - RTOOLS_VERSION: ${{ matrix.config.rtools }} - MINGW_ARCH: ${{ matrix.config.arch }} - run: ci/scripts/r_windows_build.sh - - name: Rename libarrow.zip - # So that they're unique when multiple are downloaded in the next step - shell: bash - run: mv libarrow.zip libarrow-rtools${{ matrix.config.rtools }}-${{ matrix.config.arch }}.zip - - uses: actions/upload-artifact@v3 - with: - name: libarrow-rtools${{ matrix.config.rtools }}-${{ matrix.config.arch }}.zip - path: libarrow-rtools${{ matrix.config.rtools }}-${{ matrix.config.arch }}.zip - - windows-r: - needs: [windows-cpp] - name: AMD64 Windows R ${{ matrix.config.rversion }} RTools ${{ matrix.config.rtools }} - runs-on: windows-2019 - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 75 - strategy: - fail-fast: false - matrix: - config: - - { rtools: 42, rversion: "4.2" } - - { rtools: 42, rversion: "devel" } - env: - ARROW_R_CXXFLAGS: "-Werror" - _R_CHECK_TESTS_NLINES_: 0 - steps: - - run: git config --global core.autocrlf false - - name: Checkout Arrow - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - run: mkdir r/windows - - name: Download artifacts - if: ${{ matrix.config.rtools == 42 }} - uses: actions/download-artifact@v3 - with: - name: libarrow-rtools40-ucrt64.zip - path: r/windows - - name: Unzip and rezip libarrows - shell: bash - run: | - cd r/windows - ls *.zip | xargs -n 1 unzip -uo - rm -rf *.zip - - uses: r-lib/actions/setup-r@v2 - with: - r-version: ${{ matrix.config.rversion }} - rtools-version: ${{ matrix.config.rtools }} - Ncpus: 2 - - uses: r-lib/actions/setup-r-dependencies@v2 - env: - GITHUB_PAT: "${{ github.token }}" - with: - # For some arcane reason caching does not work on the windows runners - # most likely due to https://github.com/actions/cache/issues/815 - cache: false - working-directory: 'r' - extra-packages: | - any::rcmdcheck - - name: Install MinIO - shell: bash - run: | - mkdir -p "$HOME/.local/bin" - curl \ - --output "$HOME/.local/bin/minio.exe" \ - https://dl.min.io/server/minio/release/windows-amd64/archive/minio.RELEASE.2022-05-26T05-48-41Z - chmod +x "$HOME/.local/bin/minio.exe" - echo "$HOME/.local/bin" >> $GITHUB_PATH - # TODO(ARROW-17149): figure out why the GCS tests are hanging on Windows - # - name: Install Google Cloud Storage Testbench - # shell: bash - # run: ci/scripts/install_gcs_testbench.sh default - - name: Check - shell: Rscript {0} - run: | - # Because we do R CMD build and r/windows is in .Rbuildignore, - # assemble the libarrow.zip file and pass it as an env var - setwd("r/windows") - zip("libarrow.zip", ".") - setwd("..") - - Sys.setenv( - RWINLIB_LOCAL = file.path(Sys.getenv("GITHUB_WORKSPACE"), "r", "windows", "libarrow.zip"), - MAKEFLAGS = paste0("-j", parallel::detectCores()), - ARROW_R_DEV = TRUE, - "_R_CHECK_FORCE_SUGGESTS_" = FALSE - ) - rcmdcheck::rcmdcheck(".", - build_args = '--no-build-vignettes', - args = c('--no-manual', '--as-cran', '--ignore-vignettes', '--run-donttest'), - error_on = 'warning', - check_dir = 'check', - timeout = 3600 - ) - - name: Run lintr - if: ${{ matrix.config.rversion == '4.2' }} - env: - NOT_CRAN: "true" - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - shell: Rscript {0} - working-directory: r - run: | - Sys.setenv( - RWINLIB_LOCAL = file.path(Sys.getenv("GITHUB_WORKSPACE"), "r", "windows", "libarrow.zip"), - MAKEFLAGS = paste0("-j", parallel::detectCores()), - ARROW_R_DEV = TRUE, - "_R_CHECK_FORCE_SUGGESTS_" = FALSE - ) - # we use pak for package installation since it is faster, safer and more convenient - pak::local_install() - pak::pak("lintr") - lintr::expect_lint_free() - - name: Dump install logs - shell: cmd - run: cat r/check/arrow.Rcheck/00install.out - if: always() - - name: Dump test logs - shell: bash - run: find r/check -name 'testthat.Rout*' -exec cat '{}' \; || true - if: always() diff --git a/.github/workflows/r_nightly.yml b/.github/workflows/r_nightly.yml deleted file mode 100644 index 8d10bee30d0fb..0000000000000 --- a/.github/workflows/r_nightly.yml +++ /dev/null @@ -1,192 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Upload R Nightly builds -# This workflow downloads the (nightly) binaries created in crossbow and uploads them -# to nightlies.apache.org. Due to authorization requirements, this upload can't be done -# from the crossbow repository. - -on: - workflow_dispatch: - inputs: - prefix: - description: Job prefix to use. - required: false - default: '' - keep: - description: Number of versions to keep. - required: false - default: 14 - - schedule: - #Crossbow packaging runs at 0 8 * * * - - cron: '0 14 * * *' - -permissions: - contents: read - -jobs: - upload: - if: github.repository == 'apache/arrow' - runs-on: ubuntu-latest - steps: - - name: Checkout Arrow - uses: actions/checkout@v3 - with: - fetch-depth: 1 - path: arrow - repository: apache/arrow - ref: master - submodules: recursive - - name: Checkout Crossbow - uses: actions/checkout@v3 - with: - fetch-depth: 0 - path: crossbow - repository: ursacomputing/crossbow - ref: master - - name: Set up Python - uses: actions/setup-python@v4 - with: - cache: 'pip' - python-version: 3.8 - - name: Install Archery - shell: bash - run: pip install -e arrow/dev/archery[all] - - run: mkdir -p binaries - - name: Download Artifacts - env: - PREFIX: ${{ github.event.inputs.prefix || ''}} - run: | - if [ -z $PREFIX ]; then - PREFIX=nightly-packaging-$(date +%Y-%m-%d)-0 - fi - echo $PREFIX - - archery crossbow download-artifacts -f r-binary-packages -t binaries $PREFIX - - if [ -n "$(ls -A binaries/*/*/)" ]; then - echo "Found files!" - else - echo "No files found. Stopping upload." - exit 1 - fi - - name: Cache Repo - uses: actions/cache@v3 - with: - path: repo - key: r-nightly-${{ github.run_id }} - restore-keys: r-nightly- - - name: Sync from Remote - uses: ./arrow/.github/actions/sync-nightlies - with: - switches: -avzh --update --delete --progress - local_path: repo - remote_path: ${{ secrets.NIGHTLIES_RSYNC_PATH }}/arrow/r - remote_host: ${{ secrets.NIGHTLIES_RSYNC_HOST }} - remote_port: ${{ secrets.NIGHTLIES_RSYNC_PORT }} - remote_user: ${{ secrets.NIGHTLIES_RSYNC_USER }} - remote_key: ${{ secrets.NIGHTLIES_RSYNC_KEY }} - remote_host_key: ${{ secrets.NIGHTLIES_RSYNC_HOST_KEY }} - - run: tree repo - - name: Build Repository - shell: Rscript {0} - run: | - # folder that we sync to nightlies.apache.org - repo_root <- "repo" - # The binaries are in a nested dir - # so we need to find the correct path. - art_path <- list.files("binaries", - recursive = TRUE, - include.dirs = TRUE, - pattern = "r-binary-packages$", - full.names = TRUE - ) - - current_path <- list.files(art_path, full.names = TRUE, recursive = TRUE) - files <- sub("r-(pkg|lib)", repo_root, current_path) - - # decode contrib.url from artifact name: - # bin__windows__contrib__4.1 -> bin/windows/contrib/4.1 - new_paths <- gsub("__", "/", files) - # strip superfluous nested dirs - new_paths <- sub(art_path, ".", new_paths) - dirs <- dirname(new_paths) - sapply(dirs, dir.create, recursive = TRUE, showWarnings = FALSE) - - # overwrite allows us to "force push" a new version with the same name - copy_result <- file.copy(current_path, new_paths, overwrite = TRUE) - - if (!all(copy_result)) { - stop("There was an issue while copying the files!") - } - - name: Prune Repository - shell: bash - env: - KEEP: ${{ github.event.inputs.keep || 14 }} - run: | - prune() { - # list files | retain $KEEP newest files | delete everything else - ls -t $1/arrow* | tail -n +$((KEEP + 1)) | xargs --no-run-if-empty rm - } - - # find leaf sub dirs - repo_dirs=$(find repo -type d -links 2) - - # We want to retain $keep (14) versions of each pkg/lib so we call - # prune on each leaf dir and not on repo/. - for dir in ${repo_dirs[@]}; do - prune $dir - done - - name: Update Repository Index - shell: Rscript {0} - run: | - # folder that we sync to nightlies.apache.org - repo_root <- "repo" - tools::write_PACKAGES(file.path(repo_root, "src/contrib"), - type = "source", - verbose = TRUE, - latestOnly = FALSE - ) - - repo_dirs <- list.dirs(repo_root) - # find dirs with binary R packages: e.g. */contrib/4.1 - pkg_dirs <- grep(".+contrib\\/\\d.+", repo_dirs, value = TRUE) - - - for (dir in pkg_dirs) { - on_win <- grepl("windows", dir) - tools::write_PACKAGES(dir, - type = ifelse(on_win, "win.binary", "mac.binary"), - verbose = TRUE, - latestOnly = FALSE - ) - } - - name: Show repo contents - run: tree repo - - name: Sync to Remote - uses: ./arrow/.github/actions/sync-nightlies - with: - upload: true - switches: -avzh --update --delete --progress - local_path: repo - remote_path: ${{ secrets.NIGHTLIES_RSYNC_PATH }}/arrow/r - remote_host: ${{ secrets.NIGHTLIES_RSYNC_HOST }} - remote_port: ${{ secrets.NIGHTLIES_RSYNC_PORT }} - remote_user: ${{ secrets.NIGHTLIES_RSYNC_USER }} - remote_key: ${{ secrets.NIGHTLIES_RSYNC_KEY }} - remote_host_key: ${{ secrets.NIGHTLIES_RSYNC_HOST_KEY }} diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml deleted file mode 100644 index 123c1e25a110c..0000000000000 --- a/.github/workflows/ruby.yml +++ /dev/null @@ -1,306 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: C GLib & Ruby - -on: - push: - paths: - - '.github/workflows/ruby.yml' - - 'ci/docker/**' - - 'ci/scripts/c_glib_*' - - 'ci/scripts/cpp_*' - - 'ci/scripts/msys2_*' - - 'ci/scripts/ruby_*' - - 'ci/scripts/util_*' - - 'c_glib/**' - - 'cpp/**' - - 'ruby/**' - pull_request: - paths: - - '.github/workflows/ruby.yml' - - 'ci/docker/**' - - 'ci/scripts/c_glib_*' - - 'ci/scripts/cpp_*' - - 'ci/scripts/msys2_*' - - 'ci/scripts/ruby_*' - - 'ci/scripts/util_*' - - 'c_glib/**' - - 'cpp/**' - - 'ruby/**' - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -permissions: - contents: read - -env: - DOCKER_VOLUME_PREFIX: ".docker/" - -jobs: - - ubuntu: - name: AMD64 Ubuntu ${{ matrix.ubuntu }} GLib & Ruby - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - ubuntu: - - 20.04 - env: - UBUNTU: ${{ matrix.ubuntu }} - steps: - - name: Checkout Arrow - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: recursive - - name: Cache Docker Volumes - uses: actions/cache@v3 - with: - path: .docker - key: ubuntu-${{ matrix.ubuntu }}-ruby-${{ hashFiles('cpp/**') }} - restore-keys: ubuntu-${{ matrix.ubuntu }}-ruby- - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: 3.8 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - run: | - sudo sysctl -w kernel.core_pattern="core.%e.%p" - ulimit -c unlimited - archery docker run \ - -e ARROW_FLIGHT=ON \ - -e ARROW_FLIGHT_SQL=ON \ - -e ARROW_GCS=ON \ - -e Protobuf_SOURCE=BUNDLED \ - -e gRPC_SOURCE=BUNDLED \ - ubuntu-ruby - - name: Docker Push - if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - continue-on-error: true - shell: bash - run: archery docker push ubuntu-ruby - - macos: - name: AMD64 macOS 11 GLib & Ruby - runs-on: macos-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 60 - env: - ARROW_BUILD_STATIC: OFF - ARROW_BUILD_TESTS: OFF - ARROW_BUILD_UTILITIES: OFF - ARROW_FLIGHT: ON - ARROW_FLIGHT_SQL: ON - ARROW_GANDIVA: ON - ARROW_GCS: ON - ARROW_GLIB_GTK_DOC: true - ARROW_GLIB_WERROR: true - ARROW_HOME: /usr/local - ARROW_JEMALLOC: OFF - ARROW_ORC: OFF - ARROW_PARQUET: ON - ARROW_WITH_BROTLI: ON - ARROW_WITH_LZ4: ON - ARROW_WITH_SNAPPY: ON - ARROW_WITH_ZLIB: ON - ARROW_WITH_ZSTD: ON - XML_CATALOG_FILES: /usr/local/etc/xml/catalog - steps: - - name: Checkout Arrow - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: recursive - - name: Install Homebrew Dependencies - shell: bash - run: | - rm -f /usr/local/bin/2to3 - brew update --preinstall - brew install --overwrite git - brew bundle --file=cpp/Brewfile - brew bundle --file=c_glib/Brewfile - - name: Install Ruby Dependencies - run: | - export MAKEFLAGS="-j$(sysctl -n hw.ncpu)" - bundle install --gemfile c_glib/Gemfile - bundle install --gemfile ruby/Gemfile - for ruby_package_gemfile in ruby/*/Gemfile; do \ - bundle install --gemfile ${ruby_package_gemfile} - done - - name: Setup ccache - run: | - ci/scripts/ccache_setup.sh - - name: ccache info - id: ccache-info - run: | - echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT - - name: Cache ccache - uses: actions/cache@v3 - with: - path: ${{ steps.ccache-info.outputs.cache-dir }} - key: ruby-ccache-macos-${{ hashFiles('cpp/**') }} - restore-keys: ruby-ccache-macos- - - name: Build C++ - run: | - ci/scripts/cpp_build.sh $(pwd) $(pwd)/build - - name: Build GLib - run: | - ci/scripts/c_glib_build.sh $(pwd) $(pwd)/build - - name: Test GLib - shell: bash - run: ci/scripts/c_glib_test.sh $(pwd) $(pwd)/build - - name: Test Ruby - shell: bash - run: ci/scripts/ruby_test.sh $(pwd) $(pwd)/build - - windows: - name: AMD64 Windows MinGW ${{ matrix.mingw-n-bits }} GLib & Ruby - runs-on: windows-2019 - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 90 - strategy: - fail-fast: false - matrix: - mingw-n-bits: - - 64 - ruby-version: - - "3.1" - env: - ARROW_BUILD_STATIC: OFF - ARROW_BUILD_TESTS: OFF - ARROW_BUILD_UTILITIES: OFF - ARROW_BUILD_TYPE: release - ARROW_FLIGHT: ON - ARROW_FLIGHT_SQL: ON - # ARROW-17728: SEGV on MinGW - ARROW_GANDIVA: OFF - ARROW_GCS: ON - ARROW_HDFS: OFF - ARROW_HOME: /ucrt${{ matrix.mingw-n-bits }} - ARROW_JEMALLOC: OFF - ARROW_PARQUET: ON - ARROW_PYTHON: OFF - ARROW_S3: ON - ARROW_USE_GLOG: OFF - ARROW_WITH_BROTLI: ON - ARROW_WITH_BZ2: ON - ARROW_WITH_LZ4: ON - ARROW_WITH_SNAPPY: ON - ARROW_WITH_ZLIB: ON - ARROW_WITH_ZSTD: ON - # Don't use preinstalled Boost by empty BOOST_ROOT and - # -DBoost_NO_BOOST_CMAKE=ON - BOOST_ROOT: "" - CMAKE_ARGS: >- - -DARROW_PACKAGE_PREFIX=/ucrt${{ matrix.mingw-n-bits }} - -DBoost_NO_BOOST_CMAKE=ON - CMAKE_UNITY_BUILD: ON - steps: - - name: Disable Crash Dialogs - run: | - reg add ` - "HKCU\SOFTWARE\Microsoft\Windows\Windows Error Reporting" ` - /v DontShowUI ` - /t REG_DWORD ` - /d 1 ` - /f - - name: Checkout Arrow - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: recursive - - name: Setup Ruby - uses: ruby/setup-ruby@v1 - with: - ruby-version: ${{ matrix.ruby-version }} - - name: Upgrade MSYS2 - run: | - ridk exec bash ci\scripts\msys2_system_upgrade.sh - taskkill /F /FI "MODULES eq msys-2.0.dll" - - name: Clean MSYS2 - run: | - ridk exec bash ci\scripts\msys2_system_clean.sh - - name: Setup MSYS2 - run: | - ridk exec bash ci\scripts\msys2_setup.sh ruby - - name: Cache ccache - uses: actions/cache@v3 - with: - path: ccache - key: ruby-ccache-ucrt${{ matrix.mingw-n-bits }}-${{ hashFiles('cpp/**') }} - restore-keys: ruby-ccache-ucrt${{ matrix.mingw-n-bits }}- - - name: Build C++ - run: | - $Env:CMAKE_BUILD_PARALLEL_LEVEL = $Env:NUMBER_OF_PROCESSORS - $source_dir = "$(ridk exec cygpath --unix "$(Get-Location)")" - $build_dir = "$(ridk exec cygpath --unix "$(Get-Location)\build")" - $ErrorActionPreference = "Continue" - ridk exec bash ci\scripts\cpp_build.sh "${source_dir}" "${build_dir}" - - name: Build GLib - run: | - $Env:CMAKE_BUILD_PARALLEL_LEVEL = $Env:NUMBER_OF_PROCESSORS - $source_dir = "$(ridk exec cygpath --unix "$(Get-Location)")" - $build_dir = "$(ridk exec cygpath --unix "$(Get-Location)\build")" - $ErrorActionPreference = "Continue" - ridk exec bash ci\scripts\c_glib_build.sh "${source_dir}" "${build_dir}" - - name: RubyGems info - id: rubygems-info - run: | - Write-Output "gem-dir=$(ridk exec gem env gemdir)" | ` - Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append - - name: Cache RubyGems - uses: actions/cache@v3 - with: - path: ${{ steps.rubygems-info.outputs.gem-dir }} - key: ruby-rubygems-ucrt${{ matrix.mingw-n-bits }}-${{ hashFiles('**/Gemfile', 'ruby/*/*.gemspec') }} - restore-keys: ruby-rubygems-ucrt${{ matrix.mingw-n-bits }}- - - name: Install test dependencies - run: | - bundle install --gemfile c_glib\Gemfile - bundle install --gemfile ruby\Gemfile - Get-ChildItem ruby\*\Gemfile | ` - ForEach-Object {bundle install --gemfile $_} - - name: Test GLib - run: | - $source_dir = "$(ridk exec cygpath --unix "$(Get-Location)")" - $build_dir = "$(ridk exec cygpath --unix "$(Get-Location)\build")" - $ErrorActionPreference = "Continue" - ridk exec bash ci\scripts\c_glib_test.sh "${source_dir}" "${build_dir}" - - name: Test Ruby - run: | - $Env:PKG_CONFIG_PATH = ` - "$(ridk exec cygpath --absolute --windows "${Env:ARROW_HOME}/lib/pkgconfig")" - $Env:GI_TYPELIB_PATH = ` - "$(ridk exec cygpath --absolute --windows "${Env:ARROW_HOME}/lib/girepository-1.0")" - $Env:RUBYOPTS = "-rdevkit" - $Env:MAKE = "ridk exec make" - $ErrorActionPreference = "Continue" - rake -f ruby\Rakefile