From f1a9763d46885bb7f64bc5ceaa3e46742cebaea3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Wed, 6 Nov 2019 22:55:45 +0100 Subject: [PATCH] Docker refactor (#7) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ARROW-7067: [CI] Disable code coverage on Travis-CI Closes #5778 from pitrou/ARROW-7067-travis-disable-coverage and squashes the following commits: bda5ff4bd ARROW-7067: Disable code coverage on Travis-CI Authored-by: Antoine Pitrou Signed-off-by: Antoine Pitrou * ARROW-7058: [C++] FileSystemDataSourceDiscovery should apply partition schemes relative to its base dir @nealrichardson Closes #5772 from bkietz/7058-FileSystemDataSourceDisco and squashes the following commits: c0edfa557 make base_dir of partition schemes explicitly optional f2f96897f add DCHECK for path containing selector 1bae5ff8f ARROW-7058: FilSystemDataSourceDiscovery should apply partition schemes relative to its base dir Authored-by: Benjamin Kietzman Signed-off-by: François Saint-Jacques * Refactor docker-compose file and use it with github actions. * Turn off gandiva and flight for the HDFS test [skip ci] * Missing --pyargs argument for the python test command [skip ci] * Add CentOS version to the manylinux image names [skip ci] * Fix manylinux volumes [skip ci] --- .dockerignore | 101 +- .env | 40 + .github/workflows/cpp.yml | 179 +++ .github/workflows/csharp.yml | 66 + .github/workflows/dev.yml | 115 ++ .github/workflows/go.yml | 88 ++ .github/workflows/integration.yml | 53 + .github/workflows/java.yml | 122 ++ .github/workflows/js.yml | 118 ++ .github/workflows/linux-docker-compose.yml | 58 - .github/workflows/pull_request.yml | 2 + .github/workflows/python.yml | 229 +++ .github/workflows/r.yml | 79 + .github/workflows/ruby.yml | 116 ++ .github/workflows/rust.yml | 110 ++ .github/workflows/windows-msvc-cpp.yml | 54 - .travis.yml | 326 ---- README.md | 1 - appveyor.yml | 16 - ci/appveyor-build.bat | 16 - ci/conda_env_cpp.yml | 1 + ci/conda_env_python.yml | 2 +- .../docker/conda-latest-cpp.dockerfile | 61 +- .../conda-latest-integration.dockerfile | 39 +- .../conda-latest-python-dask.dockerfile} | 34 +- .../conda-latest-python-hdfs.dockerfile | 57 +- .../conda-latest-python-pandas.dockerfile} | 29 +- .../conda-latest-python-spark.dockerfile | 50 + .../conda-latest-python-turbodbc.dockerfile | 39 +- .../docker/conda-latest-python.dockerfile | 14 +- .../docker/conda-latest-r.dockerfile | 24 +- .../docker/cuda-10.0-cpp.dockerfile | 44 +- .../docker/debian-10-cpp.dockerfile | 47 +- ci/docker/debian-10-go.dockerfile | 25 + ci/docker/debian-10-js.dockerfile | 27 + ci/docker/debian-10-rust.dockerfile | 44 + ci/docker/debian-9-java.dockerfile | 28 + .../docker/fedora-29-cpp.dockerfile | 45 +- ci/docker/linux-apt-c-glib.dockerfile | 44 + .../docker/linux-apt-cmake.dockerfile | 15 +- .../docker/linux-apt-docs.dockerfile | 55 +- ci/docker/linux-apt-fuzzit.dockerfile | 47 + .../docker/linux-apt-jni.dockerfile | 72 +- ...kerfile.lint => linux-apt-lint.dockerfile} | 58 +- .../docker/linux-apt-python-3.dockerfile | 34 +- ci/docker/linux-apt-r.dockerfile | 67 + .../{lint.sh => linux-apt-ruby.dockerfile} | 18 +- ci/docker/linux-dnf-python-3.dockerfile | 39 + .../docker/ubuntu-14.04-cpp.dockerfile | 70 +- .../docker/ubuntu-16.04-cpp.dockerfile | 57 +- ci/docker/ubuntu-18.04-cpp.dockerfile | 119 ++ ci/docker/ubuntu-18.04-csharp.dockerfile | 20 + .../ubuntu-18.04-r-sanitizer.dockerfile | 30 +- ci/docker_build_and_fuzzit.sh | 52 - ci/docker_build_cpp.sh | 82 - ci/docker_build_python.sh | 52 - ci/docker_integration.sh | 120 -- ci/docker_java_test_all.sh | 34 - .../spark => ci/etc}/ARROW-6429.patch | 0 {integration/hdfs => ci/etc}/hdfs-site.xml | 0 ci/go-build-main.bat | 26 - ci/rust-build-main.bat | 60 - .../c_glib_build.sh} | 23 +- .../c_glib_test.sh} | 36 +- ci/scripts/cpp_build.sh | 125 ++ .../cpp_test.sh} | 22 +- .../csharp_build.sh} | 11 +- .../scripts/csharp_test.sh | 11 +- .../docs_build.sh} | 28 +- .../fuzzit_build.sh} | 33 +- .../Dockerfile => ci/scripts/go_build.sh | 14 +- ci/scripts/go_test.sh | 30 + ci/scripts/install_conda.sh | 66 + .../install_iwyu.sh} | 26 +- ci/scripts/install_minio.sh | 49 + .../scripts/integration_dask.sh | 0 .../scripts/integration_hdfs.sh | 10 +- .../scripts/integration_hiveserver2.sh | 19 +- .../scripts/integration_spark.sh | 28 +- .../scripts/integration_turbodbc.sh | 27 +- ci/scripts/java_build.sh | 50 + .../java_test.sh} | 40 +- .../js_build.sh} | 17 +- .../Dockerfile => ci/scripts/js_test.sh | 14 +- .../python_benchmark.sh} | 40 +- ci/scripts/python_build.sh | 50 + .../python_test.sh} | 16 +- ci/{docker_build_r.sh => scripts/r_build.sh} | 26 +- .../r_deps.sh} | 22 +- .../r_sanitize.sh} | 14 +- .../r_test.sh} | 21 +- ci/scripts/release_test.sh | 28 + ci/scripts/ruby_test.sh | 45 + .../rust_build.sh} | 40 +- .../rust_test.sh} | 36 +- .../scripts/wait_for_it.sh | 0 ci/travis_before_script_c_glib.sh | 81 - ci/travis_before_script_cpp.sh | 217 --- ci/travis_env_common.sh | 89 -- ci/travis_install_conda.sh | 72 - ci/travis_install_linux.sh | 92 -- ci/travis_install_minio.sh | 35 - ci/travis_install_osx.sh | 43 - ci/travis_install_toolchain.sh | 39 - ci/travis_lint.sh | 79 - ci/travis_release_test.sh | 32 - ci/travis_script_c_glib.sh | 54 - ci/travis_script_gandiva_java.sh | 35 - ci/travis_script_manylinux.sh | 83 - ci/travis_script_python.sh | 271 ---- ci/travis_script_ruby.sh | 85 -- ci/travis_upload_cpp_coverage.sh | 31 - ci/travis_upload_r_coverage.R | 26 - cpp/Brewfile | 3 + cpp/CMakeLists.txt | 9 - cpp/apidoc/Doxyfile | 2 +- cpp/src/arrow/array/diff_test.cc | 10 +- cpp/src/arrow/csv/parser_test.cc | 6 +- cpp/src/arrow/dataset/discovery.cc | 23 +- cpp/src/arrow/dataset/discovery.h | 7 +- cpp/src/arrow/dataset/discovery_test.cc | 32 +- cpp/src/arrow/dataset/partition.cc | 11 +- cpp/src/arrow/dataset/partition.h | 6 +- .../arrow/dbi/hiveserver2/hiveserver2_test.cc | 10 +- .../arrow/dbi/hiveserver2/thrift_internal.cc | 2 +- cpp/src/arrow/dbi/hiveserver2/util.cc | 6 +- cpp/src/arrow/filesystem/filesystem.h | 2 +- cpp/src/arrow/io/compressed_test.cc | 94 +- cpp/src/arrow/util/compression_benchmark.cc | 18 +- cpp/src/arrow/util/compression_test.cc | 11 +- dev/README.md | 23 +- dev/archery/archery/integration/util.py | 2 +- dev/container/Dockerfile | 48 - dev/container/README.md | 76 - dev/container/script/arrow-build.sh | 33 - dev/container/script/env.sh | 27 - dev/container/script/parquet-build.sh | 32 - dev/container/script/pyarrow-build.sh | 22 - dev/docker-compose.yml | 47 - dev/docker_common/Dockerfile.xenial.base | 66 - .../install_clang_tools_xenial.sh | 24 - dev/gen_apidocs.sh | 21 - dev/gen_apidocs/Dockerfile | 63 - dev/gen_apidocs/create_documents.sh | 97 -- dev/hiveserver2/Dockerfile | 23 - dev/hiveserver2/hiveserver2.sh | 66 - dev/lint/run_iwyu.sh | 38 - dev/release/02-source.sh | 10 +- dev/release/rat_exclude_files.txt | 2 +- dev/release/verify-release-candidate.sh | 18 +- dev/run_docker_compose.sh | 41 - dev/run_site/Dockerfile | 34 - dev/run_site/run_site.sh | 27 - dev/tasks/python-wheels/travis.linux.yml | 2 +- dev/tasks/tasks.yml | 836 +++++++---- docker-compose.yml | 1335 +++++++++-------- docs/Dockerfile | 24 - go/Dockerfile | 27 - integration/Dockerfile | 52 - integration/hdfs/Dockerfile | 54 - integration/spark/Dockerfile | 57 - java/Dockerfile | 29 - java/Dockerfile.all-jdks | 35 - js/Dockerfile | 32 - python/Dockerfile.alpine | 47 - python/Dockerfile.nopandas | 24 - python/manylinux1/README.md | 4 +- python/manylinux2010/README.md | 2 +- python/pyarrow/table.pxi | 1 + python/pyarrow/tests/test_hdfs.py | 3 + r/Dockerfile | 113 -- rust/Dockerfile | 54 - rust/README.md | 6 +- 173 files changed, 4469 insertions(+), 5411 deletions(-) create mode 100644 .env create mode 100644 .github/workflows/cpp.yml create mode 100644 .github/workflows/csharp.yml create mode 100644 .github/workflows/dev.yml create mode 100644 .github/workflows/go.yml create mode 100644 .github/workflows/integration.yml create mode 100644 .github/workflows/java.yml create mode 100644 .github/workflows/js.yml delete mode 100644 .github/workflows/linux-docker-compose.yml create mode 100644 .github/workflows/python.yml create mode 100644 .github/workflows/r.yml create mode 100644 .github/workflows/ruby.yml create mode 100644 .github/workflows/rust.yml delete mode 100644 .github/workflows/windows-msvc-cpp.yml delete mode 100644 .travis.yml rename cpp/Dockerfile => ci/docker/conda-latest-cpp.dockerfile (57%) rename integration/dask/Dockerfile => ci/docker/conda-latest-integration.dockerfile (53%) rename ci/{travis_script_go.sh => docker/conda-latest-python-dask.dockerfile} (65%) mode change 100755 => 100644 rename cpp/Dockerfile.alpine => ci/docker/conda-latest-python-hdfs.dockerfile (52%) rename ci/{docker_build_and_test_cpp.sh => docker/conda-latest-python-pandas.dockerfile} (64%) mode change 100755 => 100644 create mode 100644 ci/docker/conda-latest-python-spark.dockerfile rename integration/turbodbc/Dockerfile => ci/docker/conda-latest-python-turbodbc.dockerfile (55%) rename python/Dockerfile => ci/docker/conda-latest-python.dockerfile (86%) rename r/Dockerfile.conda => ci/docker/conda-latest-r.dockerfile (77%) rename cpp/Dockerfile.ubuntu-bionic => ci/docker/cuda-10.0-cpp.dockerfile (66%) rename cpp/Dockerfile.debian-testing => ci/docker/debian-10-cpp.dockerfile (73%) create mode 100644 ci/docker/debian-10-go.dockerfile create mode 100644 ci/docker/debian-10-js.dockerfile create mode 100644 ci/docker/debian-10-rust.dockerfile create mode 100644 ci/docker/debian-9-java.dockerfile rename cpp/Dockerfile.fedora => ci/docker/fedora-29-cpp.dockerfile (67%) create mode 100644 ci/docker/linux-apt-c-glib.dockerfile rename dev/fuzzit/Dockerfile => ci/docker/linux-apt-cmake.dockerfile (73%) rename c_glib/Dockerfile => ci/docker/linux-apt-docs.dockerfile (50%) create mode 100644 ci/docker/linux-apt-fuzzit.dockerfile rename cpp/Dockerfile.cmake => ci/docker/linux-apt-jni.dockerfile (50%) rename ci/docker/{Dockerfile.lint => linux-apt-lint.dockerfile} (61%) rename dev/lint/Dockerfile => ci/docker/linux-apt-python-3.dockerfile (60%) create mode 100644 ci/docker/linux-apt-r.dockerfile rename ci/docker/{lint.sh => linux-apt-ruby.dockerfile} (80%) mode change 100755 => 100644 create mode 100644 ci/docker/linux-dnf-python-3.dockerfile rename cpp/Dockerfile.ubuntu-trusty => ci/docker/ubuntu-14.04-cpp.dockerfile (65%) rename cpp/Dockerfile.ubuntu-xenial => ci/docker/ubuntu-16.04-cpp.dockerfile (75%) create mode 100644 ci/docker/ubuntu-18.04-cpp.dockerfile create mode 100644 ci/docker/ubuntu-18.04-csharp.dockerfile rename r/Dockerfile.sanitizer => ci/docker/ubuntu-18.04-r-sanitizer.dockerfile (85%) delete mode 100755 ci/docker_build_and_fuzzit.sh delete mode 100755 ci/docker_build_cpp.sh delete mode 100755 ci/docker_build_python.sh delete mode 100755 ci/docker_integration.sh delete mode 100755 ci/docker_java_test_all.sh rename {integration/spark => ci/etc}/ARROW-6429.patch (100%) rename {integration/hdfs => ci/etc}/hdfs-site.xml (100%) delete mode 100644 ci/go-build-main.bat delete mode 100644 ci/rust-build-main.bat rename ci/{docker_build_c_glib.sh => scripts/c_glib_build.sh} (78%) rename ci/{travis_script_cpp.sh => scripts/c_glib_test.sh} (58%) create mode 100755 ci/scripts/cpp_build.sh rename ci/{travis_script_java.sh => scripts/cpp_test.sh} (70%) rename ci/{docker_build_and_test_python.sh => scripts/csharp_build.sh} (89%) rename dev/Dockerfile.context => ci/scripts/csharp_test.sh (89%) mode change 100644 => 100755 rename ci/{docker_build_sphinx.sh => scripts/docs_build.sh} (54%) rename ci/{travis_script_plasma_java_client.sh => scripts/fuzzit_build.sh} (60%) rename integration/pandas/Dockerfile => ci/scripts/go_build.sh (85%) mode change 100644 => 100755 create mode 100755 ci/scripts/go_test.sh create mode 100755 ci/scripts/install_conda.sh rename ci/{docker_install_iwyu.sh => scripts/install_iwyu.sh} (63%) create mode 100755 ci/scripts/install_minio.sh rename integration/dask/runtest.sh => ci/scripts/integration_dask.sh (100%) rename integration/hdfs/runtest.sh => ci/scripts/integration_hdfs.sh (90%) rename dev/run_site.sh => ci/scripts/integration_hiveserver2.sh (76%) rename integration/spark/runtest.sh => ci/scripts/integration_spark.sh (76%) rename integration/turbodbc/runtest.sh => ci/scripts/integration_turbodbc.sh (69%) create mode 100755 ci/scripts/java_build.sh rename ci/{docker_build_java.sh => scripts/java_test.sh} (53%) rename ci/{travis_script_js.sh => scripts/js_build.sh} (80%) rename csharp/build/docker/Dockerfile => ci/scripts/js_test.sh (87%) mode change 100644 => 100755 rename ci/{docker_install_conda.sh => scripts/python_benchmark.sh} (50%) create mode 100755 ci/scripts/python_build.sh rename ci/{travis_release_audit.sh => scripts/python_test.sh} (77%) rename ci/{docker_build_r.sh => scripts/r_build.sh} (63%) rename ci/{docker_build_r_sanitizer.sh => scripts/r_deps.sh} (75%) rename ci/{travis_before_script_js.sh => scripts/r_sanitize.sh} (82%) rename ci/{travis_before_script_ruby.sh => scripts/r_test.sh} (78%) create mode 100755 ci/scripts/release_test.sh create mode 100755 ci/scripts/ruby_test.sh rename ci/{docker_build_rust.sh => scripts/rust_build.sh} (69%) rename ci/{travis_install_clang_tools.sh => scripts/rust_test.sh} (56%) rename dev/docker_common/wait-for-it.sh => ci/scripts/wait_for_it.sh (100%) delete mode 100755 ci/travis_before_script_c_glib.sh delete mode 100755 ci/travis_before_script_cpp.sh delete mode 100755 ci/travis_env_common.sh delete mode 100755 ci/travis_install_conda.sh delete mode 100755 ci/travis_install_linux.sh delete mode 100755 ci/travis_install_minio.sh delete mode 100755 ci/travis_install_osx.sh delete mode 100755 ci/travis_install_toolchain.sh delete mode 100755 ci/travis_lint.sh delete mode 100755 ci/travis_release_test.sh delete mode 100755 ci/travis_script_c_glib.sh delete mode 100755 ci/travis_script_gandiva_java.sh delete mode 100755 ci/travis_script_manylinux.sh delete mode 100755 ci/travis_script_python.sh delete mode 100755 ci/travis_script_ruby.sh delete mode 100755 ci/travis_upload_cpp_coverage.sh delete mode 100644 ci/travis_upload_r_coverage.R delete mode 100644 dev/container/Dockerfile delete mode 100644 dev/container/README.md delete mode 100644 dev/container/script/arrow-build.sh delete mode 100644 dev/container/script/env.sh delete mode 100644 dev/container/script/parquet-build.sh delete mode 100644 dev/container/script/pyarrow-build.sh delete mode 100644 dev/docker-compose.yml delete mode 100644 dev/docker_common/Dockerfile.xenial.base delete mode 100755 dev/docker_common/install_clang_tools_xenial.sh delete mode 100755 dev/gen_apidocs.sh delete mode 100644 dev/gen_apidocs/Dockerfile delete mode 100755 dev/gen_apidocs/create_documents.sh delete mode 100644 dev/hiveserver2/Dockerfile delete mode 100755 dev/hiveserver2/hiveserver2.sh delete mode 100755 dev/lint/run_iwyu.sh delete mode 100755 dev/run_docker_compose.sh delete mode 100644 dev/run_site/Dockerfile delete mode 100755 dev/run_site/run_site.sh delete mode 100644 docs/Dockerfile delete mode 100644 go/Dockerfile delete mode 100644 integration/Dockerfile delete mode 100644 integration/hdfs/Dockerfile delete mode 100644 integration/spark/Dockerfile delete mode 100644 java/Dockerfile delete mode 100644 java/Dockerfile.all-jdks delete mode 100644 js/Dockerfile delete mode 100644 python/Dockerfile.alpine delete mode 100644 python/Dockerfile.nopandas delete mode 100644 r/Dockerfile delete mode 100644 rust/Dockerfile diff --git a/.dockerignore b/.dockerignore index 3df8c2ba5f836..b96eecdbad8f0 100644 --- a/.dockerignore +++ b/.dockerignore @@ -15,89 +15,18 @@ # specific language governing permissions and limitations # under the License. -.git -docker_cache -docs/_build - -# IDE -.idea -.vscode -*/.vscode - -# c_glib -c_glib/build -c_glib/autom4te.cache -c_glib/m4 -c_glib/*/*.o -c_glib/*/*.lo -c_glib/*/*.la -c_glib/*/.deps -c_glib/*/.libs - -# cpp -cpp/.idea -cpp/build -cpp/*-build -cpp/*_build -cpp/build-debug -cpp/build-release -cpp/build-test -cpp/Testing -cpp/thirdparty -!cpp/thirdparty/jemalloc -!cpp/thirdparty/versions.txt -!cpp/thirdparty/hadoop/include - -# python -python/build -python/dist -python/manylinux1/dist -python/*.egg-info -python/*.egg -python/*.pyc -__pycache__/ -*/__pycache__/ -*/*/__pycache__/ -*/*/*/__pycache__/ -*.py[cod] -*/*.py[cod] -*/*/*.py[cod] -*/*/*/*.py[cod] -*.so -*/*.so -*/*/*.so -*/*/*/*.so -*.dylib -*/*.dylib -*/*/*.dylib -*/*/*/*.dylib - -# JS -js/.npm -js/node_modules -js/jspm_packages - -js/logs -js/*.log -js/.esm-cache -js/npm-debug.log* -js/yarn-debug.log* -js/yarn-error.log* - -js/.grunt -js/bower_components -js/.lock-wscript -js/build/Release -js/dist -js/targets -js/test/data/**/*.json -js/test/data/**/*.arrow - -# Rust -rust/target -rust/arrow/target -rust/parquet/target -rust/datafusion/target - -# Site -site/ +# Turn .dockerignore to .dockerallow by excluding everything and explicitly +# allowing specific files and directories. This enables us to quickly add +# dependency files to the docker contect without scanning the whole directory. +# This setup requires to all of our docker containers have arrow's source +# as a mounted directory. + +# exclude everything +** + +# include explicitly +!ci/** +!c_glib/Gemfile +!python/requirements*.txt +!python/manylinux1/** +!python/manylinux2010/** diff --git a/.env b/.env new file mode 100644 index 0000000000000..c217fe4e0a750 --- /dev/null +++ b/.env @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# All of the following environment variables are required to set default values +# for the parameters in docker-compose.yml. + +ORG=arrowdev +ARCH=amd64 +CUDA=10.0 +DEBIAN=10 +UBUNTU=18.04 +FEDORA=29 +PYTHON=3.6 +RUST=nightly-2019-09-25 +GO=1.12 +NODE=11 +MAVEN=3.5.4 +JDK=8 +R=3.6.1 +PANDAS=latest +DASK=latest +TURBODBC=latest +HDFS=2.9.2 +SPARK=master +DOTNET=2.1 +R=3.6 diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml new file mode 100644 index 0000000000000..46d7dd993b165 --- /dev/null +++ b/.github/workflows/cpp.yml @@ -0,0 +1,179 @@ + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: C++ + +on: + push: + # paths: + # - 'cpp/**' + pull_request: + # paths: + # - 'cpp/**' + +jobs: + + docker: + name: ${{ matrix.name }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + image: + - amd64-conda-latest-cpp + - amd64-debian-10-cpp + # - amd64-ubuntu-16.04-cpp + - amd64-ubuntu-18.04-cpp + - amd64-ubuntu-18.04-cpp-cmake-3.2 + - amd64-fedora-29-cpp + include: + - name: AMD64 Conda C++ + image: amd64-conda-latest-cpp + service: conda-cpp + - name: AMD64 Debian 10 C++ + image: amd64-debian-10-cpp + service: debian-cpp + debian: 10 + # TODO(kszucs): earlier GCC has a compile error + # - name: AMD64 Ubuntu 16.04 C++ + # image: amd64-ubuntu-16.04-cpp + # service: ubuntu-cpp + # ubuntu: 16.04 + - name: AMD64 Ubuntu 18.04 C++ + image: amd64-ubuntu-18.04-cpp + service: ubuntu-cpp + ubuntu: 18.04 + - name: AMD64 Ubuntu 18.04 C++ CMake 3.2 + image: amd64-ubuntu-18.04-cpp-cmake-3.2 + service: ubuntu-cpp-cmake32 + ubuntu: 18.04 + - name: AMD64 Fedora 29 C++ + image: amd64-fedora-29-cpp + service: fedora-cpp + fedora: 29 + env: + DEBIAN: ${{ matrix.debian || 10 }} + UBUNTU: ${{ matrix.ubuntu || 18.04 }} + FEDORA: ${{ matrix.fedora || 29 }} + steps: + - name: Checkout Arrow + uses: actions/checkout@v1 + with: + submodules: true + - name: Docker Pull + shell: bash + run: docker-compose pull --ignore-pull-failures ${{ matrix.service }} + - name: Docker Build + shell: bash + run: docker-compose build ${{ matrix.service }} + - name: Docker Run + shell: bash + run: docker-compose run ${{ matrix.service }} + - name: Docker Push + if: github.event_name == 'push' # && github.ref == 'master' + shell: bash + run: | + docker login \ + -u ${{ secrets.DOCKERHUB_USER }} \ + -p ${{ secrets.DOCKERHUB_TOKEN }} + docker-compose push ${{ matrix.service }} + + macos: + name: AMD64 MacOS 10.15 C++ + runs-on: macos-latest + strategy: + fail-fast: false + env: + ARROW_HOME: /usr/local + # TODO(kszucs): investigate why does it fail + ARROW_JEMALLOC: OFF + ARROW_ORC: OFF + ARROW_FLIGHT: ON + ARROW_PLASMA: ON + ARROW_GANDIVA: ON + ARROW_PARQUET: ON + ARROW_WITH_ZLIB: ON + ARROW_WITH_LZ4: ON + ARROW_WITH_BZ2: ON + ARROW_WITH_ZSTD: ON + ARROW_WITH_SNAPPY: ON + ARROW_WITH_BROTLI: ON + ARROW_BUILD_TESTS: ON + steps: + - name: Checkout Arrow + uses: actions/checkout@v1 + with: + submodules: true + - name: Install Dependencies + shell: bash + # TODO(kszucs): consider to use run_brew from travis_install_osx.sh + run: brew bundle --file=cpp/Brewfile + - name: Build + shell: bash + run: ci/scripts/cpp_build.sh $(pwd) $(pwd)/cpp/build + - name: Test + shell: bash + run: ci/scripts/cpp_test.sh $(pwd) $(pwd)/cpp/build + + windows: + name: AMD64 ${{ matrix.name }} C++ + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: + - windows-latest + include: + - os: windows-latest + name: Windows 2019 + generator: Visual Studio 16 2019 + env: + CMAKE_ARGS: '-A x64' + CMAKE_GENERATOR: ${{ matrix.generator }} + CMAKE_INSTALL_LIBDIR: bin + CMAKE_INSTALL_PREFIX: /usr + ARROW_HOME: /usr + ARROW_FLIGHT: OFF + ARROW_PARQUET: OFF + ARROW_WITH_ZLIB: OFF + ARROW_WITH_LZ4: OFF + ARROW_WITH_BZ2: OFF + ARROW_WITH_ZSTD: OFF + ARROW_WITH_SNAPPY: OFF + ARROW_WITH_BROTLI: OFF + ARROW_USE_GLOG: OFF + ARROW_BUILD_TESTS: ON + ARROW_TEST_LINKAGE: static + ARROW_BOOST_USE_SHARED: OFF + ARROW_BUILD_SHARED: OFF + ARROW_USE_STATIC_CRT: ON + ARROW_VERBOSE_THIRDPARTY_BUILD: OFF + steps: + - name: Checkout Arrow + uses: actions/checkout@v1 + with: + submodules: true + - name: Dependencies + shell: bash + run: choco install dependencywalker + - name: Build + shell: bash + run: ci/scripts/cpp_build.sh $(pwd) $(pwd)/cpp/build + - name: Test + shell: bash + run: ci/scripts/cpp_test.sh $(pwd) $(pwd)/cpp/build diff --git a/.github/workflows/csharp.yml b/.github/workflows/csharp.yml new file mode 100644 index 0000000000000..32d86867b58a1 --- /dev/null +++ b/.github/workflows/csharp.yml @@ -0,0 +1,66 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: C# + +on: + push: + # paths: + # - 'csharp/**' + pull_request: + # paths: + # - 'csharp/**' + +jobs: + # the docker container fails to run because of the ubuntu host versions, see + # https://github.com/dotnet/core/issues/3509 + + all-platforms: + name: AMD64 ${{ matrix.name }} C# ${{ matrix.dotnet }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: + - ubuntu-18.04 + - macos-latest + - windows-latest + include: + - os: ubuntu-18.04 + name: Ubuntu 18.04 + dotnet: 2.2.103 + - os: macos-latest + name: MacOS 10.15 + dotnet: 2.2.103 + - os: windows-latest + name: Windows 2019 + dotnet: 2.2.103 + steps: + - name: Install csharp + uses: actions/setup-dotnet@v1 + with: + dotnet-version: ${{ matrix.dotnet }} + - name: Checkout Arrow + uses: actions/checkout@v1 + with: + submodules: true + - name: Build + shell: bash + run: ci/scripts/csharp_build.sh $(pwd) + - name: Test + shell: bash + run: ci/scripts/csharp_test.sh $(pwd) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml new file mode 100644 index 0000000000000..176f6b1580737 --- /dev/null +++ b/.github/workflows/dev.yml @@ -0,0 +1,115 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Dev + +on: + push: + pull_request: + +jobs: + + release: + name: Source Release + runs-on: ubuntu-latest + env: + GIT_AUTHOR_NAME: Github Actions + GIT_AUTHOR_EMAIL: github@actions + GIT_COMMITTER_NAME: Github Actions + GIT_COMMITTER_EMAIL: github@actions + steps: + - name: Checkout Arrow + uses: actions/checkout@v1 + with: + submodules: true + - name: Install Python + uses: actions/setup-python@v1 + with: + python-version: '3.6' + - name: Install Ruby + uses: actions/setup-ruby@v1 + with: + ruby-version: '2.6' + - name: Install Dependencies + shell: bash + run: | + pip install cython setuptools + wget -q https://packages.microsoft.com/config/ubuntu/16.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt-get install apt-transport-https + sudo apt-get update + sudo apt-get install dotnet-sdk-2.2 + - name: Run Release Test + shell: bash + run: | + ci/scripts/release_test.sh $(pwd) + + docker-lint: + name: Lint C++, Python, R, Rust, Docker, RAT + runs-on: ubuntu-latest + steps: + - name: Checkout Arrow + uses: actions/checkout@v1 + with: + submodules: true + - name: Docker Pull + run: | + docker-compose pull --ignore-pull-failures ubuntu-cpp + docker-compose pull --ignore-pull-failures ubuntu-lint + - name: Docker Build + run: | + docker-compose build ubuntu-cpp + docker-compose build ubuntu-lint + - name: Docker Run + run: docker-compose run ubuntu-lint + - name: Docker Push + if: github.event_name == 'push' # && github.ref == 'master' + shell: bash + run: | + docker login \ + -u ${{ secrets.DOCKERHUB_USER }} \ + -p ${{ secrets.DOCKERHUB_TOKEN }} + docker-compose push ubuntu-lint + + docker-docs: + name: Sphinx and API documentations + runs-on: ubuntu-latest + steps: + - name: Checkout Arrow + uses: actions/checkout@v1 + with: + submodules: true + - name: Docker Pull + run: | + docker-compose pull --ignore-pull-failures ubuntu-cpp + docker-compose pull --ignore-pull-failures ubuntu-python + docker-compose pull --ignore-pull-failures ubuntu-docs + - name: Docker Build + run: | + docker-compose build ubuntu-cpp + docker-compose build ubuntu-python + docker-compose build ubuntu-docs + - name: Docker Run + run: docker-compose run ubuntu-docs + - name: Docker Push + if: github.event_name == 'push' # && github.ref == 'master' + shell: bash + run: | + docker login \ + -u ${{ secrets.DOCKERHUB_USER }} \ + -p ${{ secrets.DOCKERHUB_TOKEN }} + docker-compose push ubuntu-docs diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml new file mode 100644 index 0000000000000..7cb1e97f876d3 --- /dev/null +++ b/.github/workflows/go.yml @@ -0,0 +1,88 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Go + +on: + push: + # paths: + # - 'go/**' + pull_request: + # paths: + # - 'go/**' + +jobs: + docker: + name: AMD64 Debian Go ${{ matrix.go }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + go: [1.12] + env: + GO: ${{ matrix.go }} + steps: + - name: Checkout Arrow + uses: actions/checkout@v1 + with: + submodules: true + - name: Docker Pull + run: docker-compose pull --ignore-pull-failures debian-go + - name: Docker Build + run: docker-compose build debian-go + - name: Docker Run + run: docker-compose run debian-go + - name: Docker Push + if: github.event_name == 'push' # && github.ref == 'master' + shell: bash + run: | + docker login \ + -u ${{ secrets.DOCKERHUB_USER }} \ + -p ${{ secrets.DOCKERHUB_TOKEN }} + docker-compose push debian-go + + macos-windows: + name: AMD64 ${{ matrix.name }} Go ${{ matrix.go }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: + - macos-latest + - windows-latest + include: + - os: macos-latest + name: MacOS 10.15 + go: 1.12 + - os: windows-latest + name: Windows 2019 + go: 1.12 + steps: + - name: Install go + uses: actions/setup-go@v1 + with: + go-version: ${{ matrix.go }} + - name: Checkout Arrow + uses: actions/checkout@v1 + with: + submodules: true + - name: Build + shell: bash + run: ci/scripts/go_build.sh . + - name: Test + shell: bash + run: ci/scripts/go_test.sh . diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml new file mode 100644 index 0000000000000..4f5f000262f75 --- /dev/null +++ b/.github/workflows/integration.yml @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Integration + +on: + push: + pull_request: + +jobs: + + docker-integration: + name: AMD64 Conda Integration Test + runs-on: ubuntu-latest + env: + MAVEN: 3.5 + steps: + - name: Checkout Arrow + uses: actions/checkout@v1 + with: + submodules: true + - name: Docker Pull + run: | + docker-compose pull --ignore-pull-failures conda-cpp + docker-compose pull --ignore-pull-failures conda-integration + - name: Docker Build + run: | + docker-compose build conda-cpp + docker-compose build conda-integration + - name: Docker Run + run: docker-compose run conda-integration + - name: Docker Push + if: github.event_name == 'push' # && github.ref == 'master' + shell: bash + run: | + docker login \ + -u ${{ secrets.DOCKERHUB_USER }} \ + -p ${{ secrets.DOCKERHUB_TOKEN }} + docker-compose push conda-integration diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml new file mode 100644 index 0000000000000..52ecadb88d792 --- /dev/null +++ b/.github/workflows/java.yml @@ -0,0 +1,122 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Java + +on: + push: + # paths: + # - 'cpp/**' + # - 'java/**' + pull_request: + # paths: + # - 'cpp/**' + # - 'java/**' + +jobs: + + docker: + name: AMD64 Debian 9 Java JDK ${{ matrix.jdk }} Maven ${{ matrix.maven }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + image: + - amd64-debian-9-java-8-maven-3.5.2 + - amd64-debian-9-java-11-maven-3.6.2 + include: + - image: amd64-debian-9-java-8-maven-3.5.2 + jdk: 8 + maven: 3.5.2 + - image: amd64-debian-9-java-11-maven-3.6.2 + jdk: 11 + maven: 3.6.2 + env: + JDK: ${{ matrix.jdk }} + MAVEN: ${{ matrix.maven }} + steps: + - name: Checkout Arrow + uses: actions/checkout@v1 + with: + submodules: true + - name: Docker Pull + run: docker-compose pull --ignore-pull-failures debian-java + - name: Docker Build + run: docker-compose build debian-java + - name: Docker Run + run: docker-compose run debian-java + - name: Docker Push + if: github.event_name == 'push' # && github.ref == 'master' + shell: bash + run: | + docker login \ + -u ${{ secrets.DOCKERHUB_USER }} \ + -p ${{ secrets.DOCKERHUB_TOKEN }} + docker-compose push debian-java + + docker-jni: + name: AMD64 Debian 9 Java JNI and Plasma + runs-on: ubuntu-latest + steps: + - name: Checkout Arrow + uses: actions/checkout@v1 + with: + submodules: true + - name: Docker Pull + run: | + docker-compose pull --ignore-pull-failures debian-java + docker-compose pull --ignore-pull-failures debian-java-jni + - name: Docker Build + run: | + docker-compose build debian-java + docker-compose build debian-java-jni + - name: Docker Run + run: | + docker-compose run debian-java-jni + - name: Docker Push + if: github.event_name == 'push' # && github.ref == 'master' + shell: bash + run: | + docker login \ + -u ${{ secrets.DOCKERHUB_USER }} \ + -p ${{ secrets.DOCKERHUB_TOKEN }} + docker-compose push debian-java-jni + + macos: + name: AMD64 MacOS 10.15 Java JDK ${{ matrix.jdk }} + runs-on: macos-latest + strategy: + fail-fast: false + matrix: + jdk: [11] + env: + ARROW_JAVA_RUN_TESTS: 1 + steps: + - name: Set up Java + uses: actions/setup-java@v1 + with: + java-version: ${{ matrix.jdk }} + - name: Checkout Arrow + uses: actions/checkout@v1 + with: + submodules: true + - name: Build + shell: bash + run: ci/scripts/java_build.sh $(pwd) + - name: Test + shell: bash + run: ci/scripts/java_test.sh $(pwd) diff --git a/.github/workflows/js.yml b/.github/workflows/js.yml new file mode 100644 index 0000000000000..bf2cc3040427e --- /dev/null +++ b/.github/workflows/js.yml @@ -0,0 +1,118 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: NodeJS + +on: + push: + # paths: + # - 'js/**' + pull_request: + # paths: + # - 'js/**' + +jobs: + docker: + name: AMD64 Debian NodeJS ${{ matrix.node }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + node: [11] + env: + NODE: ${{ matrix.node }} + steps: + - name: Checkout Arrow + uses: actions/checkout@v1 + with: + submodules: true + - name: Docker Pull + run: docker-compose pull --ignore-pull-failures debian-js + - name: Docker Build + run: docker-compose build debian-js + - name: Docker Run + run: docker-compose run debian-js + - name: Docker Push + if: github.event_name == 'push' # && github.ref == 'master' + shell: bash + run: | + docker login \ + -u ${{ secrets.DOCKERHUB_USER }} \ + -p ${{ secrets.DOCKERHUB_TOKEN }} + docker-compose push debian-js + + macos: + name: AMD64 MacOS 10.15 NodeJS ${{ matrix.node }} + runs-on: macos-latest + strategy: + fail-fast: false + matrix: + node: [11] + steps: + - name: Checkout Arrow + uses: actions/checkout@v1 + with: + submodules: true + - name: Install NodeJS + uses: actions/setup-node@v1 + with: + node-version: ${{ matrix.node }} + - name: Build + shell: bash + run: ci/scripts/js_build.sh $(pwd) + - name: Test + shell: bash + run: ci/scripts/js_test.sh $(pwd) + + # TODO(kszucs): windows build fails with: + # + # > NODE_NO_WARNINGS=1 gulp build + # 'NODE_NO_WARNINGS' is not recognized as an internal or external command, + # operable program or batch file. + # npm ERR! code ELIFECYCLE + # npm ERR! errno 1 + # npm ERR! apache-arrow@1.0.0-SNAPSHOT build: `NODE_NO_WARNINGS=1 gulp build` + # npm ERR! Exit status 1 + # npm ERR! + # npm ERR! Failed at the apache-arrow@1.0.0-SNAPSHOT build script. + # npm ERR! This is probably not a problem with npm. There is likely additional logging output above. + # + # windows: + # name: AMD64 Windows 2019 NodeJS ${{ matrix.node }} + # runs-on: windows-latest + # strategy: + # fail-fast: false + # matrix: + # node: [11] + # steps: + # - name: Checkout Arrow + # uses: actions/checkout@v1 + # with: + # submodules: true + # - name: Install NodeJS + # uses: actions/setup-node@v1 + # with: + # node-version: ${{ matrix.node }} + # - name: Install Platform Dependencies + # shell: bash + # run: npm install -g cross-env + # - name: Build + # shell: bash + # run: ci/scripts/js_build.sh $(pwd) + # - name: Test + # shell: bash + # run: ci/scripts/js_test.sh $(pwd) diff --git a/.github/workflows/linux-docker-compose.yml b/.github/workflows/linux-docker-compose.yml deleted file mode 100644 index 4431ce5cf6a64..0000000000000 --- a/.github/workflows/linux-docker-compose.yml +++ /dev/null @@ -1,58 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Linux docker-compose -on: - - push - - pull_request -jobs: - build: - name: Test - strategy: - matrix: - label: - - C++ w/ clang-7 & system packages - - Rust - - Lint, Release tests - include: - - label: C++ w/ clang-7 & system packages - image: cpp-system-deps - skip_expression: | - ${ARROW_CI_CPP_AFFECTED} != "1" - - label: Rust - image: rust - skip_expression: | - ${ARROW_CI_RUST_AFFECTED} != "1" - - label: Lint, Release tests - image: lint - # Don't skip - skip_expression: | - "1" != "1" - runs-on: ubuntu-18.04 - steps: - - uses: actions/checkout@master - with: - submodules: true - - name: docker-compose - run: | - eval "$(python ci/detect-changes.py)" - if [[ ${{ matrix.skip_expression }} ]]; then - exit - fi - - docker-compose build --pull ${{ matrix.image }} - docker-compose run ${{ matrix.image }} diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 2cdc49e69341e..109481dd2c4be 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -16,10 +16,12 @@ # under the License. name: Pull request + on: schedule: - cron: | */15 * * * * + jobs: jira-link: name: JIRA link diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml new file mode 100644 index 0000000000000..a831204fe440a --- /dev/null +++ b/.github/workflows/python.yml @@ -0,0 +1,229 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Python + +on: + push: + # paths: + # - 'cpp/**' + # - 'python/**' + pull_request: + # paths: + # - 'cpp/**' + # - 'python/**' + +jobs: + + docker: + name: ${{ matrix.name }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + image: + - amd64-conda-latest-python-2.7 + - amd64-conda-latest-python-3.6 + - amd64-conda-latest-python-3.7 + - amd64-debian-10-python-3 + # - amd64-ubuntu-16.04-python-3 + - amd64-ubuntu-18.04-python-3 + - amd64-fedora-29-python-3 + include: + - name: AMD64 Conda Python 2.7 + image: amd64-conda-latest-python-2.7 + platform: conda + python: 2.7 + - name: AMD64 Conda Python 3.6 + image: amd64-conda-latest-python-3.6 + platform: conda + python: 3.6 + - name: AMD64 Conda Python 3.7 + image: amd64-conda-latest-python-3.7 + platform: conda + python: 3.7 + - name: AMD64 Debian 10 Python 3 + image: amd64-debian-10-python-3 + platform: debian + debian: 10 + python: 3 + # - name: AMD64 Ubuntu 16.04 Python 3 + # image: amd64-ubuntu-16.04-python-3 + # platform: ubuntu + # ubuntu: 16.04 + # python: 3 + - name: AMD64 Ubuntu 18.04 Python 3 + image: amd64-ubuntu-18.04-python-3 + platform: ubuntu + ubuntu: 18.04 + python: 3 + - name: AMD64 Fedora 29 Python 3 + image: amd64-fedora-29-python-3 + platform: fedora + fedora: 29 + python: 3 + env: + PYTHON: ${{ matrix.python || 3.7 }} + DEBIAN: ${{ matrix.debian || 10 }} + UBUNTU: ${{ matrix.ubuntu || 18.04 }} + FEDORA: ${{ matrix.fedora || 29 }} + steps: + - name: Checkout Arrow + uses: actions/checkout@v1 + with: + submodules: true + - name: Docker Pull + shell: bash + run: | + docker-compose pull --ignore-pull-failures ${{ matrix.platform }}-cpp + docker-compose pull --ignore-pull-failures ${{ matrix.platform }}-python + - name: Docker Build + shell: bash + run: | + docker-compose build ${{ matrix.platform }}-cpp + docker-compose build ${{ matrix.platform }}-python + - name: Docker Run + shell: bash + run: docker-compose run ${{ matrix.platform }}-python + - name: Docker Push + if: github.event_name == 'push' # && github.ref == 'master' + shell: bash + run: | + docker login \ + -u ${{ secrets.DOCKERHUB_USER }} \ + -p ${{ secrets.DOCKERHUB_TOKEN }} + docker-compose push ${{ matrix.platform }}-python + + docker-integration: + # TODO(kszucs): we should probably have a test with pandas='master' running + # with a daily cron schedule + name: ${{ matrix.name }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + image: + - amd64-conda-latest-python-3.7-pandas-latest + - amd64-conda-latest-python-3.7-dask-latest + - amd64-conda-latest-python-3.7-turbodbc-latest + include: + - name: AMD64 Conda Python 3.7 Pandas + image: amd64-conda-latest-python-3.7-pandas-latest + platform: conda + integration: pandas + python: 3.7 + pandas: latest + - name: AMD64 Conda Python 3.7 Dask + image: amd64-conda-latest-python-3.7-dask-latest + platform: conda + integration: dask + python: 3.7 + dask: latest + - name: AMD64 Conda Python 3.7 Turbodbc + image: amd64-conda-latest-python-3.7-turbodbc-latest + platform: conda + integration: turbodbc + python: 3.7 + turbodbc: latest + env: + PYTHON: ${{ matrix.python || 3.7 }} + PANDAS: ${{ matrix.pandas || 'latest' }} + DASK: ${{ matrix.dask || 'latest' }} + TURBODBC: ${{ matrix.turbodbc || 'latest' }} + steps: + - name: Checkout Arrow + uses: actions/checkout@v1 + with: + submodules: true + - name: Docker Pull + shell: bash + run: | + docker-compose pull --ignore-pull-failures \ + ${{ matrix.platform }}-cpp + docker-compose pull --ignore-pull-failures \ + ${{ matrix.platform }}-python + docker-compose pull --ignore-pull-failures \ + ${{ matrix.platform }}-python-${{ matrix.integration }} + - name: Docker Build + shell: bash + run: | + docker-compose build \ + ${{ matrix.platform }}-cpp + docker-compose build \ + ${{ matrix.platform }}-python + docker-compose build \ + ${{ matrix.platform }}-python-${{ matrix.integration }} + - name: Docker Run + shell: bash + run: | + docker-compose run \ + ${{ matrix.platform }}-python-${{ matrix.integration }} + - name: Docker Push + if: github.event_name == 'push' # && github.ref == 'master' + shell: bash + run: | + docker login \ + -u ${{ secrets.DOCKERHUB_USER }} \ + -p ${{ secrets.DOCKERHUB_TOKEN }} + docker-compose push \ + ${{ matrix.platform }}-python-${{ matrix.integration }} + + macos: + name: AMD64 MacOS 10.15 Python ${{ matrix.python }} + runs-on: macos-latest + strategy: + fail-fast: false + matrix: + python: [3] + env: + ARROW_HOME: /usr/local + # TODO(kszucs): investigate why does it fail + ARROW_JEMALLOC: OFF + ARROW_ORC: OFF + ARROW_BUILD_TESTS: OFF + ARROW_PYTHON: ON + ARROW_PLASMA: ON + ARROW_GANDIVA: ON + ARROW_PARQUET: ON + ARROW_WITH_ZLIB: ON + ARROW_WITH_LZ4: ON + ARROW_WITH_BZ2: ON + ARROW_WITH_ZSTD: ON + ARROW_WITH_SNAPPY: ON + ARROW_WITH_BROTLI: ON + steps: + - name: Checkout Arrow + uses: actions/checkout@v1 + with: + submodules: true + - name: Install Dependencies + shell: bash + run: | + brew bundle --file=cpp/Brewfile + brew install coreutils python + pip install -r python/requirements.txt \ + -r python/requirements-test.txt \ + cython + - name: Build + shell: bash + run: | + ci/scripts/cpp_build.sh $(pwd) + ci/scripts/python_build.sh $(pwd) + - name: Test + shell: bash + run: | + ci/scripts/python_test.sh $(pwd) diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml new file mode 100644 index 0000000000000..1a974981f7ba7 --- /dev/null +++ b/.github/workflows/r.yml @@ -0,0 +1,79 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: R + +on: + push: + # paths: + # - 'cpp/**' + # - 'r/**' + pull_request: + # paths: + # - 'cpp/**' + # - 'r/**' + +jobs: + + docker: + name: ${{ matrix.name }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + image: + - amd64-conda-latest-r-3.6 + - amd64-ubuntu-18.04-r-3.6 + include: + - name: AMD64 Conda R 3.6 + image: amd64-conda-latest-r-3.6 + platform: conda + r: 3.6 + - name: AMD64 Ubuntu 18.04 R 3.6 + image: amd64-ubuntu-18.04-r-3.6 + platform: ubuntu + ubuntu: 18.04 + r: 3.6 + env: + R: ${{ matrix.r || 3.6 }} + UBUNTU: ${{ matrix.ubuntu || 18.04 }} + steps: + - name: Checkout Arrow + uses: actions/checkout@v1 + with: + submodules: true + - name: Docker Pull + shell: bash + run: | + docker-compose pull --ignore-pull-failures ${{ matrix.platform }}-cpp + docker-compose pull --ignore-pull-failures ${{ matrix.platform }}-r + - name: Docker Build + shell: bash + run: | + docker-compose build ${{ matrix.platform }}-cpp + docker-compose build ${{ matrix.platform }}-r + - name: Docker Run + shell: bash + run: docker-compose run ${{ matrix.platform }}-r + - name: Docker Push + if: github.event_name == 'push' # && github.ref == 'master' + shell: bash + run: | + docker login \ + -u ${{ secrets.DOCKERHUB_USER }} \ + -p ${{ secrets.DOCKERHUB_TOKEN }} + docker-compose push ${{ matrix.platform }}-r diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml new file mode 100644 index 0000000000000..47f922899b020 --- /dev/null +++ b/.github/workflows/ruby.yml @@ -0,0 +1,116 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: C Glib & Ruby + +on: + push: + # paths: + # - 'cpp/**' + # - 'c_glib/**' + # - 'ruby/**' + pull_request: + # paths: + # - 'cpp/**' + # - 'c_glib/**' + # - 'ruby/**' + +jobs: + + docker: + name: ${{ matrix.name }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + image: + - amd64-ubuntu-18.04-ruby + include: + - name: AMD64 Ubuntu 18.04 Glib & Ruby + image: amd64-ubuntu-18.04-ruby + ubuntu: 18.04 + platform: ubuntu + env: + UBUNTU: ${{ matrix.ubuntu || 18.04 }} + steps: + - name: Checkout Arrow + uses: actions/checkout@v1 + with: + submodules: true + - name: Docker Pull + shell: bash + run: | + docker-compose pull --ignore-pull-failures \ + ${{ matrix.platform }}-cpp + docker-compose pull --ignore-pull-failures \ + ${{ matrix.platform }}-c-glib + docker-compose pull --ignore-pull-failures \ + ${{ matrix.platform }}-ruby + - name: Docker Build + shell: bash + run: | + docker-compose build ${{ matrix.platform }}-cpp + docker-compose build ${{ matrix.platform }}-c-glib + docker-compose build ${{ matrix.platform }}-ruby + - name: Docker Run + shell: bash + run: docker-compose run ${{ matrix.platform }}-ruby + - name: Docker Push + if: github.event_name == 'push' # && github.ref == 'master' + shell: bash + run: | + docker login \ + -u ${{ secrets.DOCKERHUB_USER }} \ + -p ${{ secrets.DOCKERHUB_TOKEN }} + docker-compose push ${{ matrix.platform }}-ruby + + macos: + name: AMD64 MacOS 10.15 Glib & Ruby + runs-on: macos-latest + strategy: + fail-fast: false + env: + ARROW_HOME: /usr/local + ARROW_ORC: OFF + ARROW_JEMALLOC: OFF + ARROW_WITH_BROTLI: ON + ARROW_WITH_LZ4: ON + ARROW_WITH_SNAPPY: ON + ARROW_WITH_ZLIB: ON + ARROW_BUILD_TESTS: ON + steps: + - name: Checkout Arrow + uses: actions/checkout@v1 + with: + submodules: true + - name: Install Dependencies + shell: bash + run: | + brew bundle --file=cpp/Brewfile + brew bundle --file=c_glib/Brewfile + bundle install --gemfile ruby/red-arrow/Gemfile + - name: Build + shell: bash + run: | + ci/scripts/cpp_build.sh $(pwd) $(pwd)/cpp/build + ci/scripts/c_glib_build.sh $(pwd) $(pwd)/c_glib/build + - name: Test Glib + shell: bash + run: ci/scripts/c_glib_test.sh $(pwd) $(pwd)/c_glib/build + - name: Test Ruby + shell: bash + run: ci/scripts/ruby_test.sh $(pwd) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml new file mode 100644 index 0000000000000..f90aea5f3a170 --- /dev/null +++ b/.github/workflows/rust.yml @@ -0,0 +1,110 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Rust + +on: + push: + # paths: + # - 'rust/**' + pull_request: + # paths: + # - 'rust/**' + +jobs: + docker: + name: AMD64 Debian Rust ${{ matrix.rust }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + rust: [nightly-2019-09-25] + env: + RUST: ${{ matrix.rust }} + steps: + - name: Checkout Arrow + uses: actions/checkout@v1 + with: + submodules: true + - name: Docker Pull + run: docker-compose pull --ignore-pull-failures debian-rust + - name: Docker Build + run: docker-compose build debian-rust + - name: Docker Run + run: docker-compose run debian-rust + - name: Docker Push + if: github.event_name == 'push' # && github.ref == 'master' + shell: bash + run: | + docker login \ + -u ${{ secrets.DOCKERHUB_USER }} \ + -p ${{ secrets.DOCKERHUB_TOKEN }} + docker-compose push debian-rust + + macos: + name: AMD64 MacOS 10.15 Rust ${{ matrix.rust }} + runs-on: macos-latest + strategy: + fail-fast: false + matrix: + rust: [nightly-2019-09-25] + steps: + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: ${{ matrix.rust }} + override: true + - name: Install Flatbuffers + shell: bash + run: brew install flatbuffers + - name: Checkout Arrow + uses: actions/checkout@v1 + with: + submodules: true + - name: Build + shell: bash + run: ci/scripts/rust_build.sh $(pwd) $(pwd)/rust/target + - name: Test + shell: bash + run: ci/scripts/rust_test.sh $(pwd) + + windows: + name: AMD64 Windows 2019 Rust ${{ matrix.rust }} + runs-on: windows-latest + strategy: + fail-fast: false + matrix: + rust: [nightly-2019-09-25] + steps: + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: ${{ matrix.rust }} + override: true + - name: Install Flatbuffers + shell: bash + run: choco install flatc + - name: Checkout Arrow + uses: actions/checkout@v1 + with: + submodules: true + - name: Build + shell: bash + run: ci/scripts/rust_build.sh $(pwd) $(pwd)/rust/target + - name: Test + shell: bash + run: ci/scripts/rust_test.sh $(pwd) diff --git a/.github/workflows/windows-msvc-cpp.yml b/.github/workflows/windows-msvc-cpp.yml deleted file mode 100644 index 824bfad403a0b..0000000000000 --- a/.github/workflows/windows-msvc-cpp.yml +++ /dev/null @@ -1,54 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Windows MSVC C++ -on: - - push - - pull_request -jobs: - build: - name: Build - strategy: - matrix: - cmake-generator: - - Visual Studio 16 2019 - - Visual Studio 15 2017 - include: - - cmake-generator: Visual Studio 16 2019 - os: windows-2019 - - cmake-generator: Visual Studio 15 2017 - os: windows-2016 - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@master - - name: CMake - shell: cmd - run: | - mkdir build\cpp - cmake ^ - -G "${{ matrix.cmake-generator }}" ^ - -A x64 ^ - -DARROW_USE_GLOG=OFF ^ - -DARROW_WITH_LZ4=OFF ^ - -DARROW_WITH_ZLIB=OFF ^ - -DCMAKE_INSTALL_PREFIX=install ^ - -S cpp ^ - -B build\cpp - - name: Install - shell: cmd - run: | - cmake --build build\cpp --config Debug --target Install diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 083357bf59291..0000000000000 --- a/.travis.yml +++ /dev/null @@ -1,326 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -sudo: required -dist: xenial - -services: - - docker - -cache: - directories: - - $HOME/.ccache - - $HOME/.m2 # Maven - -git: - # ARROW-3083: We use a higher clone depth to also have the latest tag in the cloned history. - depth: 250 - -before_install: - - source ci/travis/before-install.sh - -matrix: - fast_finish: true - include: - - # ------------------------------------------------------------------------- - - name: "Lint, Release tests" - os: linux - language: python - python: "3.6" - before_script: - # Always run RAT checks, in case another build in matrix breaks RAT - - $TRAVIS_BUILD_DIR/ci/travis_release_audit.sh - - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh - script: - - $TRAVIS_BUILD_DIR/ci/travis_lint.sh - - $TRAVIS_BUILD_DIR/ci/travis_release_test.sh - - # ------------------------------------------------------------------------- - - name: "C++ unit tests w/ conda-forge toolchain, coverage" - compiler: gcc - language: cpp - os: linux - jdk: openjdk8 - env: - - ARROW_BUILD_WARNING_LEVEL=CHECKIN - - ARROW_TRAVIS_COVERAGE=1 - - ARROW_TRAVIS_FLIGHT=1 - - ARROW_TRAVIS_GANDIVA=1 - - ARROW_TRAVIS_GANDIVA_JAVA=1 - - ARROW_TRAVIS_ORC=1 - - ARROW_TRAVIS_PARQUET=1 - - ARROW_TRAVIS_PARQUET_ENCRYPTION=1 - - ARROW_TRAVIS_PLASMA=1 - - ARROW_TRAVIS_S3=1 - - ARROW_TRAVIS_USE_SYSTEM_JAVA=1 - - ARROW_TRAVIS_USE_TOOLCHAIN=1 - # TODO: This fails in re2 code - # - ARROW_TRAVIS_VALGRIND=1 - - ARROW_TRAVIS_VERBOSE=1 - # ARROW-6509: disable MIMALLOC on xenial - # - ARROW_TRAVIS_MIMALLOC=1 - before_script: - - if [ $ARROW_CI_CPP_AFFECTED != "1" ] && [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi - - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh - - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh - - $TRAVIS_BUILD_DIR/ci/travis_install_minio.sh - # If either C++ or Python changed, we must install the C++ libraries - - git submodule update --init - - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh - script: - - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh || travis_terminate 1 - - $TRAVIS_BUILD_DIR/ci/travis_script_gandiva_java.sh || travis_terminate 1 - - $TRAVIS_BUILD_DIR/ci/travis_upload_cpp_coverage.sh || travis_terminate 1 - - # ------------------------------------------------------------------------- - - name: "Python 3.6 unit tests, conda-forge toolchain, coverage" - compiler: gcc - language: cpp - os: linux - jdk: openjdk8 - env: - - ARROW_BUILD_WARNING_LEVEL=CHECKIN - - ARROW_TRAVIS_COVERAGE=1 - - ARROW_TRAVIS_FLIGHT=1 - - ARROW_TRAVIS_OPTIONAL_INSTALL=1 - - ARROW_TRAVIS_PYTHON_DOCS=1 - - ARROW_TRAVIS_PYTHON_JVM=1 - - ARROW_TRAVIS_USE_SYSTEM_JAVA=1 - - ARROW_TRAVIS_USE_TOOLCHAIN=1 - - ARROW_TRAVIS_S3=1 - # TODO(wesm): Run the benchmarks outside of Travis - # - ARROW_TRAVIS_PYTHON_BENCHMARKS=1 - before_script: - - if [ $ARROW_CI_PYTHON_AFFECTED != "1" ] && [ $ARROW_CI_DOCS_AFFECTED != "1" ]; then exit; fi - - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh - - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh - - $TRAVIS_BUILD_DIR/ci/travis_install_minio.sh - - $TRAVIS_BUILD_DIR/ci/travis_install_toolchain.sh - script: - - $TRAVIS_BUILD_DIR/ci/travis_script_java.sh || travis_terminate 1 - - export ARROW_TRAVIS_PYTHON_GANDIVA=1 - - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6 - - $TRAVIS_BUILD_DIR/ci/travis_upload_cpp_coverage.sh - - # ------------------------------------------------------------------------- - - name: "[OS X] C++ w/ XCode 9.3" - compiler: clang - osx_image: xcode9.3 - os: osx - addons: - env: - - ARROW_TRAVIS_USE_TOOLCHAIN=1 - - ARROW_TRAVIS_PLASMA=1 - - ARROW_TRAVIS_FLIGHT=1 - - ARROW_TRAVIS_ORC=1 - - ARROW_TRAVIS_S3=1 - - ARROW_TRAVIS_PARQUET=1 - - ARROW_TRAVIS_PARQUET_ENCRYPTION=1 - # TODO(ARROW-4763): llvm and llvmdev packages are in conflict: - # https://github.com/conda-forge/llvmdev-feedstock/issues/60 - # - ARROW_TRAVIS_GANDIVA=1 - # - ARROW_TRAVIS_GANDIVA_JAVA=1 - - ARROW_TRAVIS_OPTIONAL_INSTALL=1 - - ARROW_TRAVIS_VERBOSE=1 - - ARROW_BUILD_WARNING_LEVEL=CHECKIN - - ARROW_TRAVIS_MIMALLOC=1 - before_script: - - if [ $ARROW_CI_CPP_AFFECTED != "1" ] && [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi - # If either C++ or Python changed, we must install the C++ libraries - - git submodule update --init - - $TRAVIS_BUILD_DIR/ci/travis_install_minio.sh - - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh - script: - - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh || travis_terminate 1 - # Disabled because of ARROW-4763 - # - $TRAVIS_BUILD_DIR/ci/travis_script_gandiva_java.sh - - # ------------------------------------------------------------------------- - - name: "[OS X] Python w/ XCode 9.3" - compiler: clang - osx_image: xcode9.3 - os: osx - addons: - env: - - ARROW_TRAVIS_S3=1 - - ARROW_TRAVIS_PLASMA=1 - - ARROW_TRAVIS_USE_TOOLCHAIN=1 - - ARROW_BUILD_WARNING_LEVEL=CHECKIN - - ARROW_TRAVIS_OPTIONAL_INSTALL=1 - - ARROW_TRAVIS_VERBOSE=1 - - MACOSX_DEPLOYMENT_TARGET="10.9" - before_script: - script: - - if [ $ARROW_CI_PYTHON_AFFECTED != "1" ]; then exit; fi - - $TRAVIS_BUILD_DIR/ci/travis_install_minio.sh - - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6 - - # ------------------------------------------------------------------------- - - name: "Java OpenJDK8 and OpenJDK11" - language: cpp - os: linux - before_script: - - if [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi - - docker-compose build --pull java-all-jdks - script: - - docker-compose run java-all-jdks - - # ------------------------------------------------------------------------- - - name: "Protocol / Flight Integration Tests" - language: cpp - os: linux - env: ARROW_TEST_GROUP=integration - before_script: - - if [ $ARROW_CI_INTEGRATION_AFFECTED != "1" ]; then exit; fi - - docker-compose pull integration - script: - - docker-compose run integration - - # ------------------------------------------------------------------------- - - name: "NodeJS" - language: node_js - os: linux - node_js: - - '11.12' - before_script: - - if [ $ARROW_CI_JS_AFFECTED != "1" ]; then exit; fi - - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh - - $TRAVIS_BUILD_DIR/ci/travis_before_script_js.sh - script: - - $TRAVIS_BUILD_DIR/ci/travis_script_js.sh - - # ------------------------------------------------------------------------- - - name: "C++ & GLib & Ruby w/ gcc 5.4" - compiler: gcc - os: linux - env: - - ARROW_TRAVIS_ORC=1 - - ARROW_TRAVIS_GANDIVA=1 - - ARROW_TRAVIS_PARQUET=1 - - ARROW_TRAVIS_PLASMA=1 - - ARROW_TRAVIS_MIMALLOC=1 - before_script: - - if [ $ARROW_CI_RUBY_AFFECTED != "1" ]; then exit; fi - - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh - - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh - - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library - - $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh - - $TRAVIS_BUILD_DIR/ci/travis_before_script_ruby.sh - script: - - if [ $ARROW_CI_C_GLIB_AFFECTED = "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh; fi - - $TRAVIS_BUILD_DIR/ci/travis_script_ruby.sh - - # ------------------------------------------------------------------------- - - name: "[OS X] C++ & GLib & Ruby w/ XCode 10.2 & Homebrew" - compiler: clang - osx_image: xcode10.2 - os: osx - env: - - ARROW_TRAVIS_FLIGHT=1 - - ARROW_TRAVIS_GANDIVA=1 - # Disabled until either ORC-480 or ORC-561 is fixed - - ARROW_TRAVIS_ORC=0 - - ARROW_TRAVIS_PARQUET=1 - - ARROW_TRAVIS_PLASMA=1 - - ARROW_TRAVIS_PYTHON=1 - - ARROW_TRAVIS_VERBOSE=1 - - ARROW_TRAVIS_MIMALLOC=1 - addons: - before_script: - - if [ $ARROW_CI_RUBY_AFFECTED != "1" ]; then exit; fi - - $TRAVIS_BUILD_DIR/ci/travis_install_osx.sh - - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library --homebrew - - $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh - - $TRAVIS_BUILD_DIR/ci/travis_before_script_ruby.sh - script: - - if [ $ARROW_CI_C_GLIB_AFFECTED = "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh; fi - - $TRAVIS_BUILD_DIR/ci/travis_script_ruby.sh - - # ------------------------------------------------------------------------- - - name: Go - language: go - go_import_path: github.com/apache/arrow - os: linux - go: - - 1.12.x - before_script: - - if [ $ARROW_CI_GO_AFFECTED != "1" ]; then exit; fi - script: - - $TRAVIS_BUILD_DIR/ci/travis_script_go.sh - after_success: - - pushd ${TRAVIS_BUILD_DIR}/go/arrow - - bash <(curl -s https://codecov.io/bash) || echo "Codecov did not collect coverage reports" - - # ------------------------------------------------------------------------- - - name: R (with and without libarrow) - language: r - r: devel - cache: - directories: - - $HOME/.ccache - packages: true - dist: xenial - env: - - ARROW_TRAVIS_PARQUET=1 - - ARROW_TRAVIS_PARQUET_ENCRYPTION=1 - - ARROW_TRAVIS_USE_SYSTEM=1 - - ARROW_TRAVIS_MIMALLOC=1 - before_install: - # Have to copy-paste this here because of how R's build steps work - - eval `python $TRAVIS_BUILD_DIR/ci/detect-changes.py` - - if [ $ARROW_CI_R_AFFECTED != "1" ]; then exit; fi - - source ci/travis/before-install.sh - # First check that it builds without libarrow - - sudo apt-get install devscripts # For checkbashisms - - pushd ${TRAVIS_BUILD_DIR}/r - - R -e 'install.packages("remotes"); remotes::install_deps(dep = TRUE)' - - R CMD build . - - R CMD check --as-cran arrow_*tar.gz - - rm arrow_*tar.gz - - popd - # Now, proceed to install the c++ lib and the rest of the job - - | - if [ $TRAVIS_OS_NAME == "linux" ]; then - sudo bash -c "echo -e 'Acquire::Retries 10; Acquire::http::Timeout \"20\";' > /etc/apt/apt.conf.d/99-travis-retry" - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test - sudo apt-get update -qq - fi - - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh - # On Xenial, installing clang-7 leads these to get removed, - # but that breaks R, so let's reinstall them. - - sudo apt-get install gfortran gfortran-5 libblas-dev libgfortran-5-dev libgfortran3 liblapack-dev liblapack3 - - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh - - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library - - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$TRAVIS_BUILD_DIR/cpp-install/lib - - export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:$TRAVIS_BUILD_DIR/cpp-install/lib/pkgconfig - - export CXX11FLAGS=-Wall - # Add this env var so we can assert in the tests that the library is installed correctly - - export TEST_R_WITH_ARROW=TRUE - - pushd ${TRAVIS_BUILD_DIR}/r - after_success: - - Rscript ../ci/travis_upload_r_coverage.R - -after_failure: - - source ${TRAVIS_BUILD_DIR}/ci/travis/after-failure.sh - -after_script: - - source ${TRAVIS_BUILD_DIR}/ci/travis/after-script.sh - -env: - global: - - secure: "GcrPtsKUCgNY7HKYjWlHQo8SiFrShDvdZSU8t1m1FJrE+UfK0Dgh9zXmAausM8GmhqSwkF0q4UbLQf2uCnSITWKeEPAL8Mo9eu4ib+ikJx/b3Sk81frgW5ADoHfW1Eyqd8xJNIMwMegJOtRLSDqiXh1CvMlKnY8PyTOGM2DgN9ona/v6p9OFH9Qs0JhBRVXAn0S4ztjumck8E56+01hqRfxbZ88pTfpKghBxYp9PJaMjtGdomjVWlqPaWaWJj+KptT8inV9NK+TVYKx0dXWD+S1Vgr1PytQnLdILOYV23gsOBYqn33ByF/yADl4m3hUjU/qeT0Fi7aWxmVpj+oTJISOSH5N8nIsuNH8mQk2ZzzXHfV7btFvP+cOPRczadoKkT6D6cHA8nQ7b0dphC6bl6SAeSfc/cbhRT+fYnIjg8jFXC8jlyWBr7LR6GXVpc0bND7i300ITo0FuRJhy2OxqPtGo3dKLE7eAcv78tuO0OYJ/kol1PEqFdFkbYbNVbg/cFpbGqiCXDsOtPDbAGBv69YnXdVowSxxs8cRGjSkDydv6ZSytb/Zd4lH/KAomcFNk8adx12O1Lk4sbmVav1cGig5P6OcQKS0jC5IiRb4THcQzVzAkXXbaafKm5sru/NoYxhzmkyhkOc11nTYHKVng+XKWzLCNn7pTTSLitp5+xa4=" diff --git a/README.md b/README.md index ef498df1df48d..f7dbb5852c199 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,6 @@ # Apache Arrow -[![Build Status](https://travis-ci.org/apache/arrow.svg?branch=master)](https://travis-ci.org/apache/arrow) [![Build Status](https://ci.appveyor.com/api/projects/status/github/apache/arrow/branch/master?svg=true)](https://ci.appveyor.com/project/ApacheSoftwareFoundation/arrow/branch/master) [![Coverage Status](https://codecov.io/gh/apache/arrow/branch/master/graph/badge.svg)](https://codecov.io/gh/apache/arrow?branch=master) [![Fuzzit Status](https://app.fuzzit.dev/badge?org_id=yMxZh42xl9qy6bvg3EiJ&branch=master)](https://app.fuzzit.dev/admin/yMxZh42xl9qy6bvg3EiJ/dashboard) diff --git a/appveyor.yml b/appveyor.yml index a508b0ee269a6..65b44e5493394 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -90,22 +90,6 @@ environment: MINGW_PREFIX: c:\msys64\mingw64 MSYSTEM: MINGW64 USE_CLCACHE: false - - JOB: "Rust" - TARGET: x86_64-pc-windows-msvc - USE_CLCACHE: false - - JOB: "C#" - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 - USE_CLCACHE: false - - JOB: "Go" - MINGW_PACKAGE_PREFIX: mingw-w64-x86_64 - MINGW_PREFIX: c:\msys64\mingw64 - MSYSTEM: MINGW64 - USE_CLCACHE: false - GOROOT: 'c:\go112' - GOPATH: c:\gopath - GO111MODULE: on - GOTOOLDIR: '%GOROOT%\pkg\tool\windows_amd64' - PATH: '%GOPATH%\bin;%GOROOT%\bin;%PATH%' - JOB: "R with libarrow" USE_CLCACHE: false TEST_R_WITH_ARROW: "TRUE" diff --git a/ci/appveyor-build.bat b/ci/appveyor-build.bat index 09d4ac1f24f59..a306b0f43a9c7 100644 --- a/ci/appveyor-build.bat +++ b/ci/appveyor-build.bat @@ -17,10 +17,6 @@ @echo on -IF /i "%JOB%" == "C#" goto csharp -IF /i "%JOB%" == "rust" goto rust -IF /i "%JOB%" == "go" goto golang - @rem All other builds are C++ builds goto cpp @@ -34,16 +30,4 @@ if "%JOB:~,5%" == "MinGW" ( ) goto scriptexit -:csharp -call ci\appveyor-csharp-build.bat -goto scriptexit - -:rust -call ci\rust-build-main.bat -goto scriptexit - -:golang -call ci\go-build-main.bat -goto scriptexit - :scriptexit diff --git a/ci/conda_env_cpp.yml b/ci/conda_env_cpp.yml index 30cb6bbedbbc2..694a19e613388 100644 --- a/ci/conda_env_cpp.yml +++ b/ci/conda_env_cpp.yml @@ -30,6 +30,7 @@ grpc-cpp>=1.21.4 gtest=1.8.1 libprotobuf lz4-c +make ninja pkg-config python diff --git a/ci/conda_env_python.yml b/ci/conda_env_python.yml index 0e6e5bf8d5c3c..7d66c5e3717ba 100644 --- a/ci/conda_env_python.yml +++ b/ci/conda_env_python.yml @@ -19,10 +19,10 @@ cython=0.29.7 cloudpickle hypothesis numpy>=1.14 -pandas pytest pytest-faulthandler pytest-lazy-fixture pytz setuptools setuptools_scm=3.2.0 +pickle5 diff --git a/cpp/Dockerfile b/ci/docker/conda-latest-cpp.dockerfile similarity index 57% rename from cpp/Dockerfile rename to ci/docker/conda-latest-cpp.dockerfile index dcef56058901c..5d6a458acef58 100644 --- a/cpp/Dockerfile +++ b/ci/docker/conda-latest-cpp.dockerfile @@ -15,48 +15,57 @@ # specific language governing permissions and limitations # under the License. -FROM ubuntu:18.04 +ARG arch=amd64 +FROM ${arch}/ubuntu:18.04 + +# arch is unset after the FROM statement, so need to define it again +ARG arch=amd64 +ARG conda=latest +ARG minio=latest +ARG prefix=/opt/conda # install build essentials RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update -y -q && \ - apt-get install -y -q --no-install-recommends \ - ca-certificates \ - ccache \ - g++ \ - gcc \ - git \ - ninja-build \ - pkg-config \ - tzdata \ - wget \ + apt-get install -y -q wget tzdata \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* -# install conda and required packages -ARG EXTRA_CONDA_PKGS -ENV PATH=/opt/conda/bin:$PATH \ - CONDA_PREFIX=/opt/conda -COPY ci/docker_install_conda.sh \ - ci/conda_env_cpp.yml \ +ENV PATH=${prefix}/bin:$PATH \ + CONDA_PREFIX=${prefix} +# install conda and minio +COPY ci/scripts/install_conda.sh \ + ci/scripts/install_minio.sh \ + /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_conda.sh ${arch} linux ${minio} ${prefix} && \ + /arrow/ci/scripts/install_minio.sh ${arch} linux ${conda} ${prefix} + +# install the required conda packages +COPY ci/conda_env_cpp.yml \ ci/conda_env_gandiva.yml \ ci/conda_env_unix.yml \ /arrow/ci/ -RUN arrow/ci/docker_install_conda.sh && \ - conda install -q -c conda-forge \ +RUN conda install -q \ + --file arrow/ci/conda_env_unix.yml \ --file arrow/ci/conda_env_cpp.yml \ --file arrow/ci/conda_env_gandiva.yml \ - --file arrow/ci/conda_env_unix.yml \ - $EXTRA_CONDA_PKGS && \ + git compilers && \ conda clean --all +# TODO(kszucs): compilers instead of conda_env_linux -ENV CC=gcc \ - CXX=g++ \ +ENV ARROW_S3=ON \ + ARROW_ORC=ON \ ARROW_GANDIVA=ON \ + ARROW_PARQUET=ON \ + ARROW_PLASMA=ON \ + ARROW_FLIGHT=ON \ + ARROW_WITH_ZLIB=ON \ + ARROW_WITH_LZ4=ON \ + ARROW_WITH_BZ2=ON \ + ARROW_WITH_ZSTD=ON \ + ARROW_WITH_SNAPPY=ON \ + ARROW_WITH_BROTLI=ON \ ARROW_BUILD_TESTS=ON \ ARROW_DEPENDENCY_SOURCE=CONDA \ ARROW_HOME=$CONDA_PREFIX \ PARQUET_HOME=$CONDA_PREFIX - -# build and test -CMD ["arrow/ci/docker_build_and_test_cpp.sh"] diff --git a/integration/dask/Dockerfile b/ci/docker/conda-latest-integration.dockerfile similarity index 53% rename from integration/dask/Dockerfile rename to ci/docker/conda-latest-integration.dockerfile index a67c99072c491..0105d32fb20be 100644 --- a/integration/dask/Dockerfile +++ b/ci/docker/conda-latest-integration.dockerfile @@ -15,13 +15,38 @@ # specific language governing permissions and limitations # under the License. -FROM arrowdev/arrow-python-3.6:latest +ARG org +ARG arch=amd64 +ARG conda=latest +FROM ${org}/${arch}-conda-${conda}-cpp:latest -# install dask release from conda -RUN conda install -c conda-forge dask pytest=3 && \ +ARG arch=amd64 +ARG maven=3.5 +ARG node=11 +ARG jdk=8 +ARG go=1.12 + +RUN conda install -q \ + numpy \ + maven=${maven} \ + nodejs=${node} \ + openjdk=${jdk} && \ conda clean --all -# build and test -CMD ["/bin/bash", "-c", "arrow/ci/docker_build_cpp.sh && \ - arrow/ci/docker_build_python.sh && \ - arrow/integration/dask/runtest.sh"] +ENV GOROOT=/opt/go \ + GOBIN=/opt/go/bin \ + GOPATH=/go \ + PATH=/opt/go/bin:$PATH +RUN wget -nv -O - https://dl.google.com/go/go${go}.linux-${arch}.tar.gz | tar -xzf - -C /opt + +ENV ARROW_BUILD_INTEGRATION=ON \ + ARROW_FLIGHT=ON \ + ARROW_ORC=OFF \ + ARROW_DATASET=OFF \ + ARROW_GANDIVA=OFF \ + ARROW_PLASMA=OFF \ + ARROW_FILESYSTEM=OFF \ + ARROW_HDFS=OFF \ + ARROW_JEMALLOC=OFF \ + ARROW_JSON=OFF \ + ARROW_USE_GLOG=OFF diff --git a/ci/travis_script_go.sh b/ci/docker/conda-latest-python-dask.dockerfile old mode 100755 new mode 100644 similarity index 65% rename from ci/travis_script_go.sh rename to ci/docker/conda-latest-python-dask.dockerfile index 23eb29ca4426b..b47670dde94f8 --- a/ci/travis_script_go.sh +++ b/ci/docker/conda-latest-python-dask.dockerfile @@ -1,5 +1,3 @@ -#!/usr/bin/env bash - # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -17,23 +15,17 @@ # specific language governing permissions and limitations # under the License. -set -e - -GO_DIR=${TRAVIS_BUILD_DIR}/go/arrow - -pushd $GO_DIR - -go get -d -t -v ./... -go install -v ./... - -echo "" > coverage.txt - -for d in $(go list ./... | grep -v vendor); do - go test -race -coverprofile=profile.out -covermode=atomic $d - if [ -f profile.out ]; then - cat profile.out >> coverage.txt - rm profile.out +ARG org +ARG arch=amd64 +ARG conda=latest +ARG python=3.6 +FROM ${org}/${arch}-conda-${conda}-python-${python}:latest + +ARG dask=latest +RUN if [ "${dask}" = "master" ]; then \ + pip install https://github.com/dask/dask/archive/master.zip; \ + elif [ "${dask}" = "latest" ]; then \ + conda install -q dask && conda clean --all; \ + else \ + conda install -q dask=${dask} && conda clean --all; \ fi -done - -popd diff --git a/cpp/Dockerfile.alpine b/ci/docker/conda-latest-python-hdfs.dockerfile similarity index 52% rename from cpp/Dockerfile.alpine rename to ci/docker/conda-latest-python-hdfs.dockerfile index fe6d4bb0f64b5..38527244004e3 100644 --- a/cpp/Dockerfile.alpine +++ b/ci/docker/conda-latest-python-hdfs.dockerfile @@ -15,41 +15,36 @@ # specific language governing permissions and limitations # under the License. -FROM alpine:3.9 +ARG org +ARG arch=amd64 +ARG conda=latest +ARG python=3.6 +FROM ${org}/${arch}-conda-${conda}-python-${python}:latest -# Install basic build dependencies -# grpc requires libnsl-dev to be present, this cannot be installed via a bundle. -RUN apk add --no-cache -q \ - autoconf \ - bash \ - bison \ - boost-dev \ - ccache \ - cmake \ - flex \ - g++ \ - gcc \ - git \ - gzip \ - make \ - libnsl-dev \ - musl-dev \ - ninja \ - openssl-dev \ - wget \ - zlib-dev +ARG jdk=8 +ARG maven=3.5 +RUN conda install -q \ + pandas \ + openjdk=${jdk} \ + maven=${maven} && \ + conda clean --all -# Ganidva is deactivated as we don't support building LLVM via ExternalProject -# and Alpine only has LLVM 6 in its repositories yet. -# ARROW-4917: ORC fails with compiler problems +# installing libhdfs (JNI) +ARG hdfs=2.9.2 +ENV HADOOP_HOME=/opt/hadoop-${hdfs} \ + HADOOP_OPTS=-Djava.library.path=/opt/hadoop-${hdfs}/lib/native \ + PATH=$PATH:/opt/hadoop-${hdfs}/bin:/opt/hadoop-${hdfs}/sbin +RUN wget -q -O - "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=hadoop/common/hadoop-${hdfs}/hadoop-${hdfs}.tar.gz" | tar -xzf - -C /opt +COPY ci/etc/hdfs-site.xml $HADOOP_HOME/etc/hadoop/ + +# build cpp with tests ENV CC=gcc \ CXX=g++ \ ARROW_FLIGHT=OFF \ ARROW_GANDIVA=OFF \ - ARROW_ORC=OFF \ + ARROW_PLASMA=OFF \ ARROW_PARQUET=ON \ - ARROW_BUILD_TESTS=ON \ - ARROW_HOME=/usr/local - -# build and test -CMD ["arrow/ci/docker_build_and_test_cpp.sh"] + ARROW_ORC=ON \ + ARROW_HDFS=ON \ + ARROW_PYTHON=ON \ + ARROW_BUILD_TESTS=ON diff --git a/ci/docker_build_and_test_cpp.sh b/ci/docker/conda-latest-python-pandas.dockerfile old mode 100755 new mode 100644 similarity index 64% rename from ci/docker_build_and_test_cpp.sh rename to ci/docker/conda-latest-python-pandas.dockerfile index f2b9f54c88146..b4592a9b5b8f2 --- a/ci/docker_build_and_test_cpp.sh +++ b/ci/docker/conda-latest-python-pandas.dockerfile @@ -1,4 +1,3 @@ -#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -16,19 +15,17 @@ # specific language governing permissions and limitations # under the License. -set -e +ARG org +ARG arch=amd64 +ARG conda=latest +ARG python=3.6 +FROM ${org}/${arch}-conda-${conda}-python-${python}:latest -/arrow/ci/docker_build_cpp.sh -pushd /build/cpp - -export ARROW_TEST_DATA=/arrow/testing/data -export PARQUET_TEST_DATA=/arrow/cpp/submodules/parquet-testing/data - -# ARROW-5653 after install target, RPATH is modified on linux. Add this such -# that libraries are found with conda. -if [[ ! -z "${CONDA_PREFIX}" ]]; then - export LD_LIBRARY_PATH=${CONDA_PREFIX}/lib -fi - -ninja unittest -popd +ARG pandas=latest +RUN if [ "${pandas}" = "master" ]; then \ + pip install https://github.com/pandas-dev/pandas/archive/master.zip; \ + elif [ "${pandas}" = "latest" ]; then \ + conda install -q pandas && conda clean --all; \ + else \ + conda install -q pandas=${pandas} && conda clean --all; \ + fi diff --git a/ci/docker/conda-latest-python-spark.dockerfile b/ci/docker/conda-latest-python-spark.dockerfile new file mode 100644 index 0000000000000..0a7b1cf25c055 --- /dev/null +++ b/ci/docker/conda-latest-python-spark.dockerfile @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG org +ARG arch=amd64 +ARG conda=latest +ARG python=3.6 +FROM ${org}/${arch}-conda-${conda}-python-${python}:latest + +ARG jdk=8 +ARG maven=3.5 +RUN conda install -q \ + patch \ + pandas \ + openjdk=${jdk} \ + maven=${maven} && \ + conda clean --all + +# ENV MAVEN_HOME=/usr/local/maven \ +# M2_HOME=/root/.m2 \ +# PATH=/root/.m2/bin:/usr/local/maven/bin:$PATH + +# installing specific version of spark +ARG spark=master +RUN mkdir /spark && wget -q -O - https://github.com/apache/spark/archive/${spark}.tar.gz | tar -xzf - --strip-components=1 -C /spark +# patch spark to build with current Arrow Java +COPY ci/etc/ARROW-6429.patch /tmp/ +RUN patch -d /spark -p1 -i /tmp/ARROW-6429.patch && \ + rm /tmp/ARROW-6429.patch + +# build cpp with tests +ENV CC=gcc \ + CXX=g++ \ + ARROW_PYTHON=ON \ + ARROW_HDFS=ON \ + ARROW_BUILD_TESTS=OFF diff --git a/integration/turbodbc/Dockerfile b/ci/docker/conda-latest-python-turbodbc.dockerfile similarity index 55% rename from integration/turbodbc/Dockerfile rename to ci/docker/conda-latest-python-turbodbc.dockerfile index d143ea748c994..99ede793941fc 100644 --- a/integration/turbodbc/Dockerfile +++ b/ci/docker/conda-latest-python-turbodbc.dockerfile @@ -15,24 +15,43 @@ # specific language governing permissions and limitations # under the License. -FROM arrowdev/arrow-python-3.6:latest +ARG org +ARG arch=amd64 +ARG conda=latest +ARG python=3.6 +FROM ${org}/${arch}-conda-${conda}-python-${python}:latest RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update -y -q && \ apt-get install -y -q --no-install-recommends \ odbc-postgresql \ postgresql \ - sudo \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* + sudo && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* # install turbodbc dependencies from conda-forge -RUN conda install -c conda-forge pybind11 pytest pytest-cov mock unixodbc && \ +RUN conda install -c conda-forge \ + pybind11 \ + pytest-cov \ + mock \ + unixodbc && \ conda clean --all -ENV TURBODBC_TEST_CONFIGURATION_FILES "query_fixtures_postgresql.json" +RUN service postgresql start && \ + sudo -u postgres psql -U postgres -c \ + "CREATE DATABASE test_db;" && \ + sudo -u postgres psql -U postgres -c \ + "ALTER USER postgres WITH PASSWORD 'password';" + +ARG turbodbc=latest +RUN git clone --recurse-submodules https://github.com/blue-yonder/turbodbc /turbodbc && \ + if [ "${turbodbc}" = "master" ]; then \ + git -C /turbodbc checkout master; \ + elif [ "${turbodbc}" = "latest" ]; then \ + git -C /turbodbc checkout $(git describe --tags); \ + else \ + git -C /turbodbc checkout ${turbodbc}; \ + fi -# build and test -CMD ["/bin/bash", "-c", "arrow/ci/docker_build_cpp.sh && \ - arrow/ci/docker_build_python.sh && \ - arrow/integration/turbodbc/runtest.sh"] +ENV TURBODBC_TEST_CONFIGURATION_FILES "query_fixtures_postgresql.json" diff --git a/python/Dockerfile b/ci/docker/conda-latest-python.dockerfile similarity index 86% rename from python/Dockerfile rename to ci/docker/conda-latest-python.dockerfile index 71bc2b2d4ee13..859d0b7cb190b 100644 --- a/python/Dockerfile +++ b/ci/docker/conda-latest-python.dockerfile @@ -15,19 +15,21 @@ # specific language governing permissions and limitations # under the License. -FROM arrowdev/arrow-cpp:latest +ARG org +ARG arch +ARG conda +FROM ${org}/${arch}-conda-${conda}-cpp:latest # install python specific packages -ARG PYTHON_VERSION=3.6 +ARG python=3.6 COPY ci/conda_env_python.yml /arrow/ci/ RUN conda install -q \ --file arrow/ci/conda_env_python.yml \ python=$PYTHON_VERSION \ + tensorflow \ nomkl && \ conda clean --all ENV ARROW_PYTHON=ON \ - ARROW_BUILD_TESTS=OFF - -# build and test -CMD ["arrow/ci/docker_build_and_test_python.sh"] + ARROW_BUILD_TESTS=OFF \ + ARROW_TENSORFLOW=ON diff --git a/r/Dockerfile.conda b/ci/docker/conda-latest-r.dockerfile similarity index 77% rename from r/Dockerfile.conda rename to ci/docker/conda-latest-r.dockerfile index 9525646cb250a..0dcc588170519 100644 --- a/r/Dockerfile.conda +++ b/ci/docker/conda-latest-r.dockerfile @@ -15,23 +15,23 @@ # specific language governing permissions and limitations # under the License. -FROM arrowdev/arrow-cpp:latest +ARG org +ARG arch +ARG conda +FROM ${org}/${arch}-conda-${conda}-cpp:latest # install R specific packages -ARG R_VERSION=3.6.1 +ARG r=3.6.1 COPY ci/conda_env_r.yml /arrow/ci/ RUN conda install -q \ --file arrow/ci/conda_env_r.yml \ - r-base=$R_VERSION \ + r-base=$r \ nomkl && \ conda clean --all # Ensure parallel compilation of each individual package RUN printf "\nMAKEFLAGS=-j8\n" >> /opt/conda/lib/R/etc/Makeconf -ENV MAKEFLAGS=-j8 -ENV R_CONDA=1 - # Arrow build flags ENV ARROW_FLIGHT=OFF \ ARROW_GANDIVA=OFF \ @@ -42,13 +42,5 @@ ENV ARROW_FLIGHT=OFF \ ARROW_USE_ASAN=OFF \ ARROW_USE_UBSAN=OFF \ ARROW_NO_DEPRECATED_API=ON \ - ARROW_R_DEV=TRUE - -ENV ARROW_BUILD_BENCHMARKS=off -ENV ARROW_BUILD_EXAMPLES=off -ENV ARROW_BUILD_TESTS=off -ENV ARROW_BUILD_UTILITIES=off - -# build, install, test R package -CMD ["/bin/bash", "-c", "/arrow/ci/docker_build_cpp.sh && \ - /arrow/ci/docker_build_r.sh"] + ARROW_R_DEV=TRUE \ + ARROW_BUILD_TESTS=OFF diff --git a/cpp/Dockerfile.ubuntu-bionic b/ci/docker/cuda-10.0-cpp.dockerfile similarity index 66% rename from cpp/Dockerfile.ubuntu-bionic rename to ci/docker/cuda-10.0-cpp.dockerfile index 4b79527433860..7545ea4c4620e 100644 --- a/cpp/Dockerfile.ubuntu-bionic +++ b/ci/docker/cuda-10.0-cpp.dockerfile @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -FROM ubuntu:bionic +FROM nvidia/cuda:10.0-devel-ubuntu18.04 # pipefail is enabled for proper error detection in the `wget | apt-key add` # step @@ -28,26 +28,6 @@ RUN apt-get update -y -q && \ wget software-properties-common gpg-agent && \ apt-get clean && rm -rf /var/lib/apt/lists* -# Installs LLVM toolchain, for gandiva and testing other compilers -# -# Note that this is installed before the base packages to improve iteration -# while debugging package list with docker build due to slow download speed of -# llvm compared to ubuntu apt mirrors. -ARG LLVM_VERSION=7 -# Args are only exposed in the "build" step, this ensure that LLVM_VERSION is -# found in the "run" step. -ENV LLVM_VERSION=${LLVM_VERSION} -ARG LLVM_APT_URL="http://apt.llvm.org/bionic/" -ARG LLVM_APT_ARCH="llvm-toolchain-bionic-${LLVM_VERSION}" -RUN wget -q -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ - apt-add-repository -y --update "deb ${LLVM_APT_URL} ${LLVM_APT_ARCH} main" && \ - apt-get install -y -q --no-install-recommends \ - clang-${LLVM_VERSION} \ - clang-format-${LLVM_VERSION} \ - clang-tidy-${LLVM_VERSION} \ - llvm-${LLVM_VERSION}-dev && \ - apt-get clean && rm -rf /var/lib/apt/lists* - # Installs C++ toolchain and dependencies RUN apt-get update -y -q && \ apt-get install -y -q --no-install-recommends \ @@ -85,6 +65,7 @@ RUN apt-get update -y -q && \ tzdata && \ apt-get clean && rm -rf /var/lib/apt/lists* +# Prioritize system packages and local installation # The following dependencies will be downloaded due to missing/invalid packages # provided by the distribution: # - libc-ares-dev does not install CMake config files @@ -92,26 +73,15 @@ RUN apt-get update -y -q && \ # - libgtest-dev only provide sources # - libprotobuf-dev only provide sources # - thrift is too old -ENV CMAKE_ARGS="-DThrift_SOURCE=BUNDLED \ --DGTest_SOURCE=BUNDLED \ --DORC_SOURCE=BUNDLED" - -# Prioritize system packages and local installation ENV ARROW_DEPENDENCY_SOURCE=SYSTEM \ ARROW_FLIGHT=OFF \ - ARROW_GANDIVA=ON \ - ARROW_HDFS=ON \ - ARROW_ORC=ON \ - ARROW_PARQUET=ON \ + ARROW_CUDA=ON \ ARROW_PLASMA=ON \ ARROW_USE_ASAN=ON \ ARROW_USE_UBSAN=ON \ ARROW_NO_DEPRECATED_API=ON \ ARROW_INSTALL_NAME_RPATH=OFF \ - ARROW_WITH_BZ2=ON \ - ARROW_WITH_ZSTD=ON - -ENV CC=clang-${LLVM_VERSION} \ - CXX=clang++-${LLVM_VERSION} - -CMD ["arrow/ci/docker_build_and_test_cpp.sh"] + ARROW_HOME=/usr/local \ + ORC_SOURCE=BUNDLED \ + Thrift_SOURCE=BUNDLED \ + GTest_SOURCE=BUNDLED diff --git a/cpp/Dockerfile.debian-testing b/ci/docker/debian-10-cpp.dockerfile similarity index 73% rename from cpp/Dockerfile.debian-testing rename to ci/docker/debian-10-cpp.dockerfile index a7f1ef2a76f55..f7773006c56f3 100644 --- a/cpp/Dockerfile.debian-testing +++ b/ci/docker/debian-10-cpp.dockerfile @@ -15,7 +15,8 @@ # specific language governing permissions and limitations # under the License. -FROM debian:testing +ARG arch=amd64 +FROM ${arch}/debian:10 # install build essentials RUN export DEBIAN_FRONTEND=noninteractive && \ @@ -40,8 +41,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ libgoogle-glog-dev \ libgtest-dev \ liblz4-dev \ - libprotobuf-dev \ - libprotoc-dev \ libre2-dev \ libsnappy-dev \ libssl-dev \ @@ -50,29 +49,33 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ llvm-7-dev \ make \ ninja-build \ - openjdk-11-jdk \ - openjdk-11-jdk-headless \ pkg-config \ protobuf-compiler \ rapidjson-dev \ thrift-compiler \ tzdata \ - wget \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* + zlib1g-dev \ + wget && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* ENV CC=gcc \ - CXX=g++ \ - ARROW_BUILD_TESTS=ON \ - ARROW_DEPENDENCY_SOURCE=SYSTEM \ - ARROW_FLIGHT=ON \ - ARROW_GANDIVA=ON \ - ARROW_GANDIVA_JAVA=ON \ - ARROW_HOME=/usr \ - JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64 \ - CMAKE_ARGS="-Dc-ares_SOURCE=BUNDLED \ --DgRPC_SOURCE=BUNDLED \ --DORC_SOURCE=BUNDLED" - -# build and test -CMD ["arrow/ci/docker_build_and_test_cpp.sh"] + CXX=g++ \ + cares_SOURCE=BUNDLED \ + ORC_SOURCE=BUNDLED \ + gRPC_SOURCE=BUNDLED \ + Protobuf_SOURCE=BUNDLED \ + ARROW_BUILD_TESTS=ON \ + ARROW_DEPENDENCY_SOURCE=SYSTEM \ + ARROW_FLIGHT=ON \ + ARROW_PARQUET=ON \ + ARROW_PLASMA=ON \ + ARROW_ORC=ON \ + ARROW_GANDIVA=ON \ + ARROW_WITH_ZLIB=ON \ + ARROW_WITH_LZ4=ON \ + ARROW_WITH_BZ2=ON \ + ARROW_WITH_ZSTD=ON \ + ARROW_WITH_SNAPPY=ON \ + ARROW_WITH_BROTLI=ON \ + ARROW_HOME=/usr/local diff --git a/ci/docker/debian-10-go.dockerfile b/ci/docker/debian-10-go.dockerfile new file mode 100644 index 0000000000000..49ed5a51652e8 --- /dev/null +++ b/ci/docker/debian-10-go.dockerfile @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG arch=amd64 +ARG go=1.12 +FROM ${arch}/golang:${go} + +# TODO(kszucs): +# 1. add the files required to install the dependencies to .dockeignore +# 2. copy these files to their appropiate path +# 3. download and compile the dependencies diff --git a/ci/docker/debian-10-js.dockerfile b/ci/docker/debian-10-js.dockerfile new file mode 100644 index 0000000000000..a5fefdf8bd51d --- /dev/null +++ b/ci/docker/debian-10-js.dockerfile @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG arch=amd64 +ARG node=11 +FROM ${arch}/node:${node} + +ENV NODE_NO_WARNINGS=1 + +# TODO(kszucs): +# 1. add the files required to install the dependencies to .dockeignore +# 2. copy these files to their appropiate path +# 3. download and compile the dependencies diff --git a/ci/docker/debian-10-rust.dockerfile b/ci/docker/debian-10-rust.dockerfile new file mode 100644 index 0000000000000..ecdfa0c800456 --- /dev/null +++ b/ci/docker/debian-10-rust.dockerfile @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG arch=amd64 +FROM ${arch}/rust + +# install pre-requisites for building flatbuffers +RUN apt-get update -y && \ + apt-get install -y build-essential cmake && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# install flatbuffers +ARG flatbuffers=1.11.0 +RUN wget -q -O - https://github.com/google/flatbuffers/archive/v${flatbuffers}.tar.gz | tar -xzf - && \ + cd flatbuffers-${flatbuffers} && \ + cmake -G "Unix Makefiles" && \ + make install && \ + cd / && \ + rm -rf flatbuffers-${flatbuffers} + +# sadly cargo doesn't have a command to fetch and build the +# dependencies without building the library itself +ARG rust=nightly-2019-09-25 +RUN rustup default ${rust} + +# TODO(kszucs): +# 1. add the files required to install the dependencies to .dockeignore +# 2. copy these files to their appropiate path +# 3. download and compile the dependencies diff --git a/ci/docker/debian-9-java.dockerfile b/ci/docker/debian-9-java.dockerfile new file mode 100644 index 0000000000000..d4f650b92bf9c --- /dev/null +++ b/ci/docker/debian-9-java.dockerfile @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG arch=amd64 +ARG jdk=8 +ARG maven=3.5.4 +FROM ${arch}/maven:${maven}-jdk-${jdk} + +ENV ARROW_JAVA_SHADE_FLATBUFS=ON + +# TODO(kszucs): +# 1. add the files required to install the dependencies to .dockeignore +# 2. copy these files to their appropiate path +# 3. download and compile the dependencies diff --git a/cpp/Dockerfile.fedora b/ci/docker/fedora-29-cpp.dockerfile similarity index 67% rename from cpp/Dockerfile.fedora rename to ci/docker/fedora-29-cpp.dockerfile index 3e79ea29bb244..4f12878adc2cb 100644 --- a/cpp/Dockerfile.fedora +++ b/ci/docker/fedora-29-cpp.dockerfile @@ -15,7 +15,8 @@ # specific language governing permissions and limitations # under the License. -FROM fedora:29 +ARG arch +FROM ${arch}/fedora:29 # install dependencies RUN dnf update -y && \ @@ -59,19 +60,31 @@ RUN dnf update -y && \ # when built via ExternalProject: https://bugzilla.redhat.com/show_bug.cgi?id=1687844 # * protobuf libraries in Fedora 29 are too old for gRPC ENV CC=gcc \ - CXX=g++ \ - ARROW_ORC=ON \ - ARROW_DEPENDENCY_SOURCE=SYSTEM \ - ARROW_PARQUET=ON \ - ARROW_FLIGHT=ON \ - ARROW_GANDIVA=ON \ - ARROW_GANDIVA_JAVA=ON \ - ARROW_BUILD_TESTS=ON \ - ARROW_HOME=/usr/local \ - CMAKE_ARGS="\ --Dc-ares_SOURCE=BUNDLED \ --DgRPC_SOURCE=BUNDLED \ --DORC_SOURCE=BUNDLED \ --DProtobuf_SOURCE=BUNDLED" + CXX=g++ \ + ARROW_ORC=ON \ + ARROW_DEPENDENCY_SOURCE=SYSTEM \ + cares_SOURCE=BUNDLED \ + gRPC_SOURCE=BUNDLED \ + ORC_SOURCE=BUNDLED \ + Protobuf_SOURCE=BUNDLED \ + ARROW_WITH_ZLIB=ON \ + ARROW_WITH_LZ4=ON \ + ARROW_WITH_BZ2=ON \ + ARROW_WITH_ZSTD=ON \ + ARROW_WITH_SNAPPY=ON \ + ARROW_WITH_BROTLI=ON \ + ARROW_PARQUET=ON \ + ARROW_FLIGHT=ON \ + ARROW_GANDIVA=OFF \ + ARROW_GANDIVA_JAVA=ON \ + ARROW_BUILD_TESTS=ON \ + ARROW_HOME=/usr/local -CMD ["arrow/ci/docker_build_and_test_cpp.sh"] +# Gandiva test is failing with: +# Running gandiva-internals-test, redirecting output into /build/cpp/build/test-logs/gandiva-internals-test.txt (attempt 1/1) +# 1364 +# : CommandLine Error: Option 'x86-experimental-vector-widening-legalization' registered more than once! +# 1365 +# LLVM ERROR: inconsistency in registered CommandLine options +# 1366 +# /build/cpp/src/gandiva diff --git a/ci/docker/linux-apt-c-glib.dockerfile b/ci/docker/linux-apt-c-glib.dockerfile new file mode 100644 index 0000000000000..86e028b0ad311 --- /dev/null +++ b/ci/docker/linux-apt-c-glib.dockerfile @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG base +FROM ${base} + +RUN apt-get update -y -q && \ + apt-get install -y -q \ + python3 \ + python3-pip \ + gtk-doc-tools \ + libgirepository1.0-dev \ + libglib2.0-doc \ + luarocks \ + pkg-config \ + ruby-dev && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +RUN luarocks install lgi + +RUN pip3 install meson && \ + gem install bundler + +COPY c_glib/Gemfile /arrow/c_glib/ +RUN bundle install --gemfile /arrow/c_glib/Gemfile + +ENV ARROW_BUILD_TESTS=OFF \ + ARROW_BUILD_UTILITIES=OFF \ + ARROW_INSTALL_NAME_RPATH=OFF diff --git a/dev/fuzzit/Dockerfile b/ci/docker/linux-apt-cmake.dockerfile similarity index 73% rename from dev/fuzzit/Dockerfile rename to ci/docker/linux-apt-cmake.dockerfile index 447f1b844dac4..7a2024db5a9e2 100644 --- a/dev/fuzzit/Dockerfile +++ b/ci/docker/linux-apt-cmake.dockerfile @@ -15,14 +15,15 @@ # specific language governing permissions and limitations # under the License. -FROM arrowdev/arrow-cpp:latest +ARG base +FROM ${base} RUN apt-get update && \ apt-get install -y -q \ - clang-7 \ - libclang-7-dev \ - clang-format-7 \ - clang-tidy-7 \ - clang-tools-7 + libidn11 \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* -CMD ["arrow/ci/docker_build_and_fuzzit.sh"] +ARG cmake=3.2.3 +RUN wget -nv -O - https://github.com/Kitware/CMake/releases/download/v${cmake}/cmake-${cmake}-Linux-x86_64.tar.gz | tar -xzf - -C /opt +ENV PATH=/opt/cmake-${cmake}-Linux-x86_64/bin:$PATH diff --git a/c_glib/Dockerfile b/ci/docker/linux-apt-docs.dockerfile similarity index 50% rename from c_glib/Dockerfile rename to ci/docker/linux-apt-docs.dockerfile index 32fbc79d05d65..875af921e1e0c 100644 --- a/c_glib/Dockerfile +++ b/ci/docker/linux-apt-docs.dockerfile @@ -15,36 +15,45 @@ # specific language governing permissions and limitations # under the License. -FROM arrowdev/arrow-cpp:latest +ARG base +FROM ${base} +ARG jdk=8 RUN apt-get update -y -q && \ - apt-get -q install --no-install-recommends -y \ + apt-get install -y -q --no-install-recommends \ autoconf-archive \ + automake \ + doxygen \ gobject-introspection \ gtk-doc-tools \ libgirepository1.0-dev \ - libglib2.0-dev \ - pkg-config \ + libglib2.0-doc \ + libtool \ + ninja-build \ + openjdk-${jdk}-jdk-headless \ ruby-dev \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* + rsync && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* +ENV JAVA_HOME=/usr/lib/jvm/java-${jdk}-openjdk-amd64 -COPY c_glib/Gemfile /arrow/c_glib/ -RUN conda install meson=0.50.1 && \ - conda clean --all && \ - gem install bundler && \ - bundle install --gemfile arrow/c_glib/Gemfile +ARG maven=3.5.4 +RUN wget -q -O - "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=maven/maven-3/${maven}/binaries/apache-maven-${maven}-bin.tar.gz" | tar -xzf - -C /opt +ENV PATH=/opt/apache-maven-${maven}/bin:$PATH + +ARG node=11 +RUN wget -q -O - https://deb.nodesource.com/setup_${node}.x | bash - && \ + apt-get install -y nodejs && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* -# build cpp -ENV ARROW_BUILD_TESTS=OFF \ - ARROW_BUILD_UTILITIES=OFF \ - ARROW_INSTALL_NAME_RPATH=OFF \ - LD_LIBRARY_PATH="${CONDA_PREFIX}/lib" \ - PKG_CONFIG=/usr/bin/pkg-config \ - PKG_CONFIG_PATH="${CONDA_PREFIX}/lib/pkgconfig" \ - GI_TYPELIB_PATH="${CONDA_PREFIX}/lib/girepository-1.0" +RUN pip install \ + meson \ + breathe \ + ipython \ + sphinx \ + sphinx_rtd_theme -# build, install and test -CMD ["/bin/bash", "-c", "arrow/ci/docker_build_cpp.sh && \ - arrow/ci/docker_build_c_glib.sh && \ - arrow/c_glib/test/run-test.rb"] +COPY c_glib/Gemfile /arrow/c_glib/ +RUN gem install bundler && \ + bundle install --gemfile /arrow/c_glib/Gemfile diff --git a/ci/docker/linux-apt-fuzzit.dockerfile b/ci/docker/linux-apt-fuzzit.dockerfile new file mode 100644 index 0000000000000..5cb14020ae467 --- /dev/null +++ b/ci/docker/linux-apt-fuzzit.dockerfile @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG base +FROM ${base} + +RUN apt-get update && \ + apt-get install -y -q \ + clang-7 \ + libclang-7-dev \ + clang-format-7 \ + clang-tidy-7 \ + clang-tools-7 && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +RUN wget -O /usr/local/bin/fuzzit https://github.com/fuzzitdev/fuzzit/releases/latest/download/fuzzit_Linux_x86_64 && \ + chmod a+x /usr/local/bin/fuzzit + +ENV ARROW_FUZZING="ON" \ + ARROW_USE_ASAN="ON" \ + CC="clang-7" \ + CXX="clang++-7" \ + ARROW_BUILD_TYPE="RelWithDebInfo" \ + ARROW_FLIGHT="OFF" \ + ARROW_GANDIVA="OFF" \ + ARROW_ORC="OFF" \ + ARROW_PARQUET="OFF" \ + ARROW_PLASMA="OFF" \ + ARROW_WITH_BZ2="OFF" \ + ARROW_WITH_ZSTD="OFF" \ + ARROW_BUILD_BENCHMARKS="OFF" \ + ARROW_BUILD_UTILITIES="OFF" diff --git a/cpp/Dockerfile.cmake b/ci/docker/linux-apt-jni.dockerfile similarity index 50% rename from cpp/Dockerfile.cmake rename to ci/docker/linux-apt-jni.dockerfile index 9160cd7e8531b..75b3b31c8793c 100644 --- a/cpp/Dockerfile.cmake +++ b/ci/docker/linux-apt-jni.dockerfile @@ -15,54 +15,56 @@ # specific language governing permissions and limitations # under the License. -FROM ubuntu:18.04 +ARG base +FROM ${base} # install build essentials RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update -y -q && \ apt-get install -y -q --no-install-recommends \ ca-certificates \ - curl \ ccache \ + clang-7 \ + cmake \ + git \ g++ \ gcc \ - git \ - libidn11 \ + libboost-all-dev \ + libdouble-conversion-dev \ + libgflags-dev \ + libgoogle-glog-dev \ + libgtest-dev \ + liblz4-dev \ + libre2-dev \ + libsnappy-dev \ + libssl-dev \ + llvm-7-dev \ + make \ ninja-build \ pkg-config \ + protobuf-compiler \ + rapidjson-dev \ + thrift-compiler \ tzdata \ - wget \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* + zlib1g-dev \ + wget && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* -# install conda and required packages -ARG EXTRA_CONDA_PKGS -ENV PATH=/opt/conda/bin:$PATH \ - CONDA_PREFIX=/opt/conda -COPY ci/docker_install_conda.sh \ - ci/conda_env_cpp.yml \ - ci/conda_env_unix.yml \ - /arrow/ci/ -RUN arrow/ci/docker_install_conda.sh && \ - conda install -q -c conda-forge \ - --file arrow/ci/conda_env_cpp.yml \ - --file arrow/ci/conda_env_unix.yml \ - $EXTRA_CONDA_PKGS && \ - conda clean --all - -ARG CMAKE_VERSION=3.2.3 -ARG CMAKE_URL=https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz -RUN curl ${CMAKE_URL} -Lo cmake-${CMAKE_VERSION}.tar.gz && \ - mkdir /opt/cmake && tar -zxvf cmake-${CMAKE_VERSION}.tar.gz -C /opt/cmake -ENV PATH=/opt/cmake/cmake-${CMAKE_VERSION}-Linux-x86_64/bin:$PATH +ARG cmake=3.11.4 +RUN wget -nv -O - https://github.com/Kitware/CMake/releases/download/v${cmake}/cmake-${cmake}-Linux-x86_64.tar.gz | tar -xzf - -C /opt +ENV PATH=/opt/cmake-${cmake}-Linux-x86_64/bin:$PATH ENV CC=gcc \ CXX=g++ \ - ARROW_GANDIVA=OFF \ - ARROW_BUILD_TESTS=ON \ - ARROW_DEPENDENCY_SOURCE=CONDA \ - ARROW_HOME=$CONDA_PREFIX \ - PARQUET_HOME=$CONDA_PREFIX - -# build and test -CMD ["arrow/ci/docker_build_and_test_cpp.sh"] + ORC_SOURCE=BUNDLED \ + Protobuf_SOURCE=BUNDLED \ + ARROW_BUILD_TESTS=OFF \ + ARROW_FLIGHT=OFF \ + ARROW_PARQUET=OFF \ + ARROW_PLASMA=ON \ + ARROW_ORC=ON \ + ARROW_JNI=ON \ + ARROW_GANDIVA=ON \ + ARROW_GANDIVA_JAVA=ON \ + ARROW_HOME=/usr/local diff --git a/ci/docker/Dockerfile.lint b/ci/docker/linux-apt-lint.dockerfile similarity index 61% rename from ci/docker/Dockerfile.lint rename to ci/docker/linux-apt-lint.dockerfile index d24e93c14a5fb..205500ac2c603 100644 --- a/ci/docker/Dockerfile.lint +++ b/ci/docker/linux-apt-lint.dockerfile @@ -15,43 +15,47 @@ # specific language governing permissions and limitations # under the License. +ARG base FROM hadolint/hadolint:v1.17.2 AS hadolint - -FROM ubuntu:18.04 - -SHELL ["/bin/bash", "-o", "pipefail", "-c"] +FROM ${base} RUN apt-get update && \ - apt-get install -y -q \ - build-essential \ - clang-7 \ - clang-format-7 \ - clang-tidy-7 \ - clang-tools-7 \ - cmake \ - curl \ - git \ - libclang-7-dev \ - ninja-build \ - openjdk-11-jdk-headless \ - python3 \ - python3-dev \ - python3-pip \ - ruby \ + apt-get install -y -q \ + clang-7 \ + clang-format-7 \ + clang-tidy-7 \ + clang-tools-7 \ + cmake \ + curl \ + libclang-7-dev \ + openjdk-11-jdk-headless \ + python3 \ + python3-dev \ + python3-pip \ + ruby \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* -# Use python3 by default in scripts -RUN ln -s /usr/bin/python3 /usr/local/bin/python -RUN pip3 install flake8 cmake_format==0.5.2 +# Docker linter +COPY --from=hadolint /bin/hadolint /usr/bin/hadolint + +# IWYU +COPY ci/scripts/install_iwyu.sh /arrow/ci/scripts/ +RUN arrow/ci/scripts/install_iwyu.sh /tmp/iwyu /usr/local 7 # Rust linter +ARG rust=nightly-2019-09-25 RUN curl https://sh.rustup.rs -sSf | \ sh -s -- --default-toolchain stable -y ENV PATH /root/.cargo/bin:$PATH -RUN rustup component add rustfmt +RUN rustup install ${rust} && \ + rustup default ${rust} && \ + rustup component add rustfmt -# Docker linter -COPY --from=hadolint /bin/hadolint /usr/bin/hadolint +# Use python3 by default in scripts +RUN ln -s /usr/bin/python3 /usr/local/bin/python && \ + ln -s /usr/bin/pip3 /usr/local/bin/pip +RUN pip install flake8 cmake_format==0.5.2 click -CMD ["/arrow/ci/docker/lint.sh"] +ENV LC_ALL=C.UTF-8 \ + LANG=C.UTF-8 diff --git a/dev/lint/Dockerfile b/ci/docker/linux-apt-python-3.dockerfile similarity index 60% rename from dev/lint/Dockerfile rename to ci/docker/linux-apt-python-3.dockerfile index 649120657587e..974a9292d5da3 100644 --- a/dev/lint/Dockerfile +++ b/ci/docker/linux-apt-python-3.dockerfile @@ -15,19 +15,29 @@ # specific language governing permissions and limitations # under the License. -FROM arrowdev/arrow-python-3.6 +ARG base +FROM ${base} -RUN apt-get update && \ +RUN apt-get update -y -q && \ apt-get install -y -q \ - clang-7 \ - libclang-7-dev \ - clang-format-7 \ - clang-tidy-7 \ - clang-tools-7 + python3 \ + python3-pip \ + python3-dev && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* -RUN conda install flake8 && \ - conda clean --all -y +RUN ln -s /usr/bin/python3 /usr/local/bin/python && \ + ln -s /usr/bin/pip3 /usr/local/bin/pip + +COPY python/requirements.txt \ + python/requirements-test.txt \ + /arrow/python/ + +RUN pip install \ + -r arrow/python/requirements.txt \ + -r arrow/python/requirements-test.txt \ + cython setuptools + +ENV ARROW_PYTHON=ON \ + ARROW_BUILD_TESTS=OFF -ENV PATH=/opt/iwyu/bin:$PATH -ADD ci/docker_install_iwyu.sh /arrow/ci/ -RUN arrow/ci/docker_install_iwyu.sh diff --git a/ci/docker/linux-apt-r.dockerfile b/ci/docker/linux-apt-r.dockerfile new file mode 100644 index 0000000000000..c0d7755d71444 --- /dev/null +++ b/ci/docker/linux-apt-r.dockerfile @@ -0,0 +1,67 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG base +FROM ${base} + +# Build R +# [1] https://www.digitalocean.com/community/tutorials/how-to-install-r-on-ubuntu-18-04 +# [2] https://linuxize.com/post/how-to-install-r-on-ubuntu-18-04/#installing-r-packages-from-cran +ARG r=3.6 +RUN apt-get update -y && \ + apt-get install -y \ + dirmngr \ + apt-transport-https \ + software-properties-common && \ + apt-key adv \ + --keyserver keyserver.ubuntu.com \ + --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 && \ + add-apt-repository 'deb https://cloud.r-project.org/bin/linux/ubuntu bionic-cran35/' && \ + apt-get install -y \ + r-base=${r}* \ + # system libs needed by core R packages + libxml2-dev \ + libgit2-dev \ + libssl-dev \ + # install clang to mirror what was done on Travis + clang \ + clang-format \ + clang-tidy \ + # R CMD CHECK --as-cran needs pdflatex to build the package manual + texlive-latex-base && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# Ensure parallel R package installation, set CRAN repo mirror, +# and use pre-built binaries where possible +RUN printf "\ + options(Ncpus = parallel::detectCores(), \ + repos = 'https://demo.rstudiopm.com/all/__linux__/bionic/latest', \ + HTTPUserAgent = sprintf(\ + 'R/%%s R (%%s)', getRversion(), \ + paste(getRversion(), R.version\$platform, R.version\$arch, R.version\$os)))\n" \ + >> /etc/R/Rprofile.site + +# Also ensure parallel compilation of each individual package +RUN echo "MAKEFLAGS=-j8" >> /usr/lib/R/etc/Makeconf + +ENV ARROW_DEPENDENCY_SOURCE=SYSTEM \ + ARROW_PARQUET=ON \ + ARROW_USE_ASAN=OFF \ + ARROW_USE_UBSAN=OFF \ + ARROW_NO_DEPRECATED_API=ON \ + ARROW_R_DEV=TRUE diff --git a/ci/docker/lint.sh b/ci/docker/linux-apt-ruby.dockerfile old mode 100755 new mode 100644 similarity index 80% rename from ci/docker/lint.sh rename to ci/docker/linux-apt-ruby.dockerfile index 18e2bb8a5dffd..188c9711cf291 --- a/ci/docker/lint.sh +++ b/ci/docker/linux-apt-ruby.dockerfile @@ -1,4 +1,3 @@ -#!/bin/bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -16,15 +15,10 @@ # specific language governing permissions and limitations # under the License. -set -eu +# depends on a C Glib image +ARG base +FROM ${base} -: ${ARROW_HOME:=/arrow} - -install_archery() { - pip3 install -e ${ARROW_HOME}/dev/archery - export LC_ALL=C.UTF-8 - export LANG=C.UTF-8 -} - -install_archery -archery lint +# TODO(kszucs): we should install the required dependencies here +# COPY ruby/ /arrow/ +# RUN bundle install --gemfile /arrow/ruby/red-arrow/Gemfile diff --git a/ci/docker/linux-dnf-python-3.dockerfile b/ci/docker/linux-dnf-python-3.dockerfile new file mode 100644 index 0000000000000..3ecd95a1917ec --- /dev/null +++ b/ci/docker/linux-dnf-python-3.dockerfile @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG base +FROM ${base} + +RUN dnf install -y \ + python3 \ + python3-pip \ + python3-devel + +RUN ln -s /usr/bin/python3 /usr/local/bin/python && \ + ln -s /usr/bin/pip3 /usr/local/bin/pip + +COPY python/requirements.txt \ + python/requirements-test.txt \ + /arrow/python/ + +RUN pip install \ + -r arrow/python/requirements.txt \ + -r arrow/python/requirements-test.txt \ + cython + +ENV ARROW_PYTHON=ON \ + ARROW_BUILD_TESTS=OFF diff --git a/cpp/Dockerfile.ubuntu-trusty b/ci/docker/ubuntu-14.04-cpp.dockerfile similarity index 65% rename from cpp/Dockerfile.ubuntu-trusty rename to ci/docker/ubuntu-14.04-cpp.dockerfile index 6418cb848ab30..f7bcf41cf25a1 100644 --- a/cpp/Dockerfile.ubuntu-trusty +++ b/ci/docker/ubuntu-14.04-cpp.dockerfile @@ -15,7 +15,8 @@ # specific language governing permissions and limitations # under the License. -FROM ubuntu:trusty +ARG arch=amd64 +FROM ${arch}/ubuntu:14.04 SHELL ["/bin/bash", "-o", "pipefail", "-c"] @@ -41,11 +42,11 @@ RUN apt-get update -y -q && \ pkg-config \ protobuf-compiler \ python-pip \ - tzdata \ - && apt-get clean \ - && python -m pip install --upgrade pip \ - && pip install --upgrade cmake \ - && rm -rf /var/lib/apt/lists/* + tzdata && \ + apt-get clean && \ + python -m pip install --upgrade pip && \ + pip install --upgrade cmake && \ + rm -rf /var/lib/apt/lists/* # - Flight is deactivated because the OpenSSL in Ubuntu 14.04 is too old # - Disabling Gandiva since LLVM 7 require gcc >= 4.9 toolchain @@ -57,30 +58,33 @@ RUN apt-get update -y -q && \ # - libre2-dev unavailable # - liblz4-dev is too old ENV CC=gcc \ - CXX=g++ \ - ARROW_BUILD_TESTS=ON \ - ARROW_DEPENDENCY_SOURCE=SYSTEM \ - ARROW_FLIGHT=OFF \ - ARROW_WITH_OPENSSL=OFF \ - ARROW_GANDIVA=OFF \ - ARROW_GANDIVA_JAVA=OFF \ - ARROW_ORC=OFF \ - ARROW_PARQUET=ON \ - ARROW_HOME=/usr \ - CMAKE_ARGS="-DThrift_SOURCE=BUNDLED \ --DBOOST_SOURCE=BUNDLED \ --Dbenchmark_SOURCE=BUNDLED \ --DBrotli_SOURCE=BUNDLED \ --Dc-ares_SOURCE=BUNDLED \ --Dgflags_SOURCE=BUNDLED \ --DGTest_SOURCE=BUNDLED \ --DgRPC_SOURCE=BUNDLED \ --DLz4_SOURCE=BUNDLED \ --DORC_SOURCE=BUNDLED \ --DProtobuf_SOURCE=BUNDLED \ --DRapidJSON_SOURCE=BUNDLED \ --DRE2_SOURCE=BUNDLED \ --DZSTD_SOURCE=BUNDLED" - -# build and test -CMD ["arrow/ci/docker_build_and_test_cpp.sh"] + CXX=g++ \ + ARROW_BUILD_TESTS=ON \ + ARROW_DEPENDENCY_SOURCE=SYSTEM \ + Thrift_SOURCE=BUNDLED \ + BOOST_SOURCE=BUNDLED \ + benchmark_SOURCE=BUNDLED \ + Brotli_SOURCE=BUNDLED \ + RE2_SOURCE=BUNDLED \ + ZSTD_SOURCE=BUNDLED \ + Lz4_SOURCE=BUNDLED \ + cares_SOURCE=BUNDLED \ + gflags_SOURCE=BUNDLED \ + GTest_SOURCE=BUNDLED \ + gRPC_SOURCE=BUNDLED \ + ORC_SOURCE=BUNDLED \ + Protobuf_SOURCE=BUNDLED \ + RapidJSON_SOURCE=BUNDLED \ + ARROW_FLIGHT=OFF \ + ARROW_WITH_OPENSSL=OFF \ + ARROW_GANDIVA=OFF \ + ARROW_GANDIVA_JAVA=OFF \ + ARROW_ORC=OFF \ + ARROW_PARQUET=ON \ + ARROW_WITH_ZLIB=ON \ + ARROW_WITH_LZ4=ON \ + ARROW_WITH_BZ2=ON \ + ARROW_WITH_ZSTD=ON \ + ARROW_WITH_SNAPPY=ON \ + ARROW_WITH_BROTLI=ON \ + ARROW_HOME=/usr/local diff --git a/cpp/Dockerfile.ubuntu-xenial b/ci/docker/ubuntu-16.04-cpp.dockerfile similarity index 75% rename from cpp/Dockerfile.ubuntu-xenial rename to ci/docker/ubuntu-16.04-cpp.dockerfile index b8cab8599f433..44b38ff582842 100644 --- a/cpp/Dockerfile.ubuntu-xenial +++ b/ci/docker/ubuntu-16.04-cpp.dockerfile @@ -15,7 +15,8 @@ # specific language governing permissions and limitations # under the License. -FROM ubuntu:xenial +ARG arch=amd64 +FROM ${arch}/ubuntu:16.04 SHELL ["/bin/bash", "-o", "pipefail", "-c"] @@ -45,16 +46,15 @@ RUN apt-get update -y -q && \ libre2-dev \ libsnappy-dev \ libssl-dev \ - libzstd-dev \ llvm-7-dev \ make \ ninja-build \ pkg-config \ protobuf-compiler \ - python \ - tzdata \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* + python3 \ + tzdata && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* # Benchmark is deactivated as the external project requires CMake 3.6+ # Gandiva JNI is deactivated as it requires CMake 3.11+ @@ -64,25 +64,26 @@ RUN apt-get update -y -q && \ # - libboost-all-dev does not include Boost.Process, needed for Flight # unit tests, so doing vendored build by default # - liburiparser-dev is too old (< 0.9.0) -ENV CC=gcc \ - CXX=g++ \ - ARROW_BUILD_BENCHMARKS=OFF \ - ARROW_BUILD_TESTS=ON \ - ARROW_DEPENDENCY_SOURCE=SYSTEM \ - ARROW_GANDIVA=ON \ - ARROW_GANDIVA_JAVA=OFF \ - ARROW_PARQUET=ON \ - ARROW_HOME=/usr \ - ARROW_WITH_ZSTD=OFF \ - CMAKE_ARGS="-DThrift_SOURCE=BUNDLED \ --DBOOST_SOURCE=BUNDLED \ --Dc-ares_SOURCE=BUNDLED \ --Dgflags_SOURCE=BUNDLED \ --DgRPC_SOURCE=BUNDLED \ --DGTest_SOURCE=BUNDLED \ --DORC_SOURCE=BUNDLED \ --DProtobuf_SOURCE=BUNDLED \ --DRapidJSON_SOURCE=BUNDLED" - -# build and test -CMD ["arrow/ci/docker_build_and_test_cpp.sh"] +ENV ARROW_BUILD_BENCHMARKS=OFF \ + ARROW_BUILD_TESTS=ON \ + ARROW_DEPENDENCY_SOURCE=SYSTEM \ + BOOST_SOURCE=BUNDLED \ + Thrift_SOURCE=BUNDLED \ + cares_SOURCE=BUNDLED \ + gRPC_SOURCE=BUNDLED \ + ORC_SOURCE=BUNDLED \ + GTest_SOURCE=BUNDLED \ + Protobuf_SOURCE=BUNDLED \ + RapidJSON_SOURCE=BUNDLED \ + ARROW_GANDIVA_JAVA=OFF \ + ARROW_GANDIVA=ON \ + ARROW_HOME=/usr/local \ + ARROW_PARQUET=ON \ + ARROW_WITH_ZLIB=ON \ + ARROW_WITH_LZ4=ON \ + ARROW_WITH_BZ2=ON \ + ARROW_WITH_ZSTD=OFF \ + ARROW_WITH_SNAPPY=ON \ + ARROW_WITH_BROTLI=ON \ + CXX=g++ \ + CC=gcc diff --git a/ci/docker/ubuntu-18.04-cpp.dockerfile b/ci/docker/ubuntu-18.04-cpp.dockerfile new file mode 100644 index 0000000000000..324b64787a042 --- /dev/null +++ b/ci/docker/ubuntu-18.04-cpp.dockerfile @@ -0,0 +1,119 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG arch=amd64 +FROM ${arch}/ubuntu:18.04 + +# pipefail is enabled for proper error detection in the `wget | apt-key add` +# step +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update -y -q && \ + apt-get install -y -q --no-install-recommends \ + wget \ + software-properties-common \ + gpg-agent && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists* + +# Installs LLVM toolchain, for gandiva and testing other compilers +# +# Note that this is installed before the base packages to improve iteration +# while debugging package list with docker build due to slow download speed of +# llvm compared to ubuntu apt mirrors. +ARG llvm_version=7 +ARG llvm_apt_url="http://apt.llvm.org/bionic/" +ARG llvm_apt_arch="llvm-toolchain-bionic-${llvm_version}" +RUN wget -q -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ + apt-add-repository -y --update "deb ${llvm_apt_url} ${llvm_apt_arch} main" && \ + apt-get install -y -q --no-install-recommends \ + clang-${llvm_version} \ + clang-format-${llvm_version} \ + clang-tidy-${llvm_version} \ + llvm-${llvm_version}-dev && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists* + +# Installs C++ toolchain and dependencies +RUN apt-get update -y -q && \ + apt-get install -y -q --no-install-recommends \ + autoconf \ + bison \ + ca-certificates \ + ccache \ + cmake \ + flex \ + g++ \ + gcc \ + git \ + libbenchmark-dev \ + libboost-filesystem-dev \ + libboost-regex-dev \ + libboost-system-dev \ + libbrotli-dev \ + libbz2-dev \ + libdouble-conversion-dev \ + libgflags-dev \ + libgoogle-glog-dev \ + liblz4-dev \ + libprotobuf-dev \ + libprotoc-dev \ + libre2-dev \ + libsnappy-dev \ + libssl-dev \ + libzstd-dev \ + ninja-build \ + pkg-config \ + protobuf-compiler \ + rapidjson-dev \ + thrift-compiler \ + tzdata && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists* + +# Prioritize system packages and local installation +# The following dependencies will be downloaded due to missing/invalid packages +# provided by the distribution: +# - libc-ares-dev does not install CMake config files +# - flatbuffer is not packaged +# - libgtest-dev only provide sources +# - libprotobuf-dev only provide sources +# - thrift is too old +ENV ARROW_BUILD_TESTS=ON \ + ARROW_FLIGHT=OFF \ + ARROW_GANDIVA=ON \ + ARROW_HDFS=ON \ + ARROW_HOME=/usr/local \ + ARROW_INSTALL_NAME_RPATH=OFF \ + ARROW_NO_DEPRECATED_API=ON \ + ARROW_ORC=ON \ + ARROW_PARQUET=ON \ + ARROW_PLASMA=ON \ + ARROW_USE_ASAN=OFF \ + ARROW_USE_UBSAN=OFF \ + ARROW_WITH_ZLIB=ON \ + ARROW_WITH_LZ4=ON \ + ARROW_WITH_BZ2=ON \ + ARROW_WITH_ZSTD=ON \ + ARROW_WITH_SNAPPY=ON \ + ARROW_WITH_BROTLI=ON \ + ARROW_DEPENDENCY_SOURCE=SYSTEM \ + ORC_SOURCE=BUNDLED \ + GTest_SOURCE=BUNDLED \ + Thrift_SOURCE=BUNDLED diff --git a/ci/docker/ubuntu-18.04-csharp.dockerfile b/ci/docker/ubuntu-18.04-csharp.dockerfile new file mode 100644 index 0000000000000..d602958f15873 --- /dev/null +++ b/ci/docker/ubuntu-18.04-csharp.dockerfile @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG platform=bionic +ARG dotnet=2.1 +FROM mcr.microsoft.com/dotnet/core/sdk:${dotnet}-${platform} diff --git a/r/Dockerfile.sanitizer b/ci/docker/ubuntu-18.04-r-sanitizer.dockerfile similarity index 85% rename from r/Dockerfile.sanitizer rename to ci/docker/ubuntu-18.04-r-sanitizer.dockerfile index f4c18220292d8..3ae6cf802257f 100644 --- a/r/Dockerfile.sanitizer +++ b/ci/docker/ubuntu-18.04-r-sanitizer.dockerfile @@ -70,6 +70,7 @@ ENV MAKEFLAGS=-j8 RUN RDsan -e "install.packages(c('remotes', 'dplyr', 'glue'))" RUN RDsan -e "remotes::install_github('romainfrancois/decor')" +# Prioritize system packages and local installation # The following dependencies will be downloaded due to missing/invalid packages # provided by the distribution: # - libc-ares-dev does not install CMake config files @@ -77,15 +78,12 @@ RUN RDsan -e "remotes::install_github('romainfrancois/decor')" # - libgtest-dev only provide sources # - libprotobuf-dev only provide sources # - thrift is too old -ENV CMAKE_ARGS="-DThrift_SOURCE=BUNDLED \ --DFlatbuffers_SOURCE=BUNDLED \ --DGTest_SOURCE=BUNDLED \ --Dc-ares_SOURCE=BUNDLED \ --DgRPC_SOURCE=BUNDLED \ --DProtobuf_SOURCE=BUNDLED ${CMAKE_ARGS}" - -# Prioritize system packages and local installation -ENV ARROW_DEPENDENCY_SOURCE=SYSTEM \ +ENV gRPC_SOURCE=BUNDLED \ + Protobuf_SOURCE=BUNDLED \ + cares_SOURCE=BUNDLED \ + GTest_SOURCE=BUNDLED \ + Thrift_SOURCE=BUNDLED \ + ARROW_DEPENDENCY_SOURCE=SYSTEM \ ARROW_FLIGHT=OFF \ ARROW_GANDIVA=OFF \ ARROW_HDFS=OFF \ @@ -98,12 +96,10 @@ ENV ARROW_DEPENDENCY_SOURCE=SYSTEM \ ARROW_INSTALL_NAME_RPATH=OFF \ ARROW_WITH_BZ2=OFF \ ARROW_WITH_ZSTD=OFF \ - ARROW_R_DEV=TRUE + ARROW_R_DEV=TRUE \ + R_BIN=RDsan -ENV ARROW_BUILD_BENCHMARKS=off -ENV ARROW_BUILD_EXAMPLES=off -ENV ARROW_BUILD_TESTS=off -ENV ARROW_BUILD_UTILITIES=off - -CMD ["/bin/bash", "-c", "/arrow/ci/docker_build_cpp.sh && \ - /arrow/ci/docker_build_r_sanitizer.sh"] +# Ensure parallel R package installation and set CRAN repo mirror +RUN printf "options(Ncpus = parallel::detectCores(), repos = 'https://demo.rstudiopm.com/all/__linux__/bionic/latest')\n" >> /etc/R/Rprofile.site +# Also ensure parallel compilation of each individual package +RUN printf "MAKEFLAGS=-j8\n" >> /usr/lib/R/etc/Makeconf diff --git a/ci/docker_build_and_fuzzit.sh b/ci/docker_build_and_fuzzit.sh deleted file mode 100755 index b3bf1bd81fed1..0000000000000 --- a/ci/docker_build_and_fuzzit.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -euxo pipefail - -export ARROW_FUZZING="ON" -#export ARROW_DEPENDENCY_SOURCE="BUNDLED" -export ARROW_USE_ASAN="ON" -export CC="clang-7" -export CXX="clang++-7" -export ARROW_BUILD_TYPE="RelWithDebInfo" -export ARROW_FLIGHT="OFF" -export ARROW_GANDIVA="OFF" -export ARROW_ORC="OFF" -export ARROW_PARQUET="OFF" -export ARROW_PLASMA="OFF" -export ARROW_WITH_BZ2="OFF" -export ARROW_WITH_ZSTD="OFF" -export ARROW_BUILD_BENCHMARKS="OFF" -export ARROW_BUILD_UTILITIES="OFF" -/arrow/ci/docker_build_cpp.sh || exit 1 -pushd /build/cpp - -mkdir ./relwithdebinfo/out -cp ./relwithdebinfo/arrow-ipc-fuzzing-test ./relwithdebinfo/out/fuzzer -ldd ./relwithdebinfo/arrow-ipc-fuzzing-test | grep "=> /" | awk '{print $3}' | xargs -I '{}' cp -v '{}' ./relwithdebinfo/out/. -cd ./relwithdebinfo/out/ -tar -czvf fuzzer.tar.gz * -stat fuzzer.tar.gz -cd ../../ - -export TARGET_ID=apache-arrow/arrow-ipc-fuzzing - -wget -O fuzzit https://github.com/fuzzitdev/fuzzit/releases/latest/download/fuzzit_Linux_x86_64 -chmod a+x fuzzit - -./fuzzit create job --type $FUZZIT_JOB_TYPE --host bionic-llvm7 --revision $CI_ARROW_SHA --branch $CI_ARROW_BRANCH $TARGET_ID ./relwithdebinfo/out/fuzzer.tar.gz diff --git a/ci/docker_build_cpp.sh b/ci/docker_build_cpp.sh deleted file mode 100755 index 8b3f4b8a6d2d4..0000000000000 --- a/ci/docker_build_cpp.sh +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -source_dir=${1:-/arrow/cpp} -build_dir=${2:-/build/cpp} -install_dir=${3:-${ARROW_HOME:-/usr/local}} - -export CCACHE_COMPRESS=1 -export CCACHE_COMPRESSLEVEL=5 -export CCACHE_COMPILERCHECK=content -export CCACHE_DIR=/build/ccache -export PATH=/usr/lib/ccache/:$PATH - -rm -rf ${build_dir} -mkdir -p ${build_dir} -pushd ${build_dir} - -cmake -GNinja \ - -DCMAKE_INSTALL_PREFIX=${install_dir} \ - -DCMAKE_INSTALL_LIBDIR=lib \ - -DCMAKE_BUILD_TYPE=${ARROW_BUILD_TYPE:-debug} \ - -DCMAKE_CXX_FLAGS=${CXXFLAGS:-} \ - -DARROW_DEPENDENCY_SOURCE=${ARROW_DEPENDENCY_SOURCE:-AUTO} \ - -DARROW_VERBOSE_THIRDPARTY_BUILD=${ARROW_VERBOSE_THIRDPARTY_BUILD:-OFF} \ - -DBUILD_WARNING_LEVEL=${DARROW_BUILD_WARNING_LEVEL:-CHECKIN} \ - -DARROW_BUILD_TESTS=${ARROW_BUILD_TESTS:-ON} \ - -DARROW_BUILD_BENCHMARKS=${ARROW_BUILD_BENCHMARKS:-ON} \ - -DARROW_BUILD_UTILITIES=${ARROW_BUILD_UTILITIES:-ON} \ - -DARROW_BUILD_EXAMPLES=${ARROW_BUILD_EXAMPLES:-ON} \ - -DARROW_BUILD_SHARED=${ARROW_BUILD_SHARED:-ON} \ - -DARROW_BUILD_STATIC=${ARROW_BUILD_STATIC:-ON} \ - -DARROW_NO_DEPRECATED_API=${ARROW_NO_DEPRECATED_API:-OFF} \ - -DARROW_USE_ASAN=${ARROW_USE_ASAN:-OFF} \ - -DARROW_USE_UBSAN=${ARROW_USE_UBSAN:-OFF} \ - -DARROW_FLIGHT=${ARROW_FLIGHT:-ON} \ - -DARROW_JEMALLOC=${ARROW_JEMALLOC:-ON} \ - -DARROW_MIMALLOC=${ARROW_MIMALLOC:-ON} \ - -DARROW_ORC=${ARROW_ORC:-ON} \ - -DARROW_PLASMA=${ARROW_PLASMA:-ON} \ - -DARROW_PARQUET=${ARROW_PARQUET:-ON} \ - -DARROW_HDFS=${ARROW_HDFS:-OFF} \ - -DARROW_PYTHON=${ARROW_PYTHON:-OFF} \ - -DARROW_GANDIVA=${ARROW_GANDIVA:-OFF} \ - -DARROW_GANDIVA_JAVA=${ARROW_GANDIVA_JAVA:-OFF} \ - -DARROW_WITH_ZLIB=${ARROW_WITH_ZLIB:-ON} \ - -DARROW_WITH_SNAPPY=${ARROW_WITH_SNAPPY:-ON} \ - -DARROW_WITH_LZ4=${ARROW_WITH_LZ4:-ON} \ - -DARROW_WITH_BROTLI=${ARROW_WITH_BROTLI:-ON} \ - -DARROW_WITH_BZ2=${ARROW_WITH_BZ2:-ON} \ - -DARROW_WITH_ZSTD=${ARROW_WITH_ZSTD:-ON} \ - -DARROW_INSTALL_NAME_RPATH=${ARROW_INSTALL_NAME_RPATH:-ON} \ - -DARROW_EXTRA_ERROR_CONTEXT=ON \ - -DARROW_TEST_LINKAGE=${ARROW_TEST_LINKAGE:-shared} \ - -DPARQUET_REQUIRE_ENCRYPTION=${ARROW_WITH_OPENSSL:-ON} \ - -DBOOST_SOURCE=${ARROW_BOOST_SOURCE:-AUTO} \ - -Duriparser_SOURCE=AUTO \ - -DARROW_FUZZING=${ARROW_FUZZING:-OFF} \ - -DARROW_USE_ASAN=${ARROW_USE_ASAN:-OFF} \ - ${CMAKE_ARGS} \ - ${source_dir} - -ninja -ninja install - -popd diff --git a/ci/docker_build_python.sh b/ci/docker_build_python.sh deleted file mode 100755 index 98be2b4013cfa..0000000000000 --- a/ci/docker_build_python.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e - -source_dir=${1:-/arrow/python} -build_dir=${2:-/build/python} - -export PYARROW_CXXFLAGS=$CXXFLAGS -export PYARROW_CMAKE_GENERATOR=Ninja -export PYARROW_BUILD_TYPE=${PYARROW_BUILD_TYPE:-debug} - -# Feature flags -export SETUPTOOLS_SCM_VERSION_WRITE_TO_PREFIX=$build_dir -export PYARROW_WITH_ORC=${PYARROW_WITH_ORC:-1} -export PYARROW_WITH_PARQUET=${PYARROW_WITH_PARQUET:-1} -export PYARROW_WITH_PLASMA=${PYARROW_WITH_PLASMA:-1} - -# Build pyarrow -pushd ${source_dir} - # hacky again, setuptools_scm writes _generated_version.py before pyarrow - # directory is created by setuptools - rm -rf ${build_dir} - mkdir -p $build_dir/pyarrow - - relative_build_dir=$(realpath --relative-to=. $build_dir) - - # this is a nightmare, but prevents mutating the source directory - # which is bind mounted as readonly - python setup.py build_ext --build-temp $relative_build_dir \ - --build-lib $relative_build_dir \ - build_py --build-lib $relative_build_dir \ - egg_info --egg-base $relative_build_dir \ - install_lib --build-dir $relative_build_dir \ - install --single-version-externally-managed \ - --record $relative_build_dir/record.txt -popd diff --git a/ci/docker_integration.sh b/ci/docker_integration.sh deleted file mode 100755 index 3851420a172a6..0000000000000 --- a/ci/docker_integration.sh +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -eu - -BIN_DIR=/bin -CPP_DIR=/build/cpp -JAVA_DIR=/build/java -JS_DIR=/build/js - -# Copy various stuff into our Docker build directory -pushd /arrow -rsync -r \ - LICENSE.txt NOTICE.txt \ - header java js format integration testing /build -popd - -mkdir -p $BIN_DIR -mkdir -p $CPP_DIR - -# Install archery tool, require writable -e install -cp -r /arrow/dev/archery /archery -pip install -e /archery - -# archery requires NumPy -pip install numpy - -# Archery integration test environment variables -export ARROW_ROOT=/build -export ARROW_TEST_DATA=/arrow/testing/data - -# Make a minimal C++ build for running integration tests -function build_cpp() { - pushd $CPP_DIR - - cmake /arrow/cpp \ - -DCMAKE_BUILD_TYPE=debug \ - -DBOOST_SOURCE=SYSTEM \ - -DARROW_BUILD_INTEGRATION=ON \ - -DARROW_FLIGHT=ON \ - -DARROW_COMPUTE=OFF \ - -DARROW_DATASET=OFF \ - -DARROW_FILESYSTEM=OFF \ - -DARROW_HDFS=OFF \ - -DARROW_JEMALLOC=OFF \ - -DARROW_JSON=OFF \ - -DARROW_USE_GLOG=OFF - - make -j4 arrow-integration - export ARROW_CPP_EXE_PATH=$CPP_DIR/debug -} - -function build_java() { - export MAVEN_OPTS="-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn" - - export JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64 - - # /arrow/java is read-only - pushd /arrow - popd - - pushd $JAVA_DIR - mvn package -B -DskipTests -Drat.skip=true - - popd -} - -# Build Go -function build_go() { - pushd /arrow/go/arrow - - export GO111MODULE=on - export GOBIN=$BIN_DIR - which go - go version - go env - go get -v ./... - - popd -} - -# Build JS -function build_js() { - source $NVM_DIR/nvm.sh - nvm install $NODE_VERSION - nvm alias default $NODE_VERSION - nvm use default - - pushd $JS_DIR - - npm install - npm run build -- -t apache-arrow - - popd -} - -# Run integration tests -GOLD_14_1=$ARROW_TEST_DATA/arrow-ipc/integration/0.14.1 - -build_cpp -build_go -build_js -build_java - -archery integration --with-all --run-flight diff --git a/ci/docker_java_test_all.sh b/ci/docker_java_test_all.sh deleted file mode 100755 index ca48894a6e166..0000000000000 --- a/ci/docker_java_test_all.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e - -SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - -export ARROW_TEST_DATA=/arrow/testing/data - -export ARROW_JAVA_RUN_TESTS=1 - -export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 -export ARROW_JAVADOC=1 -bash $SOURCE_DIR/docker_build_java.sh - -export ARROW_JAVA_SHADE_FLATBUFS=1 -export JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64 -export ARROW_JAVADOC=0 -bash $SOURCE_DIR/docker_build_java.sh diff --git a/integration/spark/ARROW-6429.patch b/ci/etc/ARROW-6429.patch similarity index 100% rename from integration/spark/ARROW-6429.patch rename to ci/etc/ARROW-6429.patch diff --git a/integration/hdfs/hdfs-site.xml b/ci/etc/hdfs-site.xml similarity index 100% rename from integration/hdfs/hdfs-site.xml rename to ci/etc/hdfs-site.xml diff --git a/ci/go-build-main.bat b/ci/go-build-main.bat deleted file mode 100644 index 9c298089b6e6e..0000000000000 --- a/ci/go-build-main.bat +++ /dev/null @@ -1,26 +0,0 @@ -@rem Licensed to the Apache Software Foundation (ASF) under one -@rem or more contributor license agreements. See the NOTICE file -@rem distributed with this work for additional information -@rem regarding copyright ownership. The ASF licenses this file -@rem to you under the Apache License, Version 2.0 (the -@rem "License"); you may not use this file except in compliance -@rem with the License. You may obtain a copy of the License at -@rem -@rem http://www.apache.org/licenses/LICENSE-2.0 -@rem -@rem Unless required by applicable law or agreed to in writing, -@rem software distributed under the License is distributed on an -@rem "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -@rem KIND, either express or implied. See the License for the -@rem specific language governing permissions and limitations -@rem under the License. - -@rem The "main" Go build script for Windows CI - -pushd go\arrow - -%GOROOT%\bin\go version -%GOROOT%\bin\go get -v ./... || exit /B -%GOROOT%\bin\go test ./... || exit /B - -popd diff --git a/ci/rust-build-main.bat b/ci/rust-build-main.bat deleted file mode 100644 index 015ba987b5da1..0000000000000 --- a/ci/rust-build-main.bat +++ /dev/null @@ -1,60 +0,0 @@ -@rem Licensed to the Apache Software Foundation (ASF) under one -@rem or more contributor license agreements. See the NOTICE file -@rem distributed with this work for additional information -@rem regarding copyright ownership. The ASF licenses this file -@rem to you under the Apache License, Version 2.0 (the -@rem "License"); you may not use this file except in compliance -@rem with the License. You may obtain a copy of the License at -@rem -@rem http://www.apache.org/licenses/LICENSE-2.0 -@rem -@rem Unless required by applicable law or agreed to in writing, -@rem software distributed under the License is distributed on an -@rem "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -@rem KIND, either express or implied. See the License for the -@rem specific language governing permissions and limitations -@rem under the License. - -@rem The "main" Rust build script for Windows CI - -@rem Retrieve git submodules, configure env var for Parquet unit tests -git submodule update --init || exit /B -set ARROW_TEST_DATA=%CD%\testing\data -set PARQUET_TEST_DATA=%CD%\cpp\submodules\parquet-testing\data -pushd rust - -@echo =================================== -@echo Build with nightly toolchain -@echo =================================== - -rustup default nightly -rustup show -cargo build --target %TARGET% --all-targets --release || exit /B -@echo -@echo Build with no default features -@echo ------------------------------ -pushd arrow -cargo build --target %TARGET% --all-targets --no-default-features || exit /B -popd -@echo -@echo Test (release) -@echo -------------- -cargo test --target %TARGET% --release || exit /B - -@echo -@echo Run Arrow examples (release) -@echo --------------------- -pushd arrow -cargo run --example builders --target %TARGET% --release || exit /B -cargo run --example dynamic_types --target %TARGET% --release || exit /B -cargo run --example read_csv --target %TARGET% --release || exit /B -cargo run --example read_csv_infer_schema --target %TARGET% --release || exit /B -popd - -@echo -@echo Run DataFusion examples (release) -@echo --------------------- -pushd datafusion -cargo run --example csv_sql --target %TARGET% --release || exit /B -cargo run --example parquet_sql --target %TARGET% --release || exit /B -popd diff --git a/ci/docker_build_c_glib.sh b/ci/scripts/c_glib_build.sh similarity index 78% rename from ci/docker_build_c_glib.sh rename to ci/scripts/c_glib_build.sh index 0135781f6ccb2..99cf887664243 100755 --- a/ci/docker_build_c_glib.sh +++ b/ci/scripts/c_glib_build.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash - +# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -17,22 +17,25 @@ # specific language governing permissions and limitations # under the License. -set -e +set -ex -export ARROW_C_GLIB_HOME=$CONDA_PREFIX +source_dir=${1}/c_glib +build_dir=${2:-${source_dir}/build} +with_docs=${3:-false} export CFLAGS="-DARROW_NO_DEPRECATED_API" export CXXFLAGS="-DARROW_NO_DEPRECATED_API" -mkdir -p /build/c_glib +mkdir -p ${build_dir} # Build with Meson -meson --prefix=$ARROW_C_GLIB_HOME \ +meson --prefix=$ARROW_HOME \ --libdir=lib \ - /build/c_glib \ - /arrow/c_glib + -Dgtk_doc=${with_docs} \ + ${build_dir} \ + ${source_dir} -pushd /build/c_glib - ninja - ninja install +pushd ${build_dir} +ninja +ninja install popd diff --git a/ci/travis_script_cpp.sh b/ci/scripts/c_glib_test.sh similarity index 58% rename from ci/travis_script_cpp.sh rename to ci/scripts/c_glib_test.sh index 718cda882884f..0c7deb6b4035c 100755 --- a/ci/travis_script_cpp.sh +++ b/ci/scripts/c_glib_test.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash - +# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -17,25 +17,29 @@ # specific language governing permissions and limitations # under the License. -set -e +set -ex -source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh +source_dir=${1}/c_glib +build_dir=${2:-${source_dir}/build} -pushd $CPP_BUILD_DIR +export LD_LIBRARY_PATH=${ARROW_HOME}/lib:${LD_LIBRARY_PATH} +export PKG_CONFIG_PATH=${ARROW_HOME}/lib/pkgconfig +export GI_TYPELIB_PATH=${ARROW_HOME}/lib/girepository-1.0 +export PKG_CONFIG=/usr/bin/pkg-config -if [ $TRAVIS_OS_NAME == "osx" ]; then - # TODO: This does not seem to terminate - CTEST_ARGS="-E arrow-flight-test" -fi +pushd ${source_dir} -PATH=$ARROW_BUILD_TYPE:$PATH ctest -j2 --output-on-failure -L unittest ${CTEST_ARGS} +ruby test/run-test.rb -popd - -# Capture C++ coverage info (we wipe the build dir in travis_script_python.sh) -if [ "$ARROW_TRAVIS_COVERAGE" == "1" ]; then - pushd $TRAVIS_BUILD_DIR - lcov --directory . --capture --no-external --output-file $ARROW_CPP_COVERAGE_FILE \ - 2>&1 | grep -v "ignoring data for external file" +if [[ "$(uname -s)" == "Linux" ]]; then + # TODO(kszucs): on osx it fails to load 'lgi.corelgilua51' despite that lgi + # was installed by luarocks + pushd example/lua + lua write-batch.lua + lua read-batch.lua + lua write-stream.lua + lua read-stream.lua popd fi + +popd diff --git a/ci/scripts/cpp_build.sh b/ci/scripts/cpp_build.sh new file mode 100755 index 0000000000000..6e5e7410b8fae --- /dev/null +++ b/ci/scripts/cpp_build.sh @@ -0,0 +1,125 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -ex + +source_dir=${1}/cpp +build_dir=${2:-${source_dir}/build} +with_docs=${3:-false} + +# TODO(kszucs): consider to move these to CMake +if [ ! -z "${CONDA_PREFIX}" ]; then + export CMAKE_ARGS="${CMAKE_ARGS} -DCMAKE_AR=${AR} -DCMAKE_RANLIB=${RANLIB}" + export ARROW_GANDIVA_PC_CXX_FLAGS=$(echo | ${CXX} -E -Wp,-v -xc++ - 2>&1 | grep '^ ' | awk '{print "-isystem;" substr($1, 1)}' | tr '\n' ';') +elif [ -x "$(command -v xcrun)" ]; then + export ARROW_GANDIVA_PC_CXX_FLAGS="-isysroot;$(xcrun --show-sdk-path)" +fi + +# export CCACHE_COMPRESS=1 +# export CCACHE_COMPRESSLEVEL=5 +# export CCACHE_COMPILERCHECK=content +# export CCACHE_DIR=/build/ccache +# export PATH=/usr/lib/ccache/:$PATH + +mkdir -p ${build_dir} +pushd ${build_dir} + +cmake -G "${CMAKE_GENERATOR:-Ninja}" \ + -DARROW_BOOST_USE_SHARED=${ARROW_BOOST_USE_SHARED:-ON} \ + -DARROW_BUILD_BENCHMARKS=${ARROW_BUILD_BENCHMARKS:-OFF} \ + -DARROW_BUILD_EXAMPLES=${ARROW_BUILD_EXAMPLES:-OFF} \ + -DARROW_BUILD_INTEGRATION=${ARROW_BUILD_INTEGRATION:-OFF} \ + -DARROW_BUILD_SHARED=${ARROW_BUILD_SHARED:-ON} \ + -DARROW_BUILD_STATIC=${ARROW_BUILD_STATIC:-ON} \ + -DARROW_BUILD_TESTS=${ARROW_BUILD_TESTS:-OFF} \ + -DARROW_BUILD_UTILITIES=${ARROW_BUILD_UTILITIES:-ON} \ + -DARROW_COMPUTE=${ARROW_COMPUTE:-ON} \ + -DARROW_CUDA=${ARROW_CUDA:-OFF} \ + -DARROW_CXXFLAGS=${ARROW_CXXFLAGS:-} \ + -DARROW_DATASET=${ARROW_DATASET:-ON} \ + -DARROW_DEPENDENCY_SOURCE=${ARROW_DEPENDENCY_SOURCE:-AUTO} \ + -DARROW_EXTRA_ERROR_CONTEXT=${ARROW_EXTRA_ERROR_CONTEXT:-OFF} \ + -DARROW_FILESYSTEM=${ARROW_FILESYSTEM:-ON} \ + -DARROW_FLIGHT=${ARROW_FLIGHT:-OFF} \ + -DARROW_FUZZING=${ARROW_FUZZING:-OFF} \ + -DARROW_FUZZING=${ARROW_FUZZING:-OFF} \ + -DARROW_JNI=${ARROW_JNI:-OFF} \ + -DARROW_GANDIVA_JAVA=${ARROW_GANDIVA_JAVA:-OFF} \ + -DARROW_GANDIVA_PC_CXX_FLAGS=${ARROW_GANDIVA_PC_CXX_FLAGS:-} \ + -DARROW_GANDIVA=${ARROW_GANDIVA:-OFF} \ + -DARROW_HDFS=${ARROW_HDFS:-ON} \ + -DARROW_HIVESERVER2=${ARROW_HIVESERVER2:-OFF} \ + -DARROW_INSTALL_NAME_RPATH=${ARROW_INSTALL_NAME_RPATH:-ON} \ + -DARROW_JEMALLOC=${ARROW_JEMALLOC:-ON} \ + -DARROW_LARGE_MEMORY_TESTS=${ARROW_LARGE_MEMORY_TESTS:-OFF} \ + -DARROW_MIMALLOC=${ARROW_MIMALLOC:-OFF} \ + -DARROW_NO_DEPRECATED_API=${ARROW_NO_DEPRECATED_API:-OFF} \ + -DARROW_ORC=${ARROW_ORC:-OFF} \ + -DARROW_PARQUET=${ARROW_PARQUET:-OFF} \ + -DARROW_PLASMA=${ARROW_PLASMA:-OFF} \ + -DARROW_PYTHON=${ARROW_PYTHON:-OFF} \ + -DARROW_S3=${ARROW_S3:-OFF} \ + -DARROW_TEST_LINKAGE=${ARROW_TEST_LINKAGE:-shared} \ + -DARROW_USE_ASAN=${ARROW_USE_ASAN:-OFF} \ + -DARROW_USE_ASAN=${ARROW_USE_ASAN:-OFF} \ + -DARROW_USE_GLOG=${ARROW_USE_GLOG:-ON} \ + -DARROW_USE_STATIC_CRT=${ARROW_USE_STATIC_CRT:-OFF} \ + -DARROW_USE_UBSAN=${ARROW_USE_UBSAN:-OFF} \ + -DARROW_VERBOSE_THIRDPARTY_BUILD=${ARROW_VERBOSE_THIRDPARTY_BUILD:-OFF} \ + -DARROW_WITH_BROTLI=${ARROW_WITH_BROTLI:-OFF} \ + -DARROW_WITH_BZ2=${ARROW_WITH_BZ2:-OFF} \ + -DARROW_WITH_LZ4=${ARROW_WITH_LZ4:-OFF} \ + -DARROW_WITH_SNAPPY=${ARROW_WITH_SNAPPY:-OFF} \ + -DARROW_WITH_ZLIB=${ARROW_WITH_ZLIB:-OFF} \ + -DARROW_WITH_ZSTD=${ARROW_WITH_ZSTD:-OFF} \ + -Dbenchmark_SOURCE=${benchmark_SOURCE:-AUTO} \ + -DBOOST_SOURCE=${ARROW_BOOST_SOURCE:-AUTO} \ + -DBOOST_SOURCE=${BOOST_SOURCE:-AUTO} \ + -DBrotli_SOURCE=${Brotli_SOURCE:-AUTO} \ + -DBUILD_WARNING_LEVEL=${DARROW_BUILD_WARNING_LEVEL:-CHECKIN} \ + -Dc-ares_SOURCE=${cares_SOURCE:-AUTO} \ + -DCMAKE_BUILD_TYPE=${ARROW_BUILD_TYPE:-debug} \ + -DCMAKE_CXX_FLAGS=${CXXFLAGS:-} \ + -DCMAKE_INSTALL_LIBDIR=${CMAKE_INSTALL_LIBDIR:-lib} \ + -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX:-${ARROW_HOME}} \ + -Ddouble-conversion_SOURCE=${uriparser_SOURCE:-AUTO} \ + -Dgflags_SOURCE=${gflags_SOURCE:-AUTO} \ + -DgRPC_SOURCE=${gRPC_SOURCE:-AUTO} \ + -DGTest_SOURCE=${GTest_SOURCE:-AUTO} \ + -DLz4_SOURCE=${Lz4_SOURCE:-AUTO} \ + -DORC_SOURCE=${ORC_SOURCE:-AUTO} \ + -DPARQUET_REQUIRE_ENCRYPTION=${ARROW_WITH_OPENSSL:-ON} \ + -DProtobuf_SOURCE=${Protobuf_SOURCE:-AUTO} \ + -DRapidJSON_SOURCE=${RapidJSON_SOURCE:-AUTO} \ + -DRE2_SOURCE=${RE2_SOURCE:-AUTO} \ + -DThrift_SOURCE=${Thrift_SOURCE:-AUTO} \ + -Duriparser_SOURCE=${uriparser_SOURCE:-AUTO} \ + -DZSTD_SOURCE=${ZSTD_SOURCE:-AUTO} \ + ${CMAKE_ARGS} \ + ${source_dir} + +cmake --build . --target install + +popd + +if [ "${with_docs}" == "true" ]; then + pushd ${source_dir}/apidoc + doxygen + popd +fi diff --git a/ci/travis_script_java.sh b/ci/scripts/cpp_test.sh similarity index 70% rename from ci/travis_script_java.sh rename to ci/scripts/cpp_test.sh index 201c336268792..88047f9fa6262 100755 --- a/ci/travis_script_java.sh +++ b/ci/scripts/cpp_test.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash - +# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -17,18 +17,18 @@ # specific language governing permissions and limitations # under the License. -set -e +set -ex + +arrow_dir=${1} +source_dir=${1}/cpp +build_dir=${2:-${source_dir}/build} -source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh +export ARROW_TEST_DATA=${arrow_dir}/testing/data +export PARQUET_TEST_DATA=${source_dir}/submodules/parquet-testing/data +export LD_LIBRARY_PATH=${ARROW_HOME}/${CMAKE_INSTALL_LIBDIR:-lib}:${LD_LIBRARY_PATH} -JAVA_DIR=${TRAVIS_BUILD_DIR}/java -pushd $JAVA_DIR +pushd ${build_dir} -if [ "$ARROW_TRAVIS_JAVA_BUILD_ONLY" == "1" ]; then - # Save time and make build less verbose by skipping tests and style checks - $TRAVIS_MVN -DskipTests=true -Dcheckstyle.skip=true -B install -else - $TRAVIS_MVN -B install -fi +ctest --output-on-failure -j$(nproc) popd diff --git a/ci/docker_build_and_test_python.sh b/ci/scripts/csharp_build.sh similarity index 89% rename from ci/docker_build_and_test_python.sh rename to ci/scripts/csharp_build.sh index ce555b7d24ec3..5a39767948741 100755 --- a/ci/docker_build_and_test_python.sh +++ b/ci/scripts/csharp_build.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -16,8 +17,10 @@ # specific language governing permissions and limitations # under the License. -set -e +set -ex + +source_dir=${1}/csharp -/arrow/ci/docker_build_cpp.sh -/arrow/ci/docker_build_python.sh -pytest -v --pyargs pyarrow +pushd ${source_dir} +dotnet build +popd diff --git a/dev/Dockerfile.context b/ci/scripts/csharp_test.sh old mode 100644 new mode 100755 similarity index 89% rename from dev/Dockerfile.context rename to ci/scripts/csharp_test.sh index f0e7cdd6b576a..37702ceaf6d13 --- a/dev/Dockerfile.context +++ b/ci/scripts/csharp_test.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash +# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -15,7 +17,10 @@ # specific language governing permissions and limitations # under the License. -FROM ubuntu:latest +set -ex + +source_dir=${1}/csharp -COPY . /context -CMD du -sh /context/* +pushd ${source_dir} +dotnet test +popd diff --git a/ci/docker_build_sphinx.sh b/ci/scripts/docs_build.sh similarity index 54% rename from ci/docker_build_sphinx.sh rename to ci/scripts/docs_build.sh index 4f36910a31220..fc86952d18673 100755 --- a/ci/docker_build_sphinx.sh +++ b/ci/scripts/docs_build.sh @@ -18,12 +18,26 @@ set -ex -pushd /arrow/cpp/apidoc -doxygen -popd +arrow_dir=${1} +build_dir=${2} -sphinx-build -b html /arrow/docs/source /arrow/docs/_build/html +export LD_LIBRARY_PATH=${ARROW_HOME}/lib:${LD_LIBRARY_PATH} +export PKG_CONFIG_PATH=${ARROW_HOME}/lib/pkgconfig:${PKG_CONFIG_PATH} +export GI_TYPELIB_PATH=${ARROW_HOME}/lib/girepository-1.0 +export CFLAGS="-DARROW_NO_DEPRECATED_API" +export CXXFLAGS="-DARROW_NO_DEPRECATED_API" -git clone -b asf-site https://github.com/apache/arrow-site /arrow/site -mkdir -p /arrow/site/docs/latest -rsync -r /arrow/docs/_build/html/ /arrow/site/docs/latest/ +# Prose and Python +sphinx-build -b html ${arrow_dir}/docs/source ${build_dir}/sphinx + +# C++ +rsync -a --delete ${arrow_dir}/cpp/apidoc/* ${build_dir}/cpp + +# C GLib +rsync -a --delete ${ARROW_HOME}/share/gtk-doc/html/* ${build_dir}/c_glib + +# Java +rsync -a --delete ${arrow_dir}/java/target/site/apidocs ${build_dir}/java + +# Javascript +rsync -a --delete ${arrow_dir}/js/doc ${build_dir}/js diff --git a/ci/travis_script_plasma_java_client.sh b/ci/scripts/fuzzit_build.sh similarity index 60% rename from ci/travis_script_plasma_java_client.sh rename to ci/scripts/fuzzit_build.sh index 61027b58d3d4a..02de306a9c76c 100755 --- a/ci/travis_script_plasma_java_client.sh +++ b/ci/scripts/fuzzit_build.sh @@ -1,5 +1,4 @@ #!/usr/bin/env bash - # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -17,25 +16,31 @@ # specific language governing permissions and limitations # under the License. -set -e - -source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh +set -euxo pipefail -source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh +export TARGET_ID=apache-arrow/arrow-ipc-fuzzing -conda activate $CPP_TOOLCHAIN +source_dir=${1}/cpp +build_dir=${2:-${source_dir}/build} -PLASMA_JAVA_DIR=${TRAVIS_BUILD_DIR}/java/plasma +pushd ${build_dir}/relwithdebinfo -pushd $PLASMA_JAVA_DIR +mkdir -p out -$TRAVIS_MVN clean install +cp arrow-ipc-fuzzing-test out/fuzzer +ldd arrow-ipc-fuzzing-test | grep "=> /" | awk '{print $3}' | xargs -I '{}' cp -v '{}' out/ -export LD_LIBRARY_PATH=${ARROW_CPP_INSTALL}/lib:$LD_LIBRARY_PATH -export PLASMA_STORE=${ARROW_CPP_INSTALL}/bin/plasma_store_server - -ldd $PLASMA_STORE +pushd out +tar -czvf fuzzer.tar.gz * +stat fuzzer.tar.gz +popd -$TRAVIS_JAVA -cp target/test-classes:target/classes -Djava.library.path=${TRAVIS_BUILD_DIR}/cpp-build/debug/ org.apache.arrow.plasma.PlasmaClientTest +fuzzit create job \ + --type $FUZZIT_JOB_TYPE \ + --host $FUZZIT_HOST \ + --revision $CI_ARROW_SHA \ + --branch $CI_ARROW_BRANCH \ + $TARGET_ID \ + out/fuzzer.tar.gz popd diff --git a/integration/pandas/Dockerfile b/ci/scripts/go_build.sh old mode 100644 new mode 100755 similarity index 85% rename from integration/pandas/Dockerfile rename to ci/scripts/go_build.sh index a1ae23e63321f..bb53bd8213190 --- a/integration/pandas/Dockerfile +++ b/ci/scripts/go_build.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash +# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -15,7 +17,13 @@ # specific language governing permissions and limitations # under the License. -FROM arrowdev/arrow-python-3.6:latest +set -ex + +source_dir=${1}/go + +pushd ${source_dir}/arrow + +go get -d -t -v ./... +go install -v ./... -# test against pandas' master -RUN pip install git+https://github.com/pandas-dev/pandas.git +popd diff --git a/ci/scripts/go_test.sh b/ci/scripts/go_test.sh new file mode 100755 index 0000000000000..077749fc94560 --- /dev/null +++ b/ci/scripts/go_test.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -ex + +source_dir=${1}/go + +pushd ${source_dir}/arrow + +for d in $(go list ./... | grep -v vendor); do + go test $d +done + +popd diff --git a/ci/scripts/install_conda.sh b/ci/scripts/install_conda.sh new file mode 100755 index 0000000000000..04d61736a18ee --- /dev/null +++ b/ci/scripts/install_conda.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e + +declare -A archs +archs=([amd64]=x86_64 + [arm32v7]=armv7l + [ppc64le]=ppc64le + [i386]=x86) + +declare -A platforms +platforms=([windows]=Windows + [macos]=MacOSX + [linux]=Linux) + +if [ "$#" -ne 4 ]; then + echo "Usage: $0 " + exit 1 +elif [[ -z ${archs[$1]} ]]; then + echo "Unexpected architecture: ${1}" + exit 1 +elif [[ -z ${platforms[$2]} ]]; then + echo "Unexpected platform: ${2}" + exit 1 +fi + +arch=${archs[$1]} +platform=${platforms[$2]} +version=$3 +prefix=$4 + +echo "Downloading Miniconda installer..." +wget -nv https://repo.continuum.io/miniconda/Miniconda3-${version}-${platform}-${arch}.sh -O /tmp/miniconda.sh +bash /tmp/miniconda.sh -b -p ${prefix} +rm /tmp/miniconda.sh + +ln -s ${prefix}/etc/profile.d/conda.sh /etc/profile.d/conda.sh +echo "conda activate base" >> ~/.profile + +# Configure +source /etc/profile.d/conda.sh +conda config --add channels conda-forge +conda config --set channel_priority strict +conda config --set show_channel_urls True +conda config --set remote_connect_timeout_secs 12 + +# Update and clean +conda update --all -y +conda clean --all -y diff --git a/ci/docker_install_iwyu.sh b/ci/scripts/install_iwyu.sh similarity index 63% rename from ci/docker_install_iwyu.sh rename to ci/scripts/install_iwyu.sh index 5206f3b47d998..7937456d8c8c0 100755 --- a/ci/docker_install_iwyu.sh +++ b/ci/scripts/install_iwyu.sh @@ -18,22 +18,26 @@ set -eu -: ${IWYU_REPO:="https://github.com/include-what-you-use/include-what-you-use.git"} -: ${IWYU_BRANCH:="clang_7.0"} -: ${IWYU_SRC:="/tmp/iwyu"} -: ${IWYU_HOME:="/opt/iwyu"} +source_dir=${1:-/tmp/iwyu} +install_prefix=${2:-/usr/local} +llvm_major=${3:-7} -git clone "${IWYU_REPO}" "${IWYU_SRC}" -git -C "${IWYU_SRC}" checkout ${IWYU_BRANCH} +git clone --single-branch --branch "clang_${llvm_major}.0" \ + https://github.com/include-what-you-use/include-what-you-use.git ${source_dir} -mkdir -p "${IWYU_HOME}" -pushd "${IWYU_HOME}" +mkdir -p ${source_dir}/build +pushd ${source_dir}/build # Build IWYU for current Clang -export CC=clang-7 -export CXX=clang++-7 +export CC=clang-${llvm_major} +export CXX=clang++-${llvm_major} -cmake -DCMAKE_PREFIX_PATH=/usr/lib/llvm-7 "${IWYU_SRC}" +cmake -DCMAKE_PREFIX_PATH=/usr/lib/llvm-${llvm_major} \ + -DCMAKE_INSTALL_PREFIX=${install_prefix} \ + ${source_dir} make -j4 +make install popd + +rm -rf ${source_dir} diff --git a/ci/scripts/install_minio.sh b/ci/scripts/install_minio.sh new file mode 100755 index 0000000000000..9ed70afc03b8e --- /dev/null +++ b/ci/scripts/install_minio.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e + +declare -A archs +archs=([amd64]=amd64) + +declare -A platforms +platforms=([macos]=darwin + [linux]=linux) + +arch=${archs[$1]} +platform=${platforms[$2]} +version=$3 +prefix=$4 + +if [ "$#" -ne 4 ]; then + echo "Usage: $0 " + exit 1 +elif [[ -z ${archs[$1]} ]]; then + echo "Unexpected architecture: ${1}" + exit 1 +elif [[ -z ${platforms[$2]} ]]; then + echo "Unexpected platform: ${2}" + exit 1 +elif [[ ${version} != "latest" ]]; then + echo "Cannot fetch specific versions of minio, only latest is supported." + exit 1 +fi + +wget -nv -P ${prefix}/bin https://dl.min.io/server/minio/release/linux-${arch}/minio +chmod +x ${prefix}/bin/minio diff --git a/integration/dask/runtest.sh b/ci/scripts/integration_dask.sh similarity index 100% rename from integration/dask/runtest.sh rename to ci/scripts/integration_dask.sh diff --git a/integration/hdfs/runtest.sh b/ci/scripts/integration_hdfs.sh similarity index 90% rename from integration/hdfs/runtest.sh rename to ci/scripts/integration_hdfs.sh index 44afb4a1888a4..fa63d4742aab6 100755 --- a/integration/hdfs/runtest.sh +++ b/ci/scripts/integration_hdfs.sh @@ -17,21 +17,23 @@ # specific language governing permissions and limitations # under the License. -# exit on any error set -e +source_dir=${1}/cpp +build_dir=${2:-${source_dir}/build} + export CLASSPATH=`$HADOOP_HOME/bin/hadoop classpath --glob` export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop export LIBHDFS3_CONF=$HADOOP_CONF_DIR/hdfs-site.xml export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$HADOOP_HOME/lib/native/ # execute cpp tests -pushd /build/cpp - debug/arrow-io-hdfs-test +pushd ${build_dir} +debug/arrow-io-hdfs-test popd # cannot use --pyargs with custom arguments like --hdfs or --only-hdfs, because # pytest ignores them, see https://github.com/pytest-dev/pytest/issues/3517 -export PYARROW_TEST_ONLY_HDFS=ON +export PYARROW_TEST_HDFS=ON pytest -v --pyargs pyarrow.tests.test_hdfs diff --git a/dev/run_site.sh b/ci/scripts/integration_hiveserver2.sh similarity index 76% rename from dev/run_site.sh rename to ci/scripts/integration_hiveserver2.sh index 1e64dada3167e..647184c70aa40 100755 --- a/dev/run_site.sh +++ b/ci/scripts/integration_hiveserver2.sh @@ -16,6 +16,19 @@ # limitations under the License. # -# Pass the service name to run_docker_compse.sh -# Which validates environment and runs the service -exec "$(dirname ${BASH_SOURCE})"/run_docker_compose.sh run_site +set -e + +arrow_dir=${1} +source_dir=${1}/cpp +build_dir=${2:-${source_dir}/build} + +${arrow_dir}/ci/scripts/wait_for_it.sh impala:21050 -t 300 -s -- echo "impala is up" + +pushd ${build_dir} + +# ninja hiveserver2-test +debug/hiveserver2-test + +popd + + diff --git a/integration/spark/runtest.sh b/ci/scripts/integration_spark.sh similarity index 76% rename from integration/spark/runtest.sh rename to ci/scripts/integration_spark.sh index f8d2cbe9cb7bb..0b03ee2c5c293 100755 --- a/integration/spark/runtest.sh +++ b/ci/scripts/integration_spark.sh @@ -16,38 +16,38 @@ # limitations under the License. # exit on any error -set -e +set -eu -# rsynced source directory to build java libs -arrow_src=/build/java/arrow +source_dir=${1} +spark_dir=${2} -pushd $arrow_src/java - ARROW_VERSION=`mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version | sed -n -e '/^\[.*\]/ !{ /^[0-9]/ { p; q } }'` +pushd ${source_dir}/java + arrow_version=`mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version | sed -n -e '/^\[.*\]/ !{ /^[0-9]/ { p; q } }'` popd export MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=512m -Dorg.slf4j.simpleLogger.defaultLogLevel=warn" export MAVEN_OPTS="${MAVEN_OPTS} -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn" # build Spark with Arrow -pushd /spark +pushd ${spark_dir} # update Spark pom with the Arrow version just installed and build Spark, need package phase for pyspark - echo "Building Spark with Arrow $ARROW_VERSION" - mvn versions:set-property -Dproperty=arrow.version -DnewVersion=$ARROW_VERSION + echo "Building Spark with Arrow ${arrow_version}" + mvn versions:set-property -Dproperty=arrow.version -DnewVersion=${arrow_version} build/mvn -B -DskipTests package -pl sql/core -pl assembly -am - SPARK_SCALA_TESTS=( + spark_scala_tests=( "org.apache.spark.sql.execution.arrow" "org.apache.spark.sql.execution.vectorized.ColumnarBatchSuite" "org.apache.spark.sql.execution.vectorized.ArrowColumnVectorSuite") - (echo "Testing Spark:"; IFS=$'\n'; echo "${SPARK_SCALA_TESTS[*]}") + (echo "Testing Spark:"; IFS=$'\n'; echo "${spark_scala_tests[*]}") # TODO: should be able to only build spark-sql tests with adding "-pl sql/core" but not currently working - build/mvn -B -Dtest=none -DwildcardSuites=$(IFS=,; echo "${SPARK_SCALA_TESTS[*]}") test + build/mvn -B -Dtest=none -DwildcardSuites=$(IFS=,; echo "${spark_scala_tests[*]}") test # Run pyarrow related Python tests only - SPARK_PYTHON_TESTS=( + spark_python_tests=( "pyspark.sql.tests.test_arrow" "pyspark.sql.tests.test_pandas_udf" "pyspark.sql.tests.test_pandas_udf_scalar" @@ -55,6 +55,6 @@ pushd /spark "pyspark.sql.tests.test_pandas_udf_grouped_map" "pyspark.sql.tests.test_pandas_udf_window") - (echo "Testing PySpark:"; IFS=$'\n'; echo "${SPARK_PYTHON_TESTS[*]}") - python/run-tests --testnames "$(IFS=,; echo "${SPARK_PYTHON_TESTS[*]}")" --python-executables python + (echo "Testing PySpark:"; IFS=$'\n'; echo "${spark_python_tests[*]}") + python/run-tests --testnames "$(IFS=,; echo "${spark_python_tests[*]}")" --python-executables python popd diff --git a/integration/turbodbc/runtest.sh b/ci/scripts/integration_turbodbc.sh similarity index 69% rename from integration/turbodbc/runtest.sh rename to ci/scripts/integration_turbodbc.sh index 874cba0543446..8e51ceb22b35e 100755 --- a/integration/turbodbc/runtest.sh +++ b/ci/scripts/integration_turbodbc.sh @@ -19,27 +19,28 @@ set -e +source_dir=${1} +build_dir=${2:-${source_dir}/build} + # check that optional pyarrow modules are available # because pytest would just skip the pyarrow tests python -c "import pyarrow.orc" python -c "import pyarrow.parquet" -pushd /tmp -git clone https://github.com/blue-yonder/turbodbc.git -pushd turbodbc -git submodule update --init --recursive - -service postgresql start -sudo -u postgres psql -U postgres -c 'CREATE DATABASE test_db;' -sudo -u postgres psql -U postgres -c 'ALTER USER postgres WITH PASSWORD '\''password'\'';' -export ODBCSYSINI="$(pwd)/travis/odbc/" +mkdir -p ${build_dir} +pushd ${build_dir} -mkdir build -pushd build -cmake -DCMAKE_INSTALL_PREFIX=./dist -DPYTHON_EXECUTABLE=`which python` -GNinja .. +cmake -DCMAKE_INSTALL_PREFIX=${ARROW_HOME} \ + -DCMAKE_CXX_FLAGS=${CXXFLAGS} \ + -GNinja \ + ${source_dir} ninja install # TODO(ARROW-5074) -export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/opt/conda/lib" +export LD_LIBRARY_PATH="${ARROW_HOME}/lib:${LD_LIBRARY_PATH}" +export ODBCSYSINI="${source_dir}/travis/odbc/" +service postgresql start ctest --output-on-failure + +popd diff --git a/ci/scripts/java_build.sh b/ci/scripts/java_build.sh new file mode 100755 index 0000000000000..148426c052156 --- /dev/null +++ b/ci/scripts/java_build.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -ex + +arrow_dir=${1} +source_dir=${1}/java +cpp_build_dir=${2:-${arrow_dir}/cpp/build/debug} +with_docs=${3:-false} + +mvn="mvn -B -DskipTests -Drat.skip=true -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn" + +pushd ${source_dir} + +${mvn} install + +if [ "${ARROW_JAVA_SHADE_FLATBUFFERS}" == "ON" ]; then + ${mvn} -Pshade-flatbuffers install +fi + +if [ "${ARROW_GANDIVA_JAVA}" = "ON" ]; then + ${mvn} -Darrow.cpp.build.dir=${cpp_build_dir} -Parrow-jni install +fi + +if [ "${ARROW_PLASMA}" = "ON" ]; then + pushd ${source_dir}/plasma + ${mvn} clean install + popd +fi + +if [ "${with_docs}" == "true" ]; then + ${mvn} -Dcheckstyle.skip=true install site +fi + +popd diff --git a/ci/docker_build_java.sh b/ci/scripts/java_test.sh similarity index 53% rename from ci/docker_build_java.sh rename to ci/scripts/java_test.sh index a4df526336ee4..ce6f1282079b7 100755 --- a/ci/docker_build_java.sh +++ b/ci/scripts/java_test.sh @@ -16,36 +16,32 @@ # specific language governing permissions and limitations # under the License. -set -e +set -ex -export MAVEN_OPTS="-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn" +arrow_dir=${1} +source_dir=${1}/java +cpp_build_dir=${2:-${arrow_dir}/cpp/build/debug} -# /arrow/java is read-only -mkdir -p /build/java +# for jni and plasma tests +export LD_LIBRARY_PATH=${ARROW_HOME}/lib:${LD_LIBRARY_PATH} +export PLASMA_STORE=${ARROW_HOME}/bin/plasma_store_server -arrow_src=/build/java/arrow +mvn="mvn -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn" -# Remove any pre-existing artifacts -rm -rf $arrow_src +pushd ${source_dir} -pushd /arrow -rsync -a header java format integration testing $arrow_src -popd +${mvn} test -JAVA_ARGS= -if [ "$ARROW_JAVA_RUN_TESTS" != "1" ]; then - JAVA_ARGS=-DskipTests +if [ "${ARROW_GANDIVA_JAVA}" = "ON" ]; then + ${mvn} test -Parrow-jni -pl gandiva -Darrow.cpp.build.dir=${cpp_build_dir} fi -if [ "$ARROW_JAVA_SHADE_FLATBUFS" == "1" ]; then - SHADE_FLATBUFFERS=-Pshade-flatbuffers +if [ "${ARROW_PLASMA}" = "ON" ]; then + pushd ${source_dir}/plasma + java -cp target/test-classes:target/classes \ + -Djava.library.path=${cpp_build_dir} \ + org.apache.arrow.plasma.PlasmaClientTest + popd fi -pushd $arrow_src/java -mvn -B $JAVA_ARGS -Drat.skip=true install $SHADE_FLATBUFFERS - -if [ "$ARROW_JAVADOC" == "1" ]; then - export MAVEN_OPTS="$MAVEN_OPTS -Dorg.slf4j.simpleLogger.defaultLogLevel=warn" - mvn -B site -fi popd diff --git a/ci/travis_script_js.sh b/ci/scripts/js_build.sh similarity index 80% rename from ci/travis_script_js.sh rename to ci/scripts/js_build.sh index 34b07115e70b1..b2add45b024ba 100755 --- a/ci/travis_script_js.sh +++ b/ci/scripts/js_build.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash - +# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -19,14 +19,19 @@ set -ex -source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh +source_dir=${1}/js +with_docs=${2:-false} -pushd $ARROW_JS_DIR +pushd ${source_dir} +npm install +# TODO(kszucs): linting should be moved to archery npm run lint:ci npm run build -npm test -npm run test:coverage -bash <(curl -s https://codecov.io/bash) || echo "Codecov did not collect coverage reports" + +if [ "${with_docs}" == "true" ]; then + npm install typedoc + npm run doc +fi popd diff --git a/csharp/build/docker/Dockerfile b/ci/scripts/js_test.sh old mode 100644 new mode 100755 similarity index 87% rename from csharp/build/docker/Dockerfile rename to ci/scripts/js_test.sh index dca4a43f23240..1de4e1df10b76 --- a/csharp/build/docker/Dockerfile +++ b/ci/scripts/js_test.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash +# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -15,9 +17,13 @@ # specific language governing permissions and limitations # under the License. -FROM mcr.microsoft.com/dotnet/core/sdk:2.1 +set -ex + +source_dir=${1}/js + +pushd ${source_dir} -COPY csharp /arrow/csharp -WORKDIR /arrow/csharp +npm run lint +npm run test -CMD ["dotnet", "build"] +popd diff --git a/ci/docker_install_conda.sh b/ci/scripts/python_benchmark.sh similarity index 50% rename from ci/docker_install_conda.sh rename to ci/scripts/python_benchmark.sh index 73c7162b98926..23ce1bbe2d3f0 100755 --- a/ci/docker_install_conda.sh +++ b/ci/scripts/python_benchmark.sh @@ -17,26 +17,26 @@ # specific language governing permissions and limitations # under the License. -# Exit on any error -set -e +# TODO(kszucs): port it -wget --quiet https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O /tmp/miniconda.sh -bash /tmp/miniconda.sh -b -q -p ${CONDA_PREFIX:=/opt/conda} -rm /tmp/miniconda.sh +# Check the ASV benchmarking setup. +# Unfortunately this won't ensure that all benchmarks succeed +# (see https://github.com/airspeed-velocity/asv/issues/449) +source deactivate +conda create -y -q -n pyarrow_asv python=$PYTHON_VERSION +conda activate pyarrow_asv +pip install -q git+https://github.com/pitrou/asv.git@customize_commands -ln -s ${CONDA_PREFIX}/etc/profile.d/conda.sh /etc/profile.d/conda.sh -echo ". ${CONDA_PREFIX}/etc/profile.d/conda.sh" >> ~/.bashrc -echo "conda activate base" >> ~/.bashrc +export PYARROW_WITH_PARQUET=1 +export PYARROW_WITH_PLASMA=1 +export PYARROW_WITH_ORC=0 +export PYARROW_WITH_GANDIVA=0 -# Configure conda -source $MINICONDA/etc/profile.d/conda.sh -conda config --set show_channel_urls True - -# Help with SSL timeouts to S3 -conda config --set remote_connect_timeout_secs 12 - -# Setup conda-forge -conda config --add channels conda-forge - -# Update packages -conda update --all -q -y +pushd $ARROW_PYTHON_DIR +# Workaround for https://github.com/airspeed-velocity/asv/issues/631 +git fetch --depth=100 origin master:master +# Generate machine information (mandatory) +asv machine --yes +# Run benchmarks on the changeset being tested +asv run --no-pull --show-stderr --quick HEAD^! +popd # $ARROW_PYTHON_DIR diff --git a/ci/scripts/python_build.sh b/ci/scripts/python_build.sh new file mode 100755 index 0000000000000..502960e35ebb8 --- /dev/null +++ b/ci/scripts/python_build.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -ex + +source_dir=${1}/python +build_dir=${2:-${source_dir}/build} + +export PYARROW_CMAKE_GENERATOR=${CMAKE_GENERATOR:-Ninja} +export PYARROW_BUILD_TYPE=${CMAKE_BUILD_TYPE:-debug} +export PYARROW_WITH_S3=${ARROW_S3:-OFF} +export PYARROW_WITH_ORC=${ARROW_ORC:-OFF} +export PYARROW_WITH_CUDA=${ARROW_CUDA:-OFF} +export PYARROW_WITH_FLIGHT=${ARROW_FLIGHT:-OFF} +export PYARROW_WITH_PLASMA=${ARROW_PLASMA:-OFF} +export PYARROW_WITH_GANDIVA=${ARROW_GANDIVA:-OFF} +export PYARROW_WITH_PARQUET=${ARROW_PARQUET:-OFF} + +export LD_LIBRARY_PATH=${ARROW_HOME}/lib:${LD_LIBRARY_PATH} + +pushd ${source_dir} + +relative_build_dir=$(realpath --relative-to=. $build_dir) + +# not nice, but prevents mutating the mounted the source directory for docker +python setup.py build_ext --build-temp $relative_build_dir \ + --build-lib $relative_build_dir \ + build_py --build-lib $relative_build_dir \ + egg_info --egg-base $relative_build_dir \ + install_lib --build-dir $relative_build_dir \ + install --single-version-externally-managed \ + --record $relative_build_dir/record.txt + +popd diff --git a/ci/travis_release_audit.sh b/ci/scripts/python_test.sh similarity index 77% rename from ci/travis_release_audit.sh rename to ci/scripts/python_test.sh index 880e235db12de..2569c63182444 100755 --- a/ci/travis_release_audit.sh +++ b/ci/scripts/python_test.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash - +# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -17,10 +17,12 @@ # specific language governing permissions and limitations # under the License. -set -e +set -ex + +arrow_dir=${1} + +export ARROW_TEST_DATA=${arrow_dir}/testing/data +export PARQUET_TEST_DATA=${arrow_dir}/cpp/submodules/parquet-testing/data +export LD_LIBRARY_PATH=${ARROW_HOME}/lib:${LD_LIBRARY_PATH} -# Check licenses according to Apache policy -git archive HEAD --prefix=apache-arrow-raw/ | tar xf - -cp -r -L apache-arrow{-raw,} -tar czf arrow-src.tar.gz apache-arrow -./dev/release/run-rat.sh arrow-src.tar.gz +pytest -sv --pyargs pyarrow diff --git a/ci/docker_build_r.sh b/ci/scripts/r_build.sh similarity index 63% rename from ci/docker_build_r.sh rename to ci/scripts/r_build.sh index 67d96d1ed44d9..2b63b63299564 100755 --- a/ci/docker_build_r.sh +++ b/ci/scripts/r_build.sh @@ -16,27 +16,25 @@ # specific language governing permissions and limitations # under the License. -set -e - -export ARROW_HOME=$CONDA_PREFIX +set -ex : ${R_BIN:=R} -# Build arrow -pushd /arrow/r +source_dir=${1}/r + +# ARROW-6171: Because lz4 is installed in the base Ubuntu image, there's an +# issue of which library is loaded at runtime. R by default will override +# LD_LIBRARY_PATH at runtime by concatenating (in that order) +# R_LD_LIBRARY_PATH, R_JAVA_LD_LIBRARY_PATH and LD_LIBRARY_PATH. If +# R_LD_LIBRARY_PATH is not set, it'll default to a list of directories which +# contains /usr/lib/x86_64-linux-gnu. +export LD_LIBRARY_PATH=${ARROW_HOME}/lib:${LD_LIBRARY_PATH} +export R_LD_LIBRARY_PATH=${LD_LIBRARY_PATH} -if [ "$R_CONDA" = "" ]; then - # Install R package dependencies - # NOTE: any changes here should also be done in docker_build_r_sanitizer.sh - # XXX ideally this sould be done from the Dockerfile, but I can't get it to work (AP) - ${R_BIN} -e "remotes::install_deps(dependencies = TRUE)" -fi +pushd ${source_dir} make clean ${R_BIN} CMD build --keep-empty-dirs . ${R_BIN} CMD INSTALL $(ls | grep arrow_*.tar.gz) -export _R_CHECK_FORCE_SUGGESTS_=false -${R_BIN} CMD check $(ls | grep arrow_*.tar.gz) --as-cran --no-manual - popd diff --git a/ci/docker_build_r_sanitizer.sh b/ci/scripts/r_deps.sh similarity index 75% rename from ci/docker_build_r_sanitizer.sh rename to ci/scripts/r_deps.sh index 9b965a5bde428..2289d525fcfa8 100755 --- a/ci/docker_build_r_sanitizer.sh +++ b/ci/scripts/r_deps.sh @@ -16,25 +16,17 @@ # specific language governing permissions and limitations # under the License. -set -e +set -ex -: ${R_BIN:=RDsan} +: ${R_BIN:=R} -# Build arrow -pushd /arrow/r +source_dir=${1}/r + +pushd ${source_dir} # Install R package dependencies -# NOTE: any changes here should also be done in docker_build_r.sh +${R_BIN} -e "install.packages(c('remotes', 'dplyr', 'glue'))" ${R_BIN} -e "remotes::install_deps(dependencies = TRUE)" - -make clean -${R_BIN} CMD INSTALL --no-byte-compile . - -pushd tests - -export UBSAN_OPTIONS="print_stacktrace=1,suppressions=/arrow/r/tools/ubsan.supp" -${R_BIN} < testthat.R - -popd +${R_BIN} -e "remotes::install_github('romainfrancois/decor')" popd diff --git a/ci/travis_before_script_js.sh b/ci/scripts/r_sanitize.sh similarity index 82% rename from ci/travis_before_script_js.sh rename to ci/scripts/r_sanitize.sh index d80d0be467859..0072e5911331a 100755 --- a/ci/travis_before_script_js.sh +++ b/ci/scripts/r_sanitize.sh @@ -1,5 +1,4 @@ #!/usr/bin/env bash - # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -17,15 +16,16 @@ # specific language governing permissions and limitations # under the License. - set -ex -source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh +: ${R_BIN:=RDsan} + +source_dir=${1}/r -pushd $ARROW_JS_DIR +# Build arrow +pushd ${source_dir}/tests -# ARROW(ARROW-2546): Update to npm>=5.7.1 to fight EINTEGRITY problems -npm install -g npm@latest -npm install +export UBSAN_OPTIONS="print_stacktrace=1,suppressions=/arrow/r/tools/ubsan.supp" +${R_BIN} < testthat.R popd diff --git a/ci/travis_before_script_ruby.sh b/ci/scripts/r_test.sh similarity index 78% rename from ci/travis_before_script_ruby.sh rename to ci/scripts/r_test.sh index c70168715c4e0..f0a44d102b20e 100755 --- a/ci/travis_before_script_ruby.sh +++ b/ci/scripts/r_test.sh @@ -1,5 +1,4 @@ #!/usr/bin/env bash - # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -17,15 +16,17 @@ # specific language governing permissions and limitations # under the License. -set -e +set -ex -source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh +: ${R_BIN:=R} -pushd $ARROW_RUBY_DIR/red-arrow -bundle install -popd +source_dir=${1}/r + +pushd ${source_dir} -# TODO: GPU required -# pushd $ARROW_RUBY_DIR/red-arrow-gpu -# bundle install -# popd +export LD_LIBRARY_PATH=${ARROW_HOME}/lib:${LD_LIBRARY_PATH} +export R_LD_LIBRARY_PATH=${LD_LIBRARY_PATH} + +${R_BIN} CMD check --no-manual --as-cran arrow_*.tar.gz + +popd diff --git a/ci/scripts/release_test.sh b/ci/scripts/release_test.sh new file mode 100755 index 0000000000000..ae2ab328884b3 --- /dev/null +++ b/ci/scripts/release_test.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -eux + +arrow_dir=${1} + +pushd ${arrow_dir} + +dev/release/run-test.rb + +popd diff --git a/ci/scripts/ruby_test.sh b/ci/scripts/ruby_test.sh new file mode 100755 index 0000000000000..662d3ef312261 --- /dev/null +++ b/ci/scripts/ruby_test.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -ex + +source_dir=${1}/ruby +build_dir=${2:-${source_dir}/build} + +export LD_LIBRARY_PATH=${ARROW_HOME}/lib:${LD_LIBRARY_PATH} +export PKG_CONFIG_PATH=${ARROW_HOME}/lib/pkgconfig +export GI_TYPELIB_PATH=${ARROW_HOME}/lib/girepository-1.0 + +# TODO(kszucs): this should be moved into the dockerfile +bundle install --gemfile ${source_dir}/red-arrow/Gemfile + +${source_dir}/red-arrow/test/run-test.rb + +if [ ${ARROW_GANDIVA:-OFF} == "ON" ]; then + ${source_dir}/red-gandiva/test/run-test.rb +fi +if [ ${ARROW_PARQUET:-OFF} == "ON" ]; then + ${source_dir}/red-parquet/test/run-test.rb +fi +if [ ${ARROW_PLASMA:-OFF} == "ON" ]; then + ${source_dir}/red-plasma/test/run-test.rb +fi +if [ ${ARROW_CUDA:-OFF} == "ON" ]; then + ${source_dir}/red-arrow-cuda/test/run-test.rb +fi diff --git a/ci/docker_build_rust.sh b/ci/scripts/rust_build.sh similarity index 69% rename from ci/docker_build_rust.sh rename to ci/scripts/rust_build.sh index ab609977bd4a3..e845566d46b42 100755 --- a/ci/docker_build_rust.sh +++ b/ci/scripts/rust_build.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -16,49 +17,36 @@ # specific language governing permissions and limitations # under the License. -set -e +set -ex + +source_dir=${1}/rust +build_dir=${2:-${source_dir}/target} -cd /arrow/rust +export ARROW_TEST_DATA=${arrow_dir}/testing/data +export PARQUET_TEST_DATA=${arrow_dir}/cpp/submodules/parquet-testing/data +export CARGO_TARGET_DIR=${build_dir} # show activated toolchain rustup show -# clean first -cargo clean +# ensure that the build directory exists +mkdir -p ${build_dir} + +pushd ${source_dir} # flatbuffer codegen -echo Generating IPC +echo "Generating IPC" flatc --rust -o arrow/src/ipc/gen/ ../format/*.fbs # work around some bugs in flatbuffers find arrow/src/ipc/gen/ -name "*_generated.rs" -exec sed -i 's/type__type/type_type/g' {} \; -# raises on any formatting errors -echo "Running formatting checks ..." -cargo +stable fmt --all -- --check -echo "Formatting checks completed" - # build entire project RUSTFLAGS="-D warnings" cargo build --all-targets -# run tests -cargo test - # make sure we can build Arrow sub-crate without default features pushd arrow -cargo build --no-default-features -popd - -# run Arrow examples -pushd arrow -cargo run --example builders -cargo run --example dynamic_types -cargo run --example read_csv -cargo run --example read_csv_infer_schema +cargo build --all-targets --no-default-features popd -# run DataFusion examples -pushd datafusion -cargo run --example csv_sql -cargo run --example parquet_sql popd diff --git a/ci/travis_install_clang_tools.sh b/ci/scripts/rust_test.sh similarity index 56% rename from ci/travis_install_clang_tools.sh rename to ci/scripts/rust_test.sh index 856c0e842eadf..2757f6820cbcf 100755 --- a/ci/travis_install_clang_tools.sh +++ b/ci/scripts/rust_test.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash - +# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -17,13 +17,33 @@ # specific language governing permissions and limitations # under the License. - set -ex -source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh +arrow_dir=${1} +source_dir=${1}/rust +build_dir=${2:-${source_dir}/target} + +export ARROW_TEST_DATA=${arrow_dir}/testing/data +export PARQUET_TEST_DATA=${arrow_dir}/cpp/submodules/parquet-testing/data +export CARGO_TARGET_DIR=${build_dir} + +pushd ${source_dir} + +# run unit tests +cargo test + +# test arrow examples +pushd arrow +cargo run --example builders --release +cargo run --example dynamic_types --release +cargo run --example read_csv --release +cargo run --example read_csv_infer_schema --release +popd + +# test datafusion examples +pushd datafusion +cargo run --example csv_sql --release +cargo run --example parquet_sql --release +popd -wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key|sudo apt-key add - -sudo apt-add-repository -y \ - "deb https://apt.llvm.org/$DISTRO_CODENAME/ llvm-toolchain-$DISTRO_CODENAME-$ARROW_LLVM_MAJOR_VERSION main" -sudo apt-get update -qq -sudo apt-get install -q clang-$ARROW_LLVM_MAJOR_VERSION clang-format-$ARROW_LLVM_MAJOR_VERSION clang-tidy-$ARROW_LLVM_MAJOR_VERSION +popd diff --git a/dev/docker_common/wait-for-it.sh b/ci/scripts/wait_for_it.sh similarity index 100% rename from dev/docker_common/wait-for-it.sh rename to ci/scripts/wait_for_it.sh diff --git a/ci/travis_before_script_c_glib.sh b/ci/travis_before_script_c_glib.sh deleted file mode 100755 index 848992efcfb81..0000000000000 --- a/ci/travis_before_script_c_glib.sh +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh - -if [ $TRAVIS_OS_NAME = "osx" ]; then - export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:/usr/local/opt/libffi/lib/pkgconfig - export XML_CATALOG_FILES=/usr/local/etc/xml/catalog -else - source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh - conda create -n meson -y -q python=3.6 - conda activate meson - pip install meson - sudo apt-get install -y -q \ - autoconf-archive \ - gtk-doc-tools \ - libgirepository1.0-dev - conda install -q -y ninja -fi - -if [ $TRAVIS_OS_NAME = "osx" ]; then - sudo env PKG_CONFIG_PATH=$PKG_CONFIG_PATH luarocks install lgi -else - sudo apt install -y -qq luarocks - sudo luarocks install lgi -fi - -pushd $ARROW_C_GLIB_DIR - -bundle install - -export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:$ARROW_CPP_INSTALL/lib/pkgconfig -export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$ARROW_CPP_INSTALL/lib - -# Build with GNU Autotools -./autogen.sh -CONFIGURE_OPTIONS="--prefix=$ARROW_C_GLIB_INSTALL_AUTOTOOLS" -CONFIGURE_OPTIONS="$CONFIGURE_OPTIONS --enable-gtk-doc" -CONFIGURE_OPTIONS="$CONFIGURE_OPTIONS CFLAGS=-DARROW_NO_DEPRECATED_API" -CONFIGURE_OPTIONS="$CONFIGURE_OPTIONS CXXFLAGS=-DARROW_NO_DEPRECATED_API" -mkdir -p build -pushd build -../configure $CONFIGURE_OPTIONS -make -j4 -make install -popd -rm -rf build - -# Build with Meson -MESON_OPTIONS="--prefix=$ARROW_C_GLIB_INSTALL_MESON" -MESON_OPTIONS="$MESON_OPTIONS -Dgtk_doc=true" -mkdir -p build -env \ - CFLAGS="-DARROW_NO_DEPRECATED_API" \ - CXXFLAGS="-DARROW_NO_DEPRECATED_API" \ - meson build $MESON_OPTIONS -pushd build -ninja -ninja install -popd -rm -rf build - -popd diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh deleted file mode 100755 index 5204218dea9e8..0000000000000 --- a/ci/travis_before_script_cpp.sh +++ /dev/null @@ -1,217 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh - -only_library_mode=no -using_homebrew=no - -while true; do - case "$1" in - --only-library) - only_library_mode=yes - shift ;; - --homebrew) - using_homebrew=yes - shift ;; - *) break ;; - esac -done - -if [ "$only_library_mode" == "no" ]; then - source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh -fi - -if [ "$ARROW_TRAVIS_USE_TOOLCHAIN" == "1" ]; then - # Set up C++ toolchain from conda-forge packages for faster builds - source $TRAVIS_BUILD_DIR/ci/travis_install_toolchain.sh -fi - -mkdir -p $ARROW_CPP_BUILD_DIR -pushd $ARROW_CPP_BUILD_DIR - -CMAKE_COMMON_FLAGS="\ --DCMAKE_INSTALL_PREFIX=$ARROW_CPP_INSTALL \ --DARROW_WITH_BZ2=ON \ --DARROW_WITH_ZLIB=ON \ --DARROW_WITH_ZSTD=ON \ --DARROW_WITH_LZ4=ON \ --DARROW_WITH_SNAPPY=ON \ --DARROW_WITH_BROTLI=ON \ --DARROW_NO_DEPRECATED_API=ON \ --DARROW_INSTALL_NAME_RPATH=OFF \ --DARROW_EXTRA_ERROR_CONTEXT=ON" -CMAKE_LINUX_FLAGS="" -CMAKE_OSX_FLAGS="\ --DARROW_GFLAGS_USE_SHARED=OFF" - -if [ "$ARROW_TRAVIS_USE_TOOLCHAIN" == "1" ]; then - CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_JEMALLOC=ON" - CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_WITH_BZ2=ON" -fi - -if [ "$ARROW_TRAVIS_MIMALLOC" == "1" ]; then - CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_MIMALLOC=ON" -fi - -if [ $only_library_mode == "yes" ]; then - CMAKE_COMMON_FLAGS="\ - $CMAKE_COMMON_FLAGS \ - -DARROW_BUILD_UTILITIES=OFF" -else - CMAKE_COMMON_FLAGS="\ - $CMAKE_COMMON_FLAGS \ - -DARROW_BUILD_BENCHMARKS=ON \ - -DARROW_BUILD_TESTS=ON \ - -DARROW_BUILD_EXAMPLES=ON \ - -DARROW_BUILD_UTILITIES=ON" -fi - -ARROW_CXXFLAGS="" - -# Use Ninja for faster builds when using toolchain -if [ "$ARROW_TRAVIS_USE_TOOLCHAIN" == "1" ]; then - CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -GNinja" - if [ "$DISTRO_CODENAME" != "trusty" ]; then - # Make sure the toolchain linker (from binutils package) is picked up by clang - ARROW_CXXFLAGS="$ARROW_CXXFLAGS -B$CPP_TOOLCHAIN/bin" - fi - export TRAVIS_MAKE=ninja -elif [ "$using_homebrew" = "yes" ]; then - CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -GNinja" - export TRAVIS_MAKE=ninja -else - export TRAVIS_MAKE=make -fi - -if [ "$ARROW_TRAVIS_FLIGHT" == "1" ]; then - CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_FLIGHT=ON" -fi - -if [ "$ARROW_TRAVIS_INTEGRATION" == "1" ]; then - CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_BUILD_INTEGRATION=ON" -fi - -if [ "$ARROW_TRAVIS_PLASMA" == "1" ]; then - CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_PLASMA=ON" -fi - -if [ "$ARROW_TRAVIS_PLASMA_JAVA_CLIENT" == "1" ]; then - CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_PLASMA_JAVA_CLIENT=ON" -fi - -if [ "$ARROW_TRAVIS_ORC" == "1" ]; then - CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_ORC=ON" -fi - -if [ "$ARROW_TRAVIS_PYTHON" == "1" ]; then - CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_PYTHON=ON" - if [ "$using_homebrew" == "yes" ]; then - CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS \ --DPYTHON_EXECUTABLE=$(brew --prefix python)/bin/python3" - fi -fi - -if [ "$ARROW_TRAVIS_S3" == "1" ]; then - CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_S3=ON" -fi - -if [ "$ARROW_TRAVIS_PARQUET" == "1" ]; then - CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS \ --DARROW_PARQUET=ON \ --DPARQUET_BUILD_EXAMPLES=ON \ --DPARQUET_BUILD_EXECUTABLES=ON" -fi - -if [ "$ARROW_TRAVIS_PARQUET_ENCRYPTION" == "1" ]; then - CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DPARQUET_REQUIRE_ENCRYPTION=ON" -fi - -if [ "$ARROW_TRAVIS_GANDIVA" == "1" ]; then - CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_GANDIVA=ON" - if [ "$ARROW_TRAVIS_GANDIVA_JAVA" == "1" ]; then - CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_GANDIVA_JAVA=ON" - fi -fi - -if [ "$ARROW_TRAVIS_VALGRIND" == "1" ]; then - CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_TEST_MEMCHECK=ON" -fi - -if [ "$ARROW_USE_ASAN" == "1" ]; then - CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_USE_ASAN=ON" -fi - -if [ "$ARROW_USE_UBSAN" == "1" ]; then - CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_USE_UBSAN=ON" -fi - - -if [ "$ARROW_TRAVIS_COVERAGE" == "1" ]; then - CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_GENERATE_COVERAGE=ON" -fi - -if [ "$ARROW_TRAVIS_VERBOSE" == "1" ]; then - CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_VERBOSE_THIRDPARTY_BUILD=ON" -fi - -if [ "$ARROW_TRAVIS_STATIC_BOOST" == "1" ]; then - CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_BOOST_USE_SHARED=OFF" -fi - -if [ "$ARROW_TRAVIS_OPTIONAL_INSTALL" == "1" ]; then - CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_OPTIONAL_INSTALL=ON" -fi - -if [ "$ARROW_TRAVIS_USE_TOOLCHAIN" == "1" ]; then - conda activate $CPP_TOOLCHAIN -fi - -# conda-forge sets the build flags by default to -02, skip this to speed up the build -export CFLAGS=${CFLAGS//-O2} -export CXXFLAGS=${CXXFLAGS//-O2} - -if [ $TRAVIS_OS_NAME == "osx" ]; then - source $TRAVIS_BUILD_DIR/ci/travis_install_osx_sdk.sh -fi - -if [ $TRAVIS_OS_NAME == "linux" ]; then - cmake $CMAKE_COMMON_FLAGS \ - $CMAKE_LINUX_FLAGS \ - -DCMAKE_BUILD_TYPE=$ARROW_BUILD_TYPE \ - -DBUILD_WARNING_LEVEL=$ARROW_BUILD_WARNING_LEVEL \ - -DARROW_CXXFLAGS="$ARROW_CXXFLAGS" \ - $ARROW_CPP_DIR -else - cmake $CMAKE_COMMON_FLAGS \ - $CMAKE_OSX_FLAGS \ - -DCMAKE_BUILD_TYPE=$ARROW_BUILD_TYPE \ - -DBUILD_WARNING_LEVEL=$ARROW_BUILD_WARNING_LEVEL \ - $ARROW_CPP_DIR -fi - -# Build and install libraries. Configure ARROW_CPP_BUILD_TARGETS environment -# variable to only build certain targets. If you use this, you must also set -# the environment variable ARROW_TRAVIS_OPTIONAL_INSTALL=1 -time $TRAVIS_MAKE -j4 $ARROW_CPP_BUILD_TARGETS -time $TRAVIS_MAKE install - -popd diff --git a/ci/travis_env_common.sh b/ci/travis_env_common.sh deleted file mode 100755 index 453cd945ddbf6..0000000000000 --- a/ci/travis_env_common.sh +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# hide nodejs experimental-feature warnings -export NODE_NO_WARNINGS=1 -export MINICONDA=$HOME/miniconda -export CONDA_PKGS_DIRS=$HOME/.conda_packages -export CONDA_BINUTILS_VERSION=2.31 - -export ARROW_LLVM_VERSION=7.0 -export CONDA_LLVM_VERSION="7.0.*" - -# extract the major version -export ARROW_LLVM_MAJOR_VERSION=$(echo $ARROW_LLVM_VERSION | cut -d. -f1) - -export ARROW_CPP_DIR=$TRAVIS_BUILD_DIR/cpp -export ARROW_PYTHON_DIR=$TRAVIS_BUILD_DIR/python -export ARROW_C_GLIB_DIR=$TRAVIS_BUILD_DIR/c_glib -export ARROW_JAVA_DIR=${TRAVIS_BUILD_DIR}/java -export ARROW_JS_DIR=${TRAVIS_BUILD_DIR}/js -export ARROW_INTEGRATION_DIR=$TRAVIS_BUILD_DIR/integration -export ARROW_DEV_DIR=$TRAVIS_BUILD_DIR/dev -export ARROW_CROSSBOW_DIR=$TRAVIS_BUILD_DIR/dev/tasks -export ARROW_RUBY_DIR=$TRAVIS_BUILD_DIR/ruby -export ARROW_GO_DIR=${TRAVIS_BUILD_DIR}/go -export ARROW_RUST_DIR=${TRAVIS_BUILD_DIR}/rust -export ARROW_R_DIR=${TRAVIS_BUILD_DIR}/r - -export ARROW_TRAVIS_COVERAGE=${ARROW_TRAVIS_COVERAGE:=0} - -if [ "$ARROW_TRAVIS_COVERAGE" == "1" ]; then - export ARROW_CPP_COVERAGE_FILE=${TRAVIS_BUILD_DIR}/coverage.info - export ARROW_PYTHON_COVERAGE_FILE=${TRAVIS_BUILD_DIR}/.coverage -fi - -export CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/cpp-build - -export ARROW_CPP_INSTALL=$TRAVIS_BUILD_DIR/cpp-install -export ARROW_CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/cpp-build -export ARROW_C_GLIB_INSTALL_AUTOTOOLS=$TRAVIS_BUILD_DIR/c-glib-install-autotools -export ARROW_C_GLIB_INSTALL_MESON=$TRAVIS_BUILD_DIR/c-glib-install-meson - -export CMAKE_EXPORT_COMPILE_COMMANDS=1 - -export ARROW_BUILD_TYPE=${ARROW_BUILD_TYPE:=debug} -export ARROW_BUILD_WARNING_LEVEL=${ARROW_BUILD_WARNING_LEVEL:=Production} - -if [ "$ARROW_TRAVIS_USE_TOOLCHAIN" == "1" ]; then - # C++ toolchain - export CPP_TOOLCHAIN=$TRAVIS_BUILD_DIR/cpp-toolchain -fi - -if [ $TRAVIS_OS_NAME == "osx" ]; then - export GOPATH=$TRAVIS_BUILD_DIR/gopath -fi - -export ARROW_TEST_DATA=$TRAVIS_BUILD_DIR/testing/data -export PARQUET_TEST_DATA=$TRAVIS_BUILD_DIR/cpp/submodules/parquet-testing/data - -# e.g. "trusty" or "xenial" -if [ $TRAVIS_OS_NAME == "linux" ]; then - export DISTRO_CODENAME=`lsb_release -s -c` -fi - -if [ "$ARROW_TRAVIS_USE_SYSTEM_JAVA" == "1" ]; then - # Use the Ubuntu-provided OpenJDK - unset JAVA_HOME - export TRAVIS_MVN=/usr/bin/mvn - export TRAVIS_JAVA=/usr/bin/java -else - export TRAVIS_MVN=mvn - export TRAVIS_JAVA=java -fi diff --git a/ci/travis_install_conda.sh b/ci/travis_install_conda.sh deleted file mode 100755 index 205a280fffd66..0000000000000 --- a/ci/travis_install_conda.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e - -if [ $TRAVIS_OS_NAME == "linux" ]; then - MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh" -else - MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh" -fi - -function download_miniconda() { - wget --no-verbose -O miniconda.sh $MINICONDA_URL -} - -export MINICONDA=$HOME/miniconda -export CONDA_PKGS_DIRS=$HOME/.conda_packages - -# If miniconda is installed already, run the activation script -if [ -d "$MINICONDA" ]; then - source $MINICONDA/etc/profile.d/conda.sh -else - mkdir -p $CONDA_PKGS_DIRS - - CONDA_RETRIES=5 - until [ $CONDA_RETRIES -eq 0 ] - do - # If wget fails to resolve the repo.continuum.io host in the DNS, it will - # exit rather than retrying with backoff like other kinds of failures. We - # sleep for a few seconds and retry a number of times to reduce flakiness - download_miniconda && break - echo "Unable to connect to repo.continuum.io, waiting and then retrying" - CONDA_RETRIES=$[$CONDA_RETRIES - 1] - sleep 5 - done - if [ $CONDA_RETRIES -eq 0 ]; then - # If we failed to download, try again so the error message will be visible - # in Travis CI logs - download_miniconda - fi - - bash miniconda.sh -b -p $MINICONDA - source $MINICONDA/etc/profile.d/conda.sh - - conda config --set auto_update_conda false - conda info -a - - conda config --set show_channel_urls True - - # Help with SSL timeouts to S3 - conda config --set remote_connect_timeout_secs 12 - - conda config --add channels conda-forge -fi - -conda info -a diff --git a/ci/travis_install_linux.sh b/ci/travis_install_linux.sh deleted file mode 100755 index 441d50d9b16cd..0000000000000 --- a/ci/travis_install_linux.sh +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e - -source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh - -sudo apt-get install -y -qq \ - gdb binutils ccache libboost-dev libboost-filesystem-dev \ - libboost-system-dev libboost-regex-dev - -if [ "$CXX" == "g++-4.9" ]; then - sudo apt-get install -y -qq g++-4.9 -fi - -if [ "$ARROW_TRAVIS_VALGRIND" == "1" ]; then - sudo apt-get install -y -qq valgrind -fi - -if [ "$ARROW_TRAVIS_COVERAGE" == "1" ]; then - sudo apt-get install -y -qq lcov -fi - -set -x -if [ "$ARROW_TRAVIS_GANDIVA" == "1" ]; then - sudo apt-get install -y -qq llvm-$ARROW_LLVM_MAJOR_VERSION-dev -fi - -if [ "$ARROW_TRAVIS_USE_SYSTEM" == "1" ]; then - if [ "$DISTRO_CODENAME" == "xenial" ]; then - # TODO(ARROW-4761): Install libzstd-dev once we support zstd<1 - # TODO(ARROW-4762): Install rapidjson-dev once we support rapidjson<1.1.0 - export ADDITIONAL_APT_PKGS="libre2-dev libbrotli-dev liblz4-dev" - fi - sudo apt-get install -y -q --no-install-recommends \ - autoconf \ - ca-certificates \ - ccache \ - cmake \ - g++ \ - gcc \ - git \ - libboost-all-dev \ - libdouble-conversion-dev \ - libsnappy-dev \ - libssl-dev \ - libgflags-dev \ - libprotobuf-dev \ - libprotoc-dev \ - llvm-7-dev \ - clang-7 \ - protobuf-compiler \ - libbz2-dev \ - ninja-build \ - pkg-config \ - tzdata \ - flex \ - bison \ - ${ADDITIONAL_APT_PKGS} -fi - -set -x -if [ "$DISTRO_CODENAME" != "trusty" ]; then - sudo apt-get install -y -qq maven - - # Remove Travis-specific versions of Java - sudo rm -rf /usr/local/lib/jvm* - sudo rm -rf /usr/local/maven* - hash -r - unset JAVA_HOME - - which java - which mvn - java -version - mvn -v -fi diff --git a/ci/travis_install_minio.sh b/ci/travis_install_minio.sh deleted file mode 100755 index d3083ede18c0b..0000000000000 --- a/ci/travis_install_minio.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e -set -x - -if [ "$ARROW_TRAVIS_S3" == "1" ]; then - # Download the Minio S3 server into PATH - if [ $TRAVIS_OS_NAME = "osx" ]; then - MINIO_URL=https://dl.min.io/server/minio/release/darwin-amd64/minio - else - MINIO_URL=https://dl.min.io/server/minio/release/linux-amd64/minio - fi - - S3FS_DIR=~/.local/bin/ - mkdir -p $S3FS_DIR - wget --quiet --directory-prefix $S3FS_DIR $MINIO_URL - chmod +x $S3FS_DIR/minio -fi diff --git a/ci/travis_install_osx.sh b/ci/travis_install_osx.sh deleted file mode 100755 index 2d79eb017ed5c..0000000000000 --- a/ci/travis_install_osx.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -x -set -e - -if [ "$ARROW_CI_RUBY_AFFECTED" = "1" ]; then - brew_log_path=brew.log - function run_brew() { - local i=0 - local n_tries=3 - while [[ $((i++)) < ${n_tries} ]]; do - echo "${i}: brew" "$@" >> ${brew_log_path} - if gtimeout --signal=KILL 9m brew "$@" >> ${brew_log_path} 2>&1; then - break - elif [[ ${i} == ${n_tries} ]]; then - cat ${brew_log_path} - rm ${brew_log_path} - false - fi - done - } - - run_brew bundle --file=$TRAVIS_BUILD_DIR/cpp/Brewfile --verbose - run_brew bundle --file=$TRAVIS_BUILD_DIR/c_glib/Brewfile --verbose - rm ${brew_log_path} -fi \ No newline at end of file diff --git a/ci/travis_install_toolchain.sh b/ci/travis_install_toolchain.sh deleted file mode 100755 index 7951e40d3022f..0000000000000 --- a/ci/travis_install_toolchain.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh - -source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh - -if [ ! -e $CPP_TOOLCHAIN ]; then - CONDA_PACKAGES="" - - if [ "$ARROW_TRAVIS_GANDIVA" == "1" ]; then - CONDA_PACKAGES="$CONDA_PACKAGES --file=$TRAVIS_BUILD_DIR/ci/conda_env_gandiva.yml" - fi - - # Set up C++ toolchain from conda-forge packages for faster builds - time conda create -y -q -p $CPP_TOOLCHAIN \ - --file=$TRAVIS_BUILD_DIR/ci/conda_env_cpp.yml \ - --file=$TRAVIS_BUILD_DIR/ci/conda_env_unix.yml \ - $CONDA_PACKAGES \ - compilers \ - nomkl \ - python=3.6 -fi diff --git a/ci/travis_lint.sh b/ci/travis_lint.sh deleted file mode 100755 index a564bdcf61fcc..0000000000000 --- a/ci/travis_lint.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -# Disable toolchain variables in this script -export ARROW_TRAVIS_USE_TOOLCHAIN=0 -source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh - -# pytest, requests, and jira are needed to run the PR merge script unit tests -pip install pre_commit cmake_format==0.5.2 pytest requests jira -pre-commit install - -# TODO: Move more checks into pre-commit as this gives a nice summary -# and doesn't abort on the first failed check. -pre-commit run hadolint -a - -# CMake formatting check -$TRAVIS_BUILD_DIR/run-cmake-format.py --check - -# C++ code linting -if [ "$ARROW_CI_CPP_AFFECTED" != "0" ]; then - mkdir $ARROW_CPP_DIR/lint - pushd $ARROW_CPP_DIR/lint - - cmake .. -DARROW_ONLY_LINT=ON - make lint - make check-format - - python $ARROW_CPP_DIR/build-support/lint_cpp_cli.py $ARROW_CPP_DIR/src - - popd -fi - -# Python style checks -# (need Python 3 for crossbow) -FLAKE8="python3 -m flake8" -python3 -m pip install -q flake8 - -if [ "$ARROW_CI_DEV_AFFECTED" != "0" ]; then - pushd $ARROW_DEV_DIR - $FLAKE8 --count . - py.test test_merge_arrow_pr.py - popd -fi - -if [ "$ARROW_CI_INTEGRATION_AFFECTED" != "0" ]; then - $FLAKE8 --count $ARROW_INTEGRATION_DIR -fi - -if [ "$ARROW_CI_PYTHON_AFFECTED" != "0" ]; then - $FLAKE8 --count $ARROW_PYTHON_DIR - # Check Cython files with some checks turned off - $FLAKE8 --count \ - --config=$ARROW_PYTHON_DIR/.flake8.cython \ - $ARROW_PYTHON_DIR -fi - -if [ "$ARROW_CI_R_AFFECTED" != "0" ]; then - pushd $ARROW_R_DIR - ./lint.sh - popd -fi diff --git a/ci/travis_release_test.sh b/ci/travis_release_test.sh deleted file mode 100755 index 4b541525fc493..0000000000000 --- a/ci/travis_release_test.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -eux - -pip install cython -pip install setuptools - -wget -q https://packages.microsoft.com/config/ubuntu/16.04/packages-microsoft-prod.deb -sudo dpkg -i packages-microsoft-prod.deb - -sudo apt-get install apt-transport-https -sudo apt-get update -sudo apt-get install dotnet-sdk-2.2 - -dev/release/run-test.rb diff --git a/ci/travis_script_c_glib.sh b/ci/travis_script_c_glib.sh deleted file mode 100755 index c42a047ddf445..0000000000000 --- a/ci/travis_script_c_glib.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e - -source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh - -arrow_c_glib_run_test() -{ - local arrow_c_glib_lib_dir=$1 - - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$arrow_c_glib_lib_dir - export GI_TYPELIB_PATH=$arrow_c_glib_lib_dir/girepository-1.0 - test/run-test.rb - - export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:$arrow_c_glib_lib_dir/pkgconfig - - pushd example/lua - lua write-batch.lua - lua read-batch.lua - lua write-stream.lua - lua read-stream.lua - popd -} - -pushd $ARROW_C_GLIB_DIR - -export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$ARROW_CPP_INSTALL/lib -export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:$ARROW_CPP_INSTALL/lib/pkgconfig - -(arrow_c_glib_run_test $ARROW_C_GLIB_INSTALL_AUTOTOOLS/lib) -if [ -d $ARROW_C_GLIB_INSTALL_MESON/lib/$(arch)-linux-gnu ]; then - (arrow_c_glib_run_test $ARROW_C_GLIB_INSTALL_MESON/lib/$(arch)-linux-gnu) -# else # TODO: Enable this -# (arrow_c_glib_run_test $ARROW_C_GLIB_INSTALL_MESON/lib) -fi - -popd diff --git a/ci/travis_script_gandiva_java.sh b/ci/travis_script_gandiva_java.sh deleted file mode 100755 index 9ea89b774188e..0000000000000 --- a/ci/travis_script_gandiva_java.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e - -source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh -JAVA_DIR=${TRAVIS_BUILD_DIR}/java - -export MAVEN_OPTS="-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn" - -pushd $JAVA_DIR - -# build with gandiva profile -$TRAVIS_MVN -P arrow-jni -B install -DskipTests -Darrow.cpp.build.dir=$CPP_BUILD_DIR/debug - -# run gandiva tests -$TRAVIS_MVN test -P arrow-jni -pl gandiva -Darrow.cpp.build.dir=$CPP_BUILD_DIR/debug - -popd diff --git a/ci/travis_script_manylinux.sh b/ci/travis_script_manylinux.sh deleted file mode 100755 index 3c261534ababe..0000000000000 --- a/ci/travis_script_manylinux.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# NB(wesm): Here is the Travis CI entry removed in ARROW-5962 - -# - name: "[manylinux1] Python" -# language: cpp -# env: -# - PYTHON_VERSIONS="3.6,16 3.7,16" -# before_script: -# - if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then docker-compose pull python-manylinux1; fi -# script: -# - if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_manylinux.sh; fi - -set -ex - -# Testing for https://issues.apache.org/jira/browse/ARROW-2657 -# These tests cannot be run inside of the docker container, since TensorFlow -# does not run on manylinux1 - -source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh -source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh - -pushd python/manylinux1 - -cat << EOF > check_imports.py -import sys -import pyarrow -import pyarrow.orc -import pyarrow.parquet -import pyarrow.plasma - -if sys.version_info.major > 2: - import pyarrow.flight - import pyarrow.gandiva -EOF - -for PYTHON_TUPLE in ${PYTHON_VERSIONS}; do - IFS="," read PYTHON_VERSION UNICODE_WIDTH <<< $PYTHON_TUPLE - - # cleanup the artifact directory, docker writes it as root - sudo rm -rf dist - - # build the wheels - docker-compose run \ - -e PYARROW_PARALLEL=3 \ - -e PYTHON_VERSION=$PYTHON_VERSION \ - -e UNICODE_WIDTH=$UNICODE_WIDTH \ - python-manylinux1 - - # create a testing conda environment - CONDA_ENV_DIR=$TRAVIS_BUILD_DIR/pyarrow-test-$PYTHON_VERSION - conda create -y -q -p $CONDA_ENV_DIR python=$PYTHON_VERSION - conda activate $CONDA_ENV_DIR - - # install the produced wheels - pip install dist/*.whl - - # Test optional dependencies - python check_imports.py - - # Install test dependencies and run pyarrow tests - pip install -r ../requirements-test.txt - pytest --pyargs pyarrow -done - -popd diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh deleted file mode 100755 index db4f8d80275c0..0000000000000 --- a/ci/travis_script_python.sh +++ /dev/null @@ -1,271 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e - -source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh - -source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh - -export ARROW_HOME=$ARROW_CPP_INSTALL -export PARQUET_HOME=$ARROW_CPP_INSTALL -export LD_LIBRARY_PATH=$ARROW_HOME/lib:$LD_LIBRARY_PATH -export PYARROW_CXXFLAGS="-Werror" - -PYARROW_PYTEST_FLAGS=" -r sxX --durations=15 --parquet" - -PYTHON_VERSION=$1 -CONDA_ENV_DIR=$TRAVIS_BUILD_DIR/pyarrow-test-$PYTHON_VERSION - -# We should use zlib in the target Python directory to avoid loading -# the wrong libpython on macOS at run-time. Another zlib might sit in a -# directory with a different libpython3.6m.dylib, and that libpython3.6m.dylib -# may not have NumPy (which is required for python-test) -export ZLIB_HOME=$CONDA_ENV_DIR - -CONDA_FILES="" -CONDA_PACKAGES="" - -if [ "$ARROW_TRAVIS_PYTHON_GANDIVA" == "1" ]; then - CONDA_FILES="$CONDA_FILES --file=$TRAVIS_BUILD_DIR/ci/conda_env_gandiva.yml" -fi - -if [ "$ARROW_TRAVIS_PYTHON_JVM" == "1" ]; then - JPYPE_VERSION=0.6.3 - CONDA_PACKAGES="$CONDA_PACKAGES jpype1=$JPYPE_VERSION" -fi - -conda create -y -q -p $CONDA_ENV_DIR \ - --file $TRAVIS_BUILD_DIR/ci/conda_env_cpp.yml \ - --file $TRAVIS_BUILD_DIR/ci/conda_env_unix.yml \ - --file $TRAVIS_BUILD_DIR/ci/conda_env_python.yml \ - ${CONDA_FILES} \ - nomkl \ - pip \ - numpy=1.14 \ - 'libgfortran<4' \ - python=${PYTHON_VERSION} \ - compilers \ - ${CONDA_PACKAGES} - -conda activate $CONDA_ENV_DIR - -python --version -which python - -if [ "$ARROW_TRAVIS_PYTHON_DOCS" == "1" ]; then - # Install documentation dependencies - conda install -y --file ci/conda_env_sphinx.yml -fi - -# ARROW-2093: PyTorch increases the size of our conda dependency stack -# significantly, and so we have disabled these tests in Travis CI for now - -# if [ "$PYTHON_VERSION" != "2.7" ] || [ $TRAVIS_OS_NAME != "osx" ]; then -# # Install pytorch for torch tensor conversion tests -# # PyTorch seems to be broken on Python 2.7 on macOS so we skip it -# conda install -y -q pytorch torchvision -c soumith -# fi - -if [ $TRAVIS_OS_NAME != "osx" ]; then - conda install -y tensorflow - PYARROW_PYTEST_FLAGS="$PYARROW_PYTEST_FLAGS --tensorflow" -fi - -# Re-build C++ libraries with the right Python setup - -# Clear out prior build files -rm -rf $ARROW_CPP_BUILD_DIR -mkdir -p $ARROW_CPP_BUILD_DIR -pushd $ARROW_CPP_BUILD_DIR - - -# XXX Can we simply reuse CMAKE_COMMON_FLAGS from travis_before_script_cpp.sh? -CMAKE_COMMON_FLAGS="-DARROW_EXTRA_ERROR_CONTEXT=ON" - -PYTHON_CPP_BUILD_TARGETS="arrow_python-all plasma parquet" - -if [ "$ARROW_TRAVIS_S3" == "1" ]; then - CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_S3=ON" -fi - -if [ "$ARROW_TRAVIS_FLIGHT" == "1" ]; then - CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_FLIGHT=ON" -fi - -if [ "$ARROW_TRAVIS_COVERAGE" == "1" ]; then - CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_GENERATE_COVERAGE=ON" -fi - -if [ "$ARROW_TRAVIS_PYTHON_GANDIVA" == "1" ]; then - CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_GANDIVA=ON" - PYTHON_CPP_BUILD_TARGETS="$PYTHON_CPP_BUILD_TARGETS gandiva" -fi - -if [ "$ARROW_TRAVIS_VERBOSE" == "1" ]; then - CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_VERBOSE_THIRDPARTY_BUILD=ON" -fi - -if [ $TRAVIS_OS_NAME == "osx" ]; then - source $TRAVIS_BUILD_DIR/ci/travis_install_osx_sdk.sh -fi - -# conda-forge sets the build flags by default to -02, skip this to speed up the build -export CFLAGS=${CFLAGS//-O2} -export CXXFLAGS=${CXXFLAGS//-O2} - -cmake -GNinja \ - $CMAKE_COMMON_FLAGS \ - -DARROW_BUILD_TESTS=ON \ - -DARROW_BUILD_UTILITIES=OFF \ - -DARROW_OPTIONAL_INSTALL=ON \ - -DARROW_WITH_BZ2=ON \ - -DARROW_WITH_ZLIB=ON \ - -DARROW_WITH_ZSTD=ON \ - -DARROW_WITH_LZ4=ON \ - -DARROW_WITH_SNAPPY=ON \ - -DARROW_WITH_BROTLI=ON \ - -DARROW_PARQUET=on \ - -DPARQUET_REQUIRE_ENCRYPTION=on \ - -DARROW_PLASMA=on \ - -DARROW_TENSORFLOW=on \ - -DARROW_PYTHON=on \ - -DARROW_ORC=on \ - -DCMAKE_BUILD_TYPE=$ARROW_BUILD_TYPE \ - -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \ - $ARROW_CPP_DIR - -ninja $PYTHON_CPP_BUILD_TARGETS -ninja install - -popd - -# python-test isn't run by travis_script_cpp.sh, exercise it here -$ARROW_CPP_BUILD_DIR/$ARROW_BUILD_TYPE/arrow-python-test - -pushd $ARROW_PYTHON_DIR - -pip install -q pickle5 -if [ "$ARROW_TRAVIS_COVERAGE" == "1" ]; then - export PYARROW_GENERATE_COVERAGE=1 - pip install -q coverage -fi - -echo "=== pip list ===" -pip list - -export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:$ARROW_CPP_INSTALL/lib/pkgconfig - -export PYARROW_BUILD_TYPE=$ARROW_BUILD_TYPE -export PYARROW_WITH_PARQUET=1 -export PYARROW_WITH_PLASMA=1 -export PYARROW_WITH_ORC=1 -if [ "$ARROW_TRAVIS_S3" == "1" ]; then - export PYARROW_WITH_S3=1 -fi -if [ "$ARROW_TRAVIS_FLIGHT" == "1" ]; then - export PYARROW_WITH_FLIGHT=1 -fi -if [ "$ARROW_TRAVIS_PYTHON_GANDIVA" == "1" ]; then - export PYARROW_WITH_GANDIVA=1 -fi - -python setup.py develop - -# Basic sanity checks -python -c "import pyarrow.parquet" -python -c "import pyarrow.plasma" -python -c "import pyarrow.orc" -python -c "import pyarrow.fs" - -# Ensure we do eagerly import pandas (or other expensive imports) -python < scripts/test_imports.py - -echo "PLASMA_VALGRIND: $PLASMA_VALGRIND" - -# Set up huge pages for plasma test -if [ $TRAVIS_OS_NAME == "linux" ]; then - sudo sysctl -w vm.nr_hugepages=2048 - sudo mkdir -p /mnt/hugepages - sudo mount -t hugetlbfs -o uid=`id -u` -o gid=`id -g` none /mnt/hugepages - sudo bash -c "echo `id -g` > /proc/sys/vm/hugetlb_shm_group" - sudo bash -c "echo 2048 > /proc/sys/vm/nr_hugepages" -fi - -# For core dump analysis -ln -sf `which python` $TRAVIS_BUILD_DIR/current-exe - -# Need to run tests from the source tree for Cython coverage and conftest.py -if [ "$ARROW_TRAVIS_COVERAGE" == "1" ]; then - # Output Python coverage data in a persistent place - export COVERAGE_FILE=$ARROW_PYTHON_COVERAGE_FILE - python -m coverage run --append -m pytest $PYARROW_PYTEST_FLAGS pyarrow/tests -else - python -m pytest $PYARROW_PYTEST_FLAGS pyarrow/tests -fi - -if [ "$ARROW_TRAVIS_COVERAGE" == "1" ]; then - # Check Cython coverage was correctly captured in $COVERAGE_FILE - coverage report -i --include="*/lib.pyx" - coverage report -i --include="*/memory.pxi" - coverage report -i --include="*/_parquet.pyx" - # Generate XML file for CodeCov - coverage xml -i -o $TRAVIS_BUILD_DIR/coverage.xml - # Capture C++ coverage info - pushd $TRAVIS_BUILD_DIR - lcov --directory . --capture --no-external --output-file coverage-python-tests.info \ - 2>&1 | grep -v "ignoring data for external file" - lcov --add-tracefile coverage-python-tests.info \ - --output-file $ARROW_CPP_COVERAGE_FILE - rm coverage-python-tests.info - popd # $TRAVIS_BUILD_DIR -fi - -if [ "$ARROW_TRAVIS_PYTHON_DOCS" == "1" ]; then - pushd ../cpp/apidoc - doxygen - popd - cd ../docs - sphinx-build -q -b html -d _build/doctrees -W --keep-going source _build/html -fi - -popd # $ARROW_PYTHON_DIR - -if [ "$ARROW_TRAVIS_PYTHON_BENCHMARKS" == "1" ]; then - # Check the ASV benchmarking setup. - # Unfortunately this won't ensure that all benchmarks succeed - # (see https://github.com/airspeed-velocity/asv/issues/449) - source deactivate - conda create -y -q -n pyarrow_asv python=$PYTHON_VERSION - conda activate pyarrow_asv - pip install -q git+https://github.com/pitrou/asv.git@customize_commands - - export PYARROW_WITH_PARQUET=1 - export PYARROW_WITH_PLASMA=1 - export PYARROW_WITH_ORC=0 - export PYARROW_WITH_GANDIVA=0 - - pushd $ARROW_PYTHON_DIR - # Workaround for https://github.com/airspeed-velocity/asv/issues/631 - git fetch --depth=100 origin master:master - # Generate machine information (mandatory) - asv machine --yes - # Run benchmarks on the changeset being tested - asv run --no-pull --show-stderr --quick HEAD^! - popd # $ARROW_PYTHON_DIR -fi diff --git a/ci/travis_script_ruby.sh b/ci/travis_script_ruby.sh deleted file mode 100755 index 0ae85b4dc5cf6..0000000000000 --- a/ci/travis_script_ruby.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e - -source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh - -arrow_ruby_run_test() -{ - local arrow_c_glib_lib_dir="$1" - - local ld_library_path_keep="$LD_LIBRARY_PATH" - local pkg_config_path_keep="$PKG_COFNIG_PATH" - LD_LIBRARY_PATH="${arrow_c_glib_lib_dir}:${LD_LIBRARY_PATH}" - PKG_CONFIG_PATH="${arrow_c_glib_lib_dir}/pkgconfig:${PKG_CONFIG_PATH}" - export GI_TYPELIB_PATH="${arrow_c_glib_lib_dir}/girepository-1.0" - test/run-test.rb - LD_LIBRARY_PATH="$ld_library_path_keep" - PKG_CONFIG_PATH="$pkg_config_path_keep" -} - -export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$ARROW_CPP_INSTALL/lib -export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:$ARROW_CPP_INSTALL/lib/pkgconfig - -pushd $ARROW_RUBY_DIR/red-arrow -(arrow_ruby_run_test $ARROW_C_GLIB_INSTALL_AUTOTOOLS/lib) -if [ -d $ARROW_C_GLIB_INSTALL_MESON/lib/$(arch)-linux-gnu ]; then - (arrow_ruby_run_test $ARROW_C_GLIB_INSTALL_MESON/lib/$(arch)-linux-gnu) -# else # TODO: Enable this -# (arrow_ruby_run_test $ARROW_C_GLIB_INSTALL_MESON/lib) -fi -popd - -# TODO: GPU required -# pushd $ARROW_RUBY_DIR/red-arrow-gpu -# (arrow_ruby_run_test $ARROW_C_GLIB_INSTALL_AUTOTOOLS/lib) -# if [ -d $ARROW_C_GLIB_INSTALL_MESON/lib/$(arch)-linux-gnu ]; then -# (arrow_ruby_run_test $ARROW_C_GLIB_INSTALL_MESON/lib/$(arch)-linux-gnu) -# # else # TODO: Enable this -# # (arrow_ruby_run_test $ARROW_C_GLIB_INSTALL_MESON/lib) -# fi -# popd - -pushd $ARROW_RUBY_DIR/red-gandiva -(arrow_ruby_run_test $ARROW_C_GLIB_INSTALL_AUTOTOOLS/lib) -if [ -d $ARROW_C_GLIB_INSTALL_MESON/lib/$(arch)-linux-gnu ]; then - (arrow_ruby_run_test $ARROW_C_GLIB_INSTALL_MESON/lib/$(arch)-linux-gnu) -# else # TODO: Enable this -# (arrow_ruby_run_test $ARROW_C_GLIB_INSTALL_MESON/lib) -fi -popd - -pushd $ARROW_RUBY_DIR/red-parquet -(arrow_ruby_run_test $ARROW_C_GLIB_INSTALL_AUTOTOOLS/lib) -if [ -d $ARROW_C_GLIB_INSTALL_MESON/lib/$(arch)-linux-gnu ]; then - (arrow_ruby_run_test $ARROW_C_GLIB_INSTALL_MESON/lib/$(arch)-linux-gnu) -# else # TODO: Enable this -# (arrow_ruby_run_test $ARROW_C_GLIB_INSTALL_MESON/lib) -fi -popd - -pushd $ARROW_RUBY_DIR/red-plasma -(arrow_ruby_run_test $ARROW_C_GLIB_INSTALL_AUTOTOOLS/lib) -if [ -d $ARROW_C_GLIB_INSTALL_MESON/lib/$(arch)-linux-gnu ]; then - (arrow_ruby_run_test $ARROW_C_GLIB_INSTALL_MESON/lib/$(arch)-linux-gnu) -# else # TODO: Enable this -# (arrow_ruby_run_test $ARROW_C_GLIB_INSTALL_MESON/lib) -fi -popd diff --git a/ci/travis_upload_cpp_coverage.sh b/ci/travis_upload_cpp_coverage.sh deleted file mode 100755 index 38ea5d340c96f..0000000000000 --- a/ci/travis_upload_cpp_coverage.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e - -source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh - -pushd $TRAVIS_BUILD_DIR - -# Display C++ coverage summary -lcov --list $ARROW_CPP_COVERAGE_FILE -# Upload report to CodeCov, disabling gcov discovery to save time and avoid warnings -bash <(curl -s https://codecov.io/bash) -X gcov || echo "Codecov did not collect coverage reports" - -popd diff --git a/ci/travis_upload_r_coverage.R b/ci/travis_upload_r_coverage.R deleted file mode 100644 index 89c102f0f0330..0000000000000 --- a/ci/travis_upload_r_coverage.R +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -library(covr) -# Hack file paths to include the subdirectory -trace("to_codecov", quote(per_line <- function(x) { - out <- covr:::per_line(x) - setNames(out, paste0("r/", names(out))) - }), where = package_coverage) -codecov() diff --git a/cpp/Brewfile b/cpp/Brewfile index c6588873d321c..62bd487101f6d 100644 --- a/cpp/Brewfile +++ b/cpp/Brewfile @@ -18,18 +18,21 @@ brew "automake" brew "boost" brew "brotli" +brew "c-ares" brew "ccache" brew "cmake" brew "double-conversion" brew "flatbuffers" brew "git" brew "glog" +brew "google-benchmark" brew "grpc" brew "llvm@7" brew "lz4" brew "ninja" brew "numpy" brew "openssl" +brew "orc" brew "protobuf" brew "python" brew "rapidjson" diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index a60c2575f7a91..09f4e32d9e765 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -265,15 +265,6 @@ endif(UNIX) # Set up various options # -if(ARROW_BUILD_TESTS OR ARROW_BUILD_BENCHMARKS) - # Currently the compression tests require at least these libraries; bz2 and - # zstd are optional. See ARROW-3984 - set(ARROW_WITH_BROTLI ON) - set(ARROW_WITH_LZ4 ON) - set(ARROW_WITH_SNAPPY ON) - set(ARROW_WITH_ZLIB ON) -endif() - if(ARROW_BUILD_TESTS OR ARROW_BUILD_INTEGRATION) set(ARROW_JSON ON) endif() diff --git a/cpp/apidoc/Doxyfile b/cpp/apidoc/Doxyfile index 60551910b96ee..faa7f8ebfd3b9 100644 --- a/cpp/apidoc/Doxyfile +++ b/cpp/apidoc/Doxyfile @@ -75,7 +75,7 @@ PROJECT_LOGO = # entered, it will be relative to the location where doxygen was started. If # left blank the current directory will be used. -OUTPUT_DIRECTORY = +OUTPUT_DIRECTORY = $(OUTPUT_DIRECTORY) # If the CREATE_SUBDIRS tag is set to YES then doxygen will create 4096 sub- # directories (in 2 levels) under the output directory of each output format and diff --git a/cpp/src/arrow/array/diff_test.cc b/cpp/src/arrow/array/diff_test.cc index 6181a2fb8c2d3..8a39512cb2814 100644 --- a/cpp/src/arrow/array/diff_test.cc +++ b/cpp/src/arrow/array/diff_test.cc @@ -571,14 +571,15 @@ TEST_F(DiffTest, DictionaryDiffFormatter) { target_indices, target_dict, &target_)); base_->Equals(*target_, EqualOptions().diff_sink(&formatted)); - ASSERT_EQ(formatted.str(), R"(# Dictionary arrays differed + auto formatted_expected_indices = R"(# Dictionary arrays differed ## dictionary diff ## indices diff @@ -4, +4 @@ -0 @@ -6, +5 @@ +1 -)"); +)"; + ASSERT_EQ(formatted.str(), formatted_expected_indices); // differing dictionaries target_dict = ArrayFromJSON(utf8(), R"(["b", "c", "a"])"); @@ -589,14 +590,15 @@ TEST_F(DiffTest, DictionaryDiffFormatter) { formatted.str(""); base_->Equals(*target_, EqualOptions().diff_sink(&formatted)); - ASSERT_EQ(formatted.str(), R"(# Dictionary arrays differed + auto formatted_expected_values = R"(# Dictionary arrays differed ## dictionary diff @@ -0, +0 @@ -"a" @@ -3, +2 @@ +"a" ## indices diff -)"); +)"; + ASSERT_EQ(formatted.str(), formatted_expected_values); } void MakeSameLength(std::shared_ptr* a, std::shared_ptr* b) { diff --git a/cpp/src/arrow/csv/parser_test.cc b/cpp/src/arrow/csv/parser_test.cc index f988f3ce21ab1..ff86c0c39d07f 100644 --- a/cpp/src/arrow/csv/parser_test.cc +++ b/cpp/src/arrow/csv/parser_test.cc @@ -237,8 +237,9 @@ TEST(BlockParser, Basics) { { auto csv1 = MakeCSVData({"ab,cd,\n", "ef,,gh\n"}); auto csv2 = MakeCSVData({",ij,kl\n"}); + std::vector csvs = {csv1, csv2}; BlockParser parser(ParseOptions::Defaults()); - AssertParseOk(parser, {csv1, csv2}); + AssertParseOk(parser, csvs); AssertColumnsEq(parser, {{"ab", "ef", ""}, {"cd", "", "ij"}, {"", "gh", "kl"}}); AssertLastRowEq(parser, {"", "ij", "kl"}, {false, false, false}); } @@ -390,7 +391,8 @@ TEST(BlockParser, Final) { // Two blocks auto csv1 = MakeCSVData({"ab,cd\n"}); auto csv2 = MakeCSVData({"ef,"}); - AssertParseFinal(parser, {csv1, csv2}); + std::vector csvs = {csv1, csv2}; + AssertParseFinal(parser, csvs); AssertColumnsEq(parser, {{"ab", "ef"}, {"cd", ""}}); } diff --git a/cpp/src/arrow/dataset/discovery.cc b/cpp/src/arrow/dataset/discovery.cc index da1f8c592ce4a..12de3cbd3ea9d 100644 --- a/cpp/src/arrow/dataset/discovery.cc +++ b/cpp/src/arrow/dataset/discovery.cc @@ -34,25 +34,37 @@ namespace arrow { namespace dataset { FileSystemDataSourceDiscovery::FileSystemDataSourceDiscovery( - fs::FileSystem* filesystem, std::vector files, + fs::FileSystem* filesystem, std::string base_dir, std::vector files, std::shared_ptr format) - : fs_(filesystem), files_(std::move(files)), format_(std::move(format)) {} + : fs_(filesystem), + base_dir_(std::move(base_dir)), + files_(std::move(files)), + format_(std::move(format)) {} Status FileSystemDataSourceDiscovery::Make(fs::FileSystem* filesystem, + std::string base_dir, std::vector files, std::shared_ptr format, std::shared_ptr* out) { - out->reset(new FileSystemDataSourceDiscovery(filesystem, files, format)); + out->reset( + new FileSystemDataSourceDiscovery(filesystem, std::move(base_dir), files, format)); return Status::OK(); } +Status FileSystemDataSourceDiscovery::Make(fs::FileSystem* filesystem, + std::vector files, + std::shared_ptr format, + std::shared_ptr* out) { + return Make(filesystem, "", std::move(files), std::move(format), out); +} + Status FileSystemDataSourceDiscovery::Make(fs::FileSystem* filesystem, fs::Selector selector, std::shared_ptr format, std::shared_ptr* out) { std::vector files; RETURN_NOT_OK(filesystem->GetTargetStats(selector, &files)); - return Make(filesystem, files, format, out); + return Make(filesystem, std::move(selector.base_dir), std::move(files), format, out); } static inline Status InspectSchema(fs::FileSystem* fs, @@ -85,7 +97,8 @@ Status FileSystemDataSourceDiscovery::Finish(std::shared_ptr* out) { PathPartitions partitions; if (partition_scheme_ != nullptr) { - RETURN_NOT_OK(ApplyPartitionScheme(*partition_scheme_, files_, &partitions)); + RETURN_NOT_OK( + ApplyPartitionScheme(*partition_scheme_, base_dir_, files_, &partitions)); } return FileSystemBasedDataSource::Make(fs_, files_, root_partition(), diff --git a/cpp/src/arrow/dataset/discovery.h b/cpp/src/arrow/dataset/discovery.h index cbc1258e547ab..dbb596e98adb9 100644 --- a/cpp/src/arrow/dataset/discovery.h +++ b/cpp/src/arrow/dataset/discovery.h @@ -89,6 +89,10 @@ class ARROW_DS_EXPORT DataSourceDiscovery { /// of fs::FileStats or a fs::Selector. class ARROW_DS_EXPORT FileSystemDataSourceDiscovery : public DataSourceDiscovery { public: + static Status Make(fs::FileSystem* filesystem, std::string base_dir, + std::vector files, std::shared_ptr format, + std::shared_ptr* out); + static Status Make(fs::FileSystem* filesystem, std::vector files, std::shared_ptr format, std::shared_ptr* out); @@ -102,11 +106,12 @@ class ARROW_DS_EXPORT FileSystemDataSourceDiscovery : public DataSourceDiscovery Status Finish(std::shared_ptr* out) override; protected: - FileSystemDataSourceDiscovery(fs::FileSystem* filesystem, + FileSystemDataSourceDiscovery(fs::FileSystem* filesystem, std::string base_dir, std::vector files, std::shared_ptr format); fs::FileSystem* fs_; + std::string base_dir_; std::vector files_; std::shared_ptr format_; }; diff --git a/cpp/src/arrow/dataset/discovery_test.cc b/cpp/src/arrow/dataset/discovery_test.cc index 936d995c1ba79..92cf701aaac7c 100644 --- a/cpp/src/arrow/dataset/discovery_test.cc +++ b/cpp/src/arrow/dataset/discovery_test.cc @@ -20,6 +20,7 @@ #include #include +#include "arrow/dataset/partition.h" #include "arrow/dataset/test_util.h" #include "arrow/filesystem/test_util.h" @@ -31,16 +32,11 @@ class FileSystemDataSourceDiscoveryTest : public TestFileSystemBasedDataSource { void MakeDiscovery(const std::vector& files) { MakeFileSystem(files); ASSERT_OK( - FileSystemDataSourceDiscovery::Make(fs_.get(), files, format_, &discovery_)); - } - - void MakeDiscovery(const std::vector& files, fs::Selector selector) { - MakeFileSystem(files); - ASSERT_OK( - FileSystemDataSourceDiscovery::Make(fs_.get(), selector, format_, &discovery_)); + FileSystemDataSourceDiscovery::Make(fs_.get(), selector_, format_, &discovery_)); } protected: + fs::Selector selector_; std::shared_ptr discovery_; std::shared_ptr format_ = std::make_shared(); }; @@ -53,15 +49,29 @@ TEST_F(FileSystemDataSourceDiscoveryTest, Basic) { } TEST_F(FileSystemDataSourceDiscoveryTest, Selector) { - // This test ensure that the Selector is enforced. - fs::Selector selector; - selector.base_dir = "A"; - MakeDiscovery({fs::File("0"), fs::File("A/a")}, selector); + selector_.base_dir = "A"; + MakeDiscovery({fs::File("0"), fs::File("A/a")}); ASSERT_OK(discovery_->Finish(&source_)); + // "0" doesn't match selector, so it has been dropped: AssertFragmentsAreFromPath(source_->GetFragments(options_), {"A/a"}); } +TEST_F(FileSystemDataSourceDiscoveryTest, Partition) { + selector_.base_dir = "a=ignored/base"; + MakeDiscovery( + {fs::File(selector_.base_dir + "/a=1"), fs::File(selector_.base_dir + "/a=2")}); + + auto partition_scheme = + std::make_shared(schema({field("a", int32())})); + + ASSERT_OK(discovery_->SetPartitionScheme(partition_scheme)); + ASSERT_OK(discovery_->Finish(&source_)); + + AssertFragmentsAreFromPath(source_->GetFragments(options_), + {selector_.base_dir + "/a=1", selector_.base_dir + "/a=2"}); +} + TEST_F(FileSystemDataSourceDiscoveryTest, Inspect) { auto s = schema({field("f64", float64())}); format_ = std::make_shared(s); diff --git a/cpp/src/arrow/dataset/partition.cc b/cpp/src/arrow/dataset/partition.cc index 9141bb64b9996..0fa940c68b1ef 100644 --- a/cpp/src/arrow/dataset/partition.cc +++ b/cpp/src/arrow/dataset/partition.cc @@ -90,11 +90,18 @@ Result> HivePartitionScheme::Parse( Status ApplyPartitionScheme(const PartitionScheme& scheme, std::vector files, PathPartitions* out) { + return ApplyPartitionScheme(scheme, "", std::move(files), out); +} + +Status ApplyPartitionScheme(const PartitionScheme& scheme, const std::string& base_dir, + std::vector files, PathPartitions* out) { for (const auto& file : files) { - const auto& path = file.path(); + if (file.path().substr(0, base_dir.size()) != base_dir) continue; + auto path = file.path().substr(base_dir.size()); + std::shared_ptr partition; RETURN_NOT_OK(scheme.Parse(path, &partition)); - out->emplace(path, std::move(partition)); + out->emplace(std::move(path), std::move(partition)); } return Status::OK(); diff --git a/cpp/src/arrow/dataset/partition.h b/cpp/src/arrow/dataset/partition.h index 83ac5c862dbbf..012ec32914c25 100644 --- a/cpp/src/arrow/dataset/partition.h +++ b/cpp/src/arrow/dataset/partition.h @@ -34,7 +34,8 @@ namespace arrow { namespace fs { struct FileStats; -} +struct Selector; +} // namespace fs namespace dataset { @@ -171,6 +172,9 @@ using PathPartitions = std::unordered_map files, PathPartitions* out); +Status ApplyPartitionScheme(const PartitionScheme& scheme, const std::string& base_dir, + std::vector files, PathPartitions* out); + // TODO(bkietz) use RE2 and named groups to provide RegexpPartitionScheme } // namespace dataset diff --git a/cpp/src/arrow/dbi/hiveserver2/hiveserver2_test.cc b/cpp/src/arrow/dbi/hiveserver2/hiveserver2_test.cc index d1dca97c25808..c19ec1c80a7c8 100644 --- a/cpp/src/arrow/dbi/hiveserver2/hiveserver2_test.cc +++ b/cpp/src/arrow/dbi/hiveserver2/hiveserver2_test.cc @@ -204,7 +204,7 @@ class OperationTest : public HS2ClientTest {}; TEST_F(OperationTest, TestFetch) { CreateTestTable(); InsertIntoTestTable(std::vector({1, 2, 3, 4}), - std::vector({"a", "b", "c", "d"})); + std::vector({"a", "b", "c", "d"})); std::unique_ptr select_op; ASSERT_OK(session_->ExecuteStatement("select * from " + TEST_TBL + " order by int_col", @@ -223,14 +223,14 @@ TEST_F(OperationTest, TestFetch) { std::unique_ptr int_col = results->GetInt32Col(0); std::unique_ptr string_col = results->GetStringCol(1); ASSERT_EQ(int_col->data(), std::vector({1, 2})); - ASSERT_EQ(string_col->data(), std::vector({"a", "b"})); + ASSERT_EQ(string_col->data(), std::vector({"a", "b"})); ASSERT_TRUE(has_more_rows); ASSERT_OK(select_op->Fetch(2, FetchOrientation::NEXT, &results, &has_more_rows)); int_col = results->GetInt32Col(0); string_col = results->GetStringCol(1); ASSERT_EQ(int_col->data(), std::vector({3, 4})); - ASSERT_EQ(string_col->data(), std::vector({"c", "d"})); + ASSERT_EQ(string_col->data(), std::vector({"c", "d"})); ASSERT_OK(select_op->Fetch(2, FetchOrientation::NEXT, &results, &has_more_rows)); int_col = results->GetInt32Col(0); @@ -253,7 +253,7 @@ TEST_F(OperationTest, TestIsNull) { CreateTestTable(); // Insert some NULLs and ensure Column::IsNull() is correct. InsertIntoTestTable(std::vector({1, 2, 3, 4, 5, NULL_INT_VALUE}), - std::vector({"a", "b", "NULL", "d", "NULL", "f"})); + std::vector({"a", "b", "NULL", "d", "NULL", "f"})); std::unique_ptr select_nulls_op; ASSERT_OK(session_->ExecuteStatement("select * from " + TEST_TBL + " order by int_col", @@ -282,7 +282,7 @@ TEST_F(OperationTest, TestIsNull) { TEST_F(OperationTest, TestCancel) { CreateTestTable(); InsertIntoTestTable(std::vector({1, 2, 3, 4}), - std::vector({"a", "b", "c", "d"})); + std::vector({"a", "b", "c", "d"})); std::unique_ptr op; ASSERT_OK(session_->ExecuteStatement("select count(*) from " + TEST_TBL, &op)); diff --git a/cpp/src/arrow/dbi/hiveserver2/thrift_internal.cc b/cpp/src/arrow/dbi/hiveserver2/thrift_internal.cc index 7ac088cfca59a..ef8a3d6ebc5a2 100644 --- a/cpp/src/arrow/dbi/hiveserver2/thrift_internal.cc +++ b/cpp/src/arrow/dbi/hiveserver2/thrift_internal.cc @@ -199,7 +199,7 @@ Status TStatusToStatus(const hs2::TStatus& tstatus) { return Status::OK(ss.str()); } case hs2::TStatusCode::STILL_EXECUTING_STATUS: - return Status::StillExecuting(); + return Status::ExecutionError("Still executing"); case hs2::TStatusCode::ERROR_STATUS: return Status::IOError(tstatus.errorMessage); case hs2::TStatusCode::INVALID_HANDLE_STATUS: diff --git a/cpp/src/arrow/dbi/hiveserver2/util.cc b/cpp/src/arrow/dbi/hiveserver2/util.cc index 8345096937cc7..772be4e38c551 100644 --- a/cpp/src/arrow/dbi/hiveserver2/util.cc +++ b/cpp/src/arrow/dbi/hiveserver2/util.cc @@ -76,7 +76,7 @@ static size_t NumSpaces(int64_t n) { // Returns the max size needed to display a column of integer type. template -static size_t GetIntMaxSize(T* column, const string& column_name) { +static size_t GetIntMaxSize(T* column, const std::string& column_name) { size_t max_size = column_name.size(); for (int i = 0; i < column->length(); ++i) { if (!column->IsNull(i)) { @@ -113,7 +113,7 @@ void Util::PrintResults(const Operation* op, std::ostream& out) { std::vector columns; for (int i = 0; i < static_cast(column_descs.size()); i++) { - const string column_name = column_descs[i].column_name(); + const std::string column_name = column_descs[i].column_name(); switch (column_descs[i].type()->type_id()) { case ColumnType::TypeId::BOOLEAN: { BoolColumn* bool_col = results->GetBoolCol(i).release(); @@ -232,7 +232,7 @@ void Util::PrintResults(const Operation* op, std::ostream& out) { } } - string value_str = value.str(); + std::string value_str = value.str(); out << "| " << value_str << " "; int padding = static_cast(columns[j].max_size - value_str.size()); while (padding > 0) { diff --git a/cpp/src/arrow/filesystem/filesystem.h b/cpp/src/arrow/filesystem/filesystem.h index bc17346894eb9..7084f839a69f2 100644 --- a/cpp/src/arrow/filesystem/filesystem.h +++ b/cpp/src/arrow/filesystem/filesystem.h @@ -91,7 +91,7 @@ struct ARROW_EXPORT FileStats { void set_type(FileType type) { type_ = type; } /// The full file path in the filesystem - std::string path() const { return path_; } + const std::string& path() const { return path_; } void set_path(const std::string& path) { path_ = path; } /// The file base name (component after the last directory separator) diff --git a/cpp/src/arrow/io/compressed_test.cc b/cpp/src/arrow/io/compressed_test.cc index f177722f076dc..4bb3fc74598ac 100644 --- a/cpp/src/arrow/io/compressed_test.cc +++ b/cpp/src/arrow/io/compressed_test.cc @@ -173,6 +173,17 @@ class CompressedInputStreamTest : public ::testing::TestWithParam { + protected: + Compression::type GetCompression() { return GetParam(); } + + std::unique_ptr MakeCodec() { + std::unique_ptr codec; + ABORT_NOT_OK(Codec::Create(GetCompression(), &codec)); + return codec; + } +}; + TEST_P(CompressedInputStreamTest, CompressibleData) { auto codec = MakeCodec(); auto data = MakeCompressibleData(COMPRESSIBLE_DATA_SIZE); @@ -230,44 +241,6 @@ TEST_P(CompressedInputStreamTest, ConcatenatedStreams) { ASSERT_EQ(decompressed, expected); } -// NOTE: Snappy doesn't support streaming decompression - -// NOTE: BZ2 doesn't support one-shot compression - -// NOTE: LZ4 streaming decompression uses the LZ4 framing format, -// which must be tested against a streaming compressor - -INSTANTIATE_TEST_CASE_P(TestGZipInputStream, CompressedInputStreamTest, - ::testing::Values(Compression::GZIP)); - -INSTANTIATE_TEST_CASE_P(TestBrotliInputStream, CompressedInputStreamTest, - ::testing::Values(Compression::BROTLI)); - -#ifdef ARROW_WITH_ZSTD -INSTANTIATE_TEST_CASE_P(TestZSTDInputStream, CompressedInputStreamTest, - ::testing::Values(Compression::ZSTD)); -#endif - -TEST(TestSnappyInputStream, NotImplemented) { - std::unique_ptr codec; - ASSERT_OK(Codec::Create(Compression::SNAPPY, &codec)); - std::shared_ptr stream = std::make_shared(""); - std::shared_ptr compressed_stream; - ASSERT_RAISES(NotImplemented, - CompressedInputStream::Make(codec.get(), stream, &compressed_stream)); -} - -class CompressedOutputStreamTest : public ::testing::TestWithParam { - protected: - Compression::type GetCompression() { return GetParam(); } - - std::unique_ptr MakeCodec() { - std::unique_ptr codec; - ABORT_NOT_OK(Codec::Create(GetCompression(), &codec)); - return codec; - } -}; - TEST_P(CompressedOutputStreamTest, CompressibleData) { auto codec = MakeCodec(); auto data = MakeCompressibleData(COMPRESSIBLE_DATA_SIZE); @@ -284,16 +257,21 @@ TEST_P(CompressedOutputStreamTest, RandomData) { CheckCompressedOutputStream(codec.get(), data, true /* do_flush */); } -INSTANTIATE_TEST_CASE_P(TestGZipOutputStream, CompressedOutputStreamTest, - ::testing::Values(Compression::GZIP)); +// NOTES: +// - Snappy doesn't support streaming decompression +// - BZ2 doesn't support one-shot compression +// - LZ4 streaming decompression uses the LZ4 framing format, which must be tested +// against a streaming compressor -INSTANTIATE_TEST_CASE_P(TestBrotliOutputStream, CompressedOutputStreamTest, - ::testing::Values(Compression::BROTLI)); - -#ifdef ARROW_WITH_ZSTD -INSTANTIATE_TEST_CASE_P(TestZSTDOutputStream, CompressedOutputStreamTest, - ::testing::Values(Compression::ZSTD)); -#endif +#ifdef ARROW_WITH_SNAPPY +TEST(TestSnappyInputStream, NotImplemented) { + std::unique_ptr codec; + ASSERT_OK(Codec::Create(Compression::SNAPPY, &codec)); + std::shared_ptr stream = std::make_shared(""); + std::shared_ptr compressed_stream; + ASSERT_RAISES(NotImplemented, + CompressedInputStream::Make(codec.get(), stream, &compressed_stream)); +} TEST(TestSnappyOutputStream, NotImplemented) { std::unique_ptr codec; @@ -303,6 +281,28 @@ TEST(TestSnappyOutputStream, NotImplemented) { ASSERT_RAISES(NotImplemented, CompressedOutputStream::Make(codec.get(), stream, &compressed_stream)); } +#endif + +#ifdef ARROW_WITH_ZLIB +INSTANTIATE_TEST_CASE_P(TestGZipInputStream, CompressedInputStreamTest, + ::testing::Values(Compression::GZIP)); +INSTANTIATE_TEST_CASE_P(TestGZipOutputStream, CompressedOutputStreamTest, + ::testing::Values(Compression::GZIP)); +#endif + +#ifdef ARROW_WITH_BROTLI +INSTANTIATE_TEST_CASE_P(TestBrotliInputStream, CompressedInputStreamTest, + ::testing::Values(Compression::BROTLI)); +INSTANTIATE_TEST_CASE_P(TestBrotliOutputStream, CompressedOutputStreamTest, + ::testing::Values(Compression::BROTLI)); +#endif + +#ifdef ARROW_WITH_ZSTD +INSTANTIATE_TEST_CASE_P(TestZSTDInputStream, CompressedInputStreamTest, + ::testing::Values(Compression::ZSTD)); +INSTANTIATE_TEST_CASE_P(TestZSTDOutputStream, CompressedOutputStreamTest, + ::testing::Values(Compression::ZSTD)); +#endif } // namespace io } // namespace arrow diff --git a/cpp/src/arrow/util/compression_benchmark.cc b/cpp/src/arrow/util/compression_benchmark.cc index 5700b0dd0e2ea..ce89e96fdb982 100644 --- a/cpp/src/arrow/util/compression_benchmark.cc +++ b/cpp/src/arrow/util/compression_benchmark.cc @@ -183,15 +183,25 @@ static void ReferenceStreamingDecompression( StreamingDecompression(COMPRESSION, data, state); } +#ifdef ARROW_WITH_ZLIB BENCHMARK_TEMPLATE(ReferenceStreamingCompression, Compression::GZIP); -BENCHMARK_TEMPLATE(ReferenceStreamingCompression, Compression::BROTLI); -BENCHMARK_TEMPLATE(ReferenceStreamingCompression, Compression::ZSTD); -BENCHMARK_TEMPLATE(ReferenceStreamingCompression, Compression::LZ4); - BENCHMARK_TEMPLATE(ReferenceStreamingDecompression, Compression::GZIP); +#endif + +#ifdef ARROW_WITH_BROTLI +BENCHMARK_TEMPLATE(ReferenceStreamingCompression, Compression::BROTLI); BENCHMARK_TEMPLATE(ReferenceStreamingDecompression, Compression::BROTLI); +#endif + +#ifdef ARROW_WITH_ZSTD +BENCHMARK_TEMPLATE(ReferenceStreamingCompression, Compression::ZSTD); BENCHMARK_TEMPLATE(ReferenceStreamingDecompression, Compression::ZSTD); +#endif + +#ifdef ARROW_WITH_LZ4 +BENCHMARK_TEMPLATE(ReferenceStreamingCompression, Compression::LZ4); BENCHMARK_TEMPLATE(ReferenceStreamingDecompression, Compression::LZ4); +#endif #endif diff --git a/cpp/src/arrow/util/compression_test.cc b/cpp/src/arrow/util/compression_test.cc index 898cb8a9d96cc..38661955d1e4c 100644 --- a/cpp/src/arrow/util/compression_test.cc +++ b/cpp/src/arrow/util/compression_test.cc @@ -521,21 +521,26 @@ TEST_P(CodecTest, StreamingDecompressorReuse) { CheckStreamingRoundtrip(compressor, decompressor, data); } +#ifdef ARROW_WITH_ZLIB INSTANTIATE_TEST_CASE_P(TestGZip, CodecTest, ::testing::Values(Compression::GZIP)); +#endif +#ifdef ARROW_WITH_SNAPPY INSTANTIATE_TEST_CASE_P(TestSnappy, CodecTest, ::testing::Values(Compression::SNAPPY)); +#endif +#ifdef ARROW_WITH_LZ4 INSTANTIATE_TEST_CASE_P(TestLZ4, CodecTest, ::testing::Values(Compression::LZ4)); +#endif +#ifdef ARROW_WITH_BROTLI INSTANTIATE_TEST_CASE_P(TestBrotli, CodecTest, ::testing::Values(Compression::BROTLI)); +#endif -// bz2 requires a binary installation, there is no ExternalProject #if ARROW_WITH_BZ2 INSTANTIATE_TEST_CASE_P(TestBZ2, CodecTest, ::testing::Values(Compression::BZ2)); #endif -// The ExternalProject for zstd does not build on CMake < 3.7, so we do not -// require it here #ifdef ARROW_WITH_ZSTD INSTANTIATE_TEST_CASE_P(TestZSTD, CodecTest, ::testing::Values(Compression::ZSTD)); #endif diff --git a/dev/README.md b/dev/README.md index f83b75e9b84ff..79a0bb4742d75 100644 --- a/dev/README.md +++ b/dev/README.md @@ -134,6 +134,7 @@ For JavaScript-specific releases, use a different verification script: ```shell bash dev/release/js-verify-release-candidate.sh 0.7.0 0 ``` + # Integration testing Build the following base image used by multiple tests: @@ -145,7 +146,10 @@ docker build -t arrow_integration_xenial_base -f docker_common/Dockerfile.xenial ## HDFS C++ / Python support ```shell -run_docker_compose.sh hdfs_integration +docker-compose build conda-cpp +docker-compose build conda-python +docker-compose build conda-python-hdfs +docker-compose run --rm conda-python-hdfs ``` ## Apache Spark Integration Tests @@ -158,18 +162,19 @@ related unit tests in Spark for Java and Python. Any errors will exit with a non-zero value. To run, use the following command: ```shell -./run_docker_compose.sh spark_integration - +docker-compose build conda-cpp +docker-compose build conda-python +docker-compose build conda-python-spark +docker-compose run --rm conda-python-spark ``` -Alternatively, you can build and run the Docker images seperately. If you -already are building Spark, these commands will map your local Maven repo -to the image and save time by not having to download all dependencies. These -should be run in a directory one level up from your Arrow repository: +If you already are building Spark, these commands will map your local Maven +repo to the image and save time by not having to download all dependencies. +Be aware, that docker write files as root, which can cause problems for maven +on the host. ```shell -docker build -f arrow/dev/spark_integration/Dockerfile -t spark-arrow . -docker run -v $HOME/.m2:/root/.m2 spark-arrow +docker-compose run --rm -v $HOME/.m2:/root/.m2 conda-python-spark ``` NOTE: If the Java API has breaking changes, a patched version of Spark might diff --git a/dev/archery/archery/integration/util.py b/dev/archery/archery/integration/util.py index 4410295b4e528..98eb45929a27e 100644 --- a/dev/archery/archery/integration/util.py +++ b/dev/archery/archery/integration/util.py @@ -37,7 +37,7 @@ def guid(): ARROW_ROOT_DEFAULT = os.environ.get( 'ARROW_ROOT', - os.path.abspath(__file__).rsplit("/", 4)[0] + os.path.abspath(__file__).rsplit("/", 5)[0] ) diff --git a/dev/container/Dockerfile b/dev/container/Dockerfile deleted file mode 100644 index f1397f555eb03..0000000000000 --- a/dev/container/Dockerfile +++ /dev/null @@ -1,48 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -FROM ubuntu:18.04 - -RUN apt-get update && \ - apt-get install -y \ - gcc-8 \ - g++-8 \ - vim \ - git \ - wget \ - make \ - ninja-build - -ENV CC=gcc-8 -ENV CXX=g++-8 - -# Miniconda - Python 3.6, 64-bit, x86, latest -RUN wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O mconda.sh && \ - /bin/bash mconda.sh -b -p miniconda && \ - rm mconda.sh - -ENV PATH="/miniconda/bin:$PATH" - -# create conda env with deps -RUN conda create -y -q -n pyarrow-dev \ - python=3.6 numpy six setuptools cython pandas pytest \ - cmake rapidjson boost-cpp thrift-cpp snappy zlib \ - gflags brotli lz4-c zstd -c conda-forge \ - && conda clean --all - -ADD script ./script -RUN chmod u=rwx ./script/*.sh diff --git a/dev/container/README.md b/dev/container/README.md deleted file mode 100644 index d8636e95d7600..0000000000000 --- a/dev/container/README.md +++ /dev/null @@ -1,76 +0,0 @@ - - -# Apache Arrow development container -* Includes all dependencies for Arrow development -* Builds are incremental, mirrored to local file system -* Resolves [ARROW-2486](https://issues.apache.org/jira/browse/ARROW-2486) - -## Get started - -### [Install Docker](https://docs.docker.com/install/) - -### Acquire image - -``` -$ docker pull quiltdata/arrow -``` - -### Populate host directory -Keep git repos and subsequent build products in a persistent local -directory, `/io`. - -``` -$ mkdir -p io/arrow -$ git clone https://github.com/apache/arrow.git io/arrow -$ mkdir -p io/parquet-cpp -$ git clone https://github.com/apache/parquet-cpp.git io/parquet-cpp -``` -Alternatively, if you wish to use existing git repos, you can nest them -under `/io`. - -### Run container, mount `/io` as volume - -``` -$ docker run \ - --shm-size=2g \ - -v /LOCAL/PATH/TO/io:/io \ - -it quiltdata/arrow -``` - -### Use container -Run scripts to build executables. - -See also [Arrow dev docs](https://arrow.apache.org/docs/python/development.html). - -``` -$ source script/env.sh -$ script/arrow-build.sh -$ script/parquet-build.sh -$ script/pyarrow-build.sh -# run tests -$ cd /io/arrow/python -$ py.test pyarrow -``` - -## Build container - -``` -$ docker build -t USERNAME/arrow . -``` diff --git a/dev/container/script/arrow-build.sh b/dev/container/script/arrow-build.sh deleted file mode 100644 index ab448b8326f08..0000000000000 --- a/dev/container/script/arrow-build.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# See also https://arrow.apache.org/docs/python/development.html -mkdir -p /io/arrow/cpp/build -pushd /io/arrow/cpp/build -cmake -DCMAKE_BUILD_TYPE=$ARROW_BUILD_TYPE \ - -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \ - -DARROW_PYTHON=on \ - -DARROW_PLASMA=on \ - -DARROW_BUILD_TESTS=OFF \ - -DCMAKE_CXX_FLAGS=$CXXFLAGS \ - -GNinja \ - .. -ninja -ninja install -popd - diff --git a/dev/container/script/env.sh b/dev/container/script/env.sh deleted file mode 100644 index d5b670f25cab4..0000000000000 --- a/dev/container/script/env.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# See also https://arrow.apache.org/docs/python/development.html#build-and-test -conda activate pyarrow-dev -export ARROW_BUILD_TYPE=release -export PARQUET_BUILD_TOOLCHAIN=$CONDA_PREFIX -export ARROW_HOME=$CONDA_PREFIX -export PARQUET_HOME=$CONDA_PREFIX -# For newer GCC per https://arrow.apache.org/docs/python/development.html#known-issues -export CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0" -export PYARROW_CXXFLAGS=$CXXFLAGS diff --git a/dev/container/script/parquet-build.sh b/dev/container/script/parquet-build.sh deleted file mode 100644 index f06aa077d9aba..0000000000000 --- a/dev/container/script/parquet-build.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# See also https://arrow.apache.org/docs/python/development.html#build-and-test -mkdir -p /io/parquet-cpp/build -pushd /io/parquet-cpp/build -cmake -DCMAKE_BUILD_TYPE=$ARROW_BUILD_TYPE \ - -DCMAKE_INSTALL_PREFIX=$PARQUET_HOME \ - -DPARQUET_BUILD_BENCHMARKS=off \ - -DPARQUET_BUILD_EXECUTABLES=off \ - -DPARQUET_BUILD_TESTS=off \ - -DCMAKE_CXX_FLAGS=$CXXFLAGS \ - -GNinja \ - .. -ninja -ninja install -popd diff --git a/dev/container/script/pyarrow-build.sh b/dev/container/script/pyarrow-build.sh deleted file mode 100644 index 927ff79804e6a..0000000000000 --- a/dev/container/script/pyarrow-build.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# See also https://arrow.apache.org/docs/python/development.html#build-and-test -cd /io/arrow/python -python setup.py build_ext --build-type=$ARROW_BUILD_TYPE \ - --with-parquet --with-plasma --inplace diff --git a/dev/docker-compose.yml b/dev/docker-compose.yml deleted file mode 100644 index 19fda7823ac25..0000000000000 --- a/dev/docker-compose.yml +++ /dev/null @@ -1,47 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -version: '3' -services: - - impala: - image: cpcloud86/impala:java8-1 - ports: - - "21050" - hostname: impala - - hiveserver2: - links: - - impala - build: - context: hiveserver2 - volumes: - - ../..:/apache-arrow - - gen_apidocs: - build: - context: .. - dockerfile: dev/gen_apidocs/Dockerfile - volumes: - - ..:/arrow - - run_site: - build: - context: run_site - ports: - - "4000:4000" - volumes: - - ../..:/apache-arrow diff --git a/dev/docker_common/Dockerfile.xenial.base b/dev/docker_common/Dockerfile.xenial.base deleted file mode 100644 index 9a5f9c68b4f7d..0000000000000 --- a/dev/docker_common/Dockerfile.xenial.base +++ /dev/null @@ -1,66 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -FROM ubuntu:16.04 - -# Basic OS utilities -RUN apt-get update \ - && apt-get install -y \ - wget \ - git \ - pkg-config \ - build-essential \ - software-properties-common \ - ninja-build \ - && apt-get clean - -ENV PATH="/opt/conda/bin:${PATH}" - -# install conda in /home/ubuntu/miniconda -RUN wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O conda.sh \ - && /bin/bash conda.sh -b -p /opt/conda \ - && rm conda.sh \ - && conda create -y -q -c conda-forge -n pyarrow-dev \ - python=3.6 \ - ipython \ - nomkl \ - numpy \ - six \ - setuptools \ - cython \ - pandas \ - pytest \ - cmake \ - double-conversion \ - rapidjson \ - boost-cpp \ - thrift-cpp \ - snappy \ - zlib \ - gflags \ - glog \ - gtest \ - re2 \ - brotli \ - lz4-c \ - zstd \ - setuptools \ - setuptools_scm \ - && conda clean --all - -ADD docker_common/install_clang_tools_xenial.sh / -RUN /install_clang_tools_xenial.sh diff --git a/dev/docker_common/install_clang_tools_xenial.sh b/dev/docker_common/install_clang_tools_xenial.sh deleted file mode 100755 index e35e3dca3c459..0000000000000 --- a/dev/docker_common/install_clang_tools_xenial.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -wget -O - http://llvm.org/apt/llvm-snapshot.gpg.key|apt-key add - -apt-add-repository -y \ - "deb http://llvm.org/apt/xenial/ llvm-toolchain-xenial-6.0 main" -apt-get update -qq -apt-get install -y -q libclang-6.0-dev clang-6.0 clang-format-6.0 clang-tidy-6.0 diff --git a/dev/gen_apidocs.sh b/dev/gen_apidocs.sh deleted file mode 100755 index 4bba226fb511a..0000000000000 --- a/dev/gen_apidocs.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Pass the service name to run_docker_compse.sh -# Which validates environment and runs the service -exec "$(dirname ${BASH_SOURCE})"/run_docker_compose.sh gen_apidocs diff --git a/dev/gen_apidocs/Dockerfile b/dev/gen_apidocs/Dockerfile deleted file mode 100644 index 3f0fd4ad6bdab..0000000000000 --- a/dev/gen_apidocs/Dockerfile +++ /dev/null @@ -1,63 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -FROM ubuntu:18.04 - -# Basic OS dependencies -# First install JDK 8, then do a second apt call to install maven. -# Otherwise this will install openjdk 8 and openjdk 11 at once. -RUN apt-get update && \ - apt-get install -y openjdk-8-jdk-headless && \ - apt-get install -y \ - gcc-8 \ - g++-8 \ - git \ - wget \ - make \ - ninja-build \ - rsync \ - maven \ - pkg-config \ - gobject-introspection \ - libgirepository1.0-dev \ - gtk-doc-tools \ - libtool \ - autoconf-archive \ - npm - -ENV CC=gcc-8 \ - CXX=g++-8 \ - PATH=/opt/conda/bin:$PATH \ - CONDA_PREFIX=/opt/conda - -ADD ci/docker_install_conda.sh \ - ci/conda_env_cpp.yml \ - ci/conda_env_python.yml \ - /arrow/ci/ -RUN arrow/ci/docker_install_conda.sh && \ - conda install -c conda-forge \ - --file arrow/ci/conda_env_cpp.yml \ - --file arrow/ci/conda_env_python.yml \ - numpydoc \ - sphinx \ - sphinx_rtd_theme \ - doxygen \ - maven && \ - conda clean --all -y - -ADD . /arrow - -CMD arrow/dev/gen_apidocs/create_documents.sh diff --git a/dev/gen_apidocs/create_documents.sh b/dev/gen_apidocs/create_documents.sh deleted file mode 100755 index 9330075cb43b2..0000000000000 --- a/dev/gen_apidocs/create_documents.sh +++ /dev/null @@ -1,97 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -set -ex - -# Set up environment and output directory for C++ libraries - -mkdir -p apidocs-dist -export ARROW_BUILD_TYPE=release -export ARROW_HOME=/apidocs-dist -export PARQUET_HOME=/apidocs-dist -export PKG_CONFIG_PATH=/apidocs-dist/lib/pkgconfig:${PKG_CONFIG_PATH} -# For newer GCC per https://arrow.apache.org/docs/python/development.html#known-issues -export CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0" - -# Make Java documentation -# Override user.home to cache dependencies outside the Docker container -# NB: this assumes that you have arrow-site cloned in the (gitignored) site directory -pushd arrow/java -mvn -Duser.home=`pwd`/.apidocs-m2 -Drat.skip=true -Dcheckstyle.skip=true install site -mkdir -p ../site/asf-site/docs/java/ -rsync -r target/site/apidocs/ ../site/asf-site/docs/java/ -popd - -# Make Javascript documentation -pushd arrow/js -npm install -npm run doc -rsync -r doc/ ../site/asf-site/docs/js -popd - -# Make Python documentation (Depends on C++ ) -# Build Arrow C++ -export LD_LIBRARY_PATH=/apidocs-dist/lib:${CONDA_PREFIX}/lib:${LD_LIBRARY_PATH} -export PKG_CONFIG_PATH=${CONDA_PREFIX}/lib/pkgconfig:${PKG_CONFIG_PATH} -export PKG_CONFIG_PATH=/apidocs-dist/lib/pkgconfig:${PKG_CONFIG_PATH} - -CPP_BUILD_DIR=arrow/cpp/build_apidocs - -mkdir -p $CPP_BUILD_DIR -pushd $CPP_BUILD_DIR -cmake -DCMAKE_BUILD_TYPE=$ARROW_BUILD_TYPE \ - -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \ - -DARROW_PYTHON=ON \ - -DARROW_PLASMA=ON \ - -DARROW_PARQUET=ON \ - -DPARQUET_REQUIRE_ENCRYPTION=ON \ - -DARROW_ORC=ON \ - -DARROW_BUILD_TESTS=OFF \ - -GNinja \ - .. -ninja -ninja install -popd - -# Build c_glib documentation -pushd arrow/c_glib -if [ -f Makefile ]; then - # Ensure updating to prevent auto re-configure - touch configure **/Makefile - make distclean -fi -./autogen.sh -mkdir -p build_apidocs -pushd build_apidocs -../configure \ - --prefix=${ARROW_HOME} \ - --enable-gtk-doc -make -j4 GTK_DOC_V_XREF=": " -mkdir -p ../../site/asf-site/docs/c_glib -rsync -r doc/arrow-glib/html/ ../../site/asf-site/docs/c_glib/arrow-glib -rsync -r doc/parquet-glib/html/ ../../site/asf-site/docs/c_glib/parquet-glib -popd -popd - -# Make C++ documentation -pushd arrow/cpp/apidoc -rm -rf html/* -doxygen Doxyfile -mkdir -p ../../site/asf-site/docs/cpp -rsync -r html/ ../../site/asf-site/docs/cpp -popd diff --git a/dev/hiveserver2/Dockerfile b/dev/hiveserver2/Dockerfile deleted file mode 100644 index 36fa392475a7c..0000000000000 --- a/dev/hiveserver2/Dockerfile +++ /dev/null @@ -1,23 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -FROM arrow_integration_xenial_base - -ADD . /apache-arrow -WORKDIR /apache-arrow - -CMD arrow/dev/hiveserver2/hiveserver2.sh diff --git a/dev/hiveserver2/hiveserver2.sh b/dev/hiveserver2/hiveserver2.sh deleted file mode 100755 index 664b7b86a9498..0000000000000 --- a/dev/hiveserver2/hiveserver2.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Exit on any error -set -e - -# cwd is mounted from host machine to -# and contains both arrow and parquet-cpp - -# Activate conda environment -conda activate pyarrow-dev - -# Arrow build variables -export ARROW_BUILD_TYPE=debug -export ARROW_HOME=$CONDA_PREFIX - -# For newer GCC per https://arrow.apache.org/docs/python/development.html#known-issues -export CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0" -export PYARROW_CXXFLAGS=$CXXFLAGS -export PYARROW_CMAKE_GENERATOR=Ninja - -_PWD=`pwd` -ARROW_CPP_BUILD_DIR=$_PWD/arrow/cpp/hiveserver2-build -DOCKER_COMMON_DIR=$_PWD/arrow/dev/docker_common - -function cleanup { - rm -rf $ARROW_CPP_BUILD_DIR -} - -trap cleanup EXIT - -# Install arrow-cpp -mkdir -p $ARROW_CPP_BUILD_DIR -pushd $ARROW_CPP_BUILD_DIR - -cmake -GNinja \ - -DCMAKE_BUILD_TYPE=$ARROW_BUILD_TYPE \ - -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \ - -DARROW_HIVESERVER2=ON \ - -DARROW_BUILD_TESTS=ON \ - -DCMAKE_CXX_FLAGS=$CXXFLAGS \ - .. -ninja hiveserver2-test - -$DOCKER_COMMON_DIR/wait-for-it.sh impala:21050 -t 300 -s -- echo "impala is up" - -# Run C++ unit tests -export ARROW_HIVESERVER2_TEST_HOST=impala -debug/hiveserver2-test - -popd diff --git a/dev/lint/run_iwyu.sh b/dev/lint/run_iwyu.sh deleted file mode 100755 index a4040b08a10d7..0000000000000 --- a/dev/lint/run_iwyu.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -set -eux - -mkdir -p /build/lint -pushd /build/lint - -cmake -GNinja \ - -DCMAKE_BUILD_TYPE=debug \ - -DARROW_FLIGHT=ON \ - -DARROW_GANDIVA=ON \ - -DARROW_PARQUET=ON \ - -DPARQUET_REQUIRE_ENCRYPTION=ON \ - -DARROW_PYTHON=ON \ - -DCMAKE_CXX_FLAGS='-D_GLIBCXX_USE_CXX11_ABI=0' \ - -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ - /arrow/cpp - -popd - -export IWYU_COMPILATION_DATABASE_PATH=/build/lint -/arrow/cpp/build-support/iwyu/iwyu.sh all diff --git a/dev/release/02-source.sh b/dev/release/02-source.sh index f70503a9f5c70..dee1d75c51c44 100755 --- a/dev/release/02-source.sh +++ b/dev/release/02-source.sh @@ -70,11 +70,11 @@ if [ ${SOURCE_GLIB} -gt 0 ]; then git archive ${release_hash} --prefix ${archive_name}/) \ > "${SOURCE_TOP_DIR}/${archive_name}.tar" c_glib_including_configure_tar_gz=c_glib.tar.gz - "${SOURCE_TOP_DIR}/dev/run_docker_compose.sh" \ - release-source \ - /arrow/dev/release/source/build.sh \ - ${archive_name} \ - ${c_glib_including_configure_tar_gz} + docker build -t arrow-release-source "${SOURCE_TOP_DIR}/dev/release/source" + docker run \ + -v "${SOURCE_TOP_DIR}":/arrow:delegated \ + arrow-release-source \ + /arrow/dev/release/source/build.sh ${archive_name} ${c_glib_including_configure_tar_gz} rm -f "${SOURCE_TOP_DIR}/${archive_name}.tar" rm -rf ${tag}/c_glib tar xf "${SOURCE_TOP_DIR}/${c_glib_including_configure_tar_gz}" -C ${tag} diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt index 7a689d11a3653..38aa9187f53e5 100644 --- a/dev/release/rat_exclude_files.txt +++ b/dev/release/rat_exclude_files.txt @@ -139,7 +139,7 @@ dev/tasks/linux-packages/debian/source/format dev/tasks/linux-packages/debian/watch dev/tasks/conda-recipes/* docs/requirements.txt -integration/spark/ARROW-6429.patch +ci/etc/ARROW-6429.patch go/arrow/go.sum go/arrow/Gopkg.lock go/arrow/internal/cpu/* diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index c5d5a93162fe1..5c817f5d89972 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -127,12 +127,12 @@ test_binary() { } test_apt() { - for target in debian-stretch \ - debian-buster \ - ubuntu-xenial \ - ubuntu-bionic \ - ubuntu-disco; do - if ! "${SOURCE_DIR}/../run_docker_compose.sh" \ + for target in "debian:stretch" \ + "debian:buster" \ + "ubuntu:xenial" \ + "ubuntu:bionic" \ + "ubuntu:disco"; do + if ! docker run -v "${SOURCE_DIR}"/../..:/arrow:delegated \ "${target}" \ /arrow/dev/release/verify-apt.sh \ "${VERSION}" \ @@ -145,9 +145,9 @@ test_apt() { } test_yum() { - for target in centos-6 \ - centos-7; do - if ! "${SOURCE_DIR}/../run_docker_compose.sh" \ + for target in "centos:6" \ + "centos:7"; do + if ! docker run -v "${SOURCE_DIR}"/../..:/arrow:delegated \ "${target}" \ /arrow/dev/release/verify-yum.sh \ "${VERSION}" \ diff --git a/dev/run_docker_compose.sh b/dev/run_docker_compose.sh deleted file mode 100755 index 6c2f08ec25ae4..0000000000000 --- a/dev/run_docker_compose.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -set -eu - -PWD="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -ARROW_SRC=$(realpath "${PWD}/..") - -: ${DOCKER_COMPOSE:="docker-compose"} -: ${DOCKER_COMPOSE_CONF:="${ARROW_SRC}/docker-compose.yml"} - -docker_compose() { - ${DOCKER_COMPOSE} -f ${DOCKER_COMPOSE_CONF} "$@" -} - -main() { - docker_compose build "$1" - docker_compose run --rm "$@" -} - -if [ $# -lt 1 ]; then - echo "Usage: $0 " >&2 - exit 1 -fi - -main "$@" diff --git a/dev/run_site/Dockerfile b/dev/run_site/Dockerfile deleted file mode 100644 index cea4ceb72e476..0000000000000 --- a/dev/run_site/Dockerfile +++ /dev/null @@ -1,34 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -FROM ubuntu:14.04 -ADD . /apache-arrow -WORKDIR /apache-arrow -# Prerequsites for apt-add-repository -RUN apt-get update && apt-get install -y \ - software-properties-common python-software-properties -# Set up Ruby repository -RUN apt-add-repository ppa:brightbox/ruby-ng -# The publication tools -RUN apt-get update; apt-get install -y \ - apt-transport-https \ - ruby2.2-dev \ - ruby2.2 \ - zlib1g-dev \ - make \ - gcc -RUN gem install jekyll bundler -CMD arrow/dev/run_site/run_site.sh diff --git a/dev/run_site/run_site.sh b/dev/run_site/run_site.sh deleted file mode 100755 index 3f03aea6cdae7..0000000000000 --- a/dev/run_site/run_site.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# NB: this assumes that you have arrow-site cloned in the (gitignored) site directory -cd /apache-arrow/arrow/site -export GEM_HOME=$(pwd) -export BUNDLE_PATH=$(pwd) -export HOME=$(pwd) -bundle install --path . -HOST_IP=$(grep $(hostname) /etc/hosts | cut -f 1) -pushd asf-site -bundle exec jekyll serve -c ../_config.yml -s . --host ${HOST_IP} --port 4000 -popd diff --git a/dev/tasks/python-wheels/travis.linux.yml b/dev/tasks/python-wheels/travis.linux.yml index 13121a484d686..932bbf0cb218c 100644 --- a/dev/tasks/python-wheels/travis.linux.yml +++ b/dev/tasks/python-wheels/travis.linux.yml @@ -51,7 +51,7 @@ script: # don't attempt to build it, because the base image by pypa is updated # regularly and would cause undeterministic builds - - BUILD_IMAGE=python-{{ wheel_tag }} + - BUILD_IMAGE=centos-python-{{ wheel_tag }} - docker-compose pull $BUILD_IMAGE - docker-compose run -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 86459b412f0cc..3bf858061dea6 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -81,59 +81,118 @@ groups: ############################# Testing tasks ################################# docker: - - docker-r - - docker-r-conda - - docker-r-sanitizer - - docker-rust - - docker-cpp - - docker-cpp-alpine - - docker-cpp-cmake32 - - docker-cpp-release - - docker-cpp-static-only - - docker-c_glib - - docker-go - - docker-python-2.7 - - docker-python-3.6 - - docker-python-3.7 - - docker-python-2.7-alpine - - docker-python-3.6-alpine - - docker-python-2.7-nopandas - - docker-python-3.6-nopandas - - docker-java - - docker-js - - docker-docs - - docker-lint - - docker-iwyu - - docker-pandas-master - - docker-dask-integration - - docker-hdfs-integration - - docker-spark-integration - - docker-turbodbc-integration - - integration: - - docker-pandas-master - - docker-dask-integration - - docker-hdfs-integration - - docker-spark-integration - - docker-turbodbc-integration - - cpp-python: - - docker-cpp - - docker-cpp-alpine - - docker-cpp-cmake32 - - docker-python-2.7 - - docker-python-2.7-alpine - - docker-python-2.7-nopandas - - docker-python-3.6 - - docker-python-3.6-alpine - - docker-python-3.6-nopandas - - docker-python-3.7 + - test-conda-cpp + - test-debian-10-cpp + - test-ubuntu-14.04-cpp + - test-ubuntu-16.04-cpp + - test-ubuntu-18.04-cpp + - test-fedora-29-cpp + - test-ubuntu-18.04-cpp-release + - test-ubuntu-18.04-cpp-static + - test-ubuntu-18.04-cpp-cmake32 + - test-debian-c-glib + - test-ubuntu-c-glib + - test-debian-ruby + - test-ubuntu-ruby + - test-conda-python-2.7 + - test-conda-python-3.6 + - test-conda-python-3.7 + - test-debian-10-python-3 + - test-ubuntu-16.04-python-3 + - test-ubuntu-18.04-python-3 + - test-fedora-29-python-3 + - test-ubuntu-18.04-r-3.6 + - test-conda-r-3.6 + - test-ubuntu-18.04-r-sanitizer + - test-debian-10-go-1.12 + - test-debian-10-rust-nightly-2019-09-25 + - test-ubuntu-18.04-docs + - test-ubuntu-fuzzit + - test-conda-python-2.7-pandas-latest + - test-conda-python-2.7-pandas-master + - test-conda-python-3.7-pandas-latest + - test-conda-python-3.7-pandas-master + - test-conda-python-3.7-dask-latest + - test-conda-python-3.7-dask-master + - test-conda-python-3.7-turbodbc-latest + - test-conda-python-3.7-turbodbc-master + - test-conda-python-3.7-hdfs-2.9.2 + - test-conda-python-3.7-spark-master + # - test-conda-cpp-hiveserver2 + + test: + - test-conda-cpp + - test-debian-10-cpp + - test-ubuntu-14.04-cpp + - test-ubuntu-16.04-cpp + - test-ubuntu-18.04-cpp + - test-fedora-29-cpp + - test-ubuntu-18.04-cpp-release + - test-ubuntu-18.04-cpp-static + - test-ubuntu-18.04-cpp-cmake32 + - test-debian-c-glib + - test-ubuntu-c-glib + - test-debian-ruby + - test-ubuntu-ruby + - test-conda-python-2.7 + - test-conda-python-3.6 + - test-conda-python-3.7 + - test-debian-10-python-3 + - test-ubuntu-16.04-python-3 + - test-ubuntu-18.04-python-3 + - test-fedora-29-python-3 + - test-ubuntu-18.04-r-3.6 + - test-conda-r-3.6 + - test-ubuntu-18.04-r-sanitizer + - test-debian-10-go-1.12 + - test-debian-10-rust-nightly-2019-09-25 + - test-ubuntu-18.04-docs + + cpp: + - test-conda-cpp + - test-debian-10-cpp + - test-ubuntu-14.04-cpp + - test-ubuntu-16.04-cpp + - test-ubuntu-18.04-cpp + - test-fedora-29-cpp + - test-ubuntu-18.04-cpp-release + - test-ubuntu-18.04-cpp-static + - test-ubuntu-18.04-cpp-cmake32 + + c-glib: + - test-debian-c-glib + - test-ubuntu-c-glib + + python: + - test-conda-python-2.7 + - test-conda-python-3.6 + - test-conda-python-3.7 + - test-debian-10-python-3 + - test-ubuntu-16.04-python-3 + - test-ubuntu-18.04-python-3 + - test-fedora-29-python-3 r: - - docker-r - - docker-r-conda - - docker-r-sanitizer - - macos-r-autobrew + - test-ubuntu-18.04-r-3.6 + - test-conda-r-3.6 + - test-ubuntu-18.04-r-sanitizer + + ruby: + - test-debian-ruby + - test-ubuntu-ruby + + integration: + - test-conda-python-2.7-pandas-latest + - test-conda-python-2.7-pandas-master + - test-conda-python-3.7-pandas-latest + - test-conda-python-3.7-pandas-master + - test-conda-python-3.7-dask-latest + - test-conda-python-3.7-dask-master + - test-conda-python-3.7-turbodbc-latest + - test-conda-python-3.7-turbodbc-master + - test-conda-python-3.7-hdfs-2.9.2 + - test-conda-python-3.7-spark-master + # - test-conda-cpp-hiveserver2 ######################## Tasks to run regularly ############################# @@ -173,32 +232,44 @@ groups: - homebrew-cpp - gandiva-jar-trusty - gandiva-jar-osx - - docker-r - - docker-r-conda - # - docker-r-sanitizer # ARROW-6957 - macos-r-autobrew - - docker-rust - - docker-cpp - - docker-cpp-cmake32 - - docker-cpp-release - - docker-cpp-static-only - - docker-c_glib - - docker-go - - docker-python-2.7 - - docker-python-3.6 - - docker-python-3.7 - - docker-python-2.7-nopandas - - docker-python-3.6-nopandas - - docker-java - - docker-js - - docker-docs - - docker-lint - - docker-iwyu - - docker-pandas-master - - docker-dask-integration - - docker-hdfs-integration - - docker-spark-integration - - docker-turbodbc-integration + - test-conda-cpp + - test-debian-10-cpp + - test-ubuntu-14.04-cpp + - test-ubuntu-16.04-cpp + - test-ubuntu-18.04-cpp + - test-fedora-29-cpp + - test-ubuntu-18.04-cpp-release + - test-ubuntu-18.04-cpp-static + - test-ubuntu-18.04-cpp-cmake32 + - test-debian-c-glib + - test-ubuntu-c-glib + - test-debian-ruby + - test-ubuntu-ruby + - test-conda-python-2.7 + - test-conda-python-3.6 + - test-conda-python-3.7 + - test-debian-10-python-3 + - test-ubuntu-16.04-python-3 + - test-ubuntu-18.04-python-3 + - test-fedora-29-python-3 + - test-ubuntu-18.04-r-3.6 + - test-conda-r-3.6 + - test-ubuntu-18.04-r-sanitizer + - test-debian-10-go-1.12 + - test-debian-10-rust-nightly-2019-09-25 + - test-ubuntu-18.04-docs + - test-ubuntu-fuzzit + - test-conda-python-2.7-pandas-latest + - test-conda-python-2.7-pandas-master + - test-conda-python-3.7-pandas-latest + - test-conda-python-3.7-pandas-master + - test-conda-python-3.7-dask-latest + - test-conda-python-3.7-dask-master + - test-conda-python-3.7-turbodbc-latest + - test-conda-python-3.7-turbodbc-master + - test-conda-python-3.7-hdfs-2.9.2 + - test-conda-python-3.7-spark-master tasks: # arbitrary_task_name: @@ -474,6 +545,15 @@ tasks: artifacts: - pyarrow-{no_rc_version}-cp37-cp37m-macosx_10_6_intel.whl + wheel-osx-cp37m-azure: + ci: azure + platform: osx + template: python-wheels/azure.osx.yml + params: + python_version: 3.7 + artifacts: + - pyarrow-{no_rc_version}-cp37-cp37m-macosx_10_6_intel.whl + ############################## Wheel Windows ################################ wheel-win-cp36m: @@ -1374,413 +1454,505 @@ tasks: ############################## Docker tests ################################# - docker-r: + test-conda-cpp: ci: circle platform: linux template: docker-tests/circle.linux.yml params: commands: - - docker-compose pull --ignore-pull-failures cpp - - docker-compose pull --ignore-pull-failures r - - docker-compose build cpp - - docker-compose build r - - docker-compose run r - - docker-compose push --ignore-push-failures r - - docker-r-conda: + - docker-compose pull --ignore-pull-failures conda-cpp + - docker-compose build conda-cpp + - docker-compose run conda-cpp + - docker-compose push --ignore-push-failures conda-cpp + + test-debian-10-cpp: ci: circle platform: linux template: docker-tests/circle.linux.yml params: commands: - - docker-compose pull --ignore-pull-failures cpp - - docker-compose pull --ignore-pull-failures r-conda - - docker-compose build cpp - - docker-compose build r-conda - - docker-compose run r-conda - - docker-compose push --ignore-push-failures r-conda - - docker-r-sanitizer: + - export DEBIAN=10 + - docker-compose pull --ignore-pull-failures debian-cpp + - docker-compose build debian-cpp + - docker-compose run debian-cpp + - docker-compose push --ignore-push-failures debian-cpp + + test-ubuntu-14.04-cpp: ci: circle platform: linux template: docker-tests/circle.linux.yml params: commands: - - docker-compose pull --ignore-pull-failures r-sanitizer - - docker-compose build r-sanitizer - - docker-compose run r-sanitizer - - docker-compose push --ignore-push-failures r-sanitizer + - export UBUNTU=14.04 + - docker-compose pull --ignore-pull-failures ubuntu-cpp + - docker-compose build ubuntu-cpp + - docker-compose run ubuntu-cpp + - docker-compose push --ignore-push-failures ubuntu-cpp - docker-rust: + test-ubuntu-16.04-cpp: ci: circle platform: linux template: docker-tests/circle.linux.yml params: commands: - - docker-compose pull --ignore-pull-failures rust - - docker-compose build rust - - docker-compose run rust - - docker-compose push --ignore-push-failures rust + - export UBUNTU=16.04 + - docker-compose pull --ignore-pull-failures ubuntu-cpp + - docker-compose build ubuntu-cpp + - docker-compose run ubuntu-cpp + - docker-compose push --ignore-push-failures ubuntu-cpp - docker-cpp: + test-ubuntu-18.04-cpp: ci: circle platform: linux template: docker-tests/circle.linux.yml params: commands: - - docker-compose pull --ignore-pull-failures cpp - - docker-compose build cpp - - docker-compose run cpp - - docker-compose push --ignore-push-failures cpp + - export UBUNTU=18.04 + - docker-compose pull --ignore-pull-failures ubuntu-cpp + - docker-compose build ubuntu-cpp + - docker-compose run ubuntu-cpp + - docker-compose push --ignore-push-failures ubuntu-cpp - docker-cpp-release: + test-fedora-29-cpp: ci: circle platform: linux template: docker-tests/circle.linux.yml params: commands: - - docker-compose pull --ignore-pull-failures cpp-release - - docker-compose build cpp-release - - docker-compose run cpp-release - - docker-compose push --ignore-push-failures cpp-release + - export FEDORA=29 + - docker-compose pull --ignore-pull-failures fedora-cpp + - docker-compose build fedora-cpp + - docker-compose run fedora-cpp + - docker-compose push --ignore-push-failures fedora-cpp - docker-cpp-alpine: + test-ubuntu-18.04-cpp-release: ci: circle platform: linux template: docker-tests/circle.linux.yml params: commands: - - docker-compose pull --ignore-pull-failures cpp-alpine - - docker-compose build cpp-alpine - - docker-compose run cpp-alpine - - docker-compose push --ignore-push-failures cpp-alpine + - docker-compose pull --ignore-pull-failures ubuntu-cpp + - docker-compose build ubuntu-cpp + - docker-compose run -e ARROW_BUILD_TYPE=release ubuntu-cpp + - docker-compose push --ignore-push-failures ubuntu-cpp - docker-cpp-cmake32: + test-ubuntu-18.04-cpp-static: ci: circle platform: linux template: docker-tests/circle.linux.yml params: commands: - - docker-compose pull --ignore-pull-failures cpp-cmake32 - - docker-compose build cpp-cmake32 - - docker-compose run cpp-cmake32 - - docker-compose push --ignore-push-failures cpp-cmake32 + - docker-compose pull --ignore-pull-failures ubuntu-cpp + - docker-compose build ubuntu-cpp + - docker-compose run -e ARROW_BUILD_SHARED=OFF -e ARROW_BUILD_STATIC=ON -e ARROW_TEST_LINKAGE=static ubuntu-cpp + - docker-compose push --ignore-push-failures ubuntu-cpp - docker-cpp-fuzzit: + test-ubuntu-18.04-cpp-cmake32: ci: circle platform: linux template: docker-tests/circle.linux.yml params: commands: - - docker-compose pull --ignore-pull-failures cpp - - docker-compose pull --ignore-pull-failures fuzzit - - docker-compose build cpp - - docker-compose build fuzzit - - docker-compose run fuzzit - - docker-compose push --ignore-push-failures fuzzit - - docker-cpp-static-only: + - docker-compose pull --ignore-pull-failures ubuntu-cpp-cmake32 + - docker-compose build ubuntu-cpp-cmake32 + - docker-compose run ubuntu-cpp-cmake32 + - docker-compose push --ignore-push-failures ubuntu-cpp-cmake32 + + test-debian-c-glib: ci: circle platform: linux template: docker-tests/circle.linux.yml params: commands: - - docker-compose pull --ignore-pull-failures cpp - - docker-compose pull --ignore-pull-failures cpp-static-only - - docker-compose build cpp - - docker-compose run cpp-static-only - - docker-compose push --ignore-push-failures cpp-static-only - - docker-c_glib: + - docker-compose pull --ignore-pull-failures debian-cpp + - docker-compose pull --ignore-pull-failures debian-c-glib + - docker-compose build debian-cpp + - docker-compose build debian-c-glib + - docker-compose run debian-c-glib + - docker-compose push --ignore-push-failures debian-c-glib + + test-ubuntu-c-glib: ci: circle platform: linux template: docker-tests/circle.linux.yml params: commands: - - docker-compose pull --ignore-pull-failures cpp - - docker-compose pull --ignore-pull-failures c_glib - - docker-compose build cpp - - docker-compose build c_glib - - docker-compose run c_glib - - docker-compose push --ignore-push-failures c_glib - - docker-go: + - docker-compose pull --ignore-pull-failures ubuntu-cpp + - docker-compose pull --ignore-pull-failures ubuntu-c-glib + - docker-compose build ubuntu-cpp + - docker-compose build ubuntu-c-glib + - docker-compose run ubuntu-c-glib + - docker-compose push --ignore-push-failures ubuntu-c-glib + + test-debian-ruby: ci: circle platform: linux template: docker-tests/circle.linux.yml params: commands: - - docker-compose pull --ignore-pull-failures go - - docker-compose build go - - docker-compose run go - - docker-compose push --ignore-push-failures go - - docker-js: + - docker-compose pull --ignore-pull-failures debian-cpp + - docker-compose pull --ignore-pull-failures debian-c-glib + - docker-compose pull --ignore-pull-failures debian-ruby + - docker-compose build debian-cpp + - docker-compose build debian-c-glib + - docker-compose build debian-ruby + - docker-compose run debian-ruby + - docker-compose push --ignore-push-failures debian-ruby + + test-ubuntu-ruby: ci: circle platform: linux template: docker-tests/circle.linux.yml params: commands: - - docker-compose pull --ignore-pull-failures js - - docker-compose build js - - docker-compose run js - - docker-compose push --ignore-push-failures js - - docker-java: + - docker-compose pull --ignore-pull-failures ubuntu-cpp + - docker-compose pull --ignore-pull-failures ubuntu-c-glib + - docker-compose pull --ignore-pull-failures ubuntu-ruby + - docker-compose build ubuntu-cpp + - docker-compose build ubuntu-c-glib + - docker-compose build ubuntu-ruby + - docker-compose run ubuntu-ruby + - docker-compose push --ignore-push-failures ubuntu-ruby + + test-conda-python-2.7: ci: circle platform: linux template: docker-tests/circle.linux.yml params: commands: - - docker-compose pull --ignore-pull-failures java - - docker-compose build java - - docker-compose run java - - docker-compose push --ignore-push-failures java - - docker-python-2.7: + - export PYTHON=2.7 + - docker-compose pull --ignore-pull-failures conda-cpp + - docker-compose pull --ignore-pull-failures conda-python + - docker-compose build conda-cpp + - docker-compose build conda-python + - docker-compose run conda-python + - docker-compose push --ignore-push-failures conda-python + + test-conda-python-3.6: + ci: circle + platform: linux + template: docker-tests/circle.linux.yml + params: + commands: + - export PYTHON=3.6 + - docker-compose pull --ignore-pull-failures conda-cpp + - docker-compose pull --ignore-pull-failures conda-python + - docker-compose build conda-cpp + - docker-compose build conda-python + - docker-compose run conda-python + - docker-compose push --ignore-push-failures conda-python + + test-conda-python-3.7: + ci: circle + platform: linux + template: docker-tests/circle.linux.yml + params: + commands: + - export PYTHON=3.7 + - docker-compose pull --ignore-pull-failures conda-cpp + - docker-compose pull --ignore-pull-failures conda-python + - docker-compose build conda-cpp + - docker-compose build conda-python + - docker-compose run conda-python + - docker-compose push --ignore-push-failures conda-python + + test-debian-10-python-3: + ci: circle + platform: linux + template: docker-tests/circle.linux.yml + params: + commands: + - export DEBIAN=10 + - docker-compose pull --ignore-pull-failures debian-cpp + - docker-compose pull --ignore-pull-failures debian-python + - docker-compose build debian-cpp + - docker-compose build debian-python + - docker-compose run debian-python + - docker-compose push --ignore-push-failures debian-python + + test-ubuntu-16.04-python-3: ci: circle platform: linux template: docker-tests/circle.linux.yml params: - environment: - PYTHON_VERSION: 2.7 commands: - - docker-compose pull --ignore-pull-failures cpp - - docker-compose pull --ignore-pull-failures python - - docker-compose build cpp - - docker-compose build python - - docker-compose run python - - docker-compose push --ignore-push-failures python - - docker-python-3.6: + - export UBUNTU=16.04 + - docker-compose pull --ignore-pull-failures ubuntu-cpp + - docker-compose pull --ignore-pull-failures ubuntu-python + - docker-compose build ubuntu-cpp + - docker-compose build ubuntu-python + - docker-compose run ubuntu-python + - docker-compose push --ignore-push-failures ubuntu-python + + test-ubuntu-18.04-python-3: ci: circle platform: linux template: docker-tests/circle.linux.yml params: - environment: - PYTHON_VERSION: 3.6 commands: - - docker-compose pull --ignore-pull-failures cpp - - docker-compose pull --ignore-pull-failures python - - docker-compose build cpp - - docker-compose build python - - docker-compose run python - - docker-compose push --ignore-push-failures python - - docker-python-3.7: + - export UBUNTU=18.04 + - docker-compose pull --ignore-pull-failures ubuntu-cpp + - docker-compose pull --ignore-pull-failures ubuntu-python + - docker-compose build ubuntu-cpp + - docker-compose build ubuntu-python + - docker-compose run ubuntu-python + - docker-compose push --ignore-push-failures ubuntu-python + + test-fedora-29-python-3: ci: circle platform: linux template: docker-tests/circle.linux.yml params: - environment: - PYTHON_VERSION: 3.7 commands: - - docker-compose pull --ignore-pull-failures cpp - - docker-compose pull --ignore-pull-failures python - - docker-compose build cpp - - docker-compose build python - - docker-compose run python - - docker-compose push --ignore-push-failures python - - docker-python-2.7-alpine: + - export FEDORA=29 + - docker-compose pull --ignore-pull-failures fedora-cpp + - docker-compose pull --ignore-pull-failures fedora-python + - docker-compose build fedora-cpp + - docker-compose build fedora-python + - docker-compose run fedora-python + - docker-compose push --ignore-push-failures fedora-python + + test-ubuntu-18.04-r-3.6: ci: circle platform: linux template: docker-tests/circle.linux.yml params: - environment: - PYTHON_VERSION: 2.7 commands: - - docker-compose pull --ignore-pull-failures cpp-alpine - - docker-compose pull --ignore-pull-failures python-alpine - - docker-compose build cpp-alpine - - docker-compose build python-alpine - - docker-compose run python-alpine - - docker-compose push --ignore-push-failures python-alpine - - docker-python-3.6-alpine: + - export UBUNTU=18.04 R=3.6 + - docker-compose pull --ignore-pull-failures ubuntu-cpp + - docker-compose pull --ignore-pull-failures ubuntu-r + - docker-compose build ubuntu-cpp + - docker-compose build ubuntu-r + - docker-compose run ubuntu-r + - docker-compose push --ignore-push-failures ubuntu-r + + test-conda-r-3.6: ci: circle platform: linux template: docker-tests/circle.linux.yml params: - environment: - PYTHON_VERSION: 3.6 commands: - - docker-compose pull --ignore-pull-failures cpp-alpine - - docker-compose pull --ignore-pull-failures python-alpine - - docker-compose build cpp-alpine - - docker-compose build python-alpine - - docker-compose run python-alpine - - docker-compose push --ignore-push-failures python-alpine - - docker-python-2.7-nopandas: + - export R=3.6 + - docker-compose pull --ignore-pull-failures conda-cpp + - docker-compose pull --ignore-pull-failures conda-r + - docker-compose build conda-cpp + - docker-compose build conda-r + - docker-compose run conda-r + - docker-compose push --ignore-push-failures conda-r + + test-ubuntu-18.04-r-sanitizer: ci: circle platform: linux template: docker-tests/circle.linux.yml params: - environment: - PYTHON_VERSION: 2.7 commands: - - docker-compose pull --ignore-pull-failures cpp - - docker-compose pull --ignore-pull-failures python - - docker-compose pull --ignore-pull-failures python-nopandas - - docker-compose build cpp - - docker-compose build python - - docker-compose build python-nopandas - - docker-compose run python-nopandas - - docker-compose push --ignore-push-failures python-nopandas - - docker-python-3.6-nopandas: + - docker-compose pull --ignore-pull-failures ubuntu-r-sanitizer + - docker-compose build ubuntu-r-sanitizer + - docker-compose run ubuntu-r-sanitizer + - docker-compose push --ignore-push-failures ubuntu-r-sanitizer + + test-debian-10-go-1.12: ci: circle platform: linux template: docker-tests/circle.linux.yml params: - environment: - PYTHON_VERSION: 3.6 commands: - - docker-compose pull --ignore-pull-failures cpp - - docker-compose pull --ignore-pull-failures python - - docker-compose pull --ignore-pull-failures python-nopandas - - docker-compose build cpp - - docker-compose build python - - docker-compose build python-nopandas - - docker-compose run python-nopandas - - docker-compose push --ignore-push-failures python-nopandas - - ###################### Documentation building tests ######################### - - docker-docs: + - export GO=1.12 + - docker-compose pull --ignore-pull-failures debian-go + - docker-compose build debian-go + - docker-compose run debian-go + - docker-compose push --ignore-push-failures debian-go + + test-debian-10-rust-nightly-2019-09-25: ci: circle platform: linux template: docker-tests/circle.linux.yml params: - environment: - PYTHON_VERSION: 3.6 commands: - - docker-compose pull --ignore-pull-failures cpp - - docker-compose pull --ignore-pull-failures python - - docker-compose pull --ignore-pull-failures docs - - docker-compose build cpp - - docker-compose build python - - docker-compose build docs - - docker-compose run docs - - docker-compose push --ignore-push-failures docs - - ############################## Linter tests ################################# - - docker-lint: + - export RUST=nightly-2019-09-25 + - docker-compose pull --ignore-pull-failures debian-rust + - docker-compose build debian-rust + - docker-compose run debian-rust + - docker-compose push --ignore-push-failures debian-rust + + test-ubuntu-18.04-docs: ci: circle platform: linux template: docker-tests/circle.linux.yml params: - environment: - PYTHON_VERSION: 3.6 commands: - - docker-compose pull --ignore-pull-failures cpp - - docker-compose pull --ignore-pull-failures python - - docker-compose pull --ignore-pull-failures lint - - docker-compose build cpp - - docker-compose build python - - docker-compose build lint - - docker-compose run lint - - docker-compose push --ignore-push-failures lint - - docker-iwyu: + - export UBUNTU=18.04 + - docker-compose pull --ignore-pull-failures ubuntu-cpp + - docker-compose pull --ignore-pull-failures ubuntu-python + - docker-compose pull --ignore-pull-failures ubuntu-docs + - docker-compose build ubuntu-cpp + - docker-compose build ubuntu-python + - docker-compose build ubuntu-docs + - docker-compose run ubuntu-docs + - docker-compose push --ignore-push-failures ubuntu-docs + + test-ubuntu-fuzzit: ci: circle platform: linux template: docker-tests/circle.linux.yml params: - environment: - PYTHON_VERSION: 3.6 commands: - - docker-compose pull --ignore-pull-failures cpp - - docker-compose pull --ignore-pull-failures python - - docker-compose pull --ignore-pull-failures lint - - docker-compose build cpp - - docker-compose build python - - docker-compose build lint - - docker-compose run iwyu + - docker-compose pull --ignore-pull-failures ubuntu-cpp + - docker-compose pull --ignore-pull-failures ubuntu-fuzzit + - docker-compose build ubuntu-cpp + - docker-compose build ubuntu-fuzzit + - docker-compose run ubuntu-fuzzit + - docker-compose push --ignore-push-failures ubuntu-fuzzit ############################## Integration tests ############################ - docker-dask-integration: + test-conda-python-2.7-pandas-latest: + ci: circle + platform: linux + template: docker-tests/circle.linux.yml + params: + commands: + - export PYTHON=2.7 PANDAS=latest + - docker-compose pull --ignore-pull-failures conda-cpp + - docker-compose pull --ignore-pull-failures conda-python + - docker-compose build conda-cpp + - docker-compose build conda-python + - docker-compose build --no-cache conda-python-pandas + - docker-compose run conda-python-pandas + + test-conda-python-2.7-pandas-master: + ci: circle + platform: linux + template: docker-tests/circle.linux.yml + params: + commands: + - export PYTHON=2.7 PANDAS=master + - docker-compose pull --ignore-pull-failures conda-cpp + - docker-compose pull --ignore-pull-failures conda-python + - docker-compose build conda-cpp + - docker-compose build conda-python + - docker-compose build --no-cache conda-python-pandas + - docker-compose run conda-python-pandas + + test-conda-python-3.7-pandas-latest: + ci: circle + platform: linux + template: docker-tests/circle.linux.yml + params: + commands: + - export PYTHON=3.7 PANDAS=latest + - docker-compose pull --ignore-pull-failures conda-cpp + - docker-compose pull --ignore-pull-failures conda-python + - docker-compose build conda-cpp + - docker-compose build conda-python + - docker-compose build --no-cache conda-python-pandas + - docker-compose run conda-python-pandas + + test-conda-python-3.7-pandas-master: + ci: circle + platform: linux + template: docker-tests/circle.linux.yml + params: + commands: + - export PYTHON=3.7 PANDAS=master + - docker-compose pull --ignore-pull-failures conda-cpp + - docker-compose pull --ignore-pull-failures conda-python + - docker-compose build conda-cpp + - docker-compose build conda-python + - docker-compose build --no-cache conda-python-pandas + - docker-compose run conda-python-pandas + + test-conda-python-3.7-dask-latest: + ci: circle + platform: linux + template: docker-tests/circle.linux.yml + params: + commands: + - export PYTHON=3.7 DASK=latest + - docker-compose pull --ignore-pull-failures conda-cpp + - docker-compose pull --ignore-pull-failures conda-python + - docker-compose build conda-cpp + - docker-compose build conda-python + - docker-compose build --no-cache conda-python-dask + - docker-compose run conda-python-dask + + test-conda-python-3.7-dask-master: + ci: circle + platform: linux + template: docker-tests/circle.linux.yml + params: + commands: + - export PYTHON=3.7 DASK=master + - docker-compose pull --ignore-pull-failures conda-cpp + - docker-compose pull --ignore-pull-failures conda-python + - docker-compose build conda-cpp + - docker-compose build conda-python + - docker-compose build --no-cache conda-python-dask + - docker-compose run conda-python-dask + + test-conda-python-3.7-turbodbc-latest: ci: circle platform: linux template: docker-tests/circle.linux.yml params: - environment: - PYTHON_VERSION: 3.6 commands: - - docker-compose pull --ignore-pull-failures cpp - - docker-compose pull --ignore-pull-failures python - - docker-compose pull --ignore-pull-failures dask-integration - - docker-compose build cpp - - docker-compose build python - - docker-compose build dask-integration - - docker-compose run dask-integration - - docker-compose push --ignore-push-failures dask-integration - - docker-hdfs-integration: + - export PYTHON=3.7 TURBODBC=latest + - docker-compose pull --ignore-pull-failures conda-cpp + - docker-compose pull --ignore-pull-failures conda-python + - docker-compose build conda-cpp + - docker-compose build conda-python + - docker-compose build --no-cache conda-python-turbodbc + - docker-compose run conda-python-turbodbc + + test-conda-python-3.7-turbodbc-master: ci: circle platform: linux template: docker-tests/circle.linux.yml params: - environment: - PYTHON_VERSION: 3.6 commands: - - docker-compose pull --ignore-pull-failures cpp - - docker-compose pull --ignore-pull-failures python - - docker-compose pull --ignore-pull-failures hdfs-integration - - docker-compose build cpp - - docker-compose build python - - docker-compose build hdfs-integration - - docker-compose run hdfs-integration - - docker-compose push --ignore-push-failures hdfs-integration - - docker-spark-integration: + - export PYTHON=3.7 TURBODBC=master + - docker-compose pull --ignore-pull-failures conda-cpp + - docker-compose pull --ignore-pull-failures conda-python + - docker-compose build conda-cpp + - docker-compose build conda-python + - docker-compose build --no-cache conda-python-turbodbc + - docker-compose run conda-python-turbodbc + + test-conda-python-3.7-hdfs-2.9.2: ci: circle platform: linux template: docker-tests/circle.linux.yml params: - environment: - PYTHON_VERSION: 3.6 commands: - - docker-compose pull --ignore-pull-failures cpp - - docker-compose pull --ignore-pull-failures python - - docker-compose pull --ignore-pull-failures spark-integration - - docker-compose build cpp - - docker-compose build python - - docker-compose build spark-integration - - docker-compose run spark-integration - - docker-compose push --ignore-push-failures spark-integration - - docker-turbodbc-integration: + - export PYTHON=3.7 HDFS=2.9.2 + - docker-compose pull --ignore-pull-failures conda-cpp + - docker-compose pull --ignore-pull-failures conda-python + - docker-compose build conda-cpp + - docker-compose build conda-python + - docker-compose build conda-python-hdfs + - docker-compose run conda-python-hdfs + + test-conda-python-3.7-spark-master: ci: circle platform: linux template: docker-tests/circle.linux.yml params: - environment: - PYTHON_VERSION: 3.6 commands: - - docker-compose pull --ignore-pull-failures cpp - - docker-compose pull --ignore-pull-failures python - - docker-compose pull --ignore-pull-failures turbodbc-integration - - docker-compose build cpp - - docker-compose build python - - docker-compose build turbodbc-integration - - docker-compose run turbodbc-integration - - docker-compose push --ignore-push-failures turbodbc-integration - - docker-pandas-master: + - export PYTHON=3.7 SPARK=master + - docker-compose pull --ignore-pull-failures conda-cpp + - docker-compose pull --ignore-pull-failures conda-python + - docker-compose build conda-cpp + - docker-compose build conda-python + - docker-compose build --no-cache conda-python-spark + - docker-compose run conda-python-spark + + test-conda-cpp-hiveserver2: ci: circle platform: linux template: docker-tests/circle.linux.yml params: - environment: - PYTHON_VERSION: 3.6 commands: - - docker-compose pull --ignore-pull-failures cpp - - docker-compose pull --ignore-pull-failures python - - docker-compose build cpp - - docker-compose build python - - docker-compose build --no-cache pandas-master - - docker-compose run pandas-master + - docker-compose pull --ignore-pull-failures conda-cpp + - docker-compose build conda-cpp + - docker-compose build --no-cache conda-cpp-hiveserver2 + - docker-compose run conda-cpp-hiveserver2 diff --git a/docker-compose.yml b/docker-compose.yml index b6ad685da67d0..efc59055373ec 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,732 +15,843 @@ # specific language governing permissions and limitations # under the License. -# Development containers for Arrow +# The environment variables referenced from this docker-compose can be set by +# defining them before running docker-compose, the defaults are set in .env +# file. +# +# Example: +# $ ARCH=arm64v8 docker-compose build ubuntu-bionic-cpp +# $ ARCH=arm64v8 docker-compose run ubuntu-bionic-cpp version: '3.5' -# TODO(kszucs): set arrow's mount to :ro mode, once all of the builds are -# passing without write access to the source directory. The following builds -# are contaminating the source directory: -# - docs -# - python-alpine (writes .egg directory) -# - rust (writes Cargo.lock) -# - java (without the rsync trick) - -x-ubuntu-volumes: - &ubuntu-volumes - - .:/arrow:delegated - - ubuntu-cache:/build:delegated - -x-ubuntu-trusty-volumes: - &ubuntu-trusty-volumes - - .:/arrow:delegated - - ubuntu-trusty-cache:/build:delegated - -x-ubuntu-xenial-volumes: - &ubuntu-xenial-volumes - - .:/arrow:delegated - - ubuntu-xenial-cache:/build:delegated - -x-ubuntu-bionic-volumes: - &ubuntu-bionic-volumes - - .:/arrow:delegated - - ubuntu-bionic-cache:/build:delegated - -x-alpine-volumes: - &alpine-volumes - - .:/arrow:delegated - - alpine-cache:/build:delegated - -x-fedora-volumes: - &fedora-volumes - - .:/arrow:delegated - - fedora-cache:/build:delegated - -x-debian-testing-volumes: - &debian-testing-volumes - - .:/arrow:delegated - - debian-testing-cache:/build:delegated - +# TODO(kszucs): include ARCH to cache as well +# TODO(kszucs): add .ccache to the mounts volumes: - ubuntu-cache: - ubuntu-trusty-cache: - ubuntu-xenial-cache: - ubuntu-bionic-cache: - alpine-cache: - fedora-cache: - debian-testing-cache: + conda-latest-cache: + cuda-10.0-cache: + debian-9-cache: + debian-10-cache: + ubuntu-18.04-cache: + ubuntu-16.04-cache: + ubuntu-14.04-cache: + fedora-29-cache: maven-cache: - spark-cache: services: - ######################### Helper Containers ################################# - - build-context: - # The performance of the docker containers often depends on the size of the - # build context. This is a helper container that adds the full build context - # into the /context folder. - # - # One can inspect the folder and add files that should not be in the context - # to .dockerignore. - # - # Usage: - # docker-compose build build-context - # docker-compose run build-context - image: arrowdev/arrow-build-context:latest - build: - context: . - dockerfile: dev/Dockerfile.context - ######################### Language Containers ############################### + ################################# C++ ####################################### + # Release build: + # docker-compose run -e ARROW_BUILD_TYPE=release conda-cpp|debian-cpp|... + # Shared only: + # docker-compose run -e ARROW_BUILD_STATIC=OFF conda-cpp|debian-cpp|... + # Static only: + # docker-compose run \ + # -e ARROW_BUILD_SHARED=OFF \ + # -e ARROW_TEST_LINKAGE=static \ + # conda-cpp|debian-cpp|... + # Minimum boost - Ubuntu Xenial 16.04 has Boost 1.58: + # UBUNTU=16.04 docker-compose build \ + # -e ARROW_BOOST_SOURCE=SYSTEM \ + # ubuntu-cpp + + conda-cpp: + # Usage: + # docker-compose build conda-cpp + # docker-compose run --rm conda-cpp + image: ${ORG}/${ARCH}-conda-latest-cpp:latest + build: + context: . + dockerfile: ci/docker/conda-latest-cpp.dockerfile + cache_from: + - ${ORG}/${ARCH}-conda-latest-cpp:latest + args: + arch: ${ARCH} + conda: latest + prefix: /opt/conda + shm_size: &shm-size 2G + volumes: &conda-volumes + - .:/arrow:delegated + - conda-latest-cache:/build:delegated + entrypoint: &conda-entrypoint /bin/bash -c -l + command: &cpp-conda-command + ["/arrow/ci/scripts/cpp_build.sh /arrow /build/cpp && + /arrow/ci/scripts/cpp_test.sh /arrow /build/cpp"] + + cuda-cpp: + # Usage: + # docker-compose build cuda-cpp + # docker-compose run --rm cuda-cpp + # Parameters: + # ARCH: only amd64 + # CUDA: 8.0, 10.0, ... + image: ${ORG}/${ARCH}-cuda-${CUDA}-cpp:latest + build: + context: . + dockerfile: ci/docker/cuda-${CUDA}-cpp.dockerfile + cache_from: + - ${ORG}/${ARCH}-cuda-${CUDA}-cpp:latest + shm_size: *shm-size + volumes: &cuda-volumes + - .:/arrow:delegated + - cuda-${CUDA}-cache:/build:delegated + command: &cpp-command > + /bin/bash -c " + /arrow/ci/scripts/cpp_build.sh /arrow /build/cpp && + /arrow/ci/scripts/cpp_test.sh /arrow /build/cpp" - c_glib: + debian-cpp: # Usage: - # docker-compose build cpp - # docker-compose build c_glib - # docker-compose run c_glib - image: arrowdev/arrow-c_glib:latest + # docker-compose build debian-cpp + # docker-compose run --rm debian-cpp + # Parameters: + # ARCH: amd64, arm64v8, ... + # DEBIAN: 10 + image: ${ORG}/${ARCH}-debian-${DEBIAN}-cpp:latest build: context: . - dockerfile: c_glib/Dockerfile - volumes: *ubuntu-volumes + dockerfile: ci/docker/debian-${DEBIAN}-cpp.dockerfile + cache_from: + - ${ORG}/${ARCH}-debian-${DEBIAN}-cpp:latest + args: + arch: ${ARCH} + shm_size: *shm-size + volumes: &debian-volumes + - .:/arrow:delegated + - debian-${DEBIAN}-cache:/build:delegated + command: *cpp-command - cpp: + ubuntu-cpp: # Usage: - # docker-compose build cpp - # docker-compose run cpp - image: arrowdev/arrow-cpp:latest - shm_size: 2G + # docker-compose build ubuntu-cpp + # docker-compose run --rm ubuntu-cpp + # Parameters: + # ARCH: amd64, arm64v8, ... + # UBUNTU: 18.04, 16.04, 14.04 + image: ${ORG}/${ARCH}-ubuntu-${UBUNTU}-cpp:latest build: context: . - dockerfile: cpp/Dockerfile - volumes: *ubuntu-volumes + dockerfile: ci/docker/ubuntu-${UBUNTU}-cpp.dockerfile + cache_from: + - ${ORG}/${ARCH}-ubuntu-${UBUNTU}-cpp:latest + args: + arch: ${ARCH} + shm_size: *shm-size + volumes: &ubuntu-volumes + - .:/arrow:delegated + - ubuntu-${UBUNTU}-cache:/build:delegated + command: *cpp-command - cpp-system-deps: - # Usage: - # docker-compose build cpp-system-deps - # docker-compose run cpp-system-deps - image: arrowdev/arrow-cpp-system-deps:latest - shm_size: 2G - cap_add: - # LeakSanitizer and gdb requires ptrace(2) - - SYS_PTRACE + fedora-cpp: + image: ${ORG}/${ARCH}-fedora-${FEDORA}-cpp:latest build: context: . - dockerfile: cpp/Dockerfile.ubuntu-bionic + dockerfile: ci/docker/fedora-${FEDORA}-cpp.dockerfile + cache_from: + - ${ORG}/${ARCH}-fedora-${FEDORA}-cpp:latest args: - LLVM_VERSION: 7 - environment: - ARROW_USE_ASAN: "ON" - ARROW_USE_UBSAN: "ON" - volumes: + arch: ${ARCH} + shm_size: *shm-size + volumes: &fedora-volumes - .:/arrow:delegated - - ubuntu-cache:/build:delegated - - ${HOME}/.ccache:/build/ccache:cached + - fedora-${FEDORA}-cache:/build:delegated + command: *cpp-command - cpp-release: - # Usage: - # docker-compose build cpp - # docker-compose run cpp-release - image: arrowdev/arrow-cpp-release:latest - shm_size: 2G + ############################### C GLib ###################################### + + debian-c-glib: + image: ${ORG}/${ARCH}-debian-${DEBIAN}-c-glib:latest build: context: . - dockerfile: cpp/Dockerfile - environment: - ARROW_BUILD_TYPE: release - PARQUET_TEST_DATA: /arrow/cpp/submodules/parquet-testing/data - volumes: *ubuntu-volumes - - cpp-static-only: - # Usage: - # docker-compose build cpp - # docker-compose run cpp-static-only - image: arrowdev/arrow-cpp:latest - shm_size: 2G - environment: - ARROW_BUILD_SHARED: "OFF" - ARROW_TEST_LINKAGE: "static" - PARQUET_TEST_DATA: /arrow/cpp/submodules/parquet-testing/data + dockerfile: ci/docker/linux-apt-c-glib.dockerfile + cache_from: + - ${ORG}/${ARCH}-debian-${DEBIAN}-c-glib:latest + args: + base: ${ORG}/${ARCH}-debian-${DEBIAN}-cpp:latest + shm_size: *shm-size + volumes: *debian-volumes + command: &c-glib-command > + /bin/bash -c " + /arrow/ci/scripts/cpp_build.sh /arrow /build/cpp && + /arrow/ci/scripts/c_glib_build.sh /arrow /build/c_glib && + /arrow/ci/scripts/c_glib_test.sh /arrow /build/c_glib" + + ubuntu-c-glib: + image: ${ORG}/${ARCH}-ubuntu-${UBUNTU}-c-glib:latest + build: + context: . + dockerfile: ci/docker/linux-apt-c-glib.dockerfile + cache_from: + - ${ORG}/${ARCH}-ubuntu-${UBUNTU}-c-glib:latest + args: + base: ${ORG}/${ARCH}-ubuntu-${UBUNTU}-cpp:latest + shm_size: *shm-size volumes: *ubuntu-volumes + command: *c-glib-command - cpp-shared-only: - # Usage: - # docker-compose build cpp - # docker-compose run cpp-static-only - image: arrowdev/arrow-cpp:latest - shm_size: 2G - environment: - ARROW_BUILD_STATIC: "OFF" - PARQUET_TEST_DATA: /arrow/cpp/submodules/parquet-testing/data - volumes: *ubuntu-volumes + ############################### Ruby ######################################## + # Until Ruby is the only dependent implementation on top of C Glib we can + # test C Glib and Ruby in one pass. This is an optimization to avoid + # redundant (one for C GLib and one for Ruby doing the same work twice) + # builds on CI services. - cpp-cmake32: + debian-ruby: # Usage: - # docker-compose build cpp-cmake32 - # docker-compose run cpp-cmake32 - image: arrowdev/arrow-cpp-cmake32:latest - shm_size: 2G + # docker-compose build debian-cpp + # docker-compose build debian-c-glib + # docker-compose build debian-ruby + # docker-compose run --rm debian-ruby + image: ${ORG}/${ARCH}-debian-${DEBIAN}-ruby:latest build: context: . - dockerfile: cpp/Dockerfile.cmake + dockerfile: ci/docker/linux-apt-ruby.dockerfile + cache_from: + - ${ORG}/${ARCH}-debian-${DEBIAN}-ruby:latest args: - CMAKE_VERSION: 3.2.3 - environment: - # Vendor boost to avoid dealing with stale FindBoost. - ARROW_BOOST_SOURCE: "BUNDLED" - ARROW_ORC: "OFF" - # uriparser requires cmake 3.3 - ARROW_FLIGHT: "OFF" - PARQUET_TEST_DATA: /arrow/cpp/submodules/parquet-testing/data + base: ${ORG}/${ARCH}-debian-${DEBIAN}-c-glib:latest + shm_size: *shm-size + volumes: *debian-volumes + command: &ruby-command > + /bin/bash -c " + /arrow/ci/scripts/cpp_build.sh /arrow /build/cpp && + /arrow/ci/scripts/c_glib_build.sh /arrow /build/c_glib && + /arrow/ci/scripts/c_glib_test.sh /arrow /build/c_glib && + /arrow/ci/scripts/ruby_test.sh /arrow" + + ubuntu-ruby: + # Usage: + # docker-compose build debian-cpp + # docker-compose build debian-c-glib + # docker-compose build debian-ruby + # docker-compose run --rm debian-ruby + image: ${ORG}/${ARCH}-ubuntu-${UBUNTU}-ruby:latest + build: + context: . + dockerfile: ci/docker/linux-apt-ruby.dockerfile + cache_from: + - ${ORG}/${ARCH}-ubuntu-${UBUNTU}-ruby:latest + args: + base: ${ORG}/${ARCH}-ubuntu-${UBUNTU}-c-glib:latest + shm_size: *shm-size volumes: *ubuntu-volumes + command: *ruby-command - cpp-alpine: - # Usage: - # docker-compose build cpp-alpine - # docker-compose run cpp-alpine - image: arrowdev/arrow-cpp-alpine:latest - shm_size: 2G - build: - context: . - dockerfile: cpp/Dockerfile.alpine - environment: - PARQUET_TEST_DATA: /arrow/cpp/submodules/parquet-testing/data - volumes: *alpine-volumes + ############################### Python ###################################### - cpp-fedora: - # Usage: - # docker-compose build cpp-fedora - # docker-compose run cpp-fedora - image: arrowdev/arrow-cpp-fedora:latest - shm_size: 2G + conda-python: + image: ${ORG}/${ARCH}-conda-latest-python-${PYTHON}:latest build: context: . - dockerfile: cpp/Dockerfile.fedora - environment: - PARQUET_TEST_DATA: /arrow/cpp/submodules/parquet-testing/data - volumes: *fedora-volumes + dockerfile: ci/docker/conda-latest-python.dockerfile + cache_from: + - ${ORG}/${ARCH}-conda-latest-python-${PYTHON}:latest + args: + org: ${ORG} + arch: ${ARCH} + conda: latest + python: ${PYTHON} + shm_size: *shm-size + volumes: *conda-volumes + entrypoint: *conda-entrypoint + command: &python-conda-command + ["/arrow/ci/scripts/cpp_build.sh /arrow /build/cpp && + /arrow/ci/scripts/python_build.sh /arrow /build/python && + /arrow/ci/scripts/python_test.sh /arrow"] + + cuda-python: + image: ${ORG}/${ARCH}-cuda-${CUDA}-python-3:latest + build: + context: . + dockerfile: ci/docker/linux-apt-python-3.dockerfile + cache_from: + - ${ORG}/${ARCH}-cuda-${CUDA}-python-3:latest + args: + base: ${ORG}/${ARCH}-cuda-${CUDA}-cpp:latest + shm_size: *shm-size + volumes: *cuda-volumes + command: &python-command > + /bin/bash -c " + /arrow/ci/scripts/cpp_build.sh /arrow /build/cpp && + /arrow/ci/scripts/python_build.sh /arrow /build/python && + /arrow/ci/scripts/python_test.sh /arrow" + + debian-python: + image: ${ORG}/${ARCH}-debian-${DEBIAN}-python-3:latest + build: + context: . + dockerfile: ci/docker/linux-apt-python-3.dockerfile + cache_from: + - ${ORG}/${ARCH}-debian-${DEBIAN}-python-3:latest + args: + base: ${ORG}/${ARCH}-debian-${DEBIAN}-cpp:latest + shm_size: *shm-size + volumes: *debian-volumes + command: *python-command - cpp-debian-testing: - # Usage: - # docker-compose build cpp-debian-testing - # docker-compose run cpp-debian-testing - image: arrowdev/arrow-cpp-debian-testing:latest - shm_size: 2G + ubuntu-python: + image: ${ORG}/${ARCH}-ubuntu-${UBUNTU}-python-3:latest build: context: . - dockerfile: cpp/Dockerfile.debian-testing - environment: - PARQUET_TEST_DATA: /arrow/cpp/submodules/parquet-testing/data - volumes: *debian-testing-volumes + dockerfile: ci/docker/linux-apt-python-3.dockerfile + cache_from: + - ${ORG}/${ARCH}-ubuntu-${UBUNTU}-python-3:latest + args: + base: ${ORG}/${ARCH}-ubuntu-${UBUNTU}-cpp:latest + shm_size: *shm-size + volumes: *ubuntu-volumes + command: *python-command - cpp-ubuntu-trusty: - # Usage: - # docker-compose build cpp-ubuntu-trusty - # docker-compose run cpp-ubuntu-trusty - image: arrowdev/arrow-cpp-ubuntu-trusty:latest - shm_size: 2G + fedora-python: + image: ${ORG}/${ARCH}-fedora-${FEDORA}-python-3:latest build: context: . - dockerfile: cpp/Dockerfile.ubuntu-trusty - environment: - PARQUET_TEST_DATA: /arrow/cpp/submodules/parquet-testing/data - volumes: *ubuntu-trusty-volumes + dockerfile: ci/docker/linux-dnf-python-3.dockerfile + cache_from: + - ${ORG}/${ARCH}-fedora-${FEDORA}-python-3:latest + args: + base: ${ORG}/${ARCH}-fedora-${FEDORA}-cpp:latest + shm_size: *shm-size + volumes: *fedora-volumes + command: *python-command - cpp-ubuntu-xenial: - # Usage: - # docker-compose build cpp-ubuntu-xenial - # docker-compose run cpp-ubuntu-xenial - image: arrowdev/arrow-cpp-ubuntu-xenial:latest - shm_size: 2G - build: - context: . - dockerfile: cpp/Dockerfile.ubuntu-xenial - environment: - PARQUET_TEST_DATA: /arrow/cpp/submodules/parquet-testing/data - volumes: *ubuntu-xenial-volumes + ############################## Integration ################################# - cpp-minimum-boost-version: - # Ubuntu Xenial 16.04 has Boost 1.58, our minimum version - # + ubuntu-cpp-cmake32: # Usage: - # docker-compose build cpp-minimum-boost-version - # docker-compose run cpp-minimum-boost-version - image: arrowdev/arrow-cpp-ubuntu-xenial:latest - shm_size: 2G + # docker-compose build ubuntu-cpp + # docker-compose build ubuntu-cpp-cmake32 + # docker-compose run ubuntu-cpp-cmake32 + image: ${ORG}/${ARCH}-ubuntu-${UBUNTU}-cpp-cmake-3.2:latest build: context: . - dockerfile: cpp/Dockerfile.ubuntu-xenial + cache_from: + - ${ORG}/${ARCH}-ubuntu-${UBUNTU}-cpp-cmake-3.2:latest + dockerfile: ci/docker/linux-apt-cmake.dockerfile + args: + base: ${ORG}/${ARCH}-ubuntu-${UBUNTU}-cpp:latest + cmake: 3.2.3 environment: - PARQUET_TEST_DATA: /arrow/cpp/submodules/parquet-testing/data - ARROW_BOOST_SOURCE: "SYSTEM" + # Vendor boost to avoid dealing with stale FindBoost. + ARROW_BOOST_SOURCE: "BUNDLED" + double-conversion_SOURCE: "BUNDLED" + ARROW_ORC: "OFF" + ARROW_GANDIVA: "OFF" + # uriparser requires cmake 3.3 ARROW_FLIGHT: "OFF" - volumes: *ubuntu-xenial-volumes - - cpp-ubuntu-bionic: - # Usage: - # docker-compose build cpp-ubuntu-bionic - # docker-compose run cpp-ubuntu-bionic - image: arrowdev/arrow-cpp-ubuntu-bionic:latest - shm_size: 2G - build: - context: . - dockerfile: cpp/Dockerfile.ubuntu-bionic - environment: - PARQUET_TEST_DATA: /arrow/cpp/submodules/parquet-testing/data - volumes: *ubuntu-bionic-volumes - - go: - # Usage: - # docker-compose build go - # docker-compose run go - image: arrowdev/arrow-go:latest - build: - context: . - dockerfile: go/Dockerfile volumes: *ubuntu-volumes + command: *cpp-command - integration: - image: arrowdev/arrow-integration:latest - build: - context: . - dockerfile: integration/Dockerfile - volumes: - - .:/arrow:ro # ensures that docker won't contaminate the host directory - - maven-cache:/root/.m2:delegated - - java: + conda-python-pandas: + # Possible $PANDAS parameters: + # - `latest`: latest release + # - `master`: git master branch + # - ``: specific version available on conda-forge # Usage: - # docker-compose build java - # docker-compose run java - image: arrowdev/arrow-java:latest + # docker-compose build conda-cpp + # docker-compose build conda-python + # docker-compose build conda-python-pandas + # docker-compose run --rm conda-python-pandas + image: ${ORG}/${ARCH}-conda-latest-python-${PYTHON}-pandas-${PANDAS}:latest build: context: . - dockerfile: java/Dockerfile - volumes: - - .:/arrow:ro # ensures that docker won't contaminate the host directory - - maven-cache:/root/.m2:delegated - - java-all-jdks: - # Usage: - # docker-compose build java-all-jdks - # docker-compose run java-all-jdks - image: arrowdev/arrow-java-all-jdks:latest + dockerfile: ci/docker/conda-latest-python-pandas.dockerfile + cache_from: + - ${ORG}/${ARCH}-conda-latest-python-${PYTHON}-pandas-${PANDAS}:latest + args: + org: ${ORG} + arch: ${ARCH} + conda: latest + python: ${PYTHON} + pandas: ${PANDAS} + shm_size: *shm-size + volumes: *conda-volumes + entrypoint: *conda-entrypoint + command: *python-conda-command + + conda-python-dask: + # document to use --no-cache in case of DASK=master + image: ${ORG}/${ARCH}-conda-latest-python-${PYTHON}-dask-${DASK}:latest + build: + context: . + dockerfile: ci/docker/conda-latest-python-dask.dockerfile + cache_from: + - ${ORG}/${ARCH}-conda-latest-python-${PYTHON}-dask-${DASK}:latest + args: + org: ${ORG} + arch: ${ARCH} + conda: latest + python: ${PYTHON} + dask: ${DASK} + shm_size: *shm-size + volumes: *conda-volumes + entrypoint: *conda-entrypoint + command: + ["/arrow/ci/scripts/cpp_build.sh /arrow /build/cpp && + /arrow/ci/scripts/python_build.sh /arrow /build/python && + /arrow/ci/scripts/integration_dask.sh"] + + conda-python-turbodbc: + # document to use --no-cache in case of TURBODBC=master + image: ${ORG}/${ARCH}-conda-latest-python-${PYTHON}-turbodbc-${TURBODBC}:latest + build: + context: . + dockerfile: ci/docker/conda-latest-python-turbodbc.dockerfile + cache_from: + - ${ORG}/${ARCH}-conda-latest-python-${PYTHON}-turbodbc-${TURBODBC}:latest + args: + org: ${ORG} + arch: ${ARCH} + conda: latest + python: ${PYTHON} + turbodbc: ${TURBODBC} + shm_size: *shm-size + volumes: *conda-volumes + entrypoint: *conda-entrypoint + command: + ["/arrow/ci/scripts/cpp_build.sh /arrow /build/cpp && + /arrow/ci/scripts/python_build.sh /arrow /build/python && + /arrow/ci/scripts/integration_turbodbc.sh /turbodbc /build/turbodbc"] + + ########################## Python Wheels #################################### + + centos-python-manylinux1: + image: ${ORG}/amd64-centos-5.11-python-manylinux1:latest build: - context: . - dockerfile: java/Dockerfile.all-jdks + context: python/manylinux1 + dockerfile: Dockerfile-x86_64_base + cache_from: + - ${ORG}/amd64-centos-5.11-python-manylinux1:latest + shm_size: *shm-size + environment: + PYARROW_PARALLEL: 3 + PYTHON_VERSION: ${PYTHON_VERSION:-3.6} + UNICODE_WIDTH: ${UNICODE_WIDTH:-16} volumes: - - .:/arrow:ro # ensures that docker won't contaminate the host directory - - maven-cache:/root/.m2:delegated + - .:/arrow:delegated + - ./python/manylinux1:/io:delegated + command: &manylinux-command /io/build_arrow.sh - js: - image: arrowdev/arrow-js:latest + centos-python-manylinux2010: + image: ${ORG}/amd64-centos-6.10-python-manylinux2010:latest build: - context: . - dockerfile: js/Dockerfile + context: python/manylinux2010 + dockerfile: Dockerfile-x86_64_base + cache_from: + - ${ORG}/amd64-centos-6.10-python-manylinux2010:latest + shm_size: *shm-size + environment: + PYARROW_PARALLEL: 3 + PYTHON_VERSION: ${PYTHON_VERSION:-3.6} + UNICODE_WIDTH: ${UNICODE_WIDTH:-16} + volumes: + - .:/arrow:delegated + - ./python/manylinux2010:/io:delegated + command: *manylinux-command - python: - # Usage: - # export PYTHON_VERSION=2.7|3.6|3.7 - # docker-compose build cpp - # docker-compose build python - # docker-compose run python - image: arrowdev/arrow-python-${PYTHON_VERSION:-3.6}:latest - shm_size: 2G - build: - context: . - dockerfile: python/Dockerfile - args: - PYTHON_VERSION: ${PYTHON_VERSION:-3.6} - volumes: *ubuntu-volumes + ################################## R ######################################## - python-alpine: - # Usage: - # export PYTHON_VERSION=2.7|3.6 (minor version is ignored) - # docker-compose build cpp-alpine - # docker-compose build python-alpine - # docker-compose run python-alpine - image: arrowdev/arrow-python-${PYTHON_VERSION:-3.6}-alpine:latest - shm_size: 2G + conda-r: + image: ${ORG}/${ARCH}-conda-latest-r-${R}:latest build: context: . - dockerfile: python/Dockerfile.alpine + dockerfile: ci/docker/conda-latest-r.dockerfile + cache_from: + - ${ORG}/${ARCH}-conda-latest-r-${R}:latest args: - PYTHON_VERSION: ${PYTHON_VERSION:-3.6} - volumes: *alpine-volumes - - python-nopandas: - # Usage: - # export PYTHON_VERSION=2.7|3.6|3.7 - # docker-compose build cpp - # docker-compose build python - # docker-compose build python-nopandas - # docker-compose run python-nopandas - image: arrowdev/arrow-python-${PYTHON_VERSION:-3.6}-nopandas:latest - shm_size: 2G - build: - context: . - dockerfile: python/Dockerfile.nopandas + org: ${ORG} + arch: ${ARCH} + conda: latest + r: ${R} + shm_size: *shm-size + volumes: *conda-volumes + entrypoint: *conda-entrypoint + command: + ["/arrow/ci/scripts/cpp_build.sh /arrow /build/cpp && + /arrow/ci/scripts/r_build.sh /arrow && + /arrow/ci/scripts/r_test.sh /arrow"] + + ubuntu-r: + image: ${ORG}/${ARCH}-ubuntu-${UBUNTU}-r-${R}:latest + build: + context: . + dockerfile: ci/docker/linux-apt-r.dockerfile + cache_from: + - ${ORG}/${ARCH}-ubuntu-${UBUNTU}-r-${R}:latest args: - PYTHON_VERSION: ${PYTHON_VERSION:-3.6} + r: ${R} + base: ${ORG}/${ARCH}-ubuntu-${UBUNTU}-cpp:latest + shm_size: *shm-size volumes: *ubuntu-volumes - - rust: - # Usage: - # docker-compose build rust - # docker-compose run rust - image: arrowdev/arrow-rust:latest - build: - context: . - dockerfile: rust/Dockerfile - volumes: *ubuntu-volumes - - r: - # Usage: - # docker-compose build cpp - # docker-compose build r - # docker-compose run r - image: arrowdev/arrow-r:latest - build: - context: . - dockerfile: r/Dockerfile - volumes: *ubuntu-volumes - - r-sanitizer: - # Usage: - # docker-compose build r-sanitizer - # docker-compose run r-sanitizer - image: arrowdev/arrow-r-sanitizer:latest + command: > + /bin/bash -c " + /arrow/ci/scripts/cpp_build.sh /arrow /build/cpp && + /arrow/ci/scripts/r_deps.sh /arrow && + /arrow/ci/scripts/r_build.sh /arrow && + /arrow/ci/scripts/r_test.sh /arrow" + + ubuntu-r-sanitizer: + # Only 18.04 and amd64 supported + # Usage: + # docker-compose build ubuntu-r-sanitizer + # docker-compose run ubuntu-r-sanitizer + image: ${ORG}/amd64-ubuntu-18.04-r-sanitizer:latest cap_add: # LeakSanitizer and gdb requires ptrace(2) - SYS_PTRACE build: context: . - dockerfile: r/Dockerfile.sanitizer + dockerfile: ci/docker/ubuntu-18.04-r-sanitizer.dockerfile + cache_from: + - ${ORG}/amd64-ubuntu-18.04-r-sanitizer:latest volumes: *ubuntu-volumes + command: > + /bin/bash -c " + /arrow/ci/scripts/cpp_build.sh /arrow /build/cpp && + /arrow/ci/scripts/r_deps.sh /arrow && + /arrow/ci/scripts/r_build.sh /arrow && + /arrow/ci/scripts/r_sanitize.sh /arrow" - r-conda: - # Usage: - # export R_VERSION=3.5.1 - # docker-compose build cpp - # docker-compose build r-conda - # docker-compose run r-conda - image: arrowdev/arrow-r-conda-${R_VERSION:-3.5.1}:latest - shm_size: 2G + ################################ Rust ####################################### + + debian-rust: + image: ${ORG}/${ARCH}-debian-10-rust-${RUST}:latest build: context: . - dockerfile: r/Dockerfile.conda + dockerfile: ci/docker/debian-10-rust.dockerfile + cache_from: + - ${ORG}/${ARCH}-debian-10-rust-${RUST}:latest args: - R_VERSION: ${R_VERSION:-3.5.1} - volumes: *ubuntu-volumes + arch: ${ARCH} + rust: ${RUST} + shm_size: *shm-size + volumes: *debian-volumes + command: &rust-command > + /bin/bash -c " + /arrow/ci/scripts/rust_build.sh /arrow /build/rust && + /arrow/ci/scripts/rust_test.sh /arrow /build/rust" - ######################### Tools and Linters ################################# + ################################# Go ######################################## - # TODO(kszucs): site - # TODO(kszucs): {cpp,java,glib,js}-apidoc - - lint: - # Usage: - # docker-compose build lint - # docker-compose run lint - image: arrowdev/arrow-lint:latest - build: - context: . - dockerfile: ci/docker/Dockerfile.lint - volumes: *ubuntu-volumes - - iwyu: - # Usage: - # export PYTHON_VERSION=3.6 - # docker-compose build cpp - # docker-compose build python - # docker-compose build iwyu - # docker-compose run iwyu - image: arrowdev/arrow-iwyu:latest + debian-go: + image: ${ORG}/${ARCH}-debian-10-go-${GO}:latest build: context: . - dockerfile: dev/lint/Dockerfile - command: arrow/dev/lint/run_iwyu.sh - volumes: *ubuntu-volumes + dockerfile: ci/docker/debian-10-go.dockerfile + cache_from: + - ${ORG}/${ARCH}-debian-10-go-${GO}:latest + args: + arch: ${ARCH} + go: ${GO} + shm_size: *shm-size + volumes: *debian-volumes + command: &go-command > + /bin/bash -c " + /arrow/ci/scripts/go_build.sh /arrow && + /arrow/ci/scripts/go_test.sh /arrow" - fuzzit: - # Submit a fuzzing job on fuzzit.dev - # Usage: - # docker-compose build cpp - # docker-compose build fuzzit - # docker-compose run fuzzit - image: arrowdev/arrow-fuzzit:latest - build: - context: . - dockerfile: dev/fuzzit/Dockerfile - environment: - CI_ARROW_SHA: ${CI_ARROW_SHA} - CI_ARROW_BRANCH: ${CI_ARROW_BRANCH} - FUZZIT_JOB_TYPE: fuzzing - FUZZIT_API_KEY: c0b760d37db6012fcaafd8ca5f412ba7bcd297ac969650502994b51aa11798153824442f999a067e1ef67821989ed664 - volumes: *ubuntu-volumes + ############################# JavaScript #################################### - fuzzit-regression: - # Submit a regression job on fuzzit.dev - # Usage: - # docker-compose build cpp - # docker-compose build fuzzit-regression - # docker-compose run fuzzit-regression - image: arrow:fuzzit + debian-js: + image: ${ORG}/${ARCH}-debian-10-js-${NODE}:latest build: context: . - dockerfile: dev/fuzzit/Dockerfile - environment: - CI_ARROW_SHA: ${CI_ARROW_SHA} - CI_ARROW_BRANCH: ${CI_ARROW_BRANCH} - FUZZIT_JOB_TYPE: regression - FUZZIT_API_KEY: c0b760d37db6012fcaafd8ca5f412ba7bcd297ac969650502994b51aa11798153824442f999a067e1ef67821989ed664 - volumes: *ubuntu-volumes + dockerfile: ci/docker/debian-10-js.dockerfile + cache_from: + - ${ORG}/${ARCH}-debian-10-js-${NODE}:latest + args: + arch: ${ARCH} + node: ${NODE} + shm_size: *shm-size + volumes: *debian-volumes + command: &js-command > + /bin/bash -c " + /arrow/ci/scripts/js_build.sh /arrow && + /arrow/ci/scripts/js_test.sh /arrow" - fuzzit-local-regression: - # Run a local fuzzit regression job - # Usage: - # docker-compose build cpp - # docker-compose build fuzzit-local-regression - # docker-compose run fuzzit-local-regression - image: arrow:fuzzit - build: - context: . - dockerfile: dev/fuzzit/Dockerfile - environment: - CI_ARROW_SHA: ${CI_ARROW_SHA} - CI_ARROW_BRANCH: ${CI_ARROW_BRANCH} - FUZZIT_JOB_TYPE: local-regression - volumes: *ubuntu-volumes + #################################### C# ##################################### - docs: - # Usage: - # docker-compose build cpp - # docker-compose build python - # docker-compose build docs - # docker-compose run docs - image: arrowdev/arrow-docs:latest + ubuntu-csharp: + image: ${ORG}/${ARCH}-ubuntu-18.04-csharp-${DOTNET}:latest build: context: . - dockerfile: docs/Dockerfile + dockerfile: ci/docker/ubuntu-18.04-csharp.dockerfile + cache_from: + - ${ORG}/${ARCH}-ubuntu-18.04-csharp-${DOTNET}:latest + args: + arch: ${ARCH} + dotnet: ${DOTNET} + platform: bionic # use bionic-arm64v8 for ARM + shm_size: *shm-size volumes: *ubuntu-volumes + command: &csharp-command > + /bin/bash -c " + /arrow/ci/scripts/csharp_build.sh /arrow && + /arrow/ci/scripts/csharp_test.sh /arrow" - ######################### Package Builder ################################### + ################################ Java ####################################### - python-manylinux1: - # Usage: - # either build: - # $ docker-compose build python-manylinux1 - # or pull: - # $ docker-compose pull python-manylinux1 - # an then run: - # $ docker-compose run -e PYTHON_VERSION=3.7 python-manylinux1 - image: arrowdev/arrow_manylinux1_x86_64_base:0.14.1-static-zlib + debian-java: + image: ${ORG}/${ARCH}-debian-9-java-${JDK}-maven-${MAVEN}:latest build: - context: python/manylinux1 - dockerfile: Dockerfile-x86_64_base - shm_size: 2G - environment: - PYARROW_PARALLEL: 3 - PYTHON_VERSION: ${PYTHON_VERSION:-3.6} - UNICODE_WIDTH: ${UNICODE_WIDTH:-16} - volumes: + context: . + dockerfile: ci/docker/debian-9-java.dockerfile + cache_from: + - ${ORG}/${ARCH}-debian-9-java-${JDK}-maven-${MAVEN}:latest + args: + arch: ${ARCH} + jdk: ${JDK} + maven: ${MAVEN} + shm_size: *shm-size + volumes: &java-volumes - .:/arrow:delegated - - ./python/manylinux1:/io:delegated - command: /io/build_arrow.sh + - maven-cache:/root/.m2:delegated + command: &java-command > + /bin/bash -c " + /arrow/ci/scripts/java_build.sh /arrow && + /arrow/ci/scripts/java_test.sh /arrow" - python-manylinux2010: + debian-java-jni: + # Includes plasma test and jni for gandiva and orc. # Usage: - # either build: - # $ docker-compose build python-manylinux2010 - # or pull: - # $ docker-compose pull python-manylinux2010 - # an then run: - # $ docker-compose run -e PYTHON_VERSION=3.7 python-manylinux2010 - image: arrowdev/arrow_manylinux2010_x86_64_base:0.14.1-static-zlib + # docker-compose build debian-java + # docker-compose build debian-java-jni + image: ${ORG}/${ARCH}-debian-9-java-jni:latest build: - context: python/manylinux2010 - dockerfile: Dockerfile-x86_64_base - shm_size: 2G - environment: - PYARROW_PARALLEL: 3 - PYTHON_VERSION: ${PYTHON_VERSION:-3.6} - UNICODE_WIDTH: ${UNICODE_WIDTH:-16} + context: . + dockerfile: ci/docker/linux-apt-jni.dockerfile + cache_from: + - ${ORG}/${ARCH}-debian-9-java-jni:latest + args: + base: ${ORG}/${ARCH}-debian-9-java-${JDK}-maven-${MAVEN}:latest + shm_size: *shm-size volumes: - .:/arrow:delegated - - ./python/manylinux2010:/io:delegated - command: /io/build_arrow.sh - - ######################### Integration Tests ################################# + - maven-cache:/root/.m2:delegated + - debian-9-cache:/build:delegated + command: + /bin/bash -c " + /arrow/ci/scripts/cpp_build.sh /arrow /build/cpp && + /arrow/ci/scripts/java_build.sh /arrow /build/cpp/debug + /arrow/ci/scripts/java_test.sh /arrow /build/cpp/debug" - # impala: - # image: cpcloud86/impala:java8-1 - # ports: - # - "21050" - # hostname: impala + ############################## Integration ################################## - pandas-master: - # Usage: - # export PYTHON_VERSION=3.6 - # docker-compose build cpp - # docker-compose build python - # docker-compose build --no-cache pandas-master - # docker-compose run pandas-master - image: arrowdev/arrow-pandas-master:latest + conda-integration: + image: ${ORG}/${ARCH}-conda-latest-integration:latest build: context: . - dockerfile: integration/pandas/Dockerfile - shm_size: 2G - volumes: *ubuntu-volumes - - hdfs-namenode: - image: gelog/hadoop - shm_size: 2G - ports: - - "9000:9000" - - "50070:50070" - command: hdfs namenode - hostname: hdfs-namenode - - hdfs-datanode-1: - image: gelog/hadoop - command: hdfs datanode - ports: - # The host port is randomly assigned by Docker, to allow scaling - # to multiple DataNodes on the same host - - "50075" - links: - - hdfs-namenode:hdfs-namenode - - hdfs-datanode-2: - image: gelog/hadoop - command: hdfs datanode - ports: - # The host port is randomly assigned by Docker, to allow scaling - # to multiple DataNodes on the same host - - "50075" - links: - - hdfs-namenode:hdfs-namenode - - hdfs-integration: - # Usage: - # export PYTHON_VERSION=3.6 - # docker-compose build cpp - # docker-compose build python - # docker-compose build hdfs-integration - # docker-compose run hdfs-integration - image: arrowdev/arrow-hdfs-${HDFS_VERSION:-2.9.2}:latest - links: - - hdfs-namenode:hdfs-namenode - - hdfs-datanode-1:hdfs-datanode-1 - - hdfs-datanode-2:hdfs-datanode-2 + dockerfile: ci/docker/conda-latest-integration.dockerfile + cache_from: + - ${ORG}/${ARCH}-conda-latest-integration:latest + args: + org: ${ORG} + arch: ${ARCH} + conda: latest + jdk: ${JDK} + maven: ${MAVEN} + node: ${NODE} + go: ${GO} + volumes: *conda-volumes + entrypoint: *conda-entrypoint environment: - - ARROW_HDFS_TEST_HOST=hdfs-namenode - - ARROW_HDFS_TEST_PORT=9000 - - ARROW_HDFS_TEST_USER=root - build: - context: . - dockerfile: integration/hdfs/Dockerfile + # tell archery where the arrow binaries are located + ARROW_CPP_EXE_PATH: /build/cpp/debug + command: + ["/arrow/ci/scripts/cpp_build.sh /arrow /build/cpp && + /arrow/ci/scripts/go_build.sh /arrow && + /arrow/ci/scripts/java_build.sh /arrow && + /arrow/ci/scripts/js_build.sh /arrow && + pip install -e /arrow/dev/archery && + archery integration --with-all --run-flight"] + + ################################ Docs ####################################### + + ubuntu-docs: + # Usage: + # docker-compose build ubuntu-cpp + # docker-compose build ubuntu-python + # docker-compose build ubuntu-docs + # docker-compose run --rm ubuntu-docs + image: ${ORG}/${ARCH}-ubuntu-${UBUNTU}-docs:latest + build: + context: . + dockerfile: ci/docker/linux-apt-docs.dockerfile + cache_from: + - ${ORG}/${ARCH}-ubuntu-${UBUNTU}-docs:latest args: - HDFS_VERSION: ${HDFS_VERSION:-2.9.2} + jdk: ${JDK} + node: ${NODE} + base: ${ORG}/${ARCH}-ubuntu-${UBUNTU}-python-3:latest volumes: *ubuntu-volumes + command: + /bin/bash -c " + /arrow/ci/scripts/cpp_build.sh /arrow /build/cpp true && + /arrow/ci/scripts/c_glib_build.sh /arrow /build/c_glib true && + /arrow/ci/scripts/python_build.sh /arrow /build/python && + /arrow/ci/scripts/java_build.sh /arrow /build/cpp/debug true && + /arrow/ci/scripts/js_build.sh /arrow true && + /arrow/ci/scripts/docs_build.sh /arrow /build/docs" - # TODO(kszucs): pass dask version explicitly as a build argument - dask-integration: - # Usage: - # export PYTHON_VERSION=3.6 - # docker-compose build cpp - # docker-compose build python - # docker-compose build dask-integration - # docker-compose run dask-integration - image: arrowdev/arrow-dask-integration:latest - build: - context: . - dockerfile: integration/dask/Dockerfile - volumes: *ubuntu-volumes + ################################# Tools ##################################### - turbodbc-integration: + ubuntu-lint: # Usage: - # export PYTHON_VERSION=3.6 - # docker-compose build cpp - # docker-compose build python - # docker-compose build turbodbc-integration - # docker-compose run turbodbc-integration - image: arrowdev/arrow-turbodbc-integration:latest + # docker-compose build ubuntu-cpp + # docker-compose build ubuntu-lint + # docker-compose run ubuntu-lint + image: ${ORG}/${ARCH}-ubuntu-${UBUNTU}-lint:latest build: context: . - dockerfile: integration/turbodbc/Dockerfile + dockerfile: ci/docker/linux-apt-lint.dockerfile + cache_from: + - ${ORG}/${ARCH}-ubuntu-${UBUNTU}-lint:latest + args: + rust: ${RUST} + base: ${ORG}/${ARCH}-ubuntu-${UBUNTU}-cpp:latest volumes: *ubuntu-volumes + command: > + /bin/bash -c "pip install -e /arrow/dev/archery && archery lint" - spark-integration: + ubuntu-fuzzit: # Usage: - # export PYTHON_VERSION=3.6 - # docker-compose build cpp - # docker-compose build python - # docker-compose build spark-integration - # docker-compose run spark-integration - image: arrowdev/arrow-spark-${SPARK_VERSION:-master}:latest - environment: - - SPARK_VERSION=${SPARK_VERSION:-master} + # docker-compose build ubuntu-cpp + # docker-compose build ubuntu-fuzzit + # docker-compose run ubuntu-fuzzit build: context: . - dockerfile: integration/spark/Dockerfile - volumes: - - .:/arrow:ro # ensures that docker won't contaminate the host directory - - ubuntu-cache:/build:delegated - - maven-cache:/root/.m2:delegated - - spark-cache:/spark:delegated - - # TODO(kszucs): hive-integration - - ################################# Release ################################### - - release-source: - image: arrowdev/arrow-release-source:latest - build: - context: . - dockerfile: dev/release/source/Dockerfile - volumes: - - .:/arrow:delegated - - ######################## Verification Containers ############################ - - debian-stretch: - image: debian:stretch - volumes: - - .:/arrow:delegated - - debian-buster: - image: debian:buster - volumes: - - .:/arrow:delegated - - ubuntu-trusty: - image: ubuntu:trusty - volumes: - - .:/arrow:delegated - - ubuntu-xenial: - image: ubuntu:xenial - volumes: - - .:/arrow:delegated - - ubuntu-bionic: - image: ubuntu:bionic - volumes: - - .:/arrow:delegated + dockerfile: ci/docker/linux-apt-fuzzit.dockerfile + cache_from: + - ${ORG}/${ARCH}-ubuntu-${UBUNTU}-fuzzit:latest + args: + base: ${ORG}/${ARCH}-ubuntu-${UBUNTU}-cpp:latest + environment: + FUZZIT_HOST: bionic-llvm7 + CI_ARROW_SHA: ${CI_ARROW_SHA:-UNSET} + CI_ARROW_BRANCH: ${CI_ARROW_BRANCH:-UNSET} + volumes: *debian-volumes + command: > + /bin/bash -c " + /arrow/ci/scripts/cpp_build.sh /arrow /build/cpp && + /arrow/ci/scripts/fuzzit_build.sh /arrow" - ubuntu-disco: - image: ubuntu:disco - volumes: - - .:/arrow:delegated + ######################### Integration Tests ################################# - centos-6: - image: centos:6 - volumes: - - .:/arrow:delegated + postgres: + image: postgres + ports: + - 5432:5432 + environment: + POSTGRES_PASSWORD: postgres - centos-7: - image: centos:7 - volumes: - - .:/arrow:delegated + impala: + image: ibisproject/impala:latest + hostname: impala + links: + - postgres:postgres + environment: + PGPASSWORD: postgres + ports: + # HDFS + - 9020:9020 + - 50070:50070 + - 50075:50075 + - 8020:8020 + - 8042:8042 + # Hive + - 9083:9083 + # Impala + - 21000:21000 + - 21050:21050 + - 25000:25000 + - 25010:25010 + - 25020:25020 + + conda-cpp-hiveserver2: + # Usage: + # docker-compose build conda-cpp + # docker-compose build conda-cpp-hiveserver2 + # docker-compose run conda-cpp-hiveserver2 + image: ${ORG}/${ARCH}-conda-latest-cpp:latest + links: + - impala:impala + environment: + - ARROW_FLIGHT=OFF + - ARROW_GANDIVA=OFF + - ARROW_PLASMA=OFF + - ARROW_HIVESERVER2=ON + - ARROW_HIVESERVER2_TEST_HOST=impala + shm_size: *shm-size + volumes: *conda-volumes + entrypoint: *conda-entrypoint + command: + ["/arrow/ci/scripts/cpp_build.sh /arrow /build/cpp && + /arrow/ci/scripts/integration_hiveserver2.sh /arrow /build/cpp"] + + conda-python-hdfs: + # Usage: + # docker-compose build conda-cpp + # docker-compose build conda-python + # docker-compose build conda-python-hdfs + # docker-compose run conda-python-hdfs + image: ${ORG}/${ARCH}-conda-latest-python-${PYTHON}-hdfs-${HDFS}:latest + build: + context: . + dockerfile: ci/docker/conda-latest-python-hdfs.dockerfile + cache_from: + - ${ORG}/${ARCH}-conda-latest-python-${PYTHON}-hdfs-${HDFS}:latest + args: + org: ${ORG} + arch: ${ARCH} + conda: latest + python: ${PYTHON} + jdk: ${JDK} + # conda-forge doesn't have 3.5.4 so pinning explicitly, but this should + # be set to ${MAVEN} + maven: 3.5 + hdfs: ${HDFS} + links: + - impala:impala + environment: + - ARROW_HDFS_TEST_HOST=impala + - ARROW_HDFS_TEST_PORT=8020 + - ARROW_HDFS_TEST_USER=hdfs + shm_size: *shm-size + volumes: *conda-volumes + entrypoint: *conda-entrypoint + command: + ["/arrow/ci/scripts/cpp_build.sh /arrow /build/cpp && + /arrow/ci/scripts/python_build.sh /arrow /build/python && + /arrow/ci/scripts/integration_hdfs.sh /arrow /build/cpp"] + + conda-python-spark: + # Usage: + # docker-compose build conda-cpp + # docker-compose build conda-python + # docker-compose build conda-python-spark + # docker-compose run conda-python-spark + image: ${ORG}/${ARCH}-conda-latest-python-${PYTHON}-spark-${SPARK}:latest + build: + context: . + dockerfile: ci/docker/conda-latest-python-spark.dockerfile + cache_from: + - ${ORG}/${ARCH}-conda-latest-python-${PYTHON}-spark-${SPARK}:latest + args: + org: ${ORG} + arch: ${ARCH} + conda: latest + python: ${PYTHON} + jdk: ${JDK} + # conda-forge doesn't have 3.5.4 so pinning explicitly, but this should + # be set to ${MAVEN} + maven: 3.5 + spark: ${SPARK} + shm_size: *shm-size + volumes: *conda-volumes + # - maven-cache:/root/.m2:delegated + # - spark-cache:/spark:delegated + entrypoint: *conda-entrypoint + command: + ["/arrow/ci/scripts/cpp_build.sh /arrow /build/cpp && + /arrow/ci/scripts/python_build.sh /arrow /build/python && + /arrow/ci/scripts/java_build.sh /arrow && + /arrow/ci/scripts/integration_spark.sh /arrow /spark"] diff --git a/docs/Dockerfile b/docs/Dockerfile deleted file mode 100644 index 016d334f7b494..0000000000000 --- a/docs/Dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -FROM arrowdev/arrow-python-3.6:latest - -COPY ci/conda_env_sphinx.yml /arrow/ci/ -RUN conda install --file arrow/ci/conda_env_sphinx.yml && \ - conda clean --all - -CMD ["/bin/bash", "-c", "arrow/ci/docker_build_cpp.sh && arrow/ci/docker_build_python.sh && arrow/ci/docker_build_sphinx.sh"] diff --git a/go/Dockerfile b/go/Dockerfile deleted file mode 100644 index 7e1d6ea68e222..0000000000000 --- a/go/Dockerfile +++ /dev/null @@ -1,27 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -FROM golang:1.12 - -COPY go/arrow/Gopkg.lock \ - go/arrow/Gopkg.toml \ - /arrow/go/arrow/ -WORKDIR /arrow/go/arrow - -RUN go get -d -t -v ./... - -CMD ["/bin/bash", "-c", "go install -v ./... && for d in $(go list ./... | grep -v vendor); do go test $d; done"] diff --git a/integration/Dockerfile b/integration/Dockerfile deleted file mode 100644 index 5f3d2e7998139..0000000000000 --- a/integration/Dockerfile +++ /dev/null @@ -1,52 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -FROM arrowdev/arrow-cpp:latest - -# Use bash -SHELL ["/bin/bash", "-o", "pipefail", "-c"] - -# install apt extra stuff we need -RUN apt-get update -y -q && \ - apt-get install -y -q --no-install-recommends \ - curl \ - maven \ - rsync \ - tzdata \ - openjdk-11-jdk && \ - apt-get clean && rm -rf /var/lib/apt/lists/* - -# Various environment variables -ENV GOROOT=/go \ - GOPATH=/ \ - LC_ALL=C.UTF-8 \ - LANG=C.UTF-8 \ - NVM_DIR=/.nvm \ - NODE_VERSION=11.12 - -# Install Go -RUN curl -sL --output go.tar.gz https://dl.google.com/go/go1.12.7.linux-amd64.tar.gz && \ - tar -xf go.tar.gz && \ - rm go.tar.gz - -ENV PATH $PATH:$GOROOT/bin - -# Install Node with nvm -RUN mkdir $NVM_DIR && \ - curl -sL https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.0/install.sh | bash - -CMD ["arrow/ci/docker_integration.sh"] \ No newline at end of file diff --git a/integration/hdfs/Dockerfile b/integration/hdfs/Dockerfile deleted file mode 100644 index 71db503436d07..0000000000000 --- a/integration/hdfs/Dockerfile +++ /dev/null @@ -1,54 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -FROM arrowdev/arrow-python-3.6:latest - -# installing libhdfs (JNI) -ARG HADOOP_VERSION=2.9.2 -ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 \ - HADOOP_HOME=/usr/local/hadoop \ - HADOOP_OPTS=-Djava.library.path=/usr/local/hadoop/lib/native \ - PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin -RUN apt-get update -y && \ - apt-get install -y --no-install-recommends openjdk-8-jdk && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -RUN wget -q -O hadoop-$HADOOP_VERSION.tar.gz "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=hadoop/common/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz" && \ - tar -zxf /hadoop-$HADOOP_VERSION.tar.gz && \ - rm /hadoop-$HADOOP_VERSION.tar.gz && \ - mv /hadoop-$HADOOP_VERSION /usr/local/hadoop -COPY integration/hdfs/hdfs-site.xml $HADOOP_HOME/etc/hadoop/ - -# installing libhdfs3, it needs to be pinned, see ARROW-1465 and ARROW-1445 -# after the conda-forge migration it's failing with abi incompatibilities, so -# turning it off for now -# RUN conda install -y -c conda-forge libhdfs3=2.2.31 && \ -# conda clean --all - -# build cpp with tests -ENV CC=gcc \ - CXX=g++ \ - ARROW_ORC=ON \ - ARROW_HDFS=ON \ - ARROW_PYTHON=ON \ - ARROW_BUILD_TESTS=ON - -# build and test -CMD ["/bin/bash", "-c", "arrow/ci/docker_build_cpp.sh && \ - arrow/ci/docker_build_python.sh && \ - arrow/integration/hdfs/runtest.sh"] diff --git a/integration/spark/Dockerfile b/integration/spark/Dockerfile deleted file mode 100644 index a0e7185d6da39..0000000000000 --- a/integration/spark/Dockerfile +++ /dev/null @@ -1,57 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -FROM arrowdev/arrow-python-3.6:latest - -# installing java and maven -ARG MAVEN_VERSION=3.6.2 -ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 \ - MAVEN_HOME=/usr/local/maven \ - M2_HOME=/root/.m2 \ - PATH=/root/.m2/bin:/usr/local/maven/bin:$PATH -RUN apt-get update -q -y && \ - apt-get install -q -y --no-install-recommends openjdk-8-jdk && \ - wget -q -O maven-$MAVEN_VERSION.tar.gz "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=maven/maven-3/$MAVEN_VERSION/binaries/apache-maven-$MAVEN_VERSION-bin.tar.gz" && \ - tar -zxf /maven-$MAVEN_VERSION.tar.gz && \ - rm /maven-$MAVEN_VERSION.tar.gz && \ - mv /apache-maven-$MAVEN_VERSION /usr/local/maven \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - -# installing specific version of spark -ARG SPARK_VERSION=master -RUN wget -q -O /tmp/spark.tar.gz https://github.com/apache/spark/archive/$SPARK_VERSION.tar.gz && \ - mkdir /spark && \ - tar -xzf /tmp/spark.tar.gz -C /spark --strip-components=1 && \ - rm /tmp/spark.tar.gz - -# patch spark to build with current Arrow Java -COPY integration/spark/ARROW-6429.patch /tmp/ -RUN patch -d /spark -p1 -i /tmp/ARROW-6429.patch && \ - rm /tmp/ARROW-6429.patch - -# build cpp with tests -ENV CC=gcc \ - CXX=g++ \ - ARROW_PYTHON=ON \ - ARROW_HDFS=ON \ - ARROW_BUILD_TESTS=OFF - -# build and test -CMD ["/bin/bash", "-c", "arrow/ci/docker_build_cpp.sh && \ - arrow/ci/docker_build_python.sh && \ - arrow/ci/docker_build_java.sh && \ - arrow/integration/spark/runtest.sh"] diff --git a/java/Dockerfile b/java/Dockerfile deleted file mode 100644 index 9ee898f48f14a..0000000000000 --- a/java/Dockerfile +++ /dev/null @@ -1,29 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -FROM maven:3.5.2-jdk-8-slim - -# rsync is required to prevent the contamination of arrow directory -# (mounted from the host) -RUN apt-get update -y \ - && apt-get install -y --no-install-recommends rsync \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - -CMD ["/bin/bash", "-c", "arrow/ci/docker_build_java.sh && \ - cd /build/java/arrow/java && \ - mvn test"] diff --git a/java/Dockerfile.all-jdks b/java/Dockerfile.all-jdks deleted file mode 100644 index bf4e2afa22707..0000000000000 --- a/java/Dockerfile.all-jdks +++ /dev/null @@ -1,35 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -FROM ubuntu:18.04 - -# install build essentials -RUN export DEBIAN_FRONTEND=noninteractive && \ - apt-get update -y -q && \ - apt-get install -y -q --no-install-recommends \ - wget \ - software-properties-common \ - ca-certificates \ - maven \ - rsync \ - tzdata \ - openjdk-8-jdk \ - openjdk-11-jdk && \ - apt-get clean && rm -rf /var/lib/apt/lists/* - -# Test all supported JDKs -CMD ["arrow/ci/docker_java_test_all.sh"] diff --git a/js/Dockerfile b/js/Dockerfile deleted file mode 100644 index 311b4b97f548f..0000000000000 --- a/js/Dockerfile +++ /dev/null @@ -1,32 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -FROM node:11 - -# install dependencies -COPY js/.npmrc js/package.json /arrow/js/ -WORKDIR /arrow/js -RUN npm install -g npm@latest && \ - npm install - -# build -COPY LICENSE.txt /arrow/ -COPY NOTICE.txt /arrow/ -COPY js /arrow/js -RUN npm run lint && npm run build - -CMD ["npm", "run", "test"] diff --git a/python/Dockerfile.alpine b/python/Dockerfile.alpine deleted file mode 100644 index 87a1dff67acca..0000000000000 --- a/python/Dockerfile.alpine +++ /dev/null @@ -1,47 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -FROM arrowdev/arrow-cpp-alpine:latest - -SHELL ["/bin/bash", "-o", "pipefail", "-c"] - -# better compatibility for the scripts -RUN apk add --no-cache coreutils - -# install python, either python3(3.6) or python2(2.7) -ARG PYTHON_VERSION=3.6 -RUN export PYTHON_MAJOR=${PYTHON_VERSION:0:1} && \ - apk add --no-cache python${PYTHON_MAJOR}-dev && \ - python${PYTHON_MAJOR} -m ensurepip && \ - ln -sf /usr/bin/pip${PYTHON_MAJOR} /usr/bin/pip && \ - ln -sf /usr/bin/python${PYTHON_MAJOR} /usr/bin/python && \ - pip install --upgrade pip setuptools - -# install python requirements -COPY python/requirements.txt \ - python/requirements-test.txt \ - /arrow/python/ -# pandas requires numpy at build time, so install the requirements separately -RUN pip install -r /arrow/python/requirements.txt cython && \ - pip install -r /arrow/python/requirements-test.txt - -ENV ARROW_PYTHON=ON \ - PYARROW_WITH_ORC=0 \ - PYARROW_WITH_PARQUET=0 - -# build and test -CMD ["arrow/ci/docker_build_and_test_python.sh"] diff --git a/python/Dockerfile.nopandas b/python/Dockerfile.nopandas deleted file mode 100644 index 2faad084488a6..0000000000000 --- a/python/Dockerfile.nopandas +++ /dev/null @@ -1,24 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG PYTHON_VERSION=3.6 -FROM arrowdev/arrow-python-$PYTHON_VERSION:latest - -# not installing pandas doesn't mean that it's not grabbed as a transitive -# dependency, so We remove it explicitly to make sure We don't have pandas -# installed -RUN conda remove pandas && conda clean -a diff --git a/python/manylinux1/README.md b/python/manylinux1/README.md index 731dbed47dd6e..511e2e8c07e97 100644 --- a/python/manylinux1/README.md +++ b/python/manylinux1/README.md @@ -49,7 +49,7 @@ a dependency to a new version, we also need to adjust it. You can rebuild this image using ```bash -docker-compose build python-manylinux1 +docker-compose build centos-python-manylinux1 ``` For each dependency, we have a bash script in the directory `scripts/` that @@ -65,7 +65,7 @@ If you have write access to the Docker Hub Ursa Labs account, you can directly publish a build image that you built locally. ```bash -$ docker-compose push python-manylinux1 +$ docker-compose push centos-python-manylinux1 ``` ### Using quay.io to trigger and build the docker image diff --git a/python/manylinux2010/README.md b/python/manylinux2010/README.md index 3a9eee750aa1a..32d7247fd2d6e 100644 --- a/python/manylinux2010/README.md +++ b/python/manylinux2010/README.md @@ -69,7 +69,7 @@ If you have write access to the Docker Hub Ursa Labs account, you can directly publish a build image that you built locally. ```bash -$ docker push python-manylinux2010 +$ docker push centos-python-manylinux2010 The push refers to repository [arrowdev/arrow_manylinux2010_x86_64_base] a1ab88d27acc: Pushing [==============> ] 492.5MB/1.645GB [... etc. ...] diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi index 5e6078b725442..77990a1abe2cb 100644 --- a/python/pyarrow/table.pxi +++ b/python/pyarrow/table.pxi @@ -17,6 +17,7 @@ import warnings + cdef class ChunkedArray(_PandasConvertible): """ Array backed via one or more memory chunks. diff --git a/python/pyarrow/tests/test_hdfs.py b/python/pyarrow/tests/test_hdfs.py index f2430e57e0d43..da30e85017fa5 100644 --- a/python/pyarrow/tests/test_hdfs.py +++ b/python/pyarrow/tests/test_hdfs.py @@ -353,12 +353,14 @@ def test_read_write_parquet_files_with_uri(self): _pandas_api.assert_frame_equal(result, df) @pytest.mark.parquet + @pytest.mark.pandas def test_read_common_metadata_files(self): tmpdir = pjoin(self.tmp_path, 'common-metadata-' + guid()) self.hdfs.mkdir(tmpdir) test_parquet._test_read_common_metadata_files(self.hdfs, tmpdir) @pytest.mark.parquet + @pytest.mark.pandas def test_write_to_dataset_with_partitions(self): tmpdir = pjoin(self.tmp_path, 'write-partitions-' + guid()) self.hdfs.mkdir(tmpdir) @@ -366,6 +368,7 @@ def test_write_to_dataset_with_partitions(self): tmpdir, filesystem=self.hdfs) @pytest.mark.parquet + @pytest.mark.pandas def test_write_to_dataset_no_partitions(self): tmpdir = pjoin(self.tmp_path, 'write-no_partitions-' + guid()) self.hdfs.mkdir(tmpdir) diff --git a/r/Dockerfile b/r/Dockerfile deleted file mode 100644 index 55df4897c640c..0000000000000 --- a/r/Dockerfile +++ /dev/null @@ -1,113 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -FROM arrowdev/arrow-cpp:latest - -# Configure -ENV CC=gcc \ - CXX=g++ - -# r-base includes tzdata. Get around interactive stop in that package -ENV DEBIAN_FRONTEND=noninteractive -# workaround for install_github GitHub API rate limit -ENV CI=true - -# Build R -# [1] https://www.digitalocean.com/community/tutorials/how-to-install-r-on-ubuntu-18-04 -# [2] https://linuxize.com/post/how-to-install-r-on-ubuntu-18-04/#installing-r-packages-from-cran -RUN apt-get update -y && \ - apt-get install -y \ - apt-transport-https \ - software-properties-common && \ - apt-key adv \ - --keyserver keyserver.ubuntu.com \ - --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 && \ - add-apt-repository 'deb https://cloud.r-project.org/bin/linux/ubuntu bionic-cran35/' && \ - apt-get install -y r-base && \ - # system libs needed by core R packages - apt-get install -y \ - libgit2-dev \ - libssl-dev && \ - # install clang to mirror what was done on Travis - apt-get install -y \ - clang \ - clang-format \ - clang-tidy && \ - # R CMD CHECK --as-cran needs pdflatex to build the package manual - apt-get install -y \ - texlive-latex-base && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -# So that arrowExports.* files are generated -ENV ARROW_R_DEV=TRUE - -# Tell R where it can find the source code for arrow -ENV PKG_CONFIG_PATH=${PKG_CONFIG_PATH}:/build/cpp/src/arrow:/opt/conda/lib/pkgconfig -ENV LD_LIBRARY_PATH=/opt/conda/lib/:/build/cpp/src/arrow:/arrow/r/src - -# Ensure parallel R package installation, set CRAN repo mirror, -# and use pre-built binaries where possible -RUN printf "\ - options(Ncpus = parallel::detectCores(), \ - repos = 'https://demo.rstudiopm.com/all/__linux__/bionic/latest', \ - HTTPUserAgent = sprintf(\ - 'R/%%s R (%%s)', getRversion(), \ - paste(getRversion(), R.version\$platform, R.version\$arch, R.version\$os)))\n" \ - >> /etc/R/Rprofile.site - -# Also ensure parallel compilation of each individual package -RUN printf "MAKEFLAGS=-j8\n" >> /usr/lib/R/etc/Makeconf -ENV MAKEFLAGS=-j8 - -# ARROW-6171: Because lz4 is installed in the base Ubuntu image, there's an -# issue of which library is loaded at runtime. R by default will override -# LD_LIBRARY_PATH at runtime by concatenating (in that order) -# R_LD_LIBRARY_PATH, R_JAVA_LD_LIBRARY_PATH and LD_LIBRARY_PATH. If -# R_LD_LIBRARY_PATH is not set, it'll default to a list of directories which -# contains /usr/lib/x86_64-linux-gnu. -ENV R_LD_LIBRARY_PATH=$LD_LIBRARY_PATH - -# Install R package dependencies -# NOTE: any changes here should also be done in Dockerfile.sanitizer -RUN R -e "install.packages(c('remotes', 'dplyr', 'glue'))" -RUN R -e "remotes::install_github('romainfrancois/decor')" - -# Prioritize system packages and local installation -ENV ARROW_DEPENDENCY_SOURCE=SYSTEM \ - ARROW_FLIGHT=OFF \ - ARROW_GANDIVA=OFF \ - ARROW_HDFS=OFF \ - ARROW_ORC=OFF \ - ARROW_PARQUET=ON \ - ARROW_PLASMA=OFF \ - ARROW_USE_ASAN=OFF \ - ARROW_USE_UBSAN=OFF \ - ARROW_NO_DEPRECATED_API=ON \ - ARROW_INSTALL_NAME_RPATH=OFF \ - ARROW_WITH_BZ2=OFF \ - ARROW_WITH_ZSTD=OFF \ - ARROW_R_DEV=TRUE - -ENV ARROW_BUILD_BENCHMARKS=off -ENV ARROW_BUILD_EXAMPLES=off -ENV ARROW_BUILD_TESTS=off -ENV ARROW_BUILD_UTILITIES=off - -# build, install, test R package -CMD ["/bin/bash", "-c", "/arrow/ci/docker_build_cpp.sh && \ - /arrow/ci/docker_build_r.sh"] diff --git a/rust/Dockerfile b/rust/Dockerfile deleted file mode 100644 index 4b0c000cf575d..0000000000000 --- a/rust/Dockerfile +++ /dev/null @@ -1,54 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -FROM rust:1.38.0-buster -RUN rustup show - -# Install stable rustfmt -RUN rustup install stable -RUN rustup component add rustfmt - -# Install the specific version of Rust nightly that we need so that we don't need to download it for every CI run -COPY rust/rust-toolchain /tmp -RUN rustup install "$(cat /tmp/rust-toolchain)" -RUN rustup default "$(cat /tmp/rust-toolchain)" - -# Enable stable rustfmt for nightly Rust -RUN rustup component add rustfmt --toolchain stable-x86_64-unknown-linux-gnu - -# Install pre-requisites for building flatbuffers -RUN apt-get update \ - && apt-get install -y build-essential cmake \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - -# Install flatbuffers -RUN wget https://github.com/google/flatbuffers/archive/v1.11.0.tar.gz \ - && tar xzf v1.11.0.tar.gz - -WORKDIR /flatbuffers-1.11.0 - -RUN cmake -G "Unix Makefiles" \ - && make install - -WORKDIR / - -# Set environment variables for location of test data required by unit and integration tests -ENV ARROW_TEST_DATA=/arrow/testing/data -ENV PARQUET_TEST_DATA=/arrow/cpp/submodules/parquet-testing/data - -CMD ["/bin/bash", "-c", "arrow/ci/docker_build_rust.sh"] diff --git a/rust/README.md b/rust/README.md index 61c49e4103541..a9fcf8d656ba5 100644 --- a/rust/README.md +++ b/rust/README.md @@ -73,11 +73,11 @@ There are currently multiple CI systems that build the project and they all use From the root of the Arrow project, run the following command to build the Docker image that the CI system uses to build the project. ```bash -docker-compose build rust +docker-compose build debian-rust ``` Run the following command to build the project in the same way that the CI system will build the project. Note that this currently does cause some files to be written to your local workspace. ```bash -docker run -v `pwd`:/arrow -it arrowdev/arrow-rust -``` \ No newline at end of file +docker-compose run --rm debian-rust bash +```