Docker refactor (#7)

* ARROW-7067: [CI] Disable code coverage on Travis-CI Closes apache#5778 from pitrou/ARROW-7067-travis-disable-coverage and squashes the following commits: bda5ff4 <Antoine Pitrou> ARROW-7067: Disable code coverage on Travis-CI Authored-by: Antoine Pitrou <[email protected]> Signed-off-by: Antoine Pitrou <[email protected]> * ARROW-7058: [C++] FileSystemDataSourceDiscovery should apply partition schemes relative to its base dir @nealrichardson Closes apache#5772 from bkietz/7058-FileSystemDataSourceDisco and squashes the following commits: c0edfa5 <Benjamin Kietzman> make base_dir of partition schemes explicitly optional f2f9689 <Benjamin Kietzman> add DCHECK for path containing selector 1bae5ff <Benjamin Kietzman> ARROW-7058: FilSystemDataSourceDiscovery should apply partition schemes relative to its base dir Authored-by: Benjamin Kietzman <[email protected]> Signed-off-by: François Saint-Jacques <[email protected]> * Refactor docker-compose file and use it with github actions. * Turn off gandiva and flight for the HDFS test [skip ci] * Missing --pyargs argument for the python test command [skip ci] * Add CentOS version to the manylinux image names [skip ci] * Fix manylinux volumes [skip ci]
kszucs · Nov 6, 2019 · f1a9763 · f1a9763
1 parent 7f08715
commit f1a9763
Show file tree

Hide file tree

Showing 173 changed files with 4,469 additions and 5,411 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -15,89 +15,18 @@
 # specific language governing permissions and limitations
 # under the License.
 
-.git
-docker_cache
-docs/_build
-
-# IDE
-.idea
-.vscode
-*/.vscode
-
-# c_glib
-c_glib/build
-c_glib/autom4te.cache
-c_glib/m4
-c_glib/*/*.o
-c_glib/*/*.lo
-c_glib/*/*.la
-c_glib/*/.deps
-c_glib/*/.libs
-
-# cpp
-cpp/.idea
-cpp/build
-cpp/*-build
-cpp/*_build
-cpp/build-debug
-cpp/build-release
-cpp/build-test
-cpp/Testing
-cpp/thirdparty
-!cpp/thirdparty/jemalloc
-!cpp/thirdparty/versions.txt
-!cpp/thirdparty/hadoop/include
-
-# python
-python/build
-python/dist
-python/manylinux1/dist
-python/*.egg-info
-python/*.egg
-python/*.pyc
-__pycache__/
-*/__pycache__/
-*/*/__pycache__/
-*/*/*/__pycache__/
-*.py[cod]
-*/*.py[cod]
-*/*/*.py[cod]
-*/*/*/*.py[cod]
-*.so
-*/*.so
-*/*/*.so
-*/*/*/*.so
-*.dylib
-*/*.dylib
-*/*/*.dylib
-*/*/*/*.dylib
-
-# JS
-js/.npm
-js/node_modules
-js/jspm_packages
-
-js/logs
-js/*.log
-js/.esm-cache
-js/npm-debug.log*
-js/yarn-debug.log*
-js/yarn-error.log*
-
-js/.grunt
-js/bower_components
-js/.lock-wscript
-js/build/Release
-js/dist
-js/targets
-js/test/data/**/*.json
-js/test/data/**/*.arrow
-
-# Rust
-rust/target
-rust/arrow/target
-rust/parquet/target
-rust/datafusion/target
-
-# Site
-site/
+# Turn .dockerignore to .dockerallow by excluding everything and explicitly
+# allowing specific files and directories. This enables us to quickly add
+# dependency files to the docker contect without scanning the whole directory.
+# This setup requires to all of our docker containers have arrow's source
+# as a mounted directory.
+
+# exclude everything
+**
+
+# include explicitly
+!ci/**
+!c_glib/Gemfile
+!python/requirements*.txt
+!python/manylinux1/**
+!python/manylinux2010/**
diff --git a/.env b/.env
@@ -0,0 +1,40 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# All of the following environment variables are required to set default values
+# for the parameters in docker-compose.yml.
+
+ORG=arrowdev
+ARCH=amd64
+CUDA=10.0
+DEBIAN=10
+UBUNTU=18.04
+FEDORA=29
+PYTHON=3.6
+RUST=nightly-2019-09-25
+GO=1.12
+NODE=11
+MAVEN=3.5.4
+JDK=8
+R=3.6.1
+PANDAS=latest
+DASK=latest
+TURBODBC=latest
+HDFS=2.9.2
+SPARK=master
+DOTNET=2.1
+R=3.6
diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml
@@ -0,0 +1,179 @@
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: C++
+
+on:
+  push:
+    # paths:
+    #  - 'cpp/**'
+  pull_request:
+    # paths:
+    #   - 'cpp/**'
+
+jobs:
+
+  docker:
+    name: ${{ matrix.name }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        image:
+          - amd64-conda-latest-cpp
+          - amd64-debian-10-cpp
+          # - amd64-ubuntu-16.04-cpp
+          - amd64-ubuntu-18.04-cpp
+          - amd64-ubuntu-18.04-cpp-cmake-3.2
+          - amd64-fedora-29-cpp
+        include:
+          - name: AMD64 Conda C++
+            image: amd64-conda-latest-cpp
+            service: conda-cpp
+          - name: AMD64 Debian 10 C++
+            image: amd64-debian-10-cpp
+            service: debian-cpp
+            debian: 10
+          # TODO(kszucs): earlier GCC has a compile error
+          # - name: AMD64 Ubuntu 16.04 C++
+          #   image: amd64-ubuntu-16.04-cpp
+          #   service: ubuntu-cpp
+          #   ubuntu: 16.04
+          - name: AMD64 Ubuntu 18.04 C++
+            image: amd64-ubuntu-18.04-cpp
+            service: ubuntu-cpp
+            ubuntu: 18.04
+          - name: AMD64 Ubuntu 18.04 C++ CMake 3.2
+            image: amd64-ubuntu-18.04-cpp-cmake-3.2
+            service: ubuntu-cpp-cmake32
+            ubuntu: 18.04
+          - name: AMD64 Fedora 29 C++
+            image: amd64-fedora-29-cpp
+            service: fedora-cpp
+            fedora: 29
+    env:
+      DEBIAN: ${{ matrix.debian || 10 }}
+      UBUNTU: ${{ matrix.ubuntu || 18.04 }}
+      FEDORA: ${{ matrix.fedora || 29 }}
+    steps:
+      - name: Checkout Arrow
+        uses: actions/checkout@v1
+        with:
+          submodules: true
+      - name: Docker Pull
+        shell: bash
+        run: docker-compose pull --ignore-pull-failures ${{ matrix.service }}
+      - name: Docker Build
+        shell: bash
+        run: docker-compose build ${{ matrix.service }}
+      - name: Docker Run
+        shell: bash
+        run: docker-compose run ${{ matrix.service }}
+      - name: Docker Push
+        if: github.event_name == 'push' # && github.ref == 'master'
+        shell: bash
+        run: |
+          docker login \
+            -u ${{ secrets.DOCKERHUB_USER }} \
+            -p ${{ secrets.DOCKERHUB_TOKEN }}
+          docker-compose push ${{ matrix.service }}
+
+  macos:
+    name: AMD64 MacOS 10.15 C++
+    runs-on: macos-latest
+    strategy:
+      fail-fast: false
+    env:
+      ARROW_HOME: /usr/local
+      # TODO(kszucs): investigate why does it fail
+      ARROW_JEMALLOC: OFF
+      ARROW_ORC: OFF
+      ARROW_FLIGHT: ON
+      ARROW_PLASMA: ON
+      ARROW_GANDIVA: ON
+      ARROW_PARQUET: ON
+      ARROW_WITH_ZLIB: ON
+      ARROW_WITH_LZ4: ON
+      ARROW_WITH_BZ2: ON
+      ARROW_WITH_ZSTD: ON
+      ARROW_WITH_SNAPPY: ON
+      ARROW_WITH_BROTLI: ON
+      ARROW_BUILD_TESTS: ON
+    steps:
+      - name: Checkout Arrow
+        uses: actions/checkout@v1
+        with:
+          submodules: true
+      - name: Install Dependencies
+        shell: bash
+        # TODO(kszucs): consider to use run_brew from travis_install_osx.sh
+        run: brew bundle --file=cpp/Brewfile
+      - name: Build
+        shell: bash
+        run: ci/scripts/cpp_build.sh $(pwd) $(pwd)/cpp/build
+      - name: Test
+        shell: bash
+        run: ci/scripts/cpp_test.sh $(pwd) $(pwd)/cpp/build
+
+  windows:
+    name: AMD64 ${{ matrix.name }} C++
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - windows-latest
+        include:
+          - os: windows-latest
+            name: Windows 2019
+            generator: Visual Studio 16 2019
+    env:
+      CMAKE_ARGS: '-A x64'
+      CMAKE_GENERATOR: ${{ matrix.generator }}
+      CMAKE_INSTALL_LIBDIR: bin
+      CMAKE_INSTALL_PREFIX: /usr
+      ARROW_HOME: /usr
+      ARROW_FLIGHT: OFF
+      ARROW_PARQUET: OFF
+      ARROW_WITH_ZLIB: OFF
+      ARROW_WITH_LZ4: OFF
+      ARROW_WITH_BZ2: OFF
+      ARROW_WITH_ZSTD: OFF
+      ARROW_WITH_SNAPPY: OFF
+      ARROW_WITH_BROTLI: OFF
+      ARROW_USE_GLOG: OFF
+      ARROW_BUILD_TESTS: ON
+      ARROW_TEST_LINKAGE: static
+      ARROW_BOOST_USE_SHARED: OFF
+      ARROW_BUILD_SHARED: OFF
+      ARROW_USE_STATIC_CRT: ON
+      ARROW_VERBOSE_THIRDPARTY_BUILD: OFF
+    steps:
+      - name: Checkout Arrow
+        uses: actions/checkout@v1
+        with:
+          submodules: true
+      - name: Dependencies
+        shell: bash
+        run: choco install dependencywalker
+      - name: Build
+        shell: bash
+        run: ci/scripts/cpp_build.sh $(pwd) $(pwd)/cpp/build
+      - name: Test
+        shell: bash
+        run: ci/scripts/cpp_test.sh $(pwd) $(pwd)/cpp/build
diff --git a/.github/workflows/csharp.yml b/.github/workflows/csharp.yml
@@ -0,0 +1,66 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: C#
+
+on:
+  push:
+  #   paths:
+  #     - 'csharp/**'
+  pull_request:
+  #   paths:
+  #     - 'csharp/**'
+
+jobs:
+  # the docker container fails to run because of the ubuntu host versions, see
+  # https://github.com/dotnet/core/issues/3509
+
+  all-platforms:
+    name: AMD64 ${{ matrix.name }} C# ${{ matrix.dotnet }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - ubuntu-18.04
+          - macos-latest
+          - windows-latest
+        include:
+          - os: ubuntu-18.04
+            name: Ubuntu 18.04
+            dotnet: 2.2.103
+          - os: macos-latest
+            name: MacOS 10.15
+            dotnet: 2.2.103
+          - os: windows-latest
+            name: Windows 2019
+            dotnet: 2.2.103
+    steps:
+      - name: Install csharp
+        uses: actions/setup-dotnet@v1
+        with:
+          dotnet-version: ${{ matrix.dotnet }}
+      - name: Checkout Arrow
+        uses: actions/checkout@v1
+        with:
+          submodules: true
+      - name: Build
+        shell: bash
+        run: ci/scripts/csharp_build.sh $(pwd)
+      - name: Test
+        shell: bash
+        run: ci/scripts/csharp_test.sh $(pwd)