ARROW-2676: [Packaging] Deploy build artifacts to github releases

~Added a new task which trigger crossbow builds on master@crossbow.~ ~See travis output https://travis-ci.org/kszucs/crossbow/builds/388667590~ Here are the boxes we need to check: - [x] Create a separate tagged branch that contains a YAML file indicating the information about each task created as part of the run. So there should be one entry for each job that was created -- the git hash for the task, the CI service used to run the task, etc. It should also indicate if one or more artifacts are expected to be uploaded - [x] Write a status tool which can query the status of a particular run and determine if the run is complete (needs cleanup) - [x] Can we run each desired task in a particular CI service - [x] We can determine the list of created tasks associated with a particular run - [x] Tasks should be configured with the tag name, and artifacts should be uploaded to GitHub under the tag which should appear as a release on the repo - [x] Each task can upload its artifacts to a deterministic central location (e.g. GitHub), where the artifacts are not commingled with any other run -> only linux packages are failing, I suggest resolving it in a subsequent PR (issue https://issues.apache.org/jira/browse/ARROW-2713) - [x] ~~We can determine whether all the expected artifacts from a particular run have been successfully uploaded (i.e. to GitHub)~~ to be done in https://issues.apache.org/jira/browse/ARROW-2724 - [x] We can download all the artifacts from a successful run and GPG sign them for purposes of a release vote Example of artifacts available here https://github.com/kszucs/crossbow/releases Jobs and tasks here https://github.com/kszucs/crossbow/branches Job definition here https://github.com/kszucs/crossbow/blob/build-36/job.yml Author: Krisztián Szűcs <[email protected]> Author: Phillip Cloud <[email protected]> Closes #2109 from kszucs/nightly and squashes the following commits: 87860d0 <Krisztián Szűcs> batch correctly rename double extensions 83f333b <Krisztián Szűcs> don't use dirty flag in version number 0701d64 <Krisztián Szűcs> fixed conda-win deployments 062da7c <Krisztián Szűcs> conda win renames b892471 <Krisztián Szűcs> explicit task names instead of placeholder in readme; foxme note for linux-packages e4e971c <Krisztián Szűcs> don't depend on any previous commit 2e9fcf9 <Krisztián Szűcs> remove logging config f317a06 <Krisztián Szűcs> outdated readme section 337234d <Phillip Cloud> Validate submit tasks 12efaba <Phillip Cloud> Validate github token 1b3d5fb <Phillip Cloud> Code formatting cleanups 4c64db6 <Phillip Cloud> Add email property and use target repo user.email by default 9b22238 <Krisztián Szűcs> update tasks 0561e62 <Krisztián Szűcs> force update conda win assets 71b6943 <Krisztián Szűcs> postfix conda pkgs with arch bfb5eae <Krisztián Szűcs> fix osx wheel builkds 95bc3fa <Krisztián Szűcs> retrieve commit's combined status 0773f3a <Krisztián Szűcs> call rake version update af6dd48 <Krisztián Szűcs> remove flag 167a393 <Krisztián Szűcs> gry to remove osx wheel flag ccbc150 <Krisztián Szűcs> little more status context 14da2d5 <Krisztián Szűcs> build prefix fd6bd23 <Krisztián Szűcs> print build id 5c6e154 <Krisztián Szűcs> fix conda-win deployments 8c069c6 <Krisztián Szűcs> set autoincremented job_id outside of queue put 5374866 <Krisztián Szűcs> cleanup status check and artifact download 0b79d13 <Krisztián Szűcs> trying to remove python dependency from parquet-cpp's recipe 96e101a <Krisztián Szűcs> draft for downloading artifacts 55e7ccb <Krisztián Szűcs> query statuses API edeee71 <Krisztián Szűcs> refactooooor bd5ece4 <Krisztián Szűcs> rename trigger build; add license header a0a8127 <Krisztián Szűcs> tigger builds template 8a5fcb0 <Krisztián Szűcs> trigger build task; explicit task names during submission b65cb3f <Krisztián Szűcs> track remote branches 0ab0567 <Krisztián Szűcs> don't create tree based on master
apache · Jun 24, 2018 · 58a2366 · 58a2366
1 parent 6ef9888
commit 58a2366
Show file tree

Hide file tree

Showing 13 changed files with 513 additions and 281 deletions.
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
@@ -343,6 +343,9 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${ARROW_CXXFLAGS}")
 # For any C code, use the same flags.
 set(CMAKE_C_FLAGS "${CMAKE_CXX_FLAGS}")
 
+# Remove --std=c++11 to avoid errors from C compilers
+string(REPLACE "-std=c++11" "" CMAKE_C_FLAGS ${CMAKE_C_FLAGS})
+
 # Add C++-only flags, like -std=c++11
 set(CMAKE_CXX_FLAGS "${CXX_ONLY_FLAGS} ${CMAKE_CXX_FLAGS}")
 

diff --git a/dev/tasks/README.md b/dev/tasks/README.md
@@ -80,12 +80,12 @@ submission. The tasks are defined in `tasks.yml`
 6. Install the python dependencies for the script:
 
    ```bash
-   conda install -y jinja2 pygit2 click pyyaml
+   conda install -y jinja2 pygit2 click pyyaml setuptools_scm github3.py
    ```
 
    ```bash
    # pygit2 requires libgit2: http://www.pygit2.org/install.html
-   pip install -y jinja2 pygit2 click pyyaml
+   pip install -y jinja2 pygit2 click pyyaml setuptools_scm github3.py
    ```
 
 7. Try running it:
@@ -106,7 +106,7 @@ The script does the following:
    $ git clone https://github.com/kszucs/crossbow
 
    $ cd arrow/dev/tasks
-   $ python crossbow.py
+   $ python crossbow.py submit conda-win conda-linux conda-osx
    ```
 
 2. Gets the HEAD commit of the currently checked out branch and generates
@@ -115,94 +115,65 @@ The script does the following:
 
    ```bash
    git checkout ARROW-<ticket number>
-   python dev/tasks/crossbow.py --dry-run
+   python dev/tasks/crossbow.py submit --dry-run conda-linux conda-osx
    ```
 
    > Note that the arrow branch must be pushed beforehand, because the script
    > will clone the selected branch.
 
 3. Reads and renders the required build configurations with the parameters
    substituted.
-2. Create a commit per build configuration to its own branch. For example
-   to build `travis-linux-conda.yml` it will place a commit to the tip of
-   `crossbow@travis-linux-conda` branch.
+2. Create a branch per task, prefixed with the job id. For example
+   to build conda recipes on linux it will create a new branch:
+   `crossbow@build-<id>-conda-linux`.
 3. Pushes the modified branches to GitHub which triggers the builds.
    For authentication it uses github oauth tokens described in the install
    section.
 
 
-### Examples
-
-The script accepts a pattern as a first argument to narrow the build scope:
-
-Run all builds:
+### Query the build status
 
 ```bash
-$ python crossbow.py
-Repository: https://github.com/kszucs/arrow@tasks
-Commit SHA: 810a718836bb3a8cefc053055600bdcc440e6702
-Version: 0.9.1.dev48+g810a7188.d20180414
-Pushed branches:
- - travis-osx-wheel
- - travis-linux-packages
- - travis-linux-wheel
- - appveyor-win-wheel
- - appveyor-win-conda
- - travis-linux-conda
- - travis-osx-conda
+python crossbow.py status <build id / branch name>
 ```
 
-Just render without applying or committing the changes:
+### Download the build artifacts
 
 ```bash
-$ python crossbow.py --dry-run
+python crossbow.py artifacts <build id / branch name>
 ```
 
-Run only `conda` package builds but on all platforms:
+### Examples
+
+The script accepts a pattern as a first argument to narrow the build scope:
+
+Run multiple builds:
 
 ```bash
-$ python crossbow.py conda
+$ python crossbow.py submit linux-packages conda-linux wheel-win
 Repository: https://github.com/kszucs/arrow@tasks
 Commit SHA: 810a718836bb3a8cefc053055600bdcc440e6702
 Version: 0.9.1.dev48+g810a7188.d20180414
 Pushed branches:
- - appveyor-win-conda
- - travis-linux-conda
- - travis-osx-conda
+ - linux-packages
+ - conda-linux
+ - wheel-win
 ```
 
-Run `wheel` builds:
+Just render without applying or committing the changes:
 
 ```bash
-$ python crossbow.py wheel
-Repository: https://github.com/kszucs/arrow@tasks
-Commit SHA: 810a718836bb3a8cefc053055600bdcc440e6702
-Version: 0.9.1.dev48+g810a7188.d20180414
-Pushed branches:
- - travis-osx-wheel
- - travis-linux-wheel
- - appveyor-win-wheel
+$ python crossbow.py submit --dry-run task_name
 ```
 
-Run `osx` builds:
+Run only `conda` package builds but on all platforms:
 
 ```bash
-$ python crossbow.py osx
-Repository: https://github.com/kszucs/arrow@tasks
-Commit SHA: cad1df2c7f650ad3434319bbbefed0d4abe45e4a
-Version: 0.9.1.dev130+gcad1df2c.d20180414
-Pushed branches:
- - travis-osx-wheel
- - travis-osx-conda
+$ python crossbow.py submit conda-win conda-osx conda-linux
 ```
 
-Run only `linux-conda` package build:
+Run `wheel` builds:
 
 ```bash
-$ python crossbow.py linux-conda
-Repository: https://github.com/kszucs/arrow@tasks
-Commit SHA: 810a718836bb3a8cefc053055600bdcc440e6702
-Version: 0.9.1.dev48+g810a7188.d20180414
-Pushed branches:
- - travis-linux-conda
+$ python crossbow.py submit wheel-osx wheel-linux wheel-win
 ```
diff --git a/dev/tasks/conda-recipes/appveyor.yml b/dev/tasks/conda-recipes/appveyor.yml
@@ -23,7 +23,6 @@ environment:
     - TARGET_ARCH: x64
       CONDA_PY: 36
       CONDA_INSTALL_LOCN: C:\\Miniconda36-x64
-  ARROW_SRC: C:\apache-arrow
   ARROW_VERSION: {{ ARROW_VERSION }}
 
 install:
@@ -45,11 +44,28 @@ install:
 build: off
 
 test_script:
-  - git clone -b {{ ARROW_BRANCH }} {{ ARROW_REPO }} %ARROW_SRC% || exit /B
-  - git checkout {{ ARROW_SHA }} || exit /B
-  - pushd %ARROW_SRC%\dev\tasks\conda-recipes
-  - conda.exe build parquet-cpp arrow-cpp pyarrow
+  - git clone -b {{ ARROW_BRANCH }} {{ ARROW_REPO }} arrow || exit /B
+  - git -C arrow checkout {{ ARROW_SHA }} || exit /B
+  - pushd arrow\dev\tasks\conda-recipes
+  - conda.exe build --output-folder . parquet-cpp arrow-cpp pyarrow
+  - pushd win-64
+  - for %%f in (*.tar.bz2) do (
+      set %%g=%%~nf
+      ren "%%f" "%%~ng-win-64.tar.bz2"
+    )
 
+artifacts:
+  # this must be relative and child of the build C:\projects\crossbow directory
+  - path: arrow\dev\tasks\conda-recipes\win-64\*.tar.bz2
+
+deploy:
+  release: {{ BUILD_TAG }}
+  provider: GitHub
+  auth_token: "%CROSSBOW_GITHUB_TOKEN%"
+  artifact: /.*\.tar\.bz2/
+  draft: false
+  prerelease: false
+  force_update: true
 
 notifications:
   - provider: Email

diff --git a/dev/tasks/conda-recipes/parquet-cpp/meta.yaml b/dev/tasks/conda-recipes/parquet-cpp/meta.yaml
@@ -36,26 +36,21 @@ source:
 
 build:
   number: 0
-  skip: true  # [win and not (py35 and win64)]
   features:
-    - vc14  # [win and py35]
+    - vc14  # [win]
 
 requirements:
   build:
     - toolchain
     - boost-cpp 1.66.0
     - cmake
     - thrift-cpp >=0.11
-    - python            # [win]
     - arrow-cpp {{ ARROW_VERSION }}
 
   run:
     - arrow-cpp {{ ARROW_VERSION }}
 
 test:
-  requires:
-    - python {{ environ['PY_VER'] + '*' }}  # [win]
-
   commands:
     - test -f $PREFIX/lib/libparquet.so                                       # [linux]
     - test -f $PREFIX/lib/libparquet.dylib                                    # [osx]

diff --git a/dev/tasks/conda-recipes/travis.linux.yml b/dev/tasks/conda-recipes/travis.linux.yml
@@ -26,6 +26,7 @@ env:
     - CONDA_PY=35
     - CONDA_PY=36
   global:
+    - TRAVIS_TAG={{ BUILD_TAG }}
     - ARROW_VERSION={{ ARROW_VERSION }}
 
 install:
@@ -55,7 +56,22 @@ script:
   - git clone -b {{ ARROW_BRANCH }} {{ ARROW_REPO }} arrow
   - git -C arrow checkout {{ ARROW_SHA }}
   - pushd arrow/dev/tasks/conda-recipes
-  - conda build parquet-cpp arrow-cpp pyarrow
+  - conda build --output-folder . parquet-cpp arrow-cpp pyarrow
+  - |
+    pushd linux-64
+    for file in *.tar.bz2; do
+      mv "$file" "$(basename "$file" .tar.bz2)-linux-64.tar.bz2"
+    done
+    popd
+
+deploy:
+  provider: releases
+  api_key: $CROSSBOW_GITHUB_TOKEN
+  file_glob: true
+  file: $TRAVIS_BUILD_DIR/arrow/dev/tasks/conda-recipes/linux-64/*.tar.bz2
+  skip_cleanup: true
+  on:
+    tags: true
 
 notifications:
   email:

diff --git a/dev/tasks/conda-recipes/travis.osx.yml b/dev/tasks/conda-recipes/travis.osx.yml
@@ -26,6 +26,7 @@ env:
     - CONDA_PY=35
     - CONDA_PY=36
   global:
+    - TRAVIS_TAG={{ BUILD_TAG }}
     - ARROW_VERSION={{ ARROW_VERSION }}
 
 before_install:
@@ -64,7 +65,22 @@ script:
   - git clone -b {{ ARROW_BRANCH }} {{ ARROW_REPO }} arrow
   - git -C arrow checkout {{ ARROW_SHA }}
   - pushd arrow/dev/tasks/conda-recipes
-  - conda build parquet-cpp arrow-cpp pyarrow
+  - conda build --output-folder . parquet-cpp arrow-cpp pyarrow
+  - |
+    pushd osx-64
+    for file in *.tar.bz2; do
+      mv "$file" "$(basename "$file" .tar.bz2)-osx-64.tar.bz2"
+    done
+    popd
+
+deploy:
+  provider: releases
+  api_key: $CROSSBOW_GITHUB_TOKEN
+  file_glob: true
+  file: $TRAVIS_BUILD_DIR/arrow/dev/tasks/conda-recipes/osx-64/*.tar.bz2
+  skip_cleanup: true
+  on:
+    tags: true
 
 notifications:
   email:

diff --git a/dev/tasks/config-nightlies/travis.linux.yml b/dev/tasks/config-nightlies/travis.linux.yml
@@ -0,0 +1,67 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+branches:
+  # don't attempt to build branches intented for windows builds
+  except:
+    - /.*win.*/
+
+os: linux
+dist: trusty
+language: generic
+
+before_install:
+    # Install Miniconda.
+    - echo `pwd`
+    - |
+      echo ""
+      echo "Installing a fresh version of Miniconda."
+      MINICONDA_URL="https://repo.continuum.io/miniconda"
+      MINICONDA_FILE="Miniconda3-latest-Linux-x86_64.sh"
+      curl -L -O "${MINICONDA_URL}/${MINICONDA_FILE}"
+      bash $MINICONDA_FILE -b
+
+    # Configure conda.
+    - |
+      echo ""
+      echo "Configuring conda."
+      source /home/travis/miniconda3/bin/activate root
+      conda config --remove channels defaults
+      conda config --add channels defaults
+      conda config --add channels conda-forge
+      conda config --set show_channel_urls true
+
+install:
+  - conda install -y -q jinja2 pygit2 click pyyaml setuptools_scm
+
+script:
+  # fetch all branches of crossbow
+  - git config remote.origin.fetch +refs/heads/*:refs/remotes/origin/*
+
+  # clone arrow with crossbow tool
+  - pushd ..
+  - git clone -b {{ ARROW_BRANCH }} {{ ARROW_REPO }}
+
+  # submit packaging tasks
+  - |
+    python arrow/dev/tasks/crossbow.py \
+      conda-linux \
+      conda-win \
+      conda-osx \
+      wheel-linux \
+      wheel-win \
+      wheel-osx \
+      linux-packages