From 58a23668fe2cb518f7ecb41bd16886dba82ff5d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Sun, 24 Jun 2018 14:32:47 -0400 Subject: [PATCH] ARROW-2676: [Packaging] Deploy build artifacts to github releases MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ~Added a new task which trigger crossbow builds on master@crossbow.~ ~See travis output https://travis-ci.org/kszucs/crossbow/builds/388667590~ Here are the boxes we need to check: - [x] Create a separate tagged branch that contains a YAML file indicating the information about each task created as part of the run. So there should be one entry for each job that was created -- the git hash for the task, the CI service used to run the task, etc. It should also indicate if one or more artifacts are expected to be uploaded - [x] Write a status tool which can query the status of a particular run and determine if the run is complete (needs cleanup) - [x] Can we run each desired task in a particular CI service - [x] We can determine the list of created tasks associated with a particular run - [x] Tasks should be configured with the tag name, and artifacts should be uploaded to GitHub under the tag which should appear as a release on the repo - [x] Each task can upload its artifacts to a deterministic central location (e.g. GitHub), where the artifacts are not commingled with any other run -> only linux packages are failing, I suggest resolving it in a subsequent PR (issue https://issues.apache.org/jira/browse/ARROW-2713) - [x] ~~We can determine whether all the expected artifacts from a particular run have been successfully uploaded (i.e. to GitHub)~~ to be done in https://issues.apache.org/jira/browse/ARROW-2724 - [x] We can download all the artifacts from a successful run and GPG sign them for purposes of a release vote Example of artifacts available here https://github.com/kszucs/crossbow/releases Jobs and tasks here https://github.com/kszucs/crossbow/branches Job definition here https://github.com/kszucs/crossbow/blob/build-36/job.yml Author: Krisztián Szűcs Author: Phillip Cloud Closes #2109 from kszucs/nightly and squashes the following commits: 87860d00 batch correctly rename double extensions 83f333b5 don't use dirty flag in version number 0701d640 fixed conda-win deployments 062da7c7 conda win renames b892471c explicit task names instead of placeholder in readme; foxme note for linux-packages e4e971c2 don't depend on any previous commit 2e9fcf9a remove logging config f317a06b outdated readme section 337234d2 Validate submit tasks 12efaba6 Validate github token 1b3d5fb5 Code formatting cleanups 4c64db6f Add email property and use target repo user.email by default 9b222380 update tasks 0561e622 force update conda win assets 71b69436 postfix conda pkgs with arch bfb5eaec fix osx wheel builkds 95bc3faf retrieve commit's combined status 0773f3ab call rake version update af6dd480 remove flag 167a3938 gry to remove osx wheel flag ccbc1508 little more status context 14da2d5d build prefix fd6bd231 print build id 5c6e1541 fix conda-win deployments 8c069c6b set autoincremented job_id outside of queue put 53748668 cleanup status check and artifact download 0b79d13d trying to remove python dependency from parquet-cpp's recipe 96e101af draft for downloading artifacts 55e7ccb2 query statuses API edeee717 refactooooor bd5ece4a rename trigger build; add license header a0a81275 tigger builds template 8a5fcb01 trigger build task; explicit task names during submission b65cb3f2 track remote branches 0ab0567f don't create tree based on master --- cpp/CMakeLists.txt | 3 + dev/tasks/README.md | 81 +-- dev/tasks/conda-recipes/appveyor.yml | 26 +- dev/tasks/conda-recipes/parquet-cpp/meta.yaml | 7 +- dev/tasks/conda-recipes/travis.linux.yml | 18 +- dev/tasks/conda-recipes/travis.osx.yml | 18 +- dev/tasks/config-nightlies/travis.linux.yml | 67 +++ dev/tasks/crossbow.py | 487 +++++++++++------- dev/tasks/linux-packages/travis.linux.yml | 16 +- dev/tasks/python-wheels/appveyor.yml | 9 +- dev/tasks/python-wheels/travis.linux.yml | 10 + dev/tasks/python-wheels/travis.osx.yml | 21 +- dev/tasks/tasks.yml | 31 +- 13 files changed, 513 insertions(+), 281 deletions(-) create mode 100644 dev/tasks/config-nightlies/travis.linux.yml diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 5f29bdaa15c1b..9ac59e97df242 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -343,6 +343,9 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${ARROW_CXXFLAGS}") # For any C code, use the same flags. set(CMAKE_C_FLAGS "${CMAKE_CXX_FLAGS}") +# Remove --std=c++11 to avoid errors from C compilers +string(REPLACE "-std=c++11" "" CMAKE_C_FLAGS ${CMAKE_C_FLAGS}) + # Add C++-only flags, like -std=c++11 set(CMAKE_CXX_FLAGS "${CXX_ONLY_FLAGS} ${CMAKE_CXX_FLAGS}") diff --git a/dev/tasks/README.md b/dev/tasks/README.md index f1f0128f03b6e..35140fa44b33e 100644 --- a/dev/tasks/README.md +++ b/dev/tasks/README.md @@ -80,12 +80,12 @@ submission. The tasks are defined in `tasks.yml` 6. Install the python dependencies for the script: ```bash - conda install -y jinja2 pygit2 click pyyaml + conda install -y jinja2 pygit2 click pyyaml setuptools_scm github3.py ``` ```bash # pygit2 requires libgit2: http://www.pygit2.org/install.html - pip install -y jinja2 pygit2 click pyyaml + pip install -y jinja2 pygit2 click pyyaml setuptools_scm github3.py ``` 7. Try running it: @@ -106,7 +106,7 @@ The script does the following: $ git clone https://github.com/kszucs/crossbow $ cd arrow/dev/tasks - $ python crossbow.py + $ python crossbow.py submit conda-win conda-linux conda-osx ``` 2. Gets the HEAD commit of the currently checked out branch and generates @@ -115,7 +115,7 @@ The script does the following: ```bash git checkout ARROW- - python dev/tasks/crossbow.py --dry-run + python dev/tasks/crossbow.py submit --dry-run conda-linux conda-osx ``` > Note that the arrow branch must be pushed beforehand, because the script @@ -123,86 +123,57 @@ The script does the following: 3. Reads and renders the required build configurations with the parameters substituted. -2. Create a commit per build configuration to its own branch. For example - to build `travis-linux-conda.yml` it will place a commit to the tip of - `crossbow@travis-linux-conda` branch. +2. Create a branch per task, prefixed with the job id. For example + to build conda recipes on linux it will create a new branch: + `crossbow@build--conda-linux`. 3. Pushes the modified branches to GitHub which triggers the builds. For authentication it uses github oauth tokens described in the install section. -### Examples - -The script accepts a pattern as a first argument to narrow the build scope: - -Run all builds: +### Query the build status ```bash -$ python crossbow.py -Repository: https://github.com/kszucs/arrow@tasks -Commit SHA: 810a718836bb3a8cefc053055600bdcc440e6702 -Version: 0.9.1.dev48+g810a7188.d20180414 -Pushed branches: - - travis-osx-wheel - - travis-linux-packages - - travis-linux-wheel - - appveyor-win-wheel - - appveyor-win-conda - - travis-linux-conda - - travis-osx-conda +python crossbow.py status ``` -Just render without applying or committing the changes: +### Download the build artifacts ```bash -$ python crossbow.py --dry-run +python crossbow.py artifacts ``` -Run only `conda` package builds but on all platforms: +### Examples + +The script accepts a pattern as a first argument to narrow the build scope: + +Run multiple builds: ```bash -$ python crossbow.py conda +$ python crossbow.py submit linux-packages conda-linux wheel-win Repository: https://github.com/kszucs/arrow@tasks Commit SHA: 810a718836bb3a8cefc053055600bdcc440e6702 Version: 0.9.1.dev48+g810a7188.d20180414 Pushed branches: - - appveyor-win-conda - - travis-linux-conda - - travis-osx-conda + - linux-packages + - conda-linux + - wheel-win ``` -Run `wheel` builds: +Just render without applying or committing the changes: ```bash -$ python crossbow.py wheel -Repository: https://github.com/kszucs/arrow@tasks -Commit SHA: 810a718836bb3a8cefc053055600bdcc440e6702 -Version: 0.9.1.dev48+g810a7188.d20180414 -Pushed branches: - - travis-osx-wheel - - travis-linux-wheel - - appveyor-win-wheel +$ python crossbow.py submit --dry-run task_name ``` -Run `osx` builds: +Run only `conda` package builds but on all platforms: ```bash -$ python crossbow.py osx -Repository: https://github.com/kszucs/arrow@tasks -Commit SHA: cad1df2c7f650ad3434319bbbefed0d4abe45e4a -Version: 0.9.1.dev130+gcad1df2c.d20180414 -Pushed branches: - - travis-osx-wheel - - travis-osx-conda +$ python crossbow.py submit conda-win conda-osx conda-linux ``` -Run only `linux-conda` package build: +Run `wheel` builds: ```bash -$ python crossbow.py linux-conda -Repository: https://github.com/kszucs/arrow@tasks -Commit SHA: 810a718836bb3a8cefc053055600bdcc440e6702 -Version: 0.9.1.dev48+g810a7188.d20180414 -Pushed branches: - - travis-linux-conda +$ python crossbow.py submit wheel-osx wheel-linux wheel-win ``` diff --git a/dev/tasks/conda-recipes/appveyor.yml b/dev/tasks/conda-recipes/appveyor.yml index 10ef7044cebb9..3d3f3305e8662 100644 --- a/dev/tasks/conda-recipes/appveyor.yml +++ b/dev/tasks/conda-recipes/appveyor.yml @@ -23,7 +23,6 @@ environment: - TARGET_ARCH: x64 CONDA_PY: 36 CONDA_INSTALL_LOCN: C:\\Miniconda36-x64 - ARROW_SRC: C:\apache-arrow ARROW_VERSION: {{ ARROW_VERSION }} install: @@ -45,11 +44,28 @@ install: build: off test_script: - - git clone -b {{ ARROW_BRANCH }} {{ ARROW_REPO }} %ARROW_SRC% || exit /B - - git checkout {{ ARROW_SHA }} || exit /B - - pushd %ARROW_SRC%\dev\tasks\conda-recipes - - conda.exe build parquet-cpp arrow-cpp pyarrow + - git clone -b {{ ARROW_BRANCH }} {{ ARROW_REPO }} arrow || exit /B + - git -C arrow checkout {{ ARROW_SHA }} || exit /B + - pushd arrow\dev\tasks\conda-recipes + - conda.exe build --output-folder . parquet-cpp arrow-cpp pyarrow + - pushd win-64 + - for %%f in (*.tar.bz2) do ( + set %%g=%%~nf + ren "%%f" "%%~ng-win-64.tar.bz2" + ) +artifacts: + # this must be relative and child of the build C:\projects\crossbow directory + - path: arrow\dev\tasks\conda-recipes\win-64\*.tar.bz2 + +deploy: + release: {{ BUILD_TAG }} + provider: GitHub + auth_token: "%CROSSBOW_GITHUB_TOKEN%" + artifact: /.*\.tar\.bz2/ + draft: false + prerelease: false + force_update: true notifications: - provider: Email diff --git a/dev/tasks/conda-recipes/parquet-cpp/meta.yaml b/dev/tasks/conda-recipes/parquet-cpp/meta.yaml index 0f5a619010a72..e7be2a1c7d7cd 100644 --- a/dev/tasks/conda-recipes/parquet-cpp/meta.yaml +++ b/dev/tasks/conda-recipes/parquet-cpp/meta.yaml @@ -36,9 +36,8 @@ source: build: number: 0 - skip: true # [win and not (py35 and win64)] features: - - vc14 # [win and py35] + - vc14 # [win] requirements: build: @@ -46,16 +45,12 @@ requirements: - boost-cpp 1.66.0 - cmake - thrift-cpp >=0.11 - - python # [win] - arrow-cpp {{ ARROW_VERSION }} run: - arrow-cpp {{ ARROW_VERSION }} test: - requires: - - python {{ environ['PY_VER'] + '*' }} # [win] - commands: - test -f $PREFIX/lib/libparquet.so # [linux] - test -f $PREFIX/lib/libparquet.dylib # [osx] diff --git a/dev/tasks/conda-recipes/travis.linux.yml b/dev/tasks/conda-recipes/travis.linux.yml index c5266810c6dff..84ca83b27a0e9 100644 --- a/dev/tasks/conda-recipes/travis.linux.yml +++ b/dev/tasks/conda-recipes/travis.linux.yml @@ -26,6 +26,7 @@ env: - CONDA_PY=35 - CONDA_PY=36 global: + - TRAVIS_TAG={{ BUILD_TAG }} - ARROW_VERSION={{ ARROW_VERSION }} install: @@ -55,7 +56,22 @@ script: - git clone -b {{ ARROW_BRANCH }} {{ ARROW_REPO }} arrow - git -C arrow checkout {{ ARROW_SHA }} - pushd arrow/dev/tasks/conda-recipes - - conda build parquet-cpp arrow-cpp pyarrow + - conda build --output-folder . parquet-cpp arrow-cpp pyarrow + - | + pushd linux-64 + for file in *.tar.bz2; do + mv "$file" "$(basename "$file" .tar.bz2)-linux-64.tar.bz2" + done + popd + +deploy: + provider: releases + api_key: $CROSSBOW_GITHUB_TOKEN + file_glob: true + file: $TRAVIS_BUILD_DIR/arrow/dev/tasks/conda-recipes/linux-64/*.tar.bz2 + skip_cleanup: true + on: + tags: true notifications: email: diff --git a/dev/tasks/conda-recipes/travis.osx.yml b/dev/tasks/conda-recipes/travis.osx.yml index 8a38d333489cc..31c1c244b5f4f 100644 --- a/dev/tasks/conda-recipes/travis.osx.yml +++ b/dev/tasks/conda-recipes/travis.osx.yml @@ -26,6 +26,7 @@ env: - CONDA_PY=35 - CONDA_PY=36 global: + - TRAVIS_TAG={{ BUILD_TAG }} - ARROW_VERSION={{ ARROW_VERSION }} before_install: @@ -64,7 +65,22 @@ script: - git clone -b {{ ARROW_BRANCH }} {{ ARROW_REPO }} arrow - git -C arrow checkout {{ ARROW_SHA }} - pushd arrow/dev/tasks/conda-recipes - - conda build parquet-cpp arrow-cpp pyarrow + - conda build --output-folder . parquet-cpp arrow-cpp pyarrow + - | + pushd osx-64 + for file in *.tar.bz2; do + mv "$file" "$(basename "$file" .tar.bz2)-osx-64.tar.bz2" + done + popd + +deploy: + provider: releases + api_key: $CROSSBOW_GITHUB_TOKEN + file_glob: true + file: $TRAVIS_BUILD_DIR/arrow/dev/tasks/conda-recipes/osx-64/*.tar.bz2 + skip_cleanup: true + on: + tags: true notifications: email: diff --git a/dev/tasks/config-nightlies/travis.linux.yml b/dev/tasks/config-nightlies/travis.linux.yml new file mode 100644 index 0000000000000..bf2774b2ed50d --- /dev/null +++ b/dev/tasks/config-nightlies/travis.linux.yml @@ -0,0 +1,67 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +branches: + # don't attempt to build branches intented for windows builds + except: + - /.*win.*/ + +os: linux +dist: trusty +language: generic + +before_install: + # Install Miniconda. + - echo `pwd` + - | + echo "" + echo "Installing a fresh version of Miniconda." + MINICONDA_URL="https://repo.continuum.io/miniconda" + MINICONDA_FILE="Miniconda3-latest-Linux-x86_64.sh" + curl -L -O "${MINICONDA_URL}/${MINICONDA_FILE}" + bash $MINICONDA_FILE -b + + # Configure conda. + - | + echo "" + echo "Configuring conda." + source /home/travis/miniconda3/bin/activate root + conda config --remove channels defaults + conda config --add channels defaults + conda config --add channels conda-forge + conda config --set show_channel_urls true + +install: + - conda install -y -q jinja2 pygit2 click pyyaml setuptools_scm + +script: + # fetch all branches of crossbow + - git config remote.origin.fetch +refs/heads/*:refs/remotes/origin/* + + # clone arrow with crossbow tool + - pushd .. + - git clone -b {{ ARROW_BRANCH }} {{ ARROW_REPO }} + + # submit packaging tasks + - | + python arrow/dev/tasks/crossbow.py \ + conda-linux \ + conda-win \ + conda-osx \ + wheel-linux \ + wheel-win \ + wheel-osx \ + linux-packages diff --git a/dev/tasks/crossbow.py b/dev/tasks/crossbow.py index fd4f732ea823f..a2a29ebad4801 100755 --- a/dev/tasks/crossbow.py +++ b/dev/tasks/crossbow.py @@ -17,12 +17,13 @@ # specific language governing permissions and limitations # under the License. +import os import re -import sys import yaml import time import click import pygit2 +import github3 import logging from enum import Enum @@ -30,15 +31,6 @@ from textwrap import dedent from jinja2 import Template from setuptools_scm import get_version -from setuptools_scm.version import simplified_semver_version, meta - - -logging.basicConfig( - level=logging.INFO, - format="[%(asctime)s] %(levelname)s Crossbow %(message)s", - datefmt="%H:%M:%S", - stream=click.get_text_stream('stdout') -) class GitRemoteCallbacks(pygit2.RemoteCallbacks): @@ -70,134 +62,81 @@ def credentials(self, url, username_from_url, allowed_types): return None -class Platform(Enum): - # in alphabetical order - LINUX = 0 - OSX = 1 - WIN = 2 - - @property - def ci(self): - if self is self.WIN: - return 'appveyor' - else: - return 'travis' - - @property - def filename(self): - if self.ci == 'appveyor': - return 'appveyor.yml' - else: - return '.travis.yml' - +class Repo(object): -class Target(object): - - def __init__(self, repo_path, template_directory=None): + def __init__(self, repo_path): self.path = Path(repo_path).absolute() - - # relative to repository's path - if template_directory is None: - self.templates = self.path - else: - self.templates = self.path / template_directory - - # initialize a repo object to interact with arrow's git data self.repo = pygit2.Repository(str(self.path)) - msg = dedent(''' - Repository: {remote}@{branch} - Commit SHA: {sha} - Version: {version} + def __str__(self): + tpl = dedent(''' + Repo: {remote}@{branch} + Commit: {head} ''') - logging.info(msg.format( - remote=self.current_remote.url, - branch=self.current_branch.branch_name, - sha=self.sha, - version=self.version - )) + return tpl.format( + remote=self.remote.url, + branch=self.branch.branch_name, + head=self.head + ) + + def fetch(self): + self.origin.fetch() @property - def sha(self): + def head(self): """Currently checked out commit's sha""" return self.repo.head.target @property - def version(self): - """Generate version number based on version control history""" - # TODO(kszucs) use self.repo.describe() instead - return get_version(self.path) + def branch(self): + """Currently checked out branch""" + reference = self.repo.head.shorthand + return self.repo.branches[reference] @property - def current_remote(self): - remote_name = self.current_branch.upstream.remote_name + def remote(self): + """Currently checked out branch's remote counterpart""" + remote_name = self.branch.upstream.remote_name return self.repo.remotes[remote_name] @property - def current_branch(self): - reference = self.repo.head.shorthand - return self.repo.branches[reference] + def origin(self): + return self.repo.remotes['origin'] @property - def description(self): - return '[BUILD] {} of {}@{}'.format(self.version, - self.current_remote.url, - self.current_branch.branch_name) - - -class Build(object): - - def __init__(self, target, name, platform, template, **params): - assert isinstance(target, Target) - assert isinstance(platform, Platform) - - self.name = name - self.target = target - self.platform = platform - self.template = template - self.params = params - - def render(self): - path = Path(self.template) - template = Template(path.read_text()) - return template.render(**self.params) - - def config_files(self): - return {self.platform.filename: self.render()} + def email(self): + return next(self.repo.config.get_multivar('user.email')) @property - def branch(self): - return self.name + def signature(self): + name = next(self.repo.config.get_multivar('user.name')) + return pygit2.Signature(name, self.email, int(time.time())) - @property - def description(self): - return self.target.description + def parse_user_repo(self): + m = re.match('.*\/([^\/]+)\/([^\/\.]+)(\.git)?$', self.remote.url) + user, repo = m.group(1), m.group(2) + return user, repo -class Queue(object): +class Queue(Repo): def __init__(self, repo_path): - self.path = Path(repo_path).absolute() - self.repo = pygit2.Repository(str(self.path)) - self.updated_branches = [] - - def _get_parent_commit(self): - """Currently this always returns the HEAD of master""" - master = self.repo.branches['master'] - return self.repo[master.target] - - def _get_or_create_branch(self, name): - try: - return self.repo.branches[name] - except KeyError: - parent = self._get_parent_commit() - return self.repo.branches.create(name, parent) - - def _create_tree(self, files): - parent = self._get_parent_commit() + super(Queue, self).__init__(repo_path) + self._updated_refs = [] + + def next_job_id(self, prefix): + """Auto increments the branch's identifier based on the prefix""" + pattern = re.compile(prefix + '-(\d+)') + matches = list(filter(None, map(pattern.match, self.repo.branches))) + if matches: + latest = max(int(m.group(1)) for m in matches) + else: + latest = 0 + return '{}-{}'.format(prefix, latest + 1) - # creating the tree we are going to push based on master's tree - builder = self.repo.TreeBuilder(parent.tree) + def _create_branch(self, branch_name, files, parents=[], message=''): + # 1. create tree + builder = self.repo.TreeBuilder() for filename, content in files.items(): # insert the file and creating the new filetree @@ -205,119 +144,293 @@ def _create_tree(self, files): builder.insert(filename, blob_id, pygit2.GIT_FILEMODE_BLOB) tree_id = builder.write() - return tree_id - def put(self, build): - assert isinstance(build, Build) + # 2. create commit with the tree created above + author = committer = self.signature + commit_id = self.repo.create_commit(None, author, committer, message, + tree_id, parents) + commit = self.repo[commit_id] - branch = self._get_or_create_branch(build.branch) - tree_id = self._create_tree(build.config_files()) + # 3. create branch pointing to the previously created commit + branch = self.repo.create_branch(branch_name, commit) + # append to the pushable references + self._updated_refs.append('refs/heads/{}'.format(branch_name)) - # creating the new commit - timestamp = int(time.time()) + return branch - name = next(self.repo.config.get_multivar('user.name')) - email = next(self.repo.config.get_multivar('user.email')) + def _create_tag(self, tag_name, commit_id, message=''): + tag_id = self.repo.create_tag(tag_name, commit_id, + pygit2.GIT_OBJ_COMMIT, self.signature, + message) + + # append to the pushable references + self._updated_refs.append('refs/tags/{}'.format(tag_name)) + + return self.repo[tag_id] + + def put(self, job, prefix='build'): + assert isinstance(job, Job) + assert job.branch is not None - author = pygit2.Signature('crossbow', 'mailing@list.com', - int(timestamp)) - committer = pygit2.Signature(name, email, int(timestamp)) - message = build.description + # create tasks' branches + for task_name, task in job.tasks.items(): + branch = self._create_branch(task.branch, files=task.files()) + task.commit = str(branch.target) - reference = 'refs/heads/{}'.format(branch.branch_name) - commit_id = self.repo.create_commit(reference, author, committer, - message, tree_id, [branch.target]) - logging.info('{} created on {}'.format( - commit_id, branch.branch_name)) + # create job's branch + branch = self._create_branch(job.branch, files=job.files()) + self._create_tag(job.branch, branch.target) - self.updated_branches.append(branch) + return branch def push(self, token): callbacks = GitRemoteCallbacks(token) + self.origin.push(self._updated_refs, callbacks=callbacks) + self.updated_refs = [] - remote = self.repo.remotes['origin'] - refs = [branch.name for branch in self.updated_branches] - shorthands = [b.shorthand for b in self.updated_branches] - remote.push(refs, callbacks=callbacks) - self.updated_branches = [] +class Platform(Enum): + # in alphabetical order + LINUX = 0 + OSX = 1 + WIN = 2 + + @property + def ci(self): + if self is self.WIN: + return 'appveyor' + else: + return 'travis' + + @property + def filename(self): + if self.ci == 'appveyor': + return 'appveyor.yml' + else: + return '.travis.yml' + + +class Task(object): + + def __init__(self, platform, template, commit=None, branch=None, **params): + assert isinstance(platform, Platform) + assert isinstance(template, Path) + self.platform = platform + self.template = template + self.branch = branch + self.commit = commit + self.params = params + + def to_dict(self): + return {'branch': self.branch, + 'commit': str(self.commit), + 'platform': self.platform.name, + 'template': str(self.template), + 'params': self.params} + + @classmethod + def from_dict(cls, data): + return Task(platform=Platform[data['platform'].upper()], + template=Path(data['template']), + commit=data.get('commit'), + branch=data.get('branch'), + **data.get('params', {})) + + def files(self): + template = Template(self.template.read_text()) + rendered = template.render(**self.params) + return {self.platform.filename: rendered} + + +class Job(object): + + def __init__(self, tasks, branch=None): + assert all(isinstance(task, Task) for task in tasks.values()) + self.branch = branch + self.tasks = tasks + + def to_dict(self): + tasks = {name: task.to_dict() for name, task in self.tasks.items()} + return {'branch': self.branch, + 'tasks': tasks} - logging.info('\n - '.join(['\nUpdated branches:'] + shorthands)) + @classmethod + def from_dict(cls, data): + tasks = {name: Task.from_dict(task) + for name, task in data['tasks'].items()} + return Job(tasks=tasks, branch=data.get('branch')) + + def files(self): + return {'job.yml': yaml.dump(self.to_dict(), default_flow_style=False)} # this should be the mailing list MESSAGE_EMAIL = 'szucs.krisztian@gmail.com' +CWD = Path(__file__).absolute() + +DEFAULT_CONFIG_PATH = CWD.parent / 'tasks.yml' +DEFAULT_ARROW_PATH = CWD.parents[2] +DEFAULT_QUEUE_PATH = CWD.parents[3] / 'crossbow' + + +@click.group() +def crossbow(): + pass -@click.command() -@click.argument('task-regex', required=False) -@click.option('--config', help='Task configuration yml. Defaults to tasks.yml') + +def github_token_validation_callback(ctx, param, value): + if value is None: + raise click.ClickException( + 'Could not determine GitHub token. Please set the ' + 'CROSSBOW_GITHUB_TOKEN environment variable to a ' + 'valid github access token or pass one to --github-token.' + ) + return value + + +github_token = click.option( + '--github-token', + default=None, + envvar='CROSSBOW_GITHUB_TOKEN', + help='OAuth token for Github authentication', + callback=github_token_validation_callback, +) + + +def config_path_validation_callback(ctx, param, value): + with Path(value).open() as fp: + config = yaml.load(fp) + task_names = ctx.params['task_names'] + valid_tasks = set(config['tasks'].keys()) + invalid_tasks = {task for task in task_names if task not in valid_tasks} + if invalid_tasks: + raise click.ClickException( + 'Invalid task(s) {!r}. Must be one of {!r}'.format( + invalid_tasks, + valid_tasks + ) + ) + return value + + +@crossbow.command() +@click.argument('task-names', nargs=-1, required=True) +@click.option('--job-prefix', default='build', + help='Arbitrary prefix for branch names, e.g. nightly') +@click.option('--config-path', default=DEFAULT_CONFIG_PATH, + type=click.Path(exists=True), + callback=config_path_validation_callback, + help='Task configuration yml. Defaults to tasks.yml') @click.option('--dry-run/--push', default=False, help='Just display the rendered CI configurations without ' 'submitting them') -@click.option('--arrow-repo', default=None, +@click.option('--arrow-path', default=DEFAULT_ARROW_PATH, help='Arrow\'s repository path. Defaults to the repository of ' 'this script') -@click.option('--queue-repo', default=None, +@click.option('--queue-path', default=DEFAULT_QUEUE_PATH, help='The repository path used for scheduling the tasks. ' 'Defaults to crossbow directory placed next to arrow') -@click.option('--github-token', default=False, - help='Oauth token for Github authentication') -def build(task_regex, config, dry_run, arrow_repo, queue_repo, github_token): - if config is None: - config = Path(__file__).absolute().parent / 'tasks.yml' - else: - config = Path(config) - - if arrow_repo is None: - arrow_repo = Path(__file__).absolute().parents[2] - else: - arrow_repo = Path(arrow_repo) - - if queue_repo is None: - queue_repo = arrow_repo.parent / 'crossbow' - else: - queue_repo = Path(queue_repo) - - arrow = Target(arrow_repo, template_directory='cd') - queue = Queue(queue_repo) +@github_token +def submit(task_names, job_prefix, config_path, dry_run, arrow_path, + queue_path, github_token): + target = Repo(arrow_path) + queue = Queue(queue_path) + + logging.info(target) + logging.info(queue) + + queue.fetch() + + version = get_version(arrow_path, local_scheme=lambda v: '') + job_id = queue.next_job_id(prefix=job_prefix) variables = { # these should be renamed 'PLAT': 'x86_64', - 'EMAIL': MESSAGE_EMAIL, - 'BUILD_REF': arrow.sha, - 'ARROW_SHA': arrow.sha, - 'ARROW_REPO': arrow.current_remote.url, - 'ARROW_BRANCH': arrow.current_branch.branch_name, - 'ARROW_VERSION': arrow.version, - 'PYARROW_VERSION': arrow.version, + 'EMAIL': os.environ.get('CROSSBOW_EMAIL', target.email), + 'BUILD_TAG': job_id, + 'BUILD_REF': str(target.head), + 'ARROW_SHA': str(target.head), + 'ARROW_REPO': target.remote.url, + 'ARROW_BRANCH': target.branch.branch_name, + 'ARROW_VERSION': version, + 'PYARROW_VERSION': version, } - with config.open() as fp: - tasks = yaml.load(fp)['tasks'] + with Path(config_path).open() as fp: + config = yaml.load(fp) + + # create and filter tasks + tasks = {name: Task.from_dict(task) + for name, task in config['tasks'].items()} + tasks = {name: tasks[name] for name in task_names} - for task in tasks: - name = task['name'] - template = config.parent / task['template'] - platform = Platform[task['platform'].upper()] - params = task.get('params') or {} - params.update(variables) + for task_name, task in tasks.items(): + task.branch = '{}-{}'.format(job_id, task_name) + task.params.update(variables) - build = Build(arrow, name=name, platform=platform, template=template, - **params) + # create job + job = Job(tasks) + job.branch = job_id - # Regex pattern the task name is matched against - if task_regex is None or re.search(task_regex, build.name): - if dry_run: - logging.info('{}\n\n{}'.format(build.name, build.render())) - else: - queue.put(build) # create the commit + yaml_format = yaml.dump(job.to_dict(), default_flow_style=False) + click.echo(yaml_format.strip()) if not dry_run: - # push the changed branches + queue.put(job) queue.push(token=github_token) + click.echo('Pushed job identifier is: `{}`'.format(job_id)) + + +@crossbow.command() +@click.argument('job-name', required=True) +@click.option('--queue-path', default=DEFAULT_QUEUE_PATH, + help='The repository path used for scheduling the tasks. ' + 'Defaults to crossbow directory placed next to arrow') +@github_token +def status(job_name, queue_path, github_token): + queue = Queue(queue_path) + username, reponame = queue.parse_user_repo() + + gh = github3.login(token=github_token) + repo = gh.repository(username, reponame) + content = repo.file_contents('job.yml', job_name) + + job = Job.from_dict(yaml.load(content.decoded)) + + tpl = '[{:>7}] {:<24} {:<40}' + header = tpl.format('status', 'branch', 'sha') + click.echo(header) + click.echo('-' * len(header)) + + for name, task in job.tasks.items(): + commit = repo.commit(task.commit) + status = commit.status() + + click.echo(tpl.format(status.state, task.branch, task.commit)) + + +@crossbow.command() +@click.argument('job-name', required=True) +@click.option('--target-dir', default=DEFAULT_ARROW_PATH, + help='Directory to download the build artifacts') +@click.option('--queue-path', default=DEFAULT_QUEUE_PATH, + help='The repository path used for scheduling the tasks. ' + 'Defaults to crossbow directory placed next to arrow') +@github_token +def artifacts(job_name, target_dir, queue_path, github_token): + queue = Queue(queue_path) + username, reponame = queue.parse_user_repo() + + gh = github3.login(token=github_token) + repo = gh.repository(username, reponame) + release = repo.release_from_tag(job_name) + + for asset in release.assets(): + click.echo('Downloading asset {} ...'.format(asset.name)) + asset.download(target_dir / asset.name) if __name__ == '__main__': - build(auto_envvar_prefix='CROSSBOW') + crossbow(auto_envvar_prefix='CROSSBOW') diff --git a/dev/tasks/linux-packages/travis.linux.yml b/dev/tasks/linux-packages/travis.linux.yml index afab39471c923..8cc637728a367 100644 --- a/dev/tasks/linux-packages/travis.linux.yml +++ b/dev/tasks/linux-packages/travis.linux.yml @@ -22,6 +22,7 @@ language: ruby env: global: + - TRAVIS_TAG={{ BUILD_TAG }} - PLAT={{ PLAT }} - BUILD_REF={{ BUILD_REF }} - PYARROW_VERSION={{ PYARROW_VERSION }} @@ -29,11 +30,11 @@ env: matrix: include: - script: - - (cd arrow/dev/tasks/linux-packages && travis_wait 40 rake apt:build APT_TARGETS=debian-stretch,ubuntu-trusty,ubuntu-xenial PARALLEL=yes DEBUG=no) + - (cd arrow/dev/tasks/linux-packages && travis_wait 40 && rake version:update && rake apt:build APT_TARGETS=debian-stretch,ubuntu-trusty,ubuntu-xenial PARALLEL=yes DEBUG=no) - script: - - (cd arrow/dev/tasks/linux-packages && travis_wait 40 rake apt:build APT_TARGETS=ubuntu-artful PARALLEL=yes DEBUG=no) + - (cd arrow/dev/tasks/linux-packages && travis_wait 40 && rake version:update && rake apt:build APT_TARGETS=ubuntu-artful PARALLEL=yes DEBUG=no) - script: - - (cd arrow/dev/tasks/linux-packages && rake yum:build PARALLEL=yes DEBUG=no) + - (cd arrow/dev/tasks/linux-packages && rake version:update && rake yum:build PARALLEL=yes DEBUG=no) before_install: - sudo apt update -y -qq @@ -62,6 +63,15 @@ before_script: - git clone -b {{ ARROW_BRANCH }} {{ ARROW_REPO }} arrow - git -C arrow checkout {{ ARROW_SHA }} +deploy: + provider: releases + api_key: $CROSSBOW_GITHUB_TOKEN + file_glob: true + file: /path/to/pachages/*.tar.gz # FIXME(kszucs) after the builds pass + skip_cleanup: true + on: + tags: true + notifications: email: - {{ EMAIL }} diff --git a/dev/tasks/python-wheels/appveyor.yml b/dev/tasks/python-wheels/appveyor.yml index 3bd1804832f68..b3407fb178cbd 100644 --- a/dev/tasks/python-wheels/appveyor.yml +++ b/dev/tasks/python-wheels/appveyor.yml @@ -39,7 +39,6 @@ init: - set MINICONDA=C:\Miniconda35-x64 - set PATH=%MINICONDA%;%MINICONDA%/Scripts;%MINICONDA%/Library/bin;%PATH% - build_script: - mkdir wheels - git clone -b {{ ARROW_BRANCH }} {{ ARROW_REPO }} %ARROW_SRC% || exit /B @@ -52,6 +51,14 @@ after_build: artifacts: - path: wheels\*.whl +deploy: + release: {{ BUILD_TAG }} + provider: GitHub + auth_token: "%CROSSBOW_GITHUB_TOKEN%" + artifact: /.*\.whl/ + draft: false + prerelease: false + notifications: - provider: Email to: diff --git a/dev/tasks/python-wheels/travis.linux.yml b/dev/tasks/python-wheels/travis.linux.yml index 8f2435cec4e3f..2685ad4bb499a 100644 --- a/dev/tasks/python-wheels/travis.linux.yml +++ b/dev/tasks/python-wheels/travis.linux.yml @@ -16,6 +16,7 @@ env: global: + - TRAVIS_TAG={{ BUILD_TAG }} - PLAT={{ PLAT }} - BUILD_REF={{ BUILD_REF }} - PYARROW_VERSION={{ PYARROW_VERSION }} @@ -45,6 +46,15 @@ script: - sudo mv arrow/python/manylinux1/dist/* dist/ +deploy: + provider: releases + api_key: $CROSSBOW_GITHUB_TOKEN + file_glob: true + file: dist/*.whl + skip_cleanup: true + on: + tags: true + notifications: email: - {{ EMAIL }} diff --git a/dev/tasks/python-wheels/travis.osx.yml b/dev/tasks/python-wheels/travis.osx.yml index a92e274fabb83..f346297b5fb5b 100644 --- a/dev/tasks/python-wheels/travis.osx.yml +++ b/dev/tasks/python-wheels/travis.osx.yml @@ -14,12 +14,16 @@ # See the License for the specific language governing permissions and # limitations under the License. +language: generic + os: osx +osx_image: xcode8.3 + sudo: required -language: objective-c env: global: + - TRAVIS_TAG={{ BUILD_TAG }} - PLAT={{ PLAT }} - BUILD_REF={{ BUILD_REF }} - PYARROW_VERSION={{ PYARROW_VERSION }} @@ -69,13 +73,14 @@ install: - build_wheel arrow $PLAT - mv -v arrow/python/dist/* dist/ -script: - - echo "SCRIPT" - - pwd - -after_success: - - echo "After success" - - pwd +deploy: + provider: releases + api_key: $CROSSBOW_GITHUB_TOKEN + file_glob: true + file: dist/*.whl + skip_cleanup: true + on: + tags: true notifications: email: diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 890c00edd3e80..8aaf469f44ee6 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -16,36 +16,39 @@ # under the License. tasks: + # arbitrary_task_name: + # branch: defaults to name + # platform: osx|linux|win + # template: path of jinja2 templated yml + # params: optional extra parameters + # conda packages - - name: conda-linux + conda-linux: platform: linux template: conda-recipes/travis.linux.yml - params: - - name: conda-osx + + conda-osx: platform: osx template: conda-recipes/travis.osx.yml - params: - - name: conda-win + + conda-win: platform: win template: conda-recipes/appveyor.yml - params: # python wheels - - name: python-wheel-linux + wheel-linux: platform: linux template: python-wheels/travis.linux.yml - params: - - name: python-wheel-osx + + wheel-osx: platform: osx template: python-wheels/travis.osx.yml - params: - - name: python-wheel-win + + wheel-win: platform: win template: python-wheels/appveyor.yml - params: # linux packages - - name: linux-packages + linux-packages: platform: linux template: linux-packages/travis.linux.yml - params: