From 0ab0567f61c5563fb4b843830cfb5b32c189d405 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Mon, 4 Jun 2018 19:40:01 +0200
Subject: [PATCH 01/35] don't create tree based on master

---
 dev/tasks/README.md   | 4 ++--
 dev/tasks/crossbow.py | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/dev/tasks/README.md b/dev/tasks/README.md
index f1f0128f03b6e..6a3ef2179fc25 100644
--- a/dev/tasks/README.md
+++ b/dev/tasks/README.md
@@ -80,12 +80,12 @@ submission. The tasks are defined in `tasks.yml`
 6. Install the python dependencies for the script:
 
    ```bash
-   conda install -y jinja2 pygit2 click pyyaml
+   conda install -y jinja2 pygit2 click pyyaml setuptools_scm
    ```
 
    ```bash
    # pygit2 requires libgit2: http://www.pygit2.org/install.html
-   pip install -y jinja2 pygit2 click pyyaml
+   pip install -y jinja2 pygit2 click pyyaml setuptools_scm
    ```
 
 7. Try running it:
diff --git a/dev/tasks/crossbow.py b/dev/tasks/crossbow.py
index fd4f732ea823f..81d6b49733675 100755
--- a/dev/tasks/crossbow.py
+++ b/dev/tasks/crossbow.py
@@ -197,7 +197,7 @@ def _create_tree(self, files):
         parent = self._get_parent_commit()
 
         # creating the tree we are going to push based on master's tree
-        builder = self.repo.TreeBuilder(parent.tree)
+        builder = self.repo.TreeBuilder()
 
         for filename, content in files.items():
             # insert the file and creating the new filetree

From b65cb3f2fd3bdb203abe238c091a639ce6ee0a75 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Tue, 5 Jun 2018 10:48:05 +0200
Subject: [PATCH 02/35] track remote branches

---
 dev/tasks/crossbow.py | 46 ++++++++++++++++++++++++++++---------------
 1 file changed, 30 insertions(+), 16 deletions(-)

diff --git a/dev/tasks/crossbow.py b/dev/tasks/crossbow.py
index 81d6b49733675..999255c48f636 100755
--- a/dev/tasks/crossbow.py
+++ b/dev/tasks/crossbow.py
@@ -181,21 +181,36 @@ def __init__(self, repo_path):
         self.repo = pygit2.Repository(str(self.path))
         self.updated_branches = []
 
-    def _get_parent_commit(self):
-        """Currently this always returns the HEAD of master"""
-        master = self.repo.branches['master']
-        return self.repo[master.target]
+    @property
+    def origin(self):
+        return self.repo.remotes['origin']
+
+    def fetch(self):
+        self.origin.fetch()
 
     def _get_or_create_branch(self, name):
+        branches = self.repo.branches
+
+        upstream_name = '{}/{}'.format(self.origin.name, name)
+        upstream_branch = branches.remote.get(upstream_name)
+
         try:
-            return self.repo.branches[name]
+            branch = branches[name]
         except KeyError:
-            parent = self._get_parent_commit()
-            return self.repo.branches.create(name, parent)
+            if upstream_branch:
+                commit = self.repo[upstream_branch.target]
+                branch = branches.create(name, commit)
+            else:
+                master = branches['master']
+                commit = self.repo[master.target]
+                branch = branches.create(name, commit)
 
-    def _create_tree(self, files):
-        parent = self._get_parent_commit()
+        if upstream_branch:
+            branch.upstream = upstream_branch
+
+        return branch
 
+    def _create_tree(self, files):
         # creating the tree we are going to push based on master's tree
         builder = self.repo.TreeBuilder()
 
@@ -224,22 +239,19 @@ def put(self, build):
         committer = pygit2.Signature(name, email, int(timestamp))
         message = build.description
 
-        reference = 'refs/heads/{}'.format(branch.branch_name)
-        commit_id = self.repo.create_commit(reference, author, committer,
-                                            message, tree_id, [branch.target])
-        logging.info('{} created on {}'.format(
-            commit_id, branch.branch_name))
+        commit_id = self.repo.create_commit(branch.name, author, committer, message,
+                                            tree_id, [branch.target])
+        logging.info('{} created on {}'.format(commit_id, branch.branch_name))
 
         self.updated_branches.append(branch)
 
     def push(self, token):
         callbacks = GitRemoteCallbacks(token)
 
-        remote = self.repo.remotes['origin']
         refs = [branch.name for branch in self.updated_branches]
         shorthands = [b.shorthand for b in self.updated_branches]
 
-        remote.push(refs, callbacks=callbacks)
+        self.origin.push(refs, callbacks=callbacks)
         self.updated_branches = []
 
         logging.info('\n - '.join(['\nUpdated branches:'] + shorthands))
@@ -282,6 +294,8 @@ def build(task_regex, config, dry_run, arrow_repo, queue_repo, github_token):
     arrow = Target(arrow_repo, template_directory='cd')
     queue = Queue(queue_repo)
 
+    queue.fetch()
+
     variables = {
         # these should be renamed
         'PLAT': 'x86_64',

From 8a5fcb011e5558564b06c4a4dfbd860a57075d7b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Wed, 6 Jun 2018 10:57:06 +0200
Subject: [PATCH 03/35] trigger build task; explicit task names during
 submission

---
 dev/tasks/README.md   | 55 +++++++++++--------------------------------
 dev/tasks/crossbow.py | 40 +++++++++++++++----------------
 dev/tasks/tasks.yml   | 27 +++++++++++++--------
 3 files changed, 51 insertions(+), 71 deletions(-)

diff --git a/dev/tasks/README.md b/dev/tasks/README.md
index 6a3ef2179fc25..4c2fd5d2b7f3e 100644
--- a/dev/tasks/README.md
+++ b/dev/tasks/README.md
@@ -135,74 +135,47 @@ The script does the following:
 
 The script accepts a pattern as a first argument to narrow the build scope:
 
-Run all builds:
+Run multiple builds:
 
 ```bash
-$ python crossbow.py
+$ python crossbow.py linux-packages conda-linux wheel-win
 Repository: https://github.com/kszucs/arrow@tasks
 Commit SHA: 810a718836bb3a8cefc053055600bdcc440e6702
 Version: 0.9.1.dev48+g810a7188.d20180414
 Pushed branches:
- - travis-osx-wheel
- - travis-linux-packages
- - travis-linux-wheel
- - appveyor-win-wheel
- - appveyor-win-conda
- - travis-linux-conda
- - travis-osx-conda
+ - linux-packages
+ - conda-linux
+ - wheel-win
 ```
 
 Just render without applying or committing the changes:
 
 ```bash
-$ python crossbow.py --dry-run
+$ python crossbow.py --dry-run task_name
 ```
 
 Run only `conda` package builds but on all platforms:
 
 ```bash
-$ python crossbow.py conda
+$ python crossbow.py conda-win conda-osx conda-linux
 Repository: https://github.com/kszucs/arrow@tasks
 Commit SHA: 810a718836bb3a8cefc053055600bdcc440e6702
 Version: 0.9.1.dev48+g810a7188.d20180414
 Pushed branches:
- - appveyor-win-conda
- - travis-linux-conda
- - travis-osx-conda
+ - conda-win
+ - conda-osx
+ - conda-linux
 ```
 
 Run `wheel` builds:
 
 ```bash
-$ python crossbow.py wheel
+$ python crossbow.py wheel-osx wheel-linux wheel-win
 Repository: https://github.com/kszucs/arrow@tasks
 Commit SHA: 810a718836bb3a8cefc053055600bdcc440e6702
 Version: 0.9.1.dev48+g810a7188.d20180414
 Pushed branches:
- - travis-osx-wheel
- - travis-linux-wheel
- - appveyor-win-wheel
-```
-
-Run `osx` builds:
-
-```bash
-$ python crossbow.py osx
-Repository: https://github.com/kszucs/arrow@tasks
-Commit SHA: cad1df2c7f650ad3434319bbbefed0d4abe45e4a
-Version: 0.9.1.dev130+gcad1df2c.d20180414
-Pushed branches:
- - travis-osx-wheel
- - travis-osx-conda
-```
-
-Run only `linux-conda` package build:
-
-```bash
-$ python crossbow.py linux-conda
-Repository: https://github.com/kszucs/arrow@tasks
-Commit SHA: 810a718836bb3a8cefc053055600bdcc440e6702
-Version: 0.9.1.dev48+g810a7188.d20180414
-Pushed branches:
- - travis-linux-conda
+ - wheel-osx
+ - wheel-linux
+ - wheel-win
 ```
diff --git a/dev/tasks/crossbow.py b/dev/tasks/crossbow.py
index 999255c48f636..bf9fa75ff2d34 100755
--- a/dev/tasks/crossbow.py
+++ b/dev/tasks/crossbow.py
@@ -140,14 +140,15 @@ def current_branch(self):
 
     @property
     def description(self):
-        return '[BUILD] {} of {}@{}'.format(self.version,
+        return 'Version {} of {}@{}'.format(self.version,
                                             self.current_remote.url,
                                             self.current_branch.branch_name)
 
 
-class Build(object):
+class Task(object):
 
-    def __init__(self, target, name, platform, template, **params):
+    def __init__(self, target, name, platform, template, branch=None,
+                 **params):
         assert isinstance(target, Target)
         assert isinstance(platform, Platform)
 
@@ -155,6 +156,7 @@ def __init__(self, target, name, platform, template, **params):
         self.target = target
         self.platform = platform
         self.template = template
+        self.branch = branch or name
         self.params = params
 
     def render(self):
@@ -165,10 +167,6 @@ def render(self):
     def config_files(self):
         return {self.platform.filename: self.render()}
 
-    @property
-    def branch(self):
-        return self.name
-
     @property
     def description(self):
         return self.target.description
@@ -223,7 +221,7 @@ def _create_tree(self, files):
         return tree_id
 
     def put(self, build):
-        assert isinstance(build, Build)
+        assert isinstance(build, Task)
 
         branch = self._get_or_create_branch(build.branch)
         tree_id = self._create_tree(build.config_files())
@@ -262,7 +260,7 @@ def push(self, token):
 
 
 @click.command()
-@click.argument('task-regex', required=False)
+@click.argument('task-names', nargs=-1, required=True)
 @click.option('--config', help='Task configuration yml. Defaults to tasks.yml')
 @click.option('--dry-run/--push', default=False,
               help='Just display the rendered CI configurations without '
@@ -275,7 +273,7 @@ def push(self, token):
                    'Defaults to crossbow directory placed next to arrow')
 @click.option('--github-token', default=False,
               help='Oauth token for Github authentication')
-def build(task_regex, config, dry_run, arrow_repo, queue_repo, github_token):
+def submit(task_names, config, dry_run, arrow_repo, queue_repo, github_token):
     if config is None:
         config = Path(__file__).absolute().parent / 'tasks.yml'
     else:
@@ -311,22 +309,24 @@ def build(task_regex, config, dry_run, arrow_repo, queue_repo, github_token):
     with config.open() as fp:
         tasks = yaml.load(fp)['tasks']
 
+    # filter task definitions according to task_names
+    tasks = [task for task in tasks if task['name'] in task_names]
+
     for task in tasks:
         name = task['name']
         template = config.parent / task['template']
         platform = Platform[task['platform'].upper()]
-        params = task.get('params') or {}
+        branch = task.get('branch')
+        params = task.get('params', {})
         params.update(variables)
 
-        build = Build(arrow, name=name, platform=platform, template=template,
-                      **params)
+        build = Task(arrow, name=name, platform=platform, template=template,
+                     branch=branch, **params)
 
-        # Regex pattern the task name is matched against
-        if task_regex is None or re.search(task_regex, build.name):
-            if dry_run:
-                logging.info('{}\n\n{}'.format(build.name, build.render()))
-            else:
-                queue.put(build)  # create the commit
+        if dry_run:
+            logging.info('{}\n\n{}'.format(build.name, build.render()))
+        else:
+            queue.put(build)  # create the commit
 
     if not dry_run:
         # push the changed branches
@@ -334,4 +334,4 @@ def build(task_regex, config, dry_run, arrow_repo, queue_repo, github_token):
 
 
 if __name__ == '__main__':
-    build(auto_envvar_prefix='CROSSBOW')
+    submit(auto_envvar_prefix='CROSSBOW')
diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
index 890c00edd3e80..0e53c8d8e0cd5 100644
--- a/dev/tasks/tasks.yml
+++ b/dev/tasks/tasks.yml
@@ -16,36 +16,43 @@
 # under the License.
 
 tasks:
+  # supported keys
+  # - name:
+  #   branch:
+  #   platform:
+  #   template:
+  #   params:
+  #     -
+
+  # configures master to submit packaging builds
+  - name: trigger-builds
+    branch: master
+    platform: linux
+    template: trigger-builds/travis.linux.yml
+
   # conda packages
   - name: conda-linux
     platform: linux
     template: conda-recipes/travis.linux.yml
-    params:
   - name: conda-osx
     platform: osx
     template: conda-recipes/travis.osx.yml
-    params:
   - name: conda-win
     platform: win
     template: conda-recipes/appveyor.yml
-    params:
 
   # python wheels
-  - name: python-wheel-linux
+  - name: wheel-linux
     platform: linux
     template: python-wheels/travis.linux.yml
-    params:
-  - name: python-wheel-osx
+  - name: wheel-osx
     platform: osx
     template: python-wheels/travis.osx.yml
-    params:
-  - name: python-wheel-win
+  - name: wheel-win
     platform: win
     template: python-wheels/appveyor.yml
-    params:
 
   # linux packages
   - name: linux-packages
     platform: linux
     template: linux-packages/travis.linux.yml
-    params:

From a0a8127535f0f9875e7973865e7b9253f56833aa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Wed, 6 Jun 2018 11:01:11 +0200
Subject: [PATCH 04/35] tigger builds template

---
 dev/tasks/trigger-builds/travis.linux.yml | 51 +++++++++++++++++++++++
 1 file changed, 51 insertions(+)
 create mode 100644 dev/tasks/trigger-builds/travis.linux.yml

diff --git a/dev/tasks/trigger-builds/travis.linux.yml b/dev/tasks/trigger-builds/travis.linux.yml
new file mode 100644
index 0000000000000..7d2f765a7bee3
--- /dev/null
+++ b/dev/tasks/trigger-builds/travis.linux.yml
@@ -0,0 +1,51 @@
+branches:
+  # don't attempt to build branches intented for windows builds
+  except:
+    - /.*win.*/
+
+os: linux
+dist: trusty
+language: generic
+
+before_install:
+    # Install Miniconda.
+    - echo `pwd`
+    - |
+      echo ""
+      echo "Installing a fresh version of Miniconda."
+      MINICONDA_URL="https://repo.continuum.io/miniconda"
+      MINICONDA_FILE="Miniconda3-latest-Linux-x86_64.sh"
+      curl -L -O "${MINICONDA_URL}/${MINICONDA_FILE}"
+      bash $MINICONDA_FILE -b
+
+    # Configure conda.
+    - |
+      echo ""
+      echo "Configuring conda."
+      source /home/travis/miniconda3/bin/activate root
+      conda config --remove channels defaults
+      conda config --add channels defaults
+      conda config --add channels conda-forge
+      conda config --set show_channel_urls true
+
+install:
+  - conda install -y -q jinja2 pygit2 click pyyaml setuptools_scm
+
+script:
+  # fetch all branches of crossbow
+  - git config remote.origin.fetch +refs/heads/*:refs/remotes/origin/*
+
+  # clone arrow with crossbow tool
+  - pushd ..
+  - git clone -b {{ ARROW_BRANCH }} {{ ARROW_REPO }}
+
+  # submit packaging tasks
+  - |
+    python arrow/dev/tasks/crossbow.py \
+      conda-linux \
+      conda-win \
+      conda-osx \
+      wheel-linux \
+      wheel-win \
+      wheel-osx \
+      linux-packages

From bd5ece4a39ebad1a0bf169050edbe88c635d9192 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Wed, 6 Jun 2018 11:05:41 +0200
Subject: [PATCH 05/35] rename trigger build; add license header

---
 .../travis.linux.yml                             | 16 ++++++++++++++++
 dev/tasks/tasks.yml                              | 15 +++++++--------
 2 files changed, 23 insertions(+), 8 deletions(-)
 rename dev/tasks/{trigger-builds => config-nightlies}/travis.linux.yml (62%)

diff --git a/dev/tasks/trigger-builds/travis.linux.yml b/dev/tasks/config-nightlies/travis.linux.yml
similarity index 62%
rename from dev/tasks/trigger-builds/travis.linux.yml
rename to dev/tasks/config-nightlies/travis.linux.yml
index 7d2f765a7bee3..bf2774b2ed50d 100644
--- a/dev/tasks/trigger-builds/travis.linux.yml
+++ b/dev/tasks/config-nightlies/travis.linux.yml
@@ -1,3 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 branches:
   # don't attempt to build branches intented for windows builds
   except:
diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
index 0e53c8d8e0cd5..5673fa6e3de59 100644
--- a/dev/tasks/tasks.yml
+++ b/dev/tasks/tasks.yml
@@ -17,18 +17,17 @@
 
 tasks:
   # supported keys
-  # - name:
-  #   branch:
-  #   platform:
-  #   template:
-  #   params:
-  #     -
+  # - name: arbitrary task name
+  #   branch: defaults to name
+  #   platform: osx|linux|win
+  #   template: path of jinja2 templated yml
+  #   params: optional extra parameters
 
   # configures master to submit packaging builds
-  - name: trigger-builds
+  - name: config-nightlies
     branch: master
     platform: linux
-    template: trigger-builds/travis.linux.yml
+    template: config-nightlies/travis.linux.yml
 
   # conda packages
   - name: conda-linux

From edeee7175ae6cf0c65a5a41d37df6782727cd5ce Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Wed, 13 Jun 2018 01:26:00 +0200
Subject: [PATCH 06/35] refactooooor

---
 dev/tasks/conda-recipes/travis.linux.yml  |  10 +
 dev/tasks/conda-recipes/travis.osx.yml    |  10 +
 dev/tasks/crossbow.py                     | 369 +++++++++++-----------
 dev/tasks/linux-packages/travis.linux.yml |  10 +
 dev/tasks/python-wheels/appveyor.yml      |   8 +
 dev/tasks/python-wheels/travis.linux.yml  |  10 +
 dev/tasks/python-wheels/travis.osx.yml    |  16 +-
 dev/tasks/tasks.yml                       |  26 +-
 8 files changed, 261 insertions(+), 198 deletions(-)

diff --git a/dev/tasks/conda-recipes/travis.linux.yml b/dev/tasks/conda-recipes/travis.linux.yml
index c5266810c6dff..4b4a54c459bef 100644
--- a/dev/tasks/conda-recipes/travis.linux.yml
+++ b/dev/tasks/conda-recipes/travis.linux.yml
@@ -26,6 +26,7 @@ env:
     - CONDA_PY=35
     - CONDA_PY=36
   global:
+    - TRAVIS_TAG={{ BUILD_TAG }}
     - ARROW_VERSION={{ ARROW_VERSION }}
 
 install:
@@ -57,6 +58,15 @@ script:
   - pushd arrow/dev/tasks/conda-recipes
   - conda build parquet-cpp arrow-cpp pyarrow
 
+deploy:
+  provider: releases
+  api_key: $CROSSBOW_GITHUB_TOKEN
+  file_glob: true
+  file: /home/travis/miniconda3/conda-bld/linux-64/*.tar.gz
+  skip_cleanup: true
+  on:
+    tags: true
+
 notifications:
   email:
     - {{ EMAIL }}
diff --git a/dev/tasks/conda-recipes/travis.osx.yml b/dev/tasks/conda-recipes/travis.osx.yml
index 8a38d333489cc..ebc3cc3774645 100644
--- a/dev/tasks/conda-recipes/travis.osx.yml
+++ b/dev/tasks/conda-recipes/travis.osx.yml
@@ -26,6 +26,7 @@ env:
     - CONDA_PY=35
     - CONDA_PY=36
   global:
+    - TRAVIS_TAG={{ BUILD_TAG }}
     - ARROW_VERSION={{ ARROW_VERSION }}
 
 before_install:
@@ -66,6 +67,15 @@ script:
   - pushd arrow/dev/tasks/conda-recipes
   - conda build parquet-cpp arrow-cpp pyarrow
 
+deploy:
+  provider: releases
+  api_key: $CROSSBOW_GITHUB_TOKEN
+  file_glob: true
+  file: /home/travis/miniconda3/conda-bld/osx-64/*.tar.gz
+  skip_cleanup: true
+  on:
+    tags: true
+
 notifications:
   email:
     - {{ EMAIL }}
diff --git a/dev/tasks/crossbow.py b/dev/tasks/crossbow.py
index bf9fa75ff2d34..36e53805147e1 100755
--- a/dev/tasks/crossbow.py
+++ b/dev/tasks/crossbow.py
@@ -18,7 +18,6 @@
 # under the License.
 
 import re
-import sys
 import yaml
 import time
 import click
@@ -29,8 +28,8 @@
 from pathlib import Path
 from textwrap import dedent
 from jinja2 import Template
+from collections import namedtuple
 from setuptools_scm import get_version
-from setuptools_scm.version import simplified_semver_version, meta
 
 
 logging.basicConfig(
@@ -70,268 +69,278 @@ def credentials(self, url, username_from_url, allowed_types):
             return None
 
 
-class Platform(Enum):
-    # in alphabetical order
-    LINUX = 0
-    OSX = 1
-    WIN = 2
-
-    @property
-    def ci(self):
-        if self is self.WIN:
-            return 'appveyor'
-        else:
-            return 'travis'
+class Repo(object):
 
-    @property
-    def filename(self):
-        if self.ci == 'appveyor':
-            return 'appveyor.yml'
-        else:
-            return '.travis.yml'
-
-
-class Target(object):
-
-    def __init__(self, repo_path, template_directory=None):
+    def __init__(self, repo_path):
         self.path = Path(repo_path).absolute()
-
-        # relative to repository's path
-        if template_directory is None:
-            self.templates = self.path
-        else:
-            self.templates = self.path / template_directory
-
-        # initialize a repo object to interact with arrow's git data
         self.repo = pygit2.Repository(str(self.path))
 
-        msg = dedent('''
-            Repository: {remote}@{branch}
-            Commit SHA: {sha}
-            Version: {version}
+    def __str__(self):
+        tpl = dedent('''
+            Repo: {remote}@{branch}
+            Commit: {head}
         ''')
-        logging.info(msg.format(
-            remote=self.current_remote.url,
-            branch=self.current_branch.branch_name,
-            sha=self.sha,
-            version=self.version
-        ))
+        return tpl.format(
+            remote=self.remote.url,
+            branch=self.branch.branch_name,
+            head=self.head
+        )
+
+    def fetch(self):
+        self.origin.fetch()
 
     @property
-    def sha(self):
+    def head(self):
         """Currently checked out commit's sha"""
         return self.repo.head.target
 
     @property
-    def version(self):
-        """Generate version number based on version control history"""
-        # TODO(kszucs) use self.repo.describe() instead
-        return get_version(self.path)
+    def branch(self):
+        """Currently checked out branch"""
+        reference = self.repo.head.shorthand
+        return self.repo.branches[reference]
 
     @property
-    def current_remote(self):
-        remote_name = self.current_branch.upstream.remote_name
+    def remote(self):
+        """Currently checked out branch's remote counterpart"""
+        remote_name = self.branch.upstream.remote_name
         return self.repo.remotes[remote_name]
 
     @property
-    def current_branch(self):
-        reference = self.repo.head.shorthand
-        return self.repo.branches[reference]
+    def origin(self):
+        return self.repo.remotes['origin']
 
     @property
-    def description(self):
-        return 'Version {} of {}@{}'.format(self.version,
-                                            self.current_remote.url,
-                                            self.current_branch.branch_name)
+    def signature(self):
+        name = next(self.repo.config.get_multivar('user.name'))
+        email = next(self.repo.config.get_multivar('user.email'))
+        return pygit2.Signature(name, email, int(time.time()))
 
 
-class Task(object):
+class Queue(Repo):
 
-    def __init__(self, target, name, platform, template, branch=None,
-                 **params):
-        assert isinstance(target, Target)
-        assert isinstance(platform, Platform)
+    def __init__(self, repo_path):
+        super(Queue, self).__init__(repo_path)
+        self._updated_refs = []
+
+    def _next_job_id(self, prefix):
+        """Auto increments the branch's identifier based on the prefix"""
+        pattern = re.compile(prefix + '-(\d+)')
+        matches = list(filter(None, map(pattern.match, self.repo.branches)))
+        if matches:
+            latest = max(int(m.group(1)) for m in matches)
+        else:
+            latest = 0
+        return '{}-{}'.format(prefix, latest + 1)
 
-        self.name = name
-        self.target = target
-        self.platform = platform
-        self.template = template
-        self.branch = branch or name
-        self.params = params
+    def _create_branch(self, branch_name, files, parents=None, message=''):
+        # 1. create tree
+        builder = self.repo.TreeBuilder()
 
-    def render(self):
-        path = Path(self.template)
-        template = Template(path.read_text())
-        return template.render(**self.params)
+        for filename, content in files.items():
+            # insert the file and creating the new filetree
+            blob_id = self.repo.create_blob(content)
+            builder.insert(filename, blob_id, pygit2.GIT_FILEMODE_BLOB)
 
-    def config_files(self):
-        return {self.platform.filename: self.render()}
+        tree_id = builder.write()
 
-    @property
-    def description(self):
-        return self.target.description
+        # 2. create commit with the tree created above
+        author = committer = self.signature
+        parents = parents or [self.head]
+        commit_id = self.repo.create_commit(None, author, committer, message,
+                                            tree_id, parents)
+        commit = self.repo[commit_id]
 
+        # 3. create branch pointing to the previously created commit
+        branch = self.repo.create_branch(branch_name, commit)
+        # append to the pushable references
+        self._updated_refs.append('refs/heads/{}'.format(branch_name))
 
-class Queue(object):
+        return branch
 
-    def __init__(self, repo_path):
-        self.path = Path(repo_path).absolute()
-        self.repo = pygit2.Repository(str(self.path))
-        self.updated_branches = []
+    def _create_tag(self, tag_name, commit_id, message=''):
+        tag_id = self.repo.create_tag(tag_name, commit_id,
+                                      pygit2.GIT_OBJ_COMMIT, self.signature,
+                                      message)
 
-    @property
-    def origin(self):
-        return self.repo.remotes['origin']
+        # append to the pushable references
+        self._updated_refs.append('refs/tags/{}'.format(tag_name))
 
-    def fetch(self):
-        self.origin.fetch()
+        return self.repo[tag_id]
 
-    def _get_or_create_branch(self, name):
-        branches = self.repo.branches
+    def put(self, job, prefix='build'):
+        assert isinstance(job, Job)
 
-        upstream_name = '{}/{}'.format(self.origin.name, name)
-        upstream_branch = branches.remote.get(upstream_name)
+        job.branch = self._next_job_id(prefix)
 
-        try:
-            branch = branches[name]
-        except KeyError:
-            if upstream_branch:
-                commit = self.repo[upstream_branch.target]
-                branch = branches.create(name, commit)
-            else:
-                master = branches['master']
-                commit = self.repo[master.target]
-                branch = branches.create(name, commit)
+        # create tasks' branches
+        for task_name, task in job.tasks.items():
+            task.branch = '{}-{}'.format(job.branch, task_name)
+            task.params['BUILD_TAG'] = job.branch  # this is ugly
+            branch = self._create_branch(task.branch, files=task.files())
+            task.commit = str(branch.target)
 
-        if upstream_branch:
-            branch.upstream = upstream_branch
+        # create job's branch
+        branch = self._create_branch(job.branch, files=job.files())
+        tag = self._create_tag(job.branch, branch.target)
 
         return branch
 
-    def _create_tree(self, files):
-        # creating the tree we are going to push based on master's tree
-        builder = self.repo.TreeBuilder()
+    def push(self, token):
+        callbacks = GitRemoteCallbacks(token)
+        self.origin.push(self._updated_refs, callbacks=callbacks)
+        self.updated_refs = []
 
-        for filename, content in files.items():
-            # insert the file and creating the new filetree
-            blob_id = self.repo.create_blob(content)
-            builder.insert(filename, blob_id, pygit2.GIT_FILEMODE_BLOB)
+        # logging.info('\n - '.join(['\nUpdated branches:'] + shorthands))
 
-        tree_id = builder.write()
-        return tree_id
 
-    def put(self, build):
-        assert isinstance(build, Task)
+class Platform(Enum):
+    # in alphabetical order
+    LINUX = 0
+    OSX = 1
+    WIN = 2
 
-        branch = self._get_or_create_branch(build.branch)
-        tree_id = self._create_tree(build.config_files())
+    @property
+    def ci(self):
+        if self is self.WIN:
+            return 'appveyor'
+        else:
+            return 'travis'
 
-        # creating the new commit
-        timestamp = int(time.time())
+    @property
+    def filename(self):
+        if self.ci == 'appveyor':
+            return 'appveyor.yml'
+        else:
+            return '.travis.yml'
 
-        name = next(self.repo.config.get_multivar('user.name'))
-        email = next(self.repo.config.get_multivar('user.email'))
 
-        author = pygit2.Signature('crossbow', 'mailing@list.com',
-                                  int(timestamp))
-        committer = pygit2.Signature(name, email, int(timestamp))
-        message = build.description
+class Task(object):
 
-        commit_id = self.repo.create_commit(branch.name, author, committer, message,
-                                            tree_id, [branch.target])
-        logging.info('{} created on {}'.format(commit_id, branch.branch_name))
+    def __init__(self, platform, template, commit=None, branch=None, **params):
+        assert isinstance(platform, Platform)
+        assert isinstance(template, Path)
+        self.platform = platform
+        self.template = template
+        self.branch = branch
+        self.commit = commit
+        self.params = params
 
-        self.updated_branches.append(branch)
+    def to_dict(self):
+        return {'branch': self.branch,
+                'commit': str(self.commit),
+                'platform': self.platform.name,
+                'template': str(self.template),
+                'params': self.params}
 
-    def push(self, token):
-        callbacks = GitRemoteCallbacks(token)
+    @classmethod
+    def from_dict(cls, data):
+        return Task(platform=Platform[data['platform'].upper()],
+                    template=Path(data['template']),
+                    commit=data.get('commit'),
+                    branch=data.get('branch'),
+                    **data.get('params', {}))
+
+    def files(self):
+        template = Template(self.template.read_text())
+        rendered = template.render(**self.params)
+        return {self.platform.filename: rendered}
 
-        refs = [branch.name for branch in self.updated_branches]
-        shorthands = [b.shorthand for b in self.updated_branches]
 
-        self.origin.push(refs, callbacks=callbacks)
-        self.updated_branches = []
+class Job(object):
 
-        logging.info('\n - '.join(['\nUpdated branches:'] + shorthands))
+    def __init__(self, tasks, branch=None):
+        assert all(isinstance(task, Task) for task in tasks.values())
+        self.branch = branch
+        self.tasks = tasks
+
+    def to_dict(self):
+        tasks = {name: task.to_dict() for name, task in self.tasks.items()}
+        return {'branch': self.branch,
+                'tasks': tasks}
+
+    @classmethod
+    def from_dict(cls, data):
+        tasks = {name: Task.from_dict(task)
+                 for name, task in data['tasks'].items()}
+        return Job(tasks=tasks, branch=data.get('branch'))
+
+    def files(self):
+        return {'job.yml': yaml.dump(self.to_dict(), default_flow_style=False)}
 
 
 # this should be the mailing list
 MESSAGE_EMAIL = 'szucs.krisztian@gmail.com'
 
+CWD = Path(__file__).absolute()
+
+DEFAULT_CONFIG_PATH = CWD.parent / 'tasks.yml'
+DEFAULT_ARROW_PATH = CWD.parents[2]
+DEFAULT_QUEUE_PATH = CWD.parents[3] / 'crossbow'
+
 
 @click.command()
 @click.argument('task-names', nargs=-1, required=True)
-@click.option('--config', help='Task configuration yml. Defaults to tasks.yml')
+@click.option('--config-path', default=DEFAULT_CONFIG_PATH,
+              help='Task configuration yml. Defaults to tasks.yml')
 @click.option('--dry-run/--push', default=False,
               help='Just display the rendered CI configurations without '
                    'submitting them')
-@click.option('--arrow-repo', default=None,
+@click.option('--arrow-path', default=DEFAULT_ARROW_PATH,
               help='Arrow\'s repository path. Defaults to the repository of '
                    'this script')
-@click.option('--queue-repo', default=None,
+@click.option('--queue-path', default=DEFAULT_QUEUE_PATH,
               help='The repository path used for scheduling the tasks. '
                    'Defaults to crossbow directory placed next to arrow')
 @click.option('--github-token', default=False,
               help='Oauth token for Github authentication')
-def submit(task_names, config, dry_run, arrow_repo, queue_repo, github_token):
-    if config is None:
-        config = Path(__file__).absolute().parent / 'tasks.yml'
-    else:
-        config = Path(config)
+def submit(task_names, config_path, dry_run, arrow_path, queue_path,
+           github_token):
+    target = Repo(arrow_path)
+    queue = Queue(queue_path)
 
-    if arrow_repo is None:
-        arrow_repo = Path(__file__).absolute().parents[2]
-    else:
-        arrow_repo = Path(arrow_repo)
-
-    if queue_repo is None:
-        queue_repo = arrow_repo.parent / 'crossbow'
-    else:
-        queue_repo = Path(queue_repo)
-
-    arrow = Target(arrow_repo, template_directory='cd')
-    queue = Queue(queue_repo)
+    logging.info(target)
+    logging.info(queue)
 
     queue.fetch()
 
+    version = get_version(arrow_path)
+
     variables = {
         # these should be renamed
         'PLAT': 'x86_64',
         'EMAIL': MESSAGE_EMAIL,
-        'BUILD_REF': arrow.sha,
-        'ARROW_SHA': arrow.sha,
-        'ARROW_REPO': arrow.current_remote.url,
-        'ARROW_BRANCH': arrow.current_branch.branch_name,
-        'ARROW_VERSION': arrow.version,
-        'PYARROW_VERSION': arrow.version,
+        'BUILD_REF': str(target.head),
+        'ARROW_SHA': str(target.head),
+        'ARROW_REPO': target.remote.url,
+        'ARROW_BRANCH': target.branch.branch_name,
+        'ARROW_VERSION': version,
+        'PYARROW_VERSION': version,
     }
 
-    with config.open() as fp:
-        tasks = yaml.load(fp)['tasks']
+    with Path(config_path).open() as fp:
+        config = yaml.load(fp)
 
-    # filter task definitions according to task_names
-    tasks = [task for task in tasks if task['name'] in task_names]
+    # consider to set branch ids here instead of inside Queue.put
 
-    for task in tasks:
-        name = task['name']
-        template = config.parent / task['template']
-        platform = Platform[task['platform'].upper()]
-        branch = task.get('branch')
-        params = task.get('params', {})
-        params.update(variables)
+    # create and filter tasks
+    tasks = {name: Task.from_dict(task)
+             for name, task in config['tasks'].items()}
+    tasks = {name: tasks[name] for name in task_names}
 
-        build = Task(arrow, name=name, platform=platform, template=template,
-                     branch=branch, **params)
+    for task in tasks.values():
+        task.params.update(variables)
 
-        if dry_run:
-            logging.info('{}\n\n{}'.format(build.name, build.render()))
-        else:
-            queue.put(build)  # create the commit
+    # create job
+    job = Job(tasks)
 
-    if not dry_run:
-        # push the changed branches
+    if dry_run:
+        logging.info(yaml.dump(job.to_dict(), default_flow_style=False))
+    else:
+        queue.put(job)
         queue.push(token=github_token)
 
+# check status
 
 if __name__ == '__main__':
     submit(auto_envvar_prefix='CROSSBOW')
diff --git a/dev/tasks/linux-packages/travis.linux.yml b/dev/tasks/linux-packages/travis.linux.yml
index afab39471c923..1c3cef14f578e 100644
--- a/dev/tasks/linux-packages/travis.linux.yml
+++ b/dev/tasks/linux-packages/travis.linux.yml
@@ -22,6 +22,7 @@ language: ruby
 
 env:
   global:
+    - TRAVIS_TAG={{ BUILD_TAG }}
     - PLAT={{ PLAT }}
     - BUILD_REF={{ BUILD_REF }}
     - PYARROW_VERSION={{ PYARROW_VERSION }}
@@ -62,6 +63,15 @@ before_script:
   - git clone -b {{ ARROW_BRANCH }} {{ ARROW_REPO }} arrow
   - git -C arrow checkout {{ ARROW_SHA }}
 
+deploy:
+  provider: releases
+  api_key: $CROSSBOW_GITHUB_TOKEN
+  file_glob: true
+  file: /path/to/pachages/*.tar.gz
+  skip_cleanup: true
+  on:
+    tags: true
+
 notifications:
   email:
     - {{ EMAIL }}
diff --git a/dev/tasks/python-wheels/appveyor.yml b/dev/tasks/python-wheels/appveyor.yml
index 3bd1804832f68..b3d2ed2496c7d 100644
--- a/dev/tasks/python-wheels/appveyor.yml
+++ b/dev/tasks/python-wheels/appveyor.yml
@@ -52,6 +52,14 @@ after_build:
 artifacts:
   - path: wheels\*.whl
 
+deploy:
+  release: {{ BUILD_TAG }}
+  provider: GitHub
+  auth_token: $(CROSSBOW_GITHUB_TOKEN)
+  artifact: wheels\*.whl
+  draft: false
+  prerelease: false
+
 notifications:
   - provider: Email
     to:
diff --git a/dev/tasks/python-wheels/travis.linux.yml b/dev/tasks/python-wheels/travis.linux.yml
index 8f2435cec4e3f..2685ad4bb499a 100644
--- a/dev/tasks/python-wheels/travis.linux.yml
+++ b/dev/tasks/python-wheels/travis.linux.yml
@@ -16,6 +16,7 @@
 
 env:
   global:
+    - TRAVIS_TAG={{ BUILD_TAG }}
     - PLAT={{ PLAT }}
     - BUILD_REF={{ BUILD_REF }}
     - PYARROW_VERSION={{ PYARROW_VERSION }}
@@ -45,6 +46,15 @@ script:
 
   - sudo mv arrow/python/manylinux1/dist/* dist/
 
+deploy:
+  provider: releases
+  api_key: $CROSSBOW_GITHUB_TOKEN
+  file_glob: true
+  file: dist/*.whl
+  skip_cleanup: true
+  on:
+    tags: true
+
 notifications:
   email:
     - {{ EMAIL }}
diff --git a/dev/tasks/python-wheels/travis.osx.yml b/dev/tasks/python-wheels/travis.osx.yml
index a92e274fabb83..544cb96245978 100644
--- a/dev/tasks/python-wheels/travis.osx.yml
+++ b/dev/tasks/python-wheels/travis.osx.yml
@@ -20,6 +20,7 @@ language: objective-c
 
 env:
   global:
+    - TRAVIS_TAG={{ BUILD_TAG }}
     - PLAT={{ PLAT }}
     - BUILD_REF={{ BUILD_REF }}
     - PYARROW_VERSION={{ PYARROW_VERSION }}
@@ -69,13 +70,14 @@ install:
   - build_wheel arrow $PLAT
   - mv -v arrow/python/dist/* dist/
 
-script:
-  - echo "SCRIPT"
-  - pwd
-
-after_success:
-  - echo "After success"
-  - pwd
+deploy:
+  provider: releases
+  api_key: $CROSSBOW_GITHUB_TOKEN
+  file_glob: true
+  file: dist/*.whl
+  skip_cleanup: true
+  on:
+    tags: true
 
 notifications:
   email:
diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
index 5673fa6e3de59..87a493ccdcbdd 100644
--- a/dev/tasks/tasks.yml
+++ b/dev/tasks/tasks.yml
@@ -24,34 +24,38 @@ tasks:
   #   params: optional extra parameters
 
   # configures master to submit packaging builds
-  - name: config-nightlies
-    branch: master
-    platform: linux
-    template: config-nightlies/travis.linux.yml
+  # - name: config-nightlies
+  #   branch: master
+  #   platform: linux
+  #   template: config-nightlies/travis.linux.yml
 
   # conda packages
-  - name: conda-linux
+  conda-linux:
     platform: linux
     template: conda-recipes/travis.linux.yml
-  - name: conda-osx
+
+  conda-osx:
     platform: osx
     template: conda-recipes/travis.osx.yml
-  - name: conda-win
+
+  conda-win:
     platform: win
     template: conda-recipes/appveyor.yml
 
   # python wheels
-  - name: wheel-linux
+  wheel-linux:
     platform: linux
     template: python-wheels/travis.linux.yml
-  - name: wheel-osx
+
+  wheel-osx:
     platform: osx
     template: python-wheels/travis.osx.yml
-  - name: wheel-win
+
+  wheel-win:
     platform: win
     template: python-wheels/appveyor.yml
 
   # linux packages
-  - name: linux-packages
+  linux-packages:
     platform: linux
     template: linux-packages/travis.linux.yml

From 55e7ccb25437c9d9c7a4fc2e14abf10c6cd3dd1b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Wed, 13 Jun 2018 16:28:55 +0200
Subject: [PATCH 07/35] query statuses API

---
 dev/tasks/README.md                      |  4 +--
 dev/tasks/conda-recipes/appveyor.yml     |  7 ++++++
 dev/tasks/conda-recipes/travis.linux.yml |  2 +-
 dev/tasks/conda-recipes/travis.osx.yml   |  2 +-
 dev/tasks/crossbow.py                    | 31 +++++++++++++++++++++---
 dev/tasks/python-wheels/appveyor.yml     |  3 +--
 6 files changed, 39 insertions(+), 10 deletions(-)

diff --git a/dev/tasks/README.md b/dev/tasks/README.md
index 4c2fd5d2b7f3e..f5cc69eca58fb 100644
--- a/dev/tasks/README.md
+++ b/dev/tasks/README.md
@@ -80,12 +80,12 @@ submission. The tasks are defined in `tasks.yml`
 6. Install the python dependencies for the script:
 
    ```bash
-   conda install -y jinja2 pygit2 click pyyaml setuptools_scm
+   conda install -y jinja2 pygit2 click pyyaml setuptools_scm github3.py
    ```
 
    ```bash
    # pygit2 requires libgit2: http://www.pygit2.org/install.html
-   pip install -y jinja2 pygit2 click pyyaml setuptools_scm
+   pip install -y jinja2 pygit2 click pyyaml setuptools_scm github3.py
    ```
 
 7. Try running it:
diff --git a/dev/tasks/conda-recipes/appveyor.yml b/dev/tasks/conda-recipes/appveyor.yml
index 10ef7044cebb9..035773fc2952d 100644
--- a/dev/tasks/conda-recipes/appveyor.yml
+++ b/dev/tasks/conda-recipes/appveyor.yml
@@ -50,6 +50,13 @@ test_script:
   - pushd %ARROW_SRC%\dev\tasks\conda-recipes
   - conda.exe build parquet-cpp arrow-cpp pyarrow
 
+deploy:
+  release: {{ BUILD_TAG }}
+  provider: GitHub
+  auth_token: "%CROSSBOW_GITHUB_TOKEN%"
+  artifact: "%CONDA_INSTALL_LOCN%\\conda-bld\\win-64\\*.tar.bz2"
+  draft: false
+  prerelease: false
 
 notifications:
   - provider: Email
diff --git a/dev/tasks/conda-recipes/travis.linux.yml b/dev/tasks/conda-recipes/travis.linux.yml
index 4b4a54c459bef..ea9afe1c49f48 100644
--- a/dev/tasks/conda-recipes/travis.linux.yml
+++ b/dev/tasks/conda-recipes/travis.linux.yml
@@ -62,7 +62,7 @@ deploy:
   provider: releases
   api_key: $CROSSBOW_GITHUB_TOKEN
   file_glob: true
-  file: /home/travis/miniconda3/conda-bld/linux-64/*.tar.gz
+  file: /home/travis/miniconda3/conda-bld/linux-64/*.tar.bz2
   skip_cleanup: true
   on:
     tags: true
diff --git a/dev/tasks/conda-recipes/travis.osx.yml b/dev/tasks/conda-recipes/travis.osx.yml
index ebc3cc3774645..62c9891d28533 100644
--- a/dev/tasks/conda-recipes/travis.osx.yml
+++ b/dev/tasks/conda-recipes/travis.osx.yml
@@ -71,7 +71,7 @@ deploy:
   provider: releases
   api_key: $CROSSBOW_GITHUB_TOKEN
   file_glob: true
-  file: /home/travis/miniconda3/conda-bld/osx-64/*.tar.gz
+  file: /home/travis/miniconda3/conda-bld/osx-64/*.tar.bz2
   skip_cleanup: true
   on:
     tags: true
diff --git a/dev/tasks/crossbow.py b/dev/tasks/crossbow.py
index 36e53805147e1..821d72ea7578d 100755
--- a/dev/tasks/crossbow.py
+++ b/dev/tasks/crossbow.py
@@ -279,7 +279,12 @@ def files(self):
 DEFAULT_QUEUE_PATH = CWD.parents[3] / 'crossbow'
 
 
-@click.command()
+@click.group()
+def crossbow():
+    pass
+
+
+@crossbow.command()
 @click.argument('task-names', nargs=-1, required=True)
 @click.option('--config-path', default=DEFAULT_CONFIG_PATH,
               help='Task configuration yml. Defaults to tasks.yml')
@@ -292,7 +297,7 @@ def files(self):
 @click.option('--queue-path', default=DEFAULT_QUEUE_PATH,
               help='The repository path used for scheduling the tasks. '
                    'Defaults to crossbow directory placed next to arrow')
-@click.option('--github-token', default=False,
+@click.option('--github-token', default=False, envvar='CROSSBOW_GITHUB_TOKEN',
               help='Oauth token for Github authentication')
 def submit(task_names, config_path, dry_run, arrow_path, queue_path,
            github_token):
@@ -340,7 +345,25 @@ def submit(task_names, config_path, dry_run, arrow_path, queue_path,
         queue.put(job)
         queue.push(token=github_token)
 
-# check status
+
+@crossbow.command()
+@click.argument('job-name', required=True)
+@click.option('--github-token', default=False, envvar='CROSSBOW_GITHUB_TOKEN',
+              help='Oauth token for Github authentication')
+def status(job_name, github_token):
+    import github3
+
+    branch_name = job_name
+
+    gh = github3.login(token=github_token)
+    repo = gh.repository('kszucs', 'crossbow')  # FIXME(kszucs)
+    content = repo.file_contents('job.yml', branch_name)
+
+    job = Job.from_dict(yaml.load(content.decoded))
+    for name, task in job.tasks.items():
+        for status in repo.statuses(task.commit):
+            print(status)
+
 
 if __name__ == '__main__':
-    submit(auto_envvar_prefix='CROSSBOW')
+    crossbow(auto_envvar_prefix='CROSSBOW')
diff --git a/dev/tasks/python-wheels/appveyor.yml b/dev/tasks/python-wheels/appveyor.yml
index b3d2ed2496c7d..d6559d2a2ad43 100644
--- a/dev/tasks/python-wheels/appveyor.yml
+++ b/dev/tasks/python-wheels/appveyor.yml
@@ -39,7 +39,6 @@ init:
   - set MINICONDA=C:\Miniconda35-x64
   - set PATH=%MINICONDA%;%MINICONDA%/Scripts;%MINICONDA%/Library/bin;%PATH%
 
-
 build_script:
   - mkdir wheels
   - git clone -b {{ ARROW_BRANCH }} {{ ARROW_REPO }} %ARROW_SRC% || exit /B
@@ -55,7 +54,7 @@ artifacts:
 deploy:
   release: {{ BUILD_TAG }}
   provider: GitHub
-  auth_token: $(CROSSBOW_GITHUB_TOKEN)
+  auth_token: "%CROSSBOW_GITHUB_TOKEN%"
   artifact: wheels\*.whl
   draft: false
   prerelease: false

From 96e101af4c51fafe81b82e9df741e1abf56d6a6c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Wed, 13 Jun 2018 17:13:48 +0200
Subject: [PATCH 08/35] draft for downloading artifacts

---
 dev/tasks/crossbow.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/dev/tasks/crossbow.py b/dev/tasks/crossbow.py
index 821d72ea7578d..bff56794c0408 100755
--- a/dev/tasks/crossbow.py
+++ b/dev/tasks/crossbow.py
@@ -365,5 +365,22 @@ def status(job_name, github_token):
             print(status)
 
 
+@crossbow.command()
+@click.argument('job-name', required=True)
+@click.option('--github-token', default=False, envvar='CROSSBOW_GITHUB_TOKEN',
+              help='Oauth token for Github authentication')
+def artifacts(job_name, github_token):
+    import github3
+
+    branch_name = job_name
+
+    gh = github3.login(token=github_token)
+    repo = gh.repository('kszucs', 'crossbow')  # FIXME(kszucs)
+    release = repo.release_from_tag(branch_name)
+
+    for asset in release.assets():
+        asset.download()  # download to the current dir, FIXME
+
+
 if __name__ == '__main__':
     crossbow(auto_envvar_prefix='CROSSBOW')

From 0b79d13d99405d93a3551d60e394080bf06833c7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Wed, 13 Jun 2018 20:21:42 +0200
Subject: [PATCH 09/35] trying to remove python dependency from parquet-cpp's
 recipe

---
 dev/tasks/conda-recipes/appveyor.yml          | 5 ++++-
 dev/tasks/conda-recipes/parquet-cpp/meta.yaml | 7 +------
 2 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/dev/tasks/conda-recipes/appveyor.yml b/dev/tasks/conda-recipes/appveyor.yml
index 035773fc2952d..5d7e7b969da6f 100644
--- a/dev/tasks/conda-recipes/appveyor.yml
+++ b/dev/tasks/conda-recipes/appveyor.yml
@@ -50,11 +50,14 @@ test_script:
   - pushd %ARROW_SRC%\dev\tasks\conda-recipes
   - conda.exe build parquet-cpp arrow-cpp pyarrow
 
+artifacts:
+  - path: C:\bld\win-64\*.tar.bz2
+
 deploy:
   release: {{ BUILD_TAG }}
   provider: GitHub
   auth_token: "%CROSSBOW_GITHUB_TOKEN%"
-  artifact: "%CONDA_INSTALL_LOCN%\\conda-bld\\win-64\\*.tar.bz2"
+  artifact: /.*\.tar\.bz2/
   draft: false
   prerelease: false
 
diff --git a/dev/tasks/conda-recipes/parquet-cpp/meta.yaml b/dev/tasks/conda-recipes/parquet-cpp/meta.yaml
index 0f5a619010a72..e7be2a1c7d7cd 100644
--- a/dev/tasks/conda-recipes/parquet-cpp/meta.yaml
+++ b/dev/tasks/conda-recipes/parquet-cpp/meta.yaml
@@ -36,9 +36,8 @@ source:
 
 build:
   number: 0
-  skip: true  # [win and not (py35 and win64)]
   features:
-    - vc14  # [win and py35]
+    - vc14  # [win]
 
 requirements:
   build:
@@ -46,16 +45,12 @@ requirements:
     - boost-cpp 1.66.0
     - cmake
     - thrift-cpp >=0.11
-    - python            # [win]
     - arrow-cpp {{ ARROW_VERSION }}
 
   run:
     - arrow-cpp {{ ARROW_VERSION }}
 
 test:
-  requires:
-    - python {{ environ['PY_VER'] + '*' }}  # [win]
-
   commands:
     - test -f $PREFIX/lib/libparquet.so                                       # [linux]
     - test -f $PREFIX/lib/libparquet.dylib                                    # [osx]

From 53748668340febdd4c9b87e08076c568481f01d7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Wed, 13 Jun 2018 21:46:06 +0200
Subject: [PATCH 10/35] cleanup status check and artifact download

---
 dev/tasks/crossbow.py                | 56 +++++++++++++++++-----------
 dev/tasks/python-wheels/appveyor.yml |  2 +-
 2 files changed, 35 insertions(+), 23 deletions(-)

diff --git a/dev/tasks/crossbow.py b/dev/tasks/crossbow.py
index bff56794c0408..8067d4d17c55a 100755
--- a/dev/tasks/crossbow.py
+++ b/dev/tasks/crossbow.py
@@ -22,22 +22,22 @@
 import time
 import click
 import pygit2
+import github3
 import logging
 
 from enum import Enum
 from pathlib import Path
 from textwrap import dedent
 from jinja2 import Template
-from collections import namedtuple
 from setuptools_scm import get_version
 
 
-logging.basicConfig(
-    level=logging.INFO,
-    format="[%(asctime)s] %(levelname)s Crossbow %(message)s",
-    datefmt="%H:%M:%S",
-    stream=click.get_text_stream('stdout')
-)
+# logging.basicConfig(
+#     level=logging.INFO,
+#     format="[%(asctime)s] %(levelname)s Crossbow %(message)s",
+#     datefmt="%H:%M:%S",
+#     stream=click.get_text_stream('stdout')
+# )
 
 
 class GitRemoteCallbacks(pygit2.RemoteCallbacks):
@@ -116,6 +116,11 @@ def signature(self):
         email = next(self.repo.config.get_multivar('user.email'))
         return pygit2.Signature(name, email, int(time.time()))
 
+    def parse_user_repo(self):
+        m = re.match('.*\/([^\/]+)\/([^\/\.]+)(\.git)?$', self.remote.url)
+        user, repo = m.group(1), m.group(2)
+        return user, repo
+
 
 class Queue(Repo):
 
@@ -309,7 +314,7 @@ def submit(task_names, config_path, dry_run, arrow_path, queue_path,
 
     queue.fetch()
 
-    version = get_version(arrow_path)
+    version = get_version(arrow_path, local_scheme='dirty-tag')
 
     variables = {
         # these should be renamed
@@ -348,38 +353,45 @@ def submit(task_names, config_path, dry_run, arrow_path, queue_path,
 
 @crossbow.command()
 @click.argument('job-name', required=True)
+@click.option('--queue-path', default=DEFAULT_QUEUE_PATH,
+              help='The repository path used for scheduling the tasks. '
+                   'Defaults to crossbow directory placed next to arrow')
 @click.option('--github-token', default=False, envvar='CROSSBOW_GITHUB_TOKEN',
               help='Oauth token for Github authentication')
-def status(job_name, github_token):
-    import github3
-
-    branch_name = job_name
+def status(job_name, queue_path, github_token):
+    queue = Queue(queue_path)
+    username, reponame = queue.parse_user_repo()
 
     gh = github3.login(token=github_token)
-    repo = gh.repository('kszucs', 'crossbow')  # FIXME(kszucs)
-    content = repo.file_contents('job.yml', branch_name)
+    repo = gh.repository(username, reponame)
+    content = repo.file_contents('job.yml', job_name)
 
     job = Job.from_dict(yaml.load(content.decoded))
     for name, task in job.tasks.items():
         for status in repo.statuses(task.commit):
-            print(status)
+            click.echo('[{:>7}] {:<} '.format(status.state, task.branch))
 
 
 @crossbow.command()
 @click.argument('job-name', required=True)
+@click.option('--target-dir', default=DEFAULT_ARROW_PATH,
+              help='Directory to download the build artifacts')
+@click.option('--queue-path', default=DEFAULT_QUEUE_PATH,
+              help='The repository path used for scheduling the tasks. '
+                   'Defaults to crossbow directory placed next to arrow')
 @click.option('--github-token', default=False, envvar='CROSSBOW_GITHUB_TOKEN',
               help='Oauth token for Github authentication')
-def artifacts(job_name, github_token):
-    import github3
-
-    branch_name = job_name
+def artifacts(job_name, target_dir, queue_path, github_token):
+    queue = Queue(queue_path)
+    username, reponame = queue.parse_user_repo()
 
     gh = github3.login(token=github_token)
-    repo = gh.repository('kszucs', 'crossbow')  # FIXME(kszucs)
-    release = repo.release_from_tag(branch_name)
+    repo = gh.repository(username, reponame)
+    release = repo.release_from_tag(job_name)
 
     for asset in release.assets():
-        asset.download()  # download to the current dir, FIXME
+        click.echo('Downloading asset {} ...'.format(asset.name))
+        asset.download(target_dir / asset.name)
 
 
 if __name__ == '__main__':
diff --git a/dev/tasks/python-wheels/appveyor.yml b/dev/tasks/python-wheels/appveyor.yml
index d6559d2a2ad43..b3407fb178cbd 100644
--- a/dev/tasks/python-wheels/appveyor.yml
+++ b/dev/tasks/python-wheels/appveyor.yml
@@ -55,7 +55,7 @@ deploy:
   release: {{ BUILD_TAG }}
   provider: GitHub
   auth_token: "%CROSSBOW_GITHUB_TOKEN%"
-  artifact: wheels\*.whl
+  artifact: /.*\.whl/
   draft: false
   prerelease: false
 

From 8c069c6b167545ef1722632b424a4ccfc8b90fd4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Thu, 14 Jun 2018 20:16:18 +0200
Subject: [PATCH 11/35] set autoincremented job_id outside of queue put

---
 dev/tasks/crossbow.py | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/dev/tasks/crossbow.py b/dev/tasks/crossbow.py
index 8067d4d17c55a..3254c811679a1 100755
--- a/dev/tasks/crossbow.py
+++ b/dev/tasks/crossbow.py
@@ -128,7 +128,7 @@ def __init__(self, repo_path):
         super(Queue, self).__init__(repo_path)
         self._updated_refs = []
 
-    def _next_job_id(self, prefix):
+    def next_job_id(self, prefix):
         """Auto increments the branch's identifier based on the prefix"""
         pattern = re.compile(prefix + '-(\d+)')
         matches = list(filter(None, map(pattern.match, self.repo.branches)))
@@ -175,13 +175,10 @@ def _create_tag(self, tag_name, commit_id, message=''):
 
     def put(self, job, prefix='build'):
         assert isinstance(job, Job)
-
-        job.branch = self._next_job_id(prefix)
+        assert job.branch is not None
 
         # create tasks' branches
         for task_name, task in job.tasks.items():
-            task.branch = '{}-{}'.format(job.branch, task_name)
-            task.params['BUILD_TAG'] = job.branch  # this is ugly
             branch = self._create_branch(task.branch, files=task.files())
             task.commit = str(branch.target)
 
@@ -315,11 +312,13 @@ def submit(task_names, config_path, dry_run, arrow_path, queue_path,
     queue.fetch()
 
     version = get_version(arrow_path, local_scheme='dirty-tag')
+    job_id = queue.next_job_id(prefix='build')
 
     variables = {
         # these should be renamed
         'PLAT': 'x86_64',
         'EMAIL': MESSAGE_EMAIL,
+        'BUILD_TAG': job_id,
         'BUILD_REF': str(target.head),
         'ARROW_SHA': str(target.head),
         'ARROW_REPO': target.remote.url,
@@ -331,22 +330,23 @@ def submit(task_names, config_path, dry_run, arrow_path, queue_path,
     with Path(config_path).open() as fp:
         config = yaml.load(fp)
 
-    # consider to set branch ids here instead of inside Queue.put
-
     # create and filter tasks
     tasks = {name: Task.from_dict(task)
              for name, task in config['tasks'].items()}
     tasks = {name: tasks[name] for name in task_names}
 
-    for task in tasks.values():
+    for task_name, task in tasks.items():
+        task.branch = '{}-{}'.format(job_id, task_name)
         task.params.update(variables)
 
     # create job
     job = Job(tasks)
+    job.branch = job_id
+
+    yaml_format = yaml.dump(job.to_dict(), default_flow_style=False)
+    click.echo(yaml_format)
 
-    if dry_run:
-        logging.info(yaml.dump(job.to_dict(), default_flow_style=False))
-    else:
+    if not dry_run:
         queue.put(job)
         queue.push(token=github_token)
 

From 5c6e15410715f672e08443b48a771d99f6af98d1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Thu, 14 Jun 2018 20:44:45 +0200
Subject: [PATCH 12/35] fix conda-win deployments

---
 dev/tasks/conda-recipes/appveyor.yml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/dev/tasks/conda-recipes/appveyor.yml b/dev/tasks/conda-recipes/appveyor.yml
index 5d7e7b969da6f..c81244f6f1d8b 100644
--- a/dev/tasks/conda-recipes/appveyor.yml
+++ b/dev/tasks/conda-recipes/appveyor.yml
@@ -23,7 +23,6 @@ environment:
     - TARGET_ARCH: x64
       CONDA_PY: 36
       CONDA_INSTALL_LOCN: C:\\Miniconda36-x64
-  ARROW_SRC: C:\apache-arrow
   ARROW_VERSION: {{ ARROW_VERSION }}
 
 install:
@@ -45,13 +44,14 @@ install:
 build: off
 
 test_script:
-  - git clone -b {{ ARROW_BRANCH }} {{ ARROW_REPO }} %ARROW_SRC% || exit /B
-  - git checkout {{ ARROW_SHA }} || exit /B
-  - pushd %ARROW_SRC%\dev\tasks\conda-recipes
-  - conda.exe build parquet-cpp arrow-cpp pyarrow
+  - git clone -b {{ ARROW_BRANCH }} {{ ARROW_REPO }} arrow || exit /B
+  - git -C arrow checkout {{ ARROW_SHA }} || exit /B
+  - pushd arrow\dev\tasks\conda-recipes
+  - conda.exe build --output-folder . parquet-cpp arrow-cpp pyarrow
 
 artifacts:
-  - path: C:\bld\win-64\*.tar.bz2
+  # this must be relative and child of the build C:\projects\crossbow directory
+  - path: arrow\dev\tasks\conda-recipes\*.tar.bz2
 
 deploy:
   release: {{ BUILD_TAG }}

From fd6bd231d08512b9a1f11f2121d8a3a4125c676b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Thu, 14 Jun 2018 20:50:42 +0200
Subject: [PATCH 13/35] print build id

---
 dev/tasks/crossbow.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/dev/tasks/crossbow.py b/dev/tasks/crossbow.py
index 3254c811679a1..d14e57fe3a4ed 100755
--- a/dev/tasks/crossbow.py
+++ b/dev/tasks/crossbow.py
@@ -349,6 +349,7 @@ def submit(task_names, config_path, dry_run, arrow_path, queue_path,
     if not dry_run:
         queue.put(job)
         queue.push(token=github_token)
+        click.echo('Pushed job identifier is: `{}`'.format(job_id))
 
 
 @crossbow.command()

From 14da2d5d7d5e0196597117c96e3d742c869b5136 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Thu, 14 Jun 2018 21:26:30 +0200
Subject: [PATCH 14/35] build prefix

---
 dev/tasks/crossbow.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/dev/tasks/crossbow.py b/dev/tasks/crossbow.py
index d14e57fe3a4ed..33071d9511327 100755
--- a/dev/tasks/crossbow.py
+++ b/dev/tasks/crossbow.py
@@ -288,6 +288,8 @@ def crossbow():
 
 @crossbow.command()
 @click.argument('task-names', nargs=-1, required=True)
+@click.option('--job-prefix', default='build',
+               help='Arbitrary prefix for branch names, e.g. nightly')
 @click.option('--config-path', default=DEFAULT_CONFIG_PATH,
               help='Task configuration yml. Defaults to tasks.yml')
 @click.option('--dry-run/--push', default=False,
@@ -301,8 +303,8 @@ def crossbow():
                    'Defaults to crossbow directory placed next to arrow')
 @click.option('--github-token', default=False, envvar='CROSSBOW_GITHUB_TOKEN',
               help='Oauth token for Github authentication')
-def submit(task_names, config_path, dry_run, arrow_path, queue_path,
-           github_token):
+def submit(task_names, job_prefix, config_path, dry_run, arrow_path,
+           queue_path, github_token):
     target = Repo(arrow_path)
     queue = Queue(queue_path)
 
@@ -312,7 +314,7 @@ def submit(task_names, config_path, dry_run, arrow_path, queue_path,
     queue.fetch()
 
     version = get_version(arrow_path, local_scheme='dirty-tag')
-    job_id = queue.next_job_id(prefix='build')
+    job_id = queue.next_job_id(prefix=job_prefix)
 
     variables = {
         # these should be renamed

From ccbc150801fabf4822d3be784c57a2238b952569 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Thu, 14 Jun 2018 21:35:50 +0200
Subject: [PATCH 15/35] little more status context

---
 dev/tasks/crossbow.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/dev/tasks/crossbow.py b/dev/tasks/crossbow.py
index 33071d9511327..01986a670d312 100755
--- a/dev/tasks/crossbow.py
+++ b/dev/tasks/crossbow.py
@@ -370,9 +370,16 @@ def status(job_name, queue_path, github_token):
     content = repo.file_contents('job.yml', job_name)
 
     job = Job.from_dict(yaml.load(content.decoded))
+
+    tpl = '[{:>7}] {:<20} {:<10} {:<38}'
+    header = tpl.format('status', 'branch', 'github_id', 'context')
+    click.echo(header)
+    click.echo('-' * len(header))
+
     for name, task in job.tasks.items():
         for status in repo.statuses(task.commit):
-            click.echo('[{:>7}] {:<} '.format(status.state, task.branch))
+            click.echo(tpl.format(status.state, task.branch, status.id,
+                                  status.context))
 
 
 @crossbow.command()

From 167a393879061282fb3bda317843dd828d21f96f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Thu, 14 Jun 2018 22:49:36 +0200
Subject: [PATCH 16/35] gry to remove osx wheel flag

---
 dev/tasks/python-wheels/osx-build.sh | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/dev/tasks/python-wheels/osx-build.sh b/dev/tasks/python-wheels/osx-build.sh
index 43bad901bef3e..1d9f072ca9a1b 100755
--- a/dev/tasks/python-wheels/osx-build.sh
+++ b/dev/tasks/python-wheels/osx-build.sh
@@ -44,7 +44,7 @@ function build_wheel {
 
     # Arrow is 64-bit-only at the moment
     export CFLAGS="-fPIC -arch x86_64 ${CFLAGS//"-arch i386"/}"
-    export CXXFLAGS="-fPIC -arch x86_64 ${CXXFLAGS//"-arch i386"} -std=c++11"
+    export CXXFLAGS="-fPIC -arch x86_64 ${CXXFLAGS//"-arch i386"}"
 
     ./bootstrap.sh
     ./b2 tools/bcp > /dev/null 2>&1
@@ -56,7 +56,6 @@ function build_wheel {
     pushd "$arrow_boost"
     ./bootstrap.sh
     ./bjam cxxflags="${CXXFLAGS}" \
-        linkflags="-std=c++11" \
         cflags="${CFLAGS}" \
         variant=release \
         link=shared \

From af6dd48019eb3c8f7851419593860c73ba0805c0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Thu, 14 Jun 2018 23:10:19 +0200
Subject: [PATCH 17/35] remove flag

---
 cpp/CMakeLists.txt                   | 3 +++
 dev/tasks/crossbow.py                | 6 +++---
 dev/tasks/python-wheels/osx-build.sh | 3 ++-
 3 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 5f29bdaa15c1b..9ac59e97df242 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -343,6 +343,9 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${ARROW_CXXFLAGS}")
 # For any C code, use the same flags.
 set(CMAKE_C_FLAGS "${CMAKE_CXX_FLAGS}")
 
+# Remove --std=c++11 to avoid errors from C compilers
+string(REPLACE "-std=c++11" "" CMAKE_C_FLAGS ${CMAKE_C_FLAGS})
+
 # Add C++-only flags, like -std=c++11
 set(CMAKE_CXX_FLAGS "${CXX_ONLY_FLAGS} ${CMAKE_CXX_FLAGS}")
 
diff --git a/dev/tasks/crossbow.py b/dev/tasks/crossbow.py
index 01986a670d312..7307551623815 100755
--- a/dev/tasks/crossbow.py
+++ b/dev/tasks/crossbow.py
@@ -371,15 +371,15 @@ def status(job_name, queue_path, github_token):
 
     job = Job.from_dict(yaml.load(content.decoded))
 
-    tpl = '[{:>7}] {:<20} {:<10} {:<38}'
-    header = tpl.format('status', 'branch', 'github_id', 'context')
+    tpl = '[{:>7}] {:<20} {:<10} {:<30}'
+    header = tpl.format('status', 'branch', 'github_id', 'target_url')
     click.echo(header)
     click.echo('-' * len(header))
 
     for name, task in job.tasks.items():
         for status in repo.statuses(task.commit):
             click.echo(tpl.format(status.state, task.branch, status.id,
-                                  status.context))
+                                  status.target_url))
 
 
 @crossbow.command()
diff --git a/dev/tasks/python-wheels/osx-build.sh b/dev/tasks/python-wheels/osx-build.sh
index 1d9f072ca9a1b..43bad901bef3e 100755
--- a/dev/tasks/python-wheels/osx-build.sh
+++ b/dev/tasks/python-wheels/osx-build.sh
@@ -44,7 +44,7 @@ function build_wheel {
 
     # Arrow is 64-bit-only at the moment
     export CFLAGS="-fPIC -arch x86_64 ${CFLAGS//"-arch i386"/}"
-    export CXXFLAGS="-fPIC -arch x86_64 ${CXXFLAGS//"-arch i386"}"
+    export CXXFLAGS="-fPIC -arch x86_64 ${CXXFLAGS//"-arch i386"} -std=c++11"
 
     ./bootstrap.sh
     ./b2 tools/bcp > /dev/null 2>&1
@@ -56,6 +56,7 @@ function build_wheel {
     pushd "$arrow_boost"
     ./bootstrap.sh
     ./bjam cxxflags="${CXXFLAGS}" \
+        linkflags="-std=c++11" \
         cflags="${CFLAGS}" \
         variant=release \
         link=shared \

From 0773f3ab5561ecfde15676927bc92d0b61384ec9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Fri, 15 Jun 2018 10:54:28 +0200
Subject: [PATCH 18/35] call rake version update

---
 dev/tasks/README.md                       | 42 +++++++++++------------
 dev/tasks/linux-packages/travis.linux.yml |  6 ++--
 2 files changed, 23 insertions(+), 25 deletions(-)

diff --git a/dev/tasks/README.md b/dev/tasks/README.md
index f5cc69eca58fb..80169015b1f13 100644
--- a/dev/tasks/README.md
+++ b/dev/tasks/README.md
@@ -106,7 +106,7 @@ The script does the following:
    $ git clone https://github.com/kszucs/crossbow
 
    $ cd arrow/dev/tasks
-   $ python crossbow.py
+   $ python crossbow.py submit <task names>
    ```
 
 2. Gets the HEAD commit of the currently checked out branch and generates
@@ -115,7 +115,7 @@ The script does the following:
 
    ```bash
    git checkout ARROW-<ticket number>
-   python dev/tasks/crossbow.py --dry-run
+   python dev/tasks/crossbow.py submit --dry-run <task names>
    ```
 
    > Note that the arrow branch must be pushed beforehand, because the script
@@ -124,13 +124,25 @@ The script does the following:
 3. Reads and renders the required build configurations with the parameters
    substituted.
 2. Create a commit per build configuration to its own branch. For example
-   to build `travis-linux-conda.yml` it will place a commit to the tip of
-   `crossbow@travis-linux-conda` branch.
+   to build conda recipes on linux it will place a commit to the tip of
+   `crossbow@conda-linux` branch.
 3. Pushes the modified branches to GitHub which triggers the builds.
    For authentication it uses github oauth tokens described in the install
    section.
 
 
+### Query the build status
+
+```bash
+python crossbow.py status <build id / branch name>
+```
+
+### Download the build artifacts
+
+```bash
+python crossbow.py artifacts <build id / branch name>
+```
+
 ### Examples
 
 The script accepts a pattern as a first argument to narrow the build scope:
@@ -138,7 +150,7 @@ The script accepts a pattern as a first argument to narrow the build scope:
 Run multiple builds:
 
 ```bash
-$ python crossbow.py linux-packages conda-linux wheel-win
+$ python crossbow.py submit linux-packages conda-linux wheel-win
 Repository: https://github.com/kszucs/arrow@tasks
 Commit SHA: 810a718836bb3a8cefc053055600bdcc440e6702
 Version: 0.9.1.dev48+g810a7188.d20180414
@@ -151,31 +163,17 @@ Pushed branches:
 Just render without applying or committing the changes:
 
 ```bash
-$ python crossbow.py --dry-run task_name
+$ python crossbow.py submit --dry-run task_name
 ```
 
 Run only `conda` package builds but on all platforms:
 
 ```bash
-$ python crossbow.py conda-win conda-osx conda-linux
-Repository: https://github.com/kszucs/arrow@tasks
-Commit SHA: 810a718836bb3a8cefc053055600bdcc440e6702
-Version: 0.9.1.dev48+g810a7188.d20180414
-Pushed branches:
- - conda-win
- - conda-osx
- - conda-linux
+$ python crossbow.py submit conda-win conda-osx conda-linux
 ```
 
 Run `wheel` builds:
 
 ```bash
-$ python crossbow.py wheel-osx wheel-linux wheel-win
-Repository: https://github.com/kszucs/arrow@tasks
-Commit SHA: 810a718836bb3a8cefc053055600bdcc440e6702
-Version: 0.9.1.dev48+g810a7188.d20180414
-Pushed branches:
- - wheel-osx
- - wheel-linux
- - wheel-win
+$ python crossbow.py submit wheel-osx wheel-linux wheel-win
 ```
diff --git a/dev/tasks/linux-packages/travis.linux.yml b/dev/tasks/linux-packages/travis.linux.yml
index 1c3cef14f578e..6a9bb597452bd 100644
--- a/dev/tasks/linux-packages/travis.linux.yml
+++ b/dev/tasks/linux-packages/travis.linux.yml
@@ -30,11 +30,11 @@ env:
 matrix:
   include:
     - script:
-        - (cd arrow/dev/tasks/linux-packages && travis_wait 40 rake apt:build APT_TARGETS=debian-stretch,ubuntu-trusty,ubuntu-xenial PARALLEL=yes DEBUG=no)
+        - (cd arrow/dev/tasks/linux-packages && travis_wait 40 && rake version:update && rake apt:build APT_TARGETS=debian-stretch,ubuntu-trusty,ubuntu-xenial PARALLEL=yes DEBUG=no)
     - script:
-        - (cd arrow/dev/tasks/linux-packages && travis_wait 40 rake apt:build APT_TARGETS=ubuntu-artful PARALLEL=yes DEBUG=no)
+        - (cd arrow/dev/tasks/linux-packages && travis_wait 40 && rake version:update && rake apt:build APT_TARGETS=ubuntu-artful PARALLEL=yes DEBUG=no)
     - script:
-        - (cd arrow/dev/tasks/linux-packages && rake yum:build PARALLEL=yes DEBUG=no)
+        - (cd arrow/dev/tasks/linux-packages && rake version:update && rake yum:build PARALLEL=yes DEBUG=no)
 
 before_install:
   - sudo apt update -y -qq

From 95bc3fafd3d8dff37264581ee2921341962d9eaa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Sun, 17 Jun 2018 22:56:03 +0200
Subject: [PATCH 19/35] retrieve commit's combined status

---
 dev/tasks/crossbow.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/dev/tasks/crossbow.py b/dev/tasks/crossbow.py
index 7307551623815..82b967e68f9b9 100755
--- a/dev/tasks/crossbow.py
+++ b/dev/tasks/crossbow.py
@@ -371,15 +371,16 @@ def status(job_name, queue_path, github_token):
 
     job = Job.from_dict(yaml.load(content.decoded))
 
-    tpl = '[{:>7}] {:<20} {:<10} {:<30}'
-    header = tpl.format('status', 'branch', 'github_id', 'target_url')
+    tpl = '[{:>7}] {:<24} {:<40}'
+    header = tpl.format('status', 'branch', 'sha')
     click.echo(header)
     click.echo('-' * len(header))
 
     for name, task in job.tasks.items():
-        for status in repo.statuses(task.commit):
-            click.echo(tpl.format(status.state, task.branch, status.id,
-                                  status.target_url))
+        commit = repo.commit(task.commit)
+        status = commit.status()
+
+        click.echo(tpl.format(status.state, task.branch, task.commit))
 
 
 @crossbow.command()

From bfb5eaeccccf694b9406fcba585750b6f57bc4ad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Sun, 17 Jun 2018 22:57:25 +0200
Subject: [PATCH 20/35] fix osx wheel builkds

---
 dev/tasks/python-wheels/travis.osx.yml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/dev/tasks/python-wheels/travis.osx.yml b/dev/tasks/python-wheels/travis.osx.yml
index 544cb96245978..f346297b5fb5b 100644
--- a/dev/tasks/python-wheels/travis.osx.yml
+++ b/dev/tasks/python-wheels/travis.osx.yml
@@ -14,9 +14,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+language: generic
+
 os: osx
+osx_image: xcode8.3
+
 sudo: required
-language: objective-c
 
 env:
   global:

From 71b69436ed11c1ba1a34a8c3b76e8d93368fab08 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Mon, 18 Jun 2018 17:51:45 +0200
Subject: [PATCH 21/35] postfix conda pkgs with arch

---
 dev/tasks/conda-recipes/appveyor.yml     |  4 +++-
 dev/tasks/conda-recipes/travis.linux.yml | 10 ++++++++--
 dev/tasks/conda-recipes/travis.osx.yml   | 10 ++++++++--
 3 files changed, 19 insertions(+), 5 deletions(-)

diff --git a/dev/tasks/conda-recipes/appveyor.yml b/dev/tasks/conda-recipes/appveyor.yml
index c81244f6f1d8b..80787e4fea5a7 100644
--- a/dev/tasks/conda-recipes/appveyor.yml
+++ b/dev/tasks/conda-recipes/appveyor.yml
@@ -48,10 +48,12 @@ test_script:
   - git -C arrow checkout {{ ARROW_SHA }} || exit /B
   - pushd arrow\dev\tasks\conda-recipes
   - conda.exe build --output-folder . parquet-cpp arrow-cpp pyarrow
+  - pushd win-64
+  - ren *.tar.bz2 *-win-64.tar.bz2
 
 artifacts:
   # this must be relative and child of the build C:\projects\crossbow directory
-  - path: arrow\dev\tasks\conda-recipes\*.tar.bz2
+  - path: arrow\dev\tasks\conda-recipes\win-64\*.tar.bz2
 
 deploy:
   release: {{ BUILD_TAG }}
diff --git a/dev/tasks/conda-recipes/travis.linux.yml b/dev/tasks/conda-recipes/travis.linux.yml
index ea9afe1c49f48..84ca83b27a0e9 100644
--- a/dev/tasks/conda-recipes/travis.linux.yml
+++ b/dev/tasks/conda-recipes/travis.linux.yml
@@ -56,13 +56,19 @@ script:
   - git clone -b {{ ARROW_BRANCH }} {{ ARROW_REPO }} arrow
   - git -C arrow checkout {{ ARROW_SHA }}
   - pushd arrow/dev/tasks/conda-recipes
-  - conda build parquet-cpp arrow-cpp pyarrow
+  - conda build --output-folder . parquet-cpp arrow-cpp pyarrow
+  - |
+    pushd linux-64
+    for file in *.tar.bz2; do
+      mv "$file" "$(basename "$file" .tar.bz2)-linux-64.tar.bz2"
+    done
+    popd
 
 deploy:
   provider: releases
   api_key: $CROSSBOW_GITHUB_TOKEN
   file_glob: true
-  file: /home/travis/miniconda3/conda-bld/linux-64/*.tar.bz2
+  file: $TRAVIS_BUILD_DIR/arrow/dev/tasks/conda-recipes/linux-64/*.tar.bz2
   skip_cleanup: true
   on:
     tags: true
diff --git a/dev/tasks/conda-recipes/travis.osx.yml b/dev/tasks/conda-recipes/travis.osx.yml
index 62c9891d28533..31c1c244b5f4f 100644
--- a/dev/tasks/conda-recipes/travis.osx.yml
+++ b/dev/tasks/conda-recipes/travis.osx.yml
@@ -65,13 +65,19 @@ script:
   - git clone -b {{ ARROW_BRANCH }} {{ ARROW_REPO }} arrow
   - git -C arrow checkout {{ ARROW_SHA }}
   - pushd arrow/dev/tasks/conda-recipes
-  - conda build parquet-cpp arrow-cpp pyarrow
+  - conda build --output-folder . parquet-cpp arrow-cpp pyarrow
+  - |
+    pushd osx-64
+    for file in *.tar.bz2; do
+      mv "$file" "$(basename "$file" .tar.bz2)-osx-64.tar.bz2"
+    done
+    popd
 
 deploy:
   provider: releases
   api_key: $CROSSBOW_GITHUB_TOKEN
   file_glob: true
-  file: /home/travis/miniconda3/conda-bld/osx-64/*.tar.bz2
+  file: $TRAVIS_BUILD_DIR/arrow/dev/tasks/conda-recipes/osx-64/*.tar.bz2
   skip_cleanup: true
   on:
     tags: true

From 0561e62225b49c25065ccd4cb276a924a8fb0b9a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Mon, 18 Jun 2018 18:47:35 +0200
Subject: [PATCH 22/35] force update conda win assets

---
 dev/tasks/conda-recipes/appveyor.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/dev/tasks/conda-recipes/appveyor.yml b/dev/tasks/conda-recipes/appveyor.yml
index 80787e4fea5a7..340ff0059b40a 100644
--- a/dev/tasks/conda-recipes/appveyor.yml
+++ b/dev/tasks/conda-recipes/appveyor.yml
@@ -62,6 +62,7 @@ deploy:
   artifact: /.*\.tar\.bz2/
   draft: false
   prerelease: false
+  force_update: true
 
 notifications:
   - provider: Email

From 9b2223806323f93e7ed84d0cdfcdf1892da3fcea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Mon, 18 Jun 2018 19:12:20 +0200
Subject: [PATCH 23/35] update tasks

---
 dev/tasks/tasks.yml | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
index 87a493ccdcbdd..8aaf469f44ee6 100644
--- a/dev/tasks/tasks.yml
+++ b/dev/tasks/tasks.yml
@@ -16,19 +16,12 @@
 # under the License.
 
 tasks:
-  # supported keys
-  # - name: arbitrary task name
+  # arbitrary_task_name:
   #   branch: defaults to name
   #   platform: osx|linux|win
   #   template: path of jinja2 templated yml
   #   params: optional extra parameters
 
-  # configures master to submit packaging builds
-  # - name: config-nightlies
-  #   branch: master
-  #   platform: linux
-  #   template: config-nightlies/travis.linux.yml
-
   # conda packages
   conda-linux:
     platform: linux

From 4c64db6fb9f004ec1a322c22c7061eeba08429ab Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Mon, 18 Jun 2018 15:53:14 -0400
Subject: [PATCH 24/35] Add email property and use target repo user.email by
 default

---
 dev/tasks/crossbow.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/dev/tasks/crossbow.py b/dev/tasks/crossbow.py
index 82b967e68f9b9..5367533e06e47 100755
--- a/dev/tasks/crossbow.py
+++ b/dev/tasks/crossbow.py
@@ -17,6 +17,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
+import os
 import re
 import yaml
 import time
@@ -110,11 +111,14 @@ def remote(self):
     def origin(self):
         return self.repo.remotes['origin']
 
+    @property
+    def email(self):
+        return next(self.repo.config.get_multivar('user.email'))
+
     @property
     def signature(self):
         name = next(self.repo.config.get_multivar('user.name'))
-        email = next(self.repo.config.get_multivar('user.email'))
-        return pygit2.Signature(name, email, int(time.time()))
+        return pygit2.Signature(name, self.email, int(time.time()))
 
     def parse_user_repo(self):
         m = re.match('.*\/([^\/]+)\/([^\/\.]+)(\.git)?$', self.remote.url)
@@ -319,7 +323,7 @@ def submit(task_names, job_prefix, config_path, dry_run, arrow_path,
     variables = {
         # these should be renamed
         'PLAT': 'x86_64',
-        'EMAIL': MESSAGE_EMAIL,
+        'EMAIL': os.environ.get('CROSSBOW_EMAIL', target.email),
         'BUILD_TAG': job_id,
         'BUILD_REF': str(target.head),
         'ARROW_SHA': str(target.head),

From 1b3d5fb56042f029ed88b1706eaa9198b2f13ba3 Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Mon, 18 Jun 2018 15:53:53 -0400
Subject: [PATCH 25/35] Code formatting cleanups

---
 dev/tasks/crossbow.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/dev/tasks/crossbow.py b/dev/tasks/crossbow.py
index 5367533e06e47..e7bb2c12cf992 100755
--- a/dev/tasks/crossbow.py
+++ b/dev/tasks/crossbow.py
@@ -188,7 +188,7 @@ def put(self, job, prefix='build'):
 
         # create job's branch
         branch = self._create_branch(job.branch, files=job.files())
-        tag = self._create_tag(job.branch, branch.target)
+        self._create_tag(job.branch, branch.target)
 
         return branch
 
@@ -293,7 +293,7 @@ def crossbow():
 @crossbow.command()
 @click.argument('task-names', nargs=-1, required=True)
 @click.option('--job-prefix', default='build',
-               help='Arbitrary prefix for branch names, e.g. nightly')
+              help='Arbitrary prefix for branch names, e.g. nightly')
 @click.option('--config-path', default=DEFAULT_CONFIG_PATH,
               help='Task configuration yml. Defaults to tasks.yml')
 @click.option('--dry-run/--push', default=False,
@@ -350,7 +350,7 @@ def submit(task_names, job_prefix, config_path, dry_run, arrow_path,
     job.branch = job_id
 
     yaml_format = yaml.dump(job.to_dict(), default_flow_style=False)
-    click.echo(yaml_format)
+    click.echo(yaml_format.strip())
 
     if not dry_run:
         queue.put(job)

From 12efaba6b308212eb2c79fd64d18deb9ba84b636 Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Mon, 18 Jun 2018 15:54:10 -0400
Subject: [PATCH 26/35] Validate github token

---
 dev/tasks/crossbow.py | 28 ++++++++++++++++++++++------
 1 file changed, 22 insertions(+), 6 deletions(-)

diff --git a/dev/tasks/crossbow.py b/dev/tasks/crossbow.py
index e7bb2c12cf992..bd6e6806d0ebd 100755
--- a/dev/tasks/crossbow.py
+++ b/dev/tasks/crossbow.py
@@ -290,6 +290,25 @@ def crossbow():
     pass
 
 
+def github_token_validation_callback(ctx, param, value):
+    if value is None:
+        raise click.ClickException(
+            'Could not determine GitHub token. Please set the '
+            'CROSSBOW_GITHUB_TOKEN environment variable to a '
+            'valid github access token or pass one to --github-token.'
+        )
+    return value
+
+
+github_token = click.option(
+    '--github-token',
+    default=None,
+    envvar='CROSSBOW_GITHUB_TOKEN',
+    help='OAuth token for Github authentication',
+    callback=github_token_validation_callback,
+)
+
+
 @crossbow.command()
 @click.argument('task-names', nargs=-1, required=True)
 @click.option('--job-prefix', default='build',
@@ -305,8 +324,7 @@ def crossbow():
 @click.option('--queue-path', default=DEFAULT_QUEUE_PATH,
               help='The repository path used for scheduling the tasks. '
                    'Defaults to crossbow directory placed next to arrow')
-@click.option('--github-token', default=False, envvar='CROSSBOW_GITHUB_TOKEN',
-              help='Oauth token for Github authentication')
+@github_token
 def submit(task_names, job_prefix, config_path, dry_run, arrow_path,
            queue_path, github_token):
     target = Repo(arrow_path)
@@ -363,8 +381,7 @@ def submit(task_names, job_prefix, config_path, dry_run, arrow_path,
 @click.option('--queue-path', default=DEFAULT_QUEUE_PATH,
               help='The repository path used for scheduling the tasks. '
                    'Defaults to crossbow directory placed next to arrow')
-@click.option('--github-token', default=False, envvar='CROSSBOW_GITHUB_TOKEN',
-              help='Oauth token for Github authentication')
+@github_token
 def status(job_name, queue_path, github_token):
     queue = Queue(queue_path)
     username, reponame = queue.parse_user_repo()
@@ -394,8 +411,7 @@ def status(job_name, queue_path, github_token):
 @click.option('--queue-path', default=DEFAULT_QUEUE_PATH,
               help='The repository path used for scheduling the tasks. '
                    'Defaults to crossbow directory placed next to arrow')
-@click.option('--github-token', default=False, envvar='CROSSBOW_GITHUB_TOKEN',
-              help='Oauth token for Github authentication')
+@github_token
 def artifacts(job_name, target_dir, queue_path, github_token):
     queue = Queue(queue_path)
     username, reponame = queue.parse_user_repo()

From 337234d243726ca2660b0e34b0bd6e7b0cd38b10 Mon Sep 17 00:00:00 2001
From: Phillip Cloud <cpcloud@gmail.com>
Date: Mon, 18 Jun 2018 16:12:39 -0400
Subject: [PATCH 27/35] Validate submit tasks

---
 dev/tasks/crossbow.py | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/dev/tasks/crossbow.py b/dev/tasks/crossbow.py
index bd6e6806d0ebd..ffa78fdb6401b 100755
--- a/dev/tasks/crossbow.py
+++ b/dev/tasks/crossbow.py
@@ -309,11 +309,29 @@ def github_token_validation_callback(ctx, param, value):
 )
 
 
+def config_path_validation_callback(ctx, param, value):
+    with Path(value).open() as fp:
+        config = yaml.load(fp)
+    task_names = ctx.params['task_names']
+    valid_tasks = set(config['tasks'].keys())
+    invalid_tasks = {task for task in task_names if task not in valid_tasks}
+    if invalid_tasks:
+        raise click.ClickException(
+            'Invalid task(s) {!r}. Must be one of {!r}'.format(
+                invalid_tasks,
+                valid_tasks
+            )
+        )
+    return value
+
+
 @crossbow.command()
 @click.argument('task-names', nargs=-1, required=True)
 @click.option('--job-prefix', default='build',
               help='Arbitrary prefix for branch names, e.g. nightly')
 @click.option('--config-path', default=DEFAULT_CONFIG_PATH,
+              type=click.Path(exists=True),
+              callback=config_path_validation_callback,
               help='Task configuration yml. Defaults to tasks.yml')
 @click.option('--dry-run/--push', default=False,
               help='Just display the rendered CI configurations without '

From f317a06b367f2e0626051f433a40a26cb2867615 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Mon, 18 Jun 2018 22:27:38 +0200
Subject: [PATCH 28/35] outdated readme section

---
 dev/tasks/README.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/dev/tasks/README.md b/dev/tasks/README.md
index 80169015b1f13..70c1286f4071e 100644
--- a/dev/tasks/README.md
+++ b/dev/tasks/README.md
@@ -123,9 +123,9 @@ The script does the following:
 
 3. Reads and renders the required build configurations with the parameters
    substituted.
-2. Create a commit per build configuration to its own branch. For example
-   to build conda recipes on linux it will place a commit to the tip of
-   `crossbow@conda-linux` branch.
+2. Create a branch per task, prefixed with the job id. For example
+   to build conda recipes on linux it will create a new branch:
+   `crossbow@build-<id>-conda-linux`.
 3. Pushes the modified branches to GitHub which triggers the builds.
    For authentication it uses github oauth tokens described in the install
    section.

From 2e9fcf9ab9c3ed745392316e5dfbd9c7f90b149d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Mon, 18 Jun 2018 22:28:07 +0200
Subject: [PATCH 29/35] remove logging config

---
 dev/tasks/crossbow.py | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/dev/tasks/crossbow.py b/dev/tasks/crossbow.py
index ffa78fdb6401b..70301fa33de7a 100755
--- a/dev/tasks/crossbow.py
+++ b/dev/tasks/crossbow.py
@@ -33,14 +33,6 @@
 from setuptools_scm import get_version
 
 
-# logging.basicConfig(
-#     level=logging.INFO,
-#     format="[%(asctime)s] %(levelname)s Crossbow %(message)s",
-#     datefmt="%H:%M:%S",
-#     stream=click.get_text_stream('stdout')
-# )
-
-
 class GitRemoteCallbacks(pygit2.RemoteCallbacks):
 
     def __init__(self, token):

From e4e971c2130bc313e3fc924c09b63fec69473045 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Mon, 18 Jun 2018 22:41:53 +0200
Subject: [PATCH 30/35] don't depend on any previous commit

---
 dev/tasks/crossbow.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/dev/tasks/crossbow.py b/dev/tasks/crossbow.py
index 70301fa33de7a..e17b20598bef4 100755
--- a/dev/tasks/crossbow.py
+++ b/dev/tasks/crossbow.py
@@ -134,7 +134,7 @@ def next_job_id(self, prefix):
             latest = 0
         return '{}-{}'.format(prefix, latest + 1)
 
-    def _create_branch(self, branch_name, files, parents=None, message=''):
+    def _create_branch(self, branch_name, files, parents=[], message=''):
         # 1. create tree
         builder = self.repo.TreeBuilder()
 
@@ -147,7 +147,6 @@ def _create_branch(self, branch_name, files, parents=None, message=''):
 
         # 2. create commit with the tree created above
         author = committer = self.signature
-        parents = parents or [self.head]
         commit_id = self.repo.create_commit(None, author, committer, message,
                                             tree_id, parents)
         commit = self.repo[commit_id]
@@ -189,8 +188,6 @@ def push(self, token):
         self.origin.push(self._updated_refs, callbacks=callbacks)
         self.updated_refs = []
 
-        # logging.info('\n - '.join(['\nUpdated branches:'] + shorthands))
-
 
 class Platform(Enum):
     # in alphabetical order

From b892471cc6cc9a888477ef9d145bd881e712dbd1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Mon, 18 Jun 2018 22:45:47 +0200
Subject: [PATCH 31/35] explicit task names instead of placeholder in readme;
 foxme note for linux-packages

---
 dev/tasks/README.md                       | 4 ++--
 dev/tasks/linux-packages/travis.linux.yml | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/dev/tasks/README.md b/dev/tasks/README.md
index 70c1286f4071e..35140fa44b33e 100644
--- a/dev/tasks/README.md
+++ b/dev/tasks/README.md
@@ -106,7 +106,7 @@ The script does the following:
    $ git clone https://github.com/kszucs/crossbow
 
    $ cd arrow/dev/tasks
-   $ python crossbow.py submit <task names>
+   $ python crossbow.py submit conda-win conda-linux conda-osx
    ```
 
 2. Gets the HEAD commit of the currently checked out branch and generates
@@ -115,7 +115,7 @@ The script does the following:
 
    ```bash
    git checkout ARROW-<ticket number>
-   python dev/tasks/crossbow.py submit --dry-run <task names>
+   python dev/tasks/crossbow.py submit --dry-run conda-linux conda-osx
    ```
 
    > Note that the arrow branch must be pushed beforehand, because the script
diff --git a/dev/tasks/linux-packages/travis.linux.yml b/dev/tasks/linux-packages/travis.linux.yml
index 6a9bb597452bd..8cc637728a367 100644
--- a/dev/tasks/linux-packages/travis.linux.yml
+++ b/dev/tasks/linux-packages/travis.linux.yml
@@ -67,7 +67,7 @@ deploy:
   provider: releases
   api_key: $CROSSBOW_GITHUB_TOKEN
   file_glob: true
-  file: /path/to/pachages/*.tar.gz
+  file: /path/to/pachages/*.tar.gz  # FIXME(kszucs) after the builds pass
   skip_cleanup: true
   on:
     tags: true

From 062da7c7c42d9ec6a8645c143ddb69926dcd79f9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Tue, 19 Jun 2018 20:12:49 +0200
Subject: [PATCH 32/35] conda win renames

---
 dev/tasks/conda-recipes/appveyor.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/dev/tasks/conda-recipes/appveyor.yml b/dev/tasks/conda-recipes/appveyor.yml
index 340ff0059b40a..0956a8a009974 100644
--- a/dev/tasks/conda-recipes/appveyor.yml
+++ b/dev/tasks/conda-recipes/appveyor.yml
@@ -49,7 +49,8 @@ test_script:
   - pushd arrow\dev\tasks\conda-recipes
   - conda.exe build --output-folder . parquet-cpp arrow-cpp pyarrow
   - pushd win-64
-  - ren *.tar.bz2 *-win-64.tar.bz2
+  - for %%f in (*.tar.bz2) do ren "%%f" "%%~nf-win-64.tar.bz2"
+  - popd
 
 artifacts:
   # this must be relative and child of the build C:\projects\crossbow directory

From 0701d6401606878ab4340b1dcf7cfb15192ab4e4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Wed, 20 Jun 2018 16:38:33 +0200
Subject: [PATCH 33/35] fixed conda-win deployments

---
 dev/tasks/conda-recipes/appveyor.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/dev/tasks/conda-recipes/appveyor.yml b/dev/tasks/conda-recipes/appveyor.yml
index 0956a8a009974..9165a81b379dc 100644
--- a/dev/tasks/conda-recipes/appveyor.yml
+++ b/dev/tasks/conda-recipes/appveyor.yml
@@ -50,7 +50,6 @@ test_script:
   - conda.exe build --output-folder . parquet-cpp arrow-cpp pyarrow
   - pushd win-64
   - for %%f in (*.tar.bz2) do ren "%%f" "%%~nf-win-64.tar.bz2"
-  - popd
 
 artifacts:
   # this must be relative and child of the build C:\projects\crossbow directory

From 83f333b527d24f5208a68dd8e9e51a39995c1da0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Wed, 20 Jun 2018 20:23:11 +0200
Subject: [PATCH 34/35] don't use dirty flag in version number

---
 dev/tasks/crossbow.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dev/tasks/crossbow.py b/dev/tasks/crossbow.py
index e17b20598bef4..a2a29ebad4801 100755
--- a/dev/tasks/crossbow.py
+++ b/dev/tasks/crossbow.py
@@ -342,7 +342,7 @@ def submit(task_names, job_prefix, config_path, dry_run, arrow_path,
 
     queue.fetch()
 
-    version = get_version(arrow_path, local_scheme='dirty-tag')
+    version = get_version(arrow_path, local_scheme=lambda v: '')
     job_id = queue.next_job_id(prefix=job_prefix)
 
     variables = {

From 87860d009613ad6888304644f58fc1e6bcec8c69 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Wed, 20 Jun 2018 20:29:38 +0200
Subject: [PATCH 35/35] batch correctly rename double extensions

---
 dev/tasks/conda-recipes/appveyor.yml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/dev/tasks/conda-recipes/appveyor.yml b/dev/tasks/conda-recipes/appveyor.yml
index 9165a81b379dc..3d3f3305e8662 100644
--- a/dev/tasks/conda-recipes/appveyor.yml
+++ b/dev/tasks/conda-recipes/appveyor.yml
@@ -49,7 +49,10 @@ test_script:
   - pushd arrow\dev\tasks\conda-recipes
   - conda.exe build --output-folder . parquet-cpp arrow-cpp pyarrow
   - pushd win-64
-  - for %%f in (*.tar.bz2) do ren "%%f" "%%~nf-win-64.tar.bz2"
+  - for %%f in (*.tar.bz2) do (
+      set %%g=%%~nf
+      ren "%%f" "%%~ng-win-64.tar.bz2"
+    )
 
 artifacts:
   # this must be relative and child of the build C:\projects\crossbow directory