From 28d6d9757b17d21cd848ed31a11fc45ee5b956ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Thu, 24 Oct 2019 20:32:14 +0200 Subject: [PATCH] ARROW-6963: [Packaging][Wheel][OSX] Use crossbow's command to deploy artifacts from travis builds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #5726 from kszucs/osx-wheel-upload and squashes the following commits: 320b76e90 install requests security for system python 4b42380ea pass default paths as strings 80e2e5b09 better error if pygit2 is missing 1bc93a869 make pygit2 and github3 optional dependencies for crossbow Authored-by: Krisztián Szűcs Signed-off-by: Krisztián Szűcs --- dev/tasks/crossbow.py | 41 +++++++++++++++++++------- dev/tasks/python-wheels/travis.osx.yml | 29 +++++++----------- 2 files changed, 41 insertions(+), 29 deletions(-) diff --git a/dev/tasks/crossbow.py b/dev/tasks/crossbow.py index 50f965f9417a2..5fe506399a8f4 100755 --- a/dev/tasks/crossbow.py +++ b/dev/tasks/crossbow.py @@ -32,13 +32,21 @@ import click import toolz -import pygit2 -import github3 -import jira.client -from jinja2 import Template, StrictUndefined from setuptools_scm.git import parse as parse_git_version from ruamel.yaml import YAML +try: + import github3 +except ImportError: + github3 = object + +try: + import pygit2 +except ImportError: + PygitRemoteCallbacks = object +else: + PygitRemoteCallbacks = pygit2.RemoteCallbacks + CWD = Path(__file__).parent.absolute() @@ -117,6 +125,7 @@ class JiraChangelog: def __init__(self, version, username, password, server='https://issues.apache.org/jira'): + import jira.client self.server = server # clean version to the first numbers self.version = '.'.join(version.split('.')[:3]) @@ -201,7 +210,7 @@ def render(self, old_changelog, website=False): return out.getvalue().strip() -class GitRemoteCallbacks(pygit2.RemoteCallbacks): +class GitRemoteCallbacks(PygitRemoteCallbacks): def __init__(self, token): self.token = token @@ -249,10 +258,10 @@ class Repo: def __init__(self, path, github_token=None, remote_url=None, require_https=False): self.path = Path(path) - self.repo = pygit2.Repository(str(self.path)) self.github_token = github_token self.require_https = require_https self._remote_url = remote_url + self._pygit_repo = None self._github_repo = None # set by as_github_repo() self._updated_refs = [] @@ -267,6 +276,12 @@ def __str__(self): head=self.head ) + @property + def repo(self): + if self._pygit_repo is None: + self._pygit_repo = pygit2.Repository(str(self.path)) + return self._pygit_repo + @property def origin(self): remote = self.repo.remotes['origin'] @@ -398,6 +413,10 @@ def file_contents(self, commit_id, file): def _parse_github_user_repo(self): m = re.match(r'.*\/([^\/]+)\/([^\/\.]+)(\.git)?$', self.remote_url) + if m is None: + raise ValueError("Unable to parse the github owner and repository " + "from the repository's remote url '{}'" + .format(self.remote_url)) user, repo = m.group(1), m.group(2) return user, repo @@ -671,6 +690,7 @@ def __init__(self, platform, ci, template, artifacts=None, params=None): self._status = None # status cache def render_files(self, **extra_params): + from jinja2 import Template, StrictUndefined path = CWD / self.template params = toolz.merge(self.params, extra_params) template = Template(path.read_text(), undefined=StrictUndefined) @@ -1028,11 +1048,11 @@ def send(self, smtp_user, smtp_password, smtp_server, smtp_port): @click.option('--github-token', '-t', default=None, help='OAuth token for GitHub authentication') @click.option('--arrow-path', '-a', - type=click.Path(exists=True), default=DEFAULT_ARROW_PATH, + type=click.Path(exists=True), default=str(DEFAULT_ARROW_PATH), help='Arrow\'s repository path. Defaults to the repository of ' 'this script') @click.option('--queue-path', '-q', - type=click.Path(exists=True), default=DEFAULT_QUEUE_PATH, + type=click.Path(exists=True), default=str(DEFAULT_QUEUE_PATH), help='The repository path used for scheduling the tasks. ' 'Defaults to crossbow directory placed next to arrow') @click.option('--queue-remote', '-qr', default=None, @@ -1059,7 +1079,7 @@ def crossbow(ctx, github_token, arrow_path, queue_path, queue_remote, @crossbow.command() @click.option('--changelog-path', '-c', type=click.Path(exists=True), - default=DEFAULT_ARROW_PATH / 'CHANGELOG.md', + default=str(DEFAULT_ARROW_PATH / 'CHANGELOG.md'), help='Path of changelog to update') @click.option('--arrow-version', '-v', default=None, help='Set target version explicitly') @@ -1232,7 +1252,8 @@ def report(obj, job_name, sender_name, sender_email, recipient_email, @crossbow.command() @click.argument('job-name', required=True) -@click.option('-t', '--target-dir', default=DEFAULT_ARROW_PATH / 'packages', +@click.option('-t', '--target-dir', + default=str(DEFAULT_ARROW_PATH / 'packages'), type=click.Path(file_okay=False, dir_okay=True), help='Directory to download the build artifacts') @click.pass_obj diff --git a/dev/tasks/python-wheels/travis.osx.yml b/dev/tasks/python-wheels/travis.osx.yml index 7d4177fd4186a..cdf39bc58b5de 100644 --- a/dev/tasks/python-wheels/travis.osx.yml +++ b/dev/tasks/python-wheels/travis.osx.yml @@ -25,10 +25,10 @@ env: global: - PLAT=x86_64 - TRAVIS_TAG={{ task.tag }} - - MACOSX_DEPLOYMENT_TARGET="10.9" - PYARROW_VERSION={{ arrow.no_rc_version }} - PYARROW_BUILD_VERBOSE=1 - MB_PYTHON_VERSION={{ python_version }} + - MACOSX_DEPLOYMENT_TARGET="10.9" before_install: - git clone https://github.com/matthew-brett/multibuild # TODO pin it @@ -40,29 +40,26 @@ before_install: # ARROW-3976 Old versions of git can cause failures when Homebrew prints a # donation solicitation. Attempt to update git - - git --version - - brew upgrade git + - brew upgrade git python # Also remove artifacts that depend on Boost - brew uninstall boost cgal postgis sfcgal - brew update - brew upgrade cmake - travis_wait 30 brew install bison flex grpc openssl@1.1 llvm@7 zlib gperftools - # Remove shared grpc libraries installed by brew to make sure + # remove shared grpc libraries installed by brew to make sure # we are linked against the static ones. - rm -f /usr/local/opt/grpc/lib/*.dylib + # source utilities required for wheel builds - export CONFIG_PATH=`pwd`/arrow/dev/tasks/python-wheels/osx-build.sh - source multibuild/common_utils.sh - source multibuild/travis_osx_steps.sh - - before_install # Fix SSL TLS issue for Python 3.5 on macOS - pip install requests[security] install: - - mkdir -p dist - # the following functions are defined in osx-build.sh - build_wheel arrow @@ -78,19 +75,13 @@ install: # run the import tests - run_import_tests - # move built wheels to a top level directory - - mv -v arrow/python/dist/* dist/ - # reinstall openssl because travis' deployment script depends on it - brew install openssl@1.1 - -deploy: - provider: releases - api_key: $CROSSBOW_GITHUB_TOKEN - file_glob: true - file: dist/*.whl - skip_cleanup: true - on: - tags: true + # before_install activates a virtualenv but we need the system python3 + - deactivate + # crossbow dependencies for deployment + - unset MACOSX_DEPLOYMENT_TARGET + - pip3 install click ruamel.yaml setuptools_scm github3.py toolz requests[security] + - python3 arrow/dev/tasks/crossbow.py --queue-path $(pwd) --queue-remote {{ queue.remote_url }} upload-artifacts --sha {{ task.branch }} --tag {{ task.tag }} --pattern "arrow/python/dist/*.whl" notifications: email: