make pygit2 and github3 optional dependencies for crossbow [skip ci]

dev/tasks/crossbow.py

@@ -32,13 +32,23 @@
 
 import click
 import toolz
-import pygit2
-import github3
-import jira.client
-from jinja2 import Template, StrictUndefined
 from setuptools_scm.git import parse as parse_git_version
 from ruamel.yaml import YAML
 
+try:
+    import github3
+except ImportError:
+    github3 = object
+
+try:
+    import pygit2
+except ImportError:
+    PygitRepository = str
+    PygitRemoteCallbacks = object
+else:
+    PygitRepository = pygit2.Repository
+    PygitRemoteCallbacks = pygit2.RemoteCallbacks
+
 
 CWD = Path(__file__).parent.absolute()
 
@@ -117,6 +127,7 @@ class JiraChangelog:
 
     def __init__(self, version, username, password,
                  server='https://issues.apache.org/jira'):
+        import jira.client
         self.server = server
         # clean version to the first numbers
         self.version = '.'.join(version.split('.')[:3])
@@ -201,7 +212,7 @@ def render(self, old_changelog, website=False):
         return out.getvalue().strip()
 
 
-class GitRemoteCallbacks(pygit2.RemoteCallbacks):
+class GitRemoteCallbacks(PygitRemoteCallbacks):
 
     def __init__(self, token):
         self.token = token
@@ -249,7 +260,7 @@ class Repo:
     def __init__(self, path, github_token=None, remote_url=None,
                  require_https=False):
         self.path = Path(path)
-        self.repo = pygit2.Repository(str(self.path))
+        self.repo = PygitRepository(str(self.path))
         self.github_token = github_token
         self.require_https = require_https
         self._remote_url = remote_url
@@ -398,6 +409,10 @@ def file_contents(self, commit_id, file):
 
     def _parse_github_user_repo(self):
         m = re.match(r'.*\/([^\/]+)\/([^\/\.]+)(\.git)?$', self.remote_url)
+        if m is None:
+            raise ValueError("Unable to parse the github owner and repository "
+                             "from the repository's remote url '{}'"
+                             .format(self.remote_url))
         user, repo = m.group(1), m.group(2)
         return user, repo
 
@@ -671,6 +686,7 @@ def __init__(self, platform, ci, template, artifacts=None, params=None):
         self._status = None  # status cache
 
     def render_files(self, **extra_params):
+        from jinja2 import Template, StrictUndefined
         path = CWD / self.template
         params = toolz.merge(self.params, extra_params)
         template = Template(path.read_text(), undefined=StrictUndefined)

dev/tasks/python-wheels/travis.osx.yml

@@ -25,10 +25,10 @@ env:
   global:
     - PLAT=x86_64
     - TRAVIS_TAG={{ task.tag }}
-    - MACOSX_DEPLOYMENT_TARGET="10.9"
     - PYARROW_VERSION={{ arrow.no_rc_version }}
     - PYARROW_BUILD_VERBOSE=1
     - MB_PYTHON_VERSION={{ python_version }}
+    - MACOSX_DEPLOYMENT_TARGET="10.9"
 
 before_install:
   - git clone https://github.com/matthew-brett/multibuild # TODO pin it
@@ -40,29 +40,26 @@ before_install:
 
   # ARROW-3976 Old versions of git can cause failures when Homebrew prints a
   # donation solicitation. Attempt to update git
-  - git --version
-  - brew upgrade git
+  - brew upgrade git python
 
   # Also remove artifacts that depend on Boost
   - brew uninstall boost cgal postgis sfcgal
   - brew update
   - brew upgrade cmake
   - travis_wait 30 brew install bison flex grpc [email protected] llvm@7 zlib gperftools
-  # Remove shared grpc libraries installed by brew to make sure
+  # remove shared grpc libraries installed by brew to make sure
   # we are linked against the static ones.
   - rm -f /usr/local/opt/grpc/lib/*.dylib
 
+  # source utilities required for wheel builds
   - export CONFIG_PATH=`pwd`/arrow/dev/tasks/python-wheels/osx-build.sh
   - source multibuild/common_utils.sh
   - source multibuild/travis_osx_steps.sh
-
   - before_install
   # Fix SSL TLS issue for Python 3.5 on macOS
   - pip install requests[security]
 
 install:
-  - mkdir -p dist
-
   # the following functions are defined in osx-build.sh
   - build_wheel arrow
 
@@ -78,19 +75,13 @@ install:
   # run the import tests
   - run_import_tests
 
-  # move built wheels to a top level directory
-  - mv -v arrow/python/dist/* dist/
-  # reinstall openssl because travis' deployment script depends on it
   - brew install [email protected]
-
-deploy:
-  provider: releases
-  api_key: $CROSSBOW_GITHUB_TOKEN
-  file_glob: true
-  file: dist/*.whl
-  skip_cleanup: true
-  on:
-    tags: true
+  # before_install activates a virtualenv but we need the system python3
+  - deactivate
+  # crossbow dependencies for deployment
+  - unset MACOSX_DEPLOYMENT_TARGET
+  - pip3 install click ruamel.yaml setuptools_scm github3.py toolz
+  - python3 arrow/dev/tasks/crossbow.py --queue-path $(pwd) --queue-remote {{ queue.remote_url }} upload-artifacts --sha {{ task.branch }} --tag {{ task.tag }} --pattern "arrow/python/dist/*.whl"
 
 notifications:
   email: