cross repository dependency management

.travis.yml

@@ -27,7 +27,6 @@ install:
   - cp -r ../maintainer-quality-tools/ $HOME
   - mv tests/test_repo/* ./
   - export PATH=$HOME/maintainer-quality-tools/travis:$PATH
-  - git clone https://github.com/OCA/partner-contact ${HOME}/department -b 8.0 # only used if VERSION not set in env
   - travis_install_nightly 8.0 # only used if VERSION not set in env
 
 script:

README.md

@@ -12,11 +12,9 @@ To setup the TravisCI continuous integration for your project, just copy the
 content of the [`/sample_files`](https://github.com/OCA/maintainer-quality-tools/tree/master/sample_files)
 to your project’s root directory.
 
-If your project depends on other OCA/Github repositories simply add the following under `before_install` section:
+If your project depends on other OCA/Github repositories, create a file called `oca_dependencies.txt` at the root of your project and list the dependencies in there, one per line:
 
-    install:
-      - git clone https://github.com/OCA/a_project_x ${HOME}/a_project_x -b ${VERSION}
-      - git clone https://github.com/OCA/a_project_y ${HOME}/a_project_y -b ${VERSION}
+    project_name optional_repository_url optional_branch_name
 
 The addons path used will automatically consider these repositories.
 

sample_files/.travis.yml

@@ -21,8 +21,6 @@ install:
   - git clone --depth=1 https://github.com/OCA/maintainer-quality-tools.git ${HOME}/maintainer-quality-tools
   - export PATH=${HOME}/maintainer-quality-tools/travis:${PATH}
   - travis_install_nightly
-# example: dependency
-# - git clone --depth=1 https://github.com/OCA/webkit-tools -b ${VERSION} $HOME/webkit-tools
 
 script:
   - travis_run_tests

sample_files/oca_dependencies.txt

@@ -0,0 +1,15 @@
+# List the OCA project dependencies, one per line
+# Add a repository url and branch if you need a forked version
+#
+# Examples
+# ========
+#
+# To depend on the standard version of sale-workflow, use:
+# sale-workflow
+#
+# To explicitely give the URL of a fork, and still use the version specified in
+# .travis.yml, use:
+# sale-workflow https://github.com/OCA/sale-workflow
+#
+# To provide both the URL and a branch, use:
+# sale-workflow https://github.com/OCA/sale-workflow branchname

tests/test_repo/oca_dependencies.txt

@@ -0,0 +1,5 @@
+# List the OCA project dependencies, one per line
+# Add a repository url and branch if you need a forked version
+
+partner-contact https://github.com/OCA/partner-contact 8.0
+

travis/clone_oca_dependencies

@@ -0,0 +1,100 @@
+#!/usr/bin/python
+"""Usage: clone_oca_dependencies [<checkout_dir> <build_dir>]
+
+Arguments:
+
+checkout_dir: the directory in which the dependency repositories will be cloned
+build_dir: the directory in which the tested repositories have been cloned
+
+If no arguments are provided, default to the layout used in the OCA travis
+configuration.
+
+The program will process the file oca_dependencies.txt at the root of the
+tested repository, and clone the dependency repositories in checkout_dir,
+before recursively processing the oca_dependencies.txt files of the
+dependencies.
+
+The expected format for oca_dependencies.txt:
+
+* comment lines start with # and are ignored
+* a dependency line contains:
+  - the name of the OCA project
+  - (optional) the URL to the git repository (defaulting to the OCA repository)
+  - (optional) the name of the branch to use (defaulting to ${VERSION})
+"""
+from __future__ import print_function
+import sys
+import os
+import os.path as osp
+import subprocess
+import logging
+
+
+_logger = logging.getLogger()
+
+
+def parse_depfile(depfile):
+    deps = []
+    for line in depfile:
+        line = line.strip()
+        if not line or line.startswith('#'):
+            continue
+        parts = line.split()
+        repo = parts[0]
+        if len(parts) > 2:
+            branch = parts[2]
+        else:
+            branch = os.environ.get('VERSION', '8.0')
+        if len(parts) > 1:
+            url = parts[1]
+        else:
+            url = 'https://github.com/OCA/%s' % repo
+        deps.append((repo, url, branch))
+    return deps
+
+
+def git_checkout(home, reponame, url, branch):
+    checkout_dir = osp.join(home, reponame)
+    if not osp.isdir(checkout_dir):
+        command = ['git', 'clone', '-q', url, '-b', branch, checkout_dir]
+        _logger.info('Calling %s', ' '.join(command))
+        subprocess.check_call(command)
+    return checkout_dir
+
+
+def run(home, build_dir):
+    dependencies = []
+    processed = set()
+    for repo in os.listdir(build_dir):
+        _logger.info('examining %s', repo)
+        processed.add(repo)
+        depfilename = osp.join(build_dir, repo, 'oca_dependencies.txt')
+        dependencies.append(depfilename)
+    for depfilename in dependencies:
+        try:
+            with open(depfilename) as depfile:
+                deps = parse_depfile(depfile)
+        except IOError:
+            deps = []
+        for depname, url, branch in deps:
+            _logger.info('* processing %s',  depname)
+            if depname in processed:
+                continue
+            processed.add(depname)
+            checkout_dir = git_checkout(home, depname, url, branch)
+            new_dep_filename = osp.join(checkout_dir, 'oca_dependencies.txt')
+            if new_dep_filename not in dependencies:
+                dependencies.append(new_dep_filename)
+
+
+if __name__ == '__main__':
+    if len(sys.argv) == 1:
+        home = os.environ['HOME']
+        build_dir = osp.join(home, 'build/OCA')
+    elif len(sys.argv) == 2 or len(sys.argv) > 3:
+        print(__doc__)
+        sys.exit(1)
+    else:
+        home = sys.argv[1]
+        build_dir = sys.argv[2]
+    run(home, build_dir)

travis/travis_install_nightly

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-pip install -q QUnitSuite flake8 coveralls pylint
+pip install -q QUnitSuite flake8 coveralls pylint > /dev/null 2>&1
 
 # We can exit here and do nothing if this only a LINT check
 if [ "${LINT_CHECK}" == "1" ] ; then
@@ -18,20 +18,24 @@ IFS="/" read -a REPO <<< "${ODOO_REPO}"
 ODOO_URL="https://github.com/${REPO[0]}/${REPO[1]}/archive/${VERSION}.tar.gz"
 
 echo "Installing Odoo ${ODOO_URL}"
-wget -O odoo.tar.gz ${ODOO_URL}
+wget -nv -O odoo.tar.gz ${ODOO_URL}
 tar -xf odoo.tar.gz -C ${HOME}
 
-sudo apt-get -q -y install expect-dev  # provides unbuffer utility
-sudo apt-get -q -y install python-lxml  # because pip installation is slooow
+echo "Installing Python dependencies"
+sudo apt-get -qq -y install expect-dev  # provides unbuffer utility
+sudo apt-get -qq -y install python-lxml  # because pip installation is slooow
 
 # Workaround to force using system site packages (see https://github.com/Shippable/support/issues/241#issuecomment-57947925)
-rm $VIRTUAL_ENV/lib/python2.7/no-global-site-packages.txt
+rm -f $VIRTUAL_ENV/lib/python2.7/no-global-site-packages.txt
 
 pip install -q -r $(dirname ${BASH_SOURCE[0]})/requirements.txt
 
 # Use reference .coveragerc
 cp ${HOME}/maintainer-quality-tools/cfg/.coveragerc .
 
+echo "Getting dependencies"
+clone_oca_dependencies
+
 # Expected directory structure:
 #
 #    HOME/