[REF] Dependency cloning more extensible

travis/clone_oca_dependencies

@@ -34,74 +34,80 @@ import logging
 _logger = logging.getLogger()
 
 
-def parse_depfile(depfile, owner='OCA'):
+def parse_depfile(depsfile_path, owner='OCA'):
     deps = []
-    for line in depfile:
-        line = line.strip()
-        if not line or line.startswith('#'):
-            continue
-        parts = line.split()
-        repo = parts[0]
-        if len(parts) > 2:
-            branch = parts[2]
-        else:
-            branch = os.environ.get('VERSION', '8.0')
-        if len(parts) > 1:
-            url = parts[1]
-        else:
-            url = 'https://github.com/%s/%s.git' % (owner, repo)
-        deps.append((repo, url, branch))
+    if osp.isfile(depsfile_path):
+        with open(depsfile_path) as depfile:
+            for line in depfile:
+                line = line.strip()
+                if not line or line.startswith('#'):
+                    continue
+                parts = line.split()
+                repo = parts[0]
+                if len(parts) > 2:
+                    branch = parts[2]
+                else:
+                    branch = os.environ.get('VERSION', '8.0')
+                if len(parts) > 1:
+                    url = parts[1]
+                else:
+                    url = 'https://github.com/%s/%s.git' % (owner, repo)
+                deps.append((repo, url, branch))
     return deps
 
 
-def git_checkout(deps_checkout_dir, reponame, url, branch):
-    checkout_dir = osp.join(deps_checkout_dir, reponame)
-    if not osp.isdir(checkout_dir):
-        command = ['git', 'clone', '-q', url, '-b', branch, checkout_dir]
-        _logger.info('Calling %s', ' '.join(command))
+def run_shell(command):
+    _logger.info('Calling %s', ' '.join(command))
+    return subprocess.check_call(command)
+
+
+def git_clone(checkout_path, reponame, url, branch):
+    checkout_path = osp.join(checkout_path, reponame)
+    if not osp.isdir(checkout_path):
+        run_shell(['git', 'clone', '-q', url, '-b', branch, checkout_path])
+    return checkout_path
+
+
+def clone_repos(method, deps_path, deps_name, target_path):
+    """Recursively install all dependency repositories
+
+    method:   function retrieving the repository
+    deps_path:  path where the dependency file is
+    deps_name:  name of the dependency file
+    target_path:  repository destination path
+    """
+    deps_file_path = osp.join(deps_path, deps_name)
+    if osp.isfile(deps_file_path):
+        deps = parse_depfile(deps_file_path) or []
+        for dep_name, url, branch in deps:
+            _logger.info('Processing %s', dep_name)
+            checkout_path = method(
+                target_path, dep_name, url, branch)
+            if checkout_path and checkout_path != target_path:
+                # Only drill down if checkout was on another path
+                clone_repos(
+                    method, checkout_path, deps_name, target_path)
+
+
+def install_requirements(parent_path):
+    reqs_path = osp.join(parent_path, 'requirements.txt')
+    if osp.isdir(parent_path) and osp.isfile(reqs_path):
+        _logger.info('Calling %s', reqs_path)
+        command = ['pip', 'install', '-r', reqs_path]
         subprocess.check_call(command)
-    return checkout_dir
+
+        dir_entries = os.listdir(parent_path)
+        # Don't go deeper when reaching a Python module
+        if '__init__.py' not in dir_entries:
+            for subdir in dir_entries:
+                install_requirements(osp.join(parent_path, subdir))
 
 
 def run(deps_checkout_dir, build_dir):
-    dependencies = []
-    processed = set()
-    depfilename = osp.join(build_dir, 'oca_dependencies.txt')
-    dependencies.append(depfilename)
-    reqfilenames = []
-    if osp.isfile(osp.join(build_dir, 'requirements.txt')):
-        reqfilenames.append(osp.join(build_dir, 'requirements.txt'))
-    for repo in os.listdir(deps_checkout_dir):
-        _logger.info('examining %s', repo)
-        processed.add(repo)
-        depfilename = osp.join(deps_checkout_dir, repo, 'oca_dependencies.txt')
-        dependencies.append(depfilename)
-        reqfilename = osp.join(deps_checkout_dir, repo, 'requirements.txt')
-        if osp.isfile(reqfilename):
-            reqfilenames.append(reqfilename)
-    for depfilename in dependencies:
-        try:
-            with open(depfilename) as depfile:
-                deps = parse_depfile(depfile)
-        except IOError:
-            deps = []
-        for depname, url, branch in deps:
-            _logger.info('* processing %s', depname)
-            if depname in processed:
-                continue
-            processed.add(depname)
-            checkout_dir = git_checkout(deps_checkout_dir, depname,
-                                        url, branch)
-            new_dep_filename = osp.join(checkout_dir, 'oca_dependencies.txt')
-            reqfilename = osp.join(checkout_dir, 'requirements.txt')
-            if osp.isfile(reqfilename):
-                reqfilenames.append(reqfilename)
-            if new_dep_filename not in dependencies:
-                dependencies.append(new_dep_filename)
-    for reqfilename in reqfilenames:
-        command = ['pip', 'install', '-r', reqfilename]
-        _logger.info('Calling %s', ' '.join(command))
-        subprocess.check_call(command)
+    clone_repos(
+        git_clone, build_dir, 'oca_dependencies.txt', deps_checkout_dir)
+    install_requirements(deps_checkout_dir)
+    install_requirements(build_dir)
 
 
 if __name__ == '__main__':