[REF] Dependency cloning more extensible

travis/clone_oca_dependencies

@@ -34,23 +34,25 @@ import logging
 _logger = logging.getLogger()
 
 
-def parse_depfile(depfile, owner='OCA'):
+def parse_depfile(depsfile_path, owner='OCA'):
     deps = []
-    for line in depfile:
-        line = line.strip()
-        if not line or line.startswith('#'):
-            continue
-        parts = line.split()
-        repo = parts[0]
-        if len(parts) > 2:
-            branch = parts[2]
-        else:
-            branch = os.environ.get('VERSION', '8.0')
-        if len(parts) > 1:
-            url = parts[1]
-        else:
-            url = 'https://github.com/%s/%s.git' % (owner, repo)
-        deps.append((repo, url, branch))
+    if osp.isfile(depsfile_path):
+        with open(depsfile_path) as depfile:
+            for line in depfile:
+                line = line.strip()
+                if not line or line.startswith('#'):
+                    continue
+                parts = line.split()
+                repo = parts[0]
+                if len(parts) > 2:
+                    branch = parts[2]
+                else:
+                    branch = os.environ.get('VERSION', '8.0')
+                if len(parts) > 1:
+                    url = parts[1]
+                else:
+                    url = 'https://github.com/%s/%s.git' % (owner, repo)
+                deps.append((repo, url, branch))
     return deps
 
 
@@ -63,45 +65,46 @@ def git_checkout(deps_checkout_dir, reponame, url, branch):
     return checkout_dir
 
 
-def run(deps_checkout_dir, build_dir):
-    dependencies = []
-    processed = set()
-    depfilename = osp.join(build_dir, 'oca_dependencies.txt')
-    dependencies.append(depfilename)
-    reqfilenames = []
-    if osp.isfile(osp.join(build_dir, 'requirements.txt')):
-        reqfilenames.append(osp.join(build_dir, 'requirements.txt'))
-    for repo in os.listdir(deps_checkout_dir):
-        _logger.info('examining %s', repo)
-        processed.add(repo)
-        depfilename = osp.join(deps_checkout_dir, repo, 'oca_dependencies.txt')
-        dependencies.append(depfilename)
-        reqfilename = osp.join(deps_checkout_dir, repo, 'requirements.txt')
-        if osp.isfile(reqfilename):
-            reqfilenames.append(reqfilename)
-    for depfilename in dependencies:
-        try:
-            with open(depfilename) as depfile:
-                deps = parse_depfile(depfile)
-        except IOError:
-            deps = []
-        for depname, url, branch in deps:
-            _logger.info('* processing %s', depname)
-            if depname in processed:
-                continue
-            processed.add(depname)
-            checkout_dir = git_checkout(deps_checkout_dir, depname,
-                                        url, branch)
-            new_dep_filename = osp.join(checkout_dir, 'oca_dependencies.txt')
-            reqfilename = osp.join(checkout_dir, 'requirements.txt')
-            if osp.isfile(reqfilename):
-                reqfilenames.append(reqfilename)
-            if new_dep_filename not in dependencies:
-                dependencies.append(new_dep_filename)
-    for reqfilename in reqfilenames:
-        command = ['pip', 'install', '-r', reqfilename]
+def git_merge(deps_checkout_dir, reponame, url, branch):
+    checkout_dir = osp.join(deps_checkout_dir, reponame)
+    if not osp.isdir(checkout_dir):
+        command = ['git', 'merge', '-X', 'ours', '-b', branch, url]
         _logger.info('Calling %s', ' '.join(command))
         subprocess.check_call(command)
+    return checkout_dir
+
+
+def clone_repos(checkout_path, repo_path, deps_name='oca_dependencies.txt'):
+    """Clone all repos to a give directory all repos in a dependency file"""
+    deps_path = osp.join(repo_path, deps_name)
+    if osp.isdir(repo_path) and osp.isfile(deps_path):
+        deps = parse_depfile(deps_path) or []
+        for dep_name, url, branch in deps:
+            if dep_name not in os.listdir(checkout_path):
+                _logger.info('Processing %s', dep_name)
+                checkout_dir = git_checkout(
+                    checkout_path, dep_name, url, branch)
+                clone_repos(deps_checkout_dir, checkout_dir)
+
+
+def install_requirements(parent_path):
+    reqs_path = osp.join(parent_path, 'requirements.txt')
+    if osp.isdir(parent_path) and osp.isfile(reqs_path):
+        _logger.info('Calling %s', reqs_path)
+        command = ['pip', 'install', '-r', reqs_path]
+        subprocess.check_call(command)
+
+        dir_entries = os.listdir(parent_path)
+        # Don't go deeper when reaching a Python module
+        if '__init__.py' not in dir_entries:
+            for subdir in dir_entries:
+                install_requirements(osp.join(parent_path, subdir))
+
+
+def run(deps_checkout_dir, build_dir):
+    clone_repos(deps_checkout_dir, build_dir)
+    install_requirements(deps_checkout_dir)
+    install_requirements(build_dir)
 
 
 if __name__ == '__main__':