[major] executor/clean: Adding per-package cleaning, linked develspaces, and a new execution pipeline

.travis.before_install.bash

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+if [ "$TRAVIS_OS_NAME" == "linux" ]; then
+  echo "AOK"
+  #if [ "$PYTHON" == "/usr/bin/python3.4" ]; then
+    #sudo add-apt-repository ppa:fkrull/deadsnakes -y
+    #sudo apt-get update
+    #sudo apt-get install python3.4 python3-dev
+  #fi
+elif [ "$TRAVIS_OS_NAME" == "osx" ]; then
+  if [ "$PYTHON" == "/usr/local/bin/python3" ]; then
+    brew install python3
+  fi
+  sudo pip install virtualenv
+  virtualenv -p $PYTHON venv
+  source venv/bin/activate
+fi

.travis.before_script.bash

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+if [ "$TRAVIS_OS_NAME" == "linux" ]; then
+  sudo apt-get install cmake libgtest-dev build-essential python-setuptools
+elif [ "$TRAVIS_OS_NAME" == "osx" ]; then
+  # noop?
+  pip install setuptools
+fi

.travis.yml

@@ -1,12 +1,35 @@
-language: python
-python:
-  - "2.7"
-  - "3.3"
+# Travis auto-virtualenv isn't supported on OS X
+#language: python
+#python:
+  #- "2.7"
+  #- "3.3"
+language: generic
+matrix:
+  include:
+    - python: 2.7
+      language: python
+      python: "2.7"
+      os: linux
+      env: PYTHON=/usr/bin/python2.7
+    - python: 3.4
+      language: python
+      python: "3.4"
+      os: linux
+      env: PYTHON=/usr/bin/python3.4
+    - python: 2
+      os: osx
+      env: PYTHON=/usr/bin/python
+    - python: 3
+      os: osx
+      env: PYTHON=/usr/local/bin/python3
+before_install:
+  - source .travis.before_install.bash
 install:
   - pip install argparse catkin-pkg distribute PyYAML psutil
-  - pip install nose coverage flake8 --upgrade
+  - pip install nose coverage flake8 mock --upgrade
 before_script:
-  - sudo apt-get install cmake python-setuptools libgtest-dev build-essential
+  - ./.travis.before_script.bash
+  - pip install git+https://github.com/jbohren-forks/osrf_pycommon.git@dont-close-fds
   - pip install empy --upgrade
   - git clone https://github.com/ros/catkin.git /tmp/catkin_source
   - mkdir /tmp/catkin_source/build

catkin_tools/argument_parsing.py

@@ -14,6 +14,7 @@
 
 from __future__ import print_function
 
+import argparse
 import os
 import re
 import sys
@@ -22,9 +23,7 @@
 
 from catkin_tools.common import wide_log
 
-from catkin_tools.make_jobserver import initialize_jobserver
-from catkin_tools.make_jobserver import jobserver_arguments
-from catkin_tools.make_jobserver import jobserver_supported
+from catkin_tools.execution.jobs import JobServer
 
 
 def add_context_args(parser):
@@ -60,10 +59,12 @@ def add_cmake_and_make_and_catkin_make_args(parser):
     """
 
     add = parser.add_argument
-    add('-p', '--parallel-packages', '--parallel-jobs', '--parallel', dest='parallel_jobs', default=None,
-        help='Maximum number of packages which could be built in parallel (default is cpu count)')
     add('-j', '--jobs', default=None,
-        help='Limit parallel job count through the internal GNU make job server. default is cpu count')
+        help='Maximum number of build jobs to be distributed across active packages. (default is cpu count)')
+    add('-p', '--parallel-packages', metavar='PACKAGE_JOBS', dest='parallel_jobs', default=None,
+        help='Maximum number of packages allowed to be built in parallel (default is cpu count)')
+    # Deprecated flags kept for compatibility
+    add('--parallel-jobs', '--parallel', action='store_true', dest='parallel_jobs', help=argparse.SUPPRESS)
 
     add = parser.add_mutually_exclusive_group().add_argument
     add('--jobserver', dest='use_internal_make_jobserver', default=None, action='store_true',
@@ -256,7 +257,7 @@ def handle_make_arguments(
             jobs_dict['jobs'] = 1
 
     if len(jobs_dict) == 0:
-        make_args.extend(jobserver_arguments())
+        make_args.extend(JobServer.gnu_make_args())
     else:
         if 'jobs' in jobs_dict:
             make_args.append('-j{0}'.format(jobs_dict['jobs']))
@@ -282,7 +283,7 @@ def configure_make_args(make_args, use_internal_make_jobserver):
         n_cpus = cpu_count()
         jobs_flags = {
             'jobs': n_cpus,
-            'load-average': n_cpus}
+            'load-average': n_cpus + 1}
     except NotImplementedError:
         # If the number of cores cannot be determined, limit to one job
         jobs_flags = {
@@ -302,19 +303,19 @@ def configure_make_args(make_args, use_internal_make_jobserver):
         # Remove jobs flags from cli args if they're present
         make_args = re.sub(' '.join(cli_jobs_flags), '', ' '.join(make_args)).split()
 
-    # Instantiate a jobserver
-    if use_internal_make_jobserver:
-        initialize_jobserver(
-            num_jobs=jobs_flags.get('jobs', None),
-            max_load=jobs_flags.get('load-average', None))
+    # Instantiate the jobserver
+    JobServer.initialize(
+        max_jobs=jobs_flags.get('jobs', None),
+        max_load=jobs_flags.get('load-average', None),
+        gnu_make_enabled=use_internal_make_jobserver)
 
     # If the jobserver is supported
-    if jobserver_supported():
+    if JobServer.gnu_make_enabled():
         jobs_args = []
     else:
         jobs_args = cli_jobs_flags
 
-    return make_args + jobs_args, using_makeflags_jobs_flags, using_cli_flags, jobserver_supported()
+    return make_args + jobs_args, using_makeflags_jobs_flags, using_cli_flags, JobServer.gnu_make_enabled()
 
 
 def argument_preprocessor(args):

catkin_tools/common.py

@@ -17,6 +17,7 @@
 import datetime
 import os
 import re
+import subprocess
 
 from catkin_pkg.packages import find_packages
 
@@ -31,23 +32,6 @@
     string_type = str
 
 
-class FakeLock(object):
-
-    """Fake lock used to mimic a Lock but without causing synchronization"""
-
-    def acquire(self, blocking=False):
-        return True
-
-    def release(self):
-        pass
-
-    def __enter__(self):
-        pass
-
-    def __exit__(self, exc_type, exc_value, traceback):
-        pass
-
-
 def getcwd(symlinks=True):
     """Get the current working directory.
 
@@ -239,6 +223,38 @@ def get_recursive_run_depends_in_workspace(packages, ordered_packages):
     return recursive_depends
 
 
+def get_recursive_build_dependants_in_workspace(package_name, ordered_packages):
+    """Calculates the recursive build dependants of a package which are also in
+    the ordered_packages
+
+    :param package: package for which the recursive depends should be calculated
+    :type package: :py:class:`catkin_pkg.package.Package`
+    :param ordered_packages: packages in the workspace, ordered topologically,
+        stored as a list of tuples of package path and package object
+    :type ordered_packages: list(tuple(package path,
+        :py:class:`catkin_pkg.package.Package`))
+    :returns: list of package path, package object tuples which are the
+        recursive build depends for the given package
+    :rtype: list(tuple(package path, :py:class:`catkin_pkg.package.Package`))
+    """
+    workspace_packages_by_name = dict([(pkg.name, (pth, pkg)) for pth, pkg in ordered_packages])
+    packages_to_check = set([package_name])
+    recursive_dependants = list()
+
+    for pth, pkg in reversed(ordered_packages):
+        # Break if this is one to check
+        if pkg.name == package_name:
+            break
+
+        # Check if this package depends on the target package
+        deps = get_recursive_build_depends_in_workspace(pkg, ordered_packages)
+        deps_names = [p.name for _, p in deps]
+        if package_name in deps_names:
+            recursive_dependants.insert(0, (pth, pkg))
+
+    return recursive_dependants
+
+
 def is_tty(stream):
     """Returns True if the given stream is a tty, else False"""
     return hasattr(stream, 'isatty') and stream.isatty()
@@ -277,7 +293,7 @@ def terminal_width_windows():
 
 def terminal_width_linux():
     """Returns the estimated width of the terminal on linux"""
-    width = os.popen('tput cols', 'r').readline()
+    width = subprocess.Popen('tput cols', shell=True, stdout=subprocess.PIPE, close_fds=False).stdout.readline()
 
     return int(width)
 
@@ -330,6 +346,9 @@ def slice_to_printed_length(string, length):
     if not matches:
         # If no matches, then set the lookup_array to a plain range
         lookup_array = range(len(string))
+    lookup_array.append(len(string))
+    if length > len(lookup_array):
+        return string
     return string[:lookup_array[length]] + clr('@|')
 
 
@@ -426,8 +445,12 @@ def wide_log(msg, **kwargs):
     :param truncate: If True, messages wider the then terminal will be truncated
     :type truncate: bool
     """
-    global wide_log_fn
-    wide_log_fn(msg, **kwargs)
+    try:
+        global wide_log_fn
+        wide_log_fn(msg, **kwargs)
+    except IOError:
+        # This happens when someone ctrl-c's during a log message
+        pass
 
 
 def find_enclosing_package(search_start_path=None, ws_path=None, warnings=None, symlinks=True):
@@ -454,3 +477,33 @@ def find_enclosing_package(search_start_path=None, ws_path=None, warnings=None,
 def version_tuple(v):
     """Get an integer version tuple from a string."""
     return tuple(map(int, (str(v).split("."))))
+
+
+def mkdir_p(path):
+    """Equivalent to UNIX mkdir -p"""
+    if os.path.exists(path):
+        return
+    try:
+        return os.makedirs(path)
+    except OSError as exc:  # Python >2.5
+        if exc.errno == errno.EEXIST and os.path.isdir(path):
+            pass
+        else:
+            raise
+
+CATKIN_TOOLS_DIRNAME = '.catkin_tools'
+
+
+def get_linked_devel_path(devel_space_abs):
+    """The path to the hidden directory in the develspace that
+    contains the isolated devel spaces linked into the develspace."""
+    return os.path.join(
+        devel_space_abs,
+        CATKIN_TOOLS_DIRNAME)
+
+
+def get_linked_devel_package_path(devel_space_abs, package_name):
+    """The path to a given package's linked devel space"""
+    return os.path.join(
+        get_linked_devel_path(devel_space_abs),
+        package_name)

catkin_tools/context.py

@@ -59,6 +59,7 @@ class Context(object):
                    'build_space',
                    'devel_space',
                    'install_space',
+                   'link_devel',
                    'isolate_devel',
                    'install',
                    'isolate_install',
@@ -192,6 +193,7 @@ def __init__(
         devel_space=None,
         install_space=None,
         isolate_devel=False,
+        link_devel=True,
         install=False,
         isolate_install=False,
         cmake_args=None,
@@ -262,6 +264,7 @@ def __init__(
         self.blacklist = blacklist or []
 
         # Handle build options
+        self.link_devel = link_devel
         self.isolate_devel = isolate_devel
         self.install = install
         self.isolate_install = isolate_install
@@ -325,7 +328,7 @@ def load_env(self):
 
                     self.env_cmake_prefix_path = ':'.join(split_result_cmake_prefix_path[1:])
                 else:
-                    self.env_cmake_prefix_path = os.environ.get('CMAKE_PREFIX_PATH', '')
+                    self.env_cmake_prefix_path = os.environ.get('CMAKE_PREFIX_PATH', '').rstrip(':')
 
         # Add warnings based on conflicing CMAKE_PREFIX_PATH
         if self.cached_cmake_prefix_path and self.extend_path:
@@ -397,6 +400,7 @@ def summary(self, notes=[]):
                 clr("@{cf}DESTDIR:@|                     @{yf}{_Context__destdir}@|"),
             ],
             [
+                clr("@{cf}Symlink Develspaces:@|         @{yf}{_Context__link_devel}@|"),
                 clr("@{cf}Isolate Develspaces:@|         @{yf}{_Context__isolate_devel}@|"),
                 clr("@{cf}Install Packages:@|            @{yf}{_Context__install}@|"),
                 clr("@{cf}Isolate Installs:@|            @{yf}{_Context__isolate_install}@|"),
@@ -585,6 +589,16 @@ def destdir(self, value):
             raise RuntimeError("Setting of context members is not allowed while locked.")
         self.__destdir = value
 
+    @property
+    def link_devel(self):
+        return self.__link_devel
+
+    @link_devel.setter
+    def link_devel(self, value):
+        if self.__locked:
+            raise RuntimeError("Setting of context members is not allowed while locked.")
+        self.__link_devel = value
+
     @property
     def isolate_devel(self):
         return self.__isolate_devel