Merge commit 'fe74577'

* commit 'fe74577':
  Use develop mode in tox
  issue #429: fix sudo regression.
  misc: rename to scripts. tab completion!!
  core: Latch._wake improvements
  issue #498: prevent crash on double 'disconnect' signal.
  issue #413: don't double-propagate DEL_ROUTE to parent.
  issue #498: wrap Router dict mutations in a lock
  issue #429: enable en_US locale to unbreak debops test.
  issue #499: fix another mind-numbingly stupid vanilla inconsistency
  issue #497: do our best to cope with crap upstream code
  ssh: fix test to match updated log format.
  issue #429: update Changelog.
  issue #429: update Changelog.
  issue #429: teach sudo about every know i18n password string.
  issue #429: install i18n-related bits in test images.
  ssh: tidy up logs and stream names.
  tests: ensure file is closed in connection_test.
  gcloud: small updates
  tests: give ansible/gcloud/ its own requirements file.

docs/changelog.rst

@@ -248,6 +248,9 @@ Fixes
   number of 2->3 bugs were fixed, mostly in the form of Unicode/bytes
   mismatches.
 
+* `#429 <https://github.com/dw/mitogen/issues/429>`_: the ``sudo`` method can
+  now recognize internationalized password prompts.
+
 * `#362 <https://github.com/dw/mitogen/issues/362>`_,
   `#435 <https://github.com/dw/mitogen/issues/435>`_: the previous fix for slow
   Python 2.x subprocess creation on Red Hat caused newly spawned children to
@@ -464,7 +467,9 @@ bug reports, testing, features and fixes in this release contributed by
 `Strahinja Kustudic <https://github.com/kustodian>`_,
 `Tom Parker-Shemilt <https://github.com/palfrey/>`_,
 `Younès HAFRI <https://github.com/yhafri>`_,
+`@killua-eu <https://github.com/killua-eu>`_,
 `@myssa91 <https://github.com/myssa91>`_,
+`@ohmer1 <https://github.com/ohmer1>`_,
 `@s3c70r <https://github.com/s3c70r/>`_,
 `@syntonym <https://github.com/syntonym/>`_,
 `@trim777 <https://github.com/trim777/>`_,

mitogen/core.py

@@ -1703,7 +1703,7 @@ def accept(self, rfd, wfd):
 
     def __repr__(self):
         cls = type(self)
-        return '%s.%s(%r)' % (cls.__module__, cls.__name__, self.name)
+        return "%s.%s('%s')" % (cls.__module__, cls.__name__, self.name)
 
 
 class Context(object):
@@ -2220,12 +2220,8 @@ def put(self, obj):
             self._lock.release()
 
     def _wake(self, wsock, cookie):
-        try:
-            os.write(wsock.fileno(), cookie)
-        except OSError:
-            e = sys.exc_info()[1]
-            if e.args[0] != errno.EBADF:
-                raise
+        written, disconnected = io_op(os.write, wsock.fileno(), cookie)
+        assert written == len(cookie) and not disconnected
 
     def __repr__(self):
         return 'Latch(%#x, size=%d, t=%r)' % (
@@ -2423,9 +2419,10 @@ def __init__(self, broker):
         listen(broker, 'exit', self._on_broker_exit)
         self._setup_logging()
 
-        #: context ID -> Stream
+        self._write_lock = threading.Lock()
+        #: context ID -> Stream; must hold _write_lock to edit or iterate
         self._stream_by_id = {}
-        #: List of contexts to notify of shutdown.
+        #: List of contexts to notify of shutdown; must hold _write_lock
         self._context_by_id = {}
         self._last_handle = itertools.count(1000)
         #: handle -> (persistent?, func(msg))
@@ -2456,21 +2453,31 @@ def _on_del_route(self, msg):
         :class:`mitogen.parent.RouteMonitor` in an upgraded context.
         """
         LOG.error('%r._on_del_route() %r', self, msg)
-        if not msg.is_dead:
-            target_id_s, _, name = bytes_partition(msg.data, b(':'))
-            target_id = int(target_id_s, 10)
-            if target_id not in self._context_by_id:
-                LOG.debug('DEL_ROUTE for unknown ID %r: %r', target_id, msg)
-                return
+        if msg.is_dead:
+            return
 
-            fire(self._context_by_id[target_id], 'disconnect')
+        target_id_s, _, name = bytes_partition(msg.data, b(':'))
+        context = self._context_by_id.get(int(target_id_s, 10))
+        if context:
+            fire(context, 'disconnect')
+        else:
+            LOG.debug('DEL_ROUTE for unknown ID %r: %r', target_id, msg)
 
     def _on_stream_disconnect(self, stream):
-        for context in self._context_by_id.values():
-            stream_ = self._stream_by_id.get(context.context_id)
-            if stream_ is stream:
-                del self._stream_by_id[context.context_id]
-                context.on_disconnect()
+        notify = []
+        self._write_lock.acquire()
+        try:
+            for context in list(self._context_by_id.values()):
+                stream_ = self._stream_by_id.get(context.context_id)
+                if stream_ is stream:
+                    del self._stream_by_id[context.context_id]
+                    notify.append(context)
+        finally:
+            self._write_lock.release()
+
+        # Happens outside lock as e.g. RouteMonitor wants the same lock.
+        for context in notify:
+            context.on_disconnect()
 
     broker_exit_msg = 'Broker has exitted'
 
@@ -2492,14 +2499,27 @@ def myself(self):
     def context_by_id(self, context_id, via_id=None, create=True, name=None):
         """
         Messy factory/lookup function to find a context by its ID, or construct
-        it. In future this will be replaced by a much more sensible interface.
+        it. This will eventually be replaced by a more sensible interface.
         """
         context = self._context_by_id.get(context_id)
-        if create and not context:
-            context = self.context_class(self, context_id, name=name)
-            if via_id is not None:
-                context.via = self.context_by_id(via_id)
-            self._context_by_id[context_id] = context
+        if context:
+            return context
+
+        if create and via_id is not None:
+            via = self.context_by_id(via_id)
+        else:
+            via = None
+
+        self._write_lock.acquire()
+        try:
+            context = self._context_by_id.get(context_id)
+            if create and not context:
+                context = self.context_class(self, context_id, name=name)
+                context.via = via
+                self._context_by_id[context_id] = context
+        finally:
+            self._write_lock.release()
+
         return context
 
     def register(self, context, stream):
@@ -2509,8 +2529,13 @@ def register(self, context, stream):
         public while the design has not yet settled.
         """
         _v and LOG.debug('register(%r, %r)', context, stream)
-        self._stream_by_id[context.context_id] = stream
-        self._context_by_id[context.context_id] = context
+        self._write_lock.acquire()
+        try:
+            self._stream_by_id[context.context_id] = stream
+            self._context_by_id[context.context_id] = context
+        finally:
+            self._write_lock.release()
+
         self.broker.start_receive(stream)
         listen(stream, 'disconnect', lambda: self._on_stream_disconnect(stream))
 
@@ -2520,8 +2545,10 @@ def stream_by_id(self, dst_id):
         `dst_id`. If a specific route for `dst_id` is not known, a reference to
         the parent context's stream is returned.
         """
-        parent = self._stream_by_id.get(mitogen.parent_id)
-        return self._stream_by_id.get(dst_id, parent)
+        return (
+            self._stream_by_id.get(dst_id) or
+            self._stream_by_id.get(mitogen.parent_id)
+        )
 
     def del_handler(self, handle):
         """

mitogen/doas.py

@@ -66,9 +66,8 @@ def construct(self, username=None, password=None, doas_path=None,
         if incorrect_prompts is not None:
             self.incorrect_prompts = map(str.lower, incorrect_prompts)
 
-    def connect(self):
-        super(Stream, self).connect()
-        self.name = u'doas.' + mitogen.core.to_text(self.username)
+    def _get_name(self):
+        return u'doas.' + mitogen.core.to_text(self.username)
 
     def get_boot_command(self):
         bits = [self.doas_path, '-u', self.username, '--']

mitogen/docker.py

@@ -62,9 +62,8 @@ def construct(self, container=None, image=None,
         if username:
             self.username = username
 
-    def connect(self):
-        super(Stream, self).connect()
-        self.name = u'docker.' + (self.container or self.image)
+    def _get_name(self):
+        return u'docker.' + (self.container or self.image)
 
     def get_boot_command(self):
         args = ['--interactive']

mitogen/jail.py

@@ -52,9 +52,8 @@ def construct(self, container, jexec_path=None, username=None, **kwargs):
         if jexec_path:
             self.jexec_path = jexec_path
 
-    def connect(self):
-        super(Stream, self).connect()
-        self.name = u'jail.' + self.container
+    def _get_name(self):
+        return u'jail.' + self.container
 
     def get_boot_command(self):
         bits = [self.jexec_path]

mitogen/kubectl.py

@@ -55,9 +55,8 @@ def construct(self, pod, kubectl_path=None, kubectl_args=None, **kwargs):
             self.kubectl_path = kubectl_path
         self.kubectl_args = kubectl_args or []
 
-    def connect(self):
-        super(Stream, self).connect()
-        self.name = u'kubectl.%s%s' % (self.pod, self.kubectl_args)
+    def _get_name(self):
+        return u'kubectl.%s%s' % (self.pod, self.kubectl_args)
 
     def get_boot_command(self):
         bits = [self.kubectl_path] + self.kubectl_args + ['exec', '-it', self.pod]

mitogen/lxc.py

@@ -60,9 +60,8 @@ def construct(self, container, lxc_attach_path=None, **kwargs):
         if lxc_attach_path:
             self.lxc_attach_path = lxc_attach_path
 
-    def connect(self):
-        super(Stream, self).connect()
-        self.name = u'lxc.' + self.container
+    def _get_name(self):
+        return u'lxc.' + self.container
 
     def get_boot_command(self):
         bits = [

mitogen/lxd.py

@@ -61,9 +61,8 @@ def construct(self, container, lxc_path=None, **kwargs):
         if lxc_path:
             self.lxc_path = lxc_path
 
-    def connect(self):
-        super(Stream, self).connect()
-        self.name = u'lxd.' + self.container
+    def _get_name(self):
+        return u'lxd.' + self.container
 
     def get_boot_command(self):
         bits = [