When a thread is created, notify about it as soon as possible. (#430)

Fixes #106
* When a thread is created, notify about it as soon as possible.
* Properly mark internal threads as pydevd daemon threads.
Also fix tests to consider that the event related to thread
creation is given as the thread starts to run.
* Fix test flakiness and improve error message when messages are not equal.

ptvsd/_main.py

@@ -9,10 +9,10 @@
 from ptvsd.runner import run as no_debug_runner
 from ptvsd.socket import Address, create_server
 
-
 ########################
 # high-level functions
 
+
 def debug_main(address, name, kind, *extra, **kwargs):
     if kind == 'module':
         run_module(address, name, *extra, **kwargs)
@@ -23,10 +23,10 @@ def debug_main(address, name, kind, *extra, **kwargs):
 def run_main(address, name, kind, *extra, **kwargs):
     no_debug_runner(address, name, kind == 'module', *extra, **kwargs)
 
-
 ########################
 # low-level functions
 
+
 def run_module(address, modname, *extra, **kwargs):
     """Run pydevd for the given module."""
     addr = Address.from_raw(address)
@@ -129,6 +129,8 @@ def wait_for_connection(daemon, host, port):
     connection_thread = threading.Thread(target=wait_for_connection,
                                          args=(daemon, host, port),
                                          name='ptvsd.listen_for_connection')
+    connection_thread.pydev_do_not_trace = True
+    connection_thread.is_pydev_daemon_thread = True
     connection_thread.daemon = True
     connection_thread.start()
 

ptvsd/_vendored/pydevd/_pydev_bundle/pydev_is_thread_alive.py

@@ -1,23 +1,20 @@
 from _pydev_imps._pydev_saved_modules import threading
 
-# Hack for https://sw-brainwy.rhcloud.com/tracker/PyDev/363 (i.e.: calling isAlive() can throw AssertionError under some circumstances)
+# Hack for https://www.brainwy.com/tracker/PyDev/363 (i.e.: calling isAlive() can throw AssertionError under some
+# circumstances).
 # It is required to debug threads started by start_new_thread in Python 3.4
 _temp = threading.Thread()
-if hasattr(_temp, '_is_stopped'): # Python 3.4 has this
+if hasattr(_temp, '_is_stopped'): # Python 3.x has this
     def is_thread_alive(t):
-        try:
-            return not t._is_stopped
-        except:
-            return t.isAlive()
+        return not t._is_stopped
 
-elif hasattr(_temp, '_Thread__stopped'): # Python 2.7 has this
+elif hasattr(_temp, '_Thread__stopped'): # Python 2.x has this
     def is_thread_alive(t):
-        try:
-            return not t._Thread__stopped
-        except:
-            return t.isAlive()
+        return not t._Thread__stopped
+
+else: 
+    # Make it an error: we want to detect only stops (so, isAlive() can't be used because it may return True before the
+    # thread is actually running).
+    raise AssertionError('Check how to detect that a thread has been stopped.')
 
-else: # Haven't checked all other versions, so, let's use the regular isAlive call in this case.
-    def is_thread_alive(t):
-        return t.isAlive()
 del _temp

ptvsd/_vendored/pydevd/_pydev_bundle/pydev_monkey.py

@@ -2,6 +2,7 @@
 import os
 import sys
 import traceback
+from _pydev_imps._pydev_saved_modules import threading
 
 try:
     xrange
@@ -568,25 +569,44 @@ def __init__(self, original_func, args, kwargs):
         self.original_func = original_func
         self.args = args
         self.kwargs = kwargs
-        self.global_debugger = self.get_debugger()
-
-    def get_debugger(self):
-        from _pydevd_bundle.pydevd_comm import get_global_debugger
-        return get_global_debugger()
 
     def __call__(self):
-        _on_set_trace_for_new_thread(self.global_debugger)
-        global_debugger = self.global_debugger
-
-        if global_debugger is not None and global_debugger.thread_analyser is not None:
-            # we can detect start_new_thread only here
-            try:
-                from pydevd_concurrency_analyser.pydevd_concurrency_logger import log_new_thread
-                log_new_thread(global_debugger)
-            except:
-                sys.stderr.write("Failed to detect new thread for visualization")
-
-        return self.original_func(*self.args, **self.kwargs)
+        # We monkey-patch the thread creation so that this function is called in the new thread. At this point
+        # we notify of its creation and start tracing it.
+        from _pydevd_bundle.pydevd_constants import get_thread_id
+        from _pydevd_bundle.pydevd_comm import get_global_debugger
+        global_debugger = get_global_debugger()
+
+        thread_id = None
+        if global_debugger is not None:
+            # Note: if this is a thread from threading.py, we're too early in the boostrap process (because we mocked
+            # the start_new_thread internal machinery and thread._bootstrap has not finished), so, the code below needs
+            # to make sure that we use the current thread bound to the original function and not use
+            # threading.currentThread() unless we're sure it's a dummy thread.
+            t = getattr(self.original_func, '__self__', getattr(self.original_func, 'im_self', None))
+            if not isinstance(t, threading.Thread):
+                # This is not a threading.Thread but a Dummy thread (so, get it as a dummy thread using
+                # currentThread).
+                t = threading.currentThread()
+
+            if not getattr(t, 'is_pydev_daemon_thread', False):
+                thread_id = get_thread_id(t)
+                global_debugger.notify_thread_created(thread_id, t)
+                _on_set_trace_for_new_thread(global_debugger)
+
+            if getattr(global_debugger, 'thread_analyser', None) is not None:
+                try:
+                    from pydevd_concurrency_analyser.pydevd_concurrency_logger import log_new_thread
+                    log_new_thread(global_debugger, t)
+                except:
+                    sys.stderr.write("Failed to detect new thread for visualization")
+        try:
+            ret = self.original_func(*self.args, **self.kwargs)
+        finally:
+            if thread_id is not None:
+                global_debugger.notify_thread_not_alive(thread_id)
+
+        return ret
 
 
 class _NewThreadStartupWithoutTrace:
@@ -610,31 +630,35 @@ def _get_threading_modules_to_patch():
     except:
         import _thread
     threading_modules_to_patch.append(_thread)
+    threading_modules_to_patch.append(threading)
 
     return threading_modules_to_patch
 
 threading_modules_to_patch = _get_threading_modules_to_patch()
 
 
-def patch_thread_module(thread):
+def patch_thread_module(thread_module):
 
-    if getattr(thread, '_original_start_new_thread', None) is None:
-        _original_start_new_thread = thread._original_start_new_thread = thread.start_new_thread
+    if getattr(thread_module, '_original_start_new_thread', None) is None:
+        if thread_module is threading:
+            _original_start_new_thread = thread_module._original_start_new_thread = thread_module._start_new_thread
+        else:
+            _original_start_new_thread = thread_module._original_start_new_thread = thread_module.start_new_thread
     else:
-        _original_start_new_thread = thread._original_start_new_thread
+        _original_start_new_thread = thread_module._original_start_new_thread
 
     class ClassWithPydevStartNewThread:
 
         def pydev_start_new_thread(self, function, args=(), kwargs={}):
             '''
-            We need to replace the original thread.start_new_thread with this function so that threads started
+            We need to replace the original thread_module.start_new_thread with this function so that threads started
             through it and not through the threading module are properly traced.
             '''
             return _original_start_new_thread(_UseNewThreadStartup(function, args, kwargs), ())
 
-    # This is a hack for the situation where the thread.start_new_thread is declared inside a class, such as the one below
+    # This is a hack for the situation where the thread_module.start_new_thread is declared inside a class, such as the one below
     # class F(object):
-    #    start_new_thread = thread.start_new_thread
+    #    start_new_thread = thread_module.start_new_thread
     #
     #    def start_it(self):
     #        self.start_new_thread(self.function, args, kwargs)
@@ -643,10 +667,13 @@ def pydev_start_new_thread(self, function, args=(), kwargs={}):
     pydev_start_new_thread = ClassWithPydevStartNewThread().pydev_start_new_thread
 
     try:
-        # We need to replace the original thread.start_new_thread with this function so that threads started through
+        # We need to replace the original thread_module.start_new_thread with this function so that threads started through
         # it and not through the threading module are properly traced.
-        thread.start_new_thread = pydev_start_new_thread
-        thread.start_new = pydev_start_new_thread
+        if thread_module is threading:
+            thread_module._start_new_thread = pydev_start_new_thread
+        else:
+            thread_module.start_new_thread = pydev_start_new_thread
+            thread_module.start_new = pydev_start_new_thread
     except:
         pass
 
@@ -668,6 +695,11 @@ def undo_patch_thread_modules():
         except:
             pass
 
+        try:
+            t._start_new_thread = t._original_start_new_thread
+        except:
+            pass
+
 
 def disable_trace_thread_modules():
     '''

ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_constants.py

@@ -232,29 +232,28 @@ def clear_cached_thread_id(thread):
     except AttributeError:
         pass
 
+
 #=======================================================================================================================
 # get_thread_id
 #=======================================================================================================================
 def get_thread_id(thread):
     try:
         tid = thread.__pydevd_id__
         if tid is None:
-            # Fix for https://sw-brainwy.rhcloud.com/tracker/PyDev/645
+            # Fix for https://www.brainwy.com/tracker/PyDev/645
             # if __pydevd_id__ is None, recalculate it... also, use an heuristic
             # that gives us always the same id for the thread (using thread.ident or id(thread)).
             raise AttributeError()
     except AttributeError:
         _nextThreadIdLock.acquire()
         try:
-            #We do a new check with the lock in place just to be sure that nothing changed
+            # We do a new check with the lock in place just to be sure that nothing changed
             tid = getattr(thread, '__pydevd_id__', None)
             if tid is None:
                 pid = get_pid()
-                try:
-                    tid = thread.__pydevd_id__ = 'pid_%s_id_%s' % (pid, thread.get_ident())
-                except:
-                    # thread.ident isn't always there... (use id(thread) instead if it's not there).
-                    tid = thread.__pydevd_id__ = 'pid_%s_id_%s' % (pid, id(thread))
+                # Note: don't use the thread ident because if we're too early in the
+                # thread bootstrap process, the thread id could be still unset.
+                tid = thread.__pydevd_id__ = 'pid_%s_id_%s' % (pid, id(thread))
         finally:
             _nextThreadIdLock.release()
 

ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_trace_dispatch_regular.py

@@ -126,7 +126,7 @@ def __call__(self, frame, event, arg):
 
             # if thread is not alive, cancel trace_dispatch processing
             if not is_thread_alive(t):
-                py_db._process_thread_not_alive(get_thread_id(t))
+                py_db.notify_thread_not_alive(get_thread_id(t))
                 return None  # suspend tracing