From e08d53c3431c0fd0aaf7460308ba3e70584f03d2 Mon Sep 17 00:00:00 2001 From: fabioz Date: Tue, 17 Jul 2018 16:12:33 -0300 Subject: [PATCH 1/2] #72: Provide a way to set breakpoint suspension policy. --- .../pydevd_process_net_command.py | 14 ++-- .../pydevd/tests_python/debugger_unittest.py | 32 +++++++--- .../resources/_debugger_case_remote.py | 3 +- .../resources/_debugger_case_remote_1.py | 3 +- ...bugger_case_remote_unhandled_exceptions.py | 3 +- ...ugger_case_remote_unhandled_exceptions2.py | 3 +- .../_debugger_case_suspend_policy.py | 32 ++++++++++ .../pydevd/tests_python/test_debugger.py | 64 +++++++++++++++---- 8 files changed, 123 insertions(+), 31 deletions(-) create mode 100644 ptvsd/_vendored/pydevd/tests_python/resources/_debugger_case_suspend_policy.py diff --git a/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_process_net_command.py b/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_process_net_command.py index 2203ee6ac..3d8f02f40 100644 --- a/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_process_net_command.py +++ b/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_process_net_command.py @@ -253,14 +253,18 @@ def process_net_command(py_db, cmd_id, seq, text): # func name: 'None': match anything. Empty: match global, specified: only method context. # command to add some breakpoint. # text is file\tline. Add to breakpoints dictionary - suspend_policy = "NONE" + suspend_policy = "NONE" # Can be 'NONE' or 'ALL' is_logpoint = False hit_condition = None if py_db._set_breakpoints_with_id: try: - breakpoint_id, type, file, line, func_name, condition, expression, hit_condition, is_logpoint = text.split('\t', 8) + try: + breakpoint_id, type, file, line, func_name, condition, expression, hit_condition, is_logpoint, suspend_policy = text.split('\t', 9) + except ValueError: # not enough values to unpack + # No suspend_policy passed (use default). + breakpoint_id, type, file, line, func_name, condition, expression, hit_condition, is_logpoint = text.split('\t', 8) is_logpoint = is_logpoint == 'True' - except Exception: + except ValueError: # not enough values to unpack breakpoint_id, type, file, line, func_name, condition, expression = text.split('\t', 6) breakpoint_id = int(breakpoint_id) @@ -274,8 +278,8 @@ def process_net_command(py_db, cmd_id, seq, text): expression = expression.replace("@_@NEW_LINE_CHAR@_@", '\n').\ replace("@_@TAB_CHAR@_@", '\t').strip() else: - #Note: this else should be removed after PyCharm migrates to setting - #breakpoints by id (and ideally also provides func_name). + # Note: this else should be removed after PyCharm migrates to setting + # breakpoints by id (and ideally also provides func_name). type, file, line, func_name, suspend_policy, condition, expression = text.split('\t', 6) # If we don't have an id given for each breakpoint, consider # the id to be the line. diff --git a/ptvsd/_vendored/pydevd/tests_python/debugger_unittest.py b/ptvsd/_vendored/pydevd/tests_python/debugger_unittest.py index 13f8cefaf..3452afdb5 100644 --- a/ptvsd/_vendored/pydevd/tests_python/debugger_unittest.py +++ b/ptvsd/_vendored/pydevd/tests_python/debugger_unittest.py @@ -502,16 +502,25 @@ def wait_for_breakpoint_hit(self, *args, **kwargs): def wait_for_breakpoint_hit_with_suspend_type(self, reason=REASON_STOP_ON_BREAKPOINT, get_line=False, get_name=False): ''' - 108 is over - 109 is return - 111 is breakpoint + 108 is over + 109 is return + 111 is breakpoint + + :param reason: may be the actual reason (int or string) or a list of reasons. ''' self.log.append('Start: wait_for_breakpoint_hit') # wait for hit breakpoint - last = self.reader_thread.get_next_message('wait_for_breakpoint_hit. reason=%s' % (reason,)) - while not ('stop_reason="%s"' % reason) in last: + if not isinstance(reason, (list, tuple)): + reason = (reason,) + while True: last = self.reader_thread.get_next_message('wait_for_breakpoint_hit. reason=%s' % (reason,)) - + found = False + for r in reason: + if ('stop_reason="%s"' % (r,)) in last: + found = True + break + if found: + break # we have something like Date: Thu, 19 Jul 2018 11:05:19 -0300 Subject: [PATCH 2/2] Provide a CMD_GET_THREAD_STACK (#72). --- .../_pydev_bundle/pydev_console_utils.py | 50 +------ .../pydevd_additional_thread_info_regular.py | 12 +- .../pydevd/_pydevd_bundle/pydevd_comm.py | 130 ++++++++++++------ .../pydevd/_pydevd_bundle/pydevd_constants.py | 37 +++-- .../pydevd/_pydevd_bundle/pydevd_frame.py | 2 + .../pydevd_process_net_command.py | 29 +++- .../pydevd/_pydevd_bundle/pydevd_utils.py | 41 +++++- ptvsd/_vendored/pydevd/pydevd.py | 87 +++--------- ptvsd/_vendored/pydevd/pydevd_tracing.py | 8 +- .../pydevd/tests_python/debugger_unittest.py | 39 ++++-- .../_debugger_case_get_thread_stack.py | 15 ++ .../test_additional_thread_info.py | 1 - .../pydevd/tests_python/test_debugger.py | 99 +++++++++++-- .../pydevd/tests_python/test_null.py | 8 ++ 14 files changed, 361 insertions(+), 197 deletions(-) create mode 100644 ptvsd/_vendored/pydevd/tests_python/resources/_debugger_case_get_thread_stack.py create mode 100644 ptvsd/_vendored/pydevd/tests_python/test_null.py diff --git a/ptvsd/_vendored/pydevd/_pydev_bundle/pydev_console_utils.py b/ptvsd/_vendored/pydevd/_pydev_bundle/pydev_console_utils.py index 7b6a4c368..23aa41087 100644 --- a/ptvsd/_vendored/pydevd/_pydev_bundle/pydev_console_utils.py +++ b/ptvsd/_vendored/pydevd/_pydev_bundle/pydev_console_utils.py @@ -6,7 +6,7 @@ from _pydev_imps._pydev_saved_modules import thread from _pydevd_bundle import pydevd_vars from _pydevd_bundle import pydevd_xml -from _pydevd_bundle.pydevd_constants import IS_JYTHON, dict_iter_items, NEXT_VALUE_SEPARATOR +from _pydevd_bundle.pydevd_constants import IS_JYTHON, dict_iter_items, NEXT_VALUE_SEPARATOR, Null try: import cStringIO as StringIO #may not always be available @UnusedImport @@ -16,51 +16,6 @@ except: import io as StringIO -# ======================================================================================================================= -# Null -# ======================================================================================================================= -class Null: - """ - Gotten from: http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/68205 - """ - - def __init__(self, *args, **kwargs): - return None - - def __call__(self, *args, **kwargs): - return self - - def __getattr__(self, mname): - return self - - def __setattr__(self, name, value): - return self - - def __delattr__(self, name): - return self - - def __repr__(self): - return "" - - def __str__(self): - return "Null" - - def __len__(self): - return 0 - - def __getitem__(self): - return self - - def __setitem__(self, *args, **kwargs): - pass - - def write(self, *args, **kwargs): - pass - - def __nonzero__(self): - return 0 - - # ======================================================================================================================= # BaseStdIn # ======================================================================================================================= @@ -614,8 +569,9 @@ def do_connect_to_debugger(): traceback.print_exc() sys.stderr.write('pydevd is not available, cannot connect\n', ) + from _pydevd_bundle.pydevd_constants import set_thread_id from _pydev_bundle import pydev_localhost - threading.currentThread().__pydevd_id__ = "console_main" + set_thread_id(threading.currentThread(), "console_main") self.orig_find_frame = pydevd_vars.find_frame pydevd_vars.find_frame = self._findFrame diff --git a/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_additional_thread_info_regular.py b/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_additional_thread_info_regular.py index 9c961f41c..6cce214eb 100644 --- a/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_additional_thread_info_regular.py +++ b/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_additional_thread_info_regular.py @@ -115,13 +115,15 @@ def __init__(self): self.pydev_func_name = '.invalid.' # Must match the type in cython self.suspended_at_unhandled = False - def iter_frames(self, t): + def get_topmost_frame(self, thread): + ''' + Gets the topmost frame for the given thread. Note that it may be None + and callers should remove the reference to the frame as soon as possible + to avoid disturbing user code. + ''' # sys._current_frames(): dictionary with thread id -> topmost frame current_frames = _current_frames() - v = current_frames.get(t.ident) - if v is not None: - return [v] - return [] + return current_frames.get(thread.ident) def __str__(self): return 'State:%s Stop:%s Cmd: %s Kill:%s' % ( diff --git a/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_comm.py b/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_comm.py index 6580b3588..c00668f1e 100644 --- a/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_comm.py +++ b/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_comm.py @@ -111,6 +111,9 @@ def unquote(s): import StringIO #@Reimport except: import io as StringIO + +from _pydevd_bundle.pydevd_dont_trace_files import DONT_TRACE, PYDEV_FILE +get_file_type = DONT_TRACE.get CMD_RUN = 101 @@ -171,6 +174,9 @@ def unquote(s): CMD_SHOW_CYTHON_WARNING = 150 CMD_LOAD_FULL_VALUE = 151 +CMD_GET_THREAD_STACK = 152 +CMD_THREAD_DUMP_TO_STDERR = 153 # This is mostly for unit-tests to diagnose errors on ci. + CMD_REDIRECT_OUTPUT = 200 CMD_GET_NEXT_STATEMENT_TARGETS = 201 CMD_SET_PROJECT_ROOTS = 202 @@ -233,6 +239,8 @@ def unquote(s): '149': 'CMD_PROCESS_CREATED', '150': 'CMD_SHOW_CYTHON_WARNING', '151': 'CMD_LOAD_FULL_VALUE', + '152': 'CMD_GET_THREAD_STACK', + '153': 'CMD_THREAD_DUMP_TO_STDERR', '200': 'CMD_REDIRECT_OUTPUT', '201': 'CMD_GET_NEXT_STATEMENT_TARGETS', @@ -249,6 +257,7 @@ def unquote(s): from _pydev_bundle._pydev_filesystem_encoding import getfilesystemencoding file_system_encoding = getfilesystemencoding() +filesystem_encoding_is_utf8 = file_system_encoding.lower() in ('utf-8', 'utf_8', 'utf8') #--------------------------------------------------------------------------------------------------- UTILITIES @@ -637,6 +646,22 @@ def make_list_threads_message(self, seq): except: return self.make_error_message(seq, get_exception_traceback_str()) + def make_get_thread_stack_message(self, seq, thread_id, topmost_frame): + """Returns thread stack as XML """ + try: + # If frame is None, the return is an empty frame list. + cmd_text = ['' % (thread_id,)] + + if topmost_frame is not None: + try: + cmd_text.append(self.make_thread_stack_str(topmost_frame)) + finally: + topmost_frame = None + cmd_text.append('') + return NetCommand(CMD_GET_THREAD_STACK, seq, ''.join(cmd_text)) + except: + return self.make_error_message(seq, get_exception_traceback_str()) + def make_variable_changed_message(self, seq, payload): # notify debugger that value was changed successfully return NetCommand(CMD_RETURN, seq, payload) @@ -669,71 +694,94 @@ def make_thread_killed_message(self, id): except: return self.make_error_message(0, get_exception_traceback_str()) - def make_thread_suspend_str(self, thread_id, frame, stop_reason, message, suspend_type="trace"): - """ - - - - - """ - cmd_text_list = [""] - append = cmd_text_list.append + def make_thread_stack_str(self, frame): make_valid_xml_value = pydevd_xml.make_valid_xml_value - - if message: - message = make_valid_xml_value(message) - - append('' % (thread_id, stop_reason, message, suspend_type)) + cmd_text_list = [] + append = cmd_text_list.append curr_frame = frame + frame = None # Clear frame reference try: while curr_frame: - #print cmdText my_id = id(curr_frame) - #print "id is ", my_id if curr_frame.f_code is None: - break #Iron Python sometimes does not have it! + break # Iron Python sometimes does not have it! - my_name = curr_frame.f_code.co_name #method name (if in method) or ? if global - if my_name is None: - break #Iron Python sometimes does not have it! - - #print "name is ", my_name + method_name = curr_frame.f_code.co_name # method name (if in method) or ? if global + if method_name is None: + break # Iron Python sometimes does not have it! abs_path_real_path_and_base = get_abs_path_real_path_and_base_from_frame(curr_frame) - - myFile = pydevd_file_utils.norm_file_to_client(abs_path_real_path_and_base[0]) - if file_system_encoding.lower() != "utf-8" and hasattr(myFile, "decode"): - # myFile is a byte string encoded using the file system encoding + if get_file_type(abs_path_real_path_and_base[2]) == PYDEV_FILE: + # Skip pydevd files. + curr_frame = curr_frame.f_back + continue + + filename_in_utf8 = pydevd_file_utils.norm_file_to_client(abs_path_real_path_and_base[0]) + if not filesystem_encoding_is_utf8 and hasattr(filename_in_utf8, "decode"): + # filename_in_utf8 is a byte string encoded using the file system encoding # convert it to utf8 - myFile = myFile.decode(file_system_encoding).encode("utf-8") + filename_in_utf8 = filename_in_utf8.decode(file_system_encoding).encode("utf-8") - #print "file is ", myFile - #myFile = inspect.getsourcefile(curr_frame) or inspect.getfile(frame) + # print("file is ", filename_in_utf8) - myLine = str(curr_frame.f_lineno) - #print "line is ", myLine + lineno = str(curr_frame.f_lineno) + # print("line is ", lineno) - #the variables are all gotten 'on-demand' - #variables = pydevd_xml.frame_vars_to_xml(curr_frame.f_locals) - - variables = '' - append('' % (quote(myFile, '/>_= \t'), myLine)) - append(variables) + # Note: variables are all gotten 'on-demand'. + append('' % (quote(filename_in_utf8, '/>_= \t'), lineno)) append("") curr_frame = curr_frame.f_back - except : + except: traceback.print_exc() + curr_frame = None # Clear frame reference + return ''.join(cmd_text_list) + + def make_thread_suspend_str( + self, + thread_id, + frame, + stop_reason=None, + message=None, + suspend_type="trace", + ): + """ + :return str: + + + + + + + """ + make_valid_xml_value = pydevd_xml.make_valid_xml_value + cmd_text_list = [] + append = cmd_text_list.append + + cmd_text_list.append('') + if message: + message = make_valid_xml_value(message) + + append('') + append(self.make_thread_stack_str(frame)) append("") + return ''.join(cmd_text_list) def make_thread_suspend_message(self, thread_id, frame, stop_reason, message, suspend_type): try: - return NetCommand(CMD_THREAD_SUSPEND, 0, self.make_thread_suspend_str(thread_id, frame, stop_reason, message, suspend_type)) + return NetCommand(CMD_THREAD_SUSPEND, 0, self.make_thread_suspend_str( + thread_id, frame, stop_reason, message, suspend_type)) except: return self.make_error_message(0, get_exception_traceback_str()) diff --git a/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_constants.py b/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_constants.py index 2e3060484..91fcee95c 100644 --- a/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_constants.py +++ b/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_constants.py @@ -156,7 +156,7 @@ def protect_libraries_from_patching(): protect_libraries_from_patching() from _pydev_imps._pydev_saved_modules import thread -_nextThreadIdLock = thread.allocate_lock() +_thread_id_lock = thread.allocate_lock() if IS_PY3K: @@ -251,17 +251,19 @@ def get_pid(): def clear_cached_thread_id(thread): - try: - del thread.__pydevd_id__ - except AttributeError: - pass + with _thread_id_lock: + try: + if thread.__pydevd_id__ != 'console_main': + # The console_main is a special thread id used in the console and its id should never be reset + # (otherwise we may no longer be able to get its variables -- see: https://www.brainwy.com/tracker/PyDev/776). + del thread.__pydevd_id__ + except AttributeError: + pass -#======================================================================================================================= -# get_thread_id -#======================================================================================================================= def get_thread_id(thread): try: + # Fast path without getting lock. tid = thread.__pydevd_id__ if tid is None: # Fix for https://www.brainwy.com/tracker/PyDev/645 @@ -269,8 +271,7 @@ def get_thread_id(thread): # that gives us always the same id for the thread (using thread.ident or id(thread)). raise AttributeError() except AttributeError: - _nextThreadIdLock.acquire() - try: + with _thread_id_lock: # We do a new check with the lock in place just to be sure that nothing changed tid = getattr(thread, '__pydevd_id__', None) if tid is None: @@ -278,12 +279,15 @@ def get_thread_id(thread): # Note: don't use the thread ident because if we're too early in the # thread bootstrap process, the thread id could be still unset. tid = thread.__pydevd_id__ = 'pid_%s_id_%s' % (pid, id(thread)) - finally: - _nextThreadIdLock.release() return tid +def set_thread_id(thread, thread_id): + with _thread_id_lock: + thread.__pydevd_id__ = thread_id + + #======================================================================================================================= # Null #======================================================================================================================= @@ -297,6 +301,12 @@ def __init__(self, *args, **kwargs): def __call__(self, *args, **kwargs): return self + + def __enter__(self, *args, **kwargs): + return self + + def __exit__(self, *args, **kwargs): + return self def __getattr__(self, mname): if len(mname) > 4 and mname[:2] == '__' and mname[-2:] == '__': @@ -333,6 +343,9 @@ def __nonzero__(self): def __iter__(self): return iter(()) + +# Default instance +NULL = Null() def call_only_once(func): diff --git a/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_frame.py b/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_frame.py index a1d8f050f..ce7bb363f 100644 --- a/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_frame.py +++ b/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_frame.py @@ -406,6 +406,8 @@ def remove_return_values(self, main_debugger, frame): # cdef bint is_line; # cdef bint is_call; # cdef bint is_return; + # cdef bint should_stop; + # cdef dict breakpoints_for_file; # cdef str curr_func_name; # cdef bint exist_result; # cdef dict frame_skips_cache; diff --git a/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_process_net_command.py b/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_process_net_command.py index 3d8f02f40..e97841861 100644 --- a/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_process_net_command.py +++ b/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_process_net_command.py @@ -19,7 +19,8 @@ CMD_EVALUATE_CONSOLE_EXPRESSION, InternalEvaluateConsoleExpression, InternalConsoleGetCompletions, \ CMD_RUN_CUSTOM_OPERATION, InternalRunCustomOperation, CMD_IGNORE_THROWN_EXCEPTION_AT, CMD_ENABLE_DONT_TRACE, \ CMD_SHOW_RETURN_VALUES, ID_TO_MEANING, CMD_GET_DESCRIPTION, InternalGetDescription, InternalLoadFullValue, \ - CMD_LOAD_FULL_VALUE, CMD_REDIRECT_OUTPUT, CMD_GET_NEXT_STATEMENT_TARGETS, InternalGetNextStatementTargets, CMD_SET_PROJECT_ROOTS + CMD_LOAD_FULL_VALUE, CMD_REDIRECT_OUTPUT, CMD_GET_NEXT_STATEMENT_TARGETS, InternalGetNextStatementTargets, CMD_SET_PROJECT_ROOTS, \ + CMD_GET_THREAD_STACK, CMD_THREAD_DUMP_TO_STDERR from _pydevd_bundle.pydevd_constants import get_thread_id, IS_PY3K, DebugInfoHolder, dict_keys, STATE_RUN, \ NEXT_VALUE_SEPARATOR from _pydevd_bundle.pydevd_additional_thread_info import set_additional_thread_info @@ -79,6 +80,20 @@ def process_net_command(py_db, cmd_id, seq, text): # response is a list of threads cmd = py_db.cmd_factory.make_list_threads_message(seq) + elif cmd_id == CMD_GET_THREAD_STACK: + thread_id = text + + t = pydevd_find_thread_by_id(thread_id) + frame = None + if t and not getattr(t, 'pydev_do_not_trace', None): + additional_info = set_additional_thread_info(t) + frame = additional_info.get_topmost_frame(t) + try: + cmd = py_db.cmd_factory.make_get_thread_stack_message(seq, thread_id, frame) + finally: + frame = None + t = None + elif cmd_id == CMD_THREAD_KILL: int_cmd = InternalTerminateThread(text) py_db.post_internal_command(int_cmd, text) @@ -88,9 +103,12 @@ def process_net_command(py_db, cmd_id, seq, text): t = pydevd_find_thread_by_id(text) if t and not getattr(t, 'pydev_do_not_trace', None): additional_info = set_additional_thread_info(t) - for frame in additional_info.iter_frames(t): - py_db.set_trace_for_frame_and_parents(frame, overwrite_prev_trace=True) - del frame + frame = additional_info.get_topmost_frame(t) + if frame is not None: + try: + py_db.set_trace_for_frame_and_parents(frame, overwrite_prev_trace=True) + finally: + frame = None py_db.set_suspend(t, CMD_THREAD_SUSPEND) elif text.startswith('__frame__:'): @@ -764,6 +782,9 @@ def process_net_command(py_db, cmd_id, seq, text): elif cmd_id == CMD_SET_PROJECT_ROOTS: pydevd_utils.set_project_roots(text.split(u'\t')) + elif cmd_id == CMD_THREAD_DUMP_TO_STDERR: + pydevd_utils.dump_threads() + else: #I have no idea what this is all about cmd = py_db.cmd_factory.make_error_message(seq, "unexpected command " + str(cmd_id)) diff --git a/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_utils.py b/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_utils.py index f9fd2f3ea..920fff810 100644 --- a/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_utils.py +++ b/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_utils.py @@ -13,7 +13,7 @@ from _pydevd_bundle.pydevd_constants import IS_PY3K, get_global_debugger import sys from _pydev_bundle import pydev_log - +from _pydev_imps._pydev_saved_modules import threading def _normpath(filename): return pydevd_file_utils.get_abs_path_real_path_and_base_from_file(filename)[0] @@ -310,3 +310,42 @@ def is_ignored_by_filter(filename, filename_to_ignored_by_filters_cache={}): return filename_to_ignored_by_filters_cache[filename] + +def dump_threads(stream=None): + ''' + Helper to dump thread info. + ''' + if stream is None: + stream = sys.stderr + thread_id_to_name = {} + try: + for t in threading.enumerate(): + thread_id_to_name[t.ident] = '%s (daemon: %s, pydevd thread: %s)' % ( + t.name, t.daemon, getattr(t, 'is_pydev_daemon_thread', False)) + except: + pass + + stream.write('===============================================================================\n') + stream.write('Threads running\n') + stream.write('================================= Thread Dump =================================\n') + + for thread_id, stack in sys._current_frames().items(): + stream.write('\n-------------------------------------------------------------------------------\n') + stream.write(" Thread %s" % thread_id_to_name.get(thread_id, thread_id)) + stream.write('\n\n') + + for i, (filename, lineno, name, line) in enumerate(traceback.extract_stack(stack)): + + stream.write(' File "%s", line %d, in %s\n' % (filename, lineno, name)) + if line: + stream.write(" %s\n" % (line.strip())) + + if i == 0 and 'self' in stack.f_locals: + stream.write(' self: ') + try: + stream.write(str(stack.f_locals['self'])) + except: + stream.write('Unable to get str of: %s' % (type(stack.f_locals['self']),)) + stream.write('\n') + + stream.write('\n=============================== END Thread Dump ===============================') diff --git a/ptvsd/_vendored/pydevd/pydevd.py b/ptvsd/_vendored/pydevd/pydevd.py index a0fdb90dd..eb4f63b8d 100644 --- a/ptvsd/_vendored/pydevd/pydevd.py +++ b/ptvsd/_vendored/pydevd/pydevd.py @@ -14,7 +14,7 @@ from _pydevd_bundle.pydevd_constants import IS_JYTH_LESS25, IS_PYCHARM, get_thread_id, \ dict_keys, dict_iter_items, DebugInfoHolder, PYTHON_SUSPEND, STATE_SUSPEND, STATE_RUN, get_frame, xrange, \ - clear_cached_thread_id, INTERACTIVE_MODE_AVAILABLE, SHOW_DEBUG_INFO_ENV, IS_PY34_OR_GREATER, IS_PY2 + clear_cached_thread_id, INTERACTIVE_MODE_AVAILABLE, SHOW_DEBUG_INFO_ENV, IS_PY34_OR_GREATER, IS_PY2, NULL from _pydev_bundle import fix_getpass from _pydev_bundle import pydev_imports, pydev_log from _pydev_bundle._pydev_filesystem_encoding import getfilesystemencoding @@ -44,7 +44,7 @@ from pydevd_concurrency_analyser.pydevd_concurrency_logger import ThreadingLogger, AsyncioLogger, send_message, cur_time from pydevd_concurrency_analyser.pydevd_thread_wrappers import wrap_threads from pydevd_file_utils import get_fullname, rPath -import pydev_ipython +import pydev_ipython # @UnusedImport __version_info__ = (1, 2, 0) __version_info_str__ = [] @@ -430,9 +430,12 @@ def suspend_all_other_threads(self, thread_suspended_at_bp): if t is thread_suspended_at_bp: continue additional_info = set_additional_thread_info(t) - for frame in additional_info.iter_frames(t): - self.set_trace_for_frame_and_parents(frame, overwrite_prev_trace=True) - del frame + frame = additional_info.get_topmost_frame(t) + if frame is not None: + try: + self.set_trace_for_frame_and_parents(frame, overwrite_prev_trace=True) + finally: + frame = None self.set_suspend(t, CMD_THREAD_SUSPEND) @@ -444,9 +447,7 @@ def notify_thread_created(self, thread_id, thread, use_lock=True): # not be usual as it's expected that the debugger is live before other threads are created). return - if use_lock: - self._lock_running_thread_ids.acquire() - try: + with self._lock_running_thread_ids if use_lock else NULL: if thread_id in self._running_thread_ids: return @@ -457,9 +458,6 @@ def notify_thread_created(self, thread_id, thread, use_lock=True): return self._running_thread_ids[thread_id] = thread - finally: - if use_lock: - self._lock_running_thread_ids.release() self.writer.add_command(self.cmd_factory.make_thread_created_message(thread)) @@ -468,10 +466,7 @@ def notify_thread_not_alive(self, thread_id, use_lock=True): if self.writer is None: return - if use_lock: - self._lock_running_thread_ids.acquire() - - try: + with self._lock_running_thread_ids if use_lock else NULL: thread = self._running_thread_ids.pop(thread_id, None) if thread is None: return @@ -479,24 +474,19 @@ def notify_thread_not_alive(self, thread_id, use_lock=True): was_notified = thread.additional_info.pydev_notify_kill if not was_notified: thread.additional_info.pydev_notify_kill = True - finally: - if use_lock: - self._lock_running_thread_ids.release() self.writer.add_command(self.cmd_factory.make_thread_killed_message(thread_id)) def process_internal_commands(self): '''This function processes internal commands ''' - self._main_lock.acquire() - try: + with self._main_lock: self.check_output_redirect() program_threads_alive = {} all_threads = threadingEnumerate() program_threads_dead = [] - self._lock_running_thread_ids.acquire() - try: + with self._lock_running_thread_ids: for t in all_threads: if getattr(t, 'is_pydev_daemon_thread', False): pass # I.e.: skip the DummyThreads created from pydev daemon threads @@ -514,10 +504,7 @@ def process_internal_commands(self): # Fix it for all existing threads. for existing_thread in all_threads: old_thread_id = get_thread_id(existing_thread) - if old_thread_id != 'console_main': - # The console_main is a special thread id used in the console and its id should never be reset - # (otherwise we may no longer be able to get its variables -- see: https://www.brainwy.com/tracker/PyDev/776). - clear_cached_thread_id(t) + clear_cached_thread_id(t) thread_id = get_thread_id(t) if thread_id != old_thread_id: @@ -538,9 +525,8 @@ def process_internal_commands(self): for thread_id in program_threads_dead: self.notify_thread_not_alive(thread_id, use_lock=False) - finally: - self._lock_running_thread_ids.release() + # Without self._lock_running_thread_ids if len(program_threads_alive) == 0: self.finish_debugging_session() for t in all_threads: @@ -585,10 +571,6 @@ def process_internal_commands(self): for int_cmd in cmdsToReadd: queue.put(int_cmd) - - finally: - self._main_lock.release() - def disable_tracing_while_running_if_frame_eval(self): pydevd_tracing.settrace_while_running_if_frame_eval(self, self.dummy_trace_dispatch) @@ -611,12 +593,13 @@ def set_tracing_for_untraced_contexts(self, ignore_frame=None, overwrite_prev_tr if getattr(t, 'is_pydev_daemon_thread', False): continue - # TODO: optimize so that we only actually add that tracing if it's in - # the new breakpoint context. additional_info = set_additional_thread_info(t) - for frame in additional_info.iter_frames(t): - if frame is not ignore_frame: + frame = additional_info.get_topmost_frame(t) + try: + if frame is not None and frame is not ignore_frame: self.set_trace_for_frame_and_parents(frame, overwrite_prev_trace=overwrite_prev_trace) + finally: + frame = None finally: frame = None t = None @@ -1132,37 +1115,9 @@ def enable_qt_support(qt_support_mode): def dump_threads(stream=None): ''' - Helper to dump thread info. + Helper to dump thread info (default is printing to stderr). ''' - if stream is None: - stream = sys.stderr - thread_id_to_name = {} - try: - for t in threading.enumerate(): - thread_id_to_name[t.ident] = '%s (daemon: %s, pydevd thread: %s)' % ( - t.name, t.daemon, getattr(t, 'is_pydev_daemon_thread', False)) - except: - pass - - stack_trace = [ - '===============================================================================', - 'Threads running', - '================================= Thread Dump ================================='] - - for thread_id, stack in sys._current_frames().items(): - stack_trace.append('\n-------------------------------------------------------------------------------') - stack_trace.append(" Thread %s" % thread_id_to_name.get(thread_id, thread_id)) - stack_trace.append('') - - if 'self' in stack.f_locals: - stream.write(str(stack.f_locals['self']) + '\n') - - for filename, lineno, name, line in traceback.extract_stack(stack): - stack_trace.append(' File "%s", line %d, in %s' % (filename, lineno, name)) - if line: - stack_trace.append(" %s" % (line.strip())) - stack_trace.append('\n=============================== END Thread Dump ===============================') - stream.write('\n'.join(stack_trace)) + pydevd_utils.dump_threads(stream) def usage(doExit=0): diff --git a/ptvsd/_vendored/pydevd/pydevd_tracing.py b/ptvsd/_vendored/pydevd/pydevd_tracing.py index 04ed84ea3..dd2eb6eb7 100644 --- a/ptvsd/_vendored/pydevd/pydevd_tracing.py +++ b/ptvsd/_vendored/pydevd/pydevd_tracing.py @@ -124,8 +124,12 @@ def settrace_while_running_if_frame_eval(py_db, trace_func): # that's ok, no info currently set continue - for frame in additional_info.iter_frames(t): - py_db.set_trace_for_frame_and_parents(frame, overwrite_prev_trace=True, dispatch_func=trace_func) + frame = additional_info.get_topmost_frame(t) + if frame is not None: + try: + py_db.set_trace_for_frame_and_parents(frame, overwrite_prev_trace=True, dispatch_func=trace_func) + finally: + frame = None py_db.enable_cache_frames_without_breaks(False) # sometimes (when script enters new frames too fast), we can't enable tracing only in the appropriate # frame. So, if breakpoint was added during run, we should disable frame evaluation forever. diff --git a/ptvsd/_vendored/pydevd/tests_python/debugger_unittest.py b/ptvsd/_vendored/pydevd/tests_python/debugger_unittest.py index 3452afdb5..439f66b27 100644 --- a/ptvsd/_vendored/pydevd/tests_python/debugger_unittest.py +++ b/ptvsd/_vendored/pydevd/tests_python/debugger_unittest.py @@ -11,6 +11,7 @@ import sys import threading import time +import traceback from _pydev_bundle import pydev_localhost @@ -69,6 +70,9 @@ CMD_STEP_INTO_MY_CODE = 144 CMD_GET_CONCURRENCY_EVENT = 145 +CMD_GET_THREAD_STACK = 152 +CMD_THREAD_DUMP_TO_STDERR = 153 # This is mostly for unit-tests to diagnose errors on ci. + CMD_REDIRECT_OUTPUT = 200 CMD_GET_NEXT_STATEMENT_TARGETS = 201 CMD_SET_PROJECT_ROOTS = 202 @@ -140,11 +144,11 @@ def __init__(self, sock): def set_timeout(self, timeout): self.TIMEOUT = timeout - def get_next_message(self, context_messag): + def get_next_message(self, context_message): try: msg = self._queue.get(block=True, timeout=self.TIMEOUT) except: - raise AssertionError('No message was written in %s seconds. Error message:\n%s' % (self.TIMEOUT, context_messag,)) + raise AssertionError('No message was written in %s seconds. Error message:\n%s' % (self.TIMEOUT, context_message,)) else: frame = sys._getframe().f_back frame_info = '' @@ -156,7 +160,7 @@ def get_next_message(self, context_messag): frame_info += stack_msg frame = frame.f_back frame = None - sys.stdout.write('Message returned in get_next_message(): %s -- ctx: %s, asked at:\n%s\n' % (unquote_plus(unquote_plus(msg)), context_messag, frame_info)) + sys.stdout.write('Message returned in get_next_message(): %s -- ctx: %s, asked at:\n%s\n' % (unquote_plus(unquote_plus(msg)), context_message, frame_info)) return msg def run(self): @@ -289,6 +293,7 @@ def read(stream, buffer, debug_stream, stream_name): # finish successfully). initial_time = time.time() shown_intermediate = False + dumped_threads = False while True: if process.poll() is not None: break @@ -303,6 +308,17 @@ def read(stream, buffer, debug_stream, stream_name): print('Warning: writer thread exited and process still did not (%.2fs seconds elapsed).' % (time.time() - initial_time,)) shown_intermediate = True + if time.time() - initial_time > 15: + if not dumped_threads: + dumped_threads = True + # 15 seconds elapsed and it still didn't finish. Ask for a thread dump + # (we'll be able to see it later on the test output stderr). + try: + writer_thread.write_dump_threads() + except: + traceback.print_exc() + + if time.time() - initial_time > 20: process.kill() time.sleep(.2) @@ -647,6 +663,9 @@ def write_add_breakpoint(self, line, func, filename=None, hit_condition=None, is self.log.append('write_add_breakpoint: %s line: %s func: %s' % (breakpoint_id, line, func)) return breakpoint_id + def write_dump_threads(self): + self.write("%s\t%s\t" % (CMD_THREAD_DUMP_TO_STDERR, self.next_seq())) + def write_add_exception_breakpoint(self, exception): self.write("%s\t%s\t%s" % (CMD_ADD_EXCEPTION_BREAK, self.next_seq(), exception)) self.log.append('write_add_exception_breakpoint: %s' % (exception,)) @@ -714,6 +733,10 @@ def write_run_thread(self, thread_id): self.log.append('write_run_thread') self.write("%s\t%s\t%s" % (CMD_THREAD_RUN, self.next_seq(), thread_id,)) + def write_get_thread_stack(self, thread_id): + self.log.append('write_get_thread_stack') + self.write("%s\t%s\t%s" % (CMD_GET_THREAD_STACK, self.next_seq(), thread_id,)) + def write_load_source(self, filename): self.log.append('write_load_source') self.write("%s\t%s\t%s" % (CMD_LOAD_SOURCE, self.next_seq(), filename,)) @@ -751,12 +774,8 @@ def write_list_threads(self): return seq def wait_for_list_threads(self, seq): - while True: - # Note: get_next_message would timeout if there's no message. - last = self.reader_thread.get_next_message('wait_list_threads') - if last.startswith('502\t%s' % (seq,)): - return re.findall(r'\bid=\"(\w+)\"', last) - + return self.wait_for_message(lambda msg:msg.startswith('502\t%s' % (seq,))) + def wait_for_message(self, accept_message, unquote_msg=True, expect_xml=True): import untangle from io import StringIO @@ -774,7 +793,7 @@ def wait_for_message(self, accept_message, unquote_msg=True, expect_xml=True): xml = xml.decode('utf-8') xml = untangle.parse(StringIO(xml)) except: - import traceback;traceback.print_exc() + traceback.print_exc() raise AssertionError('Unable to parse:\n%s\nxml:\n%s' % (last, xml)) return xml.xml else: diff --git a/ptvsd/_vendored/pydevd/tests_python/resources/_debugger_case_get_thread_stack.py b/ptvsd/_vendored/pydevd/tests_python/resources/_debugger_case_get_thread_stack.py new file mode 100644 index 000000000..4a5ede01f --- /dev/null +++ b/ptvsd/_vendored/pydevd/tests_python/resources/_debugger_case_get_thread_stack.py @@ -0,0 +1,15 @@ +import threading +event_set = False + +def method(): + while not event_set: + import time + time.sleep(.1) + +t = threading.Thread(target=method) +t.start() + +print('break here') +event_set = True +t.join() +print('TEST SUCEEDED!') \ No newline at end of file diff --git a/ptvsd/_vendored/pydevd/tests_python/test_additional_thread_info.py b/ptvsd/_vendored/pydevd/tests_python/test_additional_thread_info.py index f161216f9..bb69c6b56 100644 --- a/ptvsd/_vendored/pydevd/tests_python/test_additional_thread_info.py +++ b/ptvsd/_vendored/pydevd/tests_python/test_additional_thread_info.py @@ -3,7 +3,6 @@ from _pydev_bundle import pydev_monkey sys.path.insert(0, os.path.split(os.path.split(__file__)[0])[0]) -from _pydevd_bundle.pydevd_constants import Null import unittest try: diff --git a/ptvsd/_vendored/pydevd/tests_python/test_debugger.py b/ptvsd/_vendored/pydevd/tests_python/test_debugger.py index fe8065345..baaa2ef76 100644 --- a/ptvsd/_vendored/pydevd/tests_python/test_debugger.py +++ b/ptvsd/_vendored/pydevd/tests_python/test_debugger.py @@ -16,8 +16,9 @@ import pytest from tests_python import debugger_unittest -from tests_python.debugger_unittest import get_free_port, CMD_SET_PROPERTY_TRACE, REASON_CAUGHT_EXCEPTION, \ - REASON_UNCAUGHT_EXCEPTION, REASON_STOP_ON_BREAKPOINT, REASON_THREAD_SUSPEND, overrides +from tests_python.debugger_unittest import (get_free_port, CMD_SET_PROPERTY_TRACE, REASON_CAUGHT_EXCEPTION, + REASON_UNCAUGHT_EXCEPTION, REASON_STOP_ON_BREAKPOINT, REASON_THREAD_SUSPEND, overrides, CMD_THREAD_CREATE, + CMD_GET_THREAD_STACK) IS_CPYTHON = platform.python_implementation() == 'CPython' IS_IRONPYTHON = platform.python_implementation() == 'IronPython' @@ -922,7 +923,8 @@ class WriterThreadCaseUnhandledExceptions(debugger_unittest.AbstractWriterThread @overrides(debugger_unittest.AbstractWriterThread.additional_output_checks) def additional_output_checks(self, stdout, stderr): if 'raise Exception' not in stderr: - raise AssertionError('Expected test to have an unhandled exception.') + raise AssertionError('Expected test to have an unhandled exception.\nstdout:\n%s\n\nstderr:\n%s' % ( + stdout, stderr)) # Don't call super (we have an unhandled exception in the stack trace). def run(self): @@ -959,7 +961,9 @@ def check_test_suceeded_msg(self, stdout, stderr): @overrides(debugger_unittest.AbstractWriterThread.additional_output_checks) def additional_output_checks(self, stdout, stderr): # Don't call super as we have an expected exception - assert 'ValueError: TEST SUCEEDED' in stderr + if 'ValueError: TEST SUCEEDED' not in stderr: + raise AssertionError('"ValueError: TEST SUCEEDED" not in stderr.\nstdout:\n%s\n\nstderr:\n%s' % ( + stdout, stderr)) def run(self): self.start_socket() @@ -988,7 +992,10 @@ def check_test_suceeded_msg(self, stdout, stderr): @overrides(debugger_unittest.AbstractWriterThread.additional_output_checks) def additional_output_checks(self, stdout, stderr): # Don't call super as we have an expected exception - assert 'ValueError: TEST SUCEEDED' in stderr + if 'ValueError: TEST SUCEEDED' not in stderr: + raise AssertionError('"ValueError: TEST SUCEEDED" not in stderr.\nstdout:\n%s\n\nstderr:\n%s' % ( + stdout, stderr)) + @overrides(debugger_unittest.AbstractWriterThread.get_environ) def get_environ(self): @@ -1036,7 +1043,9 @@ def check_test_suceeded_msg(self, stdout, stderr): @overrides(debugger_unittest.AbstractWriterThread.additional_output_checks) def additional_output_checks(self, stdout, stderr): # Don't call super as we have an expected exception - assert 'ValueError: TEST SUCEEDED' in stderr + if 'ValueError: TEST SUCEEDED' not in stderr: + raise AssertionError('"ValueError: TEST SUCEEDED" not in stderr.\nstdout:\n%s\n\nstderr:\n%s' % ( + stdout, stderr)) def run(self): self.start_socket() @@ -1954,8 +1963,9 @@ def run(self): thread_id, frame_id = self.wait_for_breakpoint_hit() seq = self.write_list_threads() - threads = self.wait_for_list_threads(seq) - assert len(threads) == 1 + msg = self.wait_for_list_threads(seq) + assert msg.thread['name'] == 'MainThread' + assert msg.thread['id'].startswith('pid') self.write_run_thread(thread_id) self.finished_ok = True @@ -2081,6 +2091,73 @@ def run(self): self.finished_ok = True +#======================================================================================================================= +# WriterCaseGetThreadStack +#====================================================================================================================== +class WriterCaseGetThreadStack(debugger_unittest.AbstractWriterThread): + + TEST_FILE = debugger_unittest._get_debugger_test_file('_debugger_case_get_thread_stack.py') + + def run(self): + self.start_socket() + self.write_add_breakpoint(12, None) + self.write_make_initial_run() + + thread_created_msgs = [self.wait_for_message(lambda msg:msg.startswith('%s\t' % (CMD_THREAD_CREATE,)))] + thread_created_msgs.append(self.wait_for_message(lambda msg:msg.startswith('%s\t' % (CMD_THREAD_CREATE,)))) + thread_id_to_name = {} + for msg in thread_created_msgs: + thread_id_to_name[msg.thread['id']] = msg.thread['name'] + assert len(thread_id_to_name) == 2 + + thread_id, _frame_id = self.wait_for_breakpoint_hit(REASON_STOP_ON_BREAKPOINT) + assert thread_id in thread_id_to_name + + for request_thread_id in thread_id_to_name: + self.write_get_thread_stack(request_thread_id) + msg = self.wait_for_message(lambda msg:msg.startswith('%s\t' % (CMD_GET_THREAD_STACK,))) + files = [frame['file'] for frame in msg.thread.frame] + assert msg.thread['id'] == request_thread_id + if not files[0].endswith('_debugger_case_get_thread_stack.py'): + raise AssertionError('Expected to find _debugger_case_get_thread_stack.py in files[0]. Found: %s' % ('\n'.join(files),)) + + if ([filename for filename in files if filename.endswith('pydevd.py')]): + raise AssertionError('Did not expect to find pydevd.py. Found: %s' % ('\n'.join(files),)) + if request_thread_id == thread_id: + assert len(msg.thread.frame) == 0 # In main thread (must have no back frames). + assert msg.thread.frame['name'] == '' + else: + assert len(msg.thread.frame) > 1 # Stopped in threading (must have back frames). + assert msg.thread.frame[0]['name'] == 'method' + + self.write_run_thread(thread_id) + + self.finished_ok = True + + +#======================================================================================================================= +# WriterCaseDumpThreadsToStderr +#====================================================================================================================== +class WriterCaseDumpThreadsToStderr(debugger_unittest.AbstractWriterThread): + + TEST_FILE = debugger_unittest._get_debugger_test_file('_debugger_case_get_thread_stack.py') + + def additional_output_checks(self, stdout, stderr): + assert 'Thread Dump' in stderr and 'Thread pydevd.CommandThread (daemon: True, pydevd thread: True)' in stderr, \ + 'Did not find thread dump in stderr. stderr:\n%s' % (stderr,) + + def run(self): + self.start_socket() + self.write_add_breakpoint(12, None) + self.write_make_initial_run() + + thread_id, _frame_id = self.wait_for_breakpoint_hit(REASON_STOP_ON_BREAKPOINT) + + self.write_dump_threads() + self.write_run_thread(thread_id) + + self.finished_ok = True + #======================================================================================================================= # Test @@ -2329,6 +2406,12 @@ def test_debug_zip_files(self): def test_case_suspension_policy(self): self.check_case(WriterCaseBreakpointSuspensionPolicy) + def test_case_get_thread_stack(self): + self.check_case(WriterCaseGetThreadStack) + + def test_case_dump_threads_to_stderr(self): + self.check_case(WriterCaseDumpThreadsToStderr) + @pytest.mark.skipif(not IS_CPYTHON, reason='CPython only test.') class TestPythonRemoteDebugger(unittest.TestCase, debugger_unittest.DebuggerRunner): diff --git a/ptvsd/_vendored/pydevd/tests_python/test_null.py b/ptvsd/_vendored/pydevd/tests_python/test_null.py new file mode 100644 index 000000000..f09da7b71 --- /dev/null +++ b/ptvsd/_vendored/pydevd/tests_python/test_null.py @@ -0,0 +1,8 @@ +def test_null(): + from _pydevd_bundle.pydevd_constants import Null + null = Null() + assert not null + assert len(null) == 0 + + with null as n: + n.write('foo') \ No newline at end of file