On Python 3.11, show column information on exceptions. Fixes #1099

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_comm.py

@@ -73,7 +73,7 @@
 from _pydevd_bundle.pydevd_constants import (DebugInfoHolder, IS_WINDOWS, IS_JYTHON, IS_WASM,
     IS_PY36_OR_GREATER, STATE_RUN, ASYNC_EVAL_TIMEOUT_SEC,
     get_global_debugger, GetGlobalDebugger, set_global_debugger,  # Keep for backward compatibility @UnusedImport
-    silence_warnings_decorator, filter_all_warnings)
+    silence_warnings_decorator, filter_all_warnings, IS_PY311_OR_GREATER)
 from _pydev_bundle.pydev_override import overrides
 import weakref
 from _pydev_bundle._pydev_completer import extract_token_and_qualifier
@@ -127,6 +127,7 @@
 if not IS_WASM:
     SO_REUSEADDR = socket_module.SO_REUSEADDR
 
+
 class ReaderThread(PyDBDaemonThread):
     ''' reader thread reads and dispatches commands in an infinite loop '''
 
@@ -1447,7 +1448,7 @@ def build_exception_info_response(dbg, thread_id, request_seq, set_additional_th
                             except:
                                 pass
 
-                    for frame_id, frame, method_name, original_filename, filename_in_utf8, lineno, _applied_mapping, show_as_current_frame in \
+                    for frame_id, frame, method_name, original_filename, filename_in_utf8, lineno, _applied_mapping, show_as_current_frame, line_col_info in \
                         iter_visible_frames_info(dbg, frames_list):
 
                         line_text = linecache.getline(original_filename, lineno)
@@ -1460,12 +1461,26 @@ def build_exception_info_response(dbg, thread_id, request_seq, set_additional_th
                         if show_as_current_frame:
                             current_paused_frame_name = method_name
                             method_name += ' (Current frame)'
-                        frames.append((filename_in_utf8, lineno, method_name, line_text))
+                        frames.append((filename_in_utf8, lineno, method_name, line_text, line_col_info))
 
                     if not source_path and frames:
                         source_path = frames[0][0]
 
-                    stack_str = ''.join(traceback.format_list(frames[-max_frames:]))
+                    if IS_PY311_OR_GREATER:
+                        stack_summary = traceback.StackSummary()
+                        for filename_in_utf8, lineno, method_name, line_text, line_col_info in frames[-max_frames:]:
+                            frame_summary = traceback.FrameSummary(filename_in_utf8, lineno, method_name, line=line_text)
+                            if line_col_info is not None:
+                                frame_summary.end_lineno = line_col_info.end_lineno
+                                frame_summary.colno = line_col_info.colno
+                                frame_summary.end_colno = line_col_info.end_colno
+                            stack_summary.append(frame_summary)
+
+                        stack_str = ''.join(stack_summary.format())
+
+                    else:
+                        # Note: remove col info (just used in 3.11).
+                        stack_str = ''.join(traceback.format_list((x[:-1] for x in frames[-max_frames:])))
 
                     try:
                         stype = frames_list.exc_type.__qualname__

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_frame_utils.py

@@ -1,5 +1,9 @@
-from _pydevd_bundle.pydevd_constants import EXCEPTION_TYPE_USER_UNHANDLED, EXCEPTION_TYPE_UNHANDLED
+from _pydevd_bundle.pydevd_constants import EXCEPTION_TYPE_USER_UNHANDLED, EXCEPTION_TYPE_UNHANDLED, \
+    IS_PY311_OR_GREATER
 from _pydev_bundle import pydev_log
+import itertools
+from collections import namedtuple
+from typing import Any, Dict
 
 
 class Frame(object):
@@ -75,6 +79,9 @@ def cached_call(obj, func, *args):
     return getattr(obj, cached_name)
 
 
+_LineColInfo = namedtuple('_LineColInfo', 'lineno, end_lineno, colno, end_colno')
+
+
 class FramesList(object):
 
     def __init__(self):
@@ -84,6 +91,7 @@ def __init__(self):
         # otherwise frame.f_lineno will be used (needed for unhandled exceptions as
         # the place where we report may be different from the place where it's raised).
         self.frame_id_to_lineno = {}
+        self.frame_id_to_line_col_info: Dict[Any, _LineColInfo] = {}
 
         self.exc_type = None
         self.exc_desc = None
@@ -209,16 +217,40 @@ def create_frames_list_from_exception_cause(trace_obj, frame, exc_type, exc_desc
         # Note: we don't use the actual tb.tb_frame because if the cause of the exception
         # uses the same frame object, the id(frame) would be the same and the frame_id_to_lineno
         # would be wrong as the same frame needs to appear with 2 different lines.
-        lst.append((_DummyFrameWrapper(tb.tb_frame, tb.tb_lineno, None), tb.tb_lineno))
+        lst.append((_DummyFrameWrapper(tb.tb_frame, tb.tb_lineno, None), tb.tb_lineno, _get_line_col_info_from_tb(tb)))
         tb = tb.tb_next
 
-    for tb_frame, tb_lineno in lst:
+    for tb_frame, tb_lineno, line_col_info in lst:
         frames_list.append(tb_frame)
         frames_list.frame_id_to_lineno[id(tb_frame)] = tb_lineno
+        frames_list.frame_id_to_line_col_info[id(tb_frame)] = line_col_info
 
     return frames_list
 
 
+if IS_PY311_OR_GREATER:
+
+    def _get_code_position(code, instruction_index):
+        if instruction_index < 0:
+            return (None, None, None, None)
+        positions_gen = code.co_positions()
+        # Note: some or all of the tuple elements can be None...
+        return next(itertools.islice(positions_gen, instruction_index // 2, None))
+
+    def _get_line_col_info_from_tb(tb):
+        positions = _get_code_position(tb.tb_frame.f_code, tb.tb_lasti)
+        if positions[0] is None:
+            return _LineColInfo(tb.tb_lineno, *positions[1:])
+        else:
+            return _LineColInfo(*positions)
+
+else:
+
+    def _get_line_col_info_from_tb(tb):
+        # Not available on older versions of Python.
+        return None
+
+
 def create_frames_list_from_traceback(trace_obj, frame, exc_type, exc_desc, exception_type=None):
     '''
     :param trace_obj:
@@ -238,16 +270,16 @@ def create_frames_list_from_traceback(trace_obj, frame, exc_type, exc_desc, exce
     if tb is not None and tb.tb_frame is not None:
         f = tb.tb_frame.f_back
         while f is not None:
-            lst.insert(0, (f, f.f_lineno))
+            lst.insert(0, (f, f.f_lineno, None))
             f = f.f_back
 
     while tb is not None:
-        lst.append((tb.tb_frame, tb.tb_lineno))
+        lst.append((tb.tb_frame, tb.tb_lineno, _get_line_col_info_from_tb(tb)))
         tb = tb.tb_next
 
     frames_list = None
 
-    for tb_frame, tb_lineno in reversed(lst):
+    for tb_frame, tb_lineno, line_col_info in reversed(lst):
         if frames_list is None and (
                 (frame is tb_frame) or
                 (frame is None) or
@@ -258,6 +290,7 @@ def create_frames_list_from_traceback(trace_obj, frame, exc_type, exc_desc, exce
         if frames_list is not None:
             frames_list.append(tb_frame)
             frames_list.frame_id_to_lineno[id(tb_frame)] = tb_lineno
+            frames_list.frame_id_to_line_col_info[id(tb_frame)] = line_col_info
 
     if frames_list is None and frame is not None:
         # Fallback (shouldn't happen in practice).

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_net_command_factory_json.py

@@ -229,7 +229,7 @@ def make_get_thread_stack_message(self, py_db, seq, thread_id, topmost_frame, fm
                 else:
                     frames_list = pydevd_frame_utils.create_frames_list_from_frame(topmost_frame)
 
-            for frame_id, frame, method_name, original_filename, filename_in_utf8, lineno, applied_mapping, show_as_current_frame in self._iter_visible_frames_info(
+            for frame_id, frame, method_name, original_filename, filename_in_utf8, lineno, applied_mapping, show_as_current_frame, line_col_info in self._iter_visible_frames_info(
                     py_db, frames_list, flatten_chained=True
                 ):
 

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_net_command_factory_xml.py

@@ -190,12 +190,13 @@ def _iter_visible_frames_info(self, py_db, frames_list, flatten_chained=False):
 
                 frame_id = id(frame)
                 lineno = frames_list.frame_id_to_lineno.get(frame_id, frame.f_lineno)
+                line_col_info = frames_list.frame_id_to_line_col_info.get(frame_id)
 
                 filename_in_utf8, lineno, changed = py_db.source_mapping.map_to_client(abs_path_real_path_and_base[0], lineno)
                 new_filename_in_utf8, applied_mapping = pydevd_file_utils.map_file_to_client(filename_in_utf8)
                 applied_mapping = applied_mapping or changed
 
-                yield frame_id, frame, method_name, abs_path_real_path_and_base[0], new_filename_in_utf8, lineno, applied_mapping, show_as_current_frame
+                yield frame_id, frame, method_name, abs_path_real_path_and_base[0], new_filename_in_utf8, lineno, applied_mapping, show_as_current_frame, line_col_info
 
             if not flatten_chained:
                 break
@@ -212,7 +213,7 @@ def make_thread_stack_str(self, py_db, frames_list):
         append = cmd_text_list.append
 
         try:
-            for frame_id, frame, method_name, _original_filename, filename_in_utf8, lineno, _applied_mapping, _show_as_current_frame in self._iter_visible_frames_info(
+            for frame_id, frame, method_name, _original_filename, filename_in_utf8, lineno, _applied_mapping, _show_as_current_frame, line_col_info in self._iter_visible_frames_info(
                     py_db, frames_list, flatten_chained=True
                 ):
 

src/debugpy/_vendored/pydevd/tests_python/test_debugger_json.py

@@ -19,7 +19,8 @@
 from _pydevd_bundle.pydevd_comm_constants import file_system_encoding
 from _pydevd_bundle.pydevd_constants import (int_types, IS_64BIT_PROCESS,
     PY_VERSION_STR, PY_IMPL_VERSION_STR, PY_IMPL_NAME, IS_PY36_OR_GREATER,
-    IS_PYPY, GENERATED_LEN_ATTR_NAME, IS_WINDOWS, IS_LINUX, IS_MAC, IS_PY38_OR_GREATER)
+    IS_PYPY, GENERATED_LEN_ATTR_NAME, IS_WINDOWS, IS_LINUX, IS_MAC, IS_PY38_OR_GREATER,
+    IS_PY311_OR_GREATER)
 from tests_python import debugger_unittest
 from tests_python.debug_constants import TEST_CHERRYPY, TEST_DJANGO, TEST_FLASK, \
     IS_CPYTHON, TEST_GEVENT, TEST_CYTHON, TODO_PY311
@@ -826,6 +827,8 @@ def additional_output_checks(writer, stdout, stderr):
         body = exc_info_response.body
         assert body.exceptionId == 'Exception'
         assert body.description == 'TEST SUCEEDED'
+        if IS_PY311_OR_GREATER:
+            assert '^^^^' in body.details.stackTrace
         assert normcase(body.details.kwargs['source']) == normcase(writer.TEST_FILE)
 
         # Check that we have the exception cause in the stack trace.