Just my code (#467)

* Add set debugger property to support VS just my code

* Just my code initial code

* Limit continue to stepping and exceptions

* Improve skipping non-user code

* Fix tests

* Just my code shoudl skip ptvsd files

* fixes and updates (#1)

* Fix typo

* Use CMD_STEP_INTO_MY_CODE and CMD_SET_PROJECT_ROOTS

* Update tests to reflect the new pydevd exception breakpoint cmd args

* Fixes and improvements

* Simplify and refactor debug options

* Test fix and fix for debug property request handler for VS

* More fixes

* Test fixes

* Address comments

* Bunch of test fixes (#2)

* Test fixes

* Restoring auto-continue till pydev side work is done.

debugger_protocol/messages/_requests.py

@@ -82,6 +82,7 @@ class Capabilities(FieldsNamespace):
         Field('supportsLoadedSourcesRequest', bool),
         Field('supportsSetExpression', bool),
         Field('supportsModulesRequest', bool),
+        Field('supportsDebuggerProperties', bool),
     ]
 
 

ptvsd/wrapper.py

@@ -11,6 +11,7 @@
 import platform
 import pydevd_file_utils
 import re
+import site
 import socket
 import sys
 import threading
@@ -111,42 +112,41 @@ def using_format(self, fmt):
 str_handlers = pydevd_extutil.EXTENSION_MANAGER_INSTANCE.type_to_instance.setdefault(pydevd_extapi.StrPresentationProvider, [])  # noqa
 str_handlers.insert(0, SafeReprPresentationProvider._instance)
 
-DONT_TRACE_FILES = set((os.path.join(*elem) for elem in [
-    ('ptvsd', 'attach_server.py'),
-    ('ptvsd', 'daemon.py'),
-    ('ptvsd', 'debugger.py'),
-    ('ptvsd', 'futures.py'),
-    ('ptvsd', 'ipcjson.py'),
-    ('ptvsd', 'pathutils.py'),
-    ('ptvsd', 'pydevd_hooks.py'),
-    ('ptvsd', 'reraise.py'),
-    ('ptvsd', 'reraise2.py'),
-    ('ptvsd', 'reraise3.py'),
-    ('ptvsd', 'runner.py'),
-    ('ptvsd', 'safe_repr.py'),
-    ('ptvsd', 'socket.py'),
-    ('ptvsd', 'untangle.py'),
-    ('ptvsd', 'version.py'),
-    ('ptvsd', 'wrapper.py'),
-    ('ptvsd', '_main.py'),
-    ('ptvsd', '_version.py'),
-    ('ptvsd', '__init__.py'),
-    ('ptvsd', '__main__.py'),
-]))
-
-
-def filter_ptvsd_files(file_path):
+PTVSD_DIR_PATH = os.path.dirname(os.path.abspath(__file__)) + os.path.sep
+NORM_PTVSD_DIR_PATH = os.path.normcase(PTVSD_DIR_PATH)
+
+
+def dont_trace_ptvsd_files(file_path):
     """
     Returns true if the file should not be traced.
     """
-    ptvsd_path_re = r"(ptvsd[\\\/].*\.py)"
-    matches = re.finditer(ptvsd_path_re, file_path)
-    return any((g in DONT_TRACE_FILES
-                for m in matches
-                for g in m.groups()))
+    return file_path.startswith(PTVSD_DIR_PATH)
+
 
+pydevd_frame.file_tracing_filter = dont_trace_ptvsd_files
 
-pydevd_frame.file_tracing_filter = filter_ptvsd_files
+
+STDLIB_PATH_PREFIXES = [os.path.normcase(sys.prefix)]
+if hasattr(sys, 'base_prefix'):
+    STDLIB_PATH_PREFIXES.append(os.path.normcase(sys.base_prefix))
+if hasattr(sys, 'real_prefix'):
+    STDLIB_PATH_PREFIXES.append(os.path.normcase(sys.real_prefix))
+
+if hasattr(site, 'getusersitepackages'):
+    site_paths = site.getusersitepackages()
+    if isinstance(site_paths, list):
+        for site_path in site_paths:
+            STDLIB_PATH_PREFIXES.append(os.path.normcase(site_path))
+    else:
+        STDLIB_PATH_PREFIXES.append(os.path.normcase(site_paths))
+
+if hasattr(site, 'getsitepackages'):
+    site_paths = site.getsitepackages()
+    if isinstance(site_paths, list):
+        for site_path in site_paths:
+            STDLIB_PATH_PREFIXES.append(os.path.normcase(site_path))
+    else:
+        STDLIB_PATH_PREFIXES.append(os.path.normcase(site_paths))
 
 
 class UnsupportedPyDevdCommandError(Exception):
@@ -446,28 +446,17 @@ def get_break_mode(self, name):
                 pass
         return 'unhandled'
 
-    def add_exception_break(self, exception, break_raised, break_uncaught):
-
-        # notify_always options:
-        #   1 is deprecated, you will see a warning message
-        #   2 notify on first raise only
-        #   3 or greater, notify always
-        notify_always = 3 if break_raised else 0
+    def add_exception_break(self, exception, break_raised, break_uncaught,
+                            skip_stdlib=False):
 
-        # notify_on_terminate options:
-        #   1 notify on terminate
-        #   Any other value do NOT notify on terminate
-        notify_on_terminate = 1 if break_uncaught else 0
+        notify_on_handled_exceptions = 1 if break_raised else 0
+        notify_on_unhandled_exceptions = 1 if break_uncaught else 0
+        ignore_libraries = 1 if skip_stdlib else 0
 
-        # ignore_libraries options:
-        #   Less than or equal to 0 DO NOT ignore libraries (required
-        #   for notify_always)
-        #   Greater than 0 ignore libraries
-        ignore_libraries = 0
         cmdargs = (
             exception,
-            notify_always,
-            notify_on_terminate,
+            notify_on_handled_exceptions,
+            notify_on_unhandled_exceptions,
             ignore_libraries,
         )
         break_mode = 'never'
@@ -482,7 +471,7 @@ def add_exception_break(self, exception, break_raised, break_uncaught):
                 pydevd_comm.CMD_ADD_EXCEPTION_BREAK, msg)
             self.exceptions[exception] = break_mode
 
-    def apply_exception_options(self, exception_options):
+    def apply_exception_options(self, exception_options, skip_stdlib=False):
         """
         Applies exception options after removing any existing exception
         breaks.
@@ -508,7 +497,7 @@ def apply_exception_options(self, exception_options):
                     is_category = True
             if is_category:
                 self.add_exception_break(
-                    'BaseException', break_raised, break_uncaught)
+                    'BaseException', break_raised, break_uncaught, skip_stdlib)
             else:
                 path_iterator = iter(exception_paths)
                 # Skip the first one. It will always be the category
@@ -520,7 +509,8 @@ def apply_exception_options(self, exception_options):
                         exception_names.append(ex_name)
                 for exception_name in exception_names:
                     self.add_exception_break(
-                        exception_name, break_raised, break_uncaught)
+                        exception_name, break_raised,
+                        break_uncaught, skip_stdlib)
 
     def _is_python_exception_category(self, option):
         """
@@ -794,6 +784,7 @@ def _parse_debug_options(opts):
 
 # TODO: Embed instead of extend (inheritance -> composition).
 
+
 class VSCodeMessageProcessorBase(ipcjson.SocketIO, ipcjson.IpcChannel):
     """The base class for VSC message processors."""
 
@@ -896,17 +887,18 @@ def _stop_event_loop(self):
 
 
 INITIALIZE_RESPONSE = dict(
-    supportsExceptionInfoRequest=True,
-    supportsConfigurationDoneRequest=True,
     supportsConditionalBreakpoints=True,
+    supportsConfigurationDoneRequest=True,
+    supportsDebuggerProperties=True,
+    supportsEvaluateForHovers=True,
+    supportsExceptionInfoRequest=True,
+    supportsExceptionOptions=True,
     supportsHitConditionalBreakpoints=True,
+    supportsLogPoints=True,
+    supportsModulesRequest=True,
+    supportsSetExpression=True,
     supportsSetVariable=True,
-    supportsExceptionOptions=True,
-    supportsEvaluateForHovers=True,
     supportsValueFormattingOptions=True,
-    supportsSetExpression=True,
-    supportsModulesRequest=True,
-    supportsLogPoints=True,
     supportTerminateDebuggee=True,
     exceptionBreakpointFilters=[
         {
@@ -1224,6 +1216,51 @@ def _process_debug_options(self, opts):
         else:
             redirect_output = ''
         self.pydevd_request(pydevd_comm.CMD_REDIRECT_OUTPUT, redirect_output)
+        self._apply_code_stepping_settings()
+
+    def _is_just_my_code_stepping_enabled(self):
+        """Returns true if just-me-code stepping is enabled.
+
+        Note: for now we consider DEBUG_STDLIB = False as just-my-code.
+        """
+        dbg_stdlib = self.debug_options.get('DEBUG_STDLIB', False)
+        return not dbg_stdlib
+
+    def _apply_code_stepping_settings(self):
+        if self._is_just_my_code_stepping_enabled():
+            vendored_pydevd = os.path.sep + \
+                              os.path.join('ptvsd', '_vendored', 'pydevd')
+            ptvsd_path = os.path.sep + 'ptvsd'
+
+            project_dirs = []
+            for path in sys.path + [os.getcwd()]:
+                is_stdlib = False
+                norm_path = os.path.normcase(path)
+                if path.endswith(ptvsd_path) or \
+                   path.endswith(vendored_pydevd):
+                    is_stdlib = True
+                else:
+                    for prefix in STDLIB_PATH_PREFIXES:
+                        if norm_path.startswith(prefix):
+                            is_stdlib = True
+                            break
+
+                if not is_stdlib and len(path) > 0:
+                    project_dirs.append(path)
+            self.pydevd_request(pydevd_comm.CMD_SET_PROJECT_ROOTS,
+                                '\t'.join(project_dirs))
+
+    def _is_stdlib(self, filepath):
+        for prefix in STDLIB_PATH_PREFIXES:
+            if prefix != '' and filepath.startswith(prefix):
+                return True
+        return filepath.startswith(NORM_PTVSD_DIR_PATH)
+
+    def _should_debug(self, filepath):
+        if self._is_just_my_code_stepping_enabled() and \
+           self._is_stdlib(filepath):
+            return False
+        return True
 
     def _initialize_path_maps(self, args):
         pathMaps = []
@@ -1783,7 +1820,10 @@ def on_next(self, request, args):
     def on_stepIn(self, request, args):
         # TODO: docstring
         tid = self.thread_map.to_pydevd(int(args['threadId']))
-        self.pydevd_notify(pydevd_comm.CMD_STEP_INTO, tid)
+        if self._is_just_my_code_stepping_enabled():
+            self.pydevd_notify(pydevd_comm.CMD_STEP_INTO_MY_CODE, tid)
+        else:
+            self.pydevd_notify(pydevd_comm.CMD_STEP_INTO, tid)
         self.send_response(request)
 
     @async_handler
@@ -1895,16 +1935,19 @@ def on_setExceptionBreakpoints(self, request, args):
         # TODO: docstring
         filters = args['filters']
         exception_options = args.get('exceptionOptions', [])
+        jmc = self._is_just_my_code_stepping_enabled()
 
         if exception_options:
-            self.exceptions_mgr.apply_exception_options(exception_options)
+            self.exceptions_mgr.apply_exception_options(
+                exception_options, jmc)
         else:
             self.exceptions_mgr.remove_all_exception_breaks()
             break_raised = 'raised' in filters
             break_uncaught = 'uncaught' in filters
             if break_raised or break_uncaught:
                 self.exceptions_mgr.add_exception_break(
-                    'BaseException', break_raised, break_uncaught)
+                    'BaseException', break_raised, break_uncaught,
+                    skip_stdlib=jmc)
         self.send_response(request)
 
     @async_handler
@@ -1985,6 +2028,16 @@ def version_str(v):
         }
         self.send_response(request, **sys_info)
 
+    # VS specific custom message handlers
+    @async_handler
+    def on_setDebuggerProperty(self, request, args):
+        if 'JustMyCodeStepping' in args:
+            jmc = int(args.get('JustMyCodeStepping', 0)) > 0
+            self.debug_options['DEBUG_STDLIB'] = not jmc
+            self._apply_code_stepping_settings()
+
+        self.send_response(request)
+
     # PyDevd protocol event handlers
 
     @pydevd_events.handler(pydevd_comm.CMD_THREAD_CREATE)
@@ -2029,16 +2082,25 @@ def on_pydevd_thread_suspend(self, seq, args):
                 pydevd_comm.CMD_STEP_INTO,
                 pydevd_comm.CMD_STEP_OVER,
                 pydevd_comm.CMD_STEP_RETURN,
+                pydevd_comm.CMD_STEP_INTO_MY_CODE,
         }
         EXCEPTION_REASONS = {
             pydevd_comm.CMD_STEP_CAUGHT_EXCEPTION,
             pydevd_comm.CMD_ADD_EXCEPTION_BREAK
         }
 
-        try:
-            vsc_tid = self.thread_map.to_vscode(pyd_tid, autogen=False)
-        except KeyError:
-            return
+        # This is needed till https://github.com/Microsoft/ptvsd/issues/477
+        # is done. Remove this after adding the appropriate pydevd commands to
+        # do step over and step out
+        xframes = list(xml.thread.frame)
+        xframe = xframes[0]
+        filepath = unquote(xframe['file'])
+        if reason in STEP_REASONS or reason in EXCEPTION_REASONS:
+            if not self._should_debug(filepath):
+                self.pydevd_notify(pydevd_comm.CMD_THREAD_RUN, pyd_tid)
+                return
+
+        vsc_tid = self.thread_map.to_vscode(pyd_tid, autogen=False)
 
         with self.stack_traces_lock:
             self.stack_traces[pyd_tid] = xml.thread.frame
@@ -2061,8 +2123,6 @@ def on_pydevd_thread_suspend(self, seq, args):
         if reason == 'exception':
             # Get exception info from frame
             try:
-                xframes = list(xml.thread.frame)
-                xframe = xframes[0]
                 pyd_fid = xframe['id']
                 cmdargs = '{}\t{}\tFRAME\t__exception__'.format(pyd_tid,
                                                                 pyd_fid)

tests/highlevel/__init__.py

@@ -17,6 +17,7 @@
     CMD_SEND_CURR_EXCEPTION_TRACE,
     CMD_THREAD_CREATE,
     CMD_GET_VARIABLE,
+    CMD_SET_PROJECT_ROOTS,
 )
 
 from ptvsd import wrapper
@@ -139,8 +140,9 @@ def __init__(self, fix):
     @contextlib.contextmanager
     def _wait_for_initialized(self):
         with self._fix.wait_for_command(CMD_REDIRECT_OUTPUT):
-            with self._fix.wait_for_command(CMD_RUN):
-                yield
+            with self._fix.wait_for_command(CMD_SET_PROJECT_ROOTS):
+                with self._fix.wait_for_command(CMD_RUN):
+                    yield
 
     def _initialize(self):
         version = self._fix.fake.VERSION