Merge pull request #1758 from fabioz/dbg_adapter_refactor_merge

Merging master > dbg_adapter_refactor.

src/ptvsd/_vendored/pydevd/.travis.yml

@@ -20,102 +20,115 @@ matrix:
     - python: 2.7
       env:
         - PYDEVD_USE_CYTHON=NO
-        - PYDEVD_TEST_JYTHON=YES
+        - PYDEVD_TEST_VM=JYTHON
         - JYTHON_URL=http://search.maven.org/remotecontent?filepath=org/python/jython-installer/2.7.1/jython-installer-2.7.1.jar
+
+    # PyPy
+    - python: 2.7
+      env:
+        - PYDEVD_PYTHON_VERSION=3.6
+        - PYDEVD_USE_CYTHON=NO
+        - PYDEVD_TEST_VM=PYPY
 
     # Python 2.6 (with and without cython)
     - python: 2.7
       env:
         - PYDEVD_PYTHON_VERSION=2.6
         - PYDEVD_USE_CYTHON=NO
-        - PYDEVD_TEST_JYTHON=NO
+        - PYDEVD_TEST_VM=CPYTHON
 #     - python: 2.7
 #       env:
 #         - PYDEVD_PYTHON_VERSION=2.6
 #         - PYDEVD_USE_CYTHON=YES
-#         - PYDEVD_TEST_JYTHON=NO
+#         - PYDEVD_TEST_VM=CPYTHON
 
     # Python 2.7 (with and without cython)
     - python: 2.7
-      env:
-        - PYDEVD_PYTHON_VERSION=2.7
-        - PYDEVD_USE_CYTHON=NO
-        - PYDEVD_TEST_JYTHON=NO
+#       env:
+#         - PYDEVD_PYTHON_VERSION=2.7
+#         - PYDEVD_USE_CYTHON=NO
+#         - PYDEVD_TEST_VM=CPYTHON
     - python: 2.7
       env:
         - PYDEVD_PYTHON_VERSION=2.7
         - PYDEVD_USE_CYTHON=YES
-        - PYDEVD_TEST_JYTHON=NO
+        - PYDEVD_TEST_VM=CPYTHON
 
     # Python 3.5 (with and without cython)
 #     - python: 2.7
 #       env:
 #         - PYDEVD_PYTHON_VERSION=3.5
 #         - PYDEVD_USE_CYTHON=NO
-#         - PYDEVD_TEST_JYTHON=NO
+#         - PYDEVD_TEST_VM=CPYTHON
     - python: 2.7
       env:
         - PYDEVD_PYTHON_VERSION=3.5
         - PYDEVD_USE_CYTHON=YES
-        - PYDEVD_TEST_JYTHON=NO
+        - PYDEVD_TEST_VM=CPYTHON
 
     # Python 3.6 (with and without cython)
-#     - python: 2.7
-#       env:
-#         - PYDEVD_PYTHON_VERSION=3.6
-#         - PYDEVD_USE_CYTHON=NO
-#         - PYDEVD_TEST_JYTHON=NO
     - python: 2.7
       env:
         - PYDEVD_PYTHON_VERSION=3.6
-        - PYDEVD_USE_CYTHON=YES
-        - PYDEVD_TEST_JYTHON=NO
+        - PYDEVD_USE_CYTHON=NO
+        - PYDEVD_TEST_VM=CPYTHON
+#     - python: 2.7
+#       env:
+#         - PYDEVD_PYTHON_VERSION=3.6
+#         - PYDEVD_USE_CYTHON=YES
+#         - PYDEVD_TEST_VM=CPYTHON
 
     # Python 3.7 (with and without cython)
-    - python: 2.7
-      env:
-        - PYDEVD_PYTHON_VERSION=3.7
-        - PYDEVD_USE_CYTHON=NO
-        - PYDEVD_TEST_JYTHON=NO
 #     - python: 2.7
 #       env:
 #         - PYDEVD_PYTHON_VERSION=3.7
-#         - PYDEVD_USE_CYTHON=YES
-#         - PYDEVD_TEST_JYTHON=NO
+#         - PYDEVD_USE_CYTHON=NO
+#         - PYDEVD_TEST_VM=CPYTHON
+    - python: 2.7
+      env:
+        - PYDEVD_PYTHON_VERSION=3.7
+        - PYDEVD_USE_CYTHON=YES
+        - PYDEVD_TEST_VM=CPYTHON
 
 before_install:
-  # CPython setup
-  - if [ "$PYDEVD_TEST_JYTHON" == "NO" ]; then wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh; fi
-  - if [ "$PYDEVD_TEST_JYTHON" == "NO" ]; then chmod +x miniconda.sh; fi
-  - if [ "$PYDEVD_TEST_JYTHON" == "NO" ]; then ./miniconda.sh -b; fi
-  - if [ "$PYDEVD_TEST_JYTHON" == "NO" ]; then export PATH=/home/travis/miniconda2/bin:$PATH; fi
-  - if [ "$PYDEVD_TEST_JYTHON" == "NO" ]; then conda update --yes conda; fi
+  # CPython / Pypy setup
+  - if [[ "$PYDEVD_TEST_VM" == "CPYTHON" || "$PYDEVD_TEST_VM" == "PYPY" ]]; then wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh; fi
+  - if [[ "$PYDEVD_TEST_VM" == "CPYTHON" || "$PYDEVD_TEST_VM" == "PYPY" ]]; then chmod +x miniconda.sh; fi
+  - if [[ "$PYDEVD_TEST_VM" == "CPYTHON" || "$PYDEVD_TEST_VM" == "PYPY" ]]; then ./miniconda.sh -b; fi
+  - if [[ "$PYDEVD_TEST_VM" == "CPYTHON" || "$PYDEVD_TEST_VM" == "PYPY" ]]; then export PATH=/home/travis/miniconda2/bin:$PATH; fi
+  - if [[ "$PYDEVD_TEST_VM" == "CPYTHON" || "$PYDEVD_TEST_VM" == "PYPY" ]]; then conda update --yes conda; fi
   # Jython setup
-  - if [ "$PYDEVD_TEST_JYTHON" == "YES" ]; then wget $JYTHON_URL -O jython_installer.jar; java -jar jython_installer.jar -s -d $HOME/jython; export PATH=$HOME/jython:$HOME/jython/bin:$PATH; fi
-  - if [ "$PYDEVD_TEST_JYTHON" == "YES" ]; then jython -c "print('')"; fi
+  - if [ "$PYDEVD_TEST_VM" == "JYTHON" ]; then wget $JYTHON_URL -O jython_installer.jar; java -jar jython_installer.jar -s -d $HOME/jython; export PATH=$HOME/jython:$HOME/jython/bin:$PATH; fi
+  - if [ "$PYDEVD_TEST_VM" == "JYTHON" ]; then jython -c "print('')"; fi
   # Fix issue with testGui
   - "export DISPLAY=:99.0"
 # Install packages
 install:
   - sudo sysctl kernel.yama.ptrace_scope=0
   # Both
   - export PYTHONPATH=.
-  # Python setup
-  - if [ "$PYDEVD_TEST_JYTHON" == "NO" ]; then conda create --yes -n build_env python=$PYDEVD_PYTHON_VERSION; fi
-  - if [ "$PYDEVD_TEST_JYTHON" == "NO" ]; then source activate build_env; fi
-  - if [ "$PYDEVD_TEST_JYTHON" == "NO" ]; then chmod +x ./.travis_install_python_deps.sh; fi
-  - if [ "$PYDEVD_TEST_JYTHON" == "NO" ]; then ./.travis_install_python_deps.sh; fi
-  - if [ "$PYDEVD_TEST_JYTHON" == "NO" ]; then source activate build_env; python build_tools/build.py; fi
+  # CPython setup
+  - if [ "$PYDEVD_TEST_VM" == "CPYTHON" ]; then conda create --yes -n build_env python=$PYDEVD_PYTHON_VERSION; fi
+  - if [ "$PYDEVD_TEST_VM" == "CPYTHON" ]; then source activate build_env; fi
+  - if [ "$PYDEVD_TEST_VM" == "CPYTHON" ]; then chmod +x ./.travis_install_python_deps.sh; fi
+  - if [ "$PYDEVD_TEST_VM" == "CPYTHON" ]; then ./.travis_install_python_deps.sh; fi
+  - if [ "$PYDEVD_TEST_VM" == "CPYTHON" ]; then source activate build_env; python build_tools/build.py; fi
+  # Pypy setup
+  - if [ "$PYDEVD_TEST_VM" == "PYPY" ]; then conda create --yes -n build_env -c conda-forge pypy3.6; fi
+  - if [ "$PYDEVD_TEST_VM" == "PYPY" ]; then source activate build_env; fi
+  - if [ "$PYDEVD_TEST_VM" == "PYPY" ]; then chmod +x ./.travis_install_pypy_deps.sh; fi
+  - if [ "$PYDEVD_TEST_VM" == "PYPY" ]; then ./.travis_install_pypy_deps.sh; fi
   # Jython setup
-  - if [ "$PYDEVD_TEST_JYTHON" == "YES" ]; then chmod +x ./.travis_install_jython_deps.sh; fi
-  - if [ "$PYDEVD_TEST_JYTHON" == "YES" ]; then ./.travis_install_jython_deps.sh; fi
+  - if [ "$PYDEVD_TEST_VM" == "JYTHON" ]; then chmod +x ./.travis_install_jython_deps.sh; fi
+  - if [ "$PYDEVD_TEST_VM" == "JYTHON" ]; then ./.travis_install_jython_deps.sh; fi
 
 # Run test
 # On local machine with jython: c:\bin\jython2.7.0\bin\jython.exe -Dpython.path=.;jython_test_deps/ant.jar;jython_test_deps/junit.jar -m pytest
 # On remove machine with python: c:\bin\python27\python.exe -m pytest
 script:
   # pytest-xdist not available for python == 2.6 and timing out without output with 2.7
-  - if [[ ("$PYDEVD_TEST_JYTHON" == "NO") && ("$PYDEVD_PYTHON_VERSION" == "2.6" || "$PYDEVD_PYTHON_VERSION" == "2.7") ]]; then source activate build_env; python -m pytest; fi
-  - if [[ ("$PYDEVD_TEST_JYTHON" == "NO") && ("$PYDEVD_PYTHON_VERSION" != "2.6" && "$PYDEVD_PYTHON_VERSION" != "2.7") ]]; then source activate build_env; python -m pytest -n auto; fi
-  - if [ "$PYDEVD_TEST_JYTHON" == "YES" ]; then jython -Dpython.path=.:jython_test_deps/ant.jar:jython_test_deps/junit.jar -m pytest; fi
+  - if [[ ("$PYDEVD_TEST_VM" == "CPYTHON") && ("$PYDEVD_PYTHON_VERSION" == "2.6" || "$PYDEVD_PYTHON_VERSION" == "2.7") ]]; then source activate build_env; python -m pytest; fi
+  - if [[ ("$PYDEVD_TEST_VM" == "CPYTHON") && ("$PYDEVD_PYTHON_VERSION" != "2.6" && "$PYDEVD_PYTHON_VERSION" != "2.7") ]]; then source activate build_env; python -m pytest -n auto; fi
+  - if [ "$PYDEVD_TEST_VM" == "PYPY" ]; then source activate build_env; pypy3 -m pytest -n auto; fi
+  - if [ "$PYDEVD_TEST_VM" == "JYTHON" ]; then jython -Dpython.path=.:jython_test_deps/ant.jar:jython_test_deps/junit.jar -m pytest; fi
 

src/ptvsd/_vendored/pydevd/.travis_install_pypy_deps.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+set -ev
+
+source activate build_env
+
+pypy3 -m ensurepip
+
+pypy3 -m pip install pytest
+pypy3 -m pip install pytest-xdist
+pypy3 -m pip install pytest-timeout
+pypy3 -m pip install colorama
+pypy3 -m pip install psutil
+pypy3 -m pip install numpy
+pypy3 -m pip install ipython
+pypy3 -m pip install untangle

src/ptvsd/_vendored/pydevd/_pydev_bundle/pydev_console_utils.py

@@ -6,8 +6,11 @@
 from _pydev_imps._pydev_saved_modules import thread
 from _pydevd_bundle import pydevd_vars
 from _pydevd_bundle import pydevd_xml
-from _pydevd_bundle.pydevd_constants import IS_JYTHON, dict_iter_items, NEXT_VALUE_SEPARATOR, Null
+from _pydevd_bundle.pydevd_constants import (IS_JYTHON, dict_iter_items, NEXT_VALUE_SEPARATOR, Null,
+    get_global_debugger)
 import signal
+from contextlib import contextmanager
+from _pydev_bundle import pydev_log
 
 try:
     import cStringIO as StringIO  # may not always be available @UnusedImport
@@ -109,31 +112,44 @@ class DebugConsoleStdIn(BaseStdIn):
         Object to be added to stdin (to emulate it as non-blocking while the next line arrives)
     '''
 
-    def __init__(self, dbg, original_stdin):
+    def __init__(self, py_db, original_stdin):
+        '''
+        :param py_db:
+            If None, get_global_debugger() is used.
+        '''
         BaseStdIn.__init__(self, original_stdin)
-        self.debugger = dbg
+        self._py_db = py_db
+        self._in_notification = 0
 
-    def __pydev_run_command(self, is_started):
+    def __send_input_requested_message(self, is_started):
         try:
-            cmd = self.debugger.cmd_factory.make_input_requested_message(is_started)
-            self.debugger.writer.add_command(cmd)
+            py_db = self._py_db
+            if py_db is None:
+                py_db = get_global_debugger()
+            cmd = py_db.cmd_factory.make_input_requested_message(is_started)
+            py_db.writer.add_command(cmd)
         except Exception:
-            traceback.print_exc()
-            return '\n'
+            pydev_log.exception()
+
+    @contextmanager
+    def notify_input_requested(self):
+        self._in_notification += 1
+        if self._in_notification == 1:
+            self.__send_input_requested_message(True)
+        try:
+            yield
+        finally:
+            self._in_notification -= 1
+            if self._in_notification == 0:
+                self.__send_input_requested_message(False)
 
     def readline(self, *args, **kwargs):
-        # Notify Java side about input and call original function
-        self.__pydev_run_command(True)
-        result = self.original_stdin.readline(*args, **kwargs)
-        self.__pydev_run_command(False)
-        return result
+        with self.notify_input_requested():
+            return self.original_stdin.readline(*args, **kwargs)
 
     def read(self, *args, **kwargs):
-        # Notify Java side about input and call original function
-        self.__pydev_run_command(True)
-        result = self.original_stdin.read(*args, **kwargs)
-        self.__pydev_run_command(False)
-        return result
+        with self.notify_input_requested():
+            return self.original_stdin.read(*args, **kwargs)
 
 
 class CodeFragment:
@@ -220,7 +236,7 @@ def create_std_in(self, debugger=None, original_std_in=None):
         if debugger is None:
             return StdIn(self, self.host, self.client_port, original_stdin=original_std_in)
         else:
-            return DebugConsoleStdIn(dbg=debugger, original_stdin=original_std_in)
+            return DebugConsoleStdIn(py_db=debugger, original_stdin=original_std_in)
 
     def add_exec(self, code_fragment, debugger=None):
         # In case sys.excepthook called, use original excepthook #PyDev-877: Debug console freezes with Python 3.5+

src/ptvsd/_vendored/pydevd/_pydev_bundle/pydev_log.py

@@ -43,7 +43,14 @@ def _pydevd_log(level, msg, *args):
                     msg = msg % args
             except:
                 msg = '%s - %s' % (msg, args)
-            DebugInfoHolder.DEBUG_STREAM.write('%s\n' % (msg,))
+            msg = '%s\n' % (msg,)
+            try:
+                DebugInfoHolder.DEBUG_STREAM.write(msg)
+            except TypeError:
+                if isinstance(msg, bytes):
+                    # Depending on the StringIO flavor, it may only accept unicode.
+                    msg = msg.decode('utf-8', 'replace')
+                    DebugInfoHolder.DEBUG_STREAM.write(msg)
             DebugInfoHolder.DEBUG_STREAM.flush()
         except:
             pass

src/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_additional_thread_info_regular.py

@@ -1,5 +1,6 @@
 import sys
-from _pydevd_bundle.pydevd_constants import STATE_RUN, PYTHON_SUSPEND, IS_JYTHON, IS_IRONPYTHON
+from _pydevd_bundle.pydevd_constants import (STATE_RUN, PYTHON_SUSPEND, IS_JYTHON,
+    USE_CUSTOM_SYS_CURRENT_FRAMES, USE_CUSTOM_SYS_CURRENT_FRAMES_MAP)
 from _pydev_bundle import pydev_log
 # IFDEF CYTHON
 # pydev_log.debug("Using Cython speedups")
@@ -9,7 +10,7 @@
 
 version = 11
 
-if not hasattr(sys, '_current_frames'):
+if USE_CUSTOM_SYS_CURRENT_FRAMES:
 
     # Some versions of Jython don't have it (but we can provide a replacement)
     if IS_JYTHON:
@@ -40,7 +41,7 @@ def _current_frames():
                 ret[thread.getId()] = frame
             return ret
 
-    elif IS_IRONPYTHON:
+    elif USE_CUSTOM_SYS_CURRENT_FRAMES_MAP:
         _tid_to_last_frame = {}
 
         # IronPython doesn't have it. Let's use our workaround...

src/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_comm.py

@@ -1155,8 +1155,11 @@ def build_exception_info_response(dbg, thread_id, request_seq, set_additional_th
                         break
                 frame = frame.f_back
 
-            while trace_obj.tb_next is not None:
-                trace_obj = trace_obj.tb_next
+            while True:
+                tb_next = getattr(trace_obj, 'tb_next', None)
+                if tb_next is None:
+                    break
+                trace_obj = tb_next
 
             info = dbg.suspended_frames_manager.get_topmost_frame_and_frame_id_to_line(thread_id)
             if info is not None:

src/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_console.py

@@ -23,6 +23,7 @@
 class ConsoleMessage:
     """Console Messages
     """
+
     def __init__(self):
         self.more = False
         # List of tuple [('error', 'error_message'), ('message_list', 'output_message')]
@@ -62,15 +63,16 @@ def to_xml(self):
 
 
 #=======================================================================================================================
-# DebugConsoleStdIn
+# _DebugConsoleStdIn
 #=======================================================================================================================
-class DebugConsoleStdIn(BaseStdIn):
+class _DebugConsoleStdIn(BaseStdIn):
 
     @overrides(BaseStdIn.readline)
     def readline(self, *args, **kwargs):
         sys.stderr.write('Warning: Reading from stdin is still not supported in this console.\n')
         return '\n'
 
+
 #=======================================================================================================================
 # DebugConsole
 #=======================================================================================================================
@@ -87,8 +89,7 @@ def create_std_in(self, *args, **kwargs):
         except:
             pass
 
-        return DebugConsoleStdIn() #If buffered, raw_input is not supported in this console.
-
+        return _DebugConsoleStdIn()  # If buffered, raw_input is not supported in this console.
 
     @overrides(InteractiveConsole.push)
     def push(self, line, frame, buffer_output=True):
@@ -121,7 +122,7 @@ def push(self, line, frame, buffer_output=True):
                 else:
                     sys.stderr.write("Internal Error: %s\n" % (exc,))
         finally:
-            #Remove frame references.
+            # Remove frame references.
             self.frame = None
             frame = None
             if buffer_output:
@@ -133,12 +134,10 @@ def push(self, line, frame, buffer_output=True):
         else:
             return more, [], []
 
-
     @overrides(BaseInterpreterInterface.do_add_exec)
     def do_add_exec(self, line):
         return InteractiveConsole.push(self, line)
 
-
     @overrides(InteractiveConsole.runcode)
     def runcode(self, code):
         """Execute a code object.
@@ -183,7 +182,7 @@ class InteractiveConsoleCache:
     interactive_console_instance = None
 
 
-#Note: On Jython 2.1 we can't use classmethod or staticmethod, so, just make the functions below free-functions.
+# Note: On Jython 2.1 we can't use classmethod or staticmethod, so, just make the functions below free-functions.
 def get_interactive_console(thread_id, frame_id, frame, console_message):
     """returns the global interactive console.
     interactive console should have been initialized by this time
@@ -198,7 +197,7 @@ def get_interactive_console(thread_id, frame_id, frame, console_message):
 
     console_stacktrace = traceback.extract_stack(frame, limit=1)
     if console_stacktrace:
-        current_context = console_stacktrace[0] # top entry from stacktrace
+        current_context = console_stacktrace[0]  # top entry from stacktrace
         context_message = 'File "%s", line %s, in %s' % (current_context[0], current_context[1], current_context[2])
         console_message.add_console_message(CONSOLE_OUTPUT, "[Current context]: %s" % (context_message,))
     return InteractiveConsoleCache.interactive_console_instance
@@ -247,7 +246,3 @@ def get_completions(frame, act_tok):
     """
     return _pydev_completer.generate_completions_as_xml(frame, act_tok)
 
-
-
-
-