Deal with __future__ imports on python -c subprocess. Fixes microsoft…

…#642

src/debugpy/_vendored/pydevd/_pydev_bundle/pydev_monkey.py

@@ -9,6 +9,7 @@
 from contextlib import contextmanager
 from _pydevd_bundle import pydevd_constants
 from _pydevd_bundle.pydevd_defaults import PydevdCustomization
+import ast
 
 try:
     xrange
@@ -75,16 +76,90 @@ def _get_setup_updated_with_protocol_and_ppid(setup, is_exec=False):
     return setup
 
 
+class _LastFutureImportFinder(ast.NodeVisitor):
+
+    def __init__(self):
+        self._last_future_import_found = None
+
+    def visit_ImportFrom(self, node):
+        if node.module == '__future__':
+            self._last_future_import_found = node
+
+    def get_last_future_import_end_line_col(self):
+        if self._last_future_import_found is None:
+            return None
+
+        node = self._last_future_import_found
+        return node.end_lineno, node.end_col_offset
+
+
+def _separate_future_imports(code):
+    '''
+    :param code:
+        The code from where we want to get the __future__ imports (note that it's possible that
+        there's no such entry).
+
+    :return tuple(str, str):
+        The return is a tuple(future_import, code).
+
+        If the future import is not available a return such as ('', code) is given, otherwise, the
+        future import will end with a ';' (so that it can be put right before the pydevd attach
+        code).
+    '''
+    try:
+        node = ast.parse(code, '<string>', 'exec')
+        visitor = _LastFutureImportFinder()
+        visitor.visit(node)
+        line_col = visitor.get_last_future_import_end_line_col()
+        if line_col is None:
+            return '', code
+
+        line, col = line_col
+        line -= 1  # ast lines are 1-based, make it 0-based.
+        offset = 0
+        for i, line_contents in enumerate(code.splitlines(True)):
+            if i == line:
+                for i in range(col, len(line_contents)):
+                    if line_contents[i] in (' ', '\t', ';', ')'):
+                        col += 1
+                    else:
+                        break
+
+                offset += col
+                future_import = code[:offset]
+                code_remainder = code[offset:]
+                if not future_import.endswith(';'):
+                    future_import += ';'
+                return future_import, code_remainder
+            else:
+                offset += len(line_contents)
+
+        # This shouldn't happen...
+        pydev_log.info('Unable to find line %s in code:\n%r', line, code)
+        return '', code
+
+    except:
+        pydev_log.exception('Error getting from __future__ imports from: %r', code)
+        return '', code
+
+
 def _get_python_c_args(host, port, code, args, setup):
     setup = _get_setup_updated_with_protocol_and_ppid(setup)
 
     # i.e.: We want to make the repr sorted so that it works in tests.
     setup_repr = setup if setup is None else (sorted_dict_repr(setup))
 
-    return ("import sys; sys.path.insert(0, r'%s'); import pydevd; pydevd.PydevdCustomization.DEFAULT_PROTOCOL=%r; "
+    future_imports = ''
+    if '__future__' in code:
+        # If the code has a __future__ import, we need to be able to strip the __future__
+        # imports from the code and add them to the start of our code snippet.
+        future_imports, code = _separate_future_imports(code)
+
+    return ("%simport sys; sys.path.insert(0, r'%s'); import pydevd; pydevd.PydevdCustomization.DEFAULT_PROTOCOL=%r; "
             "pydevd.settrace(host=%r, port=%s, suspend=False, trace_only_current_thread=False, patch_multiprocessing=True, access_token=%r, client_access_token=%r, __setup_holder__=%s); "
             "%s"
             ) % (
+               future_imports,
                pydev_src_dir,
                pydevd_constants.get_protocol(),
                host,

src/debugpy/_vendored/pydevd/tests_python/test_pydev_monkey.py

@@ -133,6 +133,70 @@ def test_monkey_patch_args_indc():
         SetupHolder.setup = original
 
 
+def test_separate_future_imports():
+    found = pydev_monkey._separate_future_imports('''from __future__ import print_function\nprint(1)''')
+    assert found == ('from __future__ import print_function;', '\nprint(1)')
+
+    found = pydev_monkey._separate_future_imports('''from __future__ import print_function;print(1)''')
+    assert found == ('from __future__ import print_function;', 'print(1)')
+
+    found = pydev_monkey._separate_future_imports('''from __future__ import (\nprint_function);print(1)''')
+    assert found == ('from __future__ import (\nprint_function);', 'print(1)')
+
+    found = pydev_monkey._separate_future_imports('''"line";from __future__ import (\n\nprint_function, absolute_imports\n);print(1)''')
+    assert found == ('"line";from __future__ import (\n\nprint_function, absolute_imports\n);', 'print(1)')
+
+
+def test_monkey_patch_args_indc_future_import():
+    original = SetupHolder.setup
+
+    try:
+        SetupHolder.setup = {'client': '127.0.0.1', 'port': '0', 'ppid': os.getpid(), 'protocol-quoted-line': True, 'skip-notify-stdin': True}
+        check = ['C:\\bin\\python.exe', '-u', '-c', 'from __future__ import print_function;connect("127.0.0.1")']
+        debug_command = (
+            "from __future__ import print_function;import sys; sys.path.insert(0, r\'%s\'); import pydevd; pydevd.PydevdCustomization.DEFAULT_PROTOCOL='quoted-line'; "
+            'pydevd.settrace(host=\'127.0.0.1\', port=0, suspend=False, trace_only_current_thread=False, patch_multiprocessing=True, access_token=None, client_access_token=None, __setup_holder__=%s); '
+            ''
+            'connect("127.0.0.1")') % (pydev_src_dir, sorted_dict_repr(SetupHolder.setup))
+        if sys.platform == "win32":
+            debug_command = debug_command.replace('"', '\\"')
+            debug_command = '"%s"' % debug_command
+        res = pydev_monkey.patch_args(check)
+        assert res == [
+            'C:\\bin\\python.exe',
+            '-u',
+            '-c',
+            debug_command
+        ]
+    finally:
+        SetupHolder.setup = original
+
+
+def test_monkey_patch_args_indc_future_import2():
+    original = SetupHolder.setup
+
+    try:
+        SetupHolder.setup = {'client': '127.0.0.1', 'port': '0', 'ppid': os.getpid(), 'protocol-quoted-line': True, 'skip-notify-stdin': True}
+        check = ['C:\\bin\\python.exe', '-u', '-c', 'from __future__ import print_function\nconnect("127.0.0.1")']
+        debug_command = (
+            "from __future__ import print_function;import sys; sys.path.insert(0, r\'%s\'); import pydevd; pydevd.PydevdCustomization.DEFAULT_PROTOCOL='quoted-line'; "
+            'pydevd.settrace(host=\'127.0.0.1\', port=0, suspend=False, trace_only_current_thread=False, patch_multiprocessing=True, access_token=None, client_access_token=None, __setup_holder__=%s); '
+            ''
+            '\nconnect("127.0.0.1")') % (pydev_src_dir, sorted_dict_repr(SetupHolder.setup))
+        if sys.platform == "win32":
+            debug_command = debug_command.replace('"', '\\"')
+            debug_command = '"%s"' % debug_command
+        res = pydev_monkey.patch_args(check)
+        assert res == [
+            'C:\\bin\\python.exe',
+            '-u',
+            '-c',
+            debug_command
+        ]
+    finally:
+        SetupHolder.setup = original
+
+
 def test_monkey_patch_args_indc2():
     original = SetupHolder.setup
 
@@ -495,7 +559,7 @@ def test_monkey_patch_c_program_arg(use_bytes):
 
     try:
         SetupHolder.setup = {'client': '127.0.0.1', 'port': '0'}
-        check = ['C:\\bin\\python.exe', '-u', 'target.py', '-c', '-áéíóú']
+        check = ['C:\\bin\\python.exe', '-u', 'target.py', '-c', '-áéíóú']
 
         encode = lambda s:s
         if use_bytes:
@@ -521,7 +585,7 @@ def test_monkey_patch_c_program_arg(use_bytes):
             '--file',
             encode('target.py'),
             encode('-c'),
-            encode('-áéíóú')
+            encode('-áéíóú')
         ]
     finally:
         SetupHolder.setup = original