From c819f4436df40d40bd6e16c5af8317efaef2dd27 Mon Sep 17 00:00:00 2001 From: Fabio Zadrozny Date: Fri, 11 Jun 2021 09:03:54 -0300 Subject: [PATCH] Deal with __future__ imports on python -c subprocess. Fixes #642 --- .../pydevd/_pydev_bundle/pydev_monkey.py | 77 ++++++++++++++++++- .../pydevd/tests_python/test_pydev_monkey.py | 68 +++++++++++++++- 2 files changed, 142 insertions(+), 3 deletions(-) diff --git a/src/debugpy/_vendored/pydevd/_pydev_bundle/pydev_monkey.py b/src/debugpy/_vendored/pydevd/_pydev_bundle/pydev_monkey.py index 148f8a3d5..382ea7f0c 100644 --- a/src/debugpy/_vendored/pydevd/_pydev_bundle/pydev_monkey.py +++ b/src/debugpy/_vendored/pydevd/_pydev_bundle/pydev_monkey.py @@ -9,6 +9,7 @@ from contextlib import contextmanager from _pydevd_bundle import pydevd_constants from _pydevd_bundle.pydevd_defaults import PydevdCustomization +import ast try: xrange @@ -75,16 +76,90 @@ def _get_setup_updated_with_protocol_and_ppid(setup, is_exec=False): return setup +class _LastFutureImportFinder(ast.NodeVisitor): + + def __init__(self): + self._last_future_import_found = None + + def visit_ImportFrom(self, node): + if node.module == '__future__': + self._last_future_import_found = node + + def get_last_future_import_end_line_col(self): + if self._last_future_import_found is None: + return None + + node = self._last_future_import_found + return node.end_lineno, node.end_col_offset + + +def _separate_future_imports(code): + ''' + :param code: + The code from where we want to get the __future__ imports (note that it's possible that + there's no such entry). + + :return tuple(str, str): + The return is a tuple(future_import, code). + + If the future import is not available a return such as ('', code) is given, otherwise, the + future import will end with a ';' (so that it can be put right before the pydevd attach + code). + ''' + try: + node = ast.parse(code, '', 'exec') + visitor = _LastFutureImportFinder() + visitor.visit(node) + line_col = visitor.get_last_future_import_end_line_col() + if line_col is None: + return '', code + + line, col = line_col + line -= 1 # ast lines are 1-based, make it 0-based. + offset = 0 + for i, line_contents in enumerate(code.splitlines(True)): + if i == line: + for i in range(col, len(line_contents)): + if line_contents[i] in (' ', '\t', ';', ')'): + col += 1 + else: + break + + offset += col + future_import = code[:offset] + code_remainder = code[offset:] + if not future_import.endswith(';'): + future_import += ';' + return future_import, code_remainder + else: + offset += len(line_contents) + + # This shouldn't happen... + pydev_log.info('Unable to find line %s in code:\n%r', line, code) + return '', code + + except: + pydev_log.exception('Error getting from __future__ imports from: %r', code) + return '', code + + def _get_python_c_args(host, port, code, args, setup): setup = _get_setup_updated_with_protocol_and_ppid(setup) # i.e.: We want to make the repr sorted so that it works in tests. setup_repr = setup if setup is None else (sorted_dict_repr(setup)) - return ("import sys; sys.path.insert(0, r'%s'); import pydevd; pydevd.PydevdCustomization.DEFAULT_PROTOCOL=%r; " + future_imports = '' + if '__future__' in code: + # If the code has a __future__ import, we need to be able to strip the __future__ + # imports from the code and add them to the start of our code snippet. + future_imports, code = _separate_future_imports(code) + + return ("%simport sys; sys.path.insert(0, r'%s'); import pydevd; pydevd.PydevdCustomization.DEFAULT_PROTOCOL=%r; " "pydevd.settrace(host=%r, port=%s, suspend=False, trace_only_current_thread=False, patch_multiprocessing=True, access_token=%r, client_access_token=%r, __setup_holder__=%s); " "%s" ) % ( + future_imports, pydev_src_dir, pydevd_constants.get_protocol(), host, diff --git a/src/debugpy/_vendored/pydevd/tests_python/test_pydev_monkey.py b/src/debugpy/_vendored/pydevd/tests_python/test_pydev_monkey.py index 744ea2a31..1c915c003 100644 --- a/src/debugpy/_vendored/pydevd/tests_python/test_pydev_monkey.py +++ b/src/debugpy/_vendored/pydevd/tests_python/test_pydev_monkey.py @@ -133,6 +133,70 @@ def test_monkey_patch_args_indc(): SetupHolder.setup = original +def test_separate_future_imports(): + found = pydev_monkey._separate_future_imports('''from __future__ import print_function\nprint(1)''') + assert found == ('from __future__ import print_function;', '\nprint(1)') + + found = pydev_monkey._separate_future_imports('''from __future__ import print_function;print(1)''') + assert found == ('from __future__ import print_function;', 'print(1)') + + found = pydev_monkey._separate_future_imports('''from __future__ import (\nprint_function);print(1)''') + assert found == ('from __future__ import (\nprint_function);', 'print(1)') + + found = pydev_monkey._separate_future_imports('''"line";from __future__ import (\n\nprint_function, absolute_imports\n);print(1)''') + assert found == ('"line";from __future__ import (\n\nprint_function, absolute_imports\n);', 'print(1)') + + +def test_monkey_patch_args_indc_future_import(): + original = SetupHolder.setup + + try: + SetupHolder.setup = {'client': '127.0.0.1', 'port': '0', 'ppid': os.getpid(), 'protocol-quoted-line': True, 'skip-notify-stdin': True} + check = ['C:\\bin\\python.exe', '-u', '-c', 'from __future__ import print_function;connect("127.0.0.1")'] + debug_command = ( + "from __future__ import print_function;import sys; sys.path.insert(0, r\'%s\'); import pydevd; pydevd.PydevdCustomization.DEFAULT_PROTOCOL='quoted-line'; " + 'pydevd.settrace(host=\'127.0.0.1\', port=0, suspend=False, trace_only_current_thread=False, patch_multiprocessing=True, access_token=None, client_access_token=None, __setup_holder__=%s); ' + '' + 'connect("127.0.0.1")') % (pydev_src_dir, sorted_dict_repr(SetupHolder.setup)) + if sys.platform == "win32": + debug_command = debug_command.replace('"', '\\"') + debug_command = '"%s"' % debug_command + res = pydev_monkey.patch_args(check) + assert res == [ + 'C:\\bin\\python.exe', + '-u', + '-c', + debug_command + ] + finally: + SetupHolder.setup = original + + +def test_monkey_patch_args_indc_future_import2(): + original = SetupHolder.setup + + try: + SetupHolder.setup = {'client': '127.0.0.1', 'port': '0', 'ppid': os.getpid(), 'protocol-quoted-line': True, 'skip-notify-stdin': True} + check = ['C:\\bin\\python.exe', '-u', '-c', 'from __future__ import print_function\nconnect("127.0.0.1")'] + debug_command = ( + "from __future__ import print_function;import sys; sys.path.insert(0, r\'%s\'); import pydevd; pydevd.PydevdCustomization.DEFAULT_PROTOCOL='quoted-line'; " + 'pydevd.settrace(host=\'127.0.0.1\', port=0, suspend=False, trace_only_current_thread=False, patch_multiprocessing=True, access_token=None, client_access_token=None, __setup_holder__=%s); ' + '' + '\nconnect("127.0.0.1")') % (pydev_src_dir, sorted_dict_repr(SetupHolder.setup)) + if sys.platform == "win32": + debug_command = debug_command.replace('"', '\\"') + debug_command = '"%s"' % debug_command + res = pydev_monkey.patch_args(check) + assert res == [ + 'C:\\bin\\python.exe', + '-u', + '-c', + debug_command + ] + finally: + SetupHolder.setup = original + + def test_monkey_patch_args_indc2(): original = SetupHolder.setup @@ -495,7 +559,7 @@ def test_monkey_patch_c_program_arg(use_bytes): try: SetupHolder.setup = {'client': '127.0.0.1', 'port': '0'} - check = ['C:\\bin\\python.exe', '-u', 'target.py', '-c', '-áéíóú'] + check = ['C:\\bin\\python.exe', '-u', 'target.py', '-c', '-áéíóú'] encode = lambda s:s if use_bytes: @@ -521,7 +585,7 @@ def test_monkey_patch_c_program_arg(use_bytes): '--file', encode('target.py'), encode('-c'), - encode('-áéíóú') + encode('-áéíóú') ] finally: SetupHolder.setup = original