From 2d52f000d92a4d7d14b0935630b62e16225aeb4f Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Sat, 5 Jan 2019 19:10:53 -0800 Subject: [PATCH 1/8] Remove two outdated parts of comments --- mypy/dmypy.py | 2 +- mypy/dmypy_server.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/mypy/dmypy.py b/mypy/dmypy.py index d8ca2a34a743..ab7de04ca0e8 100644 --- a/mypy/dmypy.py +++ b/mypy/dmypy.py @@ -1,6 +1,6 @@ """Client for mypy daemon mode. -Highly experimental! Only supports UNIX-like systems. +Experimental! This manages a daemon process which keeps useful state in memory rather than having to read it back from disk on each run. diff --git a/mypy/dmypy_server.py b/mypy/dmypy_server.py index ee1b97cbbdd9..fb55285179ee 100644 --- a/mypy/dmypy_server.py +++ b/mypy/dmypy_server.py @@ -1,7 +1,5 @@ """Server for mypy daemon mode. -Only supports UNIX-like systems. - This implements a daemon process which keeps useful state in memory to enable fine-grained incremental reprocessing of changes. """ From 47b9409f49bcf62aec6aabe8d4c319034eabbf5e Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Sun, 6 Jan 2019 17:39:52 -0800 Subject: [PATCH 2/8] Capture and report when process_options fails on the daemon --- mypy/dmypy_server.py | 18 +++++++++++----- mypy/util.py | 50 +++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 62 insertions(+), 6 deletions(-) diff --git a/mypy/dmypy_server.py b/mypy/dmypy_server.py index fb55285179ee..4c0413d6bdde 100644 --- a/mypy/dmypy_server.py +++ b/mypy/dmypy_server.py @@ -5,6 +5,7 @@ """ import base64 +import io import json import os import pickle @@ -27,6 +28,7 @@ from mypy.modulefinder import BuildSource, compute_search_paths from mypy.options import Options from mypy.typestate import reset_global_state +from mypy.util import redirect_stderr from mypy.version import __version__ @@ -268,11 +270,15 @@ def cmd_stop(self) -> Dict[str, object]: def cmd_run(self, version: str, args: Sequence[str]) -> Dict[str, object]: """Check a list of files, triggering a restart if needed.""" try: - sources, options = mypy.main.process_options( - ['-i'] + list(args), - require_targets=True, - server_options=True, - fscache=self.fscache) + # Process options can exit on improper arguments, so we need to catch that and + # capture stderr so the client can report it + stderr = io.StringIO() + with redirect_stderr(stderr): + sources, options = mypy.main.process_options( + ['-i'] + list(args), + require_targets=True, + server_options=True, + fscache=self.fscache) # Signal that we need to restart if the options have changed if self.options_snapshot != options.snapshot(): return {'restart': 'configuration changed'} @@ -286,6 +292,8 @@ def cmd_run(self, version: str, args: Sequence[str]) -> Dict[str, object]: return {'restart': 'plugins changed'} except InvalidSourceList as err: return {'out': '', 'err': str(err), 'status': 2} + except SystemExit as e: + return {'out': '', 'err': stderr.getvalue(), 'status': e.code} return self.check(sources) def cmd_check(self, files: Sequence[str]) -> Dict[str, object]: diff --git a/mypy/util.py b/mypy/util.py index 02a05f8231b7..8eb1dbd21995 100644 --- a/mypy/util.py +++ b/mypy/util.py @@ -1,10 +1,12 @@ """Utility functions with no non-trivial dependencies.""" +import contextlib import os import pathlib import re import subprocess import sys -from typing import TypeVar, List, Tuple, Optional, Dict, Sequence +from types import TracebackType +from typing import TypeVar, List, Tuple, Optional, Dict, Sequence, TextIO MYPY = False if MYPY: @@ -255,3 +257,49 @@ def hard_exit(status: int = 0) -> None: sys.stdout.flush() sys.stderr.flush() os._exit(status) + + +# The following is a backport of stream redirect utilities from Lib/contextlib.py + + +class _RedirectStream(contextlib.AbstractContextManager): + + _stream = None # type: str + + def __init__(self, new_target: TextIO) -> None: + self._new_target = new_target + # We use a list of old targets to make this CM re-entrant + self._old_targets = [] # type: List[TextIO] + + def __enter__(self) -> TextIO: + self._old_targets.append(getattr(sys, self._stream)) + setattr(sys, self._stream, self._new_target) + return self._new_target + + def __exit__(self, + exc_ty: 'Optional[Type[BaseException]]' = None, + exc_val: Optional[BaseException] = None, + exc_tb: Optional[TracebackType] = None, + ) -> bool: + setattr(sys, self._stream, self._old_targets.pop()) + return False + + +class redirect_stdout(_RedirectStream): + """Context manager for temporarily redirecting stdout to another file. + # How to send help() to stderr + with redirect_stdout(sys.stderr): + help(dir) + # How to write help() to a file + with open('help.txt', 'w') as f: + with redirect_stdout(f): + help(pow) + """ + + _stream = "stdout" + + +class redirect_stderr(_RedirectStream): + """Context manager for temporarily redirecting stderr to another file.""" + + _stream = "stderr" From 86831f2e6435027114a0e8b6cb2a9593142f9430 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Sun, 6 Jan 2019 17:48:46 -0800 Subject: [PATCH 3/8] Add test --- test-data/unit/daemon.test | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test-data/unit/daemon.test b/test-data/unit/daemon.test index 42e74b393106..aaa2cbff65bb 100644 --- a/test-data/unit/daemon.test +++ b/test-data/unit/daemon.test @@ -122,3 +122,13 @@ Daemon stopped import bar [file bar.py] pass + +[case testDaemonRunNoTarget] +$ dmypy run -- --follow-imports=error +Daemon started +usage: mypy [-h] [-v] [-V] [more options; see below] + [-m MODULE] [-p PACKAGE] [-c PROGRAM_TEXT] [files ...] +mypy: error: Missing target module, package, files, or command. +== Return code: 2 +$ dmypy stop +Daemon stopped From 84bbc9d518b43a33c4643986a49b6f9a7db733e9 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Sun, 6 Jan 2019 18:08:44 -0800 Subject: [PATCH 4/8] Make error messages nicer --- mypy/dmypy_server.py | 5 ++++- mypy/main.py | 6 ++++-- test-data/unit/daemon.test | 4 +--- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/mypy/dmypy_server.py b/mypy/dmypy_server.py index 4c0413d6bdde..b09cf6cf9bb4 100644 --- a/mypy/dmypy_server.py +++ b/mypy/dmypy_server.py @@ -4,6 +4,7 @@ to enable fine-grained incremental reprocessing of changes. """ +import argparse import base64 import io import json @@ -278,7 +279,9 @@ def cmd_run(self, version: str, args: Sequence[str]) -> Dict[str, object]: ['-i'] + list(args), require_targets=True, server_options=True, - fscache=self.fscache) + fscache=self.fscache, + program='dmypy', + header=argparse.SUPPRESS) # Signal that we need to restart if the options have changed if self.options_snapshot != options.snapshot(): return {'restart': 'configuration changed'} diff --git a/mypy/main.py b/mypy/main.py index 6f6cbf92e51f..87051eaa1d0d 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -297,6 +297,8 @@ def process_options(args: List[str], require_targets: bool = True, server_options: bool = False, fscache: Optional[FileSystemCache] = None, + program: str = 'mypy', + header: str = HEADER, ) -> Tuple[List[BuildSource], Options]: """Parse command line arguments. @@ -304,8 +306,8 @@ def process_options(args: List[str], call fscache.set_package_root() to set the cache's package root. """ - parser = argparse.ArgumentParser(prog='mypy', - usage=HEADER, + parser = argparse.ArgumentParser(prog=program, + usage=header, description=DESCRIPTION, epilog=FOOTER, fromfile_prefix_chars='@', diff --git a/test-data/unit/daemon.test b/test-data/unit/daemon.test index aaa2cbff65bb..db26cf0c8836 100644 --- a/test-data/unit/daemon.test +++ b/test-data/unit/daemon.test @@ -126,9 +126,7 @@ pass [case testDaemonRunNoTarget] $ dmypy run -- --follow-imports=error Daemon started -usage: mypy [-h] [-v] [-V] [more options; see below] - [-m MODULE] [-p PACKAGE] [-c PROGRAM_TEXT] [files ...] -mypy: error: Missing target module, package, files, or command. +dmypy: error: Missing target module, package, files, or command. == Return code: 2 $ dmypy stop Daemon stopped From 32dd04fa220c978ea4dade1843505c28c93afdd4 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Sun, 6 Jan 2019 18:42:36 -0800 Subject: [PATCH 5/8] Make more backwards compatible --- mypy/util.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mypy/util.py b/mypy/util.py index 8eb1dbd21995..8ec6f011b9c2 100644 --- a/mypy/util.py +++ b/mypy/util.py @@ -10,8 +10,10 @@ MYPY = False if MYPY: - from typing import Type + from typing import Type, ContextManager from typing_extensions import Final +else: + ContextManager = object T = TypeVar('T') @@ -262,7 +264,7 @@ def hard_exit(status: int = 0) -> None: # The following is a backport of stream redirect utilities from Lib/contextlib.py -class _RedirectStream(contextlib.AbstractContextManager): +class _RedirectStream(ContextManager): _stream = None # type: str From ab8073e3eb59d5a4b2aba6097640b0bfa66f458a Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Sun, 6 Jan 2019 18:45:39 -0800 Subject: [PATCH 6/8] Revert "Make more backwards compatible" This reverts commit 32dd04fa220c978ea4dade1843505c28c93afdd4. --- mypy/util.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/mypy/util.py b/mypy/util.py index 8ec6f011b9c2..8eb1dbd21995 100644 --- a/mypy/util.py +++ b/mypy/util.py @@ -10,10 +10,8 @@ MYPY = False if MYPY: - from typing import Type, ContextManager + from typing import Type from typing_extensions import Final -else: - ContextManager = object T = TypeVar('T') @@ -264,7 +262,7 @@ def hard_exit(status: int = 0) -> None: # The following is a backport of stream redirect utilities from Lib/contextlib.py -class _RedirectStream(ContextManager): +class _RedirectStream(contextlib.AbstractContextManager): _stream = None # type: str From c649b379a18e71462ed08e15b8b7ef3a98e3be30 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Sun, 6 Jan 2019 18:46:06 -0800 Subject: [PATCH 7/8] Make more backwards compatible --- mypy/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/util.py b/mypy/util.py index 8eb1dbd21995..8cc577775331 100644 --- a/mypy/util.py +++ b/mypy/util.py @@ -262,7 +262,7 @@ def hard_exit(status: int = 0) -> None: # The following is a backport of stream redirect utilities from Lib/contextlib.py -class _RedirectStream(contextlib.AbstractContextManager): +class _RedirectStream: _stream = None # type: str From 35b836ebec57b34855b92b673cbcc3996134c462 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Mon, 7 Jan 2019 19:04:02 -0800 Subject: [PATCH 8/8] Respond to review --- mypy/dmypy.py | 2 -- mypy/dmypy_server.py | 20 +++++++++++--------- mypy/util.py | 1 + test-data/unit/daemon.test | 2 +- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/mypy/dmypy.py b/mypy/dmypy.py index ab7de04ca0e8..06211e3fe008 100644 --- a/mypy/dmypy.py +++ b/mypy/dmypy.py @@ -1,7 +1,5 @@ """Client for mypy daemon mode. -Experimental! - This manages a daemon process which keeps useful state in memory rather than having to read it back from disk on each run. """ diff --git a/mypy/dmypy_server.py b/mypy/dmypy_server.py index b09cf6cf9bb4..01ee1e8deff1 100644 --- a/mypy/dmypy_server.py +++ b/mypy/dmypy_server.py @@ -29,7 +29,7 @@ from mypy.modulefinder import BuildSource, compute_search_paths from mypy.options import Options from mypy.typestate import reset_global_state -from mypy.util import redirect_stderr +from mypy.util import redirect_stderr, redirect_stdout from mypy.version import __version__ @@ -274,14 +274,16 @@ def cmd_run(self, version: str, args: Sequence[str]) -> Dict[str, object]: # Process options can exit on improper arguments, so we need to catch that and # capture stderr so the client can report it stderr = io.StringIO() + stdout = io.StringIO() with redirect_stderr(stderr): - sources, options = mypy.main.process_options( - ['-i'] + list(args), - require_targets=True, - server_options=True, - fscache=self.fscache, - program='dmypy', - header=argparse.SUPPRESS) + with redirect_stdout(stdout): + sources, options = mypy.main.process_options( + ['-i'] + list(args), + require_targets=True, + server_options=True, + fscache=self.fscache, + program='mypy-daemon', + header=argparse.SUPPRESS) # Signal that we need to restart if the options have changed if self.options_snapshot != options.snapshot(): return {'restart': 'configuration changed'} @@ -296,7 +298,7 @@ def cmd_run(self, version: str, args: Sequence[str]) -> Dict[str, object]: except InvalidSourceList as err: return {'out': '', 'err': str(err), 'status': 2} except SystemExit as e: - return {'out': '', 'err': stderr.getvalue(), 'status': e.code} + return {'out': stdout.getvalue(), 'err': stderr.getvalue(), 'status': e.code} return self.check(sources) def cmd_check(self, files: Sequence[str]) -> Dict[str, object]: diff --git a/mypy/util.py b/mypy/util.py index 8cc577775331..f10d588055e0 100644 --- a/mypy/util.py +++ b/mypy/util.py @@ -260,6 +260,7 @@ def hard_exit(status: int = 0) -> None: # The following is a backport of stream redirect utilities from Lib/contextlib.py +# We need this for 3.4 support. They can be removed in March 2019! class _RedirectStream: diff --git a/test-data/unit/daemon.test b/test-data/unit/daemon.test index db26cf0c8836..6a5e6fcf82e1 100644 --- a/test-data/unit/daemon.test +++ b/test-data/unit/daemon.test @@ -126,7 +126,7 @@ pass [case testDaemonRunNoTarget] $ dmypy run -- --follow-imports=error Daemon started -dmypy: error: Missing target module, package, files, or command. +mypy-daemon: error: Missing target module, package, files, or command. == Return code: 2 $ dmypy stop Daemon stopped