diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 82e8c65c4a6d..5e98ca14aaf1 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -39,24 +39,64 @@ from mypy.options import Options try: - from typed_ast import ast3 - from typed_ast.ast3 import ( - AST, - Call, - FunctionType, - Name, - Attribute, - Ellipsis as ast3_Ellipsis, - Starred, - NameConstant, - Expression as ast3_Expression, - Str, - Bytes, - Index, - Num, - UnaryOp, - USub, - ) + # Check if we can use the stdlib ast module instead of typed_ast. + if sys.version_info >= (3, 8): + import ast as ast3 + assert 'kind' in ast3.Constant._fields, \ + "This 3.8.0 alpha (%s) is too old; 3.8.0a3 required" % sys.version.split()[0] + from ast import ( + AST, + Call, + FunctionType, + Name, + Attribute, + Ellipsis as ast3_Ellipsis, + Starred, + NameConstant, + Expression as ast3_Expression, + Str, + Bytes, + Index, + Num, + UnaryOp, + USub, + ) + + def ast3_parse(source: Union[str, bytes], filename: str, mode: str, + feature_version: int = sys.version_info[1]) -> AST: + return ast3.parse(source, filename, mode, + type_comments=True, # This works the magic + feature_version=feature_version) + + NamedExpr = ast3.NamedExpr + Constant = ast3.Constant + else: + from typed_ast import ast3 + from typed_ast.ast3 import ( + AST, + Call, + FunctionType, + Name, + Attribute, + Ellipsis as ast3_Ellipsis, + Starred, + NameConstant, + Expression as ast3_Expression, + Str, + Bytes, + Index, + Num, + UnaryOp, + USub, + ) + + def ast3_parse(source: Union[str, bytes], filename: str, mode: str, + feature_version: int = sys.version_info[1]) -> AST: + return ast3.parse(source, filename, mode, feature_version=feature_version) + + # These don't exist before 3.8 + NamedExpr = Any + Constant = Any except ImportError: if sys.version_info.minor > 2: try: @@ -122,7 +162,7 @@ def parse(source: Union[str, bytes], else: assert options.python_version[0] >= 3 feature_version = options.python_version[1] - ast = ast3.parse(source, fnam, 'exec', feature_version=feature_version) + ast = ast3_parse(source, fnam, 'exec', feature_version=feature_version) tree = ASTConverter(options=options, is_stub=is_stub_file, @@ -146,7 +186,7 @@ def parse_type_comment(type_comment: str, assume_str_is_unicode: bool = True, ) -> Optional[Type]: try: - typ = ast3.parse(type_comment, '', 'eval') + typ = ast3_parse(type_comment, '', 'eval') except SyntaxError as e: if errors is not None: errors.report(line, e.offset, TYPE_COMMENT_SYNTAX_ERROR, blocker=True) @@ -366,16 +406,12 @@ def visit_Module(self, mod: ast3.Module) -> MypyFile: # arguments = (arg* args, arg? vararg, arg* kwonlyargs, expr* kw_defaults, # arg? kwarg, expr* defaults) def visit_FunctionDef(self, n: ast3.FunctionDef) -> Union[FuncDef, Decorator]: - f = self.do_func_def(n) - f.set_line(n.lineno, n.col_offset) # Overrides set_line -- can't use self.set_line - return f + return self.do_func_def(n) # AsyncFunctionDef(identifier name, arguments args, # stmt* body, expr* decorator_list, expr? returns, string? type_comment) def visit_AsyncFunctionDef(self, n: ast3.AsyncFunctionDef) -> Union[FuncDef, Decorator]: - f = self.do_func_def(n, is_coroutine=True) - f.set_line(n.lineno, n.col_offset) # Overrides set_line -- can't use self.set_line - return f + return self.do_func_def(n, is_coroutine=True) def do_func_def(self, n: Union[ast3.FunctionDef, ast3.AsyncFunctionDef], is_coroutine: bool = False) -> Union[FuncDef, Decorator]: @@ -397,7 +433,7 @@ def do_func_def(self, n: Union[ast3.FunctionDef, ast3.AsyncFunctionDef], return_type = None elif n.type_comment is not None: try: - func_type_ast = ast3.parse(n.type_comment, '', 'func_type') + func_type_ast = ast3_parse(n.type_comment, '', 'func_type') assert isinstance(func_type_ast, FunctionType) # for ellipsis arg if (len(func_type_ast.argtypes) == 1 and @@ -470,15 +506,25 @@ def do_func_def(self, n: Union[ast3.FunctionDef, ast3.AsyncFunctionDef], func_type.line = lineno if n.decorator_list: + if sys.version_info < (3, 8): + # Before 3.8, [typed_]ast the line number points to the first decorator. + # In 3.8, it points to the 'def' line, where we want it. + lineno += len(n.decorator_list) + var = Var(func_def.name()) var.is_ready = False - var.set_line(n.decorator_list[0].lineno) + var.set_line(lineno) func_def.is_decorated = True - func_def.set_line(lineno + len(n.decorator_list)) - func_def.body.set_line(func_def.get_line()) - return Decorator(func_def, self.translate_expr_list(n.decorator_list), var) + func_def.set_line(lineno, n.col_offset) + func_def.body.set_line(lineno) # TODO: Why? + + deco = Decorator(func_def, self.translate_expr_list(n.decorator_list), var) + deco.set_line(n.decorator_list[0].lineno) + return deco else: + # FuncDef overrides set_line -- can't use self.set_line + func_def.set_line(lineno, n.col_offset) return func_def def set_type_optional(self, type: Optional[Type], initializer: Optional[Expression]) -> None: @@ -568,7 +614,15 @@ def visit_ClassDef(self, n: ast3.ClassDef) -> ClassDef: metaclass=dict(keywords).get('metaclass'), keywords=keywords) cdef.decorators = self.translate_expr_list(n.decorator_list) - self.set_line(cdef, n) + if n.decorator_list and sys.version_info >= (3, 8): + # Before 3.8, n.lineno points to the first decorator; in + # 3.8, it points to the 'class' statement. We always make + # it point to the first decorator. (The node structure + # here is different than for decorated functions.) + cdef.line = n.decorator_list[0].lineno + cdef.column = n.col_offset + else: + self.set_line(cdef, n) self.class_nesting -= 1 return cdef @@ -617,6 +671,10 @@ def visit_AugAssign(self, n: ast3.AugAssign) -> OperatorAssignmentStmt: self.visit(n.value)) return self.set_line(s, n) + def visit_NamedExpr(self, n: NamedExpr) -> None: + self.fail("assignment expressions are not yet supported", n.lineno, n.col_offset) + return None + # For(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment) def visit_For(self, n: ast3.For) -> ForStmt: if n.type_comment is not None: @@ -926,6 +984,30 @@ def visit_Call(self, n: Call) -> CallExpr: cast('List[Optional[str]]', [None] * len(args)) + keyword_names) return self.set_line(e, n) + # Constant(object value) -- a constant, in Python 3.8. + def visit_Constant(self, n: Constant) -> Any: + val = n.value + e = None # type: Any + if val is None: + e = NameExpr('None') + elif isinstance(val, str): + e = StrExpr(n.s) + elif isinstance(val, bytes): + e = BytesExpr(bytes_to_human_readable_repr(n.s)) + elif isinstance(val, bool): # Must check before int! + e = NameExpr(str(val)) + elif isinstance(val, int): + e = IntExpr(val) + elif isinstance(val, float): + e = FloatExpr(val) + elif isinstance(val, complex): + e = ComplexExpr(val) + elif val is Ellipsis: + e = EllipsisExpr() + else: + raise RuntimeError('Constant not implemented for ' + str(type(val))) + return self.set_line(e, n) + # Num(object n) -- a number as a PyObject. def visit_Num(self, n: ast3.Num) -> Union[IntExpr, FloatExpr, ComplexExpr]: # The n field has the type complex, but complex isn't *really* @@ -994,7 +1076,8 @@ def visit_Bytes(self, n: ast3.Bytes) -> Union[BytesExpr, StrExpr]: # NameConstant(singleton value) def visit_NameConstant(self, n: NameConstant) -> NameExpr: - return NameExpr(str(n.value)) + e = NameExpr(str(n.value)) + return self.set_line(e, n) # Ellipsis def visit_Ellipsis(self, n: ast3_Ellipsis) -> EllipsisExpr: @@ -1094,7 +1177,7 @@ def invalid_type(self, node: AST, note: Optional[str] = None) -> RawExpressionTy def visit(self, node: ast3.expr) -> Type: ... @overload # noqa - def visit(self, node: Optional[AST]) -> Optional[Type]: ... + def visit(self, node: Optional[AST]) -> Optional[Type]: ... # noqa def visit(self, node: Optional[AST]) -> Optional[Type]: # noqa """Modified visit -- keep track of the stack of nodes""" @@ -1205,6 +1288,34 @@ def visit_NameConstant(self, n: NameConstant) -> Type: else: return UnboundType(str(n.value), line=self.line) + # Only for 3.8 and newer + def visit_Constant(self, n: Constant) -> Type: + val = n.value + if val is None: + # None is a type. + return UnboundType('None', line=self.line) + if isinstance(val, str): + # Parse forward reference. + if (n.kind and 'u' in n.kind) or self.assume_str_is_unicode: + return parse_type_string(n.s, 'builtins.unicode', self.line, n.col_offset, + assume_str_is_unicode=self.assume_str_is_unicode) + else: + return parse_type_string(n.s, 'builtins.str', self.line, n.col_offset, + assume_str_is_unicode=self.assume_str_is_unicode) + if val is Ellipsis: + # '...' is valid in some types. + return EllipsisType(line=self.line) + if isinstance(val, bool): + # Special case for True/False. + return RawExpressionType(val, 'builtins.bool', line=self.line) + if isinstance(val, (int, float, complex)): + return self.numeric_type(val, n) + if isinstance(val, bytes): + contents = bytes_to_human_readable_repr(val) + return RawExpressionType(contents, 'builtins.bytes', self.line, column=n.col_offset) + # Everything else is invalid. + return self.invalid_type(n) + # UnaryOp(op, operand) def visit_UnaryOp(self, n: UnaryOp) -> Type: # We support specifically Literal[-4] and nothing else. @@ -1216,13 +1327,11 @@ def visit_UnaryOp(self, n: UnaryOp) -> Type: return typ return self.invalid_type(n) - # Num(number n) - def visit_Num(self, n: Num) -> Type: - # The n field has the type complex, but complex isn't *really* + def numeric_type(self, value: object, n: AST) -> Type: + # The node's field has the type complex, but complex isn't *really* # a parent of int and float, and this causes isinstance below # to think that the complex branch is always picked. Avoid # this by throwing away the type. - value = n.n # type: object if isinstance(value, int): numeric_value = value # type: Optional[int] type_name = 'builtins.int' @@ -1239,24 +1348,31 @@ def visit_Num(self, n: Num) -> Type: column=getattr(n, 'col_offset', -1), ) - # Str(string s) - def visit_Str(self, n: Str) -> Type: - # Note: we transform these fallback types into the correct types in - # 'typeanal.py' -- specifically in the named_type_with_normalized_str method. - # If we're analyzing Python 3, that function will translate 'builtins.unicode' - # into 'builtins.str'. In contrast, if we're analyzing Python 2 code, we'll - # translate 'builtins.bytes' in the method below into 'builtins.str'. - if 'u' in n.kind or self.assume_str_is_unicode: - return parse_type_string(n.s, 'builtins.unicode', self.line, n.col_offset, - assume_str_is_unicode=self.assume_str_is_unicode) - else: - return parse_type_string(n.s, 'builtins.str', self.line, n.col_offset, - assume_str_is_unicode=self.assume_str_is_unicode) + if sys.version_info < (3, 8): + # Using typed_ast + + # Num(number n) + def visit_Num(self, n: Num) -> Type: + return self.numeric_type(n.n, n) + + # Str(string s) + def visit_Str(self, n: Str) -> Type: + # Note: we transform these fallback types into the correct types in + # 'typeanal.py' -- specifically in the named_type_with_normalized_str method. + # If we're analyzing Python 3, that function will translate 'builtins.unicode' + # into 'builtins.str'. In contrast, if we're analyzing Python 2 code, we'll + # translate 'builtins.bytes' in the method below into 'builtins.str'. + if 'u' in n.kind or self.assume_str_is_unicode: + return parse_type_string(n.s, 'builtins.unicode', self.line, n.col_offset, + assume_str_is_unicode=self.assume_str_is_unicode) + else: + return parse_type_string(n.s, 'builtins.str', self.line, n.col_offset, + assume_str_is_unicode=self.assume_str_is_unicode) - # Bytes(bytes s) - def visit_Bytes(self, n: Bytes) -> Type: - contents = bytes_to_human_readable_repr(n.s) - return RawExpressionType(contents, 'builtins.bytes', self.line, column=n.col_offset) + # Bytes(bytes s) + def visit_Bytes(self, n: Bytes) -> Type: + contents = bytes_to_human_readable_repr(n.s) + return RawExpressionType(contents, 'builtins.bytes', self.line, column=n.col_offset) # Subscript(expr value, slice slice, expr_context ctx) def visit_Subscript(self, n: ast3.Subscript) -> Type: diff --git a/mypy/fastparse2.py b/mypy/fastparse2.py index 46b1bd457c51..016a137c50a3 100644 --- a/mypy/fastparse2.py +++ b/mypy/fastparse2.py @@ -57,11 +57,8 @@ Attribute, Tuple as ast27_Tuple, ) - from typed_ast import ast3 - from typed_ast.ast3 import ( - FunctionType, - Ellipsis as ast3_Ellipsis, - ) + # Import ast3 from fastparse, which has special case for Python 3.8 + from mypy.fastparse import ast3, ast3_parse except ImportError: if sys.version_info.minor > 2: try: @@ -339,11 +336,11 @@ def visit_FunctionDef(self, n: ast27.FunctionDef) -> Statement: return_type = None elif type_comment is not None and len(type_comment) > 0: try: - func_type_ast = ast3.parse(type_comment, '', 'func_type') - assert isinstance(func_type_ast, FunctionType) + func_type_ast = ast3_parse(type_comment, '', 'func_type') + assert isinstance(func_type_ast, ast3.FunctionType) # for ellipsis arg if (len(func_type_ast.argtypes) == 1 and - isinstance(func_type_ast.argtypes[0], ast3_Ellipsis)): + isinstance(func_type_ast.argtypes[0], ast3.Ellipsis)): arg_types = [a.type_annotation if a.type_annotation is not None else AnyType(TypeOfAny.unannotated) diff --git a/mypy/test/helpers.py b/mypy/test/helpers.py index 385f7e938ce0..f23e3d739735 100644 --- a/mypy/test/helpers.py +++ b/mypy/test/helpers.py @@ -43,10 +43,15 @@ def assert_string_arrays_equal(expected: List[str], actual: List[str], msg: str) -> None: """Assert that two string arrays are equal. + We consider "can't" and "cannot" equivalent, by replacing the + former with the latter before comparing. + Display any differences in a human-readable form. """ actual = clean_up(actual) + actual = [line.replace("can't", "cannot") for line in actual] + expected = [line.replace("can't", "cannot") for line in expected] if actual != expected: num_skip_start = num_skipped_prefix_lines(expected, actual) diff --git a/test-data/unit/check-underscores.test b/test-data/unit/check-underscores.test index 88f95ef4d2fa..ac9fad2ca792 100644 --- a/test-data/unit/check-underscores.test +++ b/test-data/unit/check-underscores.test @@ -3,11 +3,6 @@ x = 1000_000 # E: Underscores in numeric literals are only supported in Python 3.6 and greater [out] -[case testUnderscoresSyntaxError] -# flags: --python-version 3.6 -x = 1000_000_ # E: invalid token -[out] - [case testUnderscoresBasics] # flags: --python-version 3.6 x: int diff --git a/test-data/unit/parse-errors.test b/test-data/unit/parse-errors.test index 56920af71917..5a7b5ce4c08e 100644 --- a/test-data/unit/parse-errors.test +++ b/test-data/unit/parse-errors.test @@ -14,12 +14,6 @@ def f() [out] file:1: error: invalid syntax -[case testMissingIndent] -if x: -1 -[out] -file:2: error: invalid syntax - [case testUnexpectedIndent] 1 2 @@ -52,12 +46,6 @@ file:1: error: invalid syntax [out] file:1: error: invalid syntax -[case testInvalidSuperClass] -class A(C[): - pass -[out] -file:1: error: invalid syntax - [case testMissingSuperClass] class A(: pass @@ -348,11 +336,6 @@ x[:,: [out] file:1: error: unexpected EOF while parsing -[case testPython2OctalIntLiteralInPython3] -0377 -[out] -file:1: error: invalid token - [case testInvalidEncoding] # foo # coding: uft-8 @@ -422,9 +405,3 @@ except KeyError, IndexError: pass [out] file:3: error: invalid syntax - -[case testLocalVarWithTypeOnNextLine] -x = 0 - # type: int -[out] -file:2: error: misplaced type annotation diff --git a/test-data/unit/parse.test b/test-data/unit/parse.test index 1490a7fdd741..941fbb6bceca 100644 --- a/test-data/unit/parse.test +++ b/test-data/unit/parse.test @@ -86,44 +86,6 @@ MypyFile:1( ExpressionStmt:3( StrExpr(foobar))) -[case testTripleQuotedStr] -'''''' -'''foo''' -'''foo\ -bar''' -'''\nfoo -bar''' -'''fo''bar''' -"""""" -"""foo""" -"""foo\ -bar""" -"""\nfoo -bar""" -"""fo""bar""" -[out] -MypyFile:1( - ExpressionStmt:1( - StrExpr()) - ExpressionStmt:2( - StrExpr(foo)) - ExpressionStmt:3( - StrExpr(foobar)) - ExpressionStmt:5( - StrExpr(\u000afoo\u000abar)) - ExpressionStmt:6( - StrExpr(fo''bar)) - ExpressionStmt:7( - StrExpr()) - ExpressionStmt:8( - StrExpr(foo)) - ExpressionStmt:9( - StrExpr(foobar)) - ExpressionStmt:11( - StrExpr(\u000afoo\u000abar)) - ExpressionStmt:12( - StrExpr(fo""bar))) - [case testRawStr] r'x\n\'' r"x\n\"" @@ -3269,19 +3231,6 @@ MypyFile:1( ExpressionStmt:1( NameExpr(y))) -[case testInvalidIgnoreAnnotations] -y # type: ignored -y # type: IGNORE -y # type : ignore -[out] -MypyFile:1( - ExpressionStmt:1( - NameExpr(y)) - ExpressionStmt:2( - NameExpr(y)) - ExpressionStmt:3( - NameExpr(y))) - [case testSpaceInIgnoreAnnotations] y # type: ignore # foo y #type:ignore diff --git a/test-data/unit/reports.test b/test-data/unit/reports.test index c96a90ae0b03..5e3e3dca7185 100644 --- a/test-data/unit/reports.test +++ b/test-data/unit/reports.test @@ -178,7 +178,7 @@ def bar(x): # type: (str) -> None any_f(x) - assert False + assert False any_f(x) diff --git a/test-data/unit/semanal-classes.test b/test-data/unit/semanal-classes.test index 7fee0e83a19c..d97d5d4dd83d 100644 --- a/test-data/unit/semanal-classes.test +++ b/test-data/unit/semanal-classes.test @@ -560,7 +560,7 @@ class A(None): pass class B(Any): pass class C(Callable[[], int]): pass [out] -main: error: Invalid base class +main:2: error: Invalid base class main:4: error: Invalid base class [case testTupleAsBaseClass]