From a9189eb9625e22e8a165e80416c4ad3f7770410b Mon Sep 17 00:00:00 2001 From: Jacques Wagener Date: Wed, 3 Apr 2019 02:02:04 +0200 Subject: [PATCH 01/23] Add basic Vyper AST layout. --- vyper/ast.py | 260 +++++++++++++++++++++++++++++++++++++++++++++ vyper/ast_utils.py | 91 ++++++++++++++++ 2 files changed, 351 insertions(+) create mode 100644 vyper/ast.py create mode 100644 vyper/ast_utils.py diff --git a/vyper/ast.py b/vyper/ast.py new file mode 100644 index 0000000000..d6e7af230e --- /dev/null +++ b/vyper/ast.py @@ -0,0 +1,260 @@ +from itertools import ( + chain, +) + +from vyper.exceptions import ( + CompilerPanic, +) + + +class VyperNode: + __slots__ = ('node_id', 'source_code', 'col_offset', 'lineno') + ignored_fields = ('ctx', ) + only_empty_fields = () + + @classmethod + def get_slots(cls): + return chain.from_iterable( + getattr(klass, '__slots__', []) + for klass in cls.__class__.mro(cls) + ) + + def __init__(self, **kwargs): + + for field_name, value in kwargs.items(): + if field_name in self.get_slots(): + setattr(self, field_name, value) + # elif value: + # raise CompilerPanic( + # f'Unsupported non-empty value field_name: {field_name}, ' + # f' class: {type(self)} value: {value}' + # ) + + +class Module(VyperNode): + __slots__ = ('body', ) + + +class Name(VyperNode): + __slots__ = ('id', ) + + +class Subscript(VyperNode): + __slots__ = ('slice', 'value') + + +class Index(VyperNode): + __slots__ = ('value', ) + + +class arg(VyperNode): + __slots__ = ('arg', 'annotation') + + +class Tuple(VyperNode): + __slots__ = ('elts', ) + + +class FunctionDef(VyperNode): + __slots__ = ('args', 'body', 'returns', 'name', 'decorator_list', 'pos') + + +class arguments(VyperNode): + __slots__ = ('args', 'defaults', 'default') + only_empty_fields = ('vararg', 'kwonlyargs', 'kwarg', 'kw_defaults') + + +class Import(VyperNode): + pass + + +class Call(VyperNode): + __slots__ = ('func', 'args', 'keywords', 'keyword') + + +class keyword(VyperNode): + __slots__ = ('arg', 'value') + + +class Str(VyperNode): + __slots__ = ('s', ) + + +class Compare(VyperNode): + __slots__ = ('comparators', 'ops', 'left', 'right') + + +class Num(VyperNode): + __slots__ = ('n', ) + + +class NameConstant(VyperNode): + __slots__ = ('value', ) + + +class Attribute(VyperNode): + __slots__ = ('attr', 'value',) + + +class Op(VyperNode): + __slots__ = ('op', 'left', 'right') + + +class BinOp(Op): + pass + + +class BoolOp(Op): + pass + + +class UnaryOp(Op): + operand = ('operand', ) + + +class List(VyperNode): + __slots__ = ('elts', ) + + +class Dict(VyperNode): + __slots__ = ('keys', 'values') + + +class Bytes(VyperNode): + __slots__ = ('s', ) + + +class Add(VyperNode): + pass + + +class Sub(VyperNode): + pass + + +class Mult(VyperNode): + pass + + +class Div(VyperNode): + pass + + +class Mod(VyperNode): + pass + + +class Pow(VyperNode): + pass + + +class In(VyperNode): + pass + + +class Gt(VyperNode): + pass + + +class GtE(VyperNode): + pass + + +class LtE(VyperNode): + pass + + +class Lt(VyperNode): + pass + + +class Eq(VyperNode): + pass + + +class NotEq(VyperNode): + pass + + +class And(VyperNode): + pass + + +class Or(VyperNode): + pass + + +class Not(VyperNode): + pass + + +class USub(VyperNode): + pass + + +class Expr(VyperNode): + __slots__ = ('value', ) + + +class Pass(VyperNode): + pass + + +class AnnAssign(VyperNode): + __slots__ = ('target', 'annotation', 'value', 'simple') + + +class Assign(VyperNode): + __slots__ = ('targets', 'value') + + +class If(VyperNode): + __slots__ = ('test', 'body', 'orelse') + + +class Assert(VyperNode): + __slots__ = ('test', 'msg') + + +class For(VyperNode): + __slots__ = ('iter', 'target', 'orelse', 'body') + + +class AugAssign(VyperNode): + __slots__ = ('op', 'target', 'value') + + +class Break(VyperNode): + pass + + +class Continue(VyperNode): + pass + + +class Return(VyperNode): + __slots__ = ('value', ) + + +class Delete(VyperNode): + pass + + +class stmt(VyperNode): + pass + + +class ClassDef(VyperNode): + __slots__ = ('class_type', 'name', 'body') + + +class ImportFrom(VyperNode): + pass + + +class Raise(VyperNode): + pass + + +class Slice(VyperNode): + pass diff --git a/vyper/ast_utils.py b/vyper/ast_utils.py new file mode 100644 index 0000000000..e9f54921d4 --- /dev/null +++ b/vyper/ast_utils.py @@ -0,0 +1,91 @@ +import ast as python_ast + +import vyper.ast as vyper_ast +from vyper.exceptions import ( + ParserException, + SyntaxException, +) +from vyper.parser.parser_utils import ( + annotate_and_optimize_ast, +) +from vyper.parser.pre_parser import ( + pre_parse, +) + + +def parse_python_ast(source_code, node): + if isinstance(node, list): + o = [] + for n in node: + o.append( + parse_python_ast( + source_code=source_code, + node=n, + ) + ) + return o + elif isinstance(node, python_ast.AST): + class_name = node.__class__.__name__ + if hasattr(vyper_ast, class_name): + vyper_class = getattr(vyper_ast, class_name) + init_kwargs = { + 'col_offset': getattr(node, 'col_offset', None), + 'lineno': getattr(node, 'lineno', None), + 'node_id': node.node_id + } + if isinstance(node, python_ast.ClassDef): + init_kwargs['class_type'] = node.class_type + for field_name in node._fields: + val = getattr(node, field_name) + if field_name in vyper_class.ignored_fields: + continue + elif val and field_name in vyper_class.only_empty_fields: + raise SyntaxException( + f'"{field_name}" is an unsupported attribute field ' + f'on Python AST "{class_name}" class.', node + ) + else: + init_kwargs[field_name] = parse_python_ast( + source_code=source_code, + node=val, + ) + return vyper_class(**init_kwargs) + else: + raise SyntaxException( + f'Invalid syntax (unsupported "{class_name}" Python AST node).', node + ) + else: + return node + + +def parse_to_ast(source_code): + class_types, reformatted_code = pre_parse(source_code) + if '\x00' in reformatted_code: + raise ParserException('No null bytes (\\x00) allowed in the source code.') + py_ast = python_ast.parse(reformatted_code) + annotate_and_optimize_ast(py_ast, source_code, class_types) + # Convert to Vyper AST. + vyper_ast = parse_python_ast( + source_code=source_code, + node=py_ast, + ) + return vyper_ast.body + + +def ast_to_dict(node): + skip_list = ('source_code', ) + if isinstance(node, vyper_ast.VyperNode): + o = { + f: ast_to_dict(getattr(node, f)) + for f in node.get_slots() + if f not in skip_list + } + o.update({'ast_type': node.__class__.__name__}) + return o + elif isinstance(node, list): + return [ + ast_to_dict(x) + for x in node + ] + else: + return node From 6ed5805f51b70ce22b746d5c7a2bc45e87cbefaf Mon Sep 17 00:00:00 2001 From: Jacques Wagener Date: Wed, 3 Apr 2019 17:10:34 +0200 Subject: [PATCH 02/23] Fix ast.Name statement to use kwargs instead. --- vyper/signatures/interface.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/vyper/signatures/interface.py b/vyper/signatures/interface.py index ee4fa511a4..18ad9ba5c7 100644 --- a/vyper/signatures/interface.py +++ b/vyper/signatures/interface.py @@ -1,4 +1,4 @@ -import ast +from vyper import ast import copy import importlib import os @@ -48,17 +48,17 @@ def render_return(sig): def abi_type_to_ast(atype): if atype in ('int128', 'uint256', 'bool', 'address', 'bytes32'): - return ast.Name(atype, None) + return ast.Name(id=atype) elif atype == 'decimal': - return ast.Name('int128', None) + return ast.Name(id='int128') elif atype == 'bytes': return ast.Subscript( - value=ast.Name('bytes', None), + value=ast.Name(id='bytes'), slice=ast.Index(256) ) elif atype == 'string': return ast.Subscript( - value=ast.Name('string', None), + value=ast.Name(id='string'), slice=ast.Index(256) ) else: @@ -91,11 +91,11 @@ def mk_full_signature_from_json(abi): ] ) - decorator_list = [ast.Name('public', None)] + decorator_list = [ast.Name(id='public')] if func['constant']: - decorator_list.append(ast.Name('constant', None)) + decorator_list.append(ast.Name(id='constant')) if func['payable']: - decorator_list.append(ast.Name('payable', None)) + decorator_list.append(ast.Name(id='payable')) sig = FunctionSignature.from_definition( code=ast.FunctionDef( From f356eca3f081539a3cca5b57d4125080cfe4ff50 Mon Sep 17 00:00:00 2001 From: Jacques Wagener Date: Thu, 4 Apr 2019 11:04:44 +0200 Subject: [PATCH 03/23] Use python_ast for annotate and optimise tests. --- .../test_annotate_and_optimize_ast.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/parser/parser_utils/test_annotate_and_optimize_ast.py b/tests/parser/parser_utils/test_annotate_and_optimize_ast.py index 4a51d2e32e..6593ce882a 100644 --- a/tests/parser/parser_utils/test_annotate_and_optimize_ast.py +++ b/tests/parser/parser_utils/test_annotate_and_optimize_ast.py @@ -1,5 +1,8 @@ -import ast +import ast as python_ast +from vyper.ast_utils import ( + parse_to_ast, +) from vyper.parser.parser_utils import ( annotate_and_optimize_ast, ) @@ -8,7 +11,7 @@ ) -class AssertionVisitor(ast.NodeVisitor): +class AssertionVisitor(python_ast.NodeVisitor): def assert_about_node(self, node): assert False @@ -34,11 +37,11 @@ def foo() -> int128: def get_contract_info(source_code): class_types, reformatted_code = pre_parse(source_code) - parsed_ast = ast.parse(reformatted_code) + py_ast = python_ast.parse(reformatted_code) - annotate_and_optimize_ast(parsed_ast, reformatted_code, class_types) + annotate_and_optimize_ast(py_ast, reformatted_code, class_types) - return parsed_ast, reformatted_code + return py_ast, reformatted_code def test_it_annotates_ast_with_source_code(): @@ -67,5 +70,5 @@ def test_it_rewrites_unary_subtractions(): function_def = contract_ast.body[2] return_stmt = function_def.body[0] - assert isinstance(return_stmt.value, ast.Num) + assert isinstance(return_stmt.value, python_ast.Num) assert return_stmt.value.n == -1 From 86bea461b45d53db32c172ddc549f4fa0aba22ee Mon Sep 17 00:00:00 2001 From: Jacques Wagener Date: Thu, 4 Apr 2019 11:06:03 +0200 Subject: [PATCH 04/23] Fix typo in test name. --- tests/examples/market_maker/test_on_chain_market_maker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/examples/market_maker/test_on_chain_market_maker.py b/tests/examples/market_maker/test_on_chain_market_maker.py index 376c256907..36f1d1d3ca 100644 --- a/tests/examples/market_maker/test_on_chain_market_maker.py +++ b/tests/examples/market_maker/test_on_chain_market_maker.py @@ -25,7 +25,7 @@ def erc20(get_contract): ) -def test_initial_statet(market_maker): +def test_initial_state(market_maker): assert market_maker.totalEthQty() == 0 assert market_maker.totalTokenQty() == 0 assert market_maker.invariant() == 0 From a3769093979d335ec21618425af9cd2f71d96484 Mon Sep 17 00:00:00 2001 From: Jacques Wagener Date: Thu, 4 Apr 2019 11:20:53 +0200 Subject: [PATCH 05/23] Replace ast.dump. --- vyper/parser/stmt.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/vyper/parser/stmt.py b/vyper/parser/stmt.py index 61d5153ac7..bdd3f8ce2f 100644 --- a/vyper/parser/stmt.py +++ b/vyper/parser/stmt.py @@ -1,6 +1,9 @@ -import ast import re +from vyper import ast +from vyper.ast_utils import ( + ast_to_dict, +) from vyper.exceptions import ( ConstancyViolationException, EventDeclarationException, @@ -348,7 +351,7 @@ def parse_if(self): def _clear(self): # Create zero node - none = ast.NameConstant(None) + none = ast.NameConstant(value=None) none.lineno = self.stmt.lineno none.col_offset = self.stmt.col_offset zero = Expr(none, self.context).lll_node @@ -554,15 +557,15 @@ def parse_for(self): arg1, ) - if ast.dump(arg0) != ast.dump(arg1.left): + if arg0 != arg1.left: raise StructureException( ( "Two-arg for statements of the form `for i in " "range(x, x + y): ...` must have x identical in both " "places: %r %r" ) % ( - ast.dump(arg0), - ast.dump(arg1.left) + ast_to_dict(arg0), + ast_to_dict(arg1.left) ), self.stmt.iter, ) From 7613bc7265bb7ca9da1418e9efbac64f3fcfdb89 Mon Sep 17 00:00:00 2001 From: Jacques Wagener Date: Thu, 4 Apr 2019 11:21:06 +0200 Subject: [PATCH 06/23] Move test cases to use SyntaxException. --- .../test_invalid_literal_exception.py | 5 -- .../exceptions/test_invalid_type_exception.py | 9 --- .../exceptions/test_structure_exception.py | 23 -------- .../exceptions/test_syntax_exception.py | 55 +++++++++++++++++++ .../test_annotate_and_optimize_ast.py | 3 - tests/parser/syntax/test_no_none.py | 3 +- tests/parser/syntax/utils/test_event_names.py | 3 - 7 files changed, 57 insertions(+), 44 deletions(-) create mode 100644 tests/parser/exceptions/test_syntax_exception.py diff --git a/tests/parser/exceptions/test_invalid_literal_exception.py b/tests/parser/exceptions/test_invalid_literal_exception.py index 78bc366f25..49bdac0d27 100644 --- a/tests/parser/exceptions/test_invalid_literal_exception.py +++ b/tests/parser/exceptions/test_invalid_literal_exception.py @@ -89,11 +89,6 @@ def foo(): """, """ @public -def foo(): - x = convert(-1, uint256) - """, - """ -@public def foo(): x = convert(-(-(-1)), uint256) """, diff --git a/tests/parser/exceptions/test_invalid_type_exception.py b/tests/parser/exceptions/test_invalid_type_exception.py index 39bd9eb09f..edebf42f25 100644 --- a/tests/parser/exceptions/test_invalid_type_exception.py +++ b/tests/parser/exceptions/test_invalid_type_exception.py @@ -43,9 +43,6 @@ def foo(x): pass b: map((int128, decimal), int128) """, """ -b: int128[int128: address] - """, - """ x: wei(wei) """, """ @@ -61,18 +58,12 @@ def foo(x): pass x: int128(wei ** -1) """, """ -x: int128(wei >> 3) - """, - """ x: bytes <= wei """, """ x: string <= 33 """, """ -x: bytes[1:3] - """, - """ x: bytes[33.3] """, """ diff --git a/tests/parser/exceptions/test_structure_exception.py b/tests/parser/exceptions/test_structure_exception.py index 4073f26ca7..3f24b05157 100644 --- a/tests/parser/exceptions/test_structure_exception.py +++ b/tests/parser/exceptions/test_structure_exception.py @@ -27,24 +27,6 @@ def foo(): pass send(0x1234567890123456789012345678901234567890, 5) """, """ -x: int128[5] -@public -def foo(): - self.x[2:4] = 3 - """, - """ -x: int128[5] -@public -def foo(): - z = self.x[2:4] - """, - """ -@public -def foo(): - x: int128[5] - z = x[2:4] - """, - """ @public def foo(): x: int128 = 5 @@ -147,11 +129,6 @@ def foo() -> int128: """, """ @public -def foo(): - x: address = ~self - """, - """ -@public def foo(): x = concat(b"") """, diff --git a/tests/parser/exceptions/test_syntax_exception.py b/tests/parser/exceptions/test_syntax_exception.py new file mode 100644 index 0000000000..05f9cfb3f1 --- /dev/null +++ b/tests/parser/exceptions/test_syntax_exception.py @@ -0,0 +1,55 @@ +import pytest +from pytest import ( + raises, +) + +from vyper import ( + compiler, +) +from vyper.exceptions import ( + SyntaxException, +) + +fail_list = [ + """ +x: bytes[1:3] + """, + """ +b: int128[int128: address] + """, + """ +x: int128[5] +@public +def foo(): + self.x[2:4] = 3 + """, + """ +@public +def foo(): + x: address = ~self + """, + """ +x: int128[5] +@public +def foo(): + z = self.x[2:4] + """, + """ +@public +def foo(): + x: int128[5] + z = x[2:4] + """, + """ +x: int128(wei >> 3) + """, + """ +Transfer: event({_&rom: indexed(address)}) + """, +] + + +@pytest.mark.parametrize('bad_code', fail_list) +def test_syntax_exception(bad_code): + with raises(SyntaxException): + compiler.compile_code(bad_code) diff --git a/tests/parser/parser_utils/test_annotate_and_optimize_ast.py b/tests/parser/parser_utils/test_annotate_and_optimize_ast.py index 6593ce882a..e196ce78f5 100644 --- a/tests/parser/parser_utils/test_annotate_and_optimize_ast.py +++ b/tests/parser/parser_utils/test_annotate_and_optimize_ast.py @@ -1,8 +1,5 @@ import ast as python_ast -from vyper.ast_utils import ( - parse_to_ast, -) from vyper.parser.parser_utils import ( annotate_and_optimize_ast, ) diff --git a/tests/parser/syntax/test_no_none.py b/tests/parser/syntax/test_no_none.py index e224963fa5..fc42658254 100644 --- a/tests/parser/syntax/test_no_none.py +++ b/tests/parser/syntax/test_no_none.py @@ -1,5 +1,6 @@ from vyper.exceptions import ( InvalidLiteralException, + SyntaxException, ) @@ -123,7 +124,7 @@ def foo(): for contract in contracts: assert_compile_failed( lambda: get_contract_with_gas_estimation(contract), - InvalidLiteralException + SyntaxException ) diff --git a/tests/parser/syntax/utils/test_event_names.py b/tests/parser/syntax/utils/test_event_names.py index 834901547e..0b1d9d9b1d 100644 --- a/tests/parser/syntax/utils/test_event_names.py +++ b/tests/parser/syntax/utils/test_event_names.py @@ -61,9 +61,6 @@ def foo(i: int128) -> int128: Transfer: eve.t({_from: indexed(address)}) """, InvalidTypeException), """ -Transfer: event({_&rom: indexed(address)}) - """, - """ Transfer: event({_from: i.dexed(address), _to: indexed(address),lue: uint256}) """ ] From 1cc1af1e425f32bd75fae419d4db71a486b7a822 Mon Sep 17 00:00:00 2001 From: Jacques Wagener Date: Thu, 4 Apr 2019 11:21:44 +0200 Subject: [PATCH 07/23] Finalise the AST layout. --- vyper/ast.py | 58 ++++++++++++++++++++++++++++++++++------------------ 1 file changed, 38 insertions(+), 20 deletions(-) diff --git a/vyper/ast.py b/vyper/ast.py index d6e7af230e..b469a27e1a 100644 --- a/vyper/ast.py +++ b/vyper/ast.py @@ -1,6 +1,10 @@ from itertools import ( chain, ) +from typing import ( + Any, + List as ListTyping, +) from vyper.exceptions import ( CompilerPanic, @@ -9,8 +13,8 @@ class VyperNode: __slots__ = ('node_id', 'source_code', 'col_offset', 'lineno') - ignored_fields = ('ctx', ) - only_empty_fields = () + ignored_fields = ['ctx', ] + only_empty_fields: ListTyping[Any] = [] @classmethod def get_slots(cls): @@ -24,11 +28,21 @@ def __init__(self, **kwargs): for field_name, value in kwargs.items(): if field_name in self.get_slots(): setattr(self, field_name, value) - # elif value: - # raise CompilerPanic( - # f'Unsupported non-empty value field_name: {field_name}, ' - # f' class: {type(self)} value: {value}' - # ) + elif value: + raise CompilerPanic( + f'Unsupported non-empty value field_name: {field_name}, ' + f' class: {type(self)} value: {value}' + ) + + def __eq__(self, other): + if isinstance(other, type(self)): + for field_name in self.get_slots(): + if field_name not in ('node_id', 'source_code', 'col_offset', 'lineno'): + if getattr(self, field_name, None) != getattr(other, field_name, None): + return False + return True + else: + return False class Module(VyperNode): @@ -61,11 +75,11 @@ class FunctionDef(VyperNode): class arguments(VyperNode): __slots__ = ('args', 'defaults', 'default') - only_empty_fields = ('vararg', 'kwonlyargs', 'kwarg', 'kw_defaults') + only_empty_fields = ['vararg', 'kwonlyargs', 'kwarg', 'kw_defaults'] class Import(VyperNode): - pass + __slots__ = ('names', ) class Call(VyperNode): @@ -100,16 +114,16 @@ class Op(VyperNode): __slots__ = ('op', 'left', 'right') -class BinOp(Op): - pass +class BoolOp(Op): + __slots__ = ('values', ) -class BoolOp(Op): +class BinOp(Op): pass class UnaryOp(Op): - operand = ('operand', ) + __slots__ = ('operand', ) class List(VyperNode): @@ -237,7 +251,7 @@ class Return(VyperNode): class Delete(VyperNode): - pass + __slots__ = ('targets', ) class stmt(VyperNode): @@ -248,13 +262,17 @@ class ClassDef(VyperNode): __slots__ = ('class_type', 'name', 'body') -class ImportFrom(VyperNode): - pass - - class Raise(VyperNode): - pass + __slots__ = ('exc', ) class Slice(VyperNode): - pass + only_empty_fields = ['lower'] + + +class alias(VyperNode): + __slots__ = ('name', 'asname') + + +class ImportFrom(VyperNode): + __slots__ = ('module', 'names') From 6239d2925c5605be97b564eb9492e48804b7c150 Mon Sep 17 00:00:00 2001 From: Jacques Wagener Date: Thu, 4 Apr 2019 11:22:20 +0200 Subject: [PATCH 08/23] Improve SyntaxException message. --- vyper/ast_utils.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/vyper/ast_utils.py b/vyper/ast_utils.py index e9f54921d4..792588fb97 100644 --- a/vyper/ast_utils.py +++ b/vyper/ast_utils.py @@ -31,7 +31,8 @@ def parse_python_ast(source_code, node): init_kwargs = { 'col_offset': getattr(node, 'col_offset', None), 'lineno': getattr(node, 'lineno', None), - 'node_id': node.node_id + 'node_id': node.node_id, + 'source_code': source_code } if isinstance(node, python_ast.ClassDef): init_kwargs['class_type'] = node.class_type @@ -41,8 +42,10 @@ def parse_python_ast(source_code, node): continue elif val and field_name in vyper_class.only_empty_fields: raise SyntaxException( + 'Invalid Vyper Syntax. ' f'"{field_name}" is an unsupported attribute field ' - f'on Python AST "{class_name}" class.', node + f'on Python AST "{class_name}" class.', + val ) else: init_kwargs[field_name] = parse_python_ast( From cce3f06f840a09678cf455a1cd75124431423cde Mon Sep 17 00:00:00 2001 From: Jacques Wagener Date: Thu, 4 Apr 2019 11:22:42 +0200 Subject: [PATCH 09/23] Add simple -f ast_json output. --- vyper/compiler.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/vyper/compiler.py b/vyper/compiler.py index fbc6ab3013..94f40dc8f6 100644 --- a/vyper/compiler.py +++ b/vyper/compiler.py @@ -2,11 +2,15 @@ OrderedDict, deque, ) +import json from vyper import ( compile_lll, optimizer, ) +from vyper.ast_utils import ( + ast_to_dict, +) from vyper.opcodes import ( opcodes, ) @@ -186,8 +190,17 @@ def _mk_opcodes_runtime(code, contract_name, interface_codes): return get_opcodes(code, contract_name, bytecodes_runtime=True, interface_codes=interface_codes) +def _mk_ast_json(code, contract_name, interface_codes): + o = { + 'contract_name': contract_name, + 'ast': ast_to_dict(parser.parse_to_ast(code)) + } + return json.dumps(o) + + output_formats_map = { 'abi': _mk_abi_output, + 'ast_json': _mk_ast_json, 'bytecode': _mk_bytecode_output, 'bytecode_runtime': _mk_bytecode_runtime_output, 'ir': _mk_ir_output, From 95f7023bca5c1533ef12632cd7d4a6e6388e0468 Mon Sep 17 00:00:00 2001 From: Jacques Wagener Date: Thu, 4 Apr 2019 11:24:39 +0200 Subject: [PATCH 10/23] Add SyntaxException. --- vyper/exceptions.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/vyper/exceptions.py b/vyper/exceptions.py index 0e23a018a8..5acff7ca83 100644 --- a/vyper/exceptions.py +++ b/vyper/exceptions.py @@ -84,6 +84,10 @@ class VersionException(ParserException): pass +class SyntaxException(ParserException): + pass + + class CompilerPanic(Exception): def __init__(self, message): From b36506cad60d21de0ddbdc4311296aa7d10d51d3 Mon Sep 17 00:00:00 2001 From: Jacques Wagener Date: Thu, 4 Apr 2019 11:25:19 +0200 Subject: [PATCH 11/23] Rename ast to python_ast to be explicit. --- vyper/parser/parser_utils.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/vyper/parser/parser_utils.py b/vyper/parser/parser_utils.py index 9ca2a999c5..6ad4cb1733 100644 --- a/vyper/parser/parser_utils.py +++ b/vyper/parser/parser_utils.py @@ -1,8 +1,9 @@ -import ast +import ast as python_ast from typing import ( Optional, ) +from vyper import ast from vyper.exceptions import ( InvalidLiteralException, StructureException, @@ -709,12 +710,13 @@ def make_setter(left, right, location, pos, in_function_call=False): raise Exception("Invalid type for setters") -class AnnotatingVisitor(ast.NodeTransformer): +class AnnotatingVisitor(python_ast.NodeTransformer): _source_code: str _class_types: ClassTypes def __init__(self, source_code: str, class_types: Optional[ClassTypes] = None): - self._source_code = source_code + self._source_code: str = source_code + self.counter: int = 0 if class_types is not None: self._class_types = class_types else: @@ -724,6 +726,8 @@ def generic_visit(self, node): # Decorate every node in the AST with the original source code. This is # necessary to facilitate error pretty-printing. node.source_code = self._source_code + node.node_id = self.counter + self.counter += 1 return super().generic_visit(node) @@ -736,11 +740,10 @@ def visit_ClassDef(self, node): return node -class RewriteUnarySubVisitor(ast.NodeTransformer): +class RewriteUnarySubVisitor(python_ast.NodeTransformer): def visit_UnaryOp(self, node): self.generic_visit(node) - - if isinstance(node.op, ast.USub) and isinstance(node.operand, ast.Num): + if isinstance(node.op, python_ast.USub) and isinstance(node.operand, python_ast.Num): node.operand.n = 0 - node.operand.n return node.operand else: @@ -748,7 +751,7 @@ def visit_UnaryOp(self, node): def annotate_and_optimize_ast( - parsed_ast: ast.Module, + parsed_ast: python_ast.Module, source_code: str, class_types: Optional[ClassTypes] = None, ) -> None: From b60576cffa1116a73720fd50ac3988b62058744c Mon Sep 17 00:00:00 2001 From: Jacques Wagener Date: Thu, 4 Apr 2019 11:27:18 +0200 Subject: [PATCH 12/23] Remove ast.dump exceptions. --- vyper/types/types.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/vyper/types/types.py b/vyper/types/types.py index d0a94840c0..3e5703da9f 100644 --- a/vyper/types/types.py +++ b/vyper/types/types.py @@ -1,5 +1,4 @@ import abc -import ast from collections import ( OrderedDict, ) @@ -9,6 +8,7 @@ ) import warnings +from vyper import ast from vyper.exceptions import ( InvalidTypeException, ) @@ -426,8 +426,7 @@ def parse_type(item, location, sigs=None, custom_units=None, custom_structs=None return BaseType(base_type, unit, positional) # Subscripts elif isinstance(item, ast.Subscript): - - if 'value' not in vars(item.slice): + if isinstance(item.slice, ast.Slice): raise InvalidTypeException( "Array / ByteArray access must access a single element, not a slice", item, @@ -475,7 +474,7 @@ def parse_type(item, location, sigs=None, custom_units=None, custom_structs=None " favor of named structs, see VIP300", DeprecationWarning ) - raise InvalidTypeException("Invalid type: %r" % ast.dump(item), item) + raise InvalidTypeException("Invalid type", item) elif isinstance(item, ast.Tuple): members = [ parse_type( @@ -488,7 +487,7 @@ def parse_type(item, location, sigs=None, custom_units=None, custom_structs=None ] return TupleType(members) else: - raise InvalidTypeException("Invalid type: %r" % ast.dump(item), item) + raise InvalidTypeException("Invalid type", item) # Gets the number of memory or storage keys needed to represent a given type From e504fe3e3d8d75d03e024a38556faebf80540be3 Mon Sep 17 00:00:00 2001 From: Jacques Wagener Date: Thu, 4 Apr 2019 11:27:58 +0200 Subject: [PATCH 13/23] Replace import ast with vyper ast instead. --- vyper/functions/functions.py | 2 +- vyper/functions/signatures.py | 7 ++++-- vyper/parser/constants.py | 2 +- vyper/parser/expr.py | 14 +++++------ vyper/parser/external_call.py | 5 ++-- vyper/parser/global_context.py | 11 ++++----- vyper/parser/parser.py | 34 +++----------------------- vyper/signatures/event_signature.py | 3 +-- vyper/signatures/function_signature.py | 2 +- vyper/signatures/interface.py | 2 +- vyper/types/convert.py | 2 +- 11 files changed, 29 insertions(+), 55 deletions(-) diff --git a/vyper/functions/functions.py b/vyper/functions/functions.py index f468f9077e..ab3751ef8e 100644 --- a/vyper/functions/functions.py +++ b/vyper/functions/functions.py @@ -1,6 +1,6 @@ -import ast import hashlib +from vyper import ast from vyper.exceptions import ( ConstancyViolationException, InvalidLiteralException, diff --git a/vyper/functions/signatures.py b/vyper/functions/signatures.py index 60751cadd9..223cdb27eb 100644 --- a/vyper/functions/signatures.py +++ b/vyper/functions/signatures.py @@ -1,6 +1,9 @@ -import ast import functools +from vyper import ast +from vyper.ast_utils import ( + parse_to_ast, +) from vyper.exceptions import ( InvalidLiteralException, StructureException, @@ -74,7 +77,7 @@ def process_arg(index, arg, expected_arg_typelist, function_name, context): else: # Does not work for unit-endowed types inside compound types, e.g. timestamp[2] parsed_expected_type = context.parse_type( - ast.parse(expected_arg).body[0].value, + parse_to_ast(expected_arg)[0].value, 'memory', ) if isinstance(parsed_expected_type, BaseType): diff --git a/vyper/parser/constants.py b/vyper/parser/constants.py index 9a7e1cd542..c38a5327d2 100644 --- a/vyper/parser/constants.py +++ b/vyper/parser/constants.py @@ -1,6 +1,6 @@ -import ast import copy +from vyper import ast from vyper.exceptions import ( StructureException, TypeMismatchException, diff --git a/vyper/parser/expr.py b/vyper/parser/expr.py index 7eed3bed33..fce117bfff 100644 --- a/vyper/parser/expr.py +++ b/vyper/parser/expr.py @@ -1,6 +1,6 @@ -import ast import warnings +from vyper import ast from vyper.exceptions import ( InvalidLiteralException, NonPayableViolationException, @@ -77,7 +77,7 @@ def __init__(self, expr, context): if expr_type in self.expr_table: self.lll_node = self.expr_table[expr_type]() else: - raise Exception("Unsupported operator: %r" % ast.dump(self.expr)) + raise Exception("Unsupported operator.", self.expr) def get_expr(self): return self.expr @@ -397,7 +397,7 @@ def attribute(self): def subscript(self): sub = Expr.parse_variable_location(self.expr.value, self.context) if isinstance(sub.typ, (MappingType, ListType)): - if 'value' not in vars(self.expr.slice): + if not isinstance(self.expr.slice, ast.Index): raise StructureException( "Array access must access a single element, not a slice", self.expr, @@ -466,7 +466,7 @@ def arithmetic(self): self.expr, ) - num = ast.Num(val) + num = ast.Num(n=val) num.source_code = self.expr.source_code num.lineno = self.expr.lineno num.col_offset = self.expr.col_offset @@ -964,7 +964,7 @@ def unary_operations(self): ) if operand.typ.is_literal and 'int' in operand.typ.typ: - num = ast.Num(0 - operand.value) + num = ast.Num(n=0 - operand.value) num.source_code = self.expr.source_code num.lineno = self.expr.lineno num.col_offset = self.expr.col_offset @@ -1069,7 +1069,7 @@ def dict_fail(self): " favor of named structs, see VIP300", DeprecationWarning ) - raise InvalidLiteralException("Invalid literal: %r" % ast.dump(self.expr), self.expr) + raise InvalidLiteralException("Invalid literal.", self.expr) @staticmethod def struct_literals(expr, name, context): @@ -1078,7 +1078,7 @@ def struct_literals(expr, name, context): for key, value in zip(expr.keys, expr.values): if not isinstance(key, ast.Name): raise TypeMismatchException( - "Invalid member variable for struct: %r" % vars(key).get('id', key), + "Invalid member variable for struct: %r" % getattr(key, 'id', ''), key, ) check_valid_varname( diff --git a/vyper/parser/external_call.py b/vyper/parser/external_call.py index cf96ea2747..b4990638f9 100644 --- a/vyper/parser/external_call.py +++ b/vyper/parser/external_call.py @@ -1,5 +1,4 @@ -import ast - +from vyper import ast from vyper.exceptions import ( FunctionDeclarationException, StructureException, @@ -173,4 +172,4 @@ def make_external_call(stmt_expr, context): ) else: - raise StructureException("Unsupported operator: %r" % ast.dump(stmt_expr), stmt_expr) + raise StructureException("Unsupported operator.", stmt_expr) diff --git a/vyper/parser/global_context.py b/vyper/parser/global_context.py index 8e1e86af1f..f873f4207d 100644 --- a/vyper/parser/global_context.py +++ b/vyper/parser/global_context.py @@ -1,5 +1,7 @@ -import ast - +from vyper import ast +from vyper.ast_utils import ( + parse_to_ast, +) from vyper.exceptions import ( EventDeclarationException, FunctionDeclarationException, @@ -12,7 +14,6 @@ Constants, ) from vyper.parser.parser_utils import ( - annotate_and_optimize_ast, getpos, ) from vyper.signatures.function_signature import ( @@ -243,9 +244,7 @@ def mk_getter(cls, varname, typ): # Parser for a single line @staticmethod def parse_line(code): - parsed_ast = ast.parse(code).body[0] - annotate_and_optimize_ast(parsed_ast, code) - + parsed_ast = parse_to_ast(code)[0] return parsed_ast # A struct is a list of members diff --git a/vyper/parser/parser.py b/vyper/parser/parser.py index 56c6cac703..18ccae70b2 100644 --- a/vyper/parser/parser.py +++ b/vyper/parser/parser.py @@ -1,14 +1,13 @@ -import ast import functools -from typing import ( - List, -) +from vyper import ast +from vyper.ast_utils import ( + parse_to_ast, +) from vyper.exceptions import ( EventDeclarationException, FunctionDeclarationException, InvalidLiteralException, - ParserException, StructureException, TypeMismatchException, ) @@ -29,7 +28,6 @@ MemoryAllocator, ) from vyper.parser.parser_utils import ( - annotate_and_optimize_ast, base_type_conversion, byte_array_to_num, getpos, @@ -37,9 +35,6 @@ make_setter, unwrap_location, ) -from vyper.parser.pre_parser import ( - pre_parse, -) from vyper.parser.stmt import ( Stmt, ) @@ -76,27 +71,6 @@ raise Exception("Requires python 3.6 or higher for annotation support") -def parse_to_ast(source_code: str) -> List[ast.stmt]: - """ - Parses the given vyper source code and returns a list of python AST objects - for all statements in the source. Performs pre-processing of source code - before parsing as well as post-processing of the resulting AST. - - :param source_code: The vyper source code to be parsed. - :return: The post-processed list of python AST objects for each statement in - ``source_code``. - """ - class_types, reformatted_code = pre_parse(source_code) - - if '\x00' in reformatted_code: - raise ParserException('No null bytes (\\x00) allowed in the source code.') - - parsed_ast = ast.parse(reformatted_code) - annotate_and_optimize_ast(parsed_ast, reformatted_code, class_types) - - return parsed_ast.body - - # Header code initializer_list = ['seq', ['mstore', 28, ['calldataload', 0]]] # Store limit constants at fixed addresses in memory. diff --git a/vyper/signatures/event_signature.py b/vyper/signatures/event_signature.py index 99e64a4f9c..bc289ae792 100644 --- a/vyper/signatures/event_signature.py +++ b/vyper/signatures/event_signature.py @@ -1,5 +1,4 @@ -import ast - +from vyper import ast from vyper.exceptions import ( EventDeclarationException, InvalidTypeException, diff --git a/vyper/signatures/function_signature.py b/vyper/signatures/function_signature.py index 2830dda193..6861d12c93 100644 --- a/vyper/signatures/function_signature.py +++ b/vyper/signatures/function_signature.py @@ -1,8 +1,8 @@ -import ast from collections import ( Counter, ) +from vyper import ast from vyper.exceptions import ( FunctionDeclarationException, InvalidTypeException, diff --git a/vyper/signatures/interface.py b/vyper/signatures/interface.py index 18ad9ba5c7..da22cff875 100644 --- a/vyper/signatures/interface.py +++ b/vyper/signatures/interface.py @@ -1,9 +1,9 @@ -from vyper import ast import copy import importlib import os import pkgutil +from vyper import ast from vyper.exceptions import ( ParserException, StructureException, diff --git a/vyper/types/convert.py b/vyper/types/convert.py index 55afe83094..989e2714bb 100644 --- a/vyper/types/convert.py +++ b/vyper/types/convert.py @@ -1,7 +1,7 @@ -import ast import math import warnings +from vyper import ast from vyper.exceptions import ( InvalidLiteralException, ParserException, From 0cda1acd4f3c9b3b232b7db96f6846f8b0c58ea9 Mon Sep 17 00:00:00 2001 From: Jacques Wagener Date: Thu, 4 Apr 2019 14:57:20 +0200 Subject: [PATCH 14/23] Add ast_to_string and to_python_ast. --- vyper/ast_utils.py | 51 ++++++++++++++++++++++++++++++++++++++++++++-- vyper/compiler.py | 7 +++---- 2 files changed, 52 insertions(+), 6 deletions(-) diff --git a/vyper/ast_utils.py b/vyper/ast_utils.py index 792588fb97..13c6b0997f 100644 --- a/vyper/ast_utils.py +++ b/vyper/ast_utils.py @@ -13,7 +13,7 @@ ) -def parse_python_ast(source_code, node): +def parse_python_ast(source_code: str, node: python_ast.AST): if isinstance(node, list): o = [] for n in node: @@ -79,7 +79,7 @@ def ast_to_dict(node): skip_list = ('source_code', ) if isinstance(node, vyper_ast.VyperNode): o = { - f: ast_to_dict(getattr(node, f)) + f: ast_to_dict(getattr(node, f, None)) for f in node.get_slots() if f not in skip_list } @@ -92,3 +92,50 @@ def ast_to_dict(node): ] else: return node + + +def dict_to_ast(ast_struct): + if isinstance(ast_struct, dict) and 'ast_type' in ast_struct: + vyper_class = getattr(vyper_ast, ast_struct['ast_type']) + klass = vyper_class(**{ + k: dict_to_ast(v) + for k, v in ast_struct.items() + if k in vyper_class.get_slots() + }) + return klass + elif isinstance(ast_struct, list): + return [ + dict_to_ast(x) + for x in ast_struct + ] + else: + return ast_struct + + +def to_python_ast(vyper_ast_node): + if isinstance(vyper_ast_node, list): + return [ + to_python_ast(n) + for n in vyper_ast_node + ] + elif isinstance(vyper_ast_node, vyper_ast.VyperNode): + class_name = vyper_ast_node.__class__.__name__ + if hasattr(python_ast, class_name): + py_klass = getattr(python_ast, class_name) + return py_klass(**{ + k: to_python_ast( + getattr(vyper_ast_node, k, None) + ) + for k in vyper_ast_node.get_slots() + }) + else: + return vyper_ast_node + + +def ast_to_string(vyper_ast_node): + py_ast_node = to_python_ast(vyper_ast_node) + return python_ast.dump( + python_ast.Module( + body=py_ast_node + ) + ) diff --git a/vyper/compiler.py b/vyper/compiler.py index 94f40dc8f6..45ec10a9cf 100644 --- a/vyper/compiler.py +++ b/vyper/compiler.py @@ -2,7 +2,6 @@ OrderedDict, deque, ) -import json from vyper import ( compile_lll, @@ -190,17 +189,17 @@ def _mk_opcodes_runtime(code, contract_name, interface_codes): return get_opcodes(code, contract_name, bytecodes_runtime=True, interface_codes=interface_codes) -def _mk_ast_json(code, contract_name, interface_codes): +def _mk_ast_dict(code, contract_name, interface_codes): o = { 'contract_name': contract_name, 'ast': ast_to_dict(parser.parse_to_ast(code)) } - return json.dumps(o) + return o output_formats_map = { 'abi': _mk_abi_output, - 'ast_json': _mk_ast_json, + 'ast_dict': _mk_ast_dict, 'bytecode': _mk_bytecode_output, 'bytecode_runtime': _mk_bytecode_runtime_output, 'ir': _mk_ir_output, From 7bbcfda2b7957916dabd3c9f0aaf3cb40ab60a3d Mon Sep 17 00:00:00 2001 From: Jacques Wagener Date: Thu, 4 Apr 2019 15:04:35 +0200 Subject: [PATCH 15/23] Add ast_utils tests. --- tests/parser/ast_utils/test_ast.py | 37 +++++++++ tests/parser/ast_utils/test_ast_dict.py | 81 ++++++++++++++++++++ tests/parser/ast_utils/test_ast_to_string.py | 18 +++++ vyper/ast_utils.py | 2 +- 4 files changed, 137 insertions(+), 1 deletion(-) create mode 100644 tests/parser/ast_utils/test_ast.py create mode 100644 tests/parser/ast_utils/test_ast_dict.py create mode 100644 tests/parser/ast_utils/test_ast_to_string.py diff --git a/tests/parser/ast_utils/test_ast.py b/tests/parser/ast_utils/test_ast.py new file mode 100644 index 0000000000..416fde41fa --- /dev/null +++ b/tests/parser/ast_utils/test_ast.py @@ -0,0 +1,37 @@ +from vyper.ast_utils import ( + parse_to_ast, +) + + +def test_ast_equal(): + code = """ +@public +def test() -> int128: + a: uint256 = 100 + return 123 + """ + + ast1 = parse_to_ast(code) + ast2 = parse_to_ast("\n \n" + code + "\n\n") + + assert ast1 == ast2 + + +def test_ast_unequal(): + code1 = """ +@public +def test() -> int128: + a: uint256 = 100 + return 123 + """ + code2 = """ +@public +def test() -> int128: + a: uint256 = 100 + return 121 + """ + + ast1 = parse_to_ast(code1) + ast2 = parse_to_ast(code2) + + assert ast1 != ast2 diff --git a/tests/parser/ast_utils/test_ast_dict.py b/tests/parser/ast_utils/test_ast_dict.py new file mode 100644 index 0000000000..b4d905f9cb --- /dev/null +++ b/tests/parser/ast_utils/test_ast_dict.py @@ -0,0 +1,81 @@ +from vyper import ( + compiler, +) +from vyper.ast_utils import ( + ast_to_dict, + dict_to_ast, + parse_to_ast, +) + + +def get_node_ids(ast_struct, ids=None): + if ids is None: + ids = [] + + for k, v in ast_struct.items(): + print(k, v, ids) + if isinstance(v, dict): + ids = get_node_ids(v, ids) + elif isinstance(v, list): + for x in v: + ids = get_node_ids(x, ids) + elif k == 'node_id': + ids.append(v) + return ids + + +def test_ast_to_dict_node_id(): + code = """ +@public +def test() -> int128: + a: uint256 = 100 + return 123 + """ + dict_out = compiler.compile_code(code, ['ast_dict']) + node_ids = get_node_ids(dict_out) + + assert len(node_ids) == len(set(node_ids)) + + +def test_basic_ast(): + code = """ +a: int128 + """ + dict_out = compiler.compile_code(code, ['ast_dict']) + assert dict_out['ast_dict']['ast'][0] == { + 'annotation': { + 'ast_type': 'Name', + 'col_offset': 3, + 'id': 'int128', + 'lineno': 2, + 'node_id': 4 + }, + 'ast_type': 'AnnAssign', + 'col_offset': 0, + 'lineno': 2, + 'node_id': 1, + 'simple': 1, + 'target': { + 'ast_type': 'Name', + 'col_offset': 0, + 'id': 'a', + 'lineno': 2, + 'node_id': 2 + }, + 'value': None + } + + +def test_dict_to_ast(): + code = """ +@public +def test() -> int128: + a: uint256 = 100 + return 123 + """ + + original_ast = parse_to_ast(code) + out_dict = ast_to_dict(original_ast) + new_ast = dict_to_ast(out_dict) + + assert new_ast == original_ast diff --git a/tests/parser/ast_utils/test_ast_to_string.py b/tests/parser/ast_utils/test_ast_to_string.py new file mode 100644 index 0000000000..2ea5d4d79b --- /dev/null +++ b/tests/parser/ast_utils/test_ast_to_string.py @@ -0,0 +1,18 @@ +from vyper.ast_utils import ( + ast_to_string, + parse_to_ast, +) + + +def test_ast_to_string(): + code = """ +@public +def testme(a: int128) -> int128: + return a + """ + vyper_ast = parse_to_ast(code) + assert ast_to_string(vyper_ast) == ( + "Module(body=[FunctionDef(name='testme', args=arguments(args=[arg(arg='a'," + " annotation=Name(id='int128'))], defaults=[]), body=[Return(value=Name(id='a'))]," + " decorator_list=[Name(id='public')], returns=Name(id='int128'))])" + ) diff --git a/vyper/ast_utils.py b/vyper/ast_utils.py index 13c6b0997f..bbc9cb8955 100644 --- a/vyper/ast_utils.py +++ b/vyper/ast_utils.py @@ -13,7 +13,7 @@ ) -def parse_python_ast(source_code: str, node: python_ast.AST): +def parse_python_ast(source_code, node): if isinstance(node, list): o = [] for n in node: From 468ad9a550b5ce9895a187681335574d5afd07d8 Mon Sep 17 00:00:00 2001 From: Jacques Wagener Date: Thu, 4 Apr 2019 19:24:47 +0200 Subject: [PATCH 16/23] Add basic AST json output. --- bin/vyper | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bin/vyper b/bin/vyper index 0f38732562..66cc660cf1 100755 --- a/bin/vyper +++ b/bin/vyper @@ -23,6 +23,7 @@ format_options_help = """Format to print, one or more of: bytecode_runtime - Bytecode at runtime abi - ABI in JSON format abi_python - ABI in python format + ast - AST in JSON format source_map - Vyper source map method_identifiers - Dictionary of method signature to method identifier. combined_json - All of the above format options combined as single JSON output. @@ -105,7 +106,8 @@ if __name__ == '__main__': else: # Normal output. translate_map = { 'abi_python': 'abi', - 'json': 'abi' + 'json': 'abi', + 'ast': 'ast_dict' } formats = [] orig_args = uniq(args.format.split(',')) @@ -156,7 +158,7 @@ if __name__ == '__main__': for out in out_list: for f in orig_args: o = out[translate_map.get(f, f)] - if f in ('abi', 'json'): + if f in ('abi', 'json', 'ast'): print(json.dumps(o)) elif f == 'abi_python': print(o) From 131a2acd3f8be0cc43fb227699328b533d4d0a31 Mon Sep 17 00:00:00 2001 From: Jacques Wagener Date: Mon, 15 Apr 2019 17:00:34 +0200 Subject: [PATCH 17/23] Add iterable_cast. --- vyper/utils.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/vyper/utils.py b/vyper/utils.py index 55ed4c16b2..4d1136227a 100644 --- a/vyper/utils.py +++ b/vyper/utils.py @@ -2,6 +2,7 @@ from collections import ( OrderedDict, ) +import functools import re from vyper.exceptions import ( @@ -258,3 +259,12 @@ def check_valid_varname(varname, def is_instances(instances, instance_type): return all([isinstance(inst, instance_type) for inst in instances]) + + +def iterable_cast(cast_type): + def yf(func): + @functools.wraps(func) + def f(*args, **kwargs): + return cast_type(func(*args, **kwargs)) + return f + return yf From a0584f275a8ed374f47d43245d7b149bb42d3ba5 Mon Sep 17 00:00:00 2001 From: Jacques Wagener Date: Mon, 15 Apr 2019 17:01:06 +0200 Subject: [PATCH 18/23] Add exception to get_node_ids. --- tests/parser/ast_utils/test_ast_dict.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/parser/ast_utils/test_ast_dict.py b/tests/parser/ast_utils/test_ast_dict.py index b4d905f9cb..5a7746f34f 100644 --- a/tests/parser/ast_utils/test_ast_dict.py +++ b/tests/parser/ast_utils/test_ast_dict.py @@ -13,7 +13,6 @@ def get_node_ids(ast_struct, ids=None): ids = [] for k, v in ast_struct.items(): - print(k, v, ids) if isinstance(v, dict): ids = get_node_ids(v, ids) elif isinstance(v, list): @@ -21,6 +20,10 @@ def get_node_ids(ast_struct, ids=None): ids = get_node_ids(x, ids) elif k == 'node_id': ids.append(v) + elif v is None or isinstance(v, (str, int)): + continue + else: + raise Exception('Unknown ast_struct provided.') return ids From 6426c9a811152fcf1d0aaaa6fe64f86488c911a6 Mon Sep 17 00:00:00 2001 From: Jacques Wagener Date: Mon, 15 Apr 2019 18:41:49 +0200 Subject: [PATCH 19/23] Fix slots, get_slots and only_empty_fields. --- vyper/ast.py | 65 ++++++++++++++++++++++++++-------------------------- 1 file changed, 32 insertions(+), 33 deletions(-) diff --git a/vyper/ast.py b/vyper/ast.py index b469a27e1a..9522af29ac 100644 --- a/vyper/ast.py +++ b/vyper/ast.py @@ -1,27 +1,26 @@ from itertools import ( chain, ) -from typing import ( - Any, - List as ListTyping, -) +import typing from vyper.exceptions import ( CompilerPanic, ) +BASE_NODE_ATTRIBUTES = ('node_id', 'source_code', 'col_offset', 'lineno') + class VyperNode: - __slots__ = ('node_id', 'source_code', 'col_offset', 'lineno') - ignored_fields = ['ctx', ] - only_empty_fields: ListTyping[Any] = [] + __slots__ = BASE_NODE_ATTRIBUTES + ignored_fields: typing.Tuple = ('ctx', ) + only_empty_fields: typing.Tuple = () @classmethod def get_slots(cls): - return chain.from_iterable( + return set(chain.from_iterable( getattr(klass, '__slots__', []) for klass in cls.__class__.mro(cls) - ) + )) def __init__(self, **kwargs): @@ -75,7 +74,7 @@ class FunctionDef(VyperNode): class arguments(VyperNode): __slots__ = ('args', 'defaults', 'default') - only_empty_fields = ['vararg', 'kwonlyargs', 'kwarg', 'kw_defaults'] + only_empty_fields = ('vararg', 'kwonlyargs', 'kwarg', 'kw_defaults') class Import(VyperNode): @@ -119,7 +118,7 @@ class BoolOp(Op): class BinOp(Op): - pass + __slots__ = () class UnaryOp(Op): @@ -139,71 +138,71 @@ class Bytes(VyperNode): class Add(VyperNode): - pass + __slots__ = () class Sub(VyperNode): - pass + __slots__ = () class Mult(VyperNode): - pass + __slots__ = () class Div(VyperNode): - pass + __slots__ = () class Mod(VyperNode): - pass + __slots__ = () class Pow(VyperNode): - pass + __slots__ = () class In(VyperNode): - pass + __slots__ = () class Gt(VyperNode): - pass + __slots__ = () class GtE(VyperNode): - pass + __slots__ = () class LtE(VyperNode): - pass + __slots__ = () class Lt(VyperNode): - pass + __slots__ = () class Eq(VyperNode): - pass + __slots__ = () class NotEq(VyperNode): - pass + __slots__ = () class And(VyperNode): - pass + __slots__ = () class Or(VyperNode): - pass + __slots__ = () class Not(VyperNode): - pass + __slots__ = () class USub(VyperNode): - pass + __slots__ = () class Expr(VyperNode): @@ -211,7 +210,7 @@ class Expr(VyperNode): class Pass(VyperNode): - pass + __slots__ = () class AnnAssign(VyperNode): @@ -239,11 +238,11 @@ class AugAssign(VyperNode): class Break(VyperNode): - pass + __slots__ = () class Continue(VyperNode): - pass + __slots__ = () class Return(VyperNode): @@ -255,7 +254,7 @@ class Delete(VyperNode): class stmt(VyperNode): - pass + __slots__ = () class ClassDef(VyperNode): @@ -267,7 +266,7 @@ class Raise(VyperNode): class Slice(VyperNode): - only_empty_fields = ['lower'] + only_empty_fields = ('lower', ) class alias(VyperNode): From bc0766b63ea7cb862941d60a132a08307f5e4269 Mon Sep 17 00:00:00 2001 From: Jacques Wagener Date: Mon, 15 Apr 2019 18:45:39 +0200 Subject: [PATCH 20/23] Use iterable_cast for ast_utils.py --- .../test_annotate_and_optimize_ast.py | 4 +- vyper/ast_utils.py | 125 +++++++++++------- vyper/parser/parser_utils.py | 2 +- 3 files changed, 78 insertions(+), 53 deletions(-) diff --git a/tests/parser/parser_utils/test_annotate_and_optimize_ast.py b/tests/parser/parser_utils/test_annotate_and_optimize_ast.py index e196ce78f5..10528f836b 100644 --- a/tests/parser/parser_utils/test_annotate_and_optimize_ast.py +++ b/tests/parser/parser_utils/test_annotate_and_optimize_ast.py @@ -1,7 +1,7 @@ import ast as python_ast from vyper.parser.parser_utils import ( - annotate_and_optimize_ast, + annotate_ast, ) from vyper.parser.pre_parser import ( pre_parse, @@ -36,7 +36,7 @@ def get_contract_info(source_code): class_types, reformatted_code = pre_parse(source_code) py_ast = python_ast.parse(reformatted_code) - annotate_and_optimize_ast(py_ast, reformatted_code, class_types) + annotate_ast(py_ast, reformatted_code, class_types) return py_ast, reformatted_code diff --git a/vyper/ast_utils.py b/vyper/ast_utils.py index bbc9cb8955..65f967a897 100644 --- a/vyper/ast_utils.py +++ b/vyper/ast_utils.py @@ -2,56 +2,73 @@ import vyper.ast as vyper_ast from vyper.exceptions import ( + CompilerPanic, ParserException, SyntaxException, ) from vyper.parser.parser_utils import ( - annotate_and_optimize_ast, + annotate_ast, ) from vyper.parser.pre_parser import ( pre_parse, ) +from vyper.utils import ( + iterable_cast, +) +DICT_AST_SKIPLIST = ('source_code', ) -def parse_python_ast(source_code, node): - if isinstance(node, list): - o = [] - for n in node: - o.append( + +@iterable_cast(list) +def _build_vyper_ast_list(source_code, node): + for n in node: + yield parse_python_ast( + source_code=source_code, + node=n, + ) + + +@iterable_cast(dict) +def _build_vyper_ast_init_kwargs(source_code, node, vyper_class, class_name): + yield ('col_offset', getattr(node, 'col_offset', None)) + yield ('lineno', getattr(node, 'lineno', None)) + yield ('node_id', node.node_id) + yield ('source_code', source_code) + + if isinstance(node, python_ast.ClassDef): + yield ('class_type', node.class_type) + + for field_name in node._fields: + val = getattr(node, field_name) + if field_name in vyper_class.ignored_fields: + continue + elif val and field_name in vyper_class.only_empty_fields: + raise SyntaxException( + 'Invalid Vyper Syntax. ' + f'"{field_name}" is an unsupported attribute field ' + f'on Python AST "{class_name}" class.', + val + ) + else: + yield ( + field_name, parse_python_ast( source_code=source_code, - node=n, + node=val, ) ) - return o + + +def parse_python_ast(source_code, node): + if isinstance(node, list): + return _build_vyper_ast_list(source_code, node) elif isinstance(node, python_ast.AST): class_name = node.__class__.__name__ if hasattr(vyper_ast, class_name): vyper_class = getattr(vyper_ast, class_name) - init_kwargs = { - 'col_offset': getattr(node, 'col_offset', None), - 'lineno': getattr(node, 'lineno', None), - 'node_id': node.node_id, - 'source_code': source_code - } - if isinstance(node, python_ast.ClassDef): - init_kwargs['class_type'] = node.class_type - for field_name in node._fields: - val = getattr(node, field_name) - if field_name in vyper_class.ignored_fields: - continue - elif val and field_name in vyper_class.only_empty_fields: - raise SyntaxException( - 'Invalid Vyper Syntax. ' - f'"{field_name}" is an unsupported attribute field ' - f'on Python AST "{class_name}" class.', - val - ) - else: - init_kwargs[field_name] = parse_python_ast( - source_code=source_code, - node=val, - ) + init_kwargs = _build_vyper_ast_init_kwargs( + source_code, node, vyper_class, class_name + ) return vyper_class(**init_kwargs) else: raise SyntaxException( @@ -62,11 +79,11 @@ def parse_python_ast(source_code, node): def parse_to_ast(source_code): - class_types, reformatted_code = pre_parse(source_code) - if '\x00' in reformatted_code: + if '\x00' in source_code: raise ParserException('No null bytes (\\x00) allowed in the source code.') + class_types, reformatted_code = pre_parse(source_code) py_ast = python_ast.parse(reformatted_code) - annotate_and_optimize_ast(py_ast, source_code, class_types) + annotate_ast(py_ast, source_code, class_types) # Convert to Vyper AST. vyper_ast = parse_python_ast( source_code=source_code, @@ -75,23 +92,29 @@ def parse_to_ast(source_code): return vyper_ast.body -def ast_to_dict(node): - skip_list = ('source_code', ) +@iterable_cast(list) +def _ast_to_list(node): + for x in node: + yield ast_to_dict(x) + + +@iterable_cast(dict) +def _ast_to_dict(node): + for f in node.get_slots(): + if f not in DICT_AST_SKIPLIST: + yield (f, ast_to_dict(getattr(node, f, None))) + yield ('ast_type', node.__class__.__name__) + + +def ast_to_dict(node: vyper_ast.VyperNode) -> dict: if isinstance(node, vyper_ast.VyperNode): - o = { - f: ast_to_dict(getattr(node, f, None)) - for f in node.get_slots() - if f not in skip_list - } - o.update({'ast_type': node.__class__.__name__}) - return o + return _ast_to_dict(node) elif isinstance(node, list): - return [ - ast_to_dict(x) - for x in node - ] - else: + return _ast_to_list(node) + elif node is None or isinstance(node, (str, int)): return node + else: + raise CompilerPanic('Unknown vyper AST node provided.') def dict_to_ast(ast_struct): @@ -108,8 +131,10 @@ def dict_to_ast(ast_struct): dict_to_ast(x) for x in ast_struct ] - else: + elif ast_struct is None or isinstance(ast_struct, (str, int)): return ast_struct + else: + raise CompilerPanic('Unknown ast_struct provided.') def to_python_ast(vyper_ast_node): diff --git a/vyper/parser/parser_utils.py b/vyper/parser/parser_utils.py index 6ad4cb1733..ed6739d498 100644 --- a/vyper/parser/parser_utils.py +++ b/vyper/parser/parser_utils.py @@ -750,7 +750,7 @@ def visit_UnaryOp(self, node): return node -def annotate_and_optimize_ast( +def annotate_ast( parsed_ast: python_ast.Module, source_code: str, class_types: Optional[ClassTypes] = None, From 9c33553dd2cfa0aaef73c80efbc89c297e56634d Mon Sep 17 00:00:00 2001 From: Jacques Wagener Date: Mon, 15 Apr 2019 19:44:57 +0200 Subject: [PATCH 21/23] Add typing. --- vyper/ast_utils.py | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/vyper/ast_utils.py b/vyper/ast_utils.py index 9772600659..f249b03bb8 100644 --- a/vyper/ast_utils.py +++ b/vyper/ast_utils.py @@ -1,4 +1,7 @@ import ast as python_ast +from typing import ( + Generator, +) import vyper.ast as vyper_ast from vyper.exceptions import ( @@ -16,12 +19,11 @@ iterable_cast, ) - DICT_AST_SKIPLIST = ('source_code', ) @iterable_cast(list) -def _build_vyper_ast_list(source_code, node): +def _build_vyper_ast_list(source_code: str, node: list) -> Generator: for n in node: yield parse_python_ast( source_code=source_code, @@ -30,14 +32,19 @@ def _build_vyper_ast_list(source_code, node): @iterable_cast(dict) -def _build_vyper_ast_init_kwargs(source_code, node, vyper_class, class_name): +def _build_vyper_ast_init_kwargs( + source_code: str, + node: python_ast.AST, + vyper_class: vyper_ast.VyperNode, + class_name: str +) -> Generator: yield ('col_offset', getattr(node, 'col_offset', None)) yield ('lineno', getattr(node, 'lineno', None)) - yield ('node_id', node.node_id) + yield ('node_id', node.node_id) # type: ignore yield ('source_code', source_code) if isinstance(node, python_ast.ClassDef): - yield ('class_type', node.class_type) + yield ('class_type', node.class_type) # type: ignore for field_name in node._fields: val = getattr(node, field_name) @@ -60,7 +67,7 @@ def _build_vyper_ast_init_kwargs(source_code, node, vyper_class, class_name): ) -def parse_python_ast(source_code, node): +def parse_python_ast(source_code: str, node: python_ast.Module) -> vyper_ast.Module: if isinstance(node, list): return _build_vyper_ast_list(source_code, node) elif isinstance(node, python_ast.AST): @@ -79,7 +86,7 @@ def parse_python_ast(source_code, node): return node -def parse_to_ast(source_code): +def parse_to_ast(source_code: str) -> list: if '\x00' in source_code: raise ParserException('No null bytes (\\x00) allowed in the source code.') class_types, reformatted_code = pre_parse(source_code) @@ -90,17 +97,17 @@ def parse_to_ast(source_code): source_code=source_code, node=py_ast, ) - return vyper_ast.body + return vyper_ast.body # type: ignore @iterable_cast(list) -def _ast_to_list(node): +def _ast_to_list(node: list) -> Generator: for x in node: yield ast_to_dict(x) @iterable_cast(dict) -def _ast_to_dict(node): +def _ast_to_dict(node: vyper_ast.VyperNode) -> Generator: for f in node.get_slots(): if f not in DICT_AST_SKIPLIST: yield (f, ast_to_dict(getattr(node, f, None))) @@ -118,7 +125,7 @@ def ast_to_dict(node: vyper_ast.VyperNode) -> dict: raise CompilerPanic('Unknown vyper AST node provided.') -def dict_to_ast(ast_struct): +def dict_to_ast(ast_struct: dict) -> vyper_ast.VyperNode: if isinstance(ast_struct, dict) and 'ast_type' in ast_struct: vyper_class = getattr(vyper_ast, ast_struct['ast_type']) klass = vyper_class(**{ @@ -138,7 +145,7 @@ def dict_to_ast(ast_struct): raise CompilerPanic('Unknown ast_struct provided.') -def to_python_ast(vyper_ast_node): +def to_python_ast(vyper_ast_node: vyper_ast.VyperNode) -> python_ast.AST: if isinstance(vyper_ast_node, list): return [ to_python_ast(n) @@ -154,11 +161,13 @@ def to_python_ast(vyper_ast_node): ) for k in vyper_ast_node.get_slots() }) + else: + raise CompilerPanic(f'Unknown vyper AST class "{class_name}" provided.') else: return vyper_ast_node -def ast_to_string(vyper_ast_node): +def ast_to_string(vyper_ast_node: vyper_ast.VyperNode) -> str: py_ast_node = to_python_ast(vyper_ast_node) return python_ast.dump( python_ast.Module( From 468e4994e316e19a97891f9577841b9f6c424ae3 Mon Sep 17 00:00:00 2001 From: Jacques Wagener Date: Thu, 9 May 2019 13:29:41 +0200 Subject: [PATCH 22/23] Merge branch 'master' into 1363_vyper_ast --- tests/parser/features/test_gas.py | 2 +- tests/parser/features/test_init.py | 22 +++++++++++++++ vyper/parser/events.py | 1 - vyper/parser/parser.py | 45 +++++++++++++++++++----------- 4 files changed, 52 insertions(+), 18 deletions(-) create mode 100644 tests/parser/features/test_init.py diff --git a/tests/parser/features/test_gas.py b/tests/parser/features/test_gas.py index 008c8b5ef1..6ca59c1028 100644 --- a/tests/parser/features/test_gas.py +++ b/tests/parser/features/test_gas.py @@ -31,5 +31,5 @@ def __init__(): """ parser_utils.LLLnode.repr_show_gas = True out = parse_to_lll(code) - assert '35273' in str(out)[:28] + assert '35261' in str(out)[:28] parser_utils.LLLnode.repr_show_gas = False diff --git a/tests/parser/features/test_init.py b/tests/parser/features/test_init.py new file mode 100644 index 0000000000..8cb098fad9 --- /dev/null +++ b/tests/parser/features/test_init.py @@ -0,0 +1,22 @@ +import vyper + + +def test_basic_init_function(get_contract): + code = """ +val: public(uint256) + +@public +def __init__(a: uint256): + self.val = a + """ + + c = get_contract(code, *[123]) + + assert c.val() == 123 + + # Make sure the init signature has no unecessary CALLDATLOAD copy. + opcodes = vyper.compile_code(code, ['opcodes'])['opcodes'].split(' ') + lll_return_idx = opcodes.index('JUMP') + + assert 'CALLDATALOAD' in opcodes + assert 'CALLDATALOAD' not in opcodes[:lll_return_idx] diff --git a/vyper/parser/events.py b/vyper/parser/events.py index 226c3e54ad..8b25358786 100644 --- a/vyper/parser/events.py +++ b/vyper/parser/events.py @@ -1,5 +1,4 @@ from vyper import ast - from vyper.exceptions import ( InvalidLiteralException, TypeMismatchException, diff --git a/vyper/parser/parser.py b/vyper/parser/parser.py index 84774b8b2a..2e327f0093 100644 --- a/vyper/parser/parser.py +++ b/vyper/parser/parser.py @@ -1,4 +1,7 @@ -import ast +from typing import ( + Any, + List, +) from vyper import ast from vyper.ast_utils import ( @@ -7,7 +10,6 @@ from vyper.exceptions import ( EventDeclarationException, FunctionDeclarationException, - ParserException, StructureException, ) from vyper.parser.function_definitions import ( @@ -21,9 +23,6 @@ from vyper.parser.lll_node import ( LLLnode, ) -from vyper.parser.pre_parser import ( - pre_parse, -) from vyper.signatures import ( sig_utils, ) @@ -43,6 +42,20 @@ if not hasattr(ast, 'AnnAssign'): raise Exception("Requires python 3.6 or higher for annotation support") +# Header code +STORE_CALLDATA: List[Any] = ['seq', ['mstore', 28, ['calldataload', 0]]] +# Store limit constants at fixed addresses in memory. +LIMIT_MEMORY_SET: List[Any] = [ + ['mstore', pos, limit_size] + for pos, limit_size in LOADED_LIMIT_MAP.items() +] +FUNC_INIT_LLL = LLLnode.from_list( + STORE_CALLDATA + LIMIT_MEMORY_SET, typ=None +) +INIT_FUNC_INIT_LLL = LLLnode.from_list( + ['seq'] + LIMIT_MEMORY_SET, typ=None +) + # Header code INITIALIZER_LIST = ['seq', ['mstore', 28, ['calldataload', 0]]] @@ -108,8 +121,9 @@ def parse_other_functions(o, global_ctx, default_function, runtime_only): - sub = ['seq', INITIALIZER_LLL] - add_gas = INITIALIZER_LLL.gas + sub = ['seq', FUNC_INIT_LLL] + add_gas = FUNC_INIT_LLL.gas + for _def in otherfuncs: sub.append( parse_function(_def, {**{'self': sigs}, **external_contracts}, origcode, global_ctx) @@ -178,15 +192,14 @@ def parse_tree_to_lll(code, origcode, runtime_only=False, interface_codes=None): external_contracts = parse_external_contracts(external_contracts, global_ctx) # If there is an init func... if initfunc: - o.append(INITIALIZER_LLL) - o.append( - parse_function( - initfunc[0], - {**{'self': sigs}, **external_contracts}, - origcode, - global_ctx, - ) - ) + o.append(INIT_FUNC_INIT_LLL) + o.append(parse_function( + initfunc[0], + {**{'self': sigs}, **external_contracts}, + origcode, + global_ctx, + )) + # If there are regular functions... if otherfuncs or defaultfunc: o = parse_other_functions( From 000a12ed182055c5ee5d4cecc184ff5cf753f54c Mon Sep 17 00:00:00 2001 From: Jacques Wagener Date: Thu, 9 May 2019 13:46:25 +0200 Subject: [PATCH 23/23] Lint fix. --- vyper/ast_utils.py | 2 +- vyper/parser/parser_utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/vyper/ast_utils.py b/vyper/ast_utils.py index f249b03bb8..329a73dc96 100644 --- a/vyper/ast_utils.py +++ b/vyper/ast_utils.py @@ -67,7 +67,7 @@ def _build_vyper_ast_init_kwargs( ) -def parse_python_ast(source_code: str, node: python_ast.Module) -> vyper_ast.Module: +def parse_python_ast(source_code: str, node: python_ast.AST) -> vyper_ast.VyperNode: if isinstance(node, list): return _build_vyper_ast_list(source_code, node) elif isinstance(node, python_ast.AST): diff --git a/vyper/parser/parser_utils.py b/vyper/parser/parser_utils.py index 83cdab8b0f..cace2392df 100644 --- a/vyper/parser/parser_utils.py +++ b/vyper/parser/parser_utils.py @@ -834,7 +834,7 @@ def return_check(self, node: Union[python_ast.AST, List[Any]]) -> bool: def annotate_ast( - parsed_ast: python_ast.Module, + parsed_ast: Union[python_ast.AST, python_ast.Module], source_code: str, class_types: Optional[ClassTypes] = None, ) -> None: