From 8325e39c341836de6171d0e0d3f986006d0e6d84 Mon Sep 17 00:00:00 2001 From: David Fisher Date: Thu, 9 Jun 2016 17:00:46 -0700 Subject: [PATCH 1/7] [2.7] Support the per-argument function comment syntax Adds a type_comments list of strings to the arguments object containing the per-argument type comments of a function, if any. --- ast27/Grammar/Grammar | 6 ++--- ast27/Include/Python-ast.h | 5 ++-- ast27/Parser/Python.asdl | 3 ++- ast27/Python/Python-ast.c | 40 ++++++++++++++++++++++++--- ast27/Python/ast.c | 48 ++++++++++++++++++++++++++------- ast27/Python/graminit.c | 55 ++++++++++++++++++++++++++------------ 6 files changed, 121 insertions(+), 36 deletions(-) diff --git a/ast27/Grammar/Grammar b/ast27/Grammar/Grammar index f5beb768..60dbf50e 100644 --- a/ast27/Grammar/Grammar +++ b/ast27/Grammar/Grammar @@ -25,9 +25,9 @@ decorators: decorator+ decorated: decorators (classdef | funcdef) funcdef: 'def' NAME parameters ':' [TYPE_COMMENT] suite parameters: '(' [varargslist] ')' -varargslist: ((fpdef ['=' test] ',')* - ('*' NAME [',' '**' NAME] | '**' NAME) | - fpdef ['=' test] (',' fpdef ['=' test])* [',']) +varargslist: ((fpdef ['=' test] ',' [TYPE_COMMENT])* + ('*' NAME [',' [TYPE_COMMENT] '**' NAME] [TYPE_COMMENT] | '**' NAME [TYPE_COMMENT]) | + fpdef ['=' test] (',' [TYPE_COMMENT] fpdef ['=' test])* [','] [TYPE_COMMENT]) fpdef: NAME | '(' fplist ')' fplist: fpdef (',' fpdef)* [','] diff --git a/ast27/Include/Python-ast.h b/ast27/Include/Python-ast.h index 81542294..9ff258e8 100644 --- a/ast27/Include/Python-ast.h +++ b/ast27/Include/Python-ast.h @@ -367,6 +367,7 @@ struct _arguments { identifier vararg; identifier kwarg; asdl_seq *defaults; + asdl_seq *type_comments; }; struct _keyword { @@ -531,9 +532,9 @@ comprehension_ty _Ta27_comprehension(expr_ty target, expr_ty iter, asdl_seq * if #define ExceptHandler(a0, a1, a2, a3, a4, a5) _Ta27_ExceptHandler(a0, a1, a2, a3, a4, a5) excepthandler_ty _Ta27_ExceptHandler(expr_ty type, expr_ty name, asdl_seq * body, int lineno, int col_offset, PyArena *arena); -#define arguments(a0, a1, a2, a3, a4) _Ta27_arguments(a0, a1, a2, a3, a4) +#define arguments(a0, a1, a2, a3, a4, a5) _Ta27_arguments(a0, a1, a2, a3, a4, a5) arguments_ty _Ta27_arguments(asdl_seq * args, identifier vararg, identifier kwarg, asdl_seq * - defaults, PyArena *arena); + defaults, asdl_seq * type_comments, PyArena *arena); #define keyword(a0, a1, a2) _Ta27_keyword(a0, a1, a2) keyword_ty _Ta27_keyword(identifier arg, expr_ty value, PyArena *arena); #define alias(a0, a1, a2) _Ta27_alias(a0, a1, a2) diff --git a/ast27/Parser/Python.asdl b/ast27/Parser/Python.asdl index cb17f872..24b79ff8 100644 --- a/ast27/Parser/Python.asdl +++ b/ast27/Parser/Python.asdl @@ -105,8 +105,9 @@ module Python version "$Revision$" excepthandler = ExceptHandler(expr? type, expr? name, stmt* body) attributes (int lineno, int col_offset) + -- the type comments are used only in the multiline type comment syntax arguments = (expr* args, identifier? vararg, - identifier? kwarg, expr* defaults) + identifier? kwarg, expr* defaults, string* type_comments) -- keyword arguments supplied to call keyword = (identifier arg, expr value) diff --git a/ast27/Python/Python-ast.c b/ast27/Python/Python-ast.c index c17faf16..ec9b340f 100644 --- a/ast27/Python/Python-ast.c +++ b/ast27/Python/Python-ast.c @@ -376,6 +376,7 @@ static char *arguments_fields[]={ "vararg", "kwarg", "defaults", + "type_comments", }; static PyTypeObject *keyword_type; static PyObject* ast2obj_keyword(void*); @@ -963,7 +964,7 @@ static int init_types(void) ExceptHandler_type = make_type("ExceptHandler", excepthandler_type, ExceptHandler_fields, 3); if (!ExceptHandler_type) return 0; - arguments_type = make_type("arguments", &AST_type, arguments_fields, 4); + arguments_type = make_type("arguments", &AST_type, arguments_fields, 5); if (!arguments_type) return 0; keyword_type = make_type("keyword", &AST_type, keyword_fields, 2); if (!keyword_type) return 0; @@ -2090,7 +2091,8 @@ ExceptHandler(expr_ty type, expr_ty name, asdl_seq * body, int lineno, int col_o } arguments_ty -arguments(asdl_seq * args, identifier vararg, identifier kwarg, asdl_seq * defaults, PyArena *arena) +arguments(asdl_seq * args, identifier vararg, identifier kwarg, asdl_seq * defaults, asdl_seq * + type_comments, PyArena *arena) { arguments_ty p; p = (arguments_ty)PyArena_Malloc(arena, sizeof(*p)); @@ -2100,6 +2102,7 @@ arguments(asdl_seq * args, identifier vararg, identifier kwarg, asdl_seq * defau p->vararg = vararg; p->kwarg = kwarg; p->defaults = defaults; + p->type_comments = type_comments; return p; } @@ -3304,6 +3307,11 @@ ast2obj_arguments(void* _o) if (PyObject_SetAttrString(result, "defaults", value) == -1) goto failed; Py_DECREF(value); + value = ast2obj_list(o->type_comments, ast2obj_string); + if (!value) goto failed; + if (PyObject_SetAttrString(result, "type_comments", value) == -1) + goto failed; + Py_DECREF(value); return result; failed: Py_XDECREF(value); @@ -6617,6 +6625,7 @@ obj2ast_arguments(PyObject* obj, arguments_ty* out, PyArena* arena) identifier vararg; identifier kwarg; asdl_seq* defaults; + asdl_seq* type_comments; if (PyObject_HasAttrString(obj, "args")) { int res; @@ -6690,7 +6699,32 @@ obj2ast_arguments(PyObject* obj, arguments_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"defaults\" missing from arguments"); return 1; } - *out = arguments(args, vararg, kwarg, defaults, arena); + if (PyObject_HasAttrString(obj, "type_comments")) { + int res; + Py_ssize_t len; + Py_ssize_t i; + tmp = PyObject_GetAttrString(obj, "type_comments"); + if (tmp == NULL) goto failed; + if (!PyList_Check(tmp)) { + PyErr_Format(PyExc_TypeError, "arguments field \"type_comments\" must be a list, not a %.200s", tmp->ob_type->tp_name); + goto failed; + } + len = PyList_GET_SIZE(tmp); + type_comments = asdl_seq_new(len, arena); + if (type_comments == NULL) goto failed; + for (i = 0; i < len; i++) { + string value; + res = obj2ast_string(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(type_comments, i, value); + } + Py_XDECREF(tmp); + tmp = NULL; + } else { + PyErr_SetString(PyExc_TypeError, "required field \"type_comments\" missing from arguments"); + return 1; + } + *out = arguments(args, vararg, kwarg, defaults, type_comments, arena); return 0; failed: Py_XDECREF(tmp); diff --git a/ast27/Python/ast.c b/ast27/Python/ast.c index 1100a6bc..0eca48f4 100644 --- a/ast27/Python/ast.c +++ b/ast27/Python/ast.c @@ -732,17 +732,18 @@ static arguments_ty ast_for_arguments(struct compiling *c, const node *n) { /* parameters: '(' [varargslist] ')' - varargslist: (fpdef ['=' test] ',')* ('*' NAME [',' '**' NAME] - | '**' NAME) | fpdef ['=' test] (',' fpdef ['=' test])* [','] + varargslist: ((fpdef ['=' test] ',' [TYPE_COMMENT])* + ('*' NAME [',' [TYPE_COMMENT] '**' NAME] [TYPE_COMMENT] | '**' NAME [TYPE_COMMENT]) | + fpdef ['=' test] (',' [TYPE_COMMENT] fpdef ['=' test])* [','] [TYPE_COMMENT]) */ - int i, j, k, n_args = 0, n_defaults = 0, found_default = 0; - asdl_seq *args, *defaults; + int i, j, k, l, n_args = 0, n_all_args = 0, n_defaults = 0, found_default = 0; + asdl_seq *args, *defaults, *type_comments = NULL; identifier vararg = NULL, kwarg = NULL; node *ch; if (TYPE(n) == parameters) { if (NCH(n) == 2) /* () as argument list */ - return arguments(NULL, NULL, NULL, NULL, c->c_arena); + return arguments(NULL, NULL, NULL, NULL, NULL, c->c_arena); n = CHILD(n, 1); } REQ(n, varargslist); @@ -754,7 +755,10 @@ ast_for_arguments(struct compiling *c, const node *n) n_args++; if (TYPE(ch) == EQUAL) n_defaults++; + if (TYPE(ch) == STAR || TYPE(ch) == DOUBLESTAR) + n_all_args++; } + n_all_args += n_args; args = (n_args ? asdl_seq_new(n_args, c->c_arena) : NULL); if (!args && n_args) return NULL; @@ -768,6 +772,7 @@ ast_for_arguments(struct compiling *c, const node *n) i = 0; j = 0; /* index for defaults */ k = 0; /* index for args */ + l = 0; /* index for type comments */ while (i < NCH(n)) { ch = CHILD(n, i); switch (TYPE(ch)) { @@ -834,7 +839,7 @@ ast_for_arguments(struct compiling *c, const node *n) asdl_seq_SET(args, k++, name); } - i += 2; /* the name and the comma */ + i += 1 + (TYPE(CHILD(n, i + 1)) == COMMA); /* the name and the comma, if present */ if (parenthesized && Py_Py3kWarningFlag && !ast_warn(c, ch, "parenthesized argument names " "are invalid in 3.x")) @@ -848,7 +853,7 @@ ast_for_arguments(struct compiling *c, const node *n) vararg = NEW_IDENTIFIER(CHILD(n, i+1)); if (!vararg) return NULL; - i += 3; + i += 2 + (TYPE(CHILD(n, i + 2)) == COMMA); break; case DOUBLESTAR: if (!forbidden_check(c, CHILD(n, i+1), STR(CHILD(n, i+1)))) @@ -856,7 +861,24 @@ ast_for_arguments(struct compiling *c, const node *n) kwarg = NEW_IDENTIFIER(CHILD(n, i+1)); if (!kwarg) return NULL; - i += 3; + i += 2 + (TYPE(CHILD(n, i + 2)) == COMMA); + break; + case TYPE_COMMENT: + assert(l < k + !!vararg + !!kwarg); + + if (!type_comments) { + /* lazily allocate the type_comments seq for perf reasons */ + type_comments = asdl_seq_new(n_all_args, c->c_arena); + if (!type_comments) + return NULL; + } + + while (l < k + !!vararg + !!kwarg - 1) { + asdl_seq_SET(type_comments, l++, NULL); + } + + asdl_seq_SET(type_comments, l++, NEW_TYPE_COMMENT(ch)); + i += 1; break; default: PyErr_Format(PyExc_SystemError, @@ -866,7 +888,13 @@ ast_for_arguments(struct compiling *c, const node *n) } } - return arguments(args, vararg, kwarg, defaults, c->c_arena); + if (type_comments) { + while (l < n_all_args) { + asdl_seq_SET(type_comments, l++, NULL); + } + } + + return arguments(args, vararg, kwarg, defaults, type_comments, c->c_arena); } static expr_ty @@ -1038,7 +1066,7 @@ ast_for_lambdef(struct compiling *c, const node *n) expr_ty expression; if (NCH(n) == 3) { - args = arguments(NULL, NULL, NULL, NULL, c->c_arena); + args = arguments(NULL, NULL, NULL, NULL, NULL, c->c_arena); if (!args) return NULL; expression = ast_for_expr(c, CHILD(n, 2)); diff --git a/ast27/Python/graminit.c b/ast27/Python/graminit.c index 6cdb719a..c61ce804 100644 --- a/ast27/Python/graminit.c +++ b/ast27/Python/graminit.c @@ -159,51 +159,72 @@ static arc arcs_8_0[3] = { {31, 2}, {32, 3}, }; -static arc arcs_8_1[3] = { +static arc arcs_8_1[4] = { {28, 4}, {30, 5}, + {24, 6}, {0, 1}, }; static arc arcs_8_2[1] = { - {21, 6}, + {21, 7}, }; static arc arcs_8_3[1] = { - {21, 7}, + {21, 8}, }; static arc arcs_8_4[1] = { - {29, 8}, + {29, 9}, }; -static arc arcs_8_5[4] = { +static arc arcs_8_5[5] = { {27, 1}, + {24, 10}, {31, 2}, {32, 3}, {0, 5}, }; -static arc arcs_8_6[2] = { - {30, 9}, +static arc arcs_8_6[1] = { {0, 6}, }; -static arc arcs_8_7[1] = { +static arc arcs_8_7[3] = { + {30, 11}, + {24, 6}, {0, 7}, }; static arc arcs_8_8[2] = { - {30, 5}, + {24, 6}, {0, 8}, }; -static arc arcs_8_9[1] = { +static arc arcs_8_9[3] = { + {30, 5}, + {24, 6}, + {0, 9}, +}; +static arc arcs_8_10[4] = { + {27, 1}, + {31, 2}, + {32, 3}, + {0, 10}, +}; +static arc arcs_8_11[2] = { + {24, 12}, + {32, 3}, +}; +static arc arcs_8_12[1] = { {32, 3}, }; -static state states_8[10] = { +static state states_8[13] = { {3, arcs_8_0}, - {3, arcs_8_1}, + {4, arcs_8_1}, {1, arcs_8_2}, {1, arcs_8_3}, {1, arcs_8_4}, - {4, arcs_8_5}, - {2, arcs_8_6}, - {1, arcs_8_7}, + {5, arcs_8_5}, + {1, arcs_8_6}, + {3, arcs_8_7}, {2, arcs_8_8}, - {1, arcs_8_9}, + {3, arcs_8_9}, + {4, arcs_8_10}, + {2, arcs_8_11}, + {1, arcs_8_12}, }; static arc arcs_9_0[2] = { {21, 1}, @@ -1971,7 +1992,7 @@ static dfa dfas[88] = { "\000\000\020\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {263, "parameters", 0, 4, states_7, "\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, - {264, "varargslist", 0, 10, states_8, + {264, "varargslist", 0, 13, states_8, "\000\040\040\200\001\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {265, "fpdef", 0, 4, states_9, "\000\040\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, From 3aa01a15d59272af4d4d8c44abbe51b9c16bc6fa Mon Sep 17 00:00:00 2001 From: David Fisher Date: Wed, 15 Jun 2016 16:33:00 -0700 Subject: [PATCH 2/7] [3.5] Support the per-argument function comment syntax Puts per-argument type comments into the annotation slot of the argument as a string expression. --- ast35/Grammar/Grammar | 7 ++- ast35/Python/ast.c | 57 ++++++++++++++++-- ast35/Python/graminit.c | 127 +++++++++++++++++++++++++--------------- typed_ast/ast35.py | 9 +-- 4 files changed, 143 insertions(+), 57 deletions(-) diff --git a/ast35/Grammar/Grammar b/ast35/Grammar/Grammar index bb271220..7aef9bec 100644 --- a/ast35/Grammar/Grammar +++ b/ast35/Grammar/Grammar @@ -30,9 +30,10 @@ async_funcdef: ASYNC funcdef funcdef: 'def' NAME parameters ['->' test] ':' [TYPE_COMMENT] suite parameters: '(' [typedargslist] ')' -typedargslist: (tfpdef ['=' test] (',' tfpdef ['=' test])* [',' - ['*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef]] - | '*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef) +typedargslist: (tfpdef ['=' test] (',' [TYPE_COMMENT] tfpdef ['=' test])* [',' [TYPE_COMMENT] + ['*' [tfpdef] (',' [TYPE_COMMENT] tfpdef ['=' test])* [',' [TYPE_COMMENT] '**' tfpdef] | '**' tfpdef]] [TYPE_COMMENT] + | '*' [tfpdef] (',' [TYPE_COMMENT] tfpdef ['=' test])* [',' [TYPE_COMMENT] '**' tfpdef] [TYPE_COMMENT] + | '**' tfpdef [TYPE_COMMENT]) tfpdef: NAME [':' test] varargslist: (vfpdef ['=' test] (',' vfpdef ['=' test])* [',' ['*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef]] diff --git a/ast35/Python/ast.c b/ast35/Python/ast.c index 2678233e..29eb61ca 100644 --- a/ast35/Python/ast.c +++ b/ast35/Python/ast.c @@ -1247,6 +1247,22 @@ ast_for_arg(struct compiling *c, const node *n) return ret; } +static int +set_arg_comment_annotation(struct compiling *c, arg_ty arg, node *tc) +{ + if (arg->annotation) { + ast_error(c, tc, + "annotated arg has associated type comment"); + return 0; + } + + arg->annotation = Str(NEW_TYPE_COMMENT(tc), LINENO(tc), tc->n_col_offset, c->c_arena); + if (!arg->annotation) + return 0; + + return 1; +} + /* returns -1 if failed to handle keyword only arguments returns new position to keep processing if successful (',' tfpdef ['=' test])* @@ -1304,10 +1320,16 @@ handle_keywordonly_args(struct compiling *c, const node *n, int start, if (!arg) goto error; asdl_seq_SET(kwonlyargs, j++, arg); - i += 2; /* the name and the comma */ + i += 1 + (TYPE(CHILD(n, i + 1)) == COMMA); /* the name and the comma, if present */ break; case DOUBLESTAR: return i; + case TYPE_COMMENT: + /* arg will be equal to the last argument processed */ + if (!set_arg_comment_annotation(c, arg, ch)) + return -1; + i += 1; + break; default: ast_error(c, ch, "unexpected node"); goto error; @@ -1435,7 +1457,7 @@ ast_for_arguments(struct compiling *c, const node *n) if (!arg) return NULL; asdl_seq_SET(posargs, k++, arg); - i += 2; /* the name and the comma */ + i += 1 + (TYPE(CHILD(n, i + 1)) == COMMA); /* the name and the comma, if present */ break; case STAR: if (i+1 >= NCH(n)) { @@ -1447,6 +1469,13 @@ ast_for_arguments(struct compiling *c, const node *n) if (TYPE(ch) == COMMA) { int res = 0; i += 2; /* now follows keyword only arguments */ + + if (TYPE(CHILD(n, i)) == TYPE_COMMENT) { + ast_error(c, CHILD(n, i), + "bare * has associated type comment"); + return NULL; + } + res = handle_keywordonly_args(c, n, i, kwonlyargs, kwdefaults); if (res == -1) return NULL; @@ -1457,7 +1486,15 @@ ast_for_arguments(struct compiling *c, const node *n) if (!vararg) return NULL; - i += 3; + i += 2 + (TYPE(CHILD(n, i + 2)) == COMMA); + + if (TYPE(CHILD(n, i)) == TYPE_COMMENT) { + if (!set_arg_comment_annotation(c, vararg, CHILD(n, i))) + return NULL; + + i += 1; + } + if (i < NCH(n) && (TYPE(CHILD(n, i)) == tfpdef || TYPE(CHILD(n, i)) == vfpdef)) { int res = 0; @@ -1474,7 +1511,19 @@ ast_for_arguments(struct compiling *c, const node *n) kwarg = ast_for_arg(c, ch); if (!kwarg) return NULL; - i += 3; + i += 2 + (TYPE(CHILD(n, i + 2)) == COMMA); + break; + case TYPE_COMMENT: + assert(i); + + if (kwarg) + arg = kwarg; + + /* arg will be equal to the last argument processed */ + if (!set_arg_comment_annotation(c, arg, ch)) + return NULL; + + i += 1; break; default: PyErr_Format(PyExc_SystemError, diff --git a/ast35/Python/graminit.c b/ast35/Python/graminit.c index cc9775e1..29972006 100644 --- a/ast35/Python/graminit.c +++ b/ast35/Python/graminit.c @@ -183,96 +183,131 @@ static arc arcs_9_0[3] = { {34, 2}, {35, 3}, }; -static arc arcs_9_1[3] = { +static arc arcs_9_1[4] = { {32, 4}, {33, 5}, + {28, 6}, {0, 1}, }; -static arc arcs_9_2[3] = { - {31, 6}, - {33, 7}, +static arc arcs_9_2[4] = { + {31, 7}, + {33, 8}, + {28, 6}, {0, 2}, }; static arc arcs_9_3[1] = { - {31, 8}, + {31, 9}, }; static arc arcs_9_4[1] = { - {26, 9}, + {26, 10}, }; -static arc arcs_9_5[4] = { - {31, 10}, - {34, 11}, +static arc arcs_9_5[5] = { + {28, 11}, + {31, 12}, + {34, 13}, {35, 3}, {0, 5}, }; -static arc arcs_9_6[2] = { - {33, 7}, +static arc arcs_9_6[1] = { {0, 6}, }; -static arc arcs_9_7[2] = { - {31, 12}, - {35, 3}, +static arc arcs_9_7[3] = { + {33, 8}, + {28, 6}, + {0, 7}, }; -static arc arcs_9_8[1] = { - {0, 8}, +static arc arcs_9_8[3] = { + {28, 14}, + {31, 15}, + {35, 3}, }; static arc arcs_9_9[2] = { - {33, 5}, + {28, 6}, {0, 9}, }; static arc arcs_9_10[3] = { {33, 5}, - {32, 4}, + {28, 6}, {0, 10}, }; -static arc arcs_9_11[3] = { - {31, 13}, - {33, 14}, +static arc arcs_9_11[5] = { + {31, 12}, + {34, 13}, + {35, 3}, + {28, 6}, {0, 11}, }; -static arc arcs_9_12[3] = { - {33, 7}, - {32, 15}, +static arc arcs_9_12[4] = { + {33, 5}, + {32, 4}, + {28, 6}, {0, 12}, }; -static arc arcs_9_13[2] = { - {33, 14}, +static arc arcs_9_13[4] = { + {31, 16}, + {33, 17}, + {28, 6}, {0, 13}, }; static arc arcs_9_14[2] = { - {31, 16}, + {31, 15}, {35, 3}, }; -static arc arcs_9_15[1] = { - {26, 6}, +static arc arcs_9_15[4] = { + {33, 8}, + {32, 18}, + {28, 6}, + {0, 15}, }; static arc arcs_9_16[3] = { - {33, 14}, - {32, 17}, + {33, 17}, + {28, 6}, {0, 16}, }; -static arc arcs_9_17[1] = { - {26, 13}, +static arc arcs_9_17[3] = { + {28, 19}, + {31, 20}, + {35, 3}, +}; +static arc arcs_9_18[1] = { + {26, 7}, +}; +static arc arcs_9_19[2] = { + {31, 20}, + {35, 3}, +}; +static arc arcs_9_20[4] = { + {33, 17}, + {32, 21}, + {28, 6}, + {0, 20}, +}; +static arc arcs_9_21[1] = { + {26, 16}, }; -static state states_9[18] = { +static state states_9[22] = { {3, arcs_9_0}, - {3, arcs_9_1}, - {3, arcs_9_2}, + {4, arcs_9_1}, + {4, arcs_9_2}, {1, arcs_9_3}, {1, arcs_9_4}, - {4, arcs_9_5}, - {2, arcs_9_6}, - {2, arcs_9_7}, - {1, arcs_9_8}, + {5, arcs_9_5}, + {1, arcs_9_6}, + {3, arcs_9_7}, + {3, arcs_9_8}, {2, arcs_9_9}, {3, arcs_9_10}, - {3, arcs_9_11}, - {3, arcs_9_12}, - {2, arcs_9_13}, + {5, arcs_9_11}, + {4, arcs_9_12}, + {4, arcs_9_13}, {2, arcs_9_14}, - {1, arcs_9_15}, + {4, arcs_9_15}, {3, arcs_9_16}, - {1, arcs_9_17}, + {3, arcs_9_17}, + {1, arcs_9_18}, + {2, arcs_9_19}, + {4, arcs_9_20}, + {1, arcs_9_21}, }; static arc arcs_10_0[1] = { {23, 1}, @@ -2007,7 +2042,7 @@ static dfa dfas[88] = { "\000\000\100\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {264, "parameters", 0, 4, states_8, "\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, - {265, "typedargslist", 0, 18, states_9, + {265, "typedargslist", 0, 22, states_9, "\000\000\200\000\014\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {266, "tfpdef", 0, 4, states_10, "\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, diff --git a/typed_ast/ast35.py b/typed_ast/ast35.py index f4fc17b2..6237c368 100644 --- a/typed_ast/ast35.py +++ b/typed_ast/ast35.py @@ -12,10 +12,11 @@ In particular, the `FunctionDef`, `Assign`, `For`, and `With` classes all have a `type_comment` field which contains a `str` with the text of the - type comment. `parse` has been augmented so it can parse function - signature types when called with `mode=func_type`. Finally, `Module` has - a `type_ignores` field which contains a list of lines which have been - `# type: ignored`. + type comment. Per-argument function comments are put into the annotation + field of each argument. `parse` has been augmented so it can parse + function signature types when called with `mode=func_type`. Finally, + `Module` has a `type_ignores` field which contains a list of lines which + have been `# type: ignored`. An abstract syntax tree can be generated by using the `parse()` function from this module. The result will be a tree of objects whose From 4c8a2ec37f69a4c59933639fe31334371d7862b9 Mon Sep 17 00:00:00 2001 From: David Fisher Date: Tue, 21 Jun 2016 16:01:40 -0700 Subject: [PATCH 3/7] [conversions] Convert Python 2 per-argument type comments Python 2 per-argument type comments are now converted into Python 3 annotations. --- typed_ast/conversions.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/typed_ast/conversions.py b/typed_ast/conversions.py index ce04866a..1484eef2 100644 --- a/typed_ast/conversions.py +++ b/typed_ast/conversions.py @@ -170,24 +170,34 @@ def visit_Ellipsis(self, n): return ast35.Index(ast35.Ellipsis(lineno=-1, col_offset=-1)) def visit_arguments(self, n): - def convert_arg(arg): + def convert_arg(arg, annotation): if isinstance(arg, ast27.Name): v = arg.id elif isinstance(arg, ast27.Tuple): v = self.visit(arg) else: raise RuntimeError("'{}' is not a valid argument.".format(ast27.dump(arg))) - return ast35.arg(v, None, lineno=arg.lineno, col_offset=arg.col_offset) + return ast35.arg(v, annotation, lineno=arg.lineno, col_offset=arg.col_offset) - args = [convert_arg(arg) for arg in n.args] + def get_type_comment(i): + if i < len(n.type_comments): + return ast35.Str(n.type_comments[i]) + else: + return None + + args = [convert_arg(arg, get_type_comment(i)) for i, arg in enumerate(n.args)] vararg = None if n.vararg is not None: - vararg = ast35.arg(n.vararg, None, lineno=-1, col_offset=-1) + vararg = ast35.arg(n.vararg, + get_type_comment(len(args) + 1), + lineno=-1, col_offset=-1) kwarg = None if n.kwarg is not None: - kwarg = ast35.arg(n.kwarg, None, lineno=-1, col_offset=-1) + kwarg = ast35.arg(n.kwarg, + get_type_comment(len(args) + 1 + (1 if n.vararg is not None else 0)), + lineno=-1, col_offset=-1) defaults = self.visit(n.defaults) From 314d17d7d0adcf3a755c8f7fe7d52c888643cb55 Mon Sep 17 00:00:00 2001 From: David Fisher Date: Wed, 22 Jun 2016 11:57:49 -0700 Subject: [PATCH 4/7] Respond to review feedback --- ast27/Python/ast.c | 18 +++++++++++++++--- ast35/Python/ast.c | 22 +++++++++++++++------- typed_ast/ast35.py | 17 ++++++++++------- typed_ast/conversions.py | 2 +- 4 files changed, 41 insertions(+), 18 deletions(-) diff --git a/ast27/Python/ast.c b/ast27/Python/ast.c index 0eca48f4..50a3b07f 100644 --- a/ast27/Python/ast.c +++ b/ast27/Python/ast.c @@ -765,6 +765,12 @@ ast_for_arguments(struct compiling *c, const node *n) defaults = (n_defaults ? asdl_seq_new(n_defaults, c->c_arena) : NULL); if (!defaults && n_defaults) return NULL; + /* type_comments will be lazily initialized if needed. If there are no + per-argument type comments, it will remain NULL. Otherwise, it will be + an asdl_seq with length equal to the number of args (including varargs + and kwargs, if present) and with members set to the string of each arg's + type comment, if present, or NULL otherwise. + */ /* fpdef: NAME | '(' fplist ')' fplist: fpdef (',' fpdef)* [','] @@ -839,7 +845,9 @@ ast_for_arguments(struct compiling *c, const node *n) asdl_seq_SET(args, k++, name); } - i += 1 + (TYPE(CHILD(n, i + 1)) == COMMA); /* the name and the comma, if present */ + i += 1; /* the name */ + if (TYPE(CHILD(n, i)) == COMMA) + i += 1; /* the comma, if present */ if (parenthesized && Py_Py3kWarningFlag && !ast_warn(c, ch, "parenthesized argument names " "are invalid in 3.x")) @@ -853,7 +861,9 @@ ast_for_arguments(struct compiling *c, const node *n) vararg = NEW_IDENTIFIER(CHILD(n, i+1)); if (!vararg) return NULL; - i += 2 + (TYPE(CHILD(n, i + 2)) == COMMA); + i += 2; /* the star and the name */ + if (TYPE(CHILD(n, i)) == COMMA) + i += 1; /* the comma, if present */ break; case DOUBLESTAR: if (!forbidden_check(c, CHILD(n, i+1), STR(CHILD(n, i+1)))) @@ -861,7 +871,9 @@ ast_for_arguments(struct compiling *c, const node *n) kwarg = NEW_IDENTIFIER(CHILD(n, i+1)); if (!kwarg) return NULL; - i += 2 + (TYPE(CHILD(n, i + 2)) == COMMA); + i += 2; /* the double star and the name */ + if (TYPE(CHILD(n, i)) == COMMA) + i += 1; /* the comma, if present */ break; case TYPE_COMMENT: assert(l < k + !!vararg + !!kwarg); diff --git a/ast35/Python/ast.c b/ast35/Python/ast.c index 29eb61ca..34186f07 100644 --- a/ast35/Python/ast.c +++ b/ast35/Python/ast.c @@ -1320,16 +1320,18 @@ handle_keywordonly_args(struct compiling *c, const node *n, int start, if (!arg) goto error; asdl_seq_SET(kwonlyargs, j++, arg); - i += 1 + (TYPE(CHILD(n, i + 1)) == COMMA); /* the name and the comma, if present */ + i += 1; /* the name */ + if (TYPE(CHILD(n, i)) == COMMA) + i += 1; /* the comma, if present */ break; - case DOUBLESTAR: - return i; case TYPE_COMMENT: /* arg will be equal to the last argument processed */ if (!set_arg_comment_annotation(c, arg, ch)) return -1; i += 1; break; + case DOUBLESTAR: + return i; default: ast_error(c, ch, "unexpected node"); goto error; @@ -1457,10 +1459,12 @@ ast_for_arguments(struct compiling *c, const node *n) if (!arg) return NULL; asdl_seq_SET(posargs, k++, arg); - i += 1 + (TYPE(CHILD(n, i + 1)) == COMMA); /* the name and the comma, if present */ + i += 1; /* the name */ + if (TYPE(CHILD(n, i)) == COMMA) + i += 1; /* the comma, if present */ break; case STAR: - if (i+1 >= NCH(n)) { + if (i+1 >= NCH(n) || TYPE(CHILD(n, i+1)) == TYPE_COMMENT) { ast_error(c, CHILD(n, i), "named arguments must follow bare *"); return NULL; @@ -1486,7 +1490,9 @@ ast_for_arguments(struct compiling *c, const node *n) if (!vararg) return NULL; - i += 2 + (TYPE(CHILD(n, i + 2)) == COMMA); + i += 2; /* the star and the name */ + if (TYPE(CHILD(n, i)) == COMMA) + i += 1; /* the comma, if present */ if (TYPE(CHILD(n, i)) == TYPE_COMMENT) { if (!set_arg_comment_annotation(c, vararg, CHILD(n, i))) @@ -1511,7 +1517,9 @@ ast_for_arguments(struct compiling *c, const node *n) kwarg = ast_for_arg(c, ch); if (!kwarg) return NULL; - i += 2 + (TYPE(CHILD(n, i + 2)) == COMMA); + i += 2; /* the double star and the name */ + if (TYPE(CHILD(n, i)) == COMMA) + i += 1; /* the comma, if present */ break; case TYPE_COMMENT: assert(i); diff --git a/typed_ast/ast35.py b/typed_ast/ast35.py index 6237c368..a2f9343e 100644 --- a/typed_ast/ast35.py +++ b/typed_ast/ast35.py @@ -10,13 +10,16 @@ that `ast35` provides PEP 484 type comment information as part of the AST. - In particular, the `FunctionDef`, `Assign`, `For`, and `With` classes all - have a `type_comment` field which contains a `str` with the text of the - type comment. Per-argument function comments are put into the annotation - field of each argument. `parse` has been augmented so it can parse - function signature types when called with `mode=func_type`. Finally, - `Module` has a `type_ignores` field which contains a list of lines which - have been `# type: ignored`. + In particular: + - The `FunctionDef`, `Assign`, `For`, and `With` classes all have a + `type_comment` field which contains a `str` with the text of the type + comment. + - Per-argument function comments are put into the annotation field of each + argument. + - `parse` has been augmented so it can parse function signature types when + called with `mode=func_type`. + - `Module` has a `type_ignores` field which contains a list of + lines which have been `# type: ignored`. An abstract syntax tree can be generated by using the `parse()` function from this module. The result will be a tree of objects whose diff --git a/typed_ast/conversions.py b/typed_ast/conversions.py index 1484eef2..0a74ed43 100644 --- a/typed_ast/conversions.py +++ b/typed_ast/conversions.py @@ -196,7 +196,7 @@ def get_type_comment(i): kwarg = None if n.kwarg is not None: kwarg = ast35.arg(n.kwarg, - get_type_comment(len(args) + 1 + (1 if n.vararg is not None else 0)), + get_type_comment(len(args) + 1 + (0 if n.vararg is None else 1)), lineno=-1, col_offset=-1) defaults = self.visit(n.defaults) From 97915b0158a1bf7dc9b725b3c0d385ff67ae58fb Mon Sep 17 00:00:00 2001 From: David Fisher Date: Wed, 22 Jun 2016 14:27:42 -0700 Subject: [PATCH 5/7] Fix final nits --- ast27/Parser/Python.asdl | 5 ++++- typed_ast/ast35.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/ast27/Parser/Python.asdl b/ast27/Parser/Python.asdl index 24b79ff8..cfef3c91 100644 --- a/ast27/Parser/Python.asdl +++ b/ast27/Parser/Python.asdl @@ -105,7 +105,10 @@ module Python version "$Revision$" excepthandler = ExceptHandler(expr? type, expr? name, stmt* body) attributes (int lineno, int col_offset) - -- the type comments are used only in the multiline type comment syntax + -- type_comments is used to support the per-argument type comment syntax. + -- It is either an empty list or a list with length equal to the number of + -- args (including varargs and kwargs, if present) and with members set to the + -- string of each arg's type comment, if present, or None otherwise. arguments = (expr* args, identifier? vararg, identifier? kwarg, expr* defaults, string* type_comments) diff --git a/typed_ast/ast35.py b/typed_ast/ast35.py index a2f9343e..0865a2d9 100644 --- a/typed_ast/ast35.py +++ b/typed_ast/ast35.py @@ -19,7 +19,7 @@ - `parse` has been augmented so it can parse function signature types when called with `mode=func_type`. - `Module` has a `type_ignores` field which contains a list of - lines which have been `# type: ignored`. + lines which have been `# type: ignore`d. An abstract syntax tree can be generated by using the `parse()` function from this module. The result will be a tree of objects whose From 78206fce6908e656115c40767c2fe9214fa095c2 Mon Sep 17 00:00:00 2001 From: David Fisher Date: Wed, 22 Jun 2016 14:39:41 -0700 Subject: [PATCH 6/7] tabs --- ast27/Parser/Python.asdl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ast27/Parser/Python.asdl b/ast27/Parser/Python.asdl index cfef3c91..1864bc99 100644 --- a/ast27/Parser/Python.asdl +++ b/ast27/Parser/Python.asdl @@ -106,9 +106,9 @@ module Python version "$Revision$" attributes (int lineno, int col_offset) -- type_comments is used to support the per-argument type comment syntax. - -- It is either an empty list or a list with length equal to the number of - -- args (including varargs and kwargs, if present) and with members set to the - -- string of each arg's type comment, if present, or None otherwise. + -- It is either an empty list or a list with length equal to the number of + -- args (including varargs and kwargs, if present) and with members set to the + -- string of each arg's type comment, if present, or None otherwise. arguments = (expr* args, identifier? vararg, identifier? kwarg, expr* defaults, string* type_comments) From 87e46fbaaeffce8aa8f619e3503468e6127c69ff Mon Sep 17 00:00:00 2001 From: David Fisher Date: Wed, 22 Jun 2016 14:45:06 -0700 Subject: [PATCH 7/7] update readme --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index aaa2f3d3..2bc27e59 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,8 @@ compatible with Python 3.3 - 3.5. ### Current Caveats for Use - Type comments in invalid locations produce syntax errors. +- When using per-argument function comment annotations, the type comments must + come after the argument-separating comma. ## Development Notes ### General Notes