[2.7, 3.5, conversions] Support the per-argument function comment syn…

…tax (#5) Python 2.7: Adds a type_comments list of strings to the arguments object containing the per-argument type comments of a function, if any. Python 3.5: Puts per-argument type comments into the annotation slot of the argument as a string expression. Conversions: Converts the new type_comments list to argument annotations. This change contains some autogenerated files: - `ast*/Python/graminit.c` is autogenerated from `ast*/Grammar/Grammar`. - `ast*/Include/Python-ast.h` and `ast*/Python/Python-ast.c` are autogenerated from `ast*/Parser/Python.asdl`.
python · Jun 22, 2016 · f29a0fb · f29a0fb
1 parent 651fbe5
commit f29a0fb
Show file tree

Hide file tree

Showing 12 changed files with 310 additions and 101 deletions.
diff --git a/README.md b/README.md
@@ -6,6 +6,8 @@ compatible with Python 3.3 - 3.5.
 
 ### Current Caveats for Use
 - Type comments in invalid locations produce syntax errors.
+- When using per-argument function comment annotations, the type comments must
+  come after the argument-separating comma.
 
 ## Development Notes
 ### General Notes

diff --git a/ast27/Grammar/Grammar b/ast27/Grammar/Grammar
@@ -25,9 +25,9 @@ decorators: decorator+
 decorated: decorators (classdef | funcdef)
 funcdef: 'def' NAME parameters ':' [TYPE_COMMENT] suite
 parameters: '(' [varargslist] ')'
-varargslist: ((fpdef ['=' test] ',')*
-              ('*' NAME [',' '**' NAME] | '**' NAME) |
-              fpdef ['=' test] (',' fpdef ['=' test])* [','])
+varargslist: ((fpdef ['=' test] ',' [TYPE_COMMENT])*
+              ('*' NAME [',' [TYPE_COMMENT]  '**' NAME] [TYPE_COMMENT] | '**' NAME [TYPE_COMMENT]) |
+              fpdef ['=' test] (',' [TYPE_COMMENT] fpdef ['=' test])* [','] [TYPE_COMMENT])
 fpdef: NAME | '(' fplist ')'
 fplist: fpdef (',' fpdef)* [',']
 

diff --git a/ast27/Include/Python-ast.h b/ast27/Include/Python-ast.h
@@ -367,6 +367,7 @@ struct _arguments {
         identifier vararg;
         identifier kwarg;
         asdl_seq *defaults;
+        asdl_seq *type_comments;
 };
 
 struct _keyword {
@@ -531,9 +532,9 @@ comprehension_ty _Ta27_comprehension(expr_ty target, expr_ty iter, asdl_seq * if
 #define ExceptHandler(a0, a1, a2, a3, a4, a5) _Ta27_ExceptHandler(a0, a1, a2, a3, a4, a5)
 excepthandler_ty _Ta27_ExceptHandler(expr_ty type, expr_ty name, asdl_seq * body, int lineno, int
                                      col_offset, PyArena *arena);
-#define arguments(a0, a1, a2, a3, a4) _Ta27_arguments(a0, a1, a2, a3, a4)
+#define arguments(a0, a1, a2, a3, a4, a5) _Ta27_arguments(a0, a1, a2, a3, a4, a5)
 arguments_ty _Ta27_arguments(asdl_seq * args, identifier vararg, identifier kwarg, asdl_seq *
-                             defaults, PyArena *arena);
+                             defaults, asdl_seq * type_comments, PyArena *arena);
 #define keyword(a0, a1, a2) _Ta27_keyword(a0, a1, a2)
 keyword_ty _Ta27_keyword(identifier arg, expr_ty value, PyArena *arena);
 #define alias(a0, a1, a2) _Ta27_alias(a0, a1, a2)

diff --git a/ast27/Parser/Python.asdl b/ast27/Parser/Python.asdl
@@ -105,8 +105,12 @@ module Python version "$Revision$"
 	excepthandler = ExceptHandler(expr? type, expr? name, stmt* body)
 	                attributes (int lineno, int col_offset)
 
+	-- type_comments is used to support the per-argument type comment syntax.
+	-- It is either an empty list or a list with length equal to the number of
+	-- args (including varargs and kwargs, if present) and with members set to the
+	-- string of each arg's type comment, if present, or None otherwise.
 	arguments = (expr* args, identifier? vararg, 
-		     identifier? kwarg, expr* defaults)
+		     identifier? kwarg, expr* defaults, string* type_comments)
 
         -- keyword arguments supplied to call
         keyword = (identifier arg, expr value)

diff --git a/ast27/Python/Python-ast.c b/ast27/Python/Python-ast.c
@@ -376,6 +376,7 @@ static char *arguments_fields[]={
         "vararg",
         "kwarg",
         "defaults",
+        "type_comments",
 };
 static PyTypeObject *keyword_type;
 static PyObject* ast2obj_keyword(void*);
@@ -963,7 +964,7 @@ static int init_types(void)
         ExceptHandler_type = make_type("ExceptHandler", excepthandler_type, ExceptHandler_fields,
                                        3);
         if (!ExceptHandler_type) return 0;
-        arguments_type = make_type("arguments", &AST_type, arguments_fields, 4);
+        arguments_type = make_type("arguments", &AST_type, arguments_fields, 5);
         if (!arguments_type) return 0;
         keyword_type = make_type("keyword", &AST_type, keyword_fields, 2);
         if (!keyword_type) return 0;
@@ -2090,7 +2091,8 @@ ExceptHandler(expr_ty type, expr_ty name, asdl_seq * body, int lineno, int col_o
 }
 
 arguments_ty
-arguments(asdl_seq * args, identifier vararg, identifier kwarg, asdl_seq * defaults, PyArena *arena)
+arguments(asdl_seq * args, identifier vararg, identifier kwarg, asdl_seq * defaults, asdl_seq *
+          type_comments, PyArena *arena)
 {
         arguments_ty p;
         p = (arguments_ty)PyArena_Malloc(arena, sizeof(*p));
@@ -2100,6 +2102,7 @@ arguments(asdl_seq * args, identifier vararg, identifier kwarg, asdl_seq * defau
         p->vararg = vararg;
         p->kwarg = kwarg;
         p->defaults = defaults;
+        p->type_comments = type_comments;
         return p;
 }
 
@@ -3304,6 +3307,11 @@ ast2obj_arguments(void* _o)
         if (PyObject_SetAttrString(result, "defaults", value) == -1)
                 goto failed;
         Py_DECREF(value);
+        value = ast2obj_list(o->type_comments, ast2obj_string);
+        if (!value) goto failed;
+        if (PyObject_SetAttrString(result, "type_comments", value) == -1)
+                goto failed;
+        Py_DECREF(value);
         return result;
 failed:
         Py_XDECREF(value);
@@ -6617,6 +6625,7 @@ obj2ast_arguments(PyObject* obj, arguments_ty* out, PyArena* arena)
         identifier vararg;
         identifier kwarg;
         asdl_seq* defaults;
+        asdl_seq* type_comments;
 
         if (PyObject_HasAttrString(obj, "args")) {
                 int res;
@@ -6690,7 +6699,32 @@ obj2ast_arguments(PyObject* obj, arguments_ty* out, PyArena* arena)
                 PyErr_SetString(PyExc_TypeError, "required field \"defaults\" missing from arguments");
                 return 1;
         }
-        *out = arguments(args, vararg, kwarg, defaults, arena);
+        if (PyObject_HasAttrString(obj, "type_comments")) {
+                int res;
+                Py_ssize_t len;
+                Py_ssize_t i;
+                tmp = PyObject_GetAttrString(obj, "type_comments");
+                if (tmp == NULL) goto failed;
+                if (!PyList_Check(tmp)) {
+                        PyErr_Format(PyExc_TypeError, "arguments field \"type_comments\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                        goto failed;
+                }
+                len = PyList_GET_SIZE(tmp);
+                type_comments = asdl_seq_new(len, arena);
+                if (type_comments == NULL) goto failed;
+                for (i = 0; i < len; i++) {
+                        string value;
+                        res = obj2ast_string(PyList_GET_ITEM(tmp, i), &value, arena);
+                        if (res != 0) goto failed;
+                        asdl_seq_SET(type_comments, i, value);
+                }
+                Py_XDECREF(tmp);
+                tmp = NULL;
+        } else {
+                PyErr_SetString(PyExc_TypeError, "required field \"type_comments\" missing from arguments");
+                return 1;
+        }
+        *out = arguments(args, vararg, kwarg, defaults, type_comments, arena);
         return 0;
 failed:
         Py_XDECREF(tmp);

diff --git a/ast27/Python/ast.c b/ast27/Python/ast.c
@@ -732,17 +732,18 @@ static arguments_ty
 ast_for_arguments(struct compiling *c, const node *n)
 {
     /* parameters: '(' [varargslist] ')'
-       varargslist: (fpdef ['=' test] ',')* ('*' NAME [',' '**' NAME]
-            | '**' NAME) | fpdef ['=' test] (',' fpdef ['=' test])* [',']
+       varargslist: ((fpdef ['=' test] ',' [TYPE_COMMENT])*
+                     ('*' NAME [',' [TYPE_COMMENT]  '**' NAME] [TYPE_COMMENT] | '**' NAME [TYPE_COMMENT]) |
+                     fpdef ['=' test] (',' [TYPE_COMMENT] fpdef ['=' test])* [','] [TYPE_COMMENT])
     */
-    int i, j, k, n_args = 0, n_defaults = 0, found_default = 0;
-    asdl_seq *args, *defaults;
+    int i, j, k, l, n_args = 0, n_all_args = 0, n_defaults = 0, found_default = 0;
+    asdl_seq *args, *defaults, *type_comments = NULL;
     identifier vararg = NULL, kwarg = NULL;
     node *ch;
 
     if (TYPE(n) == parameters) {
         if (NCH(n) == 2) /* () as argument list */
-            return arguments(NULL, NULL, NULL, NULL, c->c_arena);
+            return arguments(NULL, NULL, NULL, NULL, NULL, c->c_arena);
         n = CHILD(n, 1);
     }
     REQ(n, varargslist);
@@ -754,20 +755,30 @@ ast_for_arguments(struct compiling *c, const node *n)
             n_args++;
         if (TYPE(ch) == EQUAL)
             n_defaults++;
+        if (TYPE(ch) == STAR || TYPE(ch) == DOUBLESTAR)
+            n_all_args++;
     }
+    n_all_args += n_args;
     args = (n_args ? asdl_seq_new(n_args, c->c_arena) : NULL);
     if (!args && n_args)
         return NULL;
     defaults = (n_defaults ? asdl_seq_new(n_defaults, c->c_arena) : NULL);
     if (!defaults && n_defaults)
         return NULL;
+    /* type_comments will be lazily initialized if needed.  If there are no
+       per-argument type comments, it will remain NULL.  Otherwise, it will be
+       an asdl_seq with length equal to the number of args (including varargs
+       and kwargs, if present) and with members set to the string of each arg's
+       type comment, if present, or NULL otherwise.
+     */
 
     /* fpdef: NAME | '(' fplist ')'
        fplist: fpdef (',' fpdef)* [',']
     */
     i = 0;
     j = 0;  /* index for defaults */
     k = 0;  /* index for args */
+    l = 0;  /* index for type comments */
     while (i < NCH(n)) {
         ch = CHILD(n, i);
         switch (TYPE(ch)) {
@@ -834,7 +845,9 @@ ast_for_arguments(struct compiling *c, const node *n)
                     asdl_seq_SET(args, k++, name);
 
                 }
-                i += 2; /* the name and the comma */
+                i += 1; /* the name */
+                if (TYPE(CHILD(n, i)) == COMMA)
+                    i += 1; /* the comma, if present */
                 if (parenthesized && Py_Py3kWarningFlag &&
                     !ast_warn(c, ch, "parenthesized argument names "
                               "are invalid in 3.x"))
@@ -848,15 +861,36 @@ ast_for_arguments(struct compiling *c, const node *n)
                 vararg = NEW_IDENTIFIER(CHILD(n, i+1));
                 if (!vararg)
                     return NULL;
-                i += 3;
+                i += 2; /* the star and the name */
+                if (TYPE(CHILD(n, i)) == COMMA)
+                    i += 1; /* the comma, if present */
                 break;
             case DOUBLESTAR:
                 if (!forbidden_check(c, CHILD(n, i+1), STR(CHILD(n, i+1))))
                     return NULL;
                 kwarg = NEW_IDENTIFIER(CHILD(n, i+1));
                 if (!kwarg)
                     return NULL;
-                i += 3;
+                i += 2; /* the double star and the name */
+                if (TYPE(CHILD(n, i)) == COMMA)
+                    i += 1; /* the comma, if present */
+                break;
+            case TYPE_COMMENT:
+                assert(l < k + !!vararg + !!kwarg);
+
+                if (!type_comments) {
+                    /* lazily allocate the type_comments seq for perf reasons */
+                    type_comments = asdl_seq_new(n_all_args, c->c_arena);
+                    if (!type_comments)
+                        return NULL;
+                }
+
+                while (l < k + !!vararg + !!kwarg - 1) {
+                    asdl_seq_SET(type_comments, l++, NULL);
+                }
+
+                asdl_seq_SET(type_comments, l++, NEW_TYPE_COMMENT(ch));
+                i += 1;
                 break;
             default:
                 PyErr_Format(PyExc_SystemError,
@@ -866,7 +900,13 @@ ast_for_arguments(struct compiling *c, const node *n)
         }
     }
 
-    return arguments(args, vararg, kwarg, defaults, c->c_arena);
+    if (type_comments) {
+        while (l < n_all_args) {
+            asdl_seq_SET(type_comments, l++, NULL);
+        }
+    }
+
+    return arguments(args, vararg, kwarg, defaults, type_comments, c->c_arena);
 }
 
 static expr_ty
@@ -1038,7 +1078,7 @@ ast_for_lambdef(struct compiling *c, const node *n)
     expr_ty expression;
 
     if (NCH(n) == 3) {
-        args = arguments(NULL, NULL, NULL, NULL, c->c_arena);
+        args = arguments(NULL, NULL, NULL, NULL, NULL, c->c_arena);
         if (!args)
             return NULL;
         expression = ast_for_expr(c, CHILD(n, 2));

diff --git a/ast27/Python/graminit.c b/ast27/Python/graminit.c
@@ -159,51 +159,72 @@ static arc arcs_8_0[3] = {
     {31, 2},
     {32, 3},
 };
-static arc arcs_8_1[3] = {
+static arc arcs_8_1[4] = {
     {28, 4},
     {30, 5},
+    {24, 6},
     {0, 1},
 };
 static arc arcs_8_2[1] = {
-    {21, 6},
+    {21, 7},
 };
 static arc arcs_8_3[1] = {
-    {21, 7},
+    {21, 8},
 };
 static arc arcs_8_4[1] = {
-    {29, 8},
+    {29, 9},
 };
-static arc arcs_8_5[4] = {
+static arc arcs_8_5[5] = {
     {27, 1},
+    {24, 10},
     {31, 2},
     {32, 3},
     {0, 5},
 };
-static arc arcs_8_6[2] = {
-    {30, 9},
+static arc arcs_8_6[1] = {
     {0, 6},
 };
-static arc arcs_8_7[1] = {
+static arc arcs_8_7[3] = {
+    {30, 11},
+    {24, 6},
     {0, 7},
 };
 static arc arcs_8_8[2] = {
-    {30, 5},
+    {24, 6},
     {0, 8},
 };
-static arc arcs_8_9[1] = {
+static arc arcs_8_9[3] = {
+    {30, 5},
+    {24, 6},
+    {0, 9},
+};
+static arc arcs_8_10[4] = {
+    {27, 1},
+    {31, 2},
+    {32, 3},
+    {0, 10},
+};
+static arc arcs_8_11[2] = {
+    {24, 12},
+    {32, 3},
+};
+static arc arcs_8_12[1] = {
     {32, 3},
 };
-static state states_8[10] = {
+static state states_8[13] = {
     {3, arcs_8_0},
-    {3, arcs_8_1},
+    {4, arcs_8_1},
     {1, arcs_8_2},
     {1, arcs_8_3},
     {1, arcs_8_4},
-    {4, arcs_8_5},
-    {2, arcs_8_6},
-    {1, arcs_8_7},
+    {5, arcs_8_5},
+    {1, arcs_8_6},
+    {3, arcs_8_7},
     {2, arcs_8_8},
-    {1, arcs_8_9},
+    {3, arcs_8_9},
+    {4, arcs_8_10},
+    {2, arcs_8_11},
+    {1, arcs_8_12},
 };
 static arc arcs_9_0[2] = {
     {21, 1},
@@ -1971,7 +1992,7 @@ static dfa dfas[88] = {
      "\000\000\020\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"},
     {263, "parameters", 0, 4, states_7,
      "\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"},
-    {264, "varargslist", 0, 10, states_8,
+    {264, "varargslist", 0, 13, states_8,
      "\000\040\040\200\001\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"},
     {265, "fpdef", 0, 4, states_9,
      "\000\040\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"},

diff --git a/ast35/Grammar/Grammar b/ast35/Grammar/Grammar
@@ -30,9 +30,10 @@ async_funcdef: ASYNC funcdef
 funcdef: 'def' NAME parameters ['->' test] ':' [TYPE_COMMENT] suite
 
 parameters: '(' [typedargslist] ')'
-typedargslist: (tfpdef ['=' test] (',' tfpdef ['=' test])* [','
-       ['*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef]]
-     |  '*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef)
+typedargslist: (tfpdef ['=' test] (',' [TYPE_COMMENT] tfpdef ['=' test])* [',' [TYPE_COMMENT]
+       ['*' [tfpdef] (',' [TYPE_COMMENT] tfpdef ['=' test])* [',' [TYPE_COMMENT] '**' tfpdef] | '**' tfpdef]] [TYPE_COMMENT]
+     |  '*' [tfpdef] (',' [TYPE_COMMENT] tfpdef ['=' test])* [',' [TYPE_COMMENT] '**' tfpdef] [TYPE_COMMENT]
+     | '**' tfpdef [TYPE_COMMENT])
 tfpdef: NAME [':' test]
 varargslist: (vfpdef ['=' test] (',' vfpdef ['=' test])* [','
        ['*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef]]