Generate wrappers for all contracts (#2010)

* abi-gen/Py: fix return type for multi-val returns

Methods that return multiple values were broken in two ways.  One: a
spurious newline was being injected between the return type and the
colon ending the Python method prototype.  Two: the return type was
being generated as just `[TypeA, TypeB]`, whereas it should be
`Tuple[TypeA, TypeB]`.

* abi-gen/Py: fix support for arrays of structs

* abi-gen/Py: FAILING test case nested unrefd struct

When a struct contains another struct, and the inner struct is not
directly referenced by any method interface, wrapper generation is
failing to render a class to represent the inner struct.

This won't fail in CI because at this time CI doesn't run any native
Python tooling to analyze the generated code.  Running mypy locally on
the files in this commit produces the following output:

test-cli/output/python/abi_gen_dummy/__init__.py:76: error: Name 'Tuple0x246f9407' is not defined

This problem affects the generation of wrappers for the DutchAuction
contract.

* abi-gen/Py: fix nested unref'd struct failure

* abi-gen/Py: introduce newlines to quiet linter

When generating contracts with long names (eg
CoordinatorRegistryValidator), the `black` reformatter was introducing
these newlines for us, and it was moving the `# type: ignore` comment in
there such that it no longer was on the line it needed to be on.
Introducing these newlines manually (instead of letting black inject
them) allows the linter directive to stay where it needs to be.

* abi-gen/Py: declare tuples in dependency order

* abi-gen/Py: fix support for overloaded methods

* contract_wrappers.py: pylint: permit 2-char args

By default pylint says that 2 characters is too short for an argument
name, but we have some contract methods with 2-character argument names
(eg `to` in `AssetProxyOwner.getTransactionIds()`), so we want to permit
them.

* contract_wrappers.py: include all contracts

* Update CHANGELOGs

* abi-gen: rename variable

* abi-gen: refine comments

* abi-gen/Py: reword tuple class docstring

.gitignore

@@ -114,6 +114,24 @@ contracts/exchange-forwarder/generated-wrappers/
 contracts/dev-utils/generated-wrappers/
 python-packages/contract_wrappers/src/zero_ex/contract_wrappers/erc20_token/__init__.py
 python-packages/contract_wrappers/src/zero_ex/contract_wrappers/exchange/__init__.py
+python-packages/contract_wrappers/src/zero_ex/contract_wrappers/asset_proxy_owner/__init__.py
+python-packages/contract_wrappers/src/zero_ex/contract_wrappers/coordinator/__init__.py
+python-packages/contract_wrappers/src/zero_ex/contract_wrappers/coordinator_registry/__init__.py
+python-packages/contract_wrappers/src/zero_ex/contract_wrappers/dummy_erc20_token/__init__.py
+python-packages/contract_wrappers/src/zero_ex/contract_wrappers/dummy_erc721_token/__init__.py
+python-packages/contract_wrappers/src/zero_ex/contract_wrappers/dutch_auction/__init__.py
+python-packages/contract_wrappers/src/zero_ex/contract_wrappers/erc20_proxy/__init__.py
+python-packages/contract_wrappers/src/zero_ex/contract_wrappers/erc721_proxy/__init__.py
+python-packages/contract_wrappers/src/zero_ex/contract_wrappers/erc721_token/__init__.py
+python-packages/contract_wrappers/src/zero_ex/contract_wrappers/eth_balance_checker/__init__.py
+python-packages/contract_wrappers/src/zero_ex/contract_wrappers/forwarder/__init__.py
+python-packages/contract_wrappers/src/zero_ex/contract_wrappers/i_asset_proxy/__init__.py
+python-packages/contract_wrappers/src/zero_ex/contract_wrappers/i_validator/__init__.py
+python-packages/contract_wrappers/src/zero_ex/contract_wrappers/i_wallet/__init__.py
+python-packages/contract_wrappers/src/zero_ex/contract_wrappers/multi_asset_proxy/__init__.py
+python-packages/contract_wrappers/src/zero_ex/contract_wrappers/order_validator/__init__.py
+python-packages/contract_wrappers/src/zero_ex/contract_wrappers/weth9/__init__.py
+python-packages/contract_wrappers/src/zero_ex/contract_wrappers/zrx_token/__init__.py
 
 # cli test output
 packages/abi-gen/test-cli/output

packages/abi-gen-templates/Python/contract.handlebars

@@ -34,7 +34,9 @@ try:
     )
 except ImportError:
 
-    class {{contractName}}Validator(Validator):  # type: ignore
+    class {{contractName}}Validator(  # type: ignore
+        Validator
+    ):
         """No-op input validator."""
 
 
@@ -48,7 +50,7 @@ except ImportError:
 class {{contractName}}:
     """Wrapper class for {{contractName}} Solidity contract.{{docBytesIfNecessary ABIString}}"""
 {{#each methods}}
-    {{toPythonIdentifier this.name}}: {{toPythonClassname this.name}}Method
+    {{toPythonIdentifier this.languageSpecificName}}: {{toPythonClassname this.languageSpecificName}}Method
 {{/each}}
 
     def __init__(
@@ -75,7 +77,7 @@ class {{contractName}}:
         functions = self._web3_eth.contract(address=to_checksum_address(contract_address), abi={{contractName}}.abi()).functions
 
         {{#each methods}}
-        self.{{toPythonIdentifier this.name}} = {{toPythonClassname this.name}}Method(provider, contract_address, functions.{{this.name}}, validator)
+        self.{{toPythonIdentifier this.languageSpecificName}} = {{toPythonClassname this.languageSpecificName}}Method(provider, contract_address, functions.{{this.name}}, validator)
 
         {{/each}}
 {{#each events}}

packages/abi-gen-templates/Python/partials/method_class.handlebars

@@ -1,5 +1,5 @@
 
-class {{toPythonClassname this.name}}Method(ContractMethod):
+class {{toPythonClassname this.languageSpecificName}}Method(ContractMethod):
     """Various interfaces to the {{this.name}} method."""
 
     def __init__(self, provider: BaseProvider, contract_address: str, contract_function: ContractFunction, validator: Validator=None):

packages/abi-gen-templates/Python/partials/return_type.handlebars

@@ -7,8 +7,8 @@ Union[
 {{#returnType outputs.0.type outputs.0.components}}{{~/returnType~}}
 {{/singleReturnValue}}
 {{^singleReturnValue}}
-[{{#each outputs}}{{#returnType type components}}{{/returnType}}{{#unless @last}}, {{/unless}}{{/each}}]
-{{/singleReturnValue}}
+Tuple[{{#each outputs}}{{#returnType type components}}{{/returnType}}{{#unless @last}}, {{/unless}}{{/each}}]
+{{~/singleReturnValue}}
 {{else}}None
 {{/if}}{{^if this.constant}}, Union[HexBytes, bytes]]{{/if~}}
 {{else}}{{#if this.constant}}None{{else}}Union[None, Union[HexBytes, bytes]]{{/if}}{{/if~}}

packages/abi-gen/CHANGELOG.json

@@ -30,6 +30,26 @@
             {
                 "note": "added gas estimation functionality to contract methods",
                 "pr": 1996
+            },
+            {
+                "note": "Python: fixed bug with methods returning multiple values",
+                "pr": 1996
+            },
+            {
+                "note": "Python: fixed bug with methods returning arrays of structs",
+                "pr": 1996
+            },
+            {
+                "note": "Python: fixed bug with methods that return a struct that contains another struct where the inner struct was not otherwise directly referenced by any method",
+                "pr": 1996
+            },
+            {
+                "note": "Python: fixed bug with tuples sometimes being used before they were declared",
+                "pr": 1996
+            },
+            {
+                "note": "Python: fixed bug with supporting overloaded methods",
+                "pr": 1996
             }
         ]
     },

packages/abi-gen/package.json

@@ -60,6 +60,7 @@
         "@0x/types": "^2.4.1",
         "@0x/typescript-typings": "^4.2.4",
         "@0x/utils": "^4.4.2",
+        "@types/toposort": "^2.0.1",
         "chalk": "^2.3.0",
         "change-case": "^3.0.2",
         "cli-format": "^3.0.9",
@@ -70,6 +71,7 @@
         "mkdirp": "^0.5.1",
         "tmp": "^0.0.33",
         "to-snake-case": "^1.0.0",
+        "toposort": "^2.0.2",
         "yargs": "^10.0.3"
     },
     "devDependencies": {

packages/abi-gen/src/index.ts

@@ -12,12 +12,12 @@ import {
     DevdocOutput,
     EventAbi,
     MethodAbi,
-    TupleDataItem,
 } from 'ethereum-types';
 import { sync as globSync } from 'glob';
 import * as Handlebars from 'handlebars';
 import * as _ from 'lodash';
 import * as mkdirp from 'mkdirp';
+import toposort = require('toposort');
 import * as yargs from 'yargs';
 
 import { ContextData, ContractsBackend, ParamKind } from './types';
@@ -191,10 +191,15 @@ function registerPythonHelpers(): void {
     Handlebars.registerHelper('tupleDefinitions', (abisJSON: string) => {
         const abis: AbiDefinition[] = JSON.parse(abisJSON);
         // build an array of objects, each of which has one key, the Python
-        // name of a tuple, with a string value holding the Python
-        // definition of that tuple. Using a key-value object conveniently
+        // name of a tuple, with a string value holding the body of a Python
+        // class representing that tuple. Using a key-value object conveniently
         // filters duplicate references to the same tuple.
-        const tupleDefinitions: { [pythonTupleName: string]: string } = {};
+        const tupleBodies: { [pythonTupleName: string]: string } = {};
+        // build an array of tuple dependencies, whose format conforms to the
+        // expected input to toposort, a function to do a topological sort,
+        // which will help us declare tuples in the proper order, avoiding
+        // references to tuples that haven't been declared yet.
+        const tupleDependencies: Array<[string, string]> = [];
         for (const abi of abis) {
             let parameters: DataItem[] = [];
             if (abi.hasOwnProperty('inputs')) {
@@ -216,24 +221,36 @@ function registerPythonHelpers(): void {
                 parameters = parameters.concat((abi as MethodAbi).outputs);
             }
             for (const parameter of parameters) {
-                if (parameter.type === 'tuple') {
-                    tupleDefinitions[
-                        utils.makePythonTupleName((parameter as TupleDataItem).components)
-                    ] = utils.makePythonTupleClassBody((parameter as TupleDataItem).components);
-                }
+                utils.extractTuples(parameter, tupleBodies, tupleDependencies);
+            }
+        }
+        // build up a list of tuples to declare. the order they're pushed into
+        // this array is the order they will be declared.
+        const tuplesToDeclare = [];
+        // first push the ones that have dependencies
+        tuplesToDeclare.push(...toposort(tupleDependencies));
+        // then push any remaining bodies (the ones that DON'T have
+        // dependencies)
+        for (const pythonTupleName in tupleBodies) {
+            if (!tuplesToDeclare.includes(pythonTupleName)) {
+                tuplesToDeclare.push(pythonTupleName);
             }
         }
+        // now iterate over those ordered tuples-to-declare, and prefix the
+        // corresponding class bodies with their class headers, to form full
+        // class declarations.
         const tupleDeclarations = [];
-        for (const pythonTupleName in tupleDefinitions) {
-            if (tupleDefinitions[pythonTupleName]) {
+        for (const pythonTupleName of tuplesToDeclare) {
+            if (tupleBodies[pythonTupleName]) {
                 tupleDeclarations.push(
-                    `class ${pythonTupleName}(TypedDict):\n    """Python representation of a tuple or struct.\n\n    A tuple found in an ABI may have been written in Solidity as a literal\n    tuple, or it may have been written as a parameter with a Solidity\n    \`struct\`:code: data type; there's no way to tell which, based solely on the\n    ABI, and the name of a Solidity \`struct\`:code: is not conveyed through the\n    ABI.  This class represents a tuple that appeared in a method definition.\n    Its name is derived from a hash of that tuple's field names, and every\n    method whose ABI refers to a tuple with that same list of field names will\n    have a generated wrapper method that refers to this class.\n\n    Any members of type \`bytes\`:code: should be encoded as UTF-8, which can be\n    accomplished via \`str.encode("utf_8")\`:code:\n    """${
-                        tupleDefinitions[pythonTupleName]
+                    `class ${pythonTupleName}(TypedDict):\n    """Python representation of a tuple or struct.\n\n    Solidity compiler output does not include the names of structs that appear\n    in method definitions.  A tuple found in an ABI may have been written in\n    Solidity as a literal, anonymous tuple, or it may have been written as a\n    named \`struct\`:code:, but there is no way to tell from the compiler\n    output.  This class represents a tuple that appeared in a method\n    definition.  Its name is derived from a hash of that tuple's field names,\n    and every method whose ABI refers to a tuple with that same list of field\n    names will have a generated wrapper method that refers to this class.\n\n    Any members of type \`bytes\`:code: should be encoded as UTF-8, which can be\n    accomplished via \`str.encode("utf_8")\`:code:\n    """${
+                        tupleBodies[pythonTupleName]
                     }`,
                 );
             }
         }
-        return new Handlebars.SafeString(tupleDeclarations.join('\n\n'));
+        // finally, join the class declarations together for the output file
+        return new Handlebars.SafeString(tupleDeclarations.join('\n\n\n'));
     });
     Handlebars.registerHelper('docBytesIfNecessary', (abisJSON: string) => {
         const abis: AbiDefinition[] = JSON.parse(abisJSON);

packages/abi-gen/src/utils.ts

@@ -2,7 +2,7 @@ import { createHash } from 'crypto';
 
 import * as changeCase from 'change-case';
 import * as cliFormat from 'cli-format';
-import { AbiType, ConstructorAbi, DataItem } from 'ethereum-types';
+import { AbiType, ConstructorAbi, DataItem, TupleDataItem } from 'ethereum-types';
 import * as fs from 'fs';
 import * as _ from 'lodash';
 import * as path from 'path';
@@ -350,4 +350,28 @@ export const utils = {
             hangingIndent: ' '.repeat(columnsPerIndent),
         });
     },
+    extractTuples(
+        parameter: DataItem,
+        tupleBodies: { [pythonTupleName: string]: string }, // output
+        tupleDependencies: Array<[string, string]>, // output
+    ): void {
+        if (parameter.type === 'tuple' || parameter.type === 'tuple[]') {
+            const tupleDataItem = parameter as TupleDataItem; // tslint:disable-line:no-unnecessary-type-assertion
+            // without the above cast (which tslint complains about), tsc says
+            //     Argument of type 'DataItem[] | undefined' is not assignable to parameter of type 'DataItem[]'.
+            //     Type 'undefined' is not assignable to type 'DataItem[]'
+            // when the code below tries to access tupleDataItem.components.
+            const pythonTupleName = utils.makePythonTupleName(tupleDataItem.components);
+            tupleBodies[pythonTupleName] = utils.makePythonTupleClassBody(tupleDataItem.components);
+            for (const component of tupleDataItem.components) {
+                if (component.type === 'tuple' || component.type === 'tuple[]') {
+                    tupleDependencies.push([
+                        utils.makePythonTupleName((component as TupleDataItem).components), // tslint:disable-line:no-unnecessary-type-assertion
+                        pythonTupleName,
+                    ]);
+                    utils.extractTuples(component, tupleBodies, tupleDependencies);
+                }
+            }
+        }
+    },
 };