feat(gnovm): improved native bindings

.github/workflows/misc.yml → .github/workflows/lint.yml

@@ -76,3 +76,22 @@ jobs:
             env -C $path go mod tidy -v || exit 1
           done
           echo "$sums" | sha256sum -c
+  generated:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: 1.20.x
+
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Check generated files are up to date
+        run: |
+          go generate -v ./...
+          if [ "$(git status -s)" != "" ]; then
+            echo "command 'go generate' creates file that differ from git tree, please run 'go generate' and commit:"
+            git status -s
+            exit 1
+          fi

examples/gno.land/p/demo/avl/z_0_filetest.gno

@@ -267,6 +267,8 @@ func main() {
 //                 "FileName": "main.gno",
 //                 "IsMethod": false,
 //                 "Name": "init.0",
+//                 "NativeName": "",
+//                 "NativePkg": "",
 //                 "PkgPath": "gno.land/r/test",
 //                 "Source": {
 //                     "@type": "/gno.RefNode",
@@ -301,6 +303,8 @@ func main() {
 //                 "FileName": "main.gno",
 //                 "IsMethod": false,
 //                 "Name": "main",
+//                 "NativeName": "",
+//                 "NativePkg": "",
 //                 "PkgPath": "gno.land/r/test",
 //                 "Source": {
 //                     "@type": "/gno.RefNode",

examples/gno.land/p/demo/avl/z_1_filetest.gno

@@ -291,6 +291,8 @@ func main() {
 //                 "FileName": "main.gno",
 //                 "IsMethod": false,
 //                 "Name": "init.0",
+//                 "NativeName": "",
+//                 "NativePkg": "",
 //                 "PkgPath": "gno.land/r/test",
 //                 "Source": {
 //                     "@type": "/gno.RefNode",
@@ -325,6 +327,8 @@ func main() {
 //                 "FileName": "main.gno",
 //                 "IsMethod": false,
 //                 "Name": "main",
+//                 "NativeName": "",
+//                 "NativePkg": "",
 //                 "PkgPath": "gno.land/r/test",
 //                 "Source": {
 //                     "@type": "/gno.RefNode",

examples/gno.land/r/demo/boards/z_4_filetest.gno

@@ -374,7 +374,7 @@ func main() {
 //                     "Escaped": true,
 //                     "ObjectID": "336074805fc853987abe6f7fe3ad97a6a6f3077a:2"
 //                 },
-//                 "Index": "188",
+//                 "Index": "189",
 //                 "TV": null
 //             }
 //         }
@@ -541,7 +541,7 @@ func main() {
 //             },
 //             "V": {
 //                 "@type": "/gno.RefValue",
-//                 "Hash": "8164abed5231309c88497013f7da72a1b5d427b0",
+//                 "Hash": "25ffc45509708ca0ae17271cb4c3a1dfb367b965",
 //                 "ObjectID": "f6dbf411da22e67d74cd7ddba6a76cd7e14a4822:115"
 //             }
 //         },
@@ -847,7 +847,7 @@ func main() {
 //             },
 //             "V": {
 //                 "@type": "/gno.RefValue",
-//                 "Hash": "5b4b593f1d4b37cb99166247ea28174f91087fdd",
+//                 "Hash": "a8e67b9881af89ca2ec2f05778bf7528a54a5833",
 //                 "ObjectID": "f6dbf411da22e67d74cd7ddba6a76cd7e14a4822:82"
 //             }
 //         },
@@ -865,7 +865,7 @@ func main() {
 //             },
 //             "V": {
 //                 "@type": "/gno.RefValue",
-//                 "Hash": "7e9fd9bb5e90a06c7751585cd80f23aedddde25b",
+//                 "Hash": "d8ae14a4620e3c6dedabd76cd0c5d7e3c205d647",
 //                 "ObjectID": "f6dbf411da22e67d74cd7ddba6a76cd7e14a4822:83"
 //             }
 //         },

gnovm/pkg/gnolang/gno_test.go

@@ -14,7 +14,9 @@ import (
 // run empty main().
 func TestRunEmptyMain(t *testing.T) {
 	m := NewMachine("test", nil)
-	main := FuncD("main", nil, nil, nil)
+	// []Stmt{} != nil, as nil means that in the source code not even the
+	// brackets are present and is reserved for external (ie. native) functions.
+	main := FuncD("main", nil, nil, []Stmt{})
 	m.RunDeclaration(main)
 	m.RunMain()
 }

gnovm/pkg/gnolang/go2gno.go

@@ -96,9 +96,11 @@ func MustParseExpr(expr string) Expr {
 	return x
 }
 
-// filename must not include the path.
+// ParseFile uses the Go parser to parse body. It then runs [Go2Gno] on the
+// resulting AST -- the resulting FileNode is returned, together with any other
+// error (including panics, which are recovered) from [Go2Gno].
 func ParseFile(filename string, body string) (fn *FileNode, err error) {
-	// Parse src but stop after processing the imports.
+	// Use go parser to parse the body.
 	fs := token.NewFileSet()
 	f, err := parser.ParseFile(fs, filename, body, parser.ParseComments|parser.DeclarationErrors)
 	if err != nil {
@@ -112,9 +114,9 @@ func ParseFile(filename string, body string) (fn *FileNode, err error) {
 	defer func() {
 		if r := recover(); r != nil {
 			if rerr, ok := r.(error); ok {
-				err = rerr
+				err = errors.Wrap(rerr, "parsing file")
 			} else {
-				err = errors.New(fmt.Sprintf("%v", r))
+				err = errors.New(fmt.Sprintf("%v", r)).Stacktrace()
 			}
 			return
 		}
@@ -431,7 +433,10 @@ func Go2Gno(fs *token.FileSet, gon ast.Node) (n Node) {
 		}
 		name := toName(gon.Name)
 		type_ := Go2Gno(fs, gon.Type).(*FuncTypeExpr)
-		body := Go2Gno(fs, gon.Body).(*BlockStmt).Body
+		var body []Stmt
+		if gon.Body != nil {
+			body = Go2Gno(fs, gon.Body).(*BlockStmt).Body
+		}
 		return &FuncDecl{
 			IsMethod: isMethod,
 			Recv:     recv,

gnovm/pkg/gnolang/gonative.go

@@ -676,6 +676,9 @@ func go2GnoValueUpdate(alloc *Allocator, rlm *Realm, lvl int, tv *TypedValue, rv
 	return
 }
 
+// used for direct comparison to error types
+var tError = reflect.TypeOf(new(error)).Elem()
+
 // If recursive is false, this function is like go2GnoValue() but less lazy
 // (but still not recursive/eager). When recursive is false, it is for
 // converting Go types to Gno types upon an explicit conversion (via
@@ -757,6 +760,15 @@ func go2GnoValue2(alloc *Allocator, store Store, rv reflect.Value, recursive boo
 		// regardless.
 		tv.V = alloc.NewNative(rv)
 	case reflect.Interface:
+		// special case for errors, which are very often used esepcially in
+		// native bindings
+		if rv.Type() == tError {
+			tv.T = gErrorType
+			if !rv.IsNil() {
+				tv.V = alloc.NewNative(rv.Elem())
+			}
+			return
+		}
 		panic("not yet implemented")
 	case reflect.Map:
 		panic("not yet implemented")

gnovm/pkg/gnolang/machine.go

@@ -45,14 +45,15 @@ type Machine struct {
 	Context interface{}
 }
 
-// machine.Release() must be called on objects
-// created via this constructor
-// Machine with new package of given path.
-// Creates a new MemRealmer for any new realms.
-// Looks in store for package of pkgPath; if not found,
-// creates new instances as necessary.
-// If pkgPath is zero, the machine has no active package
-// and one must be set prior to usage.
+// NewMachine initializes a new gno virtual machine, acting as a shorthand
+// for [NewMachineWithOptions], setting the given options PkgPath and Store.
+//
+// The machine will run on the package at the given path, which will be
+// retrieved through the given store. If it is not set, the machine has no
+// active package, and one must be set prior to usage.
+//
+// Like for [NewMachineWithOptions], Machines initialized through this
+// constructor must be finalized with [Machine.Release].
 func NewMachine(pkgPath string, store Store) *Machine {
 	return NewMachineWithOptions(
 		MachineOptions{
@@ -61,12 +62,14 @@ func NewMachine(pkgPath string, store Store) *Machine {
 		})
 }
 
+// MachineOptions is used to pass options to [NewMachineWithOptions].
 type MachineOptions struct {
+	// Active package of the given machine; must be set before execution.
 	PkgPath       string
 	CheckTypes    bool // not yet used
 	ReadOnly      bool
-	Output        io.Writer
-	Store         Store
+	Output        io.Writer // default os.Stdout
+	Store         Store     // default NewStore(Alloc, nil, nil)
 	Context       interface{}
 	Alloc         *Allocator // or see MaxAllocBytes.
 	MaxAllocBytes int64      // or 0 for no limit.
@@ -86,6 +89,11 @@ var machinePool = sync.Pool{
 	},
 }
 
+// NewMachineWithOptions initializes a new gno virtual machine with the given
+// options.
+//
+// Machines initialized through this constructor must be finalized with
+// [Machine.Release].
 func NewMachineWithOptions(opts MachineOptions) *Machine {
 	checkTypes := opts.CheckTypes
 	readOnly := opts.ReadOnly
@@ -147,13 +155,11 @@ var (
 	valueZeroed [ValueSize]TypedValue
 )
 
-// m should not be used after this call
-// if m is nil, this will panic
-// this is on purpose, to discourage misuse
-// and prevent objects that were not taken from
-// the pool, to call Release
+// Release resets some of the values of *Machine and puts back m into the
+// machine pool; for this reason, Release() should be called as a finalizer,
+// and m should not be used after this call. Only Machines initialized with this
+// package's constructors should be released.
 func (m *Machine) Release() {
-	// copy()
 	// here we zero in the values for the next user
 	m.NumOps = 0
 	m.NumValues = 0
@@ -180,10 +186,19 @@ func (m *Machine) SetActivePackage(pv *PackageValue) {
 
 // Upon restart, preprocess all MemPackage and save blocknodes.
 // This is a temporary measure until we optimize/make-lazy.
+//
+// NOTE: package paths not beginning with gno.land will be allowed to override,
+// to support cases of stdlibs processed through [RunMemPackagesWithOverrides].
 func (m *Machine) PreprocessAllFilesAndSaveBlockNodes() {
 	ch := m.Store.IterMemPackage()
 	for memPkg := range ch {
 		fset := ParseMemPackage(memPkg)
+		// Implicitly do filterDuplicates for stdlibs.
+		// This is done as this function is at the time of writing only used
+		// in tests, ie. where we use the "override" feature for stdlibs.
+		if !strings.HasPrefix(memPkg.Path, "gno.land/") {
+			filterDuplicates(fset)
+		}
 		pn := NewPackageNode(Name(memPkg.Name), memPkg.Path, fset)
 		m.Store.SetBlockNode(pn)
 		PredefineFileSet(m.Store, pn, fset)
@@ -214,8 +229,23 @@ func (m *Machine) PreprocessAllFilesAndSaveBlockNodes() {
 // and corresponding package node, package value, and types to store. Save
 // is set to false for tests where package values may be native.
 func (m *Machine) RunMemPackage(memPkg *std.MemPackage, save bool) (*PackageNode, *PackageValue) {
+	return m.runMemPackage(memPkg, save, false)
+}
+
+// RunMemPackageWithOverrides works as [RunMemPackage], however after parsing,
+// declarations are filtered removing duplicate declarations.
+// To control which declaration overrides which, use [ReadMemPackageFromList],
+// putting the overrides at the top of the list.
+func (m *Machine) RunMemPackageWithOverrides(memPkg *std.MemPackage, save bool) (*PackageNode, *PackageValue) {
+	return m.runMemPackage(memPkg, save, true)
+}
+
+func (m *Machine) runMemPackage(memPkg *std.MemPackage, save, overrides bool) (*PackageNode, *PackageValue) {
 	// parse files.
 	files := ParseMemPackage(memPkg)
+	if overrides {
+		filterDuplicates(files)
+	}
 	// make and set package if doesn't exist.
 	pn := (*PackageNode)(nil)
 	pv := (*PackageValue)(nil)
@@ -242,6 +272,47 @@ func (m *Machine) RunMemPackage(memPkg *std.MemPackage, save bool) (*PackageNode
 	return pn, pv
 }
 
+func filterDuplicates(fset *FileSet) {
+	defined := make(map[Name]struct{}, 128)
+	for _, f := range fset.Files {
+		for i := 0; i < len(f.Decls); i++ {
+			d := f.Decls[i]
+			var name Name
+			switch d := d.(type) {
+			case *FuncDecl:
+				name = d.Name
+				if d.IsMethod {
+					name = Name(derefStar(d.Recv.Type).String()) + "." + name
+				}
+			case *TypeDecl:
+				name = d.Name
+			case *ValueDecl:
+				if len(d.NameExprs) == 1 {
+					name = d.NameExprs[0].Name
+				} else {
+					// TODO: support multiple names
+					continue
+				}
+			default:
+				continue
+			}
+			if _, ok := defined[name]; ok {
+				f.Decls = append(f.Decls[:i], f.Decls[i+1:]...)
+				i--
+			} else {
+				defined[name] = struct{}{}
+			}
+		}
+	}
+}
+
+func derefStar(x Expr) Expr {
+	if x, ok := x.(*StarExpr); ok {
+		return x.X
+	}
+	return x
+}
+
 // Tests all test files in a mempackage.
 // Assumes that the importing of packages is handled elsewhere.
 // The resulting package value and node become injected with TestMethods and