Add json(), unjson(), eval(). println(json(info)) works despite int…

… keys in the all_ids stack. Revamped errors vs normal output, don't output "nil" if not interactive. Updated extensions API. (#114)

* Add json() though... ugly but print(json(info)) works. Changed that errors are not put to stdout and nil result aren't printed except in interactive mode

* drop no_json

* shrink hashable (remove return and quote from there), augment Equals, notably Error,Function,Extension can be checked)

* added optional indent bool to json() and added eval() and unjson() being... more or less eval() for now

* Fixes #115 (but not pretty)

* update go install instructions

* only one type (extended/long) of callback, with an adapter for short ones

* pass the eval state as opaque any callback first arg

* implement custom JSON serialization for all object types. switch back to no_json, stop checking at runtime that tags are right

* collapse section of go.mod

* added tests for json of map with numeric keys, and for eval in right context

* nested array json test

* exclude pprof through no_pprof build tag, saves ~150k in binary size

* go 1.22.6 is out

* better with the new files

Makefile

@@ -1,6 +1,6 @@
 all: generate lint check test run
 
-GO_BUILD_TAGS:=no_net,no_json
+GO_BUILD_TAGS:=no_net,no_json,no_pprof
 
 run: grol
 	# Interactive debug run: use logger with file and line numbers

README.md

@@ -186,4 +186,4 @@ flags:
   -shared-state
     	All files share same interpreter state (default is new state for each)
 ```
-(excluding logger control, see `gorepl help` for all the flags, of note `-logger-no-color` will turn off colors for gorepl too, for development there are also `-profile*` options for pprof)
+(excluding logger control, see `gorepl help` for all the flags, of note `-logger-no-color` will turn off colors for gorepl too, for development there are also `-profile*` options for pprof, when building without `no_pprof`)

ast/modify_test.go

@@ -3,8 +3,6 @@ package ast
 import (
 	"reflect"
 	"testing"
-
-	"github.com/google/go-cmp/cmp"
 )
 
 func TestModify(t *testing.T) {
@@ -119,8 +117,5 @@ func TestModify(t *testing.T) {
 		if !reflect.DeepEqual(modified, tt.expected) {
 			t.Errorf("not equal.\n%#v\n-vs-\n%#v", modified, tt.expected)
 		}
-		if !cmp.Equal(modified, tt.expected) {
-			t.Errorf("not equal. %v", cmp.Diff(modified, tt.expected))
-		}
 	}
 }

eval/eval.go

@@ -35,6 +35,16 @@ func NewState() *State {
 	}
 }
 
+func NewBlankState() *State {
+	return &State{
+		env:        object.NewMacroEnvironment(), // to get empty store
+		Out:        io.Discard,
+		LogOut:     io.Discard,
+		cache:      NewCache(),
+		extensions: make(map[string]object.Extension),
+	}
+}
+
 func (s *State) ResetCache() {
 	s.cache = NewCache()
 }
@@ -398,6 +408,9 @@ func (s *State) applyExtension(fn object.Extension, args []object.Object) object
 		if i >= len(fn.ArgTypes) {
 			break
 		}
+		if fn.ArgTypes[i] == object.ANY {
+			continue
+		}
 		// Auto promote integer to float if needed.
 		if fn.ArgTypes[i] == object.FLOAT && arg.Type() == object.INTEGER {
 			args[i] = object.Float{Value: float64(arg.(object.Integer).Value)}
@@ -408,7 +421,7 @@ func (s *State) applyExtension(fn object.Extension, args []object.Object) object
 				arg.Type(), fn.Inspect())}
 		}
 	}
-	return fn.Callback(args)
+	return fn.Callback(s, fn.Name, args)
 }
 
 func (s *State) applyFunction(name string, fn object.Object, args []object.Object) object.Object {
@@ -781,17 +794,38 @@ func evalFloatInfixExpression(operator token.Type, left, right object.Object) ob
 // to the base identifiers. Used to add grol defined functions to the base environment
 // (e.g abs(), log2(), etc). Eventually we may instead `include("lib.gr")` or some such.
 func AddEvalResult(name, code string) error {
+	res, err := EvalString(NewState(), code, false)
+	if err != nil {
+		return err
+	}
+	object.AddIdentifier(name, res)
+	return nil
+}
+
+// Evals a string either from entirely blank environment or from the current environment.
+// `unjson` uses emptyEnv == true (for now, pending better/safer implementation).
+//
+//nolint:revive // eval.EvalString is fine.
+func EvalString(this any, code string, emptyEnv bool) (object.Object, error) {
 	l := lexer.New(code)
 	p := parser.New(l)
 	program := p.ParseProgram()
 	if len(p.Errors()) != 0 {
-		return fmt.Errorf("parsing error: %v", p.Errors())
+		return object.NULL, fmt.Errorf("parsing error: %v", p.Errors())
+	}
+	var evalState *State
+	if emptyEnv {
+		evalState = NewBlankState()
+	} else {
+		var ok bool
+		evalState, ok = this.(*State)
+		if !ok {
+			return object.NULL, fmt.Errorf("invalid this: %T", this)
+		}
 	}
-	st := NewState()
-	res := st.Eval(program)
+	res := evalState.Eval(program)
 	if res.Type() == object.ERROR {
-		return fmt.Errorf("eval error: %v", res.Inspect())
+		return res, fmt.Errorf("eval error: %v", res.Inspect())
 	}
-	object.AddIdentifier(name, res)
-	return nil
+	return res, nil
 }

eval/eval_test.go

@@ -828,4 +828,14 @@ func TestExtension(t *testing.T) {
 	if actual.Value != expected {
 		t.Errorf("object has wrong value. got=%q, want=%q", actual, expected)
 	}
+	input = `m={1.5:"a",2: {"str": 42, 3: pow}}; json(m)`
+	evaluated = testEval(t, input)
+	expected = `{"2":{"3":{"gofunc":"pow(float, float)"},"str":42},"1.5":"a"}`
+	actual, ok = evaluated.(object.String)
+	if !ok {
+		t.Errorf("object is not string. got=%T (%+v)", evaluated, evaluated)
+	}
+	if actual.Value != expected {
+		t.Errorf("object has wrong value.got:\n%s\n---want--\n%s", actual.Value, expected)
+	}
 }

extensions/extension.go

@@ -5,6 +5,7 @@ package extensions
 import (
 	"fmt"
 	"math"
+	"strings"
 
 	"grol.io/grol/eval"
 	"grol.io/grol/object"
@@ -32,7 +33,7 @@ func initInternal() error {
 		MinArgs:  2,
 		MaxArgs:  2,
 		ArgTypes: []object.Type{object.FLOAT, object.FLOAT},
-		Callback: pow,
+		Callback: object.ShortCallback(pow),
 	}
 	err := object.CreateFunction(cmd)
 	if err != nil {
@@ -43,13 +44,13 @@ func initInternal() error {
 		MinArgs:  1,
 		MaxArgs:  -1,
 		ArgTypes: []object.Type{object.STRING},
-		Callback: sprintf,
+		Callback: object.ShortCallback(sprintf),
 	})
 	if err != nil {
 		return err
 	}
 	// for printf, we could expose current eval "Out", but instead let's use new variadic support and define
-	// printf as print(snprintf(format,..))
+	// printf as print(snprintf(format,..)) that way the memoization of output also works out of the box.
 	err = eval.AddEvalResult("printf", "func(format, ..){print(sprintf(format, ..))}")
 	if err != nil {
 		return err
@@ -86,7 +87,7 @@ func initInternal() error {
 		{math.Ceil, "ceil"},
 		{math.Log10, "log10"},
 	} {
-		oneFloat.Callback = func(args []object.Object) object.Object {
+		oneFloat.Callback = func(_ any, _ string, args []object.Object) object.Object {
 			// Arg len check already done through MinArgs=MaxArgs=1 and
 			// type through ArgTypes: []object.Type{object.FLOAT}.
 			return object.Float{Value: function.fn(args[0].(object.Float).Value)}
@@ -99,6 +100,30 @@ func initInternal() error {
 	}
 	object.AddIdentifier("PI", object.Float{Value: math.Pi})
 	object.AddIdentifier("E", object.Float{Value: math.E}) // using uppercase so "e" isn't taken/shadowed.
+	jsonFn := object.Extension{
+		Name:     "json",
+		MinArgs:  1,
+		MaxArgs:  1,
+		ArgTypes: []object.Type{object.ANY},
+		Callback: object.ShortCallback(jsonSer),
+	}
+	err = object.CreateFunction(jsonFn)
+	if err != nil {
+		return err
+	}
+	jsonFn.Name = "eval"
+	jsonFn.Callback = evalFunc
+	jsonFn.ArgTypes = []object.Type{object.STRING}
+	err = object.CreateFunction(jsonFn)
+	if err != nil {
+		return err
+	}
+	jsonFn.Name = "unjson"
+	jsonFn.Callback = evalFunc // unjson at the moment is just (like) eval hoping that json is map/array/...
+	err = object.CreateFunction(jsonFn)
+	if err != nil {
+		return err
+	}
 	return nil
 }
 
@@ -115,3 +140,21 @@ func sprintf(args []object.Object) object.Object {
 	res := fmt.Sprintf(args[0].(object.String).Value, object.Unwrap(args[1:])...)
 	return object.String{Value: res}
 }
+
+func jsonSer(args []object.Object) object.Object {
+	w := strings.Builder{}
+	err := args[0].JSON(&w)
+	if err != nil {
+		return object.Error{Value: err.Error()}
+	}
+	return object.String{Value: w.String()}
+}
+
+func evalFunc(env any, name string, args []object.Object) object.Object {
+	s := args[0].(object.String).Value
+	res, err := eval.EvalString(env, s, name == "unjson" /* empty env */)
+	if err != nil {
+		return object.Error{Value: err.Error()}
+	}
+	return res
+}

go.mod

@@ -1,24 +1,20 @@
 module grol.io/grol
 
-go 1.22.5
+go 1.22.6
 
 require (
 	fortio.org/cli v1.8.0
 	fortio.org/log v1.16.0
+	fortio.org/sets v1.2.0
 	fortio.org/testscript v0.3.1 // only for tests
 	fortio.org/version v1.0.4
-	github.com/google/go-cmp v0.6.0 // only for tests
 )
 
-// replace fortio.org/log => ../../fortio.org/log
-
-require fortio.org/sets v1.1.1
-
 require (
 	fortio.org/struct2env v0.4.1 // indirect
 	github.com/kortschak/goroutine v1.1.2 // indirect
 	golang.org/x/crypto/x509roots/fallback v0.0.0-20240626151235-a6a393ffd658 // indirect
-	golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8 // indirect
-	golang.org/x/sys v0.21.0 // indirect
-	golang.org/x/tools v0.22.0 // indirect
+	golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
+	golang.org/x/sys v0.22.0 // indirect
+	golang.org/x/tools v0.23.0 // indirect
 )

go.sum

@@ -4,23 +4,21 @@ fortio.org/cli v1.8.0 h1:Mz1phmUwkQaXESGb1nIWBY+CHli/GYIlhwpktorh9sY=
 fortio.org/cli v1.8.0/go.mod h1:pk/JBE8LcXtNuo5Yj2bLsVbwPaHo8NWdbstSN0cpbFk=
 fortio.org/log v1.16.0 h1:GhU8/9NkYZmEIzvTN/DTMedDAStLJraWUUVUA2EbNDc=
 fortio.org/log v1.16.0/go.mod h1:t58Spg9njjymvRioh5F6qKGSupEsnMjXLGWIS1i3khE=
-fortio.org/sets v1.1.1 h1:Q7Z1Ft2lpUc1N7bfI8HofIK0QskrOflfYRyKT2LzBng=
-fortio.org/sets v1.1.1/go.mod h1:J2BwIxNOLWsSU7IMZUg541kh3Au4JEKHrghVwXs68tE=
+fortio.org/sets v1.2.0 h1:FBfC7R2xrOJtkcioUbY6WqEzdujuBoZRbSdp1fYF4Kk=
+fortio.org/sets v1.2.0/go.mod h1:J2BwIxNOLWsSU7IMZUg541kh3Au4JEKHrghVwXs68tE=
 fortio.org/struct2env v0.4.1 h1:rJludAMO5eBvpWplWEQNqoVDFZr4RWMQX7RUapgZyc0=
 fortio.org/struct2env v0.4.1/go.mod h1:lENUe70UwA1zDUCX+8AsO663QCFqYaprk5lnPhjD410=
 fortio.org/testscript v0.3.1 h1:MmRO64AsmzaU1KlYMzAbotJIMKRGxD1XXssJnBRiMGQ=
 fortio.org/testscript v0.3.1/go.mod h1:7OJ+U4avooRNqc7p/VHKJadYgj9fA6+N0SbGU8FVWGs=
 fortio.org/version v1.0.4 h1:FWUMpJ+hVTNc4RhvvOJzb0xesrlRmG/a+D6bjbQ4+5U=
 fortio.org/version v1.0.4/go.mod h1:2JQp9Ax+tm6QKiGuzR5nJY63kFeANcgrZ0osoQFDVm0=
-github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
-github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/kortschak/goroutine v1.1.2 h1:lhllcCuERxMIK5cYr8yohZZScL1na+JM5JYPRclWjck=
 github.com/kortschak/goroutine v1.1.2/go.mod h1:zKpXs1FWN/6mXasDQzfl7g0LrGFIOiA6cLs9eXKyaMY=
 golang.org/x/crypto/x509roots/fallback v0.0.0-20240626151235-a6a393ffd658 h1:i7K6wQLN/0oxF7FT3tKkfMCstxoT4VGG36YIB9ZKLzI=
 golang.org/x/crypto/x509roots/fallback v0.0.0-20240626151235-a6a393ffd658/go.mod h1:kNa9WdvYnzFwC79zRpLRMJbdEFlhyM5RPFBBZp/wWH8=
-golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8 h1:LoYXNGAShUG3m/ehNk4iFctuhGX/+R1ZpfJ4/ia80JM=
-golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI=
-golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
-golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA=
-golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c=
+golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8=
+golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY=
+golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
+golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg=
+golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI=

main.go

@@ -4,8 +4,8 @@ package main
 import (
 	"flag"
 	"fmt"
+	"io"
 	"os"
-	"runtime/pprof"
 
 	"fortio.org/cli"
 	"fortio.org/log"
@@ -19,15 +19,15 @@ func main() {
 	os.Exit(Main())
 }
 
+var hookBefore, hookAfter func() int
+
 func Main() int {
 	commandFlag := flag.String("c", "", "command/inline script to run instead of interactive mode")
 	showParse := flag.Bool("parse", false, "show parse tree")
 	format := flag.Bool("format", false, "don't execute, just parse and re format the input")
 	compact := flag.Bool("compact", false, "When printing code, use no indentation and most compact form")
 	showEval := flag.Bool("eval", true, "show eval results")
 	sharedState := flag.Bool("shared-state", false, "All files share same interpreter state (default is new state for each)")
-	cpuprofile := flag.String("profile-cpu", "", "write cpu profile to `file`")
-	memprofile := flag.String("profile-mem", "", "write memory profile to `file`")
 
 	cli.ArgsHelp = "*.gr files to interpret or `-` for stdin without prompt or no arguments for stdin repl..."
 	cli.MaxArgs = -1
@@ -39,18 +39,11 @@ func Main() int {
 		FormatOnly: *format,
 		Compact:    *compact,
 	}
-
-	if *cpuprofile != "" {
-		f, err := os.Create(*cpuprofile)
-		if err != nil {
-			return log.FErrf("can't open file for cpu profile: %v", err)
-		}
-		err = pprof.StartCPUProfile(f)
-		if err != nil {
-			return log.FErrf("can't start cpu profile: %v", err)
+	if hookBefore != nil {
+		ret := hookBefore()
+		if ret != 0 {
+			return ret
 		}
-		log.Infof("Writing cpu profile to %s", *cpuprofile)
-		defer pprof.StopCPUProfile()
 	}
 	err := extensions.Init()
 	if err != nil {
@@ -72,37 +65,38 @@ func Main() int {
 	s := eval.NewState()
 	macroState := object.NewMacroEnvironment()
 	for _, file := range flag.Args() {
-		processOneFile(file, s, macroState, options)
+		ret := processOneFile(file, s, macroState, options)
+		if ret != 0 {
+			return ret
+		}
 		if !*sharedState {
 			s = eval.NewState()
 			macroState = object.NewMacroEnvironment()
 		}
 	}
 	log.Infof("All done")
-	if *memprofile != "" {
-		f, err := os.Create(*memprofile)
-		if err != nil {
-			return log.FErrf("can't open file for mem profile: %v", err)
-		}
-		err = pprof.WriteHeapProfile(f)
-		if err != nil {
-			return log.FErrf("can't write mem profile: %v", err)
-		}
-		log.Infof("Wrote memory profile to %s", *memprofile)
-		f.Close()
+	if hookAfter != nil {
+		return hookAfter()
 	}
 	return 0
 }
 
-func processOneFile(file string, s *eval.State, macroState *object.Environment, options repl.Options) {
+func processOneStream(s *eval.State, macroState *object.Environment, in io.Reader, options repl.Options) int {
+	errs := repl.EvalAll(s, macroState, in, os.Stdout, options)
+	if len(errs) > 0 {
+		log.Errf("Errors: %v", errs)
+	}
+	return len(errs)
+}
+
+func processOneFile(file string, s *eval.State, macroState *object.Environment, options repl.Options) int {
 	if file == "-" {
 		if options.FormatOnly {
 			log.Infof("Formatting stdin")
 		} else {
 			log.Infof("Running on stdin")
 		}
-		repl.EvalAll(s, macroState, os.Stdin, os.Stdout, options)
-		return
+		return processOneStream(s, macroState, os.Stdin, options)
 	}
 	f, err := os.Open(file)
 	if err != nil {
@@ -113,6 +107,7 @@ func processOneFile(file string, s *eval.State, macroState *object.Environment,
 		verb = "Formatting"
 	}
 	log.Infof("%s %s", verb, file)
-	repl.EvalAll(s, macroState, f, os.Stdout, options)
+	code := processOneStream(s, macroState, f, options)
 	f.Close()
+	return code
 }