Simplify expression using precedence (#183)

* Simplify expression using precedence. step 1: failing test

* moved precendence to ast so it can be used to format the tree back. updated tests. failing right now for basic like (1+2)*3

* Added -parse-debug to see fully parenthesized version of ast. Fixed needParen, mostly working now

* fix prefix (mostly, except for double prefix)

* fix / vs * precedence bug, fix double prefix precendence bug

* All passing

* self review: undoing 1 readme chg

README.md

@@ -179,9 +179,8 @@ See [Open Issues](https://grol.io/grol/issues) for what's left to do
 ### CLI Usage
 
 ```
-grol 0.38.0 usage:
-	grol [flags] *.gr files to interpret or `-` for stdin without prompt
-  or no arguments for stdin repl...
+grol 0.58.0 usage:
+	grol [flags] *.gr files to interpret or `-` for stdin without prompt or no arguments for stdin repl...
 or 1 of the special arguments
 	grol {help|envhelp|version|buildinfo}
 flags:
@@ -198,15 +197,21 @@ flags:
   -history file
     	history file to use (default "~/.grol_history")
   -max-depth int
-    	Maximum interpreter depth (default 250000)
+    	Maximum interpreter depth (default 249999)
   -max-history size
     	max history size, use 0 to disable. (default 99)
+  -max-save-len int
+    	Maximum len of saved identifiers, use 0 for unlimited (default 4000)
   -no-auto
     	don't auto load/save the state to ./.gr
   -no-load-save
     	disable load/save of history
+  -panic
+    	Don't catch panic - only for development/debugging
   -parse
     	show parse tree
+  -parse-debug
+    	show all parenthesis in parse tree (default is to simplify using precedence)
   -quiet
     	Quiet mode, sets loglevel to Error (quietly) to reduces the output
   -shared-state

ast/ast.go

@@ -12,19 +12,72 @@ import (
 	"grol.io/grol/token"
 )
 
+type Priority int8
+
+const (
+	_ Priority = iota
+	LOWEST
+	ASSIGN      // =
+	OR          // ||
+	AND         // &&
+	EQUALS      // ==
+	LESSGREATER // > or <
+	SUM         // +
+	PRODUCT     // *
+	DIVIDE      // /
+	PREFIX      // -X or !X
+	CALL        // myFunction(X)
+	INDEX       // array[index]
+	DOTINDEX    // map.str access
+)
+
+var Precedences = map[token.Type]Priority{
+	token.ASSIGN:     ASSIGN,
+	token.OR:         OR,
+	token.AND:        AND,
+	token.COLON:      AND, // range operator and maps (lower than lambda)
+	token.EQ:         EQUALS,
+	token.NOTEQ:      EQUALS,
+	token.LAMBDA:     EQUALS,
+	token.LT:         LESSGREATER,
+	token.GT:         LESSGREATER,
+	token.LTEQ:       LESSGREATER,
+	token.GTEQ:       LESSGREATER,
+	token.PLUS:       SUM,
+	token.MINUS:      SUM,
+	token.BITOR:      SUM,
+	token.BITXOR:     SUM,
+	token.BITAND:     PRODUCT,
+	token.ASTERISK:   PRODUCT,
+	token.PERCENT:    PRODUCT,
+	token.LEFTSHIFT:  PRODUCT,
+	token.RIGHTSHIFT: PRODUCT,
+	token.SLASH:      DIVIDE,
+	token.INCR:       PREFIX,
+	token.DECR:       PREFIX,
+	token.LPAREN:     CALL,
+	token.LBRACKET:   INDEX,
+	token.DOT:        DOTINDEX,
+}
+
+//go:generate stringer -type=Priority
+var _ = DOTINDEX.String() // force compile error if go generate is missing.
+
 type PrintState struct {
-	Out             io.Writer
-	IndentLevel     int
-	ExpressionLevel int
-	IndentationDone bool // already put N number of tabs, reset on each new line
-	Compact         bool // don't indent at all (compact mode), no newlines, fewer spaces, no comments
-	prev            Node
-	last            string
+	Out                  io.Writer
+	IndentLevel          int
+	ExpressionPrecedence Priority
+	IndentationDone      bool // already put N number of tabs, reset on each new line
+	Compact              bool // don't indent at all (compact mode), no newlines, fewer spaces, no comments
+	AllParens            bool // print all expressions fully parenthesized.
+	prev                 Node
+	last                 string
 }
 
 func DebugString(n Node) string {
 	ps := NewPrintState()
 	ps.Compact = true
+	ps.AllParens = true
 	n.PrettyPrint(ps)
 	return ps.String()
 }
@@ -158,12 +211,12 @@ func prettyPrintLongForm(ps *PrintState, s Node, i int) {
 }
 
 func (p Statements) PrettyPrint(ps *PrintState) *PrintState {
-	oldExpressionLevel := ps.ExpressionLevel
+	oldExpressionPrecedence := ps.ExpressionPrecedence
 	if ps.IndentLevel > 0 {
 		ps.Print("{") // first statement might be a comment on same line.
 	}
 	ps.IndentLevel++
-	ps.ExpressionLevel = 0
+	ps.ExpressionPrecedence = LOWEST
 	var i int
 	for _, s := range p.Statements {
 		if ps.Compact {
@@ -179,7 +232,7 @@ func (p Statements) PrettyPrint(ps *PrintState) *PrintState {
 	}
 	ps.Println()
 	ps.IndentLevel--
-	ps.ExpressionLevel = oldExpressionLevel
+	ps.ExpressionPrecedence = oldExpressionPrecedence
 	if ps.IndentLevel > 0 {
 		ps.Print("}")
 	}
@@ -231,15 +284,27 @@ type PrefixExpression struct {
 	Right Node
 }
 
+func (ps *PrintState) needParen(t *token.Token) (bool, Priority) {
+	newPrecedence, ok := Precedences[t.Type()]
+	if !ok {
+		panic("precedence not found for " + t.Literal())
+	}
+	oldPrecedence := ps.ExpressionPrecedence
+	ps.ExpressionPrecedence = newPrecedence
+	return ps.AllParens || newPrecedence < oldPrecedence, oldPrecedence
+}
+
 func (p PrefixExpression) PrettyPrint(out *PrintState) *PrintState {
-	if out.ExpressionLevel > 0 {
+	oldPrecedence := out.ExpressionPrecedence
+	out.ExpressionPrecedence = PREFIX
+	needParen := out.AllParens || PREFIX <= oldPrecedence // double prefix like -(-a) needs parens to not become --a prefix.
+	if needParen {
 		out.Print("(")
 	}
 	out.Print(p.Literal())
-	out.ExpressionLevel++ // comment out for !(-a) to normalize to !-a
 	p.Right.PrettyPrint(out)
-	out.ExpressionLevel--
-	if out.ExpressionLevel > 0 {
+	out.ExpressionPrecedence = oldPrecedence
+	if needParen {
 		out.Print(")")
 	}
 	return out
@@ -251,14 +316,16 @@ type PostfixExpression struct {
 }
 
 func (p PostfixExpression) PrettyPrint(out *PrintState) *PrintState {
-	if out.ExpressionLevel > 0 {
+	needParen, oldPrecedence := out.needParen(p.Token)
+	if needParen {
 		out.Print("(")
 	}
 	out.Print(p.Prev.Literal())
 	out.Print(p.Literal())
-	if out.ExpressionLevel > 0 {
+	if needParen {
 		out.Print(")")
 	}
+	out.ExpressionPrecedence = oldPrecedence
 	return out
 }
 
@@ -269,13 +336,10 @@ type InfixExpression struct {
 }
 
 func (i InfixExpression) PrettyPrint(out *PrintState) *PrintState {
-	if out.ExpressionLevel > 0 { // TODO only add parens if precedence requires it.
+	needParen, oldPrecedence := out.needParen(i.Token)
+	if needParen {
 		out.Print("(")
 	}
-	isAssign := (i.Token.Type() == token.ASSIGN)
-	if !isAssign {
-		out.ExpressionLevel++
-	}
 	i.Left.PrettyPrint(out)
 	if out.Compact {
 		out.Print(i.Literal())
@@ -287,12 +351,10 @@ func (i InfixExpression) PrettyPrint(out *PrintState) *PrintState {
 	} else {
 		i.Right.PrettyPrint(out)
 	}
-	if !isAssign {
-		out.ExpressionLevel--
-	}
-	if out.ExpressionLevel > 0 {
+	if needParen {
 		out.Print(")")
 	}
+	out.ExpressionPrecedence = oldPrecedence
 	return out
 }
 
@@ -417,10 +479,10 @@ type CallExpression struct {
 func (ce CallExpression) PrettyPrint(out *PrintState) *PrintState {
 	ce.Function.PrettyPrint(out)
 	out.Print("(")
-	oldExpressionLevel := out.ExpressionLevel
-	out.ExpressionLevel = 0
+	oldExpressionPrecedence := out.ExpressionPrecedence
+	out.ExpressionPrecedence = LOWEST
 	out.ComaList(ce.Arguments)
-	out.ExpressionLevel = oldExpressionLevel
+	out.ExpressionPrecedence = oldExpressionPrecedence
 	out.Print(")")
 	return out
 }
@@ -444,18 +506,21 @@ type IndexExpression struct {
 }
 
 func (ie IndexExpression) PrettyPrint(out *PrintState) *PrintState {
-	if out.ExpressionLevel > 0 { // TODO only add parens if precedence requires it.
+	needParen, oldExpressionPrecedence := out.needParen(ie.Token)
+	if needParen {
 		out.Print("(")
 	}
 	ie.Left.PrettyPrint(out)
 	out.Print(ie.Literal())
+	out.ExpressionPrecedence = LOWEST
 	ie.Index.PrettyPrint(out)
 	if ie.Token.Type() == token.LBRACKET {
 		out.Print("]")
 	}
-	if out.ExpressionLevel > 0 {
+	if needParen {
 		out.Print(")")
 	}
+	out.ExpressionPrecedence = oldExpressionPrecedence
 	return out
 }
 

eval/eval_test.go

@@ -388,7 +388,7 @@ func TestFunctionObject(t *testing.T) {
 	if ast.DebugString(fn.Parameters[0]) != "x" {
 		t.Fatalf("parameter is not 'x'. got=%q", fn.Parameters[0])
 	}
-	expectedBody := "x+2"
+	expectedBody := "(x+2)" // DebugString adds parens.
 	got := ast.DebugString(fn.Body)
 	if got != expectedBody {
 		t.Fatalf("body is not %q. got=%q", expectedBody, got)
@@ -701,15 +701,15 @@ func TestQuote(t *testing.T) {
 		},
 		{
 			`quote(5 + 8)`,
-			`5+8`,
+			`(5+8)`,
 		},
 		{
 			`quote(foobar)`,
 			`foobar`,
 		},
 		{
 			`quote(foobar + barfoo)`,
-			`foobar+barfoo`,
+			`(foobar+barfoo)`,
 		},
 	}
 
@@ -747,11 +747,11 @@ func TestQuoteUnquote(t *testing.T) {
 		},
 		{
 			`quote(8 + unquote(4 + 4))`,
-			`8+8`,
+			`(8+8)`, // DebugString adds parens.
 		},
 		{
 			`quote(unquote(4 + 4) + 8)`,
-			`8+8`,
+			`(8+8)`,
 		},
 		{
 			`foobar = 8;
@@ -773,12 +773,12 @@ func TestQuoteUnquote(t *testing.T) {
 		},
 		{
 			`quote(unquote(quote(4 + 4)))`,
-			`4+4`,
+			`(4+4)`,
 		},
 		{
 			`quotedInfixExpression = quote(4 + 4);
             quote(unquote(4 + 4) + unquote(quotedInfixExpression))`,
-			`8+(4+4)`,
+			`(8+(4+4))`,
 		},
 	}
 	for _, tt := range tests {

eval/macro_expension_test.go

@@ -56,7 +56,7 @@ func TestDefineMacros(t *testing.T) {
 		t.Fatalf("parameter is not 'y'. got=%q", macro.Parameters[1])
 	}
 
-	expectedBody := "x+y" // or should have {}?
+	expectedBody := "(x+y)" // DebugString adds parens.
 	got := ast.DebugString(macro.Body)
 	if got != expectedBody {
 		t.Fatalf("body is not %q. got=%q", expectedBody, got)

main.go

@@ -42,6 +42,7 @@ var hookBefore, hookAfter func() int
 func Main() int {
 	commandFlag := flag.String("c", "", "command/inline script to run instead of interactive mode")
 	showParse := flag.Bool("parse", false, "show parse tree")
+	allParens := flag.Bool("parse-debug", false, "show all parenthesis in parse tree (default is to simplify using precedence)")
 	format := flag.Bool("format", false, "don't execute, just parse and re format the input")
 	compact := flag.Bool("compact", false, "When printing code, use no indentation and most compact form")
 	showEval := flag.Bool("eval", true, "show eval results")
@@ -95,6 +96,7 @@ func Main() int {
 		MaxDepth:    *maxDepth + 1,
 		MaxValueLen: *maxLen,
 		PanicOk:     *panicOk,
+		AllParens:   *allParens,
 	}
 	if hookBefore != nil {
 		ret := hookBefore()

main_test.txtar

@@ -252,8 +252,13 @@ grol -quiet -c '()=>{{"a":1,"b":2}}'
 stdout '^\(\)=>{{"a":1,"b":2}}$'
 
 # if extra paren are needed (like for a[x] in the left part of if condition) it should still parse.
-grol -quiet -c '()=> if 1+2==3 {4}'
-stdout '^\(\)=>if \(1\+2\)==3\{4}$'
+# note: no extra paren anymore.
+grol -quiet -c '()=> if 1+2 == 3 {4}'
+stdout '^\(\)=>if 1\+2==3\{4\}$'
+
+grol -quiet -c '(()=> if 1+2==3 {4})()'
+stdout '^4$'
+
 
 -- json_output --
 {