Implement a new parser on top of the new lexer

experimental/ast/pathlike.go

@@ -68,7 +68,7 @@ func unwrapPathLike[Value ~int32 | ~uint32, Kind ~int8](want Kind, p pathLike[Ki
 // wrapPath wraps a path in a pathLike.
 func wrapPath[Kind ~int8](path rawPath) pathLike[Kind] {
 	return pathLike[Kind]{
-		StartOrKind: ^int32(path.Start),
+		StartOrKind: int32(path.Start),
 		EndOrValue:  int32(path.End),
 	}
 }

experimental/parser/diagnostics_internal.go

@@ -0,0 +1,89 @@
+// Copyright 2020-2024 Buf Technologies, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package parser
+
+import (
+	"fmt"
+
+	"github.com/bufbuild/protocompile/experimental/internal/taxa"
+	"github.com/bufbuild/protocompile/experimental/report"
+)
+
+// errUnexpected is a low-level parser error for when we hit a token we don't
+// know how to handle.
+type errUnexpected struct {
+	// The unexpected thing (may be a token or AST node).
+	what report.Spanner
+
+	// The context we're in. Should be format-able with %v.
+	where taxa.Place
+	// Useful when where is an "after" position: if non-nil, this will be
+	// highlighted as "previous where.Object is here"
+	prev report.Spanner
+
+	// What we wanted vs. what we got. Got can be used to customize what gets
+	// shown, but if it's not set, we call describe(what) to get a user-visible
+	// description.
+	want taxa.Set
+	got  taxa.Subject
+}
+
+func (e errUnexpected) Error() string {
+	got := e.got
+	if got == taxa.Unknown {
+		got = taxa.Classify(e.what)
+	}
+
+	if e.where.Subject == taxa.Unknown {
+		return fmt.Sprintf("unexpected %v", got)
+	}
+
+	return fmt.Sprintf("unexpected %v %v", got, e.where)
+}
+
+func (e errUnexpected) Diagnose(d *report.Diagnostic) {
+	snippet := report.Snippet(e.what)
+	if e.want.Len() > 0 {
+		snippet = report.Snippetf(e.what, "expected %v", e.want.Join("or"))
+	}
+
+	d.With(
+		snippet,
+		report.Snippetf(e.prev, "previous %v is here", e.where.Subject),
+	)
+}
+
+// errMoreThanOne is used to diagnose the occurrence of some construct more
+// than one time, when it is expected to occur at most once.
+type errMoreThanOne struct {
+	first, second report.Spanner
+	what          taxa.Subject
+}
+
+func (e errMoreThanOne) Error() string {
+	what := e.what
+	if what == taxa.Unknown {
+		what = taxa.Classify(e.first)
+	}
+
+	return "encountered more than one " + what.String()
+}
+
+func (e errMoreThanOne) Diagnose(d *report.Diagnostic) {
+	d.With(
+		report.Snippetf(e.second, "help: consider removing this"),
+		report.Snippetf(e.first, "first one is here"),
+	)
+}

experimental/parser/diagnostics_number.go

@@ -16,8 +16,8 @@ package parser
 
 import (
 	"fmt"
-	"strings"
 
+	"github.com/bufbuild/protocompile/experimental/internal/taxa"
 	"github.com/bufbuild/protocompile/experimental/report"
 	"github.com/bufbuild/protocompile/experimental/token"
 )
@@ -30,7 +30,7 @@ type ErrInvalidNumber struct {
 // Error implements [error].
 func (e ErrInvalidNumber) Error() string {
 	switch {
-	case isFloatLiteral(e.Token):
+	case taxa.IsFloat(e.Token):
 		return "unexpected characters in floating-point literal"
 	default:
 		return "unexpected characters in integer literal"
@@ -61,7 +61,7 @@ type ErrInvalidBase struct {
 // Error implements [error].
 func (e ErrInvalidBase) Error() string {
 	switch {
-	case isFloatLiteral(e.Token):
+	case taxa.IsFloat(e.Token):
 		return "unsupported base for floating-point literal"
 	default:
 		return "unsupported base for integer literal"
@@ -82,7 +82,7 @@ func (e ErrInvalidBase) Diagnose(d *report.Diagnostic) {
 		base = fmt.Sprintf("base-%d", e.Base)
 	}
 
-	isFloat := isFloatLiteral(e.Token)
+	isFloat := taxa.IsFloat(e.Token)
 	if !isFloat && e.Base == 8 {
 		d.With(
 			report.Snippetf(e.Token, "replace `0o` with `0`"),
@@ -114,7 +114,7 @@ type ErrThousandsSep struct {
 // Error implements [error].
 func (e ErrThousandsSep) Error() string {
 	switch {
-	case isFloatLiteral(e.Token):
+	case taxa.IsFloat(e.Token):
 		return "floating-point literal contains underscores"
 	default:
 		return "integer literal contains underscores"
@@ -128,11 +128,3 @@ func (e ErrThousandsSep) Diagnose(d *report.Diagnostic) {
 		report.Note("Protobuf does not support Go/Java/Rust-style thousands separators"),
 	)
 }
-
-func isFloatLiteral(tok token.Token) bool {
-	digits := tok.Text()
-	if strings.HasPrefix(digits, "0x") || strings.HasPrefix(digits, "0X") {
-		return strings.ContainsRune(digits, '.')
-	}
-	return strings.ContainsAny(digits, ".eE")
-}

experimental/parser/lex_test.go

@@ -28,7 +28,7 @@ import (
 	"github.com/bufbuild/protocompile/internal/golden"
 )
 
-func TestRender(t *testing.T) {
+func TestLexer(t *testing.T) {
 	t.Parallel()
 
 	corpus := golden.Corpus{

experimental/parser/parse.go

@@ -0,0 +1,70 @@
+// Copyright 2020-2024 Buf Technologies, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package parser
+
+import (
+	"github.com/bufbuild/protocompile/experimental/ast"
+	"github.com/bufbuild/protocompile/experimental/internal/taxa"
+	"github.com/bufbuild/protocompile/experimental/report"
+	"github.com/bufbuild/protocompile/experimental/token"
+)
+
+// Parse lexes and parses the Protobuf file tracked by ctx.
+//
+// Diagnostics generated by this process are written to errs. Returns whether
+// parsing succeeded without errors.
+//
+// Parse will freeze the stream in ctx when it is done.
+func Parse(ctx ast.Context, errs *report.Report) (file ast.File, ok bool) {
+	prior := len(errs.Diagnostics)
+
+	Lex(ctx, errs)
+	parse(ctx, errs)
+
+	ok = true
+	for _, d := range errs.Diagnostics[prior:] {
+		ok = ok && d.Level != report.Error
+	}
+
+	return ctx.Nodes().Root(), ok
+}
+
+// parse implements the core parser loop.
+func parse(ctx ast.Context, errs *report.Report) {
+	p := &parser{
+		Context: ctx,
+		Nodes:   ctx.Nodes(),
+		Report:  errs,
+	}
+
+	defer p.CatchICE(false, nil)
+
+	c := ctx.Stream().Cursor()
+	root := ctx.Nodes().Root()
+
+	var mark token.CursorMark
+	for !c.Done() {
+		next := c.Mark()
+		if mark == next {
+			panic("protocompile/parser: parser failed to make progress; this is a bug in protocompile")
+		}
+		mark = next
+
+		node := parseDecl(p, c, taxa.TopLevel)
+		if !node.Nil() {
+			root.Append(node)
+		}
+	}
+}