gopls/internal/lsp/source: use syntax alone in FormatVarType

FormatVarType re-formats a variable type using syntax, in order to get
accurate presentation of aliases and ellipses. However, as a result it
required typed syntax trees for the type declaration, which may exist in
a distinct package from the current package.

In the near future we may not have typed syntax trees for these
packages. We could type-check on demand, but (1) that could be costly
and (2) it may break qualification using the go/types qualifier.

Instead, perform this operation using a qualifier based on syntax
and metadata, so that we only need a fully parsed file rather than a
fully type-checked package. The resulting expressions may be inaccurate
due to built-ins, "." imported packages, or missing metadata, but that
seems acceptable for the current use-cases of this function, which are
in completion and signature help.

While doing this, add a FormatNodeWithFile helper that allows formatting
a node from a *token.File, rather than *token.FileSet. This can help us
avoid relying on a global fileset. To facilitate this, move the GetLines
helper from internal/gcimporter into a shared tokeninternal package.

For golang/go#57987

Change-Id: I3b8a5256bc2261be8b5175ee360b9336228928ac
Reviewed-on: https://go-review.googlesource.com/c/tools/+/464301
Run-TryBot: Robert Findley <[email protected]>
gopls-CI: kokoro <[email protected]>
TryBot-Result: Gopher Robot <[email protected]>
Reviewed-by: Alan Donovan <[email protected]>

gopls/internal/lsp/semantic.go

@@ -220,9 +220,7 @@ type encoded struct {
 
 	ctx context.Context
 	// metadataSource is used to resolve imports
-	metadataSource interface {
-		Metadata(source.PackageID) *source.Metadata
-	}
+	metadataSource    source.MetadataSource
 	tokTypes, tokMods []string
 	pgf               *source.ParsedGoFile
 	rng               *protocol.Range

gopls/internal/lsp/source/call_hierarchy.go

@@ -231,7 +231,7 @@ func OutgoingCalls(ctx context.Context, snapshot Snapshot, fh FileHandle, pp pro
 		return nil, err
 	}
 
-	declNode, _ := FindDeclAndField([]*ast.File{declPGF.File}, declPos)
+	declNode, _, _ := FindDeclInfo([]*ast.File{declPGF.File}, declPos)
 	if declNode == nil {
 		// TODO(rfindley): why don't we return an error here, or even bug.Errorf?
 		return nil, nil

gopls/internal/lsp/source/completion/completion.go

@@ -156,7 +156,8 @@ func (ipm insensitivePrefixMatcher) Score(candidateLabel string) float32 {
 type completer struct {
 	snapshot source.Snapshot
 	pkg      source.Package
-	qf       types.Qualifier
+	qf       types.Qualifier          // for qualifying typed expressions
+	mq       source.MetadataQualifier // for syntactic qualifying
 	opts     *completionOptions
 
 	// completionContext contains information about the trigger for this
@@ -509,6 +510,7 @@ func Completion(ctx context.Context, snapshot source.Snapshot, fh source.FileHan
 		pkg:      pkg,
 		snapshot: snapshot,
 		qf:       source.Qualifier(pgf.File, pkg.GetTypes(), pkg.GetTypesInfo()),
+		mq:       source.MetadataQualifierForFile(snapshot, pgf.File, pkg.Metadata()),
 		completionContext: completionContext{
 			triggerCharacter: protoContext.TriggerCharacter,
 			triggerKind:      protoContext.TriggerKind,

gopls/internal/lsp/source/completion/format.go

@@ -81,7 +81,11 @@ func (c *completer) item(ctx context.Context, cand candidate) (CompletionItem, e
 		if _, ok := obj.Type().(*types.Struct); ok {
 			detail = "struct{...}" // for anonymous structs
 		} else if obj.IsField() {
-			detail = source.FormatVarType(ctx, c.snapshot, c.pkg, obj, c.qf)
+			var err error
+			detail, err = source.FormatVarType(ctx, c.snapshot, c.pkg, c.file, obj, c.qf, c.mq)
+			if err != nil {
+				return CompletionItem{}, err
+			}
 		}
 		if obj.IsField() {
 			kind = protocol.FieldCompletion
@@ -130,7 +134,10 @@ Suffixes:
 		switch mod {
 		case invoke:
 			if sig, ok := funcType.Underlying().(*types.Signature); ok {
-				s := source.NewSignature(ctx, c.snapshot, c.pkg, sig, nil, c.qf)
+				s, err := source.NewSignature(ctx, c.snapshot, c.pkg, c.file, sig, nil, c.qf, c.mq)
+				if err != nil {
+					return CompletionItem{}, err
+				}
 				c.functionCallSnippet("", s.TypeParams(), s.Params(), &snip)
 				if sig.Results().Len() == 1 {
 					funcType = sig.Results().At(0).Type()
@@ -243,7 +250,7 @@ Suffixes:
 		return item, nil
 	}
 
-	decl, _ := source.FindDeclAndField(pkg.GetSyntax(), obj.Pos()) // may be nil
+	decl, _, _ := source.FindDeclInfo(pkg.GetSyntax(), obj.Pos()) // may be nil
 	hover, err := source.FindHoverContext(ctx, c.snapshot, pkg, obj, decl, nil)
 	if err != nil {
 		event.Error(ctx, "failed to find Hover", err, tag.URI.Of(uri))

gopls/internal/lsp/source/completion/literal.go

@@ -202,10 +202,18 @@ func (c *completer) functionLiteral(ctx context.Context, sig *types.Signature, m
 			// If the param has no name in the signature, guess a name based
 			// on the type. Use an empty qualifier to ignore the package.
 			// For example, we want to name "http.Request" "r", not "hr".
-			name = source.FormatVarType(ctx, c.snapshot, c.pkg, p, func(p *types.Package) string {
-				return ""
-			})
-			name = abbreviateTypeName(name)
+			typeName, err := source.FormatVarType(ctx, c.snapshot, c.pkg, c.file, p,
+				func(p *types.Package) string { return "" },
+				func(source.PackageName, source.ImportPath, source.PackagePath) string { return "" })
+			if err != nil {
+				// In general, the only error we should encounter while formatting is
+				// context cancellation.
+				if ctx.Err() == nil {
+					event.Error(ctx, "formatting var type", err)
+				}
+				return
+			}
+			name = abbreviateTypeName(typeName)
 		}
 		paramNames[i] = name
 		if name != "_" {
@@ -264,7 +272,15 @@ func (c *completer) functionLiteral(ctx context.Context, sig *types.Signature, m
 		// of "i int, j int".
 		if i == sig.Params().Len()-1 || !types.Identical(p.Type(), sig.Params().At(i+1).Type()) {
 			snip.WriteText(" ")
-			typeStr := source.FormatVarType(ctx, c.snapshot, c.pkg, p, c.qf)
+			typeStr, err := source.FormatVarType(ctx, c.snapshot, c.pkg, c.file, p, c.qf, c.mq)
+			if err != nil {
+				// In general, the only error we should encounter while formatting is
+				// context cancellation.
+				if ctx.Err() == nil {
+					event.Error(ctx, "formatting var type", err)
+				}
+				return
+			}
 			if sig.Variadic() && i == sig.Params().Len()-1 {
 				typeStr = strings.Replace(typeStr, "[]", "...", 1)
 			}
@@ -314,7 +330,15 @@ func (c *completer) functionLiteral(ctx context.Context, sig *types.Signature, m
 			snip.WriteText(name + " ")
 		}
 
-		text := source.FormatVarType(ctx, c.snapshot, c.pkg, r, c.qf)
+		text, err := source.FormatVarType(ctx, c.snapshot, c.pkg, c.file, r, c.qf, c.mq)
+		if err != nil {
+			// In general, the only error we should encounter while formatting is
+			// context cancellation.
+			if ctx.Err() == nil {
+				event.Error(ctx, "formatting var type", err)
+			}
+			return
+		}
 		if tp, _ := r.Type().(*typeparams.TypeParam); tp != nil && !c.typeParamInScope(tp) {
 			snip.WritePlaceholder(func(snip *snippet.Builder) {
 				snip.WriteText(text)

gopls/internal/lsp/source/hover.go

@@ -24,6 +24,7 @@ import (
 	"golang.org/x/tools/go/types/typeutil"
 	"golang.org/x/tools/gopls/internal/lsp/protocol"
 	"golang.org/x/tools/gopls/internal/lsp/safetoken"
+	"golang.org/x/tools/gopls/internal/span"
 	"golang.org/x/tools/internal/bug"
 	"golang.org/x/tools/internal/event"
 	"golang.org/x/tools/internal/typeparams"
@@ -498,6 +499,41 @@ func objectString(obj types.Object, qf types.Qualifier, inferred *types.Signatur
 	return str
 }
 
+// parseFull fully parses the file corresponding to position pos, referenced
+// from the given srcpkg.
+//
+// It returns the resulting ParsedGoFile as well as new pos contained in the
+// parsed file.
+func parseFull(ctx context.Context, snapshot Snapshot, srcpkg Package, pos token.Pos) (*ParsedGoFile, token.Pos, error) {
+	f := srcpkg.FileSet().File(pos)
+	if f == nil {
+		return nil, 0, bug.Errorf("internal error: no file for position %d in %s", pos, srcpkg.Metadata().ID)
+	}
+
+	uri := span.URIFromPath(f.Name())
+	fh, err := snapshot.GetFile(ctx, uri)
+	if err != nil {
+		return nil, 0, err
+	}
+
+	pgf, err := snapshot.ParseGo(ctx, fh, ParseFull)
+	if err != nil {
+		return nil, 0, err
+	}
+
+	offset, err := safetoken.Offset(f, pos)
+	if err != nil {
+		return nil, 0, bug.Errorf("offset out of bounds in %q", uri)
+	}
+
+	fullPos, err := safetoken.Pos(pgf.Tok, offset)
+	if err != nil {
+		return nil, 0, err
+	}
+
+	return pgf, fullPos, nil
+}
+
 // FindHoverContext returns a HoverContext struct for an AST node and its
 // declaration object. node should be the actual node used in type checking,
 // while fullNode could be a separate node with more complete syntactic
@@ -637,7 +673,7 @@ func FindHoverContext(ctx context.Context, s Snapshot, pkg Package, obj types.Ob
 				break
 			}
 
-			_, field := FindDeclAndField(pkg.GetSyntax(), obj.Pos())
+			_, _, field := FindDeclInfo(pkg.GetSyntax(), obj.Pos())
 			if field != nil {
 				comment := field.Doc
 				if comment.Text() == "" {
@@ -893,16 +929,19 @@ func anyNonEmpty(x []string) bool {
 	return false
 }
 
-// FindDeclAndField returns the var/func/type/const Decl that declares
-// the identifier at pos, searching the given list of file syntax
-// trees. If pos is the position of an ast.Field or one of its Names
-// or Ellipsis.Elt, the field is returned, along with the innermost
-// enclosing Decl, which could be only loosely related---consider:
+// FindDeclInfo returns the syntax nodes involved in the declaration of the
+// types.Object with position pos, searching the given list of file syntax
+// trees.
+//
+// Pos may be the position of the name-defining identifier in a FuncDecl,
+// ValueSpec, TypeSpec, Field, or as a special case the position of
+// Ellipsis.Elt in an ellipsis field.
 //
-//	var decl = f(  func(field int) {}  )
+// If found, the resulting decl, spec, and field will be the inner-most
+// instance of each node type surrounding pos.
 //
-// It returns (nil, nil) if no Field or Decl is found at pos.
-func FindDeclAndField(files []*ast.File, pos token.Pos) (decl ast.Decl, field *ast.Field) {
+// It returns a nil decl if no object-defining node is found at pos.
+func FindDeclInfo(files []*ast.File, pos token.Pos) (decl ast.Decl, spec ast.Spec, field *ast.Field) {
 	// panic(found{}) breaks off the traversal and
 	// causes the function to return normally.
 	type found struct{}
@@ -933,33 +972,45 @@ func FindDeclAndField(files []*ast.File, pos token.Pos) (decl ast.Decl, field *a
 
 		switch n := n.(type) {
 		case *ast.Field:
-			checkField := func(f ast.Node) {
-				if f.Pos() == pos {
-					field = n
-					for i := len(stack) - 1; i >= 0; i-- {
-						if d, ok := stack[i].(ast.Decl); ok {
-							decl = d // innermost enclosing decl
-							break
-						}
+			findEnclosingDeclAndSpec := func() {
+				for i := len(stack) - 1; i >= 0; i-- {
+					switch n := stack[i].(type) {
+					case ast.Spec:
+						spec = n
+					case ast.Decl:
+						decl = n
+						return
 					}
+				}
+			}
+
+			// Check each field name since you can have
+			// multiple names for the same type expression.
+			for _, id := range n.Names {
+				if id.Pos() == pos {
+					field = n
+					findEnclosingDeclAndSpec()
 					panic(found{})
 				}
 			}
 
 			// Check *ast.Field itself. This handles embedded
 			// fields which have no associated *ast.Ident name.
-			checkField(n)
-
-			// Check each field name since you can have
-			// multiple names for the same type expression.
-			for _, name := range n.Names {
-				checkField(name)
+			if n.Pos() == pos {
+				field = n
+				findEnclosingDeclAndSpec()
+				panic(found{})
 			}
 
-			// Also check "X" in "...X". This makes it easy
-			// to format variadic signature params properly.
-			if ell, ok := n.Type.(*ast.Ellipsis); ok && ell.Elt != nil {
-				checkField(ell.Elt)
+			// Also check "X" in "...X". This makes it easy to format variadic
+			// signature params properly.
+			//
+			// TODO(rfindley): I don't understand this comment. How does finding the
+			// field in this case make it easier to format variadic signature params?
+			if ell, ok := n.Type.(*ast.Ellipsis); ok && ell.Elt != nil && ell.Elt.Pos() == pos {
+				field = n
+				findEnclosingDeclAndSpec()
+				panic(found{})
 			}
 
 		case *ast.FuncDecl:
@@ -969,17 +1020,19 @@ func FindDeclAndField(files []*ast.File, pos token.Pos) (decl ast.Decl, field *a
 			}
 
 		case *ast.GenDecl:
-			for _, spec := range n.Specs {
-				switch spec := spec.(type) {
+			for _, s := range n.Specs {
+				switch s := s.(type) {
 				case *ast.TypeSpec:
-					if spec.Name.Pos() == pos {
+					if s.Name.Pos() == pos {
 						decl = n
+						spec = s
 						panic(found{})
 					}
 				case *ast.ValueSpec:
-					for _, id := range spec.Names {
+					for _, id := range s.Names {
 						if id.Pos() == pos {
 							decl = n
+							spec = s
 							panic(found{})
 						}
 					}
@@ -992,5 +1045,5 @@ func FindDeclAndField(files []*ast.File, pos token.Pos) (decl ast.Decl, field *a
 		ast.Inspect(file, f)
 	}
 
-	return nil, nil
+	return nil, nil, nil
 }

gopls/internal/lsp/source/identifier.go

@@ -274,9 +274,7 @@ func findIdentifier(ctx context.Context, snapshot Snapshot, pkg Package, pgf *Pa
 	if err != nil {
 		return nil, err
 	}
-	// TODO(adonovan): there's no need to inspect the entire GetSyntax() slice:
-	// we already know it's declFile.File.
-	result.Declaration.node, _ = FindDeclAndField(declPkg.GetSyntax(), declPos) // may be nil
+	result.Declaration.node, _, _ = FindDeclInfo([]*ast.File{declFile.File}, declPos) // may be nil
 	result.Declaration.nodeFile = declFile
 
 	// Ensure that we have the full declaration, in case the declaration was

gopls/internal/lsp/source/signature_help.go

@@ -101,7 +101,7 @@ FindCall:
 		if err != nil {
 			return nil, 0, err
 		}
-		node, _ := FindDeclAndField(declPkg.GetSyntax(), obj.Pos()) // may be nil
+		node, _, _ := FindDeclInfo(declPkg.GetSyntax(), obj.Pos()) // may be nil
 		d, err := FindHoverContext(ctx, snapshot, pkg, obj, node, nil)
 		if err != nil {
 			return nil, 0, err
@@ -111,7 +111,11 @@ FindCall:
 	} else {
 		name = "func"
 	}
-	s := NewSignature(ctx, snapshot, pkg, sig, comment, qf)
+	mq := MetadataQualifierForFile(snapshot, pgf.File, pkg.Metadata())
+	s, err := NewSignature(ctx, snapshot, pkg, pgf.File, sig, comment, qf, mq)
+	if err != nil {
+		return nil, 0, err
+	}
 	paramInfo := make([]protocol.ParameterInformation, 0, len(s.params))
 	for _, p := range s.params {
 		paramInfo = append(paramInfo, protocol.ParameterInformation{Label: p})