cmd/go: use package index for std in load.loadPackageData

load.loadPackageData was only using an index for modules,
not for standard library packages. Other parts of the code were
using the index, so there was some benefit, but not as much
as you'd hope.

With the index disabled, the Script/work test takes 2.2s on my Mac.
With the index enabled before this CL, it took 2.0s.
With the index enabled after this CL, it takes 1.6s.

Before this CL, the Script/work test issued:

	 429 IsDir
	  19 IsDirWithGoFiles
	   7 Lstat
	9072 Open
	 993 ReadDir
	 256 Stat
	   7 Walk
	   3 indexModule
	  24 openIndexModule
	 525 openIndexPackage

After this CL, it issued:

	  19 IsDirWithGoFiles
	   7 Lstat
	  60 Open
	 606 ReadDir
	 256 Stat
	   7 Walk
	   3 indexModule
	  24 openIndexModule
	 525 openIndexPackage

This speedup helps the Dragonfly builder, which has very slow
file I/O and is timing out since a recent indexing change.

Times for go test -run=Script/^work$ on the Dragonfly builder:

	50s before indexing changes
	31s full module indexing of std
	46s per-package indexing of std

It cuts the time for go test -run=Script/^work$ from 44s to 20s.

For #53577.

Change-Id: I7189a77fc7fdf61de3ab3447efc4e84d1fc52c25
Reviewed-on: https://go-review.googlesource.com/c/go/+/416134
Reviewed-by: Bryan Mills <[email protected]>

src/cmd/go/internal/fsys/fsys.go

@@ -6,16 +6,65 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"internal/godebug"
 	"io/fs"
 	"io/ioutil"
+	"log"
 	"os"
+	pathpkg "path"
 	"path/filepath"
 	"runtime"
+	"runtime/debug"
 	"sort"
 	"strings"
+	"sync"
 	"time"
 )
 
+// Trace emits a trace event for the operation and file path to the trace log,
+// but only when $GODEBUG contains gofsystrace=1.
+// The traces are appended to the file named by the $GODEBUG setting gofsystracelog, or else standard error.
+// For debugging, if the $GODEBUG setting gofsystracestack is non-empty, then trace events for paths
+// matching that glob pattern (using path.Match) will be followed by a full stack trace.
+func Trace(op, path string) {
+	if !doTrace {
+		return
+	}
+	traceMu.Lock()
+	defer traceMu.Unlock()
+	fmt.Fprintf(traceFile, "%d gofsystrace %s %s\n", os.Getpid(), op, path)
+	if traceStack != "" {
+		if match, _ := pathpkg.Match(traceStack, path); match {
+			traceFile.Write(debug.Stack())
+		}
+	}
+}
+
+var (
+	doTrace    bool
+	traceStack string
+	traceFile  *os.File
+	traceMu    sync.Mutex
+)
+
+func init() {
+	if godebug.Get("gofsystrace") != "1" {
+		return
+	}
+	doTrace = true
+	traceStack = godebug.Get("gofsystracestack")
+	if f := godebug.Get("gofsystracelog"); f != "" {
+		// Note: No buffering on writes to this file, so no need to worry about closing it at exit.
+		var err error
+		traceFile, err = os.OpenFile(f, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0666)
+		if err != nil {
+			log.Fatal(err)
+		}
+	} else {
+		traceFile = os.Stderr
+	}
+}
+
 // OverlayFile is the path to a text file in the OverlayJSON format.
 // It is the value of the -overlay flag.
 var OverlayFile string
@@ -86,6 +135,7 @@ func Init(wd string) error {
 		return nil
 	}
 
+	Trace("ReadFile", OverlayFile)
 	b, err := os.ReadFile(OverlayFile)
 	if err != nil {
 		return fmt.Errorf("reading overlay file: %v", err)
@@ -191,6 +241,7 @@ func initFromJSON(overlayJSON OverlayJSON) error {
 // IsDir returns true if path is a directory on disk or in the
 // overlay.
 func IsDir(path string) (bool, error) {
+	Trace("IsDir", path)
 	path = canonicalize(path)
 
 	if _, ok := parentIsOverlayFile(path); ok {
@@ -260,6 +311,7 @@ func readDir(dir string) ([]fs.FileInfo, error) {
 // ReadDir provides a slice of fs.FileInfo entries corresponding
 // to the overlaid files in the directory.
 func ReadDir(dir string) ([]fs.FileInfo, error) {
+	Trace("ReadDir", dir)
 	dir = canonicalize(dir)
 	if _, ok := parentIsOverlayFile(dir); ok {
 		return nil, &fs.PathError{Op: "ReadDir", Path: dir, Err: errNotDir}
@@ -327,11 +379,17 @@ func OverlayPath(path string) (string, bool) {
 
 // Open opens the file at or overlaid on the given path.
 func Open(path string) (*os.File, error) {
-	return OpenFile(path, os.O_RDONLY, 0)
+	Trace("Open", path)
+	return openFile(path, os.O_RDONLY, 0)
 }
 
 // OpenFile opens the file at or overlaid on the given path with the flag and perm.
 func OpenFile(path string, flag int, perm os.FileMode) (*os.File, error) {
+	Trace("OpenFile", path)
+	return openFile(path, flag, perm)
+}
+
+func openFile(path string, flag int, perm os.FileMode) (*os.File, error) {
 	cpath := canonicalize(path)
 	if node, ok := overlay[cpath]; ok {
 		// Opening a file in the overlay.
@@ -360,6 +418,7 @@ func OpenFile(path string, flag int, perm os.FileMode) (*os.File, error) {
 // IsDirWithGoFiles reports whether dir is a directory containing Go files
 // either on disk or in the overlay.
 func IsDirWithGoFiles(dir string) (bool, error) {
+	Trace("IsDirWithGoFiles", dir)
 	fis, err := ReadDir(dir)
 	if os.IsNotExist(err) || errors.Is(err, errNotDir) {
 		return false, nil
@@ -436,6 +495,7 @@ func walk(path string, info fs.FileInfo, walkFn filepath.WalkFunc) error {
 // Walk walks the file tree rooted at root, calling walkFn for each file or
 // directory in the tree, including root.
 func Walk(root string, walkFn filepath.WalkFunc) error {
+	Trace("Walk", root)
 	info, err := Lstat(root)
 	if err != nil {
 		err = walkFn(root, nil, err)
@@ -450,11 +510,13 @@ func Walk(root string, walkFn filepath.WalkFunc) error {
 
 // lstat implements a version of os.Lstat that operates on the overlay filesystem.
 func Lstat(path string) (fs.FileInfo, error) {
+	Trace("Lstat", path)
 	return overlayStat(path, os.Lstat, "lstat")
 }
 
 // Stat implements a version of os.Stat that operates on the overlay filesystem.
 func Stat(path string) (fs.FileInfo, error) {
+	Trace("Stat", path)
 	return overlayStat(path, os.Stat, "stat")
 }
 
@@ -528,6 +590,7 @@ func (f fakeDir) Sys() any           { return nil }
 
 // Glob is like filepath.Glob but uses the overlay file system.
 func Glob(pattern string) (matches []string, err error) {
+	Trace("Glob", pattern)
 	// Check pattern is well-formed.
 	if _, err := filepath.Match(pattern, ""); err != nil {
 		return nil, err

src/cmd/go/internal/load/pkg.go

@@ -877,7 +877,14 @@ func loadPackageData(ctx context.Context, path, parentPath, parentDir, parentRoo
 			if !cfg.ModulesEnabled {
 				buildMode = build.ImportComment
 			}
-			if modroot := modload.PackageModRoot(ctx, r.path); modroot != "" {
+			modroot := modload.PackageModRoot(ctx, r.path)
+			if modroot == "" && str.HasPathPrefix(r.dir, cfg.GOROOTsrc) {
+				modroot = cfg.GOROOTsrc
+				if str.HasPathPrefix(r.dir, cfg.GOROOTsrc+string(filepath.Separator)+"cmd") {
+					modroot += string(filepath.Separator) + "cmd"
+				}
+			}
+			if modroot != "" {
 				if rp, err := modindex.GetPackage(modroot, r.dir); err == nil {
 					data.p, data.err = rp.Import(cfg.BuildContext, buildMode)
 					goto Happy

src/cmd/go/internal/modindex/read.go

@@ -179,6 +179,7 @@ func openIndexModule(modroot string, ismodcache bool) (*Module, error) {
 		err error
 	}
 	r := mcache.Do(modroot, func() any {
+		fsys.Trace("openIndexModule", modroot)
 		id, err := moduleHash(modroot, ismodcache)
 		if err != nil {
 			return result{nil, err}
@@ -212,6 +213,7 @@ func openIndexPackage(modroot, pkgdir string) (*IndexPackage, error) {
 		err error
 	}
 	r := pcache.Do([2]string{modroot, pkgdir}, func() any {
+		fsys.Trace("openIndexPackage", pkgdir)
 		id, err := dirHash(modroot, pkgdir)
 		if err != nil {
 			return result{nil, err}

src/cmd/go/internal/modindex/scan.go

@@ -46,6 +46,7 @@ func moduleWalkErr(modroot string, path string, info fs.FileInfo, err error) err
 // encoded representation. It returns ErrNotIndexed if the module can't
 // be indexed because it contains symlinks.
 func indexModule(modroot string) ([]byte, error) {
+	fsys.Trace("indexModule", modroot)
 	var packages []*rawPackage
 	err := fsys.Walk(modroot, func(path string, info fs.FileInfo, err error) error {
 		if err := moduleWalkErr(modroot, path, info, err); err != nil {
@@ -72,6 +73,7 @@ func indexModule(modroot string) ([]byte, error) {
 // encoded representation. It returns ErrNotIndexed if the package can't
 // be indexed.
 func indexPackage(modroot, pkgdir string) []byte {
+	fsys.Trace("indexPackage", pkgdir)
 	p := importRaw(modroot, relPath(pkgdir, modroot))
 	return encodePackageBytes(p)
 }

src/cmd/go/testdata/script/index.txt

@@ -0,0 +1,6 @@
+# Check that standard library packages are cached.
+go list -json math # refresh cache
+env GODEBUG=gofsystrace=1,gofsystracelog=fsys.log
+go list -json math
+! grep math/abs.go fsys.log
+grep 'openIndexPackage .*[\\/]math$' fsys.log