feat: go list all pkgs and cache them to improve build speed

packages/cache.go

@@ -0,0 +1,95 @@
+package packages
+
+import (
+	"bytes"
+	"os/exec"
+	"strings"
+	"sync"
+)
+
+// pkgPath Caches
+type Cache struct {
+	dirCache          map[string]bool
+	dirCacheMutex     sync.RWMutex
+	packageCacheMap   map[string]CacheInfo
+	packageCacheMutex sync.RWMutex
+	waitCache         sync.WaitGroup
+}
+
+type CacheInfo struct {
+	ImportPath string
+	PkgDir     string
+	PkgExport  string
+}
+
+// https://github.com/goplus/gop/issues/1710
+// Not fully optimized
+// Retrieve all imports in the specified directory and cache them
+func (c *Cache) goListExportCache(dir string, pkgs ...string) {
+	c.dirCacheMutex.Lock()
+	if c.dirCache[dir] {
+		c.dirCacheMutex.Unlock()
+		return
+	}
+	c.dirCache[dir] = true
+	c.dirCacheMutex.Unlock()
+	var stdout, stderr bytes.Buffer
+	commandStr := []string{"list", "-f", "{{.ImportPath}},{{.Dir}},{{.Export}}", "-export", "-e"}
+	commandStr = append(commandStr, pkgs...)
+	commandStr = append(commandStr, "all")
+	cmd := exec.Command("go", commandStr...)
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
+	cmd.Dir = dir
+	err := cmd.Run()
+	if err == nil {
+		ret := stdout.String()
+		for _, v := range strings.Split(ret, "\n") {
+			s := strings.Split(v, ",")
+			if len(s) != 3 {
+				continue
+			}
+			c.packageCacheMutex.Lock()
+			c.packageCacheMap[s[0]] = CacheInfo{s[0], s[1], s[2]}
+			c.packageCacheMutex.Unlock()
+		}
+	}
+}
+
+// Reduce wait time when performing `go list` on multiple pkgDir at the same time
+func (c *Cache) GoListExportCacheSync(dir string, pkgs ...string) {
+	c.waitCache.Add(1)
+	go func() {
+		defer c.waitCache.Done()
+		c.goListExportCache(dir, pkgs...)
+	}()
+}
+
+func (c *Cache) GetPkgCache(pkgPath string) (ret CacheInfo, ok bool) {
+	c.waitCache.Wait()
+	c.packageCacheMutex.RLock()
+	ret, ok = c.packageCacheMap[pkgPath]
+	c.packageCacheMutex.RUnlock()
+	return
+}
+
+func (c *Cache) DelPkgCache(pkgPath string) bool {
+	_, ok := c.GetPkgCache(pkgPath)
+	if ok {
+		c.packageCacheMutex.Lock()
+		delete(c.packageCacheMap, pkgPath)
+		c.packageCacheMutex.Unlock()
+		return true
+	}
+	return false
+}
+
+func NewGoListCache(dir string) *Cache {
+	c := &Cache{
+		dirCache:        make(map[string]bool),
+		packageCacheMap: make(map[string]CacheInfo),
+	}
+	// get the dir pkg cache
+	c.GoListExportCacheSync(dir)
+	return c
+}

packages/cache_test.go

@@ -0,0 +1,18 @@
+package packages
+
+import "testing"
+
+func TestCacheDelCache(t *testing.T) {
+	p := NewImporter(nil)
+	if !p.GetPkgCache().DelPkgCache("fmt") {
+		t.Fatal("del cache should pass!")
+	}
+	if p.GetPkgCache().DelPkgCache("fmt") {
+		t.Fatal("del cache should fail")
+	}
+}
+
+func TestRepeatLoadDir(t *testing.T) {
+	p := NewImporter(nil)
+	p.GetPkgCache().goListExportCache("", "")
+}

packages/imp.go

@@ -28,10 +28,11 @@ import (
 // ----------------------------------------------------------------------------
 
 type Importer struct {
-	loaded map[string]*types.Package
-	fset   *token.FileSet
-	dir    string
-	m      sync.RWMutex
+	loaded   map[string]*types.Package
+	fset     *token.FileSet
+	dir      string
+	m        sync.RWMutex
+	pkgCache *Cache
 }
 
 // NewImporter creates an Importer object that meets types.Importer interface.
@@ -45,7 +46,7 @@ func NewImporter(fset *token.FileSet, workDir ...string) *Importer {
 	}
 	loaded := make(map[string]*types.Package)
 	loaded["unsafe"] = types.Unsafe
-	return &Importer{loaded: loaded, fset: fset, dir: dir}
+	return &Importer{loaded: loaded, fset: fset, dir: dir, pkgCache: NewGoListCache(dir)}
 }
 
 func (p *Importer) Import(pkgPath string) (pkg *types.Package, err error) {
@@ -68,6 +69,10 @@ func (p *Importer) ImportFrom(pkgPath, dir string, mode types.ImportMode) (*type
 		return ret, nil
 	}
 	p.m.RUnlock()
+	cacheInfo, ok := p.pkgCache.GetPkgCache(pkgPath)
+	if ok {
+		return p.loadByExport(cacheInfo.PkgExport, pkgPath)
+	}
 	expfile, err := FindExport(dir, pkgPath)
 	if err != nil {
 		return nil, err
@@ -91,6 +96,10 @@ func (p *Importer) loadByExport(expfile string, pkgPath string) (ret *types.Pack
 	return
 }
 
+func (p *Importer) GetPkgCache() *Cache {
+	return p.pkgCache
+}
+
 // ----------------------------------------------------------------------------
 
 // FindExport lookups export file (.a) of a package by its pkgPath.