diff --git a/CHANGELOG.md b/CHANGELOG.md index a20f552ad7..f30cb39834 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,8 @@ NEW FEATURES: * Add CI tests against go1.10. Drop support for go1.8. ([#1620](https://github.com/golang/dep/pull/1620)) * Added `install.sh` script. ([#1533](https://github.com/golang/dep/pull/1533)) -* List out of date projects in dep status ([#1553](https://github.com/golang/dep/pull/1553)). +* List out of date projects in dep status. ([#1553](https://github.com/golang/dep/pull/1553)). +* Enabled opt-in persistent caching via $DEPCACHEAGE env var. ([#1711](https://github.com/golang/dep/pull/1711)) BUG FIXES: diff --git a/cmd/dep/main.go b/cmd/dep/main.go index e3cf27f4fe..57f0a196dd 100644 --- a/cmd/dep/main.go +++ b/cmd/dep/main.go @@ -18,6 +18,7 @@ import ( "runtime/pprof" "strings" "text/tabwriter" + "time" "github.com/golang/dep" "github.com/golang/dep/internal/fs" @@ -216,6 +217,16 @@ func (c *Config) Run() int { } } + var cacheAge time.Duration + if env := getEnv(c.Env, "DEPCACHEAGE"); env != "" { + var err error + cacheAge, err = time.ParseDuration(env) + if err != nil { + errLogger.Printf("dep: failed to parse $DEPCACHEAGE duration %q: %v\n", env, err) + return errorExitCode + } + } + // Set up dep context. ctx := &dep.Ctx{ Out: outLogger, @@ -223,6 +234,7 @@ func (c *Config) Run() int { Verbose: *verbose, DisableLocking: getEnv(c.Env, "DEPNOLOCK") != "", Cachedir: cachedir, + CacheAge: cacheAge, } GOPATHS := filepath.SplitList(getEnv(c.Env, "GOPATH")) diff --git a/context.go b/context.go index 475efb3776..9dc33dc30f 100644 --- a/context.go +++ b/context.go @@ -9,6 +9,7 @@ import ( "os" "path/filepath" "runtime" + "time" "github.com/golang/dep/gps" "github.com/golang/dep/internal/fs" @@ -34,13 +35,14 @@ import ( // } // type Ctx struct { - WorkingDir string // Where to execute. - GOPATH string // Selected Go path, containing WorkingDir. - GOPATHs []string // Other Go paths. - Out, Err *log.Logger // Required loggers. - Verbose bool // Enables more verbose logging. - DisableLocking bool // When set, no lock file will be created to protect against simultaneous dep processes. - Cachedir string // Cache directory loaded from environment. + WorkingDir string // Where to execute. + GOPATH string // Selected Go path, containing WorkingDir. + GOPATHs []string // Other Go paths. + Out, Err *log.Logger // Required loggers. + Verbose bool // Enables more verbose logging. + DisableLocking bool // When set, no lock file will be created to protect against simultaneous dep processes. + Cachedir string // Cache directory loaded from environment. + CacheAge time.Duration // Maximum valid age of cached source data. <=0: Don't cache. } // SetPaths sets the WorkingDir and GOPATHs fields. If GOPATHs is empty, then @@ -99,6 +101,7 @@ func (c *Ctx) SourceManager() (*gps.SourceMgr, error) { } return gps.NewSourceManager(gps.SourceManagerConfig{ + CacheAge: c.CacheAge, Cachedir: cachedir, Logger: c.Out, DisableLocking: c.DisableLocking, diff --git a/gps/source.go b/gps/source.go index 2deb699ec6..0d9deb14d2 100644 --- a/gps/source.go +++ b/gps/source.go @@ -67,14 +67,21 @@ type sourceCoordinator struct { psrcmut sync.Mutex // guards protoSrcs map protoSrcs map[string][]chan srcReturn cachedir string + cache sourceCache logger *log.Logger } -func newSourceCoordinator(superv *supervisor, deducer deducer, cachedir string, logger *log.Logger) *sourceCoordinator { +// newSourceCoordinator returns a new sourceCoordinator. +// Passing a nil sourceCache defaults to an in-memory cache. +func newSourceCoordinator(superv *supervisor, deducer deducer, cachedir string, cache sourceCache, logger *log.Logger) *sourceCoordinator { + if cache == nil { + cache = memoryCache{} + } return &sourceCoordinator{ supervisor: superv, deducer: deducer, cachedir: cachedir, + cache: cache, logger: logger, srcs: make(map[string]*sourceGateway), nameToURL: make(map[string]string), @@ -82,7 +89,11 @@ func newSourceCoordinator(superv *supervisor, deducer deducer, cachedir string, } } -func (sc *sourceCoordinator) close() {} +func (sc *sourceCoordinator) close() { + if err := sc.cache.close(); err != nil { + sc.logger.Println(errors.Wrap(err, "failed to close the source cache")) + } +} func (sc *sourceCoordinator) getSourceGatewayFor(ctx context.Context, id ProjectIdentifier) (*sourceGateway, error) { if err := sc.supervisor.ctx.Err(); err != nil { @@ -216,7 +227,8 @@ func (sc *sourceCoordinator) getSourceGatewayFor(ctx context.Context, id Project } src, err := m.try(ctx, sc.cachedir) if err == nil { - srcGate, err = newSourceGateway(ctx, src, sc.supervisor, sc.cachedir) + cache := sc.cache.newSingleSourceCache(id) + srcGate, err = newSourceGateway(ctx, src, sc.supervisor, sc.cachedir, cache) if err == nil { sc.srcs[url] = srcGate break @@ -260,7 +272,7 @@ type sourceGateway struct { // newSourceGateway returns a new gateway for src. If the source exists locally, // the local state may be cleaned, otherwise we ping upstream. -func newSourceGateway(ctx context.Context, src source, superv *supervisor, cachedir string) (*sourceGateway, error) { +func newSourceGateway(ctx context.Context, src source, superv *supervisor, cachedir string, cache singleSourceCache) (*sourceGateway, error) { var state sourceState local := src.existsLocally(ctx) if local { @@ -276,9 +288,9 @@ func newSourceGateway(ctx context.Context, src source, superv *supervisor, cache srcState: state, src: src, cachedir: cachedir, + cache: cache, suprvsr: superv, } - sg.cache = sg.createSingleSourceCache() if !local { if err := sg.require(ctx, sourceExistsUpstream); err != nil { @@ -542,14 +554,6 @@ func (sg *sourceGateway) disambiguateRevision(ctx context.Context, r Revision) ( return sg.src.disambiguateRevision(ctx, r) } -// createSingleSourceCache creates a singleSourceCache instance for use by -// the encapsulated source. -func (sg *sourceGateway) createSingleSourceCache() singleSourceCache { - // TODO(sdboyer) when persistent caching is ready, just drop in the creation - // of a source-specific handle here - return newMemoryCache() -} - // sourceExistsUpstream verifies that the source exists upstream and that the // upstreamURL has not changed and returns any additional sourceState, or an error. func (sg *sourceGateway) sourceExistsUpstream(ctx context.Context) (sourceState, error) { diff --git a/gps/source_cache.go b/gps/source_cache.go index 11aaefd319..1123f317d8 100644 --- a/gps/source_cache.go +++ b/gps/source_cache.go @@ -13,6 +13,16 @@ import ( "github.com/golang/dep/gps/pkgtree" ) +// sourceCache is an interface for creating singleSourceCaches, and safely +// releasing backing resources via close. +type sourceCache interface { + // newSingleSourceCache creates a new singleSourceCache for id, which + // remains valid until close is called. + newSingleSourceCache(id ProjectIdentifier) singleSourceCache + // close releases background resources. + close() error +} + // singleSourceCache provides a method set for storing and retrieving data about // a single source. type singleSourceCache interface { @@ -62,6 +72,15 @@ type singleSourceCache interface { toUnpaired(v Version) (UnpairedVersion, bool) } +// memoryCache is a sourceCache which creates singleSourceCacheMemory instances. +type memoryCache struct{} + +func (memoryCache) newSingleSourceCache(ProjectIdentifier) singleSourceCache { + return newMemoryCache() +} + +func (memoryCache) close() error { return nil } + type singleSourceCacheMemory struct { // Protects all fields. mut sync.RWMutex diff --git a/gps/source_cache_bolt.go b/gps/source_cache_bolt.go index c9764363e5..b5bfc2ab55 100644 --- a/gps/source_cache_bolt.go +++ b/gps/source_cache_bolt.go @@ -21,6 +21,10 @@ import ( "github.com/pkg/errors" ) +// boltCacheFilename is a versioned filename for the bolt cache. The version +// must be incremented whenever incompatible changes are made. +const boltCacheFilename = "bolt-v1.db" + // boltCache manages a bolt.DB cache and provides singleSourceCaches. type boltCache struct { db *bolt.DB @@ -30,7 +34,7 @@ type boltCache struct { // newBoltCache returns a new boltCache backed by a BoltDB file under the cache directory. func newBoltCache(cd string, epoch int64, logger *log.Logger) (*boltCache, error) { - path := sourceCachePath(cd, "bolt") + ".db" + path := filepath.Join(cd, boltCacheFilename) dir := filepath.Dir(path) if fi, err := os.Stat(dir); os.IsNotExist(err) { if err := os.MkdirAll(dir, os.ModeDir|os.ModePerm); err != nil { diff --git a/gps/source_cache_multi.go b/gps/source_cache_multi.go index dede66e080..52c08b2063 100644 --- a/gps/source_cache_multi.go +++ b/gps/source_cache_multi.go @@ -8,22 +8,72 @@ import ( "github.com/golang/dep/gps/pkgtree" ) -// A multiCache manages two cache levels, ephemeral in-memory and persistent on-disk. +// multiCache creates singleSourceMultiCaches, and coordinates their async updates. +type multiCache struct { + mem, disk sourceCache + // Asynchronous disk cache updates. Closed by the close method. + async chan func() + // Closed when async has completed processing. + done chan struct{} +} + +// newMultiCache returns a new multiCache backed by mem and disk sourceCaches. +// Spawns a single background goroutine which lives until close() is called. +func newMultiCache(mem, disk sourceCache) *multiCache { + m := &multiCache{ + mem: mem, + disk: disk, + async: make(chan func(), 50), + done: make(chan struct{}), + } + go m.processAsync() + return m +} + +func (c *multiCache) processAsync() { + for f := range c.async { + f() + } + close(c.done) +} + +// close releases resources after blocking until async writes complete. +func (c *multiCache) close() error { + close(c.async) + _ = c.mem.close() + <-c.done + return c.disk.close() +} + +// newSingleSourceCache returns a singleSourceMultiCache for id. +func (c *multiCache) newSingleSourceCache(id ProjectIdentifier) singleSourceCache { + return &singleSourceMultiCache{ + mem: c.mem.newSingleSourceCache(id), + disk: c.disk.newSingleSourceCache(id), + async: c.async, + } +} + +// singleSourceMultiCache manages two cache levels, ephemeral in-memory and persistent on-disk. // // The in-memory cache is always checked first, with the on-disk used as a fallback. // Values read from disk are set in-memory when an appropriate method exists. // -// Set values are cached both in-memory and on-disk. -type multiCache struct { +// Set values are cached both in-memory and on-disk. Values are set synchronously +// in-memory. Writes to the on-disk cache are asynchronous, and executed in order by a +// background goroutine. +type singleSourceMultiCache struct { mem, disk singleSourceCache + // Asynchronous disk cache updates. + async chan<- func() } -func (c *multiCache) setManifestAndLock(r Revision, ai ProjectAnalyzerInfo, m Manifest, l Lock) { +func (c *singleSourceMultiCache) setManifestAndLock(r Revision, ai ProjectAnalyzerInfo, m Manifest, l Lock) { c.mem.setManifestAndLock(r, ai, m, l) - c.disk.setManifestAndLock(r, ai, m, l) + c.async <- func() { c.disk.setManifestAndLock(r, ai, m, l) } } -func (c *multiCache) getManifestAndLock(r Revision, ai ProjectAnalyzerInfo) (Manifest, Lock, bool) { +func (c *singleSourceMultiCache) getManifestAndLock(r Revision, ai ProjectAnalyzerInfo) (Manifest, Lock, bool) { m, l, ok := c.mem.getManifestAndLock(r, ai) if ok { return m, l, true @@ -38,12 +88,12 @@ func (c *multiCache) getManifestAndLock(r Revision, ai ProjectAnalyzerInfo) (Man return nil, nil, false } -func (c *multiCache) setPackageTree(r Revision, ptree pkgtree.PackageTree) { +func (c *singleSourceMultiCache) setPackageTree(r Revision, ptree pkgtree.PackageTree) { c.mem.setPackageTree(r, ptree) - c.disk.setPackageTree(r, ptree) + c.async <- func() { c.disk.setPackageTree(r, ptree) } } -func (c *multiCache) getPackageTree(r Revision, pr ProjectRoot) (pkgtree.PackageTree, bool) { +func (c *singleSourceMultiCache) getPackageTree(r Revision, pr ProjectRoot) (pkgtree.PackageTree, bool) { ptree, ok := c.mem.getPackageTree(r, pr) if ok { return ptree, true @@ -58,17 +108,17 @@ func (c *multiCache) getPackageTree(r Revision, pr ProjectRoot) (pkgtree.Package return pkgtree.PackageTree{}, false } -func (c *multiCache) markRevisionExists(r Revision) { +func (c *singleSourceMultiCache) markRevisionExists(r Revision) { c.mem.markRevisionExists(r) - c.disk.markRevisionExists(r) + c.async <- func() { c.disk.markRevisionExists(r) } } -func (c *multiCache) setVersionMap(pvs []PairedVersion) { +func (c *singleSourceMultiCache) setVersionMap(pvs []PairedVersion) { c.mem.setVersionMap(pvs) - c.disk.setVersionMap(pvs) + c.async <- func() { c.disk.setVersionMap(pvs) } } -func (c *multiCache) getVersionsFor(rev Revision) ([]UnpairedVersion, bool) { +func (c *singleSourceMultiCache) getVersionsFor(rev Revision) ([]UnpairedVersion, bool) { uvs, ok := c.mem.getVersionsFor(rev) if ok { return uvs, true @@ -77,7 +127,7 @@ func (c *multiCache) getVersionsFor(rev Revision) ([]UnpairedVersion, bool) { return c.disk.getVersionsFor(rev) } -func (c *multiCache) getAllVersions() ([]PairedVersion, bool) { +func (c *singleSourceMultiCache) getAllVersions() ([]PairedVersion, bool) { pvs, ok := c.mem.getAllVersions() if ok { return pvs, true @@ -92,7 +142,7 @@ func (c *multiCache) getAllVersions() ([]PairedVersion, bool) { return nil, false } -func (c *multiCache) getRevisionFor(uv UnpairedVersion) (Revision, bool) { +func (c *singleSourceMultiCache) getRevisionFor(uv UnpairedVersion) (Revision, bool) { rev, ok := c.mem.getRevisionFor(uv) if ok { return rev, true @@ -101,7 +151,7 @@ func (c *multiCache) getRevisionFor(uv UnpairedVersion) (Revision, bool) { return c.disk.getRevisionFor(uv) } -func (c *multiCache) toRevision(v Version) (Revision, bool) { +func (c *singleSourceMultiCache) toRevision(v Version) (Revision, bool) { rev, ok := c.mem.toRevision(v) if ok { return rev, true @@ -110,7 +160,7 @@ func (c *multiCache) toRevision(v Version) (Revision, bool) { return c.disk.toRevision(v) } -func (c *multiCache) toUnpaired(v Version) (UnpairedVersion, bool) { +func (c *singleSourceMultiCache) toUnpaired(v Version) (UnpairedVersion, bool) { uv, ok := c.mem.toUnpaired(v) if ok { return uv, true diff --git a/gps/source_cache_test.go b/gps/source_cache_test.go index 4d93c2110c..9e15b65ac9 100644 --- a/gps/source_cache_test.go +++ b/gps/source_cache_test.go @@ -19,41 +19,46 @@ import ( ) func Test_singleSourceCache(t *testing.T) { - newMem := func(*testing.T, string, string) (singleSourceCache, func() error) { - return newMemoryCache(), func() error { return nil } + newMem := func(*testing.T, string) sourceCache { + return memoryCache{} } t.Run("mem", singleSourceCacheTest{newCache: newMem}.run) epoch := time.Now().Unix() - newBolt := func(t *testing.T, cachedir, root string) (singleSourceCache, func() error) { - pi := mkPI(root).normalize() + newBolt := func(t *testing.T, cachedir string) sourceCache { bc, err := newBoltCache(cachedir, epoch, log.New(test.Writer{TB: t}, "", 0)) if err != nil { t.Fatal(err) } - return bc.newSingleSourceCache(pi), bc.close + return bc } t.Run("bolt/keepOpen", singleSourceCacheTest{newCache: newBolt}.run) t.Run("bolt/reOpen", singleSourceCacheTest{newCache: newBolt, persistent: true}.run) - newMulti := func(t *testing.T, cachedir, root string) (singleSourceCache, func() error) { - disk, close := newBolt(t, cachedir, root) - return &multiCache{mem: newMemoryCache(), disk: disk}, close + newMulti := func(t *testing.T, cachedir string) sourceCache { + bc, err := newBoltCache(cachedir, epoch, log.New(test.Writer{TB: t}, "", 0)) + if err != nil { + t.Fatal(err) + } + return newMultiCache(memoryCache{}, bc) } t.Run("multi/keepOpen", singleSourceCacheTest{newCache: newMulti}.run) t.Run("multi/reOpen", singleSourceCacheTest{persistent: true, newCache: newMulti}.run) t.Run("multi/keepOpen/noDisk", singleSourceCacheTest{ - newCache: func(*testing.T, string, string) (singleSourceCache, func() error) { - return &multiCache{mem: newMemoryCache(), disk: discardCache{}}, func() error { return nil } + newCache: func(*testing.T, string) sourceCache { + return newMultiCache(memoryCache{}, discardCache{}) }, }.run) t.Run("multi/reOpen/noMem", singleSourceCacheTest{ persistent: true, - newCache: func(t *testing.T, cachedir, root string) (singleSourceCache, func() error) { - disk, close := newBolt(t, cachedir, root) - return &multiCache{mem: discardCache{}, disk: disk}, close + newCache: func(t *testing.T, cachedir string) sourceCache { + bc, err := newBoltCache(cachedir, epoch, log.New(test.Writer{TB: t}, "", 0)) + if err != nil { + t.Fatal(err) + } + return newMultiCache(discardCache{}, bc) }, }.run) } @@ -64,7 +69,7 @@ var testAnalyzerInfo = ProjectAnalyzerInfo{ } type singleSourceCacheTest struct { - newCache func(*testing.T, string, string) (cache singleSourceCache, close func() error) + newCache func(*testing.T, string) sourceCache persistent bool } @@ -72,6 +77,7 @@ type singleSourceCacheTest struct { // For test.persistent caches, test.newCache is periodically called mid-test to ensure persistence. func (test singleSourceCacheTest) run(t *testing.T) { const root = "example.com/test" + pi := mkPI(root).normalize() cpath, err := ioutil.TempDir("", "singlesourcecache") if err != nil { t.Fatalf("Failed to create temp cache dir: %s", err) @@ -80,9 +86,10 @@ func (test singleSourceCacheTest) run(t *testing.T) { t.Run("info", func(t *testing.T) { const rev Revision = "revision" - c, close := test.newCache(t, cpath, root) + sc := test.newCache(t, cpath) + c := sc.newSingleSourceCache(pi) defer func() { - if err := close(); err != nil { + if err := sc.close(); err != nil { t.Fatal("failed to close cache:", err) } }() @@ -121,10 +128,11 @@ func (test singleSourceCacheTest) run(t *testing.T) { c.setManifestAndLock(rev, testAnalyzerInfo, m, l) if test.persistent { - if err := close(); err != nil { + if err := sc.close(); err != nil { t.Fatal("failed to close cache:", err) } - c, close = test.newCache(t, cpath, root) + sc = test.newCache(t, cpath) + c = sc.newSingleSourceCache(pi) } gotM, gotL, ok := c.getManifestAndLock(rev, testAnalyzerInfo) @@ -165,10 +173,11 @@ func (test singleSourceCacheTest) run(t *testing.T) { c.setManifestAndLock(rev, testAnalyzerInfo, m, l) if test.persistent { - if err := close(); err != nil { + if err := sc.close(); err != nil { t.Fatal("failed to close cache:", err) } - c, close = test.newCache(t, cpath, root) + sc = test.newCache(t, cpath) + c = sc.newSingleSourceCache(pi) } gotM, gotL, ok = c.getManifestAndLock(rev, testAnalyzerInfo) @@ -182,9 +191,10 @@ func (test singleSourceCacheTest) run(t *testing.T) { }) t.Run("pkgTree", func(t *testing.T) { - c, close := test.newCache(t, cpath, root) + sc := test.newCache(t, cpath) + c := sc.newSingleSourceCache(pi) defer func() { - if err := close(); err != nil { + if err := sc.close(); err != nil { t.Fatal("failed to close cache:", err) } }() @@ -196,10 +206,11 @@ func (test singleSourceCacheTest) run(t *testing.T) { } if test.persistent { - if err := close(); err != nil { + if err := sc.close(); err != nil { t.Fatal("failed to close cache:", err) } - c, close = test.newCache(t, cpath, root) + sc = test.newCache(t, cpath) + c = sc.newSingleSourceCache(pi) } pt := pkgtree.PackageTree{ @@ -243,10 +254,11 @@ func (test singleSourceCacheTest) run(t *testing.T) { c.setPackageTree(rev, pt) if test.persistent { - if err := close(); err != nil { + if err := sc.close(); err != nil { t.Fatal("failed to close cache:", err) } - c, close = test.newCache(t, cpath, root) + sc = test.newCache(t, cpath) + c = sc.newSingleSourceCache(pi) } got, ok := c.getPackageTree(rev, root) @@ -256,10 +268,11 @@ func (test singleSourceCacheTest) run(t *testing.T) { comparePackageTree(t, pt, got) if test.persistent { - if err := close(); err != nil { + if err := sc.close(); err != nil { t.Fatal("failed to close cache:", err) } - c, close = test.newCache(t, cpath, root) + sc = test.newCache(t, cpath) + c = sc.newSingleSourceCache(pi) } pt = pkgtree.PackageTree{ @@ -273,10 +286,11 @@ func (test singleSourceCacheTest) run(t *testing.T) { c.setPackageTree(rev, pt) if test.persistent { - if err := close(); err != nil { + if err := sc.close(); err != nil { t.Fatal("failed to close cache:", err) } - c, close = test.newCache(t, cpath, root) + sc = test.newCache(t, cpath) + c = sc.newSingleSourceCache(pi) } got, ok = c.getPackageTree(rev, root) @@ -287,9 +301,10 @@ func (test singleSourceCacheTest) run(t *testing.T) { }) t.Run("versions", func(t *testing.T) { - c, close := test.newCache(t, cpath, root) + sc := test.newCache(t, cpath) + c := sc.newSingleSourceCache(pi) defer func() { - if err := close(); err != nil { + if err := sc.close(); err != nil { t.Fatal("failed to close cache:", err) } }() @@ -304,10 +319,11 @@ func (test singleSourceCacheTest) run(t *testing.T) { c.setVersionMap(versions) if test.persistent { - if err := close(); err != nil { + if err := sc.close(); err != nil { t.Fatal("failed to close cache:", err) } - c, close = test.newCache(t, cpath, root) + sc = test.newCache(t, cpath) + c = sc.newSingleSourceCache(pi) } t.Run("getAllVersions", func(t *testing.T) { @@ -549,41 +565,52 @@ func packageOrErrEqual(a, b pkgtree.PackageOrErr) bool { return true } -// discardCache discards set values and returns nothing. +// discardCache produces singleSourceDiscardCaches. type discardCache struct{} -func (discardCache) setManifestAndLock(Revision, ProjectAnalyzerInfo, Manifest, Lock) {} +func (discardCache) newSingleSourceCache(ProjectIdentifier) singleSourceCache { + return discard +} + +func (discardCache) close() error { return nil } + +var discard singleSourceCache = singleSourceDiscardCache{} + +// singleSourceDiscardCache discards set values and returns nothing. +type singleSourceDiscardCache struct{} + +func (singleSourceDiscardCache) setManifestAndLock(Revision, ProjectAnalyzerInfo, Manifest, Lock) {} -func (discardCache) getManifestAndLock(Revision, ProjectAnalyzerInfo) (Manifest, Lock, bool) { +func (singleSourceDiscardCache) getManifestAndLock(Revision, ProjectAnalyzerInfo) (Manifest, Lock, bool) { return nil, nil, false } -func (discardCache) setPackageTree(Revision, pkgtree.PackageTree) {} +func (singleSourceDiscardCache) setPackageTree(Revision, pkgtree.PackageTree) {} -func (discardCache) getPackageTree(Revision, ProjectRoot) (pkgtree.PackageTree, bool) { +func (singleSourceDiscardCache) getPackageTree(Revision, ProjectRoot) (pkgtree.PackageTree, bool) { return pkgtree.PackageTree{}, false } -func (discardCache) markRevisionExists(r Revision) {} +func (singleSourceDiscardCache) markRevisionExists(r Revision) {} -func (discardCache) setVersionMap(versionList []PairedVersion) {} +func (singleSourceDiscardCache) setVersionMap(versionList []PairedVersion) {} -func (discardCache) getVersionsFor(Revision) ([]UnpairedVersion, bool) { +func (singleSourceDiscardCache) getVersionsFor(Revision) ([]UnpairedVersion, bool) { return nil, false } -func (discardCache) getAllVersions() ([]PairedVersion, bool) { +func (singleSourceDiscardCache) getAllVersions() ([]PairedVersion, bool) { return nil, false } -func (discardCache) getRevisionFor(UnpairedVersion) (Revision, bool) { +func (singleSourceDiscardCache) getRevisionFor(UnpairedVersion) (Revision, bool) { return "", false } -func (discardCache) toRevision(v Version) (Revision, bool) { +func (singleSourceDiscardCache) toRevision(v Version) (Revision, bool) { return "", false } -func (discardCache) toUnpaired(v Version) (UnpairedVersion, bool) { +func (singleSourceDiscardCache) toUnpaired(v Version) (UnpairedVersion, bool) { return nil, false } diff --git a/gps/source_manager.go b/gps/source_manager.go index 86a0ec8fc2..277e66a6ee 100644 --- a/gps/source_manager.go +++ b/gps/source_manager.go @@ -177,9 +177,10 @@ var ErrSourceManagerIsReleased = fmt.Errorf("this SourceManager has been release // SourceManagerConfig holds configuration information for creating SourceMgrs. type SourceManagerConfig struct { - Cachedir string // Where to store local instances of upstream sources. - Logger *log.Logger // Optional info/warn logger. Discards if nil. - DisableLocking bool // True if the SourceManager should NOT use a lock file to protect the Cachedir from multiple processes. + CacheAge time.Duration // Maximum valid age of cached data. <=0: Don't cache. + Cachedir string // Where to store local instances of upstream sources. + Logger *log.Logger // Optional info/warn logger. Discards if nil. + DisableLocking bool // True if the SourceManager should NOT use a lock file to protect the Cachedir from multiple processes. } // NewSourceManager produces an instance of gps's built-in SourceManager. @@ -190,6 +191,10 @@ type SourceManagerConfig struct { // SourceManager as early as possible and use it to their ends. That way, the // solver can benefit from any caches that may have already been warmed. // +// A cacheEpoch is calculated from now()-cacheAge, and older persistent cache data +// is discarded. When cacheAge is <= 0, the persistent cache is +// not used. +// // gps's SourceManager is intended to be threadsafe (if it's not, please file a // bug!). It should be safe to reuse across concurrent solving runs, even on // unrelated projects. @@ -279,13 +284,25 @@ func NewSourceManager(c SourceManagerConfig) (*SourceMgr, error) { superv := newSupervisor(ctx) deducer := newDeductionCoordinator(superv) + var sc sourceCache + if c.CacheAge > 0 { + // Try to open the BoltDB cache from disk. + epoch := time.Now().Add(-c.CacheAge).Unix() + boltCache, err := newBoltCache(c.Cachedir, epoch, c.Logger) + if err != nil { + c.Logger.Println(errors.Wrapf(err, "failed to open persistent cache %q", c.Cachedir)) + } else { + sc = newMultiCache(memoryCache{}, boltCache) + } + } + sm := &SourceMgr{ cachedir: c.Cachedir, lf: lockfile, suprvsr: superv, cancelAll: cf, deduceCoord: deducer, - srcCoord: newSourceCoordinator(superv, deducer, c.Cachedir, c.Logger), + srcCoord: newSourceCoordinator(superv, deducer, c.Cachedir, sc, c.Logger), qch: make(chan struct{}), } @@ -379,7 +396,7 @@ func (e CouldNotCreateLockError) Error() string { return e.Err.Error() } -// Release lets go of any locks held by the SourceManager. Once called, it is no +// Release lets go of any resources held by the SourceManager. Once called, it is no // longer safe to call methods against it; all method calls will immediately // result in errors. func (sm *SourceMgr) Release() { diff --git a/gps/source_test.go b/gps/source_test.go index 636cc8cf2a..6debaee639 100644 --- a/gps/source_test.go +++ b/gps/source_test.go @@ -43,7 +43,7 @@ func testSourceGateway(t *testing.T) { superv := newSupervisor(ctx) deducer := newDeductionCoordinator(superv) logger := log.New(test.Writer{TB: t}, "", 0) - sc := newSourceCoordinator(superv, deducer, cachedir, logger) + sc := newSourceCoordinator(superv, deducer, cachedir, nil, logger) defer sc.close() id := mkPI("github.com/sdboyer/deptest")