Add cache support (#16)

* add cache library dgraph-io/ristretto

* add cache support

* make cache a part of storager instead of using global variable

* calculate path in loop instead of using closures

* make setCache() returns void

* refactor pathToId

* fix typo

* add type assertion

go.mod

@@ -5,6 +5,7 @@ go 1.15
 require (
 	github.com/beyondstorage/go-integration-test/v4 v4.1.1
 	github.com/beyondstorage/go-storage/v4 v4.2.1-0.20210709064026-793dd83d71d1
+	github.com/dgraph-io/ristretto v0.1.0
 	github.com/google/uuid v1.2.0
 	google.golang.org/api v0.50.0
 )

go.sum

@@ -50,6 +50,8 @@ github.com/beyondstorage/go-storage/v4 v4.2.1-0.20210709064026-793dd83d71d1 h1:5
 github.com/beyondstorage/go-storage/v4 v4.2.1-0.20210709064026-793dd83d71d1/go.mod h1:0fdcRCzLKMQe7Ve4zPlyTGgoPYwuINiV79Gx9tCt9tQ=
 github.com/beyondstorage/specs/go v0.0.0-20210623065218-d1c2d7d81259/go.mod h1:vF/Q0P1tCvhVAUrxg7i6NvrARRMQVTAuQdDNqpSzR1w=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
+github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
@@ -66,6 +68,12 @@ github.com/dave/rebecca v0.9.1/go.mod h1:N6XYdMD/OKw3lkF3ywh8Z6wPGuwNFDNtWYEMFWE
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dgraph-io/ristretto v0.1.0 h1:Jv3CGQHp9OjuMBSne1485aDpUkTKEcUqF+jm/LuerPI=
+github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug=
+github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
+github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
+github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
+github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
@@ -76,6 +84,7 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -168,6 +177,8 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ=
 github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=

storage.go

@@ -27,6 +27,7 @@ func (s *Storage) createDir(ctx context.Context, path string, opt pairStorageCre
 	path = s.getAbsPath(path)
 	pathUnits := strings.Split(path, "/")
 	parentsId := "root"
+
 	for _, v := range pathUnits {
 		parentsId, err = s.mkDir(ctx, parentsId, v)
 		if err != nil {
@@ -135,23 +136,37 @@ func (s *Storage) nextObjectPage(ctx context.Context, page *ObjectPage) error {
 	return nil
 }
 
-// TODO: add cache support
-// pathToId converts path to fileId, as we talked in RFC-14.
+// pathToId converts path to fileId, as we discussed in RFC-14.
 // Ref: https://github.com/beyondstorage/go-service-gdrive/blob/master/docs/rfcs/14-gdrive-for-go-storage-design.md
 // Behavior:
 // err represents the error handled in pathToId
 // fileId represents the results: fileId empty means the path is not exist, otherwise it's the fileId of input path
 func (s *Storage) pathToId(ctx context.Context, path string) (fileId string, err error) {
 	absPath := s.getAbsPath(path)
+
+	fileId, found := s.getCache(path)
+	if found {
+		return fileId, nil
+	}
+
 	pathUnits := strings.Split(absPath, "/")
 	fileId = "root"
+	cacheCurrentPath := ""
 	// Traverse the whole path, break the loop if we fails at one search
 	for _, v := range pathUnits {
 		fileId, err = s.searchContentInDir(ctx, fileId, v)
 
 		if fileId == "" || err != nil {
 			break
 		}
+
+		if cacheCurrentPath == "" {
+			cacheCurrentPath = v
+		} else {
+			cacheCurrentPath += "/" + v
+		}
+
+		s.setCache(cacheCurrentPath, fileId)
 	}
 
 	if err != nil {
@@ -211,6 +226,7 @@ func (s *Storage) stat(ctx context.Context, path string, opt pairStorageStat) (o
 // First we need make sure this file is not exist.
 // If it is, then we upload it, or we will overwrite it.
 func (s *Storage) write(ctx context.Context, path string, r io.Reader, size int64, opt pairStorageWrite) (n int64, err error) {
+
 	r = io.LimitReader(r, size)
 
 	if opt.HasIoCallback {
@@ -232,12 +248,11 @@ func (s *Storage) write(ctx context.Context, path string, r io.Reader, size int6
 		}
 
 		file := &drive.File{Name: fileName}
-		_, err = s.service.Files.Create(file).Context(ctx).Media(r).Do()
+		_, err := s.service.Files.Create(file).Context(ctx).Media(r).Do()
 
 		if err != nil {
 			return 0, err
 		}
-
 	} else {
 		// update
 		newFile := &drive.File{Name: s.getFileName(path)}

utils.go

@@ -5,20 +5,32 @@ import (
 	"fmt"
 	"strings"
 
+	"google.golang.org/api/drive/v3"
+	"google.golang.org/api/googleapi"
+	"google.golang.org/api/option"
+
+	. "github.com/dgraph-io/ristretto"
+
 	ps "github.com/beyondstorage/go-storage/v4/pairs"
 	"github.com/beyondstorage/go-storage/v4/pkg/credential"
 	"github.com/beyondstorage/go-storage/v4/services"
 	"github.com/beyondstorage/go-storage/v4/types"
-	"google.golang.org/api/drive/v3"
-	"google.golang.org/api/googleapi"
-	"google.golang.org/api/option"
+)
+
+const (
+	numCounters = 1e7     // number of keys to track frequency of (10M).
+	maxCost     = 1 << 30 // maximum cost of cache (1GB).
+	bufferItems = 64      // number of keys per Get buffer.
+	cost        = 1
+	expireTime  = 100
 )
 
 // Storage is the example client.
 type Storage struct {
 	name         string
 	workDir      string
 	service      *drive.Service
+	cache        *Cache
 	defaultPairs DefaultStoragePairs
 	features     StorageFeatures
 
@@ -55,6 +67,14 @@ func newStorager(pairs ...types.Pair) (store *Storage, err error) {
 		name:    opt.Name,
 		workDir: "/",
 	}
+
+	// Init cache for storager
+	ch, err := initCache(numCounters, maxCost, bufferItems)
+	if err != nil {
+		return nil, err
+	}
+	store.cache = ch
+
 	if opt.HasWorkDir {
 		store.workDir = opt.WorkDir
 	}
@@ -146,3 +166,30 @@ func (s *Storage) getFileName(path string) string {
 		return path
 	}
 }
+
+func initCache(nc int64, mc int64, bi int64) (cache *Cache, err error) {
+	config := &Config{
+		NumCounters: nc,
+		MaxCost:     mc,
+		BufferItems: bi,
+	}
+
+	cache, err = NewCache(config)
+
+	if err != nil {
+		return nil, err
+	}
+	return cache, nil
+}
+
+func (s *Storage) setCache(path string, fileId string) {
+	s.cache.SetWithTTL(path, fileId, cost, expireTime)
+}
+
+func (s *Storage) getCache(path string) (string, bool) {
+	id, found := s.cache.Get(path)
+	if found {
+		return id.(string), true
+	}
+	return "", false
+}