diff --git a/go.mod b/go.mod index 1e71281..672529d 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.15 require ( github.com/beyondstorage/go-integration-test/v4 v4.1.1 github.com/beyondstorage/go-storage/v4 v4.2.1-0.20210709064026-793dd83d71d1 + github.com/dgraph-io/ristretto v0.1.0 github.com/google/uuid v1.2.0 google.golang.org/api v0.50.0 ) diff --git a/go.sum b/go.sum index fd162c6..dcfa717 100644 --- a/go.sum +++ b/go.sum @@ -50,6 +50,8 @@ github.com/beyondstorage/go-storage/v4 v4.2.1-0.20210709064026-793dd83d71d1 h1:5 github.com/beyondstorage/go-storage/v4 v4.2.1-0.20210709064026-793dd83d71d1/go.mod h1:0fdcRCzLKMQe7Ve4zPlyTGgoPYwuINiV79Gx9tCt9tQ= github.com/beyondstorage/specs/go v0.0.0-20210623065218-d1c2d7d81259/go.mod h1:vF/Q0P1tCvhVAUrxg7i6NvrARRMQVTAuQdDNqpSzR1w= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -66,6 +68,12 @@ github.com/dave/rebecca v0.9.1/go.mod h1:N6XYdMD/OKw3lkF3ywh8Z6wPGuwNFDNtWYEMFWE github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/ristretto v0.1.0 h1:Jv3CGQHp9OjuMBSne1485aDpUkTKEcUqF+jm/LuerPI= +github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -76,6 +84,7 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -168,6 +177,8 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= diff --git a/storage.go b/storage.go index fd0222a..ef56b26 100644 --- a/storage.go +++ b/storage.go @@ -27,6 +27,7 @@ func (s *Storage) createDir(ctx context.Context, path string, opt pairStorageCre path = s.getAbsPath(path) pathUnits := strings.Split(path, "/") parentsId := "root" + for _, v := range pathUnits { parentsId, err = s.mkDir(ctx, parentsId, v) if err != nil { @@ -135,16 +136,22 @@ func (s *Storage) nextObjectPage(ctx context.Context, page *ObjectPage) error { return nil } -// TODO: add cache support -// pathToId converts path to fileId, as we talked in RFC-14. +// pathToId converts path to fileId, as we discussed in RFC-14. // Ref: https://github.com/beyondstorage/go-service-gdrive/blob/master/docs/rfcs/14-gdrive-for-go-storage-design.md // Behavior: // err represents the error handled in pathToId // fileId represents the results: fileId empty means the path is not exist, otherwise it's the fileId of input path func (s *Storage) pathToId(ctx context.Context, path string) (fileId string, err error) { absPath := s.getAbsPath(path) + + fileId, found := s.getCache(path) + if found { + return fileId, nil + } + pathUnits := strings.Split(absPath, "/") fileId = "root" + cacheCurrentPath := "" // Traverse the whole path, break the loop if we fails at one search for _, v := range pathUnits { fileId, err = s.searchContentInDir(ctx, fileId, v) @@ -152,6 +159,14 @@ func (s *Storage) pathToId(ctx context.Context, path string) (fileId string, err if fileId == "" || err != nil { break } + + if cacheCurrentPath == "" { + cacheCurrentPath = v + } else { + cacheCurrentPath += "/" + v + } + + s.setCache(cacheCurrentPath, fileId) } if err != nil { @@ -211,6 +226,7 @@ func (s *Storage) stat(ctx context.Context, path string, opt pairStorageStat) (o // First we need make sure this file is not exist. // If it is, then we upload it, or we will overwrite it. func (s *Storage) write(ctx context.Context, path string, r io.Reader, size int64, opt pairStorageWrite) (n int64, err error) { + r = io.LimitReader(r, size) if opt.HasIoCallback { @@ -232,12 +248,11 @@ func (s *Storage) write(ctx context.Context, path string, r io.Reader, size int6 } file := &drive.File{Name: fileName} - _, err = s.service.Files.Create(file).Context(ctx).Media(r).Do() + _, err := s.service.Files.Create(file).Context(ctx).Media(r).Do() if err != nil { return 0, err } - } else { // update newFile := &drive.File{Name: s.getFileName(path)} diff --git a/utils.go b/utils.go index 3ad2c23..5d013fb 100644 --- a/utils.go +++ b/utils.go @@ -5,13 +5,24 @@ import ( "fmt" "strings" + "google.golang.org/api/drive/v3" + "google.golang.org/api/googleapi" + "google.golang.org/api/option" + + . "github.com/dgraph-io/ristretto" + ps "github.com/beyondstorage/go-storage/v4/pairs" "github.com/beyondstorage/go-storage/v4/pkg/credential" "github.com/beyondstorage/go-storage/v4/services" "github.com/beyondstorage/go-storage/v4/types" - "google.golang.org/api/drive/v3" - "google.golang.org/api/googleapi" - "google.golang.org/api/option" +) + +const ( + numCounters = 1e7 // number of keys to track frequency of (10M). + maxCost = 1 << 30 // maximum cost of cache (1GB). + bufferItems = 64 // number of keys per Get buffer. + cost = 1 + expireTime = 100 ) // Storage is the example client. @@ -19,6 +30,7 @@ type Storage struct { name string workDir string service *drive.Service + cache *Cache defaultPairs DefaultStoragePairs features StorageFeatures @@ -55,6 +67,14 @@ func newStorager(pairs ...types.Pair) (store *Storage, err error) { name: opt.Name, workDir: "/", } + + // Init cache for storager + ch, err := initCache(numCounters, maxCost, bufferItems) + if err != nil { + return nil, err + } + store.cache = ch + if opt.HasWorkDir { store.workDir = opt.WorkDir } @@ -146,3 +166,30 @@ func (s *Storage) getFileName(path string) string { return path } } + +func initCache(nc int64, mc int64, bi int64) (cache *Cache, err error) { + config := &Config{ + NumCounters: nc, + MaxCost: mc, + BufferItems: bi, + } + + cache, err = NewCache(config) + + if err != nil { + return nil, err + } + return cache, nil +} + +func (s *Storage) setCache(path string, fileId string) { + s.cache.SetWithTTL(path, fileId, cost, expireTime) +} + +func (s *Storage) getCache(path string) (string, bool) { + id, found := s.cache.Get(path) + if found { + return id.(string), true + } + return "", false +}