diff --git a/generated.go b/generated.go index 48d8d9b..68a9fb0 100644 --- a/generated.go +++ b/generated.go @@ -29,6 +29,8 @@ const ( // ObjectMetadata stores service metadata for object. type ObjectMetadata struct { + // UploadSessionID a unique identifier for the upload session, refer to https://www.dropbox.com/developers/documentation/http/documentation#files-upload_session-start + UploadSessionID string } // GetObjectMetadata will get ObjectMetadata from Object. @@ -50,7 +52,8 @@ func setObjectMetadata(o *Object, om ObjectMetadata) { o.SetServiceMetadata(om) } -// WithDefaultStoragePairs will apply default_storage_pairs value to Options +// WithDefaultStoragePairs will apply default_storage_pairs value to Options. +// // DefaultStoragePairs set default pairs for storager actions func WithDefaultStoragePairs(v DefaultStoragePairs) Pair { return Pair{ @@ -130,13 +133,48 @@ func parsePairStorageNew(opts []Pair) (pairStorageNew, error) { // DefaultStoragePairs is default pairs for specific action type DefaultStoragePairs struct { - Create []Pair - Delete []Pair - List []Pair - Metadata []Pair - Read []Pair - Stat []Pair - Write []Pair + CommitAppend []Pair + Create []Pair + CreateAppend []Pair + Delete []Pair + List []Pair + Metadata []Pair + Read []Pair + Stat []Pair + Write []Pair + WriteAppend []Pair +} + +// pairStorageCommitAppend is the parsed struct +type pairStorageCommitAppend struct { + pairs []Pair + + // Required pairs + // Optional pairs + // Generated pairs +} + +// parsePairStorageCommitAppend will parse Pair slice into *pairStorageCommitAppend +func (s *Storage) parsePairStorageCommitAppend(opts []Pair) (pairStorageCommitAppend, error) { + result := pairStorageCommitAppend{ + pairs: opts, + } + + for _, v := range opts { + switch v.Key { + // Required pairs + // Optional pairs + // Generated pairs + default: + + if s.pairPolicy.All || s.pairPolicy.CommitAppend { + return pairStorageCommitAppend{}, services.NewPairUnsupportedError(v) + } + + } + } + + return result, nil } // pairStorageCreate is the parsed struct @@ -171,6 +209,38 @@ func (s *Storage) parsePairStorageCreate(opts []Pair) (pairStorageCreate, error) return result, nil } +// pairStorageCreateAppend is the parsed struct +type pairStorageCreateAppend struct { + pairs []Pair + + // Required pairs + // Optional pairs + // Generated pairs +} + +// parsePairStorageCreateAppend will parse Pair slice into *pairStorageCreateAppend +func (s *Storage) parsePairStorageCreateAppend(opts []Pair) (pairStorageCreateAppend, error) { + result := pairStorageCreateAppend{ + pairs: opts, + } + + for _, v := range opts { + switch v.Key { + // Required pairs + // Optional pairs + // Generated pairs + default: + + if s.pairPolicy.All || s.pairPolicy.CreateAppend { + return pairStorageCreateAppend{}, services.NewPairUnsupportedError(v) + } + + } + } + + return result, nil +} + // pairStorageDelete is the parsed struct type pairStorageDelete struct { pairs []Pair @@ -403,6 +473,63 @@ func (s *Storage) parsePairStorageWrite(opts []Pair) (pairStorageWrite, error) { return result, nil } +// pairStorageWriteAppend is the parsed struct +type pairStorageWriteAppend struct { + pairs []Pair + + // Required pairs + // Optional pairs + // Generated pairs +} + +// parsePairStorageWriteAppend will parse Pair slice into *pairStorageWriteAppend +func (s *Storage) parsePairStorageWriteAppend(opts []Pair) (pairStorageWriteAppend, error) { + result := pairStorageWriteAppend{ + pairs: opts, + } + + for _, v := range opts { + switch v.Key { + // Required pairs + // Optional pairs + // Generated pairs + default: + + if s.pairPolicy.All || s.pairPolicy.WriteAppend { + return pairStorageWriteAppend{}, services.NewPairUnsupportedError(v) + } + + } + } + + return result, nil +} + +// CommitAppend will commit and finish an append process. +// +// This function will create a context by default. +func (s *Storage) CommitAppend(o *Object, pairs ...Pair) (err error) { + ctx := context.Background() + return s.CommitAppendWithContext(ctx, o, pairs...) +} + +// CommitAppendWithContext will commit and finish an append process. +func (s *Storage) CommitAppendWithContext(ctx context.Context, o *Object, pairs ...Pair) (err error) { + pairs = append(pairs, s.defaultPairs.CommitAppend...) + var opt pairStorageCommitAppend + + defer func() { + err = s.formatError("commit_append", err) + }() + + opt, err = s.parsePairStorageCommitAppend(pairs) + if err != nil { + return + } + + return s.commitAppend(ctx, o, opt) +} + // Create will create a new object without any api call. // // This function will create a context by default. @@ -416,6 +543,31 @@ func (s *Storage) Create(path string, pairs ...Pair) (o *Object) { return s.create(path, opt) } +// CreateAppend will create an append object. +// +// This function will create a context by default. +func (s *Storage) CreateAppend(path string, pairs ...Pair) (o *Object, err error) { + ctx := context.Background() + return s.CreateAppendWithContext(ctx, path, pairs...) +} + +// CreateAppendWithContext will create an append object. +func (s *Storage) CreateAppendWithContext(ctx context.Context, path string, pairs ...Pair) (o *Object, err error) { + pairs = append(pairs, s.defaultPairs.CreateAppend...) + var opt pairStorageCreateAppend + + defer func() { + err = s.formatError("create_append", err, path) + }() + + opt, err = s.parsePairStorageCreateAppend(pairs) + if err != nil { + return + } + + return s.createAppend(ctx, path, opt) +} + // Delete will delete an Object from service. // // This function will create a context by default. @@ -565,3 +717,28 @@ func (s *Storage) WriteWithContext(ctx context.Context, path string, r io.Reader return s.write(ctx, path, r, size, opt) } + +// WriteAppend will append content to an append object. +// +// This function will create a context by default. +func (s *Storage) WriteAppend(o *Object, r io.Reader, size int64, pairs ...Pair) (n int64, err error) { + ctx := context.Background() + return s.WriteAppendWithContext(ctx, o, r, size, pairs...) +} + +// WriteAppendWithContext will append content to an append object. +func (s *Storage) WriteAppendWithContext(ctx context.Context, o *Object, r io.Reader, size int64, pairs ...Pair) (n int64, err error) { + pairs = append(pairs, s.defaultPairs.WriteAppend...) + var opt pairStorageWriteAppend + + defer func() { + err = s.formatError("write_append", err) + }() + + opt, err = s.parsePairStorageWriteAppend(pairs) + if err != nil { + return + } + + return s.writeAppend(ctx, o, r, size, opt) +} diff --git a/go.mod b/go.mod index d5b8261..8006e56 100644 --- a/go.mod +++ b/go.mod @@ -3,8 +3,8 @@ module github.com/aos-dev/go-service-dropbox go 1.14 require ( - github.com/aos-dev/go-integration-test/v3 v3.0.0 - github.com/aos-dev/go-storage/v3 v3.6.0 + github.com/aos-dev/go-integration-test/v3 v3.0.1-0.20210506085629-1ace601e00c6 + github.com/aos-dev/go-storage/v3 v3.6.1-0.20210427074731-1b3b4e86c950 github.com/dropbox/dropbox-sdk-go-unofficial v5.6.0+incompatible github.com/google/uuid v1.2.0 golang.org/x/oauth2 v0.0.0-20210413134643-5e61552d6c78 // indirect diff --git a/go.sum b/go.sum index ed1eac6..913074b 100644 --- a/go.sum +++ b/go.sum @@ -33,17 +33,14 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/Xuanwo/templateutils v0.0.0-20201216100309-46f73cd4e4b1/go.mod h1:x0qS7gfgEm24b8V9U+0zBEBAu/VpxZhO/+EWtpgEgDw= github.com/Xuanwo/templateutils v0.1.0 h1:WpkWOqQtIQ2vAIpJLa727DdN8WtxhUkkbDGa6UhntJY= github.com/Xuanwo/templateutils v0.1.0/go.mod h1:OdE0DJ+CJxDBq6psX5DPV+gOZi8bhuHuVUpPCG++Wb8= -github.com/aos-dev/go-integration-test/v3 v3.0.0 h1:rxIc7YBfiw9JS5JiVU4BZAqQqoKJemIftFaeJpYCb5M= -github.com/aos-dev/go-integration-test/v3 v3.0.0/go.mod h1:woC3E9Ld1G/Cpo2tSEQ+iwIr4MMdgf+L/8UuyJOE11Q= -github.com/aos-dev/go-storage/v3 v3.5.0/go.mod h1:PZJT0Ta7YxVM5QoYoh8Q/X4I6e/z/7gOJqm85Aib4nY= -github.com/aos-dev/go-storage/v3 v3.6.0 h1:ywjMvh320+esJH81MqB9nyuMNLW97Krujz2UiprC2ZM= -github.com/aos-dev/go-storage/v3 v3.6.0/go.mod h1:ZQwybmoCcTWUOWg+G15gT/NQJoI8G8KH1pF41TuJqYk= -github.com/aos-dev/specs/go v0.0.0-20210312090615-23109627848b/go.mod h1:XTNlLZtPA1inITyDH5hNnQXVjvvKUvo+lurs5GYB8NA= -github.com/aos-dev/specs/go v0.0.0-20210423110314-8361397c2bf3 h1:e65ozDhdfHfhnDpZF9SLcY5mwtAg/sAvNIUAkFd+4D0= -github.com/aos-dev/specs/go v0.0.0-20210423110314-8361397c2bf3/go.mod h1:gNah3KaPJEfysh7uCCX+sYjQC3g2yx2VgBkFlT945Ws= +github.com/aos-dev/go-integration-test/v3 v3.0.1-0.20210506085629-1ace601e00c6 h1:/GrJCqznHBjWWj3x2K5KU6hKG6BfR211bRJAFGdH7bc= +github.com/aos-dev/go-integration-test/v3 v3.0.1-0.20210506085629-1ace601e00c6/go.mod h1:D05/hz2AbZFBucYlzvpYBb62v17SgKUNhi3lxuUQLj4= +github.com/aos-dev/go-storage/v3 v3.6.1-0.20210427074731-1b3b4e86c950 h1:3ZsfgDAUF171rIrFdYjhr8P3WyxzR4IwxWGbGxkCRkA= +github.com/aos-dev/go-storage/v3 v3.6.1-0.20210427074731-1b3b4e86c950/go.mod h1:wzxhpUa73aCdTDRG0sHjNnnmXa9rC/ahJnk7hKOV6Pw= +github.com/aos-dev/specs/go v0.0.0-20210427074008-93183332d504 h1:vHxnQOXlz/X8/Co1ep57yc34Xso+f2+xt4Q2dMJVHA4= +github.com/aos-dev/specs/go v0.0.0-20210427074008-93183332d504/go.mod h1:gNah3KaPJEfysh7uCCX+sYjQC3g2yx2VgBkFlT945Ws= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= @@ -56,9 +53,8 @@ github.com/dave/gopackages v0.0.0-20170318123100-46e7023ec56e/go.mod h1:i00+b/gK github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= github.com/dave/kerr v0.0.0-20170318121727-bc25dd6abe8e/go.mod h1:qZqlPyPvfsDJt+3wHJ1EvSXDuVjFTK0j2p/ca+gtsb8= github.com/dave/rebecca v0.9.1/go.mod h1:N6XYdMD/OKw3lkF3ywh8Z6wPGuwNFDNtWYEMFWEmXBA= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dropbox/dropbox-sdk-go-unofficial v5.6.0+incompatible h1:DtumzkLk2zZ2SeElEr+VNz+zV7l+BTe509cV4sKPXbM= github.com/dropbox/dropbox-sdk-go-unofficial v5.6.0+incompatible/go.mod h1:lr+LhMM3F6Y3lW1T9j2U5l7QeuWm87N9+PPXo3yH4qY= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -137,7 +133,6 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= github.com/pelletier/go-toml v1.9.0 h1:NOd0BRdOKpPf0SxkL3HxSQOG7rNh+4kl6PHcBPFs7Q0= github.com/pelletier/go-toml v1.9.0/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= diff --git a/service.toml b/service.toml index ce5d7e3..b6830cb 100644 --- a/service.toml +++ b/service.toml @@ -2,6 +2,7 @@ name = "dropbox" [namespace.storage] +implement = ["appender"] [namespace.storage.new] required = ["credential"] @@ -21,4 +22,8 @@ optional = ["content_md5", "content_type", "io_callback"] [pairs.default_storage_pairs] type = "DefaultStoragePairs" -description = "set default pairs for storager actions" \ No newline at end of file +description = "set default pairs for storager actions" + +[infos.object.meta.upload-session-id] +type = "string" +description = "a unique identifier for the upload session, refer to https://www.dropbox.com/developers/documentation/http/documentation#files-upload_session-start" \ No newline at end of file diff --git a/storage.go b/storage.go index 206e62e..e1c89ee 100644 --- a/storage.go +++ b/storage.go @@ -2,6 +2,7 @@ package dropbox import ( "context" + "fmt" "io" "github.com/dropbox/dropbox-sdk-go-unofficial/dropbox" @@ -11,6 +12,54 @@ import ( . "github.com/aos-dev/go-storage/v3/types" ) +func (s *Storage) commitAppend(ctx context.Context, o *Object, opt pairStorageCommitAppend) (err error) { + if !o.Mode.IsAppend() { + err = fmt.Errorf("object not appendable") + return + } + + rp := o.GetID() + + offset, ok := o.GetAppendOffset() + if !ok { + err = fmt.Errorf("append offset is not set") + return + } + + sessionId := GetObjectMetadata(o).UploadSessionID + + cursor := &files.UploadSessionCursor{ + SessionId: sessionId, + Offset: uint64(offset), + } + + input := &files.CommitInfo{ + Path: rp, + Mode: &files.WriteMode{ + Tagged: dropbox.Tagged{ + Tag: files.WriteModeAdd, + }, + }, + } + + finishArg := &files.UploadSessionFinishArg{ + Cursor: cursor, + Commit: input, + } + + fileMetadata, err := s.client.UploadSessionFinish(finishArg, nil) + if err != nil { + return + } + + o.Mode &= ^ModeAppend + if fileMetadata != nil && fileMetadata.IsDownloadable { + o.Mode |= ModeRead + } + + return nil +} + func (s *Storage) create(path string, opt pairStorageCreate) (o *Object) { o = s.newObject(false) o.Mode = ModeRead @@ -19,6 +68,29 @@ func (s *Storage) create(path string, opt pairStorageCreate) (o *Object) { return o } +func (s *Storage) createAppend(ctx context.Context, path string, opt pairStorageCreateAppend) (o *Object, err error) { + startArg := &files.UploadSessionStartArg{ + Close: false, + } + + res, err := s.client.UploadSessionStart(startArg, nil) + if err != nil { + return + } + + sm := ObjectMetadata{ + UploadSessionID: res.SessionId, + } + + o = s.newObject(true) + o.Mode = ModeAppend + o.ID = s.getAbsPath(path) + o.Path = path + o.SetAppendOffset(0) + o.SetServiceMetadata(sm) + return o, nil +} + func (s *Storage) delete(ctx context.Context, path string, opt pairStorageDelete) (err error) { rp := s.getAbsPath(path) @@ -27,6 +99,10 @@ func (s *Storage) delete(ctx context.Context, path string, opt pairStorageDelete } _, err = s.client.DeleteV2(input) + if err != nil && checkError(err, files.DeleteErrorPathLookup, files.LookupErrorNotFound) { + // omit `path_lookup/not_found` error, ref: https://github.com/aos-dev/specs/blob/master/rfcs/46-idempotent-delete.md + err = nil + } if err != nil { return err } @@ -164,3 +240,34 @@ func (s *Storage) write(ctx context.Context, path string, r io.Reader, size int6 return size, nil } + +func (s *Storage) writeAppend(ctx context.Context, o *Object, r io.Reader, size int64, opt pairStorageWriteAppend) (n int64, err error) { + if !o.Mode.IsAppend() { + err = fmt.Errorf("object not appendable") + return + } + + sessionId := GetObjectMetadata(o).UploadSessionID + + offset := o.MustGetAppendOffset() + + cursor := &files.UploadSessionCursor{ + SessionId: sessionId, + Offset: uint64(offset), + } + + appendArg := &files.UploadSessionAppendArg{ + Cursor: cursor, + Close: false, + } + + err = s.client.UploadSessionAppendV2(appendArg, r) + if err != nil { + return + } + + offset += size + o.SetAppendOffset(offset) + + return size, nil +} diff --git a/tests/storage_test.go b/tests/storage_test.go index 7a823d1..e00d2a7 100644 --- a/tests/storage_test.go +++ b/tests/storage_test.go @@ -13,3 +13,10 @@ func TestStorage(t *testing.T) { } tests.TestStorager(t, setupTest(t)) } + +func TestAppend(t *testing.T) { + if os.Getenv("STORAGE_DROPBOX_INTEGRATION_TEST") != "on" { + t.Skipf("STORAGE_DROPBOX_INTEGRATION_TEST is not 'on', skipped") + } + tests.TestAppender(t, setupTest(t)) +} diff --git a/utils.go b/utils.go index dc80ef3..781a7d2 100644 --- a/utils.go +++ b/utils.go @@ -24,6 +24,7 @@ type Storage struct { pairPolicy typ.PairPolicy typ.UnimplementedStorager + typ.UnimplementedAppender } // String implements Storager.String @@ -110,6 +111,16 @@ func formatError(err error) error { } return err } + +func checkError(err error, codes ...string) bool { + var s strings.Builder + for _, code := range codes { + s.WriteString(code) + s.WriteString("/") + } + return strings.Contains(err.Error(), s.String()) +} + func (s *Storage) getAbsPath(path string) string { // Return workDir while input path is empty. if path == "" {