From af05fe769415f32eccbb25a9b4315e6c1a5cd36d Mon Sep 17 00:00:00 2001 From: JinnyYi Date: Tue, 27 Apr 2021 20:42:57 +0800 Subject: [PATCH 01/11] Add appender support --- generated.go | 193 ++++++++++++++++++++++++++++++++++++++++++++++++--- go.mod | 4 +- go.sum | 18 +++-- service.toml | 6 +- storage.go | 93 +++++++++++++++++++++++++ 5 files changed, 296 insertions(+), 18 deletions(-) diff --git a/generated.go b/generated.go index 48d8d9b..8a6e135 100644 --- a/generated.go +++ b/generated.go @@ -29,6 +29,8 @@ const ( // ObjectMetadata stores service metadata for object. type ObjectMetadata struct { + // UploadSessionID + UploadSessionID string } // GetObjectMetadata will get ObjectMetadata from Object. @@ -50,7 +52,8 @@ func setObjectMetadata(o *Object, om ObjectMetadata) { o.SetServiceMetadata(om) } -// WithDefaultStoragePairs will apply default_storage_pairs value to Options +// WithDefaultStoragePairs will apply default_storage_pairs value to Options. +// // DefaultStoragePairs set default pairs for storager actions func WithDefaultStoragePairs(v DefaultStoragePairs) Pair { return Pair{ @@ -130,13 +133,48 @@ func parsePairStorageNew(opts []Pair) (pairStorageNew, error) { // DefaultStoragePairs is default pairs for specific action type DefaultStoragePairs struct { - Create []Pair - Delete []Pair - List []Pair - Metadata []Pair - Read []Pair - Stat []Pair - Write []Pair + CommitAppend []Pair + Create []Pair + CreateAppend []Pair + Delete []Pair + List []Pair + Metadata []Pair + Read []Pair + Stat []Pair + Write []Pair + WriteAppend []Pair +} + +// pairStorageCommitAppend is the parsed struct +type pairStorageCommitAppend struct { + pairs []Pair + + // Required pairs + // Optional pairs + // Generated pairs +} + +// parsePairStorageCommitAppend will parse Pair slice into *pairStorageCommitAppend +func (s *Storage) parsePairStorageCommitAppend(opts []Pair) (pairStorageCommitAppend, error) { + result := pairStorageCommitAppend{ + pairs: opts, + } + + for _, v := range opts { + switch v.Key { + // Required pairs + // Optional pairs + // Generated pairs + default: + + if s.pairPolicy.All || s.pairPolicy.CommitAppend { + return pairStorageCommitAppend{}, services.NewPairUnsupportedError(v) + } + + } + } + + return result, nil } // pairStorageCreate is the parsed struct @@ -171,6 +209,38 @@ func (s *Storage) parsePairStorageCreate(opts []Pair) (pairStorageCreate, error) return result, nil } +// pairStorageCreateAppend is the parsed struct +type pairStorageCreateAppend struct { + pairs []Pair + + // Required pairs + // Optional pairs + // Generated pairs +} + +// parsePairStorageCreateAppend will parse Pair slice into *pairStorageCreateAppend +func (s *Storage) parsePairStorageCreateAppend(opts []Pair) (pairStorageCreateAppend, error) { + result := pairStorageCreateAppend{ + pairs: opts, + } + + for _, v := range opts { + switch v.Key { + // Required pairs + // Optional pairs + // Generated pairs + default: + + if s.pairPolicy.All || s.pairPolicy.CreateAppend { + return pairStorageCreateAppend{}, services.NewPairUnsupportedError(v) + } + + } + } + + return result, nil +} + // pairStorageDelete is the parsed struct type pairStorageDelete struct { pairs []Pair @@ -403,6 +473,63 @@ func (s *Storage) parsePairStorageWrite(opts []Pair) (pairStorageWrite, error) { return result, nil } +// pairStorageWriteAppend is the parsed struct +type pairStorageWriteAppend struct { + pairs []Pair + + // Required pairs + // Optional pairs + // Generated pairs +} + +// parsePairStorageWriteAppend will parse Pair slice into *pairStorageWriteAppend +func (s *Storage) parsePairStorageWriteAppend(opts []Pair) (pairStorageWriteAppend, error) { + result := pairStorageWriteAppend{ + pairs: opts, + } + + for _, v := range opts { + switch v.Key { + // Required pairs + // Optional pairs + // Generated pairs + default: + + if s.pairPolicy.All || s.pairPolicy.WriteAppend { + return pairStorageWriteAppend{}, services.NewPairUnsupportedError(v) + } + + } + } + + return result, nil +} + +// CommitAppend will commit and finish an append process. +// +// This function will create a context by default. +func (s *Storage) CommitAppend(o *Object, pairs ...Pair) (err error) { + ctx := context.Background() + return s.CommitAppendWithContext(ctx, o, pairs...) +} + +// CommitAppendWithContext will commit and finish an append process. +func (s *Storage) CommitAppendWithContext(ctx context.Context, o *Object, pairs ...Pair) (err error) { + pairs = append(pairs, s.defaultPairs.CommitAppend...) + var opt pairStorageCommitAppend + + defer func() { + err = s.formatError("commit_append", err) + }() + + opt, err = s.parsePairStorageCommitAppend(pairs) + if err != nil { + return + } + + return s.commitAppend(ctx, o, opt) +} + // Create will create a new object without any api call. // // This function will create a context by default. @@ -416,6 +543,31 @@ func (s *Storage) Create(path string, pairs ...Pair) (o *Object) { return s.create(path, opt) } +// CreateAppend will create an append object. +// +// This function will create a context by default. +func (s *Storage) CreateAppend(path string, pairs ...Pair) (o *Object, err error) { + ctx := context.Background() + return s.CreateAppendWithContext(ctx, path, pairs...) +} + +// CreateAppendWithContext will create an append object. +func (s *Storage) CreateAppendWithContext(ctx context.Context, path string, pairs ...Pair) (o *Object, err error) { + pairs = append(pairs, s.defaultPairs.CreateAppend...) + var opt pairStorageCreateAppend + + defer func() { + err = s.formatError("create_append", err, path) + }() + + opt, err = s.parsePairStorageCreateAppend(pairs) + if err != nil { + return + } + + return s.createAppend(ctx, path, opt) +} + // Delete will delete an Object from service. // // This function will create a context by default. @@ -565,3 +717,28 @@ func (s *Storage) WriteWithContext(ctx context.Context, path string, r io.Reader return s.write(ctx, path, r, size, opt) } + +// WriteAppend will append content to an append object. +// +// This function will create a context by default. +func (s *Storage) WriteAppend(o *Object, r io.Reader, size int64, pairs ...Pair) (n int64, err error) { + ctx := context.Background() + return s.WriteAppendWithContext(ctx, o, r, size, pairs...) +} + +// WriteAppendWithContext will append content to an append object. +func (s *Storage) WriteAppendWithContext(ctx context.Context, o *Object, r io.Reader, size int64, pairs ...Pair) (n int64, err error) { + pairs = append(pairs, s.defaultPairs.WriteAppend...) + var opt pairStorageWriteAppend + + defer func() { + err = s.formatError("write_append", err) + }() + + opt, err = s.parsePairStorageWriteAppend(pairs) + if err != nil { + return + } + + return s.writeAppend(ctx, o, r, size, opt) +} diff --git a/go.mod b/go.mod index d5b8261..a5e5828 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,8 @@ go 1.14 require ( github.com/aos-dev/go-integration-test/v3 v3.0.0 - github.com/aos-dev/go-storage/v3 v3.6.0 - github.com/dropbox/dropbox-sdk-go-unofficial v5.6.0+incompatible + github.com/aos-dev/go-storage/v3 v3.6.1-0.20210427074731-1b3b4e86c950 + github.com/dropbox/dropbox-sdk-go-unofficial v1.0.1-0.20210420014012-0798489e838f github.com/google/uuid v1.2.0 golang.org/x/oauth2 v0.0.0-20210413134643-5e61552d6c78 // indirect ) diff --git a/go.sum b/go.sum index ed1eac6..5e81788 100644 --- a/go.sum +++ b/go.sum @@ -39,11 +39,11 @@ github.com/Xuanwo/templateutils v0.1.0/go.mod h1:OdE0DJ+CJxDBq6psX5DPV+gOZi8bhuH github.com/aos-dev/go-integration-test/v3 v3.0.0 h1:rxIc7YBfiw9JS5JiVU4BZAqQqoKJemIftFaeJpYCb5M= github.com/aos-dev/go-integration-test/v3 v3.0.0/go.mod h1:woC3E9Ld1G/Cpo2tSEQ+iwIr4MMdgf+L/8UuyJOE11Q= github.com/aos-dev/go-storage/v3 v3.5.0/go.mod h1:PZJT0Ta7YxVM5QoYoh8Q/X4I6e/z/7gOJqm85Aib4nY= -github.com/aos-dev/go-storage/v3 v3.6.0 h1:ywjMvh320+esJH81MqB9nyuMNLW97Krujz2UiprC2ZM= -github.com/aos-dev/go-storage/v3 v3.6.0/go.mod h1:ZQwybmoCcTWUOWg+G15gT/NQJoI8G8KH1pF41TuJqYk= +github.com/aos-dev/go-storage/v3 v3.6.1-0.20210427074731-1b3b4e86c950 h1:3ZsfgDAUF171rIrFdYjhr8P3WyxzR4IwxWGbGxkCRkA= +github.com/aos-dev/go-storage/v3 v3.6.1-0.20210427074731-1b3b4e86c950/go.mod h1:wzxhpUa73aCdTDRG0sHjNnnmXa9rC/ahJnk7hKOV6Pw= github.com/aos-dev/specs/go v0.0.0-20210312090615-23109627848b/go.mod h1:XTNlLZtPA1inITyDH5hNnQXVjvvKUvo+lurs5GYB8NA= -github.com/aos-dev/specs/go v0.0.0-20210423110314-8361397c2bf3 h1:e65ozDhdfHfhnDpZF9SLcY5mwtAg/sAvNIUAkFd+4D0= -github.com/aos-dev/specs/go v0.0.0-20210423110314-8361397c2bf3/go.mod h1:gNah3KaPJEfysh7uCCX+sYjQC3g2yx2VgBkFlT945Ws= +github.com/aos-dev/specs/go v0.0.0-20210427074008-93183332d504 h1:vHxnQOXlz/X8/Co1ep57yc34Xso+f2+xt4Q2dMJVHA4= +github.com/aos-dev/specs/go v0.0.0-20210427074008-93183332d504/go.mod h1:gNah3KaPJEfysh7uCCX+sYjQC3g2yx2VgBkFlT945Ws= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= @@ -59,8 +59,8 @@ github.com/dave/rebecca v0.9.1/go.mod h1:N6XYdMD/OKw3lkF3ywh8Z6wPGuwNFDNtWYEMFWE github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dropbox/dropbox-sdk-go-unofficial v5.6.0+incompatible h1:DtumzkLk2zZ2SeElEr+VNz+zV7l+BTe509cV4sKPXbM= -github.com/dropbox/dropbox-sdk-go-unofficial v5.6.0+incompatible/go.mod h1:lr+LhMM3F6Y3lW1T9j2U5l7QeuWm87N9+PPXo3yH4qY= +github.com/dropbox/dropbox-sdk-go-unofficial v1.0.1-0.20210420014012-0798489e838f h1:AJuoBQCZiKU7zeMNImaM7JZFrElyVgZAKkmdTtnOXW8= +github.com/dropbox/dropbox-sdk-go-unofficial v1.0.1-0.20210420014012-0798489e838f/go.mod h1:6zG+Yst2Q7BA8rp69tmHlCnt7BxeCyj3rno0B7hYq8k= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -224,13 +224,15 @@ golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b h1:iFwSg7t5GZmB/Q5TjiEAsdoLDrdJRC1RiF2WhuV29Qw= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210413134643-5e61552d6c78 h1:rPRtHfUb0UKZeZ6GH4K4Nt4YRbE9V1u+QZX5upZXqJQ= golang.org/x/oauth2 v0.0.0-20210413134643-5e61552d6c78/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -267,6 +269,8 @@ golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/service.toml b/service.toml index ce5d7e3..0931300 100644 --- a/service.toml +++ b/service.toml @@ -2,6 +2,7 @@ name = "dropbox" [namespace.storage] +implement = ["appender"] [namespace.storage.new] required = ["credential"] @@ -21,4 +22,7 @@ optional = ["content_md5", "content_type", "io_callback"] [pairs.default_storage_pairs] type = "DefaultStoragePairs" -description = "set default pairs for storager actions" \ No newline at end of file +description = "set default pairs for storager actions" + +[infos.object.meta.upload-session-id] +type = "string" \ No newline at end of file diff --git a/storage.go b/storage.go index 206e62e..c2973ba 100644 --- a/storage.go +++ b/storage.go @@ -2,6 +2,7 @@ package dropbox import ( "context" + "fmt" "io" "github.com/dropbox/dropbox-sdk-go-unofficial/dropbox" @@ -11,6 +12,41 @@ import ( . "github.com/aos-dev/go-storage/v3/types" ) +func (s *Storage) commitAppend(ctx context.Context, o *Object, opt pairStorageCommitAppend) (err error) { + rp := o.GetID() + + offset, ok := o.GetAppendOffset() + if !ok { + err = fmt.Errorf("append offset is not set") + return + } + + sessionId := GetObjectMetadata(o).UploadSessionID + + cursor := &files.UploadSessionCursor{ + SessionId: sessionId, + Offset: uint64(offset), + } + + input := &files.CommitInfo{ + Path: rp, + Mode: &files.WriteMode{ + Tagged: dropbox.Tagged{ + Tag: files.WriteModeAdd, + }, + }, + } + + finishArg := &files.UploadSessionFinishArg{ + Cursor: cursor, + Commit: input, + } + + _, err = s.client.UploadSessionFinish(finishArg, nil) + + return err +} + func (s *Storage) create(path string, opt pairStorageCreate) (o *Object) { o = s.newObject(false) o.Mode = ModeRead @@ -19,6 +55,33 @@ func (s *Storage) create(path string, opt pairStorageCreate) (o *Object) { return o } +func (s *Storage) createAppend(ctx context.Context, path string, opt pairStorageCreateAppend) (o *Object, err error) { + startArg := &files.UploadSessionStartArg{ + Close: false, + SessionType: &files.UploadSessionType{ + Tagged: dropbox.Tagged{ + Tag: files.UploadSessionTypeSequential, + }, + }, + } + + res, err := s.client.UploadSessionStart(startArg, nil) + if err != nil { + return + } + + sm := ObjectMetadata{ + UploadSessionID: res.SessionId, + } + + o = s.newObject(true) + o.Mode = ModeRead | ModeAppend + o.ID = s.getAbsPath(path) + o.Path = path + o.SetServiceMetadata(sm) + return o, nil +} + func (s *Storage) delete(ctx context.Context, path string, opt pairStorageDelete) (err error) { rp := s.getAbsPath(path) @@ -164,3 +227,33 @@ func (s *Storage) write(ctx context.Context, path string, r io.Reader, size int6 return size, nil } + +func (s *Storage) writeAppend(ctx context.Context, o *Object, r io.Reader, size int64, opt pairStorageWriteAppend) (n int64, err error) { + sessionId := GetObjectMetadata(o).UploadSessionID + + offset, ok := o.GetAppendOffset() + if !ok { + err = fmt.Errorf("append offset is not set") + return + } + + cursor := &files.UploadSessionCursor{ + SessionId: sessionId, + Offset: uint64(offset), + } + + appendArg := &files.UploadSessionAppendArg{ + Cursor: cursor, + Close: false, + } + + err = s.client.UploadSessionAppendV2(appendArg, r) + if err != nil { + return + } + + offset += size + o.SetAppendOffset(offset) + + return offset, nil +} From 612dc3f278cf07c3a85ea5f4709ce808448001ef Mon Sep 17 00:00:00 2001 From: JinnyYi Date: Wed, 28 Apr 2021 15:59:45 +0800 Subject: [PATCH 02/11] add object mode check and metadata description --- generated.go | 2 +- service.toml | 3 ++- storage.go | 17 ++++++++++++++++- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/generated.go b/generated.go index 8a6e135..68a9fb0 100644 --- a/generated.go +++ b/generated.go @@ -29,7 +29,7 @@ const ( // ObjectMetadata stores service metadata for object. type ObjectMetadata struct { - // UploadSessionID + // UploadSessionID a unique identifier for the upload session, refer to https://www.dropbox.com/developers/documentation/http/documentation#files-upload_session-start UploadSessionID string } diff --git a/service.toml b/service.toml index 0931300..b6830cb 100644 --- a/service.toml +++ b/service.toml @@ -25,4 +25,5 @@ type = "DefaultStoragePairs" description = "set default pairs for storager actions" [infos.object.meta.upload-session-id] -type = "string" \ No newline at end of file +type = "string" +description = "a unique identifier for the upload session, refer to https://www.dropbox.com/developers/documentation/http/documentation#files-upload_session-start" \ No newline at end of file diff --git a/storage.go b/storage.go index c2973ba..ab01b0e 100644 --- a/storage.go +++ b/storage.go @@ -13,6 +13,11 @@ import ( ) func (s *Storage) commitAppend(ctx context.Context, o *Object, opt pairStorageCommitAppend) (err error) { + if !o.Mode.IsAppend() { + err = fmt.Errorf("object not appendable") + return + } + rp := o.GetID() offset, ok := o.GetAppendOffset() @@ -70,12 +75,17 @@ func (s *Storage) createAppend(ctx context.Context, path string, opt pairStorage return } + if res == nil { + err = fmt.Errorf("upload session start response is nil") + return + } + sm := ObjectMetadata{ UploadSessionID: res.SessionId, } o = s.newObject(true) - o.Mode = ModeRead | ModeAppend + o.Mode = ModeAppend o.ID = s.getAbsPath(path) o.Path = path o.SetServiceMetadata(sm) @@ -229,6 +239,11 @@ func (s *Storage) write(ctx context.Context, path string, r io.Reader, size int6 } func (s *Storage) writeAppend(ctx context.Context, o *Object, r io.Reader, size int64, opt pairStorageWriteAppend) (n int64, err error) { + if !o.Mode.IsAppend() { + err = fmt.Errorf("object not appendable") + return + } + sessionId := GetObjectMetadata(o).UploadSessionID offset, ok := o.GetAppendOffset() From fc8562a771586021551c0093e50a4de739801e67 Mon Sep 17 00:00:00 2001 From: JinnyYi Date: Thu, 29 Apr 2021 18:25:24 +0800 Subject: [PATCH 03/11] fix the return value of writeAppend and object mode after commitAppend --- go.mod | 4 ++-- go.sum | 21 ++++++--------------- storage.go | 16 +++++++++------- tests/storage_test.go | 7 +++++++ 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/go.mod b/go.mod index a5e5828..c7020a8 100644 --- a/go.mod +++ b/go.mod @@ -3,9 +3,9 @@ module github.com/aos-dev/go-service-dropbox go 1.14 require ( - github.com/aos-dev/go-integration-test/v3 v3.0.0 + github.com/aos-dev/go-integration-test/v3 v3.0.1-0.20210429074249-b440e34f0bc1 github.com/aos-dev/go-storage/v3 v3.6.1-0.20210427074731-1b3b4e86c950 - github.com/dropbox/dropbox-sdk-go-unofficial v1.0.1-0.20210420014012-0798489e838f + github.com/dropbox/dropbox-sdk-go-unofficial v5.6.0+incompatible github.com/google/uuid v1.2.0 golang.org/x/oauth2 v0.0.0-20210413134643-5e61552d6c78 // indirect ) diff --git a/go.sum b/go.sum index 5e81788..a6b4b71 100644 --- a/go.sum +++ b/go.sum @@ -33,15 +33,12 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/Xuanwo/templateutils v0.0.0-20201216100309-46f73cd4e4b1/go.mod h1:x0qS7gfgEm24b8V9U+0zBEBAu/VpxZhO/+EWtpgEgDw= github.com/Xuanwo/templateutils v0.1.0 h1:WpkWOqQtIQ2vAIpJLa727DdN8WtxhUkkbDGa6UhntJY= github.com/Xuanwo/templateutils v0.1.0/go.mod h1:OdE0DJ+CJxDBq6psX5DPV+gOZi8bhuHuVUpPCG++Wb8= -github.com/aos-dev/go-integration-test/v3 v3.0.0 h1:rxIc7YBfiw9JS5JiVU4BZAqQqoKJemIftFaeJpYCb5M= -github.com/aos-dev/go-integration-test/v3 v3.0.0/go.mod h1:woC3E9Ld1G/Cpo2tSEQ+iwIr4MMdgf+L/8UuyJOE11Q= -github.com/aos-dev/go-storage/v3 v3.5.0/go.mod h1:PZJT0Ta7YxVM5QoYoh8Q/X4I6e/z/7gOJqm85Aib4nY= +github.com/aos-dev/go-integration-test/v3 v3.0.1-0.20210429074249-b440e34f0bc1 h1:72qOmWtSzfKVl1lD0aDxstjWnk3dH11OM4SDK9EDKMQ= +github.com/aos-dev/go-integration-test/v3 v3.0.1-0.20210429074249-b440e34f0bc1/go.mod h1:D05/hz2AbZFBucYlzvpYBb62v17SgKUNhi3lxuUQLj4= github.com/aos-dev/go-storage/v3 v3.6.1-0.20210427074731-1b3b4e86c950 h1:3ZsfgDAUF171rIrFdYjhr8P3WyxzR4IwxWGbGxkCRkA= github.com/aos-dev/go-storage/v3 v3.6.1-0.20210427074731-1b3b4e86c950/go.mod h1:wzxhpUa73aCdTDRG0sHjNnnmXa9rC/ahJnk7hKOV6Pw= -github.com/aos-dev/specs/go v0.0.0-20210312090615-23109627848b/go.mod h1:XTNlLZtPA1inITyDH5hNnQXVjvvKUvo+lurs5GYB8NA= github.com/aos-dev/specs/go v0.0.0-20210427074008-93183332d504 h1:vHxnQOXlz/X8/Co1ep57yc34Xso+f2+xt4Q2dMJVHA4= github.com/aos-dev/specs/go v0.0.0-20210427074008-93183332d504/go.mod h1:gNah3KaPJEfysh7uCCX+sYjQC3g2yx2VgBkFlT945Ws= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -56,11 +53,10 @@ github.com/dave/gopackages v0.0.0-20170318123100-46e7023ec56e/go.mod h1:i00+b/gK github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= github.com/dave/kerr v0.0.0-20170318121727-bc25dd6abe8e/go.mod h1:qZqlPyPvfsDJt+3wHJ1EvSXDuVjFTK0j2p/ca+gtsb8= github.com/dave/rebecca v0.9.1/go.mod h1:N6XYdMD/OKw3lkF3ywh8Z6wPGuwNFDNtWYEMFWEmXBA= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dropbox/dropbox-sdk-go-unofficial v1.0.1-0.20210420014012-0798489e838f h1:AJuoBQCZiKU7zeMNImaM7JZFrElyVgZAKkmdTtnOXW8= -github.com/dropbox/dropbox-sdk-go-unofficial v1.0.1-0.20210420014012-0798489e838f/go.mod h1:6zG+Yst2Q7BA8rp69tmHlCnt7BxeCyj3rno0B7hYq8k= +github.com/dropbox/dropbox-sdk-go-unofficial v5.6.0+incompatible h1:DtumzkLk2zZ2SeElEr+VNz+zV7l+BTe509cV4sKPXbM= +github.com/dropbox/dropbox-sdk-go-unofficial v5.6.0+incompatible/go.mod h1:lr+LhMM3F6Y3lW1T9j2U5l7QeuWm87N9+PPXo3yH4qY= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -137,7 +133,6 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= github.com/pelletier/go-toml v1.9.0 h1:NOd0BRdOKpPf0SxkL3HxSQOG7rNh+4kl6PHcBPFs7Q0= github.com/pelletier/go-toml v1.9.0/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -224,15 +219,13 @@ golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201224014010-6772e930b67b h1:iFwSg7t5GZmB/Q5TjiEAsdoLDrdJRC1RiF2WhuV29Qw= -golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210413134643-5e61552d6c78 h1:rPRtHfUb0UKZeZ6GH4K4Nt4YRbE9V1u+QZX5upZXqJQ= golang.org/x/oauth2 v0.0.0-20210413134643-5e61552d6c78/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -269,8 +262,6 @@ golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/storage.go b/storage.go index ab01b0e..10c1ffd 100644 --- a/storage.go +++ b/storage.go @@ -47,7 +47,14 @@ func (s *Storage) commitAppend(ctx context.Context, o *Object, opt pairStorageCo Commit: input, } - _, err = s.client.UploadSessionFinish(finishArg, nil) + fileMetadata, err := s.client.UploadSessionFinish(finishArg, nil) + + if err == nil { + o.Mode &= ^ModeAppend + if fileMetadata != nil && fileMetadata.IsDownloadable { + o.Mode |= ModeRead + } + } return err } @@ -63,11 +70,6 @@ func (s *Storage) create(path string, opt pairStorageCreate) (o *Object) { func (s *Storage) createAppend(ctx context.Context, path string, opt pairStorageCreateAppend) (o *Object, err error) { startArg := &files.UploadSessionStartArg{ Close: false, - SessionType: &files.UploadSessionType{ - Tagged: dropbox.Tagged{ - Tag: files.UploadSessionTypeSequential, - }, - }, } res, err := s.client.UploadSessionStart(startArg, nil) @@ -270,5 +272,5 @@ func (s *Storage) writeAppend(ctx context.Context, o *Object, r io.Reader, size offset += size o.SetAppendOffset(offset) - return offset, nil + return size, nil } diff --git a/tests/storage_test.go b/tests/storage_test.go index 7a823d1..e00d2a7 100644 --- a/tests/storage_test.go +++ b/tests/storage_test.go @@ -13,3 +13,10 @@ func TestStorage(t *testing.T) { } tests.TestStorager(t, setupTest(t)) } + +func TestAppend(t *testing.T) { + if os.Getenv("STORAGE_DROPBOX_INTEGRATION_TEST") != "on" { + t.Skipf("STORAGE_DROPBOX_INTEGRATION_TEST is not 'on', skipped") + } + tests.TestAppender(t, setupTest(t)) +} From 4775dafd64ae211cc7ce9f6f0f5edf82cbb9de27 Mon Sep 17 00:00:00 2001 From: JinnyYi Date: Thu, 29 Apr 2021 18:33:19 +0800 Subject: [PATCH 04/11] add UnimplementAppender --- utils.go | 1 + 1 file changed, 1 insertion(+) diff --git a/utils.go b/utils.go index dc80ef3..d63d2b8 100644 --- a/utils.go +++ b/utils.go @@ -24,6 +24,7 @@ type Storage struct { pairPolicy typ.PairPolicy typ.UnimplementedStorager + typ.UnimplementedAppender } // String implements Storager.String From d0beb894b02186ff154c6dc908e024c11e125662 Mon Sep 17 00:00:00 2001 From: JinnyYi Date: Thu, 29 Apr 2021 18:48:41 +0800 Subject: [PATCH 05/11] set append offset in createAppend --- storage.go | 1 + 1 file changed, 1 insertion(+) diff --git a/storage.go b/storage.go index 10c1ffd..7920707 100644 --- a/storage.go +++ b/storage.go @@ -90,6 +90,7 @@ func (s *Storage) createAppend(ctx context.Context, path string, opt pairStorage o.Mode = ModeAppend o.ID = s.getAbsPath(path) o.Path = path + o.SetAppendOffset(0) o.SetServiceMetadata(sm) return o, nil } From eaa8a7d54b5069e5f693f8396837734058352ada Mon Sep 17 00:00:00 2001 From: JinnyYi Date: Thu, 6 May 2021 23:04:40 +0800 Subject: [PATCH 06/11] Idempotent storager delete operation --- go.mod | 2 +- go.sum | 4 ++-- storage.go | 5 +++++ 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index c7020a8..8006e56 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/aos-dev/go-service-dropbox go 1.14 require ( - github.com/aos-dev/go-integration-test/v3 v3.0.1-0.20210429074249-b440e34f0bc1 + github.com/aos-dev/go-integration-test/v3 v3.0.1-0.20210506085629-1ace601e00c6 github.com/aos-dev/go-storage/v3 v3.6.1-0.20210427074731-1b3b4e86c950 github.com/dropbox/dropbox-sdk-go-unofficial v5.6.0+incompatible github.com/google/uuid v1.2.0 diff --git a/go.sum b/go.sum index a6b4b71..913074b 100644 --- a/go.sum +++ b/go.sum @@ -35,8 +35,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Xuanwo/templateutils v0.1.0 h1:WpkWOqQtIQ2vAIpJLa727DdN8WtxhUkkbDGa6UhntJY= github.com/Xuanwo/templateutils v0.1.0/go.mod h1:OdE0DJ+CJxDBq6psX5DPV+gOZi8bhuHuVUpPCG++Wb8= -github.com/aos-dev/go-integration-test/v3 v3.0.1-0.20210429074249-b440e34f0bc1 h1:72qOmWtSzfKVl1lD0aDxstjWnk3dH11OM4SDK9EDKMQ= -github.com/aos-dev/go-integration-test/v3 v3.0.1-0.20210429074249-b440e34f0bc1/go.mod h1:D05/hz2AbZFBucYlzvpYBb62v17SgKUNhi3lxuUQLj4= +github.com/aos-dev/go-integration-test/v3 v3.0.1-0.20210506085629-1ace601e00c6 h1:/GrJCqznHBjWWj3x2K5KU6hKG6BfR211bRJAFGdH7bc= +github.com/aos-dev/go-integration-test/v3 v3.0.1-0.20210506085629-1ace601e00c6/go.mod h1:D05/hz2AbZFBucYlzvpYBb62v17SgKUNhi3lxuUQLj4= github.com/aos-dev/go-storage/v3 v3.6.1-0.20210427074731-1b3b4e86c950 h1:3ZsfgDAUF171rIrFdYjhr8P3WyxzR4IwxWGbGxkCRkA= github.com/aos-dev/go-storage/v3 v3.6.1-0.20210427074731-1b3b4e86c950/go.mod h1:wzxhpUa73aCdTDRG0sHjNnnmXa9rC/ahJnk7hKOV6Pw= github.com/aos-dev/specs/go v0.0.0-20210427074008-93183332d504 h1:vHxnQOXlz/X8/Co1ep57yc34Xso+f2+xt4Q2dMJVHA4= diff --git a/storage.go b/storage.go index 7920707..337828c 100644 --- a/storage.go +++ b/storage.go @@ -104,6 +104,11 @@ func (s *Storage) delete(ctx context.Context, path string, opt pairStorageDelete _, err = s.client.DeleteV2(input) if err != nil { + if deleteErr, ok := err.(files.DeleteV2APIError); ok { + if deleteErr.EndpointError.PathLookup.Tag == files.LookupErrorNotFound { + return nil + } + } return err } From 147637c05c9031a50e5be81de250095a6d5e6617 Mon Sep 17 00:00:00 2001 From: JinnyYi Date: Fri, 7 May 2021 11:29:22 +0800 Subject: [PATCH 07/11] modify delete error type --- storage.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage.go b/storage.go index 337828c..182d494 100644 --- a/storage.go +++ b/storage.go @@ -104,7 +104,7 @@ func (s *Storage) delete(ctx context.Context, path string, opt pairStorageDelete _, err = s.client.DeleteV2(input) if err != nil { - if deleteErr, ok := err.(files.DeleteV2APIError); ok { + if deleteErr, ok := err.(files.DeleteAPIError); ok { if deleteErr.EndpointError.PathLookup.Tag == files.LookupErrorNotFound { return nil } From 3966191d02d9e46f511ed6388fc268c7a8db937a Mon Sep 17 00:00:00 2001 From: JinnyYi Date: Fri, 7 May 2021 14:49:02 +0800 Subject: [PATCH 08/11] modify the delete error check --- storage.go | 8 +++----- utils.go | 5 +++++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/storage.go b/storage.go index 182d494..f997742 100644 --- a/storage.go +++ b/storage.go @@ -104,12 +104,10 @@ func (s *Storage) delete(ctx context.Context, path string, opt pairStorageDelete _, err = s.client.DeleteV2(input) if err != nil { - if deleteErr, ok := err.(files.DeleteAPIError); ok { - if deleteErr.EndpointError.PathLookup.Tag == files.LookupErrorNotFound { - return nil - } + code := files.DeleteErrorPathLookup + "/" + files.LookupErrorNotFound + if ok := checkError(err, code); !ok { + return err } - return err } return nil diff --git a/utils.go b/utils.go index d63d2b8..ec5156c 100644 --- a/utils.go +++ b/utils.go @@ -111,6 +111,11 @@ func formatError(err error) error { } return err } + +func checkError(err error, code string) bool { + return strings.Contains(err.Error(), code) +} + func (s *Storage) getAbsPath(path string) string { // Return workDir while input path is empty. if path == "" { From b8e8f6405f30e2bc0f67f504b106c2a393cc2900 Mon Sep 17 00:00:00 2001 From: JinnyYi Date: Fri, 7 May 2021 15:32:02 +0800 Subject: [PATCH 09/11] modify the delete error check --- storage.go | 3 +-- utils.go | 9 +++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/storage.go b/storage.go index f997742..9a87ae2 100644 --- a/storage.go +++ b/storage.go @@ -104,8 +104,7 @@ func (s *Storage) delete(ctx context.Context, path string, opt pairStorageDelete _, err = s.client.DeleteV2(input) if err != nil { - code := files.DeleteErrorPathLookup + "/" + files.LookupErrorNotFound - if ok := checkError(err, code); !ok { + if !checkError(err, files.DeleteErrorPathLookup, files.LookupErrorNotFound) { return err } } diff --git a/utils.go b/utils.go index ec5156c..e242103 100644 --- a/utils.go +++ b/utils.go @@ -112,8 +112,13 @@ func formatError(err error) error { return err } -func checkError(err error, code string) bool { - return strings.Contains(err.Error(), code) +func checkError(err error, codes ...string) bool { + var errMsg strings.Builder + for _, code := range codes { + errMsg.WriteString(code) + errMsg.WriteString("/") + } + return strings.Contains(err.Error(), errMsg.String()) } func (s *Storage) getAbsPath(path string) string { From d3a35a6dce97255a2c8c183006266adbac91caae Mon Sep 17 00:00:00 2001 From: JinnyYi Date: Fri, 7 May 2021 16:05:29 +0800 Subject: [PATCH 10/11] modify the appender support --- storage.go | 24 ++++++++---------------- utils.go | 8 ++++---- 2 files changed, 12 insertions(+), 20 deletions(-) diff --git a/storage.go b/storage.go index 9a87ae2..20a835e 100644 --- a/storage.go +++ b/storage.go @@ -48,15 +48,16 @@ func (s *Storage) commitAppend(ctx context.Context, o *Object, opt pairStorageCo } fileMetadata, err := s.client.UploadSessionFinish(finishArg, nil) + if err != nil { + return + } - if err == nil { - o.Mode &= ^ModeAppend - if fileMetadata != nil && fileMetadata.IsDownloadable { - o.Mode |= ModeRead - } + o.Mode &= ^ModeAppend + if fileMetadata != nil && fileMetadata.IsDownloadable { + o.Mode |= ModeRead } - return err + return nil } func (s *Storage) create(path string, opt pairStorageCreate) (o *Object) { @@ -77,11 +78,6 @@ func (s *Storage) createAppend(ctx context.Context, path string, opt pairStorage return } - if res == nil { - err = fmt.Errorf("upload session start response is nil") - return - } - sm := ObjectMetadata{ UploadSessionID: res.SessionId, } @@ -251,11 +247,7 @@ func (s *Storage) writeAppend(ctx context.Context, o *Object, r io.Reader, size sessionId := GetObjectMetadata(o).UploadSessionID - offset, ok := o.GetAppendOffset() - if !ok { - err = fmt.Errorf("append offset is not set") - return - } + offset := o.MustGetAppendOffset() cursor := &files.UploadSessionCursor{ SessionId: sessionId, diff --git a/utils.go b/utils.go index e242103..781a7d2 100644 --- a/utils.go +++ b/utils.go @@ -113,12 +113,12 @@ func formatError(err error) error { } func checkError(err error, codes ...string) bool { - var errMsg strings.Builder + var s strings.Builder for _, code := range codes { - errMsg.WriteString(code) - errMsg.WriteString("/") + s.WriteString(code) + s.WriteString("/") } - return strings.Contains(err.Error(), errMsg.String()) + return strings.Contains(err.Error(), s.String()) } func (s *Storage) getAbsPath(path string) string { From 63a5e6a00b12bf60af2c3e876415f10935d76f38 Mon Sep 17 00:00:00 2001 From: JinnyYi Date: Fri, 7 May 2021 17:27:00 +0800 Subject: [PATCH 11/11] modify delete error check --- storage.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/storage.go b/storage.go index 20a835e..e1c89ee 100644 --- a/storage.go +++ b/storage.go @@ -99,10 +99,12 @@ func (s *Storage) delete(ctx context.Context, path string, opt pairStorageDelete } _, err = s.client.DeleteV2(input) + if err != nil && checkError(err, files.DeleteErrorPathLookup, files.LookupErrorNotFound) { + // omit `path_lookup/not_found` error, ref: https://github.com/aos-dev/specs/blob/master/rfcs/46-idempotent-delete.md + err = nil + } if err != nil { - if !checkError(err, files.DeleteErrorPathLookup, files.LookupErrorNotFound) { - return err - } + return err } return nil