storage: Implement CommitAppend (#18)

* Implement commitAppend * add append related metadata Co-authored-by: JinnyYi <[email protected]>
beyondstorage · Apr 29, 2021 · 6b4ecb1 · 6b4ecb1
1 parent e7aa31a
commit 6b4ecb1
Show file tree

Hide file tree

Showing 5 changed files with 156 additions and 37 deletions.
diff --git a/generated.go b/generated.go
diff --git a/go.mod b/go.mod
@@ -5,7 +5,7 @@ go 1.14
 require (
 	github.com/aliyun/aliyun-oss-go-sdk v2.1.8+incompatible
 	github.com/aos-dev/go-integration-test/v3 v3.0.0
-	github.com/aos-dev/go-storage/v3 v3.6.0
+	github.com/aos-dev/go-storage/v3 v3.6.1-0.20210427074731-1b3b4e86c950
 	github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect
 	github.com/google/uuid v1.2.0
 	github.com/satori/go.uuid v1.2.0 // indirect

diff --git a/go.sum b/go.sum
@@ -6,11 +6,11 @@ github.com/aliyun/aliyun-oss-go-sdk v2.1.8+incompatible/go.mod h1:T/Aws4fEfogEE9
 github.com/aos-dev/go-integration-test/v3 v3.0.0 h1:rxIc7YBfiw9JS5JiVU4BZAqQqoKJemIftFaeJpYCb5M=
 github.com/aos-dev/go-integration-test/v3 v3.0.0/go.mod h1:woC3E9Ld1G/Cpo2tSEQ+iwIr4MMdgf+L/8UuyJOE11Q=
 github.com/aos-dev/go-storage/v3 v3.5.0/go.mod h1:PZJT0Ta7YxVM5QoYoh8Q/X4I6e/z/7gOJqm85Aib4nY=
-github.com/aos-dev/go-storage/v3 v3.6.0 h1:ywjMvh320+esJH81MqB9nyuMNLW97Krujz2UiprC2ZM=
-github.com/aos-dev/go-storage/v3 v3.6.0/go.mod h1:ZQwybmoCcTWUOWg+G15gT/NQJoI8G8KH1pF41TuJqYk=
+github.com/aos-dev/go-storage/v3 v3.6.1-0.20210427074731-1b3b4e86c950 h1:3ZsfgDAUF171rIrFdYjhr8P3WyxzR4IwxWGbGxkCRkA=
+github.com/aos-dev/go-storage/v3 v3.6.1-0.20210427074731-1b3b4e86c950/go.mod h1:wzxhpUa73aCdTDRG0sHjNnnmXa9rC/ahJnk7hKOV6Pw=
 github.com/aos-dev/specs/go v0.0.0-20210312090615-23109627848b/go.mod h1:XTNlLZtPA1inITyDH5hNnQXVjvvKUvo+lurs5GYB8NA=
-github.com/aos-dev/specs/go v0.0.0-20210423110314-8361397c2bf3 h1:e65ozDhdfHfhnDpZF9SLcY5mwtAg/sAvNIUAkFd+4D0=
-github.com/aos-dev/specs/go v0.0.0-20210423110314-8361397c2bf3/go.mod h1:gNah3KaPJEfysh7uCCX+sYjQC3g2yx2VgBkFlT945Ws=
+github.com/aos-dev/specs/go v0.0.0-20210427074008-93183332d504 h1:vHxnQOXlz/X8/Co1ep57yc34Xso+f2+xt4Q2dMJVHA4=
+github.com/aos-dev/specs/go v0.0.0-20210427074008-93183332d504/go.mod h1:gNah3KaPJEfysh7uCCX+sYjQC3g2yx2VgBkFlT945Ws=
 github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f h1:ZNv7On9kyUzm7fvRZumSyy/IUiSC7AzL0I1jKKtwooA=
 github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f/go.mod h1:AuiFmCCPBSrqvVMvuqFuk0qogytodnVFVSN5CeJB8Gc=
 github.com/dave/dst v0.26.2 h1:lnxLAKI3tx7MgLNVDirFCsDTlTG9nKTk7GcptKcWSwY=

diff --git a/service.toml b/service.toml
@@ -22,8 +22,11 @@ optional = ["offset", "io_callback", "size"]
 [namespace.storage.op.write]
 optional = ["content_md5", "content_type", "io_callback", "storage_class", "server_side_encryption", "server_side_data_encryption", "server_side_encryption_key_id"]
 
+[namespace.storage.op.create_append]
+optional = ["content_type", "server_side_encryption", "server_side_encryption_key_id", "server_side_data_encryption", "storage_class"]
+
 [namespace.storage.op.write_append]
-optional = ["server_side_encryption", "server_side_data_encryption", "server_side_encryption_key_id"]
+optional = ["content_md5", "io_callback"]
 
 [pairs.default_service_pairs]
 type = "DefaultServicePairs"

diff --git a/storage.go b/storage.go
@@ -14,6 +14,10 @@ import (
 	. "github.com/aos-dev/go-storage/v3/types"
 )
 
+func (s *Storage) commitAppend(ctx context.Context, o *Object, opt pairStorageCommitAppend) (err error) {
+	return
+}
+
 func (s *Storage) create(path string, opt pairStorageCreate) (o *Object) {
 	o = s.newObject(false)
 	o.Mode = ModeRead
@@ -23,11 +27,36 @@ func (s *Storage) create(path string, opt pairStorageCreate) (o *Object) {
 }
 
 func (s *Storage) createAppend(ctx context.Context, path string, opt pairStorageCreateAppend) (o *Object, err error) {
+	rp := s.getAbsPath(path)
+
+	options := make([]oss.Option, 0)
+	options = append(options, oss.ContentLength(0))
+	if opt.HasContentType {
+		options = append(options, oss.ContentType(opt.ContentType))
+	}
+	if opt.HasStorageClass {
+		options = append(options, oss.StorageClass(oss.StorageClassType(opt.StorageClass)))
+	}
+	if opt.HasServerSideEncryption {
+		options = append(options, oss.ServerSideEncryption(opt.ServerSideEncryption))
+	}
+	if opt.HasServerSideDataEncryption {
+		options = append(options, oss.ServerSideDataEncryption(opt.ServerSideDataEncryption))
+	}
+	if opt.HasServerSideEncryptionKeyID {
+		options = append(options, oss.ServerSideEncryptionKeyID(opt.ServerSideEncryptionKeyID))
+	}
+
+	offset, err := s.bucket.AppendObject(rp, nil, 0, options...)
+	if err != nil {
+		return
+	}
+
 	o = s.newObject(true)
 	o.Mode = ModeRead | ModeAppend
-	o.ID = s.getAbsPath(path)
+	o.ID = rp
 	o.Path = path
-	o.SetAppendOffset(0)
+	o.SetAppendOffset(offset)
 	return o, nil
 }
 
@@ -242,8 +271,17 @@ func (s *Storage) write(ctx context.Context, path string, r io.Reader, size int6
 }
 
 func (s *Storage) writeAppend(ctx context.Context, o *Object, r io.Reader, size int64, opt pairStorageWriteAppend) (n int64, err error) {
+	if !o.Mode.IsAppend() {
+		err = fmt.Errorf("object not appendable")
+		return
+	}
+
 	rp := o.GetID()
 
+	if opt.HasIoCallback {
+		r = iowrap.CallbackReader(r, opt.IoCallback)
+	}
+
 	offset, ok := o.GetAppendOffset()
 	if !ok {
 		err = fmt.Errorf("append offset is not set")
@@ -252,14 +290,8 @@ func (s *Storage) writeAppend(ctx context.Context, o *Object, r io.Reader, size
 
 	options := make([]oss.Option, 0)
 	options = append(options, oss.ContentLength(size))
-	if opt.HasServerSideEncryption {
-		options = append(options, oss.ServerSideEncryption(opt.ServerSideEncryption))
-	}
-	if opt.HasServerSideDataEncryption {
-		options = append(options, oss.ServerSideDataEncryption(opt.ServerSideDataEncryption))
-	}
-	if opt.HasServerSideEncryptionKeyID {
-		options = append(options, oss.ServerSideEncryptionKeyID(opt.ServerSideEncryptionKeyID))
+	if opt.HasContentMd5 {
+		options = append(options, oss.ContentMD5(opt.ContentMd5))
 	}
 
 	offset, err = s.bucket.AppendObject(rp, r, offset, options...)