Skip to content
This repository has been archived by the owner on Oct 18, 2021. It is now read-only.

Implement commitAppend #19

Merged
merged 4 commits into from
Apr 29, 2021
Merged

Implement commitAppend #19

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
4ef8755
Implement commitAppend
Apr 27, 2021
ac51326
add check parameter for append
Apr 27, 2021
1e9db7e
modify append metadata
Apr 28, 2021
770c4e6
modify append max size limit
Apr 28, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 78 additions & 5 deletions generated.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,6 @@ require (
github.com/Azure/azure-pipeline-go v0.2.3
github.com/Azure/azure-storage-blob-go v0.13.0
github.com/aos-dev/go-integration-test/v3 v3.0.0
github.com/aos-dev/go-storage/v3 v3.6.0
github.com/aos-dev/go-storage/v3 v3.6.1-0.20210427074731-1b3b4e86c950
github.com/google/uuid v1.2.0
)
8 changes: 4 additions & 4 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,11 @@ github.com/Xuanwo/templateutils v0.1.0/go.mod h1:OdE0DJ+CJxDBq6psX5DPV+gOZi8bhuH
github.com/aos-dev/go-integration-test/v3 v3.0.0 h1:rxIc7YBfiw9JS5JiVU4BZAqQqoKJemIftFaeJpYCb5M=
github.com/aos-dev/go-integration-test/v3 v3.0.0/go.mod h1:woC3E9Ld1G/Cpo2tSEQ+iwIr4MMdgf+L/8UuyJOE11Q=
github.com/aos-dev/go-storage/v3 v3.5.0/go.mod h1:PZJT0Ta7YxVM5QoYoh8Q/X4I6e/z/7gOJqm85Aib4nY=
github.com/aos-dev/go-storage/v3 v3.6.0 h1:ywjMvh320+esJH81MqB9nyuMNLW97Krujz2UiprC2ZM=
github.com/aos-dev/go-storage/v3 v3.6.0/go.mod h1:ZQwybmoCcTWUOWg+G15gT/NQJoI8G8KH1pF41TuJqYk=
github.com/aos-dev/go-storage/v3 v3.6.1-0.20210427074731-1b3b4e86c950 h1:3ZsfgDAUF171rIrFdYjhr8P3WyxzR4IwxWGbGxkCRkA=
github.com/aos-dev/go-storage/v3 v3.6.1-0.20210427074731-1b3b4e86c950/go.mod h1:wzxhpUa73aCdTDRG0sHjNnnmXa9rC/ahJnk7hKOV6Pw=
github.com/aos-dev/specs/go v0.0.0-20210312090615-23109627848b/go.mod h1:XTNlLZtPA1inITyDH5hNnQXVjvvKUvo+lurs5GYB8NA=
github.com/aos-dev/specs/go v0.0.0-20210423110314-8361397c2bf3 h1:e65ozDhdfHfhnDpZF9SLcY5mwtAg/sAvNIUAkFd+4D0=
github.com/aos-dev/specs/go v0.0.0-20210423110314-8361397c2bf3/go.mod h1:gNah3KaPJEfysh7uCCX+sYjQC3g2yx2VgBkFlT945Ws=
github.com/aos-dev/specs/go v0.0.0-20210427074008-93183332d504 h1:vHxnQOXlz/X8/Co1ep57yc34Xso+f2+xt4Q2dMJVHA4=
github.com/aos-dev/specs/go v0.0.0-20210427074008-93183332d504/go.mod h1:gNah3KaPJEfysh7uCCX+sYjQC3g2yx2VgBkFlT945Ws=
github.com/dave/dst v0.26.2 h1:lnxLAKI3tx7MgLNVDirFCsDTlTG9nKTk7GcptKcWSwY=
github.com/dave/dst v0.26.2/go.mod h1:UMDJuIRPfyUCC78eFuB+SV/WI8oDeyFDvM/JR6NI3IU=
github.com/dave/gopackages v0.0.0-20170318123100-46e7023ec56e/go.mod h1:i00+b/gKdIDIxuLDFob7ustLAVqhsZRk2qVZrArELGQ=
Expand Down
4 changes: 2 additions & 2 deletions service.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,10 @@ optional = ["offset", "io_callback", "size", "encryption_key", "encryption_scope
optional = ["content_md5", "content_type", "io_callback", "access_tier", "encryption_key", "encryption_scope"]

[namespace.storage.op.create_append]
optional = ["encryption_key", "encryption_scope"]
optional = ["content_type", "encryption_key", "encryption_scope"]

[namespace.storage.op.write_append]
optional = ["encryption_key", "encryption_scope"]
optional = ["content_md5", "encryption_key", "encryption_scope"]

[namespace.storage.op.stat]
optional = ["encryption_key", "encryption_scope"]
Expand Down
26 changes: 21 additions & 5 deletions storage.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,10 @@ import (
. "github.com/aos-dev/go-storage/v3/types"
)

func (s *Storage) commitAppend(ctx context.Context, o *Object, opt pairStorageCommitAppend) (err error) {
return
}

func (s *Storage) create(path string, opt pairStorageCreate) (o *Object) {
o = s.newObject(false)
o.Mode = ModeRead
Expand All @@ -24,14 +28,20 @@ func (s *Storage) create(path string, opt pairStorageCreate) (o *Object) {
func (s *Storage) createAppend(ctx context.Context, path string, opt pairStorageCreateAppend) (o *Object, err error) {
rp := s.getAbsPath(path)

headers := azblob.BlobHTTPHeaders{}
if opt.HasContentType {
headers.ContentType = opt.ContentType
}

var cpk azblob.ClientProvidedKeyOptions
if opt.HasEncryptionKey {
cpk, err = calculateEncryptionHeaders(opt.EncryptionKey, opt.EncryptionScope)
if err != nil {
return
}
}
_, err = s.bucket.NewAppendBlobURL(rp).Create(ctx, azblob.BlobHTTPHeaders{}, nil,

_, err = s.bucket.NewAppendBlobURL(rp).Create(ctx, headers, nil,
azblob.BlobAccessConditions{}, nil, cpk)
if err != nil {
return
Expand Down Expand Up @@ -283,6 +293,11 @@ func (s *Storage) write(ctx context.Context, path string, r io.Reader, size int6
}

func (s *Storage) writeAppend(ctx context.Context, o *Object, r io.Reader, size int64, opt pairStorageWriteAppend) (n int64, err error) {
if !o.Mode.IsAppend() {
err = fmt.Errorf("object not appendable")
return
}

rp := o.GetID()

offset, ok := o.GetAppendOffset()
Expand All @@ -299,16 +314,17 @@ func (s *Storage) writeAppend(ctx context.Context, o *Object, r io.Reader, size
}
}

var accessConditions azblob.AppendBlobAccessConditions
var ac azblob.AppendBlobAccessConditions
ac.AppendPositionAccessConditions.IfMaxSizeLessThanOrEqual = AppendBlobIfMaxSizeLessThanOrEqual
if 0 == offset {
accessConditions.AppendPositionAccessConditions.IfAppendPositionEqual = -1
ac.AppendPositionAccessConditions.IfAppendPositionEqual = -1
} else {
accessConditions.AppendPositionAccessConditions.IfAppendPositionEqual = offset
ac.AppendPositionAccessConditions.IfAppendPositionEqual = offset
}

appendResp, err := s.bucket.NewAppendBlobURL(rp).AppendBlock(
ctx, iowrap.SizedReadSeekCloser(r, size),
accessConditions, nil, cpk)
ac, nil, cpk)
if err != nil {
return
}
Expand Down
6 changes: 6 additions & 0 deletions utils.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -329,3 +329,9 @@ func calculateEncryptionHeaders(key []byte, scope string) (cpk azblob.ClientProv
}
return
}

const (
// AppendBlobIfMaxSizeLessThanOrEqual ensures that the AppendBlock operation succeeds only if the append blob's size is less than or equal to a value.
// For more information, see https://docs.microsoft.com/rest/api/storageservices/append-block.
AppendBlobIfMaxSizeLessThanOrEqual = 4 * 1024 * 1024 * 50000
)