From 07c8c43c8cb60367ca8dfa885cb83bec9bfcc526 Mon Sep 17 00:00:00 2001 From: JinnyYi Date: Wed, 9 Jun 2021 20:51:29 +0800 Subject: [PATCH 1/7] Add CreateDir --- generated.go | 144 ++++++++++++++++++++++++++++++++++++++++-- go.mod | 4 +- go.sum | 18 +++--- service.toml | 13 +++- storage.go | 44 ++++++++++++- tests/storage_test.go | 7 ++ utils.go | 1 + 7 files changed, 209 insertions(+), 22 deletions(-) diff --git a/generated.go b/generated.go index 173e689..2db9ba7 100644 --- a/generated.go +++ b/generated.go @@ -481,6 +481,7 @@ func (s *Service) ListWithContext(ctx context.Context, pairs ...Pair) (sti *Stor var ( _ Appender = &Storage{} + _ Direr = &Storage{} _ Storager = &Storage{} ) @@ -489,6 +490,7 @@ type StorageFeatures struct { LooseOperationCommitAppend bool LooseOperationCreate bool LooseOperationCreateAppend bool + LooseOperationCreateDir bool LooseOperationDelete bool LooseOperationList bool LooseOperationMetadata bool @@ -497,7 +499,8 @@ type StorageFeatures struct { LooseOperationWrite bool LooseOperationWriteAppend bool - VirtualOperationAll bool + VirtualOperationAll bool + VirtualOperationCreateDir bool VirtualPairAll bool } @@ -566,6 +569,7 @@ type DefaultStoragePairs struct { CommitAppend []Pair Create []Pair CreateAppend []Pair + CreateDir []Pair Delete []Pair List []Pair Metadata []Pair @@ -613,7 +617,9 @@ func (s *Storage) parsePairStorageCommitAppend(opts []Pair) (pairStorageCommitAp // pairStorageCreate is the parsed struct type pairStorageCreate struct { - pairs []Pair + pairs []Pair + HasObjectMode bool + ObjectMode ObjectMode } // parsePairStorageCreate will parse Pair slice into *pairStorageCreate @@ -627,6 +633,13 @@ func (s *Storage) parsePairStorageCreate(opts []Pair) (pairStorageCreate, error) isUnsupportedPair := false switch v.Key { + case "object_mode": + if result.HasObjectMode { + continue + } + result.HasObjectMode = true + result.ObjectMode = v.Value.(ObjectMode) + continue default: isUnsupportedPair = true } @@ -710,9 +723,47 @@ func (s *Storage) parsePairStorageCreateAppend(opts []Pair) (pairStorageCreateAp return result, nil } +// pairStorageCreateDir is the parsed struct +type pairStorageCreateDir struct { + pairs []Pair +} + +// parsePairStorageCreateDir will parse Pair slice into *pairStorageCreateDir +func (s *Storage) parsePairStorageCreateDir(opts []Pair) (pairStorageCreateDir, error) { + result := pairStorageCreateDir{ + pairs: opts, + } + + for _, v := range opts { + // isUnsupportedPair records whether current pair is unsupported. + isUnsupportedPair := false + + switch v.Key { + default: + isUnsupportedPair = true + } + + if !isUnsupportedPair { + continue + } + + // If user enables the loose operation feature, we will ignore PairUnsupportedError. + if s.features.LooseOperationAll || s.features.LooseOperationCreateDir { + continue + } + return pairStorageCreateDir{}, services.PairUnsupportedError{Pair: v} + } + + // Check required pairs. + + return result, nil +} + // pairStorageDelete is the parsed struct type pairStorageDelete struct { - pairs []Pair + pairs []Pair + HasObjectMode bool + ObjectMode ObjectMode } // parsePairStorageDelete will parse Pair slice into *pairStorageDelete @@ -726,6 +777,13 @@ func (s *Storage) parsePairStorageDelete(opts []Pair) (pairStorageDelete, error) isUnsupportedPair := false switch v.Key { + case "object_mode": + if result.HasObjectMode { + continue + } + result.HasObjectMode = true + result.ObjectMode = v.Value.(ObjectMode) + continue default: isUnsupportedPair = true } @@ -915,6 +973,8 @@ type pairStorageStat struct { EncryptionKey []byte HasEncryptionScope bool EncryptionScope string + HasObjectMode bool + ObjectMode ObjectMode } // parsePairStorageStat will parse Pair slice into *pairStorageStat @@ -942,6 +1002,13 @@ func (s *Storage) parsePairStorageStat(opts []Pair) (pairStorageStat, error) { result.HasEncryptionScope = true result.EncryptionScope = v.Value.(string) continue + case "object_mode": + if result.HasObjectMode { + continue + } + result.HasObjectMode = true + result.ObjectMode = v.Value.(ObjectMode) + continue default: isUnsupportedPair = true } @@ -1146,6 +1213,11 @@ func (s *Storage) CommitAppendWithContext(ctx context.Context, o *Object, pairs // Create will create a new object without any api call. // +// ## Behavior +// +// - Create SHOULD NOT send any API call. +// - Create SHOULD accept ObjectMode pair as object mode. +// // This function will create a context by default. func (s *Storage) Create(path string, pairs ...Pair) (o *Object) { pairs = append(pairs, s.defaultPairs.Create...) @@ -1182,7 +1254,47 @@ func (s *Storage) CreateAppendWithContext(ctx context.Context, path string, pair return s.createAppend(ctx, path, opt) } -// Delete will delete an Object from service. +// CreateDir will create a new dir object. +// +// This function will create a context by default. +func (s *Storage) CreateDir(path string, pairs ...Pair) (o *Object, err error) { + ctx := context.Background() + return s.CreateDirWithContext(ctx, path, pairs...) +} + +// CreateDirWithContext will create a new dir object. +func (s *Storage) CreateDirWithContext(ctx context.Context, path string, pairs ...Pair) (o *Object, err error) { + defer func() { + err = s.formatError("create_dir", err, path) + }() + // If virtual operation feature is not enabled, we will act like this operation is not implemented. + if !s.features.VirtualOperationAll && !s.features.VirtualOperationCreateDir { + err = NewOperationNotImplementedError("create_dir") + return + } + + pairs = append(pairs, s.defaultPairs.CreateDir...) + var opt pairStorageCreateDir + + opt, err = s.parsePairStorageCreateDir(pairs) + if err != nil { + return + } + + return s.createDir(ctx, path, opt) +} + +// Delete will delete an object from service. +// +// ## Behavior +// +// - Delete only delete one and only one object. +// - Service DON'T NEED to support remove all. +// - User NEED to implement remove_all by themself. +// - Delete is idempotent. +// - Successful delete always return nil error. +// - Delete SHOULD never return `ObjectNotExist` +// - Delete DON'T NEED to check the object exist or not. // // This function will create a context by default. func (s *Storage) Delete(path string, pairs ...Pair) (err error) { @@ -1190,7 +1302,17 @@ func (s *Storage) Delete(path string, pairs ...Pair) (err error) { return s.DeleteWithContext(ctx, path, pairs...) } -// DeleteWithContext will delete an Object from service. +// DeleteWithContext will delete an object from service. +// +// ## Behavior +// +// - Delete only delete one and only one object. +// - Service DON'T NEED to support remove all. +// - User NEED to implement remove_all by themself. +// - Delete is idempotent. +// - Successful delete always return nil error. +// - Delete SHOULD never return `ObjectNotExist` +// - Delete DON'T NEED to check the object exist or not. func (s *Storage) DeleteWithContext(ctx context.Context, path string, pairs ...Pair) (err error) { defer func() { err = s.formatError("delete", err, path) @@ -1272,6 +1394,12 @@ func (s *Storage) ReadWithContext(ctx context.Context, path string, w io.Writer, // Stat will stat a path to get info of an object. // +// ## Behavior +// +// - Stat SHOULD accept ObjectMode pair as hints. +// - Service COULD have different implementations for different object mode. +// - Service SHOULD check if returning ObjectMode is match +// // This function will create a context by default. func (s *Storage) Stat(path string, pairs ...Pair) (o *Object, err error) { ctx := context.Background() @@ -1279,6 +1407,12 @@ func (s *Storage) Stat(path string, pairs ...Pair) (o *Object, err error) { } // StatWithContext will stat a path to get info of an object. +// +// ## Behavior +// +// - Stat SHOULD accept ObjectMode pair as hints. +// - Service COULD have different implementations for different object mode. +// - Service SHOULD check if returning ObjectMode is match func (s *Storage) StatWithContext(ctx context.Context, path string, pairs ...Pair) (o *Object, err error) { defer func() { err = s.formatError("stat", err, path) diff --git a/go.mod b/go.mod index 9355d29..3db18f5 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.15 require ( github.com/Azure/azure-pipeline-go v0.2.3 github.com/Azure/azure-storage-blob-go v0.13.0 - github.com/beyondstorage/go-integration-test/v4 v4.0.0 - github.com/beyondstorage/go-storage/v4 v4.0.1-0.20210530044854-1c928ddbe52d + github.com/beyondstorage/go-integration-test/v4 v4.0.1-0.20210609053034-c264c36af98b + github.com/beyondstorage/go-storage/v4 v4.1.0 github.com/google/uuid v1.2.0 ) diff --git a/go.sum b/go.sum index 08f9c9e..515bab1 100644 --- a/go.sum +++ b/go.sum @@ -14,14 +14,12 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/Xuanwo/templateutils v0.1.0 h1:WpkWOqQtIQ2vAIpJLa727DdN8WtxhUkkbDGa6UhntJY= github.com/Xuanwo/templateutils v0.1.0/go.mod h1:OdE0DJ+CJxDBq6psX5DPV+gOZi8bhuHuVUpPCG++Wb8= -github.com/beyondstorage/go-integration-test/v4 v4.0.0 h1:tdXQV9yxQ3Q6p9xfyQKzK3MEo9r9j9g3uT5+3sbVtnQ= -github.com/beyondstorage/go-integration-test/v4 v4.0.0/go.mod h1:26/JF4b0XxRN0pL4kihpnVNhbbw+QWvmmvgxfnFJDfA= -github.com/beyondstorage/go-storage/v4 v4.0.0/go.mod h1:oa2dYco+xplPj99WSBnYVw/xXvRkIKWSSVDQKNZ5Kz8= -github.com/beyondstorage/go-storage/v4 v4.0.1-0.20210530044854-1c928ddbe52d h1:s9t6VNNRDqmg+PXyXtVEsxIM7xZQIJOYlma28IpkbNQ= -github.com/beyondstorage/go-storage/v4 v4.0.1-0.20210530044854-1c928ddbe52d/go.mod h1:kXMu07IDZaKtxbqI1ufuhqo0FjYe0nH7zPCbBanln/Y= -github.com/beyondstorage/specs/go v0.0.0-20210521044836-3d41c1d9c97f/go.mod h1:f5VvmLHc/dNJwl+/yAv/TOHdev3phvuEswx8DIXiSQQ= -github.com/beyondstorage/specs/go v0.0.0-20210530044123-3ff75e192bc9 h1:YSiF27cAHlDZk9q+oaEHQbA8dH8XTvYxeTOoPzNCwOQ= -github.com/beyondstorage/specs/go v0.0.0-20210530044123-3ff75e192bc9/go.mod h1:f5VvmLHc/dNJwl+/yAv/TOHdev3phvuEswx8DIXiSQQ= +github.com/beyondstorage/go-integration-test/v4 v4.0.1-0.20210609053034-c264c36af98b h1:xx05UdthvomMRHlb2CCHvI2NZ29WR9KO8gMELUPlF9w= +github.com/beyondstorage/go-integration-test/v4 v4.0.1-0.20210609053034-c264c36af98b/go.mod h1:W6Dhve1tbEpWAR1WSRuJl4UYpMNpM2QTAcb1dCpuEUI= +github.com/beyondstorage/go-storage/v4 v4.1.0 h1:O5SuSoTvs0KeXaZ/dYhpURlki6PjTNPyIzsJMY9sJWw= +github.com/beyondstorage/go-storage/v4 v4.1.0/go.mod h1:dK5DFnvKQI70bfpM1MLU9QqDYB12Z/dFV5sUnbJ/AoU= +github.com/beyondstorage/specs/go v0.0.0-20210608070420-9185b588aa58 h1:AvxsyR0bSSBi90WtYEMdAkM4Sm+Xxr7JVGXAQVVfeOo= +github.com/beyondstorage/specs/go v0.0.0-20210608070420-9185b588aa58/go.mod h1:1Az5o44awI/Ljop+ppO2djyezVfdKKb1RjJ6+M+a5XQ= github.com/dave/dst v0.26.2 h1:lnxLAKI3tx7MgLNVDirFCsDTlTG9nKTk7GcptKcWSwY= github.com/dave/dst v0.26.2/go.mod h1:UMDJuIRPfyUCC78eFuB+SV/WI8oDeyFDvM/JR6NI3IU= github.com/dave/gopackages v0.0.0-20170318123100-46e7023ec56e/go.mod h1:i00+b/gKdIDIxuLDFob7ustLAVqhsZRk2qVZrArELGQ= @@ -53,8 +51,8 @@ github.com/mattn/go-ieproxy v0.0.1 h1:qiyop7gCflfhwCzGyeT0gro3sF9AIg9HU98JORTkqf github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/pelletier/go-toml v1.9.1 h1:a6qW1EVNZWH9WGI6CsYdD8WAylkoXBS5yv0XHlh17Tc= -github.com/pelletier/go-toml v1.9.1/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pelletier/go-toml v1.9.2 h1:7NiByeVF4jKSG1lDF3X8LTIkq2/bu+1uYbIm1eS5tzk= +github.com/pelletier/go-toml v1.9.2/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= diff --git a/service.toml b/service.toml index 951aba9..7e42fa5 100644 --- a/service.toml +++ b/service.toml @@ -7,12 +7,18 @@ required = ["credential", "endpoint"] optional = ["service_features", "default_service_pairs", "http_client_options"] [namespace.storage] -implement = ["appender"] +implement = ["appender", "direr"] [namespace.storage.new] required = ["name"] optional = ["storage_features", "default_storage_pairs", "work_dir"] +[namespace.storage.op.create] +optional = ["object_mode"] + +[namespace.storage.op.delete] +optional = ["object_mode"] + [namespace.storage.op.list] optional = ["list_mode"] @@ -29,7 +35,10 @@ optional = ["content_type", "encryption_key", "encryption_scope"] optional = ["content_md5", "encryption_key", "encryption_scope"] [namespace.storage.op.stat] -optional = ["encryption_key", "encryption_scope"] +optional = ["encryption_key", "encryption_scope", "object_mode"] + +[namespace.storage.op.create_dir] +simulated = true [pairs.service_features] type = "ServiceFeatures" diff --git a/storage.go b/storage.go index 901ca0e..9207a2d 100644 --- a/storage.go +++ b/storage.go @@ -3,6 +3,7 @@ package azblob import ( "context" "encoding/base64" + "github.com/beyondstorage/go-storage/v4/pairs" "io" "strconv" @@ -18,8 +19,16 @@ func (s *Storage) commitAppend(ctx context.Context, o *Object, opt pairStorageCo } func (s *Storage) create(path string, opt pairStorageCreate) (o *Object) { - o = s.newObject(false) - o.Mode = ModeRead + + if opt.HasObjectMode && opt.ObjectMode.IsDir() { + o = s.newObject(true) + path += "/" + o.Mode = ModeDir + } else { + o = s.newObject(false) + o.Mode = ModeRead + } + o.ID = s.getAbsPath(path) o.Path = path return o @@ -55,7 +64,27 @@ func (s *Storage) createAppend(ctx context.Context, path string, opt pairStorage return o, nil } +func (s *Storage) createDir(ctx context.Context, path string, opt pairStorageCreateDir) (o *Object, err error) { + // Specify a character or string delimiter within a blob name to create a virtual hierarchy. + // ref: https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata#resource-names + path += "/" + _, err = s.Write(path, nil, 0, pairs.WithContentType("application/x-directory")) + if err != nil { + return + } + + o = s.newObject(true) + o.ID = s.getAbsPath(path) + o.Path = path + o.Mode |= ModeDir + return +} + func (s *Storage) delete(ctx context.Context, path string, opt pairStorageDelete) (err error) { + if opt.HasObjectMode && opt.ObjectMode.IsDir() { + path += "/" + } + rp := s.getAbsPath(path) _, err = s.bucket.NewBlockBlobURL(rp).Delete(ctx, @@ -207,6 +236,10 @@ func (s *Storage) read(ctx context.Context, path string, w io.Writer, opt pairSt } func (s *Storage) stat(ctx context.Context, path string, opt pairStorageStat) (o *Object, err error) { + if opt.HasObjectMode && opt.ObjectMode.IsDir() { + path += "/" + } + rp := s.getAbsPath(path) var cpk azblob.ClientProvidedKeyOptions @@ -225,7 +258,12 @@ func (s *Storage) stat(ctx context.Context, path string, opt pairStorageStat) (o o = s.newObject(true) o.ID = rp o.Path = path - o.Mode |= ModeRead + + if opt.HasObjectMode && opt.ObjectMode.IsDir() { + o.Mode |= ModeDir + } else { + o.Mode |= ModeRead + } o.SetContentLength(output.ContentLength()) o.SetLastModified(output.LastModified()) diff --git a/tests/storage_test.go b/tests/storage_test.go index f2313bb..a9e4e31 100644 --- a/tests/storage_test.go +++ b/tests/storage_test.go @@ -20,3 +20,10 @@ func TestAppend(t *testing.T) { } tests.TestAppender(t, setupTest(t)) } + +func TestDir(t *testing.T) { + if os.Getenv("STORAGE_AZBLOB_INTEGRATION_TEST") != "on" { + t.Skipf("STORAGE_AZBLOB_INTEGRATION_TEST is not 'on', skipped") + } + tests.TestDirer(t, setupTest(t)) +} diff --git a/utils.go b/utils.go index 79db82b..bcd6d20 100644 --- a/utils.go +++ b/utils.go @@ -48,6 +48,7 @@ type Storage struct { typ.UnimplementedStorager typ.UnimplementedAppender + typ.UnimplementedDirer } // String implements Storager.String From 32cc465d80ead462108346c4d5c972d8f22e22a0 Mon Sep 17 00:00:00 2001 From: JinnyYi Date: Wed, 9 Jun 2021 21:20:49 +0800 Subject: [PATCH 2/7] loose operation CreateDir --- tests/utils_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/utils_test.go b/tests/utils_test.go index 3132da9..74a05ef 100644 --- a/tests/utils_test.go +++ b/tests/utils_test.go @@ -18,6 +18,9 @@ func setupTest(t *testing.T) types.Storager { ps.WithName(os.Getenv("STORAGE_AZBLOB_NAME")), ps.WithEndpoint(os.Getenv("STORAGE_AZBLOB_ENDPOINT")), ps.WithWorkDir("/"+uuid.New().String()+"/"), + azblob.WithStorageFeatures(azblob.StorageFeatures{ + LooseOperationCreateDir: true, + }), ) if err != nil { t.Errorf("new storager: %v", err) From b0895d6f1d82ae54b5993fb5f50f6a62e372d360 Mon Sep 17 00:00:00 2001 From: JinnyYi Date: Wed, 9 Jun 2021 21:28:42 +0800 Subject: [PATCH 3/7] loose operation CreateDir --- tests/utils_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/utils_test.go b/tests/utils_test.go index 74a05ef..09b19d6 100644 --- a/tests/utils_test.go +++ b/tests/utils_test.go @@ -19,7 +19,7 @@ func setupTest(t *testing.T) types.Storager { ps.WithEndpoint(os.Getenv("STORAGE_AZBLOB_ENDPOINT")), ps.WithWorkDir("/"+uuid.New().String()+"/"), azblob.WithStorageFeatures(azblob.StorageFeatures{ - LooseOperationCreateDir: true, + VirtualOperationCreateDir: true, }), ) if err != nil { From b6b16f33c8349cb9801be204f8ff261576493b79 Mon Sep 17 00:00:00 2001 From: JinnyYi Date: Thu, 10 Jun 2021 11:32:56 +0800 Subject: [PATCH 4/7] remove content-type for createDir and normalize path --- storage.go | 25 +++++++++++++++---------- tests/utils_test.go | 2 +- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/storage.go b/storage.go index 9207a2d..e493ba3 100644 --- a/storage.go +++ b/storage.go @@ -3,7 +3,6 @@ package azblob import ( "context" "encoding/base64" - "github.com/beyondstorage/go-storage/v4/pairs" "io" "strconv" @@ -19,17 +18,19 @@ func (s *Storage) commitAppend(ctx context.Context, o *Object, opt pairStorageCo } func (s *Storage) create(path string, opt pairStorageCreate) (o *Object) { + rp := s.getAbsPath(path) if opt.HasObjectMode && opt.ObjectMode.IsDir() { - o = s.newObject(true) path += "/" + rp += "/" + o = s.newObject(true) o.Mode = ModeDir } else { o = s.newObject(false) o.Mode = ModeRead } - o.ID = s.getAbsPath(path) + o.ID = rp o.Path = path return o } @@ -65,28 +66,31 @@ func (s *Storage) createAppend(ctx context.Context, path string, opt pairStorage } func (s *Storage) createDir(ctx context.Context, path string, opt pairStorageCreateDir) (o *Object, err error) { + rp := s.getAbsPath(path) + // Specify a character or string delimiter within a blob name to create a virtual hierarchy. // ref: https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata#resource-names path += "/" - _, err = s.Write(path, nil, 0, pairs.WithContentType("application/x-directory")) + rp += "/" + _, err = s.write(ctx, rp, nil, 0, pairStorageWrite{}) if err != nil { return } o = s.newObject(true) - o.ID = s.getAbsPath(path) + o.ID = rp o.Path = path o.Mode |= ModeDir return } func (s *Storage) delete(ctx context.Context, path string, opt pairStorageDelete) (err error) { + rp := s.getAbsPath(path) + if opt.HasObjectMode && opt.ObjectMode.IsDir() { - path += "/" + rp += "/" } - rp := s.getAbsPath(path) - _, err = s.bucket.NewBlockBlobURL(rp).Delete(ctx, azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{}) if err != nil && checkError(err, azblob.ServiceCodeBlobNotFound) { @@ -236,12 +240,13 @@ func (s *Storage) read(ctx context.Context, path string, w io.Writer, opt pairSt } func (s *Storage) stat(ctx context.Context, path string, opt pairStorageStat) (o *Object, err error) { + rp := s.getAbsPath(path) + if opt.HasObjectMode && opt.ObjectMode.IsDir() { path += "/" + rp += "/" } - rp := s.getAbsPath(path) - var cpk azblob.ClientProvidedKeyOptions if opt.HasEncryptionKey { cpk, err = calculateEncryptionHeaders(opt.EncryptionKey, opt.EncryptionScope) diff --git a/tests/utils_test.go b/tests/utils_test.go index 09b19d6..cb7edf3 100644 --- a/tests/utils_test.go +++ b/tests/utils_test.go @@ -19,7 +19,7 @@ func setupTest(t *testing.T) types.Storager { ps.WithEndpoint(os.Getenv("STORAGE_AZBLOB_ENDPOINT")), ps.WithWorkDir("/"+uuid.New().String()+"/"), azblob.WithStorageFeatures(azblob.StorageFeatures{ - VirtualOperationCreateDir: true, + VirtualOperationAll: true, }), ) if err != nil { From a57126ef91a9421e07dd3cc9cab0a7000c78be01 Mon Sep 17 00:00:00 2001 From: JinnyYi Date: Thu, 10 Jun 2021 14:26:32 +0800 Subject: [PATCH 5/7] fix the path consistency issue and update createDir implement --- generated.go | 29 ++++++++++++++++++++++++++++- service.toml | 1 + storage.go | 22 ++++++++++++++++++---- 3 files changed, 47 insertions(+), 5 deletions(-) diff --git a/generated.go b/generated.go index 2db9ba7..a8441ff 100644 --- a/generated.go +++ b/generated.go @@ -725,7 +725,13 @@ func (s *Storage) parsePairStorageCreateAppend(opts []Pair) (pairStorageCreateAp // pairStorageCreateDir is the parsed struct type pairStorageCreateDir struct { - pairs []Pair + pairs []Pair + HasAccessTier bool + AccessTier string + HasEncryptionKey bool + EncryptionKey []byte + HasEncryptionScope bool + EncryptionScope string } // parsePairStorageCreateDir will parse Pair slice into *pairStorageCreateDir @@ -739,6 +745,27 @@ func (s *Storage) parsePairStorageCreateDir(opts []Pair) (pairStorageCreateDir, isUnsupportedPair := false switch v.Key { + case pairAccessTier: + if result.HasAccessTier { + continue + } + result.HasAccessTier = true + result.AccessTier = v.Value.(string) + continue + case pairEncryptionKey: + if result.HasEncryptionKey { + continue + } + result.HasEncryptionKey = true + result.EncryptionKey = v.Value.([]byte) + continue + case pairEncryptionScope: + if result.HasEncryptionScope { + continue + } + result.HasEncryptionScope = true + result.EncryptionScope = v.Value.(string) + continue default: isUnsupportedPair = true } diff --git a/service.toml b/service.toml index 7e42fa5..3ee753c 100644 --- a/service.toml +++ b/service.toml @@ -39,6 +39,7 @@ optional = ["encryption_key", "encryption_scope", "object_mode"] [namespace.storage.op.create_dir] simulated = true +optional = ["access_tier", "encryption_key", "encryption_scope"] [pairs.service_features] type = "ServiceFeatures" diff --git a/storage.go b/storage.go index e493ba3..6628f3e 100644 --- a/storage.go +++ b/storage.go @@ -21,7 +21,6 @@ func (s *Storage) create(path string, opt pairStorageCreate) (o *Object) { rp := s.getAbsPath(path) if opt.HasObjectMode && opt.ObjectMode.IsDir() { - path += "/" rp += "/" o = s.newObject(true) o.Mode = ModeDir @@ -70,9 +69,25 @@ func (s *Storage) createDir(ctx context.Context, path string, opt pairStorageCre // Specify a character or string delimiter within a blob name to create a virtual hierarchy. // ref: https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata#resource-names - path += "/" rp += "/" - _, err = s.write(ctx, rp, nil, 0, pairStorageWrite{}) + + accessTier := azblob.AccessTierNone + if opt.HasAccessTier { + accessTier = azblob.AccessTierType(opt.AccessTier) + } + + var cpk azblob.ClientProvidedKeyOptions + if opt.HasEncryptionKey { + cpk, err = calculateEncryptionHeaders(opt.EncryptionKey, opt.EncryptionScope) + if err != nil { + return + } + } + + _, err = s.bucket.NewBlockBlobURL(rp).Upload( + ctx, nil, azblob.BlobHTTPHeaders{}, + azblob.Metadata{}, azblob.BlobAccessConditions{}, + accessTier, azblob.BlobTagsMap{}, cpk) if err != nil { return } @@ -243,7 +258,6 @@ func (s *Storage) stat(ctx context.Context, path string, opt pairStorageStat) (o rp := s.getAbsPath(path) if opt.HasObjectMode && opt.ObjectMode.IsDir() { - path += "/" rp += "/" } From 533deee407dc45070056b2149a4aa623014ff9b3 Mon Sep 17 00:00:00 2001 From: JinnyYi Date: Thu, 10 Jun 2021 14:57:39 +0800 Subject: [PATCH 6/7] fix type check --- storage.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage.go b/storage.go index 6628f3e..1ff3d31 100644 --- a/storage.go +++ b/storage.go @@ -85,7 +85,7 @@ func (s *Storage) createDir(ctx context.Context, path string, opt pairStorageCre } _, err = s.bucket.NewBlockBlobURL(rp).Upload( - ctx, nil, azblob.BlobHTTPHeaders{}, + ctx, iowrap.SizedReadSeekCloser(nil, 0), azblob.BlobHTTPHeaders{}, azblob.Metadata{}, azblob.BlobAccessConditions{}, accessTier, azblob.BlobTagsMap{}, cpk) if err != nil { From 348d23181157306cce8c7f5b04703ee9affe19d8 Mon Sep 17 00:00:00 2001 From: JinnyYi Date: Thu, 10 Jun 2021 17:11:38 +0800 Subject: [PATCH 7/7] upgrage go-integration-test to 4.1.0 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 3db18f5..76e3daf 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.15 require ( github.com/Azure/azure-pipeline-go v0.2.3 github.com/Azure/azure-storage-blob-go v0.13.0 - github.com/beyondstorage/go-integration-test/v4 v4.0.1-0.20210609053034-c264c36af98b + github.com/beyondstorage/go-integration-test/v4 v4.1.0 github.com/beyondstorage/go-storage/v4 v4.1.0 github.com/google/uuid v1.2.0 ) diff --git a/go.sum b/go.sum index 515bab1..2649fcd 100644 --- a/go.sum +++ b/go.sum @@ -14,8 +14,8 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/Xuanwo/templateutils v0.1.0 h1:WpkWOqQtIQ2vAIpJLa727DdN8WtxhUkkbDGa6UhntJY= github.com/Xuanwo/templateutils v0.1.0/go.mod h1:OdE0DJ+CJxDBq6psX5DPV+gOZi8bhuHuVUpPCG++Wb8= -github.com/beyondstorage/go-integration-test/v4 v4.0.1-0.20210609053034-c264c36af98b h1:xx05UdthvomMRHlb2CCHvI2NZ29WR9KO8gMELUPlF9w= -github.com/beyondstorage/go-integration-test/v4 v4.0.1-0.20210609053034-c264c36af98b/go.mod h1:W6Dhve1tbEpWAR1WSRuJl4UYpMNpM2QTAcb1dCpuEUI= +github.com/beyondstorage/go-integration-test/v4 v4.1.0 h1:/cKM5uC+tW1mYAM/FTQlMq3EZMAb8hBm8gL4ZHk9InQ= +github.com/beyondstorage/go-integration-test/v4 v4.1.0/go.mod h1:W6Dhve1tbEpWAR1WSRuJl4UYpMNpM2QTAcb1dCpuEUI= github.com/beyondstorage/go-storage/v4 v4.1.0 h1:O5SuSoTvs0KeXaZ/dYhpURlki6PjTNPyIzsJMY9sJWw= github.com/beyondstorage/go-storage/v4 v4.1.0/go.mod h1:dK5DFnvKQI70bfpM1MLU9QqDYB12Z/dFV5sUnbJ/AoU= github.com/beyondstorage/specs/go v0.0.0-20210608070420-9185b588aa58 h1:AvxsyR0bSSBi90WtYEMdAkM4Sm+Xxr7JVGXAQVVfeOo=