diff --git a/generated.go b/generated.go index a8441ff..6566ddf 100644 --- a/generated.go +++ b/generated.go @@ -21,25 +21,9 @@ var _ httpclient.Options // Type is the type for azblob const Type = "azblob" -// Service available pairs. -const ( - // AccessTier - pairAccessTier = "azblob_access_tier" - // DefaultServicePairs set default pairs for service actions - pairDefaultServicePairs = "azblob_default_service_pairs" - // DefaultStoragePairs set default pairs for storager actions - pairDefaultStoragePairs = "azblob_default_storage_pairs" - // EncryptionKey is the customer's 32-byte AES-256 key - pairEncryptionKey = "azblob_encryption_key" - // EncryptionScope Specifies the name of the encryption scope. See https://docs.microsoft.com/en-us/azure/storage/blobs/encryption-scope-overview for details. - pairEncryptionScope = "azblob_encryption_scope" - // ServiceFeatures set service features - pairServiceFeatures = "azblob_service_features" - // StorageFeatures set storage features - pairStorageFeatures = "azblob_storage_features" -) - // ObjectMetadata stores service metadata for object. +// +// Deprecated: Use ObjectSystemMetadata instead. type ObjectMetadata struct { // AccessTier AccessTier string @@ -55,6 +39,8 @@ type ObjectMetadata struct { // // - This function should not be called by service implementer. // - The returning ObjectMetadata is read only and should not be modified. +// +// Deprecated: Use GetObjectSystemMetadata instead. func GetObjectMetadata(o *Object) ObjectMetadata { om, ok := o.GetServiceMetadata() if ok { @@ -66,16 +52,71 @@ func GetObjectMetadata(o *Object) ObjectMetadata { // setObjectMetadata will set ObjectMetadata into Object. // // - This function should only be called once, please make sure all data has been written before set. +// +// Deprecated: Use setObjectSystemMetadata instead. func setObjectMetadata(o *Object, om ObjectMetadata) { o.SetServiceMetadata(om) } +// ObjectSystemMetadata stores system metadata for object. +type ObjectSystemMetadata struct { + // AccessTier + AccessTier string + // EncryptionKeySha256 + EncryptionKeySha256 string + // EncryptionScope + EncryptionScope string + // ServerEncrypted + ServerEncrypted bool +} + +// GetObjectSystemMetadata will get ObjectSystemMetadata from Object. +// +// - This function should not be called by service implementer. +// - The returning ObjectServiceMetadata is read only and should not be modified. +func GetObjectSystemMetadata(o *Object) ObjectSystemMetadata { + sm, ok := o.GetSystemMetadata() + if ok { + return sm.(ObjectSystemMetadata) + } + return ObjectSystemMetadata{} +} + +// setObjectSystemMetadata will set ObjectSystemMetadata into Object. +// +// - This function should only be called once, please make sure all data has been written before set. +func setObjectSystemMetadata(o *Object, sm ObjectSystemMetadata) { + o.SetSystemMetadata(sm) +} + +// StorageSystemMetadata stores system metadata for storage meta. +type StorageSystemMetadata struct { +} + +// GetStorageSystemMetadata will get SystemMetadata from StorageMeta. +// +// - The returning StorageSystemMetadata is read only and should not be modified. +func GetStorageSystemMetadata(s *StorageMeta) StorageSystemMetadata { + sm, ok := s.GetSystemMetadata() + if ok { + return sm.(StorageSystemMetadata) + } + return StorageSystemMetadata{} +} + +// setStorageSystemMetadata will set SystemMetadata into StorageMeta. +// +// - This function should only be called once, please make sure all data has been written before set. +func setStorageSystemMetadata(s *StorageMeta, sm StorageSystemMetadata) { + s.SetSystemMetadata(sm) +} + // WithAccessTier will apply access_tier value to Options. // // AccessTier func WithAccessTier(v string) Pair { return Pair{ - Key: pairAccessTier, + Key: "access_tier", Value: v, } } @@ -85,7 +126,7 @@ func WithAccessTier(v string) Pair { // DefaultServicePairs set default pairs for service actions func WithDefaultServicePairs(v DefaultServicePairs) Pair { return Pair{ - Key: pairDefaultServicePairs, + Key: "default_service_pairs", Value: v, } } @@ -95,7 +136,7 @@ func WithDefaultServicePairs(v DefaultServicePairs) Pair { // DefaultStoragePairs set default pairs for storager actions func WithDefaultStoragePairs(v DefaultStoragePairs) Pair { return Pair{ - Key: pairDefaultStoragePairs, + Key: "default_storage_pairs", Value: v, } } @@ -105,7 +146,7 @@ func WithDefaultStoragePairs(v DefaultStoragePairs) Pair { // EncryptionKey is the customer's 32-byte AES-256 key func WithEncryptionKey(v []byte) Pair { return Pair{ - Key: pairEncryptionKey, + Key: "encryption_key", Value: v, } } @@ -115,7 +156,7 @@ func WithEncryptionKey(v []byte) Pair { // EncryptionScope Specifies the name of the encryption scope. See https://docs.microsoft.com/en-us/azure/storage/blobs/encryption-scope-overview for details. func WithEncryptionScope(v string) Pair { return Pair{ - Key: pairEncryptionScope, + Key: "encryption_scope", Value: v, } } @@ -125,7 +166,7 @@ func WithEncryptionScope(v string) Pair { // ServiceFeatures set service features func WithServiceFeatures(v ServiceFeatures) Pair { return Pair{ - Key: pairServiceFeatures, + Key: "service_features", Value: v, } } @@ -135,24 +176,58 @@ func WithServiceFeatures(v ServiceFeatures) Pair { // StorageFeatures set storage features func WithStorageFeatures(v StorageFeatures) Pair { return Pair{ - Key: pairStorageFeatures, + Key: "storage_features", Value: v, } } +var pairMap = map[string]string{ + "access_tier": "string", + "content_md5": "string", + "content_type": "string", + "context": "context.Context", + "continuation_token": "string", + "credential": "string", + "default_service_pairs": "DefaultServicePairs", + "default_storage_pairs": "DefaultStoragePairs", + "encryption_key": "[]byte", + "encryption_scope": "string", + "endpoint": "string", + "expire": "int", + "http_client_options": "*httpclient.Options", + "interceptor": "Interceptor", + "io_callback": "func([]byte)", + "list_mode": "ListMode", + "location": "string", + "multipart_id": "string", + "name": "string", + "object_mode": "ObjectMode", + "offset": "int64", + "service_features": "ServiceFeatures", + "size": "int64", + "storage_features": "StorageFeatures", + "work_dir": "string", +} var ( _ Servicer = &Service{} ) type ServiceFeatures struct { - LooseOperationAll bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. + LooseOperationAll bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. LooseOperationCreate bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. LooseOperationDelete bool - LooseOperationGet bool - LooseOperationList bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. + LooseOperationGet bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. + LooseOperationList bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. VirtualOperationAll bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. VirtualPairAll bool } @@ -196,7 +271,7 @@ func parsePairServiceNew(opts []Pair) (pairServiceNew, error) { result.HasEndpoint = true result.Endpoint = v.Value.(string) // Optional pairs - case pairDefaultServicePairs: + case "default_service_pairs": if result.HasDefaultServicePairs { continue } @@ -208,7 +283,7 @@ func parsePairServiceNew(opts []Pair) (pairServiceNew, error) { } result.HasHTTPClientOptions = true result.HTTPClientOptions = v.Value.(*httpclient.Options) - case pairServiceFeatures: + case "service_features": if result.HasServiceFeatures { continue } @@ -246,23 +321,10 @@ func (s *Service) parsePairServiceCreate(opts []Pair) (pairServiceCreate, error) } for _, v := range opts { - // isUnsupportedPair records whether current pair is unsupported. - isUnsupportedPair := false - switch v.Key { default: - isUnsupportedPair = true - } - - if !isUnsupportedPair { - continue + return pairServiceCreate{}, services.PairUnsupportedError{Pair: v} } - - // If user enables the loose operation feature, we will ignore PairUnsupportedError. - if s.features.LooseOperationAll || s.features.LooseOperationCreate { - continue - } - return pairServiceCreate{}, services.PairUnsupportedError{Pair: v} } // Check required pairs. @@ -282,23 +344,10 @@ func (s *Service) parsePairServiceDelete(opts []Pair) (pairServiceDelete, error) } for _, v := range opts { - // isUnsupportedPair records whether current pair is unsupported. - isUnsupportedPair := false - switch v.Key { default: - isUnsupportedPair = true + return pairServiceDelete{}, services.PairUnsupportedError{Pair: v} } - - if !isUnsupportedPair { - continue - } - - // If user enables the loose operation feature, we will ignore PairUnsupportedError. - if s.features.LooseOperationAll || s.features.LooseOperationDelete { - continue - } - return pairServiceDelete{}, services.PairUnsupportedError{Pair: v} } // Check required pairs. @@ -318,23 +367,10 @@ func (s *Service) parsePairServiceGet(opts []Pair) (pairServiceGet, error) { } for _, v := range opts { - // isUnsupportedPair records whether current pair is unsupported. - isUnsupportedPair := false - switch v.Key { default: - isUnsupportedPair = true + return pairServiceGet{}, services.PairUnsupportedError{Pair: v} } - - if !isUnsupportedPair { - continue - } - - // If user enables the loose operation feature, we will ignore PairUnsupportedError. - if s.features.LooseOperationAll || s.features.LooseOperationGet { - continue - } - return pairServiceGet{}, services.PairUnsupportedError{Pair: v} } // Check required pairs. @@ -354,23 +390,10 @@ func (s *Service) parsePairServiceList(opts []Pair) (pairServiceList, error) { } for _, v := range opts { - // isUnsupportedPair records whether current pair is unsupported. - isUnsupportedPair := false - switch v.Key { default: - isUnsupportedPair = true - } - - if !isUnsupportedPair { - continue + return pairServiceList{}, services.PairUnsupportedError{Pair: v} } - - // If user enables the loose operation feature, we will ignore PairUnsupportedError. - if s.features.LooseOperationAll || s.features.LooseOperationList { - continue - } - return pairServiceList{}, services.PairUnsupportedError{Pair: v} } // Check required pairs. @@ -486,23 +509,43 @@ var ( ) type StorageFeatures struct { - LooseOperationAll bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. + LooseOperationAll bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. LooseOperationCommitAppend bool - LooseOperationCreate bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. + LooseOperationCreate bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. LooseOperationCreateAppend bool - LooseOperationCreateDir bool - LooseOperationDelete bool - LooseOperationList bool - LooseOperationMetadata bool - LooseOperationRead bool - LooseOperationStat bool - LooseOperationWrite bool - LooseOperationWriteAppend bool - - VirtualOperationAll bool - VirtualOperationCreateDir bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. + LooseOperationCreateDir bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. + LooseOperationDelete bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. + LooseOperationList bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. + LooseOperationMetadata bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. + LooseOperationRead bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. + LooseOperationStat bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. + LooseOperationWrite bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. + LooseOperationWriteAppend bool + + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. + VirtualOperationAll bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. VirtualPairAll bool + // VirtualDir virtual_dir feature is designed for a service that doesn't have native dir support but wants to provide simulated operations. + // + // - If this feature is disabled (the default behavior), the service will behave like it doesn't have any dir support. + // - If this feature is enabled, the service will support simulated dir behavior in create_dir, create, list, delete, and so on. + // + // This feature was introduced in GSP-109. + VirtualDir bool } // pairStorageNew is the parsed struct @@ -537,13 +580,13 @@ func parsePairStorageNew(opts []Pair) (pairStorageNew, error) { result.HasName = true result.Name = v.Value.(string) // Optional pairs - case pairDefaultStoragePairs: + case "default_storage_pairs": if result.HasDefaultStoragePairs { continue } result.HasDefaultStoragePairs = true result.DefaultStoragePairs = v.Value.(DefaultStoragePairs) - case pairStorageFeatures: + case "storage_features": if result.HasStorageFeatures { continue } @@ -591,23 +634,10 @@ func (s *Storage) parsePairStorageCommitAppend(opts []Pair) (pairStorageCommitAp } for _, v := range opts { - // isUnsupportedPair records whether current pair is unsupported. - isUnsupportedPair := false - switch v.Key { default: - isUnsupportedPair = true + return pairStorageCommitAppend{}, services.PairUnsupportedError{Pair: v} } - - if !isUnsupportedPair { - continue - } - - // If user enables the loose operation feature, we will ignore PairUnsupportedError. - if s.features.LooseOperationAll || s.features.LooseOperationCommitAppend { - continue - } - return pairStorageCommitAppend{}, services.PairUnsupportedError{Pair: v} } // Check required pairs. @@ -629,9 +659,6 @@ func (s *Storage) parsePairStorageCreate(opts []Pair) (pairStorageCreate, error) } for _, v := range opts { - // isUnsupportedPair records whether current pair is unsupported. - isUnsupportedPair := false - switch v.Key { case "object_mode": if result.HasObjectMode { @@ -641,18 +668,8 @@ func (s *Storage) parsePairStorageCreate(opts []Pair) (pairStorageCreate, error) result.ObjectMode = v.Value.(ObjectMode) continue default: - isUnsupportedPair = true - } - - if !isUnsupportedPair { - continue + return pairStorageCreate{}, services.PairUnsupportedError{Pair: v} } - - // If user enables the loose operation feature, we will ignore PairUnsupportedError. - if s.features.LooseOperationAll || s.features.LooseOperationCreate { - continue - } - return pairStorageCreate{}, services.PairUnsupportedError{Pair: v} } // Check required pairs. @@ -678,9 +695,6 @@ func (s *Storage) parsePairStorageCreateAppend(opts []Pair) (pairStorageCreateAp } for _, v := range opts { - // isUnsupportedPair records whether current pair is unsupported. - isUnsupportedPair := false - switch v.Key { case "content_type": if result.HasContentType { @@ -689,14 +703,14 @@ func (s *Storage) parsePairStorageCreateAppend(opts []Pair) (pairStorageCreateAp result.HasContentType = true result.ContentType = v.Value.(string) continue - case pairEncryptionKey: + case "encryption_key": if result.HasEncryptionKey { continue } result.HasEncryptionKey = true result.EncryptionKey = v.Value.([]byte) continue - case pairEncryptionScope: + case "encryption_scope": if result.HasEncryptionScope { continue } @@ -704,18 +718,8 @@ func (s *Storage) parsePairStorageCreateAppend(opts []Pair) (pairStorageCreateAp result.EncryptionScope = v.Value.(string) continue default: - isUnsupportedPair = true + return pairStorageCreateAppend{}, services.PairUnsupportedError{Pair: v} } - - if !isUnsupportedPair { - continue - } - - // If user enables the loose operation feature, we will ignore PairUnsupportedError. - if s.features.LooseOperationAll || s.features.LooseOperationCreateAppend { - continue - } - return pairStorageCreateAppend{}, services.PairUnsupportedError{Pair: v} } // Check required pairs. @@ -725,13 +729,9 @@ func (s *Storage) parsePairStorageCreateAppend(opts []Pair) (pairStorageCreateAp // pairStorageCreateDir is the parsed struct type pairStorageCreateDir struct { - pairs []Pair - HasAccessTier bool - AccessTier string - HasEncryptionKey bool - EncryptionKey []byte - HasEncryptionScope bool - EncryptionScope string + pairs []Pair + HasAccessTier bool + AccessTier string } // parsePairStorageCreateDir will parse Pair slice into *pairStorageCreateDir @@ -741,44 +741,17 @@ func (s *Storage) parsePairStorageCreateDir(opts []Pair) (pairStorageCreateDir, } for _, v := range opts { - // isUnsupportedPair records whether current pair is unsupported. - isUnsupportedPair := false - switch v.Key { - case pairAccessTier: + case "access_tier": if result.HasAccessTier { continue } result.HasAccessTier = true result.AccessTier = v.Value.(string) continue - case pairEncryptionKey: - if result.HasEncryptionKey { - continue - } - result.HasEncryptionKey = true - result.EncryptionKey = v.Value.([]byte) - continue - case pairEncryptionScope: - if result.HasEncryptionScope { - continue - } - result.HasEncryptionScope = true - result.EncryptionScope = v.Value.(string) - continue default: - isUnsupportedPair = true + return pairStorageCreateDir{}, services.PairUnsupportedError{Pair: v} } - - if !isUnsupportedPair { - continue - } - - // If user enables the loose operation feature, we will ignore PairUnsupportedError. - if s.features.LooseOperationAll || s.features.LooseOperationCreateDir { - continue - } - return pairStorageCreateDir{}, services.PairUnsupportedError{Pair: v} } // Check required pairs. @@ -800,9 +773,6 @@ func (s *Storage) parsePairStorageDelete(opts []Pair) (pairStorageDelete, error) } for _, v := range opts { - // isUnsupportedPair records whether current pair is unsupported. - isUnsupportedPair := false - switch v.Key { case "object_mode": if result.HasObjectMode { @@ -812,18 +782,8 @@ func (s *Storage) parsePairStorageDelete(opts []Pair) (pairStorageDelete, error) result.ObjectMode = v.Value.(ObjectMode) continue default: - isUnsupportedPair = true - } - - if !isUnsupportedPair { - continue + return pairStorageDelete{}, services.PairUnsupportedError{Pair: v} } - - // If user enables the loose operation feature, we will ignore PairUnsupportedError. - if s.features.LooseOperationAll || s.features.LooseOperationDelete { - continue - } - return pairStorageDelete{}, services.PairUnsupportedError{Pair: v} } // Check required pairs. @@ -845,9 +805,6 @@ func (s *Storage) parsePairStorageList(opts []Pair) (pairStorageList, error) { } for _, v := range opts { - // isUnsupportedPair records whether current pair is unsupported. - isUnsupportedPair := false - switch v.Key { case "list_mode": if result.HasListMode { @@ -857,18 +814,8 @@ func (s *Storage) parsePairStorageList(opts []Pair) (pairStorageList, error) { result.ListMode = v.Value.(ListMode) continue default: - isUnsupportedPair = true - } - - if !isUnsupportedPair { - continue - } - - // If user enables the loose operation feature, we will ignore PairUnsupportedError. - if s.features.LooseOperationAll || s.features.LooseOperationList { - continue + return pairStorageList{}, services.PairUnsupportedError{Pair: v} } - return pairStorageList{}, services.PairUnsupportedError{Pair: v} } // Check required pairs. @@ -888,23 +835,10 @@ func (s *Storage) parsePairStorageMetadata(opts []Pair) (pairStorageMetadata, er } for _, v := range opts { - // isUnsupportedPair records whether current pair is unsupported. - isUnsupportedPair := false - switch v.Key { default: - isUnsupportedPair = true + return pairStorageMetadata{}, services.PairUnsupportedError{Pair: v} } - - if !isUnsupportedPair { - continue - } - - // If user enables the loose operation feature, we will ignore PairUnsupportedError. - if s.features.LooseOperationAll || s.features.LooseOperationMetadata { - continue - } - return pairStorageMetadata{}, services.PairUnsupportedError{Pair: v} } // Check required pairs. @@ -934,18 +868,15 @@ func (s *Storage) parsePairStorageRead(opts []Pair) (pairStorageRead, error) { } for _, v := range opts { - // isUnsupportedPair records whether current pair is unsupported. - isUnsupportedPair := false - switch v.Key { - case pairEncryptionKey: + case "encryption_key": if result.HasEncryptionKey { continue } result.HasEncryptionKey = true result.EncryptionKey = v.Value.([]byte) continue - case pairEncryptionScope: + case "encryption_scope": if result.HasEncryptionScope { continue } @@ -974,18 +905,8 @@ func (s *Storage) parsePairStorageRead(opts []Pair) (pairStorageRead, error) { result.Size = v.Value.(int64) continue default: - isUnsupportedPair = true - } - - if !isUnsupportedPair { - continue + return pairStorageRead{}, services.PairUnsupportedError{Pair: v} } - - // If user enables the loose operation feature, we will ignore PairUnsupportedError. - if s.features.LooseOperationAll || s.features.LooseOperationRead { - continue - } - return pairStorageRead{}, services.PairUnsupportedError{Pair: v} } // Check required pairs. @@ -1011,18 +932,15 @@ func (s *Storage) parsePairStorageStat(opts []Pair) (pairStorageStat, error) { } for _, v := range opts { - // isUnsupportedPair records whether current pair is unsupported. - isUnsupportedPair := false - switch v.Key { - case pairEncryptionKey: + case "encryption_key": if result.HasEncryptionKey { continue } result.HasEncryptionKey = true result.EncryptionKey = v.Value.([]byte) continue - case pairEncryptionScope: + case "encryption_scope": if result.HasEncryptionScope { continue } @@ -1037,18 +955,8 @@ func (s *Storage) parsePairStorageStat(opts []Pair) (pairStorageStat, error) { result.ObjectMode = v.Value.(ObjectMode) continue default: - isUnsupportedPair = true - } - - if !isUnsupportedPair { - continue - } - - // If user enables the loose operation feature, we will ignore PairUnsupportedError. - if s.features.LooseOperationAll || s.features.LooseOperationStat { - continue + return pairStorageStat{}, services.PairUnsupportedError{Pair: v} } - return pairStorageStat{}, services.PairUnsupportedError{Pair: v} } // Check required pairs. @@ -1080,11 +988,8 @@ func (s *Storage) parsePairStorageWrite(opts []Pair) (pairStorageWrite, error) { } for _, v := range opts { - // isUnsupportedPair records whether current pair is unsupported. - isUnsupportedPair := false - switch v.Key { - case pairAccessTier: + case "access_tier": if result.HasAccessTier { continue } @@ -1105,14 +1010,14 @@ func (s *Storage) parsePairStorageWrite(opts []Pair) (pairStorageWrite, error) { result.HasContentType = true result.ContentType = v.Value.(string) continue - case pairEncryptionKey: + case "encryption_key": if result.HasEncryptionKey { continue } result.HasEncryptionKey = true result.EncryptionKey = v.Value.([]byte) continue - case pairEncryptionScope: + case "encryption_scope": if result.HasEncryptionScope { continue } @@ -1127,18 +1032,8 @@ func (s *Storage) parsePairStorageWrite(opts []Pair) (pairStorageWrite, error) { result.IoCallback = v.Value.(func([]byte)) continue default: - isUnsupportedPair = true + return pairStorageWrite{}, services.PairUnsupportedError{Pair: v} } - - if !isUnsupportedPair { - continue - } - - // If user enables the loose operation feature, we will ignore PairUnsupportedError. - if s.features.LooseOperationAll || s.features.LooseOperationWrite { - continue - } - return pairStorageWrite{}, services.PairUnsupportedError{Pair: v} } // Check required pairs. @@ -1164,9 +1059,6 @@ func (s *Storage) parsePairStorageWriteAppend(opts []Pair) (pairStorageWriteAppe } for _, v := range opts { - // isUnsupportedPair records whether current pair is unsupported. - isUnsupportedPair := false - switch v.Key { case "content_md5": if result.HasContentMd5 { @@ -1175,14 +1067,14 @@ func (s *Storage) parsePairStorageWriteAppend(opts []Pair) (pairStorageWriteAppe result.HasContentMd5 = true result.ContentMd5 = v.Value.(string) continue - case pairEncryptionKey: + case "encryption_key": if result.HasEncryptionKey { continue } result.HasEncryptionKey = true result.EncryptionKey = v.Value.([]byte) continue - case pairEncryptionScope: + case "encryption_scope": if result.HasEncryptionScope { continue } @@ -1190,18 +1082,8 @@ func (s *Storage) parsePairStorageWriteAppend(opts []Pair) (pairStorageWriteAppe result.EncryptionScope = v.Value.(string) continue default: - isUnsupportedPair = true - } - - if !isUnsupportedPair { - continue + return pairStorageWriteAppend{}, services.PairUnsupportedError{Pair: v} } - - // If user enables the loose operation feature, we will ignore PairUnsupportedError. - if s.features.LooseOperationAll || s.features.LooseOperationWriteAppend { - continue - } - return pairStorageWriteAppend{}, services.PairUnsupportedError{Pair: v} } // Check required pairs. @@ -1294,11 +1176,6 @@ func (s *Storage) CreateDirWithContext(ctx context.Context, path string, pairs . defer func() { err = s.formatError("create_dir", err, path) }() - // If virtual operation feature is not enabled, we will act like this operation is not implemented. - if !s.features.VirtualOperationAll && !s.features.VirtualOperationCreateDir { - err = NewOperationNotImplementedError("create_dir") - return - } pairs = append(pairs, s.defaultPairs.CreateDir...) var opt pairStorageCreateDir @@ -1513,4 +1390,5 @@ func (s *Storage) WriteAppendWithContext(ctx context.Context, o *Object, r io.Re func init() { services.RegisterServicer(Type, NewServicer) services.RegisterStorager(Type, NewStorager) + services.RegisterSchema(Type, pairMap) } diff --git a/go.mod b/go.mod index 76e3daf..4c20669 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,8 @@ go 1.15 require ( github.com/Azure/azure-pipeline-go v0.2.3 github.com/Azure/azure-storage-blob-go v0.13.0 - github.com/beyondstorage/go-integration-test/v4 v4.1.0 - github.com/beyondstorage/go-storage/v4 v4.1.0 + github.com/beyondstorage/go-endpoint v1.0.1 + github.com/beyondstorage/go-integration-test/v4 v4.1.1 + github.com/beyondstorage/go-storage/v4 v4.2.0 github.com/google/uuid v1.2.0 ) diff --git a/go.sum b/go.sum index 2649fcd..32bc2a6 100644 --- a/go.sum +++ b/go.sum @@ -14,20 +14,23 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/Xuanwo/templateutils v0.1.0 h1:WpkWOqQtIQ2vAIpJLa727DdN8WtxhUkkbDGa6UhntJY= github.com/Xuanwo/templateutils v0.1.0/go.mod h1:OdE0DJ+CJxDBq6psX5DPV+gOZi8bhuHuVUpPCG++Wb8= -github.com/beyondstorage/go-integration-test/v4 v4.1.0 h1:/cKM5uC+tW1mYAM/FTQlMq3EZMAb8hBm8gL4ZHk9InQ= -github.com/beyondstorage/go-integration-test/v4 v4.1.0/go.mod h1:W6Dhve1tbEpWAR1WSRuJl4UYpMNpM2QTAcb1dCpuEUI= -github.com/beyondstorage/go-storage/v4 v4.1.0 h1:O5SuSoTvs0KeXaZ/dYhpURlki6PjTNPyIzsJMY9sJWw= -github.com/beyondstorage/go-storage/v4 v4.1.0/go.mod h1:dK5DFnvKQI70bfpM1MLU9QqDYB12Z/dFV5sUnbJ/AoU= -github.com/beyondstorage/specs/go v0.0.0-20210608070420-9185b588aa58 h1:AvxsyR0bSSBi90WtYEMdAkM4Sm+Xxr7JVGXAQVVfeOo= -github.com/beyondstorage/specs/go v0.0.0-20210608070420-9185b588aa58/go.mod h1:1Az5o44awI/Ljop+ppO2djyezVfdKKb1RjJ6+M+a5XQ= +github.com/beyondstorage/go-endpoint v1.0.1 h1:F8x2dGLMu9je6g7zPbKoxCXDlug97K26SeCx7KEHgyg= +github.com/beyondstorage/go-endpoint v1.0.1/go.mod h1:P2hknaGrziOJJKySv/XnAiVw/d3v12/LZu2gSxEx4nM= +github.com/beyondstorage/go-integration-test/v4 v4.1.1 h1:9bSXKbr6hLb4+ZsmAhWE32fvqhyrpub4U4qgBGeth4A= +github.com/beyondstorage/go-integration-test/v4 v4.1.1/go.mod h1:ihtCaOJvaHGE0v+IhY6ZUF5NU1IND6xmdrJI9Lq/jhc= +github.com/beyondstorage/go-storage/v4 v4.2.0 h1:J0xqqy4qEQRtIS2zUWMA5wRXVHx/cxX5fHsU2ezA3+I= +github.com/beyondstorage/go-storage/v4 v4.2.0/go.mod h1:rUNzOXcikYk5w0ewvNsKbztg7ndQDyDvjDuP0bznSLU= +github.com/beyondstorage/specs/go v0.0.0-20210623065218-d1c2d7d81259 h1:mW9XpHLc6pdXBRnsha1VlqF0rNsB/Oc+8l+5UYngmRA= +github.com/beyondstorage/specs/go v0.0.0-20210623065218-d1c2d7d81259/go.mod h1:vF/Q0P1tCvhVAUrxg7i6NvrARRMQVTAuQdDNqpSzR1w= github.com/dave/dst v0.26.2 h1:lnxLAKI3tx7MgLNVDirFCsDTlTG9nKTk7GcptKcWSwY= github.com/dave/dst v0.26.2/go.mod h1:UMDJuIRPfyUCC78eFuB+SV/WI8oDeyFDvM/JR6NI3IU= github.com/dave/gopackages v0.0.0-20170318123100-46e7023ec56e/go.mod h1:i00+b/gKdIDIxuLDFob7ustLAVqhsZRk2qVZrArELGQ= github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= github.com/dave/kerr v0.0.0-20170318121727-bc25dd6abe8e/go.mod h1:qZqlPyPvfsDJt+3wHJ1EvSXDuVjFTK0j2p/ca+gtsb8= github.com/dave/rebecca v0.9.1/go.mod h1:N6XYdMD/OKw3lkF3ywh8Z6wPGuwNFDNtWYEMFWEmXBA= -github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/golang/mock v1.5.0 h1:jlYHihg//f7RRwuPfptm04yp4s7O6Kw8EZiVYIGcH0g= @@ -51,19 +54,22 @@ github.com/mattn/go-ieproxy v0.0.1 h1:qiyop7gCflfhwCzGyeT0gro3sF9AIg9HU98JORTkqf github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/pelletier/go-toml v1.9.2 h1:7NiByeVF4jKSG1lDF3X8LTIkq2/bu+1uYbIm1eS5tzk= -github.com/pelletier/go-toml v1.9.2/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ= +github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -86,6 +92,7 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sys v0.0.0-20180903190138-2b024373dcd9/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200828194041-157a740278f4 h1:kCCpuwSAoYJPkNc6x0xT9yTtV4oKtARo4RGBQWOfg9E= golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/service.toml b/service.toml index 3ee753c..e593377 100644 --- a/service.toml +++ b/service.toml @@ -7,6 +7,7 @@ required = ["credential", "endpoint"] optional = ["service_features", "default_service_pairs", "http_client_options"] [namespace.storage] +features = ["virtual_dir"] implement = ["appender", "direr"] [namespace.storage.new] @@ -38,8 +39,7 @@ optional = ["content_md5", "encryption_key", "encryption_scope"] optional = ["encryption_key", "encryption_scope", "object_mode"] [namespace.storage.op.create_dir] -simulated = true -optional = ["access_tier", "encryption_key", "encryption_scope"] +optional = ["access_tier"] [pairs.service_features] type = "ServiceFeatures" diff --git a/storage.go b/storage.go index 1ff3d31..02010d2 100644 --- a/storage.go +++ b/storage.go @@ -3,11 +3,13 @@ package azblob import ( "context" "encoding/base64" + "fmt" "io" "strconv" "github.com/Azure/azure-storage-blob-go/azblob" + ps "github.com/beyondstorage/go-storage/v4/pairs" "github.com/beyondstorage/go-storage/v4/pkg/iowrap" "github.com/beyondstorage/go-storage/v4/services" . "github.com/beyondstorage/go-storage/v4/types" @@ -21,6 +23,10 @@ func (s *Storage) create(path string, opt pairStorageCreate) (o *Object) { rp := s.getAbsPath(path) if opt.HasObjectMode && opt.ObjectMode.IsDir() { + if !s.features.VirtualDir { + return + } + rp += "/" o = s.newObject(true) o.Mode = ModeDir @@ -65,6 +71,11 @@ func (s *Storage) createAppend(ctx context.Context, path string, opt pairStorage } func (s *Storage) createDir(ctx context.Context, path string, opt pairStorageCreateDir) (o *Object, err error) { + if !s.features.VirtualDir { + err = NewOperationNotImplementedError("create_dir") + return + } + rp := s.getAbsPath(path) // Specify a character or string delimiter within a blob name to create a virtual hierarchy. @@ -76,18 +87,10 @@ func (s *Storage) createDir(ctx context.Context, path string, opt pairStorageCre accessTier = azblob.AccessTierType(opt.AccessTier) } - var cpk azblob.ClientProvidedKeyOptions - if opt.HasEncryptionKey { - cpk, err = calculateEncryptionHeaders(opt.EncryptionKey, opt.EncryptionScope) - if err != nil { - return - } - } - _, err = s.bucket.NewBlockBlobURL(rp).Upload( ctx, iowrap.SizedReadSeekCloser(nil, 0), azblob.BlobHTTPHeaders{}, azblob.Metadata{}, azblob.BlobAccessConditions{}, - accessTier, azblob.BlobTagsMap{}, cpk) + accessTier, azblob.BlobTagsMap{}, azblob.ClientProvidedKeyOptions{}) if err != nil { return } @@ -103,6 +106,11 @@ func (s *Storage) delete(ctx context.Context, path string, opt pairStorageDelete rp := s.getAbsPath(path) if opt.HasObjectMode && opt.ObjectMode.IsDir() { + if !s.features.VirtualDir { + err = services.PairUnsupportedError{Pair: ps.WithObjectMode(opt.ObjectMode)} + return + } + rp += "/" } @@ -145,6 +153,10 @@ func (s *Storage) metadata(opt pairStorageMetadata) (meta *StorageMeta) { meta = NewStorageMeta() meta.Name = s.name meta.WorkDir = s.workDir + meta.SetWriteSizeMaximum(WriteSizeMaximum) + meta.SetAppendSizeMaximum(AppendSizeMaximum) + meta.SetAppendNumberMaximum(AppendNumberMaximum) + meta.SetAppendTotalSizeMaximum(AppendBlobIfMaxSizeLessThanOrEqual) return meta } @@ -258,6 +270,11 @@ func (s *Storage) stat(ctx context.Context, path string, opt pairStorageStat) (o rp := s.getAbsPath(path) if opt.HasObjectMode && opt.ObjectMode.IsDir() { + if !s.features.VirtualDir { + err = services.PairUnsupportedError{Pair: ps.WithObjectMode(opt.ObjectMode)} + return + } + rp += "/" } @@ -297,7 +314,7 @@ func (s *Storage) stat(ctx context.Context, path string, opt pairStorageStat) (o o.SetContentMd5(base64.StdEncoding.EncodeToString(v)) } - var sm ObjectMetadata + var sm ObjectSystemMetadata if v := output.AccessTier(); v != "" { sm.AccessTier = v } @@ -310,12 +327,17 @@ func (s *Storage) stat(ctx context.Context, path string, opt pairStorageStat) (o if v, err := strconv.ParseBool(output.IsServerEncrypted()); err == nil { sm.ServerEncrypted = v } - o.SetServiceMetadata(sm) + o.SetSystemMetadata(sm) return o, nil } func (s *Storage) write(ctx context.Context, path string, r io.Reader, size int64, opt pairStorageWrite) (n int64, err error) { + if size > WriteSizeMaximum { + err = fmt.Errorf("size limit exceeded: %w", services.ErrRestrictionDissatisfied) + return + } + rp := s.getAbsPath(path) if opt.HasIoCallback { @@ -356,6 +378,10 @@ func (s *Storage) write(ctx context.Context, path string, r io.Reader, size int6 } func (s *Storage) writeAppend(ctx context.Context, o *Object, r io.Reader, size int64, opt pairStorageWriteAppend) (n int64, err error) { + if size > AppendSizeMaximum { + err = fmt.Errorf("size limit exceeded: %w", services.ErrRestrictionDissatisfied) + return + } rp := o.GetID() offset, _ := o.GetAppendOffset() diff --git a/tests/utils_test.go b/tests/utils_test.go index cb7edf3..1d4671b 100644 --- a/tests/utils_test.go +++ b/tests/utils_test.go @@ -19,7 +19,7 @@ func setupTest(t *testing.T) types.Storager { ps.WithEndpoint(os.Getenv("STORAGE_AZBLOB_ENDPOINT")), ps.WithWorkDir("/"+uuid.New().String()+"/"), azblob.WithStorageFeatures(azblob.StorageFeatures{ - VirtualOperationAll: true, + VirtualDir: true, }), ) if err != nil { diff --git a/utils.go b/utils.go index bcd6d20..563cdfc 100644 --- a/utils.go +++ b/utils.go @@ -13,9 +13,9 @@ import ( "github.com/Azure/azure-pipeline-go/pipeline" "github.com/Azure/azure-storage-blob-go/azblob" + "github.com/beyondstorage/go-endpoint" ps "github.com/beyondstorage/go-storage/v4/pairs" "github.com/beyondstorage/go-storage/v4/pkg/credential" - "github.com/beyondstorage/go-storage/v4/pkg/endpoint" "github.com/beyondstorage/go-storage/v4/pkg/httpclient" "github.com/beyondstorage/go-storage/v4/services" typ "github.com/beyondstorage/go-storage/v4/types" @@ -105,7 +105,17 @@ func newServicer(pairs ...typ.Pair) (srv *Service, err error) { return nil, err } - primaryURL, _ := url.Parse(ep.String()) + var uri string + switch ep.Protocol() { + case endpoint.ProtocolHTTP: + uri, _, _ = ep.HTTP() + case endpoint.ProtocolHTTPS: + uri, _, _ = ep.HTTPS() + default: + return nil, services.PairUnsupportedError{Pair: ps.WithEndpoint(opt.Endpoint)} + } + + primaryURL, _ := url.Parse(uri) cred, err := credential.Parse(opt.Credential) if err != nil { @@ -295,7 +305,7 @@ func (s *Storage) formatFileObject(v azblob.BlobItemInternal) (o *typ.Object, er o.SetContentMd5(base64.StdEncoding.EncodeToString(v.Properties.ContentMD5)) } - var sm ObjectMetadata + var sm ObjectSystemMetadata if value := v.Properties.AccessTier; value != "" { sm.AccessTier = string(value) } @@ -308,7 +318,7 @@ func (s *Storage) formatFileObject(v azblob.BlobItemInternal) (o *typ.Object, er if v.Properties.ServerEncrypted != nil { sm.ServerEncrypted = *v.Properties.ServerEncrypted } - o.SetServiceMetadata(sm) + o.SetSystemMetadata(sm) return o, nil } @@ -336,8 +346,18 @@ func calculateEncryptionHeaders(key []byte, scope string) (cpk azblob.ClientProv const ( // AppendBlobIfMaxSizeLessThanOrEqual ensures that the AppendBlock operation succeeds only if the append blob's size is less than or equal to a value. - // For more information, see https://docs.microsoft.com/rest/api/storageservices/append-block. + // ref: https://docs.microsoft.com/rest/api/storageservices/append-block. AppendBlobIfMaxSizeLessThanOrEqual = 4 * 1024 * 1024 * 50000 + // AppendSizeMaximum is the max append size in per append operation. + // ref: https://docs.microsoft.com/rest/api/storageservices/append-block. + AppendSizeMaximum = 4 * 1024 * 1024 + // AppendNumberMaximum is the max append numbers in append operation. + // ref: https://docs.microsoft.com/rest/api/storageservices/append-block. + AppendNumberMaximum = 50000 + + // WriteSizeMaximum is the maximum size for write operation. + // ref: https://docs.microsoft.com/en-us/rest/api/storageservices/put-blob + WriteSizeMaximum = 5000 * 1024 * 1024 ) func checkError(err error, expect azblob.ServiceCodeType) bool {