From d8a47f3f41036f7a4fa9a0ba8cd7aeb44b9437bf Mon Sep 17 00:00:00 2001 From: JinnyYi <82445294+JinnyYi@users.noreply.github.com> Date: Tue, 29 Jun 2021 09:55:26 +0800 Subject: [PATCH] Implement GSP-93, GSP-97, GSP-109 and GSP-117 (#31) * Implement GSP-93 and GSP-109 * Implement GSP-117 Rename Service to System as the Opposite to Global * pkg/endpoint deprecated by go-endpoint * Implement GSP-97 Add Restrictions In Storage Metadata --- generated.go | 620 ++++++++++++++++++++---------------------- go.mod | 5 +- go.sum | 28 +- service.toml | 12 +- storage.go | 102 ++++++- tests/storage_test.go | 7 + tests/utils_test.go | 3 + utils.go | 28 +- 8 files changed, 454 insertions(+), 351 deletions(-) diff --git a/generated.go b/generated.go index ee035fa..4fda2cd 100644 --- a/generated.go +++ b/generated.go @@ -21,35 +21,9 @@ var _ httpclient.Options // Type is the type for oss const Type = "oss" -// Service available pairs. -const ( - // DefaultServicePairs set default pairs for service actions - pairDefaultServicePairs = "oss_default_service_pairs" - // DefaultStoragePairs set default pairs for storager actions - pairDefaultStoragePairs = "oss_default_storage_pairs" - // ServerSideDataEncryption specifies the encryption algorithm when server_side_encryption is KMS. Can only be set to SM4. If this is not set, AES256 will be used. - // - // For Chinese users, refer to https://help.aliyun.com/document_detail/31871.html for details. - // - // For global users, refer to https://www.alibabacloud.com/help/doc-detail/31871.htm for details, and double-check whether SM4 can be used. - pairServerSideDataEncryption = "oss_server_side_data_encryption" - // ServerSideEncryption specifies the encryption algorithm. Can be AES256, KMS or SM4. - // - // For Chinese users, refer to https://help.aliyun.com/document_detail/31871.html for details. - // - // For global users, refer to https://www.alibabacloud.com/help/doc-detail/31871.htm for details, and double-check whether SM4 can be used. - pairServerSideEncryption = "oss_server_side_encryption" - // ServerSideEncryptionKeyID is the KMS-managed user master key. Only valid when server_side_encryption is KMS. - pairServerSideEncryptionKeyID = "oss_server_side_encryption_key_id" - // ServiceFeatures set service features - pairServiceFeatures = "oss_service_features" - // StorageClass - pairStorageClass = "oss_storage_class" - // StorageFeatures set storage features - pairStorageFeatures = "oss_storage_features" -) - // ObjectMetadata stores service metadata for object. +// +// Deprecated: Use ObjectSystemMetadata instead. type ObjectMetadata struct { // ServerSideEncryption ServerSideEncryption string @@ -63,6 +37,8 @@ type ObjectMetadata struct { // // - This function should not be called by service implementer. // - The returning ObjectMetadata is read only and should not be modified. +// +// Deprecated: Use GetObjectSystemMetadata instead. func GetObjectMetadata(o *Object) ObjectMetadata { om, ok := o.GetServiceMetadata() if ok { @@ -74,16 +50,69 @@ func GetObjectMetadata(o *Object) ObjectMetadata { // setObjectMetadata will set ObjectMetadata into Object. // // - This function should only be called once, please make sure all data has been written before set. +// +// Deprecated: Use setObjectSystemMetadata instead. func setObjectMetadata(o *Object, om ObjectMetadata) { o.SetServiceMetadata(om) } +// ObjectSystemMetadata stores system metadata for object. +type ObjectSystemMetadata struct { + // ServerSideEncryption + ServerSideEncryption string + // ServerSideEncryptionKeyID + ServerSideEncryptionKeyID string + // StorageClass + StorageClass string +} + +// GetObjectSystemMetadata will get ObjectSystemMetadata from Object. +// +// - This function should not be called by service implementer. +// - The returning ObjectServiceMetadata is read only and should not be modified. +func GetObjectSystemMetadata(o *Object) ObjectSystemMetadata { + sm, ok := o.GetSystemMetadata() + if ok { + return sm.(ObjectSystemMetadata) + } + return ObjectSystemMetadata{} +} + +// setObjectSystemMetadata will set ObjectSystemMetadata into Object. +// +// - This function should only be called once, please make sure all data has been written before set. +func setObjectSystemMetadata(o *Object, sm ObjectSystemMetadata) { + o.SetSystemMetadata(sm) +} + +// StorageSystemMetadata stores system metadata for storage meta. +type StorageSystemMetadata struct { +} + +// GetStorageSystemMetadata will get SystemMetadata from StorageMeta. +// +// - The returning StorageSystemMetadata is read only and should not be modified. +func GetStorageSystemMetadata(s *StorageMeta) StorageSystemMetadata { + sm, ok := s.GetSystemMetadata() + if ok { + return sm.(StorageSystemMetadata) + } + return StorageSystemMetadata{} +} + +// setStorageSystemMetadata will set SystemMetadata into StorageMeta. +// +// - This function should only be called once, please make sure all data has been written before set. +func setStorageSystemMetadata(s *StorageMeta, sm StorageSystemMetadata) { + s.SetSystemMetadata(sm) +} + // WithDefaultServicePairs will apply default_service_pairs value to Options. // // DefaultServicePairs set default pairs for service actions func WithDefaultServicePairs(v DefaultServicePairs) Pair { return Pair{ - Key: pairDefaultServicePairs, + Key: "default_service_pairs", Value: v, } } @@ -93,7 +122,7 @@ func WithDefaultServicePairs(v DefaultServicePairs) Pair { // DefaultStoragePairs set default pairs for storager actions func WithDefaultStoragePairs(v DefaultStoragePairs) Pair { return Pair{ - Key: pairDefaultStoragePairs, + Key: "default_storage_pairs", Value: v, } } @@ -107,7 +136,7 @@ func WithDefaultStoragePairs(v DefaultStoragePairs) Pair { // For global users, refer to https://www.alibabacloud.com/help/doc-detail/31871.htm for details, and double-check whether SM4 can be used. func WithServerSideDataEncryption(v string) Pair { return Pair{ - Key: pairServerSideDataEncryption, + Key: "server_side_data_encryption", Value: v, } } @@ -121,7 +150,7 @@ func WithServerSideDataEncryption(v string) Pair { // For global users, refer to https://www.alibabacloud.com/help/doc-detail/31871.htm for details, and double-check whether SM4 can be used. func WithServerSideEncryption(v string) Pair { return Pair{ - Key: pairServerSideEncryption, + Key: "server_side_encryption", Value: v, } } @@ -131,7 +160,7 @@ func WithServerSideEncryption(v string) Pair { // ServerSideEncryptionKeyID is the KMS-managed user master key. Only valid when server_side_encryption is KMS. func WithServerSideEncryptionKeyID(v string) Pair { return Pair{ - Key: pairServerSideEncryptionKeyID, + Key: "server_side_encryption_key_id", Value: v, } } @@ -141,7 +170,7 @@ func WithServerSideEncryptionKeyID(v string) Pair { // ServiceFeatures set service features func WithServiceFeatures(v ServiceFeatures) Pair { return Pair{ - Key: pairServiceFeatures, + Key: "service_features", Value: v, } } @@ -151,7 +180,7 @@ func WithServiceFeatures(v ServiceFeatures) Pair { // StorageClass func WithStorageClass(v string) Pair { return Pair{ - Key: pairStorageClass, + Key: "storage_class", Value: v, } } @@ -161,24 +190,59 @@ func WithStorageClass(v string) Pair { // StorageFeatures set storage features func WithStorageFeatures(v StorageFeatures) Pair { return Pair{ - Key: pairStorageFeatures, + Key: "storage_features", Value: v, } } +var pairMap = map[string]string{ + "content_md5": "string", + "content_type": "string", + "context": "context.Context", + "continuation_token": "string", + "credential": "string", + "default_service_pairs": "DefaultServicePairs", + "default_storage_pairs": "DefaultStoragePairs", + "endpoint": "string", + "expire": "int", + "http_client_options": "*httpclient.Options", + "interceptor": "Interceptor", + "io_callback": "func([]byte)", + "list_mode": "ListMode", + "location": "string", + "multipart_id": "string", + "name": "string", + "object_mode": "ObjectMode", + "offset": "int64", + "server_side_data_encryption": "string", + "server_side_encryption": "string", + "server_side_encryption_key_id": "string", + "service_features": "ServiceFeatures", + "size": "int64", + "storage_class": "string", + "storage_features": "StorageFeatures", + "work_dir": "string", +} var ( _ Servicer = &Service{} ) type ServiceFeatures struct { - LooseOperationAll bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. + LooseOperationAll bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. LooseOperationCreate bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. LooseOperationDelete bool - LooseOperationGet bool - LooseOperationList bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. + LooseOperationGet bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. + LooseOperationList bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. VirtualOperationAll bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. VirtualPairAll bool } @@ -216,7 +280,7 @@ func parsePairServiceNew(opts []Pair) (pairServiceNew, error) { result.HasCredential = true result.Credential = v.Value.(string) // Optional pairs - case pairDefaultServicePairs: + case "default_service_pairs": if result.HasDefaultServicePairs { continue } @@ -234,7 +298,7 @@ func parsePairServiceNew(opts []Pair) (pairServiceNew, error) { } result.HasHTTPClientOptions = true result.HTTPClientOptions = v.Value.(*httpclient.Options) - case pairServiceFeatures: + case "service_features": if result.HasServiceFeatures { continue } @@ -269,23 +333,10 @@ func (s *Service) parsePairServiceCreate(opts []Pair) (pairServiceCreate, error) } for _, v := range opts { - // isUnsupportedPair records whether current pair is unsupported. - isUnsupportedPair := false - switch v.Key { default: - isUnsupportedPair = true + return pairServiceCreate{}, services.PairUnsupportedError{Pair: v} } - - if !isUnsupportedPair { - continue - } - - // If user enables the loose operation feature, we will ignore PairUnsupportedError. - if s.features.LooseOperationAll || s.features.LooseOperationCreate { - continue - } - return pairServiceCreate{}, services.PairUnsupportedError{Pair: v} } // Check required pairs. @@ -305,23 +356,10 @@ func (s *Service) parsePairServiceDelete(opts []Pair) (pairServiceDelete, error) } for _, v := range opts { - // isUnsupportedPair records whether current pair is unsupported. - isUnsupportedPair := false - switch v.Key { default: - isUnsupportedPair = true + return pairServiceDelete{}, services.PairUnsupportedError{Pair: v} } - - if !isUnsupportedPair { - continue - } - - // If user enables the loose operation feature, we will ignore PairUnsupportedError. - if s.features.LooseOperationAll || s.features.LooseOperationDelete { - continue - } - return pairServiceDelete{}, services.PairUnsupportedError{Pair: v} } // Check required pairs. @@ -341,23 +379,10 @@ func (s *Service) parsePairServiceGet(opts []Pair) (pairServiceGet, error) { } for _, v := range opts { - // isUnsupportedPair records whether current pair is unsupported. - isUnsupportedPair := false - switch v.Key { default: - isUnsupportedPair = true + return pairServiceGet{}, services.PairUnsupportedError{Pair: v} } - - if !isUnsupportedPair { - continue - } - - // If user enables the loose operation feature, we will ignore PairUnsupportedError. - if s.features.LooseOperationAll || s.features.LooseOperationGet { - continue - } - return pairServiceGet{}, services.PairUnsupportedError{Pair: v} } // Check required pairs. @@ -377,23 +402,10 @@ func (s *Service) parsePairServiceList(opts []Pair) (pairServiceList, error) { } for _, v := range opts { - // isUnsupportedPair records whether current pair is unsupported. - isUnsupportedPair := false - switch v.Key { default: - isUnsupportedPair = true + return pairServiceList{}, services.PairUnsupportedError{Pair: v} } - - if !isUnsupportedPair { - continue - } - - // If user enables the loose operation feature, we will ignore PairUnsupportedError. - if s.features.LooseOperationAll || s.features.LooseOperationList { - continue - } - return pairServiceList{}, services.PairUnsupportedError{Pair: v} } // Check required pairs. @@ -504,30 +516,57 @@ func (s *Service) ListWithContext(ctx context.Context, pairs ...Pair) (sti *Stor var ( _ Appender = &Storage{} + _ Direr = &Storage{} _ Multiparter = &Storage{} _ Storager = &Storage{} ) type StorageFeatures struct { - LooseOperationAll bool - LooseOperationCommitAppend bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. + LooseOperationAll bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. + LooseOperationCommitAppend bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. LooseOperationCompleteMultipart bool - LooseOperationCreate bool - LooseOperationCreateAppend bool - LooseOperationCreateMultipart bool - LooseOperationDelete bool - LooseOperationList bool - LooseOperationListMultipart bool - LooseOperationMetadata bool - LooseOperationRead bool - LooseOperationStat bool - LooseOperationWrite bool - LooseOperationWriteAppend bool - LooseOperationWriteMultipart bool - + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. + LooseOperationCreate bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. + LooseOperationCreateAppend bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. + LooseOperationCreateDir bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. + LooseOperationCreateMultipart bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. + LooseOperationDelete bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. + LooseOperationList bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. + LooseOperationListMultipart bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. + LooseOperationMetadata bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. + LooseOperationRead bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. + LooseOperationStat bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. + LooseOperationWrite bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. + LooseOperationWriteAppend bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. + LooseOperationWriteMultipart bool + + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. VirtualOperationAll bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. VirtualPairAll bool + // VirtualDir virtual_dir feature is designed for a service that doesn't have native dir support but wants to provide simulated operations. + // + // - If this feature is disabled (the default behavior), the service will behave like it doesn't have any dir support. + // - If this feature is enabled, the service will support simulated dir behavior in create_dir, create, list, delete, and so on. + // + // This feature was introduced in GSP-109. + VirtualDir bool } // pairStorageNew is the parsed struct @@ -562,13 +601,13 @@ func parsePairStorageNew(opts []Pair) (pairStorageNew, error) { result.HasName = true result.Name = v.Value.(string) // Optional pairs - case pairDefaultStoragePairs: + case "default_storage_pairs": if result.HasDefaultStoragePairs { continue } result.HasDefaultStoragePairs = true result.DefaultStoragePairs = v.Value.(DefaultStoragePairs) - case pairStorageFeatures: + case "storage_features": if result.HasStorageFeatures { continue } @@ -595,6 +634,7 @@ type DefaultStoragePairs struct { CompleteMultipart []Pair Create []Pair CreateAppend []Pair + CreateDir []Pair CreateMultipart []Pair Delete []Pair List []Pair @@ -619,23 +659,10 @@ func (s *Storage) parsePairStorageCommitAppend(opts []Pair) (pairStorageCommitAp } for _, v := range opts { - // isUnsupportedPair records whether current pair is unsupported. - isUnsupportedPair := false - switch v.Key { default: - isUnsupportedPair = true - } - - if !isUnsupportedPair { - continue - } - - // If user enables the loose operation feature, we will ignore PairUnsupportedError. - if s.features.LooseOperationAll || s.features.LooseOperationCommitAppend { - continue + return pairStorageCommitAppend{}, services.PairUnsupportedError{Pair: v} } - return pairStorageCommitAppend{}, services.PairUnsupportedError{Pair: v} } // Check required pairs. @@ -655,23 +682,10 @@ func (s *Storage) parsePairStorageCompleteMultipart(opts []Pair) (pairStorageCom } for _, v := range opts { - // isUnsupportedPair records whether current pair is unsupported. - isUnsupportedPair := false - switch v.Key { default: - isUnsupportedPair = true + return pairStorageCompleteMultipart{}, services.PairUnsupportedError{Pair: v} } - - if !isUnsupportedPair { - continue - } - - // If user enables the loose operation feature, we will ignore PairUnsupportedError. - if s.features.LooseOperationAll || s.features.LooseOperationCompleteMultipart { - continue - } - return pairStorageCompleteMultipart{}, services.PairUnsupportedError{Pair: v} } // Check required pairs. @@ -684,6 +698,8 @@ type pairStorageCreate struct { pairs []Pair HasMultipartID bool MultipartID string + HasObjectMode bool + ObjectMode ObjectMode } // parsePairStorageCreate will parse Pair slice into *pairStorageCreate @@ -693,9 +709,6 @@ func (s *Storage) parsePairStorageCreate(opts []Pair) (pairStorageCreate, error) } for _, v := range opts { - // isUnsupportedPair records whether current pair is unsupported. - isUnsupportedPair := false - switch v.Key { case "multipart_id": if result.HasMultipartID { @@ -704,19 +717,16 @@ func (s *Storage) parsePairStorageCreate(opts []Pair) (pairStorageCreate, error) result.HasMultipartID = true result.MultipartID = v.Value.(string) continue - default: - isUnsupportedPair = true - } - - if !isUnsupportedPair { - continue - } - - // If user enables the loose operation feature, we will ignore PairUnsupportedError. - if s.features.LooseOperationAll || s.features.LooseOperationCreate { + case "object_mode": + if result.HasObjectMode { + continue + } + result.HasObjectMode = true + result.ObjectMode = v.Value.(ObjectMode) continue + default: + return pairStorageCreate{}, services.PairUnsupportedError{Pair: v} } - return pairStorageCreate{}, services.PairUnsupportedError{Pair: v} } // Check required pairs. @@ -746,9 +756,6 @@ func (s *Storage) parsePairStorageCreateAppend(opts []Pair) (pairStorageCreateAp } for _, v := range opts { - // isUnsupportedPair records whether current pair is unsupported. - isUnsupportedPair := false - switch v.Key { case "content_type": if result.HasContentType { @@ -757,28 +764,28 @@ func (s *Storage) parsePairStorageCreateAppend(opts []Pair) (pairStorageCreateAp result.HasContentType = true result.ContentType = v.Value.(string) continue - case pairServerSideDataEncryption: + case "server_side_data_encryption": if result.HasServerSideDataEncryption { continue } result.HasServerSideDataEncryption = true result.ServerSideDataEncryption = v.Value.(string) continue - case pairServerSideEncryption: + case "server_side_encryption": if result.HasServerSideEncryption { continue } result.HasServerSideEncryption = true result.ServerSideEncryption = v.Value.(string) continue - case pairServerSideEncryptionKeyID: + case "server_side_encryption_key_id": if result.HasServerSideEncryptionKeyID { continue } result.HasServerSideEncryptionKeyID = true result.ServerSideEncryptionKeyID = v.Value.(string) continue - case pairStorageClass: + case "storage_class": if result.HasStorageClass { continue } @@ -786,18 +793,40 @@ func (s *Storage) parsePairStorageCreateAppend(opts []Pair) (pairStorageCreateAp result.StorageClass = v.Value.(string) continue default: - isUnsupportedPair = true + return pairStorageCreateAppend{}, services.PairUnsupportedError{Pair: v} } + } - if !isUnsupportedPair { - continue - } + // Check required pairs. + + return result, nil +} + +// pairStorageCreateDir is the parsed struct +type pairStorageCreateDir struct { + pairs []Pair + HasStorageClass bool + StorageClass string +} + +// parsePairStorageCreateDir will parse Pair slice into *pairStorageCreateDir +func (s *Storage) parsePairStorageCreateDir(opts []Pair) (pairStorageCreateDir, error) { + result := pairStorageCreateDir{ + pairs: opts, + } - // If user enables the loose operation feature, we will ignore PairUnsupportedError. - if s.features.LooseOperationAll || s.features.LooseOperationCreateAppend { + for _, v := range opts { + switch v.Key { + case "storage_class": + if result.HasStorageClass { + continue + } + result.HasStorageClass = true + result.StorageClass = v.Value.(string) continue + default: + return pairStorageCreateDir{}, services.PairUnsupportedError{Pair: v} } - return pairStorageCreateAppend{}, services.PairUnsupportedError{Pair: v} } // Check required pairs. @@ -827,9 +856,6 @@ func (s *Storage) parsePairStorageCreateMultipart(opts []Pair) (pairStorageCreat } for _, v := range opts { - // isUnsupportedPair records whether current pair is unsupported. - isUnsupportedPair := false - switch v.Key { case "content_type": if result.HasContentType { @@ -838,28 +864,28 @@ func (s *Storage) parsePairStorageCreateMultipart(opts []Pair) (pairStorageCreat result.HasContentType = true result.ContentType = v.Value.(string) continue - case pairServerSideDataEncryption: + case "server_side_data_encryption": if result.HasServerSideDataEncryption { continue } result.HasServerSideDataEncryption = true result.ServerSideDataEncryption = v.Value.(string) continue - case pairServerSideEncryption: + case "server_side_encryption": if result.HasServerSideEncryption { continue } result.HasServerSideEncryption = true result.ServerSideEncryption = v.Value.(string) continue - case pairServerSideEncryptionKeyID: + case "server_side_encryption_key_id": if result.HasServerSideEncryptionKeyID { continue } result.HasServerSideEncryptionKeyID = true result.ServerSideEncryptionKeyID = v.Value.(string) continue - case pairStorageClass: + case "storage_class": if result.HasStorageClass { continue } @@ -867,18 +893,8 @@ func (s *Storage) parsePairStorageCreateMultipart(opts []Pair) (pairStorageCreat result.StorageClass = v.Value.(string) continue default: - isUnsupportedPair = true - } - - if !isUnsupportedPair { - continue - } - - // If user enables the loose operation feature, we will ignore PairUnsupportedError. - if s.features.LooseOperationAll || s.features.LooseOperationCreateMultipart { - continue + return pairStorageCreateMultipart{}, services.PairUnsupportedError{Pair: v} } - return pairStorageCreateMultipart{}, services.PairUnsupportedError{Pair: v} } // Check required pairs. @@ -891,6 +907,8 @@ type pairStorageDelete struct { pairs []Pair HasMultipartID bool MultipartID string + HasObjectMode bool + ObjectMode ObjectMode } // parsePairStorageDelete will parse Pair slice into *pairStorageDelete @@ -900,9 +918,6 @@ func (s *Storage) parsePairStorageDelete(opts []Pair) (pairStorageDelete, error) } for _, v := range opts { - // isUnsupportedPair records whether current pair is unsupported. - isUnsupportedPair := false - switch v.Key { case "multipart_id": if result.HasMultipartID { @@ -911,19 +926,16 @@ func (s *Storage) parsePairStorageDelete(opts []Pair) (pairStorageDelete, error) result.HasMultipartID = true result.MultipartID = v.Value.(string) continue - default: - isUnsupportedPair = true - } - - if !isUnsupportedPair { - continue - } - - // If user enables the loose operation feature, we will ignore PairUnsupportedError. - if s.features.LooseOperationAll || s.features.LooseOperationDelete { + case "object_mode": + if result.HasObjectMode { + continue + } + result.HasObjectMode = true + result.ObjectMode = v.Value.(ObjectMode) continue + default: + return pairStorageDelete{}, services.PairUnsupportedError{Pair: v} } - return pairStorageDelete{}, services.PairUnsupportedError{Pair: v} } // Check required pairs. @@ -945,9 +957,6 @@ func (s *Storage) parsePairStorageList(opts []Pair) (pairStorageList, error) { } for _, v := range opts { - // isUnsupportedPair records whether current pair is unsupported. - isUnsupportedPair := false - switch v.Key { case "list_mode": if result.HasListMode { @@ -957,18 +966,8 @@ func (s *Storage) parsePairStorageList(opts []Pair) (pairStorageList, error) { result.ListMode = v.Value.(ListMode) continue default: - isUnsupportedPair = true + return pairStorageList{}, services.PairUnsupportedError{Pair: v} } - - if !isUnsupportedPair { - continue - } - - // If user enables the loose operation feature, we will ignore PairUnsupportedError. - if s.features.LooseOperationAll || s.features.LooseOperationList { - continue - } - return pairStorageList{}, services.PairUnsupportedError{Pair: v} } // Check required pairs. @@ -988,23 +987,10 @@ func (s *Storage) parsePairStorageListMultipart(opts []Pair) (pairStorageListMul } for _, v := range opts { - // isUnsupportedPair records whether current pair is unsupported. - isUnsupportedPair := false - switch v.Key { default: - isUnsupportedPair = true - } - - if !isUnsupportedPair { - continue - } - - // If user enables the loose operation feature, we will ignore PairUnsupportedError. - if s.features.LooseOperationAll || s.features.LooseOperationListMultipart { - continue + return pairStorageListMultipart{}, services.PairUnsupportedError{Pair: v} } - return pairStorageListMultipart{}, services.PairUnsupportedError{Pair: v} } // Check required pairs. @@ -1024,23 +1010,10 @@ func (s *Storage) parsePairStorageMetadata(opts []Pair) (pairStorageMetadata, er } for _, v := range opts { - // isUnsupportedPair records whether current pair is unsupported. - isUnsupportedPair := false - switch v.Key { default: - isUnsupportedPair = true + return pairStorageMetadata{}, services.PairUnsupportedError{Pair: v} } - - if !isUnsupportedPair { - continue - } - - // If user enables the loose operation feature, we will ignore PairUnsupportedError. - if s.features.LooseOperationAll || s.features.LooseOperationMetadata { - continue - } - return pairStorageMetadata{}, services.PairUnsupportedError{Pair: v} } // Check required pairs. @@ -1066,9 +1039,6 @@ func (s *Storage) parsePairStorageRead(opts []Pair) (pairStorageRead, error) { } for _, v := range opts { - // isUnsupportedPair records whether current pair is unsupported. - isUnsupportedPair := false - switch v.Key { case "io_callback": if result.HasIoCallback { @@ -1092,18 +1062,8 @@ func (s *Storage) parsePairStorageRead(opts []Pair) (pairStorageRead, error) { result.Size = v.Value.(int64) continue default: - isUnsupportedPair = true + return pairStorageRead{}, services.PairUnsupportedError{Pair: v} } - - if !isUnsupportedPair { - continue - } - - // If user enables the loose operation feature, we will ignore PairUnsupportedError. - if s.features.LooseOperationAll || s.features.LooseOperationRead { - continue - } - return pairStorageRead{}, services.PairUnsupportedError{Pair: v} } // Check required pairs. @@ -1116,6 +1076,8 @@ type pairStorageStat struct { pairs []Pair HasMultipartID bool MultipartID string + HasObjectMode bool + ObjectMode ObjectMode } // parsePairStorageStat will parse Pair slice into *pairStorageStat @@ -1125,9 +1087,6 @@ func (s *Storage) parsePairStorageStat(opts []Pair) (pairStorageStat, error) { } for _, v := range opts { - // isUnsupportedPair records whether current pair is unsupported. - isUnsupportedPair := false - switch v.Key { case "multipart_id": if result.HasMultipartID { @@ -1136,19 +1095,16 @@ func (s *Storage) parsePairStorageStat(opts []Pair) (pairStorageStat, error) { result.HasMultipartID = true result.MultipartID = v.Value.(string) continue - default: - isUnsupportedPair = true - } - - if !isUnsupportedPair { - continue - } - - // If user enables the loose operation feature, we will ignore PairUnsupportedError. - if s.features.LooseOperationAll || s.features.LooseOperationStat { + case "object_mode": + if result.HasObjectMode { + continue + } + result.HasObjectMode = true + result.ObjectMode = v.Value.(ObjectMode) continue + default: + return pairStorageStat{}, services.PairUnsupportedError{Pair: v} } - return pairStorageStat{}, services.PairUnsupportedError{Pair: v} } // Check required pairs. @@ -1182,9 +1138,6 @@ func (s *Storage) parsePairStorageWrite(opts []Pair) (pairStorageWrite, error) { } for _, v := range opts { - // isUnsupportedPair records whether current pair is unsupported. - isUnsupportedPair := false - switch v.Key { case "content_md5": if result.HasContentMd5 { @@ -1207,28 +1160,28 @@ func (s *Storage) parsePairStorageWrite(opts []Pair) (pairStorageWrite, error) { result.HasIoCallback = true result.IoCallback = v.Value.(func([]byte)) continue - case pairServerSideDataEncryption: + case "server_side_data_encryption": if result.HasServerSideDataEncryption { continue } result.HasServerSideDataEncryption = true result.ServerSideDataEncryption = v.Value.(string) continue - case pairServerSideEncryption: + case "server_side_encryption": if result.HasServerSideEncryption { continue } result.HasServerSideEncryption = true result.ServerSideEncryption = v.Value.(string) continue - case pairServerSideEncryptionKeyID: + case "server_side_encryption_key_id": if result.HasServerSideEncryptionKeyID { continue } result.HasServerSideEncryptionKeyID = true result.ServerSideEncryptionKeyID = v.Value.(string) continue - case pairStorageClass: + case "storage_class": if result.HasStorageClass { continue } @@ -1236,18 +1189,8 @@ func (s *Storage) parsePairStorageWrite(opts []Pair) (pairStorageWrite, error) { result.StorageClass = v.Value.(string) continue default: - isUnsupportedPair = true + return pairStorageWrite{}, services.PairUnsupportedError{Pair: v} } - - if !isUnsupportedPair { - continue - } - - // If user enables the loose operation feature, we will ignore PairUnsupportedError. - if s.features.LooseOperationAll || s.features.LooseOperationWrite { - continue - } - return pairStorageWrite{}, services.PairUnsupportedError{Pair: v} } // Check required pairs. @@ -1271,9 +1214,6 @@ func (s *Storage) parsePairStorageWriteAppend(opts []Pair) (pairStorageWriteAppe } for _, v := range opts { - // isUnsupportedPair records whether current pair is unsupported. - isUnsupportedPair := false - switch v.Key { case "content_md5": if result.HasContentMd5 { @@ -1290,18 +1230,8 @@ func (s *Storage) parsePairStorageWriteAppend(opts []Pair) (pairStorageWriteAppe result.IoCallback = v.Value.(func([]byte)) continue default: - isUnsupportedPair = true + return pairStorageWriteAppend{}, services.PairUnsupportedError{Pair: v} } - - if !isUnsupportedPair { - continue - } - - // If user enables the loose operation feature, we will ignore PairUnsupportedError. - if s.features.LooseOperationAll || s.features.LooseOperationWriteAppend { - continue - } - return pairStorageWriteAppend{}, services.PairUnsupportedError{Pair: v} } // Check required pairs. @@ -1323,9 +1253,6 @@ func (s *Storage) parsePairStorageWriteMultipart(opts []Pair) (pairStorageWriteM } for _, v := range opts { - // isUnsupportedPair records whether current pair is unsupported. - isUnsupportedPair := false - switch v.Key { case "content_md5": if result.HasContentMd5 { @@ -1335,18 +1262,8 @@ func (s *Storage) parsePairStorageWriteMultipart(opts []Pair) (pairStorageWriteM result.ContentMd5 = v.Value.(string) continue default: - isUnsupportedPair = true - } - - if !isUnsupportedPair { - continue - } - - // If user enables the loose operation feature, we will ignore PairUnsupportedError. - if s.features.LooseOperationAll || s.features.LooseOperationWriteMultipart { - continue + return pairStorageWriteMultipart{}, services.PairUnsupportedError{Pair: v} } - return pairStorageWriteMultipart{}, services.PairUnsupportedError{Pair: v} } // Check required pairs. @@ -1414,6 +1331,11 @@ func (s *Storage) CompleteMultipartWithContext(ctx context.Context, o *Object, p // Create will create a new object without any api call. // +// ## Behavior +// +// - Create SHOULD NOT send any API call. +// - Create SHOULD accept ObjectMode pair as object mode. +// // This function will create a context by default. func (s *Storage) Create(path string, pairs ...Pair) (o *Object) { pairs = append(pairs, s.defaultPairs.Create...) @@ -1450,6 +1372,31 @@ func (s *Storage) CreateAppendWithContext(ctx context.Context, path string, pair return s.createAppend(ctx, path, opt) } +// CreateDir will create a new dir object. +// +// This function will create a context by default. +func (s *Storage) CreateDir(path string, pairs ...Pair) (o *Object, err error) { + ctx := context.Background() + return s.CreateDirWithContext(ctx, path, pairs...) +} + +// CreateDirWithContext will create a new dir object. +func (s *Storage) CreateDirWithContext(ctx context.Context, path string, pairs ...Pair) (o *Object, err error) { + defer func() { + err = s.formatError("create_dir", err, path) + }() + + pairs = append(pairs, s.defaultPairs.CreateDir...) + var opt pairStorageCreateDir + + opt, err = s.parsePairStorageCreateDir(pairs) + if err != nil { + return + } + + return s.createDir(ctx, path, opt) +} + // CreateMultipart will create a new multipart. // // This function will create a context by default. @@ -1475,7 +1422,17 @@ func (s *Storage) CreateMultipartWithContext(ctx context.Context, path string, p return s.createMultipart(ctx, path, opt) } -// Delete will delete an Object from service. +// Delete will delete an object from service. +// +// ## Behavior +// +// - Delete only delete one and only one object. +// - Service DON'T NEED to support remove all. +// - User NEED to implement remove_all by themself. +// - Delete is idempotent. +// - Successful delete always return nil error. +// - Delete SHOULD never return `ObjectNotExist` +// - Delete DON'T NEED to check the object exist or not. // // This function will create a context by default. func (s *Storage) Delete(path string, pairs ...Pair) (err error) { @@ -1483,7 +1440,17 @@ func (s *Storage) Delete(path string, pairs ...Pair) (err error) { return s.DeleteWithContext(ctx, path, pairs...) } -// DeleteWithContext will delete an Object from service. +// DeleteWithContext will delete an object from service. +// +// ## Behavior +// +// - Delete only delete one and only one object. +// - Service DON'T NEED to support remove all. +// - User NEED to implement remove_all by themself. +// - Delete is idempotent. +// - Successful delete always return nil error. +// - Delete SHOULD never return `ObjectNotExist` +// - Delete DON'T NEED to check the object exist or not. func (s *Storage) DeleteWithContext(ctx context.Context, path string, pairs ...Pair) (err error) { defer func() { err = s.formatError("delete", err, path) @@ -1594,6 +1561,12 @@ func (s *Storage) ReadWithContext(ctx context.Context, path string, w io.Writer, // Stat will stat a path to get info of an object. // +// ## Behavior +// +// - Stat SHOULD accept ObjectMode pair as hints. +// - Service COULD have different implementations for different object mode. +// - Service SHOULD check if returning ObjectMode is match +// // This function will create a context by default. func (s *Storage) Stat(path string, pairs ...Pair) (o *Object, err error) { ctx := context.Background() @@ -1601,6 +1574,12 @@ func (s *Storage) Stat(path string, pairs ...Pair) (o *Object, err error) { } // StatWithContext will stat a path to get info of an object. +// +// ## Behavior +// +// - Stat SHOULD accept ObjectMode pair as hints. +// - Service COULD have different implementations for different object mode. +// - Service SHOULD check if returning ObjectMode is match func (s *Storage) StatWithContext(ctx context.Context, path string, pairs ...Pair) (o *Object, err error) { defer func() { err = s.formatError("stat", err, path) @@ -1703,4 +1682,5 @@ func (s *Storage) WriteMultipartWithContext(ctx context.Context, o *Object, r io func init() { services.RegisterServicer(Type, NewServicer) services.RegisterStorager(Type, NewStorager) + services.RegisterSchema(Type, pairMap) } diff --git a/go.mod b/go.mod index 3df7fdf..99804fc 100644 --- a/go.mod +++ b/go.mod @@ -5,8 +5,9 @@ go 1.14 require ( github.com/aliyun/aliyun-oss-go-sdk v2.1.8+incompatible github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect - github.com/beyondstorage/go-integration-test/v4 v4.0.0 - github.com/beyondstorage/go-storage/v4 v4.0.1-0.20210530044854-1c928ddbe52d + github.com/beyondstorage/go-endpoint v1.0.1 + github.com/beyondstorage/go-integration-test/v4 v4.1.1 + github.com/beyondstorage/go-storage/v4 v4.2.0 github.com/google/uuid v1.2.0 github.com/satori/go.uuid v1.2.0 // indirect golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect diff --git a/go.sum b/go.sum index fe9c273..2839254 100644 --- a/go.sum +++ b/go.sum @@ -4,22 +4,23 @@ github.com/aliyun/aliyun-oss-go-sdk v2.1.8+incompatible h1:hLUNPbx10wawWW7DeNExv github.com/aliyun/aliyun-oss-go-sdk v2.1.8+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f h1:ZNv7On9kyUzm7fvRZumSyy/IUiSC7AzL0I1jKKtwooA= github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f/go.mod h1:AuiFmCCPBSrqvVMvuqFuk0qogytodnVFVSN5CeJB8Gc= -github.com/beyondstorage/go-integration-test/v4 v4.0.0 h1:tdXQV9yxQ3Q6p9xfyQKzK3MEo9r9j9g3uT5+3sbVtnQ= -github.com/beyondstorage/go-integration-test/v4 v4.0.0/go.mod h1:26/JF4b0XxRN0pL4kihpnVNhbbw+QWvmmvgxfnFJDfA= -github.com/beyondstorage/go-storage/v4 v4.0.0/go.mod h1:oa2dYco+xplPj99WSBnYVw/xXvRkIKWSSVDQKNZ5Kz8= -github.com/beyondstorage/go-storage/v4 v4.0.1-0.20210530044854-1c928ddbe52d h1:s9t6VNNRDqmg+PXyXtVEsxIM7xZQIJOYlma28IpkbNQ= -github.com/beyondstorage/go-storage/v4 v4.0.1-0.20210530044854-1c928ddbe52d/go.mod h1:kXMu07IDZaKtxbqI1ufuhqo0FjYe0nH7zPCbBanln/Y= -github.com/beyondstorage/specs/go v0.0.0-20210521044836-3d41c1d9c97f/go.mod h1:f5VvmLHc/dNJwl+/yAv/TOHdev3phvuEswx8DIXiSQQ= -github.com/beyondstorage/specs/go v0.0.0-20210530044123-3ff75e192bc9 h1:YSiF27cAHlDZk9q+oaEHQbA8dH8XTvYxeTOoPzNCwOQ= -github.com/beyondstorage/specs/go v0.0.0-20210530044123-3ff75e192bc9/go.mod h1:f5VvmLHc/dNJwl+/yAv/TOHdev3phvuEswx8DIXiSQQ= +github.com/beyondstorage/go-endpoint v1.0.1 h1:F8x2dGLMu9je6g7zPbKoxCXDlug97K26SeCx7KEHgyg= +github.com/beyondstorage/go-endpoint v1.0.1/go.mod h1:P2hknaGrziOJJKySv/XnAiVw/d3v12/LZu2gSxEx4nM= +github.com/beyondstorage/go-integration-test/v4 v4.1.1 h1:9bSXKbr6hLb4+ZsmAhWE32fvqhyrpub4U4qgBGeth4A= +github.com/beyondstorage/go-integration-test/v4 v4.1.1/go.mod h1:ihtCaOJvaHGE0v+IhY6ZUF5NU1IND6xmdrJI9Lq/jhc= +github.com/beyondstorage/go-storage/v4 v4.2.0 h1:J0xqqy4qEQRtIS2zUWMA5wRXVHx/cxX5fHsU2ezA3+I= +github.com/beyondstorage/go-storage/v4 v4.2.0/go.mod h1:rUNzOXcikYk5w0ewvNsKbztg7ndQDyDvjDuP0bznSLU= +github.com/beyondstorage/specs/go v0.0.0-20210623065218-d1c2d7d81259 h1:mW9XpHLc6pdXBRnsha1VlqF0rNsB/Oc+8l+5UYngmRA= +github.com/beyondstorage/specs/go v0.0.0-20210623065218-d1c2d7d81259/go.mod h1:vF/Q0P1tCvhVAUrxg7i6NvrARRMQVTAuQdDNqpSzR1w= github.com/dave/dst v0.26.2 h1:lnxLAKI3tx7MgLNVDirFCsDTlTG9nKTk7GcptKcWSwY= github.com/dave/dst v0.26.2/go.mod h1:UMDJuIRPfyUCC78eFuB+SV/WI8oDeyFDvM/JR6NI3IU= github.com/dave/gopackages v0.0.0-20170318123100-46e7023ec56e/go.mod h1:i00+b/gKdIDIxuLDFob7ustLAVqhsZRk2qVZrArELGQ= github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= github.com/dave/kerr v0.0.0-20170318121727-bc25dd6abe8e/go.mod h1:qZqlPyPvfsDJt+3wHJ1EvSXDuVjFTK0j2p/ca+gtsb8= github.com/dave/rebecca v0.9.1/go.mod h1:N6XYdMD/OKw3lkF3ywh8Z6wPGuwNFDNtWYEMFWEmXBA= -github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/golang/mock v1.5.0 h1:jlYHihg//f7RRwuPfptm04yp4s7O6Kw8EZiVYIGcH0g= github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/google/pprof v0.0.0-20181127221834-b4f47329b966/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -37,19 +38,22 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/pelletier/go-toml v1.9.1 h1:a6qW1EVNZWH9WGI6CsYdD8WAylkoXBS5yv0XHlh17Tc= -github.com/pelletier/go-toml v1.9.1/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ= +github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -68,6 +72,8 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sys v0.0.0-20180903190138-2b024373dcd9/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/service.toml b/service.toml index 99a737c..6f04c56 100644 --- a/service.toml +++ b/service.toml @@ -7,20 +7,24 @@ required = ["credential"] optional = ["service_features", "default_service_pairs", "endpoint", "http_client_options"] [namespace.storage] -implement = ["appender", "multiparter"] +features = ["virtual_dir"] +implement = ["appender", "direr", "multiparter"] [namespace.storage.new] required = ["name"] optional = ["storage_features", "default_storage_pairs", "work_dir"] [namespace.storage.op.create] -optional = ["multipart_id"] +optional = ["multipart_id", "object_mode"] + +[namespace.storage.op.create_dir] +optional = ["storage_class"] [namespace.storage.op.delete] -optional = ["multipart_id"] +optional = ["multipart_id", "object_mode"] [namespace.storage.op.stat] -optional = ["multipart_id"] +optional = ["multipart_id", "object_mode"] [namespace.storage.op.list] optional = ["list_mode"] diff --git a/storage.go b/storage.go index 56cd316..42c6108 100644 --- a/storage.go +++ b/storage.go @@ -2,12 +2,14 @@ package oss import ( "context" + "fmt" "io" "strconv" "time" "github.com/aliyun/aliyun-oss-go-sdk/oss" + ps "github.com/beyondstorage/go-storage/v4/pairs" "github.com/beyondstorage/go-storage/v4/pkg/headers" "github.com/beyondstorage/go-storage/v4/pkg/iowrap" "github.com/beyondstorage/go-storage/v4/services" @@ -46,16 +48,27 @@ func (s *Storage) completeMultipart(ctx context.Context, o *Object, parts []*Par } func (s *Storage) create(path string, opt pairStorageCreate) (o *Object) { + rp := s.getAbsPath(path) + // Handle create multipart object separately. if opt.HasMultipartID { o = s.newObject(true) o.Mode = ModePart o.SetMultipartID(opt.MultipartID) } else { - o = s.newObject(false) - o.Mode = ModeRead + if opt.HasObjectMode && opt.ObjectMode.IsDir() { + if !s.features.VirtualDir { + return + } + rp += "/" + o = s.newObject(true) + o.Mode = ModeDir + } else { + o = s.newObject(false) + o.Mode = ModeRead + } } - o.ID = s.getAbsPath(path) + o.ID = rp o.Path = path return o } @@ -94,6 +107,35 @@ func (s *Storage) createAppend(ctx context.Context, path string, opt pairStorage return o, nil } +func (s *Storage) createDir(ctx context.Context, path string, opt pairStorageCreateDir) (o *Object, err error) { + if !s.features.VirtualDir { + err = NewOperationNotImplementedError("create_dir") + return + } + rp := s.getAbsPath(path) + + // Add `/` at the end of path to simulate directory. + // ref: https://help.aliyun.com/document_detail/31978.html#title-gkg-amg-aes + rp += "/" + + options := make([]oss.Option, 0) + options = append(options, oss.ContentLength(0)) + if opt.HasStorageClass { + options = append(options, oss.StorageClass(oss.StorageClassType(opt.StorageClass))) + } + + err = s.bucket.PutObject(rp, nil, options...) + if err != nil { + return + } + + o = s.newObject(true) + o.Path = path + o.ID = rp + o.Mode |= ModeDir + return +} + func (s *Storage) createMultipart(ctx context.Context, path string, opt pairStorageCreateMultipart) (o *Object, err error) { rp := s.getAbsPath(path) @@ -124,10 +166,6 @@ func (s *Storage) createMultipart(ctx context.Context, path string, opt pairStor o.Path = path o.Mode |= ModePart o.SetMultipartID(output.UploadID) - // set multipart restriction - o.SetMultipartNumberMaximum(multipartNumberMaximum) - o.SetMultipartNumberMaximum(multipartSizeMaximum) - o.SetMultipartSizeMinimum(multipartSizeMinimum) return o, nil } @@ -152,6 +190,15 @@ func (s *Storage) delete(ctx context.Context, path string, opt pairStorageDelete return } + if opt.HasObjectMode && opt.ObjectMode.IsDir() { + if !s.features.VirtualDir { + err = services.PairUnsupportedError{Pair: ps.WithObjectMode(opt.ObjectMode)} + return + } + + rp += "/" + } + // OSS DeleteObject is idempotent, so we don't need to check NoSuchKey error. // // References @@ -201,6 +248,14 @@ func (s *Storage) metadata(opt pairStorageMetadata) (meta *StorageMeta) { meta = NewStorageMeta() meta.Name = s.bucket.BucketName meta.WorkDir = s.workDir + // set write restriction + meta.SetWriteSizeMaximum(writeSizeMaximum) + // set append restriction + meta.SetAppendTotalSizeMaximum(appendTotalSizeMaximum) + // set multipart restrictions + meta.SetMultipartNumberMaximum(multipartNumberMaximum) + meta.SetMultipartNumberMaximum(multipartSizeMaximum) + meta.SetMultipartSizeMinimum(multipartSizeMinimum) return } @@ -390,6 +445,15 @@ func (s *Storage) stat(ctx context.Context, path string, opt pairStorageStat) (o return o, nil } + if opt.HasObjectMode && opt.ObjectMode.IsDir() { + if !s.features.VirtualDir { + err = services.PairUnsupportedError{Pair: ps.WithObjectMode(opt.ObjectMode)} + return + } + + rp += "/" + } + output, err := s.bucket.GetObjectMeta(rp) if err != nil { return nil, err @@ -398,7 +462,11 @@ func (s *Storage) stat(ctx context.Context, path string, opt pairStorageStat) (o o = s.newObject(true) o.ID = rp o.Path = path - o.Mode |= ModeRead + if opt.HasObjectMode && opt.ObjectMode.IsDir() { + o.Mode |= ModeDir + } else { + o.Mode |= ModeRead + } if v := output.Get(headers.ContentLength); v != "" { size, err := strconv.ParseInt(v, 10, 64) @@ -427,7 +495,7 @@ func (s *Storage) stat(ctx context.Context, path string, opt pairStorageStat) (o o.SetContentType(v) } - var sm ObjectMetadata + var sm ObjectSystemMetadata if v := output.Get(storageClassHeader); v != "" { sm.StorageClass = v } @@ -437,12 +505,17 @@ func (s *Storage) stat(ctx context.Context, path string, opt pairStorageStat) (o if v := output.Get(serverSideEncryptionKeyIdHeader); v != "" { sm.ServerSideEncryptionKeyID = v } - o.SetServiceMetadata(sm) + o.SetSystemMetadata(sm) return o, nil } func (s *Storage) write(ctx context.Context, path string, r io.Reader, size int64, opt pairStorageWrite) (n int64, err error) { + if size > writeSizeMaximum { + err = fmt.Errorf("size limit exceeded: %w", services.ErrRestrictionDissatisfied) + return + } + if opt.HasIoCallback { r = iowrap.CallbackReader(r, opt.IoCallback) } @@ -500,6 +573,15 @@ func (s *Storage) writeAppend(ctx context.Context, o *Object, r io.Reader, size } func (s *Storage) writeMultipart(ctx context.Context, o *Object, r io.Reader, size int64, index int, opt pairStorageWriteMultipart) (n int64, part *Part, err error) { + if index < 0 || index >= multipartNumberMaximum { + err = fmt.Errorf("multipart number limit exceeded: %w", services.ErrRestrictionDissatisfied) + return + } + if size > multipartSizeMaximum { + err = fmt.Errorf("size limit exceeded: %w", services.ErrRestrictionDissatisfied) + return + } + imur := oss.InitiateMultipartUploadResult{ Bucket: s.bucket.BucketName, Key: o.ID, diff --git a/tests/storage_test.go b/tests/storage_test.go index 4cbd418..0b0776c 100644 --- a/tests/storage_test.go +++ b/tests/storage_test.go @@ -27,3 +27,10 @@ func TestMultiparter(t *testing.T) { } tests.TestMultiparter(t, setupTest(t)) } + +func TestDirer(t *testing.T) { + if os.Getenv("STORAGE_OSS_INTEGRATION_TEST") != "on" { + t.Skipf("STORAGE_OSS_INTEGRATION_TEST is not 'on', skipped") + } + tests.TestDirer(t, setupTest(t)) +} diff --git a/tests/utils_test.go b/tests/utils_test.go index 6574b98..c803638 100644 --- a/tests/utils_test.go +++ b/tests/utils_test.go @@ -18,6 +18,9 @@ func setupTest(t *testing.T) types.Storager { ps.WithName(os.Getenv("STORAGE_OSS_NAME")), ps.WithEndpoint(os.Getenv("STORAGE_OSS_ENDPOINT")), ps.WithWorkDir("/"+uuid.New().String()+"/"), + oss.WithStorageFeatures(oss.StorageFeatures{ + VirtualDir: true, + }), ) if err != nil { t.Errorf("new storager: %v", err) diff --git a/utils.go b/utils.go index 04e1d25..ff70c71 100644 --- a/utils.go +++ b/utils.go @@ -6,9 +6,9 @@ import ( "github.com/aliyun/aliyun-oss-go-sdk/oss" + "github.com/beyondstorage/go-endpoint" ps "github.com/beyondstorage/go-storage/v4/pairs" "github.com/beyondstorage/go-storage/v4/pkg/credential" - "github.com/beyondstorage/go-storage/v4/pkg/endpoint" "github.com/beyondstorage/go-storage/v4/pkg/httpclient" "github.com/beyondstorage/go-storage/v4/services" typ "github.com/beyondstorage/go-storage/v4/types" @@ -42,6 +42,7 @@ type Storage struct { typ.UnimplementedStorager typ.UnimplementedAppender typ.UnimplementedMultiparter + typ.UnimplementedDirer } // String implements Storager.String @@ -96,12 +97,22 @@ func newServicer(pairs ...typ.Pair) (srv *Service, err error) { return nil, err } + var url string + switch ep.Protocol() { + case endpoint.ProtocolHTTP: + url, _, _ = ep.HTTP() + case endpoint.ProtocolHTTPS: + url, _, _ = ep.HTTPS() + default: + return nil, services.PairUnsupportedError{Pair: ps.WithEndpoint(opt.Endpoint)} + } + var copts []oss.ClientOption if opt.HasHTTPClientOptions { copts = append(copts, oss.HTTPClient(httpclient.New(opt.HTTPClientOptions))) } - srv.service, err = oss.New(ep.String(), ak, sk, copts...) + srv.service, err = oss.New(url, ak, sk, copts...) if err != nil { return nil, err } @@ -259,11 +270,11 @@ func (s *Storage) formatFileObject(v oss.ObjectProperties) (o *typ.Object, err e o.SetEtag(v.ETag) } - var sm ObjectMetadata + var sm ObjectSystemMetadata if value := v.Type; value != "" { sm.StorageClass = value } - o.SetServiceMetadata(sm) + o.SetSystemMetadata(sm) return } @@ -311,3 +322,12 @@ const ( // multipartSizeMinimum is the minimum size for each part, 100KB. multipartSizeMinimum = 100 * 1024 ) + +const ( + // writeSizeMaximum is the maximum size for each object with a single PUT operation, 5GB. + // ref: https://help.aliyun.com/document_detail/31978.html#title-gkg-amg-aes + writeSizeMaximum = 5 * 1024 * 1024 * 1024 + // appendSizeMaximum is the total maximum size for an append object, 5GB. + // ref: https://help.aliyun.com/document_detail/31981.html?spm=a2c4g.11186623.6.1684.479a3ea7S8dRgB#title-22f-5c3-0sv + appendTotalSizeMaximum = 5 * 1024 * 1024 * 1024 +)