diff --git a/generated.go b/generated.go index c393620..b3774ab 100644 --- a/generated.go +++ b/generated.go @@ -21,29 +21,9 @@ var _ httpclient.Options // Type is the type for gcs const Type = "gcs" -// Service available pairs. -const ( - // DefaultServicePairs set default pairs for service actions - pairDefaultServicePairs = "gcs_default_service_pairs" - // DefaultStoragePairs set default pairs for storager actions - pairDefaultStoragePairs = "gcs_default_storage_pairs" - // EncryptionKey is the customer's 32-byte AES-256 key - pairEncryptionKey = "gcs_encryption_key" - // KmsKeyName is the Cloud KMS key resource. For example, `projects/my-pet-project/locations/us-east1/keyRings/my-key-ring/cryptoKeys/my-key`. - // - // Refer to https://cloud.google.com/storage/docs/encryption/using-customer-managed-keys#add-object-key for more details. - pairKmsKeyName = "gcs_kms_key_name" - // ProjectID - pairProjectID = "gcs_project_id" - // ServiceFeatures set service features - pairServiceFeatures = "gcs_service_features" - // StorageClass - pairStorageClass = "gcs_storage_class" - // StorageFeatures set storage features - pairStorageFeatures = "gcs_storage_features" -) - // ObjectMetadata stores service metadata for object. +// +// Deprecated: Use ObjectSystemMetadata instead. type ObjectMetadata struct { // EncryptionKeySha256 EncryptionKeySha256 string @@ -55,6 +35,8 @@ type ObjectMetadata struct { // // - This function should not be called by service implementer. // - The returning ObjectMetadata is read only and should not be modified. +// +// Deprecated: Use GetObjectSystemMetadata instead. func GetObjectMetadata(o *Object) ObjectMetadata { om, ok := o.GetServiceMetadata() if ok { @@ -66,16 +48,67 @@ func GetObjectMetadata(o *Object) ObjectMetadata { // setObjectMetadata will set ObjectMetadata into Object. // // - This function should only be called once, please make sure all data has been written before set. +// +// Deprecated: Use setObjectSystemMetadata instead. func setObjectMetadata(o *Object, om ObjectMetadata) { o.SetServiceMetadata(om) } +// ObjectSystemMetadata stores system metadata for object. +type ObjectSystemMetadata struct { + // EncryptionKeySha256 + EncryptionKeySha256 string + // StorageClass + StorageClass string +} + +// GetObjectSystemMetadata will get ObjectSystemMetadata from Object. +// +// - This function should not be called by service implementer. +// - The returning ObjectServiceMetadata is read only and should not be modified. +func GetObjectSystemMetadata(o *Object) ObjectSystemMetadata { + sm, ok := o.GetSystemMetadata() + if ok { + return sm.(ObjectSystemMetadata) + } + return ObjectSystemMetadata{} +} + +// setObjectSystemMetadata will set ObjectSystemMetadata into Object. +// +// - This function should only be called once, please make sure all data has been written before set. +func setObjectSystemMetadata(o *Object, sm ObjectSystemMetadata) { + o.SetSystemMetadata(sm) +} + +// StorageSystemMetadata stores system metadata for storage meta. +type StorageSystemMetadata struct { +} + +// GetStorageSystemMetadata will get SystemMetadata from StorageMeta. +// +// - The returning StorageSystemMetadata is read only and should not be modified. +func GetStorageSystemMetadata(s *StorageMeta) StorageSystemMetadata { + sm, ok := s.GetSystemMetadata() + if ok { + return sm.(StorageSystemMetadata) + } + return StorageSystemMetadata{} +} + +// setStorageSystemMetadata will set SystemMetadata into StorageMeta. +// +// - This function should only be called once, please make sure all data has been written before set. +func setStorageSystemMetadata(s *StorageMeta, sm StorageSystemMetadata) { + s.SetSystemMetadata(sm) +} + // WithDefaultServicePairs will apply default_service_pairs value to Options. // // DefaultServicePairs set default pairs for service actions func WithDefaultServicePairs(v DefaultServicePairs) Pair { return Pair{ - Key: pairDefaultServicePairs, + Key: "default_service_pairs", Value: v, } } @@ -85,7 +118,7 @@ func WithDefaultServicePairs(v DefaultServicePairs) Pair { // DefaultStoragePairs set default pairs for storager actions func WithDefaultStoragePairs(v DefaultStoragePairs) Pair { return Pair{ - Key: pairDefaultStoragePairs, + Key: "default_storage_pairs", Value: v, } } @@ -95,7 +128,7 @@ func WithDefaultStoragePairs(v DefaultStoragePairs) Pair { // EncryptionKey is the customer's 32-byte AES-256 key func WithEncryptionKey(v []byte) Pair { return Pair{ - Key: pairEncryptionKey, + Key: "encryption_key", Value: v, } } @@ -107,7 +140,7 @@ func WithEncryptionKey(v []byte) Pair { // Refer to https://cloud.google.com/storage/docs/encryption/using-customer-managed-keys#add-object-key for more details. func WithKmsKeyName(v string) Pair { return Pair{ - Key: pairKmsKeyName, + Key: "kms_key_name", Value: v, } } @@ -117,7 +150,7 @@ func WithKmsKeyName(v string) Pair { // ProjectID func WithProjectID(v string) Pair { return Pair{ - Key: pairProjectID, + Key: "project_id", Value: v, } } @@ -127,7 +160,7 @@ func WithProjectID(v string) Pair { // ServiceFeatures set service features func WithServiceFeatures(v ServiceFeatures) Pair { return Pair{ - Key: pairServiceFeatures, + Key: "service_features", Value: v, } } @@ -137,7 +170,7 @@ func WithServiceFeatures(v ServiceFeatures) Pair { // StorageClass func WithStorageClass(v string) Pair { return Pair{ - Key: pairStorageClass, + Key: "storage_class", Value: v, } } @@ -147,24 +180,59 @@ func WithStorageClass(v string) Pair { // StorageFeatures set storage features func WithStorageFeatures(v StorageFeatures) Pair { return Pair{ - Key: pairStorageFeatures, + Key: "storage_features", Value: v, } } +var pairMap = map[string]string{ + "content_md5": "string", + "content_type": "string", + "context": "context.Context", + "continuation_token": "string", + "credential": "string", + "default_service_pairs": "DefaultServicePairs", + "default_storage_pairs": "DefaultStoragePairs", + "encryption_key": "[]byte", + "endpoint": "string", + "expire": "int", + "http_client_options": "*httpclient.Options", + "interceptor": "Interceptor", + "io_callback": "func([]byte)", + "kms_key_name": "string", + "list_mode": "ListMode", + "location": "string", + "multipart_id": "string", + "name": "string", + "object_mode": "ObjectMode", + "offset": "int64", + "project_id": "string", + "service_features": "ServiceFeatures", + "size": "int64", + "storage_class": "string", + "storage_features": "StorageFeatures", + "work_dir": "string", +} var ( _ Servicer = &Service{} ) type ServiceFeatures struct { - LooseOperationAll bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. + LooseOperationAll bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. LooseOperationCreate bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. LooseOperationDelete bool - LooseOperationGet bool - LooseOperationList bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. + LooseOperationGet bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. + LooseOperationList bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. VirtualOperationAll bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. VirtualPairAll bool } @@ -201,14 +269,14 @@ func parsePairServiceNew(opts []Pair) (pairServiceNew, error) { } result.HasCredential = true result.Credential = v.Value.(string) - case pairProjectID: + case "project_id": if result.HasProjectID { continue } result.HasProjectID = true result.ProjectID = v.Value.(string) // Optional pairs - case pairDefaultServicePairs: + case "default_service_pairs": if result.HasDefaultServicePairs { continue } @@ -220,7 +288,7 @@ func parsePairServiceNew(opts []Pair) (pairServiceNew, error) { } result.HasHTTPClientOptions = true result.HTTPClientOptions = v.Value.(*httpclient.Options) - case pairServiceFeatures: + case "service_features": if result.HasServiceFeatures { continue } @@ -232,7 +300,7 @@ func parsePairServiceNew(opts []Pair) (pairServiceNew, error) { return pairServiceNew{}, services.PairRequiredError{Keys: []string{"credential"}} } if !result.HasProjectID { - return pairServiceNew{}, services.PairRequiredError{Keys: []string{pairProjectID}} + return pairServiceNew{}, services.PairRequiredError{Keys: []string{"project_id"}} } return result, nil @@ -258,23 +326,10 @@ func (s *Service) parsePairServiceCreate(opts []Pair) (pairServiceCreate, error) } for _, v := range opts { - // isUnsupportedPair records whether current pair is unsupported. - isUnsupportedPair := false - switch v.Key { default: - isUnsupportedPair = true - } - - if !isUnsupportedPair { - continue - } - - // If user enables the loose operation feature, we will ignore PairUnsupportedError. - if s.features.LooseOperationAll || s.features.LooseOperationCreate { - continue + return pairServiceCreate{}, services.PairUnsupportedError{Pair: v} } - return pairServiceCreate{}, services.PairUnsupportedError{Pair: v} } // Check required pairs. @@ -294,23 +349,10 @@ func (s *Service) parsePairServiceDelete(opts []Pair) (pairServiceDelete, error) } for _, v := range opts { - // isUnsupportedPair records whether current pair is unsupported. - isUnsupportedPair := false - switch v.Key { default: - isUnsupportedPair = true - } - - if !isUnsupportedPair { - continue + return pairServiceDelete{}, services.PairUnsupportedError{Pair: v} } - - // If user enables the loose operation feature, we will ignore PairUnsupportedError. - if s.features.LooseOperationAll || s.features.LooseOperationDelete { - continue - } - return pairServiceDelete{}, services.PairUnsupportedError{Pair: v} } // Check required pairs. @@ -330,23 +372,10 @@ func (s *Service) parsePairServiceGet(opts []Pair) (pairServiceGet, error) { } for _, v := range opts { - // isUnsupportedPair records whether current pair is unsupported. - isUnsupportedPair := false - switch v.Key { default: - isUnsupportedPair = true - } - - if !isUnsupportedPair { - continue + return pairServiceGet{}, services.PairUnsupportedError{Pair: v} } - - // If user enables the loose operation feature, we will ignore PairUnsupportedError. - if s.features.LooseOperationAll || s.features.LooseOperationGet { - continue - } - return pairServiceGet{}, services.PairUnsupportedError{Pair: v} } // Check required pairs. @@ -366,23 +395,10 @@ func (s *Service) parsePairServiceList(opts []Pair) (pairServiceList, error) { } for _, v := range opts { - // isUnsupportedPair records whether current pair is unsupported. - isUnsupportedPair := false - switch v.Key { default: - isUnsupportedPair = true - } - - if !isUnsupportedPair { - continue - } - - // If user enables the loose operation feature, we will ignore PairUnsupportedError. - if s.features.LooseOperationAll || s.features.LooseOperationList { - continue + return pairServiceList{}, services.PairUnsupportedError{Pair: v} } - return pairServiceList{}, services.PairUnsupportedError{Pair: v} } // Check required pairs. @@ -492,22 +508,42 @@ func (s *Service) ListWithContext(ctx context.Context, pairs ...Pair) (sti *Stor } var ( + _ Direr = &Storage{} _ Storager = &Storage{} ) type StorageFeatures struct { - LooseOperationAll bool - LooseOperationCreate bool - LooseOperationDelete bool - LooseOperationList bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. + LooseOperationAll bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. + LooseOperationCreate bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. + LooseOperationCreateDir bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. + LooseOperationDelete bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. + LooseOperationList bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. LooseOperationMetadata bool - LooseOperationRead bool - LooseOperationStat bool - LooseOperationWrite bool - + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. + LooseOperationRead bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. + LooseOperationStat bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. + LooseOperationWrite bool + + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. VirtualOperationAll bool + // Deprecated: This field has been deprecated by GSP-109, planned be removed in v4.3.0. VirtualPairAll bool + // VirtualDir virtual_dir feature is designed for a service that doesn't have native dir support but wants to provide simulated operations. + // + // - If this feature is disabled (the default behavior), the service will behave like it doesn't have any dir support. + // - If this feature is enabled, the service will support simulated dir behavior in create_dir, create, list, delete, and so on. + // + // This feature was introduced in GSP-109. + VirtualDir bool } // pairStorageNew is the parsed struct @@ -542,13 +578,13 @@ func parsePairStorageNew(opts []Pair) (pairStorageNew, error) { result.HasName = true result.Name = v.Value.(string) // Optional pairs - case pairDefaultStoragePairs: + case "default_storage_pairs": if result.HasDefaultStoragePairs { continue } result.HasDefaultStoragePairs = true result.DefaultStoragePairs = v.Value.(DefaultStoragePairs) - case pairStorageFeatures: + case "storage_features": if result.HasStorageFeatures { continue } @@ -571,18 +607,21 @@ func parsePairStorageNew(opts []Pair) (pairStorageNew, error) { // DefaultStoragePairs is default pairs for specific action type DefaultStoragePairs struct { - Create []Pair - Delete []Pair - List []Pair - Metadata []Pair - Read []Pair - Stat []Pair - Write []Pair + Create []Pair + CreateDir []Pair + Delete []Pair + List []Pair + Metadata []Pair + Read []Pair + Stat []Pair + Write []Pair } // pairStorageCreate is the parsed struct type pairStorageCreate struct { - pairs []Pair + pairs []Pair + HasObjectMode bool + ObjectMode ObjectMode } // parsePairStorageCreate will parse Pair slice into *pairStorageCreate @@ -592,23 +631,49 @@ func (s *Storage) parsePairStorageCreate(opts []Pair) (pairStorageCreate, error) } for _, v := range opts { - // isUnsupportedPair records whether current pair is unsupported. - isUnsupportedPair := false - switch v.Key { + case "object_mode": + if result.HasObjectMode { + continue + } + result.HasObjectMode = true + result.ObjectMode = v.Value.(ObjectMode) + continue default: - isUnsupportedPair = true + return pairStorageCreate{}, services.PairUnsupportedError{Pair: v} } + } - if !isUnsupportedPair { - continue - } + // Check required pairs. + + return result, nil +} + +// pairStorageCreateDir is the parsed struct +type pairStorageCreateDir struct { + pairs []Pair + HasStorageClass bool + StorageClass string +} + +// parsePairStorageCreateDir will parse Pair slice into *pairStorageCreateDir +func (s *Storage) parsePairStorageCreateDir(opts []Pair) (pairStorageCreateDir, error) { + result := pairStorageCreateDir{ + pairs: opts, + } - // If user enables the loose operation feature, we will ignore PairUnsupportedError. - if s.features.LooseOperationAll || s.features.LooseOperationCreate { + for _, v := range opts { + switch v.Key { + case "storage_class": + if result.HasStorageClass { + continue + } + result.HasStorageClass = true + result.StorageClass = v.Value.(string) continue + default: + return pairStorageCreateDir{}, services.PairUnsupportedError{Pair: v} } - return pairStorageCreate{}, services.PairUnsupportedError{Pair: v} } // Check required pairs. @@ -618,7 +683,9 @@ func (s *Storage) parsePairStorageCreate(opts []Pair) (pairStorageCreate, error) // pairStorageDelete is the parsed struct type pairStorageDelete struct { - pairs []Pair + pairs []Pair + HasObjectMode bool + ObjectMode ObjectMode } // parsePairStorageDelete will parse Pair slice into *pairStorageDelete @@ -628,23 +695,17 @@ func (s *Storage) parsePairStorageDelete(opts []Pair) (pairStorageDelete, error) } for _, v := range opts { - // isUnsupportedPair records whether current pair is unsupported. - isUnsupportedPair := false - switch v.Key { - default: - isUnsupportedPair = true - } - - if !isUnsupportedPair { - continue - } - - // If user enables the loose operation feature, we will ignore PairUnsupportedError. - if s.features.LooseOperationAll || s.features.LooseOperationDelete { + case "object_mode": + if result.HasObjectMode { + continue + } + result.HasObjectMode = true + result.ObjectMode = v.Value.(ObjectMode) continue + default: + return pairStorageDelete{}, services.PairUnsupportedError{Pair: v} } - return pairStorageDelete{}, services.PairUnsupportedError{Pair: v} } // Check required pairs. @@ -666,9 +727,6 @@ func (s *Storage) parsePairStorageList(opts []Pair) (pairStorageList, error) { } for _, v := range opts { - // isUnsupportedPair records whether current pair is unsupported. - isUnsupportedPair := false - switch v.Key { case "list_mode": if result.HasListMode { @@ -678,18 +736,8 @@ func (s *Storage) parsePairStorageList(opts []Pair) (pairStorageList, error) { result.ListMode = v.Value.(ListMode) continue default: - isUnsupportedPair = true + return pairStorageList{}, services.PairUnsupportedError{Pair: v} } - - if !isUnsupportedPair { - continue - } - - // If user enables the loose operation feature, we will ignore PairUnsupportedError. - if s.features.LooseOperationAll || s.features.LooseOperationList { - continue - } - return pairStorageList{}, services.PairUnsupportedError{Pair: v} } // Check required pairs. @@ -709,23 +757,10 @@ func (s *Storage) parsePairStorageMetadata(opts []Pair) (pairStorageMetadata, er } for _, v := range opts { - // isUnsupportedPair records whether current pair is unsupported. - isUnsupportedPair := false - switch v.Key { default: - isUnsupportedPair = true + return pairStorageMetadata{}, services.PairUnsupportedError{Pair: v} } - - if !isUnsupportedPair { - continue - } - - // If user enables the loose operation feature, we will ignore PairUnsupportedError. - if s.features.LooseOperationAll || s.features.LooseOperationMetadata { - continue - } - return pairStorageMetadata{}, services.PairUnsupportedError{Pair: v} } // Check required pairs. @@ -753,11 +788,8 @@ func (s *Storage) parsePairStorageRead(opts []Pair) (pairStorageRead, error) { } for _, v := range opts { - // isUnsupportedPair records whether current pair is unsupported. - isUnsupportedPair := false - switch v.Key { - case pairEncryptionKey: + case "encryption_key": if result.HasEncryptionKey { continue } @@ -786,18 +818,8 @@ func (s *Storage) parsePairStorageRead(opts []Pair) (pairStorageRead, error) { result.Size = v.Value.(int64) continue default: - isUnsupportedPair = true - } - - if !isUnsupportedPair { - continue + return pairStorageRead{}, services.PairUnsupportedError{Pair: v} } - - // If user enables the loose operation feature, we will ignore PairUnsupportedError. - if s.features.LooseOperationAll || s.features.LooseOperationRead { - continue - } - return pairStorageRead{}, services.PairUnsupportedError{Pair: v} } // Check required pairs. @@ -807,7 +829,9 @@ func (s *Storage) parsePairStorageRead(opts []Pair) (pairStorageRead, error) { // pairStorageStat is the parsed struct type pairStorageStat struct { - pairs []Pair + pairs []Pair + HasObjectMode bool + ObjectMode ObjectMode } // parsePairStorageStat will parse Pair slice into *pairStorageStat @@ -817,23 +841,17 @@ func (s *Storage) parsePairStorageStat(opts []Pair) (pairStorageStat, error) { } for _, v := range opts { - // isUnsupportedPair records whether current pair is unsupported. - isUnsupportedPair := false - switch v.Key { - default: - isUnsupportedPair = true - } - - if !isUnsupportedPair { - continue - } - - // If user enables the loose operation feature, we will ignore PairUnsupportedError. - if s.features.LooseOperationAll || s.features.LooseOperationStat { + case "object_mode": + if result.HasObjectMode { + continue + } + result.HasObjectMode = true + result.ObjectMode = v.Value.(ObjectMode) continue + default: + return pairStorageStat{}, services.PairUnsupportedError{Pair: v} } - return pairStorageStat{}, services.PairUnsupportedError{Pair: v} } // Check required pairs. @@ -865,9 +883,6 @@ func (s *Storage) parsePairStorageWrite(opts []Pair) (pairStorageWrite, error) { } for _, v := range opts { - // isUnsupportedPair records whether current pair is unsupported. - isUnsupportedPair := false - switch v.Key { case "content_md5": if result.HasContentMd5 { @@ -883,7 +898,7 @@ func (s *Storage) parsePairStorageWrite(opts []Pair) (pairStorageWrite, error) { result.HasContentType = true result.ContentType = v.Value.(string) continue - case pairEncryptionKey: + case "encryption_key": if result.HasEncryptionKey { continue } @@ -897,14 +912,14 @@ func (s *Storage) parsePairStorageWrite(opts []Pair) (pairStorageWrite, error) { result.HasIoCallback = true result.IoCallback = v.Value.(func([]byte)) continue - case pairKmsKeyName: + case "kms_key_name": if result.HasKmsKeyName { continue } result.HasKmsKeyName = true result.KmsKeyName = v.Value.(string) continue - case pairStorageClass: + case "storage_class": if result.HasStorageClass { continue } @@ -912,18 +927,8 @@ func (s *Storage) parsePairStorageWrite(opts []Pair) (pairStorageWrite, error) { result.StorageClass = v.Value.(string) continue default: - isUnsupportedPair = true + return pairStorageWrite{}, services.PairUnsupportedError{Pair: v} } - - if !isUnsupportedPair { - continue - } - - // If user enables the loose operation feature, we will ignore PairUnsupportedError. - if s.features.LooseOperationAll || s.features.LooseOperationWrite { - continue - } - return pairStorageWrite{}, services.PairUnsupportedError{Pair: v} } // Check required pairs. @@ -933,6 +938,11 @@ func (s *Storage) parsePairStorageWrite(opts []Pair) (pairStorageWrite, error) { // Create will create a new object without any api call. // +// ## Behavior +// +// - Create SHOULD NOT send any API call. +// - Create SHOULD accept ObjectMode pair as object mode. +// // This function will create a context by default. func (s *Storage) Create(path string, pairs ...Pair) (o *Object) { pairs = append(pairs, s.defaultPairs.Create...) @@ -944,7 +954,42 @@ func (s *Storage) Create(path string, pairs ...Pair) (o *Object) { return s.create(path, opt) } -// Delete will delete an Object from service. +// CreateDir will create a new dir object. +// +// This function will create a context by default. +func (s *Storage) CreateDir(path string, pairs ...Pair) (o *Object, err error) { + ctx := context.Background() + return s.CreateDirWithContext(ctx, path, pairs...) +} + +// CreateDirWithContext will create a new dir object. +func (s *Storage) CreateDirWithContext(ctx context.Context, path string, pairs ...Pair) (o *Object, err error) { + defer func() { + err = s.formatError("create_dir", err, path) + }() + + pairs = append(pairs, s.defaultPairs.CreateDir...) + var opt pairStorageCreateDir + + opt, err = s.parsePairStorageCreateDir(pairs) + if err != nil { + return + } + + return s.createDir(ctx, path, opt) +} + +// Delete will delete an object from service. +// +// ## Behavior +// +// - Delete only delete one and only one object. +// - Service DON'T NEED to support remove all. +// - User NEED to implement remove_all by themself. +// - Delete is idempotent. +// - Successful delete always return nil error. +// - Delete SHOULD never return `ObjectNotExist` +// - Delete DON'T NEED to check the object exist or not. // // This function will create a context by default. func (s *Storage) Delete(path string, pairs ...Pair) (err error) { @@ -952,7 +997,17 @@ func (s *Storage) Delete(path string, pairs ...Pair) (err error) { return s.DeleteWithContext(ctx, path, pairs...) } -// DeleteWithContext will delete an Object from service. +// DeleteWithContext will delete an object from service. +// +// ## Behavior +// +// - Delete only delete one and only one object. +// - Service DON'T NEED to support remove all. +// - User NEED to implement remove_all by themself. +// - Delete is idempotent. +// - Successful delete always return nil error. +// - Delete SHOULD never return `ObjectNotExist` +// - Delete DON'T NEED to check the object exist or not. func (s *Storage) DeleteWithContext(ctx context.Context, path string, pairs ...Pair) (err error) { defer func() { err = s.formatError("delete", err, path) @@ -1034,6 +1089,12 @@ func (s *Storage) ReadWithContext(ctx context.Context, path string, w io.Writer, // Stat will stat a path to get info of an object. // +// ## Behavior +// +// - Stat SHOULD accept ObjectMode pair as hints. +// - Service COULD have different implementations for different object mode. +// - Service SHOULD check if returning ObjectMode is match +// // This function will create a context by default. func (s *Storage) Stat(path string, pairs ...Pair) (o *Object, err error) { ctx := context.Background() @@ -1041,6 +1102,12 @@ func (s *Storage) Stat(path string, pairs ...Pair) (o *Object, err error) { } // StatWithContext will stat a path to get info of an object. +// +// ## Behavior +// +// - Stat SHOULD accept ObjectMode pair as hints. +// - Service COULD have different implementations for different object mode. +// - Service SHOULD check if returning ObjectMode is match func (s *Storage) StatWithContext(ctx context.Context, path string, pairs ...Pair) (o *Object, err error) { defer func() { err = s.formatError("stat", err, path) @@ -1085,4 +1152,5 @@ func (s *Storage) WriteWithContext(ctx context.Context, path string, r io.Reader func init() { services.RegisterServicer(Type, NewServicer) services.RegisterStorager(Type, NewStorager) + services.RegisterSchema(Type, pairMap) } diff --git a/go.mod b/go.mod index 52039c9..fe2ef05 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,8 @@ go 1.15 require ( cloud.google.com/go/storage v1.15.0 - github.com/beyondstorage/go-integration-test/v4 v4.0.0 - github.com/beyondstorage/go-storage/v4 v4.0.1-0.20210530044854-1c928ddbe52d + github.com/beyondstorage/go-integration-test/v4 v4.1.1 + github.com/beyondstorage/go-storage/v4 v4.2.0 github.com/google/uuid v1.2.0 golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c google.golang.org/api v0.47.0 diff --git a/go.sum b/go.sum index bbee822..c61b124 100644 --- a/go.sum +++ b/go.sum @@ -43,14 +43,12 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Xuanwo/templateutils v0.1.0 h1:WpkWOqQtIQ2vAIpJLa727DdN8WtxhUkkbDGa6UhntJY= github.com/Xuanwo/templateutils v0.1.0/go.mod h1:OdE0DJ+CJxDBq6psX5DPV+gOZi8bhuHuVUpPCG++Wb8= -github.com/beyondstorage/go-integration-test/v4 v4.0.0 h1:tdXQV9yxQ3Q6p9xfyQKzK3MEo9r9j9g3uT5+3sbVtnQ= -github.com/beyondstorage/go-integration-test/v4 v4.0.0/go.mod h1:26/JF4b0XxRN0pL4kihpnVNhbbw+QWvmmvgxfnFJDfA= -github.com/beyondstorage/go-storage/v4 v4.0.0/go.mod h1:oa2dYco+xplPj99WSBnYVw/xXvRkIKWSSVDQKNZ5Kz8= -github.com/beyondstorage/go-storage/v4 v4.0.1-0.20210530044854-1c928ddbe52d h1:s9t6VNNRDqmg+PXyXtVEsxIM7xZQIJOYlma28IpkbNQ= -github.com/beyondstorage/go-storage/v4 v4.0.1-0.20210530044854-1c928ddbe52d/go.mod h1:kXMu07IDZaKtxbqI1ufuhqo0FjYe0nH7zPCbBanln/Y= -github.com/beyondstorage/specs/go v0.0.0-20210521044836-3d41c1d9c97f/go.mod h1:f5VvmLHc/dNJwl+/yAv/TOHdev3phvuEswx8DIXiSQQ= -github.com/beyondstorage/specs/go v0.0.0-20210530044123-3ff75e192bc9 h1:YSiF27cAHlDZk9q+oaEHQbA8dH8XTvYxeTOoPzNCwOQ= -github.com/beyondstorage/specs/go v0.0.0-20210530044123-3ff75e192bc9/go.mod h1:f5VvmLHc/dNJwl+/yAv/TOHdev3phvuEswx8DIXiSQQ= +github.com/beyondstorage/go-integration-test/v4 v4.1.1 h1:9bSXKbr6hLb4+ZsmAhWE32fvqhyrpub4U4qgBGeth4A= +github.com/beyondstorage/go-integration-test/v4 v4.1.1/go.mod h1:ihtCaOJvaHGE0v+IhY6ZUF5NU1IND6xmdrJI9Lq/jhc= +github.com/beyondstorage/go-storage/v4 v4.2.0 h1:J0xqqy4qEQRtIS2zUWMA5wRXVHx/cxX5fHsU2ezA3+I= +github.com/beyondstorage/go-storage/v4 v4.2.0/go.mod h1:rUNzOXcikYk5w0ewvNsKbztg7ndQDyDvjDuP0bznSLU= +github.com/beyondstorage/specs/go v0.0.0-20210623065218-d1c2d7d81259 h1:mW9XpHLc6pdXBRnsha1VlqF0rNsB/Oc+8l+5UYngmRA= +github.com/beyondstorage/specs/go v0.0.0-20210623065218-d1c2d7d81259/go.mod h1:vF/Q0P1tCvhVAUrxg7i6NvrARRMQVTAuQdDNqpSzR1w= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= @@ -65,8 +63,9 @@ github.com/dave/gopackages v0.0.0-20170318123100-46e7023ec56e/go.mod h1:i00+b/gK github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= github.com/dave/kerr v0.0.0-20170318121727-bc25dd6abe8e/go.mod h1:qZqlPyPvfsDJt+3wHJ1EvSXDuVjFTK0j2p/ca+gtsb8= github.com/dave/rebecca v0.9.1/go.mod h1:N6XYdMD/OKw3lkF3ywh8Z6wPGuwNFDNtWYEMFWEmXBA= -github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -166,19 +165,22 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/pelletier/go-toml v1.9.1 h1:a6qW1EVNZWH9WGI6CsYdD8WAylkoXBS5yv0XHlh17Tc= -github.com/pelletier/go-toml v1.9.1/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ= +github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -312,6 +314,7 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/service.toml b/service.toml index 91f6bf1..569e56e 100644 --- a/service.toml +++ b/service.toml @@ -4,16 +4,32 @@ name = "gcs" required = ["credential", "project_id"] optional = ["service_features", "default_service_pairs", "http_client_options"] +[namespace.storage] +features = ["virtual_dir"] +implement = ["direr"] + [namespace.storage.new] required = ["name"] optional = ["storage_features", "default_storage_pairs", "work_dir"] +[namespace.storage.op.create] +optional = ["object_mode"] + +[namespace.storage.op.create_dir] +optional = ["storage_class"] + +[namespace.storage.op.delete] +optional = ["object_mode"] + [namespace.storage.op.list] optional = ["list_mode"] [namespace.storage.op.read] optional = ["offset", "io_callback", "size", "encryption_key"] +[namespace.storage.op.stat] +optional = ["object_mode"] + [namespace.storage.op.write] optional = ["content_md5", "content_type", "io_callback", "storage_class", "encryption_key", "kms_key_name"] diff --git a/storage.go b/storage.go index 5eca043..53ec65f 100644 --- a/storage.go +++ b/storage.go @@ -8,22 +8,76 @@ import ( gs "cloud.google.com/go/storage" "google.golang.org/api/iterator" + ps "github.com/beyondstorage/go-storage/v4/pairs" "github.com/beyondstorage/go-storage/v4/pkg/iowrap" "github.com/beyondstorage/go-storage/v4/services" . "github.com/beyondstorage/go-storage/v4/types" ) func (s *Storage) create(path string, opt pairStorageCreate) (o *Object) { - o = s.newObject(false) - o.Mode = ModeRead - o.ID = s.getAbsPath(path) + rp := s.getAbsPath(path) + + if opt.HasObjectMode && opt.ObjectMode.IsDir() { + if !s.features.VirtualDir { + return + } + // Add `/` at the end of path to simulate a directory. + rp += "/" + o = s.newObject(true) + o.Mode = ModeDir + } else { + o = s.newObject(false) + o.Mode = ModeRead + } + + o.ID = rp o.Path = path return o } +func (s *Storage) createDir(ctx context.Context, path string, opt pairStorageCreateDir) (o *Object, err error) { + if !s.features.VirtualDir { + err = NewOperationNotImplementedError("create_dir") + return + } + + rp := s.getAbsPath(path) + + // Add `/` at the end of `path` to simulate a directory. + // ref: https://cloud.google.com/storage/docs/naming-objects + rp += "/" + + object := s.bucket.Object(rp) + w := object.NewWriter(ctx) + w.Size = 0 + if opt.HasStorageClass { + w.StorageClass = opt.StorageClass + } + + cerr := w.Close() + if cerr != nil { + err = cerr + } + + o = s.newObject(true) + o.ID = rp + o.Path = path + o.Mode |= ModeDir + return +} + func (s *Storage) delete(ctx context.Context, path string, opt pairStorageDelete) (err error) { rp := s.getAbsPath(path) + if opt.HasObjectMode && opt.ObjectMode.IsDir() { + if !s.features.VirtualDir { + err = services.PairUnsupportedError{Pair: ps.WithObjectMode(opt.ObjectMode)} + return + } + + rp += "/" + } + err = s.bucket.Object(rp).Delete(ctx) if err != nil && errors.Is(err, gs.ErrObjectNotExist) { // Omit `ErrObjectNotExist` error here. @@ -168,12 +222,31 @@ func (s *Storage) read(ctx context.Context, path string, w io.Writer, opt pairSt func (s *Storage) stat(ctx context.Context, path string, opt pairStorageStat) (o *Object, err error) { rp := s.getAbsPath(path) + if opt.HasObjectMode && opt.ObjectMode.IsDir() { + if !s.features.VirtualDir { + err = services.PairUnsupportedError{Pair: ps.WithObjectMode(opt.ObjectMode)} + return + } + + rp += "/" + } + attr, err := s.bucket.Object(rp).Attrs(ctx) if err != nil { return nil, err } - return s.formatFileObject(attr) + o, err = s.formatFileObject(attr) + if err != nil { + return nil, err + } + + if opt.HasObjectMode && opt.ObjectMode.IsDir() { + o.Path = path + o.Mode.Add(ModeDir) + } + + return o, nil } func (s *Storage) write(ctx context.Context, path string, r io.Reader, size int64, opt pairStorageWrite) (n int64, err error) { diff --git a/tests/storage_test.go b/tests/storage_test.go index fd4250c..dcf38e0 100644 --- a/tests/storage_test.go +++ b/tests/storage_test.go @@ -13,3 +13,10 @@ func TestStorage(t *testing.T) { } tests.TestStorager(t, setupTest(t)) } + +func TestDirer(t *testing.T) { + if os.Getenv("STORAGE_GCS_INTEGRATION_TEST") != "on" { + t.Skipf("STORAGE_GCS_INTEGRATION_TEST is not 'on', skipped") + } + tests.TestDirer(t, setupTest(t)) +} diff --git a/tests/utils_test.go b/tests/utils_test.go index 4016aa2..3c08c5a 100644 --- a/tests/utils_test.go +++ b/tests/utils_test.go @@ -19,6 +19,9 @@ func setupTest(t *testing.T) types.Storager { ps.WithName(os.Getenv("STORAGE_GCS_NAME")), ps.WithWorkDir("/"+uuid.New().String()+"/"), gcs.WithProjectID(os.Getenv("STORAGE_GCS_PROJECT_ID")), + gcs.WithStorageFeatures(gcs.StorageFeatures{ + VirtualDir: true, + }), ) if err != nil { t.Errorf("new storager: %v", err) diff --git a/utils.go b/utils.go index bc9275a..58f7498 100644 --- a/utils.go +++ b/utils.go @@ -50,6 +50,7 @@ type Storage struct { features StorageFeatures typ.UnimplementedStorager + typ.UnimplementedDirer } // String implements Storager.String @@ -276,14 +277,14 @@ func (s *Storage) formatFileObject(v *gs.ObjectAttrs) (o *typ.Object, err error) o.SetContentMd5(base64.StdEncoding.EncodeToString(v.MD5)) } - var sm ObjectMetadata + var sm ObjectSystemMetadata if value := v.StorageClass; value != "" { sm.StorageClass = value } if value := v.CustomerKeySHA256; value != "" { sm.EncryptionKeySha256 = value } - o.SetServiceMetadata(sm) + o.SetSystemMetadata(sm) return }