diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 233db0df2..fb2100596 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -20,3 +20,7 @@ updates: directory: "/services/bos" schedule: interval: daily + - package-ecosystem: gomod + directory: "/services/cos" + schedule: + interval: daily diff --git a/.github/workflows/services-test-cos.yml b/.github/workflows/services-test-cos.yml new file mode 100644 index 000000000..5d3ca5c80 --- /dev/null +++ b/.github/workflows/services-test-cos.yml @@ -0,0 +1,40 @@ +name: "Services Test Cos" + +on: + push: + paths: + - 'services/cos/**' + pull_request: + paths: + - 'services/cos/**' + +jobs: + services_test_cos: + name: "Services Test Cos" + runs-on: self-hosted + + strategy: + matrix: + go: [ "1.15", "1.16" ] + + steps: + - name: Set up Go 1.x + uses: actions/setup-go@v2 + with: + go-version: ${{ matrix.go }} + + - name: Load secret + uses: 1password/load-secrets-action@v1 + env: + STORAGE_COS_CREDENTIAL: op://Engineering/Cos/testing/credential + STORAGE_COS_NAME: op://Engineering/Cos/testing/name + STORAGE_COS_LOCATION: op://Engineering/Cos/testing/location + + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Test + env: + STORAGE_COS_INTEGRATION_TEST: on + working-directory: services/cos + run: make integration_test diff --git a/services/cos/.gitignore b/services/cos/.gitignore new file mode 100644 index 000000000..a2eb8ad8d --- /dev/null +++ b/services/cos/.gitignore @@ -0,0 +1,7 @@ +coverage.* +bin/ +Makefile.env + +# Jetbrain IDE +.idea +*.iml diff --git a/services/cos/CHANGELOG.md b/services/cos/CHANGELOG.md new file mode 100644 index 000000000..37f5ed2d1 --- /dev/null +++ b/services/cos/CHANGELOG.md @@ -0,0 +1,99 @@ +# Change Log + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/) +and this project adheres to [Semantic Versioning](https://semver.org/). + +## [v2.3.0] - 2021-09-13 + +### Changed + +- ci: Enable auto merge on dependabot +- ci: Cleanup Service Integration Tests (#57) +- docs: Update README (#59) + +### Upgraded + +- ci: Upgrade Xuanwo/fetch-metadata +- build(deps): bump github.com/tencentyun/cos-go-sdk-v5 to 0.7.31 (#58) + +## [v2.2.0] - 2021-07-17 + +### Added + +- ci: Add gofmt action (#37) +- ci: Add dependabot auto build support (#40) +- ci: Add diff check action (#41) + +### Changed + +- storage: Update types in service.toml to golang types (#46) +- storage: Implement GSP-654 Unify List Behavior (#46) + +### Fixed + +- ci: Fix auto-build not work correctly + +## [v2.1.0] - 2021-06-29 + +### Added + +- *: Implement GSP-87 Feature Gates (#27) +- storage: Add CreateDir (#28) +- storage: Implement GSP-97 Add Restrictions in Storage Metadata (#32) + +### Changed + +- *: Implement GSP-109 Redesign Features (#32) +- *: Implement GSP-117 Rename Service to System as the Opposite to Global (#32) + +### Upgraded + +- build(deps): bump github.com/tencentyun/cos-go-sdk-v5 to 0.7.27 (#34) + +## [v2.0.0] - 2021-05-24 + +### Added + +- storage: Implement SSE support (#17) +- services: implement GSP-47 & GSP-51 (#21) +- storage: Implement multipart support (#23) + +### Changed + +- storage: Idempotent storager delete operation (#20) +- *: Implement GSP-73 Organization rename (#24) + +## [v1.1.0] - 2021-04-24 + +### Added + +- pair: Implement default pair support for service (#4) +- storage: Implement Create API (#13) +- *: Add UnimplementedStub (#15) +- tests: Introduce STORAGE_COS_INTEGRATION_TEST (#16) +- tests: Add docs for how to run tests +- storage: Implement GSP-40 (#18) + +### Changed + +- docs: Migrate zulip to matrix +- build: Fix build scripts +- ci: Only run Integration Test while push to master + +### Upgraded + +- build(deps): bump github.com/tencentyun/cos-go-sdk-v5 from 0.7.19 to 0.7.24 + +## v1.0.0 - 2021-02-08 + +### Added + +- Implement cos services. + +[v2.3.0]: https://github.com/beyondstorage/go-service-cos/compare/v2.2.0...v2.3.0 +[v2.2.0]: https://github.com/beyondstorage/go-service-cos/compare/v2.1.0...v2.2.0 +[v2.1.0]: https://github.com/beyondstorage/go-service-cos/compare/v2.0.0...v2.1.0 +[v2.0.0]: https://github.com/beyondstorage/go-service-cos/compare/v1.1.0...v2.0.0 +[v1.1.0]: https://github.com/beyondstorage/go-service-cos/compare/v1.0.0...v1.1.0 diff --git a/services/cos/Makefile b/services/cos/Makefile new file mode 100644 index 000000000..f14abf437 --- /dev/null +++ b/services/cos/Makefile @@ -0,0 +1,43 @@ +SHELL := /bin/bash + +-include Makefile.env + +.PHONY: all check format vet lint build test generate tidy + +help: + @echo "Please use \`make \` where is one of" + @echo " check to do static check" + @echo " build to create bin directory and build" + @echo " generate to generate code" + @echo " test to run test" + +check: vet + +format: + go fmt ./... + +vet: + go vet ./... + +generate: + @echo "generate code" + go generate ./... + go fmt ./... + +build: tidy generate check + go build ./... + +test: + go test -race -coverprofile=coverage.txt -covermode=atomic -v . + go tool cover -html="coverage.txt" -o "coverage.html" + +integration_test: + go test -count=1 -race -covermode=atomic -v ./tests + +tidy: + go mod tidy + go mod verify + +clean: + @echo "clean generated files" + find . -type f -name 'generated.go' -delete diff --git a/services/cos/Makefile.env.example b/services/cos/Makefile.env.example new file mode 100644 index 000000000..81fff4682 --- /dev/null +++ b/services/cos/Makefile.env.example @@ -0,0 +1,4 @@ +export STORAGE_COS_INTEGRATION_TEST=on +export STORAGE_COS_CREDENTIAL=hmac:access_key:secret_key +export STORAGE_COS_NAME=bucketname +export STORAGE_COS_LOCATION=bucketregion diff --git a/services/cos/README.md b/services/cos/README.md new file mode 100644 index 000000000..eb585a036 --- /dev/null +++ b/services/cos/README.md @@ -0,0 +1,35 @@ +[![Services Test Cos](https://github.com/beyondstorage/go-storage/actions/workflows/services-test-cos.yml/badge.svg)](https://github.com/beyondstorage/go-storage/actions/workflows/services-test-cos.yml) + +# cos + +[COS(Cloud Object Storage)](https://cloud.tencent.com/product/cos) service support for [go-storage](https://github.com/beyondstorage/go-storage). + +## Install + +```go +go get go.beyondstorage.io/services/cos/v3 +``` + +## Usage + +```go +import ( + "log" + + _ "go.beyondstorage.io/v5/services/cos/v3" + "go.beyondstorage.io/v5/services" +) + +func main() { + store, err := services.NewStoragerFromString("cos://bucket_name/path/to/workdir?credential=hmac::") + if err != nil { + log.Fatal(err) + } + + // Write data from io.Reader into hello.txt + n, err := store.Write("hello.txt", r, length) +} +``` + +- See more examples in [go-storage-example](https://github.com/beyondstorage/go-storage-example). +- Read [more docs](https://beyondstorage.io/docs/go-storage/services/cos) about go-service-cos. diff --git a/services/cos/doc.go b/services/cos/doc.go new file mode 100644 index 000000000..006fa5740 --- /dev/null +++ b/services/cos/doc.go @@ -0,0 +1,6 @@ +/* +Package cos provided support for Tencent Cloud's Cloud Object Storage (https://intl.cloud.tencent.com/product/cos) +*/ +package cos + +//go:generate go run -tags tools go.beyondstorage.io/v5/cmd/definitions service.toml diff --git a/services/cos/error.go b/services/cos/error.go new file mode 100644 index 000000000..bd1a56e1d --- /dev/null +++ b/services/cos/error.go @@ -0,0 +1,8 @@ +package cos + +import "go.beyondstorage.io/v5/services" + +var ( + // ErrServerSideEncryptionCustomerKeyInvalid will be returned while server-side encryption customer key is invalid. + ErrServerSideEncryptionCustomerKeyInvalid = services.NewErrorCode("invalid server-side encryption customer key") +) diff --git a/services/cos/generated.go b/services/cos/generated.go new file mode 100644 index 000000000..1b1ca8ac9 --- /dev/null +++ b/services/cos/generated.go @@ -0,0 +1,1287 @@ +// Code generated by go generate via cmd/definitions; DO NOT EDIT. +package cos + +import ( + "context" + "io" + "net/http" + "strings" + "time" + + . "go.beyondstorage.io/v5/pairs" + "go.beyondstorage.io/v5/pkg/httpclient" + "go.beyondstorage.io/v5/services" + . "go.beyondstorage.io/v5/types" +) + +var ( + _ Storager + _ services.ServiceError + _ httpclient.Options + _ time.Duration + _ http.Request + _ Error +) + +// Type is the type for cos +const Type = "cos" + +// ObjectSystemMetadata stores system metadata for object. +type ObjectSystemMetadata struct { + ServerSideEncryption string + ServerSideEncryptionCosKmsKeyID string + ServerSideEncryptionCustomerAlgorithm string + ServerSideEncryptionCustomerKeyMd5 string + StorageClass string +} + +// GetObjectSystemMetadata will get ObjectSystemMetadata from Object. +// +// - This function should not be called by service implementer. +// - The returning ObjectServiceMetadata is read only and should not be modified. +func GetObjectSystemMetadata(o *Object) ObjectSystemMetadata { + sm, ok := o.GetSystemMetadata() + if ok { + return sm.(ObjectSystemMetadata) + } + return ObjectSystemMetadata{} +} + +// setObjectSystemMetadata will set ObjectSystemMetadata into Object. +// +// - This function should only be called once, please make sure all data has been written before set. +func setObjectSystemMetadata(o *Object, sm ObjectSystemMetadata) { + o.SetSystemMetadata(sm) +} + +// StorageSystemMetadata stores system metadata for object. +type StorageSystemMetadata struct { + ServerSideEncryption string + ServerSideEncryptionCosKmsKeyID string + ServerSideEncryptionCustomerAlgorithm string + ServerSideEncryptionCustomerKeyMd5 string + StorageClass string +} + +// GetStorageSystemMetadata will get StorageSystemMetadata from Storage. +// +// - This function should not be called by service implementer. +// - The returning StorageServiceMetadata is read only and should not be modified. +func GetStorageSystemMetadata(s *StorageMeta) StorageSystemMetadata { + sm, ok := s.GetSystemMetadata() + if ok { + return sm.(StorageSystemMetadata) + } + return StorageSystemMetadata{} +} + +// setStorageSystemMetadata will set StorageSystemMetadata into Storage. +// +// - This function should only be called once, please make sure all data has been written before set. +func setStorageSystemMetadata(s *StorageMeta, sm StorageSystemMetadata) { + s.SetSystemMetadata(sm) +} + +// WithDefaultServicePairs will apply default_service_pairs value to Options. +func WithDefaultServicePairs(v DefaultServicePairs) Pair { + return Pair{Key: "default_service_pairs", Value: v} +} + +// WithDefaultStoragePairs will apply default_storage_pairs value to Options. +func WithDefaultStoragePairs(v DefaultStoragePairs) Pair { + return Pair{Key: "default_storage_pairs", Value: v} +} + +// WithEnableVirtualDir will apply enable_virtual_dir value to Options. +// +// virtual_dir feature is designed for a service that doesn't have native dir support but wants to +// provide simulated operations. +// +// - If this feature is disabled (the default behavior), the service will behave like it doesn't have +// any dir support. +// - If this feature is enabled, the service will support simulated dir behavior in create_dir, create, +// list, delete, and so on. +// +// This feature was introduced in GSP-109. +func WithEnableVirtualDir() Pair { + return Pair{Key: "enable_virtual_dir", Value: true} +} + +// WithServerSideEncryption will apply server_side_encryption value to Options. +// +// the server-side encryption algorithm used when storing this object. It can be `AES-256` for SSE-COS, +// and `cos/kms` for SSE-KMS. +func WithServerSideEncryption(v string) Pair { + return Pair{Key: "server_side_encryption", Value: v} +} + +// WithServerSideEncryptionContext will apply server_side_encryption_context value to Options. +// +// specifies the COS KMS Encryption Context to use for object encryption. The value of this header +// is a base64-encoded UTF-8 string holding JSON with the encryption context key-value pairs. +func WithServerSideEncryptionContext(v string) Pair { + return Pair{Key: "server_side_encryption_context", Value: v} +} + +// WithServerSideEncryptionCosKmsKeyID will apply server_side_encryption_cos_kms_key_id +// value to Options. +// +// specifies the COS KMS key ID to use for object encryption. +func WithServerSideEncryptionCosKmsKeyID(v string) Pair { + return Pair{Key: "server_side_encryption_cos_kms_key_id", Value: v} +} + +// WithServerSideEncryptionCustomerAlgorithm will apply server_side_encryption_customer_algorithm +// value to Options. +// +// specifies the algorithm to use to when encrypting the object. Now only `AES256` is supported. +func WithServerSideEncryptionCustomerAlgorithm(v string) Pair { + return Pair{Key: "server_side_encryption_customer_algorithm", Value: v} +} + +// WithServerSideEncryptionCustomerKey will apply server_side_encryption_customer_key value +// to Options. +// +// specifies the customer-provided encryption key to encrypt/decrypt the source object. It must +// be a 32-byte AES-256 key. +func WithServerSideEncryptionCustomerKey(v []byte) Pair { + return Pair{Key: "server_side_encryption_customer_key", Value: v} +} + +// WithServiceFeatures will apply service_features value to Options. +func WithServiceFeatures(v ServiceFeatures) Pair { + return Pair{Key: "service_features", Value: v} +} + +// WithStorageClass will apply storage_class value to Options. +func WithStorageClass(v string) Pair { + return Pair{Key: "storage_class", Value: v} +} + +// WithStorageFeatures will apply storage_features value to Options. +func WithStorageFeatures(v StorageFeatures) Pair { + return Pair{Key: "storage_features", Value: v} +} + +var pairMap = map[string]string{"content_md5": "string", "content_type": "string", "context": "context.Context", "continuation_token": "string", "credential": "string", "default_content_type": "string", "default_io_callback": "func([]byte)", "default_service_pairs": "DefaultServicePairs", "default_storage_pairs": "DefaultStoragePairs", "enable_virtual_dir": "bool", "endpoint": "string", "expire": "time.Duration", "http_client_options": "*httpclient.Options", "interceptor": "Interceptor", "io_callback": "func([]byte)", "list_mode": "ListMode", "location": "string", "multipart_id": "string", "name": "string", "object_mode": "ObjectMode", "offset": "int64", "server_side_encryption": "string", "server_side_encryption_context": "string", "server_side_encryption_cos_kms_key_id": "string", "server_side_encryption_customer_algorithm": "string", "server_side_encryption_customer_key": "[]byte", "service_features": "ServiceFeatures", "size": "int64", "storage_class": "string", "storage_features": "StorageFeatures", "work_dir": "string"} +var _ Servicer = &Service{} + +type ServiceFeatures struct { +} + +// pairServiceNew is the parsed struct +type pairServiceNew struct { + pairs []Pair + + // Required pairs + HasCredential bool + Credential string + // Optional pairs + HasDefaultServicePairs bool + DefaultServicePairs DefaultServicePairs + HasEndpoint bool + Endpoint string + HasHTTPClientOptions bool + HTTPClientOptions *httpclient.Options + HasServiceFeatures bool + ServiceFeatures ServiceFeatures + // Enable features +} + +// parsePairServiceNew will parse Pair slice into *pairServiceNew +func parsePairServiceNew(opts []Pair) (pairServiceNew, error) { + result := + pairServiceNew{pairs: opts} + + for _, v := range opts { + switch v.Key { + case "credential": + if result.HasCredential { + continue + } + result.HasCredential = true + result.Credential = v.Value.(string) + case "default_service_pairs": + if result.HasDefaultServicePairs { + continue + } + result.HasDefaultServicePairs = true + result.DefaultServicePairs = v.Value.(DefaultServicePairs) + case "endpoint": + if result.HasEndpoint { + continue + } + result.HasEndpoint = true + result.Endpoint = v.Value.(string) + case "http_client_options": + if result.HasHTTPClientOptions { + continue + } + result.HasHTTPClientOptions = true + result.HTTPClientOptions = v.Value.(*httpclient.Options) + case "service_features": + if result.HasServiceFeatures { + continue + } + result.HasServiceFeatures = true + result.ServiceFeatures = v.Value.(ServiceFeatures) + } + } + // Enable features + + // Default pairs + + if !result.HasCredential { + return pairServiceNew{}, services.PairRequiredError{Keys: []string{"credential"}} + } + return result, nil +} + +// DefaultServicePairs is default pairs for specific action +type DefaultServicePairs struct { + Create []Pair + Delete []Pair + Get []Pair + List []Pair +} +type pairServiceCreate struct { + pairs []Pair + // Required pairs + HasLocation bool + Location string + // Optional pairs +} + +func (s *Service) parsePairServiceCreate(opts []Pair) (pairServiceCreate, error) { + result := + pairServiceCreate{pairs: opts} + + for _, v := range opts { + switch v.Key { + case "location": + if result.HasLocation { + continue + } + result.HasLocation = true + result.Location = v.Value.(string) + default: + return pairServiceCreate{}, services.PairUnsupportedError{Pair: v} + } + } + if !result.HasLocation { + return pairServiceCreate{}, services.PairRequiredError{Keys: []string{"location"}} + } + return result, nil +} + +type pairServiceDelete struct { + pairs []Pair + // Required pairs + HasLocation bool + Location string + // Optional pairs +} + +func (s *Service) parsePairServiceDelete(opts []Pair) (pairServiceDelete, error) { + result := + pairServiceDelete{pairs: opts} + + for _, v := range opts { + switch v.Key { + case "location": + if result.HasLocation { + continue + } + result.HasLocation = true + result.Location = v.Value.(string) + default: + return pairServiceDelete{}, services.PairUnsupportedError{Pair: v} + } + } + if !result.HasLocation { + return pairServiceDelete{}, services.PairRequiredError{Keys: []string{"location"}} + } + return result, nil +} + +type pairServiceGet struct { + pairs []Pair + // Required pairs + HasLocation bool + Location string + // Optional pairs +} + +func (s *Service) parsePairServiceGet(opts []Pair) (pairServiceGet, error) { + result := + pairServiceGet{pairs: opts} + + for _, v := range opts { + switch v.Key { + case "location": + if result.HasLocation { + continue + } + result.HasLocation = true + result.Location = v.Value.(string) + default: + return pairServiceGet{}, services.PairUnsupportedError{Pair: v} + } + } + if !result.HasLocation { + return pairServiceGet{}, services.PairRequiredError{Keys: []string{"location"}} + } + return result, nil +} + +type pairServiceList struct { + pairs []Pair + // Required pairs + // Optional pairs +} + +func (s *Service) parsePairServiceList(opts []Pair) (pairServiceList, error) { + result := + pairServiceList{pairs: opts} + + for _, v := range opts { + switch v.Key { + default: + return pairServiceList{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} +func (s *Service) Create(name string, pairs ...Pair) (store Storager, err error) { + ctx := context.Background() + return s.CreateWithContext(ctx, name, pairs...) +} +func (s *Service) CreateWithContext(ctx context.Context, name string, pairs ...Pair) (store Storager, err error) { + defer func() { + err = + s.formatError("create", err, name) + }() + + pairs = append(pairs, s.defaultPairs.Create...) + var opt pairServiceCreate + + opt, err = s.parsePairServiceCreate(pairs) + if err != nil { + return + } + return s.create(ctx, name, opt) +} +func (s *Service) Delete(name string, pairs ...Pair) (err error) { + ctx := context.Background() + return s.DeleteWithContext(ctx, name, pairs...) +} +func (s *Service) DeleteWithContext(ctx context.Context, name string, pairs ...Pair) (err error) { + defer func() { + err = + s.formatError("delete", err, name) + }() + + pairs = append(pairs, s.defaultPairs.Delete...) + var opt pairServiceDelete + + opt, err = s.parsePairServiceDelete(pairs) + if err != nil { + return + } + return s.delete(ctx, name, opt) +} +func (s *Service) Get(name string, pairs ...Pair) (store Storager, err error) { + ctx := context.Background() + return s.GetWithContext(ctx, name, pairs...) +} +func (s *Service) GetWithContext(ctx context.Context, name string, pairs ...Pair) (store Storager, err error) { + defer func() { + err = + s.formatError("get", err, name) + }() + + pairs = append(pairs, s.defaultPairs.Get...) + var opt pairServiceGet + + opt, err = s.parsePairServiceGet(pairs) + if err != nil { + return + } + return s.get(ctx, name, opt) +} +func (s *Service) List(pairs ...Pair) (sti *StoragerIterator, err error) { + ctx := context.Background() + return s.ListWithContext(ctx, pairs...) +} +func (s *Service) ListWithContext(ctx context.Context, pairs ...Pair) (sti *StoragerIterator, err error) { + defer func() { + err = + s.formatError("list", err, "") + }() + + pairs = append(pairs, s.defaultPairs.List...) + var opt pairServiceList + + opt, err = s.parsePairServiceList(pairs) + if err != nil { + return + } + return s.list(ctx, opt) +} + +var ( + _ Direr = &Storage{} + _ Multiparter = &Storage{} + _ Storager = &Storage{} +) + +type StorageFeatures struct { // virtual_dir feature is designed for a service that doesn't have native dir support but wants to + // provide simulated operations. + // + // - If this feature is disabled (the default behavior), the service will behave like it doesn't have + // any dir support. + // - If this feature is enabled, the service will support simulated dir behavior in create_dir, create, + // list, delete, and so on. + // + // This feature was introduced in GSP-109. + VirtualDir bool +} + +// pairStorageNew is the parsed struct +type pairStorageNew struct { + pairs []Pair + + // Required pairs + HasLocation bool + Location string + HasName bool + Name string + // Optional pairs + HasDefaultContentType bool + DefaultContentType string + HasDefaultIoCallback bool + DefaultIoCallback func([]byte) + HasDefaultStoragePairs bool + DefaultStoragePairs DefaultStoragePairs + HasStorageFeatures bool + StorageFeatures StorageFeatures + HasWorkDir bool + WorkDir string + // Enable features + hasEnableVirtualDir bool + EnableVirtualDir bool +} + +// parsePairStorageNew will parse Pair slice into *pairStorageNew +func parsePairStorageNew(opts []Pair) (pairStorageNew, error) { + result := + pairStorageNew{pairs: opts} + + for _, v := range opts { + switch v.Key { + case "location": + if result.HasLocation { + continue + } + result.HasLocation = true + result.Location = v.Value.(string) + case "name": + if result.HasName { + continue + } + result.HasName = true + result.Name = v.Value.(string) + case "default_content_type": + if result.HasDefaultContentType { + continue + } + result.HasDefaultContentType = true + result.DefaultContentType = v.Value.(string) + case "default_io_callback": + if result.HasDefaultIoCallback { + continue + } + result.HasDefaultIoCallback = true + result.DefaultIoCallback = v.Value.(func([]byte)) + case "default_storage_pairs": + if result.HasDefaultStoragePairs { + continue + } + result.HasDefaultStoragePairs = true + result.DefaultStoragePairs = v.Value.(DefaultStoragePairs) + case "storage_features": + if result.HasStorageFeatures { + continue + } + result.HasStorageFeatures = true + result.StorageFeatures = v.Value.(StorageFeatures) + case "work_dir": + if result.HasWorkDir { + continue + } + result.HasWorkDir = true + result.WorkDir = v.Value.(string) + case "enable_virtual_dir": + if result.hasEnableVirtualDir { + continue + } + result.hasEnableVirtualDir = true + result.EnableVirtualDir = true + } + } + // Enable features + if result.hasEnableVirtualDir { + result.HasStorageFeatures = true + result.StorageFeatures.VirtualDir = true + } + // Default pairs + if result.HasDefaultContentType { + result.HasDefaultStoragePairs = true + result.DefaultStoragePairs.CreateMultipart = append(result.DefaultStoragePairs.CreateMultipart, WithContentType(result.DefaultContentType)) + result.DefaultStoragePairs.Write = append(result.DefaultStoragePairs.Write, WithContentType(result.DefaultContentType)) + } + if result.HasDefaultIoCallback { + result.HasDefaultStoragePairs = true + result.DefaultStoragePairs.Read = append(result.DefaultStoragePairs.Read, WithIoCallback(result.DefaultIoCallback)) + result.DefaultStoragePairs.Write = append(result.DefaultStoragePairs.Write, WithIoCallback(result.DefaultIoCallback)) + } + if !result.HasLocation { + return pairStorageNew{}, services.PairRequiredError{Keys: []string{"location"}} + } + if !result.HasName { + return pairStorageNew{}, services.PairRequiredError{Keys: []string{"name"}} + } + return result, nil +} + +// DefaultStoragePairs is default pairs for specific action +type DefaultStoragePairs struct { + CompleteMultipart []Pair + Create []Pair + CreateDir []Pair + CreateMultipart []Pair + Delete []Pair + List []Pair + ListMultipart []Pair + Metadata []Pair + Read []Pair + Stat []Pair + Write []Pair + WriteMultipart []Pair +} +type pairStorageCompleteMultipart struct { + pairs []Pair + // Required pairs + // Optional pairs +} + +func (s *Storage) parsePairStorageCompleteMultipart(opts []Pair) (pairStorageCompleteMultipart, error) { + result := + pairStorageCompleteMultipart{pairs: opts} + + for _, v := range opts { + switch v.Key { + default: + return pairStorageCompleteMultipart{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} + +type pairStorageCreate struct { + pairs []Pair + // Required pairs + // Optional pairs + HasMultipartID bool + MultipartID string + HasObjectMode bool + ObjectMode ObjectMode +} + +func (s *Storage) parsePairStorageCreate(opts []Pair) (pairStorageCreate, error) { + result := + pairStorageCreate{pairs: opts} + + for _, v := range opts { + switch v.Key { + case "multipart_id": + if result.HasMultipartID { + continue + } + result.HasMultipartID = true + result.MultipartID = v.Value.(string) + case "object_mode": + if result.HasObjectMode { + continue + } + result.HasObjectMode = true + result.ObjectMode = v.Value.(ObjectMode) + default: + return pairStorageCreate{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} + +type pairStorageCreateDir struct { + pairs []Pair + // Required pairs + // Optional pairs + HasStorageClass bool + StorageClass string +} + +func (s *Storage) parsePairStorageCreateDir(opts []Pair) (pairStorageCreateDir, error) { + result := + pairStorageCreateDir{pairs: opts} + + for _, v := range opts { + switch v.Key { + case "storage_class": + if result.HasStorageClass { + continue + } + result.HasStorageClass = true + result.StorageClass = v.Value.(string) + default: + return pairStorageCreateDir{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} + +type pairStorageCreateMultipart struct { + pairs []Pair + // Required pairs + // Optional pairs + HasContentType bool + ContentType string + HasServerSideEncryption bool + ServerSideEncryption string + HasServerSideEncryptionContext bool + ServerSideEncryptionContext string + HasServerSideEncryptionCosKmsKeyID bool + ServerSideEncryptionCosKmsKeyID string + HasServerSideEncryptionCustomerAlgorithm bool + ServerSideEncryptionCustomerAlgorithm string + HasServerSideEncryptionCustomerKey bool + ServerSideEncryptionCustomerKey []byte + HasStorageClass bool + StorageClass string +} + +func (s *Storage) parsePairStorageCreateMultipart(opts []Pair) (pairStorageCreateMultipart, error) { + result := + pairStorageCreateMultipart{pairs: opts} + + for _, v := range opts { + switch v.Key { + case "content_type": + if result.HasContentType { + continue + } + result.HasContentType = true + result.ContentType = v.Value.(string) + case "server_side_encryption": + if result.HasServerSideEncryption { + continue + } + result.HasServerSideEncryption = true + result.ServerSideEncryption = v.Value.(string) + case "server_side_encryption_context": + if result.HasServerSideEncryptionContext { + continue + } + result.HasServerSideEncryptionContext = true + result.ServerSideEncryptionContext = v.Value.(string) + case "server_side_encryption_cos_kms_key_id": + if result.HasServerSideEncryptionCosKmsKeyID { + continue + } + result.HasServerSideEncryptionCosKmsKeyID = true + result.ServerSideEncryptionCosKmsKeyID = v.Value.(string) + case "server_side_encryption_customer_algorithm": + if result.HasServerSideEncryptionCustomerAlgorithm { + continue + } + result.HasServerSideEncryptionCustomerAlgorithm = true + result.ServerSideEncryptionCustomerAlgorithm = v.Value.(string) + case "server_side_encryption_customer_key": + if result.HasServerSideEncryptionCustomerKey { + continue + } + result.HasServerSideEncryptionCustomerKey = true + result.ServerSideEncryptionCustomerKey = v.Value.([]byte) + case "storage_class": + if result.HasStorageClass { + continue + } + result.HasStorageClass = true + result.StorageClass = v.Value.(string) + default: + return pairStorageCreateMultipart{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} + +type pairStorageDelete struct { + pairs []Pair + // Required pairs + // Optional pairs + HasMultipartID bool + MultipartID string + HasObjectMode bool + ObjectMode ObjectMode +} + +func (s *Storage) parsePairStorageDelete(opts []Pair) (pairStorageDelete, error) { + result := + pairStorageDelete{pairs: opts} + + for _, v := range opts { + switch v.Key { + case "multipart_id": + if result.HasMultipartID { + continue + } + result.HasMultipartID = true + result.MultipartID = v.Value.(string) + case "object_mode": + if result.HasObjectMode { + continue + } + result.HasObjectMode = true + result.ObjectMode = v.Value.(ObjectMode) + default: + return pairStorageDelete{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} + +type pairStorageList struct { + pairs []Pair + // Required pairs + // Optional pairs + HasListMode bool + ListMode ListMode +} + +func (s *Storage) parsePairStorageList(opts []Pair) (pairStorageList, error) { + result := + pairStorageList{pairs: opts} + + for _, v := range opts { + switch v.Key { + case "list_mode": + if result.HasListMode { + continue + } + result.HasListMode = true + result.ListMode = v.Value.(ListMode) + default: + return pairStorageList{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} + +type pairStorageListMultipart struct { + pairs []Pair + // Required pairs + // Optional pairs +} + +func (s *Storage) parsePairStorageListMultipart(opts []Pair) (pairStorageListMultipart, error) { + result := + pairStorageListMultipart{pairs: opts} + + for _, v := range opts { + switch v.Key { + default: + return pairStorageListMultipart{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} + +type pairStorageMetadata struct { + pairs []Pair + // Required pairs + // Optional pairs +} + +func (s *Storage) parsePairStorageMetadata(opts []Pair) (pairStorageMetadata, error) { + result := + pairStorageMetadata{pairs: opts} + + for _, v := range opts { + switch v.Key { + default: + return pairStorageMetadata{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} + +type pairStorageRead struct { + pairs []Pair + // Required pairs + // Optional pairs + HasIoCallback bool + IoCallback func([]byte) + HasOffset bool + Offset int64 + HasServerSideEncryptionCustomerAlgorithm bool + ServerSideEncryptionCustomerAlgorithm string + HasServerSideEncryptionCustomerKey bool + ServerSideEncryptionCustomerKey []byte + HasSize bool + Size int64 +} + +func (s *Storage) parsePairStorageRead(opts []Pair) (pairStorageRead, error) { + result := + pairStorageRead{pairs: opts} + + for _, v := range opts { + switch v.Key { + case "io_callback": + if result.HasIoCallback { + continue + } + result.HasIoCallback = true + result.IoCallback = v.Value.(func([]byte)) + case "offset": + if result.HasOffset { + continue + } + result.HasOffset = true + result.Offset = v.Value.(int64) + case "server_side_encryption_customer_algorithm": + if result.HasServerSideEncryptionCustomerAlgorithm { + continue + } + result.HasServerSideEncryptionCustomerAlgorithm = true + result.ServerSideEncryptionCustomerAlgorithm = v.Value.(string) + case "server_side_encryption_customer_key": + if result.HasServerSideEncryptionCustomerKey { + continue + } + result.HasServerSideEncryptionCustomerKey = true + result.ServerSideEncryptionCustomerKey = v.Value.([]byte) + case "size": + if result.HasSize { + continue + } + result.HasSize = true + result.Size = v.Value.(int64) + default: + return pairStorageRead{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} + +type pairStorageStat struct { + pairs []Pair + // Required pairs + // Optional pairs + HasMultipartID bool + MultipartID string + HasObjectMode bool + ObjectMode ObjectMode + HasServerSideEncryptionCustomerAlgorithm bool + ServerSideEncryptionCustomerAlgorithm string + HasServerSideEncryptionCustomerKey bool + ServerSideEncryptionCustomerKey []byte +} + +func (s *Storage) parsePairStorageStat(opts []Pair) (pairStorageStat, error) { + result := + pairStorageStat{pairs: opts} + + for _, v := range opts { + switch v.Key { + case "multipart_id": + if result.HasMultipartID { + continue + } + result.HasMultipartID = true + result.MultipartID = v.Value.(string) + case "object_mode": + if result.HasObjectMode { + continue + } + result.HasObjectMode = true + result.ObjectMode = v.Value.(ObjectMode) + case "server_side_encryption_customer_algorithm": + if result.HasServerSideEncryptionCustomerAlgorithm { + continue + } + result.HasServerSideEncryptionCustomerAlgorithm = true + result.ServerSideEncryptionCustomerAlgorithm = v.Value.(string) + case "server_side_encryption_customer_key": + if result.HasServerSideEncryptionCustomerKey { + continue + } + result.HasServerSideEncryptionCustomerKey = true + result.ServerSideEncryptionCustomerKey = v.Value.([]byte) + default: + return pairStorageStat{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} + +type pairStorageWrite struct { + pairs []Pair + // Required pairs + // Optional pairs + HasContentMd5 bool + ContentMd5 string + HasContentType bool + ContentType string + HasIoCallback bool + IoCallback func([]byte) + HasServerSideEncryption bool + ServerSideEncryption string + HasServerSideEncryptionContext bool + ServerSideEncryptionContext string + HasServerSideEncryptionCosKmsKeyID bool + ServerSideEncryptionCosKmsKeyID string + HasServerSideEncryptionCustomerAlgorithm bool + ServerSideEncryptionCustomerAlgorithm string + HasServerSideEncryptionCustomerKey bool + ServerSideEncryptionCustomerKey []byte + HasStorageClass bool + StorageClass string +} + +func (s *Storage) parsePairStorageWrite(opts []Pair) (pairStorageWrite, error) { + result := + pairStorageWrite{pairs: opts} + + for _, v := range opts { + switch v.Key { + case "content_md5": + if result.HasContentMd5 { + continue + } + result.HasContentMd5 = true + result.ContentMd5 = v.Value.(string) + case "content_type": + if result.HasContentType { + continue + } + result.HasContentType = true + result.ContentType = v.Value.(string) + case "io_callback": + if result.HasIoCallback { + continue + } + result.HasIoCallback = true + result.IoCallback = v.Value.(func([]byte)) + case "server_side_encryption": + if result.HasServerSideEncryption { + continue + } + result.HasServerSideEncryption = true + result.ServerSideEncryption = v.Value.(string) + case "server_side_encryption_context": + if result.HasServerSideEncryptionContext { + continue + } + result.HasServerSideEncryptionContext = true + result.ServerSideEncryptionContext = v.Value.(string) + case "server_side_encryption_cos_kms_key_id": + if result.HasServerSideEncryptionCosKmsKeyID { + continue + } + result.HasServerSideEncryptionCosKmsKeyID = true + result.ServerSideEncryptionCosKmsKeyID = v.Value.(string) + case "server_side_encryption_customer_algorithm": + if result.HasServerSideEncryptionCustomerAlgorithm { + continue + } + result.HasServerSideEncryptionCustomerAlgorithm = true + result.ServerSideEncryptionCustomerAlgorithm = v.Value.(string) + case "server_side_encryption_customer_key": + if result.HasServerSideEncryptionCustomerKey { + continue + } + result.HasServerSideEncryptionCustomerKey = true + result.ServerSideEncryptionCustomerKey = v.Value.([]byte) + case "storage_class": + if result.HasStorageClass { + continue + } + result.HasStorageClass = true + result.StorageClass = v.Value.(string) + default: + return pairStorageWrite{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} + +type pairStorageWriteMultipart struct { + pairs []Pair + // Required pairs + // Optional pairs + HasContentMd5 bool + ContentMd5 string +} + +func (s *Storage) parsePairStorageWriteMultipart(opts []Pair) (pairStorageWriteMultipart, error) { + result := + pairStorageWriteMultipart{pairs: opts} + + for _, v := range opts { + switch v.Key { + case "content_md5": + if result.HasContentMd5 { + continue + } + result.HasContentMd5 = true + result.ContentMd5 = v.Value.(string) + default: + return pairStorageWriteMultipart{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} +func (s *Storage) CompleteMultipart(o *Object, parts []*Part, pairs ...Pair) (err error) { + ctx := context.Background() + return s.CompleteMultipartWithContext(ctx, o, parts, pairs...) +} +func (s *Storage) CompleteMultipartWithContext(ctx context.Context, o *Object, parts []*Part, pairs ...Pair) (err error) { + defer func() { + err = + s.formatError("complete_multipart", err) + }() + if !o.Mode.IsPart() { + err = services.ObjectModeInvalidError{Expected: ModePart, Actual: o.Mode} + return + } + pairs = append(pairs, s.defaultPairs.CompleteMultipart...) + var opt pairStorageCompleteMultipart + + opt, err = s.parsePairStorageCompleteMultipart(pairs) + if err != nil { + return + } + return s.completeMultipart(ctx, o, parts, opt) +} +func (s *Storage) Create(path string, pairs ...Pair) (o *Object) { + pairs = append(pairs, s.defaultPairs.Create...) + var opt pairStorageCreate + + // Ignore error while handling local functions. + opt, _ = s.parsePairStorageCreate(pairs) + return s.create(path, opt) +} +func (s *Storage) CreateDir(path string, pairs ...Pair) (o *Object, err error) { + ctx := context.Background() + return s.CreateDirWithContext(ctx, path, pairs...) +} +func (s *Storage) CreateDirWithContext(ctx context.Context, path string, pairs ...Pair) (o *Object, err error) { + defer func() { + err = + s.formatError("create_dir", err, path) + }() + + pairs = append(pairs, s.defaultPairs.CreateDir...) + var opt pairStorageCreateDir + + opt, err = s.parsePairStorageCreateDir(pairs) + if err != nil { + return + } + return s.createDir(ctx, strings.ReplaceAll(path, "\\", "/"), opt) +} +func (s *Storage) CreateMultipart(path string, pairs ...Pair) (o *Object, err error) { + ctx := context.Background() + return s.CreateMultipartWithContext(ctx, path, pairs...) +} +func (s *Storage) CreateMultipartWithContext(ctx context.Context, path string, pairs ...Pair) (o *Object, err error) { + defer func() { + err = + s.formatError("create_multipart", err, path) + }() + + pairs = append(pairs, s.defaultPairs.CreateMultipart...) + var opt pairStorageCreateMultipart + + opt, err = s.parsePairStorageCreateMultipart(pairs) + if err != nil { + return + } + return s.createMultipart(ctx, strings.ReplaceAll(path, "\\", "/"), opt) +} +func (s *Storage) Delete(path string, pairs ...Pair) (err error) { + ctx := context.Background() + return s.DeleteWithContext(ctx, path, pairs...) +} +func (s *Storage) DeleteWithContext(ctx context.Context, path string, pairs ...Pair) (err error) { + defer func() { + err = + s.formatError("delete", err, path) + }() + + pairs = append(pairs, s.defaultPairs.Delete...) + var opt pairStorageDelete + + opt, err = s.parsePairStorageDelete(pairs) + if err != nil { + return + } + return s.delete(ctx, strings.ReplaceAll(path, "\\", "/"), opt) +} +func (s *Storage) List(path string, pairs ...Pair) (oi *ObjectIterator, err error) { + ctx := context.Background() + return s.ListWithContext(ctx, path, pairs...) +} +func (s *Storage) ListWithContext(ctx context.Context, path string, pairs ...Pair) (oi *ObjectIterator, err error) { + defer func() { + err = + s.formatError("list", err, path) + }() + + pairs = append(pairs, s.defaultPairs.List...) + var opt pairStorageList + + opt, err = s.parsePairStorageList(pairs) + if err != nil { + return + } + return s.list(ctx, strings.ReplaceAll(path, "\\", "/"), opt) +} +func (s *Storage) ListMultipart(o *Object, pairs ...Pair) (pi *PartIterator, err error) { + ctx := context.Background() + return s.ListMultipartWithContext(ctx, o, pairs...) +} +func (s *Storage) ListMultipartWithContext(ctx context.Context, o *Object, pairs ...Pair) (pi *PartIterator, err error) { + defer func() { + err = + s.formatError("list_multipart", err) + }() + if !o.Mode.IsPart() { + err = services.ObjectModeInvalidError{Expected: ModePart, Actual: o.Mode} + return + } + pairs = append(pairs, s.defaultPairs.ListMultipart...) + var opt pairStorageListMultipart + + opt, err = s.parsePairStorageListMultipart(pairs) + if err != nil { + return + } + return s.listMultipart(ctx, o, opt) +} +func (s *Storage) Metadata(pairs ...Pair) (meta *StorageMeta) { + pairs = append(pairs, s.defaultPairs.Metadata...) + var opt pairStorageMetadata + + // Ignore error while handling local functions. + opt, _ = s.parsePairStorageMetadata(pairs) + return s.metadata(opt) +} +func (s *Storage) Read(path string, w io.Writer, pairs ...Pair) (n int64, err error) { + ctx := context.Background() + return s.ReadWithContext(ctx, path, w, pairs...) +} +func (s *Storage) ReadWithContext(ctx context.Context, path string, w io.Writer, pairs ...Pair) (n int64, err error) { + defer func() { + err = + s.formatError("read", err, path) + }() + + pairs = append(pairs, s.defaultPairs.Read...) + var opt pairStorageRead + + opt, err = s.parsePairStorageRead(pairs) + if err != nil { + return + } + return s.read(ctx, strings.ReplaceAll(path, "\\", "/"), w, opt) +} +func (s *Storage) Stat(path string, pairs ...Pair) (o *Object, err error) { + ctx := context.Background() + return s.StatWithContext(ctx, path, pairs...) +} +func (s *Storage) StatWithContext(ctx context.Context, path string, pairs ...Pair) (o *Object, err error) { + defer func() { + err = + s.formatError("stat", err, path) + }() + + pairs = append(pairs, s.defaultPairs.Stat...) + var opt pairStorageStat + + opt, err = s.parsePairStorageStat(pairs) + if err != nil { + return + } + return s.stat(ctx, strings.ReplaceAll(path, "\\", "/"), opt) +} +func (s *Storage) Write(path string, r io.Reader, size int64, pairs ...Pair) (n int64, err error) { + ctx := context.Background() + return s.WriteWithContext(ctx, path, r, size, pairs...) +} +func (s *Storage) WriteWithContext(ctx context.Context, path string, r io.Reader, size int64, pairs ...Pair) (n int64, err error) { + defer func() { + err = + s.formatError("write", err, path) + }() + + pairs = append(pairs, s.defaultPairs.Write...) + var opt pairStorageWrite + + opt, err = s.parsePairStorageWrite(pairs) + if err != nil { + return + } + return s.write(ctx, strings.ReplaceAll(path, "\\", "/"), r, size, opt) +} +func (s *Storage) WriteMultipart(o *Object, r io.Reader, size int64, index int, pairs ...Pair) (n int64, part *Part, err error) { + ctx := context.Background() + return s.WriteMultipartWithContext(ctx, o, r, size, index, pairs...) +} +func (s *Storage) WriteMultipartWithContext(ctx context.Context, o *Object, r io.Reader, size int64, index int, pairs ...Pair) (n int64, part *Part, err error) { + defer func() { + err = + s.formatError("write_multipart", err) + }() + if !o.Mode.IsPart() { + err = services.ObjectModeInvalidError{Expected: ModePart, Actual: o.Mode} + return + } + pairs = append(pairs, s.defaultPairs.WriteMultipart...) + var opt pairStorageWriteMultipart + + opt, err = s.parsePairStorageWriteMultipart(pairs) + if err != nil { + return + } + return s.writeMultipart(ctx, o, r, size, index, opt) +} +func init() { + services.RegisterServicer(Type, NewServicer) + services.RegisterStorager(Type, NewStorager) + services.RegisterSchema(Type, pairMap) +} diff --git a/services/cos/go.mod b/services/cos/go.mod new file mode 100644 index 000000000..6a2710d6a --- /dev/null +++ b/services/cos/go.mod @@ -0,0 +1,10 @@ +module go.beyondstorage.io/services/cos/v3 + +go 1.15 + +require ( + github.com/google/uuid v1.3.0 + github.com/tencentyun/cos-go-sdk-v5 v0.7.31 + go.beyondstorage.io/credential v1.0.0 + go.beyondstorage.io/v5 v5.0.0 +) diff --git a/services/cos/go.sum b/services/cos/go.sum new file mode 100644 index 000000000..b1c07c944 --- /dev/null +++ b/services/cos/go.sum @@ -0,0 +1,118 @@ +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM= +github.com/Xuanwo/gg v0.2.0 h1:axbZmA0qmidh3s9PA86GqvBXVQ3o7Bbpf0aImGtlimA= +github.com/Xuanwo/gg v0.2.0/go.mod h1:0fLiiSxR87u2UA0ZNZiKZXuz3jnJdbDHWtU2xpdcH3s= +github.com/Xuanwo/go-bufferpool v0.2.0 h1:DXzqJD9lJufXbT/03GrcEvYOs4gXYUj9/g5yi6Q9rUw= +github.com/Xuanwo/go-bufferpool v0.2.0/go.mod h1:Mle++9GGouhOwGj52i9PJLNAPmW2nb8PWBP7JJzNCzk= +github.com/Xuanwo/templateutils v0.1.0 h1:WpkWOqQtIQ2vAIpJLa727DdN8WtxhUkkbDGa6UhntJY= +github.com/Xuanwo/templateutils v0.1.0/go.mod h1:OdE0DJ+CJxDBq6psX5DPV+gOZi8bhuHuVUpPCG++Wb8= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/dave/dst v0.26.2 h1:lnxLAKI3tx7MgLNVDirFCsDTlTG9nKTk7GcptKcWSwY= +github.com/dave/dst v0.26.2/go.mod h1:UMDJuIRPfyUCC78eFuB+SV/WI8oDeyFDvM/JR6NI3IU= +github.com/dave/gopackages v0.0.0-20170318123100-46e7023ec56e/go.mod h1:i00+b/gKdIDIxuLDFob7ustLAVqhsZRk2qVZrArELGQ= +github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= +github.com/dave/kerr v0.0.0-20170318121727-bc25dd6abe8e/go.mod h1:qZqlPyPvfsDJt+3wHJ1EvSXDuVjFTK0j2p/ca+gtsb8= +github.com/dave/rebecca v0.9.1/go.mod h1:N6XYdMD/OKw3lkF3ywh8Z6wPGuwNFDNtWYEMFWEmXBA= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= +github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/google/pprof v0.0.0-20181127221834-b4f47329b966/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/kevinburke/go-bindata v3.22.0+incompatible h1:/JmqEhIWQ7GRScV0WjX/0tqBrC5D21ALg0H0U/KZ/ts= +github.com/kevinburke/go-bindata v3.22.0+incompatible/go.mod h1:/pEEZ72flUW2p0yi30bslSp9YqD9pysLxunQDdb2CPM= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/mozillazg/go-httpheader v0.2.1 h1:geV7TrjbL8KXSyvghnFm+NyTux/hxwueTSrwhe88TQQ= +github.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISeBAdw6E61aqQma60= +github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= +github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.6 h1:lH+Snxmzl92r1jww8/jYPqKkhs3C9AF4LunzU56ZZr4= +github.com/smartystreets/goconvey v1.6.6/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.194/go.mod h1:7sCQWVkxcsR38nffDW057DRGk8mUjK1Ing/EFOK8s8Y= +github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms v1.0.194/go.mod h1:yrBKWhChnDqNz1xuXdSbWXG56XawEq0G5j1lg4VwBD4= +github.com/tencentyun/cos-go-sdk-v5 v0.7.31 h1:NujkkOKMJ3IFs1+trCwXOKRCIPQ8qI5Lxul9JkhTg6M= +github.com/tencentyun/cos-go-sdk-v5 v0.7.31/go.mod h1:4E4+bQ2gBVJcgEC9Cufwylio4mXOct2iu05WjgEBx1o= +github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M= +github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.beyondstorage.io/credential v1.0.0 h1:xJ7hBXmeUE0+rbW+RYZSz4KgHpXvc9g7oQ56f8dXdBk= +go.beyondstorage.io/credential v1.0.0/go.mod h1:7KAYievVw4a8u/eLZmnQt65Z91n84sMQj3LFbt8Xous= +go.beyondstorage.io/v5 v5.0.0 h1:k9Axfgbt+oZXoDwSBVCl1XANHSL4rkNTGP2Lz9YdJe0= +go.beyondstorage.io/v5 v5.0.0/go.mod h1:3wV9gCQnqu7tD/3LMeo2yimUKIeTSHpTc6wHSb0yY20= +golang.org/x/arch v0.0.0-20180920145803-b19384d3c130/go.mod h1:cYlCBUl1MsqxdiKgmc4uh7TxZfWSFLOGSRR090WDxt8= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180903190138-2b024373dcd9/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200509030707-2212a7e161a5/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.1.1 h1:wGiQel/hW0NnEkJUk8lbzkX2gFJU6PFxf1v5OlCfuOs= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/src-d/go-billy.v4 v4.3.0 h1:KtlZ4c1OWbIs4jCv5ZXrTqG8EQocr0g/d4DjNg70aek= +gopkg.in/src-d/go-billy.v4 v4.3.0/go.mod h1:tm33zBoOwxjYHZIE+OV8bxTWFMJLrconzFMd38aARFk= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/services/cos/iterator.go b/services/cos/iterator.go new file mode 100644 index 000000000..960b79303 --- /dev/null +++ b/services/cos/iterator.go @@ -0,0 +1,33 @@ +package cos + +type objectPageStatus struct { + delimiter string + maxKeys int + prefix string + keyMarker string + uploadIdMarker string +} + +func (i *objectPageStatus) ContinuationToken() string { + return i.uploadIdMarker +} + +type storagePageStatus struct { + marker string + maxKeys int +} + +func (i *storagePageStatus) ContinuationToken() string { + return i.marker +} + +type partPageStatus struct { + key string + uploadId string + maxParts string + partNumberMarker string +} + +func (i *partPageStatus) ContinuationToken() string { + return i.partNumberMarker +} diff --git a/services/cos/service.go b/services/cos/service.go new file mode 100644 index 000000000..30e1f12c7 --- /dev/null +++ b/services/cos/service.go @@ -0,0 +1,62 @@ +package cos + +import ( + "context" + + ps "go.beyondstorage.io/v5/pairs" + typ "go.beyondstorage.io/v5/types" +) + +func (s *Service) create(ctx context.Context, name string, opt pairServiceCreate) (store typ.Storager, err error) { + st, err := s.newStorage(ps.WithName(name), ps.WithLocation(opt.Location)) + if err != nil { + return nil, err + } + _, err = st.bucket.Put(ctx, nil) + if err != nil { + return nil, err + } + return st, nil +} + +func (s *Service) delete(ctx context.Context, name string, opt pairServiceDelete) (err error) { + store, err := s.newStorage(ps.WithName(name), ps.WithLocation(opt.Location)) + if err != nil { + return err + } + _, err = store.bucket.Delete(ctx) + if err != nil { + return err + } + return +} + +func (s *Service) get(ctx context.Context, name string, opt pairServiceGet) (store typ.Storager, err error) { + st, err := s.newStorage(ps.WithName(name), ps.WithLocation(opt.Location)) + if err != nil { + return nil, err + } + return st, nil +} + +func (s *Service) list(ctx context.Context, opt pairServiceList) (it *typ.StoragerIterator, err error) { + return typ.NewStoragerIterator(ctx, s.nextStoragePage, nil), nil +} + +func (s *Service) nextStoragePage(ctx context.Context, page *typ.StoragerPage) error { + output, _, err := s.service.Service.Get(ctx) + if err != nil { + return err + } + + for _, v := range output.Buckets { + store, err := s.newStorage(ps.WithName(v.Name), ps.WithLocation(v.Region)) + if err != nil { + return err + } + + page.Data = append(page.Data, store) + } + + return typ.IterateDone +} diff --git a/services/cos/service.toml b/services/cos/service.toml new file mode 100644 index 000000000..d97f137db --- /dev/null +++ b/services/cos/service.toml @@ -0,0 +1,89 @@ +name = "cos" + +[namespace.service] + +[namespace.service.new] +required = ["credential"] +optional = ["endpoint", "http_client_options"] + +[namespace.service.op.create] +required = ["location"] + +[namespace.service.op.delete] +required = ["location"] + +[namespace.service.op.get] +required = ["location"] + +[namespace.storage] +features = ["virtual_dir"] +implement = ["multiparter", "direr"] + +[namespace.storage.new] +required = ["name", "location"] +optional = ["work_dir"] + +[namespace.storage.op.create] +optional = ["multipart_id", "object_mode"] + +[namespace.storage.op.create_dir] +optional = ["storage_class"] + +[namespace.storage.op.delete] +optional = ["multipart_id", "object_mode"] + +[namespace.storage.op.list] +optional = ["list_mode"] + +[namespace.storage.op.read] +optional = ["offset", "io_callback", "size", "server_side_encryption_customer_algorithm", "server_side_encryption_customer_key"] + +[namespace.storage.op.stat] +optional = ["multipart_id", "object_mode", "server_side_encryption_customer_algorithm", "server_side_encryption_customer_key"] + +[namespace.storage.op.write] +optional = ["content_md5", "content_type", "io_callback", "storage_class", "server_side_encryption_customer_algorithm", "server_side_encryption_customer_key", "server_side_encryption", "server_side_encryption_cos_kms_key_id", "server_side_encryption_context"] + +[namespace.storage.op.create_multipart] +optional = ["content_type", "storage_class", "server_side_encryption_customer_algorithm", "server_side_encryption_customer_key", "server_side_encryption", "server_side_encryption_cos_kms_key_id", "server_side_encryption_context"] + +[namespace.storage.op.write_multipart] +optional = ["content_md5"] + +[pairs.storage_class] +type = "string" + +[pairs.server_side_encryption_customer_algorithm] +type = "string" +description = "specifies the algorithm to use to when encrypting the object. Now only `AES256` is supported." + +[pairs.server_side_encryption_customer_key] +type = "[]byte" +description = "specifies the customer-provided encryption key to encrypt/decrypt the source object. It must be a 32-byte AES-256 key." + +[pairs.server_side_encryption_cos_kms_key_id] +type = "string" +description = "specifies the COS KMS key ID to use for object encryption." + +[pairs.server_side_encryption_context] +type = "string" +description = "specifies the COS KMS Encryption Context to use for object encryption. The value of this header is a base64-encoded UTF-8 string holding JSON with the encryption context key-value pairs." + +[pairs.server_side_encryption] +type = "string" +description = "the server-side encryption algorithm used when storing this object. It can be `AES-256` for SSE-COS, and `cos/kms` for SSE-KMS." + +[infos.object.meta.storage-class] +type = "string" + +[infos.object.meta.server-side-encryption] +type = "string" + +[infos.object.meta.server-side-encryption-cos-kms-key-id] +type = "string" + +[infos.object.meta.server-side-encryption-customer-algorithm] +type = "string" + +[infos.object.meta.server-side-encryption-customer-key-md5] +type = "string" diff --git a/services/cos/storage.go b/services/cos/storage.go new file mode 100644 index 000000000..3aa17d179 --- /dev/null +++ b/services/cos/storage.go @@ -0,0 +1,597 @@ +package cos + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http" + "time" + + "github.com/tencentyun/cos-go-sdk-v5" + + ps "go.beyondstorage.io/v5/pairs" + "go.beyondstorage.io/v5/pkg/headers" + "go.beyondstorage.io/v5/pkg/iowrap" + "go.beyondstorage.io/v5/services" + . "go.beyondstorage.io/v5/types" +) + +func (s *Storage) completeMultipart(ctx context.Context, o *Object, parts []*Part, opt pairStorageCompleteMultipart) (err error) { + // Users should make sure the numbers of the uploaded parts are continuous and sorted in ascending order. + // ref: https://intl.cloud.tencent.com/document/product/436/7742 + upload := &cos.CompleteMultipartUploadOptions{} + for _, v := range parts { + upload.Parts = append(upload.Parts, cos.Object{ + ETag: v.ETag, + // For users the `PartNumber` is zero-based. But for COS, the effective `PartNumber` is [1, 10000]. + // Set PartNumber=v.Index+1 here to ensure pass in an effective `PartNumber` for `cos.Object`. + PartNumber: v.Index + 1, + }) + } + + _, _, err = s.object.CompleteMultipartUpload(ctx, o.ID, o.MustGetMultipartID(), upload) + if err != nil { + return + } + + o.Mode.Del(ModePart) + o.Mode.Add(ModeRead) + return +} + +func (s *Storage) create(path string, opt pairStorageCreate) (o *Object) { + rp := s.getAbsPath(path) + + // Handle create multipart object separately. + if opt.HasMultipartID { + o = s.newObject(true) + o.Mode = ModePart + o.SetMultipartID(opt.MultipartID) + } else { + if opt.HasObjectMode && opt.ObjectMode.IsDir() { + if !s.features.VirtualDir { + return + } + + rp += "/" + o = s.newObject(true) + o.Mode = ModeDir + } else { + o = s.newObject(false) + o.Mode = ModeRead + } + } + + o.ID = rp + o.Path = path + return o +} + +func (s *Storage) createDir(ctx context.Context, path string, opt pairStorageCreateDir) (o *Object, err error) { + if !s.features.VirtualDir { + err = NewOperationNotImplementedError("create_dir") + return + } + + rp := s.getAbsPath(path) + + // Add `/` at the end of `path` to simulate a directory. + // ref: https://cloud.tencent.com/document/product/436/13324 + rp += "/" + + putOptions := &cos.ObjectPutOptions{ + ObjectPutHeaderOptions: &cos.ObjectPutHeaderOptions{ + ContentLength: 0, + }, + } + if opt.HasStorageClass { + putOptions.XCosStorageClass = opt.StorageClass + } + + _, err = s.object.Put(ctx, rp, io.LimitReader(nil, 0), putOptions) + if err != nil { + return + } + + o = s.newObject(true) + o.ID = rp + o.Path = path + o.Mode |= ModeDir + return +} + +func (s *Storage) createMultipart(ctx context.Context, path string, opt pairStorageCreateMultipart) (o *Object, err error) { + rp := s.getAbsPath(path) + + input := &cos.InitiateMultipartUploadOptions{} + if opt.HasStorageClass { + input.XCosStorageClass = opt.StorageClass + } + if opt.HasContentType { + input.ContentType = opt.ContentType + } + // SSE-C + if opt.HasServerSideEncryptionCustomerAlgorithm { + input.XCosSSECustomerAglo, input.XCosSSECustomerKey, input.XCosSSECustomerKeyMD5, err = calculateEncryptionHeaders(opt.ServerSideEncryptionCustomerAlgorithm, opt.ServerSideEncryptionCustomerKey) + if err != nil { + return + } + } + // SSE-COS or SSE-KMS + if opt.HasServerSideEncryption { + input.XCosServerSideEncryption = opt.ServerSideEncryption + if opt.ServerSideEncryption == ServerSideEncryptionCosKms { + // FIXME: we can remove the usage of `XOptionHeader` when cos' SDK supports SSE-KMS + input.XOptionHeader = &http.Header{} + if opt.HasServerSideEncryptionCosKmsKeyID { + input.XOptionHeader.Set(serverSideEncryptionCosKmsKeyIdHeader, opt.ServerSideEncryptionCosKmsKeyID) + } + if opt.HasServerSideEncryptionContext { + input.XOptionHeader.Set(serverSideEncryptionContextHeader, opt.ServerSideEncryptionContext) + } + } + } + + output, _, err := s.object.InitiateMultipartUpload(ctx, rp, input) + if err != nil { + return + } + + o = s.newObject(true) + o.ID = rp + o.Path = path + o.Mode |= ModePart + o.SetMultipartID(output.UploadID) + return o, nil +} + +func (s *Storage) delete(ctx context.Context, path string, opt pairStorageDelete) (err error) { + rp := s.getAbsPath(path) + + if opt.HasMultipartID { + _, err = s.object.AbortMultipartUpload(ctx, rp, opt.MultipartID) + if err != nil && checkError(err, responseCodeNoSuchUpload) { + // Omit `NoSuchUpload` error here. + // ref: [GSP-46](https://github.com/beyondstorage/specs/blob/master/rfcs/46-idempotent-delete.md) + err = nil + } + if err != nil { + return err + } + return nil + } + + if opt.HasObjectMode && opt.ObjectMode.IsDir() { + if !s.features.VirtualDir { + err = services.PairUnsupportedError{Pair: ps.WithObjectMode(opt.ObjectMode)} + return + } + + rp += "/" + } + + _, err = s.object.Delete(ctx, rp) + if err != nil && checkError(err, responseCodeNoSuchKey) { + // Omit `NoSuchKey` error here. + // ref: [GSP-46](https://github.com/beyondstorage/specs/blob/master/rfcs/46-idempotent-delete.md) + err = nil + } + if err != nil { + return err + } + + return nil +} + +func (s *Storage) list(ctx context.Context, path string, opt pairStorageList) (oi *ObjectIterator, err error) { + input := &objectPageStatus{ + maxKeys: 200, + prefix: s.getAbsPath(path), + } + + if !opt.HasListMode { + // Support `ListModePrefix` as the default `ListMode`. + // ref: [GSP-46](https://github.com/beyondstorage/go-storage/blob/master/docs/rfcs/654-unify-list-behavior.md) + opt.ListMode = ListModePrefix + } + + var nextFn NextObjectFunc + + switch { + case opt.ListMode.IsPart(): + nextFn = s.nextPartObjectPageByPrefix + case opt.ListMode.IsDir(): + input.delimiter = "/" + nextFn = s.nextObjectPageByDir + case opt.ListMode.IsPrefix(): + nextFn = s.nextObjectPageByPrefix + default: + return nil, services.ListModeInvalidError{Actual: opt.ListMode} + } + + return NewObjectIterator(ctx, nextFn, input), nil +} + +func (s *Storage) listMultipart(ctx context.Context, o *Object, opt pairStorageListMultipart) (pi *PartIterator, err error) { + input := &partPageStatus{ + maxParts: "200", + key: o.ID, + uploadId: o.MustGetMultipartID(), + } + + return NewPartIterator(ctx, s.nextPartPage, input), nil +} + +func (s *Storage) metadata(opt pairStorageMetadata) (meta *StorageMeta) { + meta = NewStorageMeta() + meta.Name = s.name + meta.WorkDir = s.workDir + // set write restriction + meta.SetWriteSizeMaximum(writeSizeMaximum) + // set multipart restrictions + meta.SetMultipartNumberMaximum(multipartNumberMaximum) + meta.SetMultipartSizeMaximum(multipartSizeMaximum) + meta.SetMultipartSizeMinimum(multipartSizeMinimum) + return +} + +func (s *Storage) nextObjectPageByDir(ctx context.Context, page *ObjectPage) error { + input := page.Status.(*objectPageStatus) + + output, _, err := s.bucket.Get(ctx, &cos.BucketGetOptions{ + Prefix: input.prefix, + Delimiter: input.delimiter, + Marker: input.keyMarker, + MaxKeys: input.maxKeys, + }) + if err != nil { + return err + } + + for _, v := range output.CommonPrefixes { + o := s.newObject(true) + o.ID = v + o.Path = s.getRelPath(v) + o.Mode |= ModeDir + + page.Data = append(page.Data, o) + } + + for _, v := range output.Contents { + o, err := s.formatFileObject(v) + if err != nil { + return err + } + + page.Data = append(page.Data, o) + } + + if !output.IsTruncated { + return IterateDone + } + + input.keyMarker = output.NextMarker + return nil +} + +func (s *Storage) nextObjectPageByPrefix(ctx context.Context, page *ObjectPage) error { + input := page.Status.(*objectPageStatus) + + output, _, err := s.bucket.Get(ctx, &cos.BucketGetOptions{ + Prefix: input.prefix, + Marker: input.keyMarker, + MaxKeys: input.maxKeys, + }) + if err != nil { + return err + } + + for _, v := range output.Contents { + o, err := s.formatFileObject(v) + if err != nil { + return err + } + + page.Data = append(page.Data, o) + } + + if !output.IsTruncated { + return IterateDone + } + + input.keyMarker = output.NextMarker + return nil +} + +func (s *Storage) nextPartObjectPageByPrefix(ctx context.Context, page *ObjectPage) error { + input := page.Status.(*objectPageStatus) + + listInput := &cos.ListMultipartUploadsOptions{ + Prefix: input.prefix, + MaxUploads: input.maxKeys, + KeyMarker: input.keyMarker, + UploadIDMarker: input.uploadIdMarker, + } + + output, _, err := s.bucket.ListMultipartUploads(ctx, listInput) + if err != nil { + return err + } + + for _, v := range output.Uploads { + o := s.newObject(true) + o.ID = v.Key + o.Path = s.getRelPath(v.Key) + o.Mode |= ModePart + o.SetMultipartID(v.UploadID) + + page.Data = append(page.Data, o) + } + + if !output.IsTruncated { + return IterateDone + } + + input.uploadIdMarker = output.NextUploadIDMarker + input.keyMarker = output.NextKeyMarker + return nil +} + +func (s *Storage) nextPartPage(ctx context.Context, page *PartPage) error { + input := page.Status.(*partPageStatus) + + output, _, err := s.object.ListParts(ctx, input.key, input.uploadId, &cos.ObjectListPartsOptions{ + MaxParts: input.maxParts, + PartNumberMarker: input.partNumberMarker, + }) + if err != nil { + return err + } + + for _, v := range output.Parts { + p := &Part{ + // The returned `PartNumber` is [1, 10000]. + // Set Index=*v.PartNumber-1 here to make the `PartNumber` zero-based for user. + Index: v.PartNumber - 1, + Size: v.Size, + ETag: v.ETag, + } + + page.Data = append(page.Data, p) + } + + if !output.IsTruncated { + return IterateDone + } + + input.partNumberMarker = output.NextPartNumberMarker + return nil +} + +func (s *Storage) read(ctx context.Context, path string, w io.Writer, opt pairStorageRead) (n int64, err error) { + rp := s.getAbsPath(path) + + getOptions := &cos.ObjectGetOptions{} + rangeOptions := &cos.RangeOptions{} + if opt.HasOffset { + rangeOptions.HasStart = true + rangeOptions.Start = opt.Offset + } + if opt.HasSize { + rangeOptions.HasEnd = true + if opt.HasOffset { + rangeOptions.End = rangeOptions.Start + opt.Size - 1 + } else { + rangeOptions.HasStart = true + rangeOptions.Start = 0 + rangeOptions.End = opt.Size - 1 + } + } + getOptions.Range = cos.FormatRangeOptions(rangeOptions) + // SSE-C + if opt.HasServerSideEncryptionCustomerAlgorithm { + getOptions.XCosSSECustomerAglo, getOptions.XCosSSECustomerKey, getOptions.XCosSSECustomerKeyMD5, err = calculateEncryptionHeaders(opt.ServerSideEncryptionCustomerAlgorithm, opt.ServerSideEncryptionCustomerKey) + if err != nil { + return 0, err + } + } + resp, err := s.object.Get(ctx, rp, getOptions) + if err != nil { + return 0, err + } + defer resp.Body.Close() + + rc := resp.Body + if opt.HasIoCallback { + rc = iowrap.CallbackReadCloser(rc, opt.IoCallback) + } + + return io.Copy(w, rc) +} + +func (s *Storage) stat(ctx context.Context, path string, opt pairStorageStat) (o *Object, err error) { + rp := s.getAbsPath(path) + + if opt.HasMultipartID { + _, _, err = s.object.ListParts(ctx, rp, opt.MultipartID, nil) + if err != nil { + return nil, err + } + + o = s.newObject(true) + o.ID = rp + o.Path = path + o.Mode |= ModePart + o.SetMultipartID(opt.MultipartID) + return o, nil + } + + if opt.HasObjectMode && opt.ObjectMode.IsDir() { + if !s.features.VirtualDir { + err = services.PairUnsupportedError{Pair: ps.WithObjectMode(opt.ObjectMode)} + return + } + + rp += "/" + } + + headOptions := &cos.ObjectHeadOptions{} + // SSE-C + if opt.HasServerSideEncryptionCustomerAlgorithm { + headOptions.XCosSSECustomerAglo, headOptions.XCosSSECustomerKey, headOptions.XCosSSECustomerKeyMD5, err = calculateEncryptionHeaders(opt.ServerSideEncryptionCustomerAlgorithm, opt.ServerSideEncryptionCustomerKey) + if err != nil { + return + } + } + output, err := s.object.Head(ctx, rp, headOptions) + if err != nil { + return nil, err + } + + o = s.newObject(true) + o.ID = rp + o.Path = path + if opt.HasObjectMode && opt.ObjectMode.IsDir() { + o.Mode |= ModeDir + } else { + o.Mode |= ModeRead + } + + o.SetContentLength(output.ContentLength) + + // COS uses RFC1123 format in HEAD + // + // > Last-Modified: Fri, 09 Aug 2019 10:20:56 GMT + // + // ref: https://cloud.tencent.com/document/product/436/7745 + if v := output.Header.Get(headers.LastModified); v != "" { + lastModified, err := time.Parse(time.RFC1123, v) + if err != nil { + return nil, err + } + o.SetLastModified(lastModified) + } + + if v := output.Header.Get(headers.ContentType); v != "" { + o.SetContentType(v) + } + + if v := output.Header.Get(headers.ETag); v != "" { + o.SetEtag(v) + } + + var sm ObjectSystemMetadata + if v := output.Header.Get(storageClassHeader); v != "" { + sm.StorageClass = v + } + if v := output.Header.Get(serverSideEncryptionHeader); v != "" { + sm.ServerSideEncryption = v + } + if v := output.Header.Get(serverSideEncryptionCosKmsKeyIdHeader); v != "" { + sm.ServerSideEncryptionCosKmsKeyID = v + } + if v := output.Header.Get(serverSideEncryptionCustomerAlgorithmHeader); v != "" { + sm.ServerSideEncryptionCustomerAlgorithm = v + } + if v := output.Header.Get(serverSideEncryptionCustomerKeyMd5Header); v != "" { + sm.ServerSideEncryptionCustomerKeyMd5 = v + } + o.SetSystemMetadata(sm) + + return o, nil +} + +func (s *Storage) write(ctx context.Context, path string, r io.Reader, size int64, opt pairStorageWrite) (n int64, err error) { + if size > writeSizeMaximum { + err = fmt.Errorf("size limit exceeded: %w", services.ErrRestrictionDissatisfied) + return + } + + // According to GSP-751, we should allow the user to pass in a nil io.Reader. + // ref: https://github.com/beyondstorage/go-storage/blob/master/docs/rfcs/751-write-empty-file-behavior.md + if r == nil && size == 0 { + r = bytes.NewReader([]byte{}) + } else if r == nil && size != 0 { + return 0, fmt.Errorf("reader is nil but size is not 0") + } else { + r = io.LimitReader(r, size) + } + + rp := s.getAbsPath(path) + + putOptions := &cos.ObjectPutOptions{ + ObjectPutHeaderOptions: &cos.ObjectPutHeaderOptions{ + ContentLength: size, + }, + } + if opt.HasContentMd5 { + putOptions.ContentMD5 = opt.ContentMd5 + } + if opt.HasStorageClass { + putOptions.XCosStorageClass = opt.StorageClass + } + // SSE-C + if opt.HasServerSideEncryptionCustomerAlgorithm { + putOptions.XCosSSECustomerAglo, putOptions.XCosSSECustomerKey, putOptions.XCosSSECustomerKeyMD5, err = calculateEncryptionHeaders(opt.ServerSideEncryptionCustomerAlgorithm, opt.ServerSideEncryptionCustomerKey) + if err != nil { + return + } + } + // SSE-COS or SSE-KMS + if opt.HasServerSideEncryption { + putOptions.XCosServerSideEncryption = opt.ServerSideEncryption + if opt.ServerSideEncryption == ServerSideEncryptionCosKms { + // FIXME: we can remove the usage of `XOptionHeader` when cos' SDK supports SSE-KMS + putOptions.XOptionHeader = &http.Header{} + if opt.HasServerSideEncryptionCosKmsKeyID { + putOptions.XOptionHeader.Set(serverSideEncryptionCosKmsKeyIdHeader, opt.ServerSideEncryptionCosKmsKeyID) + } + if opt.HasServerSideEncryptionContext { + putOptions.XOptionHeader.Set(serverSideEncryptionContextHeader, opt.ServerSideEncryptionContext) + } + } + } + if opt.HasIoCallback { + r = iowrap.CallbackReader(r, opt.IoCallback) + } + + _, err = s.object.Put(ctx, rp, r, putOptions) + if err != nil { + return 0, err + } + return +} + +func (s *Storage) writeMultipart(ctx context.Context, o *Object, r io.Reader, size int64, index int, opt pairStorageWriteMultipart) (n int64, part *Part, err error) { + if size > multipartSizeMaximum { + err = fmt.Errorf("size limit exceeded: %w", services.ErrRestrictionDissatisfied) + return + } + if index < 0 || index >= multipartNumberMaximum { + err = fmt.Errorf("multipart number limit exceeded: %w", services.ErrRestrictionDissatisfied) + return + } + + input := &cos.ObjectUploadPartOptions{ + ContentLength: size, + } + if opt.HasContentMd5 { + input.ContentMD5 = opt.ContentMd5 + } + + // For COS, the `PartNumber` is [1, 10000]. But for users, the `PartNumber` is zero-based. + // Set PartNumber=index+1 here to ensure pass in an effective `PartNumber` for `UploadPart`. + // ref: https://cloud.tencent.com/document/product/436/7750 + output, err := s.object.UploadPart(ctx, o.ID, o.MustGetMultipartID(), index+1, r, input) + if err != nil { + return + } + + part = &Part{ + Index: index, + Size: size, + ETag: output.Header.Get("ETag"), + } + return size, part, nil +} diff --git a/services/cos/tests/README.md b/services/cos/tests/README.md new file mode 100644 index 000000000..f84a02b27 --- /dev/null +++ b/services/cos/tests/README.md @@ -0,0 +1,32 @@ +## How run integration tests + +### Run tests locally + +Copy example files and update corresponding values. + +```shell +cp Makefile.env.exmaple Makefile.env +``` + +Run tests + +```shell +make integration_test +``` + +### Run tests in CI + +Set following environment variables: + +```shell +export STORAGE_COS_INTEGRATION_TEST=on +export STORAGE_COS_CREDENTIAL=hmac:access_key:secret_key +export STORAGE_COS_NAME=bucketname +export STORAGE_COS_LOCATION=bucketname +``` + +Run tests + +```shell +make integration_test +``` diff --git a/services/cos/tests/storage_test.go b/services/cos/tests/storage_test.go new file mode 100644 index 000000000..170646b42 --- /dev/null +++ b/services/cos/tests/storage_test.go @@ -0,0 +1,29 @@ +package tests + +import ( + "os" + "testing" + + "go.beyondstorage.io/v5/tests" +) + +func TestStorage(t *testing.T) { + if os.Getenv("STORAGE_COS_INTEGRATION_TEST") != "on" { + t.Skipf("STORAGE_COS_INTEGRATION_TEST is not 'on', skipped") + } + tests.TestStorager(t, setupTest(t)) +} + +func TestMultiparter(t *testing.T) { + if os.Getenv("STORAGE_COS_INTEGRATION_TEST") != "on" { + t.Skipf("STORAGE_COS_INTEGRATION_TEST is not 'on', skipped") + } + tests.TestMultiparter(t, setupTest(t)) +} + +func TestDir(t *testing.T) { + if os.Getenv("STORAGE_COS_INTEGRATION_TEST") != "on" { + t.Skipf("STORAGE_COS_INTEGRATION_TEST is not 'on', skipped") + } + tests.TestDirer(t, setupTest(t)) +} diff --git a/services/cos/tests/utils_test.go b/services/cos/tests/utils_test.go new file mode 100644 index 000000000..10d65e367 --- /dev/null +++ b/services/cos/tests/utils_test.go @@ -0,0 +1,30 @@ +package tests + +import ( + "os" + "testing" + + "github.com/google/uuid" + + cos "go.beyondstorage.io/services/cos/v3" + ps "go.beyondstorage.io/v5/pairs" + "go.beyondstorage.io/v5/types" +) + +func setupTest(t *testing.T) types.Storager { + t.Log("Setup test for oss") + + store, err := cos.NewStorager( + ps.WithCredential(os.Getenv("STORAGE_COS_CREDENTIAL")), + ps.WithName(os.Getenv("STORAGE_COS_NAME")), + ps.WithLocation(os.Getenv("STORAGE_COS_LOCATION")), + ps.WithWorkDir("/"+uuid.New().String()+"/"), + cos.WithStorageFeatures(cos.StorageFeatures{ + VirtualDir: true, + }), + ) + if err != nil { + t.Errorf("new storager: %v", err) + } + return store +} diff --git a/services/cos/tools.go b/services/cos/tools.go new file mode 100644 index 000000000..6f27a8706 --- /dev/null +++ b/services/cos/tools.go @@ -0,0 +1,8 @@ +//go:build tools +// +build tools + +package cos + +import ( + _ "go.beyondstorage.io/v5/cmd/definitions" +) diff --git a/services/cos/utils.go b/services/cos/utils.go new file mode 100644 index 000000000..3e2df38cd --- /dev/null +++ b/services/cos/utils.go @@ -0,0 +1,340 @@ +package cos + +import ( + "crypto/md5" + "encoding/base64" + "fmt" + "net/http" + "strings" + "time" + + "github.com/tencentyun/cos-go-sdk-v5" + + "go.beyondstorage.io/credential" + ps "go.beyondstorage.io/v5/pairs" + "go.beyondstorage.io/v5/pkg/httpclient" + "go.beyondstorage.io/v5/services" + typ "go.beyondstorage.io/v5/types" +) + +// Service is the Tencent oss *Service config. +type Service struct { + service *cos.Client + client *http.Client + + defaultPairs DefaultServicePairs + features ServiceFeatures + + typ.UnimplementedServicer +} + +// String implements Servicer.String +func (s *Service) String() string { + return fmt.Sprintf("Servicer cos") +} + +// Storage is the cos object storage service. +type Storage struct { + bucket *cos.BucketService + object *cos.ObjectService + + name string + location string + workDir string + + defaultPairs DefaultStoragePairs + features StorageFeatures + + typ.UnimplementedStorager + typ.UnimplementedDirer + typ.UnimplementedMultiparter +} + +// String implements Storager.String +func (s *Storage) String() string { + return fmt.Sprintf( + "Storager cos {Name: %s, WorkDir: %s}", + s.name, s.workDir, + ) +} + +// New will create both Servicer and Storager. +func New(pairs ...typ.Pair) (_ typ.Servicer, _ typ.Storager, err error) { + return newServicerAndStorager(pairs...) +} + +// NewServicer will create Servicer only. +func NewServicer(pairs ...typ.Pair) (typ.Servicer, error) { + return newServicer(pairs...) +} + +// NewStorager will create Storager only. +func NewStorager(pairs ...typ.Pair) (typ.Storager, error) { + _, store, err := newServicerAndStorager(pairs...) + return store, err +} + +func newServicer(pairs ...typ.Pair) (srv *Service, err error) { + defer func() { + if err != nil { + err = services.InitError{Op: "new_servicer", Type: Type, Err: formatError(err), Pairs: pairs} + } + }() + + srv = &Service{} + + opt, err := parsePairServiceNew(pairs) + if err != nil { + return nil, err + } + + cp, err := credential.Parse(opt.Credential) + if err != nil { + return nil, err + } + if cp.Protocol() != credential.ProtocolHmac { + return nil, services.PairUnsupportedError{Pair: ps.WithCredential(opt.Credential)} + } + ak, sk := cp.Hmac() + + httpClient := httpclient.New(opt.HTTPClientOptions) + httpClient.Transport = &cos.AuthorizationTransport{ + Transport: httpClient.Transport, + SecretID: ak, + SecretKey: sk, + } + + srv.client = httpClient + srv.service = cos.NewClient(nil, srv.client) + + if opt.HasDefaultServicePairs { + srv.defaultPairs = opt.DefaultServicePairs + } + if opt.HasServiceFeatures { + srv.features = opt.ServiceFeatures + } + return +} + +// newServicerAndStorager will create a new Tencent oss service. +func newServicerAndStorager(pairs ...typ.Pair) (srv *Service, store *Storage, err error) { + srv, err = newServicer(pairs...) + if err != nil { + return + } + + store, err = srv.newStorage(pairs...) + if err != nil { + err = services.InitError{Op: "new_storager", Type: Type, Err: formatError(err), Pairs: pairs} + return + } + return +} + +// All available storage classes are listed here. +const ( + // ref: https://cloud.tencent.com/document/product/436/7745 + storageClassHeader = "x-cos-storage-class" + + StorageClassStandard = "STANDARD" + StorageClassStandardIA = "STANDARD_IA" + StorageClassArchive = "ARCHIVE" +) + +// ref: https://www.qcloud.com/document/product/436/7730 +func formatError(err error) error { + if _, ok := err.(services.InternalError); ok { + return err + } + + // Handle errors returned by cos. + e, ok := err.(*cos.ErrorResponse) + if !ok { + return fmt.Errorf("%w, %v", services.ErrUnexpected, err) + } + + switch e.Code { + case "": + switch e.Response.StatusCode { + case 404: + return fmt.Errorf("%w: %v", services.ErrObjectNotExist, err) + default: + return fmt.Errorf("%w, %v", services.ErrUnexpected, err) + } + case "NoSuchKey": + return fmt.Errorf("%w: %v", services.ErrObjectNotExist, err) + case "AccessDenied": + return fmt.Errorf("%w: %v", services.ErrPermissionDenied, err) + default: + return fmt.Errorf("%w, %v", services.ErrUnexpected, err) + } +} + +// newStorage will create a new client. +func (s *Service) newStorage(pairs ...typ.Pair) (st *Storage, err error) { + opt, err := parsePairStorageNew(pairs) + if err != nil { + return nil, err + } + + st = &Storage{} + + url := cos.NewBucketURL(opt.Name, opt.Location, true) + c := cos.NewClient(&cos.BaseURL{BucketURL: url}, s.client) + + st.bucket = c.Bucket + st.object = c.Object + st.name = opt.Name + st.location = opt.Location + + st.workDir = "/" + if opt.HasWorkDir { + st.workDir = opt.WorkDir + } + + if opt.HasDefaultStoragePairs { + st.defaultPairs = opt.DefaultStoragePairs + } + + if opt.HasStorageFeatures { + st.features = opt.StorageFeatures + } + return st, nil +} + +func (s *Service) formatError(op string, err error, name string) error { + if err == nil { + return nil + } + + return services.ServiceError{ + Op: op, + Err: formatError(err), + Servicer: s, + Name: name, + } +} + +// getAbsPath will calculate object storage's abs path +func (s *Storage) getAbsPath(path string) string { + prefix := strings.TrimPrefix(s.workDir, "/") + return prefix + path +} + +// getRelPath will get object storage's rel path. +func (s *Storage) getRelPath(path string) string { + prefix := strings.TrimPrefix(s.workDir, "/") + return strings.TrimPrefix(path, prefix) +} + +func (s *Storage) formatError(op string, err error, path ...string) error { + if err == nil { + return nil + } + + return services.StorageError{ + Op: op, + Err: formatError(err), + Storager: s, + Path: path, + } +} + +func (s *Storage) formatFileObject(v cos.Object) (o *typ.Object, err error) { + o = s.newObject(false) + o.ID = v.Key + o.Path = s.getRelPath(v.Key) + o.Mode |= typ.ModeRead + + o.SetContentLength(int64(v.Size)) + + // COS returns different value depends on object upload method or + // encryption method, so we can't treat this value as content-md5 + // + // ref: https://cloud.tencent.com/document/product/436/7729 + if v.ETag != "" { + o.SetEtag(v.ETag) + } + + // COS uses ISO8601 format: "2019-05-27T11:26:14.000Z" in List + // + // ref: https://cloud.tencent.com/document/product/436/7729 + if v.LastModified != "" { + t, err := time.Parse("2006-01-02T15:04:05.999Z", v.LastModified) + if err != nil { + return nil, err + } + o.SetLastModified(t) + } + + var sm ObjectSystemMetadata + if value := v.StorageClass; value != "" { + sm.StorageClass = value + } + o.SetSystemMetadata(sm) + + return o, nil +} + +func (s *Storage) newObject(done bool) *typ.Object { + return typ.NewObject(s, done) +} + +// All available server side algorithm are listed here. +const ( + // ref: https://cloud.tencent.com/document/product/436/7729 + serverSideEncryptionHeader = "x-cos-server-side-encryption" + serverSideEncryptionCosKmsKeyIdHeader = "x-cos-server-side-encryption-cos-kms-key-id" + serverSideEncryptionCustomerAlgorithmHeader = "x-cos-server-side-encryption-customer-algorithm" + serverSideEncryptionCustomerKeyMd5Header = "x-cos-server-side-encryption-customer-key-MD5" + serverSideEncryptionContextHeader = "x-cos-server-side-encryption-context" + + ServerSideEncryptionAes256 = "AES256" + ServerSideEncryptionCosKms = "cos/kms" +) + +func calculateEncryptionHeaders(algo string, key []byte) (algorithm, keyBase64, keyMD5Base64 string, err error) { + if len(key) != 32 { + err = ErrServerSideEncryptionCustomerKeyInvalid + return + } + keyBase64 = base64.StdEncoding.EncodeToString(key) + keyMD5 := md5.Sum(key) + keyMD5Base64 = base64.StdEncoding.EncodeToString(keyMD5[:]) + return +} + +// cos service response error code +// +// ref: https://cloud.tencent.com/document/product/436/7730 +const ( + // NoSuchKey the specified key does not exist. + responseCodeNoSuchKey = "NoSuchKey" + // NoSuchUpload the specified uploadId dose not exist. + responseCodeNoSuchUpload = "NoSuchUpload" +) + +func checkError(err error, code string) bool { + if e, ok := err.(*cos.ErrorResponse); ok { + return strings.Contains(e.Code, code) + } + + return false +} + +// multipartXXX are multipart upload restriction in COS, see more details at: +// https://cloud.tencent.com/document/product/436/7750 +const ( + // multipartNumberMaximum is the max part count supported. + multipartNumberMaximum = 10000 + // multipartSizeMaximum is the maximum size for each part, 5GB. + multipartSizeMaximum = 5 * 1024 * 1024 * 1024 + // multipartSizeMinimum is the minimum size for each part, 1MB. + multipartSizeMinimum = 1024 * 1024 +) + +const ( + // WriteSizeMaximum is the maximum size for write operation, 5GB. + // ref: https://cloud.tencent.com/document/product/436/7749 + writeSizeMaximum = 5 * 1024 * 1024 * 1024 +)