From 69aafc41865446de89528f844e47bf12bbe28d56 Mon Sep 17 00:00:00 2001 From: Joey-1445601153 <1445601153@qq.com> Date: Thu, 21 Oct 2021 10:55:09 +0800 Subject: [PATCH 1/4] Add qingstor back --- .github/dependabot.yml | 4 + .github/workflows/services-test-qingstor.yml | 40 + services/qingstor/.gitignore | 7 + services/qingstor/CHANGELOG.md | 152 ++ services/qingstor/LICENSE | 201 ++ services/qingstor/Makefile | 43 + services/qingstor/Makefile.env.example | 4 + services/qingstor/README.md | 35 + services/qingstor/doc.go | 7 + services/qingstor/errors.go | 22 + services/qingstor/generated.go | 1863 +++++++++++++++++ services/qingstor/go.mod | 15 + services/qingstor/go.sum | 137 ++ services/qingstor/iterator.go | 41 + services/qingstor/mock_test.go | 1454 +++++++++++++ .../rfcs/79-add-virtual-link-support.md | 56 + services/qingstor/service.go | 92 + services/qingstor/service.toml | 114 + services/qingstor/servicer_test.go | 441 ++++ services/qingstor/storage.go | 820 ++++++++ services/qingstor/storager_test.go | 562 +++++ services/qingstor/tests/README.md | 32 + services/qingstor/tests/storage_test.go | 51 + services/qingstor/tests/utils_test.go | 31 + services/qingstor/tools.go | 8 + services/qingstor/utils.go | 477 +++++ services/qingstor/utils_test.go | 207 ++ 27 files changed, 6916 insertions(+) create mode 100644 .github/workflows/services-test-qingstor.yml create mode 100644 services/qingstor/.gitignore create mode 100644 services/qingstor/CHANGELOG.md create mode 100644 services/qingstor/LICENSE create mode 100644 services/qingstor/Makefile create mode 100644 services/qingstor/Makefile.env.example create mode 100644 services/qingstor/README.md create mode 100644 services/qingstor/doc.go create mode 100644 services/qingstor/errors.go create mode 100644 services/qingstor/generated.go create mode 100644 services/qingstor/go.mod create mode 100644 services/qingstor/go.sum create mode 100644 services/qingstor/iterator.go create mode 100644 services/qingstor/mock_test.go create mode 100644 services/qingstor/rfcs/79-add-virtual-link-support.md create mode 100644 services/qingstor/service.go create mode 100644 services/qingstor/service.toml create mode 100644 services/qingstor/servicer_test.go create mode 100644 services/qingstor/storage.go create mode 100644 services/qingstor/storager_test.go create mode 100644 services/qingstor/tests/README.md create mode 100644 services/qingstor/tests/storage_test.go create mode 100644 services/qingstor/tests/utils_test.go create mode 100644 services/qingstor/tools.go create mode 100644 services/qingstor/utils.go create mode 100644 services/qingstor/utils_test.go diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 19c64dc0c..6bb6a6a15 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -64,3 +64,7 @@ updates: directory: "/services/obs" schedule: interval: daily + - package-ecosystem: gomod + directory: "/services/qingstor" + schedule: + interval: daily \ No newline at end of file diff --git a/.github/workflows/services-test-qingstor.yml b/.github/workflows/services-test-qingstor.yml new file mode 100644 index 000000000..dba208630 --- /dev/null +++ b/.github/workflows/services-test-qingstor.yml @@ -0,0 +1,40 @@ +name: "Services Test Qingstor" + +on: + push: + paths: + - 'services/qingstor/**' + pull_request: + paths: + - 'services/qingstor/**' + +jobs: + services_test_qingstor: + name: "Services Test Qingstor" + runs-on: self-hosted + + strategy: + matrix: + go: [ "1.16", "1.17" ] + + steps: + - name: Set up Go 1.x + uses: actions/setup-go@v2 + with: + go-version: ${{ matrix.go }} + + - name: Load secret + uses: 1password/load-secrets-action@v1 + env: + STORAGE_QINGSTOR_CREDENTIAL: op://Engineering/Qingstor/testing/credential + STORAGE_QINGSTOR_NAME: op://Engineering/Qingstor/testing/name + STORAGE_QINGSTOR_ENDPOINT: op://Engineering/Qingstor/testing/endpoint + + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Test + env: + STORAGE_QINGSTOR_INTEGRATION_TEST: on + working-directory: services/qingstor + run: make integration_test diff --git a/services/qingstor/.gitignore b/services/qingstor/.gitignore new file mode 100644 index 000000000..a2eb8ad8d --- /dev/null +++ b/services/qingstor/.gitignore @@ -0,0 +1,7 @@ +coverage.* +bin/ +Makefile.env + +# Jetbrain IDE +.idea +*.iml diff --git a/services/qingstor/CHANGELOG.md b/services/qingstor/CHANGELOG.md new file mode 100644 index 000000000..3ac9f1b1d --- /dev/null +++ b/services/qingstor/CHANGELOG.md @@ -0,0 +1,152 @@ +# Change Log + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/) +and this project adheres to [Semantic Versioning](https://semver.org/). + +## [v3.3.0] - 2021-09-13 + +### Added + +- rfcs: RFC-79 Add virtual link support (#79) +- feat: Implement CreateLink and setup linker test (#81) +- feat: WriteMultipart adds io_callback support (#86) +- feat: Implement StorageHTTPSigner (#88) + +### Changed + +- feat: Turn Expire into Duration for Reach (#76) +- ci: Enable auto merge for dependabot +- ci: Cleanup Service Integration Tests (#90) +- docs: Update README (#91) + +### Fixed + +- fix: Fixed append test failures (#84) + +### Upgraded + +- build(deps): Bump qingstor-sdk-go to version 4.4.0 (#87) + +## [v3.2.0] - 2021-07-22 + +### Added + +- ci: Add gofmt action (#62) +- ci: Add diff check action (#65) +- ci: Add dependabot auto build support (#66) + +### Changed + +- storage: Update types in service.toml to golang types (#71) +- storage: Update append as described in GSP-134 (#71) +- storage: Update list as described in GSP-654 (#71) +- build(deps): Migrate to go-endpoint (#74) + +### Fixed + +- ci: Fix auto-build not work correctly +- storage: Fix invalid argument for copy and move (#72) +- storage: Fix append behavior (#73) + +### Upgraded + +- build(deps): Bump github.com/google/uuid from 1.2.0 to 1.3.0 (#61) + +## [v3.1.0] - 2021-06-29 + +### Added + +- *: Implement GSP-87 Feature Gates (#53) +- storage: Implement GSP-93 Add ObjectMode Pair (#58) +- storage: Implement GSP-97 Add Restrictions In Storage Metadata (#58) + +### Changed + +- *: Implement GSP-109: Redesign Features (#58) +- *: Implement GSP-117 Rename Service to System as the Opposite to Global (#58) + +### Upgraded + +- build(deps): bump github.com/golang/mock from 1.5.0 to 1.6.0 (#56) + +## [v3.0.0] - 2021-05-24 + +### Added + +- storage: Add appender support (#40) +- *: Implement GSP-47 & GSP-51 (#46) +- storage: Implement GSP-61 Add object mode check for operations (#49) + +### Changed + +- service: Use path style instead of vhost (#43) +- service: Fix location not detected correctly (#45) +- storage: Idempotent storager delete operation (#44) +- storage: Implement GSP-62 WriteMultipart returns Part (#47) +- storage: Check if part number is valid when multipart upload (#48) +- *: Implement GSP-73 Organization rename (#51) + +## [v2.1.0] - 2021-04-24 + +### Added + +- *: Implement proposal unify object metadata (#25) +- storage: Normalize iterator next function names (#27) +- pair: Implement default pair support for service (#29) +- *: Set default pair when init (#31) +- storage: Implement Create API (#33) +- storage: Set multipart attributes when create multipart (#34) +- *: Add UnimplementedStub (#35) +- storage: Implement SSE support (#37) +- tests: Introduce STORAGE_QINGSTOR_INTEGRATION_TEST (#39) +- storage: Implement GSP-40 (#41) + +### Changed + +- storage: Clean up next page logic +- build: Make sure integration tests has been executed +- docs: Migrate zulip to matrix +- docs: Remove zulip +- ci: Only run Integration Test while push to master +- storage: Rename SSE related pairs to meet GSP-38 (#38) + +### Fixed + +- storage: Fix multipart integration tests (#36) + +### Removed + +- *: Remove parsed pairs pointer (#28) + +### Upgrade + +- build(deps): bump github.com/qingstor/qingstor-sdk-go/v4 (#26) + +## [v2.0.0] - 2021-01-17 + +### Added + +- tests: Add integration tests (#17) +- storage: Implement Fetcher (#19) +- storage: Implement proposal Unify List Operation (#20) +- *: Implement Segment API Redesign (#21) +- storage: Implement proposal Object Mode (#22) + +### Changed + +- Migrate to go-storage v3 (#23) + +## v1.0.0 - 2020-11-12 + +### Added + +- Implement qingstor services. + +[v3.3.0]: https://github.com/beyondstorage/go-service-qingstor/compare/v3.2.0...v3.3.0 +[v3.2.0]: https://github.com/beyondstorage/go-service-qingstor/compare/v3.1.0...v3.2.0 +[v3.1.0]: https://github.com/beyondstorage/go-service-qingstor/compare/v3.0.0...v3.1.0 +[v3.0.0]: https://github.com/beyondstorage/go-service-qingstor/compare/v2.1.0...v3.0.0 +[v2.1.0]: https://github.com/beyondstorage/go-service-qingstor/compare/v2.0.0...v2.1.0 +[v2.0.0]: https://github.com/beyondstorage/go-service-qingstor/compare/v1.0.0...v2.0.0 diff --git a/services/qingstor/LICENSE b/services/qingstor/LICENSE new file mode 100644 index 000000000..261eeb9e9 --- /dev/null +++ b/services/qingstor/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/services/qingstor/Makefile b/services/qingstor/Makefile new file mode 100644 index 000000000..f14abf437 --- /dev/null +++ b/services/qingstor/Makefile @@ -0,0 +1,43 @@ +SHELL := /bin/bash + +-include Makefile.env + +.PHONY: all check format vet lint build test generate tidy + +help: + @echo "Please use \`make \` where is one of" + @echo " check to do static check" + @echo " build to create bin directory and build" + @echo " generate to generate code" + @echo " test to run test" + +check: vet + +format: + go fmt ./... + +vet: + go vet ./... + +generate: + @echo "generate code" + go generate ./... + go fmt ./... + +build: tidy generate check + go build ./... + +test: + go test -race -coverprofile=coverage.txt -covermode=atomic -v . + go tool cover -html="coverage.txt" -o "coverage.html" + +integration_test: + go test -count=1 -race -covermode=atomic -v ./tests + +tidy: + go mod tidy + go mod verify + +clean: + @echo "clean generated files" + find . -type f -name 'generated.go' -delete diff --git a/services/qingstor/Makefile.env.example b/services/qingstor/Makefile.env.example new file mode 100644 index 000000000..63666b6cb --- /dev/null +++ b/services/qingstor/Makefile.env.example @@ -0,0 +1,4 @@ +export STORAGE_QINGSTOR_INTEGRATION_TEST=on +export STORAGE_QINGSTOR_CREDENTIAL=hmac:access_key:secret_key +export STORAGE_QINGSTOR_ENDPOINT=https:qingstor.com:443 +export STORAGE_QINGSTOR_NAME=bucketname diff --git a/services/qingstor/README.md b/services/qingstor/README.md new file mode 100644 index 000000000..dbba3d542 --- /dev/null +++ b/services/qingstor/README.md @@ -0,0 +1,35 @@ +[![Services Test Qingstor](https://github.com/beyondstorage/go-storage/actions/workflows/services-test-qingstor.yml/badge.svg)](https://github.com/beyondstorage/go-storage/actions/workflows/services-test-qingstor.yml) + +# qingstor + +[QingStor Object Storage](https://www.qingcloud.com/products/objectstorage/) service support for [go-storage](https://github.com/beyondstorage/go-storage). + +## Install + +```go +go get go.beyondstorage.io/services/qingstor/v4 +``` + +## Usage + +```go +import ( + "log" + + _ "go.beyondstorage.io/services/qingstor/v4" + "go.beyondstorage.io/v5/services" +) + +func main() { + store, err := services.NewStoragerFromString("qingstor://bucket_name/path/to/workdir?credential=hmac:access_key_id:secret_access_key&endpoint=https:qingstor.com") + if err != nil { + log.Fatal(err) + } + + // Write data from io.Reader into hello.txt + n, err := store.Write("hello.txt", r, length) +} +``` + +- See more examples in [go-storage-example](https://github.com/beyondstorage/go-storage-example). +- Read [more docs](https://beyondstorage.io/docs/go-storage/services/qingstor) about go-service-qingstor. diff --git a/services/qingstor/doc.go b/services/qingstor/doc.go new file mode 100644 index 000000000..4a6b5b2cb --- /dev/null +++ b/services/qingstor/doc.go @@ -0,0 +1,7 @@ +/* +Package qingstor provided support for qingstor object storage (https://www.qingcloud.com/products/qingstor/) +*/ +package qingstor + +//go:generate go run github.com/golang/mock/mockgen -package qingstor -destination mock_test.go github.com/qingstor/qingstor-sdk-go/v4/interface Service,Bucket +//go:generate go run -tags tools go.beyondstorage.io/v5/cmd/definitions service.toml diff --git a/services/qingstor/errors.go b/services/qingstor/errors.go new file mode 100644 index 000000000..39984ac0a --- /dev/null +++ b/services/qingstor/errors.go @@ -0,0 +1,22 @@ +package qingstor + +import "go.beyondstorage.io/v5/services" + +var ( + // ErrBucketNameInvalid will be returned while bucket name is invalid. + ErrBucketNameInvalid = services.NewErrorCode("invalid bucket name") + + // ErrWorkDirInvalid will be returned while work dir is invalid. + // Work dir must start and end with only one '/' + ErrWorkDirInvalid = services.NewErrorCode("invalid work dir") + + // ErrEncryptionCustomerKeyInvalid will be returned while encryption customer key is invalid. + // Encryption key must be a 32-byte AES-256 key. + ErrEncryptionCustomerKeyInvalid = services.NewErrorCode("invalid encryption customer key") + + // ErrAppendNextPositionEmpty will be returned while next append position is empty. + ErrAppendNextPositionEmpty = services.NewErrorCode("next append position is empty") + + // ErrPartNumberInvalid will be returned while part number is out of range [0, 10000] when uploading multipart. + ErrPartNumberInvalid = services.NewErrorCode("part number is out of range [0, 10000]") +) diff --git a/services/qingstor/generated.go b/services/qingstor/generated.go new file mode 100644 index 000000000..f5ca50157 --- /dev/null +++ b/services/qingstor/generated.go @@ -0,0 +1,1863 @@ +// Code generated by go generate via cmd/definitions; DO NOT EDIT. +package qingstor + +import ( + "context" + "io" + "net/http" + "strings" + "time" + + . "go.beyondstorage.io/v5/pairs" + "go.beyondstorage.io/v5/pkg/httpclient" + "go.beyondstorage.io/v5/services" + . "go.beyondstorage.io/v5/types" +) + +var ( + _ Storager + _ services.ServiceError + _ httpclient.Options + _ time.Duration + _ http.Request + _ Error +) + +// Type is the type for qingstor +const Type = "qingstor" + +// ObjectSystemMetadata stores system metadata for object. +type ObjectSystemMetadata struct { + EncryptionCustomerAlgorithm string + StorageClass string +} + +// GetObjectSystemMetadata will get ObjectSystemMetadata from Object. +// +// - This function should not be called by service implementer. +// - The returning ObjectServiceMetadata is read only and should not be modified. +func GetObjectSystemMetadata(o *Object) ObjectSystemMetadata { + sm, ok := o.GetSystemMetadata() + if ok { + return sm.(ObjectSystemMetadata) + } + return ObjectSystemMetadata{} +} + +// setObjectSystemMetadata will set ObjectSystemMetadata into Object. +// +// - This function should only be called once, please make sure all data has been written before set. +func setObjectSystemMetadata(o *Object, sm ObjectSystemMetadata) { + o.SetSystemMetadata(sm) +} + +// StorageSystemMetadata stores system metadata for object. +type StorageSystemMetadata struct { + EncryptionCustomerAlgorithm string + StorageClass string +} + +// GetStorageSystemMetadata will get StorageSystemMetadata from Storage. +// +// - This function should not be called by service implementer. +// - The returning StorageServiceMetadata is read only and should not be modified. +func GetStorageSystemMetadata(s *StorageMeta) StorageSystemMetadata { + sm, ok := s.GetSystemMetadata() + if ok { + return sm.(StorageSystemMetadata) + } + return StorageSystemMetadata{} +} + +// setStorageSystemMetadata will set StorageSystemMetadata into Storage. +// +// - This function should only be called once, please make sure all data has been written before set. +func setStorageSystemMetadata(s *StorageMeta, sm StorageSystemMetadata) { + s.SetSystemMetadata(sm) +} + +// WithCopySourceEncryptionCustomerAlgorithm will apply copy_source_encryption_customer_algorithm +// value to Options. +// +// is the encryption algorithm for the source object. Only AES256 is supported now. +func WithCopySourceEncryptionCustomerAlgorithm(v string) Pair { + return Pair{Key: "copy_source_encryption_customer_algorithm", Value: v} +} + +// WithCopySourceEncryptionCustomerKey will apply copy_source_encryption_customer_key value +// to Options. +// +// is the customer-provided encryption key for the source object. For AES256 keys, the plaintext +// must be 32 bytes long. +func WithCopySourceEncryptionCustomerKey(v []byte) Pair { + return Pair{Key: "copy_source_encryption_customer_key", Value: v} +} + +// WithDefaultServicePairs will apply default_service_pairs value to Options. +// +// set default pairs for service actions +func WithDefaultServicePairs(v DefaultServicePairs) Pair { + return Pair{Key: "default_service_pairs", Value: v} +} + +// WithDefaultStoragePairs will apply default_storage_pairs value to Options. +// +// set default pairs for storager actions +func WithDefaultStoragePairs(v DefaultStoragePairs) Pair { + return Pair{Key: "default_storage_pairs", Value: v} +} + +// WithDisableURICleaning will apply disable_uri_cleaning value to Options. +func WithDisableURICleaning() Pair { + return Pair{Key: "disable_uri_cleaning", Value: true} +} + +// WithEnableVirtualDir will apply enable_virtual_dir value to Options. +// +// virtual_dir feature is designed for a service that doesn't have native dir support but wants to +// provide simulated operations. +// +// - If this feature is disabled (the default behavior), the service will behave like it doesn't have +// any dir support. +// - If this feature is enabled, the service will support simulated dir behavior in create_dir, create, +// list, delete, and so on. +// +// This feature was introduced in GSP-109. +func WithEnableVirtualDir() Pair { + return Pair{Key: "enable_virtual_dir", Value: true} +} + +// WithEnableVirtualLink will apply enable_virtual_link value to Options. +// +// virtual_link feature is designed for a service that doesn't have native support for link. +// +// - If this feature is enabled, the service will run compatible mode: create link via native methods, +// but allow read link from old-style link object. +// - If this feature is not enabled, the service will run in native as other service. +// +// This feature was introduced in GSP-86. +func WithEnableVirtualLink() Pair { + return Pair{Key: "enable_virtual_link", Value: true} +} + +// WithEncryptionCustomerAlgorithm will apply encryption_customer_algorithm value to Options. +// +// specifies the encryption algorithm. Only AES256 is supported now. +func WithEncryptionCustomerAlgorithm(v string) Pair { + return Pair{Key: "encryption_customer_algorithm", Value: v} +} + +// WithEncryptionCustomerKey will apply encryption_customer_key value to Options. +// +// is the customer-provided encryption key. For AES256 keys, the plaintext must be 32 bytes long. +func WithEncryptionCustomerKey(v []byte) Pair { + return Pair{Key: "encryption_customer_key", Value: v} +} + +// WithServiceFeatures will apply service_features value to Options. +// +// set service features +func WithServiceFeatures(v ServiceFeatures) Pair { + return Pair{Key: "service_features", Value: v} +} + +// WithStorageClass will apply storage_class value to Options. +func WithStorageClass(v string) Pair { + return Pair{Key: "storage_class", Value: v} +} + +// WithStorageFeatures will apply storage_features value to Options. +// +// set storage features +func WithStorageFeatures(v StorageFeatures) Pair { + return Pair{Key: "storage_features", Value: v} +} + +var pairMap = map[string]string{"content_md5": "string", "content_type": "string", "context": "context.Context", "continuation_token": "string", "copy_source_encryption_customer_algorithm": "string", "copy_source_encryption_customer_key": "[]byte", "credential": "string", "default_content_type": "string", "default_io_callback": "func([]byte)", "default_service_pairs": "DefaultServicePairs", "default_storage_pairs": "DefaultStoragePairs", "disable_uri_cleaning": "bool", "enable_virtual_dir": "bool", "enable_virtual_link": "bool", "encryption_customer_algorithm": "string", "encryption_customer_key": "[]byte", "endpoint": "string", "expire": "time.Duration", "http_client_options": "*httpclient.Options", "interceptor": "Interceptor", "io_callback": "func([]byte)", "list_mode": "ListMode", "location": "string", "multipart_id": "string", "name": "string", "object_mode": "ObjectMode", "offset": "int64", "service_features": "ServiceFeatures", "size": "int64", "storage_class": "string", "storage_features": "StorageFeatures", "work_dir": "string"} +var _ Servicer = &Service{} + +type ServiceFeatures struct { +} + +// pairServiceNew is the parsed struct +type pairServiceNew struct { + pairs []Pair + + // Required pairs + HasCredential bool + Credential string + // Optional pairs + HasDefaultServicePairs bool + DefaultServicePairs DefaultServicePairs + HasEndpoint bool + Endpoint string + HasHTTPClientOptions bool + HTTPClientOptions *httpclient.Options + HasServiceFeatures bool + ServiceFeatures ServiceFeatures + // Enable features +} + +// parsePairServiceNew will parse Pair slice into *pairServiceNew +func parsePairServiceNew(opts []Pair) (pairServiceNew, error) { + result := + pairServiceNew{pairs: opts} + + for _, v := range opts { + switch v.Key { + case "credential": + if result.HasCredential { + continue + } + result.HasCredential = true + result.Credential = v.Value.(string) + case "default_service_pairs": + if result.HasDefaultServicePairs { + continue + } + result.HasDefaultServicePairs = true + result.DefaultServicePairs = v.Value.(DefaultServicePairs) + case "endpoint": + if result.HasEndpoint { + continue + } + result.HasEndpoint = true + result.Endpoint = v.Value.(string) + case "http_client_options": + if result.HasHTTPClientOptions { + continue + } + result.HasHTTPClientOptions = true + result.HTTPClientOptions = v.Value.(*httpclient.Options) + case "service_features": + if result.HasServiceFeatures { + continue + } + result.HasServiceFeatures = true + result.ServiceFeatures = v.Value.(ServiceFeatures) + } + } + // Enable features + + // Default pairs + + if !result.HasCredential { + return pairServiceNew{}, services.PairRequiredError{Keys: []string{"credential"}} + } + return result, nil +} + +// DefaultServicePairs is default pairs for specific action +type DefaultServicePairs struct { + Create []Pair + Delete []Pair + Get []Pair + List []Pair +} +type pairServiceCreate struct { + pairs []Pair + // Required pairs + HasLocation bool + Location string + // Optional pairs +} + +func (s *Service) parsePairServiceCreate(opts []Pair) (pairServiceCreate, error) { + result := + pairServiceCreate{pairs: opts} + + for _, v := range opts { + switch v.Key { + case "location": + if result.HasLocation { + continue + } + result.HasLocation = true + result.Location = v.Value.(string) + default: + return pairServiceCreate{}, services.PairUnsupportedError{Pair: v} + } + } + if !result.HasLocation { + return pairServiceCreate{}, services.PairRequiredError{Keys: []string{"location"}} + } + return result, nil +} + +type pairServiceDelete struct { + pairs []Pair + // Required pairs + // Optional pairs + HasLocation bool + Location string +} + +func (s *Service) parsePairServiceDelete(opts []Pair) (pairServiceDelete, error) { + result := + pairServiceDelete{pairs: opts} + + for _, v := range opts { + switch v.Key { + case "location": + if result.HasLocation { + continue + } + result.HasLocation = true + result.Location = v.Value.(string) + default: + return pairServiceDelete{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} + +type pairServiceGet struct { + pairs []Pair + // Required pairs + // Optional pairs + HasLocation bool + Location string +} + +func (s *Service) parsePairServiceGet(opts []Pair) (pairServiceGet, error) { + result := + pairServiceGet{pairs: opts} + + for _, v := range opts { + switch v.Key { + case "location": + if result.HasLocation { + continue + } + result.HasLocation = true + result.Location = v.Value.(string) + default: + return pairServiceGet{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} + +type pairServiceList struct { + pairs []Pair + // Required pairs + // Optional pairs + HasLocation bool + Location string +} + +func (s *Service) parsePairServiceList(opts []Pair) (pairServiceList, error) { + result := + pairServiceList{pairs: opts} + + for _, v := range opts { + switch v.Key { + case "location": + if result.HasLocation { + continue + } + result.HasLocation = true + result.Location = v.Value.(string) + default: + return pairServiceList{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} +func (s *Service) Create(name string, pairs ...Pair) (store Storager, err error) { + ctx := context.Background() + return s.CreateWithContext(ctx, name, pairs...) +} +func (s *Service) CreateWithContext(ctx context.Context, name string, pairs ...Pair) (store Storager, err error) { + defer func() { + err = + s.formatError("create", err, name) + }() + + pairs = append(pairs, s.defaultPairs.Create...) + var opt pairServiceCreate + + opt, err = s.parsePairServiceCreate(pairs) + if err != nil { + return + } + return s.create(ctx, name, opt) +} +func (s *Service) Delete(name string, pairs ...Pair) (err error) { + ctx := context.Background() + return s.DeleteWithContext(ctx, name, pairs...) +} +func (s *Service) DeleteWithContext(ctx context.Context, name string, pairs ...Pair) (err error) { + defer func() { + err = + s.formatError("delete", err, name) + }() + + pairs = append(pairs, s.defaultPairs.Delete...) + var opt pairServiceDelete + + opt, err = s.parsePairServiceDelete(pairs) + if err != nil { + return + } + return s.delete(ctx, name, opt) +} +func (s *Service) Get(name string, pairs ...Pair) (store Storager, err error) { + ctx := context.Background() + return s.GetWithContext(ctx, name, pairs...) +} +func (s *Service) GetWithContext(ctx context.Context, name string, pairs ...Pair) (store Storager, err error) { + defer func() { + err = + s.formatError("get", err, name) + }() + + pairs = append(pairs, s.defaultPairs.Get...) + var opt pairServiceGet + + opt, err = s.parsePairServiceGet(pairs) + if err != nil { + return + } + return s.get(ctx, name, opt) +} +func (s *Service) List(pairs ...Pair) (sti *StoragerIterator, err error) { + ctx := context.Background() + return s.ListWithContext(ctx, pairs...) +} +func (s *Service) ListWithContext(ctx context.Context, pairs ...Pair) (sti *StoragerIterator, err error) { + defer func() { + err = + s.formatError("list", err, "") + }() + + pairs = append(pairs, s.defaultPairs.List...) + var opt pairServiceList + + opt, err = s.parsePairServiceList(pairs) + if err != nil { + return + } + return s.list(ctx, opt) +} + +var ( + _ Appender = &Storage{} + _ Copier = &Storage{} + _ Direr = &Storage{} + _ Fetcher = &Storage{} + _ Linker = &Storage{} + _ Mover = &Storage{} + _ Multiparter = &Storage{} + _ Reacher = &Storage{} + _ StorageHTTPSigner = &Storage{} + _ Storager = &Storage{} +) + +type StorageFeatures struct { // virtual_dir feature is designed for a service that doesn't have native dir support but wants to + // provide simulated operations. + // + // - If this feature is disabled (the default behavior), the service will behave like it doesn't have + // any dir support. + // - If this feature is enabled, the service will support simulated dir behavior in create_dir, create, + // list, delete, and so on. + // + // This feature was introduced in GSP-109. + VirtualDir bool + // virtual_link feature is designed for a service that doesn't have native support for link. + // + // - If this feature is enabled, the service will run compatible mode: create link via native methods, + // but allow read link from old-style link object. + // - If this feature is not enabled, the service will run in native as other service. + // + // This feature was introduced in GSP-86. + VirtualLink bool +} + +// pairStorageNew is the parsed struct +type pairStorageNew struct { + pairs []Pair + + // Required pairs + HasName bool + Name string + // Optional pairs + HasDefaultContentType bool + DefaultContentType string + HasDefaultIoCallback bool + DefaultIoCallback func([]byte) + HasDefaultStoragePairs bool + DefaultStoragePairs DefaultStoragePairs + HasDisableURICleaning bool + DisableURICleaning bool + HasHTTPClientOptions bool + HTTPClientOptions *httpclient.Options + HasLocation bool + Location string + HasStorageFeatures bool + StorageFeatures StorageFeatures + HasWorkDir bool + WorkDir string + // Enable features + hasEnableVirtualDir bool + EnableVirtualDir bool + hasEnableVirtualLink bool + EnableVirtualLink bool +} + +// parsePairStorageNew will parse Pair slice into *pairStorageNew +func parsePairStorageNew(opts []Pair) (pairStorageNew, error) { + result := + pairStorageNew{pairs: opts} + + for _, v := range opts { + switch v.Key { + case "name": + if result.HasName { + continue + } + result.HasName = true + result.Name = v.Value.(string) + case "default_content_type": + if result.HasDefaultContentType { + continue + } + result.HasDefaultContentType = true + result.DefaultContentType = v.Value.(string) + case "default_io_callback": + if result.HasDefaultIoCallback { + continue + } + result.HasDefaultIoCallback = true + result.DefaultIoCallback = v.Value.(func([]byte)) + case "default_storage_pairs": + if result.HasDefaultStoragePairs { + continue + } + result.HasDefaultStoragePairs = true + result.DefaultStoragePairs = v.Value.(DefaultStoragePairs) + case "disable_uri_cleaning": + if result.HasDisableURICleaning { + continue + } + result.HasDisableURICleaning = true + result.DisableURICleaning = v.Value.(bool) + case "http_client_options": + if result.HasHTTPClientOptions { + continue + } + result.HasHTTPClientOptions = true + result.HTTPClientOptions = v.Value.(*httpclient.Options) + case "location": + if result.HasLocation { + continue + } + result.HasLocation = true + result.Location = v.Value.(string) + case "storage_features": + if result.HasStorageFeatures { + continue + } + result.HasStorageFeatures = true + result.StorageFeatures = v.Value.(StorageFeatures) + case "work_dir": + if result.HasWorkDir { + continue + } + result.HasWorkDir = true + result.WorkDir = v.Value.(string) + case "enable_virtual_dir": + if result.hasEnableVirtualDir { + continue + } + result.hasEnableVirtualDir = true + result.EnableVirtualDir = true + case "enable_virtual_link": + if result.hasEnableVirtualLink { + continue + } + result.hasEnableVirtualLink = true + result.EnableVirtualLink = true + } + } + // Enable features + if result.hasEnableVirtualDir { + result.HasStorageFeatures = true + result.StorageFeatures.VirtualDir = true + } + if result.hasEnableVirtualLink { + result.HasStorageFeatures = true + result.StorageFeatures.VirtualLink = true + } + // Default pairs + if result.HasDefaultContentType { + result.HasDefaultStoragePairs = true + result.DefaultStoragePairs.CreateAppend = append(result.DefaultStoragePairs.CreateAppend, WithContentType(result.DefaultContentType)) + result.DefaultStoragePairs.QuerySignHTTPWrite = append(result.DefaultStoragePairs.QuerySignHTTPWrite, WithContentType(result.DefaultContentType)) + result.DefaultStoragePairs.Write = append(result.DefaultStoragePairs.Write, WithContentType(result.DefaultContentType)) + } + if result.HasDefaultIoCallback { + result.HasDefaultStoragePairs = true + result.DefaultStoragePairs.Read = append(result.DefaultStoragePairs.Read, WithIoCallback(result.DefaultIoCallback)) + result.DefaultStoragePairs.Write = append(result.DefaultStoragePairs.Write, WithIoCallback(result.DefaultIoCallback)) + result.DefaultStoragePairs.WriteMultipart = append(result.DefaultStoragePairs.WriteMultipart, WithIoCallback(result.DefaultIoCallback)) + } + if !result.HasName { + return pairStorageNew{}, services.PairRequiredError{Keys: []string{"name"}} + } + return result, nil +} + +// DefaultStoragePairs is default pairs for specific action +type DefaultStoragePairs struct { + CommitAppend []Pair + CompleteMultipart []Pair + Copy []Pair + Create []Pair + CreateAppend []Pair + CreateDir []Pair + CreateLink []Pair + CreateMultipart []Pair + Delete []Pair + Fetch []Pair + List []Pair + ListMultipart []Pair + Metadata []Pair + Move []Pair + QuerySignHTTPDelete []Pair + QuerySignHTTPRead []Pair + QuerySignHTTPWrite []Pair + Reach []Pair + Read []Pair + Stat []Pair + Write []Pair + WriteAppend []Pair + WriteMultipart []Pair +} +type pairStorageCommitAppend struct { + pairs []Pair + // Required pairs + // Optional pairs +} + +func (s *Storage) parsePairStorageCommitAppend(opts []Pair) (pairStorageCommitAppend, error) { + result := + pairStorageCommitAppend{pairs: opts} + + for _, v := range opts { + switch v.Key { + default: + return pairStorageCommitAppend{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} + +type pairStorageCompleteMultipart struct { + pairs []Pair + // Required pairs + // Optional pairs +} + +func (s *Storage) parsePairStorageCompleteMultipart(opts []Pair) (pairStorageCompleteMultipart, error) { + result := + pairStorageCompleteMultipart{pairs: opts} + + for _, v := range opts { + switch v.Key { + default: + return pairStorageCompleteMultipart{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} + +type pairStorageCopy struct { + pairs []Pair + // Required pairs + // Optional pairs + HasCopySourceEncryptionCustomerAlgorithm bool + CopySourceEncryptionCustomerAlgorithm string + HasCopySourceEncryptionCustomerKey bool + CopySourceEncryptionCustomerKey []byte + HasEncryptionCustomerAlgorithm bool + EncryptionCustomerAlgorithm string + HasEncryptionCustomerKey bool + EncryptionCustomerKey []byte +} + +func (s *Storage) parsePairStorageCopy(opts []Pair) (pairStorageCopy, error) { + result := + pairStorageCopy{pairs: opts} + + for _, v := range opts { + switch v.Key { + case "copy_source_encryption_customer_algorithm": + if result.HasCopySourceEncryptionCustomerAlgorithm { + continue + } + result.HasCopySourceEncryptionCustomerAlgorithm = true + result.CopySourceEncryptionCustomerAlgorithm = v.Value.(string) + case "copy_source_encryption_customer_key": + if result.HasCopySourceEncryptionCustomerKey { + continue + } + result.HasCopySourceEncryptionCustomerKey = true + result.CopySourceEncryptionCustomerKey = v.Value.([]byte) + case "encryption_customer_algorithm": + if result.HasEncryptionCustomerAlgorithm { + continue + } + result.HasEncryptionCustomerAlgorithm = true + result.EncryptionCustomerAlgorithm = v.Value.(string) + case "encryption_customer_key": + if result.HasEncryptionCustomerKey { + continue + } + result.HasEncryptionCustomerKey = true + result.EncryptionCustomerKey = v.Value.([]byte) + default: + return pairStorageCopy{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} + +type pairStorageCreate struct { + pairs []Pair + // Required pairs + // Optional pairs + HasMultipartID bool + MultipartID string + HasObjectMode bool + ObjectMode ObjectMode +} + +func (s *Storage) parsePairStorageCreate(opts []Pair) (pairStorageCreate, error) { + result := + pairStorageCreate{pairs: opts} + + for _, v := range opts { + switch v.Key { + case "multipart_id": + if result.HasMultipartID { + continue + } + result.HasMultipartID = true + result.MultipartID = v.Value.(string) + case "object_mode": + if result.HasObjectMode { + continue + } + result.HasObjectMode = true + result.ObjectMode = v.Value.(ObjectMode) + default: + return pairStorageCreate{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} + +type pairStorageCreateAppend struct { + pairs []Pair + // Required pairs + // Optional pairs + HasContentType bool + ContentType string + HasStorageClass bool + StorageClass string +} + +func (s *Storage) parsePairStorageCreateAppend(opts []Pair) (pairStorageCreateAppend, error) { + result := + pairStorageCreateAppend{pairs: opts} + + for _, v := range opts { + switch v.Key { + case "content_type": + if result.HasContentType { + continue + } + result.HasContentType = true + result.ContentType = v.Value.(string) + case "storage_class": + if result.HasStorageClass { + continue + } + result.HasStorageClass = true + result.StorageClass = v.Value.(string) + default: + return pairStorageCreateAppend{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} + +type pairStorageCreateDir struct { + pairs []Pair + // Required pairs + // Optional pairs + HasStorageClass bool + StorageClass string +} + +func (s *Storage) parsePairStorageCreateDir(opts []Pair) (pairStorageCreateDir, error) { + result := + pairStorageCreateDir{pairs: opts} + + for _, v := range opts { + switch v.Key { + case "storage_class": + if result.HasStorageClass { + continue + } + result.HasStorageClass = true + result.StorageClass = v.Value.(string) + default: + return pairStorageCreateDir{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} + +type pairStorageCreateLink struct { + pairs []Pair + // Required pairs + // Optional pairs +} + +func (s *Storage) parsePairStorageCreateLink(opts []Pair) (pairStorageCreateLink, error) { + result := + pairStorageCreateLink{pairs: opts} + + for _, v := range opts { + switch v.Key { + default: + return pairStorageCreateLink{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} + +type pairStorageCreateMultipart struct { + pairs []Pair + // Required pairs + // Optional pairs + HasEncryptionCustomerAlgorithm bool + EncryptionCustomerAlgorithm string + HasEncryptionCustomerKey bool + EncryptionCustomerKey []byte +} + +func (s *Storage) parsePairStorageCreateMultipart(opts []Pair) (pairStorageCreateMultipart, error) { + result := + pairStorageCreateMultipart{pairs: opts} + + for _, v := range opts { + switch v.Key { + case "encryption_customer_algorithm": + if result.HasEncryptionCustomerAlgorithm { + continue + } + result.HasEncryptionCustomerAlgorithm = true + result.EncryptionCustomerAlgorithm = v.Value.(string) + case "encryption_customer_key": + if result.HasEncryptionCustomerKey { + continue + } + result.HasEncryptionCustomerKey = true + result.EncryptionCustomerKey = v.Value.([]byte) + default: + return pairStorageCreateMultipart{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} + +type pairStorageDelete struct { + pairs []Pair + // Required pairs + // Optional pairs + HasMultipartID bool + MultipartID string + HasObjectMode bool + ObjectMode ObjectMode +} + +func (s *Storage) parsePairStorageDelete(opts []Pair) (pairStorageDelete, error) { + result := + pairStorageDelete{pairs: opts} + + for _, v := range opts { + switch v.Key { + case "multipart_id": + if result.HasMultipartID { + continue + } + result.HasMultipartID = true + result.MultipartID = v.Value.(string) + case "object_mode": + if result.HasObjectMode { + continue + } + result.HasObjectMode = true + result.ObjectMode = v.Value.(ObjectMode) + default: + return pairStorageDelete{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} + +type pairStorageFetch struct { + pairs []Pair + // Required pairs + // Optional pairs +} + +func (s *Storage) parsePairStorageFetch(opts []Pair) (pairStorageFetch, error) { + result := + pairStorageFetch{pairs: opts} + + for _, v := range opts { + switch v.Key { + default: + return pairStorageFetch{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} + +type pairStorageList struct { + pairs []Pair + // Required pairs + // Optional pairs + HasListMode bool + ListMode ListMode +} + +func (s *Storage) parsePairStorageList(opts []Pair) (pairStorageList, error) { + result := + pairStorageList{pairs: opts} + + for _, v := range opts { + switch v.Key { + case "list_mode": + if result.HasListMode { + continue + } + result.HasListMode = true + result.ListMode = v.Value.(ListMode) + default: + return pairStorageList{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} + +type pairStorageListMultipart struct { + pairs []Pair + // Required pairs + // Optional pairs +} + +func (s *Storage) parsePairStorageListMultipart(opts []Pair) (pairStorageListMultipart, error) { + result := + pairStorageListMultipart{pairs: opts} + + for _, v := range opts { + switch v.Key { + default: + return pairStorageListMultipart{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} + +type pairStorageMetadata struct { + pairs []Pair + // Required pairs + // Optional pairs +} + +func (s *Storage) parsePairStorageMetadata(opts []Pair) (pairStorageMetadata, error) { + result := + pairStorageMetadata{pairs: opts} + + for _, v := range opts { + switch v.Key { + default: + return pairStorageMetadata{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} + +type pairStorageMove struct { + pairs []Pair + // Required pairs + // Optional pairs +} + +func (s *Storage) parsePairStorageMove(opts []Pair) (pairStorageMove, error) { + result := + pairStorageMove{pairs: opts} + + for _, v := range opts { + switch v.Key { + default: + return pairStorageMove{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} + +type pairStorageQuerySignHTTPDelete struct { + pairs []Pair + // Required pairs + // Optional pairs +} + +func (s *Storage) parsePairStorageQuerySignHTTPDelete(opts []Pair) (pairStorageQuerySignHTTPDelete, error) { + result := + pairStorageQuerySignHTTPDelete{pairs: opts} + + for _, v := range opts { + switch v.Key { + default: + return pairStorageQuerySignHTTPDelete{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} + +type pairStorageQuerySignHTTPRead struct { + pairs []Pair + // Required pairs + // Optional pairs + HasEncryptionCustomerAlgorithm bool + EncryptionCustomerAlgorithm string + HasEncryptionCustomerKey bool + EncryptionCustomerKey []byte + HasOffset bool + Offset int64 + HasSize bool + Size int64 +} + +func (s *Storage) parsePairStorageQuerySignHTTPRead(opts []Pair) (pairStorageQuerySignHTTPRead, error) { + result := + pairStorageQuerySignHTTPRead{pairs: opts} + + for _, v := range opts { + switch v.Key { + case "encryption_customer_algorithm": + if result.HasEncryptionCustomerAlgorithm { + continue + } + result.HasEncryptionCustomerAlgorithm = true + result.EncryptionCustomerAlgorithm = v.Value.(string) + case "encryption_customer_key": + if result.HasEncryptionCustomerKey { + continue + } + result.HasEncryptionCustomerKey = true + result.EncryptionCustomerKey = v.Value.([]byte) + case "offset": + if result.HasOffset { + continue + } + result.HasOffset = true + result.Offset = v.Value.(int64) + case "size": + if result.HasSize { + continue + } + result.HasSize = true + result.Size = v.Value.(int64) + default: + return pairStorageQuerySignHTTPRead{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} + +type pairStorageQuerySignHTTPWrite struct { + pairs []Pair + // Required pairs + // Optional pairs + HasContentMd5 bool + ContentMd5 string + HasContentType bool + ContentType string + HasEncryptionCustomerAlgorithm bool + EncryptionCustomerAlgorithm string + HasEncryptionCustomerKey bool + EncryptionCustomerKey []byte + HasStorageClass bool + StorageClass string +} + +func (s *Storage) parsePairStorageQuerySignHTTPWrite(opts []Pair) (pairStorageQuerySignHTTPWrite, error) { + result := + pairStorageQuerySignHTTPWrite{pairs: opts} + + for _, v := range opts { + switch v.Key { + case "content_md5": + if result.HasContentMd5 { + continue + } + result.HasContentMd5 = true + result.ContentMd5 = v.Value.(string) + case "content_type": + if result.HasContentType { + continue + } + result.HasContentType = true + result.ContentType = v.Value.(string) + case "encryption_customer_algorithm": + if result.HasEncryptionCustomerAlgorithm { + continue + } + result.HasEncryptionCustomerAlgorithm = true + result.EncryptionCustomerAlgorithm = v.Value.(string) + case "encryption_customer_key": + if result.HasEncryptionCustomerKey { + continue + } + result.HasEncryptionCustomerKey = true + result.EncryptionCustomerKey = v.Value.([]byte) + case "storage_class": + if result.HasStorageClass { + continue + } + result.HasStorageClass = true + result.StorageClass = v.Value.(string) + default: + return pairStorageQuerySignHTTPWrite{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} + +type pairStorageReach struct { + pairs []Pair + // Required pairs + HasExpire bool + Expire time.Duration + // Optional pairs +} + +func (s *Storage) parsePairStorageReach(opts []Pair) (pairStorageReach, error) { + result := + pairStorageReach{pairs: opts} + + for _, v := range opts { + switch v.Key { + case "expire": + if result.HasExpire { + continue + } + result.HasExpire = true + result.Expire = v.Value.(time.Duration) + default: + return pairStorageReach{}, services.PairUnsupportedError{Pair: v} + } + } + if !result.HasExpire { + return pairStorageReach{}, services.PairRequiredError{Keys: []string{"expire"}} + } + return result, nil +} + +type pairStorageRead struct { + pairs []Pair + // Required pairs + // Optional pairs + HasEncryptionCustomerAlgorithm bool + EncryptionCustomerAlgorithm string + HasEncryptionCustomerKey bool + EncryptionCustomerKey []byte + HasIoCallback bool + IoCallback func([]byte) + HasOffset bool + Offset int64 + HasSize bool + Size int64 +} + +func (s *Storage) parsePairStorageRead(opts []Pair) (pairStorageRead, error) { + result := + pairStorageRead{pairs: opts} + + for _, v := range opts { + switch v.Key { + case "encryption_customer_algorithm": + if result.HasEncryptionCustomerAlgorithm { + continue + } + result.HasEncryptionCustomerAlgorithm = true + result.EncryptionCustomerAlgorithm = v.Value.(string) + case "encryption_customer_key": + if result.HasEncryptionCustomerKey { + continue + } + result.HasEncryptionCustomerKey = true + result.EncryptionCustomerKey = v.Value.([]byte) + case "io_callback": + if result.HasIoCallback { + continue + } + result.HasIoCallback = true + result.IoCallback = v.Value.(func([]byte)) + case "offset": + if result.HasOffset { + continue + } + result.HasOffset = true + result.Offset = v.Value.(int64) + case "size": + if result.HasSize { + continue + } + result.HasSize = true + result.Size = v.Value.(int64) + default: + return pairStorageRead{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} + +type pairStorageStat struct { + pairs []Pair + // Required pairs + // Optional pairs + HasMultipartID bool + MultipartID string + HasObjectMode bool + ObjectMode ObjectMode +} + +func (s *Storage) parsePairStorageStat(opts []Pair) (pairStorageStat, error) { + result := + pairStorageStat{pairs: opts} + + for _, v := range opts { + switch v.Key { + case "multipart_id": + if result.HasMultipartID { + continue + } + result.HasMultipartID = true + result.MultipartID = v.Value.(string) + case "object_mode": + if result.HasObjectMode { + continue + } + result.HasObjectMode = true + result.ObjectMode = v.Value.(ObjectMode) + default: + return pairStorageStat{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} + +type pairStorageWrite struct { + pairs []Pair + // Required pairs + // Optional pairs + HasContentMd5 bool + ContentMd5 string + HasContentType bool + ContentType string + HasEncryptionCustomerAlgorithm bool + EncryptionCustomerAlgorithm string + HasEncryptionCustomerKey bool + EncryptionCustomerKey []byte + HasIoCallback bool + IoCallback func([]byte) + HasStorageClass bool + StorageClass string +} + +func (s *Storage) parsePairStorageWrite(opts []Pair) (pairStorageWrite, error) { + result := + pairStorageWrite{pairs: opts} + + for _, v := range opts { + switch v.Key { + case "content_md5": + if result.HasContentMd5 { + continue + } + result.HasContentMd5 = true + result.ContentMd5 = v.Value.(string) + case "content_type": + if result.HasContentType { + continue + } + result.HasContentType = true + result.ContentType = v.Value.(string) + case "encryption_customer_algorithm": + if result.HasEncryptionCustomerAlgorithm { + continue + } + result.HasEncryptionCustomerAlgorithm = true + result.EncryptionCustomerAlgorithm = v.Value.(string) + case "encryption_customer_key": + if result.HasEncryptionCustomerKey { + continue + } + result.HasEncryptionCustomerKey = true + result.EncryptionCustomerKey = v.Value.([]byte) + case "io_callback": + if result.HasIoCallback { + continue + } + result.HasIoCallback = true + result.IoCallback = v.Value.(func([]byte)) + case "storage_class": + if result.HasStorageClass { + continue + } + result.HasStorageClass = true + result.StorageClass = v.Value.(string) + default: + return pairStorageWrite{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} + +type pairStorageWriteAppend struct { + pairs []Pair + // Required pairs + // Optional pairs + HasContentMd5 bool + ContentMd5 string +} + +func (s *Storage) parsePairStorageWriteAppend(opts []Pair) (pairStorageWriteAppend, error) { + result := + pairStorageWriteAppend{pairs: opts} + + for _, v := range opts { + switch v.Key { + case "content_md5": + if result.HasContentMd5 { + continue + } + result.HasContentMd5 = true + result.ContentMd5 = v.Value.(string) + default: + return pairStorageWriteAppend{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} + +type pairStorageWriteMultipart struct { + pairs []Pair + // Required pairs + // Optional pairs + HasEncryptionCustomerAlgorithm bool + EncryptionCustomerAlgorithm string + HasEncryptionCustomerKey bool + EncryptionCustomerKey []byte + HasIoCallback bool + IoCallback func([]byte) +} + +func (s *Storage) parsePairStorageWriteMultipart(opts []Pair) (pairStorageWriteMultipart, error) { + result := + pairStorageWriteMultipart{pairs: opts} + + for _, v := range opts { + switch v.Key { + case "encryption_customer_algorithm": + if result.HasEncryptionCustomerAlgorithm { + continue + } + result.HasEncryptionCustomerAlgorithm = true + result.EncryptionCustomerAlgorithm = v.Value.(string) + case "encryption_customer_key": + if result.HasEncryptionCustomerKey { + continue + } + result.HasEncryptionCustomerKey = true + result.EncryptionCustomerKey = v.Value.([]byte) + case "io_callback": + if result.HasIoCallback { + continue + } + result.HasIoCallback = true + result.IoCallback = v.Value.(func([]byte)) + default: + return pairStorageWriteMultipart{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} +func (s *Storage) CommitAppend(o *Object, pairs ...Pair) (err error) { + ctx := context.Background() + return s.CommitAppendWithContext(ctx, o, pairs...) +} +func (s *Storage) CommitAppendWithContext(ctx context.Context, o *Object, pairs ...Pair) (err error) { + defer func() { + err = + s.formatError("commit_append", err) + }() + if !o.Mode.IsAppend() { + err = services.ObjectModeInvalidError{Expected: ModeAppend, Actual: o.Mode} + return + } + pairs = append(pairs, s.defaultPairs.CommitAppend...) + var opt pairStorageCommitAppend + + opt, err = s.parsePairStorageCommitAppend(pairs) + if err != nil { + return + } + return s.commitAppend(ctx, o, opt) +} +func (s *Storage) CompleteMultipart(o *Object, parts []*Part, pairs ...Pair) (err error) { + ctx := context.Background() + return s.CompleteMultipartWithContext(ctx, o, parts, pairs...) +} +func (s *Storage) CompleteMultipartWithContext(ctx context.Context, o *Object, parts []*Part, pairs ...Pair) (err error) { + defer func() { + err = + s.formatError("complete_multipart", err) + }() + if !o.Mode.IsPart() { + err = services.ObjectModeInvalidError{Expected: ModePart, Actual: o.Mode} + return + } + pairs = append(pairs, s.defaultPairs.CompleteMultipart...) + var opt pairStorageCompleteMultipart + + opt, err = s.parsePairStorageCompleteMultipart(pairs) + if err != nil { + return + } + return s.completeMultipart(ctx, o, parts, opt) +} +func (s *Storage) Copy(src string, dst string, pairs ...Pair) (err error) { + ctx := context.Background() + return s.CopyWithContext(ctx, src, dst, pairs...) +} +func (s *Storage) CopyWithContext(ctx context.Context, src string, dst string, pairs ...Pair) (err error) { + defer func() { + err = + s.formatError("copy", err, src, dst) + }() + + pairs = append(pairs, s.defaultPairs.Copy...) + var opt pairStorageCopy + + opt, err = s.parsePairStorageCopy(pairs) + if err != nil { + return + } + return s.copy(ctx, strings.ReplaceAll(src, "\\", "/"), strings.ReplaceAll(dst, "\\", "/"), opt) +} +func (s *Storage) Create(path string, pairs ...Pair) (o *Object) { + pairs = append(pairs, s.defaultPairs.Create...) + var opt pairStorageCreate + + // Ignore error while handling local functions. + opt, _ = s.parsePairStorageCreate(pairs) + return s.create(path, opt) +} +func (s *Storage) CreateAppend(path string, pairs ...Pair) (o *Object, err error) { + ctx := context.Background() + return s.CreateAppendWithContext(ctx, path, pairs...) +} +func (s *Storage) CreateAppendWithContext(ctx context.Context, path string, pairs ...Pair) (o *Object, err error) { + defer func() { + err = + s.formatError("create_append", err, path) + }() + + pairs = append(pairs, s.defaultPairs.CreateAppend...) + var opt pairStorageCreateAppend + + opt, err = s.parsePairStorageCreateAppend(pairs) + if err != nil { + return + } + return s.createAppend(ctx, strings.ReplaceAll(path, "\\", "/"), opt) +} +func (s *Storage) CreateDir(path string, pairs ...Pair) (o *Object, err error) { + ctx := context.Background() + return s.CreateDirWithContext(ctx, path, pairs...) +} +func (s *Storage) CreateDirWithContext(ctx context.Context, path string, pairs ...Pair) (o *Object, err error) { + defer func() { + err = + s.formatError("create_dir", err, path) + }() + + pairs = append(pairs, s.defaultPairs.CreateDir...) + var opt pairStorageCreateDir + + opt, err = s.parsePairStorageCreateDir(pairs) + if err != nil { + return + } + return s.createDir(ctx, strings.ReplaceAll(path, "\\", "/"), opt) +} +func (s *Storage) CreateLink(path string, target string, pairs ...Pair) (o *Object, err error) { + ctx := context.Background() + return s.CreateLinkWithContext(ctx, path, target, pairs...) +} +func (s *Storage) CreateLinkWithContext(ctx context.Context, path string, target string, pairs ...Pair) (o *Object, err error) { + defer func() { + err = + s.formatError("create_link", err, path, target) + }() + + pairs = append(pairs, s.defaultPairs.CreateLink...) + var opt pairStorageCreateLink + + opt, err = s.parsePairStorageCreateLink(pairs) + if err != nil { + return + } + return s.createLink(ctx, strings.ReplaceAll(path, "\\", "/"), strings.ReplaceAll(target, "\\", "/"), opt) +} +func (s *Storage) CreateMultipart(path string, pairs ...Pair) (o *Object, err error) { + ctx := context.Background() + return s.CreateMultipartWithContext(ctx, path, pairs...) +} +func (s *Storage) CreateMultipartWithContext(ctx context.Context, path string, pairs ...Pair) (o *Object, err error) { + defer func() { + err = + s.formatError("create_multipart", err, path) + }() + + pairs = append(pairs, s.defaultPairs.CreateMultipart...) + var opt pairStorageCreateMultipart + + opt, err = s.parsePairStorageCreateMultipart(pairs) + if err != nil { + return + } + return s.createMultipart(ctx, strings.ReplaceAll(path, "\\", "/"), opt) +} +func (s *Storage) Delete(path string, pairs ...Pair) (err error) { + ctx := context.Background() + return s.DeleteWithContext(ctx, path, pairs...) +} +func (s *Storage) DeleteWithContext(ctx context.Context, path string, pairs ...Pair) (err error) { + defer func() { + err = + s.formatError("delete", err, path) + }() + + pairs = append(pairs, s.defaultPairs.Delete...) + var opt pairStorageDelete + + opt, err = s.parsePairStorageDelete(pairs) + if err != nil { + return + } + return s.delete(ctx, strings.ReplaceAll(path, "\\", "/"), opt) +} +func (s *Storage) Fetch(path string, url string, pairs ...Pair) (err error) { + ctx := context.Background() + return s.FetchWithContext(ctx, path, url, pairs...) +} +func (s *Storage) FetchWithContext(ctx context.Context, path string, url string, pairs ...Pair) (err error) { + defer func() { + err = + s.formatError("fetch", err, path, url) + }() + + pairs = append(pairs, s.defaultPairs.Fetch...) + var opt pairStorageFetch + + opt, err = s.parsePairStorageFetch(pairs) + if err != nil { + return + } + return s.fetch(ctx, strings.ReplaceAll(path, "\\", "/"), url, opt) +} +func (s *Storage) List(path string, pairs ...Pair) (oi *ObjectIterator, err error) { + ctx := context.Background() + return s.ListWithContext(ctx, path, pairs...) +} +func (s *Storage) ListWithContext(ctx context.Context, path string, pairs ...Pair) (oi *ObjectIterator, err error) { + defer func() { + err = + s.formatError("list", err, path) + }() + + pairs = append(pairs, s.defaultPairs.List...) + var opt pairStorageList + + opt, err = s.parsePairStorageList(pairs) + if err != nil { + return + } + return s.list(ctx, strings.ReplaceAll(path, "\\", "/"), opt) +} +func (s *Storage) ListMultipart(o *Object, pairs ...Pair) (pi *PartIterator, err error) { + ctx := context.Background() + return s.ListMultipartWithContext(ctx, o, pairs...) +} +func (s *Storage) ListMultipartWithContext(ctx context.Context, o *Object, pairs ...Pair) (pi *PartIterator, err error) { + defer func() { + err = + s.formatError("list_multipart", err) + }() + if !o.Mode.IsPart() { + err = services.ObjectModeInvalidError{Expected: ModePart, Actual: o.Mode} + return + } + pairs = append(pairs, s.defaultPairs.ListMultipart...) + var opt pairStorageListMultipart + + opt, err = s.parsePairStorageListMultipart(pairs) + if err != nil { + return + } + return s.listMultipart(ctx, o, opt) +} +func (s *Storage) Metadata(pairs ...Pair) (meta *StorageMeta) { + pairs = append(pairs, s.defaultPairs.Metadata...) + var opt pairStorageMetadata + + // Ignore error while handling local functions. + opt, _ = s.parsePairStorageMetadata(pairs) + return s.metadata(opt) +} +func (s *Storage) Move(src string, dst string, pairs ...Pair) (err error) { + ctx := context.Background() + return s.MoveWithContext(ctx, src, dst, pairs...) +} +func (s *Storage) MoveWithContext(ctx context.Context, src string, dst string, pairs ...Pair) (err error) { + defer func() { + err = + s.formatError("move", err, src, dst) + }() + + pairs = append(pairs, s.defaultPairs.Move...) + var opt pairStorageMove + + opt, err = s.parsePairStorageMove(pairs) + if err != nil { + return + } + return s.move(ctx, strings.ReplaceAll(src, "\\", "/"), strings.ReplaceAll(dst, "\\", "/"), opt) +} +func (s *Storage) QuerySignHTTPDelete(path string, expire time.Duration, pairs ...Pair) (req *http.Request, err error) { + ctx := context.Background() + return s.QuerySignHTTPDeleteWithContext(ctx, path, expire, pairs...) +} +func (s *Storage) QuerySignHTTPDeleteWithContext(ctx context.Context, path string, expire time.Duration, pairs ...Pair) (req *http.Request, err error) { + defer func() { + err = + s.formatError("query_sign_http_delete", err, path) + }() + + pairs = append(pairs, s.defaultPairs.QuerySignHTTPDelete...) + var opt pairStorageQuerySignHTTPDelete + + opt, err = s.parsePairStorageQuerySignHTTPDelete(pairs) + if err != nil { + return + } + return s.querySignHTTPDelete(ctx, strings.ReplaceAll(path, "\\", "/"), expire, opt) +} +func (s *Storage) QuerySignHTTPRead(path string, expire time.Duration, pairs ...Pair) (req *http.Request, err error) { + ctx := context.Background() + return s.QuerySignHTTPReadWithContext(ctx, path, expire, pairs...) +} +func (s *Storage) QuerySignHTTPReadWithContext(ctx context.Context, path string, expire time.Duration, pairs ...Pair) (req *http.Request, err error) { + defer func() { + err = + s.formatError("query_sign_http_read", err, path) + }() + + pairs = append(pairs, s.defaultPairs.QuerySignHTTPRead...) + var opt pairStorageQuerySignHTTPRead + + opt, err = s.parsePairStorageQuerySignHTTPRead(pairs) + if err != nil { + return + } + return s.querySignHTTPRead(ctx, strings.ReplaceAll(path, "\\", "/"), expire, opt) +} +func (s *Storage) QuerySignHTTPWrite(path string, size int64, expire time.Duration, pairs ...Pair) (req *http.Request, err error) { + ctx := context.Background() + return s.QuerySignHTTPWriteWithContext(ctx, path, size, expire, pairs...) +} +func (s *Storage) QuerySignHTTPWriteWithContext(ctx context.Context, path string, size int64, expire time.Duration, pairs ...Pair) (req *http.Request, err error) { + defer func() { + err = + s.formatError("query_sign_http_write", err, path) + }() + + pairs = append(pairs, s.defaultPairs.QuerySignHTTPWrite...) + var opt pairStorageQuerySignHTTPWrite + + opt, err = s.parsePairStorageQuerySignHTTPWrite(pairs) + if err != nil { + return + } + return s.querySignHTTPWrite(ctx, strings.ReplaceAll(path, "\\", "/"), size, expire, opt) +} +func (s *Storage) Reach(path string, pairs ...Pair) (url string, err error) { + ctx := context.Background() + return s.ReachWithContext(ctx, path, pairs...) +} +func (s *Storage) ReachWithContext(ctx context.Context, path string, pairs ...Pair) (url string, err error) { + defer func() { + err = + s.formatError("reach", err, path) + }() + + pairs = append(pairs, s.defaultPairs.Reach...) + var opt pairStorageReach + + opt, err = s.parsePairStorageReach(pairs) + if err != nil { + return + } + return s.reach(ctx, strings.ReplaceAll(path, "\\", "/"), opt) +} +func (s *Storage) Read(path string, w io.Writer, pairs ...Pair) (n int64, err error) { + ctx := context.Background() + return s.ReadWithContext(ctx, path, w, pairs...) +} +func (s *Storage) ReadWithContext(ctx context.Context, path string, w io.Writer, pairs ...Pair) (n int64, err error) { + defer func() { + err = + s.formatError("read", err, path) + }() + + pairs = append(pairs, s.defaultPairs.Read...) + var opt pairStorageRead + + opt, err = s.parsePairStorageRead(pairs) + if err != nil { + return + } + return s.read(ctx, strings.ReplaceAll(path, "\\", "/"), w, opt) +} +func (s *Storage) Stat(path string, pairs ...Pair) (o *Object, err error) { + ctx := context.Background() + return s.StatWithContext(ctx, path, pairs...) +} +func (s *Storage) StatWithContext(ctx context.Context, path string, pairs ...Pair) (o *Object, err error) { + defer func() { + err = + s.formatError("stat", err, path) + }() + + pairs = append(pairs, s.defaultPairs.Stat...) + var opt pairStorageStat + + opt, err = s.parsePairStorageStat(pairs) + if err != nil { + return + } + return s.stat(ctx, strings.ReplaceAll(path, "\\", "/"), opt) +} +func (s *Storage) Write(path string, r io.Reader, size int64, pairs ...Pair) (n int64, err error) { + ctx := context.Background() + return s.WriteWithContext(ctx, path, r, size, pairs...) +} +func (s *Storage) WriteWithContext(ctx context.Context, path string, r io.Reader, size int64, pairs ...Pair) (n int64, err error) { + defer func() { + err = + s.formatError("write", err, path) + }() + + pairs = append(pairs, s.defaultPairs.Write...) + var opt pairStorageWrite + + opt, err = s.parsePairStorageWrite(pairs) + if err != nil { + return + } + return s.write(ctx, strings.ReplaceAll(path, "\\", "/"), r, size, opt) +} +func (s *Storage) WriteAppend(o *Object, r io.Reader, size int64, pairs ...Pair) (n int64, err error) { + ctx := context.Background() + return s.WriteAppendWithContext(ctx, o, r, size, pairs...) +} +func (s *Storage) WriteAppendWithContext(ctx context.Context, o *Object, r io.Reader, size int64, pairs ...Pair) (n int64, err error) { + defer func() { + err = + s.formatError("write_append", err) + }() + if !o.Mode.IsAppend() { + err = services.ObjectModeInvalidError{Expected: ModeAppend, Actual: o.Mode} + return + } + pairs = append(pairs, s.defaultPairs.WriteAppend...) + var opt pairStorageWriteAppend + + opt, err = s.parsePairStorageWriteAppend(pairs) + if err != nil { + return + } + return s.writeAppend(ctx, o, r, size, opt) +} +func (s *Storage) WriteMultipart(o *Object, r io.Reader, size int64, index int, pairs ...Pair) (n int64, part *Part, err error) { + ctx := context.Background() + return s.WriteMultipartWithContext(ctx, o, r, size, index, pairs...) +} +func (s *Storage) WriteMultipartWithContext(ctx context.Context, o *Object, r io.Reader, size int64, index int, pairs ...Pair) (n int64, part *Part, err error) { + defer func() { + err = + s.formatError("write_multipart", err) + }() + if !o.Mode.IsPart() { + err = services.ObjectModeInvalidError{Expected: ModePart, Actual: o.Mode} + return + } + pairs = append(pairs, s.defaultPairs.WriteMultipart...) + var opt pairStorageWriteMultipart + + opt, err = s.parsePairStorageWriteMultipart(pairs) + if err != nil { + return + } + return s.writeMultipart(ctx, o, r, size, index, opt) +} +func init() { + services.RegisterServicer(Type, NewServicer) + services.RegisterStorager(Type, NewStorager) + services.RegisterSchema(Type, pairMap) +} diff --git a/services/qingstor/go.mod b/services/qingstor/go.mod new file mode 100644 index 000000000..19e65e112 --- /dev/null +++ b/services/qingstor/go.mod @@ -0,0 +1,15 @@ +module go.beyondstorage.io/services/qingstor/v4 + +go 1.16 + +require ( + bou.ke/monkey v1.0.2 + github.com/golang/mock v1.6.0 + github.com/google/uuid v1.3.0 + github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 + github.com/qingstor/qingstor-sdk-go/v4 v4.4.0 + github.com/stretchr/testify v1.7.0 + go.beyondstorage.io/credential v1.0.0 + go.beyondstorage.io/endpoint v1.2.0 + go.beyondstorage.io/v5 v5.0.0 +) diff --git a/services/qingstor/go.sum b/services/qingstor/go.sum new file mode 100644 index 000000000..9e839543e --- /dev/null +++ b/services/qingstor/go.sum @@ -0,0 +1,137 @@ +bou.ke/monkey v1.0.2 h1:kWcnsrCNUatbxncxR/ThdYqbytgOIArtYWqcQLQzKLI= +bou.ke/monkey v1.0.2/go.mod h1:OqickVX3tNx6t33n1xvtTtu85YN5s6cKwVug+oHMaIA= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Xuanwo/gg v0.2.0 h1:axbZmA0qmidh3s9PA86GqvBXVQ3o7Bbpf0aImGtlimA= +github.com/Xuanwo/gg v0.2.0/go.mod h1:0fLiiSxR87u2UA0ZNZiKZXuz3jnJdbDHWtU2xpdcH3s= +github.com/Xuanwo/go-bufferpool v0.2.0 h1:DXzqJD9lJufXbT/03GrcEvYOs4gXYUj9/g5yi6Q9rUw= +github.com/Xuanwo/go-bufferpool v0.2.0/go.mod h1:Mle++9GGouhOwGj52i9PJLNAPmW2nb8PWBP7JJzNCzk= +github.com/Xuanwo/templateutils v0.1.0 h1:WpkWOqQtIQ2vAIpJLa727DdN8WtxhUkkbDGa6UhntJY= +github.com/Xuanwo/templateutils v0.1.0/go.mod h1:OdE0DJ+CJxDBq6psX5DPV+gOZi8bhuHuVUpPCG++Wb8= +github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/dave/dst v0.26.2 h1:lnxLAKI3tx7MgLNVDirFCsDTlTG9nKTk7GcptKcWSwY= +github.com/dave/dst v0.26.2/go.mod h1:UMDJuIRPfyUCC78eFuB+SV/WI8oDeyFDvM/JR6NI3IU= +github.com/dave/gopackages v0.0.0-20170318123100-46e7023ec56e/go.mod h1:i00+b/gKdIDIxuLDFob7ustLAVqhsZRk2qVZrArELGQ= +github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= +github.com/dave/kerr v0.0.0-20170318121727-bc25dd6abe8e/go.mod h1:qZqlPyPvfsDJt+3wHJ1EvSXDuVjFTK0j2p/ca+gtsb8= +github.com/dave/rebecca v0.9.1/go.mod h1:N6XYdMD/OKw3lkF3ywh8Z6wPGuwNFDNtWYEMFWEmXBA= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= +github.com/google/pprof v0.0.0-20181127221834-b4f47329b966/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/kevinburke/go-bindata v3.22.0+incompatible h1:/JmqEhIWQ7GRScV0WjX/0tqBrC5D21ALg0H0U/KZ/ts= +github.com/kevinburke/go-bindata v3.22.0+incompatible/go.mod h1:/pEEZ72flUW2p0yi30bslSp9YqD9pysLxunQDdb2CPM= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= +github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 h1:XeOYlK9W1uCmhjJSsY78Mcuh7MVkNjTzmHx1yBzizSU= +github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14/go.mod h1:jVblp62SafmidSkvWrXyxAme3gaTfEtWwRPGz5cpvHg= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/qingstor/qingstor-sdk-go/v4 v4.4.0 h1:tbItWtGB1TDfYzqK8dtm6tV+xWU5iYMwL37C6AL5dDs= +github.com/qingstor/qingstor-sdk-go/v4 v4.4.0/go.mod h1:mDVFtA7+bXQ5xoELTWkoFy1Ad13wtp8jtlnl/RU+zzM= +github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.6 h1:lH+Snxmzl92r1jww8/jYPqKkhs3C9AF4LunzU56ZZr4= +github.com/smartystreets/goconvey v1.6.6/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M= +github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.beyondstorage.io/credential v1.0.0 h1:xJ7hBXmeUE0+rbW+RYZSz4KgHpXvc9g7oQ56f8dXdBk= +go.beyondstorage.io/credential v1.0.0/go.mod h1:7KAYievVw4a8u/eLZmnQt65Z91n84sMQj3LFbt8Xous= +go.beyondstorage.io/endpoint v1.2.0 h1:/7mgKquTykeqJ9op82hso2+WQfECeywGd/Lda1N3tF4= +go.beyondstorage.io/endpoint v1.2.0/go.mod h1:oZ7Z7HZ7mAo337JBLjuCF/DM66HVEUu6+hw68c3UcLs= +go.beyondstorage.io/v5 v5.0.0 h1:k9Axfgbt+oZXoDwSBVCl1XANHSL4rkNTGP2Lz9YdJe0= +go.beyondstorage.io/v5 v5.0.0/go.mod h1:3wV9gCQnqu7tD/3LMeo2yimUKIeTSHpTc6wHSb0yY20= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= +go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/zap v1.19.0 h1:mZQZefskPPCMIBCSEH0v2/iUqqLrYtaeqwD6FUGUnFE= +go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= +golang.org/x/arch v0.0.0-20180920145803-b19384d3c130/go.mod h1:cYlCBUl1MsqxdiKgmc4uh7TxZfWSFLOGSRR090WDxt8= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180903190138-2b024373dcd9/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200509030707-2212a7e161a5/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.1.1 h1:wGiQel/hW0NnEkJUk8lbzkX2gFJU6PFxf1v5OlCfuOs= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/src-d/go-billy.v4 v4.3.0/go.mod h1:tm33zBoOwxjYHZIE+OV8bxTWFMJLrconzFMd38aARFk= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/services/qingstor/iterator.go b/services/qingstor/iterator.go new file mode 100644 index 000000000..5e7a7f5cd --- /dev/null +++ b/services/qingstor/iterator.go @@ -0,0 +1,41 @@ +package qingstor + +import ( + "strconv" +) + +type objectPageStatus struct { + delimiter string + limit int + marker string + prefix string + partIdMarker string +} + +func (i *objectPageStatus) ContinuationToken() string { + if i.partIdMarker != "" { + return i.marker + "/" + i.partIdMarker + } + return i.marker +} + +type storagePageStatus struct { + limit int + offset int + location string +} + +func (i *storagePageStatus) ContinuationToken() string { + return strconv.FormatInt(int64(i.offset), 10) +} + +type partPageStatus struct { + prefix string + limit int + partNumberMarker int + uploadID string +} + +func (i *partPageStatus) ContinuationToken() string { + return strconv.FormatInt(int64(i.partNumberMarker), 10) +} diff --git a/services/qingstor/mock_test.go b/services/qingstor/mock_test.go new file mode 100644 index 000000000..0a89529cc --- /dev/null +++ b/services/qingstor/mock_test.go @@ -0,0 +1,1454 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/qingstor/qingstor-sdk-go/v4/interface (interfaces: Service,Bucket) + +// Package qingstor is a generated GoMock package. +package qingstor + +import ( + context "context" + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + service "github.com/qingstor/qingstor-sdk-go/v4/service" +) + +// MockService is a mock of Service interface. +type MockService struct { + ctrl *gomock.Controller + recorder *MockServiceMockRecorder +} + +// MockServiceMockRecorder is the mock recorder for MockService. +type MockServiceMockRecorder struct { + mock *MockService +} + +// NewMockService creates a new mock instance. +func NewMockService(ctrl *gomock.Controller) *MockService { + mock := &MockService{ctrl: ctrl} + mock.recorder = &MockServiceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockService) EXPECT() *MockServiceMockRecorder { + return m.recorder +} + +// Bucket mocks base method. +func (m *MockService) Bucket(arg0, arg1 string) (*service.Bucket, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Bucket", arg0, arg1) + ret0, _ := ret[0].(*service.Bucket) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Bucket indicates an expected call of Bucket. +func (mr *MockServiceMockRecorder) Bucket(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Bucket", reflect.TypeOf((*MockService)(nil).Bucket), arg0, arg1) +} + +// ListBuckets mocks base method. +func (m *MockService) ListBuckets(arg0 *service.ListBucketsInput) (*service.ListBucketsOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListBuckets", arg0) + ret0, _ := ret[0].(*service.ListBucketsOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListBuckets indicates an expected call of ListBuckets. +func (mr *MockServiceMockRecorder) ListBuckets(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListBuckets", reflect.TypeOf((*MockService)(nil).ListBuckets), arg0) +} + +// ListBucketsWithContext mocks base method. +func (m *MockService) ListBucketsWithContext(arg0 context.Context, arg1 *service.ListBucketsInput) (*service.ListBucketsOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListBucketsWithContext", arg0, arg1) + ret0, _ := ret[0].(*service.ListBucketsOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListBucketsWithContext indicates an expected call of ListBucketsWithContext. +func (mr *MockServiceMockRecorder) ListBucketsWithContext(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListBucketsWithContext", reflect.TypeOf((*MockService)(nil).ListBucketsWithContext), arg0, arg1) +} + +// MockBucket is a mock of Bucket interface. +type MockBucket struct { + ctrl *gomock.Controller + recorder *MockBucketMockRecorder +} + +// MockBucketMockRecorder is the mock recorder for MockBucket. +type MockBucketMockRecorder struct { + mock *MockBucket +} + +// NewMockBucket creates a new mock instance. +func NewMockBucket(ctrl *gomock.Controller) *MockBucket { + mock := &MockBucket{ctrl: ctrl} + mock.recorder = &MockBucketMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockBucket) EXPECT() *MockBucketMockRecorder { + return m.recorder +} + +// AbortMultipartUpload mocks base method. +func (m *MockBucket) AbortMultipartUpload(arg0 string, arg1 *service.AbortMultipartUploadInput) (*service.AbortMultipartUploadOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AbortMultipartUpload", arg0, arg1) + ret0, _ := ret[0].(*service.AbortMultipartUploadOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// AbortMultipartUpload indicates an expected call of AbortMultipartUpload. +func (mr *MockBucketMockRecorder) AbortMultipartUpload(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AbortMultipartUpload", reflect.TypeOf((*MockBucket)(nil).AbortMultipartUpload), arg0, arg1) +} + +// AbortMultipartUploadWithContext mocks base method. +func (m *MockBucket) AbortMultipartUploadWithContext(arg0 context.Context, arg1 string, arg2 *service.AbortMultipartUploadInput) (*service.AbortMultipartUploadOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AbortMultipartUploadWithContext", arg0, arg1, arg2) + ret0, _ := ret[0].(*service.AbortMultipartUploadOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// AbortMultipartUploadWithContext indicates an expected call of AbortMultipartUploadWithContext. +func (mr *MockBucketMockRecorder) AbortMultipartUploadWithContext(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AbortMultipartUploadWithContext", reflect.TypeOf((*MockBucket)(nil).AbortMultipartUploadWithContext), arg0, arg1, arg2) +} + +// AppendObject mocks base method. +func (m *MockBucket) AppendObject(arg0 string, arg1 *service.AppendObjectInput) (*service.AppendObjectOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AppendObject", arg0, arg1) + ret0, _ := ret[0].(*service.AppendObjectOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// AppendObject indicates an expected call of AppendObject. +func (mr *MockBucketMockRecorder) AppendObject(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AppendObject", reflect.TypeOf((*MockBucket)(nil).AppendObject), arg0, arg1) +} + +// AppendObjectWithContext mocks base method. +func (m *MockBucket) AppendObjectWithContext(arg0 context.Context, arg1 string, arg2 *service.AppendObjectInput) (*service.AppendObjectOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AppendObjectWithContext", arg0, arg1, arg2) + ret0, _ := ret[0].(*service.AppendObjectOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// AppendObjectWithContext indicates an expected call of AppendObjectWithContext. +func (mr *MockBucketMockRecorder) AppendObjectWithContext(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AppendObjectWithContext", reflect.TypeOf((*MockBucket)(nil).AppendObjectWithContext), arg0, arg1, arg2) +} + +// CompleteMultipartUpload mocks base method. +func (m *MockBucket) CompleteMultipartUpload(arg0 string, arg1 *service.CompleteMultipartUploadInput) (*service.CompleteMultipartUploadOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CompleteMultipartUpload", arg0, arg1) + ret0, _ := ret[0].(*service.CompleteMultipartUploadOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CompleteMultipartUpload indicates an expected call of CompleteMultipartUpload. +func (mr *MockBucketMockRecorder) CompleteMultipartUpload(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CompleteMultipartUpload", reflect.TypeOf((*MockBucket)(nil).CompleteMultipartUpload), arg0, arg1) +} + +// CompleteMultipartUploadWithContext mocks base method. +func (m *MockBucket) CompleteMultipartUploadWithContext(arg0 context.Context, arg1 string, arg2 *service.CompleteMultipartUploadInput) (*service.CompleteMultipartUploadOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CompleteMultipartUploadWithContext", arg0, arg1, arg2) + ret0, _ := ret[0].(*service.CompleteMultipartUploadOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CompleteMultipartUploadWithContext indicates an expected call of CompleteMultipartUploadWithContext. +func (mr *MockBucketMockRecorder) CompleteMultipartUploadWithContext(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CompleteMultipartUploadWithContext", reflect.TypeOf((*MockBucket)(nil).CompleteMultipartUploadWithContext), arg0, arg1, arg2) +} + +// Delete mocks base method. +func (m *MockBucket) Delete() (*service.DeleteBucketOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Delete") + ret0, _ := ret[0].(*service.DeleteBucketOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Delete indicates an expected call of Delete. +func (mr *MockBucketMockRecorder) Delete() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockBucket)(nil).Delete)) +} + +// DeleteCNAME mocks base method. +func (m *MockBucket) DeleteCNAME(arg0 *service.DeleteBucketCNAMEInput) (*service.DeleteBucketCNAMEOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteCNAME", arg0) + ret0, _ := ret[0].(*service.DeleteBucketCNAMEOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeleteCNAME indicates an expected call of DeleteCNAME. +func (mr *MockBucketMockRecorder) DeleteCNAME(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteCNAME", reflect.TypeOf((*MockBucket)(nil).DeleteCNAME), arg0) +} + +// DeleteCNAMEWithContext mocks base method. +func (m *MockBucket) DeleteCNAMEWithContext(arg0 context.Context, arg1 *service.DeleteBucketCNAMEInput) (*service.DeleteBucketCNAMEOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteCNAMEWithContext", arg0, arg1) + ret0, _ := ret[0].(*service.DeleteBucketCNAMEOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeleteCNAMEWithContext indicates an expected call of DeleteCNAMEWithContext. +func (mr *MockBucketMockRecorder) DeleteCNAMEWithContext(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteCNAMEWithContext", reflect.TypeOf((*MockBucket)(nil).DeleteCNAMEWithContext), arg0, arg1) +} + +// DeleteCORS mocks base method. +func (m *MockBucket) DeleteCORS() (*service.DeleteBucketCORSOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteCORS") + ret0, _ := ret[0].(*service.DeleteBucketCORSOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeleteCORS indicates an expected call of DeleteCORS. +func (mr *MockBucketMockRecorder) DeleteCORS() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteCORS", reflect.TypeOf((*MockBucket)(nil).DeleteCORS)) +} + +// DeleteCORSWithContext mocks base method. +func (m *MockBucket) DeleteCORSWithContext(arg0 context.Context) (*service.DeleteBucketCORSOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteCORSWithContext", arg0) + ret0, _ := ret[0].(*service.DeleteBucketCORSOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeleteCORSWithContext indicates an expected call of DeleteCORSWithContext. +func (mr *MockBucketMockRecorder) DeleteCORSWithContext(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteCORSWithContext", reflect.TypeOf((*MockBucket)(nil).DeleteCORSWithContext), arg0) +} + +// DeleteExternalMirror mocks base method. +func (m *MockBucket) DeleteExternalMirror() (*service.DeleteBucketExternalMirrorOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteExternalMirror") + ret0, _ := ret[0].(*service.DeleteBucketExternalMirrorOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeleteExternalMirror indicates an expected call of DeleteExternalMirror. +func (mr *MockBucketMockRecorder) DeleteExternalMirror() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteExternalMirror", reflect.TypeOf((*MockBucket)(nil).DeleteExternalMirror)) +} + +// DeleteExternalMirrorWithContext mocks base method. +func (m *MockBucket) DeleteExternalMirrorWithContext(arg0 context.Context) (*service.DeleteBucketExternalMirrorOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteExternalMirrorWithContext", arg0) + ret0, _ := ret[0].(*service.DeleteBucketExternalMirrorOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeleteExternalMirrorWithContext indicates an expected call of DeleteExternalMirrorWithContext. +func (mr *MockBucketMockRecorder) DeleteExternalMirrorWithContext(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteExternalMirrorWithContext", reflect.TypeOf((*MockBucket)(nil).DeleteExternalMirrorWithContext), arg0) +} + +// DeleteLifecycle mocks base method. +func (m *MockBucket) DeleteLifecycle() (*service.DeleteBucketLifecycleOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteLifecycle") + ret0, _ := ret[0].(*service.DeleteBucketLifecycleOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeleteLifecycle indicates an expected call of DeleteLifecycle. +func (mr *MockBucketMockRecorder) DeleteLifecycle() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteLifecycle", reflect.TypeOf((*MockBucket)(nil).DeleteLifecycle)) +} + +// DeleteLifecycleWithContext mocks base method. +func (m *MockBucket) DeleteLifecycleWithContext(arg0 context.Context) (*service.DeleteBucketLifecycleOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteLifecycleWithContext", arg0) + ret0, _ := ret[0].(*service.DeleteBucketLifecycleOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeleteLifecycleWithContext indicates an expected call of DeleteLifecycleWithContext. +func (mr *MockBucketMockRecorder) DeleteLifecycleWithContext(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteLifecycleWithContext", reflect.TypeOf((*MockBucket)(nil).DeleteLifecycleWithContext), arg0) +} + +// DeleteLogging mocks base method. +func (m *MockBucket) DeleteLogging() (*service.DeleteBucketLoggingOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteLogging") + ret0, _ := ret[0].(*service.DeleteBucketLoggingOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeleteLogging indicates an expected call of DeleteLogging. +func (mr *MockBucketMockRecorder) DeleteLogging() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteLogging", reflect.TypeOf((*MockBucket)(nil).DeleteLogging)) +} + +// DeleteLoggingWithContext mocks base method. +func (m *MockBucket) DeleteLoggingWithContext(arg0 context.Context) (*service.DeleteBucketLoggingOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteLoggingWithContext", arg0) + ret0, _ := ret[0].(*service.DeleteBucketLoggingOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeleteLoggingWithContext indicates an expected call of DeleteLoggingWithContext. +func (mr *MockBucketMockRecorder) DeleteLoggingWithContext(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteLoggingWithContext", reflect.TypeOf((*MockBucket)(nil).DeleteLoggingWithContext), arg0) +} + +// DeleteMultipleObjects mocks base method. +func (m *MockBucket) DeleteMultipleObjects(arg0 *service.DeleteMultipleObjectsInput) (*service.DeleteMultipleObjectsOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteMultipleObjects", arg0) + ret0, _ := ret[0].(*service.DeleteMultipleObjectsOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeleteMultipleObjects indicates an expected call of DeleteMultipleObjects. +func (mr *MockBucketMockRecorder) DeleteMultipleObjects(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteMultipleObjects", reflect.TypeOf((*MockBucket)(nil).DeleteMultipleObjects), arg0) +} + +// DeleteMultipleObjectsWithContext mocks base method. +func (m *MockBucket) DeleteMultipleObjectsWithContext(arg0 context.Context, arg1 *service.DeleteMultipleObjectsInput) (*service.DeleteMultipleObjectsOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteMultipleObjectsWithContext", arg0, arg1) + ret0, _ := ret[0].(*service.DeleteMultipleObjectsOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeleteMultipleObjectsWithContext indicates an expected call of DeleteMultipleObjectsWithContext. +func (mr *MockBucketMockRecorder) DeleteMultipleObjectsWithContext(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteMultipleObjectsWithContext", reflect.TypeOf((*MockBucket)(nil).DeleteMultipleObjectsWithContext), arg0, arg1) +} + +// DeleteNotification mocks base method. +func (m *MockBucket) DeleteNotification() (*service.DeleteBucketNotificationOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteNotification") + ret0, _ := ret[0].(*service.DeleteBucketNotificationOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeleteNotification indicates an expected call of DeleteNotification. +func (mr *MockBucketMockRecorder) DeleteNotification() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteNotification", reflect.TypeOf((*MockBucket)(nil).DeleteNotification)) +} + +// DeleteNotificationWithContext mocks base method. +func (m *MockBucket) DeleteNotificationWithContext(arg0 context.Context) (*service.DeleteBucketNotificationOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteNotificationWithContext", arg0) + ret0, _ := ret[0].(*service.DeleteBucketNotificationOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeleteNotificationWithContext indicates an expected call of DeleteNotificationWithContext. +func (mr *MockBucketMockRecorder) DeleteNotificationWithContext(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteNotificationWithContext", reflect.TypeOf((*MockBucket)(nil).DeleteNotificationWithContext), arg0) +} + +// DeleteObject mocks base method. +func (m *MockBucket) DeleteObject(arg0 string) (*service.DeleteObjectOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteObject", arg0) + ret0, _ := ret[0].(*service.DeleteObjectOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeleteObject indicates an expected call of DeleteObject. +func (mr *MockBucketMockRecorder) DeleteObject(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteObject", reflect.TypeOf((*MockBucket)(nil).DeleteObject), arg0) +} + +// DeleteObjectWithContext mocks base method. +func (m *MockBucket) DeleteObjectWithContext(arg0 context.Context, arg1 string) (*service.DeleteObjectOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteObjectWithContext", arg0, arg1) + ret0, _ := ret[0].(*service.DeleteObjectOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeleteObjectWithContext indicates an expected call of DeleteObjectWithContext. +func (mr *MockBucketMockRecorder) DeleteObjectWithContext(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteObjectWithContext", reflect.TypeOf((*MockBucket)(nil).DeleteObjectWithContext), arg0, arg1) +} + +// DeletePolicy mocks base method. +func (m *MockBucket) DeletePolicy() (*service.DeleteBucketPolicyOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeletePolicy") + ret0, _ := ret[0].(*service.DeleteBucketPolicyOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeletePolicy indicates an expected call of DeletePolicy. +func (mr *MockBucketMockRecorder) DeletePolicy() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeletePolicy", reflect.TypeOf((*MockBucket)(nil).DeletePolicy)) +} + +// DeletePolicyWithContext mocks base method. +func (m *MockBucket) DeletePolicyWithContext(arg0 context.Context) (*service.DeleteBucketPolicyOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeletePolicyWithContext", arg0) + ret0, _ := ret[0].(*service.DeleteBucketPolicyOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeletePolicyWithContext indicates an expected call of DeletePolicyWithContext. +func (mr *MockBucketMockRecorder) DeletePolicyWithContext(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeletePolicyWithContext", reflect.TypeOf((*MockBucket)(nil).DeletePolicyWithContext), arg0) +} + +// DeleteReplication mocks base method. +func (m *MockBucket) DeleteReplication() (*service.DeleteBucketReplicationOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteReplication") + ret0, _ := ret[0].(*service.DeleteBucketReplicationOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeleteReplication indicates an expected call of DeleteReplication. +func (mr *MockBucketMockRecorder) DeleteReplication() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteReplication", reflect.TypeOf((*MockBucket)(nil).DeleteReplication)) +} + +// DeleteReplicationWithContext mocks base method. +func (m *MockBucket) DeleteReplicationWithContext(arg0 context.Context) (*service.DeleteBucketReplicationOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteReplicationWithContext", arg0) + ret0, _ := ret[0].(*service.DeleteBucketReplicationOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeleteReplicationWithContext indicates an expected call of DeleteReplicationWithContext. +func (mr *MockBucketMockRecorder) DeleteReplicationWithContext(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteReplicationWithContext", reflect.TypeOf((*MockBucket)(nil).DeleteReplicationWithContext), arg0) +} + +// DeleteWithContext mocks base method. +func (m *MockBucket) DeleteWithContext(arg0 context.Context) (*service.DeleteBucketOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteWithContext", arg0) + ret0, _ := ret[0].(*service.DeleteBucketOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeleteWithContext indicates an expected call of DeleteWithContext. +func (mr *MockBucketMockRecorder) DeleteWithContext(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteWithContext", reflect.TypeOf((*MockBucket)(nil).DeleteWithContext), arg0) +} + +// GetACL mocks base method. +func (m *MockBucket) GetACL() (*service.GetBucketACLOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetACL") + ret0, _ := ret[0].(*service.GetBucketACLOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetACL indicates an expected call of GetACL. +func (mr *MockBucketMockRecorder) GetACL() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetACL", reflect.TypeOf((*MockBucket)(nil).GetACL)) +} + +// GetACLWithContext mocks base method. +func (m *MockBucket) GetACLWithContext(arg0 context.Context) (*service.GetBucketACLOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetACLWithContext", arg0) + ret0, _ := ret[0].(*service.GetBucketACLOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetACLWithContext indicates an expected call of GetACLWithContext. +func (mr *MockBucketMockRecorder) GetACLWithContext(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetACLWithContext", reflect.TypeOf((*MockBucket)(nil).GetACLWithContext), arg0) +} + +// GetCNAME mocks base method. +func (m *MockBucket) GetCNAME(arg0 *service.GetBucketCNAMEInput) (*service.GetBucketCNAMEOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetCNAME", arg0) + ret0, _ := ret[0].(*service.GetBucketCNAMEOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetCNAME indicates an expected call of GetCNAME. +func (mr *MockBucketMockRecorder) GetCNAME(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCNAME", reflect.TypeOf((*MockBucket)(nil).GetCNAME), arg0) +} + +// GetCNAMEWithContext mocks base method. +func (m *MockBucket) GetCNAMEWithContext(arg0 context.Context, arg1 *service.GetBucketCNAMEInput) (*service.GetBucketCNAMEOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetCNAMEWithContext", arg0, arg1) + ret0, _ := ret[0].(*service.GetBucketCNAMEOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetCNAMEWithContext indicates an expected call of GetCNAMEWithContext. +func (mr *MockBucketMockRecorder) GetCNAMEWithContext(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCNAMEWithContext", reflect.TypeOf((*MockBucket)(nil).GetCNAMEWithContext), arg0, arg1) +} + +// GetCORS mocks base method. +func (m *MockBucket) GetCORS() (*service.GetBucketCORSOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetCORS") + ret0, _ := ret[0].(*service.GetBucketCORSOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetCORS indicates an expected call of GetCORS. +func (mr *MockBucketMockRecorder) GetCORS() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCORS", reflect.TypeOf((*MockBucket)(nil).GetCORS)) +} + +// GetCORSWithContext mocks base method. +func (m *MockBucket) GetCORSWithContext(arg0 context.Context) (*service.GetBucketCORSOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetCORSWithContext", arg0) + ret0, _ := ret[0].(*service.GetBucketCORSOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetCORSWithContext indicates an expected call of GetCORSWithContext. +func (mr *MockBucketMockRecorder) GetCORSWithContext(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCORSWithContext", reflect.TypeOf((*MockBucket)(nil).GetCORSWithContext), arg0) +} + +// GetExternalMirror mocks base method. +func (m *MockBucket) GetExternalMirror() (*service.GetBucketExternalMirrorOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetExternalMirror") + ret0, _ := ret[0].(*service.GetBucketExternalMirrorOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetExternalMirror indicates an expected call of GetExternalMirror. +func (mr *MockBucketMockRecorder) GetExternalMirror() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetExternalMirror", reflect.TypeOf((*MockBucket)(nil).GetExternalMirror)) +} + +// GetExternalMirrorWithContext mocks base method. +func (m *MockBucket) GetExternalMirrorWithContext(arg0 context.Context) (*service.GetBucketExternalMirrorOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetExternalMirrorWithContext", arg0) + ret0, _ := ret[0].(*service.GetBucketExternalMirrorOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetExternalMirrorWithContext indicates an expected call of GetExternalMirrorWithContext. +func (mr *MockBucketMockRecorder) GetExternalMirrorWithContext(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetExternalMirrorWithContext", reflect.TypeOf((*MockBucket)(nil).GetExternalMirrorWithContext), arg0) +} + +// GetLifecycle mocks base method. +func (m *MockBucket) GetLifecycle() (*service.GetBucketLifecycleOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetLifecycle") + ret0, _ := ret[0].(*service.GetBucketLifecycleOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetLifecycle indicates an expected call of GetLifecycle. +func (mr *MockBucketMockRecorder) GetLifecycle() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLifecycle", reflect.TypeOf((*MockBucket)(nil).GetLifecycle)) +} + +// GetLifecycleWithContext mocks base method. +func (m *MockBucket) GetLifecycleWithContext(arg0 context.Context) (*service.GetBucketLifecycleOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetLifecycleWithContext", arg0) + ret0, _ := ret[0].(*service.GetBucketLifecycleOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetLifecycleWithContext indicates an expected call of GetLifecycleWithContext. +func (mr *MockBucketMockRecorder) GetLifecycleWithContext(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLifecycleWithContext", reflect.TypeOf((*MockBucket)(nil).GetLifecycleWithContext), arg0) +} + +// GetLogging mocks base method. +func (m *MockBucket) GetLogging() (*service.GetBucketLoggingOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetLogging") + ret0, _ := ret[0].(*service.GetBucketLoggingOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetLogging indicates an expected call of GetLogging. +func (mr *MockBucketMockRecorder) GetLogging() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLogging", reflect.TypeOf((*MockBucket)(nil).GetLogging)) +} + +// GetLoggingWithContext mocks base method. +func (m *MockBucket) GetLoggingWithContext(arg0 context.Context) (*service.GetBucketLoggingOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetLoggingWithContext", arg0) + ret0, _ := ret[0].(*service.GetBucketLoggingOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetLoggingWithContext indicates an expected call of GetLoggingWithContext. +func (mr *MockBucketMockRecorder) GetLoggingWithContext(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLoggingWithContext", reflect.TypeOf((*MockBucket)(nil).GetLoggingWithContext), arg0) +} + +// GetNotification mocks base method. +func (m *MockBucket) GetNotification() (*service.GetBucketNotificationOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetNotification") + ret0, _ := ret[0].(*service.GetBucketNotificationOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetNotification indicates an expected call of GetNotification. +func (mr *MockBucketMockRecorder) GetNotification() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetNotification", reflect.TypeOf((*MockBucket)(nil).GetNotification)) +} + +// GetNotificationWithContext mocks base method. +func (m *MockBucket) GetNotificationWithContext(arg0 context.Context) (*service.GetBucketNotificationOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetNotificationWithContext", arg0) + ret0, _ := ret[0].(*service.GetBucketNotificationOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetNotificationWithContext indicates an expected call of GetNotificationWithContext. +func (mr *MockBucketMockRecorder) GetNotificationWithContext(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetNotificationWithContext", reflect.TypeOf((*MockBucket)(nil).GetNotificationWithContext), arg0) +} + +// GetObject mocks base method. +func (m *MockBucket) GetObject(arg0 string, arg1 *service.GetObjectInput) (*service.GetObjectOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetObject", arg0, arg1) + ret0, _ := ret[0].(*service.GetObjectOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetObject indicates an expected call of GetObject. +func (mr *MockBucketMockRecorder) GetObject(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetObject", reflect.TypeOf((*MockBucket)(nil).GetObject), arg0, arg1) +} + +// GetObjectWithContext mocks base method. +func (m *MockBucket) GetObjectWithContext(arg0 context.Context, arg1 string, arg2 *service.GetObjectInput) (*service.GetObjectOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetObjectWithContext", arg0, arg1, arg2) + ret0, _ := ret[0].(*service.GetObjectOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetObjectWithContext indicates an expected call of GetObjectWithContext. +func (mr *MockBucketMockRecorder) GetObjectWithContext(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetObjectWithContext", reflect.TypeOf((*MockBucket)(nil).GetObjectWithContext), arg0, arg1, arg2) +} + +// GetPolicy mocks base method. +func (m *MockBucket) GetPolicy() (*service.GetBucketPolicyOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetPolicy") + ret0, _ := ret[0].(*service.GetBucketPolicyOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetPolicy indicates an expected call of GetPolicy. +func (mr *MockBucketMockRecorder) GetPolicy() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPolicy", reflect.TypeOf((*MockBucket)(nil).GetPolicy)) +} + +// GetPolicyWithContext mocks base method. +func (m *MockBucket) GetPolicyWithContext(arg0 context.Context) (*service.GetBucketPolicyOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetPolicyWithContext", arg0) + ret0, _ := ret[0].(*service.GetBucketPolicyOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetPolicyWithContext indicates an expected call of GetPolicyWithContext. +func (mr *MockBucketMockRecorder) GetPolicyWithContext(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPolicyWithContext", reflect.TypeOf((*MockBucket)(nil).GetPolicyWithContext), arg0) +} + +// GetReplication mocks base method. +func (m *MockBucket) GetReplication() (*service.GetBucketReplicationOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetReplication") + ret0, _ := ret[0].(*service.GetBucketReplicationOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetReplication indicates an expected call of GetReplication. +func (mr *MockBucketMockRecorder) GetReplication() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetReplication", reflect.TypeOf((*MockBucket)(nil).GetReplication)) +} + +// GetReplicationWithContext mocks base method. +func (m *MockBucket) GetReplicationWithContext(arg0 context.Context) (*service.GetBucketReplicationOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetReplicationWithContext", arg0) + ret0, _ := ret[0].(*service.GetBucketReplicationOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetReplicationWithContext indicates an expected call of GetReplicationWithContext. +func (mr *MockBucketMockRecorder) GetReplicationWithContext(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetReplicationWithContext", reflect.TypeOf((*MockBucket)(nil).GetReplicationWithContext), arg0) +} + +// GetStatistics mocks base method. +func (m *MockBucket) GetStatistics() (*service.GetBucketStatisticsOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetStatistics") + ret0, _ := ret[0].(*service.GetBucketStatisticsOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetStatistics indicates an expected call of GetStatistics. +func (mr *MockBucketMockRecorder) GetStatistics() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetStatistics", reflect.TypeOf((*MockBucket)(nil).GetStatistics)) +} + +// GetStatisticsWithContext mocks base method. +func (m *MockBucket) GetStatisticsWithContext(arg0 context.Context) (*service.GetBucketStatisticsOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetStatisticsWithContext", arg0) + ret0, _ := ret[0].(*service.GetBucketStatisticsOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetStatisticsWithContext indicates an expected call of GetStatisticsWithContext. +func (mr *MockBucketMockRecorder) GetStatisticsWithContext(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetStatisticsWithContext", reflect.TypeOf((*MockBucket)(nil).GetStatisticsWithContext), arg0) +} + +// Head mocks base method. +func (m *MockBucket) Head() (*service.HeadBucketOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Head") + ret0, _ := ret[0].(*service.HeadBucketOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Head indicates an expected call of Head. +func (mr *MockBucketMockRecorder) Head() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Head", reflect.TypeOf((*MockBucket)(nil).Head)) +} + +// HeadObject mocks base method. +func (m *MockBucket) HeadObject(arg0 string, arg1 *service.HeadObjectInput) (*service.HeadObjectOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "HeadObject", arg0, arg1) + ret0, _ := ret[0].(*service.HeadObjectOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// HeadObject indicates an expected call of HeadObject. +func (mr *MockBucketMockRecorder) HeadObject(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HeadObject", reflect.TypeOf((*MockBucket)(nil).HeadObject), arg0, arg1) +} + +// HeadObjectWithContext mocks base method. +func (m *MockBucket) HeadObjectWithContext(arg0 context.Context, arg1 string, arg2 *service.HeadObjectInput) (*service.HeadObjectOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "HeadObjectWithContext", arg0, arg1, arg2) + ret0, _ := ret[0].(*service.HeadObjectOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// HeadObjectWithContext indicates an expected call of HeadObjectWithContext. +func (mr *MockBucketMockRecorder) HeadObjectWithContext(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HeadObjectWithContext", reflect.TypeOf((*MockBucket)(nil).HeadObjectWithContext), arg0, arg1, arg2) +} + +// HeadWithContext mocks base method. +func (m *MockBucket) HeadWithContext(arg0 context.Context) (*service.HeadBucketOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "HeadWithContext", arg0) + ret0, _ := ret[0].(*service.HeadBucketOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// HeadWithContext indicates an expected call of HeadWithContext. +func (mr *MockBucketMockRecorder) HeadWithContext(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HeadWithContext", reflect.TypeOf((*MockBucket)(nil).HeadWithContext), arg0) +} + +// ImageProcess mocks base method. +func (m *MockBucket) ImageProcess(arg0 string, arg1 *service.ImageProcessInput) (*service.ImageProcessOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ImageProcess", arg0, arg1) + ret0, _ := ret[0].(*service.ImageProcessOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ImageProcess indicates an expected call of ImageProcess. +func (mr *MockBucketMockRecorder) ImageProcess(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ImageProcess", reflect.TypeOf((*MockBucket)(nil).ImageProcess), arg0, arg1) +} + +// ImageProcessWithContext mocks base method. +func (m *MockBucket) ImageProcessWithContext(arg0 context.Context, arg1 string, arg2 *service.ImageProcessInput) (*service.ImageProcessOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ImageProcessWithContext", arg0, arg1, arg2) + ret0, _ := ret[0].(*service.ImageProcessOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ImageProcessWithContext indicates an expected call of ImageProcessWithContext. +func (mr *MockBucketMockRecorder) ImageProcessWithContext(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ImageProcessWithContext", reflect.TypeOf((*MockBucket)(nil).ImageProcessWithContext), arg0, arg1, arg2) +} + +// InitiateMultipartUpload mocks base method. +func (m *MockBucket) InitiateMultipartUpload(arg0 string, arg1 *service.InitiateMultipartUploadInput) (*service.InitiateMultipartUploadOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "InitiateMultipartUpload", arg0, arg1) + ret0, _ := ret[0].(*service.InitiateMultipartUploadOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// InitiateMultipartUpload indicates an expected call of InitiateMultipartUpload. +func (mr *MockBucketMockRecorder) InitiateMultipartUpload(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InitiateMultipartUpload", reflect.TypeOf((*MockBucket)(nil).InitiateMultipartUpload), arg0, arg1) +} + +// InitiateMultipartUploadWithContext mocks base method. +func (m *MockBucket) InitiateMultipartUploadWithContext(arg0 context.Context, arg1 string, arg2 *service.InitiateMultipartUploadInput) (*service.InitiateMultipartUploadOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "InitiateMultipartUploadWithContext", arg0, arg1, arg2) + ret0, _ := ret[0].(*service.InitiateMultipartUploadOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// InitiateMultipartUploadWithContext indicates an expected call of InitiateMultipartUploadWithContext. +func (mr *MockBucketMockRecorder) InitiateMultipartUploadWithContext(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InitiateMultipartUploadWithContext", reflect.TypeOf((*MockBucket)(nil).InitiateMultipartUploadWithContext), arg0, arg1, arg2) +} + +// ListMultipart mocks base method. +func (m *MockBucket) ListMultipart(arg0 string, arg1 *service.ListMultipartInput) (*service.ListMultipartOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListMultipart", arg0, arg1) + ret0, _ := ret[0].(*service.ListMultipartOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListMultipart indicates an expected call of ListMultipart. +func (mr *MockBucketMockRecorder) ListMultipart(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListMultipart", reflect.TypeOf((*MockBucket)(nil).ListMultipart), arg0, arg1) +} + +// ListMultipartUploads mocks base method. +func (m *MockBucket) ListMultipartUploads(arg0 *service.ListMultipartUploadsInput) (*service.ListMultipartUploadsOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListMultipartUploads", arg0) + ret0, _ := ret[0].(*service.ListMultipartUploadsOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListMultipartUploads indicates an expected call of ListMultipartUploads. +func (mr *MockBucketMockRecorder) ListMultipartUploads(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListMultipartUploads", reflect.TypeOf((*MockBucket)(nil).ListMultipartUploads), arg0) +} + +// ListMultipartUploadsWithContext mocks base method. +func (m *MockBucket) ListMultipartUploadsWithContext(arg0 context.Context, arg1 *service.ListMultipartUploadsInput) (*service.ListMultipartUploadsOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListMultipartUploadsWithContext", arg0, arg1) + ret0, _ := ret[0].(*service.ListMultipartUploadsOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListMultipartUploadsWithContext indicates an expected call of ListMultipartUploadsWithContext. +func (mr *MockBucketMockRecorder) ListMultipartUploadsWithContext(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListMultipartUploadsWithContext", reflect.TypeOf((*MockBucket)(nil).ListMultipartUploadsWithContext), arg0, arg1) +} + +// ListMultipartWithContext mocks base method. +func (m *MockBucket) ListMultipartWithContext(arg0 context.Context, arg1 string, arg2 *service.ListMultipartInput) (*service.ListMultipartOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListMultipartWithContext", arg0, arg1, arg2) + ret0, _ := ret[0].(*service.ListMultipartOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListMultipartWithContext indicates an expected call of ListMultipartWithContext. +func (mr *MockBucketMockRecorder) ListMultipartWithContext(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListMultipartWithContext", reflect.TypeOf((*MockBucket)(nil).ListMultipartWithContext), arg0, arg1, arg2) +} + +// ListObjects mocks base method. +func (m *MockBucket) ListObjects(arg0 *service.ListObjectsInput) (*service.ListObjectsOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListObjects", arg0) + ret0, _ := ret[0].(*service.ListObjectsOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListObjects indicates an expected call of ListObjects. +func (mr *MockBucketMockRecorder) ListObjects(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListObjects", reflect.TypeOf((*MockBucket)(nil).ListObjects), arg0) +} + +// ListObjectsWithContext mocks base method. +func (m *MockBucket) ListObjectsWithContext(arg0 context.Context, arg1 *service.ListObjectsInput) (*service.ListObjectsOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListObjectsWithContext", arg0, arg1) + ret0, _ := ret[0].(*service.ListObjectsOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListObjectsWithContext indicates an expected call of ListObjectsWithContext. +func (mr *MockBucketMockRecorder) ListObjectsWithContext(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListObjectsWithContext", reflect.TypeOf((*MockBucket)(nil).ListObjectsWithContext), arg0, arg1) +} + +// OptionsObject mocks base method. +func (m *MockBucket) OptionsObject(arg0 string, arg1 *service.OptionsObjectInput) (*service.OptionsObjectOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "OptionsObject", arg0, arg1) + ret0, _ := ret[0].(*service.OptionsObjectOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// OptionsObject indicates an expected call of OptionsObject. +func (mr *MockBucketMockRecorder) OptionsObject(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OptionsObject", reflect.TypeOf((*MockBucket)(nil).OptionsObject), arg0, arg1) +} + +// OptionsObjectWithContext mocks base method. +func (m *MockBucket) OptionsObjectWithContext(arg0 context.Context, arg1 string, arg2 *service.OptionsObjectInput) (*service.OptionsObjectOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "OptionsObjectWithContext", arg0, arg1, arg2) + ret0, _ := ret[0].(*service.OptionsObjectOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// OptionsObjectWithContext indicates an expected call of OptionsObjectWithContext. +func (mr *MockBucketMockRecorder) OptionsObjectWithContext(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OptionsObjectWithContext", reflect.TypeOf((*MockBucket)(nil).OptionsObjectWithContext), arg0, arg1, arg2) +} + +// Put mocks base method. +func (m *MockBucket) Put() (*service.PutBucketOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Put") + ret0, _ := ret[0].(*service.PutBucketOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Put indicates an expected call of Put. +func (mr *MockBucketMockRecorder) Put() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Put", reflect.TypeOf((*MockBucket)(nil).Put)) +} + +// PutACL mocks base method. +func (m *MockBucket) PutACL(arg0 *service.PutBucketACLInput) (*service.PutBucketACLOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PutACL", arg0) + ret0, _ := ret[0].(*service.PutBucketACLOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// PutACL indicates an expected call of PutACL. +func (mr *MockBucketMockRecorder) PutACL(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PutACL", reflect.TypeOf((*MockBucket)(nil).PutACL), arg0) +} + +// PutACLWithContext mocks base method. +func (m *MockBucket) PutACLWithContext(arg0 context.Context, arg1 *service.PutBucketACLInput) (*service.PutBucketACLOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PutACLWithContext", arg0, arg1) + ret0, _ := ret[0].(*service.PutBucketACLOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// PutACLWithContext indicates an expected call of PutACLWithContext. +func (mr *MockBucketMockRecorder) PutACLWithContext(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PutACLWithContext", reflect.TypeOf((*MockBucket)(nil).PutACLWithContext), arg0, arg1) +} + +// PutCNAME mocks base method. +func (m *MockBucket) PutCNAME(arg0 *service.PutBucketCNAMEInput) (*service.PutBucketCNAMEOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PutCNAME", arg0) + ret0, _ := ret[0].(*service.PutBucketCNAMEOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// PutCNAME indicates an expected call of PutCNAME. +func (mr *MockBucketMockRecorder) PutCNAME(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PutCNAME", reflect.TypeOf((*MockBucket)(nil).PutCNAME), arg0) +} + +// PutCNAMEWithContext mocks base method. +func (m *MockBucket) PutCNAMEWithContext(arg0 context.Context, arg1 *service.PutBucketCNAMEInput) (*service.PutBucketCNAMEOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PutCNAMEWithContext", arg0, arg1) + ret0, _ := ret[0].(*service.PutBucketCNAMEOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// PutCNAMEWithContext indicates an expected call of PutCNAMEWithContext. +func (mr *MockBucketMockRecorder) PutCNAMEWithContext(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PutCNAMEWithContext", reflect.TypeOf((*MockBucket)(nil).PutCNAMEWithContext), arg0, arg1) +} + +// PutCORS mocks base method. +func (m *MockBucket) PutCORS(arg0 *service.PutBucketCORSInput) (*service.PutBucketCORSOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PutCORS", arg0) + ret0, _ := ret[0].(*service.PutBucketCORSOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// PutCORS indicates an expected call of PutCORS. +func (mr *MockBucketMockRecorder) PutCORS(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PutCORS", reflect.TypeOf((*MockBucket)(nil).PutCORS), arg0) +} + +// PutCORSWithContext mocks base method. +func (m *MockBucket) PutCORSWithContext(arg0 context.Context, arg1 *service.PutBucketCORSInput) (*service.PutBucketCORSOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PutCORSWithContext", arg0, arg1) + ret0, _ := ret[0].(*service.PutBucketCORSOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// PutCORSWithContext indicates an expected call of PutCORSWithContext. +func (mr *MockBucketMockRecorder) PutCORSWithContext(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PutCORSWithContext", reflect.TypeOf((*MockBucket)(nil).PutCORSWithContext), arg0, arg1) +} + +// PutExternalMirror mocks base method. +func (m *MockBucket) PutExternalMirror(arg0 *service.PutBucketExternalMirrorInput) (*service.PutBucketExternalMirrorOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PutExternalMirror", arg0) + ret0, _ := ret[0].(*service.PutBucketExternalMirrorOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// PutExternalMirror indicates an expected call of PutExternalMirror. +func (mr *MockBucketMockRecorder) PutExternalMirror(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PutExternalMirror", reflect.TypeOf((*MockBucket)(nil).PutExternalMirror), arg0) +} + +// PutExternalMirrorWithContext mocks base method. +func (m *MockBucket) PutExternalMirrorWithContext(arg0 context.Context, arg1 *service.PutBucketExternalMirrorInput) (*service.PutBucketExternalMirrorOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PutExternalMirrorWithContext", arg0, arg1) + ret0, _ := ret[0].(*service.PutBucketExternalMirrorOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// PutExternalMirrorWithContext indicates an expected call of PutExternalMirrorWithContext. +func (mr *MockBucketMockRecorder) PutExternalMirrorWithContext(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PutExternalMirrorWithContext", reflect.TypeOf((*MockBucket)(nil).PutExternalMirrorWithContext), arg0, arg1) +} + +// PutLifecycle mocks base method. +func (m *MockBucket) PutLifecycle(arg0 *service.PutBucketLifecycleInput) (*service.PutBucketLifecycleOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PutLifecycle", arg0) + ret0, _ := ret[0].(*service.PutBucketLifecycleOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// PutLifecycle indicates an expected call of PutLifecycle. +func (mr *MockBucketMockRecorder) PutLifecycle(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PutLifecycle", reflect.TypeOf((*MockBucket)(nil).PutLifecycle), arg0) +} + +// PutLifecycleWithContext mocks base method. +func (m *MockBucket) PutLifecycleWithContext(arg0 context.Context, arg1 *service.PutBucketLifecycleInput) (*service.PutBucketLifecycleOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PutLifecycleWithContext", arg0, arg1) + ret0, _ := ret[0].(*service.PutBucketLifecycleOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// PutLifecycleWithContext indicates an expected call of PutLifecycleWithContext. +func (mr *MockBucketMockRecorder) PutLifecycleWithContext(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PutLifecycleWithContext", reflect.TypeOf((*MockBucket)(nil).PutLifecycleWithContext), arg0, arg1) +} + +// PutLogging mocks base method. +func (m *MockBucket) PutLogging(arg0 *service.PutBucketLoggingInput) (*service.PutBucketLoggingOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PutLogging", arg0) + ret0, _ := ret[0].(*service.PutBucketLoggingOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// PutLogging indicates an expected call of PutLogging. +func (mr *MockBucketMockRecorder) PutLogging(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PutLogging", reflect.TypeOf((*MockBucket)(nil).PutLogging), arg0) +} + +// PutLoggingWithContext mocks base method. +func (m *MockBucket) PutLoggingWithContext(arg0 context.Context, arg1 *service.PutBucketLoggingInput) (*service.PutBucketLoggingOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PutLoggingWithContext", arg0, arg1) + ret0, _ := ret[0].(*service.PutBucketLoggingOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// PutLoggingWithContext indicates an expected call of PutLoggingWithContext. +func (mr *MockBucketMockRecorder) PutLoggingWithContext(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PutLoggingWithContext", reflect.TypeOf((*MockBucket)(nil).PutLoggingWithContext), arg0, arg1) +} + +// PutNotification mocks base method. +func (m *MockBucket) PutNotification(arg0 *service.PutBucketNotificationInput) (*service.PutBucketNotificationOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PutNotification", arg0) + ret0, _ := ret[0].(*service.PutBucketNotificationOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// PutNotification indicates an expected call of PutNotification. +func (mr *MockBucketMockRecorder) PutNotification(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PutNotification", reflect.TypeOf((*MockBucket)(nil).PutNotification), arg0) +} + +// PutNotificationWithContext mocks base method. +func (m *MockBucket) PutNotificationWithContext(arg0 context.Context, arg1 *service.PutBucketNotificationInput) (*service.PutBucketNotificationOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PutNotificationWithContext", arg0, arg1) + ret0, _ := ret[0].(*service.PutBucketNotificationOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// PutNotificationWithContext indicates an expected call of PutNotificationWithContext. +func (mr *MockBucketMockRecorder) PutNotificationWithContext(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PutNotificationWithContext", reflect.TypeOf((*MockBucket)(nil).PutNotificationWithContext), arg0, arg1) +} + +// PutObject mocks base method. +func (m *MockBucket) PutObject(arg0 string, arg1 *service.PutObjectInput) (*service.PutObjectOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PutObject", arg0, arg1) + ret0, _ := ret[0].(*service.PutObjectOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// PutObject indicates an expected call of PutObject. +func (mr *MockBucketMockRecorder) PutObject(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PutObject", reflect.TypeOf((*MockBucket)(nil).PutObject), arg0, arg1) +} + +// PutObjectWithContext mocks base method. +func (m *MockBucket) PutObjectWithContext(arg0 context.Context, arg1 string, arg2 *service.PutObjectInput) (*service.PutObjectOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PutObjectWithContext", arg0, arg1, arg2) + ret0, _ := ret[0].(*service.PutObjectOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// PutObjectWithContext indicates an expected call of PutObjectWithContext. +func (mr *MockBucketMockRecorder) PutObjectWithContext(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PutObjectWithContext", reflect.TypeOf((*MockBucket)(nil).PutObjectWithContext), arg0, arg1, arg2) +} + +// PutPolicy mocks base method. +func (m *MockBucket) PutPolicy(arg0 *service.PutBucketPolicyInput) (*service.PutBucketPolicyOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PutPolicy", arg0) + ret0, _ := ret[0].(*service.PutBucketPolicyOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// PutPolicy indicates an expected call of PutPolicy. +func (mr *MockBucketMockRecorder) PutPolicy(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PutPolicy", reflect.TypeOf((*MockBucket)(nil).PutPolicy), arg0) +} + +// PutPolicyWithContext mocks base method. +func (m *MockBucket) PutPolicyWithContext(arg0 context.Context, arg1 *service.PutBucketPolicyInput) (*service.PutBucketPolicyOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PutPolicyWithContext", arg0, arg1) + ret0, _ := ret[0].(*service.PutBucketPolicyOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// PutPolicyWithContext indicates an expected call of PutPolicyWithContext. +func (mr *MockBucketMockRecorder) PutPolicyWithContext(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PutPolicyWithContext", reflect.TypeOf((*MockBucket)(nil).PutPolicyWithContext), arg0, arg1) +} + +// PutReplication mocks base method. +func (m *MockBucket) PutReplication(arg0 *service.PutBucketReplicationInput) (*service.PutBucketReplicationOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PutReplication", arg0) + ret0, _ := ret[0].(*service.PutBucketReplicationOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// PutReplication indicates an expected call of PutReplication. +func (mr *MockBucketMockRecorder) PutReplication(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PutReplication", reflect.TypeOf((*MockBucket)(nil).PutReplication), arg0) +} + +// PutReplicationWithContext mocks base method. +func (m *MockBucket) PutReplicationWithContext(arg0 context.Context, arg1 *service.PutBucketReplicationInput) (*service.PutBucketReplicationOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PutReplicationWithContext", arg0, arg1) + ret0, _ := ret[0].(*service.PutBucketReplicationOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// PutReplicationWithContext indicates an expected call of PutReplicationWithContext. +func (mr *MockBucketMockRecorder) PutReplicationWithContext(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PutReplicationWithContext", reflect.TypeOf((*MockBucket)(nil).PutReplicationWithContext), arg0, arg1) +} + +// PutWithContext mocks base method. +func (m *MockBucket) PutWithContext(arg0 context.Context) (*service.PutBucketOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PutWithContext", arg0) + ret0, _ := ret[0].(*service.PutBucketOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// PutWithContext indicates an expected call of PutWithContext. +func (mr *MockBucketMockRecorder) PutWithContext(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PutWithContext", reflect.TypeOf((*MockBucket)(nil).PutWithContext), arg0) +} + +// UploadMultipart mocks base method. +func (m *MockBucket) UploadMultipart(arg0 string, arg1 *service.UploadMultipartInput) (*service.UploadMultipartOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UploadMultipart", arg0, arg1) + ret0, _ := ret[0].(*service.UploadMultipartOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// UploadMultipart indicates an expected call of UploadMultipart. +func (mr *MockBucketMockRecorder) UploadMultipart(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UploadMultipart", reflect.TypeOf((*MockBucket)(nil).UploadMultipart), arg0, arg1) +} + +// UploadMultipartWithContext mocks base method. +func (m *MockBucket) UploadMultipartWithContext(arg0 context.Context, arg1 string, arg2 *service.UploadMultipartInput) (*service.UploadMultipartOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UploadMultipartWithContext", arg0, arg1, arg2) + ret0, _ := ret[0].(*service.UploadMultipartOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// UploadMultipartWithContext indicates an expected call of UploadMultipartWithContext. +func (mr *MockBucketMockRecorder) UploadMultipartWithContext(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UploadMultipartWithContext", reflect.TypeOf((*MockBucket)(nil).UploadMultipartWithContext), arg0, arg1, arg2) +} diff --git a/services/qingstor/rfcs/79-add-virtual-link-support.md b/services/qingstor/rfcs/79-add-virtual-link-support.md new file mode 100644 index 000000000..a8f674757 --- /dev/null +++ b/services/qingstor/rfcs/79-add-virtual-link-support.md @@ -0,0 +1,56 @@ +- Author: abyss-w +- Start Date: 2021-08-16 +- RFC PR: [beyondstorage/go-service-qingstor#79](https://github.com/beyondstorage/go-service-qingstor/pull/79) +- Tracking Issue: [beyondstorage/go-service-qingstor#64](https://github.com/beyondstorage/go-service-qingstor/issues/64) + +# RFC-79: Add Virtual Link Support + +## Background + +As you can see from the [official documentation for qingstor](https://docs.qingcloud.com/qingstor/), it does not support symbolic links itself. However, qingstor supports [user-defined object metadata](https://docs.qingcloud.com/qingstor/api/common/metadata), and we can use this to simulate the implementation of symlink. + +## Proposal + +I propose to use user-defined object metadata to support virtual link. + +```go +input := &service.PutObjectInput{ + XQSMetadata: &map[string]string{ + "x-qs-meta-bs-link-target": rt, + }, +} +``` + +- `PutObjectInput` is used to store the fields we need when calling `PutObjectWithContext` API to upload an object. +- `XQSMetadata` is a map that stores user-defined metadata. + - `"x-qs-meta-bs-link-target"` is the name of user-defined metadata, the middle `bs` is used to avoid conflicts. + - `rt` is the symlink target, it is an absolute path. + +## Rationale + +### User-defined metadata + +As the [Object Metadata](https://docs.qingcloud.com/qingstor/api/common/metadata) says,there are two types of object metadata that can be changed by the user: standard HTTP headers and user-defined metadata. We can define our own metadata to store the fields we need when uploading an object. Note that the name of the user-defined metadata must start with `x-qs-meta`. + +### Drawbacks + +As qingstor itself does not support symlink, we can only simulate it. And the object created is not really a symlink object. When we call `stat`, we can only tell if it is a symlink by using user-defined metadata, + +```go +if v, ok := metadata["x-qs-meta-bs-link-target"]; ok { + // The path is a symlink object +} +``` + +Calling `HeadObject` in `list` will increase the execution cost of `list` which we cannot afford. So we will relax the qingstor condition. We will not support getting the exact symlink object type in `list` when user has virtual link enabled, if the user wants to get the exact object mode they need to call `stat`. + +## Compatibility + +N/A + +## Implementation + +- Implement `virtual_link` in go-service-qingstor +- Support `stat` +- Setup linker tests + diff --git a/services/qingstor/service.go b/services/qingstor/service.go new file mode 100644 index 000000000..93d32fccd --- /dev/null +++ b/services/qingstor/service.go @@ -0,0 +1,92 @@ +package qingstor + +import ( + "context" + + "github.com/qingstor/qingstor-sdk-go/v4/service" + + ps "go.beyondstorage.io/v5/pairs" + . "go.beyondstorage.io/v5/types" +) + +func (s *Service) create(ctx context.Context, name string, opt pairServiceCreate) (store Storager, err error) { + // ServicePairCreate requires location, so we don't need to add location into pairs + pairs := append(opt.pairs, ps.WithName(name)) + + st, err := s.newStorage(pairs...) + if err != nil { + return + } + + _, err = st.bucket.PutWithContext(ctx) + if err != nil { + return + } + return st, nil +} + +func (s *Service) delete(ctx context.Context, name string, opt pairServiceDelete) (err error) { + pairs := append(opt.pairs, ps.WithName(name)) + + store, err := s.newStorage(pairs...) + if err != nil { + return + } + _, err = store.bucket.DeleteWithContext(ctx) + if err != nil { + return + } + return nil +} + +func (s *Service) get(ctx context.Context, name string, opt pairServiceGet) (store Storager, err error) { + pairs := append(opt.pairs, ps.WithName(name)) + + store, err = s.newStorage(pairs...) + if err != nil { + return + } + return +} + +func (s *Service) list(ctx context.Context, opt pairServiceList) (it *StoragerIterator, err error) { + input := &storagePageStatus{} + + if opt.HasLocation { + input.location = opt.Location + } + + return NewStoragerIterator(ctx, s.nextStoragePage, input), nil +} + +func (s *Service) nextStoragePage(ctx context.Context, page *StoragerPage) error { + input := page.Status.(*storagePageStatus) + + serviceInput := &service.ListBucketsInput{ + Limit: &input.offset, + Offset: &input.limit, + } + if input.location != "" { + serviceInput.Location = &input.location + } + + output, err := s.service.ListBucketsWithContext(ctx, serviceInput) + if err != nil { + return err + } + + for _, v := range output.Buckets { + store, err := s.newStorage(ps.WithName(*v.Name), ps.WithLocation(*v.Location)) + if err != nil { + return err + } + page.Data = append(page.Data, store) + } + + input.offset += len(output.Buckets) + if input.offset >= service.IntValue(output.Count) { + return IterateDone + } + + return nil +} diff --git a/services/qingstor/service.toml b/services/qingstor/service.toml new file mode 100644 index 000000000..6a07417db --- /dev/null +++ b/services/qingstor/service.toml @@ -0,0 +1,114 @@ +name = "qingstor" + +[namespace.service.new] +required = ["credential"] +optional = ["service_features", "default_service_pairs", "endpoint", "http_client_options"] + +[namespace.service.op.create] +required = ["location"] + +[namespace.service.op.delete] +optional = ["location"] + +[namespace.service.op.get] +optional = ["location"] + +[namespace.service.op.list] +optional = ["location"] + +[namespace.storage] +features = ["virtual_dir", "virtual_link"] +implement = ["appender", "copier", "direr", "fetcher", "linker", "mover", "multiparter", "reacher", "storage_http_signer"] + +[namespace.storage.new] +required = ["name"] +optional = ["storage_features", "default_storage_pairs", "disable_uri_cleaning", "http_client_options", "location", "work_dir"] + +[namespace.storage.op.create] +optional = ["multipart_id", "object_mode"] + +[namespace.storage.op.create_dir] +optional = ["storage_class"] + +[namespace.storage.op.delete] +optional = ["multipart_id", "object_mode"] + +[namespace.storage.op.stat] +optional = ["multipart_id", "object_mode"] + +[namespace.storage.op.list] +optional = ["list_mode"] + +[namespace.storage.op.reach] +required = ["expire"] + +[namespace.storage.op.read] +optional = ["offset", "io_callback", "size", "encryption_customer_algorithm", "encryption_customer_key"] + +[namespace.storage.op.write] +optional = ["content_md5", "content_type", "io_callback", "storage_class", "encryption_customer_algorithm", "encryption_customer_key"] + +[namespace.storage.op.create_append] +optional = ["content_type", "storage_class"] + +[namespace.storage.op.write_append] +optional = ["content_md5"] + +[namespace.storage.op.copy] +optional = ["encryption_customer_algorithm", "encryption_customer_key", "copy_source_encryption_customer_algorithm", "copy_source_encryption_customer_key"] + +[namespace.storage.op.create_multipart] +optional = ["encryption_customer_algorithm", "encryption_customer_key"] + +[namespace.storage.op.write_multipart] +optional = ["encryption_customer_algorithm", "encryption_customer_key", "io_callback"] + +[namespace.storage.op.query_sign_http_read] +optional = ["offset", "encryption_customer_algorithm", "encryption_customer_key", "size"] + +[namespace.storage.op.query_sign_http_write] +optional = ["content_md5", "content_type", "encryption_customer_algorithm", "encryption_customer_key", "storage_class"] + +[pairs.service_features] +type = "ServiceFeatures" +description = "set service features" + +[pairs.storage_features] +type = "StorageFeatures" +description = "set storage features" + +[pairs.encryption_customer_algorithm] +type = "string" +description = "specifies the encryption algorithm. Only AES256 is supported now." + +[pairs.encryption_customer_key] +type = "[]byte" +description = "is the customer-provided encryption key. For AES256 keys, the plaintext must be 32 bytes long." + +[pairs.copy_source_encryption_customer_algorithm] +type = "string" +description = "is the encryption algorithm for the source object. Only AES256 is supported now." + +[pairs.copy_source_encryption_customer_key] +type = "[]byte" +description = "is the customer-provided encryption key for the source object. For AES256 keys, the plaintext must be 32 bytes long." + +[pairs.disable_uri_cleaning] +type = "bool" + +[pairs.storage_class] +type = "string" + +[pairs.default_service_pairs] +type = "DefaultServicePairs" +description = "set default pairs for service actions" + +[pairs.default_storage_pairs] +type = "DefaultStoragePairs" +description = "set default pairs for storager actions" + +[infos.object.meta.storage-class] +type = "string" + +[infos.object.meta.encryption_customer_algorithm] +type = "string" diff --git a/services/qingstor/servicer_test.go b/services/qingstor/servicer_test.go new file mode 100644 index 000000000..67abef1ed --- /dev/null +++ b/services/qingstor/servicer_test.go @@ -0,0 +1,441 @@ +package qingstor + +import ( + "context" + "errors" + "fmt" + "log" + "net/http" + "reflect" + "testing" + + "bou.ke/monkey" + "github.com/golang/mock/gomock" + "github.com/google/uuid" + "github.com/qingstor/qingstor-sdk-go/v4/config" + "github.com/qingstor/qingstor-sdk-go/v4/service" + "github.com/stretchr/testify/assert" + + "go.beyondstorage.io/credential" + "go.beyondstorage.io/v5/pairs" + "go.beyondstorage.io/v5/services" + "go.beyondstorage.io/v5/types" +) + +func TestService_String(t *testing.T) { + accessKey := uuid.New().String() + secretKey := uuid.New().String() + + srv, err := NewServicer(pairs.WithCredential(credential.NewHmac(accessKey, secretKey).String())) + if err != nil { + t.Fatal(err) + } + + assert.NotEmpty(t, srv.String()) +} + +func TestService_Get(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + t.Run("with location", func(t *testing.T) { + mockService := NewMockService(ctrl) + + srv := Service{ + service: mockService, + } + + name := uuid.New().String() + location := uuid.New().String() + + mockService.EXPECT().Bucket(gomock.Any(), gomock.Any()).DoAndReturn(func(bucketName, inputLocation string) (*service.Bucket, error) { + assert.Equal(t, name, bucketName) + assert.Equal(t, location, inputLocation) + return &service.Bucket{ + Properties: &service.Properties{ + BucketName: &name, + Zone: &location, + }, + }, nil + }) + + s, err := srv.Get(name, pairs.WithLocation(location)) + assert.NoError(t, err) + assert.NotNil(t, s) + }) + + t.Run("without location", func(t *testing.T) { + mockService := NewMockService(ctrl) + + srv := &Service{} + srv.service = mockService + srv.config = &config.Config{ + AccessKeyID: uuid.New().String(), + SecretAccessKey: uuid.New().String(), + Host: uuid.New().String(), + Port: 1234, + Protocol: "https", + } + + name := uuid.New().String() + location := uuid.New().String() + + // Patch http Head. + fn := func(client *http.Client, url string) (*http.Response, error) { + header := http.Header{} + header.Set( + "location", + fmt.Sprintf("%s://%s.%s/%s", + srv.config.Protocol, location, srv.config.Host, name), + ) + return &http.Response{ + StatusCode: http.StatusMovedPermanently, + Header: header, + }, nil + } + monkey.PatchInstanceMethod(reflect.TypeOf(srv.client), "Head", fn) + + // Mock Bucket. + mockService.EXPECT().Bucket(gomock.Any(), gomock.Any()).DoAndReturn(func(bucketName, inputLocation string) (*service.Bucket, error) { + assert.Equal(t, name, bucketName) + assert.Equal(t, location, inputLocation) + return &service.Bucket{ + Properties: &service.Properties{ + BucketName: &name, + Zone: &location, + }, + }, nil + }) + + s, err := srv.Get(name) + assert.NoError(t, err) + assert.NotNil(t, s) + }) + + t.Run("invalid bucket name", func(t *testing.T) { + mockService := NewMockService(ctrl) + + srv := &Service{} + srv.service = mockService + srv.config = &config.Config{ + AccessKeyID: uuid.New().String(), + SecretAccessKey: uuid.New().String(), + Host: uuid.New().String(), + Port: 1234, + Protocol: uuid.New().String(), + } + + s, err := srv.Get("1234") + assert.Error(t, err) + assert.Nil(t, s) + }) +} + +func TestService_Create(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockService := NewMockService(ctrl) + + srv := Service{ + service: mockService, + } + + // Test case1: without location + path := uuid.New().String() + _, err := srv.Create(path) + assert.Error(t, err) + assert.True(t, errors.Is(err, services.ErrRestrictionDissatisfied)) + + // Test case2: with valid location. + path = uuid.New().String() + location := uuid.New().String() + + // Monkey the bucket's Put method + bucket := &service.Bucket{} + fn := func(*service.Bucket, context.Context) (*service.PutBucketOutput, error) { + t.Log("Bucket put has been called") + return &service.PutBucketOutput{}, nil + } + monkey.PatchInstanceMethod(reflect.TypeOf(bucket), "PutWithContext", fn) + + mockService.EXPECT().Bucket(gomock.Any(), gomock.Any()).Do(func(inputPath, inputLocation string) { + assert.Equal(t, path, inputPath) + assert.Equal(t, location, inputLocation) + }).Return(bucket, nil) + + _, err = srv.Create(path, pairs.WithLocation(location)) + assert.NoError(t, err) +} + +func TestService_Delete(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockService := NewMockService(ctrl) + + srv := Service{ + service: mockService, + } + + { + name := uuid.New().String() + location := uuid.New().String() + + // Patch bucket.Delete + bucket := &service.Bucket{} + fn := func(*service.Bucket, context.Context) (*service.DeleteBucketOutput, error) { + t.Log("Bucket delete has been called") + return nil, nil + } + monkey.PatchInstanceMethod(reflect.TypeOf(bucket), "DeleteWithContext", fn) + + mockService.EXPECT().Bucket(gomock.Any(), gomock.Any()).DoAndReturn(func(bucketName, inputLocation string) (*service.Bucket, error) { + assert.Equal(t, name, bucketName) + assert.Equal(t, location, inputLocation) + return bucket, nil + }) + + err := srv.Delete(name, pairs.WithLocation(location)) + assert.NoError(t, err) + } +} + +func TestService_List(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockService := NewMockService(ctrl) + + srv := &Service{ + service: mockService, + } + mockService.EXPECT().Bucket(gomock.Any(), gomock.Any()).DoAndReturn(func(inputName, inputLocation string) (*service.Bucket, error) { + return &service.Bucket{ + Config: &config.Config{}, + }, nil + }).AnyTimes() + + { + // Test request with location. + name := uuid.New().String() + location := uuid.New().String() + + mockService.EXPECT().ListBucketsWithContext(gomock.Eq(context.Background()), gomock.Any()).DoAndReturn(func(ctx context.Context, input *service.ListBucketsInput) (*service.ListBucketsOutput, error) { + assert.Equal(t, location, *input.Location) + return &service.ListBucketsOutput{ + Buckets: []*service.BucketType{ + {Name: &name, Location: &location}, + }, + }, nil + }) + + it, err := srv.List(pairs.WithLocation(location)) + assert.NoError(t, err) + assert.NotNil(t, it) + st, err := it.Next() + if err != nil { + t.Error(err) + } + assert.NotNil(t, st) + } + + { + // Test request without location. + name := uuid.New().String() + location := uuid.New().String() + + mockService.EXPECT().ListBucketsWithContext(gomock.Eq(context.Background()), gomock.Any()).DoAndReturn(func(ctx context.Context, input *service.ListBucketsInput) (*service.ListBucketsOutput, error) { + assert.Nil(t, input.Location) + return &service.ListBucketsOutput{ + Buckets: []*service.BucketType{ + {Name: &name, Location: &location}, + }, + }, nil + }) + + it, err := srv.List() + assert.NoError(t, err) + assert.NotNil(t, it) + // assert.Equal(t, 1, len(s)) + _, err = it.Next() + assert.NoError(t, err) + } +} + +func ExampleNew() { + _, _, err := New( + pairs.WithCredential( + credential.NewHmac("test_access_key", "test_secret_key").String(), + ), + ) + if err != nil { + log.Printf("service init failed: %v", err) + } +} + +func ExampleService_Get() { + srv, _, err := New( + pairs.WithCredential( + credential.NewHmac("test_access_key", "test_secret_key").String(), + ), + ) + if err != nil { + log.Printf("service init failed: %v", err) + } + + store, err := srv.Get("bucket_name", pairs.WithLocation("location")) + if err != nil { + log.Printf("service get bucket failed: %v", err) + } + log.Printf("%v", store) +} + +func Test_isWorkDirValid(t *testing.T) { + type args struct { + wd string + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "not a dir", + args: args{wd: "/path/to/file"}, + want: false, + }, + { + name: "not a abs dir", + args: args{wd: "path/to/file/"}, + want: false, + }, + { + name: "start with multi-slash", + args: args{wd: "///multi-slash/to/file/"}, + want: false, + }, + { + name: "end with multi-slash", + args: args{wd: "/path/to/multi-slash///"}, + want: false, + }, + { + name: "root path", + args: args{wd: "/"}, + want: true, + }, + { + name: "normal abs dir", + args: args{wd: "/path/to/dir/"}, + want: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := isWorkDirValid(tt.args.wd) + if got != tt.want { + t.Errorf("isWorkDirValid() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestService_newStorage(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + blankBucket := service.Bucket{ + Config: &config.Config{}, + } + validWorkDir, invalidWorkDir := "/valid/dir/", "invalid/dir" + type args struct { + pairs []types.Pair + } + tests := []struct { + name string + wd string + args args + wantBucket *service.Bucket + targetErr error + wantErr bool + }{ + { + name: "normal case", + wd: validWorkDir, + args: args{[]types.Pair{ + {Key: "location", Value: uuid.New().String()}, + {Key: "name", Value: uuid.New().String()}, + {Key: "work_dir", Value: validWorkDir}, + }}, + wantBucket: &blankBucket, + targetErr: nil, + wantErr: false, + }, + { + name: "invalid work dir", + wd: invalidWorkDir, + args: args{[]types.Pair{ + {Key: "location", Value: uuid.New().String()}, + {Key: "name", Value: uuid.New().String()}, + {Key: "work_dir", Value: invalidWorkDir}, + }}, + wantBucket: nil, + targetErr: ErrWorkDirInvalid, + wantErr: true, + }, + { + name: "blank work dir", + wd: "/", + args: args{[]types.Pair{ + {Key: "location", Value: uuid.New().String()}, + {Key: "name", Value: uuid.New().String()}, + }}, + wantBucket: &blankBucket, + targetErr: nil, + wantErr: false, + }, + { + name: "invalid bucket name", + wd: validWorkDir, + args: args{[]types.Pair{ + {Key: "location", Value: uuid.New().String()}, + {Key: "name", Value: "invalid bucket name"}, + {Key: "work_dir", Value: validWorkDir}, + }}, + wantBucket: nil, + targetErr: ErrBucketNameInvalid, + wantErr: true, + }, + { + name: "no pairs, fail when parse", + args: args{}, + wantBucket: nil, + targetErr: services.ErrRestrictionDissatisfied, + wantErr: true, + }, + } + + mockService := NewMockService(ctrl) + mockService.EXPECT().Bucket(gomock.Any(), gomock.Any()).DoAndReturn(func(inputName, inputLocation string) (*service.Bucket, error) { + return &blankBucket, nil + }).AnyTimes() + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := &Service{service: mockService} + gotStore, err := s.newStorage(tt.args.pairs...) + if (err != nil) != tt.wantErr { + t.Errorf("newStorage() error = %v, wantErr %v", err, tt.wantErr) + return + } + if tt.wantErr || err != nil { + assert.Nil(t, gotStore, tt.name) + assert.True(t, errors.Is(err, tt.targetErr), tt.name) + } else { + assert.Equal(t, tt.wantBucket, gotStore.bucket, tt.name) + assert.Equal(t, tt.wd, gotStore.workDir, tt.name) + } + }) + } +} diff --git a/services/qingstor/storage.go b/services/qingstor/storage.go new file mode 100644 index 000000000..2fe9231e7 --- /dev/null +++ b/services/qingstor/storage.go @@ -0,0 +1,820 @@ +package qingstor + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + "time" + + "github.com/pengsrc/go-shared/convert" + "github.com/qingstor/qingstor-sdk-go/v4/service" + + ps "go.beyondstorage.io/v5/pairs" + "go.beyondstorage.io/v5/pkg/iowrap" + "go.beyondstorage.io/v5/services" + . "go.beyondstorage.io/v5/types" +) + +func (s *Storage) commitAppend(ctx context.Context, o *Object, opt pairStorageCommitAppend) (err error) { + return +} + +func (s *Storage) completeMultipart(ctx context.Context, o *Object, parts []*Part, opt pairStorageCompleteMultipart) (err error) { + objectParts := make([]*service.ObjectPartType, 0, len(parts)) + for _, v := range parts { + objectParts = append(objectParts, &service.ObjectPartType{ + Etag: service.String(v.ETag), + PartNumber: service.Int(v.Index), + Size: service.Int64(v.Size), + }) + } + + _, err = s.bucket.CompleteMultipartUploadWithContext(ctx, o.ID, &service.CompleteMultipartUploadInput{ + UploadID: service.String(o.MustGetMultipartID()), + ObjectParts: objectParts, + }) + if err != nil { + return + } + o.Mode.Del(ModePart) + o.Mode.Add(ModeRead) + return +} + +func (s *Storage) copy(ctx context.Context, src string, dst string, opt pairStorageCopy) (err error) { + rs := s.getAbsPath(src) + rd := s.getAbsPath(dst) + + srcPath := "/" + service.StringValue(s.properties.BucketName) + "/" + url.QueryEscape(rs) + input := &service.PutObjectInput{ + XQSCopySource: &srcPath, + } + if opt.HasEncryptionCustomerAlgorithm { + input.XQSEncryptionCustomerAlgorithm, input.XQSEncryptionCustomerKey, input.XQSEncryptionCustomerKeyMD5, err = calculateEncryptionHeaders(opt.EncryptionCustomerAlgorithm, opt.EncryptionCustomerKey) + if err != nil { + return + } + } + if opt.HasCopySourceEncryptionCustomerAlgorithm { + input.XQSCopySourceEncryptionCustomerAlgorithm, input.XQSCopySourceEncryptionCustomerKey, input.XQSCopySourceEncryptionCustomerKeyMD5, err = calculateEncryptionHeaders(opt.CopySourceEncryptionCustomerAlgorithm, opt.CopySourceEncryptionCustomerKey) + if err != nil { + return + } + } + + _, err = s.bucket.PutObjectWithContext(ctx, rd, input) + if err != nil { + return + } + return nil +} + +func (s *Storage) create(path string, opt pairStorageCreate) (o *Object) { + rp := s.getAbsPath(path) + + // handle create multipart object separately + // if opt has multipartID, set object done, because we can't stat multipart object in QingStor + if opt.HasMultipartID { + o = s.newObject(true) + o.Mode = ModePart + o.SetMultipartID(opt.MultipartID) + } else { + if opt.HasObjectMode && opt.ObjectMode.IsDir() { + if !s.features.VirtualDir { + return + } + + rp += "/" + o = s.newObject(true) + o.Mode = ModeDir + } else { + o = s.newObject(false) + o.Mode = ModeRead + } + } + o.ID = rp + o.Path = path + return o +} + +func (s *Storage) createAppend(ctx context.Context, path string, opt pairStorageCreateAppend) (o *Object, err error) { + rp := s.getAbsPath(path) + + // We should set offset to 0 whether the object exists or not. + var offset int64 = 0 + + // If the object exists, we should delete it. + headInput := &service.HeadObjectInput{} + _, err = s.bucket.HeadObjectWithContext(ctx, rp, headInput) + if err == nil { + _, err = s.bucket.DeleteObject(rp) + if err != nil { + return nil, err + } + } + + input := &service.AppendObjectInput{ + Position: &offset, + } + if opt.HasContentType { + input.ContentType = &opt.ContentType + } + if opt.HasStorageClass { + input.XQSStorageClass = &opt.StorageClass + } + + output, err := s.bucket.AppendObjectWithContext(ctx, rp, input) + if err != nil { + return + } + + if output == nil || output.XQSNextAppendPosition == nil { + err = ErrAppendNextPositionEmpty + return + } else { + offset = *output.XQSNextAppendPosition + } + + o = s.newObject(true) + o.Mode = ModeRead | ModeAppend + o.ID = rp + o.Path = path + o.SetAppendOffset(offset) + // set metadata + if opt.HasContentType { + o.SetContentType(opt.ContentType) + } + var sm ObjectSystemMetadata + if opt.HasStorageClass { + sm.StorageClass = opt.StorageClass + } + return o, nil +} + +func (s *Storage) createDir(ctx context.Context, path string, opt pairStorageCreateDir) (o *Object, err error) { + if !s.features.VirtualDir { + err = NewOperationNotImplementedError("create_dir") + return + } + + rp := s.getAbsPath(path) + + // Add `/` at the end of path to simulate a directory. + // ref: https://docs.qingcloud.com/qingstor/api/object/put.html + rp += "/" + + input := &service.PutObjectInput{ + ContentLength: service.Int64(0), + } + if opt.HasStorageClass { + input.XQSStorageClass = service.String(opt.StorageClass) + } + + _, err = s.bucket.PutObjectWithContext(ctx, rp, input) + if err != nil { + return + } + + o = s.newObject(true) + o.Path = path + o.ID = rp + o.Mode = ModeDir + return +} + +// metadataLinkTargetHeader is the name of the user-defined metadata name used to store the target. +const metadataLinkTargetHeader = "x-qs-meta-bs-link-target" + +func (s *Storage) createLink(ctx context.Context, path string, target string, opt pairStorageCreateLink) (o *Object, err error) { + rt := s.getAbsPath(target) + rp := s.getAbsPath(path) + + input := &service.PutObjectInput{ + // As qingstor does not support symlink, we can only use user-defined metadata to simulate it. + // ref: https://github.com/beyondstorage/go-service-qingstor/blob/master/rfcs/79-add-virtual-link-support.md + XQSMetaData: &map[string]string{ + metadataLinkTargetHeader: rt, + }, + } + + _, err = s.bucket.PutObjectWithContext(ctx, rp, input) + if err != nil { + return nil, err + } + + o = s.newObject(true) + o.ID = rp + o.Path = path + + if !s.features.VirtualLink { + // The virtual link is not enabled, so we set the object mode to `ModeRead`. + o.Mode |= ModeRead + } else { + // qingstor does not have an absolute path, so when we call `getAbsPath`, it will remove the prefix `/`. + // To ensure that the path matches the one the user gets, we should re-add `/` here. + o.SetLinkTarget("/" + rt) + o.Mode |= ModeLink + } + return +} + +func (s *Storage) createMultipart(ctx context.Context, path string, opt pairStorageCreateMultipart) (o *Object, err error) { + input := &service.InitiateMultipartUploadInput{} + if opt.HasEncryptionCustomerAlgorithm { + input.XQSEncryptionCustomerAlgorithm, input.XQSEncryptionCustomerKey, input.XQSEncryptionCustomerKeyMD5, err = calculateEncryptionHeaders(opt.EncryptionCustomerAlgorithm, opt.EncryptionCustomerKey) + if err != nil { + return + } + } + + rp := s.getAbsPath(path) + + output, err := s.bucket.InitiateMultipartUploadWithContext(ctx, rp, input) + if err != nil { + return + } + + o = s.newObject(true) + o.ID = rp + o.Path = path + o.Mode |= ModePart + o.SetMultipartID(*output.UploadID) + + return o, nil +} + +func (s *Storage) delete(ctx context.Context, path string, opt pairStorageDelete) (err error) { + rp := s.getAbsPath(path) + + if opt.HasMultipartID { + // QingStor AbortMultipartUpload is idempotent, so we don't need to check upload_not_exists error. + // + // References + // - [GSP-46](https://github.com/beyondstorage/specs/blob/master/rfcs/46-idempotent-delete.md) + // - https://docs.qingcloud.com/qingstor/api/object/multipart/abort_multipart_upload.html + _, err = s.bucket.AbortMultipartUploadWithContext(ctx, rp, &service.AbortMultipartUploadInput{ + UploadID: service.String(opt.MultipartID), + }) + if err != nil { + return + } + return + } + + if opt.HasObjectMode && opt.ObjectMode.IsDir() { + if !s.features.VirtualDir { + err = services.PairUnsupportedError{Pair: ps.WithObjectMode(opt.ObjectMode)} + return + } + + rp += "/" + } + + // QingStor DeleteObject is idempotent, so we don't need to check object_not_exists error. + // + // - [GSP-46](https://github.com/beyondstorage/specs/blob/master/rfcs/46-idempotent-delete.md) + // - https://docs.qingcloud.com/qingstor/api/object/delete + _, err = s.bucket.DeleteObjectWithContext(ctx, rp) + if err != nil { + return + } + return nil +} + +func (s *Storage) fetch(ctx context.Context, path string, url string, opt pairStorageFetch) (err error) { + _, err = s.bucket.PutObjectWithContext(ctx, path, &service.PutObjectInput{ + XQSFetchSource: service.String(url), + }) + return err +} + +func (s *Storage) list(ctx context.Context, path string, opt pairStorageList) (oi *ObjectIterator, err error) { + input := &objectPageStatus{ + limit: 200, + prefix: s.getAbsPath(path), + } + + if !opt.HasListMode { + // Support `ListModePrefix` as the default `ListMode`. + // ref: [GSP-654](https://github.com/beyondstorage/go-storage/blob/master/docs/rfcs/654-unify-list-behavior.md) + opt.ListMode = ListModePrefix + } + + var nextFn NextObjectFunc + + switch { + case opt.ListMode.IsPart(): + nextFn = s.nextPartObjectPageByPrefix + case opt.ListMode.IsDir(): + input.delimiter = "/" + nextFn = s.nextObjectPageByDir + case opt.ListMode.IsPrefix(): + nextFn = s.nextObjectPageByPrefix + default: + return nil, services.ListModeInvalidError{Actual: opt.ListMode} + } + + return NewObjectIterator(ctx, nextFn, input), nil +} + +func (s *Storage) listMultipart(ctx context.Context, o *Object, opt pairStorageListMultipart) (pi *PartIterator, err error) { + input := &partPageStatus{ + limit: 200, + prefix: o.ID, + uploadID: o.MustGetMultipartID(), + } + + return NewPartIterator(ctx, s.nextPartPage, input), nil +} + +func (s *Storage) metadata(opt pairStorageMetadata) (meta *StorageMeta) { + meta = NewStorageMeta() + meta.Name = *s.properties.BucketName + meta.WorkDir = s.workDir + meta.SetLocation(*s.properties.Zone) + // set write restriction + meta.SetWriteSizeMaximum(writeSizeMaximum) + // set copy restriction + meta.SetCopySizeMaximum(copySizeMaximum) + // set append restrictions + meta.SetAppendSizeMaximum(appendSizeMaximum) + meta.SetAppendTotalSizeMaximum(appendTotalSizeMaximum) + // set multipart restrictions + meta.SetMultipartNumberMaximum(multipartNumberMaximum) + meta.SetMultipartSizeMaximum(multipartSizeMaximum) + meta.SetMultipartSizeMinimum(multipartSizeMinimum) + return meta +} + +func (s *Storage) move(ctx context.Context, src string, dst string, opt pairStorageMove) (err error) { + rs := s.getAbsPath(src) + rd := s.getAbsPath(dst) + + srcPath := "/" + service.StringValue(s.properties.BucketName) + "/" + url.QueryEscape(rs) + _, err = s.bucket.PutObjectWithContext(ctx, rd, &service.PutObjectInput{ + XQSMoveSource: &srcPath, + }) + if err != nil { + return + } + return nil +} + +func (s *Storage) nextObjectPageByDir(ctx context.Context, page *ObjectPage) error { + input := page.Status.(*objectPageStatus) + + output, err := s.bucket.ListObjectsWithContext(ctx, &service.ListObjectsInput{ + Delimiter: &input.delimiter, + Limit: &input.limit, + Marker: &input.marker, + Prefix: &input.prefix, + }) + if err != nil { + return err + } + + for _, v := range output.CommonPrefixes { + o := s.newObject(true) + o.ID = *v + o.Path = s.getRelPath(*v) + o.Mode |= ModeDir + + page.Data = append(page.Data, o) + } + + for _, v := range output.Keys { + // add filter to exclude dir-key itself, which would exist if created in console, see issue #365 + if convert.StringValue(v.Key) == input.prefix { + continue + } + o, err := s.formatFileObject(v) + if err != nil { + return err + } + + page.Data = append(page.Data, o) + } + + if service.StringValue(output.NextMarker) == "" { + return IterateDone + } + if !service.BoolValue(output.HasMore) { + return IterateDone + } + if len(output.Keys) == 0 { + return IterateDone + } + + input.marker = *output.NextMarker + return nil +} + +func (s *Storage) nextObjectPageByPrefix(ctx context.Context, page *ObjectPage) error { + input := page.Status.(*objectPageStatus) + + output, err := s.bucket.ListObjectsWithContext(ctx, &service.ListObjectsInput{ + Limit: &input.limit, + Marker: &input.marker, + Prefix: &input.prefix, + }) + if err != nil { + return err + } + + for _, v := range output.Keys { + o, err := s.formatFileObject(v) + if err != nil { + return err + } + + page.Data = append(page.Data, o) + } + + if service.StringValue(output.NextMarker) == "" { + return IterateDone + } + if !service.BoolValue(output.HasMore) { + return IterateDone + } + if len(output.Keys) == 0 { + return IterateDone + } + + input.marker = *output.NextMarker + return nil +} + +func (s *Storage) nextPartObjectPageByPrefix(ctx context.Context, page *ObjectPage) error { + input := page.Status.(*objectPageStatus) + + output, err := s.bucket.ListMultipartUploadsWithContext(ctx, &service.ListMultipartUploadsInput{ + KeyMarker: &input.marker, + Limit: &input.limit, + Prefix: &input.prefix, + UploadIDMarker: &input.partIdMarker, + }) + if err != nil { + return err + } + + for _, v := range output.Uploads { + o := s.newObject(true) + o.ID = *v.Key + o.Path = s.getRelPath(*v.Key) + o.Mode |= ModePart + o.SetMultipartID(*v.UploadID) + + page.Data = append(page.Data, o) + } + + nextKeyMarker := service.StringValue(output.NextKeyMarker) + nextUploadIDMarker := service.StringValue(output.NextUploadIDMarker) + + if nextKeyMarker == "" && nextUploadIDMarker == "" { + return IterateDone + } + if !service.BoolValue(output.HasMore) { + return IterateDone + } + + input.marker = nextKeyMarker + input.partIdMarker = nextUploadIDMarker + return nil +} + +func (s *Storage) nextPartPage(ctx context.Context, page *PartPage) error { + input := page.Status.(*partPageStatus) + + output, err := s.bucket.ListMultipartWithContext(ctx, input.prefix, &service.ListMultipartInput{ + Limit: &input.limit, + PartNumberMarker: &input.partNumberMarker, + UploadID: &input.uploadID, + }) + if err != nil { + return err + } + + for _, v := range output.ObjectParts { + p := &Part{ + Index: *v.PartNumber, + Size: *v.Size, + ETag: service.StringValue(v.Etag), + } + + page.Data = append(page.Data, p) + } + + // FIXME: QingStor ListMulitpart API looks like buggy. + offset := input.partNumberMarker + len(output.ObjectParts) + if offset >= service.IntValue(output.Count) { + return IterateDone + } + + input.partNumberMarker = offset + return nil +} + +func (s *Storage) querySignHTTPDelete(ctx context.Context, path string, expire time.Duration, opt pairStorageQuerySignHTTPDelete) (req *http.Request, err error) { + panic("not implemented") +} + +func (s *Storage) querySignHTTPRead(ctx context.Context, path string, expire time.Duration, opt pairStorageQuerySignHTTPRead) (req *http.Request, err error) { + pairs, err := s.parsePairStorageRead(opt.pairs) + if err != nil { + return + } + + input, err := s.formatGetObjectInput(pairs) + if err != nil { + return + } + + bucket := s.bucket.(*service.Bucket) + + rp := s.getAbsPath(path) + + r, _, err := bucket.GetObjectRequest(rp, input) + if err != nil { + return + } + if err = r.BuildWithContext(ctx); err != nil { + return + } + + if err = r.SignQuery(int(expire.Seconds())); err != nil { + return + } + + return r.HTTPRequest, nil +} + +func (s *Storage) querySignHTTPWrite(ctx context.Context, path string, size int64, expire time.Duration, opt pairStorageQuerySignHTTPWrite) (req *http.Request, err error) { + pairs, err := s.parsePairStorageWrite(opt.pairs) + if err != nil { + return + } + + input, err := s.formatPutObjectInput(size, pairs) + if err != nil { + return + } + + bucket := s.bucket.(*service.Bucket) + + rp := s.getAbsPath(path) + + r, _, err := bucket.PutObjectRequest(rp, input) + if err != nil { + return + } + if err = r.BuildWithContext(ctx); err != nil { + return + } + + if err = r.SignQuery(int(expire.Seconds())); err != nil { + return + } + + return r.HTTPRequest, nil +} + +func (s *Storage) reach(ctx context.Context, path string, opt pairStorageReach) (url string, err error) { + // FIXME: sdk should export GetObjectRequest as interface too? + bucket := s.bucket.(*service.Bucket) + + rp := s.getAbsPath(path) + + r, _, err := bucket.GetObjectRequest(rp, nil) + if err != nil { + return + } + if err = r.BuildWithContext(ctx); err != nil { + return + } + + expire := opt.Expire + if err = r.SignQuery(int(expire.Seconds())); err != nil { + return + } + return r.HTTPRequest.URL.String(), nil +} + +func (s *Storage) read(ctx context.Context, path string, w io.Writer, opt pairStorageRead) (n int64, err error) { + input, err := s.formatGetObjectInput(opt) + if err != nil { + return + } + + rp := s.getAbsPath(path) + + output, err := s.bucket.GetObjectWithContext(ctx, rp, input) + if err != nil { + return n, err + } + defer output.Body.Close() + + rc := output.Body + if opt.HasIoCallback { + rc = iowrap.CallbackReadCloser(rc, opt.IoCallback) + } + + return io.Copy(w, rc) +} + +func (s *Storage) stat(ctx context.Context, path string, opt pairStorageStat) (o *Object, err error) { + + rp := s.getAbsPath(path) + + if opt.HasMultipartID { + input := &service.ListMultipartInput{ + UploadID: service.String(opt.MultipartID), + Limit: service.Int(0), + } + _, err := s.bucket.ListMultipartWithContext(ctx, rp, input) + if err != nil { + return nil, err + } + + o = s.newObject(true) + o.ID = rp + o.Path = path + o.Mode |= ModePart + o.SetMultipartID(opt.MultipartID) + return o, nil + } + + if opt.HasObjectMode && opt.ObjectMode.IsDir() { + if !s.features.VirtualDir { + err = services.PairUnsupportedError{Pair: ps.WithObjectMode(opt.ObjectMode)} + return + } + + rp += "/" + } + + input := &service.HeadObjectInput{} + output, err := s.bucket.HeadObjectWithContext(ctx, rp, input) + if err != nil { + return + } + + o = s.newObject(true) + o.ID = rp + o.Path = path + + if output.XQSMetaData != nil { + metadata := *output.XQSMetaData + // By calling `HeadObject`, the first letter of the `key` of the object metadata will be capitalized. + if v, ok := metadata[metadataLinkTargetHeader]; ok { + // The path is a symlink object. + if !s.features.VirtualLink { + // The virtual link is not enabled, so we set the object mode to `ModeRead`. + o.Mode |= ModeRead + } else { + // qingstor does not have an absolute path, so when we call `getAbsPath`, it will remove the prefix `/`. + // To ensure that the path matches the one the user gets, we should re-add `/` here. + o.SetLinkTarget("/" + v) + o.Mode |= ModeLink + } + } + } + + if o.Mode&ModeLink == 0 && o.Mode&ModeRead == 0 { + if opt.HasObjectMode && opt.ObjectMode.IsDir() { + o.Mode |= ModeDir + } else { + o.Mode |= ModeRead + } + } + + o.SetContentLength(service.Int64Value(output.ContentLength)) + o.SetLastModified(service.TimeValue(output.LastModified)) + + if output.ContentType != nil { + o.SetContentType(service.StringValue(output.ContentType)) + } + if output.ETag != nil { + o.SetEtag(service.StringValue(output.ETag)) + } + + var sm ObjectSystemMetadata + if v := service.StringValue(output.XQSStorageClass); v != "" { + sm.StorageClass = v + } + if v := service.StringValue(output.XQSEncryptionCustomerAlgorithm); v != "" { + sm.EncryptionCustomerAlgorithm = v + } + o.SetSystemMetadata(sm) + + return o, nil +} + +func (s *Storage) write(ctx context.Context, path string, r io.Reader, size int64, opt pairStorageWrite) (n int64, err error) { + if size > writeSizeMaximum { + err = fmt.Errorf("size limit exceeded: %w", services.ErrRestrictionDissatisfied) + return + } + + // According to GSP-751, we should allow the user to pass in a nil io.Reader. + // ref: https://github.com/beyondstorage/go-storage/blob/master/docs/rfcs/751-write-empty-file-behavior.md + if r == nil && size != 0 { + return 0, fmt.Errorf("reader is nil but size is not 0") + } + + if opt.HasIoCallback { + r = iowrap.CallbackReader(r, opt.IoCallback) + } + + input, err := s.formatPutObjectInput(size, opt) + if err != nil { + return + } + input.Body = io.LimitReader(r, size) + + rp := s.getAbsPath(path) + + _, err = s.bucket.PutObjectWithContext(ctx, rp, input) + if err != nil { + return + } + return size, nil +} + +func (s *Storage) writeAppend(ctx context.Context, o *Object, r io.Reader, size int64, opt pairStorageWriteAppend) (n int64, err error) { + if size > appendSizeMaximum { + err = fmt.Errorf("size limit exceeded: %w", services.ErrRestrictionDissatisfied) + return + } + + rp := o.GetID() + + offset, _ := o.GetAppendOffset() + + input := &service.AppendObjectInput{ + Position: &offset, + ContentLength: &size, + Body: io.LimitReader(r, size), + } + if opt.HasContentMd5 { + input.ContentMD5 = &opt.ContentMd5 + } + + output, err := s.bucket.AppendObjectWithContext(ctx, rp, input) + if err != nil { + return + } + + if output == nil || output.XQSNextAppendPosition == nil { + err = ErrAppendNextPositionEmpty + return + } else { + offset = *output.XQSNextAppendPosition + } + + // We should reset the offset after calling `AppendObject` to prevent the offset being changed. + o.SetAppendOffset(offset) + + return size, nil +} + +func (s *Storage) writeMultipart(ctx context.Context, o *Object, r io.Reader, size int64, index int, opt pairStorageWriteMultipart) (n int64, part *Part, err error) { + if index < multipartNumberMinimum || index > multipartNumberMaximum { + err = ErrPartNumberInvalid + return + } + if size > multipartSizeMaximum { + err = fmt.Errorf("size limit exceeded: %w", services.ErrRestrictionDissatisfied) + return + } + + if opt.HasIoCallback { + r = iowrap.CallbackReader(r, opt.IoCallback) + } + + input := &service.UploadMultipartInput{ + PartNumber: service.Int(index), + UploadID: service.String(o.MustGetMultipartID()), + ContentLength: &size, + Body: io.LimitReader(r, size), + } + if opt.HasEncryptionCustomerAlgorithm { + input.XQSEncryptionCustomerAlgorithm, input.XQSEncryptionCustomerKey, input.XQSEncryptionCustomerKeyMD5, err = calculateEncryptionHeaders(opt.EncryptionCustomerAlgorithm, opt.EncryptionCustomerKey) + if err != nil { + return + } + } + + output, err := s.bucket.UploadMultipartWithContext(ctx, o.ID, input) + if err != nil { + return + } + + part = &Part{ + Index: index, + Size: size, + ETag: service.StringValue(output.ETag), + } + return size, part, nil +} diff --git a/services/qingstor/storager_test.go b/services/qingstor/storager_test.go new file mode 100644 index 000000000..ba233d3bf --- /dev/null +++ b/services/qingstor/storager_test.go @@ -0,0 +1,562 @@ +package qingstor + +import ( + "bytes" + "context" + "errors" + "io" + "io/ioutil" + "testing" + + "github.com/golang/mock/gomock" + "github.com/google/uuid" + "github.com/pengsrc/go-shared/convert" + qerror "github.com/qingstor/qingstor-sdk-go/v4/request/errors" + "github.com/qingstor/qingstor-sdk-go/v4/service" + "github.com/stretchr/testify/assert" + + "go.beyondstorage.io/v5/pairs" + "go.beyondstorage.io/v5/pkg/randbytes" + "go.beyondstorage.io/v5/services" + . "go.beyondstorage.io/v5/types" +) + +func TestStorage_String(t *testing.T) { + bucketName := "test_bucket" + zone := "test_zone" + c := Storage{ + workDir: "/test", + properties: &service.Properties{ + BucketName: &bucketName, + Zone: &zone, + }, + } + assert.NotEmpty(t, c.String()) +} + +func TestStorage_Metadata(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockBucket := NewMockBucket(ctrl) + + { + name := uuid.New().String() + location := uuid.New().String() + + client := Storage{ + bucket: mockBucket, + properties: &service.Properties{ + BucketName: &name, + Zone: &location, + }, + } + + m := client.Metadata() + assert.NotNil(t, m) + assert.Equal(t, name, m.Name) + assert.Equal(t, location, m.MustGetLocation()) + } +} + +func TestStorage_Copy(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockBucket := NewMockBucket(ctrl) + + name := uuid.New().String() + location := uuid.New().String() + + tests := []struct { + name string + src string + dst string + mockFn func(context.Context, string, *service.PutObjectInput) + hasError bool + wantErr error + }{ + { + "valid copy", + "test_src", "test_dst", + func(ctx context.Context, inputObjectKey string, input *service.PutObjectInput) { + assert.Equal(t, "test_dst", inputObjectKey) + assert.Equal(t, "/"+name+"/"+"test_src", *input.XQSCopySource) + }, + false, nil, + }, + } + + for _, v := range tests { + mockBucket.EXPECT().PutObjectWithContext(gomock.Eq(context.Background()), gomock.Any(), gomock.Any()).Do(v.mockFn) + + client := Storage{ + bucket: mockBucket, + properties: &service.Properties{ + BucketName: &name, + Zone: &location, + }, + } + + err := client.Copy(v.src, v.dst) + if v.hasError { + assert.Error(t, err) + assert.True(t, errors.Is(err, v.wantErr)) + } else { + assert.NoError(t, err) + } + } +} + +func TestStorage_Delete(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockBucket := NewMockBucket(ctrl) + + tests := []struct { + name string + src string + mockFn func(context.Context, string) + hasError bool + wantErr error + }{ + { + "valid delete", + "test_src", + func(ctx context.Context, inputObjectKey string) { + assert.Equal(t, "test_src", inputObjectKey) + }, + false, nil, + }, + } + + for _, v := range tests { + mockBucket.EXPECT().DeleteObjectWithContext(gomock.Eq(context.Background()), gomock.Any()).Do(v.mockFn) + + client := Storage{ + bucket: mockBucket, + } + + err := client.Delete(v.src) + if v.hasError { + assert.Error(t, err) + assert.True(t, errors.Is(err, v.wantErr)) + } else { + assert.NoError(t, err) + } + } +} + +func TestStorage_ListPrefix(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockBucket := NewMockBucket(ctrl) + + path := uuid.New().String() + key := uuid.New().String() + + mockBucket.EXPECT().ListObjectsWithContext(gomock.Eq(context.Background()), gomock.Any()). + DoAndReturn(func(ctx context.Context, input *service.ListObjectsInput) (*service.ListObjectsOutput, error) { + assert.Equal(t, path, *input.Prefix) + assert.Equal(t, 200, *input.Limit) + return &service.ListObjectsOutput{ + HasMore: service.Bool(false), + Keys: []*service.KeyType{ + { + Key: service.String(key), + }, + }, + }, nil + }) + + client := Storage{ + bucket: mockBucket, + } + + it, err := client.List(path, pairs.WithListMode(ListModePrefix)) + if err != nil { + t.Error(err) + } + object, err := it.Next() + assert.Equal(t, object.ID, key) + assert.Nil(t, err) +} + +func TestStorage_Move(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockBucket := NewMockBucket(ctrl) + + name := uuid.New().String() + location := uuid.New().String() + + tests := []struct { + name string + src string + dst string + mockFn func(context.Context, string, *service.PutObjectInput) + hasError bool + wantErr error + }{ + { + "valid copy", + "test_src", "test_dst", + func(ctx context.Context, inputObjectKey string, input *service.PutObjectInput) { + assert.Equal(t, "test_dst", inputObjectKey) + assert.Equal(t, "/"+name+"/"+"test_src", *input.XQSMoveSource) + }, + false, nil, + }, + } + + for _, v := range tests { + mockBucket.EXPECT().PutObjectWithContext(gomock.Eq(context.Background()), gomock.Any(), gomock.Any()).Do(v.mockFn) + + client := Storage{ + bucket: mockBucket, + properties: &service.Properties{ + BucketName: &name, + Zone: &location, + }, + } + + err := client.Move(v.src, v.dst) + if v.hasError { + assert.Error(t, err) + assert.True(t, errors.Is(err, v.wantErr)) + } else { + assert.NoError(t, err) + } + } +} + +func TestStorage_Read(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockBucket := NewMockBucket(ctrl) + + tests := []struct { + name string + path string + mockFn func(context.Context, string, *service.GetObjectInput) (*service.GetObjectOutput, error) + hasError bool + wantErr error + }{ + { + "valid copy", + "test_src", + func(ctx context.Context, inputPath string, input *service.GetObjectInput) (*service.GetObjectOutput, error) { + assert.Equal(t, "test_src", inputPath) + return &service.GetObjectOutput{ + Body: ioutil.NopCloser(bytes.NewBuffer([]byte("content"))), + }, nil + }, + false, nil, + }, + } + + for _, v := range tests { + mockBucket.EXPECT().GetObjectWithContext(gomock.Eq(context.Background()), gomock.Any(), gomock.Any()).DoAndReturn(v.mockFn) + + client := Storage{ + bucket: mockBucket, + } + + var buf bytes.Buffer + n, err := client.Read(v.path, &buf) + if v.hasError { + assert.Error(t, err) + assert.True(t, errors.Is(err, v.wantErr)) + } else { + assert.Equal(t, "content", buf.String()) + assert.Equal(t, int64(buf.Len()), n) + } + } +} + +func TestStorage_Stat(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockBucket := NewMockBucket(ctrl) + + tests := []struct { + name string + src string + mockFn func(context.Context, string, *service.HeadObjectInput) (*service.HeadObjectOutput, error) + hasError bool + wantErr error + }{ + { + "valid file", + "test_src", + func(ctx context.Context, objectKey string, input *service.HeadObjectInput) (*service.HeadObjectOutput, error) { + assert.Equal(t, "test_src", objectKey) + length := int64(100) + return &service.HeadObjectOutput{ + ContentLength: &length, + ContentType: convert.String("test_content_type"), + ETag: convert.String("test_etag"), + XQSStorageClass: convert.String("STANDARD"), + }, nil + }, + false, nil, + }, + } + + for _, v := range tests { + mockBucket.EXPECT().HeadObjectWithContext(gomock.Eq(context.Background()), gomock.Any(), gomock.Any()).DoAndReturn(v.mockFn) + + client := Storage{ + bucket: mockBucket, + } + + o, err := client.Stat(v.src) + if v.hasError { + assert.Error(t, err) + assert.True(t, errors.Is(err, v.wantErr)) + } else { + assert.NoError(t, err) + assert.NotNil(t, o) + assert.Equal(t, ModeRead, o.Mode) + assert.Equal(t, int64(100), o.MustGetContentLength()) + contentType, ok := o.GetContentType() + assert.True(t, ok) + assert.Equal(t, "test_content_type", contentType) + checkSum, ok := o.GetEtag() + assert.True(t, ok) + assert.Equal(t, "test_etag", checkSum) + + om := GetObjectSystemMetadata(o) + assert.Equal(t, StorageClassStandard, om.StorageClass) + } + } +} + +func TestStorage_Write(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockBucket := NewMockBucket(ctrl) + + tests := []struct { + name string + path string + size int64 + r io.Reader + mockFn func(context.Context, string, *service.PutObjectInput) (*service.PutObjectOutput, error) + hasError bool + wantErr error + }{ + { + "valid copy", + "test_src", + 100, + io.LimitReader(randbytes.NewRand(), 100), + func(ctx context.Context, inputPath string, input *service.PutObjectInput) (*service.PutObjectOutput, error) { + assert.Equal(t, "test_src", inputPath) + return nil, nil + }, + false, nil, + }, + } + + for _, v := range tests { + mockBucket.EXPECT().PutObjectWithContext(gomock.Eq(context.Background()), gomock.Any(), gomock.Any()).DoAndReturn(v.mockFn) + + client := Storage{ + bucket: mockBucket, + } + + n, err := client.Write(v.path, v.r, v.size) + if v.hasError { + assert.Error(t, err) + assert.True(t, errors.Is(err, v.wantErr)) + } else { + assert.NoError(t, err) + assert.Equal(t, v.size, n) + } + } +} + +func TestStorage_formatError(t *testing.T) { + s := &Storage{} + errCasual := errors.New("casual error") + cases := []struct { + name string + op string + err error + path string + targetErr error + targetEq bool + }{ + { + name: "nil error", + err: nil, + }, + { + name: "casual error", + op: "stat", + err: errCasual, + targetErr: services.ErrUnexpected, + targetEq: true, + }, + { + name: "not found with blank code", + op: "stat", + err: &qerror.QingStorError{ + StatusCode: 404, + Code: "", + Message: "msg", + }, + targetErr: services.ErrObjectNotExist, + targetEq: true, + }, + { + name: "not found by code", + op: "stat", + err: &qerror.QingStorError{ + StatusCode: 404, + Code: "object_not_exists", + Message: "msg", + }, + targetErr: services.ErrObjectNotExist, + targetEq: true, + }, + { + name: "permission denied by code", + op: "stat", + err: &qerror.QingStorError{ + StatusCode: 403, + Code: "permission_denied", + Message: "msg", + }, + targetErr: services.ErrPermissionDenied, + targetEq: true, + }, + { + name: "not found by code error not eq", + op: "stat", + err: qerror.QingStorError{ + StatusCode: 404, + Code: "object_not_exists", + Message: "msg", + }, + targetErr: services.ErrObjectNotExist, + targetEq: false, + }, + } + for _, tt := range cases { + err := s.formatError(tt.op, tt.err, tt.path) + if tt.err == nil { + assert.Nil(t, err, tt.name) + continue + } + + var storageErr services.StorageError + assert.True(t, errors.As(err, &storageErr), tt.name) + assert.Equal(t, tt.op, storageErr.Op, tt.name) + + assert.Equal(t, tt.targetEq, errors.Is(err, tt.targetErr), tt.name) + } +} + +func TestStorage_Fetch(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockBucket := NewMockBucket(ctrl) + + { + client := Storage{ + bucket: mockBucket, + } + + name := uuid.New().String() + url := uuid.New().String() + + mockBucket.EXPECT().PutObjectWithContext(gomock.Eq(context.Background()), gomock.Any(), gomock.Any()). + DoAndReturn(func(ctx context.Context, objectKey string, input *service.PutObjectInput) (*service.PutObjectOutput, error) { + assert.Equal(t, name, objectKey) + assert.Equal(t, *input.XQSFetchSource, url) + return &service.PutObjectOutput{}, nil + }) + err := client.Fetch(name, url) + assert.NoError(t, err) + } + + { + client := Storage{ + bucket: mockBucket, + } + + name := uuid.New().String() + url := uuid.New().String() + + mockBucket.EXPECT().PutObjectWithContext(gomock.Eq(context.Background()), gomock.Any(), gomock.Any()). + DoAndReturn(func(ctx context.Context, objectKey string, input *service.PutObjectInput) (*service.PutObjectOutput, error) { + return nil, &qerror.QingStorError{} + }) + err := client.Fetch(name, url) + assert.Error(t, err) + } +} + +func TestStorage_Create(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockBucket := NewMockBucket(ctrl) + + wd := "/test/" + c := Storage{ + workDir: wd, + bucket: mockBucket, + } + + cases := []struct { + name string + path string + multipartID string + }{ + { + name: "normal object", + path: uuid.NewString(), + multipartID: "", + }, + { + name: "multipart object", + path: uuid.NewString(), + multipartID: uuid.NewString(), + }, + } + + mockBucket.EXPECT().HeadObjectWithContext(gomock.Eq(context.Background()), gomock.Any(), gomock.Any()). + DoAndReturn(func(ctx context.Context, objectKey string, input *service.HeadObjectInput) (*service.HeadObjectOutput, error) { + return &service.HeadObjectOutput{}, nil + }).Times(1) + + for _, tt := range cases { + ps := make([]Pair, 0) + if tt.multipartID != "" { + ps = append(ps, pairs.WithMultipartID(tt.multipartID)) + } + obj := c.Create(tt.path, ps...) + assert.NotNil(t, obj) + assert.Equal(t, c.getAbsPath(tt.path), obj.ID) + assert.Equal(t, tt.path, obj.Path) + if tt.multipartID != "" { + assert.Equal(t, tt.multipartID, obj.MustGetMultipartID()) + assert.Equal(t, ModePart, obj.Mode) + } else { + assert.Equal(t, ModeRead, obj.Mode) + assert.Panics(t, func() { + obj.MustGetMultipartID() + }) + } + } +} diff --git a/services/qingstor/tests/README.md b/services/qingstor/tests/README.md new file mode 100644 index 000000000..20609c707 --- /dev/null +++ b/services/qingstor/tests/README.md @@ -0,0 +1,32 @@ +## How run integration tests + +### Run tests locally + +Copy example files and update corresponding values. + +```shell +cp Makefile.env.exmaple Makefile.env +``` + +Run tests + +```shell +make integration_test +``` + +### Run tests in CI + +Set following environment variables: + +```shell +export STORAGE_QINGSTOR_INTEGRATION_TEST=on +export STORAGE_QINGSTOR_CREDENTIAL=hamc:access_key:secret_key +export STORAGE_QINGSTOR_ENDPOINT=https:qingstor.com:443 +export STORAGE_QINGSTOR_NAME=bucketname +``` + +Run tests + +```shell +make integration_test +``` diff --git a/services/qingstor/tests/storage_test.go b/services/qingstor/tests/storage_test.go new file mode 100644 index 000000000..3fa2df176 --- /dev/null +++ b/services/qingstor/tests/storage_test.go @@ -0,0 +1,51 @@ +package tests + +import ( + "os" + "testing" + + "go.beyondstorage.io/v5/tests" +) + +func TestStorage(t *testing.T) { + if os.Getenv("STORAGE_QINGSTOR_INTEGRATION_TEST") != "on" { + t.Skipf("STORAGE_QINGSTOR_INTEGRATION_TEST is not 'on', skipped") + } + tests.TestStorager(t, setupTest(t)) +} + +func TestMultiparter(t *testing.T) { + if os.Getenv("STORAGE_QINGSTOR_INTEGRATION_TEST") != "on" { + t.Skipf("STORAGE_QINGSTOR_INTEGRATION_TEST is not 'on', skipped") + } + tests.TestMultiparter(t, setupTest(t)) +} + +func TestAppend(t *testing.T) { + if os.Getenv("STORAGE_QINGSTOR_INTEGRATION_TEST") != "on" { + t.Skipf("STORAGE_QINGSTOR_INTEGRATION_TEST is not 'on', skipped") + } + tests.TestAppender(t, setupTest(t)) +} + +func TestDirer(t *testing.T) { + if os.Getenv("STORAGE_QINGSTOR_INTEGRATION_TEST") != "on" { + t.Skipf("STORAGE_QINGSTOR_INTEGRATION_TEST is not 'on', skipped") + } + tests.TestDirer(t, setupTest(t)) +} + +func TestLinker(t *testing.T) { + if os.Getenv("STORAGE_QINGSTOR_INTEGRATION_TEST") != "on" { + t.Skipf("STORAGE_QINGSTOR_INTEGRATION_TEST is not 'on', skipped") + } + tests.TestLinker(t, setupTest(t)) +} + +func TestSigner(t *testing.T) { + if os.Getenv("STORAGE_QINGSTOR_INTEGRATION_TEST") != "on" { + t.Skipf("STORAGE_QINGSTOR_INTEGRATION_TEST is not 'on', skipped") + } + tests.TestStorageHTTPSignerRead(t, setupTest(t)) + tests.TestStorageHTTPSignerWrite(t, setupTest(t)) +} diff --git a/services/qingstor/tests/utils_test.go b/services/qingstor/tests/utils_test.go new file mode 100644 index 000000000..0b5f7b6da --- /dev/null +++ b/services/qingstor/tests/utils_test.go @@ -0,0 +1,31 @@ +package tests + +import ( + "os" + "testing" + + "github.com/google/uuid" + + "go.beyondstorage.io/services/qingstor/v4" + ps "go.beyondstorage.io/v5/pairs" + "go.beyondstorage.io/v5/types" +) + +func setupTest(t *testing.T) types.Storager { + t.Log("Setup test for qingstor") + + store, err := qingstor.NewStorager( + ps.WithCredential(os.Getenv("STORAGE_QINGSTOR_CREDENTIAL")), + ps.WithEndpoint(os.Getenv("STORAGE_QINGSTOR_ENDPOINT")), + ps.WithName(os.Getenv("STORAGE_QINGSTOR_NAME")), + ps.WithWorkDir("/"+uuid.New().String()+"/"), + qingstor.WithStorageFeatures(qingstor.StorageFeatures{ + VirtualDir: true, + VirtualLink: true, + }), + ) + if err != nil { + t.Errorf("new storager: %v", err) + } + return store +} diff --git a/services/qingstor/tools.go b/services/qingstor/tools.go new file mode 100644 index 000000000..0d3b96caf --- /dev/null +++ b/services/qingstor/tools.go @@ -0,0 +1,8 @@ +//go:build tools +// +build tools + +package qingstor + +import ( + _ "go.beyondstorage.io/v5/cmd/definitions" +) diff --git a/services/qingstor/utils.go b/services/qingstor/utils.go new file mode 100644 index 000000000..d48c47102 --- /dev/null +++ b/services/qingstor/utils.go @@ -0,0 +1,477 @@ +package qingstor + +import ( + "crypto/md5" + "encoding/base64" + "errors" + "fmt" + "net/http" + "net/url" + "regexp" + "strings" + "time" + + "github.com/pengsrc/go-shared/convert" + qsconfig "github.com/qingstor/qingstor-sdk-go/v4/config" + iface "github.com/qingstor/qingstor-sdk-go/v4/interface" + qserror "github.com/qingstor/qingstor-sdk-go/v4/request/errors" + "github.com/qingstor/qingstor-sdk-go/v4/service" + + "go.beyondstorage.io/credential" + "go.beyondstorage.io/endpoint" + ps "go.beyondstorage.io/v5/pairs" + "go.beyondstorage.io/v5/pkg/headers" + "go.beyondstorage.io/v5/pkg/httpclient" + "go.beyondstorage.io/v5/services" + typ "go.beyondstorage.io/v5/types" +) + +// Service is the qingstor service config. +type Service struct { + config *qsconfig.Config + service iface.Service + + client *http.Client + + defaultPairs DefaultServicePairs + features ServiceFeatures + + typ.UnimplementedServicer +} + +// String implements Service.String. +func (s *Service) String() string { + if s.config == nil { + return fmt.Sprintf("Servicer qingstor") + } + return fmt.Sprintf("Servicer qingstor {AccessKey: %s}", s.config.AccessKeyID) +} + +// Storage is the qingstor object storage client. +type Storage struct { + bucket iface.Bucket + config *qsconfig.Config + properties *service.Properties + + defaultPairs DefaultStoragePairs + features StorageFeatures + + // options for this storager. + workDir string // workDir dir for all operation. + + typ.UnimplementedStorager + typ.UnimplementedCopier + typ.UnimplementedFetcher + typ.UnimplementedMover + typ.UnimplementedMultiparter + typ.UnimplementedReacher + typ.UnimplementedAppender + typ.UnimplementedDirer + typ.UnimplementedLinker + typ.UnimplementedStorageHTTPSigner +} + +// String implements Storager.String +func (s *Storage) String() string { + // qingstor work dir should start and end with "/" + return fmt.Sprintf( + "Storager qingstor {Name: %s, Location: %s, WorkDir: %s}", + *s.properties.BucketName, *s.properties.Zone, s.workDir, + ) +} + +// New will create both Servicer and Storager. +func New(pairs ...typ.Pair) (typ.Servicer, typ.Storager, error) { + return newServicerAndStorager(pairs...) +} + +// NewServicer will create Servicer only. +func NewServicer(pairs ...typ.Pair) (typ.Servicer, error) { + return newServicer(pairs...) +} + +// NewStorager will create Storager only. +func NewStorager(pairs ...typ.Pair) (typ.Storager, error) { + _, store, err := newServicerAndStorager(pairs...) + return store, err +} + +func newServicer(pairs ...typ.Pair) (srv *Service, err error) { + defer func() { + if err != nil { + err = services.InitError{Op: "new_servicer", Type: Type, Err: formatError(err), Pairs: pairs} + } + }() + + opt, err := parsePairServiceNew(pairs) + if err != nil { + return nil, err + } + + srv = &Service{ + client: httpclient.New(opt.HTTPClientOptions), + } + + var cfg *qsconfig.Config + + // Set config's credential. + cp, err := credential.Parse(opt.Credential) + if err != nil { + return nil, err + } + switch cp.Protocol() { + case credential.ProtocolHmac: + cfg, err = qsconfig.New(cp.Hmac()) + if err != nil { + return nil, err + } + default: + return nil, services.PairUnsupportedError{Pair: ps.WithCredential(opt.Credential)} + } + + // Set config's endpoint + if opt.HasEndpoint { + ep, err := endpoint.Parse(opt.Endpoint) + if err != nil { + return nil, err + } + + switch ep.Protocol() { + case endpoint.ProtocolHTTPS: + _, cfg.Host, cfg.Port = ep.HTTPS() + case endpoint.ProtocolHTTP: + _, cfg.Host, cfg.Port = ep.HTTP() + default: + return nil, services.PairUnsupportedError{Pair: ps.WithEndpoint(opt.Endpoint)} + } + + cfg.Protocol = ep.Protocol() + } + // Set config's http client + cfg.Connection = srv.client + + srv.config = cfg + srv.service, _ = service.Init(cfg) + + if opt.HasDefaultServicePairs { + srv.defaultPairs = opt.DefaultServicePairs + } + if opt.HasServiceFeatures { + srv.features = opt.ServiceFeatures + } + return +} + +// New will create a new qingstor service. +func newServicerAndStorager(pairs ...typ.Pair) (srv *Service, store *Storage, err error) { + srv, err = newServicer(pairs...) + if err != nil { + return + } + + store, err = srv.newStorage(pairs...) + if err != nil { + err = services.InitError{Op: "new_storager", Type: Type, Err: formatError(err), Pairs: pairs} + return + } + return +} + +// multipartXXX are multipart upload restriction in QingStor, see more detail at: +// https://docs.qingcloud.com/qingstor/api/object/multipart/index.html#%E5%88%86%E6%AE%B5%E4%B8%8A%E4%BC%A0%E9%99%90%E5%88%B6 +const ( + // multipartNumberMinimum is the min part count supported + multipartNumberMinimum = 0 + // multipartNumberMaximum is the max part count supported + multipartNumberMaximum = 10000 + // multipartSizeMaximum is the maximum size for each part, 5GB + multipartSizeMaximum = 5 * 1024 * 1024 * 1024 + // multipartSizeMinimum is the minimum size for each part, except the last part, 4MB + multipartSizeMinimum = 4 * 1024 * 1024 +) + +const ( + // writeSizeMaximum is the maximum size for write operation, 5GB. + // ref: https://docs.qingcloud.com/qingstor/#object + writeSizeMaximum = 5 * 1024 * 1024 * 1024 + // copySizeMaximum is the maximum size for copy operation, 5GB. + // ref: https://docs.qingcloud.com/qingstor/api/object/copy + copySizeMaximum = 5 * 1024 * 1024 * 1024 + // appendSizeMaximum is the maximum append size for per append operation, 5GB. + // ref: https://docs.qingcloud.com/qingstor/api/object/append + appendSizeMaximum = 5 * 1024 * 1024 * 1024 + // appendSizeMaximum is the total maximum size for an append object, 5TB. + // ref: https://docs.qingcloud.com/qingstor/api/object/append + appendTotalSizeMaximum = 50 * 1024 * 1024 * 1024 * 1024 +) + +// bucketNameRegexp is the bucket name regexp, which indicates: +// 1. length: 6-63; +// 2. contains lowercase letters, digits and strikethrough; +// 3. starts and ends with letter or digit. +var bucketNameRegexp = regexp.MustCompile(`^[a-z\d][a-z-\d]{4,61}[a-z\d]$`) + +// IsBucketNameValid will check whether given string is a valid bucket name. +func IsBucketNameValid(s string) bool { + return bucketNameRegexp.MatchString(s) +} + +func formatError(err error) error { + if _, ok := err.(services.InternalError); ok { + return err + } + + // Handle errors returned by qingstor. + var e *qserror.QingStorError + if !errors.As(err, &e) { + return fmt.Errorf("%w: %v", services.ErrUnexpected, err) + } + + switch e.Code { + case "": + // code=="" means this response doesn't have body. + switch e.StatusCode { + case 404: + return fmt.Errorf("%w: %v", services.ErrObjectNotExist, e) + default: + return e + } + case "permission_denied": + return fmt.Errorf("%w: %v", services.ErrPermissionDenied, e) + case "object_not_exists": + return fmt.Errorf("%w: %v", services.ErrObjectNotExist, e) + default: + return fmt.Errorf("%w: %v", services.ErrUnexpected, err) + } +} + +func convertUnixTimestampToTime(v int) time.Time { + if v == 0 { + return time.Time{} + } + return time.Unix(int64(v), 0) +} + +// All available storage classes are listed here. +const ( + StorageClassStandard = "STANDARD" + StorageClassStandardIA = "STANDARD_IA" +) + +func (s *Service) newStorage(pairs ...typ.Pair) (store *Storage, err error) { + opt, err := parsePairStorageNew(pairs) + if err != nil { + return + } + + // WorkDir should be an abs path, start and ends with "/" + if opt.HasWorkDir && !isWorkDirValid(opt.WorkDir) { + err = ErrWorkDirInvalid + return + } + // set work dir into root path if no work dir passed + if !opt.HasWorkDir { + opt.WorkDir = "/" + } + + if !IsBucketNameValid(opt.Name) { + err = ErrBucketNameInvalid + return + } + + // Detect location automatically + if !opt.HasLocation { + opt.Location, err = s.detectLocation(opt.Name) + if err != nil { + return + } + } + + bucket, err := s.service.Bucket(opt.Name, opt.Location) + if err != nil { + return + } + + st := &Storage{ + bucket: bucket, + config: bucket.Config, + properties: bucket.Properties, + + workDir: "/", + } + + if opt.HasDisableURICleaning { + st.config.DisableURICleaning = opt.DisableURICleaning + } + if opt.HasDefaultStoragePairs { + st.defaultPairs = opt.DefaultStoragePairs + } + if opt.HasStorageFeatures { + st.features = opt.StorageFeatures + } + if opt.HasWorkDir { + st.workDir = opt.WorkDir + } + return st, nil +} + +func (s *Service) detectLocation(name string) (location string, err error) { + defer func() { + err = s.formatError("detect_location", err, "") + }() + + u := fmt.Sprintf("%s://%s:%d/%s", s.config.Protocol, s.config.Host, s.config.Port, name) + + r, err := s.client.Head(u) + if err != nil { + return + } + if r.StatusCode != http.StatusMovedPermanently { + err = fmt.Errorf("%w: head status is %d instead of %d", services.ErrUnexpected, r.StatusCode, http.StatusMovedPermanently) + return + } + + // Example URL: https://zone.qingstor.com/bucket + locationUrl, err := url.Parse(r.Header.Get(headers.Location)) + if err != nil { + return + } + location = strings.Split(locationUrl.Host, ".")[0] + return +} + +func (s *Service) formatError(op string, err error, name string) error { + if err == nil { + return nil + } + + return services.ServiceError{ + Op: op, + Err: formatError(err), + Servicer: s, + Name: name, + } +} + +// isWorkDirValid check qingstor work dir +// work dir must start with only one "/" (abs path), and end with only one "/" (a dir). +// If work dir is the root path, set it to "/". +func isWorkDirValid(wd string) bool { + return strings.HasPrefix(wd, "/") && // must start with "/" + strings.HasSuffix(wd, "/") && // must end with "/" + !strings.HasPrefix(wd, "//") && // not start with more than one "/" + !strings.HasSuffix(wd, "//") // not end with more than one "/" +} + +// getAbsPath will calculate object storage's abs path +func (s *Storage) getAbsPath(path string) string { + prefix := strings.TrimPrefix(s.workDir, "/") + return prefix + path +} + +// getRelPath will get object storage's rel path. +func (s *Storage) getRelPath(path string) string { + prefix := strings.TrimPrefix(s.workDir, "/") + return strings.TrimPrefix(path, prefix) +} + +func (s *Storage) formatError(op string, err error, path ...string) error { + if err == nil { + return nil + } + + return services.StorageError{ + Op: op, + Err: formatError(err), + Storager: s, + Path: path, + } +} + +func (s *Storage) newObject(done bool) *typ.Object { + return typ.NewObject(s, done) +} + +func (s *Storage) formatFileObject(v *service.KeyType) (o *typ.Object, err error) { + o = s.newObject(false) + o.ID = *v.Key + o.Path = s.getRelPath(*v.Key) + // If you have enabled virtual link, you will not get the accurate object type. + // If you want to get the exact object mode, please use `stat` + o.Mode |= typ.ModeRead + + o.SetContentLength(service.Int64Value(v.Size)) + o.SetLastModified(convertUnixTimestampToTime(service.IntValue(v.Modified))) + + if v.MimeType != nil { + o.SetContentType(service.StringValue(v.MimeType)) + } + if v.Etag != nil { + o.SetEtag(service.StringValue(v.Etag)) + } + + var sm ObjectSystemMetadata + if value := service.StringValue(v.StorageClass); value != "" { + sm.StorageClass = value + } + o.SetSystemMetadata(sm) + + return o, nil +} + +func isObjectDirectory(o *service.KeyType) bool { + return convert.StringValue(o.MimeType) == "application/x-directory" +} + +// All available SSE customer algorithms are listed here. +const ( + SseCustomerAlgorithmAes256 = "AES256" +) + +func calculateEncryptionHeaders(algo string, key []byte) (algorithm, keyBase64, keyMD5Base64 *string, err error) { + if len(key) != 32 { + err = ErrEncryptionCustomerKeyInvalid + return + } + kB64 := base64.StdEncoding.EncodeToString(key) + kMD5 := md5.Sum(key) + kMD5B64 := base64.StdEncoding.EncodeToString(kMD5[:]) + return &algo, &kB64, &kMD5B64, nil +} + +func (s *Storage) formatGetObjectInput(opt pairStorageRead) (input *service.GetObjectInput, err error) { + input = &service.GetObjectInput{} + if opt.HasEncryptionCustomerAlgorithm { + input.XQSEncryptionCustomerAlgorithm, input.XQSEncryptionCustomerKey, input.XQSEncryptionCustomerKeyMD5, err = calculateEncryptionHeaders(opt.EncryptionCustomerAlgorithm, opt.EncryptionCustomerKey) + if err != nil { + return + } + } + + if opt.HasOffset || opt.HasSize { + rs := headers.FormatRange(opt.Offset, opt.Size) + input.Range = &rs + } + + return +} + +func (s *Storage) formatPutObjectInput(size int64, opt pairStorageWrite) (input *service.PutObjectInput, err error) { + input = &service.PutObjectInput{ + ContentLength: &size, + } + if opt.HasContentMd5 { + input.ContentMD5 = service.String(opt.ContentMd5) + } + if opt.HasStorageClass { + input.XQSStorageClass = service.String(opt.StorageClass) + } + if opt.HasEncryptionCustomerAlgorithm { + input.XQSEncryptionCustomerAlgorithm, input.XQSEncryptionCustomerKey, input.XQSEncryptionCustomerKeyMD5, err = calculateEncryptionHeaders(opt.EncryptionCustomerAlgorithm, opt.EncryptionCustomerKey) + if err != nil { + return + } + } + + return +} diff --git a/services/qingstor/utils_test.go b/services/qingstor/utils_test.go new file mode 100644 index 000000000..8a5ce3c65 --- /dev/null +++ b/services/qingstor/utils_test.go @@ -0,0 +1,207 @@ +package qingstor + +import ( + "errors" + "testing" + + "github.com/golang/mock/gomock" + "github.com/google/uuid" + "github.com/pengsrc/go-shared/convert" + qserror "github.com/qingstor/qingstor-sdk-go/v4/request/errors" + "github.com/qingstor/qingstor-sdk-go/v4/service" + "github.com/stretchr/testify/assert" + + "go.beyondstorage.io/credential" + "go.beyondstorage.io/endpoint" + "go.beyondstorage.io/v5/pairs" + "go.beyondstorage.io/v5/services" +) + +func Test_New(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + // Missing required pair + _, _, err := New() + assert.Error(t, err) + assert.True(t, errors.Is(err, services.ErrRestrictionDissatisfied)) + + // Valid case + accessKey := uuid.New().String() + secretKey := uuid.New().String() + host := uuid.New().String() + name := uuid.New().String() + port := 1234 + srv, store, err := New( + pairs.WithCredential(credential.NewHmac(accessKey, secretKey).String()), + pairs.WithEndpoint(endpoint.NewHTTP(host, port).String()), + pairs.WithLocation("test"), + pairs.WithName(name), + ) + assert.NoError(t, err) + assert.NotNil(t, srv) + assert.NotNil(t, store) +} + +func TestIsBucketNameValid(t *testing.T) { + tests := []struct { + name string + args string + want bool + }{ + {"start with letter", "a-bucket-test", true}, + {"start with digit", "0-bucket-test", true}, + {"start with strike", "-bucket-test", false}, + {"end with strike", "bucket-test-", false}, + {"too short", "abcd", false}, + {"too long (64)", "abcdefghijklmnopqrstuvwxyz123456abcdefghijklmnopqrstuvwxyz123456", false}, + {"contains illegal char", "abcdefg_1234", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := IsBucketNameValid(tt.args); got != tt.want { + t.Errorf("IsBucketNameValid() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestGetAbsPath(t *testing.T) { + cases := []struct { + name string + base string + path string + expectedPath string + }{ + {"under root", "/", "abc", "abc"}, + {"under prefix", "/root", "/abc", "root/abc"}, + {"under prefix ending with /", "/root/", "abc", "root/abc"}, + {"under unexpected prefix", "//abc", "/def", "/abc/def"}, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + client := Storage{workDir: tt.base} + + gotPath := client.getAbsPath(tt.path) + assert.Equal(t, tt.expectedPath, gotPath) + }) + } +} + +func TestGetRelPath(t *testing.T) { + cases := []struct { + name string + base string + path string + expectedPath string + }{ + {"under root", "/", "abc", "abc"}, + {"under prefix", "/root", "root/abc", "/abc"}, + {"under prefix ending with /", "/root/", "root/abc", "abc"}, + {"under unexpected prefix", "//abc", "/abc/def", "/def"}, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + client := &Storage{workDir: tt.base} + + gotPath := client.getRelPath(tt.path) + assert.Equal(t, tt.expectedPath, gotPath) + }) + } +} + +func TestHandleQingStorError(t *testing.T) { + { + tests := []struct { + name string + input *qserror.QingStorError + expected error + }{ + { + "not found", + &qserror.QingStorError{ + StatusCode: 404, + Code: "", + Message: "", + RequestID: "", + ReferenceURL: "", + }, + services.ErrObjectNotExist, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.True(t, errors.Is(formatError(tt.input), tt.expected)) + }) + } + } + + { + tests := []struct { + name string + input *qserror.QingStorError + expected error + }{ + { + "permission_denied", + &qserror.QingStorError{ + StatusCode: 403, + Code: "permission_denied", + Message: "", + RequestID: "", + ReferenceURL: "", + }, + services.ErrPermissionDenied, + }, + { + "object_not_exists", + &qserror.QingStorError{ + StatusCode: 404, + Code: "object_not_exists", + Message: "", + RequestID: "", + ReferenceURL: "", + }, + services.ErrObjectNotExist, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.True(t, errors.Is(formatError(tt.input), tt.expected)) + }) + } + } +} + +func Test_isObjectDirectory(t *testing.T) { + tests := []struct { + name string + keyType string + want bool + }{ + { + name: "false", + keyType: "application/octet-stream", + want: false, + }, + { + name: "true", + keyType: "application/x-directory", + want: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + o := &service.KeyType{ + MimeType: convert.String(tt.keyType), + } + if got := isObjectDirectory(o); got != tt.want { + t.Errorf("isObjectDirectory() = %v, want %v", got, tt.want) + } + }) + } +} From e8207d63ee9f56ed101f12896c1dd1ae0b40c9d3 Mon Sep 17 00:00:00 2001 From: Joey-1445601153 <1445601153@qq.com> Date: Thu, 21 Oct 2021 11:35:13 +0800 Subject: [PATCH 2/4] Change usage of ioutil --- services/qingstor/storager_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/services/qingstor/storager_test.go b/services/qingstor/storager_test.go index ba233d3bf..f9704e3cb 100644 --- a/services/qingstor/storager_test.go +++ b/services/qingstor/storager_test.go @@ -5,7 +5,6 @@ import ( "context" "errors" "io" - "io/ioutil" "testing" "github.com/golang/mock/gomock" @@ -252,7 +251,7 @@ func TestStorage_Read(t *testing.T) { func(ctx context.Context, inputPath string, input *service.GetObjectInput) (*service.GetObjectOutput, error) { assert.Equal(t, "test_src", inputPath) return &service.GetObjectOutput{ - Body: ioutil.NopCloser(bytes.NewBuffer([]byte("content"))), + Body: io.NopCloser(bytes.NewBuffer([]byte("content"))), }, nil }, false, nil, From aaab679c85e9a7b40121f43bdc07e2c7010b6072 Mon Sep 17 00:00:00 2001 From: Joey-1445601153 <1445601153@qq.com> Date: Thu, 21 Oct 2021 12:49:54 +0800 Subject: [PATCH 3/4] Add qingstor back --- .github/dependabot.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 589b2460e..e538c8cae 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -71,4 +71,4 @@ updates: - package-ecosystem: gomod directory: "/services/qingstor" schedule: - interval: daily \ No newline at end of file + interval: daily From f31e16ebd239d7150696cd6632c317d9cc200c01 Mon Sep 17 00:00:00 2001 From: Joey-1445601153 <1445601153@qq.com> Date: Thu, 21 Oct 2021 13:25:06 +0800 Subject: [PATCH 4/4] process conflict --- .github/dependabot.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index e538c8cae..bb8f51f60 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -68,6 +68,10 @@ updates: directory: "/services/onedrive" schedule: interval: daily + - package-ecosystem: gomod + directory: "/services/oss" + schedule: + interval: daily - package-ecosystem: gomod directory: "/services/qingstor" schedule: