objstore : implement Aliyun OSS (#1573)

* add oss support Signed-off-by: wujinhu <[email protected]> * fix docs Signed-off-by: wujinhu <[email protected]> * fix Makefile Signed-off-by: wujinhu <[email protected]> * review comments Signed-off-by: wujinhu <[email protected]> * fix style Signed-off-by: wujinhu <[email protected]> * review comments Signed-off-by: wujinhu <[email protected]> * review comments Signed-off-by: wujinhu <[email protected]> * review comments Signed-off-by: wujinhu <[email protected]> * review comments Signed-off-by: wujinhu <[email protected]>
thanos-io · Oct 24, 2019 · 66b3d21 · 66b3d21
1 parent 7005b65
commit 66b3d21
Show file tree

Hide file tree

Showing 10 changed files with 406 additions and 13 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -14,6 +14,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel
 ### Added
 
 - [#1660](https://github.com/thanos-io/thanos/pull/1660) Add a new `--prometheus.ready_timeout` CLI option to the sidecar to set how long to wait until Prometheus starts up.
+- [#1573](https://github.com/thanos-io/thanos/pull/1573) `AliYun OSS` object storage, see [documents](docs/storage.md#aliyun-oss-configuration) for further information.
 
 ### Fixed
 

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -57,6 +57,7 @@ $ git push origin <your_branch_for_new_pr>
 - THANOS_SKIP_AZURE_TESTS to skip Azure tests.
 - THANOS_SKIP_SWIFT_TESTS to skip SWIFT tests.
 - THANOS_SKIP_TENCENT_COS_TESTS to skip Tencent COS tests.
+- THANOS_SKIP_ALIYUN_OSS_TESTS to skip Aliyun OSS tests.
 
 If you skip all of these, the store specific tests will be run against memory object storage only.
 CI runs GCS and inmem tests only for now. Not having these variables will produce auth errors against GCS, AWS, Azure or COS tests.

diff --git a/Makefile b/Makefile
@@ -160,8 +160,8 @@ docs: $(EMBEDMD) build
 .PHONY: check-docs
 check-docs: $(EMBEDMD) $(LICHE) build
 	@EMBEDMD_BIN="$(EMBEDMD)" scripts/genflagdocs.sh check
-	@$(LICHE) --recursive docs --exclude "cloud.tencent.com" --document-root .
-	@$(LICHE) --exclude "cloud.tencent.com|goreportcard.com" --document-root . *.md
+	@$(LICHE) --recursive docs --exclude "(cloud.tencent.com|alibabacloud.com)" --document-root .
+	@$(LICHE) --exclude "(cloud.tencent.com|goreportcard.com|alibabacloud.com)" --document-root . *.md
 
 # checks Go code comments if they have trailing period (excludes protobuffers and vendor files).
 # Comments with more than 3 spaces at beginning are omitted from the check, example: '//    - foo'.
@@ -203,17 +203,19 @@ test: check-git install-deps
 	@go install github.com/thanos-io/thanos/cmd/thanos
 	# Be careful on GOCACHE. Those tests are sometimes using built Thanos/Prometheus binaries directly. Don't cache those.
 	@rm -rf ${GOCACHE}
-	@echo ">> running all tests. Do export THANOS_SKIP_GCS_TESTS='true' or/and THANOS_SKIP_S3_AWS_TESTS='true' or/and THANOS_SKIP_AZURE_TESTS='true' and/or THANOS_SKIP_SWIFT_TESTS='true' and/or THANOS_SKIP_TENCENT_COS_TESTS='true' if you want to skip e2e tests against real store buckets"
+	@echo ">> running all tests. Do export THANOS_SKIP_GCS_TESTS='true' or/and THANOS_SKIP_S3_AWS_TESTS='true' or/and THANOS_SKIP_AZURE_TESTS='true' and/or THANOS_SKIP_SWIFT_TESTS='true' and/or THANOS_SKIP_ALIYUN_OSS_TESTS='true' and/or THANOS_SKIP_TENCENT_COS_TESTS='true' if you want to skip e2e tests against real store buckets"
 	@go test $(shell go list ./... | grep -v /vendor/);
 
 .PHONY: test-ci
 test-ci: export THANOS_SKIP_AZURE_TESTS = true
 test-ci: export THANOS_SKIP_SWIFT_TESTS = true
 test-ci: export THANOS_SKIP_TENCENT_COS_TESTS = true
+test-ci: export THANOS_SKIP_ALIYUN_OSS_TESTS = true
 test-ci:
 	@echo ">> Skipping AZURE tests"
 	@echo ">> Skipping SWIFT tests"
 	@echo ">> Skipping TENCENT tests"
+	@echo ">> Skipping ALIYUN tests"
 	$(MAKE) test
 
 .PHONY: test-local

diff --git a/docs/storage.md b/docs/storage.md
@@ -60,6 +60,7 @@ Current object storage client implementations:
 | [Azure Storage Account](./storage.md#azure) | Stable  (production usage) | yes       | @vglafirov   |
 | [OpenStack Swift](./storage.md#openstack-swift)      | Beta  (working PoCs, testing usage)               | no        | @sudhi-vm   |
 | [Tencent COS](./storage.md#tencent-cos)          | Beta  (testing usage)                   | no        | @jojohappy          |
+| [AliYun OSS](./storage.md#aliyun-oss)           | Beta  (testing usage)                   | no        | @shaulboozhiao,@wujinhu      |
 
 NOTE: Currently Thanos requires strong consistency (write-read) for object store implementation.
 
@@ -336,3 +337,20 @@ config:
 ```
 
 Set the flags `--objstore.config-file` to reference to the configuration file.
+
+##  AliYun OSS Configuration
+In order to use AliYun OSS object storage, you should first create a bucket with proper Storage Class , ACLs and get the access key on the AliYun cloud. Go to [https://www.alibabacloud.com/product/oss](https://www.alibabacloud.com/product/oss) for more detail.
+
+To use AliYun OSS object storage, please specify following yaml configuration file in `objstore.config*` flag.
+
+[embedmd]:# (flags/config_bucket_aliyunoss.txt $)
+```$
+type: ALIYUNOSS
+config:
+  endpoint: ""
+  bucket: ""
+  access_key_id: ""
+  access_key_secret: ""
+```
+
+Use --objstore.config-file to reference to this configuration file.
diff --git a/go.mod b/go.mod
@@ -4,7 +4,9 @@ require (
 	cloud.google.com/go v0.44.1
 	github.com/Azure/azure-storage-blob-go v0.7.0
 	github.com/NYTimes/gziphandler v1.1.1
+	github.com/aliyun/aliyun-oss-go-sdk v2.0.1+incompatible
 	github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878
+	github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect
 	github.com/cespare/xxhash v1.1.0
 	github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect
 	github.com/fatih/structtag v1.0.0

diff --git a/go.sum b/go.sum
@@ -35,6 +35,8 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZq
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/aliyun/aliyun-oss-go-sdk v2.0.1+incompatible h1:/MzpJOMHn/uBtd1dkS7Q9PF2ZjT6xTQMXSvv1e6ydXc=
+github.com/aliyun/aliyun-oss-go-sdk v2.0.1+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
@@ -47,6 +49,8 @@ github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:l
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/aws/aws-sdk-go v1.23.12 h1:2UnxgNO6Y5J1OrkXS8XNp0UatDxD1bWHiDT62RDPggI=
 github.com/aws/aws-sdk-go v1.23.12/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
+github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f h1:ZNv7On9kyUzm7fvRZumSyy/IUiSC7AzL0I1jKKtwooA=
+github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f/go.mod h1:AuiFmCCPBSrqvVMvuqFuk0qogytodnVFVSN5CeJB8Gc=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=

diff --git a/pkg/objstore/client/factory.go b/pkg/objstore/client/factory.go
@@ -13,6 +13,7 @@ import (
 	"github.com/thanos-io/thanos/pkg/objstore/azure"
 	"github.com/thanos-io/thanos/pkg/objstore/cos"
 	"github.com/thanos-io/thanos/pkg/objstore/gcs"
+	"github.com/thanos-io/thanos/pkg/objstore/oss"
 	"github.com/thanos-io/thanos/pkg/objstore/s3"
 	"github.com/thanos-io/thanos/pkg/objstore/swift"
 	yaml "gopkg.in/yaml.v2"
@@ -21,11 +22,12 @@ import (
 type ObjProvider string
 
 const (
-	GCS   ObjProvider = "GCS"
-	S3    ObjProvider = "S3"
-	AZURE ObjProvider = "AZURE"
-	SWIFT ObjProvider = "SWIFT"
-	COS   ObjProvider = "COS"
+	GCS       ObjProvider = "GCS"
+	S3        ObjProvider = "S3"
+	AZURE     ObjProvider = "AZURE"
+	SWIFT     ObjProvider = "SWIFT"
+	COS       ObjProvider = "COS"
+	ALIYUNOSS ObjProvider = "ALIYUNOSS"
 )
 
 type BucketConfig struct {
@@ -59,6 +61,8 @@ func NewBucket(logger log.Logger, confContentYaml []byte, reg prometheus.Registe
 		bucket, err = swift.NewContainer(logger, config)
 	case string(COS):
 		bucket, err = cos.NewBucket(logger, config, component)
+	case string(ALIYUNOSS):
+		bucket, err = oss.NewBucket(logger, config, component)
 	default:
 		return nil, errors.Errorf("bucket with type %s is not supported", bucketConf.Type)
 	}

diff --git a/pkg/objstore/objtesting/foreach.go b/pkg/objstore/objtesting/foreach.go
@@ -9,6 +9,7 @@ import (
 	"github.com/thanos-io/thanos/pkg/objstore/cos"
 	"github.com/thanos-io/thanos/pkg/objstore/gcs"
 	"github.com/thanos-io/thanos/pkg/objstore/inmem"
+	"github.com/thanos-io/thanos/pkg/objstore/oss"
 	"github.com/thanos-io/thanos/pkg/objstore/s3"
 	"github.com/thanos-io/thanos/pkg/objstore/swift"
 	"github.com/thanos-io/thanos/pkg/testutil"
@@ -114,4 +115,21 @@ func ForeachStore(t *testing.T, testFn func(t testing.TB, bkt objstore.Bucket))
 	} else {
 		t.Log("THANOS_SKIP_TENCENT_COS_TESTS envvar present. Skipping test against Tencent COS.")
 	}
+
+	// Optional OSS.
+	if _, ok := os.LookupEnv("THANOS_SKIP_ALIYUN_OSS_TESTS"); !ok {
+		bkt, closeFn, err := oss.NewTestBucket(t)
+		testutil.Ok(t, err)
+
+		ok := t.Run("AliYun oss", func(t *testing.T) {
+			testFn(t, bkt)
+		})
+
+		closeFn()
+		if !ok {
+			return
+		}
+	} else {
+		t.Log("THANOS_SKIP_ALIYUN_OSS_TESTS envvar present. Skipping test against AliYun OSS.")
+	}
 }