From 8ad99c3aa0fb0d6205b821e9e2bc1f6fedd1ab2f Mon Sep 17 00:00:00 2001 From: wujinhu Date: Thu, 26 Sep 2019 15:39:59 +0800 Subject: [PATCH] update --- .circleci/config.yml | 3 +- CHANGELOG.md | 2 - Makefile | 4 +- docs/storage.md | 20 +- go.mod | 10 +- go.sum | 21 +- pkg/objstore/client/factory.go | 16 +- pkg/objstore/objtesting/foreach.go | 19 +- pkg/objstore/oss/oss.go | 362 ----------------------------- pkg/query/api/v1_test.go | 2 - pkg/query/storeset_test.go | 2 - pkg/receive/handler.go | 1 - scripts/bucketcfggen/main.go | 13 +- 13 files changed, 18 insertions(+), 457 deletions(-) delete mode 100644 pkg/objstore/oss/oss.go diff --git a/.circleci/config.yml b/.circleci/config.yml index 69077958e6a..5676f44d661 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -42,8 +42,7 @@ jobs: echo "Skipping SWIFT tests." export THANOS_SKIP_TENCENT_COS_TESTS="true" echo "Skipping TENCENT COS tests." - export THANOS_SKIP_ALIYUN_OSS_TESTS="true" - echo "Skipping ALIYUN OSS tests." + make test # Cross build is needed for publish_release but needs to be done outside of docker. diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f21153000d..bf3ebe3dae4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,6 @@ We use *breaking* word for marking changes that are not backward compatible (rel ## Unreleased. - [#1338](https://github.com/thanos-io/thanos/pull/1338) Querier still warns on store API duplicate, but allows a single one from duplicated set. This is gracefully warn about the problematic logic and not disrupt immediately. -- [#1234](https://github.com/thanos-io/thanos/pull/1234) `AliYun OSS` object storage, see [documents](docs/storage.md#aliyun-oss-configuration) for further information. ### Fixed @@ -24,7 +23,6 @@ We use *breaking* word for marking changes that are not backward compatible (rel ### Added - - [#1097](https://github.com/thanos-io/thanos/pull/1097) Added `thanos check rules` linter for Thanos rule rules files. - [#1253](https://github.com/thanos-io/thanos/pull/1253) Add support for specifying a maximum amount of retries when using Azure Blob storage (default: no retries). diff --git a/Makefile b/Makefile index f7545825f1e..aba7962ef25 100644 --- a/Makefile +++ b/Makefile @@ -149,8 +149,8 @@ docs: $(EMBEDMD) build .PHONY: check-docs check-docs: $(EMBEDMD) $(LICHE) build @EMBEDMD_BIN="$(EMBEDMD)" scripts/genflagdocs.sh check - @$(LICHE) --recursive docs --exclude "(cloud.tencent.com|alibabacloud.com)" --document-root . - @$(LICHE) --exclude "(cloud.tencent.com|alibabacloud.com)" --document-root . *.md + @$(LICHE) --recursive docs --exclude "cloud.tencent.com" --document-root . + @$(LICHE) --exclude "cloud.tencent.com" --document-root . *.md # format formats the code (including imports format). .PHONY: format diff --git a/docs/storage.md b/docs/storage.md index 4e08f32324e..7d16c05983a 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -22,7 +22,6 @@ Current object storage client implementations: | Azure Storage Account | Stable (production usage) | yes | @vglafirov | | OpenStack Swift | Beta (working PoCs, testing usage) | no | @sudhi-vm | | Tencent COS | Beta (testing usage) | no | @jojohappy | -| AliYun OSS | Beta (testing usage) | no | @shaulboozhiao,@wujinhu | NOTE: Currently Thanos requires strong consistency (write-read) for object store implementation. @@ -308,21 +307,4 @@ config: secret_id: "" ``` -Set the flags `--objstore.config-file` to reference to the configuration file. - -## AliYun OSS Configuration -In order to use AliYun OSS object storage, you should first create a bucket with proper Storage Class , ACLs and get the access key on the AliYun cloud. Go to [https://www.alibabacloud.com/product/oss](https://www.alibabacloud.com/product/oss) for more detail. - -To use AliYun OSS object storage, please specify following yaml configuration file in `objstore.config*` flag. - -[embedmd]:# (flags/config_aliyunoss.txt $) -```$ -type: ALIYUNOSS -config: - endpoint: "" - bucket: "" - access_key_id: "" - access_key_secret: "" -``` - -Use --objstore.config-file to reference to this configuration file. +Set the flags `--objstore.config-file` to reference to the configuration file. \ No newline at end of file diff --git a/go.mod b/go.mod index 57eef80a65d..140a31b49bb 100644 --- a/go.mod +++ b/go.mod @@ -4,9 +4,7 @@ require ( cloud.google.com/go v0.34.0 github.com/Azure/azure-storage-blob-go v0.7.0 github.com/NYTimes/gziphandler v1.1.1 - github.com/aliyun/aliyun-oss-go-sdk v1.9.8 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da - github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect github.com/cespare/xxhash v1.1.0 github.com/fatih/structtag v1.0.0 github.com/fortytw2/leaktest v1.3.0 @@ -17,7 +15,6 @@ require ( github.com/google/martian v2.1.0+incompatible // indirect github.com/googleapis/gax-go v2.0.2+incompatible // indirect github.com/gophercloud/gophercloud v0.0.0-20190301152420-fca40860790e - github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 github.com/grpc-ecosystem/go-grpc-prometheus v0.0.0-20181025070259-68e3a13e4117 github.com/hashicorp/golang-lru v0.5.1 @@ -36,23 +33,18 @@ require ( github.com/pkg/errors v0.8.1 github.com/prometheus/client_golang v1.0.0 github.com/prometheus/common v0.6.0 - github.com/prometheus/procfs v0.0.3 // indirect github.com/prometheus/prometheus v2.9.2+incompatible github.com/prometheus/tsdb v0.8.0 github.com/uber-go/atomic v1.4.0 // indirect github.com/uber/jaeger-client-go v2.16.0+incompatible github.com/uber/jaeger-lib v2.0.0+incompatible go.uber.org/atomic v1.4.0 // indirect - golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 // indirect - golang.org/x/net v0.0.0-20190628185345-da137c7871d7 // indirect golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 golang.org/x/sync v0.0.0-20190423024810-112230192c58 - golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb // indirect golang.org/x/text v0.3.2 google.golang.org/api v0.3.2 google.golang.org/grpc v1.19.1 gopkg.in/alecthomas/kingpin.v2 v2.2.6 gopkg.in/fsnotify.v1 v1.4.7 - gopkg.in/fsnotify/fsnotify.v1 v1.4.7 // indirect gopkg.in/yaml.v2 v2.2.2 -) +) \ No newline at end of file diff --git a/go.sum b/go.sum index 21377ca7e86..a32857c9142 100644 --- a/go.sum +++ b/go.sum @@ -26,8 +26,6 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5Vpd github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/aliyun/aliyun-oss-go-sdk v1.9.8 h1:BOflvK0Zs/zGmoabyFIzTg5c3kguktWTXEwewwbuba0= -github.com/aliyun/aliyun-oss-go-sdk v1.9.8/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= @@ -35,8 +33,6 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aws/aws-sdk-go v0.0.0-20180507225419-00862f899353 h1:qFKf58XUUvHaEz0zFkLJsQ4dzoAyrQ8QyhK4nHGHBI4= github.com/aws/aws-sdk-go v0.0.0-20180507225419-00862f899353/go.mod h1:ZRmQr0FajVIyZ4ZzBYKG5P3ZqPz9IHG41ZoMu1ADI3k= -github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f h1:ZNv7On9kyUzm7fvRZumSyy/IUiSC7AzL0I1jKKtwooA= -github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f/go.mod h1:AuiFmCCPBSrqvVMvuqFuk0qogytodnVFVSN5CeJB8Gc= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= @@ -112,8 +108,6 @@ github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCy github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v0.0.0-20150304233714-bbcb9da2d746 h1:M6d2zDTA4cKXT6OwFsJxlo5tWrAukj3KfvJ1zcBatnA= @@ -129,8 +123,6 @@ github.com/gophercloud/gophercloud v0.0.0-20190301152420-fca40860790e h1:hQpY0g0 github.com/gophercloud/gophercloud v0.0.0-20190301152420-fca40860790e/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e h1:JKmoR8x90Iww1ks85zJ1lfDGgIiMDuIptTOhJq+zKyg= -github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 h1:pdN6V1QBWetyv/0+wjACpqVH+eVULgEjkurDLq3goeM= @@ -292,8 +284,6 @@ github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1 h1:/K3IL0Z1quvmJ github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.3 h1:CTwfnzjQ+8dS6MhHHu4YswVAD99sL2wjPqP+VkURmKE= -github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/prometheus v2.9.2+incompatible h1:5QVnXpkSsbbG59TyZ99clRfaHQy2QuIlTv6dEgS66C4= github.com/prometheus/prometheus v2.9.2+incompatible/go.mod h1:vdLuLLM0uqhLSofrQ7Nev2b/rQUyZ+pkT3zF7LB/i1g= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= @@ -308,7 +298,6 @@ github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFo github.com/samuel/go-zookeeper v0.0.0-20161028232340-1d7be4effb13 h1:4AQBn5RJY4WH8t8TLEMZUsWeXHAUcoao42TCAfpEJJE= github.com/samuel/go-zookeeper v0.0.0-20161028232340-1d7be4effb13/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/sasha-s/go-deadlock v0.0.0-20161201235124-341000892f3d/go.mod h1:StQn567HiB1fF2yJ44N9au7wOhrPS3iZqiDbRupzT10= -github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= @@ -348,8 +337,6 @@ golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f h1:R423Cnkcp5JABoeemiGEPlt9tHXFfw5kvc0yqlxRPWo= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -370,8 +357,6 @@ golang.org/x/net v0.0.0-20190522155817-f3200d17e092 h1:4QSRKanuywn15aTZvI/mIDEgP golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 h1:dfGZHvZk057jK2MCeWus/TowKpJ8y4AmooUzdBSR9GU= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7 h1:rTIdg5QFRR7XCaK4LCjBiPbx8j4DQRpdYMnGn/bJUEU= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 h1:Wo7BWFiOk0QRFMLYMqJGFMd9CgUAcGx7V+qEg/h5IBI= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -397,8 +382,6 @@ golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb h1:fgwFCsaw9buMuxNd6+DQfAuSFqbNiQZpcgJQAgJsK6k= -golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -433,8 +416,6 @@ gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/fsnotify/fsnotify.v1 v1.3.1 h1:2fkCHbPQZNYRAyRyIV9VX0bpRkxIorlQDiYRmufHnhA= gopkg.in/fsnotify/fsnotify.v1 v1.3.1/go.mod h1:Fyux9zXlo4rWoMSIzpn9fDAYjalPqJ/K1qJ27s+7ltE= -gopkg.in/fsnotify/fsnotify.v1 v1.4.7 h1:XNNYLJHt73EyYiCZi6+xjupS9CpvmiDgjPTAjrBlQbo= -gopkg.in/fsnotify/fsnotify.v1 v1.4.7/go.mod h1:Fyux9zXlo4rWoMSIzpn9fDAYjalPqJ/K1qJ27s+7ltE= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.42.0 h1:7N3gPTt50s8GuLortA00n8AqRTk75qOP98+mTPpgzRk= @@ -458,4 +439,4 @@ k8s.io/klog v0.1.0 h1:I5HMfc/DtuVaGR1KPwUrTc476K8NCqNBldC7H4dYEzk= k8s.io/klog v0.1.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/kube-openapi v0.0.0-20180629012420-d83b052f768a/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= -sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= \ No newline at end of file diff --git a/pkg/objstore/client/factory.go b/pkg/objstore/client/factory.go index 50caf4d8e7f..47a59473903 100644 --- a/pkg/objstore/client/factory.go +++ b/pkg/objstore/client/factory.go @@ -13,21 +13,19 @@ import ( "github.com/thanos-io/thanos/pkg/objstore/azure" "github.com/thanos-io/thanos/pkg/objstore/cos" "github.com/thanos-io/thanos/pkg/objstore/gcs" - "github.com/thanos-io/thanos/pkg/objstore/oss" "github.com/thanos-io/thanos/pkg/objstore/s3" "github.com/thanos-io/thanos/pkg/objstore/swift" - yaml "gopkg.in/yaml.v2" + "gopkg.in/yaml.v2" ) type ObjProvider string const ( - GCS ObjProvider = "GCS" - S3 ObjProvider = "S3" - AZURE ObjProvider = "AZURE" - SWIFT ObjProvider = "SWIFT" - COS ObjProvider = "COS" - ALIYUNOSS ObjProvider = "ALIYUNOSS" + GCS ObjProvider = "GCS" + S3 ObjProvider = "S3" + AZURE ObjProvider = "AZURE" + SWIFT ObjProvider = "SWIFT" + COS ObjProvider = "COS" ) type BucketConfig struct { @@ -61,8 +59,6 @@ func NewBucket(logger log.Logger, confContentYaml []byte, reg prometheus.Registe bucket, err = swift.NewContainer(logger, config) case string(COS): bucket, err = cos.NewBucket(logger, config, component) - case string(ALIYUNOSS): - bucket, err = oss.NewBucket(logger, config, component) default: return nil, errors.Errorf("bucket with type %s is not supported", bucketConf.Type) } diff --git a/pkg/objstore/objtesting/foreach.go b/pkg/objstore/objtesting/foreach.go index d92e45d9fdd..92ad251a6cc 100644 --- a/pkg/objstore/objtesting/foreach.go +++ b/pkg/objstore/objtesting/foreach.go @@ -11,7 +11,6 @@ import ( "github.com/thanos-io/thanos/pkg/objstore/cos" "github.com/thanos-io/thanos/pkg/objstore/gcs" "github.com/thanos-io/thanos/pkg/objstore/inmem" - "github.com/thanos-io/thanos/pkg/objstore/oss" "github.com/thanos-io/thanos/pkg/objstore/s3" "github.com/thanos-io/thanos/pkg/objstore/swift" "github.com/thanos-io/thanos/pkg/testutil" @@ -118,20 +117,4 @@ func ForeachStore(t *testing.T, testFn func(t testing.TB, bkt objstore.Bucket)) } else { t.Log("THANOS_SKIP_TENCENT_COS_TESTS envvar present. Skipping test against Tencent COS.") } - - // Optional OSS. - if _, ok := os.LookupEnv("THANOS_SKIP_ALIYUN_OSS_TESTS"); !ok { - bkt, closeFn, err := oss.NewTestBucket(t) - testutil.Ok(t, err) - - ok := t.Run("AliYun oss", func(t *testing.T) { - testFn(t, bkt) - }) - closeFn() - if !ok { - return - } - } else { - t.Log("THANOS_SKIP_ALIYUN_OSS_TESTS envvar present. Skipping test against AliYun OSS.") - } -} +} \ No newline at end of file diff --git a/pkg/objstore/oss/oss.go b/pkg/objstore/oss/oss.go deleted file mode 100644 index 7294fb18c4f..00000000000 --- a/pkg/objstore/oss/oss.go +++ /dev/null @@ -1,362 +0,0 @@ -package oss - -import ( - "context" - "fmt" - "io" - "io/ioutil" - "math" - "math/rand" - "net/http" - "os" - "strings" - "testing" - "time" - - alioss "github.com/aliyun/aliyun-oss-go-sdk/oss" - "github.com/go-kit/kit/log" - "github.com/pkg/errors" - "github.com/thanos-io/thanos/pkg/objstore" - "github.com/thanos-io/thanos/pkg/runutil" - yaml "gopkg.in/yaml.v2" -) - -type Config struct { - Endpoint string `yaml:"endpoint"` - Bucket string `yaml:"bucket"` - AccessKeyID string `yaml:"access_key_id"` - AccessKeySecret string `yaml:"access_key_secret"` -} - -type Bucket struct { - name string - logger log.Logger - client *alioss.Client - config Config - bucket *alioss.Bucket -} - -func NewTestBucket(t testing.TB) (objstore.Bucket, func(), error) { - c := configFromEnv() - err := func(conf Config) error { - if conf.Endpoint == "" { - return errors.New("no aliyun oss endpoint in config file") - } - if conf.AccessKeyID == "" { - return errors.New("access_key_id is not present in config file") - } - if conf.AccessKeySecret == "" { - return errors.New("access_key_secret is not present in config file") - } - return nil - }(c) - if err != nil { - return nil, nil, err - } - if c.Bucket != "" && os.Getenv("THANOS_ALLOW_EXISTING_BUCKET_USE") == "true" { - t.Log("ALIYUNOSS_BUCKET is defined. Normally this tests will create temporary bucket " + - "and delete it after test. Unset ALIYUNOSS_BUCKET env variable to use default logic. If you really want to run " + - "tests against provided (NOT USED!) bucket, set THANOS_ALLOW_EXISTING_BUCKET_USE=true.") - return NewTestBucketFromConfig(t, c, true) - } - return NewTestBucketFromConfig(t, c, false) -} - -func calculateChunks(name string, r io.Reader) (int, int64, error) { - switch r.(type) { - case *os.File: - f, _ := r.(*os.File) - if fileInfo, err := f.Stat(); err == nil { - s := fileInfo.Size() - return int(math.Floor(float64(s) / alioss.MaxPartSize)), s % alioss.MaxPartSize, nil - } - case *strings.Reader: - f, _ := r.(*strings.Reader) - return int(math.Floor(float64(f.Size()) / alioss.MaxPartSize)), f.Size() % alioss.MaxPartSize, nil - } - return -1, 0, errors.New("unsupported implement of io.Reader") -} - -// Upload the contents of the reader as an object into the bucket. -func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { - chunksnum, lastslice, err := calculateChunks(name, r) - if err != nil { - return err - } - - ncloser := ioutil.NopCloser(r) - if chunksnum == 0 { - if err := b.bucket.PutObject(name, ncloser); err != nil { - return errors.Wrap(err, "failed to upload oss object") - } - } else { - init, err := b.bucket.InitiateMultipartUpload(name) - if err != nil { - return errors.Wrap(err, "failed to initiate multi-part upload") - } - chunk := 0 - uploadEveryPart := func(everypartsize int64, cnk int) (alioss.UploadPart, error) { - prt, err := b.bucket.UploadPart(init, ncloser, everypartsize, cnk) - if err != nil { - if err2 := b.bucket.AbortMultipartUpload(init); err2 != nil { - return prt, errors.Wrap(err2, "failed to abort multi-part upload") - } - return prt, errors.Wrap(err, "failed to upload multi-part chunk") - } - return prt, nil - } - var parts []alioss.UploadPart - for ; chunk < chunksnum; chunk++ { - part, err := uploadEveryPart(alioss.MaxPartSize, chunk+1) - if err != nil { - return err - } - parts = append(parts, part) - } - if lastslice != 0 { - part, err := uploadEveryPart(lastslice, chunksnum+1) - if err != nil { - return errors.Wrap(err, "failed to upload the last chunk") - } - parts = append(parts, part) - } - _, err = b.bucket.CompleteMultipartUpload(init, parts) - if err != nil { - return errors.Wrap(err, "failed to set multi-part upload completive") - } - } - return nil -} - -// Delete removes the object with the given name. -func (b *Bucket) Delete(ctx context.Context, name string) error { - if err := b.bucket.DeleteObject(name); err != nil { - return errors.Wrap(err, "delete oss object") - } - return nil -} - -func configFromEnv() Config { - c := Config{ - Endpoint: os.Getenv("ALIYUNOSS_ENDPOINT"), - Bucket: os.Getenv("ALIYUNOSS_BUCKET"), - AccessKeyID: os.Getenv("ALIYUNOSS_ACCESS_KEY_ID"), - AccessKeySecret: os.Getenv("ALIYUNOSS_ACCESS_KEY_SECRET"), - } - return c -} - -func NewBucket(logger log.Logger, conf []byte, component string) (*Bucket, error) { - config, err := func(conf []byte) (Config, error) { - var config Config - if err := yaml.Unmarshal(conf, &config); err != nil { - return Config{}, err - } - - return config, nil - }(conf) - - if err != nil { - return nil, errors.Wrap(err, "parse aliyun oss config file failed") - } - if err := validate(config); err != nil { - return nil, errors.Wrap(err, "validate aliyun oss config file failed") - } - - client, err := alioss.New(config.Endpoint, config.AccessKeyID, config.AccessKeySecret) - if err != nil { - return nil, errors.Wrap(err, "create aliyun oss client failed") - } - bk, err := client.Bucket(config.Bucket) - if err != nil { - return nil, errors.Wrapf(err, "use aliyun oss bucket %s failed", config.Bucket) - } - - bkt := &Bucket{ - logger: logger, - client: client, - name: config.Bucket, - config: config, - bucket: bk, - } - return bkt, nil -} - -// Iter calls f for each entry in the given directory (not recursive). The argument to f is the full -// object name including the prefix of the inspected directory. -func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error) error { - if dir != "" { - dir = strings.TrimSuffix(dir, objstore.DirDelim) + objstore.DirDelim - } - - marker := alioss.Marker("") - for { - if err := ctx.Err(); err != nil { - return errors.Wrap(err, "context closed while iterating bucket") - } - objects, err := b.bucket.ListObjects(alioss.Prefix(dir), alioss.Delimiter(objstore.DirDelim), marker) - if err != nil { - return errors.Wrap(err, "listing aliyun oss bucket failed") - } - marker = alioss.Marker(objects.NextMarker) - - for _, object := range objects.Objects { - if err := f(object.Key); err != nil { - return errors.Wrapf(err, "callback func invoke for object %s failed ", object.Key) - } - } - - for _, object := range objects.CommonPrefixes { - if err := f(object); err != nil { - return errors.Wrapf(err, "callback func invoke for directory %s failed", object) - } - } - if !objects.IsTruncated { - break - } - } - - return nil -} - -func (b *Bucket) Name() string { - return b.name -} - -// validate checks to see the config options are set. -func validate(conf Config) error { - if conf.Endpoint == "" { - return errors.New("no aliyun oss endpoint in config file") - } - if conf.Bucket == "" { - return errors.New("no aliyun oss bucket in config file") - } - - if conf.AccessKeyID == "" { - return errors.New("access_key_id is not present in config file") - } - if conf.AccessKeySecret == "" { - return errors.New("access_key_secret is not present in config file") - } - return nil -} - -func NewTestBucketFromConfig(t testing.TB, c Config, reuseBucket bool) (objstore.Bucket, func(), error) { - if c.Bucket == "" { - src := rand.NewSource(time.Now().UnixNano()) - - bktToCreate := strings.Replace(fmt.Sprintf("test_%s_%x", strings.ToLower(t.Name()), src.Int63()), "_", "-", -1) - if len(bktToCreate) >= 63 { - bktToCreate = bktToCreate[:63] - } - testclient, err := alioss.New(c.Endpoint, c.AccessKeyID, c.AccessKeySecret) - if err != nil { - return nil, nil, errors.Wrap(err, "create aliyun oss client failed") - } - - if err := testclient.CreateBucket(bktToCreate); err != nil { - return nil, nil, errors.Wrapf(err, "create aliyun oss bucket %s failed", bktToCreate) - } - c.Bucket = bktToCreate - } - - bc, err := yaml.Marshal(c) - if err != nil { - return nil, nil, err - } - - b, err := NewBucket(log.NewNopLogger(), bc, "thanos-aliyun-oss-test") - if err != nil { - return nil, nil, err - } - - if reuseBucket { - if err := b.Iter(context.Background(), "", func(f string) error { - return errors.Errorf("bucket %s is not empty", c.Bucket) - }); err != nil { - return nil, nil, errors.Wrapf(err, "oss check bucket %s", c.Bucket) - } - - t.Log("WARNING. Reusing", c.Bucket, "Aliyun OSS bucket for OSS tests. Manual cleanup afterwards is required") - return b, func() {}, nil - } - - return b, func() { - objstore.EmptyBucket(t, context.Background(), b) - if err := b.client.DeleteBucket(c.Bucket); err != nil { - t.Logf("deleting bucket %s failed: %s", c.Bucket, err) - } - }, nil -} - -func (b *Bucket) Close() error { return nil } - -func setRange(start, end int64) (alioss.Option, error) { - var opt alioss.Option - if 0 <= start && start <= end { - opt = alioss.Range(start, end) - } else { - return nil, errors.Errorf("Invalid range specified: start=%d end=%d", start, end) - } - return opt, nil -} - -func (b *Bucket) getRange(ctx context.Context, name string, off, length int64) (io.ReadCloser, error) { - if len(name) == 0 { - return nil, errors.New("given object name should not empty") - } - - var opts []alioss.Option - if length != -1 { - opt, err := setRange(off, off+length-1) - if err != nil { - return nil, err - } - opts = append(opts, opt) - } - - resp, err := b.bucket.GetObject(name, opts...) - if err != nil { - return nil, err - } - - if _, err := resp.Read(nil); err != nil { - runutil.CloseWithLogOnErr(b.logger, resp, "oss get range obj close") - return nil, err - } - - return resp, nil -} - -// Get returns a reader for the given object name. -func (b *Bucket) Get(ctx context.Context, name string) (io.ReadCloser, error) { - return b.getRange(ctx, name, 0, -1) -} - -func (b *Bucket) GetRange(ctx context.Context, name string, off, length int64) (io.ReadCloser, error) { - return b.getRange(ctx, name, off, length) -} - -// Exists checks if the given object exists in the bucket. -func (b *Bucket) Exists(ctx context.Context, name string) (bool, error) { - exists, err := b.bucket.IsObjectExist(name) - if err != nil { - if b.IsObjNotFoundErr(err) { - return false, nil - } - return false, errors.Wrap(err, "cloud not check if object exists") - } - - return exists, nil -} - -// IsObjNotFoundErr returns true if error means that object is not found. Relevant to Get operations. -func (b *Bucket) IsObjNotFoundErr(err error) bool { - switch aliErr := err.(type) { - case alioss.ServiceError: - if aliErr.StatusCode == http.StatusNotFound { - return true - } - } - return false -} diff --git a/pkg/query/api/v1_test.go b/pkg/query/api/v1_test.go index ce43f1c392f..36fcf06644c 100644 --- a/pkg/query/api/v1_test.go +++ b/pkg/query/api/v1_test.go @@ -38,9 +38,7 @@ import ( "github.com/prometheus/prometheus/promql" "github.com/prometheus/prometheus/storage" "github.com/thanos-io/thanos/pkg/compact" - extpromhttp "github.com/thanos-io/thanos/pkg/extprom/http" - "github.com/thanos-io/thanos/pkg/query" "github.com/thanos-io/thanos/pkg/testutil" ) diff --git a/pkg/query/storeset_test.go b/pkg/query/storeset_test.go index 2ff65776cb1..4dd59b42b8e 100644 --- a/pkg/query/storeset_test.go +++ b/pkg/query/storeset_test.go @@ -12,10 +12,8 @@ import ( "sort" "github.com/fortytw2/leaktest" - "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" - "github.com/thanos-io/thanos/pkg/store/storepb" "github.com/thanos-io/thanos/pkg/testutil" "google.golang.org/grpc" diff --git a/pkg/receive/handler.go b/pkg/receive/handler.go index 9f72599406d..5e16bdd3ba1 100644 --- a/pkg/receive/handler.go +++ b/pkg/receive/handler.go @@ -23,7 +23,6 @@ import ( "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/prometheus/common/route" promtsdb "github.com/prometheus/prometheus/storage/tsdb" - terrors "github.com/prometheus/tsdb/errors" "github.com/thanos-io/thanos/pkg/runutil" "github.com/thanos-io/thanos/pkg/store/prompb" diff --git a/scripts/bucketcfggen/main.go b/scripts/bucketcfggen/main.go index 00f7f9d9e8a..72fa0a361df 100644 --- a/scripts/bucketcfggen/main.go +++ b/scripts/bucketcfggen/main.go @@ -11,13 +11,11 @@ import ( "github.com/fatih/structtag" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" - "github.com/pkg/errors" "github.com/thanos-io/thanos/pkg/objstore/azure" "github.com/thanos-io/thanos/pkg/objstore/client" "github.com/thanos-io/thanos/pkg/objstore/cos" "github.com/thanos-io/thanos/pkg/objstore/gcs" - "github.com/thanos-io/thanos/pkg/objstore/oss" "github.com/thanos-io/thanos/pkg/objstore/s3" "github.com/thanos-io/thanos/pkg/objstore/swift" kingpin "gopkg.in/alecthomas/kingpin.v2" @@ -26,12 +24,11 @@ import ( var ( configs = map[client.ObjProvider]interface{}{ - client.AZURE: azure.Config{}, - client.GCS: gcs.Config{}, - client.S3: s3.Config{}, - client.SWIFT: swift.SwiftConfig{}, - client.COS: cos.Config{}, - client.ALIYUNOSS: oss.Config{}, + client.AZURE: azure.Config{}, + client.GCS: gcs.Config{}, + client.S3: s3.Config{}, + client.SWIFT: swift.SwiftConfig{}, + client.COS: cos.Config{}, } )