cherry-pick master to release 3.1

CONTRIBUTING.md

@@ -15,7 +15,6 @@ your contribution accepted.
 Developing BR requires:
 
 * [Go 1.13+](http://golang.org/doc/code.html)
-* [Retool](https://github.com/twitchtv/retool)
 * An internet connection to download the dependencies
 
 Simply run `make` to build the program.

Makefile

@@ -36,7 +36,7 @@ test:
 	GO111MODULE=on go test ${RACEFLAG} -tags leak ./...
 
 testcover:
-	GO111MODULE=on retool do overalls \
+	GO111MODULE=on tools/bin/overalls \
 		-project=$(BR_PKG) \
 		-covermode=count \
 		-ignore='.git,vendor,tests,_tools' \
@@ -55,21 +55,20 @@ integration_test: build build_for_integration_test
 
 tools:
 	@echo "install tools..."
-	@GO111MODULE=off go get github.com/twitchtv/retool
-	@GO111MODULE=off retool sync
+	@cd tools && make
 
 check-all: static lint tidy
 	@echo "checking"
 
 check: tools check-all
 
 static: export GO111MODULE=on
-static:
+static: tools
 	@ # Not running vet and fmt through metalinter becauase it ends up looking at vendor
-	retool do goimports -w -d -format-only -local $(BR_PKG) $$($(PACKAGE_DIRECTORIES)) 2>&1 | $(GOCHECKER)
-	retool do govet --shadow $$($(PACKAGE_DIRECTORIES)) 2>&1 | $(GOCHECKER)
+	tools/bin/goimports -w -d -format-only -local $(BR_PKG) $$($(PACKAGE_DIRECTORIES)) 2>&1 | $(GOCHECKER)
+	tools/bin/govet --shadow $$($(PACKAGE_DIRECTORIES)) 2>&1 | $(GOCHECKER)
 
-	CGO_ENABLED=0 retool do golangci-lint run --enable-all --deadline 120s \
+	CGO_ENABLED=0 tools/bin/golangci-lint run --enable-all --deadline 120s \
 		--disable gochecknoglobals \
 		--disable gochecknoinits \
 		--disable interfacer \
@@ -81,13 +80,15 @@ static:
 		--disable gocognit \
 		--disable godox \
 		--disable gomnd \
-		$$($(PACKAGE_DIRECTORIES))
+		$$($(PACKAGE_DIRECTORIES)) || true
 
-lint:
+lint: tools
 	@echo "linting"
-	CGO_ENABLED=0 retool do revive -formatter friendly -config revive.toml $$($(PACKAGES))
+	CGO_ENABLED=0 tools/bin/revive -formatter friendly -config revive.toml $$($(PACKAGES))
 
 tidy:
 	@echo "go mod tidy"
 	GO111MODULE=on go mod tidy
 	git diff --quiet go.mod go.sum
+
+.PHONY: tools

README.md

@@ -1,6 +1,6 @@
 # BR
 
-[![Build Status](https://internal.pingcap.net/idc-jenkins/job/build_br_master/badge/icon)](https://internal.pingcap.net/idc-jenkins/job/build_br_master/)
+[![Build Status](https://internal.pingcap.net/idc-jenkins/job/build_br_multi_branch/job/release-3.1/badge/icon)](https://internal.pingcap.net/idc-jenkins/job/build_br_multi_branch/job/release-3.1/)
 [![codecov](https://codecov.io/gh/pingcap/br/branch/master/graph/badge.svg)](https://codecov.io/gh/pingcap/br)
 [![LICENSE](https://img.shields.io/github/license/pingcap/br.svg)](https://github.com/pingcap/br/blob/master/LICENSE)
 [![Language](https://img.shields.io/badge/Language-Go-blue.svg)](https://golang.org/)
@@ -23,9 +23,7 @@
 
 ## Building
 
-1. Install [retool](https://github.com/twitchtv/retool)
-
-2. To build binary and run test:
+To build binary and run test:
 
 ```bash
 $ make

cmd/cmd.go

@@ -47,7 +47,7 @@ const (
 )
 
 func timestampLogFileName() string {
-	return filepath.Join(os.TempDir(), "br.log."+time.Now().Format(time.RFC3339))
+	return filepath.Join(os.TempDir(), time.Now().Format("br.log.2006-01-02T15.04.05Z0700"))
 }
 
 // AddFlags adds flags to the given cmd.

go.mod

@@ -8,24 +8,22 @@ require (
 	github.com/cheggaaa/pb/v3 v3.0.1
 	github.com/coreos/go-semver v0.3.0 // indirect
 	github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect
-	github.com/fatih/color v1.9.0 // indirect
 	github.com/fsouza/fake-gcs-server v1.15.0
 	github.com/go-sql-driver/mysql v1.4.1
 	github.com/gogo/protobuf v1.3.1
 	github.com/google/btree v1.0.0
 	github.com/google/uuid v1.1.1
 	github.com/klauspost/cpuid v1.2.0 // indirect
-	github.com/mattn/go-runewidth v0.0.7 // indirect
 	github.com/montanaflynn/stats v0.5.0 // indirect
 	github.com/pingcap/check v0.0.0-20200212061837-5e12011dc712
 	github.com/pingcap/errors v0.11.5-0.20190809092503-95897b64e011
 	github.com/pingcap/kvproto v0.0.0-20200331072206-c211b473fe43
 	github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd
-	github.com/pingcap/parser v3.1.0-beta.2.0.20200318061433-f0b8f6cdca0d+incompatible
-	github.com/pingcap/pd/v3 v3.1.0-rc.0.20200410125843-8b7475c6bfd6
-	github.com/pingcap/tidb v1.1.0-beta.0.20200410101606-1347df814de0
-	github.com/pingcap/tidb-tools v4.0.0-beta.1.0.20200317092225-ed6b2a87af54+incompatible
-	github.com/pingcap/tipb v0.0.0-20200401093201-cc8b75c53383
+	github.com/pingcap/parser v3.1.0-beta.2.0.20200425032215-994651e9b6df+incompatible
+	github.com/pingcap/pd/v3 v3.1.1-0.20200426091027-e639f0b1e62b
+	github.com/pingcap/tidb v1.1.0-beta.0.20200426082429-52b31342cb0e
+	github.com/pingcap/tidb-tools v4.0.0-beta.2.0.20200426094956-7d70d54da224+incompatible
+	github.com/pingcap/tipb v0.0.0-20200426072603-ce17d2d03251
 	github.com/prometheus/client_golang v1.0.0
 	github.com/prometheus/common v0.4.1
 	github.com/sirupsen/logrus v1.4.2

go.sum

@@ -99,8 +99,6 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7
 github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0=
 github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
-github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s=
-github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsouza/fake-gcs-server v1.15.0 h1:ss/ztlt10Y64A5qslmxZKsiqW/i28t5DkRtv6qSFaLQ=
@@ -259,8 +257,6 @@ github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU=
 github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
-github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA=
-github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=
@@ -271,8 +267,6 @@ github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOA
 github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y=
 github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
-github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54=
-github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o=
 github.com/mattn/go-sqlite3 v2.0.1+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
@@ -302,15 +296,13 @@ github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:v
 github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw=
-github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.12.0 h1:Iw5WCbBcaAAd0fpRb1c9r5YCylv4XDoCSigm1zLevwU=
 github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg=
 github.com/onsi/gomega v1.4.2 h1:3mYCb7aPxS/RU7TI1y4rkEn1oKmPRjNJLNEXgw7MH2I=
 github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
-github.com/onsi/gomega v1.8.1 h1:C5Dqfs/LeauYDX0jJXIe2SWmwCbGzx9yF8C8xy3Lh34=
-github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
+github.com/onsi/gomega v1.9.0 h1:R1uwffexN6Pr340GtYRIdZmAiN4J+iw6WG4wog1DUXg=
 github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
 github.com/opentracing/basictracer-go v1.0.0 h1:YyUAhaEfjoWXclZVJ9sGoNct7j4TVk7lZWlQw5UXuoo=
 github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74=
@@ -345,22 +337,23 @@ github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9 h1:AJD9pZYm72vMgPcQDww
 github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8=
 github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd h1:CV3VsP3Z02MVtdpTMfEgRJ4T9NGgGTxdHpJerent7rM=
 github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8=
-github.com/pingcap/parser v3.1.0-beta.1.0.20200318061433-f0b8f6cdca0d+incompatible h1:+Jibmc9uklKz9/prpBggFyjZpqRM8phc1AOOJGxkP48=
-github.com/pingcap/parser v3.1.0-beta.1.0.20200318061433-f0b8f6cdca0d+incompatible/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA=
-github.com/pingcap/parser v3.1.0-beta.2.0.20200318061433-f0b8f6cdca0d+incompatible h1:cR7rWBrN8y7Uqk6UBUJ1Ai/EW2DA9yw8t0QmXl6Tr2A=
-github.com/pingcap/parser v3.1.0-beta.2.0.20200318061433-f0b8f6cdca0d+incompatible/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA=
+github.com/pingcap/parser v3.1.0-beta.2.0.20200425032215-994651e9b6df+incompatible h1:9kFDvyd1YTGin3xHetc68xAChSOgQD2vThXxBR9ccH0=
+github.com/pingcap/parser v3.1.0-beta.2.0.20200425032215-994651e9b6df+incompatible/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA=
 github.com/pingcap/pd/v3 v3.1.0-beta.2.0.20200312100832-1206736bd050 h1:mxPdR0pxnUcRfRGX2JnaLyAd9SZWeR42SzvMp4Zv3YI=
 github.com/pingcap/pd/v3 v3.1.0-beta.2.0.20200312100832-1206736bd050/go.mod h1:0HfF1LfWLMuGpui0PKhGvkXxfjv1JslMRY6B+cae3dg=
-github.com/pingcap/pd/v3 v3.1.0-rc.0.20200410125843-8b7475c6bfd6 h1:N3zllT6rtDYXPeKS/Khide/ClAH1FCNFU6/EStVwb8Y=
-github.com/pingcap/pd/v3 v3.1.0-rc.0.20200410125843-8b7475c6bfd6/go.mod h1:su5qKHuRSE0oSO8DBmck0WK17T18OPjCNi3XToM+uCM=
-github.com/pingcap/tidb v1.1.0-beta.0.20200401121410-5854181fbbe0 h1:bcl8cbL0K9oZ+vYWNJIgUH2rMkesjfsbJkpZpD2I+lg=
-github.com/pingcap/tidb v1.1.0-beta.0.20200401121410-5854181fbbe0/go.mod h1:2cL/Jdq//AUbt/m/VmOwc7wm82oLFn7o/B6fiQtOpQE=
-github.com/pingcap/tidb v1.1.0-beta.0.20200410101606-1347df814de0 h1:ZgAk3sPL0p5WEQhyRX8aSCwaDrPLqxMBitT1XP1UvqE=
-github.com/pingcap/tidb v1.1.0-beta.0.20200410101606-1347df814de0/go.mod h1:2cL/Jdq//AUbt/m/VmOwc7wm82oLFn7o/B6fiQtOpQE=
-github.com/pingcap/tidb-tools v4.0.0-beta.1.0.20200317092225-ed6b2a87af54+incompatible h1:tYADqdmWwgDOwf/qEN0trJAy6H3c3Tt/QZx1z4qVrRQ=
+github.com/pingcap/pd/v3 v3.1.1-0.20200426091027-e639f0b1e62b h1:oYWX0dRvRdN55HUmjZQST1osEj9cYOG2XUQfFXM3pbM=
+github.com/pingcap/pd/v3 v3.1.1-0.20200426091027-e639f0b1e62b/go.mod h1:su5qKHuRSE0oSO8DBmck0WK17T18OPjCNi3XToM+uCM=
+github.com/pingcap/tidb v1.1.0-beta.0.20200426082429-52b31342cb0e h1:IGNsARumk3Sqev3gtodBGL2mqqYcJ5vERE1pTF8QtxI=
+github.com/pingcap/tidb v1.1.0-beta.0.20200426082429-52b31342cb0e/go.mod h1:DnbLUiiSBC3TTeLIfsmJeVdgkmY+h3y4mL01ZmoPJ8I=
 github.com/pingcap/tidb-tools v4.0.0-beta.1.0.20200317092225-ed6b2a87af54+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM=
+github.com/pingcap/tidb-tools v4.0.0-beta.2.0.20200317092225-ed6b2a87af54+incompatible h1:G2PxT1bKamWFwKgS1vERx8BBgjgez9t/5kXW65GFzCU=
+github.com/pingcap/tidb-tools v4.0.0-beta.2.0.20200317092225-ed6b2a87af54+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM=
+github.com/pingcap/tidb-tools v4.0.0-beta.2.0.20200426094956-7d70d54da224+incompatible h1:oIJi6qQkxwGq21fZfmqhcic2OzO1nlfApJ4aqYz826Q=
+github.com/pingcap/tidb-tools v4.0.0-beta.2.0.20200426094956-7d70d54da224+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM=
 github.com/pingcap/tipb v0.0.0-20200401093201-cc8b75c53383 h1:y1ayhtouCaO0u74JNMN8s20CGJT0yIuAb8UXOYnCALc=
 github.com/pingcap/tipb v0.0.0-20200401093201-cc8b75c53383/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI=
+github.com/pingcap/tipb v0.0.0-20200426072603-ce17d2d03251 h1:DnrF3jRNUvV37jb4zGT6SL3g06b7kQcD2i1oqgSKSOE=
+github.com/pingcap/tipb v0.0.0-20200426072603-ce17d2d03251/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=

pkg/backup/client.go

@@ -215,6 +215,7 @@ func BuildBackupRangeAndSchema(
 
 		var dbData []byte
 		idAlloc := autoid.NewAllocator(storage, dbInfo.ID, false, autoid.RowIDAllocType)
+		randAlloc := autoid.NewAllocator(storage, dbInfo.ID, false, autoid.AutoRandomType)
 
 		for _, tableInfo := range dbInfo.Tables {
 			if !tableFilter.Match(&filter.Table{Schema: dbInfo.Name.L, Name: tableInfo.Name.L}) {
@@ -226,6 +227,20 @@ func BuildBackupRangeAndSchema(
 				return nil, nil, errors.Trace(err)
 			}
 			tableInfo.AutoIncID = globalAutoID
+
+			if tableInfo.PKIsHandle && tableInfo.ContainsAutoRandomBits() {
+				// this table has auto_random id, we need backup and rebase in restoration
+				var globalAutoRandID int64
+				globalAutoRandID, err = randAlloc.NextGlobalAutoID(tableInfo.ID)
+				if err != nil {
+					return nil, nil, errors.Trace(err)
+				}
+				tableInfo.AutoRandID = globalAutoRandID
+				log.Info("change table AutoRandID",
+					zap.Stringer("db", dbInfo.Name),
+					zap.Stringer("table", tableInfo.Name),
+					zap.Int64("AutoRandID", globalAutoRandID))
+			}
 			log.Info("change table AutoIncID",
 				zap.Stringer("db", dbInfo.Name),
 				zap.Stringer("table", tableInfo.Name),

pkg/restore/db.go

@@ -111,6 +111,7 @@ func (db *DB) CreateTable(ctx context.Context, table *utils.Table) error {
 		utils.EncloseName(table.Db.Name.O),
 		utils.EncloseName(table.Info.Name.O),
 		table.Info.AutoIncID)
+
 	err = db.se.Execute(ctx, alterAutoIncIDSQL)
 	if err != nil {
 		log.Error("alter AutoIncID failed",
@@ -119,6 +120,26 @@ func (db *DB) CreateTable(ctx context.Context, table *utils.Table) error {
 			zap.Stringer("table", table.Info.Name),
 			zap.Error(err))
 	}
+	if table.Info.PKIsHandle && table.Info.ContainsAutoRandomBits() {
+		// this table has auto random id, we need rebase it
+
+		// we can't merge two alter query, because
+		// it will cause Error: [ddl:8200]Unsupported multi schema change
+		alterAutoRandIDSQL := fmt.Sprintf(
+			"alter table %s.%s auto_random_base = %d",
+			utils.EncloseName(table.Db.Name.O),
+			utils.EncloseName(table.Info.Name.O),
+			table.Info.AutoRandID)
+
+		err = db.se.Execute(ctx, alterAutoRandIDSQL)
+		if err != nil {
+			log.Error("alter AutoRandID failed",
+				zap.String("query", alterAutoRandIDSQL),
+				zap.Stringer("db", table.Db.Name),
+				zap.Stringer("table", table.Info.Name),
+				zap.Error(err))
+		}
+	}
 
 	return errors.Trace(err)
 }

pkg/storage/parse.go

@@ -4,6 +4,8 @@ package storage
 
 import (
 	"net/url"
+	"reflect"
+	"strconv"
 	"strings"
 
 	"github.com/pingcap/errors"
@@ -46,19 +48,23 @@ func ParseBackend(rawURL string, options *BackendOptions) (*backup.StorageBacken
 		}
 		prefix := strings.Trim(u.Path, "/")
 		s3 := &backup.S3{Bucket: u.Host, Prefix: prefix}
-		if options != nil {
-			if err := options.S3.apply(s3); err != nil {
-				return nil, err
-			}
+		if options == nil {
+			options = &BackendOptions{}
+		}
+		ExtractQueryParameters(u, &options.S3)
+		if err := options.S3.apply(s3); err != nil {
+			return nil, err
 		}
 		return &backup.StorageBackend{Backend: &backup.StorageBackend_S3{S3: s3}}, nil
 
-	case "gcs":
+	case "gs", "gcs":
 		gcs := &backup.GCS{Bucket: u.Host, Prefix: u.Path[1:]}
-		if options != nil {
-			if err := options.GCS.apply(gcs); err != nil {
-				return nil, err
-			}
+		if options == nil {
+			options = &BackendOptions{}
+		}
+		ExtractQueryParameters(u, &options.GCS)
+		if err := options.GCS.apply(gcs); err != nil {
+			return nil, err
 		}
 		return &backup.StorageBackend{Backend: &backup.StorageBackend_Gcs{Gcs: gcs}}, nil
 
@@ -67,6 +73,57 @@ func ParseBackend(rawURL string, options *BackendOptions) (*backup.StorageBacken
 	}
 }
 
+// ExtractQueryParameters moves the query parameters of the URL into the options
+// using reflection.
+//
+// The options must be a pointer to a struct which contains only string or bool
+// fields (more types will be supported in the future), and tagged for JSON
+// serialization.
+//
+// All of the URL's query parameters will be removed after calling this method.
+func ExtractQueryParameters(u *url.URL, options interface{}) {
+	type field struct {
+		index int
+		kind  reflect.Kind
+	}
+
+	// First, find all JSON fields in the options struct type.
+	o := reflect.Indirect(reflect.ValueOf(options))
+	ty := o.Type()
+	numFields := ty.NumField()
+	tagToField := make(map[string]field, numFields)
+	for i := 0; i < numFields; i++ {
+		f := ty.Field(i)
+		tag := f.Tag.Get("json")
+		tagToField[tag] = field{index: i, kind: f.Type.Kind()}
+	}
+
+	// Then, read content from the URL into the options.
+	for key, params := range u.Query() {
+		if len(params) == 0 {
+			continue
+		}
+		param := params[0]
+		normalizedKey := strings.ToLower(strings.ReplaceAll(key, "_", "-"))
+		if f, ok := tagToField[normalizedKey]; ok {
+			field := o.Field(f.index)
+			switch f.kind {
+			case reflect.Bool:
+				if v, e := strconv.ParseBool(param); e == nil {
+					field.SetBool(v)
+				}
+			case reflect.String:
+				field.SetString(param)
+			default:
+				panic("BackendOption introduced an unsupported kind, please handle it! " + f.kind.String())
+			}
+		}
+	}
+
+	// Clean up the URL finally.
+	u.RawQuery = ""
+}
+
 // FormatBackendURL obtains the raw URL which can be used the reconstruct the
 // backend. The returned URL does not contain options for further configurating
 // the backend. This is to avoid exposing secret tokens.