From a7b498ecfc0e7b0bddf9c46cc8902d3ffc7e2ffc Mon Sep 17 00:00:00 2001 From: 3pointer Date: Sun, 26 Apr 2020 13:10:52 +0800 Subject: [PATCH] cherry-pick master to release 4.0 (#250) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * enable tidb config by default (#230) * enable tidb config by default * backup,restore: fix --checksum flag. (#223) * backup,restore: work on progress to fix a bug that causes --checksum flag won't work properly. Some code of backup and restore ignored the flag (a.k.a. config.Checksum), so when checksum is disabled, we will face failure. * backup: backup will report total bytes and kvs when checksums check disabled. Some code of backup and restore ignored the flag (a.k.a. config.Checksum), so when checksum is disabled, we will face failure. * backup: backup will report total bytes and kvs when checksums check disabled. Some code of backup and restore ignored the flag (a.k.a. config.Checksum), so when checksum is disabled, we will face failure. * backup: add log to ChecksumMatches and new version of FastChecksum. Some of log has been lose. They are in ChecksumMatches now. * restore: restore could find non-checksum tables and skip them automatically. for backup, ChecksumMatches returns error now. * misc: add document for Table::NoChecksum. * backup: omit checksum progress bar when user specify `--checksum=false`. * backup: `CopyMetaFrom` overrides original `client.Schemes` instead of append at its end. * backup: refactor about checksum logic, fix a bug. the bug would cause: when multi tables are backup, the metadata contains only one table. * backup: do some lints. * backup,restore: do some refactor so that cyclomatic complexity won't be too large. * misc: don't use underscore on receiver. * backup: print "quick checksum success" message per table. ...to make br_full_index happy! * backup: refactor a MinInt pattern. * backup: Apply suggestions from code review Co-Authored-By: kennytm Co-authored-by: 3pointer Co-authored-by: kennytm * Use table info (#231) * create with info during incremental restore * add test * fix log * address comment * fix ci * address commemnt * backup: generate backupmeta when backup empty. (#235) * cmd: don't use ':' in the default log file name (#236) * Update build status badge (#239) * pass sse_kms_key_id to S3 (#243) * redirect kvproto Signed-off-by: Yi Wu * pass sse_kms_key_id Signed-off-by: Yi Wu * fix hound Signed-off-by: Yi Wu * update kvproto Signed-off-by: Yi Wu * go mod tidy Signed-off-by: Yi Wu Co-authored-by: kennytm * storage: support placing the S3/GCS options into the storage URL (#246) * update 4.0 dependency * go mod tidy * Update README.md Co-Authored-By: Neil Shen Co-authored-by: 山岚 <36239017+YuJuncen@users.noreply.github.com> Co-authored-by: kennytm Co-authored-by: Neil Shen Co-authored-by: yiwu-arbug --- .gitignore | 1 + README.md | 2 +- cmd/cmd.go | 2 +- go.mod | 12 +- go.sum | 74 +++++++---- pkg/backup/client.go | 103 ++++++++++++---- pkg/backup/schema.go | 12 -- pkg/restore/client.go | 8 ++ pkg/restore/db.go | 22 +++- pkg/storage/parse.go | 75 ++++++++++-- pkg/storage/parse_test.go | 36 ++++-- pkg/storage/s3.go | 22 +++- pkg/task/backup.go | 75 +++++++----- pkg/task/restore.go | 57 +++++---- pkg/utils/math.go | 12 ++ pkg/utils/math_test.go | 17 +++ pkg/utils/schema.go | 5 + tests/br_alter_pk_server/run.sh | 42 ------- tests/br_backup_empty/run.sh | 34 ++++++ .../config/tidb.toml | 3 + .../config/tikv.toml | 0 tests/br_incompatible_tidb_config/run.sh | 115 ++++++++++++++++++ tests/br_s3/run.sh | 2 +- tests/br_skip_checksum/run.sh | 88 ++++++++++++++ tests/br_skip_checksum/workload | 12 ++ 25 files changed, 645 insertions(+), 186 deletions(-) create mode 100644 pkg/utils/math.go create mode 100644 pkg/utils/math_test.go delete mode 100755 tests/br_alter_pk_server/run.sh create mode 100644 tests/br_backup_empty/run.sh rename tests/{br_alter_pk_server => br_incompatible_tidb_config}/config/tidb.toml (87%) rename tests/{br_alter_pk_server => br_incompatible_tidb_config}/config/tikv.toml (100%) create mode 100755 tests/br_incompatible_tidb_config/run.sh create mode 100755 tests/br_skip_checksum/run.sh create mode 100644 tests/br_skip_checksum/workload diff --git a/.gitignore b/.gitignore index e61a56bde..cf88895a3 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ backupmeta coverage.txt docker/data/ docker/logs/ +*.swp diff --git a/README.md b/README.md index 1d9c090d2..30d5a792f 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # BR -[![Build Status](https://internal.pingcap.net/idc-jenkins/job/build_br_master/badge/icon)](https://internal.pingcap.net/idc-jenkins/job/build_br_master/) +[![Build Status](https://internal.pingcap.net/idc-jenkins/job/build_br_multi_branch/job/release-4.0/badge/icon)](https://internal.pingcap.net/idc-jenkins/job/build_br_multi_branch/job/release-4.0/) [![codecov](https://codecov.io/gh/pingcap/br/branch/master/graph/badge.svg)](https://codecov.io/gh/pingcap/br) [![LICENSE](https://img.shields.io/github/license/pingcap/br.svg)](https://github.com/pingcap/br/blob/master/LICENSE) [![Language](https://img.shields.io/badge/Language-Go-blue.svg)](https://golang.org/) diff --git a/cmd/cmd.go b/cmd/cmd.go index 87a8aadc9..ae0dd8d9d 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -47,7 +47,7 @@ const ( ) func timestampLogFileName() string { - return filepath.Join(os.TempDir(), "br.log."+time.Now().Format(time.RFC3339)) + return filepath.Join(os.TempDir(), time.Now().Format("br.log.2006-01-02T15.04.05Z0700")) } // AddFlags adds flags to the given cmd. diff --git a/go.mod b/go.mod index acfd2c74e..8d5a03b8e 100644 --- a/go.mod +++ b/go.mod @@ -8,26 +8,24 @@ require ( github.com/cheggaaa/pb/v3 v3.0.1 github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect - github.com/fatih/color v1.9.0 // indirect github.com/fsouza/fake-gcs-server v1.15.0 github.com/go-sql-driver/mysql v1.4.1 github.com/gogo/protobuf v1.3.1 github.com/google/btree v1.0.0 github.com/google/uuid v1.1.1 github.com/klauspost/cpuid v1.2.0 // indirect - github.com/mattn/go-runewidth v0.0.7 // indirect github.com/montanaflynn/stats v0.5.0 // indirect github.com/onsi/ginkgo v1.11.0 // indirect github.com/onsi/gomega v1.8.1 // indirect github.com/pingcap/check v0.0.0-20200212061837-5e12011dc712 github.com/pingcap/errors v0.11.5-0.20190809092503-95897b64e011 - github.com/pingcap/kvproto v0.0.0-20200330093347-98f910b71904 + github.com/pingcap/kvproto v0.0.0-20200424032552-6650270c39c3 github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd - github.com/pingcap/parser v0.0.0-20200326020624-68d423641be5 - github.com/pingcap/pd/v4 v4.0.0-beta.1.0.20200305072537-61d9f9cc35d3 - github.com/pingcap/tidb v0.0.0-20200401141416-959eca8f3a39 + github.com/pingcap/parser v0.0.0-20200425031156-fb338edcaac2 + github.com/pingcap/pd/v4 v4.0.0-rc.1.0.20200422143320-428acd53eba2 + github.com/pingcap/tidb v1.1.0-beta.0.20200424160056-7267747ae0ec github.com/pingcap/tidb-tools v4.0.0-rc.1.0.20200421113014-507d2bb3a15e+incompatible - github.com/pingcap/tipb v0.0.0-20200212061130-c4d518eb1d60 + github.com/pingcap/tipb v0.0.0-20200417094153-7316d94df1ee github.com/prometheus/client_golang v1.0.0 github.com/prometheus/common v0.4.1 github.com/sirupsen/logrus v1.4.2 diff --git a/go.sum b/go.sum index 1f4bde1a7..1e74fb0f2 100644 --- a/go.sum +++ b/go.sum @@ -86,6 +86,7 @@ github.com/cznic/sortutil v0.0.0-20181122101858-f5f958428db8 h1:LpMLYGyy67BoAFGd github.com/cznic/sortutil v0.0.0-20181122101858-f5f958428db8/go.mod h1:q2w6Bg5jeox1B+QkJ6Wp/+Vn0G/bo3f1uY7Fn3vivIQ= github.com/cznic/strutil v0.0.0-20171016134553-529a34b1c186/go.mod h1:AHHPPPXTw0h6pVabbcbyGRK1DckRn7r/STdZEeIDzZc= github.com/cznic/y v0.0.0-20170802143616-045f81c6662a/go.mod h1:1rk5VM7oSnA4vjp+hrLQ3HWHa+Y4yPCa3/CsJrcNnvs= +github.com/danjacques/gofslock v0.0.0-20191023191349-0a45f885bc37/go.mod h1:DC3JtzuG7kxMvJ6dZmf2ymjNyoXwgtklr7FN+Um2B0U= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -113,6 +114,7 @@ github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsouza/fake-gcs-server v1.15.0 h1:ss/ztlt10Y64A5qslmxZKsiqW/i28t5DkRtv6qSFaLQ= @@ -126,11 +128,12 @@ github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm github.com/gin-gonic/gin v1.3.0/go.mod h1:7cKuhb5qV2ggCFctp2fJQ+ErvciLZrIeoOSOm6mUr7Y= github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= github.com/gin-gonic/gin v1.5.0/go.mod h1:Nd6IXA8m5kNZdNEHMBd93KT+mdY3+bewLgRvmCsR2Do= -github.com/go-bindata/go-bindata v3.1.2+incompatible/go.mod h1:xK8Dsgwmeed+BBsSy2XTopBn/8uK2HWuGSnA11C3Joo= +github.com/go-bindata/go-bindata/v3 v3.1.3/go.mod h1:1/zrpXsLD8YDIbhZRqXzm1Ghc7NhEvIN9+Z6R5/xH4I= github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI= github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= @@ -142,9 +145,11 @@ github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwoh github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= github.com/go-openapi/spec v0.19.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= github.com/go-openapi/spec v0.19.4/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= +github.com/go-openapi/spec v0.19.7/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk= github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.8/go.mod h1:ao+8BpOPyKdpQz3AOJfbeEVpLmWAvlT1IfTe5McPyhY= github.com/go-playground/locales v0.12.1/go.mod h1:IUMDtCfWo/w/mtMfIE/IG2K+Ey3ygWanZIBtBW0W2TM= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= github.com/go-playground/overalls v0.0.0-20180201144345-22ec1a223b7c/go.mod h1:UqxAgEOt89sCiXlrc/ycnx00LVvUO/eS8tMUkWX4R7w= @@ -217,6 +222,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.9.5 h1:UImYN5qQ8tuGpGE16ZmjvcTtTw24zw1 github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.12.1 h1:zCy2xE9ablevUOrUZc3Dl72Dt+ya2FNAvC2yLYMHzi4= github.com/grpc-ecosystem/grpc-gateway v1.12.1/go.mod h1:8XEsbTttt/W+VvjtQhLACqCisSPWTxCZ7sBRjU6iH9c= +github.com/grpc-ecosystem/grpc-gateway v1.14.3 h1:OCJlWkOUoTnl0neNGlf4fUm3TmbEtguw7vR+nGtnDjY= +github.com/grpc-ecosystem/grpc-gateway v1.14.3/go.mod h1:6CwZWGDSPRJidgKAtJVvND6soZe6fT7iteq8wDPdhb0= github.com/gtank/cryptopasta v0.0.0-20170601214702-1f550f6f2f69/go.mod h1:YLEMZOtU+AZ7dhN9T/IpGhXVGly2bvkJQ+zxj3WeVQo= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= @@ -227,8 +234,6 @@ github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jeremywohl/flatten v0.0.0-20190921043622-d936035e55cf h1:Ut4tTtPNmInWiEWJRernsWm688R0RN6PFO8sZhwI0sk= -github.com/jeremywohl/flatten v0.0.0-20190921043622-d936035e55cf/go.mod h1:4AmD/VxjWcI5SRB0n6szE2A6s2fsNHDLO0nAlMHgfLQ= github.com/jinzhu/gorm v1.9.12/go.mod h1:vhTjlKSJUTWNtcbQtrMBFCxy7eXTzeCAzfL5fBZT/Qs= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= @@ -249,6 +254,7 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/juju/ratelimit v1.0.1 h1:+7AIFJVQ0EQgq/K9+0Krm7m530Du7tIz0METWzN0RgY= github.com/juju/ratelimit v1.0.1/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kevinburke/go-bindata v3.18.0+incompatible/go.mod h1:/pEEZ72flUW2p0yi30bslSp9YqD9pysLxunQDdb2CPM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= @@ -275,6 +281,7 @@ github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czP github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= @@ -296,6 +303,8 @@ github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vq github.com/mattn/go-sqlite3 v2.0.1+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mgechev/dots v0.0.0-20190921121421-c36f7dcfbb81/go.mod h1:KQ7+USdGKfpPjXk4Ga+5XxQM4Lm4e3gAogrreFAYpOg= +github.com/mgechev/revive v1.0.2/go.mod h1:rb0dQy1LVAxW9SWy5R3LPUjevzUbUS316U5MFySA2lo= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= @@ -318,6 +327,7 @@ github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef h1:K0Fn+DoFqNqktdZtdV3 github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef/go.mod h1:7WjlapSfwQyo6LNmIvEWzsW1hbBQfpUO4JWnuQRmva8= github.com/nicksnyder/go-i18n v1.10.0/go.mod h1:HrK7VCrbOvQoUAQ7Vpy7i87N7JZZZ7R2xBGjv0j365Q= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA= github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -337,7 +347,7 @@ github.com/pelletier/go-toml v1.3.0 h1:e5+lF2E4Y2WCIxBefVowBuB0iHrUH4HZ8q+6mGF7f github.com/pelletier/go-toml v1.3.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d h1:U+PMnTlV2tu7RuMK5etusZG3Cf+rpow5hqQByeCzJ2g= github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d/go.mod h1:lXfE4PvvTW5xOjO6Mba8zDPyw8M93B6AQ7frTGnMlA8= -github.com/pingcap-incubator/tidb-dashboard v0.0.0-20200302022638-35a6e979dca9/go.mod h1:YUceA4BHY/MTtp63yZLTYP22waFSwMNo9lXq2FDtzVw= +github.com/pingcap-incubator/tidb-dashboard v0.0.0-20200407064406-b2b8ad403d01/go.mod h1:77fCh8d3oKzC5ceOJWeZXAS/mLzVgdZ7rKniwmOyFuo= github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8 h1:USx2/E1bX46VG32FIw034Au6seQ2fY9NEILmNh/UlQg= github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ= github.com/pingcap/check v0.0.0-20191107115940-caf2b9e6ccf4 h1:iRtOAQ6FXkY/BGvst3CDfTva4nTqh6CL8WXvanLdbu0= @@ -362,31 +372,31 @@ github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 h1:surzm05a8C9dN github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw= github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= github.com/pingcap/kvproto v0.0.0-20200214064158-62d31900d88e/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= -github.com/pingcap/kvproto v0.0.0-20200221034943-a2aa1d1e20a8/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= -github.com/pingcap/kvproto v0.0.0-20200330093347-98f910b71904 h1:pMFUXvhJ62hX8m0Q4RsL7L+hSW1mAMG26So5eFMoAtI= -github.com/pingcap/kvproto v0.0.0-20200330093347-98f910b71904/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= +github.com/pingcap/kvproto v0.0.0-20200417092353-efbe03bcffbd/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= +github.com/pingcap/kvproto v0.0.0-20200420075417-e0c6e8842f22/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= +github.com/pingcap/kvproto v0.0.0-20200424032552-6650270c39c3 h1:EBq71gaLG7bxgCjR5DA00mzvF5dwEZBPd8xuGSztamw= +github.com/pingcap/kvproto v0.0.0-20200424032552-6650270c39c3/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9 h1:AJD9pZYm72vMgPcQDww9rkZ1DnWfl0pXV3BOWlkYIjA= github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd h1:CV3VsP3Z02MVtdpTMfEgRJ4T9NGgGTxdHpJerent7rM= github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= -github.com/pingcap/parser v0.0.0-20200326020624-68d423641be5 h1:fXVqoeYfV+xI8K2he5NNv00c6YksrjeM6+vkNo1ZK2Q= -github.com/pingcap/parser v0.0.0-20200326020624-68d423641be5/go.mod h1:9v0Edh8IbgjGYW2ArJr19E+bvL8zKahsFp+ixWeId+4= -github.com/pingcap/pd/v4 v4.0.0-beta.1.0.20200305072537-61d9f9cc35d3 h1:Yrp99FnjHAEuDrSBql2l0IqCtJX7KwJbTsD5hIArkvk= -github.com/pingcap/pd/v4 v4.0.0-beta.1.0.20200305072537-61d9f9cc35d3/go.mod h1:25GfNw6+Jcr9kca5rtmTb4gKCJ4jOpow2zV2S9Dgafs= +github.com/pingcap/parser v0.0.0-20200422082501-7329d80eaf2c/go.mod h1:9v0Edh8IbgjGYW2ArJr19E+bvL8zKahsFp+ixWeId+4= +github.com/pingcap/parser v0.0.0-20200425031156-fb338edcaac2 h1:M4ZwwFQ38yTQhqHFx7kRUfavCgsyPqhZtVA3CXxT0YY= +github.com/pingcap/parser v0.0.0-20200425031156-fb338edcaac2/go.mod h1:9v0Edh8IbgjGYW2ArJr19E+bvL8zKahsFp+ixWeId+4= +github.com/pingcap/pd/v4 v4.0.0-rc.1.0.20200422143320-428acd53eba2 h1:JTzYYukREvxVSKW/ncrzNjFitd8snoQ/Xz32pw8i+s8= +github.com/pingcap/pd/v4 v4.0.0-rc.1.0.20200422143320-428acd53eba2/go.mod h1:s+utZtXDznOiL24VK0qGmtoHjjXNsscJx3m1n8cC56s= github.com/pingcap/sysutil v0.0.0-20200206130906-2bfa6dc40bcd/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI= -github.com/pingcap/sysutil v0.0.0-20200302022240-21c8c70d0ab1 h1:YUnUZ914SHFMsOSe/xgH5DKK/thtRma8X8hcszRo3CA= -github.com/pingcap/sysutil v0.0.0-20200302022240-21c8c70d0ab1/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI= -github.com/pingcap/sysutil v0.0.0-20200309085538-962fd285f3bb h1:bDbgLaNTRNK6Qw7KjvEqqfCQstY8WMEcXyXTU7yzYKg= -github.com/pingcap/sysutil v0.0.0-20200309085538-962fd285f3bb/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI= -github.com/pingcap/tidb v0.0.0-20200401141416-959eca8f3a39 h1:nYRL69Qc4kuvp+tlDNB5wXjvDetX0J7g0DsW4RQxfXM= -github.com/pingcap/tidb v0.0.0-20200401141416-959eca8f3a39/go.mod h1:btnHsqUQvJnY18+OP2Z6MCRq1tX4B8JUCrmqctSKxOg= +github.com/pingcap/sysutil v0.0.0-20200408114249-ed3bd6f7fdb1 h1:PI8YpTl45F8ilNkrPtT4IdbcZB1SCEa+gK/U5GJYl3E= +github.com/pingcap/sysutil v0.0.0-20200408114249-ed3bd6f7fdb1/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI= +github.com/pingcap/tidb v1.1.0-beta.0.20200424160056-7267747ae0ec h1:/cIJBjoTvXxOZcZkJdwNLw75gaUz8La9O6JhpOmeTL8= +github.com/pingcap/tidb v1.1.0-beta.0.20200424160056-7267747ae0ec/go.mod h1:KJXj2xHYfl1x4zcusC2JEANzVci+ietFOMh/CAmrYdw= github.com/pingcap/tidb-tools v4.0.0-beta.1.0.20200306084441-875bd09aa3d5+incompatible h1:84F7MFMfdAYObrznvRslmVu43aoihrlL+7mMyMlOi0o= github.com/pingcap/tidb-tools v4.0.0-beta.1.0.20200306084441-875bd09aa3d5+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= github.com/pingcap/tidb-tools v4.0.0-rc.1.0.20200421113014-507d2bb3a15e+incompatible h1:+K5bqDYG5HT+GqLdx4GH5VmS84+xHgpHbGg6Xt6qQec= github.com/pingcap/tidb-tools v4.0.0-rc.1.0.20200421113014-507d2bb3a15e+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= github.com/pingcap/tipb v0.0.0-20190428032612-535e1abaa330/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= -github.com/pingcap/tipb v0.0.0-20200212061130-c4d518eb1d60 h1:aJPXrT1u4VfUSGFA2oQVwl4pOXzqe+YI6wed01cjDH4= -github.com/pingcap/tipb v0.0.0-20200212061130-c4d518eb1d60/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= +github.com/pingcap/tipb v0.0.0-20200417094153-7316d94df1ee h1:XJQ6/LGzOSc/jo33AD8t7jtc4GohxcyODsYnb+kZXJM= +github.com/pingcap/tipb v0.0.0-20200417094153-7316d94df1ee/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -421,8 +431,8 @@ github.com/shirou/gopsutil v2.19.10+incompatible h1:lA4Pi29JEVIQIgATSeftHSY0rMGI github.com/shirou/gopsutil v2.19.10+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 h1:udFKJ0aHUL60LboW/A+DfgoHVedieIzIXE8uylPue0U= github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= -github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371 h1:SWV2fHctRpRrp49VXJ6UZja7gU9QLHwRpIPBN89SKEo= -github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= +github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 h1:bUGsEnyNbVPw06Bs80sCeARAlK8lhwqGyi6UT8ymuGk= +github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/shurcooL/vfsgen v0.0.0-20181020040650-a97a25d856ca h1:3fECS8atRjByijiI8yYiuwLwQ2ZxXobW7ua/8GRB3pI= github.com/shurcooL/vfsgen v0.0.0-20181020040650-a97a25d856ca/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= @@ -460,9 +470,11 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/swaggo/files v0.0.0-20190704085106-630677cd5c14/go.mod h1:gxQT6pBGRuIGunNf/+tSOB5OHvguWi8Tbt82WOkf35E= github.com/swaggo/gin-swagger v1.2.0/go.mod h1:qlH2+W7zXGZkczuL+r2nEBR2JTT+/lX05Nn6vPhc7OI= github.com/swaggo/http-swagger v0.0.0-20200103000832-0e9263c4b516/go.mod h1:O1lAbCgAAX/KZ80LM/OXwtWFI/5TvZlwxSg8Cq08PV0= +github.com/swaggo/http-swagger v0.0.0-20200308142732-58ac5e232fba/go.mod h1:O1lAbCgAAX/KZ80LM/OXwtWFI/5TvZlwxSg8Cq08PV0= github.com/swaggo/swag v1.5.1/go.mod h1:1Bl9F/ZBpVWh22nY0zmYyASPO1lI/zIwRDrpZU+tv8Y= github.com/swaggo/swag v1.6.3/go.mod h1:wcc83tB4Mb2aNiL/HP4MFeQdpHUrca+Rp/DRNgWAUio= github.com/swaggo/swag v1.6.5/go.mod h1:Y7ZLSS0d0DdxhWGVhQdu+Bu1QhaF5k0RD7FKdiAykeY= +github.com/swaggo/swag v1.6.6-0.20200323071853-8e21f4cefeea/go.mod h1:xDhTyuFIujYiN3DKWC/H/83xcfHp+UE/IzWWampG7Zc= github.com/syndtr/goleveldb v0.0.0-20180815032940-ae2bd5eed72d h1:4J9HCZVpvDmj2tiKGSTUnb3Ok/9CEQb9oqu9LHKQQpc= github.com/syndtr/goleveldb v0.0.0-20180815032940-ae2bd5eed72d/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0= github.com/syndtr/goleveldb v1.0.1-0.20190625010220-02440ea7a285 h1:uSDYjYejelKyceA6DiCsngFof9jAyeaSyX9XC5a1a7Q= @@ -494,6 +506,7 @@ github.com/unrolled/render v0.0.0-20171102162132-65450fb6b2d3 h1:ZsIlNwu/G0zbChI github.com/unrolled/render v0.0.0-20171102162132-65450fb6b2d3/go.mod h1:tu82oB5W2ykJRVioYsB+IQKcft7ryBr7w12qMBUPyXg= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli/v2 v2.1.1/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= github.com/urfave/negroni v0.3.0 h1:PaXOb61mWeZJxc1Ji2xJjpVg9QfPo0rrB+lHyBxGNSU= github.com/urfave/negroni v0.3.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= @@ -519,6 +532,8 @@ go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/automaxprocs v1.2.0/go.mod h1:YfO3fm683kQpzETxlTGZhGIVmXAhaw3gxeBADbpZtnU= +go.uber.org/dig v1.8.0/go.mod h1:X34SnWGr8Fyla9zQNO2GSO2D+TIuqB14OS8JhYocIyw= +go.uber.org/fx v1.10.0/go.mod h1:vLRicqpG/qQEzno4SYU86iCwfT95EZza+Eba0ItuxqY= go.uber.org/goleak v0.10.0 h1:G3eWbSNIskeRqtsN/1uI5B+eP73y3JUuBsv9AZjehb4= go.uber.org/goleak v0.10.0/go.mod h1:VCZuO8V8mFPlL0F5J5GK1rtHV3DrFcQ1R8ryq7FK0aI= go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= @@ -567,6 +582,7 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= @@ -598,8 +614,8 @@ golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a h1:GuSPYbZzB5/dcLNCwLQLsg3obCJtX9IJhpXkvY7kzk0= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= @@ -633,8 +649,8 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1 h1:gZpLHxUX5BdYLA08Lj4YCJNN/jk7KtquiArPoeX0WvA= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -668,15 +684,18 @@ golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a h1:TwMENskLwU2NnWBzrJGEWHq golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191030062658-86caa796c7ab/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191107010934-f79515f33823 h1:akkRBeitX2EZP59KdtKw310CI4WGPCNPyrLbE7WZA8Y= golang.org/x/tools v0.0.0-20191107010934-f79515f33823/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191114200427-caa0b0f7d508/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2 h1:EtTFh6h4SAKemS+CURDMTDIANuduG5zKEXShyy18bGA= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200301222351-066e0c02454c h1:FD7jysxM+EJqg5UYYy3XYDsAiUickFsn4UiaanJkf8c= -golang.org/x/tools v0.0.0-20200301222351-066e0c02454c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200225230052-807dcd883420/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200325010219-a49f79bcc224/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200325203130-f53864d0dba1 h1:odiryKYJy7CjdrZxhrcE1Z8L9+kGyGZOnfpuauvdCeU= golang.org/x/tools v0.0.0-20200325203130-f53864d0dba1/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -759,6 +778,7 @@ honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXe honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3 h1:sXmLre5bzIR6ypkjXCDI3jHPssRhc8KD/Ome589sc3U= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= diff --git a/pkg/backup/client.go b/pkg/backup/client.go index 72563096b..1b016ba8b 100644 --- a/pkg/backup/client.go +++ b/pkg/backup/client.go @@ -48,6 +48,13 @@ type ClientMgr interface { Close() } +// Checksum is the checksum of some backup files calculated by CollectChecksums. +type Checksum struct { + Crc64Xor uint64 + TotalKvs uint64 + TotalBytes uint64 +} + // Maximum total sleep time(in ms) for kv/cop commands. const ( backupFineGrainedMaxBackoff = 80000 @@ -748,8 +755,59 @@ func SendBackup( return nil } -// FastChecksum check data integrity by xor all(sst_checksum) per table -func (bc *Client) FastChecksum() (bool, error) { +// ChecksumMatches tests whether the "local" checksum matches the checksum from TiKV. +func (bc *Client) ChecksumMatches(local []Checksum) (bool, error) { + if len(local) != len(bc.backupMeta.Schemas) { + return false, nil + } + + for i, schema := range bc.backupMeta.Schemas { + localChecksum := local[i] + dbInfo := &model.DBInfo{} + err := json.Unmarshal(schema.Db, dbInfo) + if err != nil { + log.Error("failed in fast checksum, and cannot parse db info.") + return false, err + } + tblInfo := &model.TableInfo{} + err = json.Unmarshal(schema.Table, tblInfo) + if err != nil { + log.Error("failed in fast checksum, and cannot parse table info.") + return false, err + } + if localChecksum.Crc64Xor != schema.Crc64Xor || + localChecksum.TotalBytes != schema.TotalBytes || + localChecksum.TotalKvs != schema.TotalKvs { + log.Error("failed in fast checksum", + zap.Stringer("db", dbInfo.Name), + zap.Stringer("table", tblInfo.Name), + zap.Uint64("origin tidb crc64", schema.Crc64Xor), + zap.Uint64("calculated crc64", localChecksum.Crc64Xor), + zap.Uint64("origin tidb total kvs", schema.TotalKvs), + zap.Uint64("calculated total kvs", localChecksum.TotalKvs), + zap.Uint64("origin tidb total bytes", schema.TotalBytes), + zap.Uint64("calculated total bytes", localChecksum.TotalBytes), + ) + return false, nil + } + log.Info("fast checksum success", + zap.String("database", dbInfo.Name.L), + zap.String("table", tblInfo.Name.L)) + } + return true, nil +} + +// CollectFileInfo collects ungrouped file summary information, like kv count and size. +func (bc *Client) CollectFileInfo() { + for _, file := range bc.backupMeta.Files { + summary.CollectSuccessUnit(summary.TotalKV, 1, file.TotalKvs) + summary.CollectSuccessUnit(summary.TotalBytes, 1, file.TotalBytes) + } +} + +// CollectChecksums check data integrity by xor all(sst_checksum) per table +// it returns the checksum of all local files. +func (bc *Client) CollectChecksums() ([]Checksum, error) { start := time.Now() defer func() { elapsed := time.Since(start) @@ -758,19 +816,20 @@ func (bc *Client) FastChecksum() (bool, error) { dbs, err := utils.LoadBackupTables(&bc.backupMeta) if err != nil { - return false, err + return nil, err } + checksums := make([]Checksum, 0, len(bc.backupMeta.Schemas)) for _, schema := range bc.backupMeta.Schemas { dbInfo := &model.DBInfo{} err = json.Unmarshal(schema.Db, dbInfo) if err != nil { - return false, err + return nil, err } tblInfo := &model.TableInfo{} err = json.Unmarshal(schema.Table, tblInfo) if err != nil { - return false, err + return nil, err } tbl := dbs[dbInfo.Name.String()].GetTable(tblInfo.Name.String()) @@ -785,25 +844,16 @@ func (bc *Client) FastChecksum() (bool, error) { summary.CollectSuccessUnit(summary.TotalKV, 1, totalKvs) summary.CollectSuccessUnit(summary.TotalBytes, 1, totalBytes) - - if schema.Crc64Xor == checksum && schema.TotalKvs == totalKvs && schema.TotalBytes == totalBytes { - log.Info("fast checksum success", zap.Stringer("db", dbInfo.Name), zap.Stringer("table", tblInfo.Name)) - } else { - log.Error("failed in fast checksum", - zap.String("database", dbInfo.Name.String()), - zap.String("table", tblInfo.Name.String()), - zap.Uint64("origin tidb crc64", schema.Crc64Xor), - zap.Uint64("calculated crc64", checksum), - zap.Uint64("origin tidb total kvs", schema.TotalKvs), - zap.Uint64("calculated total kvs", totalKvs), - zap.Uint64("origin tidb total bytes", schema.TotalBytes), - zap.Uint64("calculated total bytes", totalBytes), - ) - return false, nil + log.Info("fast checksum calculated", zap.Stringer("db", dbInfo.Name), zap.Stringer("table", tblInfo.Name)) + localChecksum := Checksum{ + Crc64Xor: checksum, + TotalKvs: totalKvs, + TotalBytes: totalBytes, } + checksums = append(checksums, localChecksum) } - return true, nil + return checksums, nil } // CompleteMeta wait response of admin checksum from TiDB to complete backup meta @@ -815,3 +865,14 @@ func (bc *Client) CompleteMeta(backupSchemas *Schemas) error { bc.backupMeta.Schemas = schemas return nil } + +// CopyMetaFrom copies schema metadata directly from pending backupSchemas, without calculating checksum. +// use this when user skip the checksum generating. +func (bc *Client) CopyMetaFrom(backupSchemas *Schemas) { + schemas := make([]*kvproto.Schema, 0, len(backupSchemas.schemas)) + for _, v := range backupSchemas.schemas { + schema := v + schemas = append(schemas, &schema) + } + bc.backupMeta.Schemas = schemas +} diff --git a/pkg/backup/schema.go b/pkg/backup/schema.go index 73a62477d..d1b5943a8 100644 --- a/pkg/backup/schema.go +++ b/pkg/backup/schema.go @@ -36,7 +36,6 @@ type Schemas struct { backupSchemaCh chan backup.Schema errCh chan error wg *sync.WaitGroup - skipChecksum bool } func newBackupSchemas() *Schemas { @@ -57,11 +56,6 @@ func (pending *Schemas) pushPending( pending.schemas[name] = schema } -// SetSkipChecksum sets whether it should skip checksum -func (pending *Schemas) SetSkipChecksum(skip bool) { - pending.skipChecksum = skip -} - // Start backups schemas func (pending *Schemas) Start( ctx context.Context, @@ -81,12 +75,6 @@ func (pending *Schemas) Start( workerPool.Apply(func() { defer pending.wg.Done() - if pending.skipChecksum { - pending.backupSchemaCh <- schema - updateCh.Inc() - return - } - start := time.Now() table := model.TableInfo{} err := json.Unmarshal(schema.Table, &table) diff --git a/pkg/restore/client.go b/pkg/restore/client.go index fde382fb0..2b685e599 100644 --- a/pkg/restore/client.go +++ b/pkg/restore/client.go @@ -650,6 +650,14 @@ func (rc *Client) ValidateChecksum( workers.Apply(func() { defer wg.Done() + if table.NoChecksum() { + log.Info("table doesn't have checksum, skipping checksum", + zap.Stringer("db", table.Db.Name), + zap.Stringer("table", table.Info.Name)) + updateCh.Inc() + return + } + startTS, err := rc.GetTS(ctx) if err != nil { errCh <- errors.Trace(err) diff --git a/pkg/restore/db.go b/pkg/restore/db.go index 6197ff7a2..4ff1b5ea2 100644 --- a/pkg/restore/db.go +++ b/pkg/restore/db.go @@ -45,7 +45,27 @@ func NewDB(g glue.Glue, store kv.Storage) (*DB, error) { // ExecDDL executes the query of a ddl job. func (db *DB) ExecDDL(ctx context.Context, ddlJob *model.Job) error { var err error - if ddlJob.BinlogInfo.TableInfo != nil { + tableInfo := ddlJob.BinlogInfo.TableInfo + dbInfo := ddlJob.BinlogInfo.DBInfo + switch ddlJob.Type { + case model.ActionCreateSchema: + err = db.se.CreateDatabase(ctx, dbInfo) + if err != nil { + log.Error("create database failed", zap.Stringer("db", dbInfo.Name), zap.Error(err)) + } + return errors.Trace(err) + case model.ActionCreateTable: + err = db.se.CreateTable(ctx, model.NewCIStr(ddlJob.SchemaName), tableInfo) + if err != nil { + log.Error("create table failed", + zap.Stringer("db", dbInfo.Name), + zap.Stringer("table", tableInfo.Name), + zap.Error(err)) + } + return errors.Trace(err) + } + + if tableInfo != nil { switchDbSQL := fmt.Sprintf("use %s;", utils.EncloseName(ddlJob.SchemaName)) err = db.se.Execute(ctx, switchDbSQL) if err != nil { diff --git a/pkg/storage/parse.go b/pkg/storage/parse.go index d75e7663d..fff518bfb 100644 --- a/pkg/storage/parse.go +++ b/pkg/storage/parse.go @@ -4,6 +4,8 @@ package storage import ( "net/url" + "reflect" + "strconv" "strings" "github.com/pingcap/errors" @@ -46,19 +48,23 @@ func ParseBackend(rawURL string, options *BackendOptions) (*backup.StorageBacken } prefix := strings.Trim(u.Path, "/") s3 := &backup.S3{Bucket: u.Host, Prefix: prefix} - if options != nil { - if err := options.S3.apply(s3); err != nil { - return nil, err - } + if options == nil { + options = &BackendOptions{} + } + ExtractQueryParameters(u, &options.S3) + if err := options.S3.apply(s3); err != nil { + return nil, err } return &backup.StorageBackend{Backend: &backup.StorageBackend_S3{S3: s3}}, nil - case "gcs": + case "gs", "gcs": gcs := &backup.GCS{Bucket: u.Host, Prefix: u.Path[1:]} - if options != nil { - if err := options.GCS.apply(gcs); err != nil { - return nil, err - } + if options == nil { + options = &BackendOptions{} + } + ExtractQueryParameters(u, &options.GCS) + if err := options.GCS.apply(gcs); err != nil { + return nil, err } return &backup.StorageBackend{Backend: &backup.StorageBackend_Gcs{Gcs: gcs}}, nil @@ -67,6 +73,57 @@ func ParseBackend(rawURL string, options *BackendOptions) (*backup.StorageBacken } } +// ExtractQueryParameters moves the query parameters of the URL into the options +// using reflection. +// +// The options must be a pointer to a struct which contains only string or bool +// fields (more types will be supported in the future), and tagged for JSON +// serialization. +// +// All of the URL's query parameters will be removed after calling this method. +func ExtractQueryParameters(u *url.URL, options interface{}) { + type field struct { + index int + kind reflect.Kind + } + + // First, find all JSON fields in the options struct type. + o := reflect.Indirect(reflect.ValueOf(options)) + ty := o.Type() + numFields := ty.NumField() + tagToField := make(map[string]field, numFields) + for i := 0; i < numFields; i++ { + f := ty.Field(i) + tag := f.Tag.Get("json") + tagToField[tag] = field{index: i, kind: f.Type.Kind()} + } + + // Then, read content from the URL into the options. + for key, params := range u.Query() { + if len(params) == 0 { + continue + } + param := params[0] + normalizedKey := strings.ToLower(strings.ReplaceAll(key, "_", "-")) + if f, ok := tagToField[normalizedKey]; ok { + field := o.Field(f.index) + switch f.kind { + case reflect.Bool: + if v, e := strconv.ParseBool(param); e == nil { + field.SetBool(v) + } + case reflect.String: + field.SetString(param) + default: + panic("BackendOption introduced an unsupported kind, please handle it! " + f.kind.String()) + } + } + } + + // Clean up the URL finally. + u.RawQuery = "" +} + // FormatBackendURL obtains the raw URL which can be used the reconstruct the // backend. The returned URL does not contain options for further configurating // the backend. This is to avoid exposing secret tokens. diff --git a/pkg/storage/parse_test.go b/pkg/storage/parse_test.go index 3f1bc4d4f..51669a806 100644 --- a/pkg/storage/parse_test.go +++ b/pkg/storage/parse_test.go @@ -4,7 +4,8 @@ package storage import ( "io/ioutil" - "os" + "net/url" + "path/filepath" "testing" . "github.com/pingcap/check" @@ -54,6 +55,17 @@ func (r *testStorageSuite) TestCreateStorage(c *C) { c.Assert(s3.Prefix, Equals, "prefix") c.Assert(s3.Endpoint, Equals, "https://s3.example.com/") + s, err = ParseBackend("s3://bucket3/prefix/path?endpoint=https://127.0.0.1:9000&force_path_style=1&SSE=aws:kms&sse-kms-key-id=TestKey&xyz=abc", nil) + c.Assert(err, IsNil) + s3 = s.GetS3() + c.Assert(s3, NotNil) + c.Assert(s3.Bucket, Equals, "bucket3") + c.Assert(s3.Prefix, Equals, "prefix/path") + c.Assert(s3.Endpoint, Equals, "https://127.0.0.1:9000") + c.Assert(s3.ForcePathStyle, IsTrue) + c.Assert(s3.Sse, Equals, "aws:kms") + c.Assert(s3.SseKmsKeyId, Equals, "TestKey") + gcsOpt := &BackendOptions{ GCS: GCSBackendOptions{ Endpoint: "https://gcs.example.com/", @@ -68,15 +80,11 @@ func (r *testStorageSuite) TestCreateStorage(c *C) { c.Assert(gcs.Endpoint, Equals, "https://gcs.example.com/") c.Assert(gcs.CredentialsBlob, Equals, "") - fakeCredentialsFile, err := ioutil.TempFile("", "fakeCredentialsFile") + fakeCredentialsFile := filepath.Join(c.MkDir(), "fakeCredentialsFile") + err = ioutil.WriteFile(fakeCredentialsFile, []byte("fakeCredentials"), 0600) c.Assert(err, IsNil) - _, err = fakeCredentialsFile.Write([]byte("fakeCredentials")) - c.Assert(err, IsNil) - defer func() { - fakeCredentialsFile.Close() - os.Remove(fakeCredentialsFile.Name()) - }() - gcsOpt.GCS.CredentialsFile = fakeCredentialsFile.Name() + + gcsOpt.GCS.CredentialsFile = fakeCredentialsFile s, err = ParseBackend("gcs://bucket/more/prefix/", gcsOpt) c.Assert(err, IsNil) @@ -86,6 +94,16 @@ func (r *testStorageSuite) TestCreateStorage(c *C) { c.Assert(gcs.Prefix, Equals, "more/prefix/") c.Assert(gcs.Endpoint, Equals, "https://gcs.example.com/") c.Assert(gcs.CredentialsBlob, Equals, "fakeCredentials") + + err = ioutil.WriteFile(fakeCredentialsFile, []byte("fakeCreds2"), 0600) + c.Assert(err, IsNil) + s, err = ParseBackend("gs://bucket4/backup/?credentials-file="+url.QueryEscape(fakeCredentialsFile), nil) + c.Assert(err, IsNil) + gcs = s.GetGcs() + c.Assert(gcs, NotNil) + c.Assert(gcs.Bucket, Equals, "bucket4") + c.Assert(gcs.Prefix, Equals, "backup/") + c.Assert(gcs.CredentialsBlob, Equals, "fakeCreds2") } func (r *testStorageSuite) TestFormatBackendURL(c *C) { diff --git a/pkg/storage/s3.go b/pkg/storage/s3.go index bf24b9a2b..00107e2b9 100644 --- a/pkg/storage/s3.go +++ b/pkg/storage/s3.go @@ -24,7 +24,8 @@ const ( s3EndpointOption = "s3.endpoint" s3RegionOption = "s3.region" s3StorageClassOption = "s3.storage-class" - s3SSEOption = "s3.sse" + s3SseOption = "s3.sse" + s3SseKmsKeyIDOption = "s3.sse-kms-key-id" s3ACLOption = "s3.acl" s3ProviderOption = "s3.provider" notFound = "NotFound" @@ -53,7 +54,8 @@ type S3BackendOptions struct { Endpoint string `json:"endpoint" toml:"endpoint"` Region string `json:"region" toml:"region"` StorageClass string `json:"storage-class" toml:"storage-class"` - SSE string `json:"sse" toml:"sse"` + Sse string `json:"sse" toml:"sse"` + SseKmsKeyID string `json:"sse-kms-key-id" toml:"sse-kms-key-id"` ACL string `json:"acl" toml:"acl"` AccessKey string `json:"access-key" toml:"access-key"` SecretAccessKey string `json:"secret-access-key" toml:"secret-access-key"` @@ -95,7 +97,8 @@ func (options *S3BackendOptions) apply(s3 *backup.S3) error { s3.Region = options.Region // StorageClass, SSE and ACL are acceptable to be empty s3.StorageClass = options.StorageClass - s3.Sse = options.SSE + s3.Sse = options.Sse + s3.SseKmsKeyId = options.SseKmsKeyID s3.Acl = options.ACL s3.AccessKey = options.AccessKey s3.SecretAccessKey = options.SecretAccessKey @@ -109,7 +112,9 @@ func defineS3Flags(flags *pflag.FlagSet) { "(experimental) Set the S3 endpoint URL, please specify the http or https scheme explicitly") flags.String(s3RegionOption, "", "(experimental) Set the S3 region, e.g. us-east-1") flags.String(s3StorageClassOption, "", "(experimental) Set the S3 storage class, e.g. STANDARD") - flags.String(s3SSEOption, "", "(experimental) Set the S3 server-side encryption algorithm, e.g. AES256") + flags.String(s3SseOption, "", "Set S3 server-side encryption, e.g. aws:kms") + flags.String(s3SseKmsKeyIDOption, "", "KMS CMK key id to use with S3 server-side encryption."+ + "Leave empty to use S3 owned key.") flags.String(s3ACLOption, "", "(experimental) Set the S3 canned ACLs, e.g. authenticated-read") flags.String(s3ProviderOption, "", "(experimental) Set the S3 provider, e.g. aws, alibaba, ceph") } @@ -124,7 +129,11 @@ func (options *S3BackendOptions) parseFromFlags(flags *pflag.FlagSet) error { if err != nil { return errors.Trace(err) } - options.SSE, err = flags.GetString(s3SSEOption) + options.Sse, err = flags.GetString(s3SseOption) + if err != nil { + return errors.Trace(err) + } + options.SseKmsKeyID, err = flags.GetString(s3SseKmsKeyIDOption) if err != nil { return errors.Trace(err) } @@ -224,6 +233,9 @@ func (rs *S3Storage) Write(ctx context.Context, file string, data []byte) error if rs.options.Sse != "" { input = input.SetServerSideEncryption(rs.options.Sse) } + if rs.options.SseKmsKeyId != "" { + input = input.SetSSEKMSKeyId(rs.options.SseKmsKeyId) + } if rs.options.StorageClass != "" { input = input.SetStorageClass(rs.options.StorageClass) } diff --git a/pkg/task/backup.go b/pkg/task/backup.go index 040be0444..c24a6c607 100644 --- a/pkg/task/backup.go +++ b/pkg/task/backup.go @@ -7,6 +7,8 @@ import ( "strconv" "time" + "github.com/pingcap/br/pkg/utils" + "github.com/pingcap/errors" kvproto "github.com/pingcap/kvproto/pkg/backup" "github.com/pingcap/log" @@ -128,7 +130,7 @@ func RunBackup(c context.Context, g glue.Glue, cmdName string, cfg *BackupConfig } // nothing to backup if ranges == nil { - return nil + return client.SaveBackupMeta(ctx, nil) } ddlJobs := make([]*model.Job, 0) @@ -180,39 +182,32 @@ func RunBackup(c context.Context, g glue.Glue, cmdName string, cfg *BackupConfig // Backup has finished updateCh.Close() - // Checksum - backupSchemasConcurrency := backup.DefaultSchemaConcurrency - if backupSchemas.Len() < backupSchemasConcurrency { - backupSchemasConcurrency = backupSchemas.Len() - } - updateCh = g.StartProgress( - ctx, "Checksum", int64(backupSchemas.Len()), !cfg.LogProgress) - backupSchemas.SetSkipChecksum(!cfg.Checksum) - backupSchemas.Start( - ctx, mgr.GetTiKV(), backupTS, uint(backupSchemasConcurrency), updateCh) - - err = client.CompleteMeta(backupSchemas) - if err != nil { - return err - } - - if cfg.LastBackupTS == 0 { - var valid bool - valid, err = client.FastChecksum() + // Checksum from server, and then fulfill the backup metadata. + if cfg.Checksum { + backupSchemasConcurrency := utils.MinInt(backup.DefaultSchemaConcurrency, backupSchemas.Len()) + updateCh = g.StartProgress( + ctx, "Checksum", int64(backupSchemas.Len()), !cfg.LogProgress) + backupSchemas.Start( + ctx, mgr.GetTiKV(), backupTS, uint(backupSchemasConcurrency), updateCh) + err = client.CompleteMeta(backupSchemas) if err != nil { return err } - if !valid { - log.Error("backup FastChecksum mismatch!") - return errors.Errorf("mismatched checksum") + // Checksum has finished + updateCh.Close() + // collect file information. + err = checkChecksums(client, cfg) + if err != nil { + return err } - } else { - // Since we don't support checksum for incremental data, fast checksum should be skipped. - log.Info("Skip fast checksum in incremental backup") + // When user specified not to calculate checksum, don't calculate checksum. + // Just... copy schemas from origin. + log.Info("Skip fast checksum because user requirement.") + client.CopyMetaFrom(backupSchemas) + // Anyway, let's collect file info for summary. + client.CollectFileInfo() } - // Checksum has finished - updateCh.Close() err = client.SaveBackupMeta(ctx, ddlJobs) if err != nil { @@ -224,6 +219,30 @@ func RunBackup(c context.Context, g glue.Glue, cmdName string, cfg *BackupConfig return nil } +// checkChecksums checks the checksum of the client, once failed, +// returning a error with message: "mismatched checksum". +func checkChecksums(client *backup.Client, cfg *BackupConfig) error { + checksums, err := client.CollectChecksums() + if err != nil { + return err + } + if cfg.LastBackupTS == 0 { + var matches bool + matches, err = client.ChecksumMatches(checksums) + if err != nil { + return err + } + if !matches { + log.Error("backup FastChecksum mismatch!") + return errors.New("mismatched checksum") + } + return nil + } + // Since we don't support checksum for incremental data, fast checksum should be skipped. + log.Info("Skip fast checksum in incremental backup") + return nil +} + // parseTSString port from tidb setSnapshotTS func parseTSString(ts string) (uint64, error) { if len(ts) == 0 { diff --git a/pkg/task/restore.go b/pkg/task/restore.go index 9dce5139e..8de9e7a37 100644 --- a/pkg/task/restore.go +++ b/pkg/task/restore.go @@ -148,16 +148,11 @@ func RunRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf if err != nil { return err } - // execute DDL first - // set max-index-length before execute DDLs and create tables - // we set this value to max(3072*4), otherwise we might not restore table - // when upstream and downstream both set this value greater than default(3072) - conf := config.GetGlobalConfig() - conf.MaxIndexLength = config.DefMaxOfMaxIndexLength - config.StoreGlobalConfig(conf) - log.Warn("set max-index-length to max(3072*4) to skip check index length in DDL") + // pre-set TiDB config for restore + enableTiDBConfig() + // execute DDL first err = client.ExecDDLs(ddlJobs) if err != nil { return errors.Trace(err) @@ -166,6 +161,8 @@ func RunRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf // nothing to restore, maybe only ddl changes in incremental restore if len(files) == 0 { log.Info("all files are filtered out from the backup archive, nothing to restore") + // even nothing to restore, we show a success message since there is no failure. + summary.SetSuccessStatus(true) return nil } @@ -229,10 +226,7 @@ func RunRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf } // Restore sst files in batch. - batchSize := int(cfg.Concurrency) - if batchSize > maxRestoreBatchSizeLimit { - batchSize = maxRestoreBatchSizeLimit // 256 - } + batchSize := utils.MinInt(int(cfg.Concurrency), maxRestoreBatchSizeLimit) tiflashStores, err := conn.GetAllTiKVStores(ctx, client.GetPDClient(), conn.TiFlashOnly) if err != nil { @@ -247,9 +241,7 @@ func RunRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf if len(ranges) == 0 { break } - if batchSize > len(ranges) { - batchSize = len(ranges) - } + batchSize = utils.MinInt(batchSize, len(ranges)) var rangeBatch []rtree.Range ranges, rangeBatch = ranges[batchSize:], ranges[0:batchSize:batchSize] @@ -292,14 +284,16 @@ func RunRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf updateCh.Close() // Checksum - updateCh = g.StartProgress( - ctx, "Checksum", int64(len(newTables)), !cfg.LogProgress) - err = client.ValidateChecksum( - ctx, mgr.GetTiKV().GetClient(), tables, newTables, updateCh) - if err != nil { - return err + if cfg.Checksum { + updateCh = g.StartProgress( + ctx, "Checksum", int64(len(newTables)), !cfg.LogProgress) + err = client.ValidateChecksum( + ctx, mgr.GetTiKV().GetClient(), tables, newTables, updateCh) + if err != nil { + return err + } + updateCh.Close() } - updateCh.Close() // Set task summary to success status. summary.SetSuccessStatus(true) @@ -467,3 +461,22 @@ func RunRestoreTiflashReplica(c context.Context, g glue.Glue, cmdName string, cf summary.SetSuccessStatus(true) return nil } + +func enableTiDBConfig() { + // set max-index-length before execute DDLs and create tables + // we set this value to max(3072*4), otherwise we might not restore table + // when upstream and downstream both set this value greater than default(3072) + conf := config.GetGlobalConfig() + conf.MaxIndexLength = config.DefMaxOfMaxIndexLength + log.Warn("set max-index-length to max(3072*4) to skip check index length in DDL") + + // we need set this to true, since all create table DDLs will create with tableInfo + // and we can handle alter drop pk/add pk DDLs with no impact + conf.AlterPrimaryKey = true + + // set this to true for some auto random DDL execute normally during incremental restore + conf.Experimental.AllowAutoRandom = true + conf.Experimental.AllowsExpressionIndex = true + + config.StoreGlobalConfig(conf) +} diff --git a/pkg/utils/math.go b/pkg/utils/math.go new file mode 100644 index 000000000..00c8dcc4b --- /dev/null +++ b/pkg/utils/math.go @@ -0,0 +1,12 @@ +// Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0. + +package utils + +// MinInt choice smaller integer from its two arguments. +func MinInt(x, y int) int { + if x < y { + return x + } + + return y +} diff --git a/pkg/utils/math_test.go b/pkg/utils/math_test.go new file mode 100644 index 000000000..90c8e6bed --- /dev/null +++ b/pkg/utils/math_test.go @@ -0,0 +1,17 @@ +// Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0. + +package utils + +import ( + . "github.com/pingcap/check" +) + +type testMathSuite struct{} + +var _ = Suite(&testMathSuite{}) + +func (*testMathSuite) TestMinInt(c *C) { + c.Assert(MinInt(1, 2), Equals, 1) + c.Assert(MinInt(2, 1), Equals, 1) + c.Assert(MinInt(1, 1), Equals, 1) +} diff --git a/pkg/utils/schema.go b/pkg/utils/schema.go index 5ac439e36..a135dac1c 100644 --- a/pkg/utils/schema.go +++ b/pkg/utils/schema.go @@ -33,6 +33,11 @@ type Table struct { TiFlashReplicas int } +// NoChecksum checks whether the table has a calculated checksum. +func (tbl *Table) NoChecksum() bool { + return tbl.Crc64Xor == 0 && tbl.TotalKvs == 0 && tbl.TotalBytes == 0 +} + // Database wraps the schema and tables of a database. type Database struct { Info *model.DBInfo diff --git a/tests/br_alter_pk_server/run.sh b/tests/br_alter_pk_server/run.sh deleted file mode 100755 index 6485a43be..000000000 --- a/tests/br_alter_pk_server/run.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash -# -# Copyright 2020 PingCAP, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# See the License for the specific language governing permissions and -# limitations under the License. - -set -eu - -cur=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) -source $cur/../_utils/run_services - -DB="$TEST_NAME" - -# prepare database -echo "Restart cluster with alter-primary-key = true" -start_services "$cur" - -run_sql "drop schema if exists $DB;" -run_sql "create schema $DB;" - -run_sql "create table $DB.a (a int primary key, b int unique);" -run_sql "insert into $DB.a values (42, 42);" - -# backup -run_br --pd $PD_ADDR backup db --db "$DB" -s "local://$TEST_DIR/$DB" - -# restore -run_sql "drop schema $DB;" -run_br --pd $PD_ADDR restore db --db "$DB" -s "local://$TEST_DIR/$DB" - -run_sql "drop schema $DB;" -echo "Restart service with alter-primary-key = false" -start_services diff --git a/tests/br_backup_empty/run.sh b/tests/br_backup_empty/run.sh new file mode 100644 index 000000000..a7fa1f233 --- /dev/null +++ b/tests/br_backup_empty/run.sh @@ -0,0 +1,34 @@ +#!/bin/sh +# +# Copyright 2020 PingCAP, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eu + +# backup empty. +echo "backup start..." +run_br --pd $PD_ADDR backup full -s "local://$TEST_DIR/empty" --ratelimit 5 --concurrency 4 +if [ $? -ne 0 ]; then + echo "TEST: [$TEST_NAME] failed on backup empty cluster!" + exit 1 +fi + +# restore empty. +echo "restore start..." +run_br restore full -s "local://$TEST_DIR/empty" --pd $PD_ADDR --ratelimit 1024 +if [ $? -ne 0 ]; then + echo "TEST: [$TEST_NAME] failed on restore empty cluster!" + exit 1 +fi + +echo "TEST: [$TEST_NAME] successed!" diff --git a/tests/br_alter_pk_server/config/tidb.toml b/tests/br_incompatible_tidb_config/config/tidb.toml similarity index 87% rename from tests/br_alter_pk_server/config/tidb.toml rename to tests/br_incompatible_tidb_config/config/tidb.toml index 30b7d4869..f649c2a28 100644 --- a/tests/br_alter_pk_server/config/tidb.toml +++ b/tests/br_incompatible_tidb_config/config/tidb.toml @@ -6,3 +6,6 @@ lease = "360s" alter-primary-key = true + +max-index-length = 12288 + diff --git a/tests/br_alter_pk_server/config/tikv.toml b/tests/br_incompatible_tidb_config/config/tikv.toml similarity index 100% rename from tests/br_alter_pk_server/config/tikv.toml rename to tests/br_incompatible_tidb_config/config/tikv.toml diff --git a/tests/br_incompatible_tidb_config/run.sh b/tests/br_incompatible_tidb_config/run.sh new file mode 100755 index 000000000..35ff00a5f --- /dev/null +++ b/tests/br_incompatible_tidb_config/run.sh @@ -0,0 +1,115 @@ +#!/bin/bash +# +# Copyright 2020 PingCAP, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eu + +cur=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) +source $cur/../_utils/run_services + +DB="$TEST_NAME" + +# prepare database +echo "Restart cluster with alter-primary-key = true, max-index-length=12288" +start_services "$cur" + +run_sql "drop schema if exists $DB;" +run_sql "create schema $DB;" + +# test alter pk issue https://github.com/pingcap/br/issues/215 +TABLE="t1" +INCREMENTAL_TABLE="t1inc" + +run_sql "create table $DB.$TABLE (a int primary key, b int unique);" +run_sql "insert into $DB.$TABLE values (42, 42);" + +# backup +run_br --pd $PD_ADDR backup db --db "$DB" -s "local://$TEST_DIR/$DB$TABLE" + +run_sql "create table $DB.$INCREMENTAL_TABLE (a int primary key, b int unique);" +run_sql "insert into $DB.$INCREMENTAL_TABLE values (42, 42);" + +# drop pk +run_sql "alter table $DB.$INCREMENTAL_TABLE drop primary key" +run_sql "drop table $DB.$INCREMENTAL_TABLE" +run_sql "create table $DB.$INCREMENTAL_TABLE like $DB.$TABLE" +run_sql "insert into $DB.$INCREMENTAL_TABLE values (42, 42);" + +# incremental backup +run_br --pd $PD_ADDR backup db --db "$DB" -s "local://$TEST_DIR/$DB$INCREMENTAL_TABLE" + +# restore +run_sql "drop schema $DB;" + +run_br --pd $PD_ADDR restore db --db "$DB" -s "local://$TEST_DIR/$DB$TABLE" + +run_br --pd $PD_ADDR restore db --db "$DB" -s "local://$TEST_DIR/$DB$INCREMENTAL_TABLE" + +run_sql "drop schema $DB;" +run_sql "create schema $DB;" + +# test max-index-length issue https://github.com/pingcap/br/issues/217 +TABLE="t2" +run_sql "create table $DB.$TABLE (a varchar(3072) primary key);" +run_sql "insert into $DB.$TABLE values ('42');" + +# backup +run_br --pd $PD_ADDR backup db --db "$DB" -s "local://$TEST_DIR/$DB$TABLE" + +# restore +run_sql "drop schema $DB;" +run_br --pd $PD_ADDR restore db --db "$DB" -s "local://$TEST_DIR/$DB$TABLE" + +run_sql "drop schema $DB;" + +# we need set auto_random to true and remove alter-primary-key otherwise we will get error +# invalid config allow-auto-random is unavailable when alter-primary-key is enabled + +# enable column attribute `auto_random` to be defined on the primary key column. +cat > $cur/config/tidb.toml << EOF +[experimental] +allow-auto-random = true +EOF + +echo "Restart cluster with allow-auto-random=true" +start_services "$cur" + +# test auto random issue issue https://github.com/pingcap/br/issues/228 +TABLE="t3" +INCREMENTAL_TABLE="t3inc" +run_sql "create schema $DB;" +run_sql "create table $DB.$TABLE (a int(11) NOT NULL /*T!30100 AUTO_RANDOM(5) */, PRIMARY KEY (a))" +run_sql "insert into $DB.$TABLE values ('42');" + +# Full backup +run_br --pd $PD_ADDR backup db --db "$DB" -s "local://$TEST_DIR/$DB$TABLE" + +run_sql "create table $DB.$INCREMENTAL_TABLE (a int(11) NOT NULL /*T!30100 AUTO_RANDOM(5) */, PRIMARY KEY (a))" +run_sql "insert into $DB.$INCREMENTAL_TABLE values ('42');" + +# incremental backup test for execute DDL +last_backup_ts=$(br validate decode --field="end-version" -s "local://$TEST_DIR/$DB$TABLE" | tail -n1) +run_br --pd $PD_ADDR backup db --db "$DB" -s "local://$TEST_DIR/$DB$INCREMENTAL_TABLE" --lastbackupts $last_backup_ts + +run_sql "drop schema $DB;" + +# full restore +run_br --pd $PD_ADDR restore db --db "$DB" -s "local://$TEST_DIR/$DB$TABLE" +# incremental restore +run_br --pd $PD_ADDR restore db --db "$DB" -s "local://$TEST_DIR/$DB$INCREMENTAL_TABLE" + +run_sql "drop schema $DB;" + +echo "Restart service with normal" +start_services diff --git a/tests/br_s3/run.sh b/tests/br_s3/run.sh index 422a1270d..394ddcf0e 100755 --- a/tests/br_s3/run.sh +++ b/tests/br_s3/run.sh @@ -58,7 +58,7 @@ done # backup full echo "backup start..." -run_br --pd $PD_ADDR backup full -s "s3://mybucket/$DB" --s3.endpoint="http://$S3_ENDPOINT" +run_br --pd $PD_ADDR backup full -s "s3://mybucket/$DB?endpoint=http://$S3_ENDPOINT" for i in $(seq $DB_COUNT); do run_sql "DROP DATABASE $DB${i};" diff --git a/tests/br_skip_checksum/run.sh b/tests/br_skip_checksum/run.sh new file mode 100755 index 000000000..f4b93adea --- /dev/null +++ b/tests/br_skip_checksum/run.sh @@ -0,0 +1,88 @@ +#!/bin/sh +# +# Copyright 2020 PingCAP, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eu +DB="$TEST_NAME" +TABLE="usertable" +DB_COUNT=3 + +for i in $(seq $DB_COUNT); do + run_sql "CREATE DATABASE $DB${i};" + go-ycsb load mysql -P tests/$TEST_NAME/workload -p mysql.host=$TIDB_IP -p mysql.port=$TIDB_PORT -p mysql.user=root -p mysql.db=$DB${i} +done + +for i in $(seq $DB_COUNT); do + row_count_ori[${i}]=$(run_sql "SELECT COUNT(*) FROM $DB${i}.$TABLE;" | awk '/COUNT/{print $2}') +done + +# backup full, skipping generate checksum. +echo "backup start..." +run_br --pd $PD_ADDR backup full -s "local://$TEST_DIR/$DB" --ratelimit 5 --concurrency 4 --checksum=false + +for i in $(seq $DB_COUNT); do + run_sql "DROP DATABASE $DB${i};" +done + +# restore full, skipping genreate checksum. +echo "restore start..." +run_br restore full -s "local://$TEST_DIR/$DB" --pd $PD_ADDR --ratelimit 1024 --checksum=false + +for i in $(seq $DB_COUNT); do + row_count_new[${i}]=$(run_sql "SELECT COUNT(*) FROM $DB${i}.$TABLE;" | awk '/COUNT/{print $2}') +done + +fail=false +for i in $(seq $DB_COUNT); do + if [ "${row_count_ori[i]}" != "${row_count_new[i]}" ];then + fail=true + echo "TEST: [$TEST_NAME] fail on database $DB${i}" + fi + echo "database $DB${i} [original] row count: ${row_count_ori[i]}, [after br] row count: ${row_count_new[i]}" +done + +if $fail; then + echo "TEST: [$TEST_NAME] failed on restore with skipping checksum!" + exit 1 +fi + +# Let drop it again. Try to restore without disable checksum. +for i in $(seq $DB_COUNT); do + run_sql "DROP DATABASE $DB${i};" +done +echo "restore(with checksum) start..." +run_br restore full -s "local://$TEST_DIR/$DB" --pd $PD_ADDR --ratelimit 1024 + +for i in $(seq $DB_COUNT); do + row_count_new[${i}]=$(run_sql "SELECT COUNT(*) FROM $DB${i}.$TABLE;" | awk '/COUNT/{print $2}') +done + +for i in $(seq $DB_COUNT); do + if [ "${row_count_ori[i]}" != "${row_count_new[i]}" ];then + fail=true + echo "TEST: [$TEST_NAME] fail on database $DB${i}" + fi + echo "database $DB${i} [original] row count: ${row_count_ori[i]}, [after br] row count: ${row_count_new[i]}" +done + +if $fail; then + echo "TEST: [$TEST_NAME] failed on restore without skipping checksum!" + exit 1 +else + echo "TEST $TEST_NAME passed." +fi + +for i in $(seq $DB_COUNT); do + run_sql "DROP DATABASE $DB${i};" +done diff --git a/tests/br_skip_checksum/workload b/tests/br_skip_checksum/workload new file mode 100644 index 000000000..84335df96 --- /dev/null +++ b/tests/br_skip_checksum/workload @@ -0,0 +1,12 @@ +recordcount=100 +operationcount=0 +workload=core + +readallfields=true + +readproportion=0 +updateproportion=0 +scanproportion=0 +insertproportion=0 + +requestdistribution=uniform \ No newline at end of file