Merge branch 'master' of ssh://github.com/pingcap/br into chunk-cp

pingcap · Jun 4, 2021 · e4924eb · e4924eb
2 parents 78dbbc2 + 05beea0
commit e4924eb
Show file tree

Hide file tree

Showing 104 changed files with 2,959 additions and 605 deletions.
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
@@ -35,7 +35,7 @@ Related changes
  - Need to cherry-pick to the release branch
  - Need to update the documentation
 
-### Release Note
+### Release note
 
  -
 

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -4,10 +4,28 @@ on:
     branches:
       - master
       - 'release-[0-9].[0-9]*'
+    paths-ignore:
+      - '**.html'
+      - '**.md'
+      - 'CNAME'
+      - 'LICENSE'
+      - 'docs/**'
+      - 'tests/**'
+      - 'docker/**'
+      - '.github/workflows/**.yml'
   pull_request:
     branches:
       - master
       - 'release-[0-9].[0-9]*'
+    paths-ignore:
+      - '**.html'
+      - '**.md'
+      - 'CNAME'
+      - 'LICENSE'
+      - 'docs/**'
+      - 'tests/**'
+      - 'docker/**'
+      - '.github/workflows/**.yml'
 
 jobs:
   compile:
@@ -30,7 +48,21 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.15
+          go-version: 1.16
 
       - name: Run build
         run: make build
+
+  compile-freebsd:
+    name: Compile for FreeBSD job
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/[email protected]
+
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.16
+
+      - name: Compile for FreeBSD
+        run: GOOS=freebsd make build
diff --git a/.github/workflows/compatible_test.yml b/.github/workflows/compatible_test.yml
@@ -5,54 +5,59 @@ on:
     branches:
       - master
       - 'release-[0-9].[0-9]*'
+    paths-ignore:
+      - '**.html'
+      - '**.md'
+      - 'CNAME'
+      - 'LICENSE'
+      - 'docs/**'
+      - 'tests/**'
+      - 'docker/**'
+      - '.github/workflows/**.yml'
   pull_request:
     branches:
       - master
       - 'release-[0-9].[0-9]*'
-  issue_comment:
-    types:
-      - created
+    paths-ignore:
+      - '**.html'
+      - '**.md'
+      - 'CNAME'
+      - 'LICENSE'
+      - 'docs/**'
+      - 'tests/**'
+      - 'docker/**'
+      - '.github/workflows/**.yml'
 
 jobs:
   build:
     runs-on: ubuntu-latest
     timeout-minutes: 25
     steps:
 
-    - uses: khan/pull-request-comment-trigger@master
-      id: check
-      with:
-        trigger: '/run-compatiblility-tests'
-
     - name: Free disk space
       run: |
         sudo rm -rf /usr/local/lib/android
         sudo rm -rf /usr/share/dotnet
         docker volume prune -f
         docker image prune -f
-      if: ${{ github.event_name == 'pull_request' || steps.check.outputs.triggered == 'true' }}
 
     - uses: actions/checkout@v2
-      if: ${{ github.event_name == 'pull_request' || steps.check.outputs.triggered == 'true' }}
 
     - name: Set up Go
       uses: actions/setup-go@v2
       with:
-        go-version: 1.15
-      if: ${{ github.event_name == 'pull_request' || steps.check.outputs.triggered == 'true' }}
+        go-version: 1.16
 
     - name: Generate compatibility test backup data
       timeout-minutes: 15
       run: sh compatibility/prepare_backup.sh
-      if: ${{ github.event_name == 'pull_request' || steps.check.outputs.triggered == 'true' }}
 
     - name: Start server
       run: |
         TAG=nightly PORT_SUFFIX=1 docker-compose -f compatibility/backup_cluster.yaml rm -s -v
         TAG=nightly PORT_SUFFIX=1 docker-compose -f compatibility/backup_cluster.yaml build
         TAG=nightly PORT_SUFFIX=1 docker-compose -f compatibility/backup_cluster.yaml up --remove-orphans -d
         TAG=nightly PORT_SUFFIX=1 docker-compose -f compatibility/backup_cluster.yaml exec -T control make compatibility_test
-      if: ${{ github.event_name == 'pull_request' || steps.check.outputs.triggered == 'true' }}
 
     - name: Collect component log
       if: ${{ failure() }}

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -14,7 +14,7 @@ your contribution accepted.
 
 Developing BR requires:
 
-* [Go 1.13+](http://golang.org/doc/code.html)
+* [Go 1.16+](http://golang.org/doc/code.html)
 * An internet connection to download the dependencies
 
 Simply run `make` to build the program.

diff --git a/Makefile b/Makefile
@@ -50,6 +50,13 @@ ifeq ("$(WITH_RACE)", "1")
 	GOBUILD  = CGO_ENABLED=1 GO111MODULE=on $(GO) build -ldflags '$(LDFLAGS)'
 endif
 
+# There is no FreeBSD environment for GitHub actions. So cross-compile on Linux
+# but that doesn't work with CGO_ENABLED=1, so disable cgo. The reason to have
+# cgo enabled on regular builds is performance.
+ifeq ("$(GOOS)", "freebsd")
+	GOBUILD  = CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '$(LDFLAGS)'
+endif
+
 all: build check test
 
 prepare:

diff --git a/README.md b/README.md
@@ -34,7 +34,7 @@ $ make
 $ make test
 ```
 
-Notice BR supports building with Go version `Go >= 1.13`
+Notice BR supports building with Go version `Go >= 1.16`
 
 When BR is built successfully, you can find binary in the `bin` directory.
 
@@ -108,15 +108,15 @@ bin/br backup table --db test \
 	-s local:///tmp/backup_test/ \
 	--pd ${PD_ADDR}:2379 \
 	--log-file backup_test.log \
-					
+
 # Let's drop the table.
 mysql -uroot --host 127.0.0.1 -P4000 -E -e "USE test; DROP TABLE order_line; show tables" -u root -p
 
 # Restore from the backup.
 bin/br restore table --db test \
-	--table order_line \ 
-	-s local:///tmp/backup_test/ \ 
-	--pd ${PD_ADDR}:2379 \ 
+	--table order_line \
+	-s local:///tmp/backup_test/ \
+	--pd ${PD_ADDR}:2379 \
 	--log-file restore_test.log
 
 # How many rows do we get after restore? Expected to be 300242 rows.

diff --git a/cmd/br/cmd.go b/cmd/br/cmd.go
@@ -15,7 +15,6 @@ import (
 	"github.com/pingcap/log"
 	tidbutils "github.com/pingcap/tidb-tools/pkg/utils"
 	"github.com/pingcap/tidb/util/logutil"
-	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 
 	"github.com/pingcap/br/pkg/gluetidb"
@@ -101,6 +100,27 @@ func AddFlags(cmd *cobra.Command) {
 // Init initializes BR cli.
 func Init(cmd *cobra.Command) (err error) {
 	initOnce.Do(func() {
+		slowLogFilename, e := cmd.Flags().GetString(FlagSlowLogFile)
+		if e != nil {
+			err = e
+			return
+		}
+		tidbLogCfg := logutil.LogConfig{}
+		if len(slowLogFilename) != 0 {
+			tidbLogCfg.SlowQueryFile = slowLogFilename
+			// Just for special grpc log file,
+			// otherwise the info will be print in stdout...
+			tidbLogCfg.File.Filename = timestampLogFileName()
+		} else {
+			// Disable annoying TiDB Log.
+			// TODO: some error logs outputs randomly, we need to fix them in TiDB.
+			tidbLogCfg.Level = "fatal"
+		}
+		e = logutil.InitLogger(&tidbLogCfg)
+		if e != nil {
+			err = e
+			return
+		}
 		// Initialize the logger.
 		conf := new(log.Config)
 		conf.Level, err = cmd.Flags().GetString(FlagLogLevel)
@@ -144,28 +164,6 @@ func Init(cmd *cobra.Command) (err error) {
 			return
 		}
 		redact.InitRedact(redactLog || redactInfoLog)
-
-		slowLogFilename, e := cmd.Flags().GetString(FlagSlowLogFile)
-		if e != nil {
-			err = e
-			return
-		}
-		tidbLogCfg := logutil.LogConfig{}
-		if len(slowLogFilename) != 0 {
-			tidbLogCfg.SlowQueryFile = slowLogFilename
-		} else {
-			// Hack! Discard slow log by setting log level to PanicLevel
-			logutil.SlowQueryLogger.SetLevel(logrus.PanicLevel)
-			// Disable annoying TiDB Log.
-			// TODO: some error logs outputs randomly, we need to fix them in TiDB.
-			tidbLogCfg.Level = "fatal"
-		}
-		e = logutil.InitLogger(&tidbLogCfg)
-		if e != nil {
-			err = e
-			return
-		}
-
 		err = startPProf(cmd)
 	})
 	return errors.Trace(err)

diff --git a/cmd/br/restore.go b/cmd/br/restore.go
@@ -127,7 +127,7 @@ func newFullRestoreCommand() *cobra.Command {
 func newDBRestoreCommand() *cobra.Command {
 	command := &cobra.Command{
 		Use:   "db",
-		Short: "restore tables in a database",
+		Short: "restore tables in a database from the backup data",
 		Args:  cobra.NoArgs,
 		RunE: func(cmd *cobra.Command, _ []string) error {
 			return runRestoreCommand(cmd, "Database restore")
@@ -140,7 +140,7 @@ func newDBRestoreCommand() *cobra.Command {
 func newTableRestoreCommand() *cobra.Command {
 	command := &cobra.Command{
 		Use:   "table",
-		Short: "restore a table",
+		Short: "restore a table from the backup data",
 		Args:  cobra.NoArgs,
 		RunE: func(cmd *cobra.Command, _ []string) error {
 			return runRestoreCommand(cmd, "Table restore")

diff --git a/compatibility/get_last_tags.sh b/compatibility/get_last_tags.sh
@@ -0,0 +1,20 @@
+# update tags
+git fetch --tags
+
+getLatestTags() {
+  release_5_branch_regex="^release-5\.[0-9].*$"
+  release_4_branch_regex="^release-4\.[0-9].*$"
+  TOTAL_TAGS=$(git for-each-ref --sort=creatordate  refs/tags | awk -F '/' '{print $3}')
+  filter='alpha'
+  # latest tags
+  TAGS=$(echo $TOTAL_TAGS | tr ' ' '\n' | grep -v $filter | tail -n3)
+  if git rev-parse --abbrev-ref HEAD | egrep -q $release_5_branch_regex
+  then
+    # If we are in release-5.0 branch, try to use latest 3 version of 5.x and last 4.x version
+    TAGS=$(echo $TOTAL_TAGS | tr ' ' '\n' | fgrep "v4." | grep -v $filter | tail -n1 && echo $TOTAL_TAGS | tr ' ' '\n' | fgrep "v5." | grep -v $filter | tail -n3)
+  elif git rev-parse --abbrev-ref HEAD | egrep -q $release_4_branch_regex
+  then
+    # If we are in release-4.0 branch, try to use latest 3 version of 4.x
+    TAGS=$(echo $TOTAL_TAGS | tr ' ' '\n' | fgrep "v4." | grep -v $filter | tail -n3)
+  fi
+}
diff --git a/compatibility/prepare_backup.sh b/compatibility/prepare_backup.sh
@@ -15,27 +15,9 @@
 
 set -eux
 
-# update tags
-git fetch --tags
+. compatibility/get_last_tags.sh
 
 TAGS="v5.0.0"
-getLatestTags() {
-  release_5_branch_regex="^release-5\.[0-9].*$"
-  release_4_branch_regex="^release-4\.[0-9].*$"
-  TOTAL_TAGS=$(git for-each-ref --sort=creatordate  refs/tags | awk -F '/' '{print $3}')
-  # latest tags
-  TAGS=$(echo $TOTAL_TAGS | tr ' ' '\n' | tail -n3)
-  if git rev-parse --abbrev-ref HEAD | egrep -q $release_5_branch_regex
-  then
-    # If we are in release-5.0 branch, try to use latest 3 version of 5.x and last 4.x version
-    TAGS=$(echo $TOTAL_TAGS | tr ' ' '\n' | fgrep "v4." | tail -n1 && echo $TOTAL_TAGS | tr ' ' '\n' | fgrep "v5." | tail -n3)
-  elif git rev-parse --abbrev-ref HEAD | egrep -q $release_4_branch_regex
-  then
-    # If we are in release-4.0 branch, try to use latest 3 version of 4.x
-    TAGS=$(echo $TOTAL_TAGS | tr ' ' '\n' | fgrep "v4." | tail -n3)
-  fi
-}
-
 getLatestTags
 echo "recent version of cluster is $TAGS"
 

diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -1,5 +1,5 @@
 # For loading data to TiDB
-FROM golang:1.13.8-buster as go-ycsb-builder
+FROM golang:1.16.4-buster as go-ycsb-builder
 WORKDIR /go/src/github.com/pingcap/
 RUN git clone https://github.com/pingcap/go-ycsb.git && \
     cd go-ycsb && \
@@ -8,7 +8,7 @@ RUN git clone https://github.com/pingcap/go-ycsb.git && \
 # For operating minio S3 compatible storage
 FROM minio/mc as mc-builder
 
-FROM golang:1.13.8-buster
+FROM golang:1.16.4-buster
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
     git \

diff --git a/errors.toml b/errors.toml
@@ -31,6 +31,11 @@ error = '''
 invalid argument
 '''
 
+["BR:Common:ErrUndefinedDbOrTable"]
+error = '''
+undefined restore databases or tables
+'''
+
 ["BR:Common:ErrUnknown"]
 error = '''
 internal error
@@ -76,11 +81,6 @@ error = '''
 key not in region
 '''
 
-["BR:KV:ErrKVNotHealth"]
-error = '''
-tikv cluster not health
-'''
-
 ["BR:KV:ErrKVNotLeader"]
 error = '''
 not leader
@@ -96,9 +96,14 @@ error = '''
 rewrite rule not found
 '''
 
+["BR:KV:ErrKVStorage"]
+error = '''
+tikv storage occur I/O error
+'''
+
 ["BR:KV:ErrKVUnknown"]
 error = '''
-unknown tikv error
+unknown error occur on tikv
 '''
 
 ["BR:KV:ErrNotTiKVStorage"]