diff --git a/cmd/raw.go b/cmd/raw.go index 5e76124b3f39c..2e8b99326efb6 100644 --- a/cmd/raw.go +++ b/cmd/raw.go @@ -68,10 +68,26 @@ func newFullBackupCommand() *cobra.Command { return err } - err = client.BackupRange([]byte(""), []byte(""), u, backupTS, rate, concurrency) + done := make(chan struct{}, 1) + + startKey := []byte("") + endKey := []byte("") + + // the count of regions need to backup + approximateRegions, err := client.GetRangeRegionCount(startKey, nil) if err != nil { return err } + + go func() { + client.PrintBackupProgress("Full Backup", int64(approximateRegions), done) + }() + + err = client.BackupRange(startKey, endKey, u, backupTS, rate, concurrency) + if err != nil { + return err + } + done <- struct{}{} return client.SaveBackupMeta(u) }, } @@ -128,10 +144,32 @@ func newTableBackupCommand() *cobra.Command { if concurrency == 0 { return errors.New("at least one thread required") } - err = client.BackupTable(db, table, u, backupTS, rate, concurrency) + ranges, err := client.GetBackupTableRanges(db, table, u, backupTS, rate, concurrency) if err != nil { return err } + done := make(chan struct{}, 1) + // the count of regions need to backup + approximateRegions := 0 + for _, r := range ranges { + regionCount, err := client.GetRangeRegionCount(r.StartKey, r.EndKey) + if err != nil { + return err + } + approximateRegions += regionCount + } + + go func() { + client.PrintBackupProgress("Table Backup", int64(approximateRegions), done) + }() + + for _, r := range ranges { + err := client.BackupRange(r.StartKey, r.EndKey, u, backupTS, rate, concurrency) + if err != nil { + return err + } + } + done <- struct{}{} return client.SaveBackupMeta(u) }, } diff --git a/go.mod b/go.mod index 91f98a49af949..c4074fa298340 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.12 require ( github.com/beorn7/perks v1.0.1 // indirect + github.com/cheggaaa/pb/v3 v3.0.1 github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect github.com/cznic/sortutil v0.0.0-20181122101858-f5f958428db8 // indirect @@ -25,9 +26,9 @@ require ( github.com/pingcap/failpoint v0.0.0-20190708053854-e7b1061e6e81 // indirect github.com/pingcap/kvproto v0.0.0-20190910074707-5a11ba1a2674 github.com/pingcap/log v0.0.0-20190715063458-479153f07ebd - github.com/pingcap/parser v0.0.0-20190912032624-978b8272c04e - github.com/pingcap/pd v0.0.0-20190806095100-82f1dd11d823 - github.com/pingcap/tidb v0.0.0-20190912055946-5c48d93368d4 + github.com/pingcap/parser v0.0.0-20190923031704-33636bc5e5d6 + github.com/pingcap/pd v1.1.0-beta.0.20190923032047-5c648dc365e0 + github.com/pingcap/tidb v0.0.0-20191010035248-d82a94fe5ea7 github.com/prometheus/client_golang v0.9.0 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 // indirect github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 // indirect diff --git a/go.sum b/go.sum index e94a16b50d9dd..372c8ebbc8460 100644 --- a/go.sum +++ b/go.sum @@ -3,6 +3,8 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/StackExchange/wmi v0.0.0-20180725035823-b12b22c5341f h1:5ZfJxyXo8KyX8DgGXC5B7ILL8y51fci/qYz2B4j8iLY= github.com/StackExchange/wmi v0.0.0-20180725035823-b12b22c5341f/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM= +github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -11,6 +13,8 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/blacktear23/go-proxyprotocol v0.0.0-20180807104634-af7a81e8dd0d h1:rQlvB2AYWme2bIB18r/SipGiMEVJYE9U0z+MGoU/LtQ= github.com/blacktear23/go-proxyprotocol v0.0.0-20180807104634-af7a81e8dd0d/go.mod h1:VKt7CNAQxpFpSDz3sXyj9hY/GbVsQCr0sB3w59nE7lU= +github.com/cheggaaa/pb/v3 v3.0.1 h1:m0BngUk2LuSRYdx4fujDKNRXNDpbNCfptPfVT2m6OJY= +github.com/cheggaaa/pb/v3 v3.0.1/go.mod h1:SqqeMF/pMOIu3xgGoxtPYhMNQP258xE4x/XRTYua+KU= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20171208011716-f6d7a1f6fbf3/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -52,6 +56,7 @@ github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385 h1:clC1lXBpe2kTj2VHdaIu9ajZQe4kcEY9j0NsnDDBZ3o= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= +github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -137,8 +142,14 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y= +github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= github.com/matttproud/golang_protobuf_extensions v1.0.0/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= @@ -181,7 +192,6 @@ github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuM github.com/pingcap/errcode v0.0.0-20180921232412-a1a7271709d9/go.mod h1:4b2X8xSqxIroj/IZ9MX/VGZhAwc11wB9wRIzHvz6SeM= github.com/pingcap/errcode v0.3.0 h1:IF6LC/4+b1KNwrMlr2rBTUrojFPMexXBcDWZSpNwxjg= github.com/pingcap/errcode v0.3.0/go.mod h1:4b2X8xSqxIroj/IZ9MX/VGZhAwc11wB9wRIzHvz6SeM= -github.com/pingcap/errors v0.10.1/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= @@ -190,24 +200,22 @@ github.com/pingcap/failpoint v0.0.0-20190708053854-e7b1061e6e81 h1:g9j4cAoWE39lh github.com/pingcap/failpoint v0.0.0-20190708053854-e7b1061e6e81/go.mod h1:DNS3Qg7bEDhU6EXNHF+XSv/PGznQaMJ5FWvctpm6pQI= github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e h1:P73/4dPCL96rGrobssy1nVy2VaVpNCuLpCbr+FEaTA8= github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw= -github.com/pingcap/kvproto v0.0.0-20190516013202-4cf58ad90b6c/go.mod h1:QMdbTAXCHzzygQzqcG9uVUgU2fKeSN1GmfMiykdSzzY= -github.com/pingcap/kvproto v0.0.0-20190904075355-9a1bd6a31da2/go.mod h1:QMdbTAXCHzzygQzqcG9uVUgU2fKeSN1GmfMiykdSzzY= +github.com/pingcap/kvproto v0.0.0-20190822090350-11ea838aedf7/go.mod h1:QMdbTAXCHzzygQzqcG9uVUgU2fKeSN1GmfMiykdSzzY= +github.com/pingcap/kvproto v0.0.0-20190910074005-0e61b6f435c1/go.mod h1:QMdbTAXCHzzygQzqcG9uVUgU2fKeSN1GmfMiykdSzzY= github.com/pingcap/kvproto v0.0.0-20190910074707-5a11ba1a2674 h1:rI76NK6QDoo4we9vCAveGr1gFbIBA3dPyTmk3wHTNjU= github.com/pingcap/kvproto v0.0.0-20190910074707-5a11ba1a2674/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= -github.com/pingcap/log v0.0.0-20190214045112-b37da76f67a7/go.mod h1:xsfkWVaFVV5B8e1K9seWfyJWFrIhbtUTAD8NV1Pq3+w= github.com/pingcap/log v0.0.0-20190715063458-479153f07ebd h1:hWDol43WY5PGhsh3+8794bFHY1bPrmu6bTalpssCrGg= github.com/pingcap/log v0.0.0-20190715063458-479153f07ebd/go.mod h1:WpHUKhNZ18v116SvGrmjkA9CBhYmuUTKL+p8JC9ANEw= -github.com/pingcap/parser v0.0.0-20190912032624-978b8272c04e h1:QeD1wC7bGElAhufSHH4JcIbs1cVdxnGWD3n3gcE5qeY= -github.com/pingcap/parser v0.0.0-20190912032624-978b8272c04e/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= -github.com/pingcap/pd v0.0.0-20190712044914-75a1f9f3062b/go.mod h1:3DlDlFT7EF64A1bmb/tulZb6wbPSagm5G4p1AlhaEDs= -github.com/pingcap/pd v0.0.0-20190806095100-82f1dd11d823 h1:mpFOR/j5MA6OobwtRJQfGcayndSYzv+HJeFdXlSDIpc= -github.com/pingcap/pd v0.0.0-20190806095100-82f1dd11d823/go.mod h1:I7TEby5BHTYIxgHszfsOJSBsk8b2Qt8QrSIgdv5n5QQ= -github.com/pingcap/tidb v0.0.0-20190912055946-5c48d93368d4 h1:7ZGadrcqJy5PP1FyJm1Va56nwL+I+/mZQuCrWr54BUQ= -github.com/pingcap/tidb v0.0.0-20190912055946-5c48d93368d4/go.mod h1:Nmo6r87GTsol+TFxvAbedf2nui/zPFz50A/Oqybw8k0= +github.com/pingcap/parser v0.0.0-20190923031704-33636bc5e5d6 h1:PyjsTUD8gJ6QGilbwiy/TTn89J84/69Pj9LixOd/fFE= +github.com/pingcap/parser v0.0.0-20190923031704-33636bc5e5d6/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= +github.com/pingcap/pd v1.1.0-beta.0.20190923032047-5c648dc365e0 h1:GIEq+wZfrl2bcJxpuSrEH4H7/nlf5YdmpS+dU9lNIt8= +github.com/pingcap/pd v1.1.0-beta.0.20190923032047-5c648dc365e0/go.mod h1:G/6rJpnYwM0LKMec2rI82/5Kg6GaZMvlfB+e6/tvYmI= +github.com/pingcap/tidb v0.0.0-20191010035248-d82a94fe5ea7 h1:VcriKhWm9w09bV7IOOMiYC/HjiHBADOCrHrzL0/X2G0= +github.com/pingcap/tidb v0.0.0-20191010035248-d82a94fe5ea7/go.mod h1:nZ0eqLl4ZcEfl8ZAHgDIJC8NtTufm2qmhfeNdxSoprg= github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible h1:MkWCxgZpJBgY2f4HtwWMMFzSBb3+JPzeJgF3VrXE/bU= github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= -github.com/pingcap/tipb v0.0.0-20190806070524-16909e03435e h1:H7meq8QPmWGImOkHTQYAWw82zwIqndJaCDPVUknOHbM= -github.com/pingcap/tipb v0.0.0-20190806070524-16909e03435e/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= +github.com/pingcap/tipb v0.0.0-20191008064422-018b2fadf414 h1:1HaTk+HEzn0rVcsAbVz9GLB9Ft3/KeSl2Sy452tDkHk= +github.com/pingcap/tipb v0.0.0-20191008064422-018b2fadf414/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -316,6 +324,7 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190909091759-094676da4a83/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7 h1:0hQKqeLdqlt5iIwVOBErRisrHJAN57yOiPRQItI20fU= golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -345,9 +354,11 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190909082730-f460065e899a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190910064555-bbd175535a8b h1:3S2h5FadpNr0zUUCVZjlKIEYF+KaX/OBplTGo89CYHI= golang.org/x/sys v0.0.0-20190910064555-bbd175535a8b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/main.go b/main.go index 290b92767a9d4..36a2fee9c65cd 100644 --- a/main.go +++ b/main.go @@ -17,7 +17,7 @@ import ( ) func main() { - conf := &log.Config{Level: "info", File: log.FileLogConfig{}} + conf := &log.Config{Level: "info", File: log.FileLogConfig{Filename:"br.log"}} lg, p, _ := log.InitLogger(conf) log.ReplaceGlobals(lg, p) diff --git a/pkg/raw/full.go b/pkg/raw/full.go index 295d5dcaf33c5..f2857c186aba5 100644 --- a/pkg/raw/full.go +++ b/pkg/raw/full.go @@ -5,9 +5,10 @@ import ( "encoding/json" "io/ioutil" "sync" + "sync/atomic" "time" - "github.com/pingcap/tidb/meta/autoid" + "github.com/cheggaaa/pb/v3" "github.com/gogo/protobuf/proto" "github.com/google/btree" @@ -18,6 +19,7 @@ import ( "github.com/pingcap/parser/model" pd "github.com/pingcap/pd/client" "github.com/pingcap/tidb/domain" + "github.com/pingcap/tidb/meta/autoid" "github.com/pingcap/tidb/session" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/store/tikv" @@ -43,6 +45,9 @@ type BackupClient struct { pdClient pd.Client backupMeta backup.BackupMeta + + // the count of regions already backup + successRegions int64 } // NewBackupClient returns a new backup client @@ -108,22 +113,22 @@ func (bc *BackupClient) SaveBackupMeta(path string) error { return ioutil.WriteFile("backupmeta", backupMetaData, 0644) } -// BackupTable backup the given table. -func (bc *BackupClient) BackupTable( +// GetBackupTableRanges gets the range of table +func (bc *BackupClient) GetBackupTableRanges( dbName, tableName string, path string, backupTS uint64, rateLimit uint64, concurrency uint32, -) error { +) ([]Range, error) { dbSession, err := session.CreateSession(bc.backer.GetTiKV()) if err != nil { - return errors.Trace(err) + return nil, errors.Trace(err) } do := domain.GetDomain(dbSession.(sessionctx.Context)) info, err := do.GetSnapshotInfoSchema(backupTS) if err != nil { - return errors.Trace(err) + return nil, errors.Trace(err) } var dbInfo *model.DBInfo @@ -131,29 +136,29 @@ func (bc *BackupClient) BackupTable( cDBName := model.NewCIStr(dbName) dbInfo, exist := info.SchemaByName(cDBName) if !exist { - return errors.Errorf("schema %s not found", dbName) + return nil, errors.Errorf("schema %s not found", dbName) } cTableName := model.NewCIStr(tableName) table, err := info.TableByName(cDBName, cTableName) if err != nil { - return errors.Trace(err) + return nil, errors.Trace(err) } tableInfo = table.Meta() idAlloc := autoid.NewAllocator(bc.backer.GetTiKV(), dbInfo.ID, false) globalAutoID, err := idAlloc.NextGlobalAutoID(tableInfo.ID) if err != nil { - return errors.Trace(err) + return nil, errors.Trace(err) } tableInfo.AutoIncID = globalAutoID dbData, err := json.Marshal(dbInfo) if err != nil { - return errors.Trace(err) + return nil, errors.Trace(err) } tableData, err := json.Marshal(tableInfo) if err != nil { - return errors.Trace(err) + return nil, errors.Trace(err) } // Save schema. @@ -173,25 +178,25 @@ func (bc *BackupClient) BackupTable( // TODO: We may need to include [t, t) in order to // backup global index. - ranges := buildTableRanges(tableInfo) - for _, r := range ranges { - start, end := r.Range() - err = bc.BackupRange(start, end, path, backupTS, rateLimit, concurrency) - if err != nil { - return err - } + tableRanges := buildTableRanges(tableInfo) + ranges := make([]Range, 0, len(tableRanges)) + for _, r := range tableRanges { + ranges = append(ranges, r.Range()) } - return nil + return ranges, nil } type tableRange struct { startID, endID int64 } -func (tr tableRange) Range() ([]byte, []byte) { +func (tr tableRange) Range() Range { startKey := tablecodec.GenTablePrefix(tr.startID) endKey := tablecodec.GenTablePrefix(tr.endID) - return []byte(startKey), []byte(endKey) + return Range{ + StartKey: []byte(startKey), + EndKey: []byte(endKey), + } } func buildTableRanges(tbl *model.TableInfo) []tableRange { @@ -305,7 +310,8 @@ func (bc *BackupClient) BackupRange( Concurrency: concurrency, } push := newPushDown(ctx, bc.backer, len(allStores)) - results, err := push.pushBackup(req, allStores...) + + results, err := push.pushBackup(req, &bc.successRegions, allStores...) if err != nil { return err } @@ -566,3 +572,41 @@ func (bc *BackupClient) handleFineGrained( } return max, nil } + +// PrintBackupProgress prints progress of backup +func (bc *BackupClient) PrintBackupProgress(barName string, count int64, done <-chan struct{}) { + tmpl := `{{string . "barName" | red}} {{ bar . "<" "-" (cycle . "↖" "↗" "↘" "↙" ) "." ">"}} {{percent .}}` + bar := pb.ProgressBarTemplate(tmpl).Start64(count) + bar.Set("barName", barName) + bar.Start() + + t := time.NewTicker(time.Second) + defer t.Stop() + + for { + select { + case <-done: + bar.SetCurrent(count) + bar.Finish() + return + case <-t.C: + regions := atomic.LoadInt64(&bc.successRegions) + if regions <= count { + bar.SetCurrent(regions) + } else { + bar.SetCurrent(count) + } + } + } + +} + +// GetRangeRegionCount get region count by scanRegions(startKey, endKey) +func (bc *BackupClient) GetRangeRegionCount(startKey, endKey []byte) (int, error) { + // TODO find an efficient way to get range regionCount + regions, _, err := bc.pdClient.ScanRegions(bc.ctx, startKey, endKey, 0) + if err != nil { + return 0, errors.Trace(err) + } + return len(regions), nil +} diff --git a/pkg/raw/push.go b/pkg/raw/push.go index 4672737ab5958..edabc165a1a73 100644 --- a/pkg/raw/push.go +++ b/pkg/raw/push.go @@ -3,7 +3,7 @@ package raw import ( "context" "sync" - + "sync/atomic" // "github.com/pingcap/kvproto/pkg/errorpb" "github.com/pingcap/br/pkg/meta" "github.com/pingcap/errors" @@ -35,6 +35,7 @@ func newPushDown(ctx context.Context, backer *meta.Backer, cap int) *pushDown { // FullBackup make a full backup of a tikv cluster. func (push *pushDown) pushBackup( req backup.BackupRequest, + successRegions *int64, stores ...*metapb.Store, ) (RangeTree, error) { // Push down backup tasks to all tikv instances. @@ -74,6 +75,7 @@ func (push *pushDown) pushBackup( } if resp.GetError() == nil { // None error means range has been backuped successfully. + atomic.AddInt64(successRegions, 1) res.putOk( resp.GetStartKey(), resp.GetEndKey(), resp.GetFiles()) } else {