Skip to content

Commit

Permalink
Merge pull request pingcap#11 from 3pointer/backup_progress
Browse files Browse the repository at this point in the history
Backup progress
  • Loading branch information
3pointer authored Oct 12, 2019
2 parents 87977db + 38f2834 commit 5293cb4
Show file tree
Hide file tree
Showing 6 changed files with 138 additions and 42 deletions.
42 changes: 40 additions & 2 deletions cmd/raw.go
Original file line number Diff line number Diff line change
Expand Up @@ -68,10 +68,26 @@ func newFullBackupCommand() *cobra.Command {
return err
}

err = client.BackupRange([]byte(""), []byte(""), u, backupTS, rate, concurrency)
done := make(chan struct{}, 1)

startKey := []byte("")
endKey := []byte("")

// the count of regions need to backup
approximateRegions, err := client.GetRangeRegionCount(startKey, nil)
if err != nil {
return err
}

go func() {
client.PrintBackupProgress("Full Backup", int64(approximateRegions), done)
}()

err = client.BackupRange(startKey, endKey, u, backupTS, rate, concurrency)
if err != nil {
return err
}
done <- struct{}{}
return client.SaveBackupMeta(u)
},
}
Expand Down Expand Up @@ -128,10 +144,32 @@ func newTableBackupCommand() *cobra.Command {
if concurrency == 0 {
return errors.New("at least one thread required")
}
err = client.BackupTable(db, table, u, backupTS, rate, concurrency)
ranges, err := client.GetBackupTableRanges(db, table, u, backupTS, rate, concurrency)
if err != nil {
return err
}
done := make(chan struct{}, 1)
// the count of regions need to backup
approximateRegions := 0
for _, r := range ranges {
regionCount, err := client.GetRangeRegionCount(r.StartKey, r.EndKey)
if err != nil {
return err
}
approximateRegions += regionCount
}

go func() {
client.PrintBackupProgress("Table Backup", int64(approximateRegions), done)
}()

for _, r := range ranges {
err := client.BackupRange(r.StartKey, r.EndKey, u, backupTS, rate, concurrency)
if err != nil {
return err
}
}
done <- struct{}{}
return client.SaveBackupMeta(u)
},
}
Expand Down
7 changes: 4 additions & 3 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ go 1.12

require (
github.com/beorn7/perks v1.0.1 // indirect
github.com/cheggaaa/pb/v3 v3.0.1
github.com/coreos/go-semver v0.3.0 // indirect
github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect
github.com/cznic/sortutil v0.0.0-20181122101858-f5f958428db8 // indirect
Expand All @@ -25,9 +26,9 @@ require (
github.com/pingcap/failpoint v0.0.0-20190708053854-e7b1061e6e81 // indirect
github.com/pingcap/kvproto v0.0.0-20190910074707-5a11ba1a2674
github.com/pingcap/log v0.0.0-20190715063458-479153f07ebd
github.com/pingcap/parser v0.0.0-20190912032624-978b8272c04e
github.com/pingcap/pd v0.0.0-20190806095100-82f1dd11d823
github.com/pingcap/tidb v0.0.0-20190912055946-5c48d93368d4
github.com/pingcap/parser v0.0.0-20190923031704-33636bc5e5d6
github.com/pingcap/pd v1.1.0-beta.0.20190923032047-5c648dc365e0
github.com/pingcap/tidb v0.0.0-20191010035248-d82a94fe5ea7
github.com/prometheus/client_golang v0.9.0
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 // indirect
Expand Down
37 changes: 24 additions & 13 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/StackExchange/wmi v0.0.0-20180725035823-b12b22c5341f h1:5ZfJxyXo8KyX8DgGXC5B7ILL8y51fci/qYz2B4j8iLY=
github.com/StackExchange/wmi v0.0.0-20180725035823-b12b22c5341f/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM=
github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
Expand All @@ -11,6 +13,8 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
github.com/blacktear23/go-proxyprotocol v0.0.0-20180807104634-af7a81e8dd0d h1:rQlvB2AYWme2bIB18r/SipGiMEVJYE9U0z+MGoU/LtQ=
github.com/blacktear23/go-proxyprotocol v0.0.0-20180807104634-af7a81e8dd0d/go.mod h1:VKt7CNAQxpFpSDz3sXyj9hY/GbVsQCr0sB3w59nE7lU=
github.com/cheggaaa/pb/v3 v3.0.1 h1:m0BngUk2LuSRYdx4fujDKNRXNDpbNCfptPfVT2m6OJY=
github.com/cheggaaa/pb/v3 v3.0.1/go.mod h1:SqqeMF/pMOIu3xgGoxtPYhMNQP258xE4x/XRTYua+KU=
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
github.com/chzyer/readline v0.0.0-20171208011716-f6d7a1f6fbf3/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
Expand Down Expand Up @@ -52,6 +56,7 @@ github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385 h1:clC1lXBpe2kTj2VHdaIu9ajZQe4kcEY9j0NsnDDBZ3o=
github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM=
github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
Expand Down Expand Up @@ -137,8 +142,14 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU=
github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y=
github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o=
github.com/matttproud/golang_protobuf_extensions v1.0.0/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
Expand Down Expand Up @@ -181,7 +192,6 @@ github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuM
github.com/pingcap/errcode v0.0.0-20180921232412-a1a7271709d9/go.mod h1:4b2X8xSqxIroj/IZ9MX/VGZhAwc11wB9wRIzHvz6SeM=
github.com/pingcap/errcode v0.3.0 h1:IF6LC/4+b1KNwrMlr2rBTUrojFPMexXBcDWZSpNwxjg=
github.com/pingcap/errcode v0.3.0/go.mod h1:4b2X8xSqxIroj/IZ9MX/VGZhAwc11wB9wRIzHvz6SeM=
github.com/pingcap/errors v0.10.1/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
Expand All @@ -190,24 +200,22 @@ github.com/pingcap/failpoint v0.0.0-20190708053854-e7b1061e6e81 h1:g9j4cAoWE39lh
github.com/pingcap/failpoint v0.0.0-20190708053854-e7b1061e6e81/go.mod h1:DNS3Qg7bEDhU6EXNHF+XSv/PGznQaMJ5FWvctpm6pQI=
github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e h1:P73/4dPCL96rGrobssy1nVy2VaVpNCuLpCbr+FEaTA8=
github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw=
github.com/pingcap/kvproto v0.0.0-20190516013202-4cf58ad90b6c/go.mod h1:QMdbTAXCHzzygQzqcG9uVUgU2fKeSN1GmfMiykdSzzY=
github.com/pingcap/kvproto v0.0.0-20190904075355-9a1bd6a31da2/go.mod h1:QMdbTAXCHzzygQzqcG9uVUgU2fKeSN1GmfMiykdSzzY=
github.com/pingcap/kvproto v0.0.0-20190822090350-11ea838aedf7/go.mod h1:QMdbTAXCHzzygQzqcG9uVUgU2fKeSN1GmfMiykdSzzY=
github.com/pingcap/kvproto v0.0.0-20190910074005-0e61b6f435c1/go.mod h1:QMdbTAXCHzzygQzqcG9uVUgU2fKeSN1GmfMiykdSzzY=
github.com/pingcap/kvproto v0.0.0-20190910074707-5a11ba1a2674 h1:rI76NK6QDoo4we9vCAveGr1gFbIBA3dPyTmk3wHTNjU=
github.com/pingcap/kvproto v0.0.0-20190910074707-5a11ba1a2674/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w=
github.com/pingcap/log v0.0.0-20190214045112-b37da76f67a7/go.mod h1:xsfkWVaFVV5B8e1K9seWfyJWFrIhbtUTAD8NV1Pq3+w=
github.com/pingcap/log v0.0.0-20190715063458-479153f07ebd h1:hWDol43WY5PGhsh3+8794bFHY1bPrmu6bTalpssCrGg=
github.com/pingcap/log v0.0.0-20190715063458-479153f07ebd/go.mod h1:WpHUKhNZ18v116SvGrmjkA9CBhYmuUTKL+p8JC9ANEw=
github.com/pingcap/parser v0.0.0-20190912032624-978b8272c04e h1:QeD1wC7bGElAhufSHH4JcIbs1cVdxnGWD3n3gcE5qeY=
github.com/pingcap/parser v0.0.0-20190912032624-978b8272c04e/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA=
github.com/pingcap/pd v0.0.0-20190712044914-75a1f9f3062b/go.mod h1:3DlDlFT7EF64A1bmb/tulZb6wbPSagm5G4p1AlhaEDs=
github.com/pingcap/pd v0.0.0-20190806095100-82f1dd11d823 h1:mpFOR/j5MA6OobwtRJQfGcayndSYzv+HJeFdXlSDIpc=
github.com/pingcap/pd v0.0.0-20190806095100-82f1dd11d823/go.mod h1:I7TEby5BHTYIxgHszfsOJSBsk8b2Qt8QrSIgdv5n5QQ=
github.com/pingcap/tidb v0.0.0-20190912055946-5c48d93368d4 h1:7ZGadrcqJy5PP1FyJm1Va56nwL+I+/mZQuCrWr54BUQ=
github.com/pingcap/tidb v0.0.0-20190912055946-5c48d93368d4/go.mod h1:Nmo6r87GTsol+TFxvAbedf2nui/zPFz50A/Oqybw8k0=
github.com/pingcap/parser v0.0.0-20190923031704-33636bc5e5d6 h1:PyjsTUD8gJ6QGilbwiy/TTn89J84/69Pj9LixOd/fFE=
github.com/pingcap/parser v0.0.0-20190923031704-33636bc5e5d6/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA=
github.com/pingcap/pd v1.1.0-beta.0.20190923032047-5c648dc365e0 h1:GIEq+wZfrl2bcJxpuSrEH4H7/nlf5YdmpS+dU9lNIt8=
github.com/pingcap/pd v1.1.0-beta.0.20190923032047-5c648dc365e0/go.mod h1:G/6rJpnYwM0LKMec2rI82/5Kg6GaZMvlfB+e6/tvYmI=
github.com/pingcap/tidb v0.0.0-20191010035248-d82a94fe5ea7 h1:VcriKhWm9w09bV7IOOMiYC/HjiHBADOCrHrzL0/X2G0=
github.com/pingcap/tidb v0.0.0-20191010035248-d82a94fe5ea7/go.mod h1:nZ0eqLl4ZcEfl8ZAHgDIJC8NtTufm2qmhfeNdxSoprg=
github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible h1:MkWCxgZpJBgY2f4HtwWMMFzSBb3+JPzeJgF3VrXE/bU=
github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM=
github.com/pingcap/tipb v0.0.0-20190806070524-16909e03435e h1:H7meq8QPmWGImOkHTQYAWw82zwIqndJaCDPVUknOHbM=
github.com/pingcap/tipb v0.0.0-20190806070524-16909e03435e/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI=
github.com/pingcap/tipb v0.0.0-20191008064422-018b2fadf414 h1:1HaTk+HEzn0rVcsAbVz9GLB9Ft3/KeSl2Sy452tDkHk=
github.com/pingcap/tipb v0.0.0-20191008064422-018b2fadf414/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
Expand Down Expand Up @@ -316,6 +324,7 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190909091759-094676da4a83/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7 h1:0hQKqeLdqlt5iIwVOBErRisrHJAN57yOiPRQItI20fU=
golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
Expand Down Expand Up @@ -345,9 +354,11 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190909082730-f460065e899a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190910064555-bbd175535a8b h1:3S2h5FadpNr0zUUCVZjlKIEYF+KaX/OBplTGo89CYHI=
golang.org/x/sys v0.0.0-20190910064555-bbd175535a8b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
Expand Down
2 changes: 1 addition & 1 deletion main.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ import (
)

func main() {
conf := &log.Config{Level: "info", File: log.FileLogConfig{}}
conf := &log.Config{Level: "info", File: log.FileLogConfig{Filename:"br.log"}}
lg, p, _ := log.InitLogger(conf)
log.ReplaceGlobals(lg, p)

Expand Down
88 changes: 66 additions & 22 deletions pkg/raw/full.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,10 @@ import (
"encoding/json"
"io/ioutil"
"sync"
"sync/atomic"
"time"

"github.com/pingcap/tidb/meta/autoid"
"github.com/cheggaaa/pb/v3"

"github.com/gogo/protobuf/proto"
"github.com/google/btree"
Expand All @@ -18,6 +19,7 @@ import (
"github.com/pingcap/parser/model"
pd "github.com/pingcap/pd/client"
"github.com/pingcap/tidb/domain"
"github.com/pingcap/tidb/meta/autoid"
"github.com/pingcap/tidb/session"
"github.com/pingcap/tidb/sessionctx"
"github.com/pingcap/tidb/store/tikv"
Expand All @@ -43,6 +45,9 @@ type BackupClient struct {
pdClient pd.Client

backupMeta backup.BackupMeta

// the count of regions already backup
successRegions int64
}

// NewBackupClient returns a new backup client
Expand Down Expand Up @@ -108,52 +113,52 @@ func (bc *BackupClient) SaveBackupMeta(path string) error {
return ioutil.WriteFile("backupmeta", backupMetaData, 0644)
}

// BackupTable backup the given table.
func (bc *BackupClient) BackupTable(
// GetBackupTableRanges gets the range of table
func (bc *BackupClient) GetBackupTableRanges(
dbName, tableName string,
path string,
backupTS uint64,
rateLimit uint64,
concurrency uint32,
) error {
) ([]Range, error) {
dbSession, err := session.CreateSession(bc.backer.GetTiKV())
if err != nil {
return errors.Trace(err)
return nil, errors.Trace(err)
}
do := domain.GetDomain(dbSession.(sessionctx.Context))
info, err := do.GetSnapshotInfoSchema(backupTS)
if err != nil {
return errors.Trace(err)
return nil, errors.Trace(err)
}

var dbInfo *model.DBInfo
var tableInfo *model.TableInfo
cDBName := model.NewCIStr(dbName)
dbInfo, exist := info.SchemaByName(cDBName)
if !exist {
return errors.Errorf("schema %s not found", dbName)
return nil, errors.Errorf("schema %s not found", dbName)
}
cTableName := model.NewCIStr(tableName)
table, err := info.TableByName(cDBName, cTableName)
if err != nil {
return errors.Trace(err)
return nil, errors.Trace(err)
}
tableInfo = table.Meta()
idAlloc := autoid.NewAllocator(bc.backer.GetTiKV(), dbInfo.ID, false)
globalAutoID, err := idAlloc.NextGlobalAutoID(tableInfo.ID)
if err != nil {
return errors.Trace(err)
return nil, errors.Trace(err)
}
tableInfo.AutoIncID = globalAutoID

dbData, err := json.Marshal(dbInfo)
if err != nil {
return errors.Trace(err)
return nil, errors.Trace(err)
}

tableData, err := json.Marshal(tableInfo)
if err != nil {
return errors.Trace(err)
return nil, errors.Trace(err)
}

// Save schema.
Expand All @@ -173,25 +178,25 @@ func (bc *BackupClient) BackupTable(

// TODO: We may need to include [t<tableID>, t<tableID+1>) in order to
// backup global index.
ranges := buildTableRanges(tableInfo)
for _, r := range ranges {
start, end := r.Range()
err = bc.BackupRange(start, end, path, backupTS, rateLimit, concurrency)
if err != nil {
return err
}
tableRanges := buildTableRanges(tableInfo)
ranges := make([]Range, 0, len(tableRanges))
for _, r := range tableRanges {
ranges = append(ranges, r.Range())
}
return nil
return ranges, nil
}

type tableRange struct {
startID, endID int64
}

func (tr tableRange) Range() ([]byte, []byte) {
func (tr tableRange) Range() Range {
startKey := tablecodec.GenTablePrefix(tr.startID)
endKey := tablecodec.GenTablePrefix(tr.endID)
return []byte(startKey), []byte(endKey)
return Range{
StartKey: []byte(startKey),
EndKey: []byte(endKey),
}
}

func buildTableRanges(tbl *model.TableInfo) []tableRange {
Expand Down Expand Up @@ -305,7 +310,8 @@ func (bc *BackupClient) BackupRange(
Concurrency: concurrency,
}
push := newPushDown(ctx, bc.backer, len(allStores))
results, err := push.pushBackup(req, allStores...)

results, err := push.pushBackup(req, &bc.successRegions, allStores...)
if err != nil {
return err
}
Expand Down Expand Up @@ -566,3 +572,41 @@ func (bc *BackupClient) handleFineGrained(
}
return max, nil
}

// PrintBackupProgress prints progress of backup
func (bc *BackupClient) PrintBackupProgress(barName string, count int64, done <-chan struct{}) {
tmpl := `{{string . "barName" | red}} {{ bar . "<" "-" (cycle . "↖" "↗" "↘" "↙" ) "." ">"}} {{percent .}}`
bar := pb.ProgressBarTemplate(tmpl).Start64(count)
bar.Set("barName", barName)
bar.Start()

t := time.NewTicker(time.Second)
defer t.Stop()

for {
select {
case <-done:
bar.SetCurrent(count)
bar.Finish()
return
case <-t.C:
regions := atomic.LoadInt64(&bc.successRegions)
if regions <= count {
bar.SetCurrent(regions)
} else {
bar.SetCurrent(count)
}
}
}

}

// GetRangeRegionCount get region count by scanRegions(startKey, endKey)
func (bc *BackupClient) GetRangeRegionCount(startKey, endKey []byte) (int, error) {
// TODO find an efficient way to get range regionCount
regions, _, err := bc.pdClient.ScanRegions(bc.ctx, startKey, endKey, 0)
if err != nil {
return 0, errors.Trace(err)
}
return len(regions), nil
}
Loading

0 comments on commit 5293cb4

Please sign in to comment.