Remove compact tables and checksum tables (pingcap#26)

* Remove compact tables and checksum tables 1. TiKV can only do full compaction in import mode, so it is meaningless to compact tables one by one. 2. We can do checksum in TiDB, so we don't need an extra checksum command here. * Remove timer
YuJuncen · Apr 3, 2018 · b2ec4d9 · b2ec4d9
1 parent ed5c67c
commit b2ec4d9
Show file tree

Hide file tree

Showing 3 changed files with 14 additions and 74 deletions.
diff --git a/lightning/config/config.go b/lightning/config/config.go
@@ -33,9 +33,8 @@ type Config struct {
 	PostRestore  PostRestore     `toml:"post-restore"`
 
 	// command line flags
-	DoCompact  string
-	DoChecksum string
-	file       string
+	ConfigFile string
+	DoCompact  bool
 }
 
 type Lightning struct {
@@ -69,15 +68,14 @@ func LoadConfig(args []string) (*Config, error) {
 	cfg.FlagSet = flag.NewFlagSet("lightning", flag.ContinueOnError)
 	fs := cfg.FlagSet
 
-	fs.StringVar(&cfg.file, "c", "tidb-lightning.toml", "tidb-lightning configuration file")
-	fs.StringVar(&cfg.DoCompact, "compact", "", "do manual compact for tables which in comma separated format, like foo.bar1, foo.bar2")
-	fs.StringVar(&cfg.DoChecksum, "checksum", "", "do manual checksum for tables which in comma separated format, like foo.bar1,foo.bar2")
+	fs.StringVar(&cfg.ConfigFile, "c", "tidb-lightning.toml", "tidb-lightning configuration file")
+	fs.BoolVar(&cfg.DoCompact, "compact", false, "do manual compaction on the target cluster")
 
 	if err := fs.Parse(args); err != nil {
 		return nil, errors.Trace(err)
 	}
 
-	data, err := ioutil.ReadFile(cfg.file)
+	data, err := ioutil.ReadFile(cfg.ConfigFile)
 	if err != nil {
 		return nil, errors.Trace(err)
 	}

diff --git a/lightning/lightning.go b/lightning/lightning.go
@@ -4,10 +4,8 @@ import (
 	"fmt"
 	"net/http"
 	"runtime"
-	"strings"
 	"sync"
-
-	"github.com/pingcap/tidb/tablecodec"
+	"time"
 
 	"github.com/juju/errors"
 	uuid "github.com/satori/go.uuid"
@@ -61,24 +59,14 @@ func New(cfg *config.Config) *Lightning {
 func (l *Lightning) Run() {
 	runtime.GOMAXPROCS(runtime.NumCPU())
 
-	if l.cfg.DoCompact != "" {
-		tables := strings.Split(l.cfg.DoCompact, ",")
-		err := l.doCompact(tables)
+	if l.cfg.DoCompact {
+		err := l.doCompact()
 		if err != nil {
 			log.Errorf("compact error %s", errors.ErrorStack(err))
 		}
 		return
 	}
 
-	if l.cfg.DoChecksum != "" {
-		tables := strings.Split(l.cfg.DoChecksum, ",")
-		err := l.doChecksum(tables)
-		if err != nil {
-			log.Errorf("checksum error %s", errors.ErrorStack(err))
-		}
-		return
-	}
-
 	l.wg.Add(1)
 	go func() {
 		defer l.wg.Done()
@@ -102,56 +90,19 @@ func (l *Lightning) run() {
 	return
 }
 
-func (l *Lightning) doCompact(tables []string) error {
+func (l *Lightning) doCompact() error {
 	cli, err := kv.NewKVDeliverClient(context.Background(), uuid.Nil, l.cfg.ImportServer.Addr, l.cfg.TiDB.PdAddr)
 	if err != nil {
 		return errors.Trace(err)
 	}
 	defer cli.Close()
 
-	tidbMgr, err := restore.NewTiDBManager(l.cfg.TiDB.PdAddr)
-	if err != nil {
-		return errors.Trace(err)
-	}
-	defer tidbMgr.Close()
-
-	for _, table := range tables {
-		log.Infof("begin compaction for table %s", table)
-
-		// table must contains only one dot or we don't know how to split it.
-		if strings.Count(table, ".") != 1 {
-			log.Warnf("tables %s contains not dot or more than one dot which is not allowed", table)
-			continue
-		}
-
-		split := strings.Split(table, ".")
-		tableInfo, err := tidbMgr.GetTableByName(split[0], split[1])
-		if err != nil {
-			return errors.Trace(err)
-		}
-
-		start := tablecodec.GenTablePrefix(tableInfo.ID)
-		end := tablecodec.GenTablePrefix(tableInfo.ID + 1)
-		if err := cli.Compact(start, end); err != nil {
-			return errors.Trace(err)
-		}
-		log.Infof("finished compaction for table %s", table)
-	}
-
-	log.Info("compact done")
-	return nil
-}
-
-func (l *Lightning) doChecksum(tables []string) error {
-	results, err := restore.DoChecksum(l.cfg.TiDB, tables)
-	if err != nil {
+	start := time.Now()
+	if err := cli.Compact([]byte{}, []byte{}); err != nil {
 		return errors.Trace(err)
 	}
 
-	for _, result := range results {
-		log.Infof("table %s.%s remote(from tidb) checksum %d,  total_kvs, total_bytes %d",
-			result.Schema, result.Table, result.Checksum, result.TotalKVs, result.TotalBytes)
-	}
+	fmt.Println("compact takes", time.Since(start))
 	return nil
 }
 

diff --git a/lightning/restore/restore.go b/lightning/restore/restore.go
@@ -20,7 +20,6 @@ import (
 	"github.com/pingcap/tidb-lightning/lightning/mydump"
 	verify "github.com/pingcap/tidb-lightning/lightning/verification"
 	tidbcfg "github.com/pingcap/tidb/config"
-	"github.com/pingcap/tidb/tablecodec"
 )
 
 var (
@@ -220,16 +219,8 @@ func (rc *RestoreControlloer) compact(ctx context.Context) error {
 	}
 	defer cli.Close()
 
-	for _, table := range rc.dbInfo.Tables {
-		timer := time.Now()
-		log.Infof("[%s] compact", table.Name)
-		start := tablecodec.GenTablePrefix(table.ID)
-		end := tablecodec.GenTablePrefix(table.ID + 1)
-		err = cli.Compact(start, end)
-		if err != nil {
-			return errors.Trace(err)
-		}
-		log.Infof("[%s] compact takes %v", table.Name, time.Since(timer))
+	if err := cli.Compact([]byte{}, []byte{}); err != nil {
+		return errors.Trace(err)
 	}
 	return nil
 }