syncer: improve performance (#594)

pingcap · Apr 12, 2020 · d79bfc9 · d79bfc9
1 parent f69b2ca
commit d79bfc9
Show file tree

Hide file tree

Showing 9 changed files with 686 additions and 239 deletions.
diff --git a/dm/config/task.go b/dm/config/task.go
@@ -57,6 +57,7 @@ var (
 	// SyncerConfig
 	defaultWorkerCount = 16
 	defaultBatch       = 100
+	defaultQueueSize   = 5120
 )
 
 // Meta represents binlog's meta pos
@@ -193,6 +194,8 @@ type SyncerConfig struct {
 	MetaFile    string `yaml:"meta-file" toml:"meta-file" json:"meta-file"` // meta filename, used only when load SubConfig directly
 	WorkerCount int    `yaml:"worker-count" toml:"worker-count" json:"worker-count"`
 	Batch       int    `yaml:"batch" toml:"batch" json:"batch"`
+	QueueSize   int    `yaml:"queue-size" toml:"queue-size" json:"queue-size"`
+
 	// deprecated
 	MaxRetry int `yaml:"max-retry" toml:"max-retry" json:"max-retry"`
 
@@ -208,6 +211,7 @@ func defaultSyncerConfig() SyncerConfig {
 	return SyncerConfig{
 		WorkerCount: defaultWorkerCount,
 		Batch:       defaultBatch,
+		QueueSize:   defaultQueueSize,
 	}
 }