diff --git a/pkg/lightning/backend/local/local.go b/pkg/lightning/backend/local/local.go index f4921da0f..eb7ab3780 100644 --- a/pkg/lightning/backend/local/local.go +++ b/pkg/lightning/backend/local/local.go @@ -82,7 +82,8 @@ const ( gRPCBackOffMaxDelay = 10 * time.Minute // See: https://github.com/tikv/tikv/blob/e030a0aae9622f3774df89c62f21b2171a72a69e/etc/config-template.toml#L360 - regionMaxKeyCount = 1_440_000 + // lower the max-key-count to avoid tikv trigger region auto split + regionMaxKeyCount = 1_280_000 defaultRegionSplitSize = 96 * units.MiB propRangeIndex = "tikv.range_index" @@ -782,7 +783,12 @@ func (local *local) WriteToTiKV( size := int64(0) totalCount := int64(0) firstLoop := true - regionMaxSize := regionSplitSize * 4 / 3 + // if region-split-size <= 96MiB, we bump the threshold a bit to avoid too many retry split + // because the range-properties is not 100% accurate + regionMaxSize := regionSplitSize + if regionSplitSize <= defaultRegionSplitSize { + regionMaxSize = regionSplitSize * 4 / 3 + } for iter.First(); iter.Valid(); iter.Next() { size += int64(len(iter.Key()) + len(iter.Value())) diff --git a/pkg/lightning/restore/table_restore.go b/pkg/lightning/restore/table_restore.go index 8664943e7..a60d34dcf 100644 --- a/pkg/lightning/restore/table_restore.go +++ b/pkg/lightning/restore/table_restore.go @@ -998,8 +998,8 @@ func estimateCompactionThreshold(cp *checkpoints.TableCheckpoint, factor int64) threshold := totalRawFileSize / 512 threshold = utils.NextPowerOfTwo(threshold) if threshold < compactionLowerThreshold { - // disable compaction if threshold is smaller than lower bound - threshold = 0 + // too may small SST files will cause inaccuracy of region range estimation, + threshold = compactionLowerThreshold } else if threshold > compactionUpperThreshold { threshold = compactionUpperThreshold }