diff --git a/statistics/handle/bootstrap.go b/statistics/handle/bootstrap.go
index bc5cff9a9084f..6a1930b44aa87 100644
--- a/statistics/handle/bootstrap.go
+++ b/statistics/handle/bootstrap.go
@@ -179,7 +179,7 @@ func (h *Handle) initStatsHistograms(is infoschema.InfoSchema, cache *statsCache
 }
 
 func (h *Handle) initStatsTopN4Chunk(cache *statsCache, iter *chunk.Iterator4Chunk) {
-	affectedIndexes := make(map[int64]*statistics.Index)
+	affectedIndexes := make(map[*statistics.Index]struct{})
 	for row := iter.Begin(); row != iter.End(); row = iter.Next() {
 		table, ok := cache.tables[row.GetInt64(0)]
 		if !ok {
@@ -192,12 +192,12 @@ func (h *Handle) initStatsTopN4Chunk(cache *statsCache, iter *chunk.Iterator4Chu
 		if idx.TopN == nil {
 			idx.TopN = statistics.NewTopN(32)
 		}
-		affectedIndexes[row.GetInt64(1)] = idx
+		affectedIndexes[idx] = struct{}{}
 		data := make([]byte, len(row.GetBytes(2)))
 		copy(data, row.GetBytes(2))
 		idx.TopN.AppendTopN(data, row.GetUint64(3))
 	}
-	for _, idx := range affectedIndexes {
+	for idx := range affectedIndexes {
 		idx.TopN.Sort()
 	}
 }