util/ranger: fix wrong range calculation of prefix index when appending ranges to point ranges (#26066)

util/ranger/detacher.go

@@ -268,8 +268,21 @@ func (d *rangeDetacher) detachCNFCondAndBuildRangeForIndex(conditions []expressi
 	res.EqOrInCount = eqOrInCount
 	ranges, err = d.buildCNFIndexRange(tpSlice, eqOrInCount, accessConds)
 	if err != nil {
-		return res, err
+		return nil, err
+	}
+
+	// Though ranges are built from equal/in conditions, some range may not be a single point after UnionRanges in buildCNFIndexRange.
+	// In order to prepare for the following appendRanges2PointRanges, we set d.mergeConsecutive to false and call buildCNFIndexRange
+	// again to get pointRanges, in which each range must be a single point. If we use ranges rather than pointRanges when calling
+	// appendRanges2PointRanges, wrong ranges would be calculated as issue https://github.com/pingcap/tidb/issues/26029 describes.
+	mergeConsecutive := d.mergeConsecutive
+	d.mergeConsecutive = false
+	pointRanges, err := d.buildCNFIndexRange(tpSlice, eqOrInCount, accessConds)
+	if err != nil {
+		return nil, err
 	}
+	d.mergeConsecutive = mergeConsecutive
+
 	res.Ranges = ranges
 	res.AccessConds = accessConds
 	res.RemainedConds = filterConds
@@ -293,6 +306,7 @@ func (d *rangeDetacher) detachCNFCondAndBuildRangeForIndex(conditions []expressi
 			}
 			if len(pointRes.Ranges[0].LowVal) > eqOrInCount {
 				res = pointRes
+				pointRanges = pointRes.Ranges
 				eqOrInCount = len(res.Ranges[0].LowVal)
 				newConditions = newConditions[:0]
 				newConditions = append(newConditions, conditions[:offset]...)
@@ -314,7 +328,7 @@ func (d *rangeDetacher) detachCNFCondAndBuildRangeForIndex(conditions []expressi
 				return &DetachRangeResult{}, nil
 			}
 			if len(tailRes.AccessConds) > 0 {
-				res.Ranges = appendRanges2PointRanges(res.Ranges, tailRes.Ranges)
+				res.Ranges = appendRanges2PointRanges(pointRanges, tailRes.Ranges)
 				res.AccessConds = append(res.AccessConds, tailRes.AccessConds...)
 			}
 			res.RemainedConds = append(res.RemainedConds, tailRes.RemainedConds...)

util/ranger/ranger_test.go

@@ -1754,3 +1754,41 @@ func (s *testRangerSuite) TestIndexRangeForDecimal(c *C) {
 		testKit.MustQuery(tt).Check(testkit.Rows(output[i].Result...))
 	}
 }
+
+func (s *testRangerSuite) TestPrefixIndexAppendPointRanges(c *C) {
+	defer testleak.AfterTest(c)()
+	dom, store, err := newDomainStoreWithBootstrap(c)
+	defer func() {
+		dom.Close()
+		store.Close()
+	}()
+	c.Assert(err, IsNil)
+	testKit := testkit.NewTestKit(c, store)
+	testKit.MustExec("USE test")
+	testKit.MustExec("DROP TABLE IF EXISTS IDT_20755")
+	testKit.MustExec("CREATE TABLE `IDT_20755` (\n" +
+		"  `COL1` varchar(20) DEFAULT NULL,\n" +
+		"  `COL2` tinyint(16) DEFAULT NULL,\n" +
+		"  `COL3` timestamp NULL DEFAULT NULL,\n" +
+		"  KEY `u_m_col` (`COL1`(10),`COL2`,`COL3`)\n" +
+		") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin")
+	testKit.MustExec("INSERT INTO IDT_20755 VALUES(\"牾窓螎刳闌蜹瑦詬鍖湪槢壿玟瞏膍敗特森撇縆\", 73, \"2010-06-03 07:29:05\")")
+	testKit.MustExec("INSERT INTO IDT_20755 VALUES(\"xxxxxxxxxxxxxxx\", 73, \"2010-06-03 07:29:05\")")
+
+	var input []string
+	var output []struct {
+		SQL    string
+		Plan   []string
+		Result []string
+	}
+	s.testData.GetTestCases(c, &input, &output)
+	for i, tt := range input {
+		s.testData.OnRecord(func() {
+			output[i].SQL = tt
+			output[i].Plan = s.testData.ConvertRowsToStrings(testKit.MustQuery("explain format = 'brief' " + tt).Rows())
+			output[i].Result = s.testData.ConvertRowsToStrings(testKit.MustQuery(tt).Rows())
+		})
+		testKit.MustQuery("explain format = 'brief' " + tt).Check(testkit.Rows(output[i].Plan...))
+		testKit.MustQuery(tt).Check(testkit.Rows(output[i].Result...))
+	}
+}

util/ranger/testdata/ranger_suite_in.json

@@ -101,5 +101,13 @@
       "select * from t2 use index(idx) where a = 1 and b <= -1;",
       "select * from t2 use index(idx) where a = 1 and b >= -1;"
     ]
+  },
+  {
+    "name": "TestPrefixIndexAppendPointRanges",
+    "cases": [
+      "select * from IDT_20755 use index (u_m_col) where col1 in (\"牾窓螎刳闌蜹瑦詬鍖湪槢壿玟瞏膍敗特森撇縆\", \"物碃貞枕騫摨聫嚣蜻禼担堋黕詖蝒毎槒阆畒郒\", \"剮毵樍穋摻瀽鬦擀钟鷫產冖悄乮曙枱诠鑡轰砠\") and col2 in (72, 39, 73) and col3 != \"2024-10-19 08:55:32\"",
+      "select * from IDT_20755 use index (u_m_col) where col1 = \"xxxxxxxxxxxxxxx\" and col2 in (72, 73) and col3 != \"2024-10-19 08:55:32\"",
+      "select * from IDT_20755 use index (u_m_col) where col1 = \"xxxxxxxxxxxxxxx\" and col2 in (72, 73, 74) and col3 != \"2024-10-19 08:55:32\""
+    ]
   }
 ]

util/ranger/testdata/ranger_suite_out.json

@@ -648,5 +648,46 @@
         ]
       }
     ]
+  },
+  {
+    "Name": "TestPrefixIndexAppendPointRanges",
+    "Cases": [
+      {
+        "SQL": "select * from IDT_20755 use index (u_m_col) where col1 in (\"牾窓螎刳闌蜹瑦詬鍖湪槢壿玟瞏膍敗特森撇縆\", \"物碃貞枕騫摨聫嚣蜻禼担堋黕詖蝒毎槒阆畒郒\", \"剮毵樍穋摻瀽鬦擀钟鷫產冖悄乮曙枱诠鑡轰砠\") and col2 in (72, 39, 73) and col3 != \"2024-10-19 08:55:32\"",
+        "Plan": [
+          "IndexLookUp 5.99 root  ",
+          "├─IndexRangeScan(Build) 5.99 cop[tikv] table:IDT_20755, index:u_m_col(COL1, COL2, COL3) range:[\"剮毵樍穋摻瀽鬦擀钟鷫\" 39 -inf,\"剮毵樍穋摻瀽鬦擀钟鷫\" 39 2024-10-19 08:55:32), (\"剮毵樍穋摻瀽鬦擀钟鷫\" 39 2024-10-19 08:55:32,\"剮毵樍穋摻瀽鬦擀钟鷫\" 39 +inf], [\"剮毵樍穋摻瀽鬦擀钟鷫\" 72 -inf,\"剮毵樍穋摻瀽鬦擀钟鷫\" 72 2024-10-19 08:55:32), (\"剮毵樍穋摻瀽鬦擀钟鷫\" 72 2024-10-19 08:55:32,\"剮毵樍穋摻瀽鬦擀钟鷫\" 72 +inf], [\"剮毵樍穋摻瀽鬦擀钟鷫\" 73 -inf,\"剮毵樍穋摻瀽鬦擀钟鷫\" 73 2024-10-19 08:55:32), (\"剮毵樍穋摻瀽鬦擀钟鷫\" 73 2024-10-19 08:55:32,\"剮毵樍穋摻瀽鬦擀钟鷫\" 73 +inf], [\"物碃貞枕騫摨聫嚣蜻禼\" 39 -inf,\"物碃貞枕騫摨聫嚣蜻禼\" 39 2024-10-19 08:55:32), (\"物碃貞枕騫摨聫嚣蜻禼\" 39 2024-10-19 08:55:32,\"物碃貞枕騫摨聫嚣蜻禼\" 39 +inf], [\"物碃貞枕騫摨聫嚣蜻禼\" 72 -inf,\"物碃貞枕騫摨聫嚣蜻禼\" 72 2024-10-19 08:55:32), (\"物碃貞枕騫摨聫嚣蜻禼\" 72 2024-10-19 08:55:32,\"物碃貞枕騫摨聫嚣蜻禼\" 72 +inf], [\"物碃貞枕騫摨聫嚣蜻禼\" 73 -inf,\"物碃貞枕騫摨聫嚣蜻禼\" 73 2024-10-19 08:55:32), (\"物碃貞枕騫摨聫嚣蜻禼\" 73 2024-10-19 08:55:32,\"物碃貞枕騫摨聫嚣蜻禼\" 73 +inf], [\"牾窓螎刳闌蜹瑦詬鍖湪\" 39 -inf,\"牾窓螎刳闌蜹瑦詬鍖湪\" 39 2024-10-19 08:55:32), (\"牾窓螎刳闌蜹瑦詬鍖湪\" 39 2024-10-19 08:55:32,\"牾窓螎刳闌蜹瑦詬鍖湪\" 39 +inf], [\"牾窓螎刳闌蜹瑦詬鍖湪\" 72 -inf,\"牾窓螎刳闌蜹瑦詬鍖湪\" 72 2024-10-19 08:55:32), (\"牾窓螎刳闌蜹瑦詬鍖湪\" 72 2024-10-19 08:55:32,\"牾窓螎刳闌蜹瑦詬鍖湪\" 72 +inf], [\"牾窓螎刳闌蜹瑦詬鍖湪\" 73 -inf,\"牾窓螎刳闌蜹瑦詬鍖湪\" 73 2024-10-19 08:55:32), (\"牾窓螎刳闌蜹瑦詬鍖湪\" 73 2024-10-19 08:55:32,\"牾窓螎刳闌蜹瑦詬鍖湪\" 73 +inf], keep order:false, stats:pseudo",
+          "└─Selection(Probe) 5.99 cop[tikv]  in(test.idt_20755.col1, \"牾窓螎刳闌蜹瑦詬鍖湪槢壿玟瞏膍敗特森撇縆\", \"物碃貞枕騫摨聫嚣蜻禼担堋黕詖蝒毎槒阆畒郒\", \"剮毵樍穋摻瀽鬦擀钟鷫產冖悄乮曙枱诠鑡轰砠\")",
+          "  └─TableRowIDScan 5.99 cop[tikv] table:IDT_20755 keep order:false, stats:pseudo"
+        ],
+        "Result": [
+          "牾窓螎刳闌蜹瑦詬鍖湪槢壿玟瞏膍敗特森撇縆 73 2010-06-03 07:29:05"
+        ]
+      },
+      {
+        "SQL": "select * from IDT_20755 use index (u_m_col) where col1 = \"xxxxxxxxxxxxxxx\" and col2 in (72, 73) and col3 != \"2024-10-19 08:55:32\"",
+        "Plan": [
+          "IndexLookUp 1.33 root  ",
+          "├─IndexRangeScan(Build) 1.33 cop[tikv] table:IDT_20755, index:u_m_col(COL1, COL2, COL3) range:[\"xxxxxxxxxx\" 72 -inf,\"xxxxxxxxxx\" 72 2024-10-19 08:55:32), (\"xxxxxxxxxx\" 72 2024-10-19 08:55:32,\"xxxxxxxxxx\" 72 +inf], [\"xxxxxxxxxx\" 73 -inf,\"xxxxxxxxxx\" 73 2024-10-19 08:55:32), (\"xxxxxxxxxx\" 73 2024-10-19 08:55:32,\"xxxxxxxxxx\" 73 +inf], keep order:false, stats:pseudo",
+          "└─Selection(Probe) 1.33 cop[tikv]  eq(test.idt_20755.col1, \"xxxxxxxxxxxxxxx\")",
+          "  └─TableRowIDScan 1.33 cop[tikv] table:IDT_20755 keep order:false, stats:pseudo"
+        ],
+        "Result": [
+          "xxxxxxxxxxxxxxx 73 2010-06-03 07:29:05"
+        ]
+      },
+      {
+        "SQL": "select * from IDT_20755 use index (u_m_col) where col1 = \"xxxxxxxxxxxxxxx\" and col2 in (72, 73, 74) and col3 != \"2024-10-19 08:55:32\"",
+        "Plan": [
+          "IndexLookUp 2.00 root  ",
+          "├─IndexRangeScan(Build) 2.00 cop[tikv] table:IDT_20755, index:u_m_col(COL1, COL2, COL3) range:[\"xxxxxxxxxx\" 72 -inf,\"xxxxxxxxxx\" 72 2024-10-19 08:55:32), (\"xxxxxxxxxx\" 72 2024-10-19 08:55:32,\"xxxxxxxxxx\" 72 +inf], [\"xxxxxxxxxx\" 73 -inf,\"xxxxxxxxxx\" 73 2024-10-19 08:55:32), (\"xxxxxxxxxx\" 73 2024-10-19 08:55:32,\"xxxxxxxxxx\" 73 +inf], [\"xxxxxxxxxx\" 74 -inf,\"xxxxxxxxxx\" 74 2024-10-19 08:55:32), (\"xxxxxxxxxx\" 74 2024-10-19 08:55:32,\"xxxxxxxxxx\" 74 +inf], keep order:false, stats:pseudo",
+          "└─Selection(Probe) 2.00 cop[tikv]  eq(test.idt_20755.col1, \"xxxxxxxxxxxxxxx\")",
+          "  └─TableRowIDScan 2.00 cop[tikv] table:IDT_20755 keep order:false, stats:pseudo"
+        ],
+        "Result": [
+          "xxxxxxxxxxxxxxx 73 2010-06-03 07:29:05"
+        ]
+      }
+    ]
   }
 ]