From bf166d9c5d4b3162c29f204b33749da7e4230d8f Mon Sep 17 00:00:00 2001 From: Zhou Kunqin <25057648+time-and-fate@users.noreply.github.com> Date: Fri, 5 Jan 2024 14:51:34 +0800 Subject: [PATCH] planner, statistics: support estimation for mv index access path (#49852) close pingcap/tidb#46539 --- pkg/executor/test/analyzetest/analyze_test.go | 160 +++++++-------- pkg/planner/cardinality/row_count_index.go | 34 ++-- pkg/planner/cardinality/selectivity.go | 155 +++++++++++++- pkg/planner/core/find_best_task.go | 9 +- pkg/planner/core/indexmerge_path.go | 46 +++-- pkg/statistics/cmsketch.go | 4 +- pkg/statistics/table.go | 46 ++++- .../planner/core/casetest/index/index.result | 62 +++--- .../physicalplantest/physical_plan.result | 52 ++--- .../r/planner/core/indexmerge_path.result | 190 ++++++++++-------- .../t/planner/core/indexmerge_path.test | 9 +- 11 files changed, 498 insertions(+), 269 deletions(-) diff --git a/pkg/executor/test/analyzetest/analyze_test.go b/pkg/executor/test/analyzetest/analyze_test.go index b7e4694f2cedf..1a765ec0c67f9 100644 --- a/pkg/executor/test/analyzetest/analyze_test.go +++ b/pkg/executor/test/analyzetest/analyze_test.go @@ -2930,56 +2930,56 @@ func TestAnalyzeMVIndex(t *testing.T) { tk.MustExec("set session tidb_stats_load_sync_wait = 0") tk.MustQuery("explain format = brief select * from t where 1 member of (j->'$.signed')").Check(testkit.Rows( "IndexMerge 0.03 root type: union", - "├─IndexRangeScan(Build) 0.03 cop[tikv] table:t, index:ij_signed(cast(json_extract(`j`, _utf8mb4'$.signed') as signed array)) range:[1,1], keep order:false, stats:partial[ia:allEvicted, ij_signed:allEvicted, j:unInitialized]", - "└─TableRowIDScan(Probe) 0.03 cop[tikv] table:t keep order:false, stats:partial[ia:allEvicted, ij_signed:allEvicted, j:unInitialized]", + "├─IndexRangeScan(Build) 0.03 cop[tikv] table:t, index:ij_signed(cast(json_extract(`j`, _utf8mb4'$.signed') as signed array)) range:[1,1], keep order:false, stats:partial[ia:allEvicted, ij_signed:allEvicted, ij_unsigned:allEvicted...(more: 3 allEvicted, 1 unInitialized)]", + "└─TableRowIDScan(Probe) 0.03 cop[tikv] table:t keep order:false, stats:partial[ia:allEvicted, ij_signed:allEvicted, ij_unsigned:allEvicted...(more: 3 allEvicted, 1 unInitialized)]", )) tk.MustQuery("explain format = brief select * from t where 1 member of (j->'$.unsigned')").Check(testkit.Rows( "IndexMerge 0.03 root type: union", - "├─IndexRangeScan(Build) 0.03 cop[tikv] table:t, index:ij_unsigned(cast(json_extract(`j`, _utf8mb4'$.unsigned') as unsigned array)) range:[1,1], keep order:false, stats:partial[ia:allEvicted, ij_unsigned:allEvicted, j:unInitialized]", - "└─TableRowIDScan(Probe) 0.03 cop[tikv] table:t keep order:false, stats:partial[ia:allEvicted, ij_unsigned:allEvicted, j:unInitialized]", + "├─IndexRangeScan(Build) 0.03 cop[tikv] table:t, index:ij_unsigned(cast(json_extract(`j`, _utf8mb4'$.unsigned') as unsigned array)) range:[1,1], keep order:false, stats:partial[ia:allEvicted, ij_signed:allEvicted, ij_unsigned:allEvicted...(more: 3 allEvicted, 1 unInitialized)]", + "└─TableRowIDScan(Probe) 0.03 cop[tikv] table:t keep order:false, stats:partial[ia:allEvicted, ij_signed:allEvicted, ij_unsigned:allEvicted...(more: 3 allEvicted, 1 unInitialized)]", )) tk.MustQuery("explain format = brief select * from t where 10.01 member of (j->'$.dbl')").Check(testkit.Rows( "TableReader 21.60 root data:Selection", "└─Selection 21.60 cop[tikv] json_memberof(cast(10.01, json BINARY), json_extract(test.t.j, \"$.dbl\"))", - " └─TableFullScan 27.00 cop[tikv] table:t keep order:false, stats:partial[ia:allEvicted, j:unInitialized]", + " └─TableFullScan 27.00 cop[tikv] table:t keep order:false, stats:partial[ia:allEvicted, ij_signed:allEvicted, ij_unsigned:allEvicted...(more: 2 allEvicted, 1 unInitialized)]", )) tk.MustQuery("explain format = brief select * from t where '1' member of (j->'$.bin')").Check(testkit.Rows( "IndexMerge 0.03 root type: union", - "├─IndexRangeScan(Build) 0.03 cop[tikv] table:t, index:ij_binary(cast(json_extract(`j`, _utf8mb4'$.bin') as binary(50) array)) range:[\"1\",\"1\"], keep order:false, stats:partial[ia:allEvicted, ij_binary:allEvicted, j:unInitialized]", - "└─TableRowIDScan(Probe) 0.03 cop[tikv] table:t keep order:false, stats:partial[ia:allEvicted, ij_binary:allEvicted, j:unInitialized]", + "├─IndexRangeScan(Build) 0.03 cop[tikv] table:t, index:ij_binary(cast(json_extract(`j`, _utf8mb4'$.bin') as binary(50) array)) range:[\"1\",\"1\"], keep order:false, stats:partial[ia:allEvicted, ij_signed:allEvicted, ij_unsigned:allEvicted...(more: 3 allEvicted, 1 unInitialized)]", + "└─TableRowIDScan(Probe) 0.03 cop[tikv] table:t keep order:false, stats:partial[ia:allEvicted, ij_signed:allEvicted, ij_unsigned:allEvicted...(more: 3 allEvicted, 1 unInitialized)]", )) tk.MustQuery("explain format = brief select * from t where '1' member of (j->'$.char')").Check(testkit.Rows( "IndexMerge 0.03 root type: union", - "├─IndexRangeScan(Build) 0.03 cop[tikv] table:t, index:ij_char(cast(json_extract(`j`, _utf8mb4'$.char') as char(50) array)) range:[\"1\",\"1\"], keep order:false, stats:partial[ia:allEvicted, ij_char:allEvicted, j:unInitialized]", - "└─TableRowIDScan(Probe) 0.03 cop[tikv] table:t keep order:false, stats:partial[ia:allEvicted, ij_char:allEvicted, j:unInitialized]", + "├─IndexRangeScan(Build) 0.03 cop[tikv] table:t, index:ij_char(cast(json_extract(`j`, _utf8mb4'$.char') as char(50) array)) range:[\"1\",\"1\"], keep order:false, stats:partial[ia:allEvicted, ij_signed:allEvicted, ij_unsigned:allEvicted...(more: 3 allEvicted, 1 unInitialized)]", + "└─TableRowIDScan(Probe) 0.03 cop[tikv] table:t keep order:false, stats:partial[ia:allEvicted, ij_signed:allEvicted, ij_unsigned:allEvicted...(more: 3 allEvicted, 1 unInitialized)]", )) // 3.2. emulate the background async loading require.NoError(t, h.LoadNeededHistograms()) // 3.3. now, stats on all indexes should be loaded - tk.MustQuery("explain format = brief select * from t where 1 member of (j->'$.signed')").Check(testkit.Rows( - "IndexMerge 0.03 root type: union", - "├─IndexRangeScan(Build) 0.03 cop[tikv] table:t, index:ij_signed(cast(json_extract(`j`, _utf8mb4'$.signed') as signed array)) range:[1,1], keep order:false, stats:partial[j:unInitialized]", - "└─TableRowIDScan(Probe) 0.03 cop[tikv] table:t keep order:false, stats:partial[j:unInitialized]", + tk.MustQuery("explain format = brief select /*+ use_index_merge(t, ij_signed) */ * from t where 1 member of (j->'$.signed')").Check(testkit.Rows( + "IndexMerge 27.00 root type: union", + "├─IndexRangeScan(Build) 27.00 cop[tikv] table:t, index:ij_signed(cast(json_extract(`j`, _utf8mb4'$.signed') as signed array)) range:[1,1], keep order:false, stats:partial[j:unInitialized]", + "└─TableRowIDScan(Probe) 27.00 cop[tikv] table:t keep order:false, stats:partial[j:unInitialized]", )) - tk.MustQuery("explain format = brief select * from t where 1 member of (j->'$.unsigned')").Check(testkit.Rows( - "IndexMerge 0.03 root type: union", - "├─IndexRangeScan(Build) 0.03 cop[tikv] table:t, index:ij_unsigned(cast(json_extract(`j`, _utf8mb4'$.unsigned') as unsigned array)) range:[1,1], keep order:false, stats:partial[j:unInitialized]", - "└─TableRowIDScan(Probe) 0.03 cop[tikv] table:t keep order:false, stats:partial[j:unInitialized]", + tk.MustQuery("explain format = brief select /*+ use_index_merge(t, ij_unsigned) */* from t where 1 member of (j->'$.unsigned')").Check(testkit.Rows( + "IndexMerge 18.00 root type: union", + "├─IndexRangeScan(Build) 18.00 cop[tikv] table:t, index:ij_unsigned(cast(json_extract(`j`, _utf8mb4'$.unsigned') as unsigned array)) range:[1,1], keep order:false, stats:partial[j:unInitialized]", + "└─TableRowIDScan(Probe) 18.00 cop[tikv] table:t keep order:false, stats:partial[j:unInitialized]", )) - tk.MustQuery("explain format = brief select * from t where 10.01 member of (j->'$.dbl')").Check(testkit.Rows( + tk.MustQuery("explain format = brief select /*+ use_index_merge(t, ij_double) */ * from t where 10.01 member of (j->'$.dbl')").Check(testkit.Rows( "TableReader 21.60 root data:Selection", "└─Selection 21.60 cop[tikv] json_memberof(cast(10.01, json BINARY), json_extract(test.t.j, \"$.dbl\"))", " └─TableFullScan 27.00 cop[tikv] table:t keep order:false, stats:partial[j:unInitialized]", )) - tk.MustQuery("explain format = brief select * from t where '1' member of (j->'$.bin')").Check(testkit.Rows( - "IndexMerge 0.03 root type: union", - "├─IndexRangeScan(Build) 0.03 cop[tikv] table:t, index:ij_binary(cast(json_extract(`j`, _utf8mb4'$.bin') as binary(50) array)) range:[\"1\",\"1\"], keep order:false, stats:partial[j:unInitialized]", - "└─TableRowIDScan(Probe) 0.03 cop[tikv] table:t keep order:false, stats:partial[j:unInitialized]", + tk.MustQuery("explain format = brief select /*+ use_index_merge(t, ij_binary) */ * from t where '1' member of (j->'$.bin')").Check(testkit.Rows( + "IndexMerge 14.83 root type: union", + "├─IndexRangeScan(Build) 14.83 cop[tikv] table:t, index:ij_binary(cast(json_extract(`j`, _utf8mb4'$.bin') as binary(50) array)) range:[\"1\",\"1\"], keep order:false, stats:partial[j:unInitialized]", + "└─TableRowIDScan(Probe) 14.83 cop[tikv] table:t keep order:false, stats:partial[j:unInitialized]", )) - tk.MustQuery("explain format = brief select * from t where '1' member of (j->'$.char')").Check(testkit.Rows( - "IndexMerge 0.03 root type: union", - "├─IndexRangeScan(Build) 0.03 cop[tikv] table:t, index:ij_char(cast(json_extract(`j`, _utf8mb4'$.char') as char(50) array)) range:[\"1\",\"1\"], keep order:false, stats:partial[j:unInitialized]", - "└─TableRowIDScan(Probe) 0.03 cop[tikv] table:t keep order:false, stats:partial[j:unInitialized]", + tk.MustQuery("explain format = brief select /*+ use_index_merge(t, ij_char) */ * from t where '1' member of (j->'$.char')").Check(testkit.Rows( + "IndexMerge 13.50 root type: union", + "├─IndexRangeScan(Build) 13.50 cop[tikv] table:t, index:ij_char(cast(json_extract(`j`, _utf8mb4'$.char') as char(50) array)) range:[\"1\",\"1\"], keep order:false, stats:partial[j:unInitialized]", + "└─TableRowIDScan(Probe) 13.50 cop[tikv] table:t keep order:false, stats:partial[j:unInitialized]", )) // 3.4. clean up the stats and re-analyze the table @@ -2988,24 +2988,24 @@ func TestAnalyzeMVIndex(t *testing.T) { // 3.5. turn on the sync loading, stats on mv indexes should be loaded tk.MustExec("set session tidb_stats_load_sync_wait = 1000") tk.MustQuery("explain format = brief select * from t where 1 member of (j->'$.signed')").Check(testkit.Rows( - "IndexMerge 0.03 root type: union", - "├─IndexRangeScan(Build) 0.03 cop[tikv] table:t, index:ij_signed(cast(json_extract(`j`, _utf8mb4'$.signed') as signed array)) range:[1,1], keep order:false, stats:partial[ia:allEvicted, j:unInitialized]", - "└─TableRowIDScan(Probe) 0.03 cop[tikv] table:t keep order:false, stats:partial[ia:allEvicted, j:unInitialized]", + "IndexMerge 3.84 root type: union", + "├─IndexRangeScan(Build) 3.84 cop[tikv] table:t, index:ij_signed(cast(json_extract(`j`, _utf8mb4'$.signed') as signed array)) range:[1,1], keep order:false, stats:partial[ia:allEvicted, j:unInitialized]", + "└─TableRowIDScan(Probe) 3.84 cop[tikv] table:t keep order:false, stats:partial[ia:allEvicted, j:unInitialized]", )) tk.MustQuery("explain format = brief select * from t where 1 member of (j->'$.unsigned')").Check(testkit.Rows( - "IndexMerge 0.03 root type: union", - "├─IndexRangeScan(Build) 0.03 cop[tikv] table:t, index:ij_unsigned(cast(json_extract(`j`, _utf8mb4'$.unsigned') as unsigned array)) range:[1,1], keep order:false, stats:partial[ia:allEvicted, j:unInitialized]", - "└─TableRowIDScan(Probe) 0.03 cop[tikv] table:t keep order:false, stats:partial[ia:allEvicted, j:unInitialized]", + "IndexMerge 3.60 root type: union", + "├─IndexRangeScan(Build) 3.60 cop[tikv] table:t, index:ij_unsigned(cast(json_extract(`j`, _utf8mb4'$.unsigned') as unsigned array)) range:[1,1], keep order:false, stats:partial[ia:allEvicted, j:unInitialized]", + "└─TableRowIDScan(Probe) 3.60 cop[tikv] table:t keep order:false, stats:partial[ia:allEvicted, j:unInitialized]", )) tk.MustQuery("explain format = brief select * from t where '1' member of (j->'$.bin')").Check(testkit.Rows( - "IndexMerge 0.03 root type: union", - "├─IndexRangeScan(Build) 0.03 cop[tikv] table:t, index:ij_binary(cast(json_extract(`j`, _utf8mb4'$.bin') as binary(50) array)) range:[\"1\",\"1\"], keep order:false, stats:partial[ia:allEvicted, j:unInitialized]", - "└─TableRowIDScan(Probe) 0.03 cop[tikv] table:t keep order:false, stats:partial[ia:allEvicted, j:unInitialized]", + "IndexMerge 1.55 root type: union", + "├─IndexRangeScan(Build) 1.55 cop[tikv] table:t, index:ij_binary(cast(json_extract(`j`, _utf8mb4'$.bin') as binary(50) array)) range:[\"1\",\"1\"], keep order:false, stats:partial[ia:allEvicted, j:unInitialized]", + "└─TableRowIDScan(Probe) 1.55 cop[tikv] table:t keep order:false, stats:partial[ia:allEvicted, j:unInitialized]", )) tk.MustQuery("explain format = brief select * from t where '1' member of (j->'$.char')").Check(testkit.Rows( - "IndexMerge 0.03 root type: union", - "├─IndexRangeScan(Build) 0.03 cop[tikv] table:t, index:ij_char(cast(json_extract(`j`, _utf8mb4'$.char') as char(50) array)) range:[\"1\",\"1\"], keep order:false, stats:partial[ia:allEvicted, j:unInitialized]", - "└─TableRowIDScan(Probe) 0.03 cop[tikv] table:t keep order:false, stats:partial[ia:allEvicted, j:unInitialized]", + "IndexMerge 1.93 root type: union", + "├─IndexRangeScan(Build) 1.93 cop[tikv] table:t, index:ij_char(cast(json_extract(`j`, _utf8mb4'$.char') as char(50) array)) range:[\"1\",\"1\"], keep order:false, stats:partial[ia:allEvicted, j:unInitialized]", + "└─TableRowIDScan(Probe) 1.93 cop[tikv] table:t keep order:false, stats:partial[ia:allEvicted, j:unInitialized]", )) // 4. check stats content in the memory @@ -3024,59 +3024,59 @@ func TestAnalyzeMVIndex(t *testing.T) { tk.MustQuery("show stats_topn").Check(testkit.Rows( // db_name, table_name, partition_name, column_name, is_index, value, count "test t ia 1 1 27", - "test t ij_signed 1 -40000 16", - "test t ij_signed 1 -300 1", - "test t ij_signed 1 -5 11", + "test t ij_signed 1 0 27", + "test t ij_signed 1 1 27", + "test t ij_signed 1 2 27", "test t ij_unsigned 1 0 27", "test t ij_unsigned 1 3 27", "test t ij_unsigned 1 4 27", "test t ij_double 1 -21.5 27", - "test t ij_double 1 -12.000005 8", "test t ij_double 1 0 27", - "test t ij_binary 1 0000 26", - "test t ij_binary 1 1234 19", - "test t ij_binary 1 3796 1", - "test t ij_char 1 !@#$ 24", - "test t ij_char 1 %*$%#@qwe 2", - "test t ij_char 1 %*asdf@ 1", + "test t ij_double 1 2.15 27", + "test t ij_binary 1 aaaaaa 27", + "test t ij_binary 1 bbbb 27", + "test t ij_binary 1 ccc 27", + "test t ij_char 1 aaa 27", + "test t ij_char 1 asdf 27", + "test t ij_char 1 cccccc 27", )) tk.MustQuery("show stats_buckets").Check(testkit.Rows( // db_name, table_name, partition_name, column_name, is_index, bucket_id, count, repeats, lower_bound, upper_bound, ndv - "test t ij_signed 1 0 27 27 0 0 0", - "test t ij_signed 1 1 54 27 1 1 0", - "test t ij_signed 1 2 81 27 2 2 0", - "test t ij_signed 1 3 107 26 4 4 0", - "test t ij_signed 1 4 123 16 5 5 0", - "test t ij_signed 1 5 124 1 100 100 0", - "test t ij_signed 1 6 151 27 300 300 0", - "test t ij_signed 1 7 162 11 13245 13245 0", + "test t ij_signed 1 0 16 16 -40000 -40000 0", + "test t ij_signed 1 1 17 1 -300 -300 0", + "test t ij_signed 1 2 28 11 -5 -5 0", + "test t ij_signed 1 3 54 26 4 4 0", + "test t ij_signed 1 4 70 16 5 5 0", + "test t ij_signed 1 5 71 1 100 100 0", + "test t ij_signed 1 6 98 27 300 300 0", + "test t ij_signed 1 7 109 11 13245 13245 0", "test t ij_unsigned 1 0 16 16 12 12 0", "test t ij_unsigned 1 1 43 27 600 600 0", "test t ij_unsigned 1 2 54 11 3112 3112 0", - "test t ij_double 1 0 19 19 0.000005 0.000005 0", - "test t ij_double 1 1 46 27 2.15 2.15 0", - "test t ij_double 1 2 73 27 10.555555 10.555555 0", - "test t ij_double 1 3 92 19 10.9876 10.9876 0", - "test t ij_binary 1 0 8 8 5678 5678 0", - "test t ij_binary 1 1 35 27 aaaaaa aaaaaa 0", - "test t ij_binary 1 2 59 24 asdf asdf 0", - "test t ij_binary 1 3 86 27 bbbb bbbb 0", - "test t ij_binary 1 4 113 27 ccc ccc 0", - "test t ij_binary 1 5 116 3 egfb egfb 0", - "test t ij_binary 1 6 124 8 ghjk ghjk 0", - "test t ij_binary 1 7 127 3 nfre nfre 0", - "test t ij_binary 1 8 154 27 ppp ppp 0", - "test t ij_binary 1 9 178 24 qwer qwer 0", - "test t ij_binary 1 10 186 8 yuiop yuiop 0", - "test t ij_binary 1 11 213 27 zzzz zzzz 0", - "test t ij_char 1 0 27 27 aaa aaa 0", - "test t ij_char 1 1 54 27 asdf asdf 0", - "test t ij_char 1 2 81 27 cccccc cccccc 0", - "test t ij_char 1 3 108 27 eee eee 0", - "test t ij_char 1 4 110 2 k!@cvd k!@cvd 0", - "test t ij_char 1 5 111 1 kicvd kicvd 0", - "test t ij_char 1 6 135 24 qwer qwer 0", - "test t ij_char 1 7 162 27 yuiop yuiop 0", + "test t ij_double 1 0 8 8 -12.000005 -12.000005 0", + "test t ij_double 1 1 27 19 0.000005 0.000005 0", + "test t ij_double 1 2 54 27 10.555555 10.555555 0", + "test t ij_double 1 3 73 19 10.9876 10.9876 0", + "test t ij_binary 1 0 26 26 0000 0000 0", + "test t ij_binary 1 1 45 19 1234 1234 0", + "test t ij_binary 1 2 46 1 3796 3796 0", + "test t ij_binary 1 3 54 8 5678 5678 0", + "test t ij_binary 1 4 78 24 asdf asdf 0", + "test t ij_binary 1 5 81 3 egfb egfb 0", + "test t ij_binary 1 6 89 8 ghjk ghjk 0", + "test t ij_binary 1 7 92 3 nfre nfre 0", + "test t ij_binary 1 8 119 27 ppp ppp 0", + "test t ij_binary 1 9 143 24 qwer qwer 0", + "test t ij_binary 1 10 151 8 yuiop yuiop 0", + "test t ij_binary 1 11 178 27 zzzz zzzz 0", + "test t ij_char 1 0 24 24 !@#$ !@#$ 0", + "test t ij_char 1 1 26 2 %*$%#@qwe %*$%#@qwe 0", + "test t ij_char 1 2 27 1 %*asdf@ %*asdf@ 0", + "test t ij_char 1 3 54 27 eee eee 0", + "test t ij_char 1 4 56 2 k!@cvd k!@cvd 0", + "test t ij_char 1 5 57 1 kicvd kicvd 0", + "test t ij_char 1 6 81 24 qwer qwer 0", + "test t ij_char 1 7 108 27 yuiop yuiop 0", )) } diff --git a/pkg/planner/cardinality/row_count_index.go b/pkg/planner/cardinality/row_count_index.go index 6c809df236f71..467b73f2fa81e 100644 --- a/pkg/planner/cardinality/row_count_index.go +++ b/pkg/planner/cardinality/row_count_index.go @@ -50,20 +50,16 @@ func GetRowCountByIndexRanges(sctx sessionctx.Context, coll *statistics.HistColl sc := sctx.GetSessionVars().StmtCtx idx, ok := coll.Indices[idxID] colNames := make([]string, 0, 8) - isMVIndex := false if ok { if idx.Info != nil { name = idx.Info.Name.O for _, col := range idx.Info.Columns { colNames = append(colNames, col.Name.O) } - isMVIndex = idx.Info.MVIndex } } recordUsedItemStatsStatus(sctx, idx, coll.PhysicalID, idxID) - // For the mv index case, now we have supported collecting stats and async loading stats, but sync loading and - // estimation is not well-supported, so we keep mv index using pseudo estimation for this period of time. - if !ok || idx.IsInvalid(sctx, coll.Pseudo) || isMVIndex { + if !ok || idx.IsInvalid(sctx, coll.Pseudo) { colsLen := -1 if idx != nil && idx.Info.Unique { colsLen = len(idx.Info.Columns) @@ -74,17 +70,18 @@ func GetRowCountByIndexRanges(sctx sessionctx.Context, coll *statistics.HistColl } return result, err } + realtimeCnt, modifyCount := coll.GetScaledRealtimeAndModifyCnt(idx) if sctx.GetSessionVars().StmtCtx.EnableOptimizerDebugTrace { debugtrace.RecordAnyValuesWithNames(sctx, "Histogram NotNull Count", idx.Histogram.NotNullCount(), "TopN total count", idx.TopN.TotalCount(), - "Increase Factor", idx.GetIncreaseFactor(coll.RealtimeCount), + "Increase Factor", idx.GetIncreaseFactor(realtimeCnt), ) } if idx.CMSketch != nil && idx.StatsVer == statistics.Version1 { result, err = getIndexRowCountForStatsV1(sctx, coll, idxID, indexRanges) } else { - result, err = getIndexRowCountForStatsV2(sctx, idx, coll, indexRanges, coll.RealtimeCount, coll.ModifyCount) + result, err = getIndexRowCountForStatsV2(sctx, idx, coll, indexRanges, realtimeCnt, modifyCount) } if sc.EnableOptimizerCETrace { ceTraceRange(sctx, coll.PhysicalID, colNames, indexRanges, "Index Stats", uint64(result)) @@ -118,7 +115,8 @@ func getIndexRowCountForStatsV1(sctx sessionctx.Context, coll *statistics.HistCo // on single-column index, use previous way as well, because CMSketch does not contain null // values in this case. if rangePosition == 0 || isSingleColIdxNullRange(idx, ran) { - count, err := getIndexRowCountForStatsV2(sctx, idx, nil, []*ranger.Range{ran}, coll.RealtimeCount, coll.ModifyCount) + realtimeCnt, modifyCount := coll.GetScaledRealtimeAndModifyCnt(idx) + count, err := getIndexRowCountForStatsV2(sctx, idx, nil, []*ranger.Range{ran}, realtimeCnt, modifyCount) if err != nil { return 0, errors.Trace(err) } @@ -432,13 +430,15 @@ func expBackoffEstimation(sctx sessionctx.Context, idx *statistics.Index, coll * } colID := colsIDs[i] var ( - count float64 - err error - foundStats bool + count float64 + selectivity float64 + err error + foundStats bool ) if col, ok := coll.Columns[colID]; ok && !col.IsInvalid(sctx, coll.Pseudo) { foundStats = true count, err = GetRowCountByColumnRanges(sctx, coll, colID, tmpRan) + selectivity = count / float64(coll.RealtimeCount) } if idxIDs, ok := coll.ColID2IdxIDs[colID]; ok && !foundStats && len(indexRange.LowVal) > 1 { // Note the `len(indexRange.LowVal) > 1` condition here, it means we only recursively call @@ -448,11 +448,17 @@ func expBackoffEstimation(sctx sessionctx.Context, idx *statistics.Index, coll * if idxID == idx.Histogram.ID { continue } + idxStats, ok := coll.Indices[idxID] + if !ok || idxStats.IsInvalid(sctx, coll.Pseudo) { + continue + } foundStats = true count, err = GetRowCountByIndexRanges(sctx, coll, idxID, tmpRan) if err == nil { break } + realtimeCnt, _ := coll.GetScaledRealtimeAndModifyCnt(idxStats) + selectivity = count / float64(realtimeCnt) } } if !foundStats { @@ -461,15 +467,11 @@ func expBackoffEstimation(sctx sessionctx.Context, idx *statistics.Index, coll * if err != nil { return 0, false, err } - singleColumnEstResults = append(singleColumnEstResults, count) + singleColumnEstResults = append(singleColumnEstResults, selectivity) } // Sort them. slices.Sort(singleColumnEstResults) l := len(singleColumnEstResults) - // Convert the first 4 to selectivity results. - for i := 0; i < l && i < 4; i++ { - singleColumnEstResults[i] = singleColumnEstResults[i] / float64(coll.RealtimeCount) - } failpoint.Inject("cleanEstResults", func() { singleColumnEstResults = singleColumnEstResults[:0] l = 0 diff --git a/pkg/planner/cardinality/selectivity.go b/pkg/planner/cardinality/selectivity.go index f801c1cebd7ca..782fd544260ff 100644 --- a/pkg/planner/cardinality/selectivity.go +++ b/pkg/planner/cardinality/selectivity.go @@ -23,6 +23,7 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/tidb/pkg/expression" "github.com/pingcap/tidb/pkg/parser/ast" + "github.com/pingcap/tidb/pkg/parser/model" planutil "github.com/pingcap/tidb/pkg/planner/util" "github.com/pingcap/tidb/pkg/planner/util/debugtrace" "github.com/pingcap/tidb/pkg/sessionctx" @@ -32,6 +33,7 @@ import ( "github.com/pingcap/tidb/pkg/util/codec" "github.com/pingcap/tidb/pkg/util/collate" "github.com/pingcap/tidb/pkg/util/logutil" + "github.com/pingcap/tidb/pkg/util/mathutil" "github.com/pingcap/tidb/pkg/util/ranger" "go.uber.org/zap" "golang.org/x/exp/maps" @@ -157,6 +159,21 @@ func Selectivity( slices.Sort(idxIDs) for _, id := range idxIDs { idxStats := coll.Indices[id] + idxInfo := idxStats.Info + if idxInfo.MVIndex { + totalSelectivity, mask, ok := getMaskAndSelectivityForMVIndex(ctx, coll, id, remainedExprs) + if !ok { + continue + } + nodes = append(nodes, &StatsNode{ + Tp: IndexType, + ID: id, + mask: mask, + numCols: len(idxInfo.Columns), + Selectivity: totalSelectivity, + }) + continue + } idxCols := findPrefixOfIndexByCol(ctx, extractedCols, coll.Idx2ColumnIDs[id], id2Paths[idxStats.ID]) if len(idxCols) > 0 { lengths := make([]int, 0, len(idxCols)) @@ -417,6 +434,83 @@ OUTER: return ret, nodes, nil } +// CalcTotalSelectivityForMVIdxPath calculates the total selectivity for the given partial paths of an MV index merge path. +// It corresponds with the meaning of AccessPath.CountAfterAccess, as used in buildPartialPathUp4MVIndex. +// It uses the independence assumption to estimate the selectivity. +func CalcTotalSelectivityForMVIdxPath( + coll *statistics.HistColl, + partialPaths []*planutil.AccessPath, + isIntersection bool, +) float64 { + selectivities := make([]float64, 0, len(partialPaths)) + for _, path := range partialPaths { + // For a partial path, we distinguish between two cases. + // 1. We will access a single value on the virtual column of the mv index. + // In this case, handles from a single partial path must be unique. + // The CountAfterAccess of a partial path will never be larger than the table total row count. + // For an index merge path with only one partial path, the CountAfterAccess will be exactly the same as the + // CountAfterAccess of the partial path (currently there's no index filter for partial path of mv index merge + // path). + // 2. We use the mv index as if it's a non-MV index, which means the virtual column is not involved in the access + // conditions. + // In this case, we may read repeated handles from a single partial path. + // The CountAfterAccess of a partial path might be larger than the table total row count. + // For an index merge path with only one partial path, the CountAfterAccess might be less than the CountAfterAccess + // of the partial path + // For example: + // create table t(a int, d json, index iad(a, (cast(d->'$.b' as signed array)))); + // insert into t value(1,'{"b":[1,2,3,4]}'), (2,'{"b":[3,4,5,6]}'); + // The index has 8 entries. + // Case 1: + // select * from t use index (iad) where a = 1 and 1 member of (d->'$.b'); + // IndexMerge + // ├─IndexRangeScan RowCount:1 Range:[1 1,1 1] + // └─TableRowIDScan RowCount:1 + // Case 2: + // select * from t use index (iad) where a = 1; + // IndexMerge + // ├─IndexRangeScan RowCount:4 Range:[1,1] + // └─TableRowIDScan RowCount:1 + // From the example, it should be obvious that we need different total row count to calculate the selectivity of + // the access conditions: + // Case 1: Here we should use the table total row count + // Selectivity( a = 1 and 1 member of (d->'$.b') ) = 1 / 2 + // Case 2: Here we should use the index total row count + // Selectivity( a = 1 ) = 4 / 8 + var virtualCol *expression.Column + for _, col := range coll.MVIdx2Columns[path.Index.ID] { + if col.VirtualExpr != nil { + virtualCol = col + break + } + } + cols := expression.ExtractColumnsFromExpressions(nil, path.AccessConds, func(column *expression.Column) bool { + return virtualCol != nil && column.UniqueID == virtualCol.UniqueID + }) + realtimeCount := coll.RealtimeCount + // If we can't find the virtual column from the access conditions, it's the case 2. + if len(cols) == 0 { + realtimeCount, _ = coll.GetScaledRealtimeAndModifyCnt(coll.Indices[path.Index.ID]) + } + sel := path.CountAfterAccess / float64(realtimeCount) + sel = mathutil.Clamp(sel, 0, 1) + selectivities = append(selectivities, sel) + } + var totalSelectivity float64 + if isIntersection { + totalSelectivity = 1 + for _, sel := range selectivities { + totalSelectivity *= sel + } + } else { + totalSelectivity = 0 + for _, sel := range selectivities { + totalSelectivity = (sel + totalSelectivity) - totalSelectivity*sel + } + } + return totalSelectivity +} + // StatsNode is used for calculating selectivity. type StatsNode struct { // Ranges contains all the Ranges we got. @@ -621,6 +715,36 @@ func getMaskAndRanges(ctx sessionctx.Context, exprs []expression.Expression, ran return mask, ranges, false, nil } +func getMaskAndSelectivityForMVIndex( + ctx sessionctx.Context, + coll *statistics.HistColl, + id int64, + exprs []expression.Expression, +) (float64, int64, bool) { + cols := coll.MVIdx2Columns[id] + if len(cols) == 0 { + return 1.0, 0, false + } + // You can find more examples and explanations in comments for collectFilters4MVIndex() and + // buildPartialPaths4MVIndex() in planner/core. + accessConds, _ := CollectFilters4MVIndex(ctx, exprs, cols) + paths, isIntersection, ok, err := BuildPartialPaths4MVIndex(ctx, accessConds, cols, coll.Indices[id].Info, coll) + if err != nil || !ok { + return 1.0, 0, false + } + totalSelectivity := CalcTotalSelectivityForMVIdxPath(coll, paths, isIntersection) + var mask int64 + for i := range exprs { + for _, accessCond := range accessConds { + if exprs[i].Equal(ctx, accessCond) { + mask |= 1 << uint64(i) + break + } + } + } + return totalSelectivity, mask, true +} + // GetSelectivityByFilter try to estimate selectivity of expressions by evaluate the expressions using TopN, Histogram buckets boundaries and NULL. // Currently, this method can only handle expressions involving a single column. func GetSelectivityByFilter(sctx sessionctx.Context, coll *statistics.HistColl, filters []expression.Expression) (ok bool, selectivity float64, err error) { @@ -820,10 +944,11 @@ func getEqualCondSelectivity(sctx sessionctx.Context, coll *statistics.HistColl, } val := types.NewBytesDatum(bytes) if outOfRangeOnIndex(idx, val) { + realtimeCnt, _ := coll.GetScaledRealtimeAndModifyCnt(idx) // When the value is out of range, we could not found this value in the CM Sketch, // so we use heuristic methods to estimate the selectivity. if idx.NDV > 0 && coverAll { - return outOfRangeEQSelectivity(sctx, idx.NDV, coll.RealtimeCount, int64(idx.TotalRowCount())), nil + return outOfRangeEQSelectivity(sctx, idx.NDV, realtimeCnt, int64(idx.TotalRowCount())), nil } // The equal condition only uses prefix columns of the index. colIDs := coll.Idx2ColumnIDs[idx.ID] @@ -836,7 +961,7 @@ func getEqualCondSelectivity(sctx sessionctx.Context, coll *statistics.HistColl, ndv = max(ndv, col.Histogram.NDV) } } - return outOfRangeEQSelectivity(sctx, ndv, coll.RealtimeCount, int64(idx.TotalRowCount())), nil + return outOfRangeEQSelectivity(sctx, ndv, realtimeCnt, int64(idx.TotalRowCount())), nil } minRowCount, crossValidSelectivity, err := crossValidationSelectivity(sctx, coll, idx, usedColsLen, idxPointRange) @@ -942,3 +1067,29 @@ func crossValidationSelectivity( } return minRowCount, crossValidationSelectivity, nil } + +// CollectFilters4MVIndex and BuildPartialPaths4MVIndex are for matching JSON expressions against mv index. +// This logic is shared between the estimation logic and the access path generation logic. But the two functions are +// defined in planner/core package and hard to move here. So we use this trick to avoid the import cycle. +var ( + CollectFilters4MVIndex func( + sctx sessionctx.Context, + filters []expression.Expression, + idxCols []*expression.Column, + ) ( + accessFilters, + remainingFilters []expression.Expression, + ) + BuildPartialPaths4MVIndex func( + sctx sessionctx.Context, + accessFilters []expression.Expression, + idxCols []*expression.Column, + mvIndex *model.IndexInfo, + histColl *statistics.HistColl, + ) ( + partialPaths []*planutil.AccessPath, + isIntersection bool, + ok bool, + err error, + ) +) diff --git a/pkg/planner/core/find_best_task.go b/pkg/planner/core/find_best_task.go index 8da95ef920fc8..6a8fdc6c981c2 100644 --- a/pkg/planner/core/find_best_task.go +++ b/pkg/planner/core/find_best_task.go @@ -2565,7 +2565,14 @@ func (ds *DataSource) getOriginalPhysicalIndexScan(prop *property.PhysicalProper ds.StatsInfo(), ds.tableStats, ds.statisticTable, path, prop.ExpectedCnt, isMatchProp && prop.SortItems[0].Desc) } - is.SetStats(ds.tableStats.ScaleByExpectCnt(rowCount)) + // ScaleByExpectCnt only allows to scale the row count smaller than the table total row count. + // But for MV index, it's possible that the IndexRangeScan row count is larger than the table total row count. + // Please see the Case 2 in CalcTotalSelectivityForMVIdxPath for an example. + if idx.MVIndex && rowCount > ds.tableStats.RowCount { + is.SetStats(ds.tableStats.Scale(rowCount / ds.tableStats.RowCount)) + } else { + is.SetStats(ds.tableStats.ScaleByExpectCnt(rowCount)) + } usedStats := ds.SCtx().GetSessionVars().StmtCtx.GetUsedStatsInfo(false) if usedStats != nil && usedStats[is.physicalTableID] != nil { is.usedStatsInfo = usedStats[is.physicalTableID] diff --git a/pkg/planner/core/indexmerge_path.go b/pkg/planner/core/indexmerge_path.go index 1c47d905fd8c0..a0f0798da8f9f 100644 --- a/pkg/planner/core/indexmerge_path.go +++ b/pkg/planner/core/indexmerge_path.go @@ -38,6 +38,12 @@ import ( "go.uber.org/zap" ) +func init() { + cardinality.CollectFilters4MVIndex = collectFilters4MVIndex + cardinality.BuildPartialPaths4MVIndex = buildPartialPaths4MVIndex + statistics.PrepareCols4MVIndex = prepareCols4MVIndex +} + // generateIndexMergePath generates IndexMerge AccessPaths on this DataSource. func (ds *DataSource) generateIndexMergePath() error { if ds.SCtx().GetSessionVars().StmtCtx.EnableOptimizerDebugTrace { @@ -87,6 +93,21 @@ func (ds *DataSource) generateIndexMergePath() error { return err } + // Because index merge access paths are built from ds.allConds, sometimes they can help us consider more filters than + // the ds.stats, which is calculated from ds.pushedDownConds before this point. + // So we use a simple and naive method to update ds.stats here using the largest row count from index merge paths. + // This can help to avoid some cases where the row count of operator above IndexMerge is larger than IndexMerge. + // TODO: Probably we should directly consider ds.allConds when calculating ds.stats in the future. + if len(ds.possibleAccessPaths) > regularPathCount && len(ds.allConds) > len(ds.pushedDownConds) { + var maxRowCount float64 + for i := regularPathCount; i < len(ds.possibleAccessPaths); i++ { + maxRowCount = max(maxRowCount, ds.possibleAccessPaths[i].CountAfterAccess) + } + if ds.StatsInfo().RowCount > maxRowCount { + ds.SetStats(ds.tableStats.ScaleByExpectCnt(maxRowCount)) + } + } + // If without hints, it means that `enableIndexMerge` is true if len(ds.indexMergeHints) == 0 { return nil @@ -106,15 +127,6 @@ func (ds *DataSource) generateIndexMergePath() error { } else { ds.possibleAccessPaths = ds.possibleAccessPaths[regularPathCount:] } - minRowCount := ds.possibleAccessPaths[0].CountAfterAccess - for _, path := range ds.possibleAccessPaths { - if minRowCount < path.CountAfterAccess { - minRowCount = path.CountAfterAccess - } - } - if ds.StatsInfo().RowCount > minRowCount { - ds.SetStats(ds.tableStats.ScaleByExpectCnt(minRowCount)) - } return nil } @@ -1145,23 +1157,13 @@ func (*DataSource) buildPartialPathUp4MVIndex( partialPaths []*util.AccessPath, isIntersection bool, remainingFilters []expression.Expression, - _ *statistics.HistColl, + histColl *statistics.HistColl, ) *util.AccessPath { indexMergePath := &util.AccessPath{PartialIndexPaths: partialPaths, IndexMergeAccessMVIndex: true} indexMergePath.IndexMergeIsIntersection = isIntersection indexMergePath.TableFilters = remainingFilters - - // TODO: use a naive estimation strategy here now for simplicity, make it more accurate. - minEstRows, maxEstRows := math.MaxFloat64, -1.0 - for _, p := range indexMergePath.PartialIndexPaths { - minEstRows = math.Min(minEstRows, p.CountAfterAccess) - maxEstRows = math.Max(maxEstRows, p.CountAfterAccess) - } - if indexMergePath.IndexMergeIsIntersection { - indexMergePath.CountAfterAccess = minEstRows - } else { - indexMergePath.CountAfterAccess = maxEstRows - } + indexMergePath.CountAfterAccess = float64(histColl.RealtimeCount) * + cardinality.CalcTotalSelectivityForMVIdxPath(histColl, partialPaths, isIntersection) return indexMergePath } diff --git a/pkg/statistics/cmsketch.go b/pkg/statistics/cmsketch.go index eecffae89e09d..ff30aea0c89e4 100644 --- a/pkg/statistics/cmsketch.go +++ b/pkg/statistics/cmsketch.go @@ -861,8 +861,8 @@ func SortTopnMeta(topnMetas []TopNMeta) { // TopnMetaCompare compare topnMeta func TopnMetaCompare(i, j TopNMeta) int { - c := cmp.Compare(i.Count, j.Count) - if c == 0 { + c := cmp.Compare(j.Count, i.Count) + if c != 0 { return c } return bytes.Compare(i.Encoded, j.Encoded) diff --git a/pkg/statistics/table.go b/pkg/statistics/table.go index 64f017e17a353..e556b4f234dd8 100644 --- a/pkg/statistics/table.go +++ b/pkg/statistics/table.go @@ -106,7 +106,11 @@ type HistColl struct { Idx2ColumnIDs map[int64][]int64 // ColID2IdxIDs maps the column id to a list index ids whose first column is it. It's used to calculate the selectivity in planner. ColID2IdxIDs map[int64][]int64 - PhysicalID int64 + // MVIdx2Columns maps the index id to its columns by expression.Column. + // For normal index, the column id is enough, as we already have in Idx2ColumnIDs. But currently, mv index needs more + // information to match the filter against the mv index columns, and we need this map to provide this information. + MVIdx2Columns map[int64][]*expression.Column + PhysicalID int64 // TODO: add AnalyzeCount here RealtimeCount int64 // RealtimeCount is the current table row count, maintained by applying stats delta based on AnalyzeCount. ModifyCount int64 // Total modify count in a table. @@ -432,6 +436,29 @@ func (coll *HistColl) GetAnalyzeRowCount() float64 { return -1 } +// GetScaledRealtimeAndModifyCnt scale the RealtimeCount and ModifyCount for some special indexes where the total row +// count is different from the total row count of the table. Currently, only the mv index is this case. +// Because we will use the RealtimeCount and ModifyCount during the estimation for ranges on this index (like the upper +// bound for the out-of-range estimation logic and the IncreaseFactor logic), we can't directly use the RealtimeCount and +// ModifyCount of the table. Instead, we should scale them before using. +// For example, if the table analyze row count is 1000 and realtime row count is 1500, and the mv index total count is 5000, +// when calculating the IncreaseFactor, it should be 1500/1000 = 1.5 for normal columns/indexes, and we should use the +// same 1.5 for mv index. But obviously, use 1500/5000 would be wrong, the correct calculation should be 7500/5000 = 1.5. +// So we add this function to get this 7500. +func (coll *HistColl) GetScaledRealtimeAndModifyCnt(idxStats *Index) (realtimeCnt, modifyCnt int64) { + // In theory, we can apply this scale logic on all indexes. But currently, we only apply it on the mv index to avoid + // any unexpected changes caused by factors like precision difference. + if idxStats.Info == nil || !idxStats.Info.MVIndex || !idxStats.IsFullLoad() { + return coll.RealtimeCount, coll.ModifyCount + } + analyzeRowCount := coll.GetAnalyzeRowCount() + if analyzeRowCount <= 0 { + return coll.RealtimeCount, coll.ModifyCount + } + scale := idxStats.TotalRowCount() / analyzeRowCount + return int64(float64(coll.RealtimeCount) * scale), int64(float64(coll.ModifyCount) * scale) +} + // GetStatsHealthy calculates stats healthy if the table stats is not pseudo. // If the table stats is pseudo, it returns 0, false, otherwise it returns stats healthy, true. func (t *Table) GetStatsHealthy() (int64, bool) { @@ -568,6 +595,7 @@ func (coll *HistColl) GenerateHistCollFromColumnInfo(tblInfo *model.TableInfo, c newIdxHistMap := make(map[int64]*Index) idx2Columns := make(map[int64][]int64) colID2IdxIDs := make(map[int64][]int64) + mvIdx2Columns := make(map[int64][]*expression.Column) for id, idxHist := range coll.Indices { idxInfo := idxID2idxInfo[id] if idxInfo == nil { @@ -588,6 +616,12 @@ func (coll *HistColl) GenerateHistCollFromColumnInfo(tblInfo *model.TableInfo, c colID2IdxIDs[ids[0]] = append(colID2IdxIDs[ids[0]], idxHist.ID) newIdxHistMap[idxHist.ID] = idxHist idx2Columns[idxHist.ID] = ids + if idxInfo.MVIndex { + cols, ok := PrepareCols4MVIndex(tblInfo, idxInfo, columns) + if ok { + mvIdx2Columns[id] = cols + } + } } for _, idxIDs := range colID2IdxIDs { slices.Sort(idxIDs) @@ -602,6 +636,7 @@ func (coll *HistColl) GenerateHistCollFromColumnInfo(tblInfo *model.TableInfo, c Indices: newIdxHistMap, ColID2IdxIDs: colID2IdxIDs, Idx2ColumnIDs: idx2Columns, + MVIdx2Columns: mvIdx2Columns, } return newColl } @@ -683,3 +718,12 @@ func CheckAnalyzeVerOnTable(tbl *Table, version *int) bool { // This table has no statistics yet. We can directly return true. return true } + +// PrepareCols4MVIndex helps to identify the columns of an MV index. We need this information for estimation. +// This logic is shared between the estimation logic and the access path generation logic. We'd like to put the mv index +// related functions together in the planner/core package. So we use this trick here to avoid the import cycle. +var PrepareCols4MVIndex func( + tableInfo *model.TableInfo, + mvIndex *model.IndexInfo, + tblCols []*expression.Column, +) (idxCols []*expression.Column, ok bool) diff --git a/tests/integrationtest/r/planner/core/casetest/index/index.result b/tests/integrationtest/r/planner/core/casetest/index/index.result index 189f0f94fa719..43eb6d11c0637 100644 --- a/tests/integrationtest/r/planner/core/casetest/index/index.result +++ b/tests/integrationtest/r/planner/core/casetest/index/index.result @@ -35,104 +35,104 @@ drop table if exists t2; create table t2(pk int primary key, a json, b json, c int, d int, e int, index idx(c, (cast(a as signed array))), index idx2((cast(b as signed array)), c), index idx3(c, d), index idx4(d)); explain select /*+ use_index_merge(t2, idx2, idx) */ * from t2 where (1 member of (a) and c=1) or (2 member of (b) and c=1); -- 1: OR index merge from multi complicated mv index (memberof); id estRows task access object operator info -IndexMerge_8 0.10 root type: union +IndexMerge_8 0.20 root type: union ├─IndexRangeScan_5(Build) 0.10 cop[tikv] table:t2, index:idx(c, cast(`a` as signed array)) range:[1 1,1 1], keep order:false, stats:pseudo ├─IndexRangeScan_6(Build) 0.10 cop[tikv] table:t2, index:idx2(cast(`b` as signed array), c) range:[2 1,2 1], keep order:false, stats:pseudo -└─TableRowIDScan_7(Probe) 0.10 cop[tikv] table:t2 keep order:false, stats:pseudo +└─TableRowIDScan_7(Probe) 0.20 cop[tikv] table:t2 keep order:false, stats:pseudo explain select /*+ use_index_merge(t2, idx2, idx) */ * from t2 where (1 member of (a) and c=1) or (2 member of (b) and c=1); -- 2: OR index merge from multi complicated mv index (memberof); id estRows task access object operator info -IndexMerge_8 0.10 root type: union +IndexMerge_8 0.20 root type: union ├─IndexRangeScan_5(Build) 0.10 cop[tikv] table:t2, index:idx(c, cast(`a` as signed array)) range:[1 1,1 1], keep order:false, stats:pseudo ├─IndexRangeScan_6(Build) 0.10 cop[tikv] table:t2, index:idx2(cast(`b` as signed array), c) range:[2 1,2 1], keep order:false, stats:pseudo -└─TableRowIDScan_7(Probe) 0.10 cop[tikv] table:t2 keep order:false, stats:pseudo +└─TableRowIDScan_7(Probe) 0.20 cop[tikv] table:t2 keep order:false, stats:pseudo explain select /*+ use_index_merge(t2, idx2, idx) */ * from t2 where (1 member of (a) and c=1 and d=2) or (2 member of (b) and c=3 and d=2); -- 3: OR index merge from multi complicated mv index (memberof)，while each DNF item contains redundant condition, which should be remained as table filters; id estRows task access object operator info -IndexMerge_9 0.10 root type: union +IndexMerge_9 0.20 root type: union ├─IndexRangeScan_5(Build) 0.10 cop[tikv] table:t2, index:idx(c, cast(`a` as signed array)) range:[1 1,1 1], keep order:false, stats:pseudo ├─IndexRangeScan_6(Build) 0.10 cop[tikv] table:t2, index:idx2(cast(`b` as signed array), c) range:[2 3,2 3], keep order:false, stats:pseudo -└─Selection_8(Probe) 0.10 cop[tikv] or(and(json_memberof(cast(1, json BINARY), planner__core__casetest__index__index.t2.a), and(eq(planner__core__casetest__index__index.t2.c, 1), eq(planner__core__casetest__index__index.t2.d, 2))), and(json_memberof(cast(2, json BINARY), planner__core__casetest__index__index.t2.b), and(eq(planner__core__casetest__index__index.t2.c, 3), eq(planner__core__casetest__index__index.t2.d, 2)))) - └─TableRowIDScan_7 0.10 cop[tikv] table:t2 keep order:false, stats:pseudo +└─Selection_8(Probe) 0.20 cop[tikv] or(and(json_memberof(cast(1, json BINARY), planner__core__casetest__index__index.t2.a), and(eq(planner__core__casetest__index__index.t2.c, 1), eq(planner__core__casetest__index__index.t2.d, 2))), and(json_memberof(cast(2, json BINARY), planner__core__casetest__index__index.t2.b), and(eq(planner__core__casetest__index__index.t2.c, 3), eq(planner__core__casetest__index__index.t2.d, 2)))) + └─TableRowIDScan_7 0.20 cop[tikv] table:t2 keep order:false, stats:pseudo explain select /*+ use_index_merge(t2, idx2, idx) */ * from t2 where ( json_contains(a, '[1, 2, 3]') and c=1 and d=2) or (2 member of (b) and c=3 and d=2); -- 4: OR index merge from multi complicated mv index (memberof)，make full use of DNF item's condition even if the predicate is intersection case (json_contains); id estRows task access object operator info IndexMerge_9 0.01 root type: union ├─IndexRangeScan_5(Build) 10.00 cop[tikv] table:t2, index:idx(c, cast(`a` as signed array)) range:[1,1], keep order:false, stats:pseudo ├─IndexRangeScan_6(Build) 0.10 cop[tikv] table:t2, index:idx2(cast(`b` as signed array), c) range:[2 3,2 3], keep order:false, stats:pseudo └─Selection_8(Probe) 0.01 cop[tikv] or(and(json_contains(planner__core__casetest__index__index.t2.a, cast("[1, 2, 3]", json BINARY)), and(eq(planner__core__casetest__index__index.t2.c, 1), eq(planner__core__casetest__index__index.t2.d, 2))), and(json_memberof(cast(2, json BINARY), planner__core__casetest__index__index.t2.b), and(eq(planner__core__casetest__index__index.t2.c, 3), eq(planner__core__casetest__index__index.t2.d, 2)))) - └─TableRowIDScan_7 10.00 cop[tikv] table:t2 keep order:false, stats:pseudo + └─TableRowIDScan_7 10.10 cop[tikv] table:t2 keep order:false, stats:pseudo explain select /*+ use_index_merge(t2, idx2, idx) */ * from t2 where ( json_overlaps(a, '[1, 2, 3]') and c=1 and d=2) or (2 member of (b) and c=3 and d=2); -- 5: OR index merge from multi complicated mv index (memberof)，make full use of DNF item's condition even if the predicate is intersection case (json_contains); id estRows task access object operator info -Selection_5 0.08 root or(and(json_overlaps(planner__core__casetest__index__index.t2.a, cast("[1, 2, 3]", json BINARY)), and(eq(planner__core__casetest__index__index.t2.c, 1), eq(planner__core__casetest__index__index.t2.d, 2))), and(json_memberof(cast(2, json BINARY), planner__core__casetest__index__index.t2.b), and(eq(planner__core__casetest__index__index.t2.c, 3), eq(planner__core__casetest__index__index.t2.d, 2)))) -└─IndexMerge_11 0.10 root type: union +Selection_5 0.32 root or(and(json_overlaps(planner__core__casetest__index__index.t2.a, cast("[1, 2, 3]", json BINARY)), and(eq(planner__core__casetest__index__index.t2.c, 1), eq(planner__core__casetest__index__index.t2.d, 2))), and(json_memberof(cast(2, json BINARY), planner__core__casetest__index__index.t2.b), and(eq(planner__core__casetest__index__index.t2.c, 3), eq(planner__core__casetest__index__index.t2.d, 2)))) +└─IndexMerge_11 0.40 root type: union ├─IndexRangeScan_6(Build) 0.10 cop[tikv] table:t2, index:idx(c, cast(`a` as signed array)) range:[1 1,1 1], keep order:false, stats:pseudo ├─IndexRangeScan_7(Build) 0.10 cop[tikv] table:t2, index:idx(c, cast(`a` as signed array)) range:[1 2,1 2], keep order:false, stats:pseudo ├─IndexRangeScan_8(Build) 0.10 cop[tikv] table:t2, index:idx(c, cast(`a` as signed array)) range:[1 3,1 3], keep order:false, stats:pseudo ├─IndexRangeScan_9(Build) 0.10 cop[tikv] table:t2, index:idx2(cast(`b` as signed array), c) range:[2 3,2 3], keep order:false, stats:pseudo - └─TableRowIDScan_10(Probe) 0.10 cop[tikv] table:t2 keep order:false, stats:pseudo + └─TableRowIDScan_10(Probe) 0.40 cop[tikv] table:t2 keep order:false, stats:pseudo explain select /*+ use_index_merge(t2, idx2, idx, idx4) */ * from t2 where ( json_contains(a, '[1, 2, 3]') and d=2) or (2 member of (b) and c=3 and d=2); -- 6: OR index merge from multi complicated mv index (memberof)，make full use of other DNF items even if one of the DNF items fails; id estRows task access object operator info IndexMerge_9 0.01 root type: union ├─IndexRangeScan_5(Build) 10.00 cop[tikv] table:t2, index:idx4(d) range:[2,2], keep order:false, stats:pseudo ├─IndexRangeScan_6(Build) 0.10 cop[tikv] table:t2, index:idx2(cast(`b` as signed array), c) range:[2 3,2 3], keep order:false, stats:pseudo └─Selection_8(Probe) 0.01 cop[tikv] or(and(json_contains(planner__core__casetest__index__index.t2.a, cast("[1, 2, 3]", json BINARY)), eq(planner__core__casetest__index__index.t2.d, 2)), and(json_memberof(cast(2, json BINARY), planner__core__casetest__index__index.t2.b), and(eq(planner__core__casetest__index__index.t2.c, 3), eq(planner__core__casetest__index__index.t2.d, 2)))) - └─TableRowIDScan_7 10.00 cop[tikv] table:t2 keep order:false, stats:pseudo + └─TableRowIDScan_7 10.10 cop[tikv] table:t2 keep order:false, stats:pseudo explain select /*+ use_index_merge(t2, idx2, idx) */ * from t2 where (1 member of (a) and 1 member of (b) and c=3) or (3 member of (b) and c=4); -- 7: OR index merge from multi complicated mv index (memberof)，each DNF item can be more complicated like a another embedded CNF member-of composition.; id estRows task access object operator info -IndexMerge_9 0.10 root type: union +IndexMerge_9 0.20 root type: union ├─IndexRangeScan_5(Build) 0.10 cop[tikv] table:t2, index:idx(c, cast(`a` as signed array)) range:[3 1,3 1], keep order:false, stats:pseudo ├─IndexRangeScan_6(Build) 0.10 cop[tikv] table:t2, index:idx2(cast(`b` as signed array), c) range:[3 4,3 4], keep order:false, stats:pseudo -└─Selection_8(Probe) 0.10 cop[tikv] or(and(json_memberof(cast(1, json BINARY), planner__core__casetest__index__index.t2.a), and(json_memberof(cast(1, json BINARY), planner__core__casetest__index__index.t2.b), eq(planner__core__casetest__index__index.t2.c, 3))), and(json_memberof(cast(3, json BINARY), planner__core__casetest__index__index.t2.b), eq(planner__core__casetest__index__index.t2.c, 4))) - └─TableRowIDScan_7 0.10 cop[tikv] table:t2 keep order:false, stats:pseudo +└─Selection_8(Probe) 0.20 cop[tikv] or(and(json_memberof(cast(1, json BINARY), planner__core__casetest__index__index.t2.a), and(json_memberof(cast(1, json BINARY), planner__core__casetest__index__index.t2.b), eq(planner__core__casetest__index__index.t2.c, 3))), and(json_memberof(cast(3, json BINARY), planner__core__casetest__index__index.t2.b), eq(planner__core__casetest__index__index.t2.c, 4))) + └─TableRowIDScan_7 0.20 cop[tikv] table:t2 keep order:false, stats:pseudo explain select /*+ use_index_merge(t2, idx2, idx) */ * from t2 where (1 member of (a) and 1 member of (b) and c=3) or (3 member of (b) and c=4) or e=1; -- 8: OR index merge from multi complicated mv index (memberof), each DNF item should be strict or lax used as index partial path.; id estRows task access object operator info -TableReader_7 25.98 root data:Selection_6 -└─Selection_6 25.98 cop[tikv] or(and(json_memberof(cast(1, json BINARY), planner__core__casetest__index__index.t2.a), and(json_memberof(cast(1, json BINARY), planner__core__casetest__index__index.t2.b), eq(planner__core__casetest__index__index.t2.c, 3))), or(and(json_memberof(cast(3, json BINARY), planner__core__casetest__index__index.t2.b), eq(planner__core__casetest__index__index.t2.c, 4)), eq(planner__core__casetest__index__index.t2.e, 1))) +TableReader_7 10.18 root data:Selection_6 +└─Selection_6 10.18 cop[tikv] or(and(json_memberof(cast(1, json BINARY), planner__core__casetest__index__index.t2.a), and(json_memberof(cast(1, json BINARY), planner__core__casetest__index__index.t2.b), eq(planner__core__casetest__index__index.t2.c, 3))), or(and(json_memberof(cast(3, json BINARY), planner__core__casetest__index__index.t2.b), eq(planner__core__casetest__index__index.t2.c, 4)), eq(planner__core__casetest__index__index.t2.e, 1))) └─TableFullScan_5 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo explain select /*+ use_index_merge(t2, idx2, idx, idx4) */ * from t2 where (1 member of (a) and 1 member of (b) and c=3) or (3 member of (b) and c=4) or d=1; -- 9: OR index merge from multi complicated mv index (memberof), each DNF item should be strict or lax used as index partial path, specify the index in index merge hint; id estRows task access object operator info -IndexMerge_10 0.03 root type: union +IndexMerge_10 0.01 root type: union ├─IndexRangeScan_5(Build) 10.00 cop[tikv] table:t2, index:idx4(d) range:[1,1], keep order:false, stats:pseudo ├─IndexRangeScan_6(Build) 0.10 cop[tikv] table:t2, index:idx(c, cast(`a` as signed array)) range:[3 1,3 1], keep order:false, stats:pseudo ├─IndexRangeScan_7(Build) 0.10 cop[tikv] table:t2, index:idx2(cast(`b` as signed array), c) range:[3 4,3 4], keep order:false, stats:pseudo -└─Selection_9(Probe) 0.03 cop[tikv] or(and(json_memberof(cast(1, json BINARY), planner__core__casetest__index__index.t2.a), and(json_memberof(cast(1, json BINARY), planner__core__casetest__index__index.t2.b), eq(planner__core__casetest__index__index.t2.c, 3))), or(and(json_memberof(cast(3, json BINARY), planner__core__casetest__index__index.t2.b), eq(planner__core__casetest__index__index.t2.c, 4)), eq(planner__core__casetest__index__index.t2.d, 1))) - └─TableRowIDScan_8 10.00 cop[tikv] table:t2 keep order:false, stats:pseudo +└─Selection_9(Probe) 0.01 cop[tikv] or(and(json_memberof(cast(1, json BINARY), planner__core__casetest__index__index.t2.a), and(json_memberof(cast(1, json BINARY), planner__core__casetest__index__index.t2.b), eq(planner__core__casetest__index__index.t2.c, 3))), or(and(json_memberof(cast(3, json BINARY), planner__core__casetest__index__index.t2.b), eq(planner__core__casetest__index__index.t2.c, 4)), eq(planner__core__casetest__index__index.t2.d, 1))) + └─TableRowIDScan_8 10.20 cop[tikv] table:t2 keep order:false, stats:pseudo drop table if exists t1, t2; create table t1(pk int primary key, a json, b json, c int, d int, index idx((cast(a as signed array))), index idx2((cast(b as signed array)))); create table t2(pk int primary key, a json, b json, c int, d int, index idx(c, (cast(a as signed array))), index idx2((cast(b as signed array)), c), index idx3(c, d), index idx4(d)); explain select /*+ use_index_merge(t1, idx2, idx) */ * from t1 where 1 member of (a) and 2 member of (b); -- 1: AND index merge from multi member mv index predicate, since member of is single partial path, it can be merged with outer index merge.; id estRows task access object operator info -IndexMerge_8 10.00 root type: intersection +IndexMerge_8 0.01 root type: intersection ├─IndexRangeScan_5(Build) 10.00 cop[tikv] table:t1, index:idx(cast(`a` as signed array)) range:[1,1], keep order:false, stats:pseudo ├─IndexRangeScan_6(Build) 10.00 cop[tikv] table:t1, index:idx2(cast(`b` as signed array)) range:[2,2], keep order:false, stats:pseudo -└─TableRowIDScan_7(Probe) 10.00 cop[tikv] table:t1 keep order:false, stats:pseudo +└─TableRowIDScan_7(Probe) 0.01 cop[tikv] table:t1 keep order:false, stats:pseudo explain select /*+ use_index_merge(t2, idx2, idx) */ * from t2 where 1 member of (a) and c=1 and 2 member of (b); -- 2: AND index merge from multi complicated mv index; id estRows task access object operator info -IndexMerge_8 0.10 root type: intersection +IndexMerge_8 0.00 root type: intersection ├─IndexRangeScan_5(Build) 0.10 cop[tikv] table:t2, index:idx(c, cast(`a` as signed array)) range:[1 1,1 1], keep order:false, stats:pseudo ├─IndexRangeScan_6(Build) 0.10 cop[tikv] table:t2, index:idx2(cast(`b` as signed array), c) range:[2 1,2 1], keep order:false, stats:pseudo -└─TableRowIDScan_7(Probe) 0.10 cop[tikv] table:t2 keep order:false, stats:pseudo +└─TableRowIDScan_7(Probe) 0.00 cop[tikv] table:t2 keep order:false, stats:pseudo explain select /*+ use_index_merge(t2, idx2, idx, idx4) */ * from t2 where 1 member of (a) and c=1 and 2 member of (b) and d=3; -- 3: AND index merge from multi complicated mv indexes and normal indexes; id estRows task access object operator info -IndexMerge_9 0.10 root type: intersection +IndexMerge_9 0.00 root type: intersection ├─IndexRangeScan_5(Build) 10.00 cop[tikv] table:t2, index:idx4(d) range:[3,3], keep order:false, stats:pseudo ├─IndexRangeScan_6(Build) 0.10 cop[tikv] table:t2, index:idx(c, cast(`a` as signed array)) range:[1 1,1 1], keep order:false, stats:pseudo ├─IndexRangeScan_7(Build) 0.10 cop[tikv] table:t2, index:idx2(cast(`b` as signed array), c) range:[2 1,2 1], keep order:false, stats:pseudo -└─TableRowIDScan_8(Probe) 0.10 cop[tikv] table:t2 keep order:false, stats:pseudo +└─TableRowIDScan_8(Probe) 0.00 cop[tikv] table:t2 keep order:false, stats:pseudo explain select /*+ use_index_merge(t2, idx2, idx, idx3) */ * from t2 where json_contains(a, '[1, 2, 3]') and c=1 and 2 member of (b) and d=3; -- 4: AND index merge from multi complicated mv indexes (json_contains (intersection))and normal indexes; id estRows task access object operator info -IndexMerge_11 0.10 root type: intersection +IndexMerge_11 0.00 root type: intersection ├─IndexRangeScan_5(Build) 0.10 cop[tikv] table:t2, index:idx3(c, d) range:[1 3,1 3], keep order:false, stats:pseudo ├─IndexRangeScan_6(Build) 0.10 cop[tikv] table:t2, index:idx(c, cast(`a` as signed array)) range:[1 1,1 1], keep order:false, stats:pseudo ├─IndexRangeScan_7(Build) 0.10 cop[tikv] table:t2, index:idx(c, cast(`a` as signed array)) range:[1 2,1 2], keep order:false, stats:pseudo ├─IndexRangeScan_8(Build) 0.10 cop[tikv] table:t2, index:idx(c, cast(`a` as signed array)) range:[1 3,1 3], keep order:false, stats:pseudo ├─IndexRangeScan_9(Build) 0.10 cop[tikv] table:t2, index:idx2(cast(`b` as signed array), c) range:[2 1,2 1], keep order:false, stats:pseudo -└─TableRowIDScan_10(Probe) 0.10 cop[tikv] table:t2 keep order:false, stats:pseudo +└─TableRowIDScan_10(Probe) 0.00 cop[tikv] table:t2 keep order:false, stats:pseudo explain select /*+ use_index_merge(t2, idx2, idx, idx3) */ * from t2 where json_overlaps(a, '[1, 2, 3]') and c=1 and 2 member of (b) and d=3; -- 5: AND index merge from multi complicated mv indexes (json_overlap (intersection))and normal indexes; id estRows task access object operator info -Selection_5 0.06 root json_overlaps(planner__core__casetest__index__index.t2.a, cast("[1, 2, 3]", json BINARY)) -└─IndexMerge_9 0.10 root type: intersection +Selection_5 0.00 root json_overlaps(planner__core__casetest__index__index.t2.a, cast("[1, 2, 3]", json BINARY)) +└─IndexMerge_9 0.00 root type: intersection ├─IndexRangeScan_6(Build) 0.10 cop[tikv] table:t2, index:idx3(c, d) range:[1 3,1 3], keep order:false, stats:pseudo ├─IndexRangeScan_7(Build) 0.10 cop[tikv] table:t2, index:idx2(cast(`b` as signed array), c) range:[2 1,2 1], keep order:false, stats:pseudo - └─TableRowIDScan_8(Probe) 0.10 cop[tikv] table:t2 keep order:false, stats:pseudo + └─TableRowIDScan_8(Probe) 0.00 cop[tikv] table:t2 keep order:false, stats:pseudo explain select /*+ use_index_merge(t2, idx2, idx) */ * from t2 where 1 member of (a) and c=1 and c=2; -- 6: AND index merge from multi complicated mv indexes (empty range); id estRows task access object operator info TableDual_5 0.00 root rows:0 diff --git a/tests/integrationtest/r/planner/core/casetest/physicalplantest/physical_plan.result b/tests/integrationtest/r/planner/core/casetest/physicalplantest/physical_plan.result index 47c63d2efe4f9..b83a1fe725cc1 100644 --- a/tests/integrationtest/r/planner/core/casetest/physicalplantest/physical_plan.result +++ b/tests/integrationtest/r/planner/core/casetest/physicalplantest/physical_plan.result @@ -3679,18 +3679,18 @@ show warnings; Level Code Message explain format = 'brief' select /*+ use_index(t, kj) */ * from t where (1 member of (j)) limit 1; id estRows task access object operator info -IndexMerge 0.00 root type: union, limit embedded(offset:0, count:1) -├─Limit(Build) 0.00 cop[tikv] offset:0, count:1 -│ └─IndexRangeScan 0.00 cop[tikv] table:t, index:kj(cast(`j` as signed array)) range:[1,1], keep order:false, stats:pseudo -└─TableRowIDScan(Probe) 0.00 cop[tikv] table:t keep order:false, stats:pseudo +IndexMerge 1.00 root type: union, limit embedded(offset:0, count:1) +├─Limit(Build) 1.00 cop[tikv] offset:0, count:1 +│ └─IndexRangeScan 1.00 cop[tikv] table:t, index:kj(cast(`j` as signed array)) range:[1,1], keep order:false, stats:pseudo +└─TableRowIDScan(Probe) 1.00 cop[tikv] table:t keep order:false, stats:pseudo show warnings; Level Code Message explain format = 'brief' select /*+ use_index(t, kj) */ * from t where json_contains(j, '[1, 2, 3]') limit 1; id estRows task access object operator info IndexMerge 0.00 root type: intersection, limit embedded(offset:0, count:1) -├─IndexRangeScan(Build) 0.00 cop[tikv] table:t, index:kj(cast(`j` as signed array)) range:[1,1], keep order:false, stats:pseudo -├─IndexRangeScan(Build) 0.00 cop[tikv] table:t, index:kj(cast(`j` as signed array)) range:[2,2], keep order:false, stats:pseudo -├─IndexRangeScan(Build) 0.00 cop[tikv] table:t, index:kj(cast(`j` as signed array)) range:[3,3], keep order:false, stats:pseudo +├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:kj(cast(`j` as signed array)) range:[1,1], keep order:false, stats:pseudo +├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:kj(cast(`j` as signed array)) range:[2,2], keep order:false, stats:pseudo +├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:kj(cast(`j` as signed array)) range:[3,3], keep order:false, stats:pseudo └─TableRowIDScan(Probe) 0.00 cop[tikv] table:t keep order:false, stats:pseudo show warnings; Level Code Message @@ -3698,46 +3698,46 @@ explain format = 'brief' select /*+ use_index(t, kj) */ * from t where json_over id estRows task access object operator info Limit 1.00 root offset:0, count:1 └─Selection 1.00 root json_overlaps(planner__core__casetest__physicalplantest__physical_plan.t.j, cast("[1, 2, 3]", json BINARY)) - └─IndexMerge 0.00 root type: union - ├─IndexRangeScan(Build) 0.00 cop[tikv] table:t, index:kj(cast(`j` as signed array)) range:[1,1], keep order:false, stats:pseudo - ├─IndexRangeScan(Build) 0.00 cop[tikv] table:t, index:kj(cast(`j` as signed array)) range:[2,2], keep order:false, stats:pseudo - ├─IndexRangeScan(Build) 0.00 cop[tikv] table:t, index:kj(cast(`j` as signed array)) range:[3,3], keep order:false, stats:pseudo - └─TableRowIDScan(Probe) 0.00 cop[tikv] table:t keep order:false, stats:pseudo + └─IndexMerge 1.00 root type: union + ├─IndexRangeScan(Build) 0.33 cop[tikv] table:t, index:kj(cast(`j` as signed array)) range:[1,1], keep order:false, stats:pseudo + ├─IndexRangeScan(Build) 0.33 cop[tikv] table:t, index:kj(cast(`j` as signed array)) range:[2,2], keep order:false, stats:pseudo + ├─IndexRangeScan(Build) 0.33 cop[tikv] table:t, index:kj(cast(`j` as signed array)) range:[3,3], keep order:false, stats:pseudo + └─TableRowIDScan(Probe) 1.00 cop[tikv] table:t keep order:false, stats:pseudo show warnings; Level Code Message Warning 1105 Scalar function 'json_overlaps'(signature: Unspecified, return type: bigint(20)) is not supported to push down to storage layer now. explain format = 'brief' select /*+ use_index(t, kj) */ * from t where (1 member of (j) and a=1 ) limit 1; id estRows task access object operator info -Limit 1.00 root offset:0, count:1 -└─IndexMerge 0.00 root type: union - ├─IndexRangeScan(Build) 1.25 cop[tikv] table:t, index:kj(cast(`j` as signed array)) range:[1,1], keep order:false, stats:pseudo - └─Limit(Probe) 0.00 cop[tikv] offset:0, count:1 - └─Selection 0.00 cop[tikv] eq(planner__core__casetest__physicalplantest__physical_plan.t.a, 1) - └─TableRowIDScan 1.25 cop[tikv] table:t keep order:false, stats:pseudo +Limit 0.01 root offset:0, count:1 +└─IndexMerge 0.01 root type: union + ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:kj(cast(`j` as signed array)) range:[1,1], keep order:false, stats:pseudo + └─Limit(Probe) 0.01 cop[tikv] offset:0, count:1 + └─Selection 0.01 cop[tikv] eq(planner__core__casetest__physicalplantest__physical_plan.t.a, 1) + └─TableRowIDScan 10.00 cop[tikv] table:t keep order:false, stats:pseudo show warnings; Level Code Message explain format = 'brief' select /*+ use_index(t, kj) */ * from t where json_contains(j, '[1, 2, 3]') and a=1 limit 1; id estRows task access object operator info -Limit 1.00 root offset:0, count:1 +Limit 0.00 root offset:0, count:1 └─IndexMerge 0.00 root type: intersection - ├─IndexRangeScan(Build) 1.25 cop[tikv] table:t, index:kj(cast(`j` as signed array)) range:[1,1], keep order:false, stats:pseudo - ├─IndexRangeScan(Build) 1.25 cop[tikv] table:t, index:kj(cast(`j` as signed array)) range:[2,2], keep order:false, stats:pseudo - ├─IndexRangeScan(Build) 1.25 cop[tikv] table:t, index:kj(cast(`j` as signed array)) range:[3,3], keep order:false, stats:pseudo + ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:kj(cast(`j` as signed array)) range:[1,1], keep order:false, stats:pseudo + ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:kj(cast(`j` as signed array)) range:[2,2], keep order:false, stats:pseudo + ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:kj(cast(`j` as signed array)) range:[3,3], keep order:false, stats:pseudo └─Limit(Probe) 0.00 cop[tikv] offset:0, count:1 └─Selection 0.00 cop[tikv] eq(planner__core__casetest__physicalplantest__physical_plan.t.a, 1) - └─TableRowIDScan 1.25 cop[tikv] table:t keep order:false, stats:pseudo + └─TableRowIDScan 0.00 cop[tikv] table:t keep order:false, stats:pseudo show warnings; Level Code Message explain format = 'brief' select /*+ use_index(t, kj) */ * from t where json_overlaps(j, '[1, 2, 3]') and a=1 limit 1; id estRows task access object operator info Limit 1.00 root offset:0, count:1 └─Selection 1.00 root json_overlaps(planner__core__casetest__physicalplantest__physical_plan.t.j, cast("[1, 2, 3]", json BINARY)) - └─IndexMerge 1.00 root type: union + └─IndexMerge 0.00 root type: union ├─IndexRangeScan(Build) 1.00 cop[tikv] table:t, index:kj(cast(`j` as signed array)) range:[1,1], keep order:false, stats:pseudo ├─IndexRangeScan(Build) 1.00 cop[tikv] table:t, index:kj(cast(`j` as signed array)) range:[2,2], keep order:false, stats:pseudo ├─IndexRangeScan(Build) 1.00 cop[tikv] table:t, index:kj(cast(`j` as signed array)) range:[3,3], keep order:false, stats:pseudo - └─Selection(Probe) 1.00 cop[tikv] eq(planner__core__casetest__physicalplantest__physical_plan.t.a, 1) - └─TableRowIDScan 1.00 cop[tikv] table:t keep order:false, stats:pseudo + └─Selection(Probe) 0.00 cop[tikv] eq(planner__core__casetest__physicalplantest__physical_plan.t.a, 1) + └─TableRowIDScan 3.00 cop[tikv] table:t keep order:false, stats:pseudo show warnings; Level Code Message Warning 1105 Scalar function 'json_overlaps'(signature: Unspecified, return type: bigint(20)) is not supported to push down to storage layer now. diff --git a/tests/integrationtest/r/planner/core/indexmerge_path.result b/tests/integrationtest/r/planner/core/indexmerge_path.result index d89e1f168c9d1..e21684ca0dff8 100644 --- a/tests/integrationtest/r/planner/core/indexmerge_path.result +++ b/tests/integrationtest/r/planner/core/indexmerge_path.result @@ -50,9 +50,9 @@ TableReader 2658.67 root data:Selection └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo explain format = 'brief' select /*+ use_index_merge(t, j0_1) */ * from t where (1 member of (j0->'$.path1')) and (2 member of (j1)) and a<10; id estRows task access object operator info -IndexMerge 2.66 root type: union +IndexMerge 0.00 root type: union ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:j0_1(cast(json_extract(`j0`, _utf8mb4'$.path1') as signed array)) range:[1,1], keep order:false, stats:pseudo -└─Selection(Probe) 2.66 cop[tikv] json_memberof(cast(2, json BINARY), planner__core__indexmerge_path.t.j1), lt(planner__core__indexmerge_path.t.a, 10) +└─Selection(Probe) 0.00 cop[tikv] json_memberof(cast(2, json BINARY), planner__core__indexmerge_path.t.j1), lt(planner__core__indexmerge_path.t.a, 10) └─TableRowIDScan 10.00 cop[tikv] table:t keep order:false, stats:pseudo explain format = 'brief' select /*+ use_index(t, j0_0) */ * from t where (1 member of (j0->'$.path0')); id estRows task access object operator info @@ -89,59 +89,59 @@ IndexMerge 3.32 root type: union └─TableRowIDScan 10.00 cop[tikv] table:t keep order:false, stats:pseudo explain format = 'brief' select /*+ use_index_merge(t, j1) */ * from t where (1 member of (j0->'$.path1')) and (2 member of (j1)) and a<10; id estRows task access object operator info -IndexMerge 2.66 root type: union +IndexMerge 0.00 root type: union ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:j0_1(cast(json_extract(`j0`, _utf8mb4'$.path1') as signed array)) range:[1,1], keep order:false, stats:pseudo -└─Selection(Probe) 2.66 cop[tikv] json_memberof(cast(2, json BINARY), planner__core__indexmerge_path.t.j1), lt(planner__core__indexmerge_path.t.a, 10) +└─Selection(Probe) 0.00 cop[tikv] json_memberof(cast(2, json BINARY), planner__core__indexmerge_path.t.j1), lt(planner__core__indexmerge_path.t.a, 10) └─TableRowIDScan 10.00 cop[tikv] table:t keep order:false, stats:pseudo explain format = 'brief' select /*+ use_index_merge(t, j0_0) */ * from t where json_contains((j0->'$.path0'), '[1, 2, 3]'); id estRows task access object operator info -IndexMerge 10.00 root type: intersection +IndexMerge 0.00 root type: intersection ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:j0_0(cast(json_extract(`j0`, _utf8mb4'$.path0') as signed array)) range:[1,1], keep order:false, stats:pseudo ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:j0_0(cast(json_extract(`j0`, _utf8mb4'$.path0') as signed array)) range:[2,2], keep order:false, stats:pseudo ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:j0_0(cast(json_extract(`j0`, _utf8mb4'$.path0') as signed array)) range:[3,3], keep order:false, stats:pseudo -└─TableRowIDScan(Probe) 10.00 cop[tikv] table:t keep order:false, stats:pseudo +└─TableRowIDScan(Probe) 0.00 cop[tikv] table:t keep order:false, stats:pseudo explain format = 'brief' select /*+ use_index_merge(t, j0_0) */ * from t where json_overlaps((j0->'$.path0'), '[1, 2, 3]'); id estRows task access object operator info -Selection 8.00 root json_overlaps(json_extract(planner__core__indexmerge_path.t.j0, "$.path0"), cast("[1, 2, 3]", json BINARY)) -└─IndexMerge 10.00 root type: union +Selection 23.98 root json_overlaps(json_extract(planner__core__indexmerge_path.t.j0, "$.path0"), cast("[1, 2, 3]", json BINARY)) +└─IndexMerge 29.97 root type: union ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:j0_0(cast(json_extract(`j0`, _utf8mb4'$.path0') as signed array)) range:[1,1], keep order:false, stats:pseudo ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:j0_0(cast(json_extract(`j0`, _utf8mb4'$.path0') as signed array)) range:[2,2], keep order:false, stats:pseudo ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:j0_0(cast(json_extract(`j0`, _utf8mb4'$.path0') as signed array)) range:[3,3], keep order:false, stats:pseudo - └─TableRowIDScan(Probe) 10.00 cop[tikv] table:t keep order:false, stats:pseudo + └─TableRowIDScan(Probe) 29.97 cop[tikv] table:t keep order:false, stats:pseudo explain format = 'brief' select /*+ use_index_merge(t, j0_0) */ * from t where json_overlaps('[1, 2, 3]', (j0->'$.path0')); id estRows task access object operator info -Selection 8.00 root json_overlaps(cast("[1, 2, 3]", json BINARY), json_extract(planner__core__indexmerge_path.t.j0, "$.path0")) -└─IndexMerge 10.00 root type: union +Selection 23.98 root json_overlaps(cast("[1, 2, 3]", json BINARY), json_extract(planner__core__indexmerge_path.t.j0, "$.path0")) +└─IndexMerge 29.97 root type: union ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:j0_0(cast(json_extract(`j0`, _utf8mb4'$.path0') as signed array)) range:[1,1], keep order:false, stats:pseudo ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:j0_0(cast(json_extract(`j0`, _utf8mb4'$.path0') as signed array)) range:[2,2], keep order:false, stats:pseudo ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:j0_0(cast(json_extract(`j0`, _utf8mb4'$.path0') as signed array)) range:[3,3], keep order:false, stats:pseudo - └─TableRowIDScan(Probe) 10.00 cop[tikv] table:t keep order:false, stats:pseudo + └─TableRowIDScan(Probe) 29.97 cop[tikv] table:t keep order:false, stats:pseudo explain format = 'brief' select /*+ use_index_merge(t, j0_0) */ * from t where json_contains((j0->'$.path0'), '[1, 2, 3]') and a<10; id estRows task access object operator info -IndexMerge 3.32 root type: intersection +IndexMerge 0.00 root type: intersection ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:j0_0(cast(json_extract(`j0`, _utf8mb4'$.path0') as signed array)) range:[1,1], keep order:false, stats:pseudo ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:j0_0(cast(json_extract(`j0`, _utf8mb4'$.path0') as signed array)) range:[2,2], keep order:false, stats:pseudo ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:j0_0(cast(json_extract(`j0`, _utf8mb4'$.path0') as signed array)) range:[3,3], keep order:false, stats:pseudo -└─Selection(Probe) 3.32 cop[tikv] lt(planner__core__indexmerge_path.t.a, 10) - └─TableRowIDScan 10.00 cop[tikv] table:t keep order:false, stats:pseudo +└─Selection(Probe) 0.00 cop[tikv] lt(planner__core__indexmerge_path.t.a, 10) + └─TableRowIDScan 0.00 cop[tikv] table:t keep order:false, stats:pseudo explain format = 'brief' select /*+ use_index_merge(t, j0_0) */ * from t where json_overlaps((j0->'$.path0'), '[1, 2, 3]') and a<10; id estRows task access object operator info -Selection 8.00 root json_overlaps(json_extract(planner__core__indexmerge_path.t.j0, "$.path0"), cast("[1, 2, 3]", json BINARY)) -└─IndexMerge 3.32 root type: union +Selection 23.98 root json_overlaps(json_extract(planner__core__indexmerge_path.t.j0, "$.path0"), cast("[1, 2, 3]", json BINARY)) +└─IndexMerge 9.96 root type: union ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:j0_0(cast(json_extract(`j0`, _utf8mb4'$.path0') as signed array)) range:[1,1], keep order:false, stats:pseudo ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:j0_0(cast(json_extract(`j0`, _utf8mb4'$.path0') as signed array)) range:[2,2], keep order:false, stats:pseudo ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:j0_0(cast(json_extract(`j0`, _utf8mb4'$.path0') as signed array)) range:[3,3], keep order:false, stats:pseudo - └─Selection(Probe) 3.32 cop[tikv] lt(planner__core__indexmerge_path.t.a, 10) - └─TableRowIDScan 10.00 cop[tikv] table:t keep order:false, stats:pseudo + └─Selection(Probe) 9.96 cop[tikv] lt(planner__core__indexmerge_path.t.a, 10) + └─TableRowIDScan 29.97 cop[tikv] table:t keep order:false, stats:pseudo explain format = 'brief' select /*+ use_index_merge(t, j0_0) */ * from t where json_overlaps('[1, 2, 3]', (j0->'$.path0')) and a<10; id estRows task access object operator info -Selection 8.00 root json_overlaps(cast("[1, 2, 3]", json BINARY), json_extract(planner__core__indexmerge_path.t.j0, "$.path0")) -└─IndexMerge 3.32 root type: union +Selection 23.98 root json_overlaps(cast("[1, 2, 3]", json BINARY), json_extract(planner__core__indexmerge_path.t.j0, "$.path0")) +└─IndexMerge 9.96 root type: union ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:j0_0(cast(json_extract(`j0`, _utf8mb4'$.path0') as signed array)) range:[1,1], keep order:false, stats:pseudo ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:j0_0(cast(json_extract(`j0`, _utf8mb4'$.path0') as signed array)) range:[2,2], keep order:false, stats:pseudo ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:j0_0(cast(json_extract(`j0`, _utf8mb4'$.path0') as signed array)) range:[3,3], keep order:false, stats:pseudo - └─Selection(Probe) 3.32 cop[tikv] lt(planner__core__indexmerge_path.t.a, 10) - └─TableRowIDScan 10.00 cop[tikv] table:t keep order:false, stats:pseudo + └─Selection(Probe) 9.96 cop[tikv] lt(planner__core__indexmerge_path.t.a, 10) + └─TableRowIDScan 29.97 cop[tikv] table:t keep order:false, stats:pseudo explain format = 'brief' select /*+ use_index_merge(t, j0_0) */ * from t where json_contains((j0->'$.path0'), '1'); id estRows task access object operator info IndexMerge 10.00 root type: intersection @@ -192,36 +192,36 @@ IndexMerge 3.32 root type: union └─TableRowIDScan 10.00 cop[tikv] table:t keep order:false, stats:pseudo explain format = 'brief' select /*+ use_index_merge(t, j0_string) */ * from t where json_contains((j0->'$.path_string'), '["a", "b", "c"]'); id estRows task access object operator info -IndexMerge 10.00 root type: intersection +IndexMerge 0.00 root type: intersection ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:j0_string(cast(json_extract(`j0`, _utf8mb4'$.path_string') as char(10) array)) range:["a","a"], keep order:false, stats:pseudo ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:j0_string(cast(json_extract(`j0`, _utf8mb4'$.path_string') as char(10) array)) range:["b","b"], keep order:false, stats:pseudo ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:j0_string(cast(json_extract(`j0`, _utf8mb4'$.path_string') as char(10) array)) range:["c","c"], keep order:false, stats:pseudo -└─TableRowIDScan(Probe) 10.00 cop[tikv] table:t keep order:false, stats:pseudo +└─TableRowIDScan(Probe) 0.00 cop[tikv] table:t keep order:false, stats:pseudo explain format = 'brief' select /*+ use_index_merge(t, j0_string) */ * from t where json_contains((j0->'$.path_string'), '["a", "b", "c"]') and a<10; id estRows task access object operator info -IndexMerge 3.32 root type: intersection +IndexMerge 0.00 root type: intersection ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:j0_string(cast(json_extract(`j0`, _utf8mb4'$.path_string') as char(10) array)) range:["a","a"], keep order:false, stats:pseudo ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:j0_string(cast(json_extract(`j0`, _utf8mb4'$.path_string') as char(10) array)) range:["b","b"], keep order:false, stats:pseudo ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:j0_string(cast(json_extract(`j0`, _utf8mb4'$.path_string') as char(10) array)) range:["c","c"], keep order:false, stats:pseudo -└─Selection(Probe) 3.32 cop[tikv] lt(planner__core__indexmerge_path.t.a, 10) - └─TableRowIDScan 10.00 cop[tikv] table:t keep order:false, stats:pseudo +└─Selection(Probe) 0.00 cop[tikv] lt(planner__core__indexmerge_path.t.a, 10) + └─TableRowIDScan 0.00 cop[tikv] table:t keep order:false, stats:pseudo explain format = 'brief' select /*+ use_index_merge(t, j0_string) */ * from t where json_overlaps((j0->'$.path_string'), '["a", "b", "c"]'); id estRows task access object operator info -Selection 8.00 root json_overlaps(json_extract(planner__core__indexmerge_path.t.j0, "$.path_string"), cast("["a", "b", "c"]", json BINARY)) -└─IndexMerge 10.00 root type: union +Selection 23.98 root json_overlaps(json_extract(planner__core__indexmerge_path.t.j0, "$.path_string"), cast("["a", "b", "c"]", json BINARY)) +└─IndexMerge 29.97 root type: union ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:j0_string(cast(json_extract(`j0`, _utf8mb4'$.path_string') as char(10) array)) range:["a","a"], keep order:false, stats:pseudo ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:j0_string(cast(json_extract(`j0`, _utf8mb4'$.path_string') as char(10) array)) range:["b","b"], keep order:false, stats:pseudo ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:j0_string(cast(json_extract(`j0`, _utf8mb4'$.path_string') as char(10) array)) range:["c","c"], keep order:false, stats:pseudo - └─TableRowIDScan(Probe) 10.00 cop[tikv] table:t keep order:false, stats:pseudo + └─TableRowIDScan(Probe) 29.97 cop[tikv] table:t keep order:false, stats:pseudo explain format = 'brief' select /*+ use_index_merge(t, j0_string) */ * from t where json_overlaps((j0->'$.path_string'), '["a", "b", "c"]') and a<10; id estRows task access object operator info -Selection 8.00 root json_overlaps(json_extract(planner__core__indexmerge_path.t.j0, "$.path_string"), cast("["a", "b", "c"]", json BINARY)) -└─IndexMerge 3.32 root type: union +Selection 23.98 root json_overlaps(json_extract(planner__core__indexmerge_path.t.j0, "$.path_string"), cast("["a", "b", "c"]", json BINARY)) +└─IndexMerge 9.96 root type: union ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:j0_string(cast(json_extract(`j0`, _utf8mb4'$.path_string') as char(10) array)) range:["a","a"], keep order:false, stats:pseudo ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:j0_string(cast(json_extract(`j0`, _utf8mb4'$.path_string') as char(10) array)) range:["b","b"], keep order:false, stats:pseudo ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:j0_string(cast(json_extract(`j0`, _utf8mb4'$.path_string') as char(10) array)) range:["c","c"], keep order:false, stats:pseudo - └─Selection(Probe) 3.32 cop[tikv] lt(planner__core__indexmerge_path.t.a, 10) - └─TableRowIDScan 10.00 cop[tikv] table:t keep order:false, stats:pseudo + └─Selection(Probe) 9.96 cop[tikv] lt(planner__core__indexmerge_path.t.a, 10) + └─TableRowIDScan 29.97 cop[tikv] table:t keep order:false, stats:pseudo explain format = 'brief' select /*+ use_index_merge(t, j0_date) */ * from t where ("2023-01-01" member of (j0->'$.path_date')); id estRows task access object operator info TableReader 8000.00 root data:Selection @@ -234,76 +234,76 @@ TableReader 2658.67 root data:Selection └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo explain format = 'brief' select /*+ use_index_merge(t, j0_date) */ * from t where json_contains((j0->'$.path_date'), json_array(cast('2023-01-01' as date), cast('2023-01-02' as date), cast('2023-01-03' as date))); id estRows task access object operator info -IndexMerge 10.00 root type: intersection +IndexMerge 0.00 root type: intersection ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:j0_date(cast(json_extract(`j0`, _utf8mb4'$.path_date') as date array)) range:[2023-01-01,2023-01-01], keep order:false, stats:pseudo ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:j0_date(cast(json_extract(`j0`, _utf8mb4'$.path_date') as date array)) range:[2023-01-02,2023-01-02], keep order:false, stats:pseudo ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:j0_date(cast(json_extract(`j0`, _utf8mb4'$.path_date') as date array)) range:[2023-01-03,2023-01-03], keep order:false, stats:pseudo -└─TableRowIDScan(Probe) 10.00 cop[tikv] table:t keep order:false, stats:pseudo +└─TableRowIDScan(Probe) 0.00 cop[tikv] table:t keep order:false, stats:pseudo explain format = 'brief' select /*+ use_index_merge(t, j0_date) */ * from t where json_contains((j0->'$.path_date'), json_array(cast('2023-01-01' as date), cast('2023-01-02' as date), cast('2023-01-03' as date))) and a<10; id estRows task access object operator info -IndexMerge 3.32 root type: intersection +IndexMerge 0.00 root type: intersection ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:j0_date(cast(json_extract(`j0`, _utf8mb4'$.path_date') as date array)) range:[2023-01-01,2023-01-01], keep order:false, stats:pseudo ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:j0_date(cast(json_extract(`j0`, _utf8mb4'$.path_date') as date array)) range:[2023-01-02,2023-01-02], keep order:false, stats:pseudo ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:j0_date(cast(json_extract(`j0`, _utf8mb4'$.path_date') as date array)) range:[2023-01-03,2023-01-03], keep order:false, stats:pseudo -└─Selection(Probe) 3.32 cop[tikv] lt(planner__core__indexmerge_path.t.a, 10) - └─TableRowIDScan 10.00 cop[tikv] table:t keep order:false, stats:pseudo +└─Selection(Probe) 0.00 cop[tikv] lt(planner__core__indexmerge_path.t.a, 10) + └─TableRowIDScan 0.00 cop[tikv] table:t keep order:false, stats:pseudo explain format = 'brief' select /*+ use_index_merge(t, j0_date) */ * from t where json_overlaps((j0->'$.path_date'), json_array(cast('2023-01-01' as date), cast('2023-01-02' as date), cast('2023-01-03' as date))); id estRows task access object operator info -Selection 8.00 root json_overlaps(json_extract(planner__core__indexmerge_path.t.j0, "$.path_date"), json_array(cast(2023-01-01, json BINARY), cast(2023-01-02, json BINARY), cast(2023-01-03, json BINARY))) -└─IndexMerge 10.00 root type: union +Selection 23.98 root json_overlaps(json_extract(planner__core__indexmerge_path.t.j0, "$.path_date"), json_array(cast(2023-01-01, json BINARY), cast(2023-01-02, json BINARY), cast(2023-01-03, json BINARY))) +└─IndexMerge 29.97 root type: union ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:j0_date(cast(json_extract(`j0`, _utf8mb4'$.path_date') as date array)) range:[2023-01-01,2023-01-01], keep order:false, stats:pseudo ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:j0_date(cast(json_extract(`j0`, _utf8mb4'$.path_date') as date array)) range:[2023-01-02,2023-01-02], keep order:false, stats:pseudo ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:j0_date(cast(json_extract(`j0`, _utf8mb4'$.path_date') as date array)) range:[2023-01-03,2023-01-03], keep order:false, stats:pseudo - └─TableRowIDScan(Probe) 10.00 cop[tikv] table:t keep order:false, stats:pseudo + └─TableRowIDScan(Probe) 29.97 cop[tikv] table:t keep order:false, stats:pseudo explain format = 'brief' select /*+ use_index_merge(t, j0_date) */ * from t where json_overlaps((j0->'$.path_date'), json_array(cast('2023-01-01' as date), cast('2023-01-02' as date), cast('2023-01-03' as date))) and a<10; id estRows task access object operator info -Selection 8.00 root json_overlaps(json_extract(planner__core__indexmerge_path.t.j0, "$.path_date"), json_array(cast(2023-01-01, json BINARY), cast(2023-01-02, json BINARY), cast(2023-01-03, json BINARY))) -└─IndexMerge 3.32 root type: union +Selection 23.98 root json_overlaps(json_extract(planner__core__indexmerge_path.t.j0, "$.path_date"), json_array(cast(2023-01-01, json BINARY), cast(2023-01-02, json BINARY), cast(2023-01-03, json BINARY))) +└─IndexMerge 9.96 root type: union ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:j0_date(cast(json_extract(`j0`, _utf8mb4'$.path_date') as date array)) range:[2023-01-01,2023-01-01], keep order:false, stats:pseudo ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:j0_date(cast(json_extract(`j0`, _utf8mb4'$.path_date') as date array)) range:[2023-01-02,2023-01-02], keep order:false, stats:pseudo ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:j0_date(cast(json_extract(`j0`, _utf8mb4'$.path_date') as date array)) range:[2023-01-03,2023-01-03], keep order:false, stats:pseudo - └─Selection(Probe) 3.32 cop[tikv] lt(planner__core__indexmerge_path.t.a, 10) - └─TableRowIDScan 10.00 cop[tikv] table:t keep order:false, stats:pseudo + └─Selection(Probe) 9.96 cop[tikv] lt(planner__core__indexmerge_path.t.a, 10) + └─TableRowIDScan 29.97 cop[tikv] table:t keep order:false, stats:pseudo drop table if exists t; create table t(a int, b int, c int, j json, index idx1((cast(j as signed array))), index idx2(a, b, (cast(j as signed array)), c)); explain format = 'brief' select /*+ use_index_merge(t, idx1) */ * from t where (1 member of (j)) or (2 member of (j)); id estRows task access object operator info -IndexMerge 10.00 root type: union +IndexMerge 19.99 root type: union ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:idx1(cast(`j` as signed array)) range:[1,1], keep order:false, stats:pseudo ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:idx1(cast(`j` as signed array)) range:[2,2], keep order:false, stats:pseudo -└─TableRowIDScan(Probe) 10.00 cop[tikv] table:t keep order:false, stats:pseudo +└─TableRowIDScan(Probe) 19.99 cop[tikv] table:t keep order:false, stats:pseudo explain format = 'brief' select /*+ use_index_merge(t, idx1) */ * from t where ((1 member of (j)) or (2 member of (j))) and (a > 10); id estRows task access object operator info -IndexMerge 3.33 root type: union +IndexMerge 6.66 root type: union ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:idx1(cast(`j` as signed array)) range:[1,1], keep order:false, stats:pseudo ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:idx1(cast(`j` as signed array)) range:[2,2], keep order:false, stats:pseudo -└─Selection(Probe) 3.33 cop[tikv] gt(planner__core__indexmerge_path.t.a, 10) - └─TableRowIDScan 10.00 cop[tikv] table:t keep order:false, stats:pseudo +└─Selection(Probe) 6.66 cop[tikv] gt(planner__core__indexmerge_path.t.a, 10) + └─TableRowIDScan 19.99 cop[tikv] table:t keep order:false, stats:pseudo explain format = 'brief' select /*+ use_index_merge(t, idx1) */ * from t where (json_overlaps(j, '[1, 2]')) or (json_overlaps(j, '[3, 4]')); id estRows task access object operator info -Selection 8.00 root or(json_overlaps(planner__core__indexmerge_path.t.j, cast("[1, 2]", json BINARY)), json_overlaps(planner__core__indexmerge_path.t.j, cast("[3, 4]", json BINARY))) -└─IndexMerge 10.00 root type: union +Selection 31.95 root or(json_overlaps(planner__core__indexmerge_path.t.j, cast("[1, 2]", json BINARY)), json_overlaps(planner__core__indexmerge_path.t.j, cast("[3, 4]", json BINARY))) +└─IndexMerge 39.94 root type: union ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:idx1(cast(`j` as signed array)) range:[1,1], keep order:false, stats:pseudo ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:idx1(cast(`j` as signed array)) range:[2,2], keep order:false, stats:pseudo ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:idx1(cast(`j` as signed array)) range:[3,3], keep order:false, stats:pseudo ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:idx1(cast(`j` as signed array)) range:[4,4], keep order:false, stats:pseudo - └─TableRowIDScan(Probe) 10.00 cop[tikv] table:t keep order:false, stats:pseudo + └─TableRowIDScan(Probe) 39.94 cop[tikv] table:t keep order:false, stats:pseudo explain format = 'brief' select /*+ use_index_merge(t, idx1) */ * from t where ((json_overlaps(j, '[1, 2]')) or (json_overlaps(j, '[3, 4]'))) and (a > 10); id estRows task access object operator info -Selection 8.00 root or(json_overlaps(planner__core__indexmerge_path.t.j, cast("[1, 2]", json BINARY)), json_overlaps(planner__core__indexmerge_path.t.j, cast("[3, 4]", json BINARY))) -└─IndexMerge 3.33 root type: union +Selection 31.95 root or(json_overlaps(planner__core__indexmerge_path.t.j, cast("[1, 2]", json BINARY)), json_overlaps(planner__core__indexmerge_path.t.j, cast("[3, 4]", json BINARY))) +└─IndexMerge 13.31 root type: union ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:idx1(cast(`j` as signed array)) range:[1,1], keep order:false, stats:pseudo ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:idx1(cast(`j` as signed array)) range:[2,2], keep order:false, stats:pseudo ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:idx1(cast(`j` as signed array)) range:[3,3], keep order:false, stats:pseudo ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:idx1(cast(`j` as signed array)) range:[4,4], keep order:false, stats:pseudo - └─Selection(Probe) 3.33 cop[tikv] gt(planner__core__indexmerge_path.t.a, 10) - └─TableRowIDScan 10.00 cop[tikv] table:t keep order:false, stats:pseudo + └─Selection(Probe) 13.31 cop[tikv] gt(planner__core__indexmerge_path.t.a, 10) + └─TableRowIDScan 39.94 cop[tikv] table:t keep order:false, stats:pseudo explain format = 'brief' select /*+ use_index_merge(t, idx1) */ * from t where (json_contains(j, '[1, 2]')) or (json_contains(j, '[3, 4]')); id estRows task access object operator info -TableReader 9600.00 root data:Selection -└─Selection 9600.00 cop[tikv] or(json_contains(planner__core__indexmerge_path.t.j, cast("[1, 2]", json BINARY)), json_contains(planner__core__indexmerge_path.t.j, cast("[3, 4]", json BINARY))) +TableReader 0.02 root data:Selection +└─Selection 0.02 cop[tikv] or(json_contains(planner__core__indexmerge_path.t.j, cast("[1, 2]", json BINARY)), json_contains(planner__core__indexmerge_path.t.j, cast("[3, 4]", json BINARY))) └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo explain format = 'brief' select /*+ use_index_merge(t, idx2) */ * from t where (a=1 and b=2 and (3 member of (j))) or (a=11 and b=12 and (13 member of (j))); id estRows task access object operator info @@ -403,7 +403,7 @@ create table t(a int, j json, index i_int((cast(j->'$.int' as signed array)))); explain format = 'brief' select (j->'$.int') from t where (1 member of (j->'$.int')); id estRows task access object operator info -Projection 8000.00 root json_extract(planner__core__indexmerge_path.t.j, $.int)->Column#5 +Projection 10.00 root json_extract(planner__core__indexmerge_path.t.j, $.int)->Column#5 └─IndexMerge 10.00 root type: union ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:i_int(cast(json_extract(`j`, _utf8mb4'$.int') as signed array)) range:[1,1], keep order:false, stats:pseudo └─TableRowIDScan(Probe) 10.00 cop[tikv] table:t keep order:false, stats:pseudo @@ -420,53 +420,53 @@ IndexMerge 3.32 root type: union └─TableRowIDScan 10.00 cop[tikv] table:t keep order:false, stats:pseudo explain format = 'brief' select (j->'$.int') from t where json_contains((j->'$.int'), '[1, 2, 3]'); id estRows task access object operator info -Projection 8000.00 root json_extract(planner__core__indexmerge_path.t.j, $.int)->Column#5 -└─IndexMerge 10.00 root type: intersection +Projection 0.00 root json_extract(planner__core__indexmerge_path.t.j, $.int)->Column#5 +└─IndexMerge 0.00 root type: intersection ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:i_int(cast(json_extract(`j`, _utf8mb4'$.int') as signed array)) range:[1,1], keep order:false, stats:pseudo ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:i_int(cast(json_extract(`j`, _utf8mb4'$.int') as signed array)) range:[2,2], keep order:false, stats:pseudo ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:i_int(cast(json_extract(`j`, _utf8mb4'$.int') as signed array)) range:[3,3], keep order:false, stats:pseudo - └─TableRowIDScan(Probe) 10.00 cop[tikv] table:t keep order:false, stats:pseudo + └─TableRowIDScan(Probe) 0.00 cop[tikv] table:t keep order:false, stats:pseudo explain format = 'brief' select * from t where json_contains((j->'$.int'), '[1, 2, 3]'); id estRows task access object operator info -IndexMerge 10.00 root type: intersection +IndexMerge 0.00 root type: intersection ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:i_int(cast(json_extract(`j`, _utf8mb4'$.int') as signed array)) range:[1,1], keep order:false, stats:pseudo ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:i_int(cast(json_extract(`j`, _utf8mb4'$.int') as signed array)) range:[2,2], keep order:false, stats:pseudo ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:i_int(cast(json_extract(`j`, _utf8mb4'$.int') as signed array)) range:[3,3], keep order:false, stats:pseudo -└─TableRowIDScan(Probe) 10.00 cop[tikv] table:t keep order:false, stats:pseudo +└─TableRowIDScan(Probe) 0.00 cop[tikv] table:t keep order:false, stats:pseudo explain format = 'brief' select * from t where json_contains((j->'$.int'), '[1, 2, 3]') and a<10; id estRows task access object operator info -IndexMerge 3.32 root type: intersection +IndexMerge 0.00 root type: intersection ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:i_int(cast(json_extract(`j`, _utf8mb4'$.int') as signed array)) range:[1,1], keep order:false, stats:pseudo ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:i_int(cast(json_extract(`j`, _utf8mb4'$.int') as signed array)) range:[2,2], keep order:false, stats:pseudo ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:i_int(cast(json_extract(`j`, _utf8mb4'$.int') as signed array)) range:[3,3], keep order:false, stats:pseudo -└─Selection(Probe) 3.32 cop[tikv] lt(planner__core__indexmerge_path.t.a, 10) - └─TableRowIDScan 10.00 cop[tikv] table:t keep order:false, stats:pseudo +└─Selection(Probe) 0.00 cop[tikv] lt(planner__core__indexmerge_path.t.a, 10) + └─TableRowIDScan 0.00 cop[tikv] table:t keep order:false, stats:pseudo explain format = 'brief' select (j->'$.int') from t where json_overlaps((j->'$.int'), '[1, 2, 3]'); id estRows task access object operator info -Projection 8000.00 root json_extract(planner__core__indexmerge_path.t.j, $.int)->Column#5 -└─Selection 8000.00 root json_overlaps(json_extract(planner__core__indexmerge_path.t.j, "$.int"), cast("[1, 2, 3]", json BINARY)) - └─IndexMerge 10.00 root type: union +Projection 23.98 root json_extract(planner__core__indexmerge_path.t.j, $.int)->Column#5 +└─Selection 23.98 root json_overlaps(json_extract(planner__core__indexmerge_path.t.j, "$.int"), cast("[1, 2, 3]", json BINARY)) + └─IndexMerge 29.97 root type: union ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:i_int(cast(json_extract(`j`, _utf8mb4'$.int') as signed array)) range:[1,1], keep order:false, stats:pseudo ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:i_int(cast(json_extract(`j`, _utf8mb4'$.int') as signed array)) range:[2,2], keep order:false, stats:pseudo ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:i_int(cast(json_extract(`j`, _utf8mb4'$.int') as signed array)) range:[3,3], keep order:false, stats:pseudo - └─TableRowIDScan(Probe) 10.00 cop[tikv] table:t keep order:false, stats:pseudo + └─TableRowIDScan(Probe) 29.97 cop[tikv] table:t keep order:false, stats:pseudo explain format = 'brief' select * from t where json_overlaps((j->'$.int'), '[1, 2, 3]'); id estRows task access object operator info -Selection 8000.00 root json_overlaps(json_extract(planner__core__indexmerge_path.t.j, "$.int"), cast("[1, 2, 3]", json BINARY)) -└─IndexMerge 10.00 root type: union +Selection 23.98 root json_overlaps(json_extract(planner__core__indexmerge_path.t.j, "$.int"), cast("[1, 2, 3]", json BINARY)) +└─IndexMerge 29.97 root type: union ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:i_int(cast(json_extract(`j`, _utf8mb4'$.int') as signed array)) range:[1,1], keep order:false, stats:pseudo ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:i_int(cast(json_extract(`j`, _utf8mb4'$.int') as signed array)) range:[2,2], keep order:false, stats:pseudo ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:i_int(cast(json_extract(`j`, _utf8mb4'$.int') as signed array)) range:[3,3], keep order:false, stats:pseudo - └─TableRowIDScan(Probe) 10.00 cop[tikv] table:t keep order:false, stats:pseudo + └─TableRowIDScan(Probe) 29.97 cop[tikv] table:t keep order:false, stats:pseudo explain format = 'brief' select * from t where json_overlaps((j->'$.int'), '[1, 2, 3]') and a<10; id estRows task access object operator info -Selection 2658.67 root json_overlaps(json_extract(planner__core__indexmerge_path.t.j, "$.int"), cast("[1, 2, 3]", json BINARY)) -└─IndexMerge 3.32 root type: union +Selection 23.98 root json_overlaps(json_extract(planner__core__indexmerge_path.t.j, "$.int"), cast("[1, 2, 3]", json BINARY)) +└─IndexMerge 9.96 root type: union ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:i_int(cast(json_extract(`j`, _utf8mb4'$.int') as signed array)) range:[1,1], keep order:false, stats:pseudo ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:i_int(cast(json_extract(`j`, _utf8mb4'$.int') as signed array)) range:[2,2], keep order:false, stats:pseudo ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:i_int(cast(json_extract(`j`, _utf8mb4'$.int') as signed array)) range:[3,3], keep order:false, stats:pseudo - └─Selection(Probe) 3.32 cop[tikv] lt(planner__core__indexmerge_path.t.a, 10) - └─TableRowIDScan 10.00 cop[tikv] table:t keep order:false, stats:pseudo + └─Selection(Probe) 9.96 cop[tikv] lt(planner__core__indexmerge_path.t.a, 10) + └─TableRowIDScan 29.97 cop[tikv] table:t keep order:false, stats:pseudo drop table if exists t; create table t(j json, index kj((cast(j as signed array)))); prepare st from 'select /*+ use_index_merge(t, kj) */ * from t where (1 member of (j))'; @@ -532,8 +532,8 @@ TableReader 3323.33 root data:Selection └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo explain format = 'brief' select /*+ use_index_merge(t, kj) */ * from t where (1 member of (j)) or a=10; id estRows task access object operator info -TableReader 8002.00 root data:Selection -└─Selection 8002.00 cop[tikv] or(json_memberof(cast(1, json BINARY), planner__core__indexmerge_path.t.j), eq(planner__core__indexmerge_path.t.a, 10)) +TableReader 19.99 root data:Selection +└─Selection 19.99 cop[tikv] or(json_memberof(cast(1, json BINARY), planner__core__indexmerge_path.t.j), eq(planner__core__indexmerge_path.t.a, 10)) └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo drop table if exists t; create table t(a int, j json, index kj((cast(j as signed array)))); @@ -545,13 +545,13 @@ IndexMerge 10.00 root type: union ALTER TABLE t ALTER INDEX kj INVISIBLE; explain format='brief' select /*+ use_index(t, kj) */ * from t where (1 member of (j)); id estRows task access object operator info -TableReader 8000.00 root data:Selection -└─Selection 8000.00 cop[tikv] json_memberof(cast(1, json BINARY), planner__core__indexmerge_path.t.j) +TableReader 10.00 root data:Selection +└─Selection 10.00 cop[tikv] json_memberof(cast(1, json BINARY), planner__core__indexmerge_path.t.j) └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo explain format='brief' select /*+ use_index_merge(t, kj) */ * from t where (1 member of (j)); id estRows task access object operator info -TableReader 8000.00 root data:Selection -└─Selection 8000.00 cop[tikv] json_memberof(cast(1, json BINARY), planner__core__indexmerge_path.t.j) +TableReader 10.00 root data:Selection +└─Selection 10.00 cop[tikv] json_memberof(cast(1, json BINARY), planner__core__indexmerge_path.t.j) └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo ALTER TABLE t ALTER INDEX kj VISIBLE; explain format='brief' select /*+ use_index(t, kj) */ * from t where (1 member of (j)); @@ -656,3 +656,19 @@ a 1 select /*+ use_index_merge(t, j0_0) */ a from t where ('1' member of (j0->'$.path0')); a +drop table if exists t; +create table t(a int, b int, c int, d json, index iad(a, (cast(d->'$.b' as signed array)))); +insert into t value(1,1,1, '{"b":[1,2,3,4]}'); +insert into t value(2,2,2, '{"b":[3,4,5,6]}'); +set tidb_analyze_version=2; +analyze table t; +explain select * from t use index (iad) where a = 1; +id estRows task access object operator info +IndexMerge_7 1.00 root type: union +├─IndexRangeScan_5(Build) 4.00 cop[tikv] table:t, index:iad(a, cast(json_extract(`d`, _utf8mb4'$.b') as signed array)) range:[1,1], keep order:false +└─TableRowIDScan_6(Probe) 1.00 cop[tikv] table:t keep order:false +explain select * from t use index (iad) where a = 1 and (2 member of (d->'$.b')); +id estRows task access object operator info +IndexMerge_7 1.00 root type: union +├─IndexRangeScan_5(Build) 1.00 cop[tikv] table:t, index:iad(a, cast(json_extract(`d`, _utf8mb4'$.b') as signed array)) range:[1 2,1 2], keep order:false, stats:partial[d:unInitialized] +└─TableRowIDScan_6(Probe) 1.00 cop[tikv] table:t keep order:false, stats:partial[d:unInitialized] diff --git a/tests/integrationtest/t/planner/core/indexmerge_path.test b/tests/integrationtest/t/planner/core/indexmerge_path.test index 6d82ad3981aa0..0cd63b4bcd296 100644 --- a/tests/integrationtest/t/planner/core/indexmerge_path.test +++ b/tests/integrationtest/t/planner/core/indexmerge_path.test @@ -229,4 +229,11 @@ select /*+ no_index_merge() */ a from t where ('1' member of (j0->'$.path0')); select /*+ use_index_merge(t, j0_0) */ a from t where (1 member of (j0->'$.path0')); select /*+ use_index_merge(t, j0_0) */ a from t where ('1' member of (j0->'$.path0')); - +drop table if exists t; +create table t(a int, b int, c int, d json, index iad(a, (cast(d->'$.b' as signed array)))); +insert into t value(1,1,1, '{"b":[1,2,3,4]}'); +insert into t value(2,2,2, '{"b":[3,4,5,6]}'); +set tidb_analyze_version=2; +analyze table t; +explain select * from t use index (iad) where a = 1; +explain select * from t use index (iad) where a = 1 and (2 member of (d->'$.b'));