From a22c66e1090599fd21ea53f5060c2660b025e023 Mon Sep 17 00:00:00 2001 From: Song Gao Date: Mon, 13 Feb 2023 14:04:08 +0800 Subject: [PATCH] This is an automated cherry-pick of #41319 Signed-off-by: ti-chi-bot --- executor/index_advise_test.go | 134 +++++++++++++++++++++++++ planner/core/exhaust_physical_plans.go | 73 ++++++++++++++ sessionctx/variable/session.go | 51 ++++++++++ sessionctx/variable/sysvar.go | 30 ++++++ sessionctx/variable/tidb_vars.go | 19 ++++ 5 files changed, 307 insertions(+) diff --git a/executor/index_advise_test.go b/executor/index_advise_test.go index 3415ffe83537b..5b5838a0fc4cc 100644 --- a/executor/index_advise_test.go +++ b/executor/index_advise_test.go @@ -65,3 +65,137 @@ func TestIndexAdvise(t *testing.T) { require.Equal(t, uint64(4), ia.MaxIndexNum.PerTable) require.Equal(t, uint64(5), ia.MaxIndexNum.PerDB) } +<<<<<<< HEAD +======= + +func TestIndexJoinProjPattern(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec(`create table t1( +pnbrn_cnaps varchar(5) not null, +new_accno varchar(18) not null, +primary key(pnbrn_cnaps,new_accno) nonclustered +);`) + tk.MustExec(`create table t2( +pnbrn_cnaps varchar(5) not null, +txn_accno varchar(18) not null, +txn_dt date not null, +yn_frz varchar(1) default null +);`) + tk.MustExec(`insert into t1(pnbrn_cnaps,new_accno) values ("40001","123")`) + tk.MustExec(`insert into t2(pnbrn_cnaps, txn_accno, txn_dt, yn_frz) values ("40001","123","20221201","0");`) + + sql := `update +/*+ inl_join(a) */ +t2 b, +( +select t1.pnbrn_cnaps, +t1.new_accno +from t1 +where t1.pnbrn_cnaps = '40001' +) a +set b.yn_frz = '1' +where b.txn_dt = str_to_date('20221201', '%Y%m%d') +and b.pnbrn_cnaps = a.pnbrn_cnaps +and b.txn_accno = a.new_accno;` + rows := [][]interface{}{ + {"Update_8"}, + {"└─IndexJoin_14"}, + {" ├─TableReader_25(Build)"}, + {" │ └─Selection_24"}, + {" │ └─TableFullScan_23"}, + {" └─IndexReader_12(Probe)"}, + {" └─Selection_11"}, + {" └─IndexRangeScan_10"}, + } + tk.MustExec("set @@session.tidb_enable_inl_join_inner_multi_pattern='ON'") + tk.MustQuery("explain "+sql).CheckAt([]int{0}, rows) + rows = [][]interface{}{ + {"Update_8"}, + {"└─HashJoin_10"}, + {" ├─IndexReader_17(Build)"}, + {" │ └─IndexRangeScan_16"}, + {" └─TableReader_14(Probe)"}, + {" └─Selection_13"}, + {" └─TableFullScan_12"}, + } + tk.MustExec("set @@session.tidb_enable_inl_join_inner_multi_pattern='OFF'") + tk.MustQuery("explain "+sql).CheckAt([]int{0}, rows) + + tk.MustExec("set @@session.tidb_enable_inl_join_inner_multi_pattern='ON'") + tk.MustExec(sql) + tk.MustQuery("select yn_frz from t2").Check(testkit.Rows("1")) +} + +func TestIndexJoinSelPattern(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec(` create table tbl_miss( +id bigint(20) unsigned not null +,txn_dt date default null +,perip_sys_uuid varchar(32) not null +,rvrs_idr varchar(1) not null +,primary key(id) clustered +,key idx1 (txn_dt, perip_sys_uuid, rvrs_idr) +); +`) + tk.MustExec(`insert into tbl_miss (id,txn_dt,perip_sys_uuid,rvrs_idr) values (1,"20221201","123","1");`) + tk.MustExec(`create table tbl_src( +txn_dt date default null +,uuid varchar(32) not null +,rvrs_idr char(1) +,expd_inf varchar(5000) +,primary key(uuid,rvrs_idr) nonclustered +); +`) + tk.MustExec(`insert into tbl_src (txn_dt,uuid,rvrs_idr) values ("20221201","123","1");`) + sql := `select /*+ use_index(mis,) inl_join(src) */ + * + from tbl_miss mis + ,tbl_src src + where src.txn_dt >= str_to_date('20221201', '%Y%m%d') + and mis.id between 1 and 10000 + and mis.perip_sys_uuid = src.uuid + and mis.rvrs_idr = src.rvrs_idr + and mis.txn_dt = src.txn_dt + and ( + case when isnull(src.expd_inf) = 1 then '' + else + substr(concat_ws('',src.expd_inf,'~~'), + instr(concat_ws('',src.expd_inf,'~~'),'~~a4') + 4, + instr(substr(concat_ws('',src.expd_inf,'~~'), + instr(concat_ws('',src.expd_inf,'~~'),'~~a4') + 4, length(concat_ws('',src.expd_inf,'~~'))),'~~') -1) + end + ) != '01';` + rows := [][]interface{}{ + {"HashJoin_9"}, + {"├─TableReader_12(Build)"}, + {"│ └─Selection_11"}, + {"│ └─TableRangeScan_10"}, + {"└─Selection_13(Probe)"}, + {" └─TableReader_16"}, + {" └─Selection_15"}, + {" └─TableFullScan_14"}, + } + tk.MustExec("set @@session.tidb_enable_inl_join_inner_multi_pattern='OFF'") + tk.MustQuery("explain "+sql).CheckAt([]int{0}, rows) + rows = [][]interface{}{ + {"IndexJoin_13"}, + {"├─TableReader_25(Build)"}, + {"│ └─Selection_24"}, + {"│ └─TableRangeScan_23"}, + {"└─Selection_12(Probe)"}, + {" └─IndexLookUp_11"}, + {" ├─IndexRangeScan_8(Build)"}, + {" └─Selection_10(Probe)"}, + {" └─TableRowIDScan_9"}, + } + tk.MustExec("set @@session.tidb_enable_inl_join_inner_multi_pattern='ON'") + tk.MustQuery("explain "+sql).CheckAt([]int{0}, rows) + tk.MustQuery(sql).Check(testkit.Rows("1 2022-12-01 123 1 2022-12-01 123 1 ")) + tk.MustExec("set @@session.tidb_enable_inl_join_inner_multi_pattern='OFF'") + tk.MustQuery(sql).Check(testkit.Rows("1 2022-12-01 123 1 2022-12-01 123 1 ")) +} +>>>>>>> 982a6163a1 (sysvar: introduce variable tidb_enable_inl_join_inner_multi_pattern (#41319)) diff --git a/planner/core/exhaust_physical_plans.go b/planner/core/exhaust_physical_plans.go index 768a8c20fc0b5..6c0ff8e4e7868 100644 --- a/planner/core/exhaust_physical_plans.go +++ b/planner/core/exhaust_physical_plans.go @@ -734,6 +734,35 @@ func (p *LogicalJoin) getIndexJoinByOuterIdx(prop *property.PhysicalProperty, ou return nil } } +<<<<<<< HEAD +======= + case *LogicalProjection: + if !p.ctx.GetSessionVars().EnableINLJoinInnerMultiPattern { + return nil + } + // For now, we only allow proj with all Column expression can be the inner side of index join + for _, expr := range child.Exprs { + if _, ok := expr.(*expression.Column); !ok { + return nil + } + } + wrapper.proj = child + ds, isDataSource := wrapper.proj.Children()[0].(*DataSource) + if !isDataSource { + return nil + } + wrapper.ds = ds + case *LogicalSelection: + if !p.ctx.GetSessionVars().EnableINLJoinInnerMultiPattern { + return nil + } + wrapper.sel = child + ds, isDataSource := wrapper.sel.Children()[0].(*DataSource) + if !isDataSource { + return nil + } + wrapper.ds = ds +>>>>>>> 982a6163a1 (sysvar: introduce variable tidb_enable_inl_join_inner_multi_pattern (#41319)) } var avgInnerRowCnt float64 if outerChild.statsInfo().RowCount > 0 { @@ -1042,6 +1071,50 @@ func (p *LogicalJoin) constructInnerTableScanTask( return t } +<<<<<<< HEAD +======= +func (p *LogicalJoin) constructInnerByWrapper(wrapper *indexJoinInnerChildWrapper, child PhysicalPlan) PhysicalPlan { + if !p.ctx.GetSessionVars().EnableINLJoinInnerMultiPattern { + if wrapper.us != nil { + return p.constructInnerUnionScan(wrapper.us, child) + } + return child + } + if wrapper.us != nil { + return p.constructInnerUnionScan(wrapper.us, child) + } else if wrapper.proj != nil { + return p.constructInnerProj(wrapper.proj, child) + } else if wrapper.sel != nil { + return p.constructInnerSel(wrapper.sel, child) + } + return child +} + +func (p *LogicalJoin) constructInnerSel(sel *LogicalSelection, child PhysicalPlan) PhysicalPlan { + if sel == nil { + return child + } + physicalSel := PhysicalSelection{ + Conditions: sel.Conditions, + }.Init(sel.ctx, sel.stats, sel.blockOffset, nil) + physicalSel.SetChildren(child) + return physicalSel +} + +func (p *LogicalJoin) constructInnerProj(proj *LogicalProjection, child PhysicalPlan) PhysicalPlan { + if proj == nil { + return child + } + physicalProj := PhysicalProjection{ + Exprs: proj.Exprs, + CalculateNoDelay: proj.CalculateNoDelay, + AvoidColumnEvaluator: proj.AvoidColumnEvaluator, + }.Init(proj.ctx, proj.stats, proj.blockOffset, nil) + physicalProj.SetChildren(child) + return physicalProj +} + +>>>>>>> 982a6163a1 (sysvar: introduce variable tidb_enable_inl_join_inner_multi_pattern (#41319)) func (p *LogicalJoin) constructInnerUnionScan(us *LogicalUnionScan, reader PhysicalPlan) PhysicalPlan { if us == nil { return reader diff --git a/sessionctx/variable/session.go b/sessionctx/variable/session.go index 47d7eef068ed2..6c8722ac7e92b 100644 --- a/sessionctx/variable/session.go +++ b/sessionctx/variable/session.go @@ -1319,6 +1319,57 @@ type SessionVars struct { // StoreBatchSize indicates the batch size limit of store batch, set this field to 0 to disable store batch. StoreBatchSize int +<<<<<<< HEAD +======= + + // shardRand is used by TxnCtx, for the GetCurrentShard() method. + shardRand *rand.Rand + + // Resource group name + ResourceGroupName string + + // ProtectedTSList holds a list of timestamps that should delay GC. + ProtectedTSList protectedTSList + + // PessimisticTransactionAggressiveLocking controls whether aggressive locking for pessimistic transaction + // is enabled. + PessimisticTransactionAggressiveLocking bool + + // EnableINLJoinInnerMultiPattern indicates whether enable multi pattern for index join inner side + // For now it is not public to user + EnableINLJoinInnerMultiPattern bool +} + +// planReplayerSessionFinishedTaskKeyLen is used to control the max size for the finished plan replayer task key in session +// in order to control the used memory +const planReplayerSessionFinishedTaskKeyLen = 128 + +// AddPlanReplayerFinishedTaskKey record finished task key in session +func (s *SessionVars) AddPlanReplayerFinishedTaskKey(key replayer.PlanReplayerTaskKey) { + if len(s.PlanReplayerFinishedTaskKey) >= planReplayerSessionFinishedTaskKeyLen { + s.initializePlanReplayerFinishedTaskKey() + } + s.PlanReplayerFinishedTaskKey[key] = struct{}{} +} + +func (s *SessionVars) initializePlanReplayerFinishedTaskKey() { + s.PlanReplayerFinishedTaskKey = make(map[replayer.PlanReplayerTaskKey]struct{}, planReplayerSessionFinishedTaskKeyLen) +} + +// CheckPlanReplayerFinishedTaskKey check whether the key exists +func (s *SessionVars) CheckPlanReplayerFinishedTaskKey(key replayer.PlanReplayerTaskKey) bool { + if s.PlanReplayerFinishedTaskKey == nil { + s.initializePlanReplayerFinishedTaskKey() + return false + } + _, ok := s.PlanReplayerFinishedTaskKey[key] + return ok +} + +// IsPlanReplayerCaptureEnabled indicates whether capture or continues capture enabled +func (s *SessionVars) IsPlanReplayerCaptureEnabled() bool { + return s.EnablePlanReplayerCapture || s.EnablePlanReplayedContinuesCapture +>>>>>>> 982a6163a1 (sysvar: introduce variable tidb_enable_inl_join_inner_multi_pattern (#41319)) } // GetNewChunkWithCapacity Attempt to request memory from the chunk pool diff --git a/sessionctx/variable/sysvar.go b/sessionctx/variable/sysvar.go index 25e868b7c21f2..6c90676e92129 100644 --- a/sessionctx/variable/sysvar.go +++ b/sessionctx/variable/sysvar.go @@ -2237,6 +2237,36 @@ var defaultSysVars = []*SysVar{ return strconv.Itoa(int(TTLDeleteWorkerCount.Load())), nil }, }, +<<<<<<< HEAD +======= + {Scope: ScopeGlobal, Name: TiDBEnableResourceControl, Value: BoolToOnOff(DefTiDBEnableResourceControl), Type: TypeBool, SetGlobal: func(ctx context.Context, vars *SessionVars, s string) error { + if TiDBOptOn(s) != EnableResourceControl.Load() { + EnableResourceControl.Store(TiDBOptOn(s)) + (*SetGlobalResourceControl.Load())(TiDBOptOn(s)) + logutil.BgLogger().Info("set resource control", zap.Bool("enable", TiDBOptOn(s))) + } + return nil + }, GetGlobal: func(ctx context.Context, vars *SessionVars) (string, error) { + return BoolToOnOff(EnableResourceControl.Load()), nil + }}, + {Scope: ScopeGlobal | ScopeSession, Name: TiDBPessimisticTransactionAggressiveLocking, Value: BoolToOnOff(DefTiDBPessimisticTransactionAggressiveLocking), Type: TypeBool, SetSession: func(s *SessionVars, val string) error { + s.PessimisticTransactionAggressiveLocking = TiDBOptOn(val) + return nil + }}, + {Scope: ScopeGlobal | ScopeSession, Name: TiDBEnablePlanCacheForParamLimit, Value: BoolToOnOff(DefTiDBEnablePlanCacheForParamLimit), Type: TypeBool, SetSession: func(s *SessionVars, val string) error { + s.EnablePlanCacheForParamLimit = TiDBOptOn(val) + return nil + }}, + {Scope: ScopeGlobal | ScopeSession, Name: TiDBEnableINLJoinInnerMultiPattern, Value: BoolToOnOff(false), Type: TypeBool, + SetSession: func(s *SessionVars, val string) error { + s.EnableINLJoinInnerMultiPattern = TiDBOptOn(val) + return nil + }, + GetSession: func(s *SessionVars) (string, error) { + return BoolToOnOff(s.EnableINLJoinInnerMultiPattern), nil + }, + }, +>>>>>>> 982a6163a1 (sysvar: introduce variable tidb_enable_inl_join_inner_multi_pattern (#41319)) } // FeedbackProbability points to the FeedbackProbability in statistics package. diff --git a/sessionctx/variable/tidb_vars.go b/sessionctx/variable/tidb_vars.go index a5758ced155ac..3185089b76214 100644 --- a/sessionctx/variable/tidb_vars.go +++ b/sessionctx/variable/tidb_vars.go @@ -786,6 +786,25 @@ const ( // TiDBStoreBatchSize indicates the batch size of coprocessor in the same store. TiDBStoreBatchSize = "tidb_store_batch_size" +<<<<<<< HEAD +======= + + // MppExchangeCompressionMode indicates the data compression method in mpp exchange operator + MppExchangeCompressionMode = "mpp_exchange_compression_mode" + + // MppVersion indicates the mpp-version used to build mpp plan + MppVersion = "mpp_version" + + // TiDBPessimisticTransactionAggressiveLocking controls whether aggressive locking for pessimistic transaction + // is enabled. + TiDBPessimisticTransactionAggressiveLocking = "tidb_pessimistic_txn_aggressive_locking" + + // TiDBEnablePlanCacheForParamLimit controls whether prepare statement with parameterized limit can be cached + TiDBEnablePlanCacheForParamLimit = "tidb_enable_plan_cache_for_param_limit" + + // TiDBEnableINLJoinInnerMultiPattern indicates whether enable multi pattern for inner side of inl join + TiDBEnableINLJoinInnerMultiPattern = "tidb_enable_inl_join_inner_multi_pattern" +>>>>>>> 982a6163a1 (sysvar: introduce variable tidb_enable_inl_join_inner_multi_pattern (#41319)) ) // TiDB vars that have only global scope