From a00f0fe28e92c86c1969812347303c7b36d5ad2b Mon Sep 17 00:00:00 2001 From: tiancaiamao Date: Wed, 30 Nov 2022 16:40:01 +0800 Subject: [PATCH 1/4] autoid_service,meta/autoid: distinguish retriable error from non-retriable error (#39215) close pingcap/tidb#39214 --- DEPS.bzl | 4 +-- autoid_service/BUILD.bazel | 1 + autoid_service/autoid.go | 26 ++++++++++++++++--- executor/autoidtest/BUILD.bazel | 1 + executor/autoidtest/autoid_test.go | 13 ++++++++++ go.mod | 2 +- go.sum | 4 +-- meta/autoid/BUILD.bazel | 3 ++- meta/autoid/autoid_service.go | 40 ++++++++++++++++++++++-------- server/http_status.go | 4 ++- 10 files changed, 76 insertions(+), 22 deletions(-) diff --git a/DEPS.bzl b/DEPS.bzl index 351129c23b51f..c6ff78e509ea0 100644 --- a/DEPS.bzl +++ b/DEPS.bzl @@ -2915,8 +2915,8 @@ def go_deps(): name = "com_github_pingcap_kvproto", build_file_proto_mode = "disable_global", importpath = "github.com/pingcap/kvproto", - sum = "h1:LzIZsQpXQlj8yF7+yvyOg680OaPq7bmPuDuszgXfHsw=", - version = "v0.0.0-20221129023506-621ec37aac7a", + sum = "h1:46ZD6xzQWJ8Jkeal/U7SqkX030Mgs8DAn6QV/9zbqOQ=", + version = "v0.0.0-20221130022225-6c56ac56fe5f", ) go_repository( name = "com_github_pingcap_log", diff --git a/autoid_service/BUILD.bazel b/autoid_service/BUILD.bazel index c973afd85c4e7..6f1a13742ca80 100644 --- a/autoid_service/BUILD.bazel +++ b/autoid_service/BUILD.bazel @@ -15,6 +15,7 @@ go_library( "//util/logutil", "//util/mathutil", "@com_github_pingcap_errors//:errors", + "@com_github_pingcap_failpoint//:failpoint", "@com_github_pingcap_kvproto//pkg/autoid", "@io_etcd_go_etcd_client_v3//:client", "@org_golang_google_grpc//:grpc", diff --git a/autoid_service/autoid.go b/autoid_service/autoid.go index fd0feae76f6bf..1a4d2b426263e 100644 --- a/autoid_service/autoid.go +++ b/autoid_service/autoid.go @@ -22,6 +22,7 @@ import ( "time" "github.com/pingcap/errors" + "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/autoid" "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/kv" @@ -401,9 +402,16 @@ func (s *Service) getAlloc(dbID, tblID int64, isUnsigned bool) *autoIDValue { func (s *Service) allocAutoID(ctx context.Context, req *autoid.AutoIDRequest) (*autoid.AutoIDResponse, error) { if s.leaderShip != nil && !s.leaderShip.IsOwner() { + logutil.BgLogger().Info("[autoid service] Alloc AutoID fail, not leader") return nil, errors.New("not leader") } + failpoint.Inject("mockErr", func(val failpoint.Value) { + if val.(bool) { + failpoint.Return(nil, errors.New("mock reload failed")) + } + }) + val := s.getAlloc(req.DbID, req.TblID, req.IsUnsigned) if req.N == 0 { @@ -426,10 +434,13 @@ func (s *Service) allocAutoID(ctx context.Context, req *autoid.AutoIDRequest) (* val.end = currentEnd return nil }) + if err != nil { + return &autoid.AutoIDResponse{Errmsg: []byte(err.Error())}, nil + } return &autoid.AutoIDResponse{ Min: currentEnd, Max: currentEnd, - }, err + }, nil } val.Lock() @@ -443,10 +454,13 @@ func (s *Service) allocAutoID(ctx context.Context, req *autoid.AutoIDRequest) (* min, max, err = val.alloc4Signed(ctx, s.store, req.DbID, req.TblID, req.IsUnsigned, req.N, req.Increment, req.Offset) } + if err != nil { + return &autoid.AutoIDResponse{Errmsg: []byte(err.Error())}, nil + } return &autoid.AutoIDResponse{ Min: min, Max: max, - }, err + }, nil } func (alloc *autoIDValue) forceRebase(ctx context.Context, store kv.Storage, dbID, tblID, requiredBase int64, isUnsigned bool) error { @@ -478,6 +492,7 @@ func (alloc *autoIDValue) forceRebase(ctx context.Context, store kv.Storage, dbI // req.N = 0 is handled specially, it is used to return the current auto ID value. func (s *Service) Rebase(ctx context.Context, req *autoid.RebaseRequest) (*autoid.RebaseResponse, error) { if s.leaderShip != nil && !s.leaderShip.IsOwner() { + logutil.BgLogger().Info("[autoid service] Rebase() fail, not leader") return nil, errors.New("not leader") } @@ -485,7 +500,7 @@ func (s *Service) Rebase(ctx context.Context, req *autoid.RebaseRequest) (*autoi if req.Force { err := val.forceRebase(ctx, s.store, req.DbID, req.TblID, req.Base, req.IsUnsigned) if err != nil { - return nil, errors.Trace(err) + return &autoid.RebaseResponse{Errmsg: []byte(err.Error())}, nil } } @@ -495,5 +510,8 @@ func (s *Service) Rebase(ctx context.Context, req *autoid.RebaseRequest) (*autoi } else { err = val.rebase4Signed(ctx, s.store, req.DbID, req.TblID, req.Base) } - return &autoid.RebaseResponse{}, err + if err != nil { + return &autoid.RebaseResponse{Errmsg: []byte(err.Error())}, nil + } + return &autoid.RebaseResponse{}, nil } diff --git a/executor/autoidtest/BUILD.bazel b/executor/autoidtest/BUILD.bazel index dd467855dc8d9..0f5bf6c434f91 100644 --- a/executor/autoidtest/BUILD.bazel +++ b/executor/autoidtest/BUILD.bazel @@ -17,6 +17,7 @@ go_test( "//sessionctx/variable", "//testkit", "//testkit/testutil", + "@com_github_pingcap_failpoint//:failpoint", "@com_github_stretchr_testify//require", "@com_github_tikv_client_go_v2//tikv", "@org_uber_go_goleak//:goleak", diff --git a/executor/autoidtest/autoid_test.go b/executor/autoidtest/autoid_test.go index f52f63b9c59a2..fd74018b162fb 100644 --- a/executor/autoidtest/autoid_test.go +++ b/executor/autoidtest/autoid_test.go @@ -21,6 +21,7 @@ import ( "strings" "testing" + "github.com/pingcap/failpoint" ddltestutil "github.com/pingcap/tidb/ddl/testutil" "github.com/pingcap/tidb/parser/mysql" "github.com/pingcap/tidb/session" @@ -736,3 +737,15 @@ func TestAlterTableAutoIDCache(t *testing.T) { _, err = tk.Exec("alter table t_473 auto_id_cache = 1") require.Error(t, err) } + +func TestMockAutoIDServiceError(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("USE test;") + tk.MustExec("create table t_mock_err (id int key auto_increment) auto_id_cache 1") + + failpoint.Enable("github.com/pingcap/tidb/autoid_service/mockErr", `return(true)`) + defer failpoint.Disable("github.com/pingcap/tidb/autoid_service/mockErr") + // Cover a bug that the autoid client retry non-retryable errors forever cause dead loop. + tk.MustExecToErr("insert into t_mock_err values (),()") // mock error, instead of dead loop +} diff --git a/go.mod b/go.mod index 502ca90e47ea9..8c68ec0d5137c 100644 --- a/go.mod +++ b/go.mod @@ -68,7 +68,7 @@ require ( github.com/pingcap/errors v0.11.5-0.20220729040631-518f63d66278 github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 github.com/pingcap/fn v0.0.0-20200306044125-d5540d389059 - github.com/pingcap/kvproto v0.0.0-20221129023506-621ec37aac7a + github.com/pingcap/kvproto v0.0.0-20221130022225-6c56ac56fe5f github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c github.com/pingcap/sysutil v0.0.0-20220114020952-ea68d2dbf5b4 github.com/pingcap/tidb/parser v0.0.0-20211011031125-9b13dc409c5e diff --git a/go.sum b/go.sum index dc6888f53d3b2..31210331beeb6 100644 --- a/go.sum +++ b/go.sum @@ -779,8 +779,8 @@ github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 h1:surzm05a8C9dN github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw= github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= github.com/pingcap/kvproto v0.0.0-20221026112947-f8d61344b172/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= -github.com/pingcap/kvproto v0.0.0-20221129023506-621ec37aac7a h1:LzIZsQpXQlj8yF7+yvyOg680OaPq7bmPuDuszgXfHsw= -github.com/pingcap/kvproto v0.0.0-20221129023506-621ec37aac7a/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= +github.com/pingcap/kvproto v0.0.0-20221130022225-6c56ac56fe5f h1:46ZD6xzQWJ8Jkeal/U7SqkX030Mgs8DAn6QV/9zbqOQ= +github.com/pingcap/kvproto v0.0.0-20221130022225-6c56ac56fe5f/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20200511115504-543df19646ad/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM= diff --git a/meta/autoid/BUILD.bazel b/meta/autoid/BUILD.bazel index 7490d65691e4c..50e53258f305b 100644 --- a/meta/autoid/BUILD.bazel +++ b/meta/autoid/BUILD.bazel @@ -12,6 +12,7 @@ go_library( visibility = ["//visibility:public"], deps = [ "//autoid_service", + "//config", "//errno", "//kv", "//meta", @@ -31,7 +32,7 @@ go_library( "@com_github_tikv_client_go_v2//util", "@io_etcd_go_etcd_client_v3//:client", "@org_golang_google_grpc//:grpc", - "@org_golang_google_grpc//credentials/insecure", + "@org_golang_google_grpc//credentials", "@org_uber_go_zap//:zap", ], ) diff --git a/meta/autoid/autoid_service.go b/meta/autoid/autoid_service.go index e524397e0041f..0b0f4946b3d0c 100644 --- a/meta/autoid/autoid_service.go +++ b/meta/autoid/autoid_service.go @@ -23,12 +23,13 @@ import ( "github.com/opentracing/opentracing-go" "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/autoid" + "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/metrics" "github.com/pingcap/tidb/util/logutil" clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/credentials" ) var _ Allocator = &singlePointAlloc{} @@ -77,13 +78,23 @@ func (d *clientDiscover) GetClient(ctx context.Context) (autoid.AutoIDAllocClien if err != nil { return nil, errors.Trace(err) } - if len(resp.Kvs) == 0 { return nil, errors.New("autoid service leader not found") } addr := string(resp.Kvs[0].Value) - grpcConn, err := grpc.Dial(addr, grpc.WithTransportCredentials(insecure.NewCredentials())) + opt := grpc.WithInsecure() + security := config.GetGlobalConfig().Security + if len(security.ClusterSSLCA) != 0 { + clusterSecurity := security.ClusterSecurity() + tlsConfig, err := clusterSecurity.ToTLSConfig() + if err != nil { + return nil, errors.Trace(err) + } + opt = grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig)) + } + logutil.BgLogger().Info("[autoid client] connect to leader", zap.String("addr", addr)) + grpcConn, err := grpc.Dial(addr, opt) if err != nil { return nil, errors.Trace(err) } @@ -128,11 +139,14 @@ retry: if err != nil { if strings.Contains(err.Error(), "rpc error") { time.Sleep(backoffDuration) - sp.resetConn() + sp.resetConn(err) goto retry } return 0, 0, errors.Trace(err) } + if len(resp.Errmsg) != 0 { + return 0, 0, errors.Trace(errors.New(string(resp.Errmsg))) + } du := time.Since(start) metrics.AutoIDReqDuration.Observe(du.Seconds()) @@ -142,7 +156,9 @@ retry: const backoffDuration = 200 * time.Millisecond -func (sp *singlePointAlloc) resetConn() { +func (sp *singlePointAlloc) resetConn(reason error) { + logutil.BgLogger().Info("[autoid client] reset grpc connection", + zap.String("reason", reason.Error())) var grpcConn *grpc.ClientConn sp.mu.Lock() grpcConn = sp.mu.ClientConn @@ -153,9 +169,7 @@ func (sp *singlePointAlloc) resetConn() { if grpcConn != nil { err := grpcConn.Close() if err != nil { - logutil.BgLogger().Warn("[autoid client] AllocAutoID grpc error, reconnect", zap.Error(err)) - } else { - logutil.BgLogger().Info("[autoid client] AllocAutoID grpc error, reconnect") + logutil.BgLogger().Warn("[autoid client] close grpc connection error", zap.Error(err)) } } } @@ -186,7 +200,8 @@ retry: if err != nil { return errors.Trace(err) } - _, err = cli.Rebase(ctx, &autoid.RebaseRequest{ + var resp *autoid.RebaseResponse + resp, err = cli.Rebase(ctx, &autoid.RebaseRequest{ DbID: sp.dbID, TblID: sp.tblID, Base: newBase, @@ -196,13 +211,16 @@ retry: if err != nil { if strings.Contains(err.Error(), "rpc error") { time.Sleep(backoffDuration) - sp.resetConn() + sp.resetConn(err) goto retry } return errors.Trace(err) } + if len(resp.Errmsg) != 0 { + return errors.Trace(errors.New(string(resp.Errmsg))) + } sp.lastAllocated = newBase - return err + return nil } // ForceRebase set the next global auto ID to newBase. diff --git a/server/http_status.go b/server/http_status.go index 5f02e1762d4ac..2f31fa4a423d2 100644 --- a/server/http_status.go +++ b/server/http_status.go @@ -477,7 +477,9 @@ func (s *Server) startStatusServerAndRPCServer(serverMux *http.ServeMux) { logutil.BgLogger().Error("tikv store not etcd background", zap.Error(err)) break } - service := autoid.New(s.statusListener.Addr().String(), etcdAddr, store, ebd.TLSConfig()) + selfAddr := s.statusListener.Addr().String() + service := autoid.New(selfAddr, etcdAddr, store, ebd.TLSConfig()) + logutil.BgLogger().Info("register auto service at", zap.String("addr", selfAddr)) pb.RegisterAutoIDAllocServer(grpcServer, service) s.autoIDService = service break From 31d943ffdffa0183da6675c16235bf1f85fbadc2 Mon Sep 17 00:00:00 2001 From: Chengpeng Yan <41809508+Reminiscent@users.noreply.github.com> Date: Wed, 30 Nov 2022 17:36:01 +0800 Subject: [PATCH 2/4] Design: add the design doc for view hint (#39313) ref pingcap/tidb#37887 --- docs/design/2022-11-22-view-hint.md | 134 ++++++++++++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100644 docs/design/2022-11-22-view-hint.md diff --git a/docs/design/2022-11-22-view-hint.md b/docs/design/2022-11-22-view-hint.md new file mode 100644 index 0000000000000..b780208a13f45 --- /dev/null +++ b/docs/design/2022-11-22-view-hint.md @@ -0,0 +1,134 @@ +# View Hint Design Doc +- Author: Reminiscent +- Tracking Issue: https://github.com/pingcap/tidb/issues/37887 + +## Background +Hints that specify a table generally refer to tables in the DELETE, SELECT, or UPDATE query block in which the hint occurs, not to tables inside any views referenced by the statement. So we introduce the view hint to specify the table in view instead of embedding the hint in the view. + +In Oracle, there are three ways to use the global hint. (Node: the `{}` part is only used for explanation) +```SQL +CREATE OR REPLACE VIEW v AS +SELECT {SEL$2} * from e1 join (select {SEL$3} * from e3) e2 on e1.a = e2.a; + +SELECT {SEL$1} * FROM v; + +A. SELECT /*+ INDEX(v.e2.e3 idx) */ * FROM v; // /*+ INDEX(@SEL$1 v.e2.e3 idx) */ + +B. SELECT /*+ INDEX(@SEL$2 e2.e3 idx) */ * FROM v; + +C. SELECT /*+ INDEX(@SEL$3 e3 idx) */ * FROM v; +``` + +Compared with TiDB, Oracle has two differences: +1. Oracle can use `alias.table` to represent in subquery, such as e2.e3. Besides, TiDB can use `db.table` to represent a table. +2. The count for query block number treats view like a subquery, which means the select parts in view are counted. + +Based on the difference, there are some reasons why TiDB can not just use the grammar from Oracle: +1. Compatibility + 1. Grammar. + 1. We can not use the `alias.table` to represent in subquery, such as e2.e3. + 2. We can use `db.table` to represent a table. So if we want to use the `view.table` to represent a table in view, we should change the grammar or it will conflict with db.table. + 2. The count for the query block. + 1. Previously, the query block in view would not be counted. But now, if we take the view into consideration, it will change the origin count. For example, in the following part. The count of the query block for the `select a from t1` will be changed from `@SEL_2` to `@SEL_3`. So if we use the query block related hints for this part, it will be invalid or represent the content in the view. + +```SQL +CREATE OR REPLACE VIEW v AS +SELECT {SEL$2} * FROM t; + +SELECT {SEL$1} * FROM v JOIN (select {SEL$3} a from t1) t2 on v.a = t2.a; +``` + +So based on the above reasons, we should introduce another way to let hint take effect in the view. + +## Detailed Design +### How does origin hint framework work? +1. Parser: parse the sql text and get the basic information about the hint. Handle hint syntax error in this phase. +2. Optimizer: + 1. Divide and mark the query block. Besides, group the hints in the same query blocks. + 2. In the plan builder phase, when we try to build select. We will handle the hints in the current query block. Including doing some simple checks and building the hints structure which can be used by planner. + 3. When we build some logical operators, we will use the hints which belongs to the current query block. And tt will use the table in hint to match the table in the plan node. For example, when we build the `DataSource` operator, it will generate the possible access path based on the index hints. When we build the `Aggregation` operator, it will set the aggregation algorithm based on the agg hints. And for the `Join` operator, it will store the hint in the join node and use the hint information in the physical optimization phase. The warning about which table is not used in the hint will be recorded in this phase. + 4. Use the hint information in the physical optimization phase to determine which physical algorithm should be used. And if the hint can not take effect, it will report warning. For example, if the join can not use the index join, but we set the index join hint in the sql text. It will report related warnings. + +### View Hint Design +Based on the goal and current infrastructure for hint. I extend the current usage of the qb_name hint to a bigger scope to support the view hint. + +An example to show the usage of the current `qb_name` hint. +```SQL +select /*+ stream_agg(@qb) merge_join(t1@qb)*/ * from (select /*+ qb_name(qb) */ count(*) from t1 join t2 on t1.a = t2.a) tt; +``` +1. First, we define the name for some query blocks. +2. Then we can use the query block name to represent the query block. + +Based on the meaning of `qb_name` hint now, we can expand it to support the view. The basic idea is the same here. We define the query block name in the view first. And then we can use the query block name to represent the contents in the view. Now the grammar is expanded from +`qb_name(name)` in the query block which you want to rename +To +`qb_name(name, viewName@queryBlockNum . {viewName}@queryBlockNum . ...)` in the first query block to represent any query block. Besides, we will reset the count for query block in every view. It means, for every view, it always counts from 1 and it will not effect the outer part. +For example: +```SQL +create table t(a int, b int); +create table t1(a int, b int); +create table t2(a int, b int); + +create view v as select {@SEL_1}{5} t.a, t.b from t join (select {@SEL_2}{6} t1.a from t1 join t2 on t1.b=t2.b) tt on t.a = tt.a; + +create view v1 as select {@SEL_1}{3} t.a, t.b from t join (select {@SEL_2}{4} from t1 join v on t1.b=v.b) tt on t.a = tt.a; + +create view v2 as select {@SEL_1}{1} t.a, t.b from t join (select {@SEL_2}{2} t1.a from t1 join v1 join v3 on t1.b=v1.b) tt on t.a = tt.a; + +select {@SEL_1} * from v2; + +/* We can use the following part to represent the {1} - {6} */ +1: qb_name(v2_sel1, v2@sel_1 . @sel_1) +2: qb_name(v2_sel2, v2@sel_1 . @sel_2) +3: qb_name(v1_sel1, v2@sel_1 . v1@sel_2 . @sel_1) +4: qb_name(v1_sel2, v2@sel_1 . v1@sel_2 . @sel_2) +5: qb_name(v_sel1, v2@sel_1 . v1@sel_2 . v@sel_2 . @sel_1) +6: qb_name(v_sel2, v2@sel_1 . v1@sel_2 . v@sel_2 . @sel_2) +``` +Take the previous as example: +```SQL +CREATE OR REPLACE VIEW v AS +SELECT * from e1 join (select count(*) from e3) e2 on e1.a = e2.a; + + +/* In Oracle */ +A1. SELECT /*+ INDEX(v.e2.e3 idx) */ * FROM v; + +A2. SELECT /*+ INDEX(@SEL$1 v.e2.e3 idx) */ * FROM v; + +B. SELECT /*+ INDEX(@SEL$2 e2.e3 idx) */ * FROM v; + +C. SELECT /*+ INDEX(@SEL$3 e3 idx) */ * FROM v; + +/* In TiDB */ +SELECT /*+ qb_name(viewSub, v@sel_1 . @sel_2) use_index(e3@viewSub, idx) hash_agg(viewSub) */ * FROM v; +``` + +### Implementation +Parser part is easy to implement. Just to expand the origin `qb_name` hint grammar. The only problem maybe is how to express the nested view(use dot or blank or something else). + +For the planner part: +1. At the beginning of the optimization, we should handle the query block name hint for view and the other method hints for view. And group these hints based on the query block name. +2. When we try to build the data source from the view, we have to traverse all of the query blocks for views. Check whether the view name in hint can match the data source or not. If there are some hints that can match, we pass it to the `buildDataSourceFromView`. +3. When we try to build the view plan, we first handle the hints which are passed by the caller. Distinguish which hints belong to the current view and which belongs to the nested view. If the hint belongs to the current view, we transform the hint to the normal hint. If the hints belong to the nested view. Then we will do the same thing, like step2. + +Besides the planner part, we need support to show the query block for a sql to increase usability. The user can copy the result and use it in hint directly. + +### Support Scope +1. We can support almost all physical algorithm's hints. Like join hints/ agg hints/ index etc. +2. Do not support the leading hints which may be across the view. But we can support the leading hint in the same view. + +### Pros and Cons +Pros: +1. No compatibility problems. Just expand the usage of the existing hint. +2. It is easier to implement. It can use the origin hints' infrastructure as much as possible. +3. It can support almost all the hints which can take effect in the query block. Oracle can only support the join order, join method and access path hints. + +Cons: +1. It may not be easy to write the query block name hint for a view. +2. The user should define the query block name hint first. + +## Reference +[Oracle Global Hint](https://docs.oracle.com/cd/E18283_01/server.112/e16638/hintsref.htm#i27644) + + From 8e4748a21125e3d6fc2a686c9670c2deedf1d471 Mon Sep 17 00:00:00 2001 From: Xiaoju Wu Date: Wed, 30 Nov 2022 18:22:01 +0800 Subject: [PATCH 3/4] *: make auto-analyze concurrency configurable (#37078) close pingcap/tidb#37183 --- executor/adapter.go | 16 ++++++++++++++-- sessionctx/variable/sysvar.go | 2 ++ sessionctx/variable/tidb_vars.go | 6 ++++++ 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/executor/adapter.go b/executor/adapter.go index a7942f2638a02..c047c2beaa05c 100644 --- a/executor/adapter.go +++ b/executor/adapter.go @@ -469,8 +469,20 @@ func (a *ExecStmt) Exec(ctx context.Context) (_ sqlexec.RecordSet, err error) { if !ok { oriIso = "REPEATABLE-READ" } - terror.Log(sctx.GetSessionVars().SetSystemVar(variable.TiDBBuildStatsConcurrency, "1")) - sctx.GetSessionVars().SetDistSQLScanConcurrency(1) + autoConcurrency, err1 := sctx.GetSessionVars().GetSessionOrGlobalSystemVar(ctx, variable.TiDBAutoBuildStatsConcurrency) + terror.Log(err1) + if err1 == nil { + terror.Log(sctx.GetSessionVars().SetSystemVar(variable.TiDBBuildStatsConcurrency, autoConcurrency)) + } + sVal, err2 := sctx.GetSessionVars().GetSessionOrGlobalSystemVar(ctx, variable.TiDBSysProcScanConcurrency) + terror.Log(err2) + if err2 == nil { + concurrency, err3 := strconv.ParseInt(sVal, 10, 64) + terror.Log(err3) + if err3 == nil { + sctx.GetSessionVars().SetDistSQLScanConcurrency(int(concurrency)) + } + } sctx.GetSessionVars().SetIndexSerialScanConcurrency(1) terror.Log(sctx.GetSessionVars().SetSystemVar(variable.TxnIsolation, ast.ReadCommitted)) defer func() { diff --git a/sessionctx/variable/sysvar.go b/sessionctx/variable/sysvar.go index 1a641811ac20f..5af72b6ae21fb 100644 --- a/sessionctx/variable/sysvar.go +++ b/sessionctx/variable/sysvar.go @@ -1122,6 +1122,8 @@ var defaultSysVars = []*SysVar{ }, GetGlobal: func(_ context.Context, s *SessionVars) (string, error) { return BoolToOnOff(EnableTmpStorageOnOOM.Load()), nil }}, + {Scope: ScopeGlobal, Name: TiDBAutoBuildStatsConcurrency, Value: strconv.Itoa(DefTiDBAutoBuildStatsConcurrency), Type: TypeInt, MinValue: 1, MaxValue: MaxConfigurableConcurrency}, + {Scope: ScopeGlobal, Name: TiDBSysProcScanConcurrency, Value: strconv.Itoa(DefTiDBSysProcScanConcurrency), Type: TypeInt, MinValue: 1, MaxValue: MaxConfigurableConcurrency}, {Scope: ScopeGlobal, Name: TiDBMemoryUsageAlarmRatio, Value: strconv.FormatFloat(DefMemoryUsageAlarmRatio, 'f', -1, 64), Type: TypeFloat, MinValue: 0.0, MaxValue: 1.0, SetGlobal: func(_ context.Context, s *SessionVars, val string) error { MemoryUsageAlarmRatio.Store(tidbOptFloat64(val, DefMemoryUsageAlarmRatio)) return nil diff --git a/sessionctx/variable/tidb_vars.go b/sessionctx/variable/tidb_vars.go index 6cad5f23b623d..faf116a2632ae 100644 --- a/sessionctx/variable/tidb_vars.go +++ b/sessionctx/variable/tidb_vars.go @@ -849,6 +849,10 @@ const ( TiDBDDLEnableFastReorg = "tidb_ddl_enable_fast_reorg" // TiDBDDLDiskQuota used to set disk quota for lightning add index. TiDBDDLDiskQuota = "tidb_ddl_disk_quota" + // TiDBAutoBuildStatsConcurrency is used to set the build concurrency of auto-analyze. + TiDBAutoBuildStatsConcurrency = "tidb_auto_build_stats_concurrency" + // TiDBSysProcScanConcurrency is used to set the scan concurrency of for backend system processes, like auto-analyze. + TiDBSysProcScanConcurrency = "tidb_sysproc_scan_concurrency" // TiDBServerMemoryLimit indicates the memory limit of the tidb-server instance. TiDBServerMemoryLimit = "tidb_server_memory_limit" // TiDBServerMemoryLimitSessMinSize indicates the minimal memory used of a session, that becomes a candidate for session kill. @@ -1084,6 +1088,8 @@ const ( MaxDDLReorgBatchSize int32 = 10240 MinDDLReorgBatchSize int32 = 32 MinExpensiveQueryTimeThreshold uint64 = 10 // 10s + DefTiDBAutoBuildStatsConcurrency = 1 + DefTiDBSysProcScanConcurrency = 1 DefTiDBRcWriteCheckTs = false DefTiDBForeignKeyChecks = false DefTiDBAnalyzePartitionConcurrency = 1 From f5487e3f2596cdd63ae9840606c7fff3fb539e03 Mon Sep 17 00:00:00 2001 From: crazycs Date: Wed, 30 Nov 2022 18:46:01 +0800 Subject: [PATCH 4/4] br: add foreign key test for br (#39427) --- br/pkg/gluetidb/glue.go | 2 ++ br/tests/br_foreign_key/run.sh | 55 ++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 br/tests/br_foreign_key/run.sh diff --git a/br/pkg/gluetidb/glue.go b/br/pkg/gluetidb/glue.go index 38bf4114e7694..3b1675921afd3 100644 --- a/br/pkg/gluetidb/glue.go +++ b/br/pkg/gluetidb/glue.go @@ -208,6 +208,8 @@ func (gs *tidbSession) CreateTables(ctx context.Context, tables map[string][]*mo d := domain.GetDomain(gs.se).DDL() var dbName model.CIStr + // Disable foreign key check when batch create tables. + gs.se.GetSessionVars().ForeignKeyChecks = false for db, tablesInDB := range tables { dbName = model.NewCIStr(db) queryBuilder := strings.Builder{} diff --git a/br/tests/br_foreign_key/run.sh b/br/tests/br_foreign_key/run.sh new file mode 100644 index 0000000000000..1eaaa168f7bbe --- /dev/null +++ b/br/tests/br_foreign_key/run.sh @@ -0,0 +1,55 @@ +#!/bin/sh +# +# Copyright 2022 PingCAP, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eu +DB="$TEST_NAME" + +run_sql "set @@global.tidb_enable_foreign_key=1;" +run_sql "set @@global.foreign_key_checks=1;" +run_sql "set @@foreign_key_checks=1;" +run_sql "create schema $DB;" +run_sql "create table $DB.t1 (id int key);" +run_sql "create table $DB.t2 (id int key, a int, b int, foreign key fk_1 (a) references t1(id) ON UPDATE SET NULL ON DELETE SET NULL, foreign key fk_2 (b) references t1(id) ON DELETE CASCADE ON UPDATE CASCADE);" +run_sql "insert into $DB.t1 values (1), (2), (3);" +run_sql "insert into $DB.t2 values (1, 1, 1), (2, 2, 2), (3, 3, 3);" +run_sql "update $DB.t1 set id=id+10 where id in (1, 3);" +run_sql "delete from $DB.t1 where id = 2;" + +echo "backup start..." +run_br backup db --db "$DB" -s "local://$TEST_DIR/$DB" --pd $PD_ADDR + +run_sql "drop schema $DB;" + +echo "restore start..." +run_br restore db --db $DB -s "local://$TEST_DIR/$DB" --pd $PD_ADDR + +set -x + +run_sql "select count(*) from $DB.t1;" +check_contains 'count(*): 2' + +run_sql "select count(*) from $DB.t2;" +check_contains 'count(*): 2' + +run_sql "select id, a, b from $DB.t2;" +check_contains 'id: 1' +check_contains 'id: 3' +check_contains 'a: NULL' +check_contains 'b: 11' +check_contains 'b: 13' +check_contains 'b: 14' + +run_sql "drop schema $DB"