From 39186240dda5413f93f3e86302f87fa6a51649e5 Mon Sep 17 00:00:00 2001 From: crazycs Date: Wed, 18 Jul 2018 00:18:14 +0800 Subject: [PATCH 01/40] add get all server info api --- ddl/ddl.go | 57 ++++++++++++++++++++++ ddl/mock.go | 20 ++++++++ ddl/syncer.go | 107 ++++++++++++++++++++++++++++++++++++++--- ddl/util/util.go | 7 +++ domain/domain.go | 4 ++ server/http_handler.go | 49 +++++++++++++++++++ server/http_status.go | 2 + 7 files changed, 240 insertions(+), 6 deletions(-) diff --git a/ddl/ddl.go b/ddl/ddl.go index 8c2fb80c81260..80af765c85150 100644 --- a/ddl/ddl.go +++ b/ddl/ddl.go @@ -26,6 +26,7 @@ import ( "github.com/juju/errors" "github.com/ngaut/pools" "github.com/pingcap/tidb/ast" + "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/ddl/util" "github.com/pingcap/tidb/infoschema" "github.com/pingcap/tidb/kv" @@ -207,6 +208,11 @@ type DDL interface { // SetBinlogClient sets the binlog client for DDL worker. It's exported for testing. SetBinlogClient(interface{}) + + GetServerInfo() *util.DDLServerInfo + StoreServerInfoToPD() error + GetOwnerServerInfo() (*util.DDLServerInfo, error) + GetAllServerInfo() (map[string]*util.DDLServerInfo, error) } // ddl is used to handle the statements that define the structure or schema of the database. @@ -376,6 +382,10 @@ func (d *ddl) close() { if err != nil { log.Errorf("[ddl] remove self version path failed %v", err) } + err = d.schemaSyncer.RemoveSelfServerInfo() + if err != nil { + log.Errorf("[ddl] remove self server info path failed %v", err) + } for _, worker := range d.workers { worker.close() @@ -510,6 +520,53 @@ func (d *ddl) SetBinlogClient(binlogCli interface{}) { d.binlogCli = binlogCli } +func (d *ddl) GetServerInfo() *util.DDLServerInfo { + info := &util.DDLServerInfo{} + cfg := config.GetGlobalConfig() + info.IP = cfg.Host + info.StatusPort = cfg.Status.StatusPort + info.ID = d.uuid + info.IsOwner = d.isOwner() + return info +} + +func (d *ddl) GetOwnerServerInfo() (*util.DDLServerInfo, error) { + ctx := context.Background() + ddlOwnerID, err := d.ownerManager.GetOwnerID(ctx) + if err != nil { + return nil, errors.Trace(err) + } + ownerInfo, err := d.schemaSyncer.GetDDLServerInfoFromPD(ctx, ddlOwnerID) + if err != nil { + return nil, errors.Trace(err) + } + return ownerInfo, nil +} + +func (d *ddl) GetAllServerInfo() (map[string]*util.DDLServerInfo, error) { + ctx := context.Background() + ddlOwnerID, err := d.ownerManager.GetOwnerID(ctx) + if err != nil { + return nil, errors.Trace(err) + } + AllDDLServerInfo, err := d.schemaSyncer.GetAllDDLServerInfoFromPD(ctx, ddlOwnerID) + if err != nil { + return nil, errors.Trace(err) + } + if _, ok := AllDDLServerInfo[ddlOwnerID]; ok { + AllDDLServerInfo[ddlOwnerID].IsOwner = true + } + return AllDDLServerInfo, nil +} + +func (d *ddl) StoreServerInfoToPD() error { + info := d.GetServerInfo() + ctx := context.Background() + // Owner will change , we will check ownerID to confirm which server is owner + info.IsOwner = false + return d.schemaSyncer.UpdateSelfServerInfo(ctx, info) +} + // DDL error codes. const ( codeInvalidWorker terror.ErrCode = 1 diff --git a/ddl/mock.go b/ddl/mock.go index 70562e9a496cd..083275bde550d 100644 --- a/ddl/mock.go +++ b/ddl/mock.go @@ -20,6 +20,7 @@ import ( "github.com/coreos/etcd/clientv3" "github.com/juju/errors" "github.com/pingcap/tidb/ast" + "github.com/pingcap/tidb/ddl/util" "github.com/pingcap/tidb/model" "github.com/pingcap/tidb/sessionctx" "golang.org/x/net/context" @@ -32,6 +33,7 @@ const mockCheckVersInterval = 2 * time.Millisecond type mockSchemaSyncer struct { selfSchemaVersion int64 globalVerCh chan clientv3.WatchResponse + selfServerInfo *util.DDLServerInfo } // NewMockSchemaSyncer creates a new mock SchemaSyncer. @@ -102,6 +104,24 @@ func (s *mockSchemaSyncer) OwnerCheckAllVersions(ctx context.Context, latestVer } } +func (s *mockSchemaSyncer) GetDDLServerInfoFromPD(ctx context.Context, ddlID string) (*util.DDLServerInfo, error) { + return s.selfServerInfo, nil +} + +func (s *mockSchemaSyncer) GetAllDDLServerInfoFromPD(ctx context.Context, ddlID string) (map[string]*util.DDLServerInfo, error) { + allDDLInfo := make(map[string]*util.DDLServerInfo) + allDDLInfo[s.selfServerInfo.ID] = s.selfServerInfo + return allDDLInfo, nil +} + +func (s *mockSchemaSyncer) UpdateSelfServerInfo(ctx context.Context, info *util.DDLServerInfo) error { + s.selfServerInfo = info + return nil +} +func (s *mockSchemaSyncer) RemoveSelfServerInfo() error { + return nil +} + type mockDelRange struct { } diff --git a/ddl/syncer.go b/ddl/syncer.go index 7549604d25e35..f9759144ce3bb 100644 --- a/ddl/syncer.go +++ b/ddl/syncer.go @@ -20,11 +20,14 @@ import ( "sync" "time" + "encoding/json" "github.com/coreos/etcd/clientv3" "github.com/coreos/etcd/clientv3/concurrency" "github.com/juju/errors" + "github.com/pingcap/tidb/ddl/util" "github.com/pingcap/tidb/metrics" "github.com/pingcap/tidb/owner" + "github.com/pingcap/tidb/util/hack" log "github.com/sirupsen/logrus" "golang.org/x/net/context" ) @@ -33,6 +36,8 @@ const ( // DDLAllSchemaVersions is the path on etcd that is used to store all servers current schema versions. // It's exported for testing. DDLAllSchemaVersions = "/tidb/ddl/all_schema_versions" + //DDLServerInformation store DDL server information such as IP,port and so on + DDLServerInformation = "/tidb/ddl/info" // DDLGlobalSchemaVersion is the path on etcd that is used to store the latest schema versions. // It's exported for testing. DDLGlobalSchemaVersion = "/tidb/ddl/global_schema_version" @@ -83,13 +88,22 @@ type SchemaSyncer interface { // the latest schema version. If the result is false, wait for a while and check again util the processing time reach 2 * lease. // It returns until all servers' versions are equal to the latest version or the ctx is done. OwnerCheckAllVersions(ctx context.Context, latestVer int64) error + + GetDDLServerInfoFromPD(ctx context.Context, ddlID string) (*util.DDLServerInfo, error) + + GetAllDDLServerInfoFromPD(ctx context.Context, ddlID string) (map[string]*util.DDLServerInfo, error) + + UpdateSelfServerInfo(ctx context.Context, info *util.DDLServerInfo) error + + RemoveSelfServerInfo() error } type schemaVersionSyncer struct { - selfSchemaVerPath string - etcdCli *clientv3.Client - session *concurrency.Session - mu struct { + selfSchemaVerPath string + selfServerInfoPath string + etcdCli *clientv3.Client + session *concurrency.Session + mu struct { sync.RWMutex globalVerCh clientv3.WatchChan } @@ -98,8 +112,9 @@ type schemaVersionSyncer struct { // NewSchemaSyncer creates a new SchemaSyncer. func NewSchemaSyncer(etcdCli *clientv3.Client, id string) SchemaSyncer { return &schemaVersionSyncer{ - etcdCli: etcdCli, - selfSchemaVerPath: fmt.Sprintf("%s/%s", DDLAllSchemaVersions, id), + etcdCli: etcdCli, + selfSchemaVerPath: fmt.Sprintf("%s/%s", DDLAllSchemaVersions, id), + selfServerInfoPath: fmt.Sprintf("%s/%s", DDLServerInformation, id), } } @@ -153,6 +168,86 @@ func (s *schemaVersionSyncer) Init(ctx context.Context) error { return errors.Trace(err) } +func (s *schemaVersionSyncer) GetDDLServerInfoFromPD(ctx context.Context, ddlID string) (*util.DDLServerInfo, error) { + var err error + var resp *clientv3.GetResponse + ddlPath := fmt.Sprintf("%s/%s", DDLServerInformation, ddlID) + for { + if isContextDone(ctx) { + err = errors.Trace(ctx.Err()) + return nil, err + } + + resp, err = s.etcdCli.Get(ctx, ddlPath) + if err != nil { + continue + } + if err == nil && len(resp.Kvs) > 0 { + info := &util.DDLServerInfo{} + err := json.Unmarshal(resp.Kvs[0].Value, info) + if err != nil { + return nil, err + } + return info, nil + } + time.Sleep(200 * time.Millisecond) + } +} + +func (s *schemaVersionSyncer) GetAllDDLServerInfoFromPD(ctx context.Context, ddlID string) (map[string]*util.DDLServerInfo, error) { + var err error + allDDLInfo := make(map[string]*util.DDLServerInfo) + for { + if isContextDone(ctx) { + // ctx is canceled or timeout. + err = errors.Trace(ctx.Err()) + return nil, err + } + + resp, err := s.etcdCli.Get(ctx, DDLServerInformation, clientv3.WithPrefix()) + if err != nil { + log.Infof("[syncer] get all ddl server info failed %v, continue checking.", err) + time.Sleep(200 * time.Millisecond) + continue + } + + for _, kv := range resp.Kvs { + info := &util.DDLServerInfo{} + err := json.Unmarshal(kv.Value, info) + if err != nil { + log.Infof("[syncer] get all ddl server info, ddl %s json.Unmarshal %v failed %v, continue checking.", kv.Key, kv.Value, err) + return nil, err + } + allDDLInfo[info.ID] = info + } + return allDDLInfo, nil + } +} + +func (s *schemaVersionSyncer) UpdateSelfServerInfo(ctx context.Context, info *util.DDLServerInfo) error { + infoBuf, err := json.Marshal(info) + if err != nil { + return errors.Trace(err) + } + err = s.putKV(ctx, keyOpDefaultRetryCnt, s.selfServerInfoPath, hack.String(infoBuf)) + return errors.Trace(err) +} + +func (s *schemaVersionSyncer) RemoveSelfServerInfo() error { + var err error + ctx := context.Background() + for i := 0; i < keyOpDefaultRetryCnt; i++ { + childCtx, cancel := context.WithTimeout(ctx, keyOpDefaultTimeout) + _, err = s.etcdCli.Delete(childCtx, s.selfServerInfoPath) + cancel() + if err == nil { + return nil + } + log.Warnf("[syncer] remove server info path %s failed %v no.%d", s.selfServerInfoPath, err, i) + } + return errors.Trace(err) +} + // Done implements SchemaSyncer.Done interface. func (s *schemaVersionSyncer) Done() <-chan struct{} { return s.session.Done() diff --git a/ddl/util/util.go b/ddl/util/util.go index db3e09fdb3d3f..c1efc49abb311 100644 --- a/ddl/util/util.go +++ b/ddl/util/util.go @@ -109,3 +109,10 @@ func UpdateDeleteRange(ctx sessionctx.Context, dr DelRangeTask, newStartKey, old _, err := ctx.(sqlexec.SQLExecutor).Execute(context.TODO(), sql) return errors.Trace(err) } + +type DDLServerInfo struct { + ID string `json:"ddl_id"` + IP string `json:"ip"` + StatusPort uint `json:"status_port"` + IsOwner bool `json:"is_owner,omitempty"` +} diff --git a/domain/domain.go b/domain/domain.go index 9336c09b3f3e7..eea8f9bfe46db 100644 --- a/domain/domain.go +++ b/domain/domain.go @@ -500,6 +500,10 @@ func (do *Domain) Init(ddlLease time.Duration, sysFactory func(*Domain) (pools.R if err != nil { return errors.Trace(err) } + err = do.ddl.StoreServerInfoToPD() + if err != nil { + return errors.Trace(err) + } err = do.Reload() if err != nil { return errors.Trace(err) diff --git a/server/http_handler.go b/server/http_handler.go index 6ac958a3faf3d..21e5bdaaad232 100644 --- a/server/http_handler.go +++ b/server/http_handler.go @@ -32,6 +32,7 @@ import ( "github.com/pingcap/kvproto/pkg/kvrpcpb" "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/tidb/config" + "github.com/pingcap/tidb/ddl/util" "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/infoschema" "github.com/pingcap/tidb/kv" @@ -334,6 +335,14 @@ type ddlHistoryJobHandler struct { *tikvHandlerTool } +type ddlServerInfoHandler struct { + *tikvHandlerTool +} + +type ddlAllServerInfoHandler struct { + *tikvHandlerTool +} + // valueHandle is the handler for get value. type valueHandler struct { } @@ -1249,3 +1258,43 @@ func (h *mvccTxnHandler) handleMvccGetByTxn(params map[string]string) (interface } return h.getMvccByStartTs(uint64(startTS), startKey, endKey) } + +// ServeHTTP handles request of ddl server info. +func (h ddlServerInfoHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { + session, err := session.CreateSession(h.store.(kv.Storage)) + if err != nil { + writeError(w, errors.New("create session error")) + return + } + ddl := domain.GetDomain(session.(sessionctx.Context)).DDL() + clusterInfo := make(map[string]*util.DDLServerInfo) + selfInfo := ddl.GetServerInfo() + if !selfInfo.IsOwner { + ownerInfo, err := ddl.GetOwnerServerInfo() + if err != nil { + writeError(w, errors.New("ddl server information not found")) + return + } + // ownerInfo.IsOwner is false because the server may no the owner when store the info to PD, + ownerInfo.IsOwner = true + clusterInfo[ownerInfo.ID] = ownerInfo + } + clusterInfo[selfInfo.ID] = selfInfo + writeData(w, clusterInfo) +} + +// ServeHTTP handles request of all ddl servers info. +func (h ddlAllServerInfoHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { + session, err := session.CreateSession(h.store.(kv.Storage)) + if err != nil { + writeError(w, errors.New("create session error")) + return + } + ddl := domain.GetDomain(session.(sessionctx.Context)).DDL() + clusterInfo, err := ddl.GetAllServerInfo() + if err != nil { + writeError(w, errors.New("ddl server information not found")) + return + } + writeData(w, clusterInfo) +} diff --git a/server/http_status.go b/server/http_status.go index 0c6012fa31b49..53b59270525bb 100644 --- a/server/http_status.go +++ b/server/http_status.go @@ -52,6 +52,8 @@ func (s *Server) startHTTPServer() { router.Handle("/schema/{db}/{table}", schemaHandler{tikvHandlerTool}) router.Handle("/tables/{colID}/{colTp}/{colFlag}/{colLen}", valueHandler{}) router.Handle("/ddl/history", ddlHistoryJobHandler{tikvHandlerTool}) + router.Handle("/ddl/info", ddlServerInfoHandler{tikvHandlerTool}) + router.Handle("/ddl/info/all", ddlAllServerInfoHandler{tikvHandlerTool}) if s.cfg.Store == "tikv" { // HTTP path for tikv router.Handle("/tables/{db}/{table}/regions", tableHandler{tikvHandlerTool, opTableRegions}) From 2370bb0d6eeebe4c724cb83e9138e76dc9e52957 Mon Sep 17 00:00:00 2001 From: crazycs Date: Wed, 18 Jul 2018 01:38:55 +0800 Subject: [PATCH 02/40] rename and add comment --- ddl/ddl.go | 14 +++++++++++--- ddl/mock.go | 12 +++++++++--- ddl/syncer.go | 21 +++++++++++++-------- ddl/util/util.go | 9 ++++++++- server/http_handler.go | 1 - 5 files changed, 41 insertions(+), 16 deletions(-) diff --git a/ddl/ddl.go b/ddl/ddl.go index 80af765c85150..f69ce8ceb8b17 100644 --- a/ddl/ddl.go +++ b/ddl/ddl.go @@ -39,6 +39,7 @@ import ( "github.com/pingcap/tidb/sessionctx/binloginfo" "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/terror" + "github.com/pingcap/tidb/util/printer" log "github.com/sirupsen/logrus" "github.com/twinj/uuid" "golang.org/x/net/context" @@ -209,9 +210,13 @@ type DDL interface { // SetBinlogClient sets the binlog client for DDL worker. It's exported for testing. SetBinlogClient(interface{}) + // GetServerInfo get self DDL Server static information and isOwner information. GetServerInfo() *util.DDLServerInfo + // StoreServerInfoToPD store self DDL server static information to PD when DDL server Started. StoreServerInfoToPD() error + // GetOwnerServerInfo get owner DDL Server static information from PD. GetOwnerServerInfo() (*util.DDLServerInfo, error) + // GetAllServerInfo get all DDL Server static information from PD. GetAllServerInfo() (map[string]*util.DDLServerInfo, error) } @@ -525,6 +530,8 @@ func (d *ddl) GetServerInfo() *util.DDLServerInfo { cfg := config.GetGlobalConfig() info.IP = cfg.Host info.StatusPort = cfg.Status.StatusPort + info.Version = mysql.ServerVersion + info.GitHash = printer.TiDBGitHash info.ID = d.uuid info.IsOwner = d.isOwner() return info @@ -536,10 +543,11 @@ func (d *ddl) GetOwnerServerInfo() (*util.DDLServerInfo, error) { if err != nil { return nil, errors.Trace(err) } - ownerInfo, err := d.schemaSyncer.GetDDLServerInfoFromPD(ctx, ddlOwnerID) + ownerInfo, err := d.schemaSyncer.GetServerInfoFromPD(ctx, ddlOwnerID) if err != nil { return nil, errors.Trace(err) } + ownerInfo.IsOwner = true return ownerInfo, nil } @@ -549,7 +557,7 @@ func (d *ddl) GetAllServerInfo() (map[string]*util.DDLServerInfo, error) { if err != nil { return nil, errors.Trace(err) } - AllDDLServerInfo, err := d.schemaSyncer.GetAllDDLServerInfoFromPD(ctx, ddlOwnerID) + AllDDLServerInfo, err := d.schemaSyncer.GetAllServerInfoFromPD(ctx, ddlOwnerID) if err != nil { return nil, errors.Trace(err) } @@ -562,7 +570,7 @@ func (d *ddl) GetAllServerInfo() (map[string]*util.DDLServerInfo, error) { func (d *ddl) StoreServerInfoToPD() error { info := d.GetServerInfo() ctx := context.Background() - // Owner will change , we will check ownerID to confirm which server is owner + // Owner will change , we will check ownerID to confirm which server is owner. info.IsOwner = false return d.schemaSyncer.UpdateSelfServerInfo(ctx, info) } diff --git a/ddl/mock.go b/ddl/mock.go index 083275bde550d..72d6f6e3fa50b 100644 --- a/ddl/mock.go +++ b/ddl/mock.go @@ -33,7 +33,8 @@ const mockCheckVersInterval = 2 * time.Millisecond type mockSchemaSyncer struct { selfSchemaVersion int64 globalVerCh chan clientv3.WatchResponse - selfServerInfo *util.DDLServerInfo + // selfServerInfo used to save self DDL server information. + selfServerInfo *util.DDLServerInfo } // NewMockSchemaSyncer creates a new mock SchemaSyncer. @@ -104,20 +105,25 @@ func (s *mockSchemaSyncer) OwnerCheckAllVersions(ctx context.Context, latestVer } } -func (s *mockSchemaSyncer) GetDDLServerInfoFromPD(ctx context.Context, ddlID string) (*util.DDLServerInfo, error) { +// GetServerInfoFromPD implements SchemaSyncer.GetServerInfoFromPD interface. +func (s *mockSchemaSyncer) GetServerInfoFromPD(ctx context.Context, _ string) (*util.DDLServerInfo, error) { return s.selfServerInfo, nil } -func (s *mockSchemaSyncer) GetAllDDLServerInfoFromPD(ctx context.Context, ddlID string) (map[string]*util.DDLServerInfo, error) { +// GetAllServerInfoFromPD implements SchemaSyncer.GetAllServerInfoFromPD interface. +func (s *mockSchemaSyncer) GetAllServerInfoFromPD(ctx context.Context, _ string) (map[string]*util.DDLServerInfo, error) { allDDLInfo := make(map[string]*util.DDLServerInfo) allDDLInfo[s.selfServerInfo.ID] = s.selfServerInfo return allDDLInfo, nil } +// UpdateSelfServerInfo implements SchemaSyncer.UpdateSelfServerInfo interface. func (s *mockSchemaSyncer) UpdateSelfServerInfo(ctx context.Context, info *util.DDLServerInfo) error { s.selfServerInfo = info return nil } + +// RemoveSelfServerInfo implements SchemaSyncer.RemoveSelfServerInfo interface. func (s *mockSchemaSyncer) RemoveSelfServerInfo() error { return nil } diff --git a/ddl/syncer.go b/ddl/syncer.go index f9759144ce3bb..5b68ac9a74bdc 100644 --- a/ddl/syncer.go +++ b/ddl/syncer.go @@ -89,12 +89,13 @@ type SchemaSyncer interface { // It returns until all servers' versions are equal to the latest version or the ctx is done. OwnerCheckAllVersions(ctx context.Context, latestVer int64) error - GetDDLServerInfoFromPD(ctx context.Context, ddlID string) (*util.DDLServerInfo, error) - - GetAllDDLServerInfoFromPD(ctx context.Context, ddlID string) (map[string]*util.DDLServerInfo, error) - + // GetServerInfoFromPD get the DDL_id server information from PD. + GetServerInfoFromPD(ctx context.Context, id string) (*util.DDLServerInfo, error) + // GetAllServerInfoFromPD get all DDL servers information from PD. + GetAllServerInfoFromPD(ctx context.Context, id string) (map[string]*util.DDLServerInfo, error) + // UpdateSelfServerInfo store DDL server information to PD. UpdateSelfServerInfo(ctx context.Context, info *util.DDLServerInfo) error - + // RemoveSelfServerInfo remove DDL server information from PD. RemoveSelfServerInfo() error } @@ -168,10 +169,11 @@ func (s *schemaVersionSyncer) Init(ctx context.Context) error { return errors.Trace(err) } -func (s *schemaVersionSyncer) GetDDLServerInfoFromPD(ctx context.Context, ddlID string) (*util.DDLServerInfo, error) { +// GetServerInfoFromPD implements SchemaSyncer.GetServerInfoFromPD interface. +func (s *schemaVersionSyncer) GetServerInfoFromPD(ctx context.Context, id string) (*util.DDLServerInfo, error) { var err error var resp *clientv3.GetResponse - ddlPath := fmt.Sprintf("%s/%s", DDLServerInformation, ddlID) + ddlPath := fmt.Sprintf("%s/%s", DDLServerInformation, id) for { if isContextDone(ctx) { err = errors.Trace(ctx.Err()) @@ -194,7 +196,8 @@ func (s *schemaVersionSyncer) GetDDLServerInfoFromPD(ctx context.Context, ddlID } } -func (s *schemaVersionSyncer) GetAllDDLServerInfoFromPD(ctx context.Context, ddlID string) (map[string]*util.DDLServerInfo, error) { +// GetAllServerInfoFromPD implements SchemaSyncer.GetAllServerInfoFromPD interface. +func (s *schemaVersionSyncer) GetAllServerInfoFromPD(ctx context.Context, id string) (map[string]*util.DDLServerInfo, error) { var err error allDDLInfo := make(map[string]*util.DDLServerInfo) for { @@ -224,6 +227,7 @@ func (s *schemaVersionSyncer) GetAllDDLServerInfoFromPD(ctx context.Context, ddl } } +// UpdateSelfServerInfo implements SchemaSyncer.UpdateSelfServerInfo interface. func (s *schemaVersionSyncer) UpdateSelfServerInfo(ctx context.Context, info *util.DDLServerInfo) error { infoBuf, err := json.Marshal(info) if err != nil { @@ -233,6 +237,7 @@ func (s *schemaVersionSyncer) UpdateSelfServerInfo(ctx context.Context, info *ut return errors.Trace(err) } +// RemoveSelfServerInfo implements SchemaSyncer.RemoveSelfServerInfo interface. func (s *schemaVersionSyncer) RemoveSelfServerInfo() error { var err error ctx := context.Background() diff --git a/ddl/util/util.go b/ddl/util/util.go index c1efc49abb311..dc262beeea877 100644 --- a/ddl/util/util.go +++ b/ddl/util/util.go @@ -110,9 +110,16 @@ func UpdateDeleteRange(ctx sessionctx.Context, dr DelRangeTask, newStartKey, old return errors.Trace(err) } +// DDLServerInfo is DDL server static information. +// DDLServerInfo will store into PD when server start up and delete when DDL closed. +// It will not update when DDL server running. So please only put static information in DDLServerInfo struct. type DDLServerInfo struct { ID string `json:"ddl_id"` + Version string `json:"version"` + GitHash string `json:"git_hash"` IP string `json:"ip"` StatusPort uint `json:"status_port"` - IsOwner bool `json:"is_owner,omitempty"` + // IsOwner will set false and will omit when json marshal and then store to PD. + // IsOwner is used when do http-api request ddl/info to indicate which server is Owner. + IsOwner bool `json:"is_owner,omitempty"` } diff --git a/server/http_handler.go b/server/http_handler.go index 21e5bdaaad232..28b1d02a0a25c 100644 --- a/server/http_handler.go +++ b/server/http_handler.go @@ -1276,7 +1276,6 @@ func (h ddlServerInfoHandler) ServeHTTP(w http.ResponseWriter, req *http.Request return } // ownerInfo.IsOwner is false because the server may no the owner when store the info to PD, - ownerInfo.IsOwner = true clusterInfo[ownerInfo.ID] = ownerInfo } clusterInfo[selfInfo.ID] = selfInfo From b574aff9f1b6ab309101aaedbec4168ae260c74d Mon Sep 17 00:00:00 2001 From: crazycs Date: Wed, 18 Jul 2018 09:19:11 +0800 Subject: [PATCH 03/40] add cluster info --- ddl/util/util.go | 11 ++++++++--- server/http_handler.go | 28 ++++++++++++++++++++++++++-- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/ddl/util/util.go b/ddl/util/util.go index dc262beeea877..5e7ce61cbc8e1 100644 --- a/ddl/util/util.go +++ b/ddl/util/util.go @@ -114,12 +114,17 @@ func UpdateDeleteRange(ctx sessionctx.Context, dr DelRangeTask, newStartKey, old // DDLServerInfo will store into PD when server start up and delete when DDL closed. // It will not update when DDL server running. So please only put static information in DDLServerInfo struct. type DDLServerInfo struct { - ID string `json:"ddl_id"` - Version string `json:"version"` - GitHash string `json:"git_hash"` + ID string `json:"ddl_id"` + ServerVersionInfo IP string `json:"ip"` StatusPort uint `json:"status_port"` // IsOwner will set false and will omit when json marshal and then store to PD. // IsOwner is used when do http-api request ddl/info to indicate which server is Owner. IsOwner bool `json:"is_owner,omitempty"` } + +// ServerVersionInfo is the server version and git_hash +type ServerVersionInfo struct { + Version string `json:"version"` + GitHash string `json:"git_hash"` +} diff --git a/server/http_handler.go b/server/http_handler.go index 28b1d02a0a25c..941f34910e9ff 100644 --- a/server/http_handler.go +++ b/server/http_handler.go @@ -1282,6 +1282,14 @@ func (h ddlServerInfoHandler) ServeHTTP(w http.ResponseWriter, req *http.Request writeData(w, clusterInfo) } +// ClusterServerInfo is only used to report cluster servers info when do http request +type ClusterServerInfo struct { + ServersNum int `json:"servers_num"` + IsAllServerVersionConsistent bool `json:"is_all_server_version_consistent"` + AllServersVersion []util.ServerVersionInfo `json:"all_servers_version"` + AllServersInfo map[string]*util.DDLServerInfo `json:"all_servers_info"` +} + // ServeHTTP handles request of all ddl servers info. func (h ddlAllServerInfoHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { session, err := session.CreateSession(h.store.(kv.Storage)) @@ -1290,10 +1298,26 @@ func (h ddlAllServerInfoHandler) ServeHTTP(w http.ResponseWriter, req *http.Requ return } ddl := domain.GetDomain(session.(sessionctx.Context)).DDL() - clusterInfo, err := ddl.GetAllServerInfo() + + allServersInfo, err := ddl.GetAllServerInfo() if err != nil { writeError(w, errors.New("ddl server information not found")) return } - writeData(w, clusterInfo) + allVersionsMap := map[util.ServerVersionInfo]struct{}{} + allVersions := []util.ServerVersionInfo{} + for _, v := range allServersInfo { + if _, ok := allVersionsMap[v.ServerVersionInfo]; ok { + continue + } + allVersionsMap[v.ServerVersionInfo] = struct{}{} + allVersions = append(allVersions, v.ServerVersionInfo) + } + clusterServerInfo := ClusterServerInfo{ + ServersNum: len(allServersInfo), + IsAllServerVersionConsistent: len(allVersions) == 1, + AllServersVersion: allVersions, + AllServersInfo: allServersInfo, + } + writeData(w, clusterServerInfo) } From aca6b97d34717d71bfc00ea91e5e84d9d2239109 Mon Sep 17 00:00:00 2001 From: crazycs Date: Wed, 18 Jul 2018 09:42:01 +0800 Subject: [PATCH 04/40] remove isOwner from server static info --- ddl/ddl.go | 13 +----------- ddl/mock.go | 2 +- ddl/syncer.go | 4 ++-- ddl/util/util.go | 3 --- server/http_handler.go | 48 +++++++++++++++++++++++++++++------------- 5 files changed, 37 insertions(+), 33 deletions(-) diff --git a/ddl/ddl.go b/ddl/ddl.go index f69ce8ceb8b17..22ae5213c9406 100644 --- a/ddl/ddl.go +++ b/ddl/ddl.go @@ -533,7 +533,6 @@ func (d *ddl) GetServerInfo() *util.DDLServerInfo { info.Version = mysql.ServerVersion info.GitHash = printer.TiDBGitHash info.ID = d.uuid - info.IsOwner = d.isOwner() return info } @@ -547,31 +546,21 @@ func (d *ddl) GetOwnerServerInfo() (*util.DDLServerInfo, error) { if err != nil { return nil, errors.Trace(err) } - ownerInfo.IsOwner = true return ownerInfo, nil } func (d *ddl) GetAllServerInfo() (map[string]*util.DDLServerInfo, error) { ctx := context.Background() - ddlOwnerID, err := d.ownerManager.GetOwnerID(ctx) + AllDDLServerInfo, err := d.schemaSyncer.GetAllServerInfoFromPD(ctx) if err != nil { return nil, errors.Trace(err) } - AllDDLServerInfo, err := d.schemaSyncer.GetAllServerInfoFromPD(ctx, ddlOwnerID) - if err != nil { - return nil, errors.Trace(err) - } - if _, ok := AllDDLServerInfo[ddlOwnerID]; ok { - AllDDLServerInfo[ddlOwnerID].IsOwner = true - } return AllDDLServerInfo, nil } func (d *ddl) StoreServerInfoToPD() error { info := d.GetServerInfo() ctx := context.Background() - // Owner will change , we will check ownerID to confirm which server is owner. - info.IsOwner = false return d.schemaSyncer.UpdateSelfServerInfo(ctx, info) } diff --git a/ddl/mock.go b/ddl/mock.go index 72d6f6e3fa50b..ad9a9e2e691d7 100644 --- a/ddl/mock.go +++ b/ddl/mock.go @@ -111,7 +111,7 @@ func (s *mockSchemaSyncer) GetServerInfoFromPD(ctx context.Context, _ string) (* } // GetAllServerInfoFromPD implements SchemaSyncer.GetAllServerInfoFromPD interface. -func (s *mockSchemaSyncer) GetAllServerInfoFromPD(ctx context.Context, _ string) (map[string]*util.DDLServerInfo, error) { +func (s *mockSchemaSyncer) GetAllServerInfoFromPD(ctx context.Context) (map[string]*util.DDLServerInfo, error) { allDDLInfo := make(map[string]*util.DDLServerInfo) allDDLInfo[s.selfServerInfo.ID] = s.selfServerInfo return allDDLInfo, nil diff --git a/ddl/syncer.go b/ddl/syncer.go index 5b68ac9a74bdc..4e07c084c094a 100644 --- a/ddl/syncer.go +++ b/ddl/syncer.go @@ -92,7 +92,7 @@ type SchemaSyncer interface { // GetServerInfoFromPD get the DDL_id server information from PD. GetServerInfoFromPD(ctx context.Context, id string) (*util.DDLServerInfo, error) // GetAllServerInfoFromPD get all DDL servers information from PD. - GetAllServerInfoFromPD(ctx context.Context, id string) (map[string]*util.DDLServerInfo, error) + GetAllServerInfoFromPD(ctx context.Context) (map[string]*util.DDLServerInfo, error) // UpdateSelfServerInfo store DDL server information to PD. UpdateSelfServerInfo(ctx context.Context, info *util.DDLServerInfo) error // RemoveSelfServerInfo remove DDL server information from PD. @@ -197,7 +197,7 @@ func (s *schemaVersionSyncer) GetServerInfoFromPD(ctx context.Context, id string } // GetAllServerInfoFromPD implements SchemaSyncer.GetAllServerInfoFromPD interface. -func (s *schemaVersionSyncer) GetAllServerInfoFromPD(ctx context.Context, id string) (map[string]*util.DDLServerInfo, error) { +func (s *schemaVersionSyncer) GetAllServerInfoFromPD(ctx context.Context) (map[string]*util.DDLServerInfo, error) { var err error allDDLInfo := make(map[string]*util.DDLServerInfo) for { diff --git a/ddl/util/util.go b/ddl/util/util.go index 5e7ce61cbc8e1..b54ed47b38932 100644 --- a/ddl/util/util.go +++ b/ddl/util/util.go @@ -118,9 +118,6 @@ type DDLServerInfo struct { ServerVersionInfo IP string `json:"ip"` StatusPort uint `json:"status_port"` - // IsOwner will set false and will omit when json marshal and then store to PD. - // IsOwner is used when do http-api request ddl/info to indicate which server is Owner. - IsOwner bool `json:"is_owner,omitempty"` } // ServerVersionInfo is the server version and git_hash diff --git a/server/http_handler.go b/server/http_handler.go index 941f34910e9ff..f8c999abb2b43 100644 --- a/server/http_handler.go +++ b/server/http_handler.go @@ -1259,6 +1259,15 @@ func (h *mvccTxnHandler) handleMvccGetByTxn(params map[string]string) (interface return h.getMvccByStartTs(uint64(startTS), startKey, endKey) } +// ClusterServerInfo is only used to report cluster servers info when do http request +type ClusterServerInfo struct { + ServersNum int `json:"servers_num,omitempty"` + OwnerID string `json:"owner_id"` + IsAllServerVersionConsistent bool `json:"is_all_server_version_consistent,omitempty"` + AllServersDiffVersions []util.ServerVersionInfo `json:"all_servers_diff_versions,omitempty"` + AllServersInfo map[string]*util.DDLServerInfo `json:"all_servers_info,omitempty"` +} + // ServeHTTP handles request of ddl server info. func (h ddlServerInfoHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { session, err := session.CreateSession(h.store.(kv.Storage)) @@ -1267,27 +1276,28 @@ func (h ddlServerInfoHandler) ServeHTTP(w http.ResponseWriter, req *http.Request return } ddl := domain.GetDomain(session.(sessionctx.Context)).DDL() - clusterInfo := make(map[string]*util.DDLServerInfo) + allServersInfo := make(map[string]*util.DDLServerInfo) + ownerID, err := ddl.OwnerManager().GetOwnerID(context.Background()) + if err != nil { + writeError(w, errors.New("ddl server information not found")) + return + } selfInfo := ddl.GetServerInfo() - if !selfInfo.IsOwner { + if selfInfo.ID != ownerID { ownerInfo, err := ddl.GetOwnerServerInfo() if err != nil { writeError(w, errors.New("ddl server information not found")) return } // ownerInfo.IsOwner is false because the server may no the owner when store the info to PD, - clusterInfo[ownerInfo.ID] = ownerInfo + allServersInfo[ownerInfo.ID] = ownerInfo } - clusterInfo[selfInfo.ID] = selfInfo - writeData(w, clusterInfo) -} - -// ClusterServerInfo is only used to report cluster servers info when do http request -type ClusterServerInfo struct { - ServersNum int `json:"servers_num"` - IsAllServerVersionConsistent bool `json:"is_all_server_version_consistent"` - AllServersVersion []util.ServerVersionInfo `json:"all_servers_version"` - AllServersInfo map[string]*util.DDLServerInfo `json:"all_servers_info"` + allServersInfo[selfInfo.ID] = selfInfo + clusterServerInfo := ClusterServerInfo{ + OwnerID: ownerID, + AllServersInfo: allServersInfo, + } + writeData(w, clusterServerInfo) } // ServeHTTP handles request of all ddl servers info. @@ -1304,6 +1314,11 @@ func (h ddlAllServerInfoHandler) ServeHTTP(w http.ResponseWriter, req *http.Requ writeError(w, errors.New("ddl server information not found")) return } + ownerID, err := ddl.OwnerManager().GetOwnerID(context.Background()) + if err != nil { + writeError(w, errors.New("ddl server information not found")) + return + } allVersionsMap := map[util.ServerVersionInfo]struct{}{} allVersions := []util.ServerVersionInfo{} for _, v := range allServersInfo { @@ -1314,10 +1329,13 @@ func (h ddlAllServerInfoHandler) ServeHTTP(w http.ResponseWriter, req *http.Requ allVersions = append(allVersions, v.ServerVersionInfo) } clusterServerInfo := ClusterServerInfo{ - ServersNum: len(allServersInfo), + ServersNum: len(allServersInfo), + OwnerID: ownerID, IsAllServerVersionConsistent: len(allVersions) == 1, - AllServersVersion: allVersions, AllServersInfo: allServersInfo, } + if !clusterServerInfo.IsAllServerVersionConsistent { + clusterServerInfo.AllServersDiffVersions = allVersions + } writeData(w, clusterServerInfo) } From d5a1922aa2dc00a2d9fbebb1bf3d47d0b8025c31 Mon Sep 17 00:00:00 2001 From: crazycs Date: Wed, 18 Jul 2018 10:11:35 +0800 Subject: [PATCH 05/40] refine comment --- ddl/ddl.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddl/ddl.go b/ddl/ddl.go index 22ae5213c9406..3463e8c090bd3 100644 --- a/ddl/ddl.go +++ b/ddl/ddl.go @@ -210,7 +210,7 @@ type DDL interface { // SetBinlogClient sets the binlog client for DDL worker. It's exported for testing. SetBinlogClient(interface{}) - // GetServerInfo get self DDL Server static information and isOwner information. + // GetServerInfo get self DDL Server static information. GetServerInfo() *util.DDLServerInfo // StoreServerInfoToPD store self DDL server static information to PD when DDL server Started. StoreServerInfoToPD() error From 24eea211bea9bf2ae20504bd93cae5306ea32458 Mon Sep 17 00:00:00 2001 From: crazycs Date: Wed, 18 Jul 2018 11:11:35 +0800 Subject: [PATCH 06/40] add lease to server info and change tidb_http_api.md --- ddl/ddl.go | 9 +++++---- ddl/syncer.go | 2 +- ddl/util/util.go | 3 ++- docs/tidb_http_api.md | 12 ++++++++++++ 4 files changed, 20 insertions(+), 6 deletions(-) diff --git a/ddl/ddl.go b/ddl/ddl.go index 3463e8c090bd3..afa7f6d44dce2 100644 --- a/ddl/ddl.go +++ b/ddl/ddl.go @@ -210,13 +210,13 @@ type DDL interface { // SetBinlogClient sets the binlog client for DDL worker. It's exported for testing. SetBinlogClient(interface{}) - // GetServerInfo get self DDL Server static information. + // GetServerInfo get self DDL server static information. GetServerInfo() *util.DDLServerInfo // StoreServerInfoToPD store self DDL server static information to PD when DDL server Started. StoreServerInfoToPD() error - // GetOwnerServerInfo get owner DDL Server static information from PD. + // GetOwnerServerInfo get owner DDL server static information from PD. GetOwnerServerInfo() (*util.DDLServerInfo, error) - // GetAllServerInfo get all DDL Server static information from PD. + // GetAllServerInfo get all DDL servers static information from PD. GetAllServerInfo() (map[string]*util.DDLServerInfo, error) } @@ -528,8 +528,9 @@ func (d *ddl) SetBinlogClient(binlogCli interface{}) { func (d *ddl) GetServerInfo() *util.DDLServerInfo { info := &util.DDLServerInfo{} cfg := config.GetGlobalConfig() - info.IP = cfg.Host + info.IP = cfg.AdvertiseAddress info.StatusPort = cfg.Status.StatusPort + info.Lease = cfg.Lease info.Version = mysql.ServerVersion info.GitHash = printer.TiDBGitHash info.ID = d.uuid diff --git a/ddl/syncer.go b/ddl/syncer.go index 4e07c084c094a..b1be99e611b60 100644 --- a/ddl/syncer.go +++ b/ddl/syncer.go @@ -218,7 +218,7 @@ func (s *schemaVersionSyncer) GetAllServerInfoFromPD(ctx context.Context) (map[s info := &util.DDLServerInfo{} err := json.Unmarshal(kv.Value, info) if err != nil { - log.Infof("[syncer] get all ddl server info, ddl %s json.Unmarshal %v failed %v, continue checking.", kv.Key, kv.Value, err) + log.Infof("[syncer] get all ddl server info, ddl %s json.Unmarshal %v failed %v.", kv.Key, kv.Value, err) return nil, err } allDDLInfo[info.ID] = info diff --git a/ddl/util/util.go b/ddl/util/util.go index b54ed47b38932..9dfd9064312bc 100644 --- a/ddl/util/util.go +++ b/ddl/util/util.go @@ -114,10 +114,11 @@ func UpdateDeleteRange(ctx sessionctx.Context, dr DelRangeTask, newStartKey, old // DDLServerInfo will store into PD when server start up and delete when DDL closed. // It will not update when DDL server running. So please only put static information in DDLServerInfo struct. type DDLServerInfo struct { - ID string `json:"ddl_id"` ServerVersionInfo + ID string `json:"ddl_id"` IP string `json:"ip"` StatusPort uint `json:"status_port"` + Lease string `json:"lease"` } // ServerVersionInfo is the server version and git_hash diff --git a/docs/tidb_http_api.md b/docs/tidb_http_api.md index 2136d7eca10aa..4a6d4aba75182 100644 --- a/docs/tidb_http_api.md +++ b/docs/tidb_http_api.md @@ -96,6 +96,18 @@ timezone.* curl http://{TiDBIP}:10080/settings ``` +1. Get TiDB server information and owner server infomation. + + ```shell + curl http://{TiDBIP}:10080/ddl/info + ``` + +1. Get TiDB cluster all servers information. + + ```shell + curl http://{TiDBIP}:10080/ddl/info/all + ``` + 1. Enable/Disable TiDB server general log ```shell From f0007a4c22ce115b31b79ac7250c5beac0c0bf84 Mon Sep 17 00:00:00 2001 From: crazycs Date: Wed, 18 Jul 2018 13:39:27 +0800 Subject: [PATCH 07/40] refine log --- ddl/syncer.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ddl/syncer.go b/ddl/syncer.go index b1be99e611b60..50bc1588112a2 100644 --- a/ddl/syncer.go +++ b/ddl/syncer.go @@ -182,17 +182,19 @@ func (s *schemaVersionSyncer) GetServerInfoFromPD(ctx context.Context, id string resp, err = s.etcdCli.Get(ctx, ddlPath) if err != nil { + log.Infof("[syncer] get ddl server info, ddl %s failed %v, continue checking.", ddlPath, err) + time.Sleep(200 * time.Millisecond) continue } if err == nil && len(resp.Kvs) > 0 { info := &util.DDLServerInfo{} err := json.Unmarshal(resp.Kvs[0].Value, info) if err != nil { + log.Infof("[syncer] get ddl server info, ddl %s json.Unmarshal %v failed %v.", resp.Kvs[0].Key, resp.Kvs[0].Value, err) return nil, err } return info, nil } - time.Sleep(200 * time.Millisecond) } } From 159a7b4d624b747fcd37a9b85e660fc558dbebad Mon Sep 17 00:00:00 2001 From: crazycs Date: Wed, 18 Jul 2018 13:53:45 +0800 Subject: [PATCH 08/40] refine http-api ddl/info --- server/http_handler.go | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/server/http_handler.go b/server/http_handler.go index f8c999abb2b43..ec65f63bac5d8 100644 --- a/server/http_handler.go +++ b/server/http_handler.go @@ -1259,13 +1259,11 @@ func (h *mvccTxnHandler) handleMvccGetByTxn(params map[string]string) (interface return h.getMvccByStartTs(uint64(startTS), startKey, endKey) } -// ClusterServerInfo is only used to report cluster servers info when do http request -type ClusterServerInfo struct { - ServersNum int `json:"servers_num,omitempty"` - OwnerID string `json:"owner_id"` - IsAllServerVersionConsistent bool `json:"is_all_server_version_consistent,omitempty"` - AllServersDiffVersions []util.ServerVersionInfo `json:"all_servers_diff_versions,omitempty"` - AllServersInfo map[string]*util.DDLServerInfo `json:"all_servers_info,omitempty"` +// ReportServerInfo is only used to report the servers info when do http request +type ReportServerInfo struct { + IsOwner bool `json:"is_owner"` + SelfServerInfo *util.DDLServerInfo `json:"self_server_info"` + OwnerServerInfo *util.DDLServerInfo `json:"owner_server_info,omitempty"` } // ServeHTTP handles request of ddl server info. @@ -1276,28 +1274,34 @@ func (h ddlServerInfoHandler) ServeHTTP(w http.ResponseWriter, req *http.Request return } ddl := domain.GetDomain(session.(sessionctx.Context)).DDL() - allServersInfo := make(map[string]*util.DDLServerInfo) ownerID, err := ddl.OwnerManager().GetOwnerID(context.Background()) if err != nil { writeError(w, errors.New("ddl server information not found")) return } selfInfo := ddl.GetServerInfo() - if selfInfo.ID != ownerID { + reportServerInfo := ReportServerInfo{ + IsOwner: selfInfo.ID == ownerID, + SelfServerInfo: selfInfo, + } + if !reportServerInfo.IsOwner { ownerInfo, err := ddl.GetOwnerServerInfo() if err != nil { writeError(w, errors.New("ddl server information not found")) return } - // ownerInfo.IsOwner is false because the server may no the owner when store the info to PD, - allServersInfo[ownerInfo.ID] = ownerInfo - } - allServersInfo[selfInfo.ID] = selfInfo - clusterServerInfo := ClusterServerInfo{ - OwnerID: ownerID, - AllServersInfo: allServersInfo, + reportServerInfo.OwnerServerInfo = ownerInfo } - writeData(w, clusterServerInfo) + writeData(w, reportServerInfo) +} + +// ClusterServerInfo is only used to report cluster servers info when do http request +type ClusterServerInfo struct { + ServersNum int `json:"servers_num,omitempty"` + OwnerID string `json:"owner_id"` + IsAllServerVersionConsistent bool `json:"is_all_server_version_consistent,omitempty"` + AllServersDiffVersions []util.ServerVersionInfo `json:"all_servers_diff_versions,omitempty"` + AllServersInfo map[string]*util.DDLServerInfo `json:"all_servers_info,omitempty"` } // ServeHTTP handles request of all ddl servers info. From f1437099591c0b52aa605b639101cca5bb763703 Mon Sep 17 00:00:00 2001 From: crazycs Date: Thu, 19 Jul 2018 11:14:50 +0800 Subject: [PATCH 09/40] refine code --- ddl/ddl.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/ddl/ddl.go b/ddl/ddl.go index afa7f6d44dce2..8aed5085d6528 100644 --- a/ddl/ddl.go +++ b/ddl/ddl.go @@ -526,14 +526,15 @@ func (d *ddl) SetBinlogClient(binlogCli interface{}) { } func (d *ddl) GetServerInfo() *util.DDLServerInfo { - info := &util.DDLServerInfo{} cfg := config.GetGlobalConfig() - info.IP = cfg.AdvertiseAddress - info.StatusPort = cfg.Status.StatusPort - info.Lease = cfg.Lease + info := &util.DDLServerInfo{ + IP: cfg.AdvertiseAddress, + StatusPort: cfg.Status.StatusPort, + Lease: cfg.Lease, + ID: d.uuid, + } info.Version = mysql.ServerVersion info.GitHash = printer.TiDBGitHash - info.ID = d.uuid return info } From 8bd6c3d43f0c5ba199a7323cb51bc99b85ba7a57 Mon Sep 17 00:00:00 2001 From: crazycs Date: Thu, 19 Jul 2018 19:37:30 +0800 Subject: [PATCH 10/40] rename variable --- server/http_handler.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/http_handler.go b/server/http_handler.go index ec65f63bac5d8..ee038a516ed02 100644 --- a/server/http_handler.go +++ b/server/http_handler.go @@ -1268,12 +1268,12 @@ type ReportServerInfo struct { // ServeHTTP handles request of ddl server info. func (h ddlServerInfoHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { - session, err := session.CreateSession(h.store.(kv.Storage)) + se, err := session.CreateSession(h.store.(kv.Storage)) if err != nil { writeError(w, errors.New("create session error")) return } - ddl := domain.GetDomain(session.(sessionctx.Context)).DDL() + ddl := domain.GetDomain(se.(sessionctx.Context)).DDL() ownerID, err := ddl.OwnerManager().GetOwnerID(context.Background()) if err != nil { writeError(w, errors.New("ddl server information not found")) @@ -1306,12 +1306,12 @@ type ClusterServerInfo struct { // ServeHTTP handles request of all ddl servers info. func (h ddlAllServerInfoHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { - session, err := session.CreateSession(h.store.(kv.Storage)) + se, err := session.CreateSession(h.store.(kv.Storage)) if err != nil { writeError(w, errors.New("create session error")) return } - ddl := domain.GetDomain(session.(sessionctx.Context)).DDL() + ddl := domain.GetDomain(se.(sessionctx.Context)).DDL() allServersInfo, err := ddl.GetAllServerInfo() if err != nil { From 80cdd6ac06ad9779e15cfb009f6424222794b9f6 Mon Sep 17 00:00:00 2001 From: crazycs Date: Sun, 22 Jul 2018 14:20:50 +0800 Subject: [PATCH 11/40] refactor code --- ddl/ddl.go | 64 +++++------------------------------------ ddl/mock.go | 4 +-- ddl/syncer.go | 18 ++++++------ domain/domain.go | 3 +- domain/info.go | 65 ++++++++++++++++++++++++++++++++++++++++++ server/http_handler.go | 6 ++-- 6 files changed, 88 insertions(+), 72 deletions(-) create mode 100644 domain/info.go diff --git a/ddl/ddl.go b/ddl/ddl.go index fbd8acca778b2..2d057ab7f0ca1 100644 --- a/ddl/ddl.go +++ b/ddl/ddl.go @@ -26,7 +26,6 @@ import ( "github.com/juju/errors" "github.com/ngaut/pools" "github.com/pingcap/tidb/ast" - "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/ddl/util" "github.com/pingcap/tidb/infoschema" "github.com/pingcap/tidb/kv" @@ -39,7 +38,6 @@ import ( "github.com/pingcap/tidb/sessionctx/binloginfo" "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/terror" - "github.com/pingcap/tidb/util/printer" log "github.com/sirupsen/logrus" "github.com/twinj/uuid" "golang.org/x/net/context" @@ -206,18 +204,11 @@ type DDL interface { OwnerManager() owner.Manager // GetTableMaxRowID gets table max row ID. GetTableMaxRowID(startTS uint64, tblInfo *model.TableInfo) (int64, bool, error) + // GetID gets the ddl ID + GetID() string // SetBinlogClient sets the binlog client for DDL worker. It's exported for testing. SetBinlogClient(interface{}) - - // GetServerInfo get self DDL server static information. - GetServerInfo() *util.DDLServerInfo - // StoreServerInfoToPD store self DDL server static information to PD when DDL server Started. - StoreServerInfoToPD() error - // GetOwnerServerInfo get owner DDL server static information from PD. - GetOwnerServerInfo() (*util.DDLServerInfo, error) - // GetAllServerInfo get all DDL servers static information from PD. - GetAllServerInfo() (map[string]*util.DDLServerInfo, error) } // ddl is used to handle the statements that define the structure or schema of the database. @@ -387,11 +378,6 @@ func (d *ddl) close() { if err != nil { log.Errorf("[ddl] remove self version path failed %v", err) } - err = d.schemaSyncer.RemoveSelfServerInfo() - if err != nil { - log.Errorf("[ddl] remove self server info path failed %v", err) - } - for _, worker := range d.workers { worker.close() } @@ -445,6 +431,11 @@ func (d *ddl) OwnerManager() owner.Manager { return d.ownerManager } +// GetID implements DDL.GetID interface. +func (d *ddl) GetID() string { + return d.uuid +} + func checkJobMaxInterval(job *model.Job) time.Duration { // The job of adding index takes more time to process. // So it uses the longer time. @@ -525,47 +516,6 @@ func (d *ddl) SetBinlogClient(binlogCli interface{}) { d.binlogCli = binlogCli } -func (d *ddl) GetServerInfo() *util.DDLServerInfo { - cfg := config.GetGlobalConfig() - info := &util.DDLServerInfo{ - ID: d.uuid, - IP: cfg.AdvertiseAddress, - StatusPort: cfg.Status.StatusPort, - Lease: cfg.Lease, - } - info.Version = mysql.ServerVersion - info.GitHash = printer.TiDBGitHash - return info -} - -func (d *ddl) GetOwnerServerInfo() (*util.DDLServerInfo, error) { - ctx := context.Background() - ddlOwnerID, err := d.ownerManager.GetOwnerID(ctx) - if err != nil { - return nil, errors.Trace(err) - } - ownerInfo, err := d.schemaSyncer.GetServerInfoFromPD(ctx, ddlOwnerID) - if err != nil { - return nil, errors.Trace(err) - } - return ownerInfo, nil -} - -func (d *ddl) GetAllServerInfo() (map[string]*util.DDLServerInfo, error) { - ctx := context.Background() - AllDDLServerInfo, err := d.schemaSyncer.GetAllServerInfoFromPD(ctx) - if err != nil { - return nil, errors.Trace(err) - } - return AllDDLServerInfo, nil -} - -func (d *ddl) StoreServerInfoToPD() error { - info := d.GetServerInfo() - ctx := context.Background() - return d.schemaSyncer.UpdateSelfServerInfo(ctx, info) -} - // DDL error codes. const ( codeInvalidWorker terror.ErrCode = 1 diff --git a/ddl/mock.go b/ddl/mock.go index ad9a9e2e691d7..4b1d14e0c4e77 100644 --- a/ddl/mock.go +++ b/ddl/mock.go @@ -117,8 +117,8 @@ func (s *mockSchemaSyncer) GetAllServerInfoFromPD(ctx context.Context) (map[stri return allDDLInfo, nil } -// UpdateSelfServerInfo implements SchemaSyncer.UpdateSelfServerInfo interface. -func (s *mockSchemaSyncer) UpdateSelfServerInfo(ctx context.Context, info *util.DDLServerInfo) error { +// StoreSelfServerInfo implements SchemaSyncer.StoreSelfServerInfo interface. +func (s *mockSchemaSyncer) StoreSelfServerInfo(ctx context.Context, info *util.DDLServerInfo) error { s.selfServerInfo = info return nil } diff --git a/ddl/syncer.go b/ddl/syncer.go index 50bc1588112a2..7c07c7e85e400 100644 --- a/ddl/syncer.go +++ b/ddl/syncer.go @@ -14,13 +14,13 @@ package ddl import ( + "encoding/json" "fmt" "math" "strconv" "sync" "time" - "encoding/json" "github.com/coreos/etcd/clientv3" "github.com/coreos/etcd/clientv3/concurrency" "github.com/juju/errors" @@ -93,8 +93,8 @@ type SchemaSyncer interface { GetServerInfoFromPD(ctx context.Context, id string) (*util.DDLServerInfo, error) // GetAllServerInfoFromPD get all DDL servers information from PD. GetAllServerInfoFromPD(ctx context.Context) (map[string]*util.DDLServerInfo, error) - // UpdateSelfServerInfo store DDL server information to PD. - UpdateSelfServerInfo(ctx context.Context, info *util.DDLServerInfo) error + // StoreSelfServerInfo store DDL server information to PD. + StoreSelfServerInfo(ctx context.Context, info *util.DDLServerInfo) error // RemoveSelfServerInfo remove DDL server information from PD. RemoveSelfServerInfo() error } @@ -186,12 +186,12 @@ func (s *schemaVersionSyncer) GetServerInfoFromPD(ctx context.Context, id string time.Sleep(200 * time.Millisecond) continue } - if err == nil && len(resp.Kvs) > 0 { + if len(resp.Kvs) > 0 { info := &util.DDLServerInfo{} err := json.Unmarshal(resp.Kvs[0].Value, info) if err != nil { log.Infof("[syncer] get ddl server info, ddl %s json.Unmarshal %v failed %v.", resp.Kvs[0].Key, resp.Kvs[0].Value, err) - return nil, err + return nil, errors.Trace(err) } return info, nil } @@ -221,7 +221,7 @@ func (s *schemaVersionSyncer) GetAllServerInfoFromPD(ctx context.Context) (map[s err := json.Unmarshal(kv.Value, info) if err != nil { log.Infof("[syncer] get all ddl server info, ddl %s json.Unmarshal %v failed %v.", kv.Key, kv.Value, err) - return nil, err + return nil, errors.Trace(err) } allDDLInfo[info.ID] = info } @@ -229,8 +229,8 @@ func (s *schemaVersionSyncer) GetAllServerInfoFromPD(ctx context.Context) (map[s } } -// UpdateSelfServerInfo implements SchemaSyncer.UpdateSelfServerInfo interface. -func (s *schemaVersionSyncer) UpdateSelfServerInfo(ctx context.Context, info *util.DDLServerInfo) error { +// StoreSelfServerInfo implements SchemaSyncer.StoreSelfServerInfo interface. +func (s *schemaVersionSyncer) StoreSelfServerInfo(ctx context.Context, info *util.DDLServerInfo) error { infoBuf, err := json.Marshal(info) if err != nil { return errors.Trace(err) @@ -383,7 +383,7 @@ func (s *schemaVersionSyncer) MustGetGlobalVersion(ctx context.Context) (int64, if err != nil { continue } - if err == nil && len(resp.Kvs) > 0 { + if len(resp.Kvs) > 0 { var ver int ver, err = strconv.Atoi(string(resp.Kvs[0].Value)) if err == nil { diff --git a/domain/domain.go b/domain/domain.go index eea8f9bfe46db..6ecef0c3ff92a 100644 --- a/domain/domain.go +++ b/domain/domain.go @@ -390,6 +390,7 @@ func (do *Domain) mustRestartSyncer() error { // Close closes the Domain and release its resource. func (do *Domain) Close() { if do.ddl != nil { + RemoveServerInfoFromPD(do.ddl) terror.Log(errors.Trace(do.ddl.Stop())) } close(do.exit) @@ -500,7 +501,7 @@ func (do *Domain) Init(ddlLease time.Duration, sysFactory func(*Domain) (pools.R if err != nil { return errors.Trace(err) } - err = do.ddl.StoreServerInfoToPD() + err = StoreServerInfoToPD(do.ddl) if err != nil { return errors.Trace(err) } diff --git a/domain/info.go b/domain/info.go new file mode 100644 index 0000000000000..a0314590cac84 --- /dev/null +++ b/domain/info.go @@ -0,0 +1,65 @@ +package domain + +import ( + "github.com/juju/errors" + "github.com/pingcap/tidb/config" + "github.com/pingcap/tidb/ddl" + "github.com/pingcap/tidb/ddl/util" + "github.com/pingcap/tidb/mysql" + "github.com/pingcap/tidb/util/printer" + log "github.com/sirupsen/logrus" + "golang.org/x/net/context" +) + +//GetServerInfo gets self DDL server static information. +func GetServerInfo(ddlID string) *util.DDLServerInfo { + cfg := config.GetGlobalConfig() + info := &util.DDLServerInfo{ + ID: ddlID, + IP: cfg.AdvertiseAddress, + StatusPort: cfg.Status.StatusPort, + Lease: cfg.Lease, + } + info.Version = mysql.ServerVersion + info.GitHash = printer.TiDBGitHash + return info +} + +// GetOwnerServerInfo gets owner DDL server static information from PD. +func GetOwnerServerInfo(d ddl.DDL) (*util.DDLServerInfo, error) { + ctx := context.Background() + ddlOwnerID, err := d.OwnerManager().GetOwnerID(ctx) + if err != nil { + return nil, errors.Trace(err) + } + ownerInfo, err := d.SchemaSyncer().GetServerInfoFromPD(ctx, ddlOwnerID) + if err != nil { + return nil, errors.Trace(err) + } + return ownerInfo, nil +} + +// GetAllServerInfo gets all DDL servers static information from PD. +func GetAllServerInfo(d ddl.DDL) (map[string]*util.DDLServerInfo, error) { + ctx := context.Background() + AllDDLServerInfo, err := d.SchemaSyncer().GetAllServerInfoFromPD(ctx) + if err != nil { + return nil, errors.Trace(err) + } + return AllDDLServerInfo, nil +} + +// StoreServerInfoToPD stores self DDL server static information to PD when domain Init. +func StoreServerInfoToPD(d ddl.DDL) error { + info := GetServerInfo(d.GetID()) + ctx := context.Background() + return d.SchemaSyncer().StoreSelfServerInfo(ctx, info) +} + +// RemoveServerInfoFromPD remove self DDL server static information from PD when domain close. +func RemoveServerInfoFromPD(d ddl.DDL) { + err := d.SchemaSyncer().RemoveSelfServerInfo() + if err != nil { + log.Errorf("[ddl] remove self server info failed %v", err) + } +} diff --git a/server/http_handler.go b/server/http_handler.go index ee038a516ed02..5ffcccd941e06 100644 --- a/server/http_handler.go +++ b/server/http_handler.go @@ -1279,13 +1279,13 @@ func (h ddlServerInfoHandler) ServeHTTP(w http.ResponseWriter, req *http.Request writeError(w, errors.New("ddl server information not found")) return } - selfInfo := ddl.GetServerInfo() + selfInfo := domain.GetServerInfo(ddl.GetID()) reportServerInfo := ReportServerInfo{ IsOwner: selfInfo.ID == ownerID, SelfServerInfo: selfInfo, } if !reportServerInfo.IsOwner { - ownerInfo, err := ddl.GetOwnerServerInfo() + ownerInfo, err := domain.GetOwnerServerInfo(ddl) if err != nil { writeError(w, errors.New("ddl server information not found")) return @@ -1313,7 +1313,7 @@ func (h ddlAllServerInfoHandler) ServeHTTP(w http.ResponseWriter, req *http.Requ } ddl := domain.GetDomain(se.(sessionctx.Context)).DDL() - allServersInfo, err := ddl.GetAllServerInfo() + allServersInfo, err := domain.GetAllServerInfo(ddl) if err != nil { writeError(w, errors.New("ddl server information not found")) return From 839f298ecc6db05674a1e8e3835b3334e491eb69 Mon Sep 17 00:00:00 2001 From: crazycs Date: Mon, 23 Jul 2018 20:42:40 +0800 Subject: [PATCH 12/40] refactor and add comment --- ddl/mock.go | 8 ++++---- ddl/syncer.go | 18 +++++++++--------- domain/info.go | 17 +++++++++++++++-- server/http_handler.go | 8 ++++---- 4 files changed, 32 insertions(+), 19 deletions(-) diff --git a/ddl/mock.go b/ddl/mock.go index 4b1d14e0c4e77..86d4f0bc3a512 100644 --- a/ddl/mock.go +++ b/ddl/mock.go @@ -117,14 +117,14 @@ func (s *mockSchemaSyncer) GetAllServerInfoFromPD(ctx context.Context) (map[stri return allDDLInfo, nil } -// StoreSelfServerInfo implements SchemaSyncer.StoreSelfServerInfo interface. -func (s *mockSchemaSyncer) StoreSelfServerInfo(ctx context.Context, info *util.DDLServerInfo) error { +// StoreSelfServerInfoToPD implements SchemaSyncer.StoreSelfServerInfoToPD interface. +func (s *mockSchemaSyncer) StoreSelfServerInfoToPD(ctx context.Context, info *util.DDLServerInfo) error { s.selfServerInfo = info return nil } -// RemoveSelfServerInfo implements SchemaSyncer.RemoveSelfServerInfo interface. -func (s *mockSchemaSyncer) RemoveSelfServerInfo() error { +// RemoveSelfServerInfoFromPD implements SchemaSyncer.RemoveSelfServerInfoFromPD interface. +func (s *mockSchemaSyncer) RemoveSelfServerInfoFromPD() error { return nil } diff --git a/ddl/syncer.go b/ddl/syncer.go index 7c07c7e85e400..2182d3553aa42 100644 --- a/ddl/syncer.go +++ b/ddl/syncer.go @@ -36,7 +36,7 @@ const ( // DDLAllSchemaVersions is the path on etcd that is used to store all servers current schema versions. // It's exported for testing. DDLAllSchemaVersions = "/tidb/ddl/all_schema_versions" - //DDLServerInformation store DDL server information such as IP,port and so on + // DDLServerInformation store DDL server information such as IP, port and so on. DDLServerInformation = "/tidb/ddl/info" // DDLGlobalSchemaVersion is the path on etcd that is used to store the latest schema versions. // It's exported for testing. @@ -93,10 +93,10 @@ type SchemaSyncer interface { GetServerInfoFromPD(ctx context.Context, id string) (*util.DDLServerInfo, error) // GetAllServerInfoFromPD get all DDL servers information from PD. GetAllServerInfoFromPD(ctx context.Context) (map[string]*util.DDLServerInfo, error) - // StoreSelfServerInfo store DDL server information to PD. - StoreSelfServerInfo(ctx context.Context, info *util.DDLServerInfo) error - // RemoveSelfServerInfo remove DDL server information from PD. - RemoveSelfServerInfo() error + // StoreSelfServerInfoToPD store DDL server information to PD. + StoreSelfServerInfoToPD(ctx context.Context, info *util.DDLServerInfo) error + // RemoveSelfServerInfoFromPD remove DDL server information from PD. + RemoveSelfServerInfoFromPD() error } type schemaVersionSyncer struct { @@ -229,8 +229,8 @@ func (s *schemaVersionSyncer) GetAllServerInfoFromPD(ctx context.Context) (map[s } } -// StoreSelfServerInfo implements SchemaSyncer.StoreSelfServerInfo interface. -func (s *schemaVersionSyncer) StoreSelfServerInfo(ctx context.Context, info *util.DDLServerInfo) error { +// StoreSelfServerInfoToPD implements SchemaSyncer.StoreSelfServerInfoToPD interface. +func (s *schemaVersionSyncer) StoreSelfServerInfoToPD(ctx context.Context, info *util.DDLServerInfo) error { infoBuf, err := json.Marshal(info) if err != nil { return errors.Trace(err) @@ -239,8 +239,8 @@ func (s *schemaVersionSyncer) StoreSelfServerInfo(ctx context.Context, info *uti return errors.Trace(err) } -// RemoveSelfServerInfo implements SchemaSyncer.RemoveSelfServerInfo interface. -func (s *schemaVersionSyncer) RemoveSelfServerInfo() error { +// RemoveSelfServerInfoFromPD implements SchemaSyncer.RemoveSelfServerInfoFromPD interface. +func (s *schemaVersionSyncer) RemoveSelfServerInfoFromPD() error { var err error ctx := context.Background() for i := 0; i < keyOpDefaultRetryCnt; i++ { diff --git a/domain/info.go b/domain/info.go index a0314590cac84..085225fcb6bc1 100644 --- a/domain/info.go +++ b/domain/info.go @@ -1,3 +1,16 @@ +// Copyright 2018 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + package domain import ( @@ -53,12 +66,12 @@ func GetAllServerInfo(d ddl.DDL) (map[string]*util.DDLServerInfo, error) { func StoreServerInfoToPD(d ddl.DDL) error { info := GetServerInfo(d.GetID()) ctx := context.Background() - return d.SchemaSyncer().StoreSelfServerInfo(ctx, info) + return d.SchemaSyncer().StoreSelfServerInfoToPD(ctx, info) } // RemoveServerInfoFromPD remove self DDL server static information from PD when domain close. func RemoveServerInfoFromPD(d ddl.DDL) { - err := d.SchemaSyncer().RemoveSelfServerInfo() + err := d.SchemaSyncer().RemoveSelfServerInfoFromPD() if err != nil { log.Errorf("[ddl] remove self server info failed %v", err) } diff --git a/server/http_handler.go b/server/http_handler.go index 5ffcccd941e06..488aee49595e4 100644 --- a/server/http_handler.go +++ b/server/http_handler.go @@ -1268,12 +1268,12 @@ type ReportServerInfo struct { // ServeHTTP handles request of ddl server info. func (h ddlServerInfoHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { - se, err := session.CreateSession(h.store.(kv.Storage)) + do, err := session.GetDomain(h.store.(kv.Storage)) if err != nil { writeError(w, errors.New("create session error")) return } - ddl := domain.GetDomain(se.(sessionctx.Context)).DDL() + ddl := do.DDL() ownerID, err := ddl.OwnerManager().GetOwnerID(context.Background()) if err != nil { writeError(w, errors.New("ddl server information not found")) @@ -1306,12 +1306,12 @@ type ClusterServerInfo struct { // ServeHTTP handles request of all ddl servers info. func (h ddlAllServerInfoHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { - se, err := session.CreateSession(h.store.(kv.Storage)) + do, err := session.GetDomain(h.store.(kv.Storage)) if err != nil { writeError(w, errors.New("create session error")) return } - ddl := domain.GetDomain(se.(sessionctx.Context)).DDL() + ddl := do.DDL() allServersInfo, err := domain.GetAllServerInfo(ddl) if err != nil { From d0c1925874685c5bc37bdcca6b373dfd117bf943 Mon Sep 17 00:00:00 2001 From: crazycs Date: Thu, 2 Aug 2018 21:27:58 +0800 Subject: [PATCH 13/40] refine code --- ddl/mock.go | 20 +++++++++--------- ddl/syncer.go | 48 +++++++++++++++++++++--------------------- ddl/util/util.go | 8 +++---- domain/info.go | 22 +++++++++---------- server/http_handler.go | 16 +++++++------- server/http_status.go | 6 ++++-- 6 files changed, 61 insertions(+), 59 deletions(-) diff --git a/ddl/mock.go b/ddl/mock.go index 86d4f0bc3a512..b6f600fe558fb 100644 --- a/ddl/mock.go +++ b/ddl/mock.go @@ -34,7 +34,7 @@ type mockSchemaSyncer struct { selfSchemaVersion int64 globalVerCh chan clientv3.WatchResponse // selfServerInfo used to save self DDL server information. - selfServerInfo *util.DDLServerInfo + selfServerInfo *util.ServerInfo } // NewMockSchemaSyncer creates a new mock SchemaSyncer. @@ -105,26 +105,26 @@ func (s *mockSchemaSyncer) OwnerCheckAllVersions(ctx context.Context, latestVer } } -// GetServerInfoFromPD implements SchemaSyncer.GetServerInfoFromPD interface. -func (s *mockSchemaSyncer) GetServerInfoFromPD(ctx context.Context, _ string) (*util.DDLServerInfo, error) { +// GetServerInfo implements SchemaSyncer.GetServerInfo interface. +func (s *mockSchemaSyncer) GetServerInfo(ctx context.Context, _ string) (*util.ServerInfo, error) { return s.selfServerInfo, nil } -// GetAllServerInfoFromPD implements SchemaSyncer.GetAllServerInfoFromPD interface. -func (s *mockSchemaSyncer) GetAllServerInfoFromPD(ctx context.Context) (map[string]*util.DDLServerInfo, error) { - allDDLInfo := make(map[string]*util.DDLServerInfo) +// GetAllServerInfo implements SchemaSyncer.GetAllServerInfo interface. +func (s *mockSchemaSyncer) GetAllServerInfo(ctx context.Context) (map[string]*util.ServerInfo, error) { + allDDLInfo := make(map[string]*util.ServerInfo) allDDLInfo[s.selfServerInfo.ID] = s.selfServerInfo return allDDLInfo, nil } -// StoreSelfServerInfoToPD implements SchemaSyncer.StoreSelfServerInfoToPD interface. -func (s *mockSchemaSyncer) StoreSelfServerInfoToPD(ctx context.Context, info *util.DDLServerInfo) error { +// StoreServerInfo implements SchemaSyncer.StoreServerInfo interface. +func (s *mockSchemaSyncer) StoreServerInfo(ctx context.Context, info *util.ServerInfo) error { s.selfServerInfo = info return nil } -// RemoveSelfServerInfoFromPD implements SchemaSyncer.RemoveSelfServerInfoFromPD interface. -func (s *mockSchemaSyncer) RemoveSelfServerInfoFromPD() error { +// RemoveServerInfo implements SchemaSyncer.RemoveServerInfo interface. +func (s *mockSchemaSyncer) RemoveServerInfo() error { return nil } diff --git a/ddl/syncer.go b/ddl/syncer.go index 2182d3553aa42..6224c533345d3 100644 --- a/ddl/syncer.go +++ b/ddl/syncer.go @@ -36,8 +36,8 @@ const ( // DDLAllSchemaVersions is the path on etcd that is used to store all servers current schema versions. // It's exported for testing. DDLAllSchemaVersions = "/tidb/ddl/all_schema_versions" - // DDLServerInformation store DDL server information such as IP, port and so on. - DDLServerInformation = "/tidb/ddl/info" + // ServerInformation store DDL server information such as IP, port and so on. + ServerInformation = "/tidb/ddl/info" // DDLGlobalSchemaVersion is the path on etcd that is used to store the latest schema versions. // It's exported for testing. DDLGlobalSchemaVersion = "/tidb/ddl/global_schema_version" @@ -89,14 +89,14 @@ type SchemaSyncer interface { // It returns until all servers' versions are equal to the latest version or the ctx is done. OwnerCheckAllVersions(ctx context.Context, latestVer int64) error - // GetServerInfoFromPD get the DDL_id server information from PD. - GetServerInfoFromPD(ctx context.Context, id string) (*util.DDLServerInfo, error) - // GetAllServerInfoFromPD get all DDL servers information from PD. - GetAllServerInfoFromPD(ctx context.Context) (map[string]*util.DDLServerInfo, error) - // StoreSelfServerInfoToPD store DDL server information to PD. - StoreSelfServerInfoToPD(ctx context.Context, info *util.DDLServerInfo) error - // RemoveSelfServerInfoFromPD remove DDL server information from PD. - RemoveSelfServerInfoFromPD() error + // GetServerInfo gets the DDL_id server information from PD. + GetServerInfo(ctx context.Context, id string) (*util.ServerInfo, error) + // GetAllServerInfo gets all DDL servers information from PD. + GetAllServerInfo(ctx context.Context) (map[string]*util.ServerInfo, error) + // StoreServerInfo stores DDL server information to PD. + StoreServerInfo(ctx context.Context, info *util.ServerInfo) error + // RemoveServerInfo removes DDL server information from PD. + RemoveServerInfo() error } type schemaVersionSyncer struct { @@ -115,7 +115,7 @@ func NewSchemaSyncer(etcdCli *clientv3.Client, id string) SchemaSyncer { return &schemaVersionSyncer{ etcdCli: etcdCli, selfSchemaVerPath: fmt.Sprintf("%s/%s", DDLAllSchemaVersions, id), - selfServerInfoPath: fmt.Sprintf("%s/%s", DDLServerInformation, id), + selfServerInfoPath: fmt.Sprintf("%s/%s", ServerInformation, id), } } @@ -169,11 +169,11 @@ func (s *schemaVersionSyncer) Init(ctx context.Context) error { return errors.Trace(err) } -// GetServerInfoFromPD implements SchemaSyncer.GetServerInfoFromPD interface. -func (s *schemaVersionSyncer) GetServerInfoFromPD(ctx context.Context, id string) (*util.DDLServerInfo, error) { +// GetServerInfo implements SchemaSyncer.GetServerInfo interface. +func (s *schemaVersionSyncer) GetServerInfo(ctx context.Context, id string) (*util.ServerInfo, error) { var err error var resp *clientv3.GetResponse - ddlPath := fmt.Sprintf("%s/%s", DDLServerInformation, id) + ddlPath := fmt.Sprintf("%s/%s", ServerInformation, id) for { if isContextDone(ctx) { err = errors.Trace(ctx.Err()) @@ -187,7 +187,7 @@ func (s *schemaVersionSyncer) GetServerInfoFromPD(ctx context.Context, id string continue } if len(resp.Kvs) > 0 { - info := &util.DDLServerInfo{} + info := &util.ServerInfo{} err := json.Unmarshal(resp.Kvs[0].Value, info) if err != nil { log.Infof("[syncer] get ddl server info, ddl %s json.Unmarshal %v failed %v.", resp.Kvs[0].Key, resp.Kvs[0].Value, err) @@ -198,10 +198,10 @@ func (s *schemaVersionSyncer) GetServerInfoFromPD(ctx context.Context, id string } } -// GetAllServerInfoFromPD implements SchemaSyncer.GetAllServerInfoFromPD interface. -func (s *schemaVersionSyncer) GetAllServerInfoFromPD(ctx context.Context) (map[string]*util.DDLServerInfo, error) { +// GetAllServerInfo implements SchemaSyncer.GetAllServerInfo interface. +func (s *schemaVersionSyncer) GetAllServerInfo(ctx context.Context) (map[string]*util.ServerInfo, error) { var err error - allDDLInfo := make(map[string]*util.DDLServerInfo) + allDDLInfo := make(map[string]*util.ServerInfo) for { if isContextDone(ctx) { // ctx is canceled or timeout. @@ -209,7 +209,7 @@ func (s *schemaVersionSyncer) GetAllServerInfoFromPD(ctx context.Context) (map[s return nil, err } - resp, err := s.etcdCli.Get(ctx, DDLServerInformation, clientv3.WithPrefix()) + resp, err := s.etcdCli.Get(ctx, ServerInformation, clientv3.WithPrefix()) if err != nil { log.Infof("[syncer] get all ddl server info failed %v, continue checking.", err) time.Sleep(200 * time.Millisecond) @@ -217,7 +217,7 @@ func (s *schemaVersionSyncer) GetAllServerInfoFromPD(ctx context.Context) (map[s } for _, kv := range resp.Kvs { - info := &util.DDLServerInfo{} + info := &util.ServerInfo{} err := json.Unmarshal(kv.Value, info) if err != nil { log.Infof("[syncer] get all ddl server info, ddl %s json.Unmarshal %v failed %v.", kv.Key, kv.Value, err) @@ -229,8 +229,8 @@ func (s *schemaVersionSyncer) GetAllServerInfoFromPD(ctx context.Context) (map[s } } -// StoreSelfServerInfoToPD implements SchemaSyncer.StoreSelfServerInfoToPD interface. -func (s *schemaVersionSyncer) StoreSelfServerInfoToPD(ctx context.Context, info *util.DDLServerInfo) error { +// StoreServerInfo implements SchemaSyncer.StoreServerInfo interface. +func (s *schemaVersionSyncer) StoreServerInfo(ctx context.Context, info *util.ServerInfo) error { infoBuf, err := json.Marshal(info) if err != nil { return errors.Trace(err) @@ -239,8 +239,8 @@ func (s *schemaVersionSyncer) StoreSelfServerInfoToPD(ctx context.Context, info return errors.Trace(err) } -// RemoveSelfServerInfoFromPD implements SchemaSyncer.RemoveSelfServerInfoFromPD interface. -func (s *schemaVersionSyncer) RemoveSelfServerInfoFromPD() error { +// RemoveServerInfo implements SchemaSyncer.RemoveServerInfo interface. +func (s *schemaVersionSyncer) RemoveServerInfo() error { var err error ctx := context.Background() for i := 0; i < keyOpDefaultRetryCnt; i++ { diff --git a/ddl/util/util.go b/ddl/util/util.go index 1330c2681130f..b72f896a50e4c 100644 --- a/ddl/util/util.go +++ b/ddl/util/util.go @@ -110,10 +110,10 @@ func UpdateDeleteRange(ctx sessionctx.Context, dr DelRangeTask, newStartKey, old return errors.Trace(err) } -// DDLServerInfo is DDL server static information. -// DDLServerInfo will store into PD when server start up and delete when DDL closed. -// It will not update when DDL server running. So please only put static information in DDLServerInfo struct. -type DDLServerInfo struct { +// ServerInfo is DDL server static information. +// ServerInfo will store into PD when server start up and delete when DDL closed. +// It will not update when DDL server running. So please only put static information in ServerInfo struct. +type ServerInfo struct { ServerVersionInfo ID string `json:"ddl_id"` IP string `json:"ip"` diff --git a/domain/info.go b/domain/info.go index 085225fcb6bc1..8eedc20a4eedb 100644 --- a/domain/info.go +++ b/domain/info.go @@ -25,9 +25,9 @@ import ( ) //GetServerInfo gets self DDL server static information. -func GetServerInfo(ddlID string) *util.DDLServerInfo { +func GetServerInfo(ddlID string) *util.ServerInfo { cfg := config.GetGlobalConfig() - info := &util.DDLServerInfo{ + info := &util.ServerInfo{ ID: ddlID, IP: cfg.AdvertiseAddress, StatusPort: cfg.Status.StatusPort, @@ -39,13 +39,13 @@ func GetServerInfo(ddlID string) *util.DDLServerInfo { } // GetOwnerServerInfo gets owner DDL server static information from PD. -func GetOwnerServerInfo(d ddl.DDL) (*util.DDLServerInfo, error) { +func GetOwnerServerInfo(d ddl.DDL) (*util.ServerInfo, error) { ctx := context.Background() ddlOwnerID, err := d.OwnerManager().GetOwnerID(ctx) if err != nil { return nil, errors.Trace(err) } - ownerInfo, err := d.SchemaSyncer().GetServerInfoFromPD(ctx, ddlOwnerID) + ownerInfo, err := d.SchemaSyncer().GetServerInfo(ctx, ddlOwnerID) if err != nil { return nil, errors.Trace(err) } @@ -53,25 +53,25 @@ func GetOwnerServerInfo(d ddl.DDL) (*util.DDLServerInfo, error) { } // GetAllServerInfo gets all DDL servers static information from PD. -func GetAllServerInfo(d ddl.DDL) (map[string]*util.DDLServerInfo, error) { +func GetAllServerInfo(d ddl.DDL) (map[string]*util.ServerInfo, error) { ctx := context.Background() - AllDDLServerInfo, err := d.SchemaSyncer().GetAllServerInfoFromPD(ctx) + AllServerInfo, err := d.SchemaSyncer().GetAllServerInfo(ctx) if err != nil { return nil, errors.Trace(err) } - return AllDDLServerInfo, nil + return AllServerInfo, nil } -// StoreServerInfoToPD stores self DDL server static information to PD when domain Init. +// StoreServerInfo stores self DDL server static information to PD when domain Init. func StoreServerInfoToPD(d ddl.DDL) error { info := GetServerInfo(d.GetID()) ctx := context.Background() - return d.SchemaSyncer().StoreSelfServerInfoToPD(ctx, info) + return d.SchemaSyncer().StoreServerInfo(ctx, info) } -// RemoveServerInfoFromPD remove self DDL server static information from PD when domain close. +// RemoveServerInfo remove self DDL server static information from PD when domain close. func RemoveServerInfoFromPD(d ddl.DDL) { - err := d.SchemaSyncer().RemoveSelfServerInfoFromPD() + err := d.SchemaSyncer().RemoveServerInfo() if err != nil { log.Errorf("[ddl] remove self server info failed %v", err) } diff --git a/server/http_handler.go b/server/http_handler.go index 488aee49595e4..8e0f6713d8ffe 100644 --- a/server/http_handler.go +++ b/server/http_handler.go @@ -1261,9 +1261,9 @@ func (h *mvccTxnHandler) handleMvccGetByTxn(params map[string]string) (interface // ReportServerInfo is only used to report the servers info when do http request type ReportServerInfo struct { - IsOwner bool `json:"is_owner"` - SelfServerInfo *util.DDLServerInfo `json:"self_server_info"` - OwnerServerInfo *util.DDLServerInfo `json:"owner_server_info,omitempty"` + IsOwner bool `json:"is_owner"` + SelfServerInfo *util.ServerInfo `json:"self_server_info"` + OwnerServerInfo *util.ServerInfo `json:"owner_server_info,omitempty"` } // ServeHTTP handles request of ddl server info. @@ -1297,11 +1297,11 @@ func (h ddlServerInfoHandler) ServeHTTP(w http.ResponseWriter, req *http.Request // ClusterServerInfo is only used to report cluster servers info when do http request type ClusterServerInfo struct { - ServersNum int `json:"servers_num,omitempty"` - OwnerID string `json:"owner_id"` - IsAllServerVersionConsistent bool `json:"is_all_server_version_consistent,omitempty"` - AllServersDiffVersions []util.ServerVersionInfo `json:"all_servers_diff_versions,omitempty"` - AllServersInfo map[string]*util.DDLServerInfo `json:"all_servers_info,omitempty"` + ServersNum int `json:"servers_num,omitempty"` + OwnerID string `json:"owner_id"` + IsAllServerVersionConsistent bool `json:"is_all_server_version_consistent,omitempty"` + AllServersDiffVersions []util.ServerVersionInfo `json:"all_servers_diff_versions,omitempty"` + AllServersInfo map[string]*util.ServerInfo `json:"all_servers_info,omitempty"` } // ServeHTTP handles request of all ddl servers info. diff --git a/server/http_status.go b/server/http_status.go index 53b59270525bb..3662a81fab821 100644 --- a/server/http_status.go +++ b/server/http_status.go @@ -52,8 +52,10 @@ func (s *Server) startHTTPServer() { router.Handle("/schema/{db}/{table}", schemaHandler{tikvHandlerTool}) router.Handle("/tables/{colID}/{colTp}/{colFlag}/{colLen}", valueHandler{}) router.Handle("/ddl/history", ddlHistoryJobHandler{tikvHandlerTool}) - router.Handle("/ddl/info", ddlServerInfoHandler{tikvHandlerTool}) - router.Handle("/ddl/info/all", ddlAllServerInfoHandler{tikvHandlerTool}) + + // HTTP path for get server info + router.Handle("/info", ddlServerInfoHandler{tikvHandlerTool}) + router.Handle("/info/all", ddlAllServerInfoHandler{tikvHandlerTool}) if s.cfg.Store == "tikv" { // HTTP path for tikv router.Handle("/tables/{db}/{table}/regions", tableHandler{tikvHandlerTool, opTableRegions}) From 5fdba0b7060dd18c6879cf593f727219a3db054e Mon Sep 17 00:00:00 2001 From: crazycs Date: Thu, 2 Aug 2018 21:58:53 +0800 Subject: [PATCH 14/40] refine code --- ddl/syncer.go | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/ddl/syncer.go b/ddl/syncer.go index 6224c533345d3..00728b2a198b9 100644 --- a/ddl/syncer.go +++ b/ddl/syncer.go @@ -242,16 +242,7 @@ func (s *schemaVersionSyncer) StoreServerInfo(ctx context.Context, info *util.Se // RemoveServerInfo implements SchemaSyncer.RemoveServerInfo interface. func (s *schemaVersionSyncer) RemoveServerInfo() error { var err error - ctx := context.Background() - for i := 0; i < keyOpDefaultRetryCnt; i++ { - childCtx, cancel := context.WithTimeout(ctx, keyOpDefaultTimeout) - _, err = s.etcdCli.Delete(childCtx, s.selfServerInfoPath) - cancel() - if err == nil { - return nil - } - log.Warnf("[syncer] remove server info path %s failed %v no.%d", s.selfServerInfoPath, err, i) - } + err = removePath(s.selfServerInfoPath, s.etcdCli) return errors.Trace(err) } @@ -341,15 +332,21 @@ func (s *schemaVersionSyncer) RemoveSelfVersionPath() error { metrics.DeploySyncerHistogram.WithLabelValues(metrics.SyncerClear, metrics.RetLabel(err)).Observe(time.Since(startTime).Seconds()) }() + err = removePath(s.selfSchemaVerPath, s.etcdCli) + return errors.Trace(err) +} + +func removePath(path string,etcdCli *clientv3.Client) error { + var err error ctx := context.Background() for i := 0; i < keyOpDefaultRetryCnt; i++ { childCtx, cancel := context.WithTimeout(ctx, keyOpDefaultTimeout) - _, err = s.etcdCli.Delete(childCtx, s.selfSchemaVerPath) + _, err = etcdCli.Delete(childCtx, path) cancel() if err == nil { return nil } - log.Warnf("[syncer] remove schema version path %s failed %v no.%d", s.selfSchemaVerPath, err, i) + log.Warnf("[syncer] remove path %s failed %v no.%d", path, err, i) } return errors.Trace(err) } From 73ea84fb9e5266eb2b818b5df9fa175dbb25feea Mon Sep 17 00:00:00 2001 From: crazycs Date: Thu, 2 Aug 2018 22:18:11 +0800 Subject: [PATCH 15/40] add http test --- server/http_handler_test.go | 40 +++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/server/http_handler_test.go b/server/http_handler_test.go index 3404055719e82..cb45eab3b31d3 100644 --- a/server/http_handler_test.go +++ b/server/http_handler_test.go @@ -46,6 +46,7 @@ import ( "github.com/pingcap/tidb/tablecodec" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/codec" + "github.com/pingcap/tidb/util/printer" log "github.com/sirupsen/logrus" ) @@ -645,3 +646,42 @@ func (ts *HTTPHandlerTestSuite) TestPprof(c *C) { } log.Fatalf("Failed to get profile for %d retries in every 10 ms", retryTime) } + +func (ts *HTTPHandlerTestSuite) TestServerInfo(c *C) { + ts.startServer(c) + defer ts.stopServer(c) + resp, err := http.Get("http://127.0.0.1:10090/info") + c.Assert(err, IsNil) + defer resp.Body.Close() + c.Assert(resp.StatusCode, Equals, http.StatusOK) + decoder := json.NewDecoder(resp.Body) + + serverInfo := ReportServerInfo{} + err = decoder.Decode(&serverInfo) + c.Assert(err, IsNil) + + cfg := config.GetGlobalConfig() + c.Assert(serverInfo.IsOwner, IsTrue) + c.Assert(serverInfo.SelfServerInfo.IP, Equals, cfg.AdvertiseAddress) + c.Assert(serverInfo.SelfServerInfo.StatusPort, Equals, cfg.Status.StatusPort) + c.Assert(serverInfo.SelfServerInfo.Lease, Equals, cfg.Lease) + c.Assert(serverInfo.SelfServerInfo.Version, Equals, mysql.ServerVersion) + c.Assert(serverInfo.SelfServerInfo.GitHash, Equals, printer.TiDBGitHash) +} + +func (ts *HTTPHandlerTestSuite) TestAllServerInfo(c *C) { + ts.startServer(c) + defer ts.stopServer(c) + resp, err := http.Get("http://127.0.0.1:10090/info/all") + c.Assert(err, IsNil) + defer resp.Body.Close() + c.Assert(resp.StatusCode, Equals, http.StatusOK) + decoder := json.NewDecoder(resp.Body) + + allServerInfo := ClusterServerInfo{} + err = decoder.Decode(&allServerInfo) + c.Assert(err, IsNil) + + c.Assert(allServerInfo.IsAllServerVersionConsistent, IsTrue) + c.Assert(allServerInfo.ServersNum, Equals, 1) +} From 440953f64c9f68d4f14800f12e31e37189b7d7c6 Mon Sep 17 00:00:00 2001 From: crazycs Date: Fri, 3 Aug 2018 00:13:27 +0800 Subject: [PATCH 16/40] fmt code --- ddl/syncer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddl/syncer.go b/ddl/syncer.go index 00728b2a198b9..6b2ee4e3c149a 100644 --- a/ddl/syncer.go +++ b/ddl/syncer.go @@ -336,7 +336,7 @@ func (s *schemaVersionSyncer) RemoveSelfVersionPath() error { return errors.Trace(err) } -func removePath(path string,etcdCli *clientv3.Client) error { +func removePath(path string, etcdCli *clientv3.Client) error { var err error ctx := context.Background() for i := 0; i < keyOpDefaultRetryCnt; i++ { From 9815df26bcaf201df370f9671aa062e0be42d226 Mon Sep 17 00:00:00 2001 From: crazycs Date: Sun, 5 Aug 2018 16:59:20 +0800 Subject: [PATCH 17/40] add test and comment --- docs/tidb_http_api.md | 4 ++-- domain/info.go | 12 ++++++------ server/http_handler.go | 4 ++-- server/http_handler_test.go | 26 ++++++++++++++++++++++++++ 4 files changed, 36 insertions(+), 10 deletions(-) diff --git a/docs/tidb_http_api.md b/docs/tidb_http_api.md index 73ab2a6cb4d78..8e8e76dc5f6c1 100644 --- a/docs/tidb_http_api.md +++ b/docs/tidb_http_api.md @@ -99,13 +99,13 @@ timezone.* 1. Get TiDB server information and owner server infomation. ```shell - curl http://{TiDBIP}:10080/ddl/info + curl http://{TiDBIP}:10080/info ``` 1. Get TiDB cluster all servers information. ```shell - curl http://{TiDBIP}:10080/ddl/info/all + curl http://{TiDBIP}:10080/info/all ``` 1. Enable/Disable TiDB server general log diff --git a/domain/info.go b/domain/info.go index 8eedc20a4eedb..1369167b6f9e1 100644 --- a/domain/info.go +++ b/domain/info.go @@ -38,8 +38,8 @@ func GetServerInfo(ddlID string) *util.ServerInfo { return info } -// GetOwnerServerInfo gets owner DDL server static information from PD. -func GetOwnerServerInfo(d ddl.DDL) (*util.ServerInfo, error) { +// GetOwnerServerInfoFromPD gets owner DDL server static information from PD. +func GetOwnerServerInfoFromPD(d ddl.DDL) (*util.ServerInfo, error) { ctx := context.Background() ddlOwnerID, err := d.OwnerManager().GetOwnerID(ctx) if err != nil { @@ -52,8 +52,8 @@ func GetOwnerServerInfo(d ddl.DDL) (*util.ServerInfo, error) { return ownerInfo, nil } -// GetAllServerInfo gets all DDL servers static information from PD. -func GetAllServerInfo(d ddl.DDL) (map[string]*util.ServerInfo, error) { +// GetAllServerInfoFromPD gets all DDL servers static information from PD. +func GetAllServerInfoFromPD(d ddl.DDL) (map[string]*util.ServerInfo, error) { ctx := context.Background() AllServerInfo, err := d.SchemaSyncer().GetAllServerInfo(ctx) if err != nil { @@ -62,14 +62,14 @@ func GetAllServerInfo(d ddl.DDL) (map[string]*util.ServerInfo, error) { return AllServerInfo, nil } -// StoreServerInfo stores self DDL server static information to PD when domain Init. +// StoreServerInfoToPD stores self DDL server static information to PD when domain Init. func StoreServerInfoToPD(d ddl.DDL) error { info := GetServerInfo(d.GetID()) ctx := context.Background() return d.SchemaSyncer().StoreServerInfo(ctx, info) } -// RemoveServerInfo remove self DDL server static information from PD when domain close. +// RemoveServerInfoFromPD remove self DDL server static information from PD when domain close. func RemoveServerInfoFromPD(d ddl.DDL) { err := d.SchemaSyncer().RemoveServerInfo() if err != nil { diff --git a/server/http_handler.go b/server/http_handler.go index 8e0f6713d8ffe..079e382002340 100644 --- a/server/http_handler.go +++ b/server/http_handler.go @@ -1285,7 +1285,7 @@ func (h ddlServerInfoHandler) ServeHTTP(w http.ResponseWriter, req *http.Request SelfServerInfo: selfInfo, } if !reportServerInfo.IsOwner { - ownerInfo, err := domain.GetOwnerServerInfo(ddl) + ownerInfo, err := domain.GetOwnerServerInfoFromPD(ddl) if err != nil { writeError(w, errors.New("ddl server information not found")) return @@ -1313,7 +1313,7 @@ func (h ddlAllServerInfoHandler) ServeHTTP(w http.ResponseWriter, req *http.Requ } ddl := do.DDL() - allServersInfo, err := domain.GetAllServerInfo(ddl) + allServersInfo, err := domain.GetAllServerInfoFromPD(ddl) if err != nil { writeError(w, errors.New("ddl server information not found")) return diff --git a/server/http_handler_test.go b/server/http_handler_test.go index cb45eab3b31d3..4c8d06b85fc1f 100644 --- a/server/http_handler_test.go +++ b/server/http_handler_test.go @@ -667,6 +667,14 @@ func (ts *HTTPHandlerTestSuite) TestServerInfo(c *C) { c.Assert(serverInfo.SelfServerInfo.Lease, Equals, cfg.Lease) c.Assert(serverInfo.SelfServerInfo.Version, Equals, mysql.ServerVersion) c.Assert(serverInfo.SelfServerInfo.GitHash, Equals, printer.TiDBGitHash) + + s, _ := session.CreateSession(ts.server.newTikvHandlerTool().store.(kv.Storage)) + defer s.Close() + store := domain.GetDomain(s.(sessionctx.Context)).Store() + do, err := session.GetDomain(store.(kv.Storage)) + c.Assert(err, IsNil) + ddl := do.DDL() + c.Assert(serverInfo.SelfServerInfo.ID, Equals, ddl.GetID()) } func (ts *HTTPHandlerTestSuite) TestAllServerInfo(c *C) { @@ -684,4 +692,22 @@ func (ts *HTTPHandlerTestSuite) TestAllServerInfo(c *C) { c.Assert(allServerInfo.IsAllServerVersionConsistent, IsTrue) c.Assert(allServerInfo.ServersNum, Equals, 1) + + s, _ := session.CreateSession(ts.server.newTikvHandlerTool().store.(kv.Storage)) + defer s.Close() + store := domain.GetDomain(s.(sessionctx.Context)).Store() + do, err := session.GetDomain(store.(kv.Storage)) + c.Assert(err, IsNil) + ddl := do.DDL() + c.Assert(allServerInfo.OwnerID, Equals, ddl.GetID()) + serverInfo, ok := allServerInfo.AllServersInfo[ddl.GetID()] + c.Assert(ok, Equals, true) + + cfg := config.GetGlobalConfig() + c.Assert(serverInfo.IP, Equals, cfg.AdvertiseAddress) + c.Assert(serverInfo.StatusPort, Equals, cfg.Status.StatusPort) + c.Assert(serverInfo.Lease, Equals, cfg.Lease) + c.Assert(serverInfo.Version, Equals, mysql.ServerVersion) + c.Assert(serverInfo.GitHash, Equals, printer.TiDBGitHash) + c.Assert(serverInfo.ID, Equals, ddl.GetID()) } From 321cf318bd15c40a4ad4bc789995b5827d5edf47 Mon Sep 17 00:00:00 2001 From: crazycs Date: Sun, 5 Aug 2018 17:05:12 +0800 Subject: [PATCH 18/40] update test --- server/http_handler_test.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/server/http_handler_test.go b/server/http_handler_test.go index 4c8d06b85fc1f..afaa3750751cf 100644 --- a/server/http_handler_test.go +++ b/server/http_handler_test.go @@ -668,9 +668,7 @@ func (ts *HTTPHandlerTestSuite) TestServerInfo(c *C) { c.Assert(serverInfo.SelfServerInfo.Version, Equals, mysql.ServerVersion) c.Assert(serverInfo.SelfServerInfo.GitHash, Equals, printer.TiDBGitHash) - s, _ := session.CreateSession(ts.server.newTikvHandlerTool().store.(kv.Storage)) - defer s.Close() - store := domain.GetDomain(s.(sessionctx.Context)).Store() + store := ts.server.newTikvHandlerTool().store.(kv.Storage) do, err := session.GetDomain(store.(kv.Storage)) c.Assert(err, IsNil) ddl := do.DDL() @@ -693,9 +691,7 @@ func (ts *HTTPHandlerTestSuite) TestAllServerInfo(c *C) { c.Assert(allServerInfo.IsAllServerVersionConsistent, IsTrue) c.Assert(allServerInfo.ServersNum, Equals, 1) - s, _ := session.CreateSession(ts.server.newTikvHandlerTool().store.(kv.Storage)) - defer s.Close() - store := domain.GetDomain(s.(sessionctx.Context)).Store() + store := ts.server.newTikvHandlerTool().store.(kv.Storage) do, err := session.GetDomain(store.(kv.Storage)) c.Assert(err, IsNil) ddl := do.DDL() From 027c9832e4aef57782c8e0ab1c9179319426d9f1 Mon Sep 17 00:00:00 2001 From: crazycs Date: Sun, 5 Aug 2018 17:46:18 +0800 Subject: [PATCH 19/40] refactor code --- ddl/syncer.go | 65 ++++++++++++++++++++++++--------------------------- 1 file changed, 30 insertions(+), 35 deletions(-) diff --git a/ddl/syncer.go b/ddl/syncer.go index 6b2ee4e3c149a..ab15988cdac1b 100644 --- a/ddl/syncer.go +++ b/ddl/syncer.go @@ -169,64 +169,59 @@ func (s *schemaVersionSyncer) Init(ctx context.Context) error { return errors.Trace(err) } -// GetServerInfo implements SchemaSyncer.GetServerInfo interface. -func (s *schemaVersionSyncer) GetServerInfo(ctx context.Context, id string) (*util.ServerInfo, error) { +func (s *schemaVersionSyncer) getInfo(ctx context.Context, key string, opts ...clientv3.OpOption) (map[string]*util.ServerInfo, error) { var err error - var resp *clientv3.GetResponse - ddlPath := fmt.Sprintf("%s/%s", ServerInformation, id) + allInfo := make(map[string]*util.ServerInfo) + for { if isContextDone(ctx) { err = errors.Trace(ctx.Err()) return nil, err } - resp, err = s.etcdCli.Get(ctx, ddlPath) + resp, err := s.etcdCli.Get(ctx, key, opts...) if err != nil { - log.Infof("[syncer] get ddl server info, ddl %s failed %v, continue checking.", ddlPath, err) + log.Infof("[syncer] get %s failed %v, continue checking.", key, err) time.Sleep(200 * time.Millisecond) continue } - if len(resp.Kvs) > 0 { + + for _, kv := range resp.Kvs { info := &util.ServerInfo{} - err := json.Unmarshal(resp.Kvs[0].Value, info) + err := json.Unmarshal(kv.Value, info) if err != nil { - log.Infof("[syncer] get ddl server info, ddl %s json.Unmarshal %v failed %v.", resp.Kvs[0].Key, resp.Kvs[0].Value, err) + log.Infof("[syncer] get %s, json.Unmarshal %v failed %v.", kv.Key, kv.Value, err) return nil, errors.Trace(err) } - return info, nil + allInfo[info.ID] = info } + return allInfo, nil } } +// GetServerInfo implements SchemaSyncer.GetServerInfo interface. +func (s *schemaVersionSyncer) GetServerInfo(ctx context.Context, id string) (*util.ServerInfo, error) { + var err error + ddlPath := fmt.Sprintf("%s/%s", ServerInformation, id) + allInfo, err := s.getInfo(ctx, ddlPath) + if err != nil { + return nil, errors.Trace(err) + } + info, ok := allInfo[id] + if !ok { + return nil, errors.New(fmt.Sprintf("[syncer] get %s failed", ddlPath)) + } + return info, nil +} + // GetAllServerInfo implements SchemaSyncer.GetAllServerInfo interface. func (s *schemaVersionSyncer) GetAllServerInfo(ctx context.Context) (map[string]*util.ServerInfo, error) { var err error - allDDLInfo := make(map[string]*util.ServerInfo) - for { - if isContextDone(ctx) { - // ctx is canceled or timeout. - err = errors.Trace(ctx.Err()) - return nil, err - } - - resp, err := s.etcdCli.Get(ctx, ServerInformation, clientv3.WithPrefix()) - if err != nil { - log.Infof("[syncer] get all ddl server info failed %v, continue checking.", err) - time.Sleep(200 * time.Millisecond) - continue - } - - for _, kv := range resp.Kvs { - info := &util.ServerInfo{} - err := json.Unmarshal(kv.Value, info) - if err != nil { - log.Infof("[syncer] get all ddl server info, ddl %s json.Unmarshal %v failed %v.", kv.Key, kv.Value, err) - return nil, errors.Trace(err) - } - allDDLInfo[info.ID] = info - } - return allDDLInfo, nil + allInfo, err := s.getInfo(ctx, ServerInformation, clientv3.WithPrefix()) + if err != nil { + return nil, errors.Trace(err) } + return allInfo, nil } // StoreServerInfo implements SchemaSyncer.StoreServerInfo interface. From c2ff1e59e100a97c5229b73700887387844537b8 Mon Sep 17 00:00:00 2001 From: crazycs Date: Tue, 7 Aug 2018 01:46:39 +0800 Subject: [PATCH 20/40] refactor code --- ddl/mock.go | 26 ------- ddl/syncer.go | 106 ++++---------------------- ddl/util/util.go | 17 ----- domain/domain.go | 13 +++- domain/info.go | 146 ++++++++++++++++++++++++++++++------ server/http_handler.go | 42 +++++------ server/http_handler_test.go | 4 +- 7 files changed, 169 insertions(+), 185 deletions(-) diff --git a/ddl/mock.go b/ddl/mock.go index b6f600fe558fb..70562e9a496cd 100644 --- a/ddl/mock.go +++ b/ddl/mock.go @@ -20,7 +20,6 @@ import ( "github.com/coreos/etcd/clientv3" "github.com/juju/errors" "github.com/pingcap/tidb/ast" - "github.com/pingcap/tidb/ddl/util" "github.com/pingcap/tidb/model" "github.com/pingcap/tidb/sessionctx" "golang.org/x/net/context" @@ -33,8 +32,6 @@ const mockCheckVersInterval = 2 * time.Millisecond type mockSchemaSyncer struct { selfSchemaVersion int64 globalVerCh chan clientv3.WatchResponse - // selfServerInfo used to save self DDL server information. - selfServerInfo *util.ServerInfo } // NewMockSchemaSyncer creates a new mock SchemaSyncer. @@ -105,29 +102,6 @@ func (s *mockSchemaSyncer) OwnerCheckAllVersions(ctx context.Context, latestVer } } -// GetServerInfo implements SchemaSyncer.GetServerInfo interface. -func (s *mockSchemaSyncer) GetServerInfo(ctx context.Context, _ string) (*util.ServerInfo, error) { - return s.selfServerInfo, nil -} - -// GetAllServerInfo implements SchemaSyncer.GetAllServerInfo interface. -func (s *mockSchemaSyncer) GetAllServerInfo(ctx context.Context) (map[string]*util.ServerInfo, error) { - allDDLInfo := make(map[string]*util.ServerInfo) - allDDLInfo[s.selfServerInfo.ID] = s.selfServerInfo - return allDDLInfo, nil -} - -// StoreServerInfo implements SchemaSyncer.StoreServerInfo interface. -func (s *mockSchemaSyncer) StoreServerInfo(ctx context.Context, info *util.ServerInfo) error { - s.selfServerInfo = info - return nil -} - -// RemoveServerInfo implements SchemaSyncer.RemoveServerInfo interface. -func (s *mockSchemaSyncer) RemoveServerInfo() error { - return nil -} - type mockDelRange struct { } diff --git a/ddl/syncer.go b/ddl/syncer.go index ab15988cdac1b..5bd022a266f86 100644 --- a/ddl/syncer.go +++ b/ddl/syncer.go @@ -14,7 +14,6 @@ package ddl import ( - "encoding/json" "fmt" "math" "strconv" @@ -24,10 +23,8 @@ import ( "github.com/coreos/etcd/clientv3" "github.com/coreos/etcd/clientv3/concurrency" "github.com/juju/errors" - "github.com/pingcap/tidb/ddl/util" "github.com/pingcap/tidb/metrics" "github.com/pingcap/tidb/owner" - "github.com/pingcap/tidb/util/hack" log "github.com/sirupsen/logrus" "golang.org/x/net/context" ) @@ -36,8 +33,6 @@ const ( // DDLAllSchemaVersions is the path on etcd that is used to store all servers current schema versions. // It's exported for testing. DDLAllSchemaVersions = "/tidb/ddl/all_schema_versions" - // ServerInformation store DDL server information such as IP, port and so on. - ServerInformation = "/tidb/ddl/info" // DDLGlobalSchemaVersion is the path on etcd that is used to store the latest schema versions. // It's exported for testing. DDLGlobalSchemaVersion = "/tidb/ddl/global_schema_version" @@ -88,15 +83,6 @@ type SchemaSyncer interface { // the latest schema version. If the result is false, wait for a while and check again util the processing time reach 2 * lease. // It returns until all servers' versions are equal to the latest version or the ctx is done. OwnerCheckAllVersions(ctx context.Context, latestVer int64) error - - // GetServerInfo gets the DDL_id server information from PD. - GetServerInfo(ctx context.Context, id string) (*util.ServerInfo, error) - // GetAllServerInfo gets all DDL servers information from PD. - GetAllServerInfo(ctx context.Context) (map[string]*util.ServerInfo, error) - // StoreServerInfo stores DDL server information to PD. - StoreServerInfo(ctx context.Context, info *util.ServerInfo) error - // RemoveServerInfo removes DDL server information from PD. - RemoveServerInfo() error } type schemaVersionSyncer struct { @@ -113,13 +99,18 @@ type schemaVersionSyncer struct { // NewSchemaSyncer creates a new SchemaSyncer. func NewSchemaSyncer(etcdCli *clientv3.Client, id string) SchemaSyncer { return &schemaVersionSyncer{ - etcdCli: etcdCli, - selfSchemaVerPath: fmt.Sprintf("%s/%s", DDLAllSchemaVersions, id), - selfServerInfoPath: fmt.Sprintf("%s/%s", ServerInformation, id), + etcdCli: etcdCli, + selfSchemaVerPath: fmt.Sprintf("%s/%s", DDLAllSchemaVersions, id), } } func (s *schemaVersionSyncer) putKV(ctx context.Context, retryCnt int, key, val string, + opts ...clientv3.OpOption) error { + return PutKV(ctx, s.etcdCli, retryCnt, key, val, opts...) +} + +// PutKV put key value to etcd. +func PutKV(ctx context.Context, etcdCli *clientv3.Client, retryCnt int, key, val string, opts ...clientv3.OpOption) error { var err error for i := 0; i < retryCnt; i++ { @@ -128,12 +119,12 @@ func (s *schemaVersionSyncer) putKV(ctx context.Context, retryCnt int, key, val } childCtx, cancel := context.WithTimeout(ctx, keyOpDefaultTimeout) - _, err = s.etcdCli.Put(childCtx, key, val, opts...) + _, err = etcdCli.Put(childCtx, key, val, opts...) cancel() if err == nil { return nil } - log.Warnf("[syncer] put schema version %s failed %v no.%d", val, err, i) + log.Warnf("[syncer] put key: %s value: %s failed %v no.%d", key, val, err, i) time.Sleep(keyOpRetryInterval) } return errors.Trace(err) @@ -169,78 +160,6 @@ func (s *schemaVersionSyncer) Init(ctx context.Context) error { return errors.Trace(err) } -func (s *schemaVersionSyncer) getInfo(ctx context.Context, key string, opts ...clientv3.OpOption) (map[string]*util.ServerInfo, error) { - var err error - allInfo := make(map[string]*util.ServerInfo) - - for { - if isContextDone(ctx) { - err = errors.Trace(ctx.Err()) - return nil, err - } - - resp, err := s.etcdCli.Get(ctx, key, opts...) - if err != nil { - log.Infof("[syncer] get %s failed %v, continue checking.", key, err) - time.Sleep(200 * time.Millisecond) - continue - } - - for _, kv := range resp.Kvs { - info := &util.ServerInfo{} - err := json.Unmarshal(kv.Value, info) - if err != nil { - log.Infof("[syncer] get %s, json.Unmarshal %v failed %v.", kv.Key, kv.Value, err) - return nil, errors.Trace(err) - } - allInfo[info.ID] = info - } - return allInfo, nil - } -} - -// GetServerInfo implements SchemaSyncer.GetServerInfo interface. -func (s *schemaVersionSyncer) GetServerInfo(ctx context.Context, id string) (*util.ServerInfo, error) { - var err error - ddlPath := fmt.Sprintf("%s/%s", ServerInformation, id) - allInfo, err := s.getInfo(ctx, ddlPath) - if err != nil { - return nil, errors.Trace(err) - } - info, ok := allInfo[id] - if !ok { - return nil, errors.New(fmt.Sprintf("[syncer] get %s failed", ddlPath)) - } - return info, nil -} - -// GetAllServerInfo implements SchemaSyncer.GetAllServerInfo interface. -func (s *schemaVersionSyncer) GetAllServerInfo(ctx context.Context) (map[string]*util.ServerInfo, error) { - var err error - allInfo, err := s.getInfo(ctx, ServerInformation, clientv3.WithPrefix()) - if err != nil { - return nil, errors.Trace(err) - } - return allInfo, nil -} - -// StoreServerInfo implements SchemaSyncer.StoreServerInfo interface. -func (s *schemaVersionSyncer) StoreServerInfo(ctx context.Context, info *util.ServerInfo) error { - infoBuf, err := json.Marshal(info) - if err != nil { - return errors.Trace(err) - } - err = s.putKV(ctx, keyOpDefaultRetryCnt, s.selfServerInfoPath, hack.String(infoBuf)) - return errors.Trace(err) -} - -// RemoveServerInfo implements SchemaSyncer.RemoveServerInfo interface. -func (s *schemaVersionSyncer) RemoveServerInfo() error { - var err error - err = removePath(s.selfServerInfoPath, s.etcdCli) - return errors.Trace(err) -} - // Done implements SchemaSyncer.Done interface. func (s *schemaVersionSyncer) Done() <-chan struct{} { return s.session.Done() @@ -327,11 +246,12 @@ func (s *schemaVersionSyncer) RemoveSelfVersionPath() error { metrics.DeploySyncerHistogram.WithLabelValues(metrics.SyncerClear, metrics.RetLabel(err)).Observe(time.Since(startTime).Seconds()) }() - err = removePath(s.selfSchemaVerPath, s.etcdCli) + err = RemovePath(s.selfSchemaVerPath, s.etcdCli) return errors.Trace(err) } -func removePath(path string, etcdCli *clientv3.Client) error { +// RemovePath remove key value from etcd. +func RemovePath(path string, etcdCli *clientv3.Client) error { var err error ctx := context.Background() for i := 0; i < keyOpDefaultRetryCnt; i++ { diff --git a/ddl/util/util.go b/ddl/util/util.go index b72f896a50e4c..db3e09fdb3d3f 100644 --- a/ddl/util/util.go +++ b/ddl/util/util.go @@ -109,20 +109,3 @@ func UpdateDeleteRange(ctx sessionctx.Context, dr DelRangeTask, newStartKey, old _, err := ctx.(sqlexec.SQLExecutor).Execute(context.TODO(), sql) return errors.Trace(err) } - -// ServerInfo is DDL server static information. -// ServerInfo will store into PD when server start up and delete when DDL closed. -// It will not update when DDL server running. So please only put static information in ServerInfo struct. -type ServerInfo struct { - ServerVersionInfo - ID string `json:"ddl_id"` - IP string `json:"ip"` - StatusPort uint `json:"status_port"` - Lease string `json:"lease"` -} - -// ServerVersionInfo is the server version and git_hash. -type ServerVersionInfo struct { - Version string `json:"version"` - GitHash string `json:"git_hash"` -} diff --git a/domain/domain.go b/domain/domain.go index c2e47ef7d8b71..83f7d997207cf 100644 --- a/domain/domain.go +++ b/domain/domain.go @@ -52,6 +52,7 @@ type Domain struct { statsHandle unsafe.Pointer statsLease time.Duration ddl ddl.DDL + info *infoSyncer m sync.Mutex SchemaValidator SchemaValidator sysSessionPool *pools.ResourcePool @@ -251,6 +252,11 @@ func (do *Domain) DDL() ddl.DDL { return do.ddl } +// InfoSyncer gets infoSyncer from domain +func (do *Domain) InfoSyncer() *infoSyncer { + return do.info +} + // Store gets KV store from domain. func (do *Domain) Store() kv.Storage { return do.store @@ -390,9 +396,11 @@ func (do *Domain) mustRestartSyncer() error { // Close closes the Domain and release its resource. func (do *Domain) Close() { if do.ddl != nil { - RemoveServerInfoFromPD(do.ddl) terror.Log(errors.Trace(do.ddl.Stop())) } + if do.info != nil { + do.info.RemoveServerInfoFromPD() + } close(do.exit) if do.etcdClient != nil { terror.Log(errors.Trace(do.etcdClient.Close())) @@ -501,7 +509,8 @@ func (do *Domain) Init(ddlLease time.Duration, sysFactory func(*Domain) (pools.R if err != nil { return errors.Trace(err) } - err = StoreServerInfoToPD(do.ddl) + do.info = newInfoSyncer(do.ddl.GetID(), do.etcdClient) + err = do.info.StoreServerInfoToPD() if err != nil { return errors.Trace(err) } diff --git a/domain/info.go b/domain/info.go index 1369167b6f9e1..feef573b5bbcb 100644 --- a/domain/info.go +++ b/domain/info.go @@ -14,21 +14,61 @@ package domain import ( + "encoding/json" + "fmt" + "time" + + "github.com/coreos/etcd/clientv3" "github.com/juju/errors" "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/ddl" - "github.com/pingcap/tidb/ddl/util" "github.com/pingcap/tidb/mysql" + "github.com/pingcap/tidb/util/hack" "github.com/pingcap/tidb/util/printer" log "github.com/sirupsen/logrus" "golang.org/x/net/context" ) -//GetServerInfo gets self DDL server static information. -func GetServerInfo(ddlID string) *util.ServerInfo { +const ( + // ServerInformation store server information such as IP, port and so on. + ServerInformation = "/tidb/server/info" + + keyOpDefaultRetryCnt = 3 +) + +// infoSyncer stores server info to PD when server start and delete when server down. +type infoSyncer struct { + etcdCli *clientv3.Client + info *ServerInfo +} + +// ServerInfo is server static information. +// It will not update when server running. So please only put static information in ServerInfo struct. +type ServerInfo struct { + ServerVersionInfo + ID string `json:"ddl_id"` + IP string `json:"ip"` + StatusPort uint `json:"status_port"` + Lease string `json:"lease"` +} + +// ServerVersionInfo is the server version and git_hash. +type ServerVersionInfo struct { + Version string `json:"version"` + GitHash string `json:"git_hash"` +} + +func newInfoSyncer(id string, etcdCli *clientv3.Client) *infoSyncer { + return &infoSyncer{ + etcdCli: etcdCli, + info: getServerInfo(id), + } +} + +func getServerInfo(id string) *ServerInfo { cfg := config.GetGlobalConfig() - info := &util.ServerInfo{ - ID: ddlID, + info := &ServerInfo{ + ID: id, IP: cfg.AdvertiseAddress, StatusPort: cfg.Status.StatusPort, Lease: cfg.Lease, @@ -38,41 +78,99 @@ func GetServerInfo(ddlID string) *util.ServerInfo { return info } -// GetOwnerServerInfoFromPD gets owner DDL server static information from PD. -func GetOwnerServerInfoFromPD(d ddl.DDL) (*util.ServerInfo, error) { +//GetServerInfo gets self server static information. +func (is *infoSyncer) GetServerInfo() *ServerInfo { + return is.info +} + +// GetOwnerServerInfoFromPD gets owner server static information from PD. +func (is *infoSyncer) GetOwnerServerInfoFromPD(ownerID string) (*ServerInfo, error) { + if is.etcdCli == nil || ownerID == is.info.ID { + return is.info, nil + } ctx := context.Background() - ddlOwnerID, err := d.OwnerManager().GetOwnerID(ctx) + key := fmt.Sprintf("%s/%s", ServerInformation, ownerID) + allInfo, err := getInfo(ctx, is.etcdCli, key) if err != nil { return nil, errors.Trace(err) } - ownerInfo, err := d.SchemaSyncer().GetServerInfo(ctx, ddlOwnerID) - if err != nil { - return nil, errors.Trace(err) + info, ok := allInfo[ownerID] + if !ok { + return nil, errors.New(fmt.Sprintf("[syncer] get %s failed", key)) } - return ownerInfo, nil + return info, nil } -// GetAllServerInfoFromPD gets all DDL servers static information from PD. -func GetAllServerInfoFromPD(d ddl.DDL) (map[string]*util.ServerInfo, error) { +// GetAllServerInfoFromPD gets all servers static information from PD. +func (is *infoSyncer) GetAllServerInfoFromPD() (map[string]*ServerInfo, error) { + allInfo := make(map[string]*ServerInfo) + if is.etcdCli == nil { + allInfo[is.info.ID] = is.info + return allInfo, nil + } ctx := context.Background() - AllServerInfo, err := d.SchemaSyncer().GetAllServerInfo(ctx) + allInfo, err := getInfo(ctx, is.etcdCli, ServerInformation, clientv3.WithPrefix()) if err != nil { return nil, errors.Trace(err) } - return AllServerInfo, nil + return allInfo, nil } -// StoreServerInfoToPD stores self DDL server static information to PD when domain Init. -func StoreServerInfoToPD(d ddl.DDL) error { - info := GetServerInfo(d.GetID()) +// StoreServerInfoToPD stores self server static information to PD when domain Init. +func (is *infoSyncer) StoreServerInfoToPD() error { + if is.etcdCli == nil { + return nil + } + infoBuf, err := json.Marshal(is.info) + if err != nil { + return errors.Trace(err) + } ctx := context.Background() - return d.SchemaSyncer().StoreServerInfo(ctx, info) + key := fmt.Sprintf("%s/%s", ServerInformation, is.info.ID) + err = ddl.PutKV(ctx, is.etcdCli, keyOpDefaultRetryCnt, key, hack.String(infoBuf)) + return errors.Trace(err) } -// RemoveServerInfoFromPD remove self DDL server static information from PD when domain close. -func RemoveServerInfoFromPD(d ddl.DDL) { - err := d.SchemaSyncer().RemoveServerInfo() +// RemoveServerInfoFromPD remove self server static information from PD when domain close. +func (is *infoSyncer) RemoveServerInfoFromPD() { + if is.etcdCli == nil { + return + } + key := fmt.Sprintf("%s/%s", ServerInformation, is.info.ID) + err := ddl.RemovePath(key, is.etcdCli) if err != nil { - log.Errorf("[ddl] remove self server info failed %v", err) + log.Errorf("[infoSyncer] remove self server info failed %v", err) + } +} + +func getInfo(ctx context.Context, etcdCli *clientv3.Client, key string, opts ...clientv3.OpOption) (map[string]*ServerInfo, error) { + var err error + allInfo := make(map[string]*ServerInfo) + + for { + select { + case <-ctx.Done(): + err = errors.Trace(ctx.Err()) + return nil, err + default: + } + + resp, err := etcdCli.Get(ctx, key, opts...) + if err != nil { + log.Infof("[infoSyncer] get %s failed %v, continue checking.", key, err) + time.Sleep(200 * time.Millisecond) + continue + } + + for _, kv := range resp.Kvs { + info := &ServerInfo{} + err := json.Unmarshal(kv.Value, info) + if err != nil { + log.Infof("[infoSyncer] get %s, json.Unmarshal %v failed %v.", kv.Key, kv.Value, err) + return nil, errors.Trace(err) + } + allInfo[info.ID] = info + } + return allInfo, nil } } diff --git a/server/http_handler.go b/server/http_handler.go index 079e382002340..e2618e380fe16 100644 --- a/server/http_handler.go +++ b/server/http_handler.go @@ -32,7 +32,6 @@ import ( "github.com/pingcap/kvproto/pkg/kvrpcpb" "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/tidb/config" - "github.com/pingcap/tidb/ddl/util" "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/infoschema" "github.com/pingcap/tidb/kv" @@ -1259,11 +1258,11 @@ func (h *mvccTxnHandler) handleMvccGetByTxn(params map[string]string) (interface return h.getMvccByStartTs(uint64(startTS), startKey, endKey) } -// ReportServerInfo is only used to report the servers info when do http request -type ReportServerInfo struct { - IsOwner bool `json:"is_owner"` - SelfServerInfo *util.ServerInfo `json:"self_server_info"` - OwnerServerInfo *util.ServerInfo `json:"owner_server_info,omitempty"` +// reportServerInfo is only used to report the servers info when do http request +type reportServerInfo struct { + IsOwner bool `json:"is_owner"` + SelfServerInfo *domain.ServerInfo `json:"self_server_info"` + OwnerServerInfo *domain.ServerInfo `json:"owner_server_info,omitempty"` } // ServeHTTP handles request of ddl server info. @@ -1274,18 +1273,19 @@ func (h ddlServerInfoHandler) ServeHTTP(w http.ResponseWriter, req *http.Request return } ddl := do.DDL() + is := do.InfoSyncer() ownerID, err := ddl.OwnerManager().GetOwnerID(context.Background()) if err != nil { writeError(w, errors.New("ddl server information not found")) return } - selfInfo := domain.GetServerInfo(ddl.GetID()) - reportServerInfo := ReportServerInfo{ + selfInfo := is.GetServerInfo() + reportServerInfo := reportServerInfo{ IsOwner: selfInfo.ID == ownerID, SelfServerInfo: selfInfo, } if !reportServerInfo.IsOwner { - ownerInfo, err := domain.GetOwnerServerInfoFromPD(ddl) + ownerInfo, err := is.GetOwnerServerInfoFromPD(ownerID) if err != nil { writeError(w, errors.New("ddl server information not found")) return @@ -1295,13 +1295,13 @@ func (h ddlServerInfoHandler) ServeHTTP(w http.ResponseWriter, req *http.Request writeData(w, reportServerInfo) } -// ClusterServerInfo is only used to report cluster servers info when do http request -type ClusterServerInfo struct { - ServersNum int `json:"servers_num,omitempty"` - OwnerID string `json:"owner_id"` - IsAllServerVersionConsistent bool `json:"is_all_server_version_consistent,omitempty"` - AllServersDiffVersions []util.ServerVersionInfo `json:"all_servers_diff_versions,omitempty"` - AllServersInfo map[string]*util.ServerInfo `json:"all_servers_info,omitempty"` +// clusterServerInfo is only used to report cluster servers info when do http request +type clusterServerInfo struct { + ServersNum int `json:"servers_num,omitempty"` + OwnerID string `json:"owner_id"` + IsAllServerVersionConsistent bool `json:"is_all_server_version_consistent,omitempty"` + AllServersDiffVersions []domain.ServerVersionInfo `json:"all_servers_diff_versions,omitempty"` + AllServersInfo map[string]*domain.ServerInfo `json:"all_servers_info,omitempty"` } // ServeHTTP handles request of all ddl servers info. @@ -1312,8 +1312,8 @@ func (h ddlAllServerInfoHandler) ServeHTTP(w http.ResponseWriter, req *http.Requ return } ddl := do.DDL() - - allServersInfo, err := domain.GetAllServerInfoFromPD(ddl) + is := do.InfoSyncer() + allServersInfo, err := is.GetAllServerInfoFromPD() if err != nil { writeError(w, errors.New("ddl server information not found")) return @@ -1323,8 +1323,8 @@ func (h ddlAllServerInfoHandler) ServeHTTP(w http.ResponseWriter, req *http.Requ writeError(w, errors.New("ddl server information not found")) return } - allVersionsMap := map[util.ServerVersionInfo]struct{}{} - allVersions := []util.ServerVersionInfo{} + allVersionsMap := map[domain.ServerVersionInfo]struct{}{} + allVersions := []domain.ServerVersionInfo{} for _, v := range allServersInfo { if _, ok := allVersionsMap[v.ServerVersionInfo]; ok { continue @@ -1332,7 +1332,7 @@ func (h ddlAllServerInfoHandler) ServeHTTP(w http.ResponseWriter, req *http.Requ allVersionsMap[v.ServerVersionInfo] = struct{}{} allVersions = append(allVersions, v.ServerVersionInfo) } - clusterServerInfo := ClusterServerInfo{ + clusterServerInfo := clusterServerInfo{ ServersNum: len(allServersInfo), OwnerID: ownerID, IsAllServerVersionConsistent: len(allVersions) == 1, diff --git a/server/http_handler_test.go b/server/http_handler_test.go index afaa3750751cf..1838e2fb24eda 100644 --- a/server/http_handler_test.go +++ b/server/http_handler_test.go @@ -656,7 +656,7 @@ func (ts *HTTPHandlerTestSuite) TestServerInfo(c *C) { c.Assert(resp.StatusCode, Equals, http.StatusOK) decoder := json.NewDecoder(resp.Body) - serverInfo := ReportServerInfo{} + serverInfo := reportServerInfo{} err = decoder.Decode(&serverInfo) c.Assert(err, IsNil) @@ -684,7 +684,7 @@ func (ts *HTTPHandlerTestSuite) TestAllServerInfo(c *C) { c.Assert(resp.StatusCode, Equals, http.StatusOK) decoder := json.NewDecoder(resp.Body) - allServerInfo := ClusterServerInfo{} + allServerInfo := clusterServerInfo{} err = decoder.Decode(&allServerInfo) c.Assert(err, IsNil) From 8baa5be71d3347298870e906a7abcc8f55f6f839 Mon Sep 17 00:00:00 2001 From: crazycs Date: Tue, 7 Aug 2018 01:50:14 +0800 Subject: [PATCH 21/40] refine code --- ddl/syncer.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/ddl/syncer.go b/ddl/syncer.go index 5bd022a266f86..afddb80040563 100644 --- a/ddl/syncer.go +++ b/ddl/syncer.go @@ -86,11 +86,10 @@ type SchemaSyncer interface { } type schemaVersionSyncer struct { - selfSchemaVerPath string - selfServerInfoPath string - etcdCli *clientv3.Client - session *concurrency.Session - mu struct { + selfSchemaVerPath string + etcdCli *clientv3.Client + session *concurrency.Session + mu struct { sync.RWMutex globalVerCh clientv3.WatchChan } From be06280a3108ddd11dc08b0c9d19936abc802619 Mon Sep 17 00:00:00 2001 From: crazycs Date: Tue, 7 Aug 2018 09:08:47 +0800 Subject: [PATCH 22/40] refine code and add comment --- ddl/ddl.go | 1 + ddl/syncer.go | 21 +++++++++++---------- domain/info.go | 8 +++----- server/http_handler.go | 20 ++++++++++---------- server/http_handler_test.go | 2 +- 5 files changed, 26 insertions(+), 26 deletions(-) diff --git a/ddl/ddl.go b/ddl/ddl.go index 19b6c2c8655e0..9adb67b393595 100644 --- a/ddl/ddl.go +++ b/ddl/ddl.go @@ -388,6 +388,7 @@ func (d *ddl) close() { if err != nil { log.Errorf("[ddl] remove self version path failed %v", err) } + for _, worker := range d.workers { worker.close() } diff --git a/ddl/syncer.go b/ddl/syncer.go index afddb80040563..6212f6c418406 100644 --- a/ddl/syncer.go +++ b/ddl/syncer.go @@ -38,9 +38,10 @@ const ( DDLGlobalSchemaVersion = "/tidb/ddl/global_schema_version" // InitialVersion is the initial schema version for every server. // It's exported for testing. - InitialVersion = "0" + InitialVersion = "0" + // KeyOpDefaultRetryCnt is the default retry count for etcd store. + KeyOpDefaultRetryCnt = 3 putKeyNoRetry = 1 - keyOpDefaultRetryCnt = 3 putKeyRetryUnlimited = math.MaxInt64 keyOpDefaultTimeout = 2 * time.Second keyOpRetryInterval = 30 * time.Millisecond @@ -123,7 +124,7 @@ func PutKV(ctx context.Context, etcdCli *clientv3.Client, retryCnt int, key, val if err == nil { return nil } - log.Warnf("[syncer] put key: %s value: %s failed %v no.%d", key, val, err, i) + log.Warnf("[etcdCli] put key: %s value: %s failed %v no.%d", key, val, err, i) time.Sleep(keyOpRetryInterval) } return errors.Trace(err) @@ -154,7 +155,7 @@ func (s *schemaVersionSyncer) Init(ctx context.Context) error { s.mu.globalVerCh = s.etcdCli.Watch(ctx, DDLGlobalSchemaVersion) s.mu.Unlock() - err = s.putKV(ctx, keyOpDefaultRetryCnt, s.selfSchemaVerPath, InitialVersion, + err = s.putKV(ctx, KeyOpDefaultRetryCnt, s.selfSchemaVerPath, InitialVersion, clientv3.WithLease(s.session.Lease())) return errors.Trace(err) } @@ -245,22 +246,22 @@ func (s *schemaVersionSyncer) RemoveSelfVersionPath() error { metrics.DeploySyncerHistogram.WithLabelValues(metrics.SyncerClear, metrics.RetLabel(err)).Observe(time.Since(startTime).Seconds()) }() - err = RemovePath(s.selfSchemaVerPath, s.etcdCli) + err = DeleteKey(s.selfSchemaVerPath, s.etcdCli) return errors.Trace(err) } -// RemovePath remove key value from etcd. -func RemovePath(path string, etcdCli *clientv3.Client) error { +// DeleteKey delete key value from etcd. +func DeleteKey(key string, etcdCli *clientv3.Client) error { var err error ctx := context.Background() - for i := 0; i < keyOpDefaultRetryCnt; i++ { + for i := 0; i < KeyOpDefaultRetryCnt; i++ { childCtx, cancel := context.WithTimeout(ctx, keyOpDefaultTimeout) - _, err = etcdCli.Delete(childCtx, path) + _, err = etcdCli.Delete(childCtx, key) cancel() if err == nil { return nil } - log.Warnf("[syncer] remove path %s failed %v no.%d", path, err, i) + log.Warnf("[etcd] remove key %s failed %v no.%d", key, err, i) } return errors.Trace(err) } diff --git a/domain/info.go b/domain/info.go index feef573b5bbcb..c57824ec54c8b 100644 --- a/domain/info.go +++ b/domain/info.go @@ -32,8 +32,6 @@ import ( const ( // ServerInformation store server information such as IP, port and so on. ServerInformation = "/tidb/server/info" - - keyOpDefaultRetryCnt = 3 ) // infoSyncer stores server info to PD when server start and delete when server down. @@ -96,7 +94,7 @@ func (is *infoSyncer) GetOwnerServerInfoFromPD(ownerID string) (*ServerInfo, err } info, ok := allInfo[ownerID] if !ok { - return nil, errors.New(fmt.Sprintf("[syncer] get %s failed", key)) + return nil, errors.New(fmt.Sprintf("[infoSyncer] get %s failed", key)) } return info, nil } @@ -127,7 +125,7 @@ func (is *infoSyncer) StoreServerInfoToPD() error { } ctx := context.Background() key := fmt.Sprintf("%s/%s", ServerInformation, is.info.ID) - err = ddl.PutKV(ctx, is.etcdCli, keyOpDefaultRetryCnt, key, hack.String(infoBuf)) + err = ddl.PutKV(ctx, is.etcdCli, ddl.KeyOpDefaultRetryCnt, key, hack.String(infoBuf)) return errors.Trace(err) } @@ -137,7 +135,7 @@ func (is *infoSyncer) RemoveServerInfoFromPD() { return } key := fmt.Sprintf("%s/%s", ServerInformation, is.info.ID) - err := ddl.RemovePath(key, is.etcdCli) + err := ddl.DeleteKey(key, is.etcdCli) if err != nil { log.Errorf("[infoSyncer] remove self server info failed %v", err) } diff --git a/server/http_handler.go b/server/http_handler.go index e2618e380fe16..23f8bfb0af967 100644 --- a/server/http_handler.go +++ b/server/http_handler.go @@ -1280,23 +1280,23 @@ func (h ddlServerInfoHandler) ServeHTTP(w http.ResponseWriter, req *http.Request return } selfInfo := is.GetServerInfo() - reportServerInfo := reportServerInfo{ + reportInfo := reportServerInfo{ IsOwner: selfInfo.ID == ownerID, SelfServerInfo: selfInfo, } - if !reportServerInfo.IsOwner { + if !reportInfo.IsOwner { ownerInfo, err := is.GetOwnerServerInfoFromPD(ownerID) if err != nil { writeError(w, errors.New("ddl server information not found")) return } - reportServerInfo.OwnerServerInfo = ownerInfo + reportInfo.OwnerServerInfo = ownerInfo } - writeData(w, reportServerInfo) + writeData(w, reportInfo) } -// clusterServerInfo is only used to report cluster servers info when do http request -type clusterServerInfo struct { +// ClusterServerInfo is only used to report cluster servers info when do http request +type ClusterServerInfo struct { ServersNum int `json:"servers_num,omitempty"` OwnerID string `json:"owner_id"` IsAllServerVersionConsistent bool `json:"is_all_server_version_consistent,omitempty"` @@ -1332,14 +1332,14 @@ func (h ddlAllServerInfoHandler) ServeHTTP(w http.ResponseWriter, req *http.Requ allVersionsMap[v.ServerVersionInfo] = struct{}{} allVersions = append(allVersions, v.ServerVersionInfo) } - clusterServerInfo := clusterServerInfo{ + clusterInfo := ClusterServerInfo{ ServersNum: len(allServersInfo), OwnerID: ownerID, IsAllServerVersionConsistent: len(allVersions) == 1, AllServersInfo: allServersInfo, } - if !clusterServerInfo.IsAllServerVersionConsistent { - clusterServerInfo.AllServersDiffVersions = allVersions + if !clusterInfo.IsAllServerVersionConsistent { + clusterInfo.AllServersDiffVersions = allVersions } - writeData(w, clusterServerInfo) + writeData(w, clusterInfo) } diff --git a/server/http_handler_test.go b/server/http_handler_test.go index 1838e2fb24eda..15e5aa3f85ef0 100644 --- a/server/http_handler_test.go +++ b/server/http_handler_test.go @@ -684,7 +684,7 @@ func (ts *HTTPHandlerTestSuite) TestAllServerInfo(c *C) { c.Assert(resp.StatusCode, Equals, http.StatusOK) decoder := json.NewDecoder(resp.Body) - allServerInfo := clusterServerInfo{} + allServerInfo := ClusterServerInfo{} err = decoder.Decode(&allServerInfo) c.Assert(err, IsNil) From 38e30f1a89842418d9c554e79a6dd1ba76f43628 Mon Sep 17 00:00:00 2001 From: crazycs Date: Tue, 7 Aug 2018 09:52:10 +0800 Subject: [PATCH 23/40] update test and comment --- ddl/syncer.go | 2 +- domain/domain.go | 2 +- domain/info.go | 9 ++++----- server/http_handler.go | 6 +++--- server/http_handler_test.go | 12 ++++++------ 5 files changed, 15 insertions(+), 16 deletions(-) diff --git a/ddl/syncer.go b/ddl/syncer.go index 6212f6c418406..f15914dc32f12 100644 --- a/ddl/syncer.go +++ b/ddl/syncer.go @@ -261,7 +261,7 @@ func DeleteKey(key string, etcdCli *clientv3.Client) error { if err == nil { return nil } - log.Warnf("[etcd] remove key %s failed %v no.%d", key, err, i) + log.Warnf("[etcd] delete key %s failed %v no.%d", key, err, i) } return errors.Trace(err) } diff --git a/domain/domain.go b/domain/domain.go index 83f7d997207cf..7a90f801a4897 100644 --- a/domain/domain.go +++ b/domain/domain.go @@ -509,7 +509,7 @@ func (do *Domain) Init(ddlLease time.Duration, sysFactory func(*Domain) (pools.R if err != nil { return errors.Trace(err) } - do.info = newInfoSyncer(do.ddl.GetID(), do.etcdClient) + do.info = NewInfoSyncer(do.ddl.GetID(), do.etcdClient) err = do.info.StoreServerInfoToPD() if err != nil { return errors.Trace(err) diff --git a/domain/info.go b/domain/info.go index c57824ec54c8b..353b273cf1a82 100644 --- a/domain/info.go +++ b/domain/info.go @@ -56,7 +56,8 @@ type ServerVersionInfo struct { GitHash string `json:"git_hash"` } -func newInfoSyncer(id string, etcdCli *clientv3.Client) *infoSyncer { +// NewInfoSyncer return new infoSyncer. It export for tidb-test test. +func NewInfoSyncer(id string, etcdCli *clientv3.Client) *infoSyncer { return &infoSyncer{ etcdCli: etcdCli, info: getServerInfo(id), @@ -88,11 +89,11 @@ func (is *infoSyncer) GetOwnerServerInfoFromPD(ownerID string) (*ServerInfo, err } ctx := context.Background() key := fmt.Sprintf("%s/%s", ServerInformation, ownerID) - allInfo, err := getInfo(ctx, is.etcdCli, key) + infoMap, err := getInfo(ctx, is.etcdCli, key) if err != nil { return nil, errors.Trace(err) } - info, ok := allInfo[ownerID] + info, ok := infoMap[ownerID] if !ok { return nil, errors.New(fmt.Sprintf("[infoSyncer] get %s failed", key)) } @@ -144,7 +145,6 @@ func (is *infoSyncer) RemoveServerInfoFromPD() { func getInfo(ctx context.Context, etcdCli *clientv3.Client, key string, opts ...clientv3.OpOption) (map[string]*ServerInfo, error) { var err error allInfo := make(map[string]*ServerInfo) - for { select { case <-ctx.Done(): @@ -159,7 +159,6 @@ func getInfo(ctx context.Context, etcdCli *clientv3.Client, key string, opts ... time.Sleep(200 * time.Millisecond) continue } - for _, kv := range resp.Kvs { info := &ServerInfo{} err := json.Unmarshal(kv.Value, info) diff --git a/server/http_handler.go b/server/http_handler.go index 23f8bfb0af967..1bd2f137b8082 100644 --- a/server/http_handler.go +++ b/server/http_handler.go @@ -1295,8 +1295,8 @@ func (h ddlServerInfoHandler) ServeHTTP(w http.ResponseWriter, req *http.Request writeData(w, reportInfo) } -// ClusterServerInfo is only used to report cluster servers info when do http request -type ClusterServerInfo struct { +// clusterServerInfo is only used to report cluster servers info when do http request +type clusterServerInfo struct { ServersNum int `json:"servers_num,omitempty"` OwnerID string `json:"owner_id"` IsAllServerVersionConsistent bool `json:"is_all_server_version_consistent,omitempty"` @@ -1332,7 +1332,7 @@ func (h ddlAllServerInfoHandler) ServeHTTP(w http.ResponseWriter, req *http.Requ allVersionsMap[v.ServerVersionInfo] = struct{}{} allVersions = append(allVersions, v.ServerVersionInfo) } - clusterInfo := ClusterServerInfo{ + clusterInfo := clusterServerInfo{ ServersNum: len(allServersInfo), OwnerID: ownerID, IsAllServerVersionConsistent: len(allVersions) == 1, diff --git a/server/http_handler_test.go b/server/http_handler_test.go index 15e5aa3f85ef0..ee6890016a872 100644 --- a/server/http_handler_test.go +++ b/server/http_handler_test.go @@ -684,19 +684,19 @@ func (ts *HTTPHandlerTestSuite) TestAllServerInfo(c *C) { c.Assert(resp.StatusCode, Equals, http.StatusOK) decoder := json.NewDecoder(resp.Body) - allServerInfo := ClusterServerInfo{} - err = decoder.Decode(&allServerInfo) + clusterInfo := clusterServerInfo{} + err = decoder.Decode(&clusterInfo) c.Assert(err, IsNil) - c.Assert(allServerInfo.IsAllServerVersionConsistent, IsTrue) - c.Assert(allServerInfo.ServersNum, Equals, 1) + c.Assert(clusterInfo.IsAllServerVersionConsistent, IsTrue) + c.Assert(clusterInfo.ServersNum, Equals, 1) store := ts.server.newTikvHandlerTool().store.(kv.Storage) do, err := session.GetDomain(store.(kv.Storage)) c.Assert(err, IsNil) ddl := do.DDL() - c.Assert(allServerInfo.OwnerID, Equals, ddl.GetID()) - serverInfo, ok := allServerInfo.AllServersInfo[ddl.GetID()] + c.Assert(clusterInfo.OwnerID, Equals, ddl.GetID()) + serverInfo, ok := clusterInfo.AllServersInfo[ddl.GetID()] c.Assert(ok, Equals, true) cfg := config.GetGlobalConfig() From c81426787973dce0d8a78e7d7e6a4193fb4e9793 Mon Sep 17 00:00:00 2001 From: crazycs Date: Tue, 7 Aug 2018 12:35:56 +0800 Subject: [PATCH 24/40] refine code --- ddl/ddl.go | 2 +- ddl/syncer.go | 26 +++++++++++--------------- domain/domain.go | 6 +++--- domain/info.go | 42 ++++++++++++++++++++---------------------- server/http_handler.go | 14 ++++++++------ server/http_status.go | 4 ++-- 6 files changed, 45 insertions(+), 49 deletions(-) diff --git a/ddl/ddl.go b/ddl/ddl.go index d76fcd12f61db..30a9c26c5dd3b 100644 --- a/ddl/ddl.go +++ b/ddl/ddl.go @@ -216,7 +216,7 @@ type DDL interface { SchemaSyncer() SchemaSyncer // OwnerManager gets the owner manager. OwnerManager() owner.Manager - // GetID gets the ddl ID + // GetID gets the ddl ID. GetID() string // GetTableMaxRowID gets the max row ID of a normal table or a partition. GetTableMaxRowID(startTS uint64, tbl table.Table) (int64, bool, error) diff --git a/ddl/syncer.go b/ddl/syncer.go index f15914dc32f12..22e15038cddf3 100644 --- a/ddl/syncer.go +++ b/ddl/syncer.go @@ -41,9 +41,10 @@ const ( InitialVersion = "0" // KeyOpDefaultRetryCnt is the default retry count for etcd store. KeyOpDefaultRetryCnt = 3 + // KeyOpDefaultTimeout is the default time out for etcd store. + KeyOpDefaultTimeout = 2 * time.Second putKeyNoRetry = 1 putKeyRetryUnlimited = math.MaxInt64 - keyOpDefaultTimeout = 2 * time.Second keyOpRetryInterval = 30 * time.Millisecond checkVersInterval = 20 * time.Millisecond ) @@ -104,12 +105,7 @@ func NewSchemaSyncer(etcdCli *clientv3.Client, id string) SchemaSyncer { } } -func (s *schemaVersionSyncer) putKV(ctx context.Context, retryCnt int, key, val string, - opts ...clientv3.OpOption) error { - return PutKV(ctx, s.etcdCli, retryCnt, key, val, opts...) -} - -// PutKV put key value to etcd. +// PutKV puts key value to etcd. func PutKV(ctx context.Context, etcdCli *clientv3.Client, retryCnt int, key, val string, opts ...clientv3.OpOption) error { var err error @@ -118,7 +114,7 @@ func PutKV(ctx context.Context, etcdCli *clientv3.Client, retryCnt int, key, val return errors.Trace(ctx.Err()) } - childCtx, cancel := context.WithTimeout(ctx, keyOpDefaultTimeout) + childCtx, cancel := context.WithTimeout(ctx, KeyOpDefaultTimeout) _, err = etcdCli.Put(childCtx, key, val, opts...) cancel() if err == nil { @@ -155,7 +151,7 @@ func (s *schemaVersionSyncer) Init(ctx context.Context) error { s.mu.globalVerCh = s.etcdCli.Watch(ctx, DDLGlobalSchemaVersion) s.mu.Unlock() - err = s.putKV(ctx, KeyOpDefaultRetryCnt, s.selfSchemaVerPath, InitialVersion, + err = PutKV(ctx, s.etcdCli, KeyOpDefaultRetryCnt, s.selfSchemaVerPath, InitialVersion, clientv3.WithLease(s.session.Lease())) return errors.Trace(err) } @@ -181,9 +177,9 @@ func (s *schemaVersionSyncer) Restart(ctx context.Context) error { } s.session = session - childCtx, cancel := context.WithTimeout(ctx, keyOpDefaultTimeout) + childCtx, cancel := context.WithTimeout(ctx, KeyOpDefaultTimeout) defer cancel() - err = s.putKV(childCtx, putKeyRetryUnlimited, s.selfSchemaVerPath, InitialVersion, + err = PutKV(childCtx, s.etcdCli, putKeyRetryUnlimited, s.selfSchemaVerPath, InitialVersion, clientv3.WithLease(s.session.Lease())) return errors.Trace(err) @@ -221,7 +217,7 @@ func (s *schemaVersionSyncer) WatchGlobalSchemaVer(ctx context.Context) { func (s *schemaVersionSyncer) UpdateSelfVersion(ctx context.Context, version int64) error { startTime := time.Now() ver := strconv.FormatInt(version, 10) - err := s.putKV(ctx, putKeyNoRetry, s.selfSchemaVerPath, ver, + err := PutKV(ctx, s.etcdCli, putKeyNoRetry, s.selfSchemaVerPath, ver, clientv3.WithLease(s.session.Lease())) metrics.UpdateSelfVersionHistogram.WithLabelValues(metrics.RetLabel(err)).Observe(time.Since(startTime).Seconds()) @@ -232,7 +228,7 @@ func (s *schemaVersionSyncer) UpdateSelfVersion(ctx context.Context, version int func (s *schemaVersionSyncer) OwnerUpdateGlobalVersion(ctx context.Context, version int64) error { startTime := time.Now() ver := strconv.FormatInt(version, 10) - err := s.putKV(ctx, putKeyRetryUnlimited, DDLGlobalSchemaVersion, ver) + err := PutKV(ctx, s.etcdCli, putKeyRetryUnlimited, DDLGlobalSchemaVersion, ver) metrics.OwnerHandleSyncerHistogram.WithLabelValues(metrics.OwnerUpdateGlobalVersion, metrics.RetLabel(err)).Observe(time.Since(startTime).Seconds()) return errors.Trace(err) @@ -250,12 +246,12 @@ func (s *schemaVersionSyncer) RemoveSelfVersionPath() error { return errors.Trace(err) } -// DeleteKey delete key value from etcd. +// DeleteKey deletes key value from etcd. func DeleteKey(key string, etcdCli *clientv3.Client) error { var err error ctx := context.Background() for i := 0; i < KeyOpDefaultRetryCnt; i++ { - childCtx, cancel := context.WithTimeout(ctx, keyOpDefaultTimeout) + childCtx, cancel := context.WithTimeout(ctx, KeyOpDefaultTimeout) _, err = etcdCli.Delete(childCtx, key) cancel() if err == nil { diff --git a/domain/domain.go b/domain/domain.go index 7a90f801a4897..8cb696dc349e9 100644 --- a/domain/domain.go +++ b/domain/domain.go @@ -252,7 +252,7 @@ func (do *Domain) DDL() ddl.DDL { return do.ddl } -// InfoSyncer gets infoSyncer from domain +// InfoSyncer gets infoSyncer from domain. func (do *Domain) InfoSyncer() *infoSyncer { return do.info } @@ -399,7 +399,7 @@ func (do *Domain) Close() { terror.Log(errors.Trace(do.ddl.Stop())) } if do.info != nil { - do.info.RemoveServerInfoFromPD() + do.info.RemoveServerInfo() } close(do.exit) if do.etcdClient != nil { @@ -510,7 +510,7 @@ func (do *Domain) Init(ddlLease time.Duration, sysFactory func(*Domain) (pools.R return errors.Trace(err) } do.info = NewInfoSyncer(do.ddl.GetID(), do.etcdClient) - err = do.info.StoreServerInfoToPD() + err = do.info.StoreServerInfo(ctx) if err != nil { return errors.Trace(err) } diff --git a/domain/info.go b/domain/info.go index 353b273cf1a82..552ae98d706d7 100644 --- a/domain/info.go +++ b/domain/info.go @@ -36,8 +36,9 @@ const ( // infoSyncer stores server info to PD when server start and delete when server down. type infoSyncer struct { - etcdCli *clientv3.Client - info *ServerInfo + etcdCli *clientv3.Client + info *ServerInfo + serverInfoPath string } // ServerInfo is server static information. @@ -59,8 +60,9 @@ type ServerVersionInfo struct { // NewInfoSyncer return new infoSyncer. It export for tidb-test test. func NewInfoSyncer(id string, etcdCli *clientv3.Client) *infoSyncer { return &infoSyncer{ - etcdCli: etcdCli, - info: getServerInfo(id), + etcdCli: etcdCli, + info: getServerInfo(id), + serverInfoPath: fmt.Sprintf("%s/%s", ServerInformation, id), } } @@ -82,12 +84,11 @@ func (is *infoSyncer) GetServerInfo() *ServerInfo { return is.info } -// GetOwnerServerInfoFromPD gets owner server static information from PD. -func (is *infoSyncer) GetOwnerServerInfoFromPD(ownerID string) (*ServerInfo, error) { +// GetOwnerServerInfo gets owner server static information from PD. +func (is *infoSyncer) GetOwnerServerInfo(ctx context.Context, ownerID string) (*ServerInfo, error) { if is.etcdCli == nil || ownerID == is.info.ID { return is.info, nil } - ctx := context.Background() key := fmt.Sprintf("%s/%s", ServerInformation, ownerID) infoMap, err := getInfo(ctx, is.etcdCli, key) if err != nil { @@ -95,19 +96,18 @@ func (is *infoSyncer) GetOwnerServerInfoFromPD(ownerID string) (*ServerInfo, err } info, ok := infoMap[ownerID] if !ok { - return nil, errors.New(fmt.Sprintf("[infoSyncer] get %s failed", key)) + return nil, errors.Errorf("[infoSyncer] get %s failed", key) } return info, nil } -// GetAllServerInfoFromPD gets all servers static information from PD. -func (is *infoSyncer) GetAllServerInfoFromPD() (map[string]*ServerInfo, error) { +// GetAllServerInfo gets all servers static information from PD. +func (is *infoSyncer) GetAllServerInfo(ctx context.Context) (map[string]*ServerInfo, error) { allInfo := make(map[string]*ServerInfo) if is.etcdCli == nil { allInfo[is.info.ID] = is.info return allInfo, nil } - ctx := context.Background() allInfo, err := getInfo(ctx, is.etcdCli, ServerInformation, clientv3.WithPrefix()) if err != nil { return nil, errors.Trace(err) @@ -115,8 +115,8 @@ func (is *infoSyncer) GetAllServerInfoFromPD() (map[string]*ServerInfo, error) { return allInfo, nil } -// StoreServerInfoToPD stores self server static information to PD when domain Init. -func (is *infoSyncer) StoreServerInfoToPD() error { +// StoreServerInfo stores self server static information to PD when domain Init. +func (is *infoSyncer) StoreServerInfo(ctx context.Context) error { if is.etcdCli == nil { return nil } @@ -124,19 +124,16 @@ func (is *infoSyncer) StoreServerInfoToPD() error { if err != nil { return errors.Trace(err) } - ctx := context.Background() - key := fmt.Sprintf("%s/%s", ServerInformation, is.info.ID) - err = ddl.PutKV(ctx, is.etcdCli, ddl.KeyOpDefaultRetryCnt, key, hack.String(infoBuf)) + err = ddl.PutKV(ctx, is.etcdCli, ddl.KeyOpDefaultRetryCnt, is.serverInfoPath, hack.String(infoBuf)) return errors.Trace(err) } -// RemoveServerInfoFromPD remove self server static information from PD when domain close. -func (is *infoSyncer) RemoveServerInfoFromPD() { +// RemoveServerInfo remove self server static information from PD when domain close. +func (is *infoSyncer) RemoveServerInfo() { if is.etcdCli == nil { return } - key := fmt.Sprintf("%s/%s", ServerInformation, is.info.ID) - err := ddl.DeleteKey(key, is.etcdCli) + err := ddl.DeleteKey(is.serverInfoPath, is.etcdCli) if err != nil { log.Errorf("[infoSyncer] remove self server info failed %v", err) } @@ -152,8 +149,9 @@ func getInfo(ctx context.Context, etcdCli *clientv3.Client, key string, opts ... return nil, err default: } - - resp, err := etcdCli.Get(ctx, key, opts...) + childCtx, cancel := context.WithTimeout(ctx, ddl.KeyOpDefaultTimeout) + resp, err := etcdCli.Get(childCtx, key, opts...) + cancel() if err != nil { log.Infof("[infoSyncer] get %s failed %v, continue checking.", key, err) time.Sleep(200 * time.Millisecond) diff --git a/server/http_handler.go b/server/http_handler.go index 1bd2f137b8082..ff2fb6b0ae40d 100644 --- a/server/http_handler.go +++ b/server/http_handler.go @@ -1258,7 +1258,7 @@ func (h *mvccTxnHandler) handleMvccGetByTxn(params map[string]string) (interface return h.getMvccByStartTs(uint64(startTS), startKey, endKey) } -// reportServerInfo is only used to report the servers info when do http request +// reportServerInfo is only used to report the servers info when do http request. type reportServerInfo struct { IsOwner bool `json:"is_owner"` SelfServerInfo *domain.ServerInfo `json:"self_server_info"` @@ -1274,7 +1274,8 @@ func (h ddlServerInfoHandler) ServeHTTP(w http.ResponseWriter, req *http.Request } ddl := do.DDL() is := do.InfoSyncer() - ownerID, err := ddl.OwnerManager().GetOwnerID(context.Background()) + ctx := context.Background() + ownerID, err := ddl.OwnerManager().GetOwnerID(ctx) if err != nil { writeError(w, errors.New("ddl server information not found")) return @@ -1285,7 +1286,7 @@ func (h ddlServerInfoHandler) ServeHTTP(w http.ResponseWriter, req *http.Request SelfServerInfo: selfInfo, } if !reportInfo.IsOwner { - ownerInfo, err := is.GetOwnerServerInfoFromPD(ownerID) + ownerInfo, err := is.GetOwnerServerInfo(ctx, ownerID) if err != nil { writeError(w, errors.New("ddl server information not found")) return @@ -1295,7 +1296,7 @@ func (h ddlServerInfoHandler) ServeHTTP(w http.ResponseWriter, req *http.Request writeData(w, reportInfo) } -// clusterServerInfo is only used to report cluster servers info when do http request +// clusterServerInfo is only used to report cluster servers info when do http request. type clusterServerInfo struct { ServersNum int `json:"servers_num,omitempty"` OwnerID string `json:"owner_id"` @@ -1313,12 +1314,13 @@ func (h ddlAllServerInfoHandler) ServeHTTP(w http.ResponseWriter, req *http.Requ } ddl := do.DDL() is := do.InfoSyncer() - allServersInfo, err := is.GetAllServerInfoFromPD() + ctx := context.Background() + allServersInfo, err := is.GetAllServerInfo(ctx) if err != nil { writeError(w, errors.New("ddl server information not found")) return } - ownerID, err := ddl.OwnerManager().GetOwnerID(context.Background()) + ownerID, err := ddl.OwnerManager().GetOwnerID(ctx) if err != nil { writeError(w, errors.New("ddl server information not found")) return diff --git a/server/http_status.go b/server/http_status.go index 3662a81fab821..e434689fc8eda 100644 --- a/server/http_status.go +++ b/server/http_status.go @@ -53,11 +53,11 @@ func (s *Server) startHTTPServer() { router.Handle("/tables/{colID}/{colTp}/{colFlag}/{colLen}", valueHandler{}) router.Handle("/ddl/history", ddlHistoryJobHandler{tikvHandlerTool}) - // HTTP path for get server info + // HTTP path for get server info. router.Handle("/info", ddlServerInfoHandler{tikvHandlerTool}) router.Handle("/info/all", ddlAllServerInfoHandler{tikvHandlerTool}) if s.cfg.Store == "tikv" { - // HTTP path for tikv + // HTTP path for tikv. router.Handle("/tables/{db}/{table}/regions", tableHandler{tikvHandlerTool, opTableRegions}) router.Handle("/tables/{db}/{table}/scatter", tableHandler{tikvHandlerTool, opTableScatter}) router.Handle("/tables/{db}/{table}/stop-scatter", tableHandler{tikvHandlerTool, opStopTableScatter}) From 9d091c95a4a5b5f4c7591403f16bba9da1f3acb8 Mon Sep 17 00:00:00 2001 From: crazycs Date: Wed, 8 Aug 2018 10:16:39 +0800 Subject: [PATCH 25/40] rename function --- ddl/syncer.go | 18 +++++++++--------- domain/info.go | 4 ++-- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/ddl/syncer.go b/ddl/syncer.go index 22e15038cddf3..204f9c7de6522 100644 --- a/ddl/syncer.go +++ b/ddl/syncer.go @@ -105,8 +105,8 @@ func NewSchemaSyncer(etcdCli *clientv3.Client, id string) SchemaSyncer { } } -// PutKV puts key value to etcd. -func PutKV(ctx context.Context, etcdCli *clientv3.Client, retryCnt int, key, val string, +// PutKVToEtcd puts key value to etcd. +func PutKVToEtcd(ctx context.Context, etcdCli *clientv3.Client, retryCnt int, key, val string, opts ...clientv3.OpOption) error { var err error for i := 0; i < retryCnt; i++ { @@ -151,7 +151,7 @@ func (s *schemaVersionSyncer) Init(ctx context.Context) error { s.mu.globalVerCh = s.etcdCli.Watch(ctx, DDLGlobalSchemaVersion) s.mu.Unlock() - err = PutKV(ctx, s.etcdCli, KeyOpDefaultRetryCnt, s.selfSchemaVerPath, InitialVersion, + err = PutKVToEtcd(ctx, s.etcdCli, KeyOpDefaultRetryCnt, s.selfSchemaVerPath, InitialVersion, clientv3.WithLease(s.session.Lease())) return errors.Trace(err) } @@ -179,7 +179,7 @@ func (s *schemaVersionSyncer) Restart(ctx context.Context) error { childCtx, cancel := context.WithTimeout(ctx, KeyOpDefaultTimeout) defer cancel() - err = PutKV(childCtx, s.etcdCli, putKeyRetryUnlimited, s.selfSchemaVerPath, InitialVersion, + err = PutKVToEtcd(childCtx, s.etcdCli, putKeyRetryUnlimited, s.selfSchemaVerPath, InitialVersion, clientv3.WithLease(s.session.Lease())) return errors.Trace(err) @@ -217,7 +217,7 @@ func (s *schemaVersionSyncer) WatchGlobalSchemaVer(ctx context.Context) { func (s *schemaVersionSyncer) UpdateSelfVersion(ctx context.Context, version int64) error { startTime := time.Now() ver := strconv.FormatInt(version, 10) - err := PutKV(ctx, s.etcdCli, putKeyNoRetry, s.selfSchemaVerPath, ver, + err := PutKVToEtcd(ctx, s.etcdCli, putKeyNoRetry, s.selfSchemaVerPath, ver, clientv3.WithLease(s.session.Lease())) metrics.UpdateSelfVersionHistogram.WithLabelValues(metrics.RetLabel(err)).Observe(time.Since(startTime).Seconds()) @@ -228,7 +228,7 @@ func (s *schemaVersionSyncer) UpdateSelfVersion(ctx context.Context, version int func (s *schemaVersionSyncer) OwnerUpdateGlobalVersion(ctx context.Context, version int64) error { startTime := time.Now() ver := strconv.FormatInt(version, 10) - err := PutKV(ctx, s.etcdCli, putKeyRetryUnlimited, DDLGlobalSchemaVersion, ver) + err := PutKVToEtcd(ctx, s.etcdCli, putKeyRetryUnlimited, DDLGlobalSchemaVersion, ver) metrics.OwnerHandleSyncerHistogram.WithLabelValues(metrics.OwnerUpdateGlobalVersion, metrics.RetLabel(err)).Observe(time.Since(startTime).Seconds()) return errors.Trace(err) @@ -242,12 +242,12 @@ func (s *schemaVersionSyncer) RemoveSelfVersionPath() error { metrics.DeploySyncerHistogram.WithLabelValues(metrics.SyncerClear, metrics.RetLabel(err)).Observe(time.Since(startTime).Seconds()) }() - err = DeleteKey(s.selfSchemaVerPath, s.etcdCli) + err = DeleteKeyFromEtcd(s.selfSchemaVerPath, s.etcdCli) return errors.Trace(err) } -// DeleteKey deletes key value from etcd. -func DeleteKey(key string, etcdCli *clientv3.Client) error { +// DeleteKeyFromEtcd deletes key value from etcd. +func DeleteKeyFromEtcd(key string, etcdCli *clientv3.Client) error { var err error ctx := context.Background() for i := 0; i < KeyOpDefaultRetryCnt; i++ { diff --git a/domain/info.go b/domain/info.go index 552ae98d706d7..1e99a7a44763a 100644 --- a/domain/info.go +++ b/domain/info.go @@ -124,7 +124,7 @@ func (is *infoSyncer) StoreServerInfo(ctx context.Context) error { if err != nil { return errors.Trace(err) } - err = ddl.PutKV(ctx, is.etcdCli, ddl.KeyOpDefaultRetryCnt, is.serverInfoPath, hack.String(infoBuf)) + err = ddl.PutKVToEtcd(ctx, is.etcdCli, ddl.KeyOpDefaultRetryCnt, is.serverInfoPath, hack.String(infoBuf)) return errors.Trace(err) } @@ -133,7 +133,7 @@ func (is *infoSyncer) RemoveServerInfo() { if is.etcdCli == nil { return } - err := ddl.DeleteKey(is.serverInfoPath, is.etcdCli) + err := ddl.DeleteKeyFromEtcd(is.serverInfoPath, is.etcdCli) if err != nil { log.Errorf("[infoSyncer] remove self server info failed %v", err) } From ea275c16def335ad02404e792758f6b11172203e Mon Sep 17 00:00:00 2001 From: crazycs Date: Wed, 8 Aug 2018 10:18:48 +0800 Subject: [PATCH 26/40] refine log --- ddl/syncer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddl/syncer.go b/ddl/syncer.go index 204f9c7de6522..a4423dc7e5ef6 100644 --- a/ddl/syncer.go +++ b/ddl/syncer.go @@ -257,7 +257,7 @@ func DeleteKeyFromEtcd(key string, etcdCli *clientv3.Client) error { if err == nil { return nil } - log.Warnf("[etcd] delete key %s failed %v no.%d", key, err, i) + log.Warnf("[etcdCli] delete key %s failed %v no.%d", key, err, i) } return errors.Trace(err) } From 33d26a0914b3f7ad42336978915ddd4e2856f363 Mon Sep 17 00:00:00 2001 From: crazycs Date: Wed, 8 Aug 2018 11:04:04 +0800 Subject: [PATCH 27/40] refine code --- domain/domain.go | 4 ++-- domain/info.go | 20 ++++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/domain/domain.go b/domain/domain.go index 8cb696dc349e9..bf242d95e35a6 100644 --- a/domain/domain.go +++ b/domain/domain.go @@ -52,7 +52,7 @@ type Domain struct { statsHandle unsafe.Pointer statsLease time.Duration ddl ddl.DDL - info *infoSyncer + info *InfoSyncer m sync.Mutex SchemaValidator SchemaValidator sysSessionPool *pools.ResourcePool @@ -253,7 +253,7 @@ func (do *Domain) DDL() ddl.DDL { } // InfoSyncer gets infoSyncer from domain. -func (do *Domain) InfoSyncer() *infoSyncer { +func (do *Domain) InfoSyncer() *InfoSyncer { return do.info } diff --git a/domain/info.go b/domain/info.go index 1e99a7a44763a..a16488177bf1e 100644 --- a/domain/info.go +++ b/domain/info.go @@ -34,8 +34,8 @@ const ( ServerInformation = "/tidb/server/info" ) -// infoSyncer stores server info to PD when server start and delete when server down. -type infoSyncer struct { +// InfoSyncer stores server info to PD when server start and delete when server down. +type InfoSyncer struct { etcdCli *clientv3.Client info *ServerInfo serverInfoPath string @@ -57,9 +57,9 @@ type ServerVersionInfo struct { GitHash string `json:"git_hash"` } -// NewInfoSyncer return new infoSyncer. It export for tidb-test test. -func NewInfoSyncer(id string, etcdCli *clientv3.Client) *infoSyncer { - return &infoSyncer{ +// NewInfoSyncer return new InfoSyncer. It export for tidb-test test. +func NewInfoSyncer(id string, etcdCli *clientv3.Client) *InfoSyncer { + return &InfoSyncer{ etcdCli: etcdCli, info: getServerInfo(id), serverInfoPath: fmt.Sprintf("%s/%s", ServerInformation, id), @@ -80,12 +80,12 @@ func getServerInfo(id string) *ServerInfo { } //GetServerInfo gets self server static information. -func (is *infoSyncer) GetServerInfo() *ServerInfo { +func (is *InfoSyncer) GetServerInfo() *ServerInfo { return is.info } // GetOwnerServerInfo gets owner server static information from PD. -func (is *infoSyncer) GetOwnerServerInfo(ctx context.Context, ownerID string) (*ServerInfo, error) { +func (is *InfoSyncer) GetOwnerServerInfo(ctx context.Context, ownerID string) (*ServerInfo, error) { if is.etcdCli == nil || ownerID == is.info.ID { return is.info, nil } @@ -102,7 +102,7 @@ func (is *infoSyncer) GetOwnerServerInfo(ctx context.Context, ownerID string) (* } // GetAllServerInfo gets all servers static information from PD. -func (is *infoSyncer) GetAllServerInfo(ctx context.Context) (map[string]*ServerInfo, error) { +func (is *InfoSyncer) GetAllServerInfo(ctx context.Context) (map[string]*ServerInfo, error) { allInfo := make(map[string]*ServerInfo) if is.etcdCli == nil { allInfo[is.info.ID] = is.info @@ -116,7 +116,7 @@ func (is *infoSyncer) GetAllServerInfo(ctx context.Context) (map[string]*ServerI } // StoreServerInfo stores self server static information to PD when domain Init. -func (is *infoSyncer) StoreServerInfo(ctx context.Context) error { +func (is *InfoSyncer) StoreServerInfo(ctx context.Context) error { if is.etcdCli == nil { return nil } @@ -129,7 +129,7 @@ func (is *infoSyncer) StoreServerInfo(ctx context.Context) error { } // RemoveServerInfo remove self server static information from PD when domain close. -func (is *infoSyncer) RemoveServerInfo() { +func (is *InfoSyncer) RemoveServerInfo() { if is.etcdCli == nil { return } From 750407c8a4c2c14be0856cb435023d69091f41eb Mon Sep 17 00:00:00 2001 From: crazycs Date: Wed, 8 Aug 2018 15:41:55 +0800 Subject: [PATCH 28/40] refine code and add comment --- ddl/syncer.go | 3 +++ docs/tidb_http_api.md | 2 +- domain/info.go | 31 +++++++++++++++++-------------- server/http_handler.go | 4 +++- 4 files changed, 24 insertions(+), 16 deletions(-) diff --git a/ddl/syncer.go b/ddl/syncer.go index a4423dc7e5ef6..d62f3adab8d70 100644 --- a/ddl/syncer.go +++ b/ddl/syncer.go @@ -106,6 +106,9 @@ func NewSchemaSyncer(etcdCli *clientv3.Client, id string) SchemaSyncer { } // PutKVToEtcd puts key value to etcd. +// etcdCli is client of etcd. +// retryCnt is retry time when When an error occurs. +// opts is configures of etcd Operations. func PutKVToEtcd(ctx context.Context, etcdCli *clientv3.Client, retryCnt int, key, val string, opts ...clientv3.OpOption) error { var err error diff --git a/docs/tidb_http_api.md b/docs/tidb_http_api.md index 8e8e76dc5f6c1..46d4ff33adff7 100644 --- a/docs/tidb_http_api.md +++ b/docs/tidb_http_api.md @@ -96,7 +96,7 @@ timezone.* curl http://{TiDBIP}:10080/settings ``` -1. Get TiDB server information and owner server infomation. +1. Get TiDB server information and owner server information. ```shell curl http://{TiDBIP}:10080/info diff --git a/domain/info.go b/domain/info.go index a16488177bf1e..a609c1224c3cc 100644 --- a/domain/info.go +++ b/domain/info.go @@ -30,11 +30,11 @@ import ( ) const ( - // ServerInformation store server information such as IP, port and so on. - ServerInformation = "/tidb/server/info" + // ServerInformationPath store server information such as IP, port and so on. + ServerInformationPath = "/tidb/server/info" ) -// InfoSyncer stores server info to PD when server start and delete when server down. +// InfoSyncer stores server info to Etcd when when the tidb-server starts and delete when tidb-server shuts down. type InfoSyncer struct { etcdCli *clientv3.Client info *ServerInfo @@ -47,6 +47,7 @@ type ServerInfo struct { ServerVersionInfo ID string `json:"ddl_id"` IP string `json:"ip"` + Port uint `json:"listening_port"` StatusPort uint `json:"status_port"` Lease string `json:"lease"` } @@ -57,12 +58,12 @@ type ServerVersionInfo struct { GitHash string `json:"git_hash"` } -// NewInfoSyncer return new InfoSyncer. It export for tidb-test test. +// NewInfoSyncer return new InfoSyncer. It is exported for testing. func NewInfoSyncer(id string, etcdCli *clientv3.Client) *InfoSyncer { return &InfoSyncer{ etcdCli: etcdCli, info: getServerInfo(id), - serverInfoPath: fmt.Sprintf("%s/%s", ServerInformation, id), + serverInfoPath: fmt.Sprintf("%s/%s", ServerInformationPath, id), } } @@ -71,6 +72,7 @@ func getServerInfo(id string) *ServerInfo { info := &ServerInfo{ ID: id, IP: cfg.AdvertiseAddress, + Port: cfg.Port, StatusPort: cfg.Status.StatusPort, Lease: cfg.Lease, } @@ -84,38 +86,38 @@ func (is *InfoSyncer) GetServerInfo() *ServerInfo { return is.info } -// GetOwnerServerInfo gets owner server static information from PD. -func (is *InfoSyncer) GetOwnerServerInfo(ctx context.Context, ownerID string) (*ServerInfo, error) { - if is.etcdCli == nil || ownerID == is.info.ID { +// GetServerInfoByID gets owner server static information from Etcd. +func (is *InfoSyncer) GetServerInfoByID(ctx context.Context, id string) (*ServerInfo, error) { + if is.etcdCli == nil || id == is.info.ID { return is.info, nil } - key := fmt.Sprintf("%s/%s", ServerInformation, ownerID) + key := fmt.Sprintf("%s/%s", ServerInformationPath, id) infoMap, err := getInfo(ctx, is.etcdCli, key) if err != nil { return nil, errors.Trace(err) } - info, ok := infoMap[ownerID] + info, ok := infoMap[id] if !ok { return nil, errors.Errorf("[infoSyncer] get %s failed", key) } return info, nil } -// GetAllServerInfo gets all servers static information from PD. +// GetAllServerInfo gets all servers static information from Etcd. func (is *InfoSyncer) GetAllServerInfo(ctx context.Context) (map[string]*ServerInfo, error) { allInfo := make(map[string]*ServerInfo) if is.etcdCli == nil { allInfo[is.info.ID] = is.info return allInfo, nil } - allInfo, err := getInfo(ctx, is.etcdCli, ServerInformation, clientv3.WithPrefix()) + allInfo, err := getInfo(ctx, is.etcdCli, ServerInformationPath, clientv3.WithPrefix()) if err != nil { return nil, errors.Trace(err) } return allInfo, nil } -// StoreServerInfo stores self server static information to PD when domain Init. +// StoreServerInfo stores self server static information to Etcd when domain Init. func (is *InfoSyncer) StoreServerInfo(ctx context.Context) error { if is.etcdCli == nil { return nil @@ -128,7 +130,7 @@ func (is *InfoSyncer) StoreServerInfo(ctx context.Context) error { return errors.Trace(err) } -// RemoveServerInfo remove self server static information from PD when domain close. +// RemoveServerInfo remove self server static information from Etcd. func (is *InfoSyncer) RemoveServerInfo() { if is.etcdCli == nil { return @@ -139,6 +141,7 @@ func (is *InfoSyncer) RemoveServerInfo() { } } +// getInfo gets server information from Etcd according to the key and opts. func getInfo(ctx context.Context, etcdCli *clientv3.Client, key string, opts ...clientv3.OpOption) (map[string]*ServerInfo, error) { var err error allInfo := make(map[string]*ServerInfo) diff --git a/server/http_handler.go b/server/http_handler.go index ff2fb6b0ae40d..32721dafe3eee 100644 --- a/server/http_handler.go +++ b/server/http_handler.go @@ -1286,9 +1286,10 @@ func (h ddlServerInfoHandler) ServeHTTP(w http.ResponseWriter, req *http.Request SelfServerInfo: selfInfo, } if !reportInfo.IsOwner { - ownerInfo, err := is.GetOwnerServerInfo(ctx, ownerID) + ownerInfo, err := is.GetServerInfoByID(ctx, ownerID) if err != nil { writeError(w, errors.New("ddl server information not found")) + log.Error(err) return } reportInfo.OwnerServerInfo = ownerInfo @@ -1318,6 +1319,7 @@ func (h ddlAllServerInfoHandler) ServeHTTP(w http.ResponseWriter, req *http.Requ allServersInfo, err := is.GetAllServerInfo(ctx) if err != nil { writeError(w, errors.New("ddl server information not found")) + log.Error(err) return } ownerID, err := ddl.OwnerManager().GetOwnerID(ctx) From ec488e9d3737cb14a9caa674569aa0c2d764039d Mon Sep 17 00:00:00 2001 From: crazycs Date: Wed, 8 Aug 2018 20:48:49 +0800 Subject: [PATCH 29/40] refine code and comment --- ddl/syncer.go | 22 ++++++++++------------ domain/info.go | 33 +++++++++++++++++++++------------ 2 files changed, 31 insertions(+), 24 deletions(-) diff --git a/ddl/syncer.go b/ddl/syncer.go index d62f3adab8d70..f640edb495801 100644 --- a/ddl/syncer.go +++ b/ddl/syncer.go @@ -38,13 +38,11 @@ const ( DDLGlobalSchemaVersion = "/tidb/ddl/global_schema_version" // InitialVersion is the initial schema version for every server. // It's exported for testing. - InitialVersion = "0" - // KeyOpDefaultRetryCnt is the default retry count for etcd store. - KeyOpDefaultRetryCnt = 3 - // KeyOpDefaultTimeout is the default time out for etcd store. - KeyOpDefaultTimeout = 2 * time.Second + InitialVersion = "0" putKeyNoRetry = 1 + keyOpDefaultRetryCnt = 3 putKeyRetryUnlimited = math.MaxInt64 + keyOpDefaultTimeout = 2 * time.Second keyOpRetryInterval = 30 * time.Millisecond checkVersInterval = 20 * time.Millisecond ) @@ -117,7 +115,7 @@ func PutKVToEtcd(ctx context.Context, etcdCli *clientv3.Client, retryCnt int, ke return errors.Trace(ctx.Err()) } - childCtx, cancel := context.WithTimeout(ctx, KeyOpDefaultTimeout) + childCtx, cancel := context.WithTimeout(ctx, keyOpDefaultTimeout) _, err = etcdCli.Put(childCtx, key, val, opts...) cancel() if err == nil { @@ -154,7 +152,7 @@ func (s *schemaVersionSyncer) Init(ctx context.Context) error { s.mu.globalVerCh = s.etcdCli.Watch(ctx, DDLGlobalSchemaVersion) s.mu.Unlock() - err = PutKVToEtcd(ctx, s.etcdCli, KeyOpDefaultRetryCnt, s.selfSchemaVerPath, InitialVersion, + err = PutKVToEtcd(ctx, s.etcdCli, keyOpDefaultRetryCnt, s.selfSchemaVerPath, InitialVersion, clientv3.WithLease(s.session.Lease())) return errors.Trace(err) } @@ -180,7 +178,7 @@ func (s *schemaVersionSyncer) Restart(ctx context.Context) error { } s.session = session - childCtx, cancel := context.WithTimeout(ctx, KeyOpDefaultTimeout) + childCtx, cancel := context.WithTimeout(ctx, keyOpDefaultTimeout) defer cancel() err = PutKVToEtcd(childCtx, s.etcdCli, putKeyRetryUnlimited, s.selfSchemaVerPath, InitialVersion, clientv3.WithLease(s.session.Lease())) @@ -245,16 +243,16 @@ func (s *schemaVersionSyncer) RemoveSelfVersionPath() error { metrics.DeploySyncerHistogram.WithLabelValues(metrics.SyncerClear, metrics.RetLabel(err)).Observe(time.Since(startTime).Seconds()) }() - err = DeleteKeyFromEtcd(s.selfSchemaVerPath, s.etcdCli) + err = DeleteKeyFromEtcd(s.selfSchemaVerPath, s.etcdCli, keyOpDefaultRetryCnt, keyOpDefaultTimeout) return errors.Trace(err) } // DeleteKeyFromEtcd deletes key value from etcd. -func DeleteKeyFromEtcd(key string, etcdCli *clientv3.Client) error { +func DeleteKeyFromEtcd(key string, etcdCli *clientv3.Client, retryCnt int, timeout time.Duration) error { var err error ctx := context.Background() - for i := 0; i < KeyOpDefaultRetryCnt; i++ { - childCtx, cancel := context.WithTimeout(ctx, KeyOpDefaultTimeout) + for i := 0; i < retryCnt; i++ { + childCtx, cancel := context.WithTimeout(ctx, timeout) _, err = etcdCli.Delete(childCtx, key) cancel() if err == nil { diff --git a/domain/info.go b/domain/info.go index a609c1224c3cc..8f865800655ce 100644 --- a/domain/info.go +++ b/domain/info.go @@ -32,6 +32,10 @@ import ( const ( // ServerInformationPath store server information such as IP, port and so on. ServerInformationPath = "/tidb/server/info" + // keyOpDefaultRetryCnt is the default retry count for etcd store. + keyOpDefaultRetryCnt = 2 + // keyOpDefaultTimeout is the default time out for etcd store. + keyOpDefaultTimeout = 1 * time.Second ) // InfoSyncer stores server info to Etcd when when the tidb-server starts and delete when tidb-server shuts down. @@ -86,13 +90,13 @@ func (is *InfoSyncer) GetServerInfo() *ServerInfo { return is.info } -// GetServerInfoByID gets owner server static information from Etcd. +// GetServerInfoByID gets server static information from Etcd. func (is *InfoSyncer) GetServerInfoByID(ctx context.Context, id string) (*ServerInfo, error) { if is.etcdCli == nil || id == is.info.ID { return is.info, nil } key := fmt.Sprintf("%s/%s", ServerInformationPath, id) - infoMap, err := getInfo(ctx, is.etcdCli, key) + infoMap, err := getInfo(ctx, is.etcdCli, key, keyOpDefaultRetryCnt, keyOpDefaultTimeout) if err != nil { return nil, errors.Trace(err) } @@ -110,7 +114,7 @@ func (is *InfoSyncer) GetAllServerInfo(ctx context.Context) (map[string]*ServerI allInfo[is.info.ID] = is.info return allInfo, nil } - allInfo, err := getInfo(ctx, is.etcdCli, ServerInformationPath, clientv3.WithPrefix()) + allInfo, err := getInfo(ctx, is.etcdCli, ServerInformationPath, keyOpDefaultRetryCnt, keyOpDefaultTimeout, clientv3.WithPrefix()) if err != nil { return nil, errors.Trace(err) } @@ -126,7 +130,7 @@ func (is *InfoSyncer) StoreServerInfo(ctx context.Context) error { if err != nil { return errors.Trace(err) } - err = ddl.PutKVToEtcd(ctx, is.etcdCli, ddl.KeyOpDefaultRetryCnt, is.serverInfoPath, hack.String(infoBuf)) + err = ddl.PutKVToEtcd(ctx, is.etcdCli, keyOpDefaultRetryCnt, is.serverInfoPath, hack.String(infoBuf)) return errors.Trace(err) } @@ -135,34 +139,38 @@ func (is *InfoSyncer) RemoveServerInfo() { if is.etcdCli == nil { return } - err := ddl.DeleteKeyFromEtcd(is.serverInfoPath, is.etcdCli) + err := ddl.DeleteKeyFromEtcd(is.serverInfoPath, is.etcdCli, keyOpDefaultRetryCnt, keyOpDefaultTimeout) if err != nil { - log.Errorf("[infoSyncer] remove self server info failed %v", err) + log.Errorf("[infoSyncer] remove server info failed %v", err) } } // getInfo gets server information from Etcd according to the key and opts. -func getInfo(ctx context.Context, etcdCli *clientv3.Client, key string, opts ...clientv3.OpOption) (map[string]*ServerInfo, error) { +func getInfo(ctx context.Context, etcdCli *clientv3.Client, key string, retryCnt int, timeout time.Duration, opts ...clientv3.OpOption) (map[string]*ServerInfo, error) { var err error + var resp *clientv3.GetResponse allInfo := make(map[string]*ServerInfo) - for { + for i := 0; i < retryCnt; i++ { select { case <-ctx.Done(): err = errors.Trace(ctx.Err()) return nil, err default: } - childCtx, cancel := context.WithTimeout(ctx, ddl.KeyOpDefaultTimeout) - resp, err := etcdCli.Get(childCtx, key, opts...) + childCtx, cancel := context.WithTimeout(ctx, timeout) + resp, err = etcdCli.Get(childCtx, key, opts...) cancel() if err != nil { - log.Infof("[infoSyncer] get %s failed %v, continue checking.", key, err) + // Reduce the number of logs. + if i%2 == 1 { + log.Infof("[infoSyncer] get %s failed %v, continue checking.", key, err) + } time.Sleep(200 * time.Millisecond) continue } for _, kv := range resp.Kvs { info := &ServerInfo{} - err := json.Unmarshal(kv.Value, info) + err = json.Unmarshal(kv.Value, info) if err != nil { log.Infof("[infoSyncer] get %s, json.Unmarshal %v failed %v.", kv.Key, kv.Value, err) return nil, errors.Trace(err) @@ -171,4 +179,5 @@ func getInfo(ctx context.Context, etcdCli *clientv3.Client, key string, opts ... } return allInfo, nil } + return nil, errors.Trace(err) } From b40cf74eeddd238232b380dc0cf555a6cccf8a18 Mon Sep 17 00:00:00 2001 From: crazycs Date: Wed, 8 Aug 2018 23:36:11 +0800 Subject: [PATCH 30/40] refine code --- domain/info.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/domain/info.go b/domain/info.go index 8f865800655ce..b362c54ca3e6a 100644 --- a/domain/info.go +++ b/domain/info.go @@ -161,10 +161,7 @@ func getInfo(ctx context.Context, etcdCli *clientv3.Client, key string, retryCnt resp, err = etcdCli.Get(childCtx, key, opts...) cancel() if err != nil { - // Reduce the number of logs. - if i%2 == 1 { - log.Infof("[infoSyncer] get %s failed %v, continue checking.", key, err) - } + log.Infof("[infoSyncer] get %s failed %v, continue checking.", key, err) time.Sleep(200 * time.Millisecond) continue } From 7403bb1523f38aa8400e12a14e94af6764dda50d Mon Sep 17 00:00:00 2001 From: crazycs Date: Thu, 9 Aug 2018 14:43:04 +0800 Subject: [PATCH 31/40] only return self server info when do /info http request --- docs/tidb_http_api.md | 2 +- server/http_handler.go | 38 +++++++++++++------------------------ server/http_handler_test.go | 18 +++++++++--------- server/http_status.go | 4 ++-- 4 files changed, 25 insertions(+), 37 deletions(-) diff --git a/docs/tidb_http_api.md b/docs/tidb_http_api.md index 46d4ff33adff7..819c7c1933c0a 100644 --- a/docs/tidb_http_api.md +++ b/docs/tidb_http_api.md @@ -96,7 +96,7 @@ timezone.* curl http://{TiDBIP}:10080/settings ``` -1. Get TiDB server information and owner server information. +1. Get TiDB server information. ```shell curl http://{TiDBIP}:10080/info diff --git a/server/http_handler.go b/server/http_handler.go index 32721dafe3eee..04ac3c808ad49 100644 --- a/server/http_handler.go +++ b/server/http_handler.go @@ -334,11 +334,11 @@ type ddlHistoryJobHandler struct { *tikvHandlerTool } -type ddlServerInfoHandler struct { +type serverInfoHandler struct { *tikvHandlerTool } -type ddlAllServerInfoHandler struct { +type allServerInfoHandler struct { *tikvHandlerTool } @@ -1258,15 +1258,14 @@ func (h *mvccTxnHandler) handleMvccGetByTxn(params map[string]string) (interface return h.getMvccByStartTs(uint64(startTS), startKey, endKey) } -// reportServerInfo is only used to report the servers info when do http request. -type reportServerInfo struct { - IsOwner bool `json:"is_owner"` - SelfServerInfo *domain.ServerInfo `json:"self_server_info"` - OwnerServerInfo *domain.ServerInfo `json:"owner_server_info,omitempty"` +// serverInfo is used to report the servers info when do http request. +type serverInfo struct { + IsOwner bool `json:"is_owner"` + *domain.ServerInfo } // ServeHTTP handles request of ddl server info. -func (h ddlServerInfoHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { +func (h serverInfoHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { do, err := session.GetDomain(h.store.(kv.Storage)) if err != nil { writeError(w, errors.New("create session error")) @@ -1280,24 +1279,13 @@ func (h ddlServerInfoHandler) ServeHTTP(w http.ResponseWriter, req *http.Request writeError(w, errors.New("ddl server information not found")) return } - selfInfo := is.GetServerInfo() - reportInfo := reportServerInfo{ - IsOwner: selfInfo.ID == ownerID, - SelfServerInfo: selfInfo, - } - if !reportInfo.IsOwner { - ownerInfo, err := is.GetServerInfoByID(ctx, ownerID) - if err != nil { - writeError(w, errors.New("ddl server information not found")) - log.Error(err) - return - } - reportInfo.OwnerServerInfo = ownerInfo - } - writeData(w, reportInfo) + info := serverInfo{} + info.ServerInfo = is.GetServerInfo() + info.IsOwner = info.ID == ownerID + writeData(w, info) } -// clusterServerInfo is only used to report cluster servers info when do http request. +// clusterServerInfo is used to report cluster servers info when do http request. type clusterServerInfo struct { ServersNum int `json:"servers_num,omitempty"` OwnerID string `json:"owner_id"` @@ -1307,7 +1295,7 @@ type clusterServerInfo struct { } // ServeHTTP handles request of all ddl servers info. -func (h ddlAllServerInfoHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { +func (h allServerInfoHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { do, err := session.GetDomain(h.store.(kv.Storage)) if err != nil { writeError(w, errors.New("create session error")) diff --git a/server/http_handler_test.go b/server/http_handler_test.go index ee6890016a872..0dc9b279a7100 100644 --- a/server/http_handler_test.go +++ b/server/http_handler_test.go @@ -656,23 +656,23 @@ func (ts *HTTPHandlerTestSuite) TestServerInfo(c *C) { c.Assert(resp.StatusCode, Equals, http.StatusOK) decoder := json.NewDecoder(resp.Body) - serverInfo := reportServerInfo{} - err = decoder.Decode(&serverInfo) + info := serverInfo{} + err = decoder.Decode(&info) c.Assert(err, IsNil) cfg := config.GetGlobalConfig() - c.Assert(serverInfo.IsOwner, IsTrue) - c.Assert(serverInfo.SelfServerInfo.IP, Equals, cfg.AdvertiseAddress) - c.Assert(serverInfo.SelfServerInfo.StatusPort, Equals, cfg.Status.StatusPort) - c.Assert(serverInfo.SelfServerInfo.Lease, Equals, cfg.Lease) - c.Assert(serverInfo.SelfServerInfo.Version, Equals, mysql.ServerVersion) - c.Assert(serverInfo.SelfServerInfo.GitHash, Equals, printer.TiDBGitHash) + c.Assert(info.IsOwner, IsTrue) + c.Assert(info.IP, Equals, cfg.AdvertiseAddress) + c.Assert(info.StatusPort, Equals, cfg.Status.StatusPort) + c.Assert(info.Lease, Equals, cfg.Lease) + c.Assert(info.Version, Equals, mysql.ServerVersion) + c.Assert(info.GitHash, Equals, printer.TiDBGitHash) store := ts.server.newTikvHandlerTool().store.(kv.Storage) do, err := session.GetDomain(store.(kv.Storage)) c.Assert(err, IsNil) ddl := do.DDL() - c.Assert(serverInfo.SelfServerInfo.ID, Equals, ddl.GetID()) + c.Assert(info.ID, Equals, ddl.GetID()) } func (ts *HTTPHandlerTestSuite) TestAllServerInfo(c *C) { diff --git a/server/http_status.go b/server/http_status.go index e434689fc8eda..357594d3cb695 100644 --- a/server/http_status.go +++ b/server/http_status.go @@ -54,8 +54,8 @@ func (s *Server) startHTTPServer() { router.Handle("/ddl/history", ddlHistoryJobHandler{tikvHandlerTool}) // HTTP path for get server info. - router.Handle("/info", ddlServerInfoHandler{tikvHandlerTool}) - router.Handle("/info/all", ddlAllServerInfoHandler{tikvHandlerTool}) + router.Handle("/info", serverInfoHandler{tikvHandlerTool}) + router.Handle("/info/all", allServerInfoHandler{tikvHandlerTool}) if s.cfg.Store == "tikv" { // HTTP path for tikv. router.Handle("/tables/{db}/{table}/regions", tableHandler{tikvHandlerTool, opTableRegions}) From 3801efec4a3e43f93c8af6d9bcbf085847ae15da Mon Sep 17 00:00:00 2001 From: crazycs Date: Thu, 9 Aug 2018 16:01:50 +0800 Subject: [PATCH 32/40] refine code and comment --- server/http_handler.go | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/server/http_handler.go b/server/http_handler.go index 04ac3c808ad49..520d094eee03d 100644 --- a/server/http_handler.go +++ b/server/http_handler.go @@ -1271,24 +1271,17 @@ func (h serverInfoHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { writeError(w, errors.New("create session error")) return } - ddl := do.DDL() - is := do.InfoSyncer() - ctx := context.Background() - ownerID, err := ddl.OwnerManager().GetOwnerID(ctx) - if err != nil { - writeError(w, errors.New("ddl server information not found")) - return - } info := serverInfo{} - info.ServerInfo = is.GetServerInfo() - info.IsOwner = info.ID == ownerID + info.ServerInfo = do.InfoSyncer().GetServerInfo() + info.IsOwner = do.DDL().OwnerManager().IsOwner() writeData(w, info) } // clusterServerInfo is used to report cluster servers info when do http request. type clusterServerInfo struct { - ServersNum int `json:"servers_num,omitempty"` - OwnerID string `json:"owner_id"` + ServersNum int `json:"servers_num,omitempty"` + OwnerID string `json:"owner_id"` + // IsAllServerVersionConsistent indicates whether all tidb servers version is consistent. IsAllServerVersionConsistent bool `json:"is_all_server_version_consistent,omitempty"` AllServersDiffVersions []domain.ServerVersionInfo `json:"all_servers_diff_versions,omitempty"` AllServersInfo map[string]*domain.ServerInfo `json:"all_servers_info,omitempty"` @@ -1301,18 +1294,17 @@ func (h allServerInfoHandler) ServeHTTP(w http.ResponseWriter, req *http.Request writeError(w, errors.New("create session error")) return } - ddl := do.DDL() - is := do.InfoSyncer() ctx := context.Background() - allServersInfo, err := is.GetAllServerInfo(ctx) + allServersInfo, err := do.InfoSyncer().GetAllServerInfo(ctx) if err != nil { writeError(w, errors.New("ddl server information not found")) log.Error(err) return } - ownerID, err := ddl.OwnerManager().GetOwnerID(ctx) + ownerID, err := do.DDL().OwnerManager().GetOwnerID(ctx) if err != nil { writeError(w, errors.New("ddl server information not found")) + log.Error(err) return } allVersionsMap := map[domain.ServerVersionInfo]struct{}{} @@ -1330,6 +1322,7 @@ func (h allServerInfoHandler) ServeHTTP(w http.ResponseWriter, req *http.Request IsAllServerVersionConsistent: len(allVersions) == 1, AllServersInfo: allServersInfo, } + // if IsAllServerVersionConsistent is false, return the all tidb servers version. if !clusterInfo.IsAllServerVersionConsistent { clusterInfo.AllServersDiffVersions = allVersions } From b27b80408560e275fa33ac3de089b080c44c6857 Mon Sep 17 00:00:00 2001 From: crazycs Date: Thu, 9 Aug 2018 16:08:36 +0800 Subject: [PATCH 33/40] refine comment --- server/http_handler.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/http_handler.go b/server/http_handler.go index 520d094eee03d..2c9be6877619d 100644 --- a/server/http_handler.go +++ b/server/http_handler.go @@ -1279,9 +1279,8 @@ func (h serverInfoHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { // clusterServerInfo is used to report cluster servers info when do http request. type clusterServerInfo struct { - ServersNum int `json:"servers_num,omitempty"` - OwnerID string `json:"owner_id"` - // IsAllServerVersionConsistent indicates whether all tidb servers version is consistent. + ServersNum int `json:"servers_num,omitempty"` + OwnerID string `json:"owner_id"` IsAllServerVersionConsistent bool `json:"is_all_server_version_consistent,omitempty"` AllServersDiffVersions []domain.ServerVersionInfo `json:"all_servers_diff_versions,omitempty"` AllServersInfo map[string]*domain.ServerInfo `json:"all_servers_info,omitempty"` @@ -1319,6 +1318,7 @@ func (h allServerInfoHandler) ServeHTTP(w http.ResponseWriter, req *http.Request clusterInfo := clusterServerInfo{ ServersNum: len(allServersInfo), OwnerID: ownerID, + // len(allVersions) = 1 indicates there has only 1 tidb version in cluster, so all server versions are consistent. IsAllServerVersionConsistent: len(allVersions) == 1, AllServersInfo: allServersInfo, } From 11dfd860f1ba17a7af35f9ded99f8e5594eed89a Mon Sep 17 00:00:00 2001 From: crazycs Date: Thu, 9 Aug 2018 22:13:18 +0800 Subject: [PATCH 34/40] store server info with lease --- ddl/mock.go | 5 +++++ ddl/syncer.go | 7 +++++++ domain/domain.go | 3 ++- domain/info.go | 42 ++++++++++++++++++++++++++---------------- server/http_handler.go | 2 ++ 5 files changed, 42 insertions(+), 17 deletions(-) diff --git a/ddl/mock.go b/ddl/mock.go index 70562e9a496cd..89ad340af72a2 100644 --- a/ddl/mock.go +++ b/ddl/mock.go @@ -39,6 +39,11 @@ func NewMockSchemaSyncer() SchemaSyncer { return &mockSchemaSyncer{} } +// GetSessionLeaseID implements SchemaSyncer.GetSessionLeaseID interface. +func (s *mockSchemaSyncer) GetSessionLeaseID() clientv3.LeaseID { + return 0 +} + // Init implements SchemaSyncer.Init interface. func (s *mockSchemaSyncer) Init(ctx context.Context) error { s.globalVerCh = make(chan clientv3.WatchResponse, 1) diff --git a/ddl/syncer.go b/ddl/syncer.go index f640edb495801..3a245fdfa69c7 100644 --- a/ddl/syncer.go +++ b/ddl/syncer.go @@ -83,6 +83,8 @@ type SchemaSyncer interface { // the latest schema version. If the result is false, wait for a while and check again util the processing time reach 2 * lease. // It returns until all servers' versions are equal to the latest version or the ctx is done. OwnerCheckAllVersions(ctx context.Context, latestVer int64) error + // GetSession return the session lease id of SchemaSyncer. + GetSessionLeaseID() clientv3.LeaseID } type schemaVersionSyncer struct { @@ -127,6 +129,11 @@ func PutKVToEtcd(ctx context.Context, etcdCli *clientv3.Client, retryCnt int, ke return errors.Trace(err) } +// GetSessionLeaseID implements SchemaSyncer.GetSessionLeaseID interface. +func (s *schemaVersionSyncer) GetSessionLeaseID() clientv3.LeaseID { + return s.session.Lease() +} + // Init implements SchemaSyncer.Init interface. func (s *schemaVersionSyncer) Init(ctx context.Context) error { startTime := time.Now() diff --git a/domain/domain.go b/domain/domain.go index bf242d95e35a6..7adcab8f4592b 100644 --- a/domain/domain.go +++ b/domain/domain.go @@ -365,6 +365,7 @@ func (do *Domain) loadSchemaInLoop(lease time.Duration) { break } do.SchemaValidator.Restart() + do.info.Restart(syncer.GetSessionLeaseID()) case <-do.exit: return } @@ -509,7 +510,7 @@ func (do *Domain) Init(ddlLease time.Duration, sysFactory func(*Domain) (pools.R if err != nil { return errors.Trace(err) } - do.info = NewInfoSyncer(do.ddl.GetID(), do.etcdClient) + do.info = NewInfoSyncer(do.ddl.GetID(), do.etcdClient, do.ddl.SchemaSyncer().GetSessionLeaseID()) err = do.info.StoreServerInfo(ctx) if err != nil { return errors.Trace(err) diff --git a/domain/info.go b/domain/info.go index b362c54ca3e6a..f978a7f66efed 100644 --- a/domain/info.go +++ b/domain/info.go @@ -43,6 +43,8 @@ type InfoSyncer struct { etcdCli *clientv3.Client info *ServerInfo serverInfoPath string + // InfoSyncer will reuse the leaseID of ddl.SchemaSyncer. + sessionLeaseID clientv3.LeaseID } // ServerInfo is server static information. @@ -63,28 +65,15 @@ type ServerVersionInfo struct { } // NewInfoSyncer return new InfoSyncer. It is exported for testing. -func NewInfoSyncer(id string, etcdCli *clientv3.Client) *InfoSyncer { +func NewInfoSyncer(id string, etcdCli *clientv3.Client, leaseID clientv3.LeaseID) *InfoSyncer { return &InfoSyncer{ etcdCli: etcdCli, info: getServerInfo(id), serverInfoPath: fmt.Sprintf("%s/%s", ServerInformationPath, id), + sessionLeaseID: leaseID, } } -func getServerInfo(id string) *ServerInfo { - cfg := config.GetGlobalConfig() - info := &ServerInfo{ - ID: id, - IP: cfg.AdvertiseAddress, - Port: cfg.Port, - StatusPort: cfg.Status.StatusPort, - Lease: cfg.Lease, - } - info.Version = mysql.ServerVersion - info.GitHash = printer.TiDBGitHash - return info -} - //GetServerInfo gets self server static information. func (is *InfoSyncer) GetServerInfo() *ServerInfo { return is.info @@ -130,7 +119,7 @@ func (is *InfoSyncer) StoreServerInfo(ctx context.Context) error { if err != nil { return errors.Trace(err) } - err = ddl.PutKVToEtcd(ctx, is.etcdCli, keyOpDefaultRetryCnt, is.serverInfoPath, hack.String(infoBuf)) + err = ddl.PutKVToEtcd(ctx, is.etcdCli, keyOpDefaultRetryCnt, is.serverInfoPath, hack.String(infoBuf), clientv3.WithLease(is.sessionLeaseID)) return errors.Trace(err) } @@ -145,6 +134,12 @@ func (is *InfoSyncer) RemoveServerInfo() { } } +// Restart restart the info syncer with new session leaseID and store server info to etcd again. +func (is *InfoSyncer) Restart(leaseID clientv3.LeaseID) { + is.sessionLeaseID = leaseID + is.StoreServerInfo(context.Background()) +} + // getInfo gets server information from Etcd according to the key and opts. func getInfo(ctx context.Context, etcdCli *clientv3.Client, key string, retryCnt int, timeout time.Duration, opts ...clientv3.OpOption) (map[string]*ServerInfo, error) { var err error @@ -178,3 +173,18 @@ func getInfo(ctx context.Context, etcdCli *clientv3.Client, key string, retryCnt } return nil, errors.Trace(err) } + +// getServerInfo gets self tidb server information. +func getServerInfo(id string) *ServerInfo { + cfg := config.GetGlobalConfig() + info := &ServerInfo{ + ID: id, + IP: cfg.AdvertiseAddress, + Port: cfg.Port, + StatusPort: cfg.Status.StatusPort, + Lease: cfg.Lease, + } + info.Version = mysql.ServerVersion + info.GitHash = printer.TiDBGitHash + return info +} diff --git a/server/http_handler.go b/server/http_handler.go index 2c9be6877619d..74ab47660004e 100644 --- a/server/http_handler.go +++ b/server/http_handler.go @@ -1269,6 +1269,7 @@ func (h serverInfoHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { do, err := session.GetDomain(h.store.(kv.Storage)) if err != nil { writeError(w, errors.New("create session error")) + log.Error(err) return } info := serverInfo{} @@ -1291,6 +1292,7 @@ func (h allServerInfoHandler) ServeHTTP(w http.ResponseWriter, req *http.Request do, err := session.GetDomain(h.store.(kv.Storage)) if err != nil { writeError(w, errors.New("create session error")) + log.Error(err) return } ctx := context.Background() From c4586ea536253ed5ba4de2fd5390d83faa4ce1e6 Mon Sep 17 00:00:00 2001 From: crazycs Date: Fri, 10 Aug 2018 11:10:23 +0800 Subject: [PATCH 35/40] use new seesion for server info syncer --- ddl/mock.go | 5 ----- ddl/syncer.go | 7 ------- domain/domain.go | 7 ++++--- domain/info.go | 50 ++++++++++++++++++++++++++++++++++++++---------- 4 files changed, 44 insertions(+), 25 deletions(-) diff --git a/ddl/mock.go b/ddl/mock.go index 89ad340af72a2..70562e9a496cd 100644 --- a/ddl/mock.go +++ b/ddl/mock.go @@ -39,11 +39,6 @@ func NewMockSchemaSyncer() SchemaSyncer { return &mockSchemaSyncer{} } -// GetSessionLeaseID implements SchemaSyncer.GetSessionLeaseID interface. -func (s *mockSchemaSyncer) GetSessionLeaseID() clientv3.LeaseID { - return 0 -} - // Init implements SchemaSyncer.Init interface. func (s *mockSchemaSyncer) Init(ctx context.Context) error { s.globalVerCh = make(chan clientv3.WatchResponse, 1) diff --git a/ddl/syncer.go b/ddl/syncer.go index d012ff91de2da..929be95d9a9f5 100644 --- a/ddl/syncer.go +++ b/ddl/syncer.go @@ -83,8 +83,6 @@ type SchemaSyncer interface { // the latest schema version. If the result is false, wait for a while and check again util the processing time reach 2 * lease. // It returns until all servers' versions are equal to the latest version or the ctx is done. OwnerCheckAllVersions(ctx context.Context, latestVer int64) error - // GetSession return the session lease id of SchemaSyncer. - GetSessionLeaseID() clientv3.LeaseID } type schemaVersionSyncer struct { @@ -129,11 +127,6 @@ func PutKVToEtcd(ctx context.Context, etcdCli *clientv3.Client, retryCnt int, ke return errors.Trace(err) } -// GetSessionLeaseID implements SchemaSyncer.GetSessionLeaseID interface. -func (s *schemaVersionSyncer) GetSessionLeaseID() clientv3.LeaseID { - return s.session.Lease() -} - // Init implements SchemaSyncer.Init interface. func (s *schemaVersionSyncer) Init(ctx context.Context) error { startTime := time.Now() diff --git a/domain/domain.go b/domain/domain.go index 7adcab8f4592b..7f80aa0baf5f0 100644 --- a/domain/domain.go +++ b/domain/domain.go @@ -365,7 +365,8 @@ func (do *Domain) loadSchemaInLoop(lease time.Duration) { break } do.SchemaValidator.Restart() - do.info.Restart(syncer.GetSessionLeaseID()) + case <-do.info.Done(): + do.info.Restart(context.Background()) case <-do.exit: return } @@ -510,8 +511,8 @@ func (do *Domain) Init(ddlLease time.Duration, sysFactory func(*Domain) (pools.R if err != nil { return errors.Trace(err) } - do.info = NewInfoSyncer(do.ddl.GetID(), do.etcdClient, do.ddl.SchemaSyncer().GetSessionLeaseID()) - err = do.info.StoreServerInfo(ctx) + do.info = NewInfoSyncer(do.ddl.GetID(), do.etcdClient) + err = do.info.Init(ctx) if err != nil { return errors.Trace(err) } diff --git a/domain/info.go b/domain/info.go index f978a7f66efed..589a32b16528d 100644 --- a/domain/info.go +++ b/domain/info.go @@ -19,10 +19,12 @@ import ( "time" "github.com/coreos/etcd/clientv3" + "github.com/coreos/etcd/clientv3/concurrency" "github.com/juju/errors" "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/ddl" "github.com/pingcap/tidb/mysql" + "github.com/pingcap/tidb/owner" "github.com/pingcap/tidb/util/hack" "github.com/pingcap/tidb/util/printer" log "github.com/sirupsen/logrus" @@ -38,13 +40,15 @@ const ( keyOpDefaultTimeout = 1 * time.Second ) +// InfoSessionTTL is the etcd session's TTL in seconds. It's exported for testing. +var InfoSessionTTL = 10 * 60 + // InfoSyncer stores server info to Etcd when when the tidb-server starts and delete when tidb-server shuts down. type InfoSyncer struct { etcdCli *clientv3.Client info *ServerInfo serverInfoPath string - // InfoSyncer will reuse the leaseID of ddl.SchemaSyncer. - sessionLeaseID clientv3.LeaseID + session *concurrency.Session } // ServerInfo is server static information. @@ -65,15 +69,19 @@ type ServerVersionInfo struct { } // NewInfoSyncer return new InfoSyncer. It is exported for testing. -func NewInfoSyncer(id string, etcdCli *clientv3.Client, leaseID clientv3.LeaseID) *InfoSyncer { +func NewInfoSyncer(id string, etcdCli *clientv3.Client) *InfoSyncer { return &InfoSyncer{ etcdCli: etcdCli, info: getServerInfo(id), serverInfoPath: fmt.Sprintf("%s/%s", ServerInformationPath, id), - sessionLeaseID: leaseID, } } +// Init creates a new etcd session and stores server info to etcd. +func (is *InfoSyncer) Init(ctx context.Context) error { + return errors.Trace(is.newSessionAndStoreServerInfo(ctx, owner.NewSessionDefaultRetryCnt)) +} + //GetServerInfo gets self server static information. func (is *InfoSyncer) GetServerInfo() *ServerInfo { return is.info @@ -110,8 +118,8 @@ func (is *InfoSyncer) GetAllServerInfo(ctx context.Context) (map[string]*ServerI return allInfo, nil } -// StoreServerInfo stores self server static information to Etcd when domain Init. -func (is *InfoSyncer) StoreServerInfo(ctx context.Context) error { +// storeServerInfo stores self server static information to Etcd. +func (is *InfoSyncer) storeServerInfo(ctx context.Context) error { if is.etcdCli == nil { return nil } @@ -119,7 +127,7 @@ func (is *InfoSyncer) StoreServerInfo(ctx context.Context) error { if err != nil { return errors.Trace(err) } - err = ddl.PutKVToEtcd(ctx, is.etcdCli, keyOpDefaultRetryCnt, is.serverInfoPath, hack.String(infoBuf), clientv3.WithLease(is.sessionLeaseID)) + err = ddl.PutKVToEtcd(ctx, is.etcdCli, keyOpDefaultRetryCnt, is.serverInfoPath, hack.String(infoBuf), clientv3.WithLease(is.session.Lease())) return errors.Trace(err) } @@ -134,10 +142,32 @@ func (is *InfoSyncer) RemoveServerInfo() { } } +// Done returns a channel that closes when the info syncer is no longer being refreshed. +func (is InfoSyncer) Done() <-chan struct{} { + if is.etcdCli == nil { + return make(chan struct{}, 1) + } + return is.session.Done() +} + // Restart restart the info syncer with new session leaseID and store server info to etcd again. -func (is *InfoSyncer) Restart(leaseID clientv3.LeaseID) { - is.sessionLeaseID = leaseID - is.StoreServerInfo(context.Background()) +func (is *InfoSyncer) Restart(ctx context.Context) error { + return errors.Trace(is.newSessionAndStoreServerInfo(ctx, owner.NewSessionRetryUnlimited)) +} + +// newSessionAndStoreServerInfo creates a new etcd session and stores server info to etcd. +func (is *InfoSyncer) newSessionAndStoreServerInfo(ctx context.Context, retryCnt int) error { + if is.etcdCli == nil { + return nil + } + var err error + logPrefix := fmt.Sprintf("[InfoSyncer] %s", is.serverInfoPath) + is.session, err = owner.NewSession(ctx, logPrefix, is.etcdCli, retryCnt, InfoSessionTTL) + if err != nil { + return errors.Trace(err) + } + err = is.storeServerInfo(ctx) + return errors.Trace(err) } // getInfo gets server information from Etcd according to the key and opts. From b7b35e3210197d6902f357823973acfe339edff5 Mon Sep 17 00:00:00 2001 From: crazycs Date: Fri, 10 Aug 2018 23:32:11 +0800 Subject: [PATCH 36/40] address comment --- domain/info.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/domain/info.go b/domain/info.go index 589a32b16528d..b35eb8b8b4556 100644 --- a/domain/info.go +++ b/domain/info.go @@ -43,7 +43,7 @@ const ( // InfoSessionTTL is the etcd session's TTL in seconds. It's exported for testing. var InfoSessionTTL = 10 * 60 -// InfoSyncer stores server info to Etcd when when the tidb-server starts and delete when tidb-server shuts down. +// InfoSyncer stores server info to etcd when when the tidb-server starts and delete when tidb-server shuts down. type InfoSyncer struct { etcdCli *clientv3.Client info *ServerInfo @@ -87,7 +87,7 @@ func (is *InfoSyncer) GetServerInfo() *ServerInfo { return is.info } -// GetServerInfoByID gets server static information from Etcd. +// GetServerInfoByID gets server static information from etcd. func (is *InfoSyncer) GetServerInfoByID(ctx context.Context, id string) (*ServerInfo, error) { if is.etcdCli == nil || id == is.info.ID { return is.info, nil @@ -104,7 +104,7 @@ func (is *InfoSyncer) GetServerInfoByID(ctx context.Context, id string) (*Server return info, nil } -// GetAllServerInfo gets all servers static information from Etcd. +// GetAllServerInfo gets all servers static information from etcd. func (is *InfoSyncer) GetAllServerInfo(ctx context.Context) (map[string]*ServerInfo, error) { allInfo := make(map[string]*ServerInfo) if is.etcdCli == nil { @@ -118,7 +118,7 @@ func (is *InfoSyncer) GetAllServerInfo(ctx context.Context) (map[string]*ServerI return allInfo, nil } -// storeServerInfo stores self server static information to Etcd. +// storeServerInfo stores self server static information to etcd. func (is *InfoSyncer) storeServerInfo(ctx context.Context) error { if is.etcdCli == nil { return nil @@ -131,7 +131,7 @@ func (is *InfoSyncer) storeServerInfo(ctx context.Context) error { return errors.Trace(err) } -// RemoveServerInfo remove self server static information from Etcd. +// RemoveServerInfo remove self server static information from etcd. func (is *InfoSyncer) RemoveServerInfo() { if is.etcdCli == nil { return @@ -170,7 +170,7 @@ func (is *InfoSyncer) newSessionAndStoreServerInfo(ctx context.Context, retryCnt return errors.Trace(err) } -// getInfo gets server information from Etcd according to the key and opts. +// getInfo gets server information from etcd according to the key and opts. func getInfo(ctx context.Context, etcdCli *clientv3.Client, key string, retryCnt int, timeout time.Duration, opts ...clientv3.OpOption) (map[string]*ServerInfo, error) { var err error var resp *clientv3.GetResponse From 6060b5740cc522753f1803090e658d0adbdcfec5 Mon Sep 17 00:00:00 2001 From: crazycs Date: Mon, 13 Aug 2018 17:32:00 +0800 Subject: [PATCH 37/40] address comment --- ddl/syncer.go | 4 ++-- domain/info.go | 12 ++++++------ server/http_handler.go | 2 ++ 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/ddl/syncer.go b/ddl/syncer.go index 929be95d9a9f5..8b74b1706290c 100644 --- a/ddl/syncer.go +++ b/ddl/syncer.go @@ -121,7 +121,7 @@ func PutKVToEtcd(ctx context.Context, etcdCli *clientv3.Client, retryCnt int, ke if err == nil { return nil } - log.Warnf("[etcdCli] put key: %s value: %s failed %v no.%d", key, val, err, i) + log.Warnf("[etcd-cli] put key: %s value: %s failed %v no.%d", key, val, err, i) time.Sleep(keyOpRetryInterval) } return errors.Trace(err) @@ -259,7 +259,7 @@ func DeleteKeyFromEtcd(key string, etcdCli *clientv3.Client, retryCnt int, timeo if err == nil { return nil } - log.Warnf("[etcdCli] delete key %s failed %v no.%d", key, err, i) + log.Warnf("[etcd-cli] delete key %s failed %v no.%d", key, err, i) } return errors.Trace(err) } diff --git a/domain/info.go b/domain/info.go index b35eb8b8b4556..bbcd95a60afb4 100644 --- a/domain/info.go +++ b/domain/info.go @@ -82,7 +82,7 @@ func (is *InfoSyncer) Init(ctx context.Context) error { return errors.Trace(is.newSessionAndStoreServerInfo(ctx, owner.NewSessionDefaultRetryCnt)) } -//GetServerInfo gets self server static information. +// GetServerInfo gets self server static information. func (is *InfoSyncer) GetServerInfo() *ServerInfo { return is.info } @@ -99,7 +99,7 @@ func (is *InfoSyncer) GetServerInfoByID(ctx context.Context, id string) (*Server } info, ok := infoMap[id] if !ok { - return nil, errors.Errorf("[infoSyncer] get %s failed", key) + return nil, errors.Errorf("[info-syncer] get %s failed", key) } return info, nil } @@ -138,7 +138,7 @@ func (is *InfoSyncer) RemoveServerInfo() { } err := ddl.DeleteKeyFromEtcd(is.serverInfoPath, is.etcdCli, keyOpDefaultRetryCnt, keyOpDefaultTimeout) if err != nil { - log.Errorf("[infoSyncer] remove server info failed %v", err) + log.Errorf("[info-syncer] remove server info failed %v", err) } } @@ -161,7 +161,7 @@ func (is *InfoSyncer) newSessionAndStoreServerInfo(ctx context.Context, retryCnt return nil } var err error - logPrefix := fmt.Sprintf("[InfoSyncer] %s", is.serverInfoPath) + logPrefix := fmt.Sprintf("[Info-syncer] %s", is.serverInfoPath) is.session, err = owner.NewSession(ctx, logPrefix, is.etcdCli, retryCnt, InfoSessionTTL) if err != nil { return errors.Trace(err) @@ -186,7 +186,7 @@ func getInfo(ctx context.Context, etcdCli *clientv3.Client, key string, retryCnt resp, err = etcdCli.Get(childCtx, key, opts...) cancel() if err != nil { - log.Infof("[infoSyncer] get %s failed %v, continue checking.", key, err) + log.Infof("[info-syncer] get %s failed %v, continue checking.", key, err) time.Sleep(200 * time.Millisecond) continue } @@ -194,7 +194,7 @@ func getInfo(ctx context.Context, etcdCli *clientv3.Client, key string, retryCnt info := &ServerInfo{} err = json.Unmarshal(kv.Value, info) if err != nil { - log.Infof("[infoSyncer] get %s, json.Unmarshal %v failed %v.", kv.Key, kv.Value, err) + log.Infof("[info-syncer] get %s, json.Unmarshal %v failed %v.", kv.Key, kv.Value, err) return nil, errors.Trace(err) } allInfo[info.ID] = info diff --git a/server/http_handler.go b/server/http_handler.go index 74ab47660004e..e3b2551968a60 100644 --- a/server/http_handler.go +++ b/server/http_handler.go @@ -1302,7 +1302,9 @@ func (h allServerInfoHandler) ServeHTTP(w http.ResponseWriter, req *http.Request log.Error(err) return } + ctx, cancel := context.WithTimeout(ctx, 3*time.Second) ownerID, err := do.DDL().OwnerManager().GetOwnerID(ctx) + cancel() if err != nil { writeError(w, errors.New("ddl server information not found")) log.Error(err) From 2a97788f50f7528f5d825649a3d86c96f340d632 Mon Sep 17 00:00:00 2001 From: crazycs Date: Tue, 14 Aug 2018 11:19:38 +0800 Subject: [PATCH 38/40] address comment --- ddl/syncer.go | 2 +- domain/info.go | 4 ++-- util/memory/action.go | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ddl/syncer.go b/ddl/syncer.go index 8b74b1706290c..50b1c97e5a7b2 100644 --- a/ddl/syncer.go +++ b/ddl/syncer.go @@ -105,7 +105,7 @@ func NewSchemaSyncer(etcdCli *clientv3.Client, id string) SchemaSyncer { // PutKVToEtcd puts key value to etcd. // etcdCli is client of etcd. -// retryCnt is retry time when When an error occurs. +// retryCnt is retry time when an error occurs. // opts is configures of etcd Operations. func PutKVToEtcd(ctx context.Context, etcdCli *clientv3.Client, retryCnt int, key, val string, opts ...clientv3.OpOption) error { diff --git a/domain/info.go b/domain/info.go index bbcd95a60afb4..41b4afac83f4d 100644 --- a/domain/info.go +++ b/domain/info.go @@ -43,7 +43,7 @@ const ( // InfoSessionTTL is the etcd session's TTL in seconds. It's exported for testing. var InfoSessionTTL = 10 * 60 -// InfoSyncer stores server info to etcd when when the tidb-server starts and delete when tidb-server shuts down. +// InfoSyncer stores server info to etcd when the tidb-server starts and delete when tidb-server shuts down. type InfoSyncer struct { etcdCli *clientv3.Client info *ServerInfo @@ -52,7 +52,7 @@ type InfoSyncer struct { } // ServerInfo is server static information. -// It will not update when server running. So please only put static information in ServerInfo struct. +// It will not be updated when tidb-server running. So please only put static information in ServerInfo struct. type ServerInfo struct { ServerVersionInfo ID string `json:"ddl_id"` diff --git a/util/memory/action.go b/util/memory/action.go index d06b33c22d733..06a2d39deded8 100644 --- a/util/memory/action.go +++ b/util/memory/action.go @@ -45,13 +45,13 @@ func (a *LogOnExceed) Action(t *Tracker) { } } -// PanicOnExceed panics when when memory usage exceeds memory quota. +// PanicOnExceed panics when memory usage exceeds memory quota. type PanicOnExceed struct { mutex sync.Mutex // For synchronization. acted bool } -// Action panics when when memory usage exceeds memory quota. +// Action panics when memory usage exceeds memory quota. func (a *PanicOnExceed) Action(t *Tracker) { a.mutex.Lock() if a.acted { From 2da0623c4c55af28e8a209c4dfaeaee87c3d82e4 Mon Sep 17 00:00:00 2001 From: crazycs Date: Tue, 14 Aug 2018 15:09:33 +0800 Subject: [PATCH 39/40] address comment --- domain/info.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/domain/info.go b/domain/info.go index 41b4afac83f4d..1bba31e6ea2e1 100644 --- a/domain/info.go +++ b/domain/info.go @@ -41,7 +41,7 @@ const ( ) // InfoSessionTTL is the etcd session's TTL in seconds. It's exported for testing. -var InfoSessionTTL = 10 * 60 +var InfoSessionTTL = 1 * 60 // InfoSyncer stores server info to etcd when the tidb-server starts and delete when tidb-server shuts down. type InfoSyncer struct { From 26c8b256ccefe5a3e640e05009d7f9c352c69c65 Mon Sep 17 00:00:00 2001 From: crazycs Date: Tue, 14 Aug 2018 18:54:11 +0800 Subject: [PATCH 40/40] address comment --- domain/info.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/domain/info.go b/domain/info.go index 1bba31e6ea2e1..6ab5a5f5d5bdb 100644 --- a/domain/info.go +++ b/domain/info.go @@ -152,7 +152,7 @@ func (is InfoSyncer) Done() <-chan struct{} { // Restart restart the info syncer with new session leaseID and store server info to etcd again. func (is *InfoSyncer) Restart(ctx context.Context) error { - return errors.Trace(is.newSessionAndStoreServerInfo(ctx, owner.NewSessionRetryUnlimited)) + return errors.Trace(is.newSessionAndStoreServerInfo(ctx, owner.NewSessionDefaultRetryCnt)) } // newSessionAndStoreServerInfo creates a new etcd session and stores server info to etcd.