From b24d2e1f99618449c893cd2e6e0c6fe92a472c75 Mon Sep 17 00:00:00 2001 From: fabian4 Date: Wed, 20 Sep 2023 17:49:32 +0800 Subject: [PATCH 01/21] feat: add related cache for additional instance data. --- cache/service/instance.go | 5 ++++- common/model/naming.go | 8 ++++++++ store/discover_api.go | 1 + store/mysql/scripts/polaris_server.sql | 17 +++++++++++++++++ 4 files changed, 30 insertions(+), 1 deletion(-) diff --git a/cache/service/instance.go b/cache/service/instance.go index a16ca6cac..784944aba 100644 --- a/cache/service/instance.go +++ b/cache/service/instance.go @@ -51,7 +51,9 @@ type instanceCache struct { // service id -> [instanceid ->instance] services *utils.SyncMap[string, *ServiceInstances] // service id -> [instanceCount] - instanceCounts *utils.SyncMap[string, *model.InstanceCount] + instanceCounts *utils.SyncMap[string, *model.InstanceCount] + // service id -> [instanceConsole] + instanceConsoles *utils.SyncMap[string, *model.InstanceConsole] instancePorts *instancePorts disableBusiness bool needMeta bool @@ -77,6 +79,7 @@ func (ic *instanceCache) Initialize(opt map[string]interface{}) error { ic.services = utils.NewSyncMap[string, *ServiceInstances]() ic.instanceCounts = utils.NewSyncMap[string, *model.InstanceCount]() ic.instancePorts = newInstancePorts() + ic.instanceConsoles = utils.NewSyncMap[string, *model.InstanceConsole]() if opt == nil { return nil } diff --git a/common/model/naming.go b/common/model/naming.go index 68e652257..576275b71 100644 --- a/common/model/naming.go +++ b/common/model/naming.go @@ -861,6 +861,14 @@ type InstanceEvent struct { MetaData map[string]string } +// instance info from console +type InstanceConsole struct { + id string + isolate bool + weight int8 + metadata map[string]string +} + // InjectMetadata 从context中获取metadata并注入到事件对象 func (i *InstanceEvent) InjectMetadata(ctx context.Context) { value := ctx.Value(CtxEventKeyMetadata) diff --git a/store/discover_api.go b/store/discover_api.go index 5c3463873..7292a88f5 100644 --- a/store/discover_api.go +++ b/store/discover_api.go @@ -121,6 +121,7 @@ type InstanceStore interface { // GetMoreInstances 根据mtime获取增量instances，返回所有store的变更信息 // 此方法用于 cache 增量更新，需要注意 mtime 应为数据库时间戳 GetMoreInstances(tx Tx, mtime time.Time, firstUpdate, needMeta bool, serviceID []string) (map[string]*model.Instance, error) + GetMoreInstanceConsoles(tx Tx, mtime time.Time, firstUpdate, needMeta bool, serviceID []string) (map[string]*instanceConsole, error) // SetInstanceHealthStatus 设置实例的健康状态 SetInstanceHealthStatus(instanceID string, flag int, revision string) error // BatchSetInstanceHealthStatus 批量设置实例的健康状态 diff --git a/store/mysql/scripts/polaris_server.sql b/store/mysql/scripts/polaris_server.sql index a88e66068..2acc14ae9 100644 --- a/store/mysql/scripts/polaris_server.sql +++ b/store/mysql/scripts/polaris_server.sql @@ -57,6 +57,23 @@ CREATE TABLE `instance` KEY `host` (`host`) ) ENGINE = InnoDB; +-- -------------------------------------------------------- +-- +-- Table structure `instance_console` +-- +CREATE TABLE `instance_console` +( + `id` VARCHAR(128) NOT NULL COMMENT 'Unique ID', + `isolate` TINYINT(4) DEFAULT NULL COMMENT 'Example isolation status flag, 0 is not isolated, 1 is isolated', + `weight` SMALLINT(6) DEFAULT NULL COMMENT 'The weight of the instance is mainly used for LoadBalance, default is 100', + `metadata` LONGTEXT COLLATE utf8_bin DEFAULT NULL COMMENT 'Instance metadata', + `flag` TINYINT(4) NOT NULL DEFAULT '0' COMMENT 'Logic delete flag, 0 means visible, 1 means that it has been logically deleted', + `ctime` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time', + `mtime` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Last updated time', + PRIMARY KEY (`id`), + KEY `mtime` (`mtime`) +) ENGINE = InnoDB; + -- -------------------------------------------------------- -- -- Table structure `health_check` From 9e821e476af4314dcf337905b7042f44bff0f911 Mon Sep 17 00:00:00 2001 From: fabian4 Date: Fri, 22 Sep 2023 15:25:10 +0800 Subject: [PATCH 02/21] feat: add related cache for additional instance data. --- cache/service/instance.go | 11 ++++++++++ common/model/naming.go | 8 +++---- store/discover_api.go | 2 +- store/mysql/instance.go | 45 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 61 insertions(+), 5 deletions(-) diff --git a/cache/service/instance.go b/cache/service/instance.go index 784944aba..1ce886703 100644 --- a/cache/service/instance.go +++ b/cache/service/instance.go @@ -194,6 +194,17 @@ func (ic *instanceCache) handleUpdate(start time.Time, tx store.Tx) ([]*eventhub return nil, nil, -1, err } + instanceConsoles, err := ic.storage.GetMoreInstanceConsoles(tx, ic.LastFetchTime(), ic.IsFirstUpdate(), + ic.needMeta, ic.systemServiceID) + if err != nil { + log.Error("[Cache][InstanceConsole] update get storage more", zap.Error(err)) + return nil, nil, -1, err + } + for _, item := range instanceConsoles { + //Todo: check validation + ic.instanceConsoles.Store(item.Id, item) + } + events, lastMtimes, update, del := ic.setInstances(instances) log.Info("[Cache][Instance] get more instances", zap.Int("pull-from-store", len(instances)), zap.Int("update", update), zap.Int("delete", del), diff --git a/common/model/naming.go b/common/model/naming.go index 576275b71..777d0ff2c 100644 --- a/common/model/naming.go +++ b/common/model/naming.go @@ -863,10 +863,10 @@ type InstanceEvent struct { // instance info from console type InstanceConsole struct { - id string - isolate bool - weight int8 - metadata map[string]string + Id string + Isolate bool + Weight int8 + Metadata map[string]string } // InjectMetadata 从context中获取metadata并注入到事件对象 diff --git a/store/discover_api.go b/store/discover_api.go index 7292a88f5..b08e912c4 100644 --- a/store/discover_api.go +++ b/store/discover_api.go @@ -121,7 +121,7 @@ type InstanceStore interface { // GetMoreInstances 根据mtime获取增量instances，返回所有store的变更信息 // 此方法用于 cache 增量更新，需要注意 mtime 应为数据库时间戳 GetMoreInstances(tx Tx, mtime time.Time, firstUpdate, needMeta bool, serviceID []string) (map[string]*model.Instance, error) - GetMoreInstanceConsoles(tx Tx, mtime time.Time, firstUpdate, needMeta bool, serviceID []string) (map[string]*instanceConsole, error) + GetMoreInstanceConsoles(tx Tx, mtime time.Time, firstUpdate, needMeta bool, serviceID []string) (map[string]*model.InstanceConsole, error) // SetInstanceHealthStatus 设置实例的健康状态 SetInstanceHealthStatus(instanceID string, flag int, revision string) error // BatchSetInstanceHealthStatus 批量设置实例的健康状态 diff --git a/store/mysql/instance.go b/store/mysql/instance.go index b55221d97..a2c49f964 100644 --- a/store/mysql/instance.go +++ b/store/mysql/instance.go @@ -508,6 +508,51 @@ func (ins *instanceStore) GetMoreInstances(tx store.Tx, mtime time.Time, firstUp return instances, nil } +// GetMoreInstancesConsoles 获取增量修改数据 +func (ins *instanceStore) GetMoreInstancesConsoles(tx store.Tx, mtime time.Time, firstUpdate, needMeta bool, + serviceID []string) (map[string]*model.InstanceConsole, error) { + + dbTx, _ := tx.GetDelegateTx().(*BaseTx) + + str := `select id, isolate, weight, metadata from instance where instance.mtime >= FROM_UNIXTIME(?) ` + + if firstUpdate { + str += " and flag != 1" + } + + rows, err := dbTx.Query(str, timeToTimestamp(mtime)) + if err != nil { + log.Errorf("[Store][database] get more instanceConsoles query err: %s", err.Error()) + return nil, err + } + + out := make(map[string]*model.InstanceConsole) + + if rows == nil { + return out, err + } + + defer rows.Close() + var item model.InstanceConsole + + for rows.Next() { + err := rows.Scan(&item.Id, &item.Isolate, &item.Metadata, &item.Weight) + if err != nil { + log.Errorf("[Store][database] fetch instanceConsole rows err: %s", err.Error()) + return out, err + } + + out[item.Id] = &item + } + + if err := rows.Err(); err != nil { + log.Errorf("[Store][database] instanceConsole rows catch err: %s", err.Error()) + return out, err + } + + return out, err +} + // GetInstanceMeta 根据实例ID获取实例的metadata func (ins *instanceStore) GetInstanceMeta(instanceID string) (map[string]string, error) { str := "select `mkey`, `mvalue` from instance_metadata where id = ?" From 3118f500d8ef5184242acf21a3fbf88ece97aeef Mon Sep 17 00:00:00 2001 From: fabian4 Date: Fri, 22 Sep 2023 17:30:17 +0800 Subject: [PATCH 03/21] feat: add related cache for additional instance data. --- cache/service/instance.go | 3 ++- store/mysql/instance.go | 17 +++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/cache/service/instance.go b/cache/service/instance.go index 1ce886703..c0308a972 100644 --- a/cache/service/instance.go +++ b/cache/service/instance.go @@ -78,8 +78,8 @@ func (ic *instanceCache) Initialize(opt map[string]interface{}) error { ic.ids = utils.NewSyncMap[string, *model.Instance]() ic.services = utils.NewSyncMap[string, *ServiceInstances]() ic.instanceCounts = utils.NewSyncMap[string, *model.InstanceCount]() - ic.instancePorts = newInstancePorts() ic.instanceConsoles = utils.NewSyncMap[string, *model.InstanceConsole]() + ic.instancePorts = newInstancePorts() if opt == nil { return nil } @@ -218,6 +218,7 @@ func (ic *instanceCache) Clear() error { ic.ids = utils.NewSyncMap[string, *model.Instance]() ic.services = utils.NewSyncMap[string, *ServiceInstances]() ic.instanceCounts = utils.NewSyncMap[string, *model.InstanceCount]() + ic.instanceConsoles = utils.NewSyncMap[string, *model.InstanceConsole]() ic.instancePorts.reset() ic.instanceCount = 0 return nil diff --git a/store/mysql/instance.go b/store/mysql/instance.go index a2c49f964..3fb05481d 100644 --- a/store/mysql/instance.go +++ b/store/mysql/instance.go @@ -19,6 +19,7 @@ package sqldb import ( "database/sql" + "encoding/json" "errors" "fmt" "strings" @@ -1059,6 +1060,22 @@ func batchAddInstanceCheck(tx *BaseTx, instances []*model.Instance) error { return err } +// addInstanceConsole +func addInstanceConsole(tx *BaseTx, instanceConsole *model.InstanceConsole) error { + + str := "insert into instance_console(`id`, `isolate`, `weight`, `metadata`, `ctime`, `mtime`) values " + str += "(?, ?, ?, ?, sysdate(), sysdate())" + metadata, err := json.Marshal(&instanceConsole.Metadata) + if err != nil { + log.Errorf("[Store][database] instance_console metadata phrase err: %s", err.Error()) + return err + } + + _, err := tx.Exec(str, instanceConsole.Id, instanceConsole.Isolate, string(metadata), instanceConsole.Weight) + + return err +} + // addInstanceMeta 往表中加入instance meta数据 func addInstanceMeta(tx *BaseTx, id string, meta map[string]string) error { if len(meta) == 0 { From 7f02df5e070c7fe83a106a076e6a8e31ac21e94b Mon Sep 17 00:00:00 2001 From: fabian4 Date: Sat, 23 Sep 2023 13:15:51 +0800 Subject: [PATCH 04/21] feat: add related cache for additional instance data. --- store/mysql/instance.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/store/mysql/instance.go b/store/mysql/instance.go index 3fb05481d..30d4feab3 100644 --- a/store/mysql/instance.go +++ b/store/mysql/instance.go @@ -1071,7 +1071,7 @@ func addInstanceConsole(tx *BaseTx, instanceConsole *model.InstanceConsole) erro return err } - _, err := tx.Exec(str, instanceConsole.Id, instanceConsole.Isolate, string(metadata), instanceConsole.Weight) + _, err = tx.Exec(str, instanceConsole.Id, instanceConsole.Isolate, string(metadata), instanceConsole.Weight) return err } From f9336bc74f4cd299866a32f907ed5e068d5d511f Mon Sep 17 00:00:00 2001 From: fabian4 Date: Mon, 25 Sep 2023 13:57:50 +0800 Subject: [PATCH 05/21] feat: add related cache for additional instance data. --- store/boltdb/instance.go | 7 +++++++ store/mysql/instance.go | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/store/boltdb/instance.go b/store/boltdb/instance.go index 4b3e2c183..17e172d0e 100644 --- a/store/boltdb/instance.go +++ b/store/boltdb/instance.go @@ -572,6 +572,13 @@ func (i *instanceStore) GetMoreInstances(tx store.Tx, mtime time.Time, firstUpda return toInstance(instances), nil } +// GetMoreInstancesConsoles 获取增量修改数据 +func (ins *instanceStore) GetMoreInstanceConsoles(tx store.Tx, mtime time.Time, firstUpdate, needMeta bool, + serviceID []string) (map[string]*model.InstanceConsole, error) { + //Todo: get from boltdb + return nil, nil +} + // BatchSetInstanceHealthStatus 批量设置实例的健康状态 func (i *instanceStore) BatchSetInstanceHealthStatus(ids []interface{}, healthy int, revision string) error { for _, id := range ids { diff --git a/store/mysql/instance.go b/store/mysql/instance.go index 30d4feab3..1fcbb41e2 100644 --- a/store/mysql/instance.go +++ b/store/mysql/instance.go @@ -510,7 +510,7 @@ func (ins *instanceStore) GetMoreInstances(tx store.Tx, mtime time.Time, firstUp } // GetMoreInstancesConsoles 获取增量修改数据 -func (ins *instanceStore) GetMoreInstancesConsoles(tx store.Tx, mtime time.Time, firstUpdate, needMeta bool, +func (ins *instanceStore) GetMoreInstanceConsoles(tx store.Tx, mtime time.Time, firstUpdate, needMeta bool, serviceID []string) (map[string]*model.InstanceConsole, error) { dbTx, _ := tx.GetDelegateTx().(*BaseTx) From 7cd8ff9cff57b13e05c2242b5e69bd3f3ef10d30 Mon Sep 17 00:00:00 2001 From: fabian4 Date: Mon, 25 Sep 2023 16:44:59 +0800 Subject: [PATCH 06/21] feat: add related cache for additional instance data. --- store/boltdb/instance.go | 18 ++++++++++++++++++ store/discover_api.go | 3 +++ 2 files changed, 21 insertions(+) diff --git a/store/boltdb/instance.go b/store/boltdb/instance.go index 17e172d0e..4b037d39f 100644 --- a/store/boltdb/instance.go +++ b/store/boltdb/instance.go @@ -579,6 +579,24 @@ func (ins *instanceStore) GetMoreInstanceConsoles(tx store.Tx, mtime time.Time, return nil, nil } +// DeleteInstanceConsole 逻辑删除instanceConsole +func (ins *instanceStore) DeleteInstanceConsole(instanceConsoleID string) error { + //Todo: delete from boltdb + return nil +} + +// CleanInstanceConsole 物理删除instanceConsole +func (ins *instanceStore) CleanInstanceConsole(instanceConsoleID string) error { + //Todo: clean from boltdb + return nil +} + +// UpdateInstanceConsole 更新instanceConsole +func (ins *instanceStore) UpdateInstanceConsole(instance *model.InstanceConsole) error { + //Todo: update from boltdb + return nil +} + // BatchSetInstanceHealthStatus 批量设置实例的健康状态 func (i *instanceStore) BatchSetInstanceHealthStatus(ids []interface{}, healthy int, revision string) error { for _, id := range ids { diff --git a/store/discover_api.go b/store/discover_api.go index b08e912c4..dbd5c32c7 100644 --- a/store/discover_api.go +++ b/store/discover_api.go @@ -97,12 +97,15 @@ type InstanceStore interface { BatchAddInstances(instances []*model.Instance) error // UpdateInstance 更新实例 UpdateInstance(instance *model.Instance) error + UpdateInstanceConsole(instance *model.InstanceConsole) error // DeleteInstance 删除一个实例，实际是把valid置为false DeleteInstance(instanceID string) error + DeleteInstanceConsole(instanceConsoleID string) error // BatchDeleteInstances 批量删除实例，flag=1 BatchDeleteInstances(ids []interface{}) error // CleanInstance 清空一个实例，真正删除 CleanInstance(instanceID string) error + CleanInstanceConsole(instanceConsoleID string) error // BatchGetInstanceIsolate 检查ID是否存在，并且返回存在的ID，以及ID的隔离状态 BatchGetInstanceIsolate(ids map[string]bool) (map[string]bool, error) // GetInstancesBrief 获取实例关联的token From b17cbee6feb6a322bec85033012e4960a07d177b Mon Sep 17 00:00:00 2001 From: fabian4 Date: Tue, 26 Sep 2023 15:54:45 +0800 Subject: [PATCH 07/21] feat: add related cache for additional instance data. --- common/model/naming.go | 2 +- store/boltdb/instance.go | 2 +- store/discover_api.go | 2 +- store/mysql/instance.go | 73 ++++++++++++++++++++++++++++++++++++---- 4 files changed, 69 insertions(+), 10 deletions(-) diff --git a/common/model/naming.go b/common/model/naming.go index 777d0ff2c..61662f4dc 100644 --- a/common/model/naming.go +++ b/common/model/naming.go @@ -866,7 +866,7 @@ type InstanceConsole struct { Id string Isolate bool Weight int8 - Metadata map[string]string + Metadata string } // InjectMetadata 从context中获取metadata并注入到事件对象 diff --git a/store/boltdb/instance.go b/store/boltdb/instance.go index 4b037d39f..93f4a2e45 100644 --- a/store/boltdb/instance.go +++ b/store/boltdb/instance.go @@ -592,7 +592,7 @@ func (ins *instanceStore) CleanInstanceConsole(instanceConsoleID string) error { } // UpdateInstanceConsole 更新instanceConsole -func (ins *instanceStore) UpdateInstanceConsole(instance *model.InstanceConsole) error { +func (ins *instanceStore) UpdateInstanceConsole(instanceConsole *model.InstanceConsole) error { //Todo: update from boltdb return nil } diff --git a/store/discover_api.go b/store/discover_api.go index dbd5c32c7..c6fbc1210 100644 --- a/store/discover_api.go +++ b/store/discover_api.go @@ -97,7 +97,7 @@ type InstanceStore interface { BatchAddInstances(instances []*model.Instance) error // UpdateInstance 更新实例 UpdateInstance(instance *model.Instance) error - UpdateInstanceConsole(instance *model.InstanceConsole) error + UpdateInstanceConsole(instanceConsole *model.InstanceConsole) error // DeleteInstance 删除一个实例，实际是把valid置为false DeleteInstance(instanceID string) error DeleteInstanceConsole(instanceConsoleID string) error diff --git a/store/mysql/instance.go b/store/mysql/instance.go index 1fcbb41e2..c92c2852e 100644 --- a/store/mysql/instance.go +++ b/store/mysql/instance.go @@ -19,7 +19,6 @@ package sqldb import ( "database/sql" - "encoding/json" "errors" "fmt" "strings" @@ -1065,17 +1064,77 @@ func addInstanceConsole(tx *BaseTx, instanceConsole *model.InstanceConsole) erro str := "insert into instance_console(`id`, `isolate`, `weight`, `metadata`, `ctime`, `mtime`) values " str += "(?, ?, ?, ?, sysdate(), sysdate())" - metadata, err := json.Marshal(&instanceConsole.Metadata) - if err != nil { - log.Errorf("[Store][database] instance_console metadata phrase err: %s", err.Error()) - return err - } - _, err = tx.Exec(str, instanceConsole.Id, instanceConsole.Isolate, string(metadata), instanceConsole.Weight) + _, err := tx.Exec(str, instanceConsole.Id, instanceConsole.Isolate, instanceConsole.Metadata, instanceConsole.Weight) return err } +// DeleteInstanceConsole 逻辑删除instanceConsole +func (ins *instanceStore) DeleteInstanceConsole(instanceConsoleID string) error { + if instanceConsoleID == "" { + return errors.New("delete InstanceConsole Missing instanceConsole id") + } + return RetryTransaction("deleteInstanceConsole", func() error { + return ins.master.processWithTransaction("deleteInstance", func(tx *BaseTx) error { + str := "update instance_console set flag = 1, mtime = sysdate() where `id` = ?" + if _, err := tx.Exec(str, instanceConsoleID); err != nil { + return store.Error(err) + } + + if err := tx.Commit(); err != nil { + log.Errorf("[Store][database] delete instanceConsole commit tx err: %s", err.Error()) + return err + } + + return nil + }) + }) +} + +// CleanInstanceConsole 物理删除instanceConsole +func (ins *instanceStore) CleanInstanceConsole(instanceConsoleID string) error { + if instanceConsoleID == "" { + return errors.New("clean InstanceConsole Missing instanceConsole id") + } + return RetryTransaction("cleanInstanceConsole", func() error { + return ins.master.processWithTransaction("cleanInstanceConsole", func(tx *BaseTx) error { + str := "delete from instance_console where id = ? and flag = 1" + if _, err := tx.Exec(str, instanceConsoleID); err != nil { + return store.Error(err) + } + + if err := tx.Commit(); err != nil { + log.Errorf("[Store][database] clean instanceConsole commit tx err: %s", err.Error()) + return err + } + + return nil + }) + }) +} + +// UpdateInstanceConsole 更新instanceConsole +func (ins *instanceStore) UpdateInstanceConsole(instanceConsole *model.InstanceConsole) error { + + return RetryTransaction("UpdateInstanceConsole", func() error { + return ins.master.processWithTransaction("UpdateInstanceConsole", func(tx *BaseTx) error { + str := `update instance_console set isolate = ?, weight = ?, metadata = ?, mtime = sysdate() where id = ?` + + if _, err := tx.Exec(str, instanceConsole.Isolate, instanceConsole.Weight, instanceConsole.Metadata, instanceConsole.Id); err != nil { + return store.Error(err) + } + + if err := tx.Commit(); err != nil { + log.Errorf("[Store][database] update instanceConsole commit tx err: %s", err.Error()) + return err + } + + return nil + }) + }) +} + // addInstanceMeta 往表中加入instance meta数据 func addInstanceMeta(tx *BaseTx, id string, meta map[string]string) error { if len(meta) == 0 { From b3e121e1a5301f403d69da525fe3d737815adaba Mon Sep 17 00:00:00 2001 From: fabian4 Date: Tue, 26 Sep 2023 16:21:11 +0800 Subject: [PATCH 08/21] feat: add related cache for additional instance data. --- store/mock/api_mock.go | 57 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/store/mock/api_mock.go b/store/mock/api_mock.go index ed865ed41..a860feae7 100644 --- a/store/mock/api_mock.go +++ b/store/mock/api_mock.go @@ -361,6 +361,20 @@ func (mr *MockStoreMockRecorder) CleanInstance(instanceID interface{}) *gomock.C return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CleanInstance", reflect.TypeOf((*MockStore)(nil).CleanInstance), instanceID) } +// CleanInstanceConsole mocks base method. +func (m *MockStore) CleanInstanceConsole(instanceConsoleID string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CleanInstanceConsole", instanceConsoleID) + ret0, _ := ret[0].(error) + return ret0 +} + +// CleanInstanceConsole indicates an expected call of CleanInstanceConsole. +func (mr *MockStoreMockRecorder) CleanInstanceConsole(instanceConsoleID interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CleanInstanceConsole", reflect.TypeOf((*MockStore)(nil).CleanInstanceConsole), instanceConsoleID) +} + // CountConfigFileEachGroup mocks base method. func (m *MockStore) CountConfigFileEachGroup() (map[string]map[string]int64, error) { m.ctrl.T.Helper() @@ -704,6 +718,20 @@ func (mr *MockStoreMockRecorder) DeleteInstance(instanceID interface{}) *gomock. return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteInstance", reflect.TypeOf((*MockStore)(nil).DeleteInstance), instanceID) } +// DeleteInstanceConsole mocks base method. +func (m *MockStore) DeleteInstanceConsole(instanceConsoleID string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteInstanceConsole", instanceConsoleID) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteInstanceConsole indicates an expected call of DeleteInstanceConsole. +func (mr *MockStoreMockRecorder) DeleteInstanceConsole(instanceConsoleID interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteInstanceConsole", reflect.TypeOf((*MockStore)(nil).DeleteInstanceConsole), instanceConsoleID) +} + // DeleteRateLimit mocks base method. func (m *MockStore) DeleteRateLimit(limiting *model.RateLimit) error { m.ctrl.T.Helper() @@ -1325,6 +1353,21 @@ func (mr *MockStoreMockRecorder) GetMoreConfigGroup(firstUpdate, mtime interface return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetMoreConfigGroup", reflect.TypeOf((*MockStore)(nil).GetMoreConfigGroup), firstUpdate, mtime) } +// GetMoreInstanceConsoles mocks base method. +func (m *MockStore) GetMoreInstanceConsoles(tx store.Tx, mtime time.Time, firstUpdate, needMeta bool, serviceID []string) (map[string]*model.InstanceConsole, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetMoreInstanceConsoles", tx, mtime, firstUpdate, needMeta, serviceID) + ret0, _ := ret[0].(map[string]*model.InstanceConsole) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetMoreInstanceConsoles indicates an expected call of GetMoreInstanceConsoles. +func (mr *MockStoreMockRecorder) GetMoreInstanceConsoles(tx, mtime, firstUpdate, needMeta, serviceID interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetMoreInstanceConsoles", reflect.TypeOf((*MockStore)(nil).GetMoreInstanceConsoles), tx, mtime, firstUpdate, needMeta, serviceID) +} + // GetMoreInstances mocks base method. func (m *MockStore) GetMoreInstances(tx store.Tx, mtime time.Time, firstUpdate, needMeta bool, serviceID []string) (map[string]*model.Instance, error) { m.ctrl.T.Helper() @@ -2369,6 +2412,20 @@ func (mr *MockStoreMockRecorder) UpdateInstance(instance interface{}) *gomock.Ca return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateInstance", reflect.TypeOf((*MockStore)(nil).UpdateInstance), instance) } +// UpdateInstanceConsole mocks base method. +func (m *MockStore) UpdateInstanceConsole(instanceConsole *model.InstanceConsole) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateInstanceConsole", instanceConsole) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpdateInstanceConsole indicates an expected call of UpdateInstanceConsole. +func (mr *MockStoreMockRecorder) UpdateInstanceConsole(instanceConsole interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateInstanceConsole", reflect.TypeOf((*MockStore)(nil).UpdateInstanceConsole), instanceConsole) +} + // UpdateNamespace mocks base method. func (m *MockStore) UpdateNamespace(namespace *model.Namespace) error { m.ctrl.T.Helper() From ac81b30be15ad49be4dc38755bdd99e188422706 Mon Sep 17 00:00:00 2001 From: fabian4 Date: Wed, 27 Sep 2023 17:16:05 +0800 Subject: [PATCH 09/21] feat: add related cache for additional instance data. --- auth/defaultauth/common_test.go | 8 ++++++++ cache/service/instance_test.go | 30 ++++++++++++++++++++++++++++++ cache/service/service_test.go | 1 + 3 files changed, 39 insertions(+) diff --git a/auth/defaultauth/common_test.go b/auth/defaultauth/common_test.go index 3806527f2..76e06a13f 100644 --- a/auth/defaultauth/common_test.go +++ b/auth/defaultauth/common_test.go @@ -109,6 +109,14 @@ func initCache(ctrl *gomock.Controller) (*cache.Config, *storemock.MockStore) { Valid: true, }, }, nil).AnyTimes() + storage.EXPECT().GetMoreInstanceConsoles(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(map[string]*model.InstanceConsole{ + "123": { + Id: "123", + Isolate: false, + Weight: 100, + Metadata: "{a: 1}", + }, + }, nil).AnyTimes() storage.EXPECT().GetUnixSecond(gomock.Any()).AnyTimes().Return(time.Now().Unix(), nil) return cfg, storage diff --git a/cache/service/instance_test.go b/cache/service/instance_test.go index e1522fcf8..4b1a45e41 100644 --- a/cache/service/instance_test.go +++ b/cache/service/instance_test.go @@ -127,6 +127,9 @@ func TestInstanceCache_Update(t *testing.T) { gomock.InOrder(storage.EXPECT(). GetMoreInstances(gomock.Any(), gomock.Any(), ic.IsFirstUpdate(), ic.needMeta, ic.systemServiceID). Return(ret, nil)) + gomock.InOrder(storage.EXPECT(). + GetMoreInstanceConsoles(gomock.Any(), gomock.Any(), ic.IsFirstUpdate(), ic.needMeta, ic.systemServiceID). + Return(nil, nil)) gomock.InOrder(storage.EXPECT().GetInstancesCountTx(gomock.Any()).Return(uint32(15), nil)) if err := ic.Update(); err != nil { t.Fatalf("error: %s", err.Error()) @@ -145,6 +148,9 @@ func TestInstanceCache_Update(t *testing.T) { gomock.InOrder(storage.EXPECT(). GetMoreInstances(gomock.Any(), gomock.Any(), ic.IsFirstUpdate(), ic.needMeta, ic.systemServiceID). Return(nil, nil)) + gomock.InOrder(storage.EXPECT(). + GetMoreInstanceConsoles(gomock.Any(), gomock.Any(), ic.IsFirstUpdate(), ic.needMeta, ic.systemServiceID). + Return(nil, nil)) if err := ic.Update(); err != nil { t.Fatalf("error: %s", err.Error()) } @@ -165,6 +171,9 @@ func TestInstanceCache_Update(t *testing.T) { storage.EXPECT(). GetMoreInstances(gomock.Any(), gomock.Any(), gomock.Any(), ic.needMeta, ic.systemServiceID). Return(instances, nil), + storage.EXPECT(). + GetMoreInstanceConsoles(gomock.Any(), gomock.Any(), ic.IsFirstUpdate(), ic.needMeta, ic.systemServiceID). + Return(nil, nil), storage.EXPECT().GetUnixSecond(gomock.Any()).Return(maxMtime.Unix(), nil).AnyTimes(), ) if err := ic.Update(); err != nil { @@ -186,6 +195,9 @@ func TestInstanceCache_Update2(t *testing.T) { gomock.InOrder(storage.EXPECT(). GetMoreInstances(gomock.Any(), gomock.Any(), ic.IsFirstUpdate(), ic.needMeta, ic.systemServiceID). Return(nil, fmt.Errorf("storage get error"))) + gomock.InOrder(storage.EXPECT(). + GetMoreInstanceConsoles(gomock.Any(), gomock.Any(), ic.IsFirstUpdate(), ic.needMeta, ic.systemServiceID). + Return(nil, nil)) gomock.InOrder(storage.EXPECT().GetInstancesCountTx(gomock.Any()).Return(uint32(0), fmt.Errorf("storage get error"))) if err := ic.Update(); err != nil { t.Logf("pass: %s", err.Error()) @@ -216,6 +228,9 @@ func TestInstanceCache_Update2(t *testing.T) { gomock.InOrder(storage.EXPECT(). GetMoreInstances(gomock.Any(), gomock.Any(), ic.IsFirstUpdate(), ic.needMeta, ic.systemServiceID). Return(instances, nil)) + gomock.InOrder(storage.EXPECT(). + GetMoreInstanceConsoles(gomock.Any(), gomock.Any(), ic.IsFirstUpdate(), ic.needMeta, ic.systemServiceID). + Return(nil, nil)) if err := ic.Update(); err != nil { t.Fatalf("error: %s", err.Error()) } @@ -231,6 +246,9 @@ func TestInstanceCache_Update2(t *testing.T) { instances := genModelInstances("service-a", 20) queryCount := int32(0) storage.EXPECT().GetInstancesCountTx(gomock.Any()).Return(uint32(0), nil).AnyTimes() + storage.EXPECT(). + GetMoreInstanceConsoles(gomock.Any(), gomock.Any(), ic.IsFirstUpdate(), ic.needMeta, ic.systemServiceID). + Return(nil, nil) storage.EXPECT(). GetMoreInstances(gomock.Any(), gomock.Any(), ic.IsFirstUpdate(), ic.needMeta, ic.systemServiceID). DoAndReturn(func(tx store.Tx, mtime time.Time, firstUpdate, needMeta bool, svcIds []string) (map[string]*model.Instance, error) { @@ -257,6 +275,9 @@ func TestInstanceCache_GetInstance(t *testing.T) { gomock.InOrder(storage.EXPECT(). GetMoreInstances(gomock.Any(), gomock.Any(), ic.IsFirstUpdate(), ic.needMeta, ic.systemServiceID). Return(instances, nil)) + gomock.InOrder(storage.EXPECT(). + GetMoreInstanceConsoles(gomock.Any(), gomock.Any(), ic.IsFirstUpdate(), ic.needMeta, ic.systemServiceID). + Return(nil, nil)) gomock.InOrder(storage.EXPECT().GetInstancesCountTx(gomock.Any()).Return(uint32(10), nil)) if err := ic.Update(); err != nil { t.Fatalf("error: %s", err.Error()) @@ -309,6 +330,9 @@ func TestInstanceCache_GetServicePorts(t *testing.T) { gomock.InOrder(storage.EXPECT(). GetMoreInstances(gomock.Any(), gomock.Any(), ic.IsFirstUpdate(), ic.needMeta, ic.systemServiceID). Return(instances, nil)) + gomock.InOrder(storage.EXPECT(). + GetMoreInstanceConsoles(gomock.Any(), gomock.Any(), ic.IsFirstUpdate(), ic.needMeta, ic.systemServiceID). + Return(nil, nil)) gomock.InOrder(storage.EXPECT().GetInstancesCountTx(gomock.Any()).Return(uint32(10), nil)) if err := ic.Update(); err != nil { t.Fatalf("error: %s", err.Error()) @@ -360,6 +384,9 @@ func TestInstanceCache_fillIntrnalLabels(t *testing.T) { gomock.InOrder(storage.EXPECT(). GetMoreInstances(gomock.Any(), gomock.Any(), ic.IsFirstUpdate(), ic.needMeta, ic.systemServiceID). Return(instances, nil)) + gomock.InOrder(storage.EXPECT(). + GetMoreInstanceConsoles(gomock.Any(), gomock.Any(), ic.IsFirstUpdate(), ic.needMeta, ic.systemServiceID). + Return(nil, nil)) gomock.InOrder(storage.EXPECT().GetInstancesCountTx(gomock.Any()).Return(uint32(10), nil)) if err := ic.Update(); err != nil { t.Fatalf("error: %s", err.Error()) @@ -403,6 +430,9 @@ func TestGetInstancesByServiceID(t *testing.T) { gomock.InOrder(storage.EXPECT(). GetMoreInstances(gomock.Any(), gomock.Any(), ic.IsFirstUpdate(), ic.needMeta, ic.systemServiceID). Return(ret, nil)) + gomock.InOrder(storage.EXPECT(). + GetMoreInstanceConsoles(gomock.Any(), gomock.Any(), ic.IsFirstUpdate(), ic.needMeta, ic.systemServiceID). + Return(nil, nil)) gomock.InOrder(storage.EXPECT(). GetInstancesCountTx(gomock.Any()). Return(uint32(instances1Count+instances2Count), nil)) diff --git a/cache/service/service_test.go b/cache/service/service_test.go index f87a92f7d..e13062a94 100644 --- a/cache/service/service_test.go +++ b/cache/service/service_test.go @@ -402,6 +402,7 @@ func TestServiceCache_GetServicesByFilter(t *testing.T) { mockStore.EXPECT().GetInstancesCountTx(gomock.Any()).Return(uint32(len(instances)), nil).AnyTimes() mockStore.EXPECT().GetMoreServices(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(services, nil).AnyTimes() mockStore.EXPECT().GetMoreInstances(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(instances, nil).AnyTimes() + mockStore.EXPECT().GetMoreInstanceConsoles(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil).AnyTimes() ic.setInstances(instances) hostToService := make(map[string]string) From 65049f45256fbcaed90d28d2e704800000bd8f1c Mon Sep 17 00:00:00 2001 From: fabian4 Date: Wed, 27 Sep 2023 17:53:38 +0800 Subject: [PATCH 10/21] feat: add related cache for additional instance data. --- apiserver/eurekaserver/write_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/apiserver/eurekaserver/write_test.go b/apiserver/eurekaserver/write_test.go index 5cdff5f1c..02eb16605 100644 --- a/apiserver/eurekaserver/write_test.go +++ b/apiserver/eurekaserver/write_test.go @@ -108,6 +108,10 @@ func TestEurekaServer_renew(t *testing.T) { insId: ins, disableBeatInsId: disableBeatIns, }, nil) + mockStore.EXPECT(). + GetMoreInstanceConsoles(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + AnyTimes(). + Return(nil, nil)) mockStore.EXPECT().StartReadTx().Return(mockTx, nil).AnyTimes() mockStore.EXPECT(). GetMoreServices(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). From 2c5f961eb099503e4ebe742070e0da8e74d30986 Mon Sep 17 00:00:00 2001 From: fabian4 Date: Wed, 27 Sep 2023 19:32:43 +0800 Subject: [PATCH 11/21] feat: add related cache for additional instance data. --- apiserver/eurekaserver/write_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apiserver/eurekaserver/write_test.go b/apiserver/eurekaserver/write_test.go index 02eb16605..ebc379e9e 100644 --- a/apiserver/eurekaserver/write_test.go +++ b/apiserver/eurekaserver/write_test.go @@ -109,9 +109,9 @@ func TestEurekaServer_renew(t *testing.T) { disableBeatInsId: disableBeatIns, }, nil) mockStore.EXPECT(). - GetMoreInstanceConsoles(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). - AnyTimes(). - Return(nil, nil)) + GetMoreInstanceConsoles(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + AnyTimes(). + Return(nil, nil) mockStore.EXPECT().StartReadTx().Return(mockTx, nil).AnyTimes() mockStore.EXPECT(). GetMoreServices(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). From 5058981ab41ebd482e325618bda8a3c29b01306a Mon Sep 17 00:00:00 2001 From: fabian4 Date: Tue, 10 Oct 2023 19:37:16 +0800 Subject: [PATCH 12/21] feat: add related cache for additional instance data. --- cache/service/instance.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/cache/service/instance.go b/cache/service/instance.go index 9a650a3dc..86d69dab2 100644 --- a/cache/service/instance.go +++ b/cache/service/instance.go @@ -194,16 +194,16 @@ func (ic *instanceCache) handleUpdate(start time.Time, tx store.Tx) ([]*eventhub return nil, nil, -1, err } - instanceConsoles, err := ic.storage.GetMoreInstanceConsoles(tx, ic.LastFetchTime(), ic.IsFirstUpdate(), - ic.needMeta, ic.systemServiceID) - if err != nil { - log.Error("[Cache][InstanceConsole] update get storage more", zap.Error(err)) - return nil, nil, -1, err - } - for _, item := range instanceConsoles { - //Todo: check validation - ic.instanceConsoles.Store(item.Id, item) - } + //instanceConsoles, err := ic.storage.GetMoreInstanceConsoles(tx, ic.LastFetchTime(), ic.IsFirstUpdate(), + // ic.needMeta, ic.systemServiceID) + //if err != nil { + // log.Error("[Cache][InstanceConsole] update get storage more", zap.Error(err)) + // return nil, nil, -1, err + //} + //for _, item := range instanceConsoles { + // //Todo: check validation + // ic.instanceConsoles.Store(item.Id, item) + //} events, lastMtimes, update, del := ic.setInstances(instances) log.Info("[Cache][Instance] get more instances", From 7f034888a4586f8a32d95faa91921409089b6559 Mon Sep 17 00:00:00 2001 From: fabian4 Date: Tue, 10 Oct 2023 19:42:27 +0800 Subject: [PATCH 13/21] feat: add related cache for additional instance data. --- release/conf/polaris-server.yaml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/release/conf/polaris-server.yaml b/release/conf/polaris-server.yaml index ccff8cd55..4e459fc5a 100644 --- a/release/conf/polaris-server.yaml +++ b/release/conf/polaris-server.yaml @@ -476,22 +476,22 @@ maintain: # Storage configuration store: # Standalone file storage plugin - name: boltdbStore - option: - path: ./polaris.bolt +# name: boltdbStore +# option: +# path: ./polaris.bolt ## Database storage plugin - # name: defaultStore - # option: - # master: - # dbType: mysql - # dbName: polaris_server - # dbUser: ##DB_USER## - # dbPwd: ##DB_PWD## - # dbAddr: ##DB_ADDR## - # maxOpenConns: 300 - # maxIdleConns: 50 - # connMaxLifetime: 300 # Unit second - # txIsolationLevel: 2 #LevelReadCommitted + name: defaultStore + option: + master: + dbType: mysql + dbName: polaris_server + dbUser: root + dbPwd: root + dbAddr: 192.168.100.7:3306 + maxOpenConns: 300 + maxIdleConns: 50 + connMaxLifetime: 300 # Unit second + txIsolationLevel: 2 #LevelReadCommitted # polaris-server plugin settings plugin: crypto: From 022dcb8b61fa7c02f3d4afde6179495110acc806 Mon Sep 17 00:00:00 2001 From: fabian4 Date: Tue, 10 Oct 2023 22:19:29 +0800 Subject: [PATCH 14/21] feat: add related cache for additional instance data. --- cache/service/instance.go | 20 ++++++++++---------- store/mysql/instance.go | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/cache/service/instance.go b/cache/service/instance.go index 86d69dab2..9a650a3dc 100644 --- a/cache/service/instance.go +++ b/cache/service/instance.go @@ -194,16 +194,16 @@ func (ic *instanceCache) handleUpdate(start time.Time, tx store.Tx) ([]*eventhub return nil, nil, -1, err } - //instanceConsoles, err := ic.storage.GetMoreInstanceConsoles(tx, ic.LastFetchTime(), ic.IsFirstUpdate(), - // ic.needMeta, ic.systemServiceID) - //if err != nil { - // log.Error("[Cache][InstanceConsole] update get storage more", zap.Error(err)) - // return nil, nil, -1, err - //} - //for _, item := range instanceConsoles { - // //Todo: check validation - // ic.instanceConsoles.Store(item.Id, item) - //} + instanceConsoles, err := ic.storage.GetMoreInstanceConsoles(tx, ic.LastFetchTime(), ic.IsFirstUpdate(), + ic.needMeta, ic.systemServiceID) + if err != nil { + log.Error("[Cache][InstanceConsole] update get storage more", zap.Error(err)) + return nil, nil, -1, err + } + for _, item := range instanceConsoles { + //Todo: check validation + ic.instanceConsoles.Store(item.Id, item) + } events, lastMtimes, update, del := ic.setInstances(instances) log.Info("[Cache][Instance] get more instances", diff --git a/store/mysql/instance.go b/store/mysql/instance.go index c92c2852e..0baf7488f 100644 --- a/store/mysql/instance.go +++ b/store/mysql/instance.go @@ -514,7 +514,7 @@ func (ins *instanceStore) GetMoreInstanceConsoles(tx store.Tx, mtime time.Time, dbTx, _ := tx.GetDelegateTx().(*BaseTx) - str := `select id, isolate, weight, metadata from instance where instance.mtime >= FROM_UNIXTIME(?) ` + str := `select id, isolate, weight, metadata from instance_console where instance_console.mtime >= FROM_UNIXTIME(?) ` if firstUpdate { str += " and flag != 1" From c0c337fd3fc2b556cb7654de2891c2c8a4447913 Mon Sep 17 00:00:00 2001 From: fabian4 Date: Tue, 10 Oct 2023 22:26:03 +0800 Subject: [PATCH 15/21] feat: add related cache for additional instance data. --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 0acde0031..c77f0991e 100644 --- a/go.mod +++ b/go.mod @@ -83,7 +83,7 @@ require ( require ( github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/agiledragon/gomonkey/v2 v2.10.1 - github.com/polarismesh/specification v1.4.1-alpha + github.com/polarismesh/specification v1.4.1 ) require ( From 967abd29275869a1d3b170cde30170f92cc42366 Mon Sep 17 00:00:00 2001 From: fabian4 Date: Tue, 10 Oct 2023 22:34:03 +0800 Subject: [PATCH 16/21] feat: add related cache for additional instance data. --- go.sum | 2 ++ 1 file changed, 2 insertions(+) diff --git a/go.sum b/go.sum index 7dab81380..051e4df0d 100644 --- a/go.sum +++ b/go.sum @@ -325,6 +325,8 @@ github.com/polarismesh/go-restful-openapi/v2 v2.0.0-20220928152401-083908d10219 github.com/polarismesh/go-restful-openapi/v2 v2.0.0-20220928152401-083908d10219/go.mod h1:4WhwBysTom9Eoy0hQ4W69I0FmO+T0EpjEW9/5sgHoUk= github.com/polarismesh/specification v1.4.1-alpha h1:4y7/cDN50o0JrFyEu6j78u4trEsessKAjS/7YPtlwVo= github.com/polarismesh/specification v1.4.1-alpha/go.mod h1:rDvMMtl5qebPmqiBLNa5Ps0XtwkP31ZLirbH4kXA0YU= +github.com/polarismesh/specification v1.4.1 h1:lTZqeyUhhWuKyr6NDKBwmUrNfcUDvKLxWT/uOq71T5A= +github.com/polarismesh/specification v1.4.1/go.mod h1:rDvMMtl5qebPmqiBLNa5Ps0XtwkP31ZLirbH4kXA0YU= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= From e8133ad79efdf8b0f6d89564ec7af07ccee4262a Mon Sep 17 00:00:00 2001 From: fabian4 Date: Tue, 10 Oct 2023 22:40:08 +0800 Subject: [PATCH 17/21] feat: add related cache for additional instance data. --- release/conf/polaris-server.yaml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/release/conf/polaris-server.yaml b/release/conf/polaris-server.yaml index 4e459fc5a..ccff8cd55 100644 --- a/release/conf/polaris-server.yaml +++ b/release/conf/polaris-server.yaml @@ -476,22 +476,22 @@ maintain: # Storage configuration store: # Standalone file storage plugin -# name: boltdbStore -# option: -# path: ./polaris.bolt + name: boltdbStore + option: + path: ./polaris.bolt ## Database storage plugin - name: defaultStore - option: - master: - dbType: mysql - dbName: polaris_server - dbUser: root - dbPwd: root - dbAddr: 192.168.100.7:3306 - maxOpenConns: 300 - maxIdleConns: 50 - connMaxLifetime: 300 # Unit second - txIsolationLevel: 2 #LevelReadCommitted + # name: defaultStore + # option: + # master: + # dbType: mysql + # dbName: polaris_server + # dbUser: ##DB_USER## + # dbPwd: ##DB_PWD## + # dbAddr: ##DB_ADDR## + # maxOpenConns: 300 + # maxIdleConns: 50 + # connMaxLifetime: 300 # Unit second + # txIsolationLevel: 2 #LevelReadCommitted # polaris-server plugin settings plugin: crypto: From 1ee4956fbba5489b1a37dc0e160dd9e61faa862e Mon Sep 17 00:00:00 2001 From: fabian4 Date: Fri, 13 Oct 2023 16:15:16 +0800 Subject: [PATCH 18/21] feat: add related cache for additional instance data. --- cache/service/instance_test.go | 44 +++++++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/cache/service/instance_test.go b/cache/service/instance_test.go index 4b1a45e41..28010e00a 100644 --- a/cache/service/instance_test.go +++ b/cache/service/instance_test.go @@ -92,6 +92,21 @@ func genModelInstances(label string, total int) map[string]*model.Instance { return out } +func genModelInstancesConsole(label string, total int) map[string]*model.InstanceConsole { + out := make(map[string]*model.InstanceConsole) + for i := 0; i < total; i++ { + entry := &model.InstanceConsole{ + Id: fmt.Sprintf("InstanceConsole-%s-%d", label, i), + Isolate: false, + Weight: 100, + Metadata: "Metadata", + } + out[entry.Id] = entry + } + + return out +} + // 对instanceCache的缓存数据进行计数统计 func iteratorInstances(ic *instanceCache) (int, int) { instancesCount := 0 @@ -116,6 +131,7 @@ func TestInstanceCache_Update(t *testing.T) { ret := make(map[string]*model.Instance) instances1 := genModelInstances("service1", 10) // 每次gen为一个服务的 instances2 := genModelInstances("service2", 5) + instanceConsoles := genModelInstancesConsole("console", 3) for id, instance := range instances1 { ret[id] = instance @@ -129,7 +145,7 @@ func TestInstanceCache_Update(t *testing.T) { Return(ret, nil)) gomock.InOrder(storage.EXPECT(). GetMoreInstanceConsoles(gomock.Any(), gomock.Any(), ic.IsFirstUpdate(), ic.needMeta, ic.systemServiceID). - Return(nil, nil)) + Return(instanceConsoles, nil)) gomock.InOrder(storage.EXPECT().GetInstancesCountTx(gomock.Any()).Return(uint32(15), nil)) if err := ic.Update(); err != nil { t.Fatalf("error: %s", err.Error()) @@ -164,6 +180,7 @@ func TestInstanceCache_Update(t *testing.T) { t.Run("lastMtime可以正常更新", func(t *testing.T) { _ = ic.Clear() instances := genModelInstances("services", 10) + instanceConsoles := genModelInstancesConsole("console", 3) maxMtime := time.Now() instances[fmt.Sprintf("instanceID-%s-%d", "services", 5)].ModifyTime = maxMtime @@ -173,7 +190,7 @@ func TestInstanceCache_Update(t *testing.T) { Return(instances, nil), storage.EXPECT(). GetMoreInstanceConsoles(gomock.Any(), gomock.Any(), ic.IsFirstUpdate(), ic.needMeta, ic.systemServiceID). - Return(nil, nil), + Return(instanceConsoles, nil), storage.EXPECT().GetUnixSecond(gomock.Any()).Return(maxMtime.Unix(), nil).AnyTimes(), ) if err := ic.Update(); err != nil { @@ -209,9 +226,13 @@ func TestInstanceCache_Update2(t *testing.T) { t.Run("更新数据，再删除部分数据，缓存正常", func(t *testing.T) { _ = ic.Clear() instances := genModelInstances("service-a", 20) + instanceConsoles := genModelInstancesConsole("console", 3) gomock.InOrder(storage.EXPECT(). GetMoreInstances(gomock.Any(), gomock.Any(), ic.IsFirstUpdate(), ic.needMeta, ic.systemServiceID). Return(instances, nil)) + gomock.InOrder(storage.EXPECT(). + GetMoreInstanceConsoles(gomock.Any(), gomock.Any(), ic.IsFirstUpdate(), ic.needMeta, ic.systemServiceID). + Return(instanceConsoles, nil)) if err := ic.Update(); err != nil { t.Fatalf("error: %s", err.Error()) } @@ -230,7 +251,7 @@ func TestInstanceCache_Update2(t *testing.T) { Return(instances, nil)) gomock.InOrder(storage.EXPECT(). GetMoreInstanceConsoles(gomock.Any(), gomock.Any(), ic.IsFirstUpdate(), ic.needMeta, ic.systemServiceID). - Return(nil, nil)) + Return(instanceConsoles, nil)) if err := ic.Update(); err != nil { t.Fatalf("error: %s", err.Error()) } @@ -244,11 +265,13 @@ func TestInstanceCache_Update2(t *testing.T) { t.Run("对账发现缓存数据数量和存储层不一致", func(t *testing.T) { _ = ic.Clear() instances := genModelInstances("service-a", 20) + instanceConsoles := genModelInstancesConsole("console", 3) + queryCount := int32(0) storage.EXPECT().GetInstancesCountTx(gomock.Any()).Return(uint32(0), nil).AnyTimes() storage.EXPECT(). GetMoreInstanceConsoles(gomock.Any(), gomock.Any(), ic.IsFirstUpdate(), ic.needMeta, ic.systemServiceID). - Return(nil, nil) + Return(instanceConsoles, nil) storage.EXPECT(). GetMoreInstances(gomock.Any(), gomock.Any(), ic.IsFirstUpdate(), ic.needMeta, ic.systemServiceID). DoAndReturn(func(tx store.Tx, mtime time.Time, firstUpdate, needMeta bool, svcIds []string) (map[string]*model.Instance, error) { @@ -272,12 +295,14 @@ func TestInstanceCache_GetInstance(t *testing.T) { t.Run("缓存有数据，可以正常获取到数据", func(t *testing.T) { _ = ic.Clear() instances := genModelInstances("my-services", 10) + instanceConsoles := genModelInstancesConsole("console", 3) + gomock.InOrder(storage.EXPECT(). GetMoreInstances(gomock.Any(), gomock.Any(), ic.IsFirstUpdate(), ic.needMeta, ic.systemServiceID). Return(instances, nil)) gomock.InOrder(storage.EXPECT(). GetMoreInstanceConsoles(gomock.Any(), gomock.Any(), ic.IsFirstUpdate(), ic.needMeta, ic.systemServiceID). - Return(nil, nil)) + Return(instanceConsoles, nil)) gomock.InOrder(storage.EXPECT().GetInstancesCountTx(gomock.Any()).Return(uint32(10), nil)) if err := ic.Update(); err != nil { t.Fatalf("error: %s", err.Error()) @@ -299,6 +324,7 @@ func TestInstanceCache_GetServicePorts(t *testing.T) { t.Run("缓存有数据，可以正常获取到服务的端口列表", func(t *testing.T) { _ = ic.Clear() instances := genModelInstances("my-services", 10) + instanceConsoles := genModelInstancesConsole("console", 3) ports := make(map[string][]*model.ServicePort) @@ -332,7 +358,7 @@ func TestInstanceCache_GetServicePorts(t *testing.T) { Return(instances, nil)) gomock.InOrder(storage.EXPECT(). GetMoreInstanceConsoles(gomock.Any(), gomock.Any(), ic.IsFirstUpdate(), ic.needMeta, ic.systemServiceID). - Return(nil, nil)) + Return(instanceConsoles, nil)) gomock.InOrder(storage.EXPECT().GetInstancesCountTx(gomock.Any()).Return(uint32(10), nil)) if err := ic.Update(); err != nil { t.Fatalf("error: %s", err.Error()) @@ -355,6 +381,7 @@ func TestInstanceCache_fillIntrnalLabels(t *testing.T) { t.Run("向实例Metadata中自动注入北极星默认label信息", func(t *testing.T) { _ = ic.Clear() instances := genModelInstances("inject-internal-label", 10) + instanceConsoles := genModelInstancesConsole("console", 3) ports := make(map[string][]string) @@ -386,7 +413,7 @@ func TestInstanceCache_fillIntrnalLabels(t *testing.T) { Return(instances, nil)) gomock.InOrder(storage.EXPECT(). GetMoreInstanceConsoles(gomock.Any(), gomock.Any(), ic.IsFirstUpdate(), ic.needMeta, ic.systemServiceID). - Return(nil, nil)) + Return(instanceConsoles, nil)) gomock.InOrder(storage.EXPECT().GetInstancesCountTx(gomock.Any()).Return(uint32(10), nil)) if err := ic.Update(); err != nil { t.Fatalf("error: %s", err.Error()) @@ -418,6 +445,7 @@ func TestGetInstancesByServiceID(t *testing.T) { instances1 := genModelInstances("my-services", instances1Count) instances2 := genModelInstances("my-services-a", instances2Count) // instances2 = append(instances2, instances1...) + instanceConsoles := genModelInstancesConsole("console", 3) ret := make(map[string]*model.Instance) for id, instance := range instances1 { @@ -432,7 +460,7 @@ func TestGetInstancesByServiceID(t *testing.T) { Return(ret, nil)) gomock.InOrder(storage.EXPECT(). GetMoreInstanceConsoles(gomock.Any(), gomock.Any(), ic.IsFirstUpdate(), ic.needMeta, ic.systemServiceID). - Return(nil, nil)) + Return(instanceConsoles, nil)) gomock.InOrder(storage.EXPECT(). GetInstancesCountTx(gomock.Any()). Return(uint32(instances1Count+instances2Count), nil)) From fff45cfc4a56dd6d2526397d6837591544e1fe01 Mon Sep 17 00:00:00 2001 From: fabian4 Date: Fri, 13 Oct 2023 16:38:19 +0800 Subject: [PATCH 19/21] feat: add related cache for additional instance data. --- cache/service/instance.go | 14 ++++++++++++++ cache/service/instance_test.go | 18 ++++++++++++------ 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/cache/service/instance.go b/cache/service/instance.go index 9a650a3dc..baccb277e 100644 --- a/cache/service/instance.go +++ b/cache/service/instance.go @@ -445,6 +445,20 @@ func (ic *instanceCache) GetInstance(instanceID string) *model.Instance { return value } +// GetInstanceConsole 根据实例ID获取实例数据 +func (ic *instanceCache) GetInstanceConsole(instanceConsoleID string) *model.InstanceConsole { + if instanceConsoleID == "" { + return nil + } + + value, ok := ic.instanceConsoles.Load(instanceConsoleID) + if !ok { + return nil + } + + return value +} + // GetInstancesByServiceID 根据ServiceID获取实例数据 func (ic *instanceCache) GetInstancesByServiceID(serviceID string) []*model.Instance { if serviceID == "" { diff --git a/cache/service/instance_test.go b/cache/service/instance_test.go index 28010e00a..6a2a3b473 100644 --- a/cache/service/instance_test.go +++ b/cache/service/instance_test.go @@ -152,7 +152,8 @@ func TestInstanceCache_Update(t *testing.T) { } servicesCount, instancesCount := iteratorInstances(ic) - if servicesCount == 2 && instancesCount == 10+5 { // gen两次，有两个不同服务 + instanceConsoleCounts := ic.instanceConsoles.Len() + if servicesCount == 2 && instancesCount == 10+5 && instanceConsoleCounts == 3 { // gen两次，有两个不同服务 t.Logf("pass") } else { t.Fatalf("error: %d, %d", servicesCount, instancesCount) @@ -172,7 +173,8 @@ func TestInstanceCache_Update(t *testing.T) { } servicesCount, instancesCount := iteratorInstances(ic) - if servicesCount != 0 || instancesCount != 0 { + instanceConsoleCounts := ic.instanceConsoles.Len() + if servicesCount != 0 || instancesCount != 0 || instanceConsoleCounts != 0 { t.Fatalf("error: %d %d", servicesCount, instancesCount) } }) @@ -265,13 +267,9 @@ func TestInstanceCache_Update2(t *testing.T) { t.Run("对账发现缓存数据数量和存储层不一致", func(t *testing.T) { _ = ic.Clear() instances := genModelInstances("service-a", 20) - instanceConsoles := genModelInstancesConsole("console", 3) queryCount := int32(0) storage.EXPECT().GetInstancesCountTx(gomock.Any()).Return(uint32(0), nil).AnyTimes() - storage.EXPECT(). - GetMoreInstanceConsoles(gomock.Any(), gomock.Any(), ic.IsFirstUpdate(), ic.needMeta, ic.systemServiceID). - Return(instanceConsoles, nil) storage.EXPECT(). GetMoreInstances(gomock.Any(), gomock.Any(), ic.IsFirstUpdate(), ic.needMeta, ic.systemServiceID). DoAndReturn(func(tx store.Tx, mtime time.Time, firstUpdate, needMeta bool, svcIds []string) (map[string]*model.Instance, error) { @@ -315,6 +313,14 @@ func TestInstanceCache_GetInstance(t *testing.T) { if instance := ic.GetInstance("test-instance-xx"); instance != nil { t.Fatalf("error") } + + if instanceConsole := ic.GetInstanceConsole(instanceConsoles[fmt.Sprintf("InstanceConsole-%s-%d", "console", 2)].Id); instanceConsole == nil { + t.Fatalf("error") + } + + if instanceConsole := ic.GetInstance("test-instanceConsole-xx"); instanceConsole != nil { + t.Fatalf("error") + } }) } From 5e19f35d76a97911c6e4007e383aff31eb74ea8b Mon Sep 17 00:00:00 2001 From: fabian4 Date: Tue, 24 Sep 2024 22:52:04 +0800 Subject: [PATCH 20/21] feat: add related cache for additional instance data. --- auth/policy/helper.go | 17 +++++++++++++++++ auth/policy/role.go | 17 +++++++++++++++++ store/mysql/scripts/delta/v1_18_1-v1_18_2.sql | 16 ++++++++++++++++ 3 files changed, 50 insertions(+) diff --git a/auth/policy/helper.go b/auth/policy/helper.go index e77f14de1..fc986bd3f 100644 --- a/auth/policy/helper.go +++ b/auth/policy/helper.go @@ -1,3 +1,20 @@ +/** + * Tencent is pleased to support the open source community by making Polaris available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * Licensed under the BSD 3-Clause License (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://opensource.org/licenses/BSD-3-Clause + * + * Unless required by applicable law or agreed to in writing, software distributed + * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + package policy import ( diff --git a/auth/policy/role.go b/auth/policy/role.go index 2417883ba..cc6d0fb91 100644 --- a/auth/policy/role.go +++ b/auth/policy/role.go @@ -1,3 +1,20 @@ +/** + * Tencent is pleased to support the open source community by making Polaris available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * Licensed under the BSD 3-Clause License (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://opensource.org/licenses/BSD-3-Clause + * + * Unless required by applicable law or agreed to in writing, software distributed + * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + package policy import ( diff --git a/store/mysql/scripts/delta/v1_18_1-v1_18_2.sql b/store/mysql/scripts/delta/v1_18_1-v1_18_2.sql index 49041073d..1535f73ed 100644 --- a/store/mysql/scripts/delta/v1_18_1-v1_18_2.sql +++ b/store/mysql/scripts/delta/v1_18_1-v1_18_2.sql @@ -1,3 +1,19 @@ +/** + * Tencent is pleased to support the open source community by making Polaris available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * Licensed under the BSD 3-Clause License (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://opensource.org/licenses/BSD-3-Clause + * + * Unless required by applicable law or agreed to in writing, software distributed + * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ /* 角色数据 */ CREATE TABLE `auth_role` ( From 05b2e00a911080d429bfc5be6adac8a3d7c33313 Mon Sep 17 00:00:00 2001 From: fabian4 Date: Wed, 25 Sep 2024 21:10:23 +0800 Subject: [PATCH 21/21] feat: add related cache for additional instance data. --- auth/user/common_test.go | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 auth/user/common_test.go diff --git a/auth/user/common_test.go b/auth/user/common_test.go deleted file mode 100644 index e69de29bb..000000000