From db3018df0ef423e6c59a4b8131f5550e5e1f7269 Mon Sep 17 00:00:00 2001 From: JmPotato Date: Fri, 13 Jan 2023 14:54:43 +0800 Subject: [PATCH 1/4] Introduce the resource unit model and config Signed-off-by: JmPotato --- pkg/mcs/resource_manager/client/config.go | 83 ++++++++++++++++++ pkg/mcs/resource_manager/client/model.go | 100 ++++++++++++++++++++++ 2 files changed, 183 insertions(+) create mode 100644 pkg/mcs/resource_manager/client/config.go create mode 100644 pkg/mcs/resource_manager/client/model.go diff --git a/pkg/mcs/resource_manager/client/config.go b/pkg/mcs/resource_manager/client/config.go new file mode 100644 index 00000000000..1e13a18c12a --- /dev/null +++ b/pkg/mcs/resource_manager/client/config.go @@ -0,0 +1,83 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS,g +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package client + +const ( + defaultReadBaseCost = 1 + defaultReadCostPerByte = 1. / 1024 / 1024 + defaultWriteBaseCost = 5 + defaultWriteCostPerByte = 10. / 1024 / 1024 + defaultWriteCPUMsCost = 1 +) + +// RequestUnitConfig is the configuration of the request units, which determines the coefficients of +// the RRU and WRU cost. This configuration should be modified carefully. +type RequestUnitConfig struct { + // ReadBaseCost is the base cost for a read request. No matter how many bytes read/written or + // the CPU times taken for a request, this cost is inevitable. + ReadBaseCost float64 `toml:"read-base-cost" json:"read-base-cost"` + // ReadCostPerByte is the cost for each byte read. It's 1 MiB = 1 RRU by default. + ReadCostPerByte float64 `toml:"read-cost-per-byte" json:"read-cost-per-byte"` + // WriteBaseCost is the base cost for a write request. No matter how many bytes read/written or + // the CPU times taken for a request, this cost is inevitable. + WriteBaseCost float64 `toml:"write-base-cost" json:"write-base-cost"` + // WriteCostPerByte is the cost for each byte written. It's 1 MiB = 10 WRU by default. + WriteCostPerByte float64 `toml:"write-cost-per-byte" json:"write-cost-per-byte"` + // WriteCPUMsCost is the cost for each millisecond of CPU time taken by a write request. + // It's 1 millisecond = 1 WRU by default. + WriteCPUMsCost float64 `toml:"write-cpu-ms-cost" json:"write-cpu-ms-cost"` +} + +// DefaultRequestUnitConfig returns the default request unit configuration. +func DefaultRequestUnitConfig() *RequestUnitConfig { + return &RequestUnitConfig{ + ReadBaseCost: defaultReadBaseCost, + ReadCostPerByte: defaultReadCostPerByte, + WriteBaseCost: defaultWriteBaseCost, + WriteCostPerByte: defaultWriteCostPerByte, + WriteCPUMsCost: defaultWriteCPUMsCost, + } +} + +// Config is the configuration of the resource units, which gives the read/write request +// units or request resource cost standards. It should be calculated by a given `RequestUnitConfig` +// or `RequestResourceConfig`. +type Config struct { + ReadBaseCost RequestUnit + ReadBytesCost RequestUnit + WriteBaseCost RequestUnit + WriteBytesCost RequestUnit + WriteCPUMsCost RequestUnit + // TODO: add SQL computing CPU cost. +} + +// DefaultConfig returns the default configuration. +func DefaultConfig() *Config { + cfg := generateConfig( + DefaultRequestUnitConfig(), + ) + return cfg +} + +func generateConfig(ruConfig *RequestUnitConfig) *Config { + cfg := &Config{ + ReadBaseCost: RequestUnit(ruConfig.ReadBaseCost), + ReadBytesCost: RequestUnit(ruConfig.ReadCostPerByte), + WriteBaseCost: RequestUnit(ruConfig.WriteBaseCost), + WriteBytesCost: RequestUnit(ruConfig.WriteCostPerByte), + WriteCPUMsCost: RequestUnit(ruConfig.WriteCPUMsCost), + } + return cfg +} diff --git a/pkg/mcs/resource_manager/client/model.go b/pkg/mcs/resource_manager/client/model.go new file mode 100644 index 00000000000..927fb5f72a0 --- /dev/null +++ b/pkg/mcs/resource_manager/client/model.go @@ -0,0 +1,100 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS,g +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package client + +import ( + "context" + + rmpb "github.com/pingcap/kvproto/pkg/resource_manager" +) + +// RequestUnit is the basic unit of the resource request management, which has two types: +// - RRU: read request unit +// - WRU: write request unit +type RequestUnit float64 + +// RequestInfo is the interface of the request information provider. A request should be +// able tell whether it's a write request and if so, the written bytes would also be provided. +type RequestInfo interface { + IsWrite() bool + WriteBytes() uint64 +} + +// ResponseInfo is the interface of the response information provider. A response should be +// able tell how many bytes it read and KV CPU cost in milliseconds. +type ResponseInfo interface { + ReadBytes() uint64 + KVCPUMs() uint64 +} + +// ResourceCalculator is used to calculate the resource consumption of a request. +type ResourceCalculator interface { + Trickle(map[rmpb.ResourceType]float64, map[rmpb.RequestUnitType]float64, context.Context) + BeforeKVRequest(map[rmpb.ResourceType]float64, map[rmpb.RequestUnitType]float64, RequestInfo) + AfterKVRequest(map[rmpb.ResourceType]float64, map[rmpb.RequestUnitType]float64, RequestInfo, ResponseInfo) +} + +// KVCalculator is used to calculate the KV request consumption. +type KVCalculator struct { + *Config +} + +func newKVCalculator(cfg *Config) *KVCalculator { + return &KVCalculator{Config: cfg} +} + +func (kc *KVCalculator) Trickle(consumption *rmpb.Consumption, ctx context.Context) { +} + +func (kc *KVCalculator) BeforeKVRequest(consumption *rmpb.Consumption, req RequestInfo) { + if req.IsWrite() { + consumption.KvWriteRpcCount += 1 + writeBytes := float64(req.WriteBytes()) + consumption.WriteBytes += writeBytes + consumption.WRU += float64(kc.WriteBaseCost) + float64(kc.WriteBytesCost)*writeBytes + } else { + consumption.KvReadRpcCount += 1 + consumption.RRU += float64(kc.ReadBaseCost) + } +} +func (kc *KVCalculator) AfterKVRequest(consumption *rmpb.Consumption, req RequestInfo, res ResponseInfo) { + if req.IsWrite() { + kvCPUMs := float64(res.KVCPUMs()) + consumption.TotalCpuTimeMs += kvCPUMs + consumption.WRU += float64(kc.WriteCPUMsCost) * kvCPUMs + } else { + readBytes := float64(res.ReadBytes()) + consumption.ReadBytes += readBytes + consumption.RRU += float64(kc.ReadBytesCost) * readBytes + } +} + +type SQLCPUCalculator struct { + *Config +} + +func newSQLCPUCalculator(cfg *Config) *SQLCPUCalculator { + return &SQLCPUCalculator{Config: cfg} +} + +// TODO: calculate the SQL CPU cost and related resource consumption. +func (dsc *SQLCPUCalculator) Trickle(resource map[rmpb.ResourceType]float64, ru map[rmpb.RequestUnitType]float64, ctx context.Context) { +} + +func (dsc *SQLCPUCalculator) BeforeKVRequest(resource map[rmpb.ResourceType]float64, ru map[rmpb.RequestUnitType]float64, req RequestInfo) { +} + +func (dsc *SQLCPUCalculator) AfterKVRequest(resource map[rmpb.ResourceType]float64, ru map[rmpb.RequestUnitType]float64, req RequestInfo, res ResponseInfo) { +} From 2971a2f4f478d57479ab0c23965e79c00ac29bf5 Mon Sep 17 00:00:00 2001 From: JmPotato Date: Fri, 13 Jan 2023 15:52:29 +0800 Subject: [PATCH 2/4] Make the check happy Signed-off-by: JmPotato --- pkg/mcs/resource_manager/client/model.go | 40 ++++++++++++++++-------- 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/pkg/mcs/resource_manager/client/model.go b/pkg/mcs/resource_manager/client/model.go index 927fb5f72a0..9dacf107c02 100644 --- a/pkg/mcs/resource_manager/client/model.go +++ b/pkg/mcs/resource_manager/client/model.go @@ -41,23 +41,31 @@ type ResponseInfo interface { // ResourceCalculator is used to calculate the resource consumption of a request. type ResourceCalculator interface { - Trickle(map[rmpb.ResourceType]float64, map[rmpb.RequestUnitType]float64, context.Context) + // Trickle is used to calculate the resource consumption periodically rather than on the request path. + // It's mainly used to calculate like the SQL CPU cost. + Trickle(context.Context, map[rmpb.ResourceType]float64, map[rmpb.RequestUnitType]float64) + // BeforeKVRequest is used to calculate the resource consumption before the KV request. + // It's mainly used to calculate the base and write request cost. BeforeKVRequest(map[rmpb.ResourceType]float64, map[rmpb.RequestUnitType]float64, RequestInfo) + // AfterKVRequest is used to calculate the resource consumption after the KV request. + // It's mainly used to calculate the read request cost and KV CPU cost. AfterKVRequest(map[rmpb.ResourceType]float64, map[rmpb.RequestUnitType]float64, RequestInfo, ResponseInfo) } -// KVCalculator is used to calculate the KV request consumption. +// KVCalculator is used to calculate the KV-side consumption. type KVCalculator struct { *Config } -func newKVCalculator(cfg *Config) *KVCalculator { - return &KVCalculator{Config: cfg} -} +// func newKVCalculator(cfg *Config) *KVCalculator { +// return &KVCalculator{Config: cfg} +// } -func (kc *KVCalculator) Trickle(consumption *rmpb.Consumption, ctx context.Context) { +// Trickle ... +func (kc *KVCalculator) Trickle(ctx context.Context, consumption *rmpb.Consumption) { } +// BeforeKVRequest ... func (kc *KVCalculator) BeforeKVRequest(consumption *rmpb.Consumption, req RequestInfo) { if req.IsWrite() { consumption.KvWriteRpcCount += 1 @@ -69,6 +77,8 @@ func (kc *KVCalculator) BeforeKVRequest(consumption *rmpb.Consumption, req Reque consumption.RRU += float64(kc.ReadBaseCost) } } + +// AfterKVRequest ... func (kc *KVCalculator) AfterKVRequest(consumption *rmpb.Consumption, req RequestInfo, res ResponseInfo) { if req.IsWrite() { kvCPUMs := float64(res.KVCPUMs()) @@ -81,20 +91,24 @@ func (kc *KVCalculator) AfterKVRequest(consumption *rmpb.Consumption, req Reques } } -type SQLCPUCalculator struct { +// SQLCalculator is used to calculate the SQL-side consumption. +type SQLCalculator struct { *Config } -func newSQLCPUCalculator(cfg *Config) *SQLCPUCalculator { - return &SQLCPUCalculator{Config: cfg} -} +// func newSQLCalculator(cfg *Config) *SQLCalculator { +// return &SQLCalculator{Config: cfg} +// } +// Trickle ... // TODO: calculate the SQL CPU cost and related resource consumption. -func (dsc *SQLCPUCalculator) Trickle(resource map[rmpb.ResourceType]float64, ru map[rmpb.RequestUnitType]float64, ctx context.Context) { +func (dsc *SQLCalculator) Trickle(ctx context.Context, resource map[rmpb.ResourceType]float64, ru map[rmpb.RequestUnitType]float64) { } -func (dsc *SQLCPUCalculator) BeforeKVRequest(resource map[rmpb.ResourceType]float64, ru map[rmpb.RequestUnitType]float64, req RequestInfo) { +// BeforeKVRequest ... +func (dsc *SQLCalculator) BeforeKVRequest(resource map[rmpb.ResourceType]float64, ru map[rmpb.RequestUnitType]float64, req RequestInfo) { } -func (dsc *SQLCPUCalculator) AfterKVRequest(resource map[rmpb.ResourceType]float64, ru map[rmpb.RequestUnitType]float64, req RequestInfo, res ResponseInfo) { +// AfterKVRequest ... +func (dsc *SQLCalculator) AfterKVRequest(resource map[rmpb.ResourceType]float64, ru map[rmpb.RequestUnitType]float64, req RequestInfo, res ResponseInfo) { } From e275c0d4907ecb73eac6804e721d74a6c942b379 Mon Sep 17 00:00:00 2001 From: JmPotato Date: Fri, 13 Jan 2023 16:48:45 +0800 Subject: [PATCH 3/4] Adjust the WRU coefficients to smaller values Signed-off-by: JmPotato --- pkg/mcs/resource_manager/client/config.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/mcs/resource_manager/client/config.go b/pkg/mcs/resource_manager/client/config.go index 1e13a18c12a..7cfcd639b42 100644 --- a/pkg/mcs/resource_manager/client/config.go +++ b/pkg/mcs/resource_manager/client/config.go @@ -17,8 +17,8 @@ package client const ( defaultReadBaseCost = 1 defaultReadCostPerByte = 1. / 1024 / 1024 - defaultWriteBaseCost = 5 - defaultWriteCostPerByte = 10. / 1024 / 1024 + defaultWriteBaseCost = 3 + defaultWriteCostPerByte = 5. / 1024 / 1024 defaultWriteCPUMsCost = 1 ) @@ -33,7 +33,7 @@ type RequestUnitConfig struct { // WriteBaseCost is the base cost for a write request. No matter how many bytes read/written or // the CPU times taken for a request, this cost is inevitable. WriteBaseCost float64 `toml:"write-base-cost" json:"write-base-cost"` - // WriteCostPerByte is the cost for each byte written. It's 1 MiB = 10 WRU by default. + // WriteCostPerByte is the cost for each byte written. It's 1 MiB = 5 WRU by default. WriteCostPerByte float64 `toml:"write-cost-per-byte" json:"write-cost-per-byte"` // WriteCPUMsCost is the cost for each millisecond of CPU time taken by a write request. // It's 1 millisecond = 1 WRU by default. From e48b2fc5ebb43cb0e1f44405462579eefb760b0c Mon Sep 17 00:00:00 2001 From: JmPotato Date: Fri, 13 Jan 2023 17:08:58 +0800 Subject: [PATCH 4/4] Address the comments Signed-off-by: JmPotato --- pkg/mcs/resource_manager/client/config.go | 14 +++++----- pkg/mcs/resource_manager/client/model.go | 32 ++++++++++++++--------- 2 files changed, 27 insertions(+), 19 deletions(-) diff --git a/pkg/mcs/resource_manager/client/config.go b/pkg/mcs/resource_manager/client/config.go index 7cfcd639b42..dd578d56557 100644 --- a/pkg/mcs/resource_manager/client/config.go +++ b/pkg/mcs/resource_manager/client/config.go @@ -17,9 +17,9 @@ package client const ( defaultReadBaseCost = 1 defaultReadCostPerByte = 1. / 1024 / 1024 + defaultReadCPUMsCost = 1 defaultWriteBaseCost = 3 defaultWriteCostPerByte = 5. / 1024 / 1024 - defaultWriteCPUMsCost = 1 ) // RequestUnitConfig is the configuration of the request units, which determines the coefficients of @@ -30,14 +30,14 @@ type RequestUnitConfig struct { ReadBaseCost float64 `toml:"read-base-cost" json:"read-base-cost"` // ReadCostPerByte is the cost for each byte read. It's 1 MiB = 1 RRU by default. ReadCostPerByte float64 `toml:"read-cost-per-byte" json:"read-cost-per-byte"` + // ReadCPUMsCost is the cost for each millisecond of CPU time taken by a read request. + // It's 1 millisecond = 1 RRU by default. + ReadCPUMsCost float64 `toml:"read-cpu-ms-cost" json:"read-cpu-ms-cost"` // WriteBaseCost is the base cost for a write request. No matter how many bytes read/written or // the CPU times taken for a request, this cost is inevitable. WriteBaseCost float64 `toml:"write-base-cost" json:"write-base-cost"` // WriteCostPerByte is the cost for each byte written. It's 1 MiB = 5 WRU by default. WriteCostPerByte float64 `toml:"write-cost-per-byte" json:"write-cost-per-byte"` - // WriteCPUMsCost is the cost for each millisecond of CPU time taken by a write request. - // It's 1 millisecond = 1 WRU by default. - WriteCPUMsCost float64 `toml:"write-cpu-ms-cost" json:"write-cpu-ms-cost"` } // DefaultRequestUnitConfig returns the default request unit configuration. @@ -45,9 +45,9 @@ func DefaultRequestUnitConfig() *RequestUnitConfig { return &RequestUnitConfig{ ReadBaseCost: defaultReadBaseCost, ReadCostPerByte: defaultReadCostPerByte, + ReadCPUMsCost: defaultReadCPUMsCost, WriteBaseCost: defaultWriteBaseCost, WriteCostPerByte: defaultWriteCostPerByte, - WriteCPUMsCost: defaultWriteCPUMsCost, } } @@ -57,9 +57,9 @@ func DefaultRequestUnitConfig() *RequestUnitConfig { type Config struct { ReadBaseCost RequestUnit ReadBytesCost RequestUnit + ReadCPUMsCost RequestUnit WriteBaseCost RequestUnit WriteBytesCost RequestUnit - WriteCPUMsCost RequestUnit // TODO: add SQL computing CPU cost. } @@ -75,9 +75,9 @@ func generateConfig(ruConfig *RequestUnitConfig) *Config { cfg := &Config{ ReadBaseCost: RequestUnit(ruConfig.ReadBaseCost), ReadBytesCost: RequestUnit(ruConfig.ReadCostPerByte), + ReadCPUMsCost: RequestUnit(ruConfig.ReadCPUMsCost), WriteBaseCost: RequestUnit(ruConfig.WriteBaseCost), WriteBytesCost: RequestUnit(ruConfig.WriteCostPerByte), - WriteCPUMsCost: RequestUnit(ruConfig.WriteCPUMsCost), } return cfg } diff --git a/pkg/mcs/resource_manager/client/model.go b/pkg/mcs/resource_manager/client/model.go index 9dacf107c02..64a6303774c 100644 --- a/pkg/mcs/resource_manager/client/model.go +++ b/pkg/mcs/resource_manager/client/model.go @@ -43,13 +43,13 @@ type ResponseInfo interface { type ResourceCalculator interface { // Trickle is used to calculate the resource consumption periodically rather than on the request path. // It's mainly used to calculate like the SQL CPU cost. - Trickle(context.Context, map[rmpb.ResourceType]float64, map[rmpb.RequestUnitType]float64) + Trickle(context.Context, *rmpb.Consumption) // BeforeKVRequest is used to calculate the resource consumption before the KV request. // It's mainly used to calculate the base and write request cost. - BeforeKVRequest(map[rmpb.ResourceType]float64, map[rmpb.RequestUnitType]float64, RequestInfo) + BeforeKVRequest(*rmpb.Consumption, RequestInfo) // AfterKVRequest is used to calculate the resource consumption after the KV request. // It's mainly used to calculate the read request cost and KV CPU cost. - AfterKVRequest(map[rmpb.ResourceType]float64, map[rmpb.RequestUnitType]float64, RequestInfo, ResponseInfo) + AfterKVRequest(*rmpb.Consumption, RequestInfo, ResponseInfo) } // KVCalculator is used to calculate the KV-side consumption. @@ -57,6 +57,8 @@ type KVCalculator struct { *Config } +var _ ResourceCalculator = (*KVCalculator)(nil) + // func newKVCalculator(cfg *Config) *KVCalculator { // return &KVCalculator{Config: cfg} // } @@ -69,26 +71,30 @@ func (kc *KVCalculator) Trickle(ctx context.Context, consumption *rmpb.Consumpti func (kc *KVCalculator) BeforeKVRequest(consumption *rmpb.Consumption, req RequestInfo) { if req.IsWrite() { consumption.KvWriteRpcCount += 1 + // Write bytes are knowable in advance, so we can calculate the WRU cost here. writeBytes := float64(req.WriteBytes()) consumption.WriteBytes += writeBytes consumption.WRU += float64(kc.WriteBaseCost) + float64(kc.WriteBytesCost)*writeBytes } else { consumption.KvReadRpcCount += 1 + // Read bytes could not be known before the request is executed, + // so we only add the base cost here. consumption.RRU += float64(kc.ReadBaseCost) } } // AfterKVRequest ... func (kc *KVCalculator) AfterKVRequest(consumption *rmpb.Consumption, req RequestInfo, res ResponseInfo) { - if req.IsWrite() { + // For now, we can only collect the KV CPU cost for a read request. + if !req.IsWrite() { kvCPUMs := float64(res.KVCPUMs()) consumption.TotalCpuTimeMs += kvCPUMs - consumption.WRU += float64(kc.WriteCPUMsCost) * kvCPUMs - } else { - readBytes := float64(res.ReadBytes()) - consumption.ReadBytes += readBytes - consumption.RRU += float64(kc.ReadBytesCost) * readBytes + consumption.RRU += float64(kc.ReadCPUMsCost) * kvCPUMs } + // A write request may also read data, which should be counted into the RRU cost. + readBytes := float64(res.ReadBytes()) + consumption.ReadBytes += readBytes + consumption.RRU += float64(kc.ReadBytesCost) * readBytes } // SQLCalculator is used to calculate the SQL-side consumption. @@ -96,19 +102,21 @@ type SQLCalculator struct { *Config } +var _ ResourceCalculator = (*SQLCalculator)(nil) + // func newSQLCalculator(cfg *Config) *SQLCalculator { // return &SQLCalculator{Config: cfg} // } // Trickle ... // TODO: calculate the SQL CPU cost and related resource consumption. -func (dsc *SQLCalculator) Trickle(ctx context.Context, resource map[rmpb.ResourceType]float64, ru map[rmpb.RequestUnitType]float64) { +func (dsc *SQLCalculator) Trickle(ctx context.Context, consumption *rmpb.Consumption) { } // BeforeKVRequest ... -func (dsc *SQLCalculator) BeforeKVRequest(resource map[rmpb.ResourceType]float64, ru map[rmpb.RequestUnitType]float64, req RequestInfo) { +func (dsc *SQLCalculator) BeforeKVRequest(consumption *rmpb.Consumption, req RequestInfo) { } // AfterKVRequest ... -func (dsc *SQLCalculator) AfterKVRequest(resource map[rmpb.ResourceType]float64, ru map[rmpb.RequestUnitType]float64, req RequestInfo, res ResponseInfo) { +func (dsc *SQLCalculator) AfterKVRequest(consumption *rmpb.Consumption, req RequestInfo, res ResponseInfo) { }