From 1dec52f3b8ba6d260a33379e44f2a24a555b639d Mon Sep 17 00:00:00 2001 From: linzhecheng Date: Wed, 12 Jun 2024 18:07:44 +0800 Subject: [PATCH] feat: support cpu provision of aqc Signed-off-by: linzhecheng --- ...t.kubewharf.io_adminqosconfigurations.yaml | 61 ++++++++++ pkg/apis/config/v1alpha1/adminqos.go | 32 ++++++ .../config/v1alpha1/zz_generated.deepcopy.go | 108 ++++++++++++++++++ pkg/apis/workload/v1alpha1/types.go | 29 +++++ .../clientset/versioned/fake/register.go | 14 +-- .../clientset/versioned/scheme/register.go | 14 +-- 6 files changed, 244 insertions(+), 14 deletions(-) diff --git a/config/crd/bases/config.katalyst.kubewharf.io_adminqosconfigurations.yaml b/config/crd/bases/config.katalyst.kubewharf.io_adminqosconfigurations.yaml index ccce31f..ef92944 100644 --- a/config/crd/bases/config.katalyst.kubewharf.io_adminqosconfigurations.yaml +++ b/config/crd/bases/config.katalyst.kubewharf.io_adminqosconfigurations.yaml @@ -76,6 +76,67 @@ spec: to ensure that shared_cores pods can suppress and preempt reclaimed_cores pods. type: boolean + cpuProvisionConfig: + description: optional + properties: + constraints: + items: + properties: + name: + description: ControlKnobName defines available + control knob key for provision policy + type: string + restrictControlKnobMaxLowerGap: + description: RestrictControlKnobMaxUpperGap + is the maximum downward offset value from + the baseline + type: number + restrictControlKnobMaxLowerGapRatio: + description: RestrictControlKnobMaxUpperGap + is the maximum downward offset ratio from + the baseline + type: number + restrictControlKnobMaxUpperGap: + description: RestrictControlKnobMaxUpperGap + is the maximum upward offset value from the + baseline + type: number + restrictControlKnobMaxUpperGapRatio: + description: RestrictControlKnobMaxUpperGap + is the maximum upward offset ratio from the + baseline + type: number + required: + - name + type: object + type: array + indicatorTargets: + items: + properties: + regionType: + description: QoSRegionType declares pre-defined + region types + type: string + targets: + items: + properties: + name: + type: string + target: + type: number + required: + - name + - target + type: object + type: array + required: + - regionType + - targets + type: object + type: array + type: object + required: + - cpuProvisionConfig type: object memoryAdvisorConfig: properties: diff --git a/pkg/apis/config/v1alpha1/adminqos.go b/pkg/apis/config/v1alpha1/adminqos.go index 7e1a408..07b4522 100644 --- a/pkg/apis/config/v1alpha1/adminqos.go +++ b/pkg/apis/config/v1alpha1/adminqos.go @@ -20,6 +20,8 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/kubewharf/katalyst-api/pkg/apis/workload/v1alpha1" ) // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object @@ -131,6 +133,36 @@ type CPUAdvisorConfig struct { // we will rely on kernel features to ensure that shared_cores pods can suppress and preempt reclaimed_cores pods. // +optional AllowSharedCoresOverlapReclaimedCores *bool `json:"allowSharedCoresOverlapReclaimedCores,omitempty"` + + // optional + CPUProvisionConfig *CPUProvisionConfig `json:"cpuProvisionConfig"` +} + +type RegionIndicators struct { + regionType v1alpha1.QoSRegionType `json:"regionType"` + Targets []IndicatorTargetConfiguration `json:"targets"` +} + +type IndicatorTargetConfiguration struct { + Name v1alpha1.ServiceSystemIndicatorName `json:"name"` + Target float64 `json:"target"` +} + +type ControlKnobConstraints struct { + Name v1alpha1.ControlKnobName `json:"name"` + // RestrictControlKnobMaxUpperGap is the maximum upward offset value from the baseline + RestrictControlKnobMaxUpperGap *float64 `json:"restrictControlKnobMaxUpperGap,omitempty"` + // RestrictControlKnobMaxUpperGap is the maximum downward offset value from the baseline + RestrictControlKnobMaxLowerGap *float64 `json:"restrictControlKnobMaxLowerGap,omitempty"` + // RestrictControlKnobMaxUpperGap is the maximum upward offset ratio from the baseline + RestrictControlKnobMaxUpperGapRatio *float64 `json:"restrictControlKnobMaxUpperGapRatio,omitempty"` + // RestrictControlKnobMaxUpperGap is the maximum downward offset ratio from the baseline + RestrictControlKnobMaxLowerGapRatio *float64 `json:"restrictControlKnobMaxLowerGapRatio,omitempty"` +} + +type CPUProvisionConfig struct { + RegionIndicators []RegionIndicators `json:"indicatorTargets,omitempty"` + Constraints []ControlKnobConstraints `json:"constraints,omitempty"` } type MemoryAdvisorConfig struct { diff --git a/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go index cfaedc4..ec8dbce 100644 --- a/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go @@ -362,6 +362,11 @@ func (in *CPUAdvisorConfig) DeepCopyInto(out *CPUAdvisorConfig) { *out = new(bool) **out = **in } + if in.CPUProvisionConfig != nil { + in, out := &in.CPUProvisionConfig, &out.CPUProvisionConfig + *out = new(CPUProvisionConfig) + (*in).DeepCopyInto(*out) + } return } @@ -498,6 +503,36 @@ func (in *CPUPressureEvictionConfig) DeepCopy() *CPUPressureEvictionConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CPUProvisionConfig) DeepCopyInto(out *CPUProvisionConfig) { + *out = *in + if in.RegionIndicators != nil { + in, out := &in.RegionIndicators, &out.RegionIndicators + *out = make([]RegionIndicators, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Constraints != nil { + in, out := &in.Constraints, &out.Constraints + *out = make([]ControlKnobConstraints, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CPUProvisionConfig. +func (in *CPUProvisionConfig) DeepCopy() *CPUProvisionConfig { + if in == nil { + return nil + } + out := new(CPUProvisionConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CPUSystemPressureEvictionConfig) DeepCopyInto(out *CPUSystemPressureEvictionConfig) { *out = *in @@ -601,6 +636,42 @@ func (in *CgroupConfig) DeepCopy() *CgroupConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ControlKnobConstraints) DeepCopyInto(out *ControlKnobConstraints) { + *out = *in + if in.RestrictControlKnobMaxUpperGap != nil { + in, out := &in.RestrictControlKnobMaxUpperGap, &out.RestrictControlKnobMaxUpperGap + *out = new(float64) + **out = **in + } + if in.RestrictControlKnobMaxLowerGap != nil { + in, out := &in.RestrictControlKnobMaxLowerGap, &out.RestrictControlKnobMaxLowerGap + *out = new(float64) + **out = **in + } + if in.RestrictControlKnobMaxUpperGapRatio != nil { + in, out := &in.RestrictControlKnobMaxUpperGapRatio, &out.RestrictControlKnobMaxUpperGapRatio + *out = new(float64) + **out = **in + } + if in.RestrictControlKnobMaxLowerGapRatio != nil { + in, out := &in.RestrictControlKnobMaxLowerGapRatio, &out.RestrictControlKnobMaxLowerGapRatio + *out = new(float64) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControlKnobConstraints. +func (in *ControlKnobConstraints) DeepCopy() *ControlKnobConstraints { + if in == nil { + return nil + } + out := new(ControlKnobConstraints) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CustomNodeConfig) DeepCopyInto(out *CustomNodeConfig) { *out = *in @@ -875,6 +946,22 @@ func (in *GlobalResourcePortraitConfiguration) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IndicatorTargetConfiguration) DeepCopyInto(out *IndicatorTargetConfiguration) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IndicatorTargetConfiguration. +func (in *IndicatorTargetConfiguration) DeepCopy() *IndicatorTargetConfiguration { + if in == nil { + return nil + } + out := new(IndicatorTargetConfiguration) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KatalystCustomConfig) DeepCopyInto(out *KatalystCustomConfig) { *out = *in @@ -1404,6 +1491,27 @@ func (in *RefaultPolicyConf) DeepCopy() *RefaultPolicyConf { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RegionIndicators) DeepCopyInto(out *RegionIndicators) { + *out = *in + if in.Targets != nil { + in, out := &in.Targets, &out.Targets + *out = make([]IndicatorTargetConfiguration, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RegionIndicators. +func (in *RegionIndicators) DeepCopy() *RegionIndicators { + if in == nil { + return nil + } + out := new(RegionIndicators) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ResourcePortraitConfig) DeepCopyInto(out *ResourcePortraitConfig) { *out = *in diff --git a/pkg/apis/workload/v1alpha1/types.go b/pkg/apis/workload/v1alpha1/types.go index ad552ed..9b327ff 100644 --- a/pkg/apis/workload/v1alpha1/types.go +++ b/pkg/apis/workload/v1alpha1/types.go @@ -322,3 +322,32 @@ type TestExtendedIndicators struct { type TestIndicators struct { TestIndicator string `json:"testIndicator"` } + +// QoSRegionType declares pre-defined region types +type QoSRegionType string + +const ( + // QoSRegionTypeShare for each share pool + QoSRegionTypeShare QoSRegionType = "share" + + // QoSRegionTypeIsolation for each isolation pool + QoSRegionTypeIsolation QoSRegionType = "isolation" + + // QoSRegionTypeDedicatedNumaExclusive for each dedicated core with numa binding + // and numa exclusive container + QoSRegionTypeDedicatedNumaExclusive QoSRegionType = "dedicated-numa-exclusive" +) + +// ControlKnobName defines available control knob key for provision policy +type ControlKnobName string + +const ( + // ControlKnobNonReclaimedCPURequirement refers to cpu requirement of non-reclaimed workloads, like shared_cores and dedicated_cores + ControlKnobNonReclaimedCPURequirement ControlKnobName = "non-reclaimed-cpu-requirement" + + // ControlKnobNonReclaimedCPURequirementUpper refers to the upper cpu size, for isolated pods now + ControlKnobNonReclaimedCPURequirementUpper ControlKnobName = "non-reclaimed-cpu-requirement-upper" + + // ControlKnobNonReclaimedCPURequirementLower refers to the lower cpu size, for isolated pods now + ControlKnobNonReclaimedCPURequirementLower ControlKnobName = "non-reclaimed-cpu-requirement-lower" +) diff --git a/pkg/client/clientset/versioned/fake/register.go b/pkg/client/clientset/versioned/fake/register.go index 0f7d311..9aaf89e 100644 --- a/pkg/client/clientset/versioned/fake/register.go +++ b/pkg/client/clientset/versioned/fake/register.go @@ -51,14 +51,14 @@ var localSchemeBuilder = runtime.SchemeBuilder{ // AddToScheme adds all types of this clientset into the given scheme. This allows composition // of clientsets, like in: // -// import ( -// "k8s.io/client-go/kubernetes" -// clientsetscheme "k8s.io/client-go/kubernetes/scheme" -// aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" -// ) +// import ( +// "k8s.io/client-go/kubernetes" +// clientsetscheme "k8s.io/client-go/kubernetes/scheme" +// aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" +// ) // -// kclientset, _ := kubernetes.NewForConfig(c) -// _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) +// kclientset, _ := kubernetes.NewForConfig(c) +// _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) // // After this, RawExtensions in Kubernetes types will serialize kube-aggregator types // correctly. diff --git a/pkg/client/clientset/versioned/scheme/register.go b/pkg/client/clientset/versioned/scheme/register.go index 17ad6fb..ffa9cdb 100644 --- a/pkg/client/clientset/versioned/scheme/register.go +++ b/pkg/client/clientset/versioned/scheme/register.go @@ -51,14 +51,14 @@ var localSchemeBuilder = runtime.SchemeBuilder{ // AddToScheme adds all types of this clientset into the given scheme. This allows composition // of clientsets, like in: // -// import ( -// "k8s.io/client-go/kubernetes" -// clientsetscheme "k8s.io/client-go/kubernetes/scheme" -// aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" -// ) +// import ( +// "k8s.io/client-go/kubernetes" +// clientsetscheme "k8s.io/client-go/kubernetes/scheme" +// aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" +// ) // -// kclientset, _ := kubernetes.NewForConfig(c) -// _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) +// kclientset, _ := kubernetes.NewForConfig(c) +// _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) // // After this, RawExtensions in Kubernetes types will serialize kube-aggregator types // correctly.