From 83d619d159e6bff0dba32ba4ccd65bbc1bf3ea12 Mon Sep 17 00:00:00 2001 From: Peter Hunt Date: Wed, 9 Oct 2024 15:25:50 -0400 Subject: [PATCH 1/4] kubeapiserver: add minimum kubelet version Signed-off-by: Peter Hunt --- kubecontrolplane/v1/types.go | 6 ++++++ kubecontrolplane/v1/zz_generated.swagger_doc_generated.go | 1 + openapi/generated_openapi/zz_generated.openapi.go | 7 +++++++ openapi/openapi.json | 4 ++++ 4 files changed, 18 insertions(+) diff --git a/kubecontrolplane/v1/types.go b/kubecontrolplane/v1/types.go index b9cdcc213b8..3fe564ffcc6 100644 --- a/kubecontrolplane/v1/types.go +++ b/kubecontrolplane/v1/types.go @@ -62,6 +62,12 @@ type KubeAPIServerConfig struct { // TODO this needs to be removed. APIServerArguments map[string]Arguments `json:"apiServerArguments"` + + // MinimumKubeletVersion is the lowest version of a kubelet that can meaningfully join the cluster. + // Specifically, the apiserver will deny most authorization requests of kubelets that are older + // than the specified version, only allowing the kubelet to get and update its node object, and perform + // subjectaccessreviews. + MinimumKubeletVersion string `json:"minimumKubeletVersion,omitempty"` } // Arguments masks the value so protobuf can generate diff --git a/kubecontrolplane/v1/zz_generated.swagger_doc_generated.go b/kubecontrolplane/v1/zz_generated.swagger_doc_generated.go index 906bb271b00..75264a69bff 100644 --- a/kubecontrolplane/v1/zz_generated.swagger_doc_generated.go +++ b/kubecontrolplane/v1/zz_generated.swagger_doc_generated.go @@ -33,6 +33,7 @@ var map_KubeAPIServerConfig = map[string]string{ "projectConfig": "projectConfig feeds an admission plugin", "serviceAccountPublicKeyFiles": "serviceAccountPublicKeyFiles is a list of files, each containing a PEM-encoded public RSA key. (If any file contains a private key, the public portion of the key is used) The list of public keys is used to verify presented service account tokens. Each key is tried in order until the list is exhausted or verification succeeds. If no keys are specified, no service account authentication will be available.", "oauthConfig": "oauthConfig, if present start the /oauth endpoint in this process", + "minimumKubeletVersion": "MinimumKubeletVersion is the lowest version of a kubelet that can meaningfully join the cluster. Specifically, the apiserver will deny most authorization requests of kubelets that are older than the specified version, only allowing the kubelet to get and update its node object, and perform subjectaccessreviews.", } func (KubeAPIServerConfig) SwaggerDoc() map[string]string { diff --git a/openapi/generated_openapi/zz_generated.openapi.go b/openapi/generated_openapi/zz_generated.openapi.go index 53d69dce110..b1348c22847 100644 --- a/openapi/generated_openapi/zz_generated.openapi.go +++ b/openapi/generated_openapi/zz_generated.openapi.go @@ -26499,6 +26499,13 @@ func schema_openshift_api_kubecontrolplane_v1_KubeAPIServerConfig(ref common.Ref }, }, }, + "minimumKubeletVersion": { + SchemaProps: spec.SchemaProps{ + Description: "MinimumKubeletVersion is the lowest version of a kubelet that can meaningfully join the cluster. Specifically, the apiserver will deny most authorization requests of kubelets that are older than the specified version, only allowing the kubelet to get and update its node object, and perform subjectaccessreviews.", + Type: []string{"string"}, + Format: "", + }, + }, }, Required: []string{"servingInfo", "corsAllowedOrigins", "auditConfig", "storageConfig", "admission", "kubeClientConfig", "authConfig", "aggregatorConfig", "kubeletClientInfo", "servicesSubnet", "servicesNodePortRange", "consolePublicURL", "userAgentMatchingConfig", "imagePolicyConfig", "projectConfig", "serviceAccountPublicKeyFiles", "oauthConfig", "apiServerArguments"}, }, diff --git a/openapi/openapi.json b/openapi/openapi.json index 4fe157bf21e..df66fce7d43 100644 --- a/openapi/openapi.json +++ b/openapi/openapi.json @@ -14786,6 +14786,10 @@ "default": {}, "$ref": "#/definitions/com.github.openshift.api.kubecontrolplane.v1.KubeletConnectionInfo" }, + "minimumKubeletVersion": { + "description": "MinimumKubeletVersion is the lowest version of a kubelet that can meaningfully join the cluster. Specifically, the apiserver will deny most authorization requests of kubelets that are older than the specified version, only allowing the kubelet to get and update its node object, and perform subjectaccessreviews.", + "type": "string" + }, "oauthConfig": { "description": "oauthConfig, if present start the /oauth endpoint in this process", "$ref": "#/definitions/com.github.openshift.api.osin.v1.OAuthConfig" From ffa1b52de601376880d7f84aba1aca0e3dd2eff6 Mon Sep 17 00:00:00 2001 From: Peter Hunt Date: Fri, 11 Oct 2024 16:55:50 -0400 Subject: [PATCH 2/4] config/types_node: add minimumKubeletVersion Signed-off-by: Peter Hunt --- config/v1/types_node.go | 8 ++++++++ .../0000_10_config-operator_01_nodes.crd.yaml | 8 ++++++++ .../nodes.config.openshift.io/AAA_ungated.yaml | 8 ++++++++ config/v1/zz_generated.swagger_doc_generated.go | 5 +++-- openapi/generated_openapi/zz_generated.openapi.go | 7 +++++++ openapi/openapi.json | 4 ++++ .../crds/0000_10_config-operator_01_nodes.crd.yaml | 8 ++++++++ 7 files changed, 46 insertions(+), 2 deletions(-) diff --git a/config/v1/types_node.go b/config/v1/types_node.go index b3b1b62c4df..b5dd457847b 100644 --- a/config/v1/types_node.go +++ b/config/v1/types_node.go @@ -46,6 +46,14 @@ type NodeSpec struct { // the status and corresponding reaction of the cluster // +optional WorkerLatencyProfile WorkerLatencyProfileType `json:"workerLatencyProfile,omitempty"` + + // MinimumKubeletVersion is the lowest version of a kubelet that can meaningfully join the cluster. + // Specifically, the apiserver will deny most authorization requests of kubelets that are older + // than the specified version, only allowing the kubelet to get and update its node object, and perform + // subjectaccessreviews. + // +kubebuilder:validation:Pattern=`^[0-9]*\.[0-9]*\.[0-9]*$` + // +optional + MinimumKubeletVersion string `json:"minimumKubeletVersion,omitempty"` } type NodeStatus struct { diff --git a/config/v1/zz_generated.crd-manifests/0000_10_config-operator_01_nodes.crd.yaml b/config/v1/zz_generated.crd-manifests/0000_10_config-operator_01_nodes.crd.yaml index 87de7f1b93d..9d2f140a0a4 100644 --- a/config/v1/zz_generated.crd-manifests/0000_10_config-operator_01_nodes.crd.yaml +++ b/config/v1/zz_generated.crd-manifests/0000_10_config-operator_01_nodes.crd.yaml @@ -52,6 +52,14 @@ spec: - v2 - "" type: string + minimumKubeletVersion: + description: |- + MinimumKubeletVersion is the lowest version of a kubelet that can meaningfully join the cluster. + Specifically, the apiserver will deny most authorization requests of kubelets that are older + than the specified version, only allowing the kubelet to get and update its node object, and perform + subjectaccessreviews. + pattern: ^[0-9]*\.[0-9]*\.[0-9]*$ + type: string workerLatencyProfile: description: |- WorkerLatencyProfile determins the how fast the kubelet is updating diff --git a/config/v1/zz_generated.featuregated-crd-manifests/nodes.config.openshift.io/AAA_ungated.yaml b/config/v1/zz_generated.featuregated-crd-manifests/nodes.config.openshift.io/AAA_ungated.yaml index e7a9a23a9ab..36ffe6f249a 100644 --- a/config/v1/zz_generated.featuregated-crd-manifests/nodes.config.openshift.io/AAA_ungated.yaml +++ b/config/v1/zz_generated.featuregated-crd-manifests/nodes.config.openshift.io/AAA_ungated.yaml @@ -53,6 +53,14 @@ spec: - v2 - "" type: string + minimumKubeletVersion: + description: |- + MinimumKubeletVersion is the lowest version of a kubelet that can meaningfully join the cluster. + Specifically, the apiserver will deny most authorization requests of kubelets that are older + than the specified version, only allowing the kubelet to get and update its node object, and perform + subjectaccessreviews. + pattern: ^[0-9]*\.[0-9]*\.[0-9]*$ + type: string workerLatencyProfile: description: |- WorkerLatencyProfile determins the how fast the kubelet is updating diff --git a/config/v1/zz_generated.swagger_doc_generated.go b/config/v1/zz_generated.swagger_doc_generated.go index ea3a424046c..2ce5ea99f67 100644 --- a/config/v1/zz_generated.swagger_doc_generated.go +++ b/config/v1/zz_generated.swagger_doc_generated.go @@ -2088,8 +2088,9 @@ func (NodeList) SwaggerDoc() map[string]string { } var map_NodeSpec = map[string]string{ - "cgroupMode": "CgroupMode determines the cgroups version on the node", - "workerLatencyProfile": "WorkerLatencyProfile determins the how fast the kubelet is updating the status and corresponding reaction of the cluster", + "cgroupMode": "CgroupMode determines the cgroups version on the node", + "workerLatencyProfile": "WorkerLatencyProfile determins the how fast the kubelet is updating the status and corresponding reaction of the cluster", + "minimumKubeletVersion": "MinimumKubeletVersion is the lowest version of a kubelet that can meaningfully join the cluster. Specifically, the apiserver will deny most authorization requests of kubelets that are older than the specified version, only allowing the kubelet to get and update its node object, and perform subjectaccessreviews.", } func (NodeSpec) SwaggerDoc() map[string]string { diff --git a/openapi/generated_openapi/zz_generated.openapi.go b/openapi/generated_openapi/zz_generated.openapi.go index b1348c22847..ae21056c5d5 100644 --- a/openapi/generated_openapi/zz_generated.openapi.go +++ b/openapi/generated_openapi/zz_generated.openapi.go @@ -15552,6 +15552,13 @@ func schema_openshift_api_config_v1_NodeSpec(ref common.ReferenceCallback) commo Format: "", }, }, + "minimumKubeletVersion": { + SchemaProps: spec.SchemaProps{ + Description: "MinimumKubeletVersion is the lowest version of a kubelet that can meaningfully join the cluster. Specifically, the apiserver will deny most authorization requests of kubelets that are older than the specified version, only allowing the kubelet to get and update its node object, and perform subjectaccessreviews.", + Type: []string{"string"}, + Format: "", + }, + }, }, }, }, diff --git a/openapi/openapi.json b/openapi/openapi.json index df66fce7d43..19ac8961af7 100644 --- a/openapi/openapi.json +++ b/openapi/openapi.json @@ -8270,6 +8270,10 @@ "description": "CgroupMode determines the cgroups version on the node", "type": "string" }, + "minimumKubeletVersion": { + "description": "MinimumKubeletVersion is the lowest version of a kubelet that can meaningfully join the cluster. Specifically, the apiserver will deny most authorization requests of kubelets that are older than the specified version, only allowing the kubelet to get and update its node object, and perform subjectaccessreviews.", + "type": "string" + }, "workerLatencyProfile": { "description": "WorkerLatencyProfile determins the how fast the kubelet is updating the status and corresponding reaction of the cluster", "type": "string" diff --git a/payload-manifests/crds/0000_10_config-operator_01_nodes.crd.yaml b/payload-manifests/crds/0000_10_config-operator_01_nodes.crd.yaml index 87de7f1b93d..9d2f140a0a4 100644 --- a/payload-manifests/crds/0000_10_config-operator_01_nodes.crd.yaml +++ b/payload-manifests/crds/0000_10_config-operator_01_nodes.crd.yaml @@ -52,6 +52,14 @@ spec: - v2 - "" type: string + minimumKubeletVersion: + description: |- + MinimumKubeletVersion is the lowest version of a kubelet that can meaningfully join the cluster. + Specifically, the apiserver will deny most authorization requests of kubelets that are older + than the specified version, only allowing the kubelet to get and update its node object, and perform + subjectaccessreviews. + pattern: ^[0-9]*\.[0-9]*\.[0-9]*$ + type: string workerLatencyProfile: description: |- WorkerLatencyProfile determins the how fast the kubelet is updating From 0a8ec780d4898b94d1fe391e1f13074b445c380b Mon Sep 17 00:00:00 2001 From: Peter Hunt Date: Tue, 22 Oct 2024 15:20:44 -0400 Subject: [PATCH 3/4] feature gates: Add MinimumKubeletVersion gate Signed-off-by: Peter Hunt --- .../MinimumKubeletVersion.yaml | 14 ++ config/v1/types_node.go | 1 + ...perator_01_nodes-CustomNoUpgrade.crd.yaml} | 1 + ..._config-operator_01_nodes-Default.crd.yaml | 137 +++++++++++++++++ ...ator_01_nodes-DevPreviewNoUpgrade.crd.yaml | 145 ++++++++++++++++++ ...tor_01_nodes-TechPreviewNoUpgrade.crd.yaml | 145 ++++++++++++++++++ ..._generated.featuregated-crd-manifests.yaml | 3 +- .../AAA_ungated.yaml | 8 - .../MinimumKubeletVersion.yaml | 145 ++++++++++++++++++ features.md | 1 + features/features.go | 8 + kubecontrolplane/v1/types.go | 1 + ...perator_01_nodes-CustomNoUpgrade.crd.yaml} | 1 + ..._config-operator_01_nodes-Default.crd.yaml | 137 +++++++++++++++++ ...ator_01_nodes-DevPreviewNoUpgrade.crd.yaml | 145 ++++++++++++++++++ ...tor_01_nodes-TechPreviewNoUpgrade.crd.yaml | 145 ++++++++++++++++++ .../featureGate-Hypershift-Default.yaml | 3 + ...reGate-Hypershift-DevPreviewNoUpgrade.yaml | 3 + ...eGate-Hypershift-TechPreviewNoUpgrade.yaml | 3 + .../featureGate-SelfManagedHA-Default.yaml | 3 + ...ate-SelfManagedHA-DevPreviewNoUpgrade.yaml | 3 + ...te-SelfManagedHA-TechPreviewNoUpgrade.yaml | 3 + 22 files changed, 1046 insertions(+), 9 deletions(-) create mode 100644 config/v1/tests/nodes.config.openshift.io/MinimumKubeletVersion.yaml rename config/v1/zz_generated.crd-manifests/{0000_10_config-operator_01_nodes.crd.yaml => 0000_10_config-operator_01_nodes-CustomNoUpgrade.crd.yaml} (99%) create mode 100644 config/v1/zz_generated.crd-manifests/0000_10_config-operator_01_nodes-Default.crd.yaml create mode 100644 config/v1/zz_generated.crd-manifests/0000_10_config-operator_01_nodes-DevPreviewNoUpgrade.crd.yaml create mode 100644 config/v1/zz_generated.crd-manifests/0000_10_config-operator_01_nodes-TechPreviewNoUpgrade.crd.yaml create mode 100644 config/v1/zz_generated.featuregated-crd-manifests/nodes.config.openshift.io/MinimumKubeletVersion.yaml rename payload-manifests/crds/{0000_10_config-operator_01_nodes.crd.yaml => 0000_10_config-operator_01_nodes-CustomNoUpgrade.crd.yaml} (99%) create mode 100644 payload-manifests/crds/0000_10_config-operator_01_nodes-Default.crd.yaml create mode 100644 payload-manifests/crds/0000_10_config-operator_01_nodes-DevPreviewNoUpgrade.crd.yaml create mode 100644 payload-manifests/crds/0000_10_config-operator_01_nodes-TechPreviewNoUpgrade.crd.yaml diff --git a/config/v1/tests/nodes.config.openshift.io/MinimumKubeletVersion.yaml b/config/v1/tests/nodes.config.openshift.io/MinimumKubeletVersion.yaml new file mode 100644 index 00000000000..41ed1e8c072 --- /dev/null +++ b/config/v1/tests/nodes.config.openshift.io/MinimumKubeletVersion.yaml @@ -0,0 +1,14 @@ +apiVersion: apiextensions.k8s.io/v1 # Hack because controller-gen complains if we don't have this +name: "Node" +crdName: nodes.config.openshift.io +tests: + onCreate: + - name: Should be able to create a minimal Node + initial: | + apiVersion: config.openshift.io/v1 + kind: Node + spec: {} # No spec is required for a Node + expected: | + apiVersion: config.openshift.io/v1 + kind: Node + spec: {} diff --git a/config/v1/types_node.go b/config/v1/types_node.go index b5dd457847b..20b536a2bf2 100644 --- a/config/v1/types_node.go +++ b/config/v1/types_node.go @@ -52,6 +52,7 @@ type NodeSpec struct { // than the specified version, only allowing the kubelet to get and update its node object, and perform // subjectaccessreviews. // +kubebuilder:validation:Pattern=`^[0-9]*\.[0-9]*\.[0-9]*$` + // +openshift:enable:FeatureGate=MinimumKubeletVersion // +optional MinimumKubeletVersion string `json:"minimumKubeletVersion,omitempty"` } diff --git a/config/v1/zz_generated.crd-manifests/0000_10_config-operator_01_nodes.crd.yaml b/config/v1/zz_generated.crd-manifests/0000_10_config-operator_01_nodes-CustomNoUpgrade.crd.yaml similarity index 99% rename from config/v1/zz_generated.crd-manifests/0000_10_config-operator_01_nodes.crd.yaml rename to config/v1/zz_generated.crd-manifests/0000_10_config-operator_01_nodes-CustomNoUpgrade.crd.yaml index 9d2f140a0a4..121a47ac41a 100644 --- a/config/v1/zz_generated.crd-manifests/0000_10_config-operator_01_nodes.crd.yaml +++ b/config/v1/zz_generated.crd-manifests/0000_10_config-operator_01_nodes-CustomNoUpgrade.crd.yaml @@ -7,6 +7,7 @@ metadata: include.release.openshift.io/ibm-cloud-managed: "true" include.release.openshift.io/self-managed-high-availability: "true" release.openshift.io/bootstrap-required: "true" + release.openshift.io/feature-set: CustomNoUpgrade name: nodes.config.openshift.io spec: group: config.openshift.io diff --git a/config/v1/zz_generated.crd-manifests/0000_10_config-operator_01_nodes-Default.crd.yaml b/config/v1/zz_generated.crd-manifests/0000_10_config-operator_01_nodes-Default.crd.yaml new file mode 100644 index 00000000000..b79a394c6e1 --- /dev/null +++ b/config/v1/zz_generated.crd-manifests/0000_10_config-operator_01_nodes-Default.crd.yaml @@ -0,0 +1,137 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + api-approved.openshift.io: https://github.com/openshift/api/pull/1107 + api.openshift.io/merged-by-featuregates: "true" + include.release.openshift.io/ibm-cloud-managed: "true" + include.release.openshift.io/self-managed-high-availability: "true" + release.openshift.io/bootstrap-required: "true" + release.openshift.io/feature-set: Default + name: nodes.config.openshift.io +spec: + group: config.openshift.io + names: + kind: Node + listKind: NodeList + plural: nodes + singular: node + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + description: |- + Node holds cluster-wide information about node specific features. + + Compatibility level 1: Stable within a major release for a minimum of 12 months or 3 minor releases (whichever is longer). + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: spec holds user settable values for configuration + properties: + cgroupMode: + description: CgroupMode determines the cgroups version on the node + enum: + - v1 + - v2 + - "" + type: string + workerLatencyProfile: + description: |- + WorkerLatencyProfile determins the how fast the kubelet is updating + the status and corresponding reaction of the cluster + enum: + - Default + - MediumUpdateAverageReaction + - LowUpdateSlowReaction + type: string + type: object + status: + description: status holds observed values. + properties: + conditions: + description: conditions contain the details and the current state + of the nodes.config object + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/v1/zz_generated.crd-manifests/0000_10_config-operator_01_nodes-DevPreviewNoUpgrade.crd.yaml b/config/v1/zz_generated.crd-manifests/0000_10_config-operator_01_nodes-DevPreviewNoUpgrade.crd.yaml new file mode 100644 index 00000000000..3b6f33dffaa --- /dev/null +++ b/config/v1/zz_generated.crd-manifests/0000_10_config-operator_01_nodes-DevPreviewNoUpgrade.crd.yaml @@ -0,0 +1,145 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + api-approved.openshift.io: https://github.com/openshift/api/pull/1107 + api.openshift.io/merged-by-featuregates: "true" + include.release.openshift.io/ibm-cloud-managed: "true" + include.release.openshift.io/self-managed-high-availability: "true" + release.openshift.io/bootstrap-required: "true" + release.openshift.io/feature-set: DevPreviewNoUpgrade + name: nodes.config.openshift.io +spec: + group: config.openshift.io + names: + kind: Node + listKind: NodeList + plural: nodes + singular: node + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + description: |- + Node holds cluster-wide information about node specific features. + + Compatibility level 1: Stable within a major release for a minimum of 12 months or 3 minor releases (whichever is longer). + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: spec holds user settable values for configuration + properties: + cgroupMode: + description: CgroupMode determines the cgroups version on the node + enum: + - v1 + - v2 + - "" + type: string + minimumKubeletVersion: + description: |- + MinimumKubeletVersion is the lowest version of a kubelet that can meaningfully join the cluster. + Specifically, the apiserver will deny most authorization requests of kubelets that are older + than the specified version, only allowing the kubelet to get and update its node object, and perform + subjectaccessreviews. + pattern: ^[0-9]*\.[0-9]*\.[0-9]*$ + type: string + workerLatencyProfile: + description: |- + WorkerLatencyProfile determins the how fast the kubelet is updating + the status and corresponding reaction of the cluster + enum: + - Default + - MediumUpdateAverageReaction + - LowUpdateSlowReaction + type: string + type: object + status: + description: status holds observed values. + properties: + conditions: + description: conditions contain the details and the current state + of the nodes.config object + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/v1/zz_generated.crd-manifests/0000_10_config-operator_01_nodes-TechPreviewNoUpgrade.crd.yaml b/config/v1/zz_generated.crd-manifests/0000_10_config-operator_01_nodes-TechPreviewNoUpgrade.crd.yaml new file mode 100644 index 00000000000..3e92f0df978 --- /dev/null +++ b/config/v1/zz_generated.crd-manifests/0000_10_config-operator_01_nodes-TechPreviewNoUpgrade.crd.yaml @@ -0,0 +1,145 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + api-approved.openshift.io: https://github.com/openshift/api/pull/1107 + api.openshift.io/merged-by-featuregates: "true" + include.release.openshift.io/ibm-cloud-managed: "true" + include.release.openshift.io/self-managed-high-availability: "true" + release.openshift.io/bootstrap-required: "true" + release.openshift.io/feature-set: TechPreviewNoUpgrade + name: nodes.config.openshift.io +spec: + group: config.openshift.io + names: + kind: Node + listKind: NodeList + plural: nodes + singular: node + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + description: |- + Node holds cluster-wide information about node specific features. + + Compatibility level 1: Stable within a major release for a minimum of 12 months or 3 minor releases (whichever is longer). + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: spec holds user settable values for configuration + properties: + cgroupMode: + description: CgroupMode determines the cgroups version on the node + enum: + - v1 + - v2 + - "" + type: string + minimumKubeletVersion: + description: |- + MinimumKubeletVersion is the lowest version of a kubelet that can meaningfully join the cluster. + Specifically, the apiserver will deny most authorization requests of kubelets that are older + than the specified version, only allowing the kubelet to get and update its node object, and perform + subjectaccessreviews. + pattern: ^[0-9]*\.[0-9]*\.[0-9]*$ + type: string + workerLatencyProfile: + description: |- + WorkerLatencyProfile determins the how fast the kubelet is updating + the status and corresponding reaction of the cluster + enum: + - Default + - MediumUpdateAverageReaction + - LowUpdateSlowReaction + type: string + type: object + status: + description: status holds observed values. + properties: + conditions: + description: conditions contain the details and the current state + of the nodes.config object + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/v1/zz_generated.featuregated-crd-manifests.yaml b/config/v1/zz_generated.featuregated-crd-manifests.yaml index 6b8dfd3f007..abfea5eaf0e 100644 --- a/config/v1/zz_generated.featuregated-crd-manifests.yaml +++ b/config/v1/zz_generated.featuregated-crd-manifests.yaml @@ -383,7 +383,8 @@ nodes.config.openshift.io: CRDName: nodes.config.openshift.io Capability: "" Category: "" - FeatureGates: [] + FeatureGates: + - MinimumKubeletVersion FilenameOperatorName: config-operator FilenameOperatorOrdering: "01" FilenameRunLevel: "0000_10" diff --git a/config/v1/zz_generated.featuregated-crd-manifests/nodes.config.openshift.io/AAA_ungated.yaml b/config/v1/zz_generated.featuregated-crd-manifests/nodes.config.openshift.io/AAA_ungated.yaml index 36ffe6f249a..e7a9a23a9ab 100644 --- a/config/v1/zz_generated.featuregated-crd-manifests/nodes.config.openshift.io/AAA_ungated.yaml +++ b/config/v1/zz_generated.featuregated-crd-manifests/nodes.config.openshift.io/AAA_ungated.yaml @@ -53,14 +53,6 @@ spec: - v2 - "" type: string - minimumKubeletVersion: - description: |- - MinimumKubeletVersion is the lowest version of a kubelet that can meaningfully join the cluster. - Specifically, the apiserver will deny most authorization requests of kubelets that are older - than the specified version, only allowing the kubelet to get and update its node object, and perform - subjectaccessreviews. - pattern: ^[0-9]*\.[0-9]*\.[0-9]*$ - type: string workerLatencyProfile: description: |- WorkerLatencyProfile determins the how fast the kubelet is updating diff --git a/config/v1/zz_generated.featuregated-crd-manifests/nodes.config.openshift.io/MinimumKubeletVersion.yaml b/config/v1/zz_generated.featuregated-crd-manifests/nodes.config.openshift.io/MinimumKubeletVersion.yaml new file mode 100644 index 00000000000..2a9d9f372a9 --- /dev/null +++ b/config/v1/zz_generated.featuregated-crd-manifests/nodes.config.openshift.io/MinimumKubeletVersion.yaml @@ -0,0 +1,145 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + api-approved.openshift.io: https://github.com/openshift/api/pull/1107 + api.openshift.io/filename-cvo-runlevel: "0000_10" + api.openshift.io/filename-operator: config-operator + api.openshift.io/filename-ordering: "01" + feature-gate.release.openshift.io/MinimumKubeletVersion: "true" + release.openshift.io/bootstrap-required: "true" + name: nodes.config.openshift.io +spec: + group: config.openshift.io + names: + kind: Node + listKind: NodeList + plural: nodes + singular: node + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + description: |- + Node holds cluster-wide information about node specific features. + + Compatibility level 1: Stable within a major release for a minimum of 12 months or 3 minor releases (whichever is longer). + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: spec holds user settable values for configuration + properties: + cgroupMode: + description: CgroupMode determines the cgroups version on the node + enum: + - v1 + - v2 + - "" + type: string + minimumKubeletVersion: + description: |- + MinimumKubeletVersion is the lowest version of a kubelet that can meaningfully join the cluster. + Specifically, the apiserver will deny most authorization requests of kubelets that are older + than the specified version, only allowing the kubelet to get and update its node object, and perform + subjectaccessreviews. + pattern: ^[0-9]*\.[0-9]*\.[0-9]*$ + type: string + workerLatencyProfile: + description: |- + WorkerLatencyProfile determins the how fast the kubelet is updating + the status and corresponding reaction of the cluster + enum: + - Default + - MediumUpdateAverageReaction + - LowUpdateSlowReaction + type: string + type: object + status: + description: status holds observed values. + properties: + conditions: + description: conditions contain the details and the current state + of the nodes.config object + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/features.md b/features.md index ba175a0c146..e7fb01bc075 100644 --- a/features.md +++ b/features.md @@ -29,6 +29,7 @@ | ManagedBootImagesAWS| | | Enabled | Enabled | Enabled | Enabled | | MaxUnavailableStatefulSet| | | Enabled | Enabled | Enabled | Enabled | | MetricsCollectionProfiles| | | Enabled | Enabled | Enabled | Enabled | +| MinimumKubeletVersion| | | Enabled | Enabled | Enabled | Enabled | | MixedCPUsAllocation| | | Enabled | Enabled | Enabled | Enabled | | NetworkSegmentation| | | Enabled | Enabled | Enabled | Enabled | | NewOLM| | | Enabled | Enabled | Enabled | Enabled | diff --git a/features/features.go b/features/features.go index 60dc3182ab8..2eba54e2c53 100644 --- a/features/features.go +++ b/features/features.go @@ -654,4 +654,12 @@ var ( enhancementPR(legacyFeatureGateWithoutEnhancement). enableIn(configv1.DevPreviewNoUpgrade, configv1.TechPreviewNoUpgrade). mustRegister() + + FeatureGateMinimumKubeletVersion = newFeatureGate("MinimumKubeletVersion"). + reportProblemsToJiraComponent("Node"). + contactPerson("haircommander"). + productScope(ocpSpecific). + enableIn(configv1.DevPreviewNoUpgrade, configv1.TechPreviewNoUpgrade). + enhancementPR("https://github.com/openshift/enhancements/pull/1697"). + mustRegister() ) diff --git a/kubecontrolplane/v1/types.go b/kubecontrolplane/v1/types.go index 3fe564ffcc6..31a0a8b7826 100644 --- a/kubecontrolplane/v1/types.go +++ b/kubecontrolplane/v1/types.go @@ -67,6 +67,7 @@ type KubeAPIServerConfig struct { // Specifically, the apiserver will deny most authorization requests of kubelets that are older // than the specified version, only allowing the kubelet to get and update its node object, and perform // subjectaccessreviews. + // +openshift:enable:FeatureGate=MinimumKubeletVersion MinimumKubeletVersion string `json:"minimumKubeletVersion,omitempty"` } diff --git a/payload-manifests/crds/0000_10_config-operator_01_nodes.crd.yaml b/payload-manifests/crds/0000_10_config-operator_01_nodes-CustomNoUpgrade.crd.yaml similarity index 99% rename from payload-manifests/crds/0000_10_config-operator_01_nodes.crd.yaml rename to payload-manifests/crds/0000_10_config-operator_01_nodes-CustomNoUpgrade.crd.yaml index 9d2f140a0a4..121a47ac41a 100644 --- a/payload-manifests/crds/0000_10_config-operator_01_nodes.crd.yaml +++ b/payload-manifests/crds/0000_10_config-operator_01_nodes-CustomNoUpgrade.crd.yaml @@ -7,6 +7,7 @@ metadata: include.release.openshift.io/ibm-cloud-managed: "true" include.release.openshift.io/self-managed-high-availability: "true" release.openshift.io/bootstrap-required: "true" + release.openshift.io/feature-set: CustomNoUpgrade name: nodes.config.openshift.io spec: group: config.openshift.io diff --git a/payload-manifests/crds/0000_10_config-operator_01_nodes-Default.crd.yaml b/payload-manifests/crds/0000_10_config-operator_01_nodes-Default.crd.yaml new file mode 100644 index 00000000000..b79a394c6e1 --- /dev/null +++ b/payload-manifests/crds/0000_10_config-operator_01_nodes-Default.crd.yaml @@ -0,0 +1,137 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + api-approved.openshift.io: https://github.com/openshift/api/pull/1107 + api.openshift.io/merged-by-featuregates: "true" + include.release.openshift.io/ibm-cloud-managed: "true" + include.release.openshift.io/self-managed-high-availability: "true" + release.openshift.io/bootstrap-required: "true" + release.openshift.io/feature-set: Default + name: nodes.config.openshift.io +spec: + group: config.openshift.io + names: + kind: Node + listKind: NodeList + plural: nodes + singular: node + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + description: |- + Node holds cluster-wide information about node specific features. + + Compatibility level 1: Stable within a major release for a minimum of 12 months or 3 minor releases (whichever is longer). + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: spec holds user settable values for configuration + properties: + cgroupMode: + description: CgroupMode determines the cgroups version on the node + enum: + - v1 + - v2 + - "" + type: string + workerLatencyProfile: + description: |- + WorkerLatencyProfile determins the how fast the kubelet is updating + the status and corresponding reaction of the cluster + enum: + - Default + - MediumUpdateAverageReaction + - LowUpdateSlowReaction + type: string + type: object + status: + description: status holds observed values. + properties: + conditions: + description: conditions contain the details and the current state + of the nodes.config object + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/payload-manifests/crds/0000_10_config-operator_01_nodes-DevPreviewNoUpgrade.crd.yaml b/payload-manifests/crds/0000_10_config-operator_01_nodes-DevPreviewNoUpgrade.crd.yaml new file mode 100644 index 00000000000..3b6f33dffaa --- /dev/null +++ b/payload-manifests/crds/0000_10_config-operator_01_nodes-DevPreviewNoUpgrade.crd.yaml @@ -0,0 +1,145 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + api-approved.openshift.io: https://github.com/openshift/api/pull/1107 + api.openshift.io/merged-by-featuregates: "true" + include.release.openshift.io/ibm-cloud-managed: "true" + include.release.openshift.io/self-managed-high-availability: "true" + release.openshift.io/bootstrap-required: "true" + release.openshift.io/feature-set: DevPreviewNoUpgrade + name: nodes.config.openshift.io +spec: + group: config.openshift.io + names: + kind: Node + listKind: NodeList + plural: nodes + singular: node + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + description: |- + Node holds cluster-wide information about node specific features. + + Compatibility level 1: Stable within a major release for a minimum of 12 months or 3 minor releases (whichever is longer). + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: spec holds user settable values for configuration + properties: + cgroupMode: + description: CgroupMode determines the cgroups version on the node + enum: + - v1 + - v2 + - "" + type: string + minimumKubeletVersion: + description: |- + MinimumKubeletVersion is the lowest version of a kubelet that can meaningfully join the cluster. + Specifically, the apiserver will deny most authorization requests of kubelets that are older + than the specified version, only allowing the kubelet to get and update its node object, and perform + subjectaccessreviews. + pattern: ^[0-9]*\.[0-9]*\.[0-9]*$ + type: string + workerLatencyProfile: + description: |- + WorkerLatencyProfile determins the how fast the kubelet is updating + the status and corresponding reaction of the cluster + enum: + - Default + - MediumUpdateAverageReaction + - LowUpdateSlowReaction + type: string + type: object + status: + description: status holds observed values. + properties: + conditions: + description: conditions contain the details and the current state + of the nodes.config object + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/payload-manifests/crds/0000_10_config-operator_01_nodes-TechPreviewNoUpgrade.crd.yaml b/payload-manifests/crds/0000_10_config-operator_01_nodes-TechPreviewNoUpgrade.crd.yaml new file mode 100644 index 00000000000..3e92f0df978 --- /dev/null +++ b/payload-manifests/crds/0000_10_config-operator_01_nodes-TechPreviewNoUpgrade.crd.yaml @@ -0,0 +1,145 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + api-approved.openshift.io: https://github.com/openshift/api/pull/1107 + api.openshift.io/merged-by-featuregates: "true" + include.release.openshift.io/ibm-cloud-managed: "true" + include.release.openshift.io/self-managed-high-availability: "true" + release.openshift.io/bootstrap-required: "true" + release.openshift.io/feature-set: TechPreviewNoUpgrade + name: nodes.config.openshift.io +spec: + group: config.openshift.io + names: + kind: Node + listKind: NodeList + plural: nodes + singular: node + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + description: |- + Node holds cluster-wide information about node specific features. + + Compatibility level 1: Stable within a major release for a minimum of 12 months or 3 minor releases (whichever is longer). + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: spec holds user settable values for configuration + properties: + cgroupMode: + description: CgroupMode determines the cgroups version on the node + enum: + - v1 + - v2 + - "" + type: string + minimumKubeletVersion: + description: |- + MinimumKubeletVersion is the lowest version of a kubelet that can meaningfully join the cluster. + Specifically, the apiserver will deny most authorization requests of kubelets that are older + than the specified version, only allowing the kubelet to get and update its node object, and perform + subjectaccessreviews. + pattern: ^[0-9]*\.[0-9]*\.[0-9]*$ + type: string + workerLatencyProfile: + description: |- + WorkerLatencyProfile determins the how fast the kubelet is updating + the status and corresponding reaction of the cluster + enum: + - Default + - MediumUpdateAverageReaction + - LowUpdateSlowReaction + type: string + type: object + status: + description: status holds observed values. + properties: + conditions: + description: conditions contain the details and the current state + of the nodes.config object + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/payload-manifests/featuregates/featureGate-Hypershift-Default.yaml b/payload-manifests/featuregates/featureGate-Hypershift-Default.yaml index 69580a44def..7b55f2621b0 100644 --- a/payload-manifests/featuregates/featureGate-Hypershift-Default.yaml +++ b/payload-manifests/featuregates/featureGate-Hypershift-Default.yaml @@ -97,6 +97,9 @@ { "name": "MetricsCollectionProfiles" }, + { + "name": "MinimumKubeletVersion" + }, { "name": "MixedCPUsAllocation" }, diff --git a/payload-manifests/featuregates/featureGate-Hypershift-DevPreviewNoUpgrade.yaml b/payload-manifests/featuregates/featureGate-Hypershift-DevPreviewNoUpgrade.yaml index 3194eb13a60..b369d077324 100644 --- a/payload-manifests/featuregates/featureGate-Hypershift-DevPreviewNoUpgrade.yaml +++ b/payload-manifests/featuregates/featureGate-Hypershift-DevPreviewNoUpgrade.yaml @@ -149,6 +149,9 @@ { "name": "MetricsCollectionProfiles" }, + { + "name": "MinimumKubeletVersion" + }, { "name": "MixedCPUsAllocation" }, diff --git a/payload-manifests/featuregates/featureGate-Hypershift-TechPreviewNoUpgrade.yaml b/payload-manifests/featuregates/featureGate-Hypershift-TechPreviewNoUpgrade.yaml index 0a28367e4d5..6d51a62ab78 100644 --- a/payload-manifests/featuregates/featureGate-Hypershift-TechPreviewNoUpgrade.yaml +++ b/payload-manifests/featuregates/featureGate-Hypershift-TechPreviewNoUpgrade.yaml @@ -149,6 +149,9 @@ { "name": "MetricsCollectionProfiles" }, + { + "name": "MinimumKubeletVersion" + }, { "name": "MixedCPUsAllocation" }, diff --git a/payload-manifests/featuregates/featureGate-SelfManagedHA-Default.yaml b/payload-manifests/featuregates/featureGate-SelfManagedHA-Default.yaml index 470103706a9..086b456cf36 100644 --- a/payload-manifests/featuregates/featureGate-SelfManagedHA-Default.yaml +++ b/payload-manifests/featuregates/featureGate-SelfManagedHA-Default.yaml @@ -100,6 +100,9 @@ { "name": "MetricsCollectionProfiles" }, + { + "name": "MinimumKubeletVersion" + }, { "name": "MixedCPUsAllocation" }, diff --git a/payload-manifests/featuregates/featureGate-SelfManagedHA-DevPreviewNoUpgrade.yaml b/payload-manifests/featuregates/featureGate-SelfManagedHA-DevPreviewNoUpgrade.yaml index 3fbc04b059f..8ae7a62f460 100644 --- a/payload-manifests/featuregates/featureGate-SelfManagedHA-DevPreviewNoUpgrade.yaml +++ b/payload-manifests/featuregates/featureGate-SelfManagedHA-DevPreviewNoUpgrade.yaml @@ -149,6 +149,9 @@ { "name": "MetricsCollectionProfiles" }, + { + "name": "MinimumKubeletVersion" + }, { "name": "MixedCPUsAllocation" }, diff --git a/payload-manifests/featuregates/featureGate-SelfManagedHA-TechPreviewNoUpgrade.yaml b/payload-manifests/featuregates/featureGate-SelfManagedHA-TechPreviewNoUpgrade.yaml index ca83b159207..d29348d6199 100644 --- a/payload-manifests/featuregates/featureGate-SelfManagedHA-TechPreviewNoUpgrade.yaml +++ b/payload-manifests/featuregates/featureGate-SelfManagedHA-TechPreviewNoUpgrade.yaml @@ -149,6 +149,9 @@ { "name": "MetricsCollectionProfiles" }, + { + "name": "MinimumKubeletVersion" + }, { "name": "MixedCPUsAllocation" }, From 808d8814f38a1eaa4491c42d62274c6577abe880 Mon Sep 17 00:00:00 2001 From: origin-release-container Date: Mon, 28 Oct 2024 17:52:55 +0000 Subject: [PATCH 4/4] update api validation on minimumKubeletVersion Signed-off-by: Peter Hunt --- .../MinimumKubeletVersion.yaml | 30 +++++++++++++++++++ config/v1/types_node.go | 16 ++++++++-- ...operator_01_nodes-CustomNoUpgrade.crd.yaml | 17 +++++++++-- ...ator_01_nodes-DevPreviewNoUpgrade.crd.yaml | 17 +++++++++-- ...tor_01_nodes-TechPreviewNoUpgrade.crd.yaml | 17 +++++++++-- .../MinimumKubeletVersion.yaml | 17 +++++++++-- .../v1/zz_generated.swagger_doc_generated.go | 2 +- kubecontrolplane/v1/types.go | 16 ++++++++-- .../v1/zz_generated.swagger_doc_generated.go | 2 +- .../generated_openapi/zz_generated.openapi.go | 6 ++-- openapi/openapi.json | 10 ++++--- ...operator_01_nodes-CustomNoUpgrade.crd.yaml | 17 +++++++++-- ...ator_01_nodes-DevPreviewNoUpgrade.crd.yaml | 17 +++++++++-- ...tor_01_nodes-TechPreviewNoUpgrade.crd.yaml | 17 +++++++++-- 14 files changed, 174 insertions(+), 27 deletions(-) diff --git a/config/v1/tests/nodes.config.openshift.io/MinimumKubeletVersion.yaml b/config/v1/tests/nodes.config.openshift.io/MinimumKubeletVersion.yaml index 41ed1e8c072..f57370016b0 100644 --- a/config/v1/tests/nodes.config.openshift.io/MinimumKubeletVersion.yaml +++ b/config/v1/tests/nodes.config.openshift.io/MinimumKubeletVersion.yaml @@ -1,6 +1,7 @@ apiVersion: apiextensions.k8s.io/v1 # Hack because controller-gen complains if we don't have this name: "Node" crdName: nodes.config.openshift.io +featureGate: MinimumKubeletVersion tests: onCreate: - name: Should be able to create a minimal Node @@ -12,3 +13,32 @@ tests: apiVersion: config.openshift.io/v1 kind: Node spec: {} + - name: Should be able to create an empty minimumKubeletVersion + initial: | + apiVersion: config.openshift.io/v1 + kind: Node + spec: + minimumKubeletVersion: "" + expected: | + apiVersion: config.openshift.io/v1 + kind: Node + spec: + minimumKubeletVersion: "" + - name: Should be able to create a minimumKubeletVersion + initial: | + apiVersion: config.openshift.io/v1 + kind: Node + spec: + minimumKubeletVersion: 1.30.0 + expected: | + apiVersion: config.openshift.io/v1 + kind: Node + spec: + minimumKubeletVersion: 1.30.0 + - name: Should fail to create a bogus version + initial: | + apiVersion: config.openshift.io/v1 + kind: Node + spec: + minimumKubeletVersion: bogus + expectedError: "Invalid value: \"string\": minmumKubeletVersion must be in a semver compatible format of x.y.z, or empty" diff --git a/config/v1/types_node.go b/config/v1/types_node.go index 20b536a2bf2..a50328c91f6 100644 --- a/config/v1/types_node.go +++ b/config/v1/types_node.go @@ -47,14 +47,24 @@ type NodeSpec struct { // +optional WorkerLatencyProfile WorkerLatencyProfileType `json:"workerLatencyProfile,omitempty"` - // MinimumKubeletVersion is the lowest version of a kubelet that can meaningfully join the cluster. + // minimumKubeletVersion is the lowest version of a kubelet that can join the cluster. // Specifically, the apiserver will deny most authorization requests of kubelets that are older // than the specified version, only allowing the kubelet to get and update its node object, and perform // subjectaccessreviews. - // +kubebuilder:validation:Pattern=`^[0-9]*\.[0-9]*\.[0-9]*$` + // This means any kubelet that attempts to join the cluster will not be able to run any assigned workloads, + // and will eventually be marked as not ready. + // Its max length is 8, so maximum version allowed is either "9.999.99" or "99.99.99". + // Since the kubelet reports the version of the kubernetes release, not Openshift, this field references + // the underlying kubernetes version this version of Openshift is based off of. + // In other words: if an admin wishes to ensure no nodes run an older version than Openshift 4.17, then + // they should set the minimumKubeletVersion to 1.30.0. + // When comparing versions, the kubelet's version is stripped of any contents outside of major.minor.patch version. + // Thus, a kubelet with version "1.0.0-ec.0" will be compatible with minimumKubeletVersion "1.0.0" or earlier. + // +kubebuilder:validation:XValidation:rule="self == \"\" || self.matches('^[0-9]*.[0-9]*.[0-9]*$')",message="minmumKubeletVersion must be in a semver compatible format of x.y.z, or empty" + // +kubebuilder:validation:MaxLength:=8 // +openshift:enable:FeatureGate=MinimumKubeletVersion // +optional - MinimumKubeletVersion string `json:"minimumKubeletVersion,omitempty"` + MinimumKubeletVersion string `json:"minimumKubeletVersion"` } type NodeStatus struct { diff --git a/config/v1/zz_generated.crd-manifests/0000_10_config-operator_01_nodes-CustomNoUpgrade.crd.yaml b/config/v1/zz_generated.crd-manifests/0000_10_config-operator_01_nodes-CustomNoUpgrade.crd.yaml index 121a47ac41a..469400577ae 100644 --- a/config/v1/zz_generated.crd-manifests/0000_10_config-operator_01_nodes-CustomNoUpgrade.crd.yaml +++ b/config/v1/zz_generated.crd-manifests/0000_10_config-operator_01_nodes-CustomNoUpgrade.crd.yaml @@ -55,12 +55,25 @@ spec: type: string minimumKubeletVersion: description: |- - MinimumKubeletVersion is the lowest version of a kubelet that can meaningfully join the cluster. + minimumKubeletVersion is the lowest version of a kubelet that can join the cluster. Specifically, the apiserver will deny most authorization requests of kubelets that are older than the specified version, only allowing the kubelet to get and update its node object, and perform subjectaccessreviews. - pattern: ^[0-9]*\.[0-9]*\.[0-9]*$ + This means any kubelet that attempts to join the cluster will not be able to run any assigned workloads, + and will eventually be marked as not ready. + Its max length is 8, so maximum version allowed is either "9.999.99" or "99.99.99". + Since the kubelet reports the version of the kubernetes release, not Openshift, this field references + the underlying kubernetes version this version of Openshift is based off of. + In other words: if an admin wishes to ensure no nodes run an older version than Openshift 4.17, then + they should set the minimumKubeletVersion to 1.30.0. + When comparing versions, the kubelet's version is stripped of any contents outside of major.minor.patch version. + Thus, a kubelet with version "1.0.0-ec.0" will be compatible with minimumKubeletVersion "1.0.0" or earlier. + maxLength: 8 type: string + x-kubernetes-validations: + - message: minmumKubeletVersion must be in a semver compatible format + of x.y.z, or empty + rule: self == "" || self.matches('^[0-9]*.[0-9]*.[0-9]*$') workerLatencyProfile: description: |- WorkerLatencyProfile determins the how fast the kubelet is updating diff --git a/config/v1/zz_generated.crd-manifests/0000_10_config-operator_01_nodes-DevPreviewNoUpgrade.crd.yaml b/config/v1/zz_generated.crd-manifests/0000_10_config-operator_01_nodes-DevPreviewNoUpgrade.crd.yaml index 3b6f33dffaa..99b124d5728 100644 --- a/config/v1/zz_generated.crd-manifests/0000_10_config-operator_01_nodes-DevPreviewNoUpgrade.crd.yaml +++ b/config/v1/zz_generated.crd-manifests/0000_10_config-operator_01_nodes-DevPreviewNoUpgrade.crd.yaml @@ -55,12 +55,25 @@ spec: type: string minimumKubeletVersion: description: |- - MinimumKubeletVersion is the lowest version of a kubelet that can meaningfully join the cluster. + minimumKubeletVersion is the lowest version of a kubelet that can join the cluster. Specifically, the apiserver will deny most authorization requests of kubelets that are older than the specified version, only allowing the kubelet to get and update its node object, and perform subjectaccessreviews. - pattern: ^[0-9]*\.[0-9]*\.[0-9]*$ + This means any kubelet that attempts to join the cluster will not be able to run any assigned workloads, + and will eventually be marked as not ready. + Its max length is 8, so maximum version allowed is either "9.999.99" or "99.99.99". + Since the kubelet reports the version of the kubernetes release, not Openshift, this field references + the underlying kubernetes version this version of Openshift is based off of. + In other words: if an admin wishes to ensure no nodes run an older version than Openshift 4.17, then + they should set the minimumKubeletVersion to 1.30.0. + When comparing versions, the kubelet's version is stripped of any contents outside of major.minor.patch version. + Thus, a kubelet with version "1.0.0-ec.0" will be compatible with minimumKubeletVersion "1.0.0" or earlier. + maxLength: 8 type: string + x-kubernetes-validations: + - message: minmumKubeletVersion must be in a semver compatible format + of x.y.z, or empty + rule: self == "" || self.matches('^[0-9]*.[0-9]*.[0-9]*$') workerLatencyProfile: description: |- WorkerLatencyProfile determins the how fast the kubelet is updating diff --git a/config/v1/zz_generated.crd-manifests/0000_10_config-operator_01_nodes-TechPreviewNoUpgrade.crd.yaml b/config/v1/zz_generated.crd-manifests/0000_10_config-operator_01_nodes-TechPreviewNoUpgrade.crd.yaml index 3e92f0df978..8db838df772 100644 --- a/config/v1/zz_generated.crd-manifests/0000_10_config-operator_01_nodes-TechPreviewNoUpgrade.crd.yaml +++ b/config/v1/zz_generated.crd-manifests/0000_10_config-operator_01_nodes-TechPreviewNoUpgrade.crd.yaml @@ -55,12 +55,25 @@ spec: type: string minimumKubeletVersion: description: |- - MinimumKubeletVersion is the lowest version of a kubelet that can meaningfully join the cluster. + minimumKubeletVersion is the lowest version of a kubelet that can join the cluster. Specifically, the apiserver will deny most authorization requests of kubelets that are older than the specified version, only allowing the kubelet to get and update its node object, and perform subjectaccessreviews. - pattern: ^[0-9]*\.[0-9]*\.[0-9]*$ + This means any kubelet that attempts to join the cluster will not be able to run any assigned workloads, + and will eventually be marked as not ready. + Its max length is 8, so maximum version allowed is either "9.999.99" or "99.99.99". + Since the kubelet reports the version of the kubernetes release, not Openshift, this field references + the underlying kubernetes version this version of Openshift is based off of. + In other words: if an admin wishes to ensure no nodes run an older version than Openshift 4.17, then + they should set the minimumKubeletVersion to 1.30.0. + When comparing versions, the kubelet's version is stripped of any contents outside of major.minor.patch version. + Thus, a kubelet with version "1.0.0-ec.0" will be compatible with minimumKubeletVersion "1.0.0" or earlier. + maxLength: 8 type: string + x-kubernetes-validations: + - message: minmumKubeletVersion must be in a semver compatible format + of x.y.z, or empty + rule: self == "" || self.matches('^[0-9]*.[0-9]*.[0-9]*$') workerLatencyProfile: description: |- WorkerLatencyProfile determins the how fast the kubelet is updating diff --git a/config/v1/zz_generated.featuregated-crd-manifests/nodes.config.openshift.io/MinimumKubeletVersion.yaml b/config/v1/zz_generated.featuregated-crd-manifests/nodes.config.openshift.io/MinimumKubeletVersion.yaml index 2a9d9f372a9..ffce7121a41 100644 --- a/config/v1/zz_generated.featuregated-crd-manifests/nodes.config.openshift.io/MinimumKubeletVersion.yaml +++ b/config/v1/zz_generated.featuregated-crd-manifests/nodes.config.openshift.io/MinimumKubeletVersion.yaml @@ -55,12 +55,25 @@ spec: type: string minimumKubeletVersion: description: |- - MinimumKubeletVersion is the lowest version of a kubelet that can meaningfully join the cluster. + minimumKubeletVersion is the lowest version of a kubelet that can join the cluster. Specifically, the apiserver will deny most authorization requests of kubelets that are older than the specified version, only allowing the kubelet to get and update its node object, and perform subjectaccessreviews. - pattern: ^[0-9]*\.[0-9]*\.[0-9]*$ + This means any kubelet that attempts to join the cluster will not be able to run any assigned workloads, + and will eventually be marked as not ready. + Its max length is 8, so maximum version allowed is either "9.999.99" or "99.99.99". + Since the kubelet reports the version of the kubernetes release, not Openshift, this field references + the underlying kubernetes version this version of Openshift is based off of. + In other words: if an admin wishes to ensure no nodes run an older version than Openshift 4.17, then + they should set the minimumKubeletVersion to 1.30.0. + When comparing versions, the kubelet's version is stripped of any contents outside of major.minor.patch version. + Thus, a kubelet with version "1.0.0-ec.0" will be compatible with minimumKubeletVersion "1.0.0" or earlier. + maxLength: 8 type: string + x-kubernetes-validations: + - message: minmumKubeletVersion must be in a semver compatible format + of x.y.z, or empty + rule: self == "" || self.matches('^[0-9]*.[0-9]*.[0-9]*$') workerLatencyProfile: description: |- WorkerLatencyProfile determins the how fast the kubelet is updating diff --git a/config/v1/zz_generated.swagger_doc_generated.go b/config/v1/zz_generated.swagger_doc_generated.go index 2ce5ea99f67..145a7e4c041 100644 --- a/config/v1/zz_generated.swagger_doc_generated.go +++ b/config/v1/zz_generated.swagger_doc_generated.go @@ -2090,7 +2090,7 @@ func (NodeList) SwaggerDoc() map[string]string { var map_NodeSpec = map[string]string{ "cgroupMode": "CgroupMode determines the cgroups version on the node", "workerLatencyProfile": "WorkerLatencyProfile determins the how fast the kubelet is updating the status and corresponding reaction of the cluster", - "minimumKubeletVersion": "MinimumKubeletVersion is the lowest version of a kubelet that can meaningfully join the cluster. Specifically, the apiserver will deny most authorization requests of kubelets that are older than the specified version, only allowing the kubelet to get and update its node object, and perform subjectaccessreviews.", + "minimumKubeletVersion": "minimumKubeletVersion is the lowest version of a kubelet that can join the cluster. Specifically, the apiserver will deny most authorization requests of kubelets that are older than the specified version, only allowing the kubelet to get and update its node object, and perform subjectaccessreviews. This means any kubelet that attempts to join the cluster will not be able to run any assigned workloads, and will eventually be marked as not ready. Its max length is 8, so maximum version allowed is either \"9.999.99\" or \"99.99.99\". Since the kubelet reports the version of the kubernetes release, not Openshift, this field references the underlying kubernetes version this version of Openshift is based off of. In other words: if an admin wishes to ensure no nodes run an older version than Openshift 4.17, then they should set the minimumKubeletVersion to 1.30.0. When comparing versions, the kubelet's version is stripped of any contents outside of major.minor.patch version. Thus, a kubelet with version \"1.0.0-ec.0\" will be compatible with minimumKubeletVersion \"1.0.0\" or earlier.", } func (NodeSpec) SwaggerDoc() map[string]string { diff --git a/kubecontrolplane/v1/types.go b/kubecontrolplane/v1/types.go index 31a0a8b7826..6d29f42e3fc 100644 --- a/kubecontrolplane/v1/types.go +++ b/kubecontrolplane/v1/types.go @@ -63,12 +63,24 @@ type KubeAPIServerConfig struct { // TODO this needs to be removed. APIServerArguments map[string]Arguments `json:"apiServerArguments"` - // MinimumKubeletVersion is the lowest version of a kubelet that can meaningfully join the cluster. + // minimumKubeletVersion is the lowest version of a kubelet that can join the cluster. // Specifically, the apiserver will deny most authorization requests of kubelets that are older // than the specified version, only allowing the kubelet to get and update its node object, and perform // subjectaccessreviews. + // This means any kubelet that attempts to join the cluster will not be able to run any assigned workloads, + // and will eventually be marked as not ready. + // Its max length is 8, so maximum version allowed is either "9.999.99" or "99.99.99". + // Since the kubelet reports the version of the kubernetes release, not Openshift, this field references + // the underlying kubernetes version this version of Openshift is based off of. + // In other words: if an admin wishes to ensure no nodes run an older version than Openshift 4.17, then + // they should set the minimumKubeletVersion to 1.30.0. + // When comparing versions, the kubelet's version is stripped of any contents outside of major.minor.patch version. + // Thus, a kubelet with version "1.0.0-ec.0" will be compatible with minimumKubeletVersion "1.0.0" or earlier. + // +kubebuilder:validation:XValidation:rule="self == \"\" || self.matches('^[0-9]*.[0-9]*.[0-9]*$')",message="minmumKubeletVersion must be in a semver compatible format of x.y.z, or empty" + // +kubebuilder:validation:MaxLength:=8 // +openshift:enable:FeatureGate=MinimumKubeletVersion - MinimumKubeletVersion string `json:"minimumKubeletVersion,omitempty"` + // +optional + MinimumKubeletVersion string `json:"minimumKubeletVersion"` } // Arguments masks the value so protobuf can generate diff --git a/kubecontrolplane/v1/zz_generated.swagger_doc_generated.go b/kubecontrolplane/v1/zz_generated.swagger_doc_generated.go index 75264a69bff..5ecdd058392 100644 --- a/kubecontrolplane/v1/zz_generated.swagger_doc_generated.go +++ b/kubecontrolplane/v1/zz_generated.swagger_doc_generated.go @@ -33,7 +33,7 @@ var map_KubeAPIServerConfig = map[string]string{ "projectConfig": "projectConfig feeds an admission plugin", "serviceAccountPublicKeyFiles": "serviceAccountPublicKeyFiles is a list of files, each containing a PEM-encoded public RSA key. (If any file contains a private key, the public portion of the key is used) The list of public keys is used to verify presented service account tokens. Each key is tried in order until the list is exhausted or verification succeeds. If no keys are specified, no service account authentication will be available.", "oauthConfig": "oauthConfig, if present start the /oauth endpoint in this process", - "minimumKubeletVersion": "MinimumKubeletVersion is the lowest version of a kubelet that can meaningfully join the cluster. Specifically, the apiserver will deny most authorization requests of kubelets that are older than the specified version, only allowing the kubelet to get and update its node object, and perform subjectaccessreviews.", + "minimumKubeletVersion": "minimumKubeletVersion is the lowest version of a kubelet that can join the cluster. Specifically, the apiserver will deny most authorization requests of kubelets that are older than the specified version, only allowing the kubelet to get and update its node object, and perform subjectaccessreviews. This means any kubelet that attempts to join the cluster will not be able to run any assigned workloads, and will eventually be marked as not ready. Its max length is 8, so maximum version allowed is either \"9.999.99\" or \"99.99.99\". Since the kubelet reports the version of the kubernetes release, not Openshift, this field references the underlying kubernetes version this version of Openshift is based off of. In other words: if an admin wishes to ensure no nodes run an older version than Openshift 4.17, then they should set the minimumKubeletVersion to 1.30.0. When comparing versions, the kubelet's version is stripped of any contents outside of major.minor.patch version. Thus, a kubelet with version \"1.0.0-ec.0\" will be compatible with minimumKubeletVersion \"1.0.0\" or earlier.", } func (KubeAPIServerConfig) SwaggerDoc() map[string]string { diff --git a/openapi/generated_openapi/zz_generated.openapi.go b/openapi/generated_openapi/zz_generated.openapi.go index ae21056c5d5..d500ddeee7e 100644 --- a/openapi/generated_openapi/zz_generated.openapi.go +++ b/openapi/generated_openapi/zz_generated.openapi.go @@ -15554,7 +15554,8 @@ func schema_openshift_api_config_v1_NodeSpec(ref common.ReferenceCallback) commo }, "minimumKubeletVersion": { SchemaProps: spec.SchemaProps{ - Description: "MinimumKubeletVersion is the lowest version of a kubelet that can meaningfully join the cluster. Specifically, the apiserver will deny most authorization requests of kubelets that are older than the specified version, only allowing the kubelet to get and update its node object, and perform subjectaccessreviews.", + Description: "minimumKubeletVersion is the lowest version of a kubelet that can join the cluster. Specifically, the apiserver will deny most authorization requests of kubelets that are older than the specified version, only allowing the kubelet to get and update its node object, and perform subjectaccessreviews. This means any kubelet that attempts to join the cluster will not be able to run any assigned workloads, and will eventually be marked as not ready. Its max length is 8, so maximum version allowed is either \"9.999.99\" or \"99.99.99\". Since the kubelet reports the version of the kubernetes release, not Openshift, this field references the underlying kubernetes version this version of Openshift is based off of. In other words: if an admin wishes to ensure no nodes run an older version than Openshift 4.17, then they should set the minimumKubeletVersion to 1.30.0. When comparing versions, the kubelet's version is stripped of any contents outside of major.minor.patch version. Thus, a kubelet with version \"1.0.0-ec.0\" will be compatible with minimumKubeletVersion \"1.0.0\" or earlier.", + Default: "", Type: []string{"string"}, Format: "", }, @@ -26508,7 +26509,8 @@ func schema_openshift_api_kubecontrolplane_v1_KubeAPIServerConfig(ref common.Ref }, "minimumKubeletVersion": { SchemaProps: spec.SchemaProps{ - Description: "MinimumKubeletVersion is the lowest version of a kubelet that can meaningfully join the cluster. Specifically, the apiserver will deny most authorization requests of kubelets that are older than the specified version, only allowing the kubelet to get and update its node object, and perform subjectaccessreviews.", + Description: "minimumKubeletVersion is the lowest version of a kubelet that can join the cluster. Specifically, the apiserver will deny most authorization requests of kubelets that are older than the specified version, only allowing the kubelet to get and update its node object, and perform subjectaccessreviews. This means any kubelet that attempts to join the cluster will not be able to run any assigned workloads, and will eventually be marked as not ready. Its max length is 8, so maximum version allowed is either \"9.999.99\" or \"99.99.99\". Since the kubelet reports the version of the kubernetes release, not Openshift, this field references the underlying kubernetes version this version of Openshift is based off of. In other words: if an admin wishes to ensure no nodes run an older version than Openshift 4.17, then they should set the minimumKubeletVersion to 1.30.0. When comparing versions, the kubelet's version is stripped of any contents outside of major.minor.patch version. Thus, a kubelet with version \"1.0.0-ec.0\" will be compatible with minimumKubeletVersion \"1.0.0\" or earlier.", + Default: "", Type: []string{"string"}, Format: "", }, diff --git a/openapi/openapi.json b/openapi/openapi.json index 19ac8961af7..5decc918e27 100644 --- a/openapi/openapi.json +++ b/openapi/openapi.json @@ -8271,8 +8271,9 @@ "type": "string" }, "minimumKubeletVersion": { - "description": "MinimumKubeletVersion is the lowest version of a kubelet that can meaningfully join the cluster. Specifically, the apiserver will deny most authorization requests of kubelets that are older than the specified version, only allowing the kubelet to get and update its node object, and perform subjectaccessreviews.", - "type": "string" + "description": "minimumKubeletVersion is the lowest version of a kubelet that can join the cluster. Specifically, the apiserver will deny most authorization requests of kubelets that are older than the specified version, only allowing the kubelet to get and update its node object, and perform subjectaccessreviews. This means any kubelet that attempts to join the cluster will not be able to run any assigned workloads, and will eventually be marked as not ready. Its max length is 8, so maximum version allowed is either \"9.999.99\" or \"99.99.99\". Since the kubelet reports the version of the kubernetes release, not Openshift, this field references the underlying kubernetes version this version of Openshift is based off of. In other words: if an admin wishes to ensure no nodes run an older version than Openshift 4.17, then they should set the minimumKubeletVersion to 1.30.0. When comparing versions, the kubelet's version is stripped of any contents outside of major.minor.patch version. Thus, a kubelet with version \"1.0.0-ec.0\" will be compatible with minimumKubeletVersion \"1.0.0\" or earlier.", + "type": "string", + "default": "" }, "workerLatencyProfile": { "description": "WorkerLatencyProfile determins the how fast the kubelet is updating the status and corresponding reaction of the cluster", @@ -14791,8 +14792,9 @@ "$ref": "#/definitions/com.github.openshift.api.kubecontrolplane.v1.KubeletConnectionInfo" }, "minimumKubeletVersion": { - "description": "MinimumKubeletVersion is the lowest version of a kubelet that can meaningfully join the cluster. Specifically, the apiserver will deny most authorization requests of kubelets that are older than the specified version, only allowing the kubelet to get and update its node object, and perform subjectaccessreviews.", - "type": "string" + "description": "minimumKubeletVersion is the lowest version of a kubelet that can join the cluster. Specifically, the apiserver will deny most authorization requests of kubelets that are older than the specified version, only allowing the kubelet to get and update its node object, and perform subjectaccessreviews. This means any kubelet that attempts to join the cluster will not be able to run any assigned workloads, and will eventually be marked as not ready. Its max length is 8, so maximum version allowed is either \"9.999.99\" or \"99.99.99\". Since the kubelet reports the version of the kubernetes release, not Openshift, this field references the underlying kubernetes version this version of Openshift is based off of. In other words: if an admin wishes to ensure no nodes run an older version than Openshift 4.17, then they should set the minimumKubeletVersion to 1.30.0. When comparing versions, the kubelet's version is stripped of any contents outside of major.minor.patch version. Thus, a kubelet with version \"1.0.0-ec.0\" will be compatible with minimumKubeletVersion \"1.0.0\" or earlier.", + "type": "string", + "default": "" }, "oauthConfig": { "description": "oauthConfig, if present start the /oauth endpoint in this process", diff --git a/payload-manifests/crds/0000_10_config-operator_01_nodes-CustomNoUpgrade.crd.yaml b/payload-manifests/crds/0000_10_config-operator_01_nodes-CustomNoUpgrade.crd.yaml index 121a47ac41a..469400577ae 100644 --- a/payload-manifests/crds/0000_10_config-operator_01_nodes-CustomNoUpgrade.crd.yaml +++ b/payload-manifests/crds/0000_10_config-operator_01_nodes-CustomNoUpgrade.crd.yaml @@ -55,12 +55,25 @@ spec: type: string minimumKubeletVersion: description: |- - MinimumKubeletVersion is the lowest version of a kubelet that can meaningfully join the cluster. + minimumKubeletVersion is the lowest version of a kubelet that can join the cluster. Specifically, the apiserver will deny most authorization requests of kubelets that are older than the specified version, only allowing the kubelet to get and update its node object, and perform subjectaccessreviews. - pattern: ^[0-9]*\.[0-9]*\.[0-9]*$ + This means any kubelet that attempts to join the cluster will not be able to run any assigned workloads, + and will eventually be marked as not ready. + Its max length is 8, so maximum version allowed is either "9.999.99" or "99.99.99". + Since the kubelet reports the version of the kubernetes release, not Openshift, this field references + the underlying kubernetes version this version of Openshift is based off of. + In other words: if an admin wishes to ensure no nodes run an older version than Openshift 4.17, then + they should set the minimumKubeletVersion to 1.30.0. + When comparing versions, the kubelet's version is stripped of any contents outside of major.minor.patch version. + Thus, a kubelet with version "1.0.0-ec.0" will be compatible with minimumKubeletVersion "1.0.0" or earlier. + maxLength: 8 type: string + x-kubernetes-validations: + - message: minmumKubeletVersion must be in a semver compatible format + of x.y.z, or empty + rule: self == "" || self.matches('^[0-9]*.[0-9]*.[0-9]*$') workerLatencyProfile: description: |- WorkerLatencyProfile determins the how fast the kubelet is updating diff --git a/payload-manifests/crds/0000_10_config-operator_01_nodes-DevPreviewNoUpgrade.crd.yaml b/payload-manifests/crds/0000_10_config-operator_01_nodes-DevPreviewNoUpgrade.crd.yaml index 3b6f33dffaa..99b124d5728 100644 --- a/payload-manifests/crds/0000_10_config-operator_01_nodes-DevPreviewNoUpgrade.crd.yaml +++ b/payload-manifests/crds/0000_10_config-operator_01_nodes-DevPreviewNoUpgrade.crd.yaml @@ -55,12 +55,25 @@ spec: type: string minimumKubeletVersion: description: |- - MinimumKubeletVersion is the lowest version of a kubelet that can meaningfully join the cluster. + minimumKubeletVersion is the lowest version of a kubelet that can join the cluster. Specifically, the apiserver will deny most authorization requests of kubelets that are older than the specified version, only allowing the kubelet to get and update its node object, and perform subjectaccessreviews. - pattern: ^[0-9]*\.[0-9]*\.[0-9]*$ + This means any kubelet that attempts to join the cluster will not be able to run any assigned workloads, + and will eventually be marked as not ready. + Its max length is 8, so maximum version allowed is either "9.999.99" or "99.99.99". + Since the kubelet reports the version of the kubernetes release, not Openshift, this field references + the underlying kubernetes version this version of Openshift is based off of. + In other words: if an admin wishes to ensure no nodes run an older version than Openshift 4.17, then + they should set the minimumKubeletVersion to 1.30.0. + When comparing versions, the kubelet's version is stripped of any contents outside of major.minor.patch version. + Thus, a kubelet with version "1.0.0-ec.0" will be compatible with minimumKubeletVersion "1.0.0" or earlier. + maxLength: 8 type: string + x-kubernetes-validations: + - message: minmumKubeletVersion must be in a semver compatible format + of x.y.z, or empty + rule: self == "" || self.matches('^[0-9]*.[0-9]*.[0-9]*$') workerLatencyProfile: description: |- WorkerLatencyProfile determins the how fast the kubelet is updating diff --git a/payload-manifests/crds/0000_10_config-operator_01_nodes-TechPreviewNoUpgrade.crd.yaml b/payload-manifests/crds/0000_10_config-operator_01_nodes-TechPreviewNoUpgrade.crd.yaml index 3e92f0df978..8db838df772 100644 --- a/payload-manifests/crds/0000_10_config-operator_01_nodes-TechPreviewNoUpgrade.crd.yaml +++ b/payload-manifests/crds/0000_10_config-operator_01_nodes-TechPreviewNoUpgrade.crd.yaml @@ -55,12 +55,25 @@ spec: type: string minimumKubeletVersion: description: |- - MinimumKubeletVersion is the lowest version of a kubelet that can meaningfully join the cluster. + minimumKubeletVersion is the lowest version of a kubelet that can join the cluster. Specifically, the apiserver will deny most authorization requests of kubelets that are older than the specified version, only allowing the kubelet to get and update its node object, and perform subjectaccessreviews. - pattern: ^[0-9]*\.[0-9]*\.[0-9]*$ + This means any kubelet that attempts to join the cluster will not be able to run any assigned workloads, + and will eventually be marked as not ready. + Its max length is 8, so maximum version allowed is either "9.999.99" or "99.99.99". + Since the kubelet reports the version of the kubernetes release, not Openshift, this field references + the underlying kubernetes version this version of Openshift is based off of. + In other words: if an admin wishes to ensure no nodes run an older version than Openshift 4.17, then + they should set the minimumKubeletVersion to 1.30.0. + When comparing versions, the kubelet's version is stripped of any contents outside of major.minor.patch version. + Thus, a kubelet with version "1.0.0-ec.0" will be compatible with minimumKubeletVersion "1.0.0" or earlier. + maxLength: 8 type: string + x-kubernetes-validations: + - message: minmumKubeletVersion must be in a semver compatible format + of x.y.z, or empty + rule: self == "" || self.matches('^[0-9]*.[0-9]*.[0-9]*$') workerLatencyProfile: description: |- WorkerLatencyProfile determins the how fast the kubelet is updating