From 352fd958011d5f16e21a395f6947f05aeaff4da3 Mon Sep 17 00:00:00 2001 From: Francesco Pantano Date: Tue, 3 Dec 2024 12:18:03 +0100 Subject: [PATCH] Add Topology CRD Topology CRD is made by two different structs imported and abstracted from PodSpec [1]: 1. TopologySpreadConstraint 2. Affinity/AntiAffinity The above seems enough to draft a dedicated CR instead of exposing those parameters through the service operators' API. In addition, Affinity/AntiAffinity is wrapped in lib-common and not imported as is from PodSpec. [1] https://pkg.go.dev/k8s.io/api/core/v1#PodSpec Signed-off-by: Francesco Pantano --- PROJECT | 13 + .../topology.openstack.org_topologies.yaml | 346 ++++++++++++++++++ apis/go.mod | 2 + apis/go.sum | 4 +- apis/redis/v1beta1/redis_webhook.go | 28 +- apis/topology/v1beta1/groupversion_info.go | 36 ++ apis/topology/v1beta1/topology_types.go | 104 ++++++ .../topology/v1beta1/zz_generated.deepcopy.go | 141 +++++++ .../topology.openstack.org_topologies.yaml | 346 ++++++++++++++++++ config/crd/kustomization.yaml | 3 + .../cainjection_in_topology_topologies.yaml | 7 + .../webhook_in_topology_topologies.yaml | 16 + config/rbac/role.yaml | 27 ++ .../rbac/topology_topology_editor_role.yaml | 31 ++ .../rbac/topology_topology_viewer_role.yaml | 27 ++ config/samples/kustomization.yaml | 1 + config/samples/topology_v1beta1_topology.yaml | 12 + controllers/network/dnsmasq_controller.go | 5 +- controllers/topology/suite_test.go | 80 ++++ controllers/topology/topology_controller.go | 189 ++++++++++ go.mod | 2 + go.sum | 4 +- main.go | 18 + pkg/dnsmasq/deployment.go | 8 +- pkg/memcached/statefulset.go | 3 +- 25 files changed, 1430 insertions(+), 23 deletions(-) create mode 100644 apis/bases/topology.openstack.org_topologies.yaml create mode 100644 apis/topology/v1beta1/groupversion_info.go create mode 100644 apis/topology/v1beta1/topology_types.go create mode 100644 apis/topology/v1beta1/zz_generated.deepcopy.go create mode 100644 config/crd/bases/topology.openstack.org_topologies.yaml create mode 100644 config/crd/patches/cainjection_in_topology_topologies.yaml create mode 100644 config/crd/patches/webhook_in_topology_topologies.yaml create mode 100644 config/rbac/topology_topology_editor_role.yaml create mode 100644 config/rbac/topology_topology_viewer_role.yaml create mode 100644 config/samples/topology_v1beta1_topology.yaml create mode 100644 controllers/topology/suite_test.go create mode 100644 controllers/topology/topology_controller.go diff --git a/PROJECT b/PROJECT index 4a77a99c..7e0651f3 100644 --- a/PROJECT +++ b/PROJECT @@ -1,3 +1,7 @@ +# Code generated by tool. DO NOT EDIT. +# This file is used to track the info used to scaffold your project +# and allow the plugins properly work. +# More info: https://book.kubebuilder.io/reference/project-config.html domain: openstack.org layout: - go.kubebuilder.io/v3 @@ -116,4 +120,13 @@ resources: defaulting: true validation: true webhookVersion: v1 +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: openstack.org + group: topology + kind: Topology + path: github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1 + version: v1beta1 version: "3" diff --git a/apis/bases/topology.openstack.org_topologies.yaml b/apis/bases/topology.openstack.org_topologies.yaml new file mode 100644 index 00000000..ad3ee777 --- /dev/null +++ b/apis/bases/topology.openstack.org_topologies.yaml @@ -0,0 +1,346 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.11.1 + creationTimestamp: null + name: topologies.topology.openstack.org +spec: + group: topology.openstack.org + names: + kind: Topology + listKind: TopologyList + plural: topologies + singular: topology + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: Topology is the Schema for the topologies API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: TopologySpec defines the desired state of Topology + properties: + affinity: + description: PodScheduling - + properties: + preferred: + description: Rules - + properties: + selectorKey: + type: string + selectorValues: + items: + type: string + type: array + topologyKey: + description: https://github.com/kubernetes/api/blob/master/core/v1/well_known_labels.go#L20 + type: string + weight: + format: int32 + type: integer + type: object + required: + description: Rules - + properties: + selectorKey: + type: string + selectorValues: + items: + type: string + type: array + topologyKey: + description: https://github.com/kubernetes/api/blob/master/core/v1/well_known_labels.go#L20 + type: string + weight: + format: int32 + type: integer + type: object + type: object + antiAffinity: + description: PodScheduling - + properties: + preferred: + description: Rules - + properties: + selectorKey: + type: string + selectorValues: + items: + type: string + type: array + topologyKey: + description: https://github.com/kubernetes/api/blob/master/core/v1/well_known_labels.go#L20 + type: string + weight: + format: int32 + type: integer + type: object + required: + description: Rules - + properties: + selectorKey: + type: string + selectorValues: + items: + type: string + type: array + topologyKey: + description: https://github.com/kubernetes/api/blob/master/core/v1/well_known_labels.go#L20 + type: string + weight: + format: int32 + type: integer + type: object + type: object + topologySpreadConstraint: + description: APITopologySpreadConstraint exposes topologySpreadConstraint + that are applied to the StatefulSet + items: + description: TopologySpreadConstraint specifies how to spread matching + pods among the given topology. + properties: + labelSelector: + description: LabelSelector is used to find matching pods. Pods + that match this label selector are counted to determine the + number of pods in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values array + must be non-empty. If the operator is Exists or + DoesNotExist, the values array must be empty. This + array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field is + "key", the operator is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: "MatchLabelKeys is a set of pod label keys to select + the pods over which spreading will be calculated. The keys + are used to lookup values from the incoming pod labels, those + key-value labels are ANDed with labelSelector to select the + group of existing pods over which spreading will be calculated + for the incoming pod. The same key is forbidden to exist in + both MatchLabelKeys and LabelSelector. MatchLabelKeys cannot + be set when LabelSelector isn't set. Keys that don't exist + in the incoming pod labels will be ignored. A null or empty + list means only match against labelSelector. \n This is a + beta field and requires the MatchLabelKeysInPodTopologySpread + feature gate to be enabled (enabled by default)." + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: 'MaxSkew describes the degree to which pods may + be unevenly distributed. When `whenUnsatisfiable=DoNotSchedule`, + it is the maximum permitted difference between the number + of matching pods in the target topology and the global minimum. + The global minimum is the minimum number of matching pods + in an eligible domain or zero if the number of eligible domains + is less than MinDomains. For example, in a 3-zone cluster, + MaxSkew is set to 1, and pods with the same labelSelector + spread as 2/2/1: In this case, the global minimum is 1. | + zone1 | zone2 | zone3 | | P P | P P | P | - if MaxSkew + is 1, incoming pod can only be scheduled to zone3 to become + 2/2/2; scheduling it onto zone1(zone2) would make the ActualSkew(3-1) + on zone1(zone2) violate MaxSkew(1). - if MaxSkew is 2, incoming + pod can be scheduled onto any zone. When `whenUnsatisfiable=ScheduleAnyway`, + it is used to give higher precedence to topologies that satisfy + it. It''s a required field. Default value is 1 and 0 is not + allowed.' + format: int32 + type: integer + minDomains: + description: "MinDomains indicates a minimum number of eligible + domains. When the number of eligible domains with matching + topology keys is less than minDomains, Pod Topology Spread + treats \"global minimum\" as 0, and then the calculation of + Skew is performed. And when the number of eligible domains + with matching topology keys equals or greater than minDomains, + this value has no effect on scheduling. As a result, when + the number of eligible domains is less than minDomains, scheduler + won't schedule more than maxSkew Pods to those domains. If + value is nil, the constraint behaves as if MinDomains is equal + to 1. Valid values are integers greater than 0. When value + is not nil, WhenUnsatisfiable must be DoNotSchedule. \n For + example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains + is set to 5 and pods with the same labelSelector spread as + 2/2/2: | zone1 | zone2 | zone3 | | P P | P P | P P | + The number of domains is less than 5(MinDomains), so \"global + minimum\" is treated as 0. In this situation, new pod with + the same labelSelector cannot be scheduled, because computed + skew will be 3(3 - 0) if new Pod is scheduled to any of the + three zones, it will violate MaxSkew. \n This is a beta field + and requires the MinDomainsInPodTopologySpread feature gate + to be enabled (enabled by default)." + format: int32 + type: integer + nodeAffinityPolicy: + description: "NodeAffinityPolicy indicates how we will treat + Pod's nodeAffinity/nodeSelector when calculating pod topology + spread skew. Options are: - Honor: only nodes matching nodeAffinity/nodeSelector + are included in the calculations. - Ignore: nodeAffinity/nodeSelector + are ignored. All nodes are included in the calculations. \n + If this value is nil, the behavior is equivalent to the Honor + policy. This is a beta-level feature default enabled by the + NodeInclusionPolicyInPodTopologySpread feature flag." + type: string + nodeTaintsPolicy: + description: "NodeTaintsPolicy indicates how we will treat node + taints when calculating pod topology spread skew. Options + are: - Honor: nodes without taints, along with tainted nodes + for which the incoming pod has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. + \n If this value is nil, the behavior is equivalent to the + Ignore policy. This is a beta-level feature default enabled + by the NodeInclusionPolicyInPodTopologySpread feature flag." + type: string + topologyKey: + description: TopologyKey is the key of node labels. Nodes that + have a label with this key and identical values are considered + to be in the same topology. We consider each + as a "bucket", and try to put balanced number of pods into + each bucket. We define a domain as a particular instance of + a topology. Also, we define an eligible domain as a domain + whose nodes meet the requirements of nodeAffinityPolicy and + nodeTaintsPolicy. e.g. If TopologyKey is "kubernetes.io/hostname", + each Node is a domain of that topology. And, if TopologyKey + is "topology.kubernetes.io/zone", each zone is a domain of + that topology. It's a required field. + type: string + whenUnsatisfiable: + description: 'WhenUnsatisfiable indicates how to deal with a + pod if it doesn''t satisfy the spread constraint. - DoNotSchedule + (default) tells the scheduler not to schedule it. - ScheduleAnyway + tells the scheduler to schedule the pod in any location, but + giving higher precedence to topologies that would help reduce + the skew. A constraint is considered "Unsatisfiable" for an + incoming pod if and only if every possible node assignment + for that pod would violate "MaxSkew" on some topology. For + example, in a 3-zone cluster, MaxSkew is set to 1, and pods + with the same labelSelector spread as 3/1/1: | zone1 | zone2 + | zone3 | | P P P | P | P | If WhenUnsatisfiable is + set to DoNotSchedule, incoming pod can only be scheduled to + zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on + zone2(zone3) satisfies MaxSkew(1). In other words, the cluster + can still be imbalanced, but scheduler won''t make it *more* + imbalanced. It''s a required field.' + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + type: object + status: + description: TopologyStatus defines the observed state of Topology + properties: + conditions: + description: Conditions + items: + description: Condition defines an observation of a API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. + type: string + severity: + description: Severity provides a classification of Reason code, + so the current situation is immediately understandable and + could act accordingly. It is meant for situations where Status=False + and it should be indicated if it is just informational, warning + (next reconciliation might fix it) or an error (e.g. DB create + issue and no actions to automatically resolve the issue can/should + be done). For conditions where Status=Unknown or Status=True + the Severity should be SeverityNone. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + hash: + description: Hash of the topology configuration + type: string + observedGeneration: + description: ObservedGeneration - the most recent generation observed + for this service. If the observed generation is less than the spec + generation, then the controller has not processed the latest changes + injected by the opentack-operator in the top-level CR (e.g. the + ContainerImage) + format: int64 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/apis/go.mod b/apis/go.mod index d52171a7..8a8099ba 100644 --- a/apis/go.mod +++ b/apis/go.mod @@ -76,3 +76,5 @@ require ( ) replace github.com/openshift/api => github.com/openshift/api v0.0.0-20240830023148-b7d0481c9094 //allow-merging + +replace github.com/openstack-k8s-operators/lib-common/modules/common => github.com/fmount/lib-common/modules/common v0.0.0-20241203102750-0b9fe14de0b0 diff --git a/apis/go.sum b/apis/go.sum index dcd73ec3..53e450d0 100644 --- a/apis/go.sum +++ b/apis/go.sum @@ -12,6 +12,8 @@ github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= +github.com/fmount/lib-common/modules/common v0.0.0-20241203102750-0b9fe14de0b0 h1:Agq7NNy3L6PBgGKPfS0tRNRAnT5lNQ/OrbvE+6gF6Ag= +github.com/fmount/lib-common/modules/common v0.0.0-20241203102750-0b9fe14de0b0/go.mod h1:YpNTuJhDWhbXM50O3qBkhO7M+OOyRmWkNVmJ4y3cyFs= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= @@ -76,8 +78,6 @@ github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= github.com/openshift/api v0.0.0-20240830023148-b7d0481c9094 h1:J1wuGhVxpsHykZBa6Beb1gQ96Ptej9AE/BvwCBiRj1E= github.com/openshift/api v0.0.0-20240830023148-b7d0481c9094/go.mod h1:CxgbWAlvu2iQB0UmKTtRu1YfepRg1/vJ64n2DlIEVz4= -github.com/openstack-k8s-operators/lib-common/modules/common v0.5.1-0.20241114091812-6dc9fd0961dc h1:Ufa/q/nC9wmKblvsc0kJppsXHOJoY4fbUamb3ItWCOk= -github.com/openstack-k8s-operators/lib-common/modules/common v0.5.1-0.20241114091812-6dc9fd0961dc/go.mod h1:YpNTuJhDWhbXM50O3qBkhO7M+OOyRmWkNVmJ4y3cyFs= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= diff --git a/apis/redis/v1beta1/redis_webhook.go b/apis/redis/v1beta1/redis_webhook.go index ec14dcd6..72a33bb7 100644 --- a/apis/redis/v1beta1/redis_webhook.go +++ b/apis/redis/v1beta1/redis_webhook.go @@ -25,8 +25,8 @@ package v1beta1 import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/validation/field" ctrl "sigs.k8s.io/controller-runtime" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/webhook" @@ -91,21 +91,21 @@ var _ webhook.Validator = &Redis{} func (r *Redis) ValidateCreate() (admission.Warnings, error) { redislog.Info("validate create", "name", r.Name) - var allErrs field.ErrorList - var allWarn []string + var allErrs field.ErrorList + var allWarn []string - allErrs = common_webhook.ValidateDNS1123Label( - field.NewPath("metadata").Child("name"), - []string{r.Name}, - CrMaxLengthCorrection) // omit issue with statefulset pod label "controller-revision-hash": "-" + allErrs = common_webhook.ValidateDNS1123Label( + field.NewPath("metadata").Child("name"), + []string{r.Name}, + CrMaxLengthCorrection) // omit issue with statefulset pod label "controller-revision-hash": "-" - if len(allErrs) != 0 { - return allWarn, apierrors.NewInvalid( - schema.GroupKind{Group: "redis.openstack.org", Kind: "Redis"}, - r.Name, allErrs) - } + if len(allErrs) != 0 { + return allWarn, apierrors.NewInvalid( + schema.GroupKind{Group: "redis.openstack.org", Kind: "Redis"}, + r.Name, allErrs) + } - return allWarn, nil + return allWarn, nil } diff --git a/apis/topology/v1beta1/groupversion_info.go b/apis/topology/v1beta1/groupversion_info.go new file mode 100644 index 00000000..fc7b91a7 --- /dev/null +++ b/apis/topology/v1beta1/groupversion_info.go @@ -0,0 +1,36 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package v1beta1 contains API Schema definitions for the topology v1beta1 API group +// +kubebuilder:object:generate=true +// +groupName=topology.openstack.org +package v1beta1 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // GroupVersion is group version used to register these objects + GroupVersion = schema.GroupVersion{Group: "topology.openstack.org", Version: "v1beta1"} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) diff --git a/apis/topology/v1beta1/topology_types.go b/apis/topology/v1beta1/topology_types.go new file mode 100644 index 00000000..022d88e2 --- /dev/null +++ b/apis/topology/v1beta1/topology_types.go @@ -0,0 +1,104 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "context" + "github.com/openstack-k8s-operators/lib-common/modules/common/affinity" + "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/helper" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +// TopologySpec defines the desired state of Topology +type TopologySpec struct { + // +kubebuilder:validation:Optional + // APITopologySpreadConstraint exposes topologySpreadConstraint that are + // applied to the StatefulSet + TopologySpreadConstraint *[]corev1.TopologySpreadConstraint `json:"topologySpreadConstraint,omitempty"` + + // APIAffinity exposes PodAffinity and PodAntiaffinity overrides that are applied + // to the StatefulSet + // +optional + APIAffinity *affinity.Overrides `json:",inline"` + + //TODO: We could add NodeSelector here as it belongs to the same APIGroup +} + +// TopologyStatus defines the observed state of Topology +type TopologyStatus struct { + // Conditions + Conditions condition.Conditions `json:"conditions,omitempty" optional:"true"` + + // Hash of the topology configuration + Hash string `json:"hash,omitempty"` + + // ObservedGeneration - the most recent generation observed for this + // service. If the observed generation is less than the spec generation, + // then the controller has not processed the latest changes injected by + // the opentack-operator in the top-level CR (e.g. the ContainerImage) + ObservedGeneration int64 `json:"observedGeneration,omitempty"` +} + +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status + +// Topology is the Schema for the topologies API +type Topology struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec TopologySpec `json:"spec,omitempty"` + Status TopologyStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// TopologyList contains a list of Topology +type TopologyList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []Topology `json:"items"` +} + +func init() { + SchemeBuilder.Register(&Topology{}, &TopologyList{}) +} + +// IsReady returns true if TopologySpreadConstraint reconciled successfully +func (instance Topology) IsReady() bool { + return instance.Status.Conditions.IsTrue(condition.ReadyCondition) +} + +// GetTopologyByName - a function exposed to the service operators +// that need to retrieve the referenced topology by name +func GetTopologyByName( + ctx context.Context, + h *helper.Helper, + name string, + namespace string, +) (*Topology, string, error) { + + topology := &Topology{} + err := h.GetClient().Get(ctx, types.NamespacedName{Name: name, Namespace: namespace}, topology) + if err != nil { + return topology, "", err + } + return topology, topology.Status.Hash, nil +} diff --git a/apis/topology/v1beta1/zz_generated.deepcopy.go b/apis/topology/v1beta1/zz_generated.deepcopy.go new file mode 100644 index 00000000..40bfa7ce --- /dev/null +++ b/apis/topology/v1beta1/zz_generated.deepcopy.go @@ -0,0 +1,141 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package v1beta1 + +import ( + "github.com/openstack-k8s-operators/lib-common/modules/common/affinity" + "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "k8s.io/api/core/v1" + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Topology) DeepCopyInto(out *Topology) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Topology. +func (in *Topology) DeepCopy() *Topology { + if in == nil { + return nil + } + out := new(Topology) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Topology) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TopologyList) DeepCopyInto(out *TopologyList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Topology, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TopologyList. +func (in *TopologyList) DeepCopy() *TopologyList { + if in == nil { + return nil + } + out := new(TopologyList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *TopologyList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TopologySpec) DeepCopyInto(out *TopologySpec) { + *out = *in + if in.TopologySpreadConstraint != nil { + in, out := &in.TopologySpreadConstraint, &out.TopologySpreadConstraint + *out = new([]v1.TopologySpreadConstraint) + if **in != nil { + in, out := *in, *out + *out = make([]v1.TopologySpreadConstraint, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + } + if in.APIAffinity != nil { + in, out := &in.APIAffinity, &out.APIAffinity + *out = new(affinity.Overrides) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TopologySpec. +func (in *TopologySpec) DeepCopy() *TopologySpec { + if in == nil { + return nil + } + out := new(TopologySpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TopologyStatus) DeepCopyInto(out *TopologyStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make(condition.Conditions, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TopologyStatus. +func (in *TopologyStatus) DeepCopy() *TopologyStatus { + if in == nil { + return nil + } + out := new(TopologyStatus) + in.DeepCopyInto(out) + return out +} diff --git a/config/crd/bases/topology.openstack.org_topologies.yaml b/config/crd/bases/topology.openstack.org_topologies.yaml new file mode 100644 index 00000000..ad3ee777 --- /dev/null +++ b/config/crd/bases/topology.openstack.org_topologies.yaml @@ -0,0 +1,346 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.11.1 + creationTimestamp: null + name: topologies.topology.openstack.org +spec: + group: topology.openstack.org + names: + kind: Topology + listKind: TopologyList + plural: topologies + singular: topology + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: Topology is the Schema for the topologies API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: TopologySpec defines the desired state of Topology + properties: + affinity: + description: PodScheduling - + properties: + preferred: + description: Rules - + properties: + selectorKey: + type: string + selectorValues: + items: + type: string + type: array + topologyKey: + description: https://github.com/kubernetes/api/blob/master/core/v1/well_known_labels.go#L20 + type: string + weight: + format: int32 + type: integer + type: object + required: + description: Rules - + properties: + selectorKey: + type: string + selectorValues: + items: + type: string + type: array + topologyKey: + description: https://github.com/kubernetes/api/blob/master/core/v1/well_known_labels.go#L20 + type: string + weight: + format: int32 + type: integer + type: object + type: object + antiAffinity: + description: PodScheduling - + properties: + preferred: + description: Rules - + properties: + selectorKey: + type: string + selectorValues: + items: + type: string + type: array + topologyKey: + description: https://github.com/kubernetes/api/blob/master/core/v1/well_known_labels.go#L20 + type: string + weight: + format: int32 + type: integer + type: object + required: + description: Rules - + properties: + selectorKey: + type: string + selectorValues: + items: + type: string + type: array + topologyKey: + description: https://github.com/kubernetes/api/blob/master/core/v1/well_known_labels.go#L20 + type: string + weight: + format: int32 + type: integer + type: object + type: object + topologySpreadConstraint: + description: APITopologySpreadConstraint exposes topologySpreadConstraint + that are applied to the StatefulSet + items: + description: TopologySpreadConstraint specifies how to spread matching + pods among the given topology. + properties: + labelSelector: + description: LabelSelector is used to find matching pods. Pods + that match this label selector are counted to determine the + number of pods in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values array + must be non-empty. If the operator is Exists or + DoesNotExist, the values array must be empty. This + array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field is + "key", the operator is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: "MatchLabelKeys is a set of pod label keys to select + the pods over which spreading will be calculated. The keys + are used to lookup values from the incoming pod labels, those + key-value labels are ANDed with labelSelector to select the + group of existing pods over which spreading will be calculated + for the incoming pod. The same key is forbidden to exist in + both MatchLabelKeys and LabelSelector. MatchLabelKeys cannot + be set when LabelSelector isn't set. Keys that don't exist + in the incoming pod labels will be ignored. A null or empty + list means only match against labelSelector. \n This is a + beta field and requires the MatchLabelKeysInPodTopologySpread + feature gate to be enabled (enabled by default)." + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: 'MaxSkew describes the degree to which pods may + be unevenly distributed. When `whenUnsatisfiable=DoNotSchedule`, + it is the maximum permitted difference between the number + of matching pods in the target topology and the global minimum. + The global minimum is the minimum number of matching pods + in an eligible domain or zero if the number of eligible domains + is less than MinDomains. For example, in a 3-zone cluster, + MaxSkew is set to 1, and pods with the same labelSelector + spread as 2/2/1: In this case, the global minimum is 1. | + zone1 | zone2 | zone3 | | P P | P P | P | - if MaxSkew + is 1, incoming pod can only be scheduled to zone3 to become + 2/2/2; scheduling it onto zone1(zone2) would make the ActualSkew(3-1) + on zone1(zone2) violate MaxSkew(1). - if MaxSkew is 2, incoming + pod can be scheduled onto any zone. When `whenUnsatisfiable=ScheduleAnyway`, + it is used to give higher precedence to topologies that satisfy + it. It''s a required field. Default value is 1 and 0 is not + allowed.' + format: int32 + type: integer + minDomains: + description: "MinDomains indicates a minimum number of eligible + domains. When the number of eligible domains with matching + topology keys is less than minDomains, Pod Topology Spread + treats \"global minimum\" as 0, and then the calculation of + Skew is performed. And when the number of eligible domains + with matching topology keys equals or greater than minDomains, + this value has no effect on scheduling. As a result, when + the number of eligible domains is less than minDomains, scheduler + won't schedule more than maxSkew Pods to those domains. If + value is nil, the constraint behaves as if MinDomains is equal + to 1. Valid values are integers greater than 0. When value + is not nil, WhenUnsatisfiable must be DoNotSchedule. \n For + example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains + is set to 5 and pods with the same labelSelector spread as + 2/2/2: | zone1 | zone2 | zone3 | | P P | P P | P P | + The number of domains is less than 5(MinDomains), so \"global + minimum\" is treated as 0. In this situation, new pod with + the same labelSelector cannot be scheduled, because computed + skew will be 3(3 - 0) if new Pod is scheduled to any of the + three zones, it will violate MaxSkew. \n This is a beta field + and requires the MinDomainsInPodTopologySpread feature gate + to be enabled (enabled by default)." + format: int32 + type: integer + nodeAffinityPolicy: + description: "NodeAffinityPolicy indicates how we will treat + Pod's nodeAffinity/nodeSelector when calculating pod topology + spread skew. Options are: - Honor: only nodes matching nodeAffinity/nodeSelector + are included in the calculations. - Ignore: nodeAffinity/nodeSelector + are ignored. All nodes are included in the calculations. \n + If this value is nil, the behavior is equivalent to the Honor + policy. This is a beta-level feature default enabled by the + NodeInclusionPolicyInPodTopologySpread feature flag." + type: string + nodeTaintsPolicy: + description: "NodeTaintsPolicy indicates how we will treat node + taints when calculating pod topology spread skew. Options + are: - Honor: nodes without taints, along with tainted nodes + for which the incoming pod has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. + \n If this value is nil, the behavior is equivalent to the + Ignore policy. This is a beta-level feature default enabled + by the NodeInclusionPolicyInPodTopologySpread feature flag." + type: string + topologyKey: + description: TopologyKey is the key of node labels. Nodes that + have a label with this key and identical values are considered + to be in the same topology. We consider each + as a "bucket", and try to put balanced number of pods into + each bucket. We define a domain as a particular instance of + a topology. Also, we define an eligible domain as a domain + whose nodes meet the requirements of nodeAffinityPolicy and + nodeTaintsPolicy. e.g. If TopologyKey is "kubernetes.io/hostname", + each Node is a domain of that topology. And, if TopologyKey + is "topology.kubernetes.io/zone", each zone is a domain of + that topology. It's a required field. + type: string + whenUnsatisfiable: + description: 'WhenUnsatisfiable indicates how to deal with a + pod if it doesn''t satisfy the spread constraint. - DoNotSchedule + (default) tells the scheduler not to schedule it. - ScheduleAnyway + tells the scheduler to schedule the pod in any location, but + giving higher precedence to topologies that would help reduce + the skew. A constraint is considered "Unsatisfiable" for an + incoming pod if and only if every possible node assignment + for that pod would violate "MaxSkew" on some topology. For + example, in a 3-zone cluster, MaxSkew is set to 1, and pods + with the same labelSelector spread as 3/1/1: | zone1 | zone2 + | zone3 | | P P P | P | P | If WhenUnsatisfiable is + set to DoNotSchedule, incoming pod can only be scheduled to + zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on + zone2(zone3) satisfies MaxSkew(1). In other words, the cluster + can still be imbalanced, but scheduler won''t make it *more* + imbalanced. It''s a required field.' + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + type: object + status: + description: TopologyStatus defines the observed state of Topology + properties: + conditions: + description: Conditions + items: + description: Condition defines an observation of a API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. + type: string + severity: + description: Severity provides a classification of Reason code, + so the current situation is immediately understandable and + could act accordingly. It is meant for situations where Status=False + and it should be indicated if it is just informational, warning + (next reconciliation might fix it) or an error (e.g. DB create + issue and no actions to automatically resolve the issue can/should + be done). For conditions where Status=Unknown or Status=True + the Severity should be SeverityNone. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + hash: + description: Hash of the topology configuration + type: string + observedGeneration: + description: ObservedGeneration - the most recent generation observed + for this service. If the observed generation is less than the spec + generation, then the controller has not processed the latest changes + injected by the opentack-operator in the top-level CR (e.g. the + ContainerImage) + format: int64 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 7a56abf6..12cbc7c1 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -11,6 +11,7 @@ resources: - bases/network.openstack.org_netconfigs.yaml - bases/network.openstack.org_ipsets.yaml - bases/network.openstack.org_reservations.yaml +- bases/topology.openstack.org_topologies.yaml #+kubebuilder:scaffold:crdkustomizeresource patchesStrategicMerge: @@ -24,6 +25,7 @@ patchesStrategicMerge: #- patches/webhook_in_netconfigs.yaml #- patches/webhook_in_reservations.yaml #- patches/webhook_in_ipsets.yaml +#- patches/webhook_in_topologies.yaml #+kubebuilder:scaffold:crdkustomizewebhookpatch # [CERTMANAGER] To enable cert-manager, uncomment all the sections with [CERTMANAGER] prefix. @@ -36,6 +38,7 @@ patchesStrategicMerge: #- patches/cainjection_in_netconfigs.yaml #- patches/cainjection_in_reservations.yaml #- patches/cainjection_in_ipsets.yaml +#- patches/cainjection_in_topologies.yaml #+kubebuilder:scaffold:crdkustomizecainjectionpatch # the following config is for teaching kustomize how to do kustomization for CRDs. diff --git a/config/crd/patches/cainjection_in_topology_topologies.yaml b/config/crd/patches/cainjection_in_topology_topologies.yaml new file mode 100644 index 00000000..e38d7aff --- /dev/null +++ b/config/crd/patches/cainjection_in_topology_topologies.yaml @@ -0,0 +1,7 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + name: topologies.topology.openstack.org diff --git a/config/crd/patches/webhook_in_topology_topologies.yaml b/config/crd/patches/webhook_in_topology_topologies.yaml new file mode 100644 index 00000000..e72222f8 --- /dev/null +++ b/config/crd/patches/webhook_in_topology_topologies.yaml @@ -0,0 +1,16 @@ +# The following patch enables a conversion webhook for the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: topologies.topology.openstack.org +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + namespace: system + name: webhook-service + path: /convert + conversionReviewVersions: + - v1 diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 4cfd5180..c3d4a026 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -394,3 +394,30 @@ rules: - securitycontextconstraints verbs: - use +- apiGroups: + - topology.openstack.org + resources: + - topologies + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - topology.openstack.org + resources: + - topologies/finalizers + verbs: + - patch + - update +- apiGroups: + - topology.openstack.org + resources: + - topologies/status + verbs: + - get + - patch + - update diff --git a/config/rbac/topology_topology_editor_role.yaml b/config/rbac/topology_topology_editor_role.yaml new file mode 100644 index 00000000..bf0ae185 --- /dev/null +++ b/config/rbac/topology_topology_editor_role.yaml @@ -0,0 +1,31 @@ +# permissions for end users to edit topologies. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: topology-editor-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: infra-operator + app.kubernetes.io/part-of: infra-operator + app.kubernetes.io/managed-by: kustomize + name: topology-editor-role +rules: +- apiGroups: + - topology.openstack.org + resources: + - topologies + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - topology.openstack.org + resources: + - topologies/status + verbs: + - get diff --git a/config/rbac/topology_topology_viewer_role.yaml b/config/rbac/topology_topology_viewer_role.yaml new file mode 100644 index 00000000..fff61412 --- /dev/null +++ b/config/rbac/topology_topology_viewer_role.yaml @@ -0,0 +1,27 @@ +# permissions for end users to view topologies. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: topology-viewer-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: infra-operator + app.kubernetes.io/part-of: infra-operator + app.kubernetes.io/managed-by: kustomize + name: topology-viewer-role +rules: +- apiGroups: + - topology.openstack.org + resources: + - topologies + verbs: + - get + - list + - watch +- apiGroups: + - topology.openstack.org + resources: + - topologies/status + verbs: + - get diff --git a/config/samples/kustomization.yaml b/config/samples/kustomization.yaml index 7f7b6ecc..547b33c6 100644 --- a/config/samples/kustomization.yaml +++ b/config/samples/kustomization.yaml @@ -9,4 +9,5 @@ resources: - network_v1beta1_netconfig.yaml - network_v1beta1_ipset.yaml - network_v1beta1_reservation.yaml +- topology_v1beta1_topology.yaml #+kubebuilder:scaffold:manifestskustomizesamples diff --git a/config/samples/topology_v1beta1_topology.yaml b/config/samples/topology_v1beta1_topology.yaml new file mode 100644 index 00000000..e45117fd --- /dev/null +++ b/config/samples/topology_v1beta1_topology.yaml @@ -0,0 +1,12 @@ +apiVersion: topology.openstack.org/v1beta1 +kind: Topology +metadata: + labels: + app.kubernetes.io/name: topology + app.kubernetes.io/instance: topology-sample + app.kubernetes.io/part-of: infra-operator + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/created-by: infra-operator + name: topology-sample +spec: + # TODO(user): Add fields here diff --git a/controllers/network/dnsmasq_controller.go b/controllers/network/dnsmasq_controller.go index bf7c50a9..b8e46beb 100644 --- a/controllers/network/dnsmasq_controller.go +++ b/controllers/network/dnsmasq_controller.go @@ -433,7 +433,10 @@ func (r *DNSMasqReconciler) reconcileNormal(ctx context.Context, instance *netwo instance.Status.Conditions.MarkTrue(condition.CreateServiceReadyCondition, condition.CreateServiceReadyMessage) // Define a new Deployment object - deplDef := dnsmasq.Deployment(instance, instance.Status.Hash[common.InputHashName], serviceLabels, serviceAnnotations, configMaps) + deplDef, err := dnsmasq.Deployment(instance, instance.Status.Hash[common.InputHashName], serviceLabels, serviceAnnotations, configMaps) + if err != nil { + return ctrlResult, err + } depl := deployment.NewDeployment( deplDef, time.Duration(5)*time.Second, diff --git a/controllers/topology/suite_test.go b/controllers/topology/suite_test.go new file mode 100644 index 00000000..a549b1bc --- /dev/null +++ b/controllers/topology/suite_test.go @@ -0,0 +1,80 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package topology + +import ( + "path/filepath" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + topologyv1beta1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" + //+kubebuilder:scaffold:imports +) + +// These tests use Ginkgo (BDD-style Go testing framework). Refer to +// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. + +var cfg *rest.Config +var k8sClient client.Client +var testEnv *envtest.Environment + +func TestAPIs(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecs(t, "Controller Suite") +} + +var _ = BeforeSuite(func() { + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")}, + ErrorIfCRDPathMissing: true, + } + + var err error + // cfg is defined in this file globally. + cfg, err = testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + + err = topologyv1beta1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + //+kubebuilder:scaffold:scheme + + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient).NotTo(BeNil()) + +}) + +var _ = AfterSuite(func() { + By("tearing down the test environment") + err := testEnv.Stop() + Expect(err).NotTo(HaveOccurred()) +}) diff --git a/controllers/topology/topology_controller.go b/controllers/topology/topology_controller.go new file mode 100644 index 00000000..55ec1116 --- /dev/null +++ b/controllers/topology/topology_controller.go @@ -0,0 +1,189 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package topology + +import ( + "context" + + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + + "github.com/go-logr/logr" + topologyv1beta1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" + common "github.com/openstack-k8s-operators/lib-common/modules/common" + condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + helper "github.com/openstack-k8s-operators/lib-common/modules/common/helper" + util "github.com/openstack-k8s-operators/lib-common/modules/common/util" + k8s_errors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/client-go/kubernetes" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +// TopologyReconciler reconciles a Topology object +type TopologyReconciler struct { + client.Client + Scheme *runtime.Scheme + Kclient kubernetes.Interface +} + +// GetLogger returns a logger object with a prefix of "controller.name" and additional controller context fields +func (r *TopologyReconciler) GetLogger(ctx context.Context) logr.Logger { + return log.FromContext(ctx).WithName("Controllers").WithName("Topology") +} + +// +kubebuilder:rbac:groups=topology.openstack.org,resources=topologies,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=topology.openstack.org,resources=topologies/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=topology.openstack.org,resources=topologies/finalizers,verbs=update;patch + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +// +// For more details, check Reconcile and its Result here: +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.14.4/pkg/reconcile +func (r *TopologyReconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ctrl.Result, _err error) { + Log := r.GetLogger(ctx) + + instance := &topologyv1beta1.Topology{} + err := r.Client.Get(ctx, req.NamespacedName, instance) + if err != nil { + if k8s_errors.IsNotFound(err) { + // Request object not found, could have been deleted after reconcile request. + // Owned objects are automatically garbage collected. + // For additional cleanup logic use finalizers. Return and don't requeue. + return ctrl.Result{}, nil + } + // Error reading the object - requeue the request. + return ctrl.Result{}, err + } + + helper, err := helper.NewHelper( + instance, + r.Client, + r.Kclient, + r.Scheme, + Log, + ) + if err != nil { + return ctrl.Result{}, err + } + + // initialize status if Conditions is nil, but do not reset if it already + // exists + isNewInstance := instance.Status.Conditions == nil + if isNewInstance { + instance.Status.Conditions = condition.Conditions{} + } + + // Save a copy of the condtions so that we can restore the LastTransitionTime + // when a condition's state doesn't change. + savedConditions := instance.Status.Conditions.DeepCopy() + + // Always patch the instance status when exiting this function so we can + // persist any changes. + defer func() { + condition.RestoreLastTransitionTimes( + &instance.Status.Conditions, savedConditions) + if instance.Status.Conditions.IsUnknown(condition.ReadyCondition) { + instance.Status.Conditions.Set( + instance.Status.Conditions.Mirror(condition.ReadyCondition)) + } + err := helper.PatchInstance(ctx, instance) + if err != nil { + _err = err + return + } + }() + + // initialize status + cl := condition.CreateList( + condition.UnknownCondition(condition.ReadyCondition, condition.InitReason, condition.ReadyInitMessage), + condition.UnknownCondition(condition.ServiceConfigReadyCondition, condition.InitReason, condition.ServiceConfigReadyInitMessage), + ) + + instance.Status.Conditions.Init(&cl) + instance.Status.ObservedGeneration = instance.Generation + + // If we're not deleting this and the service object doesn't have our finalizer, add it. + if instance.DeletionTimestamp.IsZero() && controllerutil.AddFinalizer(instance, helper.GetFinalizer()) || isNewInstance { + return ctrl.Result{}, err + } + + // Handle service delete + if !instance.DeletionTimestamp.IsZero() { + return r.reconcileDelete(ctx, instance, helper) + } + + // Handle non-deleted clusters + return r.reconcileNormal(ctx, instance, helper) +} + +func (r *TopologyReconciler) reconcileNormal( + ctx context.Context, + instance *topologyv1beta1.Topology, + helper *helper.Helper, +) (ctrl.Result, error) { + + Log := r.GetLogger(ctx) + + Log.Info("Reconciling Service") + + hash, err := util.ObjectHash(instance.Spec) + if err != nil { + return ctrl.Result{}, err + } + if hash != instance.Status.Hash { + instance.Status.Hash = hash + Log.Info("Input maps hash", "HashName", common.InputHashName, "Hash", hash) + } + + instance.Status.Conditions.MarkTrue(condition.ServiceConfigReadyCondition, condition.InputReadyMessage) + + // We reached the end of the Reconcile, update the Ready condition based on + // the sub conditions + if instance.Status.Conditions.AllSubConditionIsTrue() { + instance.Status.Conditions.MarkTrue( + condition.ReadyCondition, condition.ReadyMessage) + } + Log.Info("Reconciled Topology successfully") + return ctrl.Result{}, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *TopologyReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&topologyv1beta1.Topology{}). + Complete(r) +} + +// reconcileDelete - +func (r *TopologyReconciler) reconcileDelete( + ctx context.Context, + instance *topologyv1beta1.Topology, + helper *helper.Helper, +) (ctrl.Result, error) { + Log := r.GetLogger(ctx) + + Log.Info("Reconciling Service delete") + + // Service is deleted so remove the finalizer. + controllerutil.RemoveFinalizer(instance, helper.GetFinalizer()) + Log.Info("Reconciled Service delete successfully") + + return ctrl.Result{}, nil +} diff --git a/go.mod b/go.mod index e4c31a21..b0d18a36 100644 --- a/go.mod +++ b/go.mod @@ -86,3 +86,5 @@ replace github.com/openshift/api => github.com/openshift/api v0.0.0-202408300231 // custom RabbitmqClusterSpecCore for OpenStackControlplane (v2.6.0_patches_tag) replace github.com/rabbitmq/cluster-operator/v2 => github.com/openstack-k8s-operators/rabbitmq-cluster-operator/v2 v2.6.1-0.20241017142550-a3524acedd49 //allow-merging + +replace github.com/openstack-k8s-operators/lib-common/modules/common => github.com/fmount/lib-common/modules/common v0.0.0-20241203102750-0b9fe14de0b0 diff --git a/go.sum b/go.sum index 70914b75..a93e0c65 100644 --- a/go.sum +++ b/go.sum @@ -12,6 +12,8 @@ github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= +github.com/fmount/lib-common/modules/common v0.0.0-20241203102750-0b9fe14de0b0 h1:Agq7NNy3L6PBgGKPfS0tRNRAnT5lNQ/OrbvE+6gF6Ag= +github.com/fmount/lib-common/modules/common v0.0.0-20241203102750-0b9fe14de0b0/go.mod h1:YpNTuJhDWhbXM50O3qBkhO7M+OOyRmWkNVmJ4y3cyFs= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= @@ -76,8 +78,6 @@ github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= github.com/openshift/api v0.0.0-20240830023148-b7d0481c9094 h1:J1wuGhVxpsHykZBa6Beb1gQ96Ptej9AE/BvwCBiRj1E= github.com/openshift/api v0.0.0-20240830023148-b7d0481c9094/go.mod h1:CxgbWAlvu2iQB0UmKTtRu1YfepRg1/vJ64n2DlIEVz4= -github.com/openstack-k8s-operators/lib-common/modules/common v0.5.1-0.20241114091812-6dc9fd0961dc h1:Ufa/q/nC9wmKblvsc0kJppsXHOJoY4fbUamb3ItWCOk= -github.com/openstack-k8s-operators/lib-common/modules/common v0.5.1-0.20241114091812-6dc9fd0961dc/go.mod h1:YpNTuJhDWhbXM50O3qBkhO7M+OOyRmWkNVmJ4y3cyFs= github.com/openstack-k8s-operators/lib-common/modules/test v0.5.1-0.20241114091812-6dc9fd0961dc h1:knyjd0eg4DyY+dTDHSrE9QwrZ0mtr7MpASCrmhW+5pw= github.com/openstack-k8s-operators/lib-common/modules/test v0.5.1-0.20241114091812-6dc9fd0961dc/go.mod h1:LV0jo5etIsGyINpmB37i4oWR8zU6ApIuh7fsqGGA41o= github.com/openstack-k8s-operators/rabbitmq-cluster-operator/v2 v2.6.1-0.20241017142550-a3524acedd49 h1:/7SnnHfGCH/dwuZFNUx54zw4cnwv2w6hjONq16aoowM= diff --git a/main.go b/main.go index 79de7b84..36061e1e 100644 --- a/main.go +++ b/main.go @@ -42,16 +42,19 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/config" k8s_networkv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" + instancehav1 "github.com/openstack-k8s-operators/infra-operator/apis/instanceha/v1beta1" memcachedv1 "github.com/openstack-k8s-operators/infra-operator/apis/memcached/v1beta1" networkv1 "github.com/openstack-k8s-operators/infra-operator/apis/network/v1beta1" rabbitmqv1beta1 "github.com/openstack-k8s-operators/infra-operator/apis/rabbitmq/v1beta1" redisv1 "github.com/openstack-k8s-operators/infra-operator/apis/redis/v1beta1" + topologyv1beta1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" instancehacontrollers "github.com/openstack-k8s-operators/infra-operator/controllers/instanceha" memcachedcontrollers "github.com/openstack-k8s-operators/infra-operator/controllers/memcached" networkcontrollers "github.com/openstack-k8s-operators/infra-operator/controllers/network" rabbitmqcontrollers "github.com/openstack-k8s-operators/infra-operator/controllers/rabbitmq" rediscontrollers "github.com/openstack-k8s-operators/infra-operator/controllers/redis" + topologycontrollers "github.com/openstack-k8s-operators/infra-operator/controllers/topology" //+kubebuilder:scaffold:imports ) @@ -70,6 +73,7 @@ func init() { utilruntime.Must(redisv1.AddToScheme(scheme)) utilruntime.Must(networkv1.AddToScheme(scheme)) utilruntime.Must(k8s_networkv1.AddToScheme(scheme)) + utilruntime.Must(topologyv1beta1.AddToScheme(scheme)) //+kubebuilder:scaffold:scheme } @@ -206,6 +210,13 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "IPSet") os.Exit(1) } + if err = (&topologycontrollers.TopologyReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "Topology") + os.Exit(1) + } // Acquire environmental defaults and initialize operator defaults with them memcachedv1.SetupDefaults() @@ -247,6 +258,13 @@ func main() { checker = mgr.GetWebhookServer().StartedChecker() } + if err = (&topologycontrollers.TopologyReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "Topology") + os.Exit(1) + } //+kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("healthz", checker); err != nil { diff --git a/pkg/dnsmasq/deployment.go b/pkg/dnsmasq/deployment.go index 57ddf5df..b7c966c7 100644 --- a/pkg/dnsmasq/deployment.go +++ b/pkg/dnsmasq/deployment.go @@ -45,7 +45,7 @@ func Deployment( labels map[string]string, annotations map[string]string, cms *corev1.ConfigMapList, -) *appsv1.Deployment { +) (*appsv1.Deployment, error) { terminationGracePeriodSeconds := int64(10) livenessProbe := &corev1.Probe{ @@ -166,16 +166,18 @@ func Deployment( // If possible two pods of the same service should not // run on the same worker node. If this is not possible // the get still created on the same worker node. - deployment.Spec.Template.Spec.Affinity = affinity.DistributePods( + var err error + deployment.Spec.Template.Spec.Affinity, err = affinity.DistributePods( common.AppSelector, []string{ ServiceName, }, corev1.LabelHostname, + nil, ) if instance.Spec.NodeSelector != nil { deployment.Spec.Template.Spec.NodeSelector = *instance.Spec.NodeSelector } - return deployment + return deployment, err } diff --git a/pkg/memcached/statefulset.go b/pkg/memcached/statefulset.go index 2d4b6197..8c7a0879 100644 --- a/pkg/memcached/statefulset.go +++ b/pkg/memcached/statefulset.go @@ -104,12 +104,13 @@ func StatefulSet( // If possible two pods of the same service should not // run on the same worker node. If this is not possible // the get still created on the same worker node. - sfs.Spec.Template.Spec.Affinity = affinity.DistributePods( + sfs.Spec.Template.Spec.Affinity, _ = affinity.DistributePods( common.AppSelector, []string{ m.Name, }, corev1.LabelHostname, + nil, ) if m.Spec.NodeSelector != nil { sfs.Spec.Template.Spec.NodeSelector = *m.Spec.NodeSelector