From 2bcccd88d20b44adf5c75da986182ae2b4755d13 Mon Sep 17 00:00:00 2001 From: Christian Schwede Date: Thu, 8 Feb 2024 08:07:41 +0100 Subject: [PATCH] Add DNSData entries for SwiftStorage service pods Storage service hostnames need to be resolvable on the dataplane nodes as well. They are used within the Swift rings to be able to allow node IP changes without updating and redistributing Swift rings. This patch creates DNSData entries for every storage service pod, similar to [1]. It only creates DNS for the storage network, as this is the only one that should be used within the storage backend services. There is no delete function to scale down, as we don't support scaling down for SwiftStorage instances. However, the created DNSData CRs are owned by the SwiftStorage instance, thus being deleted properly if the instance is deleted. [1] https://github.com/openstack-k8s-operators/ovn-operator/pull/154 --- config/rbac/role.yaml | 12 +++++ controllers/swiftstorage_controller.go | 73 ++++++++++++++++++++++++++ go.mod | 1 + go.sum | 2 + main.go | 2 + pkg/swiftstorage/dnsdata.go | 56 ++++++++++++++++++++ pkg/swiftstorage/funcs.go | 2 +- 7 files changed, 147 insertions(+), 1 deletion(-) create mode 100644 pkg/swiftstorage/dnsdata.go diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 8e631754..d19b8f23 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -167,6 +167,18 @@ rules: - get - list - watch +- apiGroups: + - network.openstack.org + resources: + - dnsdata + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - networking.k8s.io resources: diff --git a/controllers/swiftstorage_controller.go b/controllers/swiftstorage_controller.go index 288e3472..5b48dd04 100644 --- a/controllers/swiftstorage_controller.go +++ b/controllers/swiftstorage_controller.go @@ -19,7 +19,9 @@ package controllers import ( "context" "encoding/json" + "errors" "fmt" + "strings" "time" "github.com/go-logr/logr" @@ -45,6 +47,7 @@ import ( "github.com/openstack-k8s-operators/lib-common/modules/common/configmap" "github.com/openstack-k8s-operators/lib-common/modules/common/env" "github.com/openstack-k8s-operators/lib-common/modules/common/networkattachment" + "github.com/openstack-k8s-operators/lib-common/modules/common/pod" ) // SwiftStorageReconciler reconciles a SwiftStorage object @@ -72,6 +75,7 @@ type Netconfig struct { //+kubebuilder:rbac:groups=core,resources=configmaps,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=k8s.cni.cncf.io,resources=network-attachment-definitions,verbs=get;list;watch //+kubebuilder:rbac:groups=memcached.openstack.org,resources=memcacheds,verbs=get;list;watch; +//+kubebuilder:rbac:groups=network.openstack.org,resources=dnsdata,verbs=get;list;watch;create;update;patch;delete // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. @@ -238,6 +242,53 @@ func (r *SwiftStorageReconciler) Reconcile(ctx context.Context, req ctrl.Request if err != nil { return ctrl.Result{}, err } + + // When the cluster is attached to an external network, create DNS record for every + // cluster member so it can be resolved from outside cluster (edpm nodes) + podList, err := pod.GetPodListWithLabel(ctx, helper, instance.Namespace, serviceLabels) + if err != nil { + return ctrl.Result{}, err + } + + for _, swiftPod := range podList.Items { + dnsIP := "" + if len(instance.Spec.NetworkAttachments) > 0 { + // Currently only IPv4 is supported + dnsIP, err = getPodIPv4InNetwork(swiftPod, instance.Namespace, "storage") + if err != nil { + return ctrl.Result{}, err + } + } + + if len(dnsIP) == 0 { + // If this is reached it means that no IP was found in the network + // or no networkAttachment exists. Try to use podIP if possible + if len(swiftPod.Status.PodIP) > 0 { + dnsIP = swiftPod.Status.PodIP + } + } + + if len(dnsIP) == 0 { + return ctrl.Result{}, errors.New("Unable to get any IP address for pod") + } + + hostName := fmt.Sprintf("%s.%s.%s.svc", swiftPod.Name, instance.Name, swiftPod.Namespace) + + // Create DNSData CR + err = swiftstorage.DNSData( + ctx, + helper, + hostName, + dnsIP, + instance, + swiftPod, + serviceLabels, + ) + if err != nil { + return ctrl.Result{}, err + } + } + instance.Status.Conditions.MarkTrue(condition.ReadyCondition, condition.ReadyMessage) instance.Status.Conditions.MarkTrue(swiftv1beta1.SwiftStorageReadyCondition, condition.ReadyMessage) if err := r.Status().Update(ctx, instance); err != nil { @@ -259,3 +310,25 @@ func (r *SwiftStorageReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&networkingv1.NetworkPolicy{}). Complete(r) } + +func getPodIPv4InNetwork(swiftPod corev1.Pod, namespace string, networkAttachment string) (string, error) { + networkName := fmt.Sprintf("%s/%s", namespace, networkAttachment) + netStat, err := networkattachment.GetNetworkStatusFromAnnotation(swiftPod.Annotations) + if err != nil { + err = fmt.Errorf("Error while getting the Network Status for pod %s: %v", swiftPod.Name, err) + return "", err + } + for _, net := range netStat { + if net.Name == networkName { + for _, ip := range net.IPs { + if !strings.Contains(ip, ":") { + return ip, nil + } + } + } + } + + // If this is reached it means that no IP was found, construct error and return + err = fmt.Errorf("Error while getting IPv4 address from pod %s in network %s", swiftPod.Name, networkAttachment) + return "", err +} diff --git a/go.mod b/go.mod index 07abd71e..16530395 100644 --- a/go.mod +++ b/go.mod @@ -57,6 +57,7 @@ require ( github.com/spf13/pflag v1.0.5 // indirect go.uber.org/multierr v1.10.0 // indirect go.uber.org/zap v1.26.0 // indirect + golang.org/x/exp v0.0.0-20240119083558-1b970713d09a // indirect golang.org/x/net v0.20.0 // indirect golang.org/x/oauth2 v0.7.0 // indirect golang.org/x/sys v0.16.0 // indirect diff --git a/go.sum b/go.sum index d7fad2f1..840ba0ca 100644 --- a/go.sum +++ b/go.sum @@ -337,6 +337,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20240119083558-1b970713d09a h1:Q8/wZp0KX97QFTc2ywcOE0YRjZPVIx+MXInMzdvQqcA= +golang.org/x/exp v0.0.0-20240119083558-1b970713d09a/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= diff --git a/main.go b/main.go index 0927737d..b9dd20a2 100644 --- a/main.go +++ b/main.go @@ -41,6 +41,7 @@ import ( networkv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" memcachedv1 "github.com/openstack-k8s-operators/infra-operator/apis/memcached/v1beta1" + infranetworkv1 "github.com/openstack-k8s-operators/infra-operator/apis/network/v1beta1" swiftv1beta1 "github.com/openstack-k8s-operators/swift-operator/api/v1beta1" "github.com/openstack-k8s-operators/swift-operator/controllers" //+kubebuilder:scaffold:imports @@ -57,6 +58,7 @@ func init() { utilruntime.Must(keystonev1beta1.AddToScheme(scheme)) utilruntime.Must(memcachedv1.AddToScheme(scheme)) utilruntime.Must(networkv1.AddToScheme(scheme)) + utilruntime.Must(infranetworkv1.AddToScheme(scheme)) //+kubebuilder:scaffold:scheme } diff --git a/pkg/swiftstorage/dnsdata.go b/pkg/swiftstorage/dnsdata.go new file mode 100644 index 00000000..255816be --- /dev/null +++ b/pkg/swiftstorage/dnsdata.go @@ -0,0 +1,56 @@ +package swiftstorage + +import ( + "context" + "fmt" + + infranetworkv1 "github.com/openstack-k8s-operators/infra-operator/apis/network/v1beta1" + "github.com/openstack-k8s-operators/lib-common/modules/common/helper" + swiftv1 "github.com/openstack-k8s-operators/swift-operator/api/v1beta1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +// DNSData - Create DNS entry that openstack dnsmasq will resolve +func DNSData( + ctx context.Context, + helper *helper.Helper, + hostName string, + ip string, + instance *swiftv1.SwiftStorage, + swiftPod corev1.Pod, + serviceLabels map[string]string, +) error { + dnsHostCname := infranetworkv1.DNSHost{ + IP: ip, + Hostnames: []string{ + hostName, + }, + } + + // Create DNSData object + dnsData := &infranetworkv1.DNSData{ + ObjectMeta: metav1.ObjectMeta{ + Name: swiftPod.Name, + Namespace: swiftPod.Namespace, + Labels: serviceLabels, + }, + } + dnsHosts := []infranetworkv1.DNSHost{dnsHostCname} + + _, err := controllerutil.CreateOrPatch(ctx, helper.GetClient(), dnsData, func() error { + dnsData.Spec.Hosts = dnsHosts + // TODO: use value from DNSMasq instance instead of hardcode + dnsData.Spec.DNSDataLabelSelectorValue = "dnsdata" + err := controllerutil.SetControllerReference(helper.GetBeforeObject(), dnsData, helper.GetScheme()) + if err != nil { + return err + } + return nil + }) + if err != nil { + return fmt.Errorf("Error creating DNSData %s: %w", dnsData.Name, err) + } + return nil +} diff --git a/pkg/swiftstorage/funcs.go b/pkg/swiftstorage/funcs.go index b774ca0e..015baa86 100644 --- a/pkg/swiftstorage/funcs.go +++ b/pkg/swiftstorage/funcs.go @@ -53,7 +53,7 @@ func DeviceList(ctx context.Context, h *helper.Helper, instance *swiftv1beta1.Sw } weight = weight / (1000 * 1000 * 1000) // 10GiB gets a weight of 10 etc. // CSV: region,zone,hostname,devicename,weight - devices.WriteString(fmt.Sprintf("1,1,%s-%d.%s,%s,%d\n", instance.Name, replica, instance.Name, "d1", weight)) + devices.WriteString(fmt.Sprintf("1,1,%s-%d.%s.%s.svc,%s,%d\n", instance.Name, replica, instance.Name, instance.Namespace, "d1", weight)) } return devices.String() }