Add DNSData entries for SwiftStorage service pods

Storage service hostnames need to be resolvable on the dataplane nodes
as well. They are used within the Swift rings to be able to allow node
IP changes without updating and redistributing Swift rings.

This patch creates DNSData entries for every storage service pod,
similar to [1]. It only creates DNS for the storage network, as this is
the only one that should be used within the storage backend services.

There is no delete function to scale down, as we don't support scaling
down for SwiftStorage instances. However, the created DNSData CRs are
owned by the SwiftStorage instance, thus being deleted properly if the
instance is deleted.

[1] openstack-k8s-operators/ovn-operator#154

config/rbac/role.yaml

@@ -167,6 +167,18 @@ rules:
   - get
   - list
   - watch
+- apiGroups:
+  - network.openstack.org
+  resources:
+  - dnsdata
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - networking.k8s.io
   resources:

controllers/swiftstorage_controller.go

@@ -19,7 +19,9 @@ package controllers
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
+	"strings"
 	"time"
 
 	"github.com/go-logr/logr"
@@ -45,6 +47,7 @@ import (
 	"github.com/openstack-k8s-operators/lib-common/modules/common/configmap"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/env"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/networkattachment"
+	"github.com/openstack-k8s-operators/lib-common/modules/common/pod"
 )
 
 // SwiftStorageReconciler reconciles a SwiftStorage object
@@ -72,6 +75,7 @@ type Netconfig struct {
 //+kubebuilder:rbac:groups=core,resources=configmaps,verbs=get;list;watch;create;update;patch;delete
 //+kubebuilder:rbac:groups=k8s.cni.cncf.io,resources=network-attachment-definitions,verbs=get;list;watch
 //+kubebuilder:rbac:groups=memcached.openstack.org,resources=memcacheds,verbs=get;list;watch;
+//+kubebuilder:rbac:groups=network.openstack.org,resources=dnsdata,verbs=get;list;watch;create;update;patch;delete
 
 // Reconcile is part of the main kubernetes reconciliation loop which aims to
 // move the current state of the cluster closer to the desired state.
@@ -238,6 +242,53 @@ func (r *SwiftStorageReconciler) Reconcile(ctx context.Context, req ctrl.Request
 		if err != nil {
 			return ctrl.Result{}, err
 		}
+
+		// When the cluster is attached to an external network, create DNS record for every
+		// cluster member so it can be resolved from outside cluster (edpm nodes)
+		podList, err := pod.GetPodListWithLabel(ctx, helper, instance.Namespace, serviceLabels)
+		if err != nil {
+			return ctrl.Result{}, err
+		}
+
+		for _, swiftPod := range podList.Items {
+			dnsIP := ""
+			if len(instance.Spec.NetworkAttachments) > 0 {
+				// Currently only IPv4 is supported
+				dnsIP, err = getPodIPv4InNetwork(swiftPod, instance.Namespace, "storage")
+				if err != nil {
+					return ctrl.Result{}, err
+				}
+			}
+
+			if len(dnsIP) == 0 {
+				// If this is reached it means that no IP was found in the network
+				// or no networkAttachment exists. Try to use podIP if possible
+				if len(swiftPod.Status.PodIP) > 0 {
+					dnsIP = swiftPod.Status.PodIP
+				}
+			}
+
+			if len(dnsIP) == 0 {
+				return ctrl.Result{}, errors.New("Unable to get any IP address for pod")
+			}
+
+			hostName := fmt.Sprintf("%s.%s.%s.svc", swiftPod.Name, instance.Name, swiftPod.Namespace)
+
+			// Create DNSData CR
+			err = swiftstorage.DNSData(
+				ctx,
+				helper,
+				hostName,
+				dnsIP,
+				instance,
+				swiftPod,
+				serviceLabels,
+			)
+			if err != nil {
+				return ctrl.Result{}, err
+			}
+		}
+
 		instance.Status.Conditions.MarkTrue(condition.ReadyCondition, condition.ReadyMessage)
 		instance.Status.Conditions.MarkTrue(swiftv1beta1.SwiftStorageReadyCondition, condition.ReadyMessage)
 		if err := r.Status().Update(ctx, instance); err != nil {
@@ -259,3 +310,25 @@ func (r *SwiftStorageReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		Owns(&networkingv1.NetworkPolicy{}).
 		Complete(r)
 }
+
+func getPodIPv4InNetwork(swiftPod corev1.Pod, namespace string, networkAttachment string) (string, error) {
+	networkName := fmt.Sprintf("%s/%s", namespace, networkAttachment)
+	netStat, err := networkattachment.GetNetworkStatusFromAnnotation(swiftPod.Annotations)
+	if err != nil {
+		err = fmt.Errorf("Error while getting the Network Status for pod %s: %v", swiftPod.Name, err)
+		return "", err
+	}
+	for _, net := range netStat {
+		if net.Name == networkName {
+			for _, ip := range net.IPs {
+				if !strings.Contains(ip, ":") {
+					return ip, nil
+				}
+			}
+		}
+	}
+
+	// If this is reached it means that no IP was found, construct error and return
+	err = fmt.Errorf("Error while getting IPv4 address from pod %s in network %s", swiftPod.Name, networkAttachment)
+	return "", err
+}

go.mod

@@ -57,6 +57,7 @@ require (
 	github.com/spf13/pflag v1.0.5 // indirect
 	go.uber.org/multierr v1.10.0 // indirect
 	go.uber.org/zap v1.26.0 // indirect
+	golang.org/x/exp v0.0.0-20240119083558-1b970713d09a // indirect
 	golang.org/x/net v0.20.0 // indirect
 	golang.org/x/oauth2 v0.7.0 // indirect
 	golang.org/x/sys v0.16.0 // indirect

go.sum

@@ -337,6 +337,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/exp v0.0.0-20240119083558-1b970713d09a h1:Q8/wZp0KX97QFTc2ywcOE0YRjZPVIx+MXInMzdvQqcA=
+golang.org/x/exp v0.0.0-20240119083558-1b970713d09a/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=

main.go

@@ -41,6 +41,7 @@ import (
 
 	networkv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
 	memcachedv1 "github.com/openstack-k8s-operators/infra-operator/apis/memcached/v1beta1"
+	infranetworkv1 "github.com/openstack-k8s-operators/infra-operator/apis/network/v1beta1"
 	swiftv1beta1 "github.com/openstack-k8s-operators/swift-operator/api/v1beta1"
 	"github.com/openstack-k8s-operators/swift-operator/controllers"
 	//+kubebuilder:scaffold:imports
@@ -57,6 +58,7 @@ func init() {
 	utilruntime.Must(keystonev1beta1.AddToScheme(scheme))
 	utilruntime.Must(memcachedv1.AddToScheme(scheme))
 	utilruntime.Must(networkv1.AddToScheme(scheme))
+	utilruntime.Must(infranetworkv1.AddToScheme(scheme))
 	//+kubebuilder:scaffold:scheme
 }
 

pkg/swiftstorage/dnsdata.go

@@ -0,0 +1,56 @@
+package swiftstorage
+
+import (
+	"context"
+	"fmt"
+
+	infranetworkv1 "github.com/openstack-k8s-operators/infra-operator/apis/network/v1beta1"
+	"github.com/openstack-k8s-operators/lib-common/modules/common/helper"
+	swiftv1 "github.com/openstack-k8s-operators/swift-operator/api/v1beta1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+)
+
+// DNSData - Create DNS entry that openstack dnsmasq will resolve
+func DNSData(
+	ctx context.Context,
+	helper *helper.Helper,
+	hostName string,
+	ip string,
+	instance *swiftv1.SwiftStorage,
+	swiftPod corev1.Pod,
+	serviceLabels map[string]string,
+) error {
+	dnsHostCname := infranetworkv1.DNSHost{
+		IP: ip,
+		Hostnames: []string{
+			hostName,
+		},
+	}
+
+	// Create DNSData object
+	dnsData := &infranetworkv1.DNSData{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      swiftPod.Name,
+			Namespace: swiftPod.Namespace,
+			Labels:    serviceLabels,
+		},
+	}
+	dnsHosts := []infranetworkv1.DNSHost{dnsHostCname}
+
+	_, err := controllerutil.CreateOrPatch(ctx, helper.GetClient(), dnsData, func() error {
+		dnsData.Spec.Hosts = dnsHosts
+		// TODO: use value from DNSMasq instance instead of hardcode
+		dnsData.Spec.DNSDataLabelSelectorValue = "dnsdata"
+		err := controllerutil.SetControllerReference(helper.GetBeforeObject(), dnsData, helper.GetScheme())
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+	if err != nil {
+		return fmt.Errorf("Error creating DNSData %s: %w", dnsData.Name, err)
+	}
+	return nil
+}

pkg/swiftstorage/funcs.go

@@ -53,7 +53,7 @@ func DeviceList(ctx context.Context, h *helper.Helper, instance *swiftv1beta1.Sw
 		}
 		weight = weight / (1000 * 1000 * 1000) // 10GiB gets a weight of 10 etc.
 		// CSV: region,zone,hostname,devicename,weight
-		devices.WriteString(fmt.Sprintf("1,1,%s-%d.%s,%s,%d\n", instance.Name, replica, instance.Name, "d1", weight))
+		devices.WriteString(fmt.Sprintf("1,1,%s-%d.%s.%s.svc,%s,%d\n", instance.Name, replica, instance.Name, instance.Namespace, "d1", weight))
 	}
 	return devices.String()
 }