From f8b1577c82ea51b78aef13841bf6bdc974cc81c2 Mon Sep 17 00:00:00 2001 From: Miguel Duarte Barroso Date: Fri, 30 Sep 2022 17:02:12 +0200 Subject: [PATCH] controller, network status: update pod after interface add/remove This commit updates the pods network-status annotation whenever an interface is hot-plugged to the pod (or removed from it ...). Signed-off-by: Miguel Duarte Barroso --- pkg/annotations/dynamic-network-status.go | 79 ++++++++++ .../dynamic-network-status_test.go | 136 ++++++++++++++++++ .../network-selection-elements_test.go | 8 +- pkg/controller/pod.go | 49 +++++-- pkg/controller/pod_test.go | 9 +- 5 files changed, 264 insertions(+), 17 deletions(-) create mode 100644 pkg/annotations/dynamic-network-status.go create mode 100644 pkg/annotations/dynamic-network-status_test.go diff --git a/pkg/annotations/dynamic-network-status.go b/pkg/annotations/dynamic-network-status.go new file mode 100644 index 00000000..85da7dad --- /dev/null +++ b/pkg/annotations/dynamic-network-status.go @@ -0,0 +1,79 @@ +package annotations + +import ( + "encoding/json" + "fmt" + + corev1 "k8s.io/api/core/v1" + + nettypes "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" + nadutils "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/utils" + multusapi "gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/server/api" +) + +func AddDynamicIfaceToStatus(currentPod *corev1.Pod, networkSelectionElement *nettypes.NetworkSelectionElement, response *multusapi.Response) (string, error) { + currentIfaceStatus, err := podDynamicNetworkStatus(currentPod) + if err != nil { + return "", err + } + + if response != nil && response.Result != nil { + newIfaceStatus, err := nadutils.CreateNetworkStatus( + response.Result, + NamespacedName(networkSelectionElement.Namespace, networkSelectionElement.Name), + false, + nil, + ) + if err != nil { + return "", fmt.Errorf("failed to create NetworkStatus from the response: %v", err) + } + + newIfaceString, err := json.Marshal(append(currentIfaceStatus, *newIfaceStatus)) + if err != nil { + return "", fmt.Errorf("failed to marshall the dynamic networks status after interface creation") + } + return string(newIfaceString), nil + } + return "", fmt.Errorf("got an empty response from multus: %+v", response) +} + +func DeleteDynamicIfaceFromStatus(currentPod *corev1.Pod, networkSelectionElement *nettypes.NetworkSelectionElement) (string, error) { + currentIfaceStatus, err := podDynamicNetworkStatus(currentPod) + if err != nil { + return "", err + } + + netName := NamespacedName(networkSelectionElement.Namespace, networkSelectionElement.Name) + var newIfaceStatus []nettypes.NetworkStatus + newIfaceStatus = make([]nettypes.NetworkStatus, 0) + for i := range currentIfaceStatus { + if currentIfaceStatus[i].Name == netName && currentIfaceStatus[i].Interface == networkSelectionElement.InterfaceRequest { + continue + } + newIfaceStatus = append(newIfaceStatus, currentIfaceStatus[i]) + } + + newIfaceString, err := json.Marshal(newIfaceStatus) + if err != nil { + return "", fmt.Errorf("failed to marshall the dynamic networks status after deleting interface") + } + return string(newIfaceString), nil +} + +func podDynamicNetworkStatus(currentPod *corev1.Pod) ([]nettypes.NetworkStatus, error) { + var currentIfaceStatus []nettypes.NetworkStatus + if currentIfaceStatusString, wasFound := currentPod.Annotations[nettypes.NetworkStatusAnnot]; wasFound { + if err := json.Unmarshal([]byte(currentIfaceStatusString), ¤tIfaceStatus); err != nil { + return nil, fmt.Errorf("could not unmarshall the current dynamic annotations for pod %s: %v", podNameAndNs(currentPod), err) + } + } + return currentIfaceStatus, nil +} + +func podNameAndNs(currentPod *corev1.Pod) string { + return fmt.Sprintf("%s/%s", currentPod.GetNamespace(), currentPod.GetName()) +} + +func NamespacedName(podNamespace string, podName string) string { + return fmt.Sprintf("%s/%s", podNamespace, podName) +} diff --git a/pkg/annotations/dynamic-network-status_test.go b/pkg/annotations/dynamic-network-status_test.go new file mode 100644 index 00000000..22ea5e24 --- /dev/null +++ b/pkg/annotations/dynamic-network-status_test.go @@ -0,0 +1,136 @@ +package annotations + +import ( + "encoding/json" + "net" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + cni100 "github.com/containernetworking/cni/pkg/types/100" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + nadv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" + "gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/server/api" +) + +var _ = Describe("NetworkStatusFromResponse", func() { + const ( + ifaceName = "ens32" + namespace = "ns1" + networkName = "tenantnetwork" + podName = "tpod" + ) + + DescribeTable("add dynamic interface to network status", func(initialNetStatus []nadv1.NetworkStatus, resultIPs []string, expectedNetworkStatus string) { + const ( + ifaceToAdd = "newiface" + macAddr = "02:03:04:05:06:07" + ) + Expect( + AddDynamicIfaceToStatus( + newPod(podName, namespace, initialNetStatus...), + newNetworkSelectionElementWithIface(networkName, ifaceName, namespace), + newResponse(ifaceToAdd, macAddr, resultIPs...), + ), + ).To(Equal(expectedNetworkStatus)) + }, + Entry("initial empty pod", []nadv1.NetworkStatus{}, nil, `[{"name":"ns1/tenantnetwork","interface":"newiface","mac":"02:03:04:05:06:07","dns":{}}]`), + Entry("pod with a network present in the network status", []nadv1.NetworkStatus{ + { + Name: "net1", + Interface: "iface1", + Mac: "00:00:00:20:10:00", + }}, + nil, + `[{"name":"net1","interface":"iface1","mac":"00:00:00:20:10:00","dns":{}},{"name":"ns1/tenantnetwork","interface":"newiface","mac":"02:03:04:05:06:07","dns":{}}]`), + Entry("result with IPs", []nadv1.NetworkStatus{ + { + Name: "net1", + Interface: "iface1", + Mac: "00:00:00:20:10:00", + }}, + []string{"10.10.10.10/24"}, + `[{"name":"net1","interface":"iface1","mac":"00:00:00:20:10:00","dns":{}},{"name":"ns1/tenantnetwork","interface":"newiface","ips":["10.10.10.10"],"mac":"02:03:04:05:06:07","dns":{}}]`)) + + DescribeTable("remove an interface to the current network status", func(initialNetStatus []nadv1.NetworkStatus, networkName, ifaceToRemove, expectedNetworkStatus string) { + Expect( + DeleteDynamicIfaceFromStatus( + newPod(podName, namespace, initialNetStatus...), + newNetworkSelectionElementWithIface(networkName, ifaceToRemove, namespace), + ), + ).To(Equal(expectedNetworkStatus)) + }, + Entry("when there aren't any existing interfaces", nil, "net1", "iface1", "[]"), + Entry("when we remove all the currently existing interfaces", []nadv1.NetworkStatus{ + { + Name: NamespacedName(namespace, networkName), + Interface: "iface1", + Mac: "00:00:00:20:10:00", + }}, networkName, "iface1", "[]"), + Entry("when there is *not* a matching interface to remove", []nadv1.NetworkStatus{ + { + Name: NamespacedName(namespace, networkName), + Interface: "iface1", + Mac: "00:00:00:20:10:00", + }}, "net2", "iface1", `[{"name":"ns1/tenantnetwork","interface":"iface1","mac":"00:00:00:20:10:00","dns":{}}]`), + Entry("when we remove one of the existing interfaces", []nadv1.NetworkStatus{ + { + Name: NamespacedName(namespace, networkName), + Interface: "iface1", + Mac: "00:00:00:20:10:00", + }, + { + Name: NamespacedName(namespace, "net2"), + Interface: "iface2", + Mac: "aa:bb:cc:20:10:00", + }, + }, "net2", "iface2", `[{"name":"ns1/tenantnetwork","interface":"iface1","mac":"00:00:00:20:10:00","dns":{}}]`)) +}) + +func newPod(podName string, namespace string, netStatus ...nadv1.NetworkStatus) *corev1.Pod { + status, err := json.Marshal(netStatus) + if err != nil { + return nil + } + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + Namespace: namespace, + Annotations: map[string]string{ + nadv1.NetworkStatusAnnot: string(status), + }, + }, + } +} + +func newResponse(ifaceName string, macAddr string, ips ...string) *api.Response { + var ipConfs []*cni100.IPConfig + for i := range ips { + ipConfs = append(ipConfs, &cni100.IPConfig{Address: *ipNet(ips[i])}) + } + + const sandboxPath = "/over/there" + ifaces := []*cni100.Interface{{ + Name: ifaceName, + Mac: macAddr, + Sandbox: sandboxPath, + }} + return &api.Response{ + Result: &cni100.Result{ + CNIVersion: "1.0.0", + Interfaces: ifaces, + IPs: ipConfs, + }} +} + +func ipNet(ipString string) *net.IPNet { + ip, network, err := net.ParseCIDR(ipString) + if err != nil { + return nil + } + network.IP = ip + return network +} diff --git a/pkg/annotations/network-selection-elements_test.go b/pkg/annotations/network-selection-elements_test.go index 76e0630d..e6a2e8a7 100644 --- a/pkg/annotations/network-selection-elements_test.go +++ b/pkg/annotations/network-selection-elements_test.go @@ -1,11 +1,13 @@ package annotations import ( - v1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" "strings" "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + v1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" ) func TestController(t *testing.T) { diff --git a/pkg/controller/pod.go b/pkg/controller/pod.go index 38ca8c73..3b7741d9 100644 --- a/pkg/controller/pod.go +++ b/pkg/controller/pod.go @@ -1,6 +1,7 @@ package controller import ( + "context" "encoding/json" "fmt" "reflect" @@ -8,6 +9,7 @@ import ( "time" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/wait" v1coreinformerfactory "k8s.io/client-go/informers" @@ -174,7 +176,7 @@ func (pnc *PodNetworksController) handleResult(err error, dynamicAttachmentReque currentRetries := pnc.workqueue.NumRequeues(dynamicAttachmentRequest) if currentRetries <= maxRetries { - klog.Errorf("re-queued request for: %v", dynamicAttachmentRequest) + klog.Errorf("re-queued request for: %v. Error: %v", dynamicAttachmentRequest, err) pnc.workqueue.AddRateLimited(dynamicAttachmentRequest) return } @@ -196,7 +198,7 @@ func (pnc *PodNetworksController) handlePodUpdate(oldObj interface{}, newObj int } podNamespace := oldPod.GetNamespace() podName := oldPod.GetName() - klog.V(logging.Debug).Infof("pod [%s] updated", namespacedName(podNamespace, podName)) + klog.V(logging.Debug).Infof("pod [%s] updated", annotations.NamespacedName(podNamespace, podName)) oldNetworkSelectionElements, err := networkSelectionElements(oldPod.Annotations, podNamespace) if err != nil { @@ -211,7 +213,7 @@ func (pnc *PodNetworksController) handlePodUpdate(oldObj interface{}, newObj int } toAdd := exclusiveNetworks(newNetworkSelectionElements, oldNetworkSelectionElements) - klog.Infof("%d attachments to add to pod %s", len(toAdd), namespacedName(podNamespace, podName)) + klog.Infof("%d attachments to add to pod %s", len(toAdd), annotations.NamespacedName(podNamespace, podName)) netnsPath, err := pnc.netnsPath(newPod) if err != nil { @@ -230,7 +232,7 @@ func (pnc *PodNetworksController) handlePodUpdate(oldObj interface{}, newObj int } toRemove := exclusiveNetworks(oldNetworkSelectionElements, newNetworkSelectionElements) - klog.Infof("%d attachments to remove from pod %s", len(toRemove), namespacedName(podNamespace, podName)) + klog.Infof("%d attachments to remove from pod %s", len(toRemove), annotations.NamespacedName(podNamespace, podName)) if len(toRemove) > 0 { pnc.workqueue.Add( &DynamicAttachmentRequest{ @@ -243,10 +245,6 @@ func (pnc *PodNetworksController) handlePodUpdate(oldObj interface{}, newObj int } } -func namespacedName(podNamespace string, podName string) string { - return fmt.Sprintf("%s/%s", podNamespace, podName) -} - func (pnc *PodNetworksController) addNetworks(dynamicAttachmentRequest *DynamicAttachmentRequest, pod *corev1.Pod) error { for i := range dynamicAttachmentRequest.AttachmentNames { netToAdd := dynamicAttachmentRequest.AttachmentNames[i] @@ -274,6 +272,15 @@ func (pnc *PodNetworksController) addNetworks(dynamicAttachmentRequest *DynamicA } klog.Infof("response: %v", *response.Result) + newIfaceStatus, err := annotations.AddDynamicIfaceToStatus(pod, netToAdd, response) + if err != nil { + return fmt.Errorf("failed to compute the updated network status: %v", err) + } + + if err := pnc.updatePodNetworkStatus(pod, newIfaceStatus); err != nil { + return err + } + pnc.Eventf(pod, corev1.EventTypeNormal, "AddedInterface", addIfaceEventFormat(pod, netToAdd)) } @@ -307,12 +314,34 @@ func (pnc *PodNetworksController) removeNetworks(dynamicAttachmentRequest *Dynam } klog.Infof("response: %v", *response) + newIfaceStatus, err := annotations.DeleteDynamicIfaceFromStatus(pod, netToRemove) + if err != nil { + return fmt.Errorf( + "failed to compute the dynamic network attachments after deleting network: %s, iface: %s: %v", + netToRemove.Name, + netToRemove.InterfaceRequest, + err, + ) + } + if err := pnc.updatePodNetworkStatus(pod, newIfaceStatus); err != nil { + return err + } + pnc.Eventf(pod, corev1.EventTypeNormal, "RemovedInterface", removeIfaceEventFormat(pod, netToRemove)) } return nil } +func (pnc *PodNetworksController) updatePodNetworkStatus(pod *corev1.Pod, newIfaceStatus string) error { + pod.Annotations[nadv1.NetworkStatusAnnot] = newIfaceStatus + + if _, err := pnc.k8sClientSet.CoreV1().Pods(pod.GetNamespace()).Update(context.Background(), pod, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("failed to update pod's network-status annotations for %s: %v", pod.GetName(), err) + } + return nil +} + func networkSelectionElements(podAnnotations map[string]string, podNamespace string) ([]*nadv1.NetworkSelectionElement, error) { podNetworks, ok := podAnnotations[nadv1.NetworkAttachmentAnnot] if !ok { @@ -408,7 +437,7 @@ func podContainerID(pod *corev1.Pod) string { func addIfaceEventFormat(pod *corev1.Pod, network *nadv1.NetworkSelectionElement) string { return fmt.Sprintf( "pod [%s]: added interface %s to network: %s", - namespacedName(pod.GetNamespace(), pod.GetName()), + annotations.NamespacedName(pod.GetNamespace(), pod.GetName()), network.InterfaceRequest, network.Name, ) @@ -417,7 +446,7 @@ func addIfaceEventFormat(pod *corev1.Pod, network *nadv1.NetworkSelectionElement func removeIfaceEventFormat(pod *corev1.Pod, network *nadv1.NetworkSelectionElement) string { return fmt.Sprintf( "pod [%s]: removed interface %s from network: %s", - namespacedName(pod.GetNamespace(), pod.GetName()), + annotations.NamespacedName(pod.GetNamespace(), pod.GetName()), network.InterfaceRequest, network.Name, ) diff --git a/pkg/controller/pod_test.go b/pkg/controller/pod_test.go index 378f8037..5955b3b3 100644 --- a/pkg/controller/pod_test.go +++ b/pkg/controller/pod_test.go @@ -29,6 +29,7 @@ import ( nadinformers "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions" multusapi "gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/server/api" + "github.com/maiqueb/multus-dynamic-networks-controller/pkg/annotations" "github.com/maiqueb/multus-dynamic-networks-controller/pkg/cri" fakecri "github.com/maiqueb/multus-dynamic-networks-controller/pkg/cri/fake" "github.com/maiqueb/multus-dynamic-networks-controller/pkg/multuscni" @@ -121,8 +122,8 @@ var _ = Describe("Dynamic Attachment controller", func() { return status }()).Should( And( - WithTransform(networkStatusNames, ContainElements(namespacedName(namespace, networkName))), - Not(WithTransform(networkStatusNames, ContainElements(namespacedName(namespace, networkToAdd))))), + WithTransform(networkStatusNames, ContainElements(annotations.NamespacedName(namespace, networkName))), + Not(WithTransform(networkStatusNames, ContainElements(annotations.NamespacedName(namespace, networkToAdd))))), ) }) @@ -143,7 +144,7 @@ var _ = Describe("Dynamic Attachment controller", func() { It("an `AddedInterface` event is seen in the event recorded ", func() { expectedEventPayload := fmt.Sprintf( "Normal AddedInterface pod [%s]: added interface %s to network: %s", - namespacedName(namespace, podName), + annotations.NamespacedName(namespace, podName), "net1", networkToAdd, ) @@ -164,7 +165,7 @@ var _ = Describe("Dynamic Attachment controller", func() { It("an `RemovedInterface` event is seen in the event recorded ", func() { expectedEventPayload := fmt.Sprintf( "Normal RemovedInterface pod [%s]: removed interface %s from network: %s", - namespacedName(namespace, podName), + annotations.NamespacedName(namespace, podName), "net0", networkName, )