controller: generate stable annotations for pod routes

Signed-off-by: zhangzujian <[email protected]>

pkg/apis/kubeovn/v1/vpc.go

@@ -69,6 +69,10 @@ type BFDPort struct {
 	NodeSelector *metav1.LabelSelector `json:"nodeSelector,omitempty"`
 }
 
+func (p *BFDPort) IsEnabled() bool {
+	return p != nil && p.Enabled
+}
+
 type VpcPeering struct {
 	RemoteVpc      string `json:"remoteVpc,omitempty"`
 	LocalConnectIP string `json:"localConnectIP,omitempty"`
@@ -98,6 +102,14 @@ type BFDPortStatus struct {
 	Nodes []string `json:"nodes,omitempty"`
 }
 
+func (s BFDPortStatus) IsEmpty() bool {
+	return s.Name == "" && s.IP == "" && len(s.Nodes) == 0
+}
+
+func (s *BFDPortStatus) Clear() {
+	s.Name, s.IP, s.Nodes = "", "", nil
+}
+
 type VpcStatus struct {
 	// Conditions represents the latest state of the object
 	// +optional

pkg/controller/vpc.go

@@ -154,6 +154,9 @@ func (c *Controller) handleUpdateVpcStatus(key string) error {
 
 	vpc.Status.DefaultLogicalSwitch = defaultSubnet
 	vpc.Status.Subnets = subnets
+	if !vpc.Spec.BFDPort.IsEnabled() && !vpc.Status.BFDPort.IsEmpty() {
+		vpc.Status.BFDPort.Clear()
+	}
 	bytes, err := vpc.Status.Bytes()
 	if err != nil {
 		klog.Error(err)

pkg/util/pod_routes.go

@@ -1,8 +1,10 @@
 package util
 
 import (
+	"cmp"
 	"encoding/json"
 	"fmt"
+	"slices"
 
 	"github.com/kubeovn/kube-ovn/pkg/request"
 )
@@ -48,6 +50,14 @@ func (r PodRoutes) ToAnnotations() (map[string]string, error) {
 			continue
 		}
 
+		// sort routes to ensure the result is stable
+		slices.SortFunc(routes, func(a, b request.Route) int {
+			if n := cmp.Compare(a.Destination, b.Destination); n != 0 {
+				return n
+			}
+			return cmp.Compare(a.Gateway, b.Gateway)
+		})
+
 		// no error will be returned here
 		buf, _ := json.Marshal(routes)
 		annotations[fmt.Sprintf(RoutesAnnotationTemplate, provider)] = string(buf)

pkg/util/pod_routes_test.go

@@ -1,6 +1,7 @@
 package util
 
 import (
+	"fmt"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -24,6 +25,10 @@ func TestPodRoutes(t *testing.T) {
 	annotations, err = routes.ToAnnotations()
 	require.NoError(t, err)
 	require.Len(t, annotations, 1)
+	require.Equal(t,
+		annotations[fmt.Sprintf(RoutesAnnotationTemplate, "foo")],
+		`[{"dst":"0.0.0.1","gw":"1.1.1.1"},{"dst":"0.0.1.0/24","gw":"1.1.1.1"},{"dst":"0.1.0.0/16","gw":"1.1.1.2"}]`,
+	)
 
 	routes.Add("foo", "0.0.0.1", "")
 	routes.Add("foo", "", "1.1.1.3")

test/e2e/framework/pod.go

@@ -175,6 +175,6 @@ func CheckPodEgressRoutes(ns, pod string, ipv4, ipv6 bool, ttl int, expectedHops
 			lines := strings.Split(strings.TrimSpace(output), "\n")
 			fields := strings.Fields(lines[len(lines)-1])
 			return len(fields) > 2 && slices.Contains(expectedHops, fields[1]), nil
-		}, "")
+		}, fmt.Sprintf("expected hops: %s", strings.Join(expectedHops, ", ")))
 	}
 }

test/e2e/vpc-egress-gateway/e2e_test.go

@@ -357,6 +357,8 @@ func generateSubnetFromDockerNetwork(subnetName string, network *dockernetwork.I
 }
 
 func checkEgressAccess(f *framework.Framework, namespaceName, svrPodName, image, svrPort string, svrIPs, intIPs, extIPs []string, subnetName string, snat bool) {
+	ginkgo.GinkgoHelper()
+
 	podName := "pod-" + framework.RandomSuffix()
 	ginkgo.By("Creating client pod " + podName + " within subnet " + subnetName)
 	annotations := map[string]string{util.LogicalSwitchAnnotation: subnetName}