Delete only 1 cilium pod

Signed-off-by: naoki-take <[email protected]>
cybozu-go · Sep 4, 2024 · 4333b7f · 4333b7f
1 parent 59fdc63
commit 4333b7f
Show file tree

Hide file tree

Showing 3 changed files with 72 additions and 11 deletions.
diff --git a/dctest/before_test.go b/dctest/before_test.go
@@ -22,6 +22,8 @@ func runBeforeSuite() {
 
 	SetDefaultEventuallyPollingInterval(time.Second)
 	SetDefaultEventuallyTimeout(10 * time.Minute)
+	SetDefaultConsistentlyDuration(time.Second)
+	SetDefaultConsistentlyPollingInterval(100 * time.Millisecond)
 
 	data, err := os.ReadFile(machinesFile)
 	Expect(err).NotTo(HaveOccurred())

diff --git a/dctest/l4lb_test.go b/dctest/l4lb_test.go
@@ -78,6 +78,14 @@ func testL4LB() {
 
 			return exec.Command("ip", "netns", "exec", "external", "curl", targetIPForLocal, "-m", "5").Run()
 		}).Should(Succeed())
+		Consistently(func() error {
+			err := exec.Command("ip", "netns", "exec", "external", "curl", targetIP, "-m", "5").Run()
+			if err != nil {
+				return err
+			}
+
+			return exec.Command("ip", "netns", "exec", "external", "curl", targetIPForLocal, "-m", "5").Run()
+		}).Should(Succeed())
 
 		By("access service from external(Inbound packets have the tos)")
 		Expect(exec.Command("ip", "netns", "exec", "external",
@@ -90,6 +98,14 @@ func testL4LB() {
 
 			return exec.Command("ip", "netns", "exec", "external", "curl", targetIPForLocal, "-m", "5").Run()
 		}).Should(Succeed())
+		Consistently(func() error {
+			err := exec.Command("ip", "netns", "exec", "external", "curl", targetIP, "-m", "5").Run()
+			if err != nil {
+				return err
+			}
+
+			return exec.Command("ip", "netns", "exec", "external", "curl", targetIPForLocal, "-m", "5").Run()
+		}).Should(Succeed())
 		Expect(exec.Command("ip", "netns", "exec", "external",
 			"iptables", "-t", "mangle", "-D", "OUTPUT", "-p", "TCP", "--dport", "80", "-j", "TOS", "--set-tos", "0x20").Run()).ShouldNot(HaveOccurred())
 
@@ -131,20 +147,26 @@ func testL4LB() {
 		Expect(err).NotTo(HaveOccurred(), "stderr: %s", stderr)
 
 		By("access service from a Pod")
+		stdout, stderr, err = execAt(bootServers[0], "kubectl", "-n", ns, "get", "pods", "-l", "app.kubernetes.io/name=ubuntu-l4lb-client", "-o", "json")
+		Expect(err).ShouldNot(HaveOccurred(), "stdout=%s, stderr=%s", stdout, stderr)
+		podList = &corev1.PodList{}
+		err = json.Unmarshal(stdout, podList)
+		Expect(err).ShouldNot(HaveOccurred())
+		Expect(len(podList.Items)).To(Equal(1))
+		podName := podList.Items[0].Name
+
 		Eventually(func() error {
-			stdout, stderr, err := execAt(bootServers[0], "kubectl", "-n", ns, "get", "pods", "-l", "app.kubernetes.io/name=ubuntu-l4lb-client", "-o", "json")
+			stdout, stderr, err = execAt(bootServers[0], "kubectl", "exec", "-n", ns, podName, "--", "curl", targetIP, "-m", "5")
 			if err != nil {
 				return fmt.Errorf("stdout: %s, stderr: %s, err: %v", stdout, stderr, err)
 			}
-			podList := &corev1.PodList{}
-			if err := json.Unmarshal(stdout, podList); err != nil {
-				return err
-			}
-			if len(podList.Items) != 1 {
-				return fmt.Errorf("podList length is not 1: %d", len(podList.Items))
+			stdout, stderr, err = execAt(bootServers[0], "kubectl", "exec", "-n", ns, podName, "--", "curl", targetIPForLocal, "-m", "5")
+			if err != nil {
+				return fmt.Errorf("stdout: %s, stderr: %s, err: %v", stdout, stderr, err)
 			}
-			podName := podList.Items[0].Name
-
+			return nil
+		}).Should(Succeed())
+		Consistently(func() error {
 			stdout, stderr, err = execAt(bootServers[0], "kubectl", "exec", "-n", ns, podName, "--", "curl", targetIP, "-m", "5")
 			if err != nil {
 				return fmt.Errorf("stdout: %s, stderr: %s, err: %v", stdout, stderr, err)

diff --git a/dctest/upgrade_test.go b/dctest/upgrade_test.go
@@ -173,7 +173,14 @@ func testUpgrade() {
 	})
 
 	It("should update cilium-agent", func() {
-		stdout, stderr, err := execAt(bootServers[0], "kubectl", "delete", "pod", "-n=kube-system", "-l=app.kubernetes.io/name=cilium-agent")
+		stdout, stderr, err := execAt(bootServers[0], "kubectl", "-n=kube-system", "get", "pods", "-l=app.kubernetes.io/name=cilium-agent", "-o=json")
+		Expect(err).NotTo(HaveOccurred(), "stdout=%s, stderr=%s", stdout, stderr)
+		podList := new(corev1.PodList)
+		err = json.Unmarshal(stdout, podList)
+		Expect(err).NotTo(HaveOccurred(), "data=%s", stdout)
+		Expect(len(podList.Items)).To(BeNumerically(">", 0))
+		podName := podList.Items[0].Name
+		stdout, stderr, err = execAt(bootServers[0], "kubectl", "delete", "pod", "-n=kube-system", podName)
 		Expect(err).ShouldNot(HaveOccurred(), "stdout=%s, stderr=%s", stdout, stderr)
 	})
 
@@ -273,7 +280,7 @@ func testUpgrade() {
 				case "squid-exporter":
 					return checkVersionInDeployment("internet-egress", "squid", newImage)
 				case "cilium":
-					return checkVersionInDaemonSet("kube-system", "cilium", newImage)
+					return checkVersionInDaemonSetPartial("kube-system", "cilium", newImage, 1)
 				case "cilium-operator-generic":
 					return checkVersionInDeployment("kube-system", "cilium-operator", newImage)
 				case "hubble-relay":
@@ -443,6 +450,36 @@ func checkVersionInDaemonSet(namespace, dsName, image string) error {
 	return nil
 }
 
+func checkVersionInDaemonSetPartial(namespace, dsName, image string, desiredNumber int32) error {
+	stdout, _, err := execAt(bootServers[0], "kubectl", "get", "ds", "-n", namespace, dsName, "-o", "json")
+	if err != nil {
+		return err
+	}
+	ds := new(appsv1.DaemonSet)
+	err = json.Unmarshal(stdout, ds)
+	if err != nil {
+		return err
+	}
+	found := false
+	for _, c := range ds.Spec.Template.Spec.Containers {
+		if c.Image == image {
+			found = true
+		}
+	}
+	if !found {
+		return fmt.Errorf("%s not found in %s", image, dsName)
+	}
+	if ds.Status.DesiredNumberScheduled != ds.Status.NumberAvailable {
+		return fmt.Errorf("%s %s is not updated completely. desired number scheduled is %d, but actual available is %d",
+			dsName, image, ds.Status.DesiredNumberScheduled, ds.Status.NumberAvailable)
+	}
+	if desiredNumber != ds.Status.UpdatedNumberScheduled {
+		return fmt.Errorf("%s %s is not updated completely. desired number scheduled is %d, but actual updated is %d",
+			dsName, image, desiredNumber, ds.Status.UpdatedNumberScheduled)
+	}
+	return nil
+}
+
 func checkVersionInDeployment(namespace, deploymentName, image string) error {
 	stdout, _, err := execAt(bootServers[0], "kubectl", "get", "deployment", "-n", namespace, deploymentName, "-o", "json")
 	if err != nil {