From 11fc5d1ebe7b3e1619b1bc7293d8782ed807f527 Mon Sep 17 00:00:00 2001
From: erabii <eugen.rabii@gmail.com>
Date: Thu, 5 Oct 2023 15:41:29 +0300
Subject: [PATCH] fabric8-configmap-event-reload-patch-refactor part 1 (#1464)

---
 .../event/reload/ConfigMapEventReloadIT.java  | 156 ++++++++---------
 .../configmap/event/reload/TestUtil.java      | 161 ++++++++++++++++++
 .../src/test/resources/deployment.yaml        |   6 +-
 3 files changed, 232 insertions(+), 91 deletions(-)
 create mode 100644 spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-configmap-event-reload/src/test/java/org/springframework/cloud/kubernetes/fabric8/configmap/event/reload/TestUtil.java

diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-configmap-event-reload/src/test/java/org/springframework/cloud/kubernetes/fabric8/configmap/event/reload/ConfigMapEventReloadIT.java b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-configmap-event-reload/src/test/java/org/springframework/cloud/kubernetes/fabric8/configmap/event/reload/ConfigMapEventReloadIT.java
index 5029dbf8b6..8476daaaa3 100644
--- a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-configmap-event-reload/src/test/java/org/springframework/cloud/kubernetes/fabric8/configmap/event/reload/ConfigMapEventReloadIT.java
+++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-configmap-event-reload/src/test/java/org/springframework/cloud/kubernetes/fabric8/configmap/event/reload/ConfigMapEventReloadIT.java
@@ -18,18 +18,11 @@
 
 import java.io.InputStream;
 import java.time.Duration;
-import java.util.ArrayList;
-import java.util.List;
 import java.util.Map;
-import java.util.Objects;
 import java.util.Set;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.locks.LockSupport;
 
 import io.fabric8.kubernetes.api.model.ConfigMap;
 import io.fabric8.kubernetes.api.model.ConfigMapBuilder;
-import io.fabric8.kubernetes.api.model.EnvVar;
-import io.fabric8.kubernetes.api.model.EnvVarBuilder;
 import io.fabric8.kubernetes.api.model.ObjectMetaBuilder;
 import io.fabric8.kubernetes.api.model.Service;
 import io.fabric8.kubernetes.api.model.apps.Deployment;
@@ -41,18 +34,22 @@
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
 import org.testcontainers.k3s.K3sContainer;
-import reactor.netty.http.client.HttpClient;
-import reactor.util.retry.Retry;
-import reactor.util.retry.RetryBackoffSpec;
 
 import org.springframework.cloud.kubernetes.integration.tests.commons.Commons;
 import org.springframework.cloud.kubernetes.integration.tests.commons.Phase;
 import org.springframework.cloud.kubernetes.integration.tests.commons.fabric8_client.Util;
 import org.springframework.http.HttpMethod;
-import org.springframework.http.client.reactive.ReactorClientHttpConnector;
 import org.springframework.web.reactive.function.client.WebClient;
 
 import static org.awaitility.Awaitility.await;
+import static org.springframework.cloud.kubernetes.fabric8.configmap.event.reload.TestUtil.builder;
+import static org.springframework.cloud.kubernetes.fabric8.configmap.event.reload.TestUtil.patchOne;
+import static org.springframework.cloud.kubernetes.fabric8.configmap.event.reload.TestUtil.patchThree;
+import static org.springframework.cloud.kubernetes.fabric8.configmap.event.reload.TestUtil.patchTwo;
+import static org.springframework.cloud.kubernetes.fabric8.configmap.event.reload.TestUtil.reCreateConfigMaps;
+import static org.springframework.cloud.kubernetes.fabric8.configmap.event.reload.TestUtil.replaceConfigMap;
+import static org.springframework.cloud.kubernetes.fabric8.configmap.event.reload.TestUtil.retrySpec;
+import static org.springframework.cloud.kubernetes.integration.tests.commons.Commons.pomVersion;
 
 /**
  * @author wind57
@@ -61,6 +58,8 @@ class ConfigMapEventReloadIT {
 
 	private static final String IMAGE_NAME = "spring-cloud-kubernetes-fabric8-client-configmap-event-reload";
 
+	private static final String DOCKER_IMAGE = "docker.io/springcloud/" + IMAGE_NAME + ":" + pomVersion();
+
 	private static final String NAMESPACE = "default";
 
 	private static final K3sContainer K3S = Commons.container();
@@ -81,6 +80,8 @@ static void beforeAll() throws Exception {
 		util.createNamespace("left");
 		util.createNamespace("right");
 		util.setUpClusterWide(NAMESPACE, Set.of("left", "right"));
+
+		manifests(Phase.CREATE);
 	}
 
 	@AfterAll
@@ -89,6 +90,8 @@ static void afterAll() throws Exception {
 		util.deleteNamespace("right");
 		Commons.cleanUp(IMAGE_NAME, K3S);
 		Commons.systemPrune();
+
+		manifests(Phase.DELETE);
 	}
 
 	/**
@@ -101,7 +104,6 @@ static void afterAll() throws Exception {
 	 */
 	@Test
 	void testInformFromOneNamespaceEventNotTriggered() {
-		manifests("one", Phase.CREATE, false);
 		Commons.assertReloadLogStatements("added configmap informer for namespace",
 				"added secret informer for namespace", IMAGE_NAME);
 
@@ -122,30 +124,37 @@ void testInformFromOneNamespaceEventNotTriggered() {
 				.withMetadata(new ObjectMetaBuilder().withNamespace("right").withName("right-configmap").build())
 				.withData(Map.of("right.value", "right-after-change")).build();
 
-		replaceConfigMap(rightConfigMapAfterChange);
-
-		// wait dummy for 5 seconds
-		LockSupport.parkNanos(TimeUnit.SECONDS.toNanos(5));
+		replaceConfigMap(client, rightConfigMapAfterChange, "right");
 
 		webClient = builder().baseUrl("http://localhost/left").build();
-		result = webClient.method(HttpMethod.GET).retrieve().bodyToMono(String.class).retryWhen(retrySpec()).block();
-		// left configmap has not changed, no restart of app has happened
-		Assertions.assertEquals("left-initial", result);
 
-		manifests("one", Phase.DELETE, false);
+		WebClient finalWebClient = webClient;
+		await().pollInterval(Duration.ofSeconds(1)).atMost(Duration.ofSeconds(30)).until(() -> {
+			String innerResult = finalWebClient.method(HttpMethod.GET).retrieve().bodyToMono(String.class)
+					.retryWhen(retrySpec()).block();
+			// left configmap has not changed, no restart of app has happened
+			return "left-initial".equals(innerResult);
+		});
+
+		testInformFromOneNamespaceEventTriggered();
+		testInform();
+		testInformFromOneNamespaceEventTriggeredSecretsDisabled();
 	}
 
 	/**
 	 * <pre>
-	 *     - there are two namespaces : left and right
-	 *     - each of the namespaces has one configmap
-	 *     - we watch the "right" namespace and make a change in the configmap in the same namespace
-	 *     - as such, event is triggered and we see the updated value
+	 * - there are two namespaces : left and right
+	 * - each of the namespaces has one configmap
+	 * - we watch the "right" namespace and make a change in the configmap in the same
+	 * namespace
+	 * - as such, event is triggered and we see the updated value
 	 * </pre>
 	 */
-	@Test
 	void testInformFromOneNamespaceEventTriggered() {
-		manifests("two", Phase.CREATE, false);
+
+		reCreateConfigMaps(util, client);
+		patchOne(util, DOCKER_IMAGE, IMAGE_NAME, NAMESPACE);
+
 		Commons.assertReloadLogStatements("added configmap informer for namespace",
 				"added secret informer for namespace", IMAGE_NAME);
 
@@ -160,7 +169,7 @@ void testInformFromOneNamespaceEventTriggered() {
 				.withMetadata(new ObjectMetaBuilder().withNamespace("right").withName("right-configmap").build())
 				.withData(Map.of("right.value", "right-after-change")).build();
 
-		replaceConfigMap(rightConfigMapAfterChange);
+		replaceConfigMap(client, rightConfigMapAfterChange, "right");
 
 		String[] resultAfterChange = new String[1];
 		await().pollInterval(Duration.ofSeconds(3)).atMost(Duration.ofSeconds(90)).until(() -> {
@@ -171,22 +180,23 @@ void testInformFromOneNamespaceEventTriggered() {
 			return innerResult != null;
 		});
 		Assertions.assertEquals("right-after-change", resultAfterChange[0]);
-
-		manifests("two", Phase.DELETE, false);
 	}
 
 	/**
 	 * <pre>
-	 *     - there are two namespaces : left and right (though we do not care about the left one)
-	 *     - left has one configmap : left-configmap
-	 *     - right has two configmaps: right-configmap, right-configmap-with-label
-	 *     - we watch the "right" namespace, but enable tagging; which means that only
-	 *       right-configmap-with-label triggers changes.
+	 * - there are two namespaces : left and right (though we do not care about the left
+	 * one)
+	 * - left has one configmap : left-configmap
+	 * - right has two configmaps: right-configmap, right-configmap-with-label
+	 * - we watch the "right" namespace, but enable tagging; which means that only
+	 * right-configmap-with-label triggers changes.
 	 * </pre>
 	 */
-	@Test
 	void testInform() {
-		manifests("three", Phase.CREATE, false);
+
+		reCreateConfigMaps(util, client);
+		patchTwo(util, DOCKER_IMAGE, IMAGE_NAME, NAMESPACE);
+
 		Commons.assertReloadLogStatements("added configmap informer for namespace",
 				"added secret informer for namespace", IMAGE_NAME);
 
@@ -207,15 +217,14 @@ void testInform() {
 				.withMetadata(new ObjectMetaBuilder().withNamespace("right").withName("right-configmap").build())
 				.withData(Map.of("right.value", "right-after-change")).build();
 
-		replaceConfigMap(rightConfigMapAfterChange);
-
-		// sleep for 5 seconds
-		LockSupport.parkNanos(TimeUnit.SECONDS.toNanos(5));
+		replaceConfigMap(client, rightConfigMapAfterChange, "right");
 
 		// nothing changes in our app, because we are watching only labeled configmaps
-		rightResult = rightWebClient.method(HttpMethod.GET).retrieve().bodyToMono(String.class).retryWhen(retrySpec())
-				.block();
-		Assertions.assertEquals("right-initial", rightResult);
+		await().pollInterval(Duration.ofSeconds(1)).atMost(Duration.ofSeconds(30)).until(() -> {
+			String innerRightResult = rightWebClient.method(HttpMethod.GET).retrieve().bodyToMono(String.class)
+					.retryWhen(retrySpec()).block();
+			return "right-initial".equals(innerRightResult);
+		});
 
 		// then deploy a new version of right-with-label-configmap
 		ConfigMap rightWithLabelConfigMapAfterChange = new ConfigMapBuilder()
@@ -223,7 +232,7 @@ void testInform() {
 						new ObjectMetaBuilder().withNamespace("right").withName("right-configmap-with-label").build())
 				.withData(Map.of("right.with.label.value", "right-with-label-after-change")).build();
 
-		replaceConfigMap(rightWithLabelConfigMapAfterChange);
+		replaceConfigMap(client, rightWithLabelConfigMapAfterChange, "right");
 
 		// since we have changed a labeled configmap, app will restart and pick up the new
 		// value
@@ -242,22 +251,23 @@ void testInform() {
 		rightResult = rightWebClient.method(HttpMethod.GET).retrieve().bodyToMono(String.class).retryWhen(retrySpec())
 				.block();
 		Assertions.assertEquals("right-after-change", rightResult);
-
-		manifests("three", Phase.DELETE, false);
 	}
 
 	/**
 	 * <pre>
-	 *     - there are two namespaces : left and right
-	 *     - each of the namespaces has one configmap
-	 *     - secrets are disabled
-	 *     - we watch the "right" namespace and make a change in the configmap in the same namespace
-	 *     - as such, event is triggered and we see the updated value
+	 * - there are two namespaces : left and right
+	 * - each of the namespaces has one configmap
+	 * - secrets are disabled
+	 * - we watch the "right" namespace and make a change in the configmap in the same
+	 * namespace
+	 * - as such, event is triggered and we see the updated value
 	 * </pre>
 	 */
-	@Test
 	void testInformFromOneNamespaceEventTriggeredSecretsDisabled() {
-		manifests("two", Phase.CREATE, true);
+
+		reCreateConfigMaps(util, client);
+		patchThree(util, DOCKER_IMAGE, IMAGE_NAME, NAMESPACE);
+
 		Commons.assertReloadLogStatements("added configmap informer for namespace",
 				"added secret informer for namespace", IMAGE_NAME);
 
@@ -272,7 +282,7 @@ void testInformFromOneNamespaceEventTriggeredSecretsDisabled() {
 				.withMetadata(new ObjectMetaBuilder().withNamespace("right").withName("right-configmap").build())
 				.withData(Map.of("right.value", "right-after-change")).build();
 
-		replaceConfigMap(rightConfigMapAfterChange);
+		replaceConfigMap(client, rightConfigMapAfterChange, "right");
 
 		String[] resultAfterChange = new String[1];
 		await().pollInterval(Duration.ofSeconds(3)).atMost(Duration.ofSeconds(90)).until(() -> {
@@ -284,10 +294,9 @@ void testInformFromOneNamespaceEventTriggeredSecretsDisabled() {
 		});
 		Assertions.assertEquals("right-after-change", resultAfterChange[0]);
 
-		manifests("two", Phase.DELETE, true);
 	}
 
-	private static void manifests(String activeProfile, Phase phase, boolean secretsDisabled) {
+	private static void manifests(Phase phase) {
 
 		InputStream deploymentStream = util.inputStream("deployment.yaml");
 		InputStream serviceStream = util.inputStream("service.yaml");
@@ -298,21 +307,6 @@ private static void manifests(String activeProfile, Phase phase, boolean secrets
 
 		Deployment deployment = Serialization.unmarshal(deploymentStream, Deployment.class);
 
-		List<EnvVar> envVars = new ArrayList<>(
-				deployment.getSpec().getTemplate().getSpec().getContainers().get(0).getEnv());
-		EnvVar activeProfileProperty = new EnvVarBuilder().withName("SPRING_PROFILES_ACTIVE").withValue(activeProfile)
-				.build();
-		envVars.add(activeProfileProperty);
-
-		if (secretsDisabled) {
-			EnvVar secretsDisabledEnvVar = new EnvVarBuilder().withName("SPRING_CLOUD_KUBERNETES_SECRETS_ENABLED")
-					.withValue("FALSE").build();
-			envVars.add(secretsDisabledEnvVar);
-			deployment.getSpec().getTemplate().getSpec().getContainers().get(0).setEnv(envVars);
-		}
-
-		deployment.getSpec().getTemplate().getSpec().getContainers().get(0).setEnv(envVars);
-
 		Service service = Serialization.unmarshal(serviceStream, Service.class);
 		Ingress ingress = Serialization.unmarshal(ingressStream, Ingress.class);
 		ConfigMap leftConfigMap = Serialization.unmarshal(leftConfigMapStream, ConfigMap.class);
@@ -322,32 +316,16 @@ private static void manifests(String activeProfile, Phase phase, boolean secrets
 		if (phase.equals(Phase.CREATE)) {
 			util.createAndWait("left", leftConfigMap, null);
 			util.createAndWait("right", rightConfigMap, null);
-			if ("three".equals(activeProfile)) {
-				util.createAndWait("right", rightWithLabelConfigMap, null);
-			}
+			util.createAndWait("right", rightWithLabelConfigMap, null);
 			util.createAndWait(NAMESPACE, null, deployment, service, ingress, true);
 		}
 		else {
 			util.deleteAndWait("left", leftConfigMap, null);
 			util.deleteAndWait("right", rightConfigMap, null);
-			if ("three".equals(activeProfile)) {
-				util.deleteAndWait("right", rightWithLabelConfigMap, null);
-			}
+			util.deleteAndWait("right", rightWithLabelConfigMap, null);
 			util.deleteAndWait(NAMESPACE, deployment, service, ingress);
 		}
 
 	}
 
-	private WebClient.Builder builder() {
-		return WebClient.builder().clientConnector(new ReactorClientHttpConnector(HttpClient.create()));
-	}
-
-	private RetryBackoffSpec retrySpec() {
-		return Retry.fixedDelay(120, Duration.ofSeconds(2)).filter(Objects::nonNull);
-	}
-
-	private static void replaceConfigMap(ConfigMap configMap) {
-		client.configMaps().inNamespace("right").resource(configMap).createOrReplace();
-	}
-
 }
diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-configmap-event-reload/src/test/java/org/springframework/cloud/kubernetes/fabric8/configmap/event/reload/TestUtil.java b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-configmap-event-reload/src/test/java/org/springframework/cloud/kubernetes/fabric8/configmap/event/reload/TestUtil.java
new file mode 100644
index 0000000000..992b242eac
--- /dev/null
+++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-configmap-event-reload/src/test/java/org/springframework/cloud/kubernetes/fabric8/configmap/event/reload/TestUtil.java
@@ -0,0 +1,161 @@
+/*
+ * Copyright 2013-2023 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.cloud.kubernetes.fabric8.configmap.event.reload;
+
+import java.io.InputStream;
+import java.time.Duration;
+import java.util.Map;
+import java.util.Objects;
+
+import io.fabric8.kubernetes.api.model.ConfigMap;
+import io.fabric8.kubernetes.client.KubernetesClient;
+import io.fabric8.kubernetes.client.utils.Serialization;
+import reactor.netty.http.client.HttpClient;
+import reactor.util.retry.Retry;
+import reactor.util.retry.RetryBackoffSpec;
+
+import org.springframework.cloud.kubernetes.integration.tests.commons.fabric8_client.Util;
+import org.springframework.http.client.reactive.ReactorClientHttpConnector;
+import org.springframework.web.reactive.function.client.WebClient;
+
+/**
+ * @author wind57
+ */
+final class TestUtil {
+
+	private static final Map<String, String> POD_LABELS = Map.of("app",
+			"spring-cloud-kubernetes-fabric8-client-configmap-event-reload");
+
+	private static final String BODY_ONE = """
+			{
+				"spec": {
+					"template": {
+						"spec": {
+							"containers": [{
+								"name": "spring-cloud-kubernetes-fabric8-client-configmap-event-reload",
+								"image": "image_name_here",
+								"env": [
+								{
+									"name": "LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_CLOUD_KUBERNETES_FABRIC8_DISCOVERY",
+									"value": "DEBUG"
+								},
+								{
+									"name": "SPRING_PROFILES_ACTIVE",
+									"value": "two"
+								}
+								]
+							}]
+						}
+					}
+				}
+			}
+						""";
+
+	private static final String BODY_TWO = """
+			{
+				"spec": {
+					"template": {
+						"spec": {
+							"containers": [{
+								"name": "spring-cloud-kubernetes-fabric8-client-configmap-event-reload",
+								"image": "image_name_here",
+								"env": [
+								{
+									"name": "LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_CLOUD_KUBERNETES_FABRIC8_DISCOVERY",
+									"value": "DEBUG"
+								},
+								{
+									"name": "SPRING_PROFILES_ACTIVE",
+									"value": "three"
+								}
+								]
+							}]
+						}
+					}
+				}
+			}
+						""";
+
+	private static final String BODY_THREE = """
+			{
+				"spec": {
+					"template": {
+						"spec": {
+							"containers": [{
+								"name": "spring-cloud-kubernetes-fabric8-client-configmap-event-reload",
+								"image": "image_name_here",
+								"env": [
+								{
+									"name": "LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_CLOUD_KUBERNETES_FABRIC8_DISCOVERY",
+									"value": "DEBUG"
+								},
+								{
+									"name": "SPRING_PROFILES_ACTIVE",
+									"value": "two"
+								},
+								{
+									"name": "SPRING_CLOUD_KUBERNETES_SECRETS_ENABLED",
+									"value": "FALSE"
+								}
+								]
+							}]
+						}
+					}
+				}
+			}
+						""";
+
+	private TestUtil() {
+
+	}
+
+	static void reCreateConfigMaps(Util util, KubernetesClient client) {
+		InputStream leftConfigMapStream = util.inputStream("left-configmap.yaml");
+		InputStream rightConfigMapStream = util.inputStream("right-configmap.yaml");
+
+		ConfigMap leftConfigMap = Serialization.unmarshal(leftConfigMapStream, ConfigMap.class);
+		ConfigMap rightConfigMap = Serialization.unmarshal(rightConfigMapStream, ConfigMap.class);
+
+		replaceConfigMap(client, leftConfigMap, "left");
+		replaceConfigMap(client, rightConfigMap, "right");
+	}
+
+	static void replaceConfigMap(KubernetesClient client, ConfigMap configMap, String namespace) {
+		client.configMaps().inNamespace(namespace).resource(configMap).createOrReplace();
+	}
+
+	static void patchOne(Util util, String dockerImage, String deploymentName, String namespace) {
+		util.patchWithReplace(dockerImage, deploymentName, namespace, BODY_ONE, POD_LABELS);
+	}
+
+	static void patchTwo(Util util, String dockerImage, String deploymentName, String namespace) {
+		util.patchWithReplace(dockerImage, deploymentName, namespace, BODY_TWO, POD_LABELS);
+	}
+
+	static void patchThree(Util util, String dockerImage, String deploymentName, String namespace) {
+		util.patchWithReplace(dockerImage, deploymentName, namespace, BODY_THREE, POD_LABELS);
+	}
+
+	static WebClient.Builder builder() {
+		return WebClient.builder().clientConnector(new ReactorClientHttpConnector(HttpClient.create()));
+	}
+
+	static RetryBackoffSpec retrySpec() {
+		return Retry.fixedDelay(120, Duration.ofSeconds(2)).filter(Objects::nonNull);
+	}
+
+}
diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-configmap-event-reload/src/test/resources/deployment.yaml b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-configmap-event-reload/src/test/resources/deployment.yaml
index a7fe7eaca3..d234ee8934 100644
--- a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-configmap-event-reload/src/test/resources/deployment.yaml
+++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-configmap-event-reload/src/test/resources/deployment.yaml
@@ -1,7 +1,7 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: spring-cloud-kubernetes-fabric8-client-configmap-deployment-event-reload
+  name: spring-cloud-kubernetes-fabric8-client-configmap-event-reload
 spec:
   selector:
     matchLabels:
@@ -28,4 +28,6 @@ spec:
             - containerPort: 8080
           env:
             - name: LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_CLOUD_KUBERNETES_FABRIC8_CONFIG_RELOAD
-              value: DEBUG
+              value: "DEBUG"
+            - name: SPRING_PROFILES_ACTIVE
+              value: "one"