From 35181c228070766e779996ad6e4f1c24a7747ef7 Mon Sep 17 00:00:00 2001 From: David Kornel Date: Wed, 28 Aug 2024 09:40:12 +0200 Subject: [PATCH] Delete wait async batch (#174) ## Description Wait async when deletion resources. During experiments with streams-e2e I was able to save more then `3min` (per test) when I had to delete all operators etc... I also changed log4j2.properties to log thread ID instead of name to do not mess log output. ## Type of Change Please delete options that are not relevant. * New feature (non-breaking change which adds functionality) ## Checklist - [x] My code follows the style guidelines of this project - [x] I have performed a self-review of my own code - [x] I have commented my code, particularly in hard-to-understand areas - [x] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [x] I have added tests that prove my fix is effective or that my feature works - [x] New and existing unit/integration tests pass locally with my changes Signed-off-by: David Kornel --- .../resources/KubeResourceManager.java | 31 ++++++++++++++----- .../src/main/resources/log4j2.properties | 2 +- .../src/main/resources/log4j2.properties | 2 +- .../src/main/resources/log4j2.properties | 2 +- 4 files changed, 26 insertions(+), 11 deletions(-) diff --git a/test-frame-common/src/main/java/io/skodjob/testframe/resources/KubeResourceManager.java b/test-frame-common/src/main/java/io/skodjob/testframe/resources/KubeResourceManager.java index 66517ba..6701a4d 100644 --- a/test-frame-common/src/main/java/io/skodjob/testframe/resources/KubeResourceManager.java +++ b/test-frame-common/src/main/java/io/skodjob/testframe/resources/KubeResourceManager.java @@ -372,20 +372,23 @@ private void createOrUpdateResource(boolean async, */ @SafeVarargs public final void deleteResource(T... resources) { + List> waitExecutors = new LinkedList<>(); for (T resource : resources) { ResourceType type = findResourceType(resource); LoggerUtils.logResource("Deleting", resource); try { if (type == null) { client.getClient().resource(resource).delete(); - assertTrue(waitResourceCondition(resource, ResourceCondition.deletion()), - String.format("Timed out deleting %s/%s in %s", resource.getKind(), - resource.getMetadata().getName(), resource.getMetadata().getNamespace())); + waitExecutors.add(CompletableFuture.runAsync(() -> + assertTrue(waitResourceCondition(resource, ResourceCondition.deletion()), + String.format("Timed out deleting %s/%s in %s", resource.getKind(), + resource.getMetadata().getName(), resource.getMetadata().getNamespace())))); } else { type.delete(resource); - assertTrue(waitResourceCondition(resource, ResourceCondition.deletion()), - String.format("Timed out deleting %s/%s in %s", resource.getKind(), - resource.getMetadata().getName(), resource.getMetadata().getNamespace())); + waitExecutors.add(CompletableFuture.runAsync(() -> + assertTrue(waitResourceCondition(resource, ResourceCondition.deletion()), + String.format("Timed out deleting %s/%s in %s", resource.getKind(), + resource.getMetadata().getName(), resource.getMetadata().getNamespace())))); } } catch (Exception e) { if (resource.getMetadata().getNamespace() == null) { @@ -396,6 +399,9 @@ public final void deleteResource(T... resources) { resource.getMetadata().getName(), resource.getMetadata().getNamespace(), e); } } + if (!waitExecutors.isEmpty()) { + CompletableFuture.allOf(waitExecutors.toArray(new CompletableFuture[0])).join(); + } deleteCallbacks.forEach(callback -> callback.accept(resource)); } } @@ -474,12 +480,18 @@ public void deleteResources() { new AtomicInteger(0); while (STORED_RESOURCES.containsKey(getTestContext().getDisplayName()) && numberOfResources.get() > 0) { Stack> s = STORED_RESOURCES.get(getTestContext().getDisplayName()); - + List> waitExecutors = new LinkedList<>(); while (!s.isEmpty()) { ResourceItem resourceItem = s.pop(); try { - resourceItem.throwableRunner().run(); + waitExecutors.add(CompletableFuture.runAsync(() -> { + try { + resourceItem.throwableRunner().run(); + } catch (Exception e) { + throw new RuntimeException(e); + } + })); } catch (Exception e) { LOGGER.error(e.getMessage(), e); } @@ -490,6 +502,9 @@ public void deleteResources() { } }); } + if (!waitExecutors.isEmpty()) { + CompletableFuture.allOf(waitExecutors.toArray(new CompletableFuture[0])).join(); + } } STORED_RESOURCES.remove(getTestContext().getDisplayName()); LoggerUtils.logSeparator(); diff --git a/test-frame-common/src/main/resources/log4j2.properties b/test-frame-common/src/main/resources/log4j2.properties index d89b4aa..c03dd25 100644 --- a/test-frame-common/src/main/resources/log4j2.properties +++ b/test-frame-common/src/main/resources/log4j2.properties @@ -3,7 +3,7 @@ name = TFConfig appender.console.type = Console appender.console.name = STDOUT appender.console.layout.type = PatternLayout -appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss}{GMT} [%thread] %highlight{%-5p} [%c{1}:%L] %m%n +appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss}{GMT} [%tid] %highlight{%-5p} [%c{1}:%L] %m%n appender.rolling.type = RollingFile appender.rolling.name = RollingFile diff --git a/test-frame-log-collector/src/main/resources/log4j2.properties b/test-frame-log-collector/src/main/resources/log4j2.properties index 49129d8..256f35a 100644 --- a/test-frame-log-collector/src/main/resources/log4j2.properties +++ b/test-frame-log-collector/src/main/resources/log4j2.properties @@ -3,7 +3,7 @@ name = LCConfig appender.console.type = Console appender.console.name = STDOUT appender.console.layout.type = PatternLayout -appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss}{GMT} [%thread] %highlight{%-5p} [%c{1}:%L] %m%n +appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss}{GMT} [%tid] %highlight{%-5p} [%c{1}:%L] %m%n appender.rolling.type = RollingFile appender.rolling.name = RollingFile diff --git a/test-frame-metrics-collector/src/main/resources/log4j2.properties b/test-frame-metrics-collector/src/main/resources/log4j2.properties index b1fdf17..9d52d5d 100644 --- a/test-frame-metrics-collector/src/main/resources/log4j2.properties +++ b/test-frame-metrics-collector/src/main/resources/log4j2.properties @@ -3,7 +3,7 @@ name = MCConfig appender.console.type = Console appender.console.name = STDOUT appender.console.layout.type = PatternLayout -appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss}{GMT} [%thread] %highlight{%-5p} [%c{1}:%L] %m%n +appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss}{GMT} [%tid] %highlight{%-5p} [%c{1}:%L] %m%n appender.rolling.type = RollingFile appender.rolling.name = RollingFile