From fe05cea619b1a5b806d623230a6cb33dcb1ff65e Mon Sep 17 00:00:00 2001 From: Fabio Burzigotti Date: Tue, 22 Feb 2022 16:30:19 +0100 Subject: [PATCH] Adding support to skip tests based on provided xtf..subId --- junit5/pom.xml | 7 ++ .../cz/xtf/junit5/annotations/SkipFor.java | 19 ++- .../junit5/extensions/SkipForCondition.java | 21 ++-- .../extensions/SkipForConditionTest.java | 113 ++++++++++++++++++ 4 files changed, 150 insertions(+), 10 deletions(-) create mode 100644 junit5/src/test/java/cz/xtf/junit5/extensions/SkipForConditionTest.java diff --git a/junit5/pom.xml b/junit5/pom.xml index f2c45a70..83200053 100644 --- a/junit5/pom.xml +++ b/junit5/pom.xml @@ -42,6 +42,13 @@ org.assertj assertj-core + + + uk.org.webcompere + system-stubs-jupiter + 2.0.1 + test + diff --git a/junit5/src/main/java/cz/xtf/junit5/annotations/SkipFor.java b/junit5/src/main/java/cz/xtf/junit5/annotations/SkipFor.java index eb9c77d3..afe4f982 100644 --- a/junit5/src/main/java/cz/xtf/junit5/annotations/SkipFor.java +++ b/junit5/src/main/java/cz/xtf/junit5/annotations/SkipFor.java @@ -19,7 +19,8 @@ /** * Name or regexp pattern matching name of the image. For example "eap73-openjdk11-openshift-rhel8" or "eap73-openjdk11-.*" *

- * Only one of {@code name}, {@code imageMetadataLabelName} and {@code imageMetadataLabelArchitecture} can be presented. + * Only one of {@code name}, {@code imageMetadataLabelName}, {@code imageMetadataLabelArchitecture} and + * {@code subId} can be presented. */ String name() default ""; @@ -27,7 +28,8 @@ * Name or regexp pattern matching name in {@code Docker Labels} in image metadata * For example "jboss-eap-7/eap73-openjdk11-openshift-rhel8" or "eap73-openjdk11-.*". *

- * Only one of {@code name}, {@code imageMetadataLabelName} and {@code imageMetadataLabelArchitecture} can be presented. + * Only one of {@code name}, {@code imageMetadataLabelName}, {@code imageMetadataLabelArchitecture} and + * {@code subId} can be presented. */ String imageMetadataLabelName() default ""; @@ -35,10 +37,21 @@ * Architecture or regexp pattern matching architecture in {@code Docker Labels} in image metadata * For example "x86_64" or "x86_.*". *

- * Only one of {@code name}, {@code imageMetadataLabelName} and {@code imageMetadataLabelArchitecture} can be presented. + * Only one of {@code name}, {@code imageMetadataLabelName}, {@code imageMetadataLabelArchitecture} and + * {@code subId} can be presented. */ String imageMetadataLabelArchitecture() default ""; + /** + * Name or regexp pattern matching the value of the {@code xtf.>.subid} XTF property + * For example, ".*74.*" will match and skip the annotated test when xtf.eap.subid is set to "74-openjdk11", + * "74-openj9-11" or "74". + *

+ * Only one of {@code imageMetadataLabelName}, {@code name}, {@code imageMetadataLabelArchitecture} and + * {@code subId} can be presented. + */ + String subId() default ""; + String image(); String reason() default ""; diff --git a/junit5/src/main/java/cz/xtf/junit5/extensions/SkipForCondition.java b/junit5/src/main/java/cz/xtf/junit5/extensions/SkipForCondition.java index 0d5ab15b..0adfe045 100644 --- a/junit5/src/main/java/cz/xtf/junit5/extensions/SkipForCondition.java +++ b/junit5/src/main/java/cz/xtf/junit5/extensions/SkipForCondition.java @@ -2,12 +2,15 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Stream; +import org.apache.commons.lang3.StringUtils; import org.junit.jupiter.api.extension.ConditionEvaluationResult; import org.junit.jupiter.api.extension.ExecutionCondition; import org.junit.jupiter.api.extension.ExtensionContext; import org.junit.platform.commons.support.AnnotationSupport; +import cz.xtf.core.config.XTFConfig; import cz.xtf.core.image.Image; import cz.xtf.core.openshift.OpenShifts; import cz.xtf.junit5.annotations.SkipFor; @@ -35,12 +38,9 @@ public ConditionEvaluationResult evaluateExecutionCondition(ExtensionContext con } public static ConditionEvaluationResult resolve(SkipFor skipFor) { - if ((skipFor.name().equals("") == skipFor.imageMetadataLabelName().equals("") == skipFor - .imageMetadataLabelArchitecture().equals("")) || - (!skipFor.name().equals("") && !skipFor.imageMetadataLabelName().equals("") - && !skipFor.imageMetadataLabelArchitecture().equals(""))) { + if (detectMultipleSkipForCriteria(skipFor)) { throw new RuntimeException( - "Only one of 'name','imageMetadataLabelName' and 'imageMetadataLabelArchitecture' can be presented in 'SkipFor' annotation."); + "Only one of 'name', 'imageMetadataLabelName', 'imageMetadataLabelArchitecture' and 'subId' can be presented in 'SkipFor' annotation."); } Image image = Image.resolve(skipFor.image()); @@ -51,11 +51,12 @@ public static ConditionEvaluationResult resolve(SkipFor skipFor) { } else if (!skipFor.imageMetadataLabelName().equals("")) { DockerImageMetadata metadata = DockerImageMetadata.get(OpenShifts.master(), image); matcher = Pattern.compile(skipFor.imageMetadataLabelName()).matcher(metadata.labels().get("name")); - } else { + } else if (!skipFor.imageMetadataLabelArchitecture().equals("")) { DockerImageMetadata metadata = DockerImageMetadata.get(OpenShifts.master(), image); matcher = Pattern.compile(skipFor.imageMetadataLabelArchitecture()).matcher(metadata.labels().get("architecture")); + } else { + matcher = Pattern.compile(skipFor.subId()).matcher(XTFConfig.get("xtf." + skipFor.image() + ".subid")); } - if (matcher.matches()) { String reason = skipFor.reason().equals("") ? "" : " (" + skipFor.reason() + ")"; return ConditionEvaluationResult @@ -64,4 +65,10 @@ public static ConditionEvaluationResult resolve(SkipFor skipFor) { return ConditionEvaluationResult.enabled("Image '" + image.getRepo() + "' is expected to contain tested feature."); } } + + private static boolean detectMultipleSkipForCriteria(SkipFor skipFor) { + return Stream + .of(skipFor.name(), skipFor.imageMetadataLabelName(), skipFor.subId(), skipFor.imageMetadataLabelArchitecture()) + .filter(c -> StringUtils.isNotBlank(c)).count() > 1; + } } diff --git a/junit5/src/test/java/cz/xtf/junit5/extensions/SkipForConditionTest.java b/junit5/src/test/java/cz/xtf/junit5/extensions/SkipForConditionTest.java new file mode 100644 index 00000000..43449003 --- /dev/null +++ b/junit5/src/test/java/cz/xtf/junit5/extensions/SkipForConditionTest.java @@ -0,0 +1,113 @@ +package cz.xtf.junit5.extensions; + +import java.util.Arrays; +import java.util.stream.Stream; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ConditionEvaluationResult; +import org.junit.jupiter.api.extension.ExtendWith; + +import cz.xtf.junit5.annotations.SkipFor; +import uk.org.webcompere.systemstubs.jupiter.SystemStub; +import uk.org.webcompere.systemstubs.jupiter.SystemStubsExtension; +import uk.org.webcompere.systemstubs.properties.SystemProperties; + +@ExtendWith(SystemStubsExtension.class) +class SkipForConditionTest { + + @SystemStub + private SystemProperties systemProperties; + + @BeforeEach + void before() { + systemProperties.set("xtf.eap.subid", "74-openjdk11"); + systemProperties.set("xtf.eap.74-openjdk11.image", + "registry-proxy.engineering.redhat.com/rh-osbs/jboss-eap-7-eap74-openjdk8-openshift-rhel7:7.4.0-6"); + } + + @SkipFor(image = "eap", name = ".*eap-74.*", reason = "This test is skipped based on the image name.") + class WellAnnotatedImageNameBasedSkipTest { + + } + + @SkipFor(image = "eap", imageMetadataLabelName = "centos7/s2i-base-centos7", reason = "This test is skipped based on the image docker labels") + class WellAnnotatedImageMetadataLabelBasedSkipTest { + + } + + @SkipFor(image = "eap", imageMetadataLabelArchitecture = "s390x", reason = "This test is skipped based on the image metadata label architecture") + class WellAnnotatedImageMetadataLabelArchitectureBasedSkipTest { + + } + + @SkipFor(image = "eap", subId = ".*74.*", reason = "This test is skipped based on the image product subId") + class WellAnnotatedSubIdBasedSkipTest { + + } + + @SkipFor(image = "eap", name = ".*eap-xp1.*", imageMetadataLabelName = "centos7/s2i-base-centos7", reason = "This test SHOULD BE skipped based on the image name.") + class BadlyAnnotatedImageNameBasedSkipTest { + + } + + @SkipFor(image = "eap", name = ".*eap-xp1.*", imageMetadataLabelName = "centos7/s2i-base-centos7", reason = "This test SHOULD BE skipped based on the image docker labels") + class BadlyAnnotatedImageMetadataLabelBasedSkipTest { + + } + + @SkipFor(image = "eap", name = ".*eap-xp4.*", imageMetadataLabelArchitecture = "s390x", reason = "This test SHOULD BE skipped based on the image metadata label architecture") + class BadlyAnnotatedImageMetadataLabelArchitectureBasedSkipTest { + + } + + @SkipFor(image = "eap", name = ".*eap-xp1.*", imageMetadataLabelName = "centos7/s2i-base-centos7", subId = ".*74.*", reason = "This test SHOULD BE skipped based on the image product subId") + class BadlyAnnotatedSubIdBasedSkipTest { + + } + + @Test + void testUniqueCriteriaResolutionOnWellAnnotatedClasses() { + Stream workingClasses = Stream.of( + WellAnnotatedImageNameBasedSkipTest.class, + WellAnnotatedImageMetadataLabelBasedSkipTest.class, + WellAnnotatedImageMetadataLabelArchitectureBasedSkipTest.class, + WellAnnotatedSubIdBasedSkipTest.class); + workingClasses.forEach(k -> { + try { + SkipForCondition.resolve((SkipFor) Arrays.stream(k.getAnnotationsByType(SkipFor.class)).findFirst().get()); + } catch (RuntimeException re) { + // we should be able to do better things here, as for instance using a specific Exception type + if (re.getMessage().equals( + "Only one of 'name', 'imageMetadataLabelName', 'imageMetadataLabelArchitecture' and 'subId' can be presented in 'SkipFor' annotation.")) { + Assertions.fail(String.format("No exception is expected when resolving %s \"@SkipFor\" annotation", + k.getSimpleName())); + } + } + }); + } + + @Test + void testUniqueCriteriaResolutionOnBadlyAnnotatedClasses() { + Stream workingClasses = Stream.of( + BadlyAnnotatedImageNameBasedSkipTest.class, + BadlyAnnotatedImageMetadataLabelBasedSkipTest.class, + BadlyAnnotatedImageMetadataLabelArchitectureBasedSkipTest.class, + BadlyAnnotatedSubIdBasedSkipTest.class); + workingClasses.forEach(k -> { + Exception exception = Assertions.assertThrows(RuntimeException.class, () -> SkipForCondition + .resolve((SkipFor) Arrays.stream(k.getAnnotationsByType(SkipFor.class)).findFirst().get())); + Assertions.assertEquals( + "Only one of 'name', 'imageMetadataLabelName', 'imageMetadataLabelArchitecture' and 'subId' can be presented in 'SkipFor' annotation.", + exception.getMessage()); + }); + } + + @Test + void testSubIdBasedSkipForResolution() { + ConditionEvaluationResult conditionEvaluationResult = SkipForCondition.resolve( + Arrays.stream(WellAnnotatedSubIdBasedSkipTest.class.getAnnotationsByType(SkipFor.class)).findFirst().get()); + Assertions.assertTrue(conditionEvaluationResult.isDisabled(), "This test should be disabled via \"subId\""); + } +}