Fail when returning collections in Resteasy Reactive JAXB

Fix quarkusio#33865

extensions/resteasy-reactive/quarkus-resteasy-reactive-jaxb/deployment/src/main/java/io/quarkus/resteasy/reactive/jaxb/deployment/ResteasyReactiveJaxbProcessor.java

@@ -1,5 +1,7 @@
 package io.quarkus.resteasy.reactive.jaxb.deployment;
 
+import static org.jboss.resteasy.reactive.common.processor.ResteasyReactiveDotNames.REST_RESPONSE;
+
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -8,6 +10,7 @@
 import java.util.Locale;
 import java.util.Set;
 
+import jakarta.enterprise.inject.spi.DeploymentException;
 import jakarta.ws.rs.Priorities;
 import jakarta.ws.rs.RuntimeType;
 import jakarta.ws.rs.core.MediaType;
@@ -87,6 +90,13 @@ void registerClassesToBeBound(ResteasyReactiveResourceMethodEntriesBuildItem res
             if (effectiveReturnType != null) {
                 // When using "application/xml", the return type needs to be registered
                 if (producesXml(resourceInfo)) {
+                    if (!isTypeCompatibleWithJaxb(methodInfo.returnType())) {
+                        throw new DeploymentException(
+                                "Cannot directly return collections or arrays using JAXB. You need to wrap it "
+                                        + "into a root element class. Problematic method is '"
+                                        + entry.getActualClassInfo().name() + "." + methodInfo.name() + "'");
+                    }
+
                     classesInfo.add(effectiveReturnType);
                 }
 
@@ -154,23 +164,16 @@ private ClassInfo getEffectiveClassInfo(Type type, IndexView indexView) {
         }
 
         Type effectiveType = type;
-        if (effectiveType.name().equals(ResteasyReactiveDotNames.REST_RESPONSE) ||
-                effectiveType.name().equals(ResteasyReactiveDotNames.UNI) ||
-                effectiveType.name().equals(ResteasyReactiveDotNames.COMPLETABLE_FUTURE) ||
-                effectiveType.name().equals(ResteasyReactiveDotNames.COMPLETION_STAGE) ||
-                effectiveType.name().equals(ResteasyReactiveDotNames.REST_MULTI) ||
-                effectiveType.name().equals(ResteasyReactiveDotNames.MULTI)) {
+        if (isContainerType(effectiveType)) {
             if (effectiveType.kind() != Type.Kind.PARAMETERIZED_TYPE) {
                 return null;
             }
 
             effectiveType = type.asParameterizedType().arguments().get(0);
         }
-        if (effectiveType.name().equals(ResteasyReactiveDotNames.SET) ||
-                effectiveType.name().equals(ResteasyReactiveDotNames.COLLECTION) ||
-                effectiveType.name().equals(ResteasyReactiveDotNames.LIST)) {
+        if (isCollectionType(effectiveType)) {
             effectiveType = effectiveType.asParameterizedType().arguments().get(0);
-        } else if (effectiveType.name().equals(ResteasyReactiveDotNames.MAP)) {
+        } else if (isMapType(effectiveType)) {
             effectiveType = effectiveType.asParameterizedType().arguments().get(1);
         }
 
@@ -220,4 +223,43 @@ private List<String> toClasses(Collection<ClassInfo> classesInfo) {
 
         return classes;
     }
+
+    private boolean isCollectionType(Type type) {
+        return type.name().equals(ResteasyReactiveDotNames.SET) ||
+                type.name().equals(ResteasyReactiveDotNames.COLLECTION) ||
+                type.name().equals(ResteasyReactiveDotNames.LIST);
+    }
+
+    private boolean isMapType(Type type) {
+        return type.name().equals(ResteasyReactiveDotNames.MAP);
+    }
+
+    private boolean isContainerType(Type type) {
+        return type.name().equals(REST_RESPONSE) ||
+                type.name().equals(ResteasyReactiveDotNames.UNI) ||
+                type.name().equals(ResteasyReactiveDotNames.COMPLETABLE_FUTURE) ||
+                type.name().equals(ResteasyReactiveDotNames.COMPLETION_STAGE) ||
+                type.name().equals(ResteasyReactiveDotNames.REST_MULTI) ||
+                type.name().equals(ResteasyReactiveDotNames.MULTI);
+    }
+
+    private boolean isTypeCompatibleWithJaxb(Type type) {
+        if (type.kind() == Type.Kind.PRIMITIVE) {
+            return true;
+        }
+
+        if (type.kind() == Type.Kind.ARRAY || isCollectionType(type) || isMapType(type)) {
+            return false;
+        }
+
+        if (isContainerType(type)) {
+            if (type.kind() != Type.Kind.PARAMETERIZED_TYPE) {
+                return true;
+            }
+
+            return isTypeCompatibleWithJaxb(type.asParameterizedType().arguments().get(0));
+        }
+
+        return true;
+    }
 }

extensions/resteasy-reactive/quarkus-resteasy-reactive-jaxb/deployment/src/test/java/io/quarkus/resteasy/reactive/jaxb/deployment/test/FailWhenUsingListTest.java

@@ -0,0 +1,41 @@
+package io.quarkus.resteasy.reactive.jaxb.deployment.test;
+
+import java.util.List;
+
+import jakarta.enterprise.inject.spi.DeploymentException;
+import jakarta.ws.rs.GET;
+import jakarta.ws.rs.Path;
+import jakarta.ws.rs.Produces;
+import jakarta.ws.rs.core.MediaType;
+
+import org.jboss.shrinkwrap.api.ShrinkWrap;
+import org.jboss.shrinkwrap.api.spec.JavaArchive;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+import io.quarkus.test.QuarkusUnitTest;
+
+public class FailWhenUsingListTest {
+
+    @RegisterExtension
+    static QuarkusUnitTest test = new QuarkusUnitTest()
+            .setExpectedException(DeploymentException.class)
+            .setArchiveProducer(() -> ShrinkWrap.create(JavaArchive.class)
+                    .addClasses(GreetingResource.class));
+
+    @Test
+    void shouldFailWithDeploymentException() {
+        Assertions.fail("The test case should not be invoked as it should fail with a deployment exception.");
+    }
+
+    @Path("/greeting")
+    public static class GreetingResource {
+
+        @GET
+        @Produces(MediaType.APPLICATION_XML)
+        public List<String> hello() {
+            return List.of("1", "2", "3");
+        }
+    }
+}

extensions/resteasy-reactive/quarkus-resteasy-reactive-jaxb/deployment/src/test/java/io/quarkus/resteasy/reactive/jaxb/deployment/test/FailWhenUsingRestResponseTest.java

@@ -0,0 +1,43 @@
+package io.quarkus.resteasy.reactive.jaxb.deployment.test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import jakarta.enterprise.inject.spi.DeploymentException;
+import jakarta.ws.rs.GET;
+import jakarta.ws.rs.Path;
+import jakarta.ws.rs.Produces;
+import jakarta.ws.rs.core.MediaType;
+
+import org.jboss.resteasy.reactive.RestResponse;
+import org.jboss.shrinkwrap.api.ShrinkWrap;
+import org.jboss.shrinkwrap.api.spec.JavaArchive;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+import io.quarkus.test.QuarkusUnitTest;
+
+public class FailWhenUsingRestResponseTest {
+
+    @RegisterExtension
+    static QuarkusUnitTest test = new QuarkusUnitTest()
+            .setExpectedException(DeploymentException.class)
+            .setArchiveProducer(() -> ShrinkWrap.create(JavaArchive.class)
+                    .addClasses(GreetingResource.class));
+
+    @Test
+    void shouldFailWithDeploymentException() {
+        Assertions.fail("The test case should not be invoked as it should fail with a deployment exception.");
+    }
+
+    @Path("/greeting")
+    public static class GreetingResource {
+
+        @GET
+        @Produces(MediaType.APPLICATION_XML)
+        public RestResponse<Map<String, String>> hello() {
+            return RestResponse.ok(new HashMap<>());
+        }
+    }
+}