Reproducer for quarkus/issues/15025: Quarkus keeps dead database connect

004-quarkus-HHH-and-HV/README.md

@@ -4,4 +4,5 @@ Test class annotated with `@QuarkusTestResource(H2DatabaseTestResource.class)` s
 
 Module that covers integration with some Hibernate features like:
 - Reproducer for [14201](https://github.com/quarkusio/quarkus/issues/14201) and [14881](https://github.com/quarkusio/quarkus/issues/14881): possible data loss bug in hibernate. This is covered under the Java package `io.quarkus.qe.hibernate.items`.
-- Reproducer for [QUARKUS-661](https://issues.redhat.com/browse/QUARKUS-661): @TransactionScoped Context does not call @Predestroy on TransactionScoped Beans. This is covered under the Java package `io.quarkus.qe.hibernate.transaction`.
+- Reproducer for [QUARKUS-661](https://issues.redhat.com/browse/QUARKUS-661): @TransactionScoped Context does not call @Predestroy on TransactionScoped Beans. This is covered under the Java package `io.quarkus.qe.hibernate.transaction`.
+- Reproducer for [15025](https://github.com/quarkusio/quarkus/issues/15025): Quarkus keeps dead database connections in its connection pool. This is covered under the Java package `io.quarkus.qe.hibernate.connections`.

004-quarkus-HHH-and-HV/pom.xml

@@ -16,6 +16,14 @@
             <groupId>io.quarkus</groupId>
             <artifactId>quarkus-resteasy</artifactId>
         </dependency>
+        <dependency>
+            <groupId>io.quarkus</groupId>
+            <artifactId>quarkus-smallrye-metrics</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>io.quarkus</groupId>
+            <artifactId>quarkus-smallrye-health</artifactId>
+        </dependency>
         <dependency>
             <groupId>io.quarkus</groupId>
             <artifactId>quarkus-junit5</artifactId>
@@ -43,6 +51,11 @@
             <artifactId>quarkus-test-h2</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.awaitility</groupId>
+            <artifactId>awaitility</artifactId>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
 </project>

004-quarkus-HHH-and-HV/src/main/resources/application.properties

@@ -4,4 +4,8 @@ quarkus.hibernate-orm.dialect=org.hibernate.dialect.H2Dialect
 quarkus.datasource.jdbc.min-size=3
 quarkus.datasource.jdbc.max-size=10
 quarkus.hibernate-orm.database.generation=drop-and-create
-quarkus.hibernate-orm.sql-load-script=import.sql
+quarkus.hibernate-orm.sql-load-script=import.sql
+
+quarkus.datasource.health.enabled=true
+quarkus.datasource.metrics.enabled=true
+quarkus.datasource.jdbc.enable-metrics=true

004-quarkus-HHH-and-HV/src/test/java/io/quarkus/qe/hibernate/connections/ConnectionPoolHandlingTest.java

@@ -0,0 +1,78 @@
+package io.quarkus.qe.hibernate.connections;
+
+import static io.restassured.RestAssured.when;
+import static org.hamcrest.Matchers.containsString;
+
+import java.time.Duration;
+
+import org.apache.http.HttpStatus;
+import org.awaitility.Awaitility;
+import org.eclipse.microprofile.config.inject.ConfigProperty;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import io.quarkus.qe.hibernate.resources.CustomH2DatabaseTestResource;
+import io.quarkus.test.junit.QuarkusTest;
+
+/**
+ * Reproducer for https://github.com/quarkusio/quarkus/issues/15025.
+ */
+@QuarkusTest
+public class ConnectionPoolHandlingTest {
+
+    private static final String EXPECTED_COUNTER = "vendor_agroal_available_count{datasource=\"default\"} %s.0";
+
+    @ConfigProperty(name = "quarkus.datasource.jdbc.min-size")
+    int minConnections;
+
+    CustomH2DatabaseTestResource database;
+    Thread restartDatabaseJob;
+
+    @BeforeEach
+    public void setup() {
+        restartDatabaseJob = new Thread(this::restartDatabaseIndefinitely);
+    }
+
+    @AfterEach
+    public void tearDown() {
+        if (restartDatabaseJob == null) {
+            restartDatabaseJob.interrupt();
+        }
+    }
+
+    @Test
+    public void connectionPoolShouldRemoveDeadDatabaseConnections() throws InterruptedException {
+        whenRestartDatabaseManyTimes();
+        thenVendorAgroalAvailableCountShouldNeverBeUpToMinConnections();
+    }
+
+    private void whenRestartDatabaseManyTimes() {
+        restartDatabaseJob.start();
+    }
+
+    private void thenVendorAgroalAvailableCountShouldNeverBeUpToMinConnections() {
+        // Wait 4 seconds to start checking and up to 15 seconds to verify the assertion (as the counter can increase over time)
+        Awaitility.await().pollDelay(Duration.ofSeconds(8)).atMost(Duration.ofSeconds(30))
+                .untilAsserted(() -> {
+                    when().get("/q/metrics").then().statusCode(HttpStatus.SC_OK)
+                            .and().body(containsString(String.format(EXPECTED_COUNTER, minConnections)));
+        });
+
+    }
+
+    private void restartDatabaseIndefinitely() {
+        try {
+            while (true) {
+                database.stop();
+                database.start();
+                Thread.sleep(1000);
+                // call health check to acquire the new connection
+                when().get("/q/health");
+            }
+
+        } catch (InterruptedException e) {
+            // stopped in tearDown
+        }
+    }
+}

004-quarkus-HHH-and-HV/src/test/java/io/quarkus/qe/hibernate/resources/CustomH2DatabaseTestResource.java

@@ -0,0 +1,34 @@
+package io.quarkus.qe.hibernate.resources;
+
+import java.lang.reflect.Field;
+
+import io.quarkus.test.h2.H2DatabaseTestResource;
+
+public class CustomH2DatabaseTestResource extends H2DatabaseTestResource {
+
+    @Override
+    public void inject(Object testInstance) {
+        super.inject(testInstance);
+
+        Class<?> c = testInstance.getClass();
+        while (c != Object.class) {
+            for (Field f : c.getDeclaredFields()) {
+                if (f.getType().isAssignableFrom(CustomH2DatabaseTestResource.class)) {
+                    setFieldValue(f, testInstance, this);
+                }
+            }
+
+            c = c.getSuperclass();
+        }
+    }
+
+    private void setFieldValue(Field f, Object testInstance, Object value) {
+        try {
+            f.setAccessible(true);
+            f.set(testInstance, value);
+            return;
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+}

004-quarkus-HHH-and-HV/src/test/java/io/quarkus/qe/hibernate/validator/TestResources.java

@@ -1,8 +1,8 @@
 package io.quarkus.qe.hibernate.validator;
 
+import io.quarkus.qe.hibernate.resources.CustomH2DatabaseTestResource;
 import io.quarkus.test.common.QuarkusTestResource;
-import io.quarkus.test.h2.H2DatabaseTestResource;
 
-@QuarkusTestResource(H2DatabaseTestResource.class)
+@QuarkusTestResource(CustomH2DatabaseTestResource.class)
 public class TestResources {
 }