diff --git a/docs/src/main/asciidoc/databases-dev-services.adoc b/docs/src/main/asciidoc/databases-dev-services.adoc index 7d2f131f238e4..0c6424616383f 100644 --- a/docs/src/main/asciidoc/databases-dev-services.adoc +++ b/docs/src/main/asciidoc/databases-dev-services.adoc @@ -59,6 +59,58 @@ ibmcom/db2:11.5.0.0a mcr.microsoft.com/mssql/server:2017-CU12 ---- +== Mapping volumes into Dev Services for Database + +Mapping volumes from the Docker host's filesystem to the containers is handy to provide files like scripts or configuration, but also to preserve database data and reuse it after an application restart. + +[NOTE] +==== +Mapping volumes will only work in Dev Services with a container-based database like PostgreSQL. +==== + +Dev Services volumes can be mapped to the filesystem or the classpath: + +[source,properties] +---- +# Using a filesystem volume: +quarkus.datasource.devservices.volumes."/path/from"=/container/to <1> +# Using a classpath volume: +quarkus.datasource.devservices.volumes."classpath:./file"=/container/to <2> +---- + +<1> The file or folder "/path/from" from the local machine will be accessible at "/container/to" in the container. +<2> When using classpath volumes, the location has to start with "classpath:". The file or folder "./file" from the project's classpath will be accessible at "/container/to" in the container. + +IMPORTANT: when using a classpath volume, the container will only be granted read permission. On the other hand, when using a filesystem volume, the container will be granted read and write permission. + +=== Example of mapping volumes to persist the database data + +Let's see an example using PostgreSQL where we'll map a file system volume to keep the database data permantently and use it: + +[source,properties] +---- +quarkus.datasource.db-kind=postgresql +quarkus.datasource.devservices.volumes."/local/test/data"=/var/lib/postgresql/data +---- + +The appropriate in-container location varies depending on the database vendor. For PostgresSQL is "/var/lib/postgresql/data", but for MySQL, you would need this configuration instead: + +[source,properties] +---- +quarkus.datasource.db-kind=mysql +quarkus.datasource.devservices.volumes."/local/test/data"=/var/lib/mysql +---- + +When starting Dev Services (for example, in tests or in DEV mode), you will see that the folder "/local/test/data" will be created at your file sytem and that will contain all the database data. When rerunning again the same dev services, this data will contain all the data you might have created beforehand. + +[IMPORTANT] +==== +When using Dev Services with Hibernate ORM, by default Quarkus will wipe out the database on application startup, which will wipe out the database data on your Docker host's filesystem. +Configure `quarkus.hibernate-orm.database.generation=none` or `quarkus.hibernate-orm.database.generation=validate` to avoid this behavior. + +Also, using Flyway to migrate your schema when starting the application will modify the database data on your Docker hosts's file system. +==== + == Database Vendor Specific Configuration All services based on containers are run using Testcontainers but Quarkus is not using the Testcontainers JDBC driver. diff --git a/extensions/datasource/deployment-spi/src/main/java/io/quarkus/datasource/deployment/spi/DevServicesDatasourceContainerConfig.java b/extensions/datasource/deployment-spi/src/main/java/io/quarkus/datasource/deployment/spi/DevServicesDatasourceContainerConfig.java index abba376e6eaf0..e4b533af72e13 100644 --- a/extensions/datasource/deployment-spi/src/main/java/io/quarkus/datasource/deployment/spi/DevServicesDatasourceContainerConfig.java +++ b/extensions/datasource/deployment-spi/src/main/java/io/quarkus/datasource/deployment/spi/DevServicesDatasourceContainerConfig.java @@ -15,6 +15,7 @@ public class DevServicesDatasourceContainerConfig { private final Optional username; private final Optional password; private final Optional initScriptPath; + private final Map volumes; public DevServicesDatasourceContainerConfig(Optional imageName, Map containerProperties, @@ -24,7 +25,8 @@ public DevServicesDatasourceContainerConfig(Optional imageName, Optional dbName, Optional username, Optional password, - Optional initScriptPath) { + Optional initScriptPath, + Map volumes) { this.imageName = imageName; this.containerProperties = containerProperties; this.additionalJdbcUrlProperties = additionalJdbcUrlProperties; @@ -34,6 +36,7 @@ public DevServicesDatasourceContainerConfig(Optional imageName, this.username = username; this.password = password; this.initScriptPath = initScriptPath; + this.volumes = volumes; } public Optional getImageName() { @@ -71,4 +74,8 @@ public Optional getPassword() { public Optional getInitScriptPath() { return initScriptPath; } + + public Map getVolumes() { + return volumes; + } } diff --git a/extensions/datasource/deployment/src/main/java/io/quarkus/datasource/deployment/devservices/DevServicesDatasourceProcessor.java b/extensions/datasource/deployment/src/main/java/io/quarkus/datasource/deployment/devservices/DevServicesDatasourceProcessor.java index 65bc997799ab9..83af2b468d637 100644 --- a/extensions/datasource/deployment/src/main/java/io/quarkus/datasource/deployment/devservices/DevServicesDatasourceProcessor.java +++ b/extensions/datasource/deployment/src/main/java/io/quarkus/datasource/deployment/devservices/DevServicesDatasourceProcessor.java @@ -266,6 +266,7 @@ private RunningDevService startDevDb(String dbName, + " (" + defaultDbKind.get() + ") starting:", consoleInstalledBuildItem, loggingSetupBuildItem); + try { DevServicesDatasourceContainerConfig containerConfig = new DevServicesDatasourceContainerConfig( dataSourceBuildTimeConfig.devservices.imageName, @@ -276,7 +277,8 @@ private RunningDevService startDevDb(String dbName, dataSourceBuildTimeConfig.devservices.dbName, dataSourceBuildTimeConfig.devservices.username, dataSourceBuildTimeConfig.devservices.password, - dataSourceBuildTimeConfig.devservices.initScriptPath); + dataSourceBuildTimeConfig.devservices.initScriptPath, + dataSourceBuildTimeConfig.devservices.volumes); DevServicesDatasourceProvider.RunningDevServicesDatasource datasource = devDbProvider .startDatabase( diff --git a/extensions/datasource/runtime/src/main/java/io/quarkus/datasource/runtime/DevServicesBuildTimeConfig.java b/extensions/datasource/runtime/src/main/java/io/quarkus/datasource/runtime/DevServicesBuildTimeConfig.java index 9a2e33823e824..a35e46039a823 100644 --- a/extensions/datasource/runtime/src/main/java/io/quarkus/datasource/runtime/DevServicesBuildTimeConfig.java +++ b/extensions/datasource/runtime/src/main/java/io/quarkus/datasource/runtime/DevServicesBuildTimeConfig.java @@ -80,8 +80,21 @@ public class DevServicesBuildTimeConfig { /** * Path to a SQL script that will be loaded from the classpath and applied to the Dev Service database * - * If the provider is not container based (e.g. a H2 or Derby Database) then this has no effect. + * If the provider is not container based (e.g. an H2 or Derby Database) then this has no effect. */ @ConfigItem public Optional initScriptPath; + + /** + * The volumes to be mapped to the container. The map key corresponds to the host location and the map value is the + * container location. If the host location starts with "classpath:", then the mapping will load the resource from the + * classpath with read-only permission. + * + * When using a file system location, the volume will be created with read-write permission, so the data in your file + * system might be wiped out or altered. + * + * If the provider is not container based (e.g. an H2 or Derby Database) then this has no effect. + */ + @ConfigItem + public Map volumes; } diff --git a/extensions/devservices/common/src/main/java/io/quarkus/devservices/common/Volumes.java b/extensions/devservices/common/src/main/java/io/quarkus/devservices/common/Volumes.java new file mode 100644 index 0000000000000..39219be41eed6 --- /dev/null +++ b/extensions/devservices/common/src/main/java/io/quarkus/devservices/common/Volumes.java @@ -0,0 +1,36 @@ +package io.quarkus.devservices.common; + +import java.net.URL; +import java.util.Map; + +import org.testcontainers.containers.BindMode; +import org.testcontainers.containers.GenericContainer; + +public final class Volumes { + + private static final String CLASSPATH = "classpath:"; + private static final String EMPTY = ""; + + private Volumes() { + + } + + public static void addVolumes(GenericContainer container, Map volumes) { + for (Map.Entry volume : volumes.entrySet()) { + String hostLocation = volume.getKey(); + BindMode bindMode = BindMode.READ_WRITE; + if (volume.getKey().startsWith(CLASSPATH)) { + URL url = Thread.currentThread().getContextClassLoader() + .getResource(hostLocation.replaceFirst(CLASSPATH, EMPTY)); + if (url == null) { + throw new IllegalStateException("Classpath resource at '" + hostLocation + "' not found!"); + } + + hostLocation = url.getPath(); + bindMode = BindMode.READ_ONLY; + } + + container.withFileSystemBind(hostLocation, volume.getValue(), bindMode); + } + } +} diff --git a/extensions/devservices/db2/src/main/java/io/quarkus/devservices/db2/deployment/DB2DevServicesProcessor.java b/extensions/devservices/db2/src/main/java/io/quarkus/devservices/db2/deployment/DB2DevServicesProcessor.java index de239be3b9d4c..fc75323b23ced 100644 --- a/extensions/devservices/db2/src/main/java/io/quarkus/devservices/db2/deployment/DB2DevServicesProcessor.java +++ b/extensions/devservices/db2/src/main/java/io/quarkus/devservices/db2/deployment/DB2DevServicesProcessor.java @@ -22,6 +22,7 @@ import io.quarkus.devservices.common.ConfigureUtil; import io.quarkus.devservices.common.ContainerShutdownCloseable; import io.quarkus.devservices.common.Labels; +import io.quarkus.devservices.common.Volumes; import io.quarkus.runtime.LaunchMode; public class DB2DevServicesProcessor { @@ -50,6 +51,7 @@ public RunningDevServicesDatasource startDatabase(Optional username, Opt .withDatabaseName(effectiveDbName) .withReuse(true); Labels.addDataSourceLabel(container, datasourceName); + Volumes.addVolumes(container, containerConfig.getVolumes()); containerConfig.getAdditionalJdbcUrlProperties().forEach(container::withUrlParam); containerConfig.getCommand().ifPresent(container::setCommand); diff --git a/extensions/devservices/mariadb/src/main/java/io/quarkus/devservices/mariadb/deployment/MariaDBDevServicesProcessor.java b/extensions/devservices/mariadb/src/main/java/io/quarkus/devservices/mariadb/deployment/MariaDBDevServicesProcessor.java index c308925a2d671..e3b07f6d3125c 100644 --- a/extensions/devservices/mariadb/src/main/java/io/quarkus/devservices/mariadb/deployment/MariaDBDevServicesProcessor.java +++ b/extensions/devservices/mariadb/src/main/java/io/quarkus/devservices/mariadb/deployment/MariaDBDevServicesProcessor.java @@ -22,6 +22,7 @@ import io.quarkus.devservices.common.ConfigureUtil; import io.quarkus.devservices.common.ContainerShutdownCloseable; import io.quarkus.devservices.common.Labels; +import io.quarkus.devservices.common.Volumes; import io.quarkus.runtime.LaunchMode; public class MariaDBDevServicesProcessor { @@ -53,6 +54,7 @@ public RunningDevServicesDatasource startDatabase(Optional username, Opt .withDatabaseName(effectiveDbName) .withReuse(true); Labels.addDataSourceLabel(container, datasourceName); + Volumes.addVolumes(container, containerConfig.getVolumes()); if (containerConfig.getContainerProperties().containsKey(MY_CNF_CONFIG_OVERRIDE_PARAM_NAME)) { container.withConfigurationOverride( @@ -62,7 +64,6 @@ public RunningDevServicesDatasource startDatabase(Optional username, Opt containerConfig.getAdditionalJdbcUrlProperties().forEach(container::withUrlParam); containerConfig.getCommand().ifPresent(container::setCommand); containerConfig.getInitScriptPath().ifPresent(container::withInitScript); - container.start(); LOG.info("Dev Services for MariaDB started."); diff --git a/extensions/devservices/mssql/src/main/java/io/quarkus/devservices/mssql/deployment/MSSQLDevServicesProcessor.java b/extensions/devservices/mssql/src/main/java/io/quarkus/devservices/mssql/deployment/MSSQLDevServicesProcessor.java index 5f133ae371ff6..3da12d579152f 100644 --- a/extensions/devservices/mssql/src/main/java/io/quarkus/devservices/mssql/deployment/MSSQLDevServicesProcessor.java +++ b/extensions/devservices/mssql/src/main/java/io/quarkus/devservices/mssql/deployment/MSSQLDevServicesProcessor.java @@ -20,6 +20,7 @@ import io.quarkus.devservices.common.ConfigureUtil; import io.quarkus.devservices.common.ContainerShutdownCloseable; import io.quarkus.devservices.common.Labels; +import io.quarkus.devservices.common.Volumes; import io.quarkus.runtime.LaunchMode; public class MSSQLDevServicesProcessor { @@ -46,6 +47,7 @@ public RunningDevServicesDatasource startDatabase(Optional username, Opt container.withPassword(effectivePassword) .withReuse(true); Labels.addDataSourceLabel(container, datasourceName); + Volumes.addVolumes(container, containerConfig.getVolumes()); containerConfig.getAdditionalJdbcUrlProperties().forEach(container::withUrlParam); containerConfig.getCommand().ifPresent(container::setCommand); diff --git a/extensions/devservices/mysql/src/main/java/io/quarkus/devservices/mysql/deployment/MySQLDevServicesProcessor.java b/extensions/devservices/mysql/src/main/java/io/quarkus/devservices/mysql/deployment/MySQLDevServicesProcessor.java index 41d62223c327b..f59ac3f9cb4a1 100644 --- a/extensions/devservices/mysql/src/main/java/io/quarkus/devservices/mysql/deployment/MySQLDevServicesProcessor.java +++ b/extensions/devservices/mysql/src/main/java/io/quarkus/devservices/mysql/deployment/MySQLDevServicesProcessor.java @@ -22,6 +22,7 @@ import io.quarkus.devservices.common.ConfigureUtil; import io.quarkus.devservices.common.ContainerShutdownCloseable; import io.quarkus.devservices.common.Labels; +import io.quarkus.devservices.common.Volumes; import io.quarkus.runtime.LaunchMode; public class MySQLDevServicesProcessor { @@ -52,6 +53,7 @@ public RunningDevServicesDatasource startDatabase(Optional username, Opt .withDatabaseName(effectiveDbName) .withReuse(true); Labels.addDataSourceLabel(container, datasourceName); + Volumes.addVolumes(container, containerConfig.getVolumes()); if (containerConfig.getContainerProperties().containsKey(MY_CNF_CONFIG_OVERRIDE_PARAM_NAME)) { container.withConfigurationOverride( diff --git a/extensions/devservices/oracle/src/main/java/io/quarkus/devservices/oracle/deployment/OracleDevServicesProcessor.java b/extensions/devservices/oracle/src/main/java/io/quarkus/devservices/oracle/deployment/OracleDevServicesProcessor.java index fa4ff0f5766dc..8e88e884170a5 100644 --- a/extensions/devservices/oracle/src/main/java/io/quarkus/devservices/oracle/deployment/OracleDevServicesProcessor.java +++ b/extensions/devservices/oracle/src/main/java/io/quarkus/devservices/oracle/deployment/OracleDevServicesProcessor.java @@ -22,6 +22,7 @@ import io.quarkus.devservices.common.ConfigureUtil; import io.quarkus.devservices.common.ContainerShutdownCloseable; import io.quarkus.devservices.common.Labels; +import io.quarkus.devservices.common.Volumes; import io.quarkus.runtime.LaunchMode; public class OracleDevServicesProcessor { @@ -53,6 +54,7 @@ public RunningDevServicesDatasource startDatabase(Optional username, Opt .withDatabaseName(effectiveDbName) .withReuse(true); Labels.addDataSourceLabel(container, datasourceName); + Volumes.addVolumes(container, containerConfig.getVolumes()); // We need to limit the maximum amount of CPUs being used by the container; // otherwise the hardcoded memory configuration of the DB might not be enough to successfully boot it. diff --git a/extensions/devservices/postgresql/src/main/java/io/quarkus/devservices/postgresql/deployment/PostgresqlDevServicesProcessor.java b/extensions/devservices/postgresql/src/main/java/io/quarkus/devservices/postgresql/deployment/PostgresqlDevServicesProcessor.java index a4c9c9bf1c224..b4235ce941207 100644 --- a/extensions/devservices/postgresql/src/main/java/io/quarkus/devservices/postgresql/deployment/PostgresqlDevServicesProcessor.java +++ b/extensions/devservices/postgresql/src/main/java/io/quarkus/devservices/postgresql/deployment/PostgresqlDevServicesProcessor.java @@ -5,12 +5,14 @@ import static io.quarkus.datasource.deployment.spi.DatabaseDefaultSetupConfig.DEFAULT_DATABASE_USERNAME; import java.time.Duration; +import java.time.temporal.ChronoUnit; import java.util.List; import java.util.Optional; import java.util.OptionalInt; import org.jboss.logging.Logger; import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy; import org.testcontainers.utility.DockerImageName; import io.quarkus.datasource.common.runtime.DatabaseKind; @@ -24,6 +26,7 @@ import io.quarkus.devservices.common.ConfigureUtil; import io.quarkus.devservices.common.ContainerShutdownCloseable; import io.quarkus.devservices.common.Labels; +import io.quarkus.devservices.common.Volumes; import io.quarkus.runtime.LaunchMode; public class PostgresqlDevServicesProcessor { @@ -57,6 +60,7 @@ public RunningDevServicesDatasource startDatabase(Optional username, Opt .withDatabaseName(effectiveDbName) .withReuse(true); Labels.addDataSourceLabel(container, datasourceName); + Volumes.addVolumes(container, containerConfig.getVolumes()); containerConfig.getAdditionalJdbcUrlProperties().forEach(container::withUrlParam); containerConfig.getCommand().ifPresent(container::setCommand); @@ -77,6 +81,10 @@ public RunningDevServicesDatasource startDatabase(Optional username, Opt } private static class QuarkusPostgreSQLContainer extends PostgreSQLContainer { + + private static final String READY_REGEX = ".*database system is ready to accept connections.*\\s"; + private static final String SKIPPING_INITIALIZATION_REGEX = ".*PostgreSQL Database directory appears to contain a database; Skipping initialization:*\\s"; + private final OptionalInt fixedExposedPort; private final boolean useSharedNetwork; @@ -88,6 +96,15 @@ public QuarkusPostgreSQLContainer(Optional imageName, OptionalInt fixedE .asCompatibleSubstituteFor(DockerImageName.parse(PostgreSQLContainer.IMAGE))); this.fixedExposedPort = fixedExposedPort; this.useSharedNetwork = useSharedNetwork; + // Workaround for https://github.com/testcontainers/testcontainers-java/issues/4799. + // The motivation of this custom wait strategy is that Testcontainers fails to start a Postgresql database when it + // has been already initialized. + // This custom wait strategy will work fine regardless of the state of the Postgresql database. + // More information in the issue ticket in Testcontainers. + this.waitStrategy = new LogMessageWaitStrategy() + .withRegEx("(" + READY_REGEX + ")?(" + SKIPPING_INITIALIZATION_REGEX + ")?") + .withTimes(2) + .withStartupTimeout(Duration.of(60L, ChronoUnit.SECONDS)); } @Override diff --git a/extensions/jdbc/jdbc-postgresql/deployment/src/test/java/io/quarkus/jdbc/postgresql/deployment/DevServicesPostgresqlDatasourceWithVolumeTestCase.java b/extensions/jdbc/jdbc-postgresql/deployment/src/test/java/io/quarkus/jdbc/postgresql/deployment/DevServicesPostgresqlDatasourceWithVolumeTestCase.java new file mode 100644 index 0000000000000..e00dfd1761f03 --- /dev/null +++ b/extensions/jdbc/jdbc-postgresql/deployment/src/test/java/io/quarkus/jdbc/postgresql/deployment/DevServicesPostgresqlDatasourceWithVolumeTestCase.java @@ -0,0 +1,44 @@ +package io.quarkus.jdbc.postgresql.deployment; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.sql.CallableStatement; +import java.sql.Connection; +import java.sql.ResultSet; + +import javax.sql.DataSource; + +import jakarta.inject.Inject; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import io.quarkus.test.QuarkusUnitTest; + +public class DevServicesPostgresqlDatasourceWithVolumeTestCase { + + @RegisterExtension + static QuarkusUnitTest test = new QuarkusUnitTest() + .overrideConfigKey("quarkus.datasource.db-kind", "postgresql") + // The official postgres image will execute all the scripts in the folder "docker-entrypoint-initdb.d" + .overrideConfigKey("quarkus.datasource.devservices.volumes.\"classpath:./init-db.sql\"", + "/docker-entrypoint-initdb.d/init-db.sql"); + + @Inject + DataSource ds; + + @Test + @DisplayName("Test if volume is mounted successfully") + public void testDatasource() throws Exception { + int result = 0; + try (Connection con = ds.getConnection(); + CallableStatement cs = con.prepareCall("SELECT my_func()"); + ResultSet rs = cs.executeQuery()) { + if (rs.next()) { + result = rs.getInt(1); + } + } + assertEquals(100, result, "The init script should have been executed"); + } +}