From 0aa84a097e7a1b29070ac1395c2c8212c28f8e2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edd=C3=BA=20Mel=C3=A9ndez?= Date: Thu, 12 May 2022 14:41:40 -0500 Subject: [PATCH 1/3] Add PostgreSQLWaitStrategy A more generic wait strategy for postgres. i.e Postgres has different output when the container database has data that doesn't match with the default wait strategy. First evaluation looks for (db doesn't contain data): * PostgreSQL init process complete * database system is ready to accept connections Second evaluation looks for (db contains data): * PostgreSQL Database directory appears to contain a database * database system is ready to accept connections See gh-5359 --- .../jdbc/JdbcDatabaseDelegate.java | 1 + .../containers/PostgreSQLContainer.java | 8 +- .../containers/PostgreSQLWaitStrategy.java | 88 +++++++++++++++++++ .../postgresql/SimplePostgreSQLTest.java | 38 +++++++- .../src/test/resources/Dockerfile-142 | 15 ++++ .../src/test/resources/Dockerfile-92 | 15 ++++ 6 files changed, 157 insertions(+), 8 deletions(-) create mode 100644 modules/postgresql/src/main/java/org/testcontainers/containers/PostgreSQLWaitStrategy.java create mode 100644 modules/postgresql/src/test/resources/Dockerfile-142 create mode 100644 modules/postgresql/src/test/resources/Dockerfile-92 diff --git a/modules/jdbc/src/main/java/org/testcontainers/jdbc/JdbcDatabaseDelegate.java b/modules/jdbc/src/main/java/org/testcontainers/jdbc/JdbcDatabaseDelegate.java index 6173a8f6a37..1a70e5ca357 100644 --- a/modules/jdbc/src/main/java/org/testcontainers/jdbc/JdbcDatabaseDelegate.java +++ b/modules/jdbc/src/main/java/org/testcontainers/jdbc/JdbcDatabaseDelegate.java @@ -1,6 +1,7 @@ package org.testcontainers.jdbc; import lombok.extern.slf4j.Slf4j; +import org.testcontainers.containers.ContainerState; import org.testcontainers.containers.JdbcDatabaseContainer; import org.testcontainers.delegate.AbstractDatabaseDelegate; import org.testcontainers.exception.ConnectionCreationException; diff --git a/modules/postgresql/src/main/java/org/testcontainers/containers/PostgreSQLContainer.java b/modules/postgresql/src/main/java/org/testcontainers/containers/PostgreSQLContainer.java index 741a8c4139a..1eadd69e3bb 100644 --- a/modules/postgresql/src/main/java/org/testcontainers/containers/PostgreSQLContainer.java +++ b/modules/postgresql/src/main/java/org/testcontainers/containers/PostgreSQLContainer.java @@ -1,13 +1,10 @@ package org.testcontainers.containers; import org.jetbrains.annotations.NotNull; -import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy; import org.testcontainers.utility.DockerImageName; -import java.time.Duration; import java.util.Set; -import static java.time.temporal.ChronoUnit.SECONDS; import static java.util.Collections.singleton; /** @@ -49,10 +46,7 @@ public PostgreSQLContainer(final DockerImageName dockerImageName) { dockerImageName.assertCompatibleWith(DEFAULT_IMAGE_NAME); - this.waitStrategy = new LogMessageWaitStrategy() - .withRegEx(".*database system is ready to accept connections.*\\s") - .withTimes(2) - .withStartupTimeout(Duration.of(60, SECONDS)); + this.waitStrategy = new PostgreSQLWaitStrategy(dockerImageName.getVersionPart()); this.setCommand("postgres", "-c", FSYNC_OFF_OPTION); addExposedPort(POSTGRESQL_PORT); diff --git a/modules/postgresql/src/main/java/org/testcontainers/containers/PostgreSQLWaitStrategy.java b/modules/postgresql/src/main/java/org/testcontainers/containers/PostgreSQLWaitStrategy.java new file mode 100644 index 00000000000..9e42cffd265 --- /dev/null +++ b/modules/postgresql/src/main/java/org/testcontainers/containers/PostgreSQLWaitStrategy.java @@ -0,0 +1,88 @@ +package org.testcontainers.containers; + +import com.github.dockerjava.api.command.LogContainerCmd; +import lombok.SneakyThrows; +import org.testcontainers.DockerClientFactory; +import org.testcontainers.containers.output.FrameConsumerResultCallback; +import org.testcontainers.containers.output.OutputFrame; +import org.testcontainers.containers.output.WaitingConsumer; +import org.testcontainers.containers.wait.strategy.AbstractWaitStrategy; +import org.testcontainers.utility.ComparableVersion; + +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.function.Predicate; + +import static org.testcontainers.containers.output.OutputFrame.OutputType.STDERR; +import static org.testcontainers.containers.output.OutputFrame.OutputType.STDOUT; + +public class PostgreSQLWaitStrategy extends AbstractWaitStrategy { + + private final String version; + + public PostgreSQLWaitStrategy(String version) { + this.version = version; + } + + @Override + @SneakyThrows(IOException.class) + protected void waitUntilReady() { + int times = 1; + long limit = 15; + boolean isAtLeastMajorVersion94 = new ComparableVersion(this.version).isGreaterThanOrEqualTo("9.4"); + + List> regExs = new ArrayList<>(); + + List first = new ArrayList<>(); + first.add(".*PostgreSQL init process complete.*$"); + first.add(".*database system is ready to accept connections.*$"); + regExs.add(first); + + List second = new ArrayList<>(); + if (isAtLeastMajorVersion94) { + second.add(".*PostgreSQL Database directory appears to contain a database.*$"); + } + second.add(".*database system is ready to accept connections.*$"); + regExs.add(second); + + boolean success = true; + for (List ex : regExs) { + WaitingConsumer waitingConsumer = new WaitingConsumer(); + try (FrameConsumerResultCallback callback = new FrameConsumerResultCallback()) { + callback.addConsumer(STDOUT, waitingConsumer); + callback.addConsumer(STDERR, waitingConsumer); + success = true; + + try { + for (String regEx : ex) { + LogContainerCmd cmd = DockerClientFactory.instance().client().logContainerCmd(waitStrategyTarget.getContainerId()) + .withFollowStream(true) + .withSince(0) + .withStdOut(true) + .withStdErr(true); + cmd.exec(callback); + + Predicate waitPredicate = outputFrame -> + // (?s) enables line terminator matching (equivalent to Pattern.DOTALL) + outputFrame.getUtf8String().matches("(?s)" + regEx); + + waitingConsumer.waitUntil(waitPredicate, limit, TimeUnit.SECONDS, times); + } + } catch (TimeoutException e) { + success = false; + } + if (success) { + break; + } + } + } + if (!success) { + throw new ContainerLaunchException("Timed out waiting for log output matching '" + "." + "'"); + } + } + +} diff --git a/modules/postgresql/src/test/java/org/testcontainers/junit/postgresql/SimplePostgreSQLTest.java b/modules/postgresql/src/test/java/org/testcontainers/junit/postgresql/SimplePostgreSQLTest.java index 95e1fdf10fd..c483921a0ab 100644 --- a/modules/postgresql/src/test/java/org/testcontainers/junit/postgresql/SimplePostgreSQLTest.java +++ b/modules/postgresql/src/test/java/org/testcontainers/junit/postgresql/SimplePostgreSQLTest.java @@ -3,14 +3,17 @@ import org.junit.Test; import org.testcontainers.containers.PostgreSQLContainer; import org.testcontainers.db.AbstractContainerDatabaseTest; +import org.testcontainers.images.builder.ImageFromDockerfile; +import org.testcontainers.utility.DockerImageName; +import java.nio.file.Paths; import java.sql.ResultSet; import java.sql.SQLException; import java.util.logging.Level; import java.util.logging.LogManager; import static org.hamcrest.CoreMatchers.containsString; -import static org.junit.Assert.assertThat; +import static org.hamcrest.MatcherAssert.assertThat; import static org.rnorth.visibleassertions.VisibleAssertions.assertEquals; import static org.rnorth.visibleassertions.VisibleAssertions.assertNotEquals; import static org.testcontainers.PostgreSQLTestImages.POSTGRES_TEST_IMAGE; @@ -79,4 +82,37 @@ public void testWithAdditionalUrlParamInJdbcUrl() { assertThat(jdbcUrl, containsString("charSet=UNICODE")); } } + + @Test + public void test92WithDataAlreadyInTheContainer() throws SQLException { + ImageFromDockerfile image = new ImageFromDockerfile("postgres-with-data:9.2") + .withDockerfile(Paths.get("src/test/resources/Dockerfile-92")); + + DockerImageName postgresImage = DockerImageName.parse(image.get()).asCompatibleSubstituteFor("postgres"); + try (PostgreSQLContainer postgres = new PostgreSQLContainer<>(postgresImage)) { + postgres.start(); + + ResultSet resultSet = performQuery(postgres, "SELECT foo FROM bar"); + + String firstColumnValue = resultSet.getString(1); + assertEquals("Value from init script should equal real value", "hello world", firstColumnValue); + } + } + + @Test + public void test142WithDataAlreadyInTheContainer() throws SQLException { + ImageFromDockerfile image = new ImageFromDockerfile("postgres-with-data:14.2") + .withDockerfile(Paths.get("src/test/resources/Dockerfile-142")); + + DockerImageName postgresImage = DockerImageName.parse(image.get()).asCompatibleSubstituteFor("postgres"); + try (PostgreSQLContainer postgres = new PostgreSQLContainer<>(postgresImage)) { + postgres.start(); + + ResultSet resultSet = performQuery(postgres, "SELECT foo FROM bar"); + + String firstColumnValue = resultSet.getString(1); + assertEquals("Value from init script should equal real value", "hello world", firstColumnValue); + } + } + } diff --git a/modules/postgresql/src/test/resources/Dockerfile-142 b/modules/postgresql/src/test/resources/Dockerfile-142 new file mode 100644 index 00000000000..c0469a80f32 --- /dev/null +++ b/modules/postgresql/src/test/resources/Dockerfile-142 @@ -0,0 +1,15 @@ +FROM postgres:14.2 AS dumper + +COPY somepath/init_postgresql.sql /docker-entrypoint-initdb.d + +RUN ["sed", "-i", "s/exec \"$@\"/echo \"skipping...\"/", "/usr/local/bin/docker-entrypoint.sh"] + +ENV POSTGRES_USER=test +ENV POSTGRES_PASSWORD=test +ENV PGDATA=/data + +RUN ["/usr/local/bin/docker-entrypoint.sh", "postgres"] + +FROM postgres:14.2 + +COPY --from=dumper /data $PGDATA diff --git a/modules/postgresql/src/test/resources/Dockerfile-92 b/modules/postgresql/src/test/resources/Dockerfile-92 new file mode 100644 index 00000000000..0cdcc76fc69 --- /dev/null +++ b/modules/postgresql/src/test/resources/Dockerfile-92 @@ -0,0 +1,15 @@ +FROM postgres:9.2 AS dumper + +COPY somepath/init_postgresql.sql /docker-entrypoint-initdb.d + +RUN ["sed", "-i", "s/exec \"$@\"/echo \"skipping...\"/", "/usr/local/bin/docker-entrypoint.sh"] + +ENV POSTGRES_USER=test +ENV POSTGRES_PASSWORD=test +ENV PGDATA=/data + +RUN ["/usr/local/bin/docker-entrypoint.sh", "postgres"] + +FROM postgres:9.2 + +COPY --from=dumper /data $PGDATA From 2ba7111818b01dac5e4e2b698f6617cf1c3f3265 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edd=C3=BA=20Mel=C3=A9ndez?= Date: Mon, 16 May 2022 15:00:31 -0500 Subject: [PATCH 2/3] Create MultiLogMessageWaitStrategy --- .../jdbc/JdbcDatabaseDelegate.java | 1 - .../MultiLogMessageWaitStrategy.java | 73 +++++++++++++++++++ .../containers/PostgreSQLWaitStrategy.java | 71 +++--------------- 3 files changed, 83 insertions(+), 62 deletions(-) create mode 100644 modules/postgresql/src/main/java/org/testcontainers/containers/MultiLogMessageWaitStrategy.java diff --git a/modules/jdbc/src/main/java/org/testcontainers/jdbc/JdbcDatabaseDelegate.java b/modules/jdbc/src/main/java/org/testcontainers/jdbc/JdbcDatabaseDelegate.java index 1a70e5ca357..6173a8f6a37 100644 --- a/modules/jdbc/src/main/java/org/testcontainers/jdbc/JdbcDatabaseDelegate.java +++ b/modules/jdbc/src/main/java/org/testcontainers/jdbc/JdbcDatabaseDelegate.java @@ -1,7 +1,6 @@ package org.testcontainers.jdbc; import lombok.extern.slf4j.Slf4j; -import org.testcontainers.containers.ContainerState; import org.testcontainers.containers.JdbcDatabaseContainer; import org.testcontainers.delegate.AbstractDatabaseDelegate; import org.testcontainers.exception.ConnectionCreationException; diff --git a/modules/postgresql/src/main/java/org/testcontainers/containers/MultiLogMessageWaitStrategy.java b/modules/postgresql/src/main/java/org/testcontainers/containers/MultiLogMessageWaitStrategy.java new file mode 100644 index 00000000000..8160cea6e31 --- /dev/null +++ b/modules/postgresql/src/main/java/org/testcontainers/containers/MultiLogMessageWaitStrategy.java @@ -0,0 +1,73 @@ +package org.testcontainers.containers; + +import com.github.dockerjava.api.command.LogContainerCmd; +import lombok.SneakyThrows; +import org.testcontainers.DockerClientFactory; +import org.testcontainers.containers.output.FrameConsumerResultCallback; +import org.testcontainers.containers.output.OutputFrame; +import org.testcontainers.containers.output.WaitingConsumer; +import org.testcontainers.containers.wait.strategy.AbstractWaitStrategy; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.function.Predicate; + +import static org.testcontainers.containers.output.OutputFrame.OutputType.STDERR; +import static org.testcontainers.containers.output.OutputFrame.OutputType.STDOUT; + +public class MultiLogMessageWaitStrategy extends AbstractWaitStrategy { + + private List> regExs = new ArrayList<>(); + + private final int times = 1; + + private final long limit = 15; + + @Override + @SneakyThrows(IOException.class) + protected void waitUntilReady() { + boolean success = true; + for (List ex : this.regExs) { + WaitingConsumer waitingConsumer = new WaitingConsumer(); + try (FrameConsumerResultCallback callback = new FrameConsumerResultCallback()) { + callback.addConsumer(STDOUT, waitingConsumer); + callback.addConsumer(STDERR, waitingConsumer); + success = true; + + try { + for (String regEx : ex) { + LogContainerCmd cmd = DockerClientFactory.instance().client().logContainerCmd(waitStrategyTarget.getContainerId()) + .withFollowStream(true) + .withSince(0) + .withStdOut(true) + .withStdErr(true); + cmd.exec(callback); + + Predicate waitPredicate = outputFrame -> + // (?s) enables line terminator matching (equivalent to Pattern.DOTALL) + outputFrame.getUtf8String().matches("(?s)" + regEx); + + waitingConsumer.waitUntil(waitPredicate, limit, TimeUnit.SECONDS, times); + } + } catch (TimeoutException e) { + success = false; + } + if (success) { + break; + } + } + } + if (!success) { + throw new ContainerLaunchException("Timed out waiting for log output matching '" + "." + "'"); + } + } + + public MultiLogMessageWaitStrategy withRegEx(List regExs) { + this.regExs.add(regExs); + return this; + } + +} diff --git a/modules/postgresql/src/main/java/org/testcontainers/containers/PostgreSQLWaitStrategy.java b/modules/postgresql/src/main/java/org/testcontainers/containers/PostgreSQLWaitStrategy.java index 9e42cffd265..ddd60ad7749 100644 --- a/modules/postgresql/src/main/java/org/testcontainers/containers/PostgreSQLWaitStrategy.java +++ b/modules/postgresql/src/main/java/org/testcontainers/containers/PostgreSQLWaitStrategy.java @@ -1,24 +1,10 @@ package org.testcontainers.containers; -import com.github.dockerjava.api.command.LogContainerCmd; -import lombok.SneakyThrows; -import org.testcontainers.DockerClientFactory; -import org.testcontainers.containers.output.FrameConsumerResultCallback; -import org.testcontainers.containers.output.OutputFrame; -import org.testcontainers.containers.output.WaitingConsumer; import org.testcontainers.containers.wait.strategy.AbstractWaitStrategy; import org.testcontainers.utility.ComparableVersion; -import java.io.IOException; -import java.time.Duration; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.function.Predicate; - -import static org.testcontainers.containers.output.OutputFrame.OutputType.STDERR; -import static org.testcontainers.containers.output.OutputFrame.OutputType.STDOUT; public class PostgreSQLWaitStrategy extends AbstractWaitStrategy { @@ -29,60 +15,23 @@ public PostgreSQLWaitStrategy(String version) { } @Override - @SneakyThrows(IOException.class) protected void waitUntilReady() { - int times = 1; - long limit = 15; boolean isAtLeastMajorVersion94 = new ComparableVersion(this.version).isGreaterThanOrEqualTo("9.4"); - List> regExs = new ArrayList<>(); - - List first = new ArrayList<>(); - first.add(".*PostgreSQL init process complete.*$"); - first.add(".*database system is ready to accept connections.*$"); - regExs.add(first); + List firstAttempt = new ArrayList<>(); + firstAttempt.add(".*PostgreSQL init process complete.*$"); + firstAttempt.add(".*database system is ready to accept connections.*$"); - List second = new ArrayList<>(); + List secondAttempt = new ArrayList<>(); if (isAtLeastMajorVersion94) { - second.add(".*PostgreSQL Database directory appears to contain a database.*$"); + secondAttempt.add(".*PostgreSQL Database directory appears to contain a database.*$"); } - second.add(".*database system is ready to accept connections.*$"); - regExs.add(second); - - boolean success = true; - for (List ex : regExs) { - WaitingConsumer waitingConsumer = new WaitingConsumer(); - try (FrameConsumerResultCallback callback = new FrameConsumerResultCallback()) { - callback.addConsumer(STDOUT, waitingConsumer); - callback.addConsumer(STDERR, waitingConsumer); - success = true; - - try { - for (String regEx : ex) { - LogContainerCmd cmd = DockerClientFactory.instance().client().logContainerCmd(waitStrategyTarget.getContainerId()) - .withFollowStream(true) - .withSince(0) - .withStdOut(true) - .withStdErr(true); - cmd.exec(callback); + secondAttempt.add(".*database system is ready to accept connections.*$"); - Predicate waitPredicate = outputFrame -> - // (?s) enables line terminator matching (equivalent to Pattern.DOTALL) - outputFrame.getUtf8String().matches("(?s)" + regEx); - - waitingConsumer.waitUntil(waitPredicate, limit, TimeUnit.SECONDS, times); - } - } catch (TimeoutException e) { - success = false; - } - if (success) { - break; - } - } - } - if (!success) { - throw new ContainerLaunchException("Timed out waiting for log output matching '" + "." + "'"); - } + new MultiLogMessageWaitStrategy() + .withRegEx(firstAttempt) + .withRegEx(secondAttempt) + .waitUntilReady(this.waitStrategyTarget); } } From fc195601b84005cb98f5239a7c0ae8173164a886 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edd=C3=BA=20Mel=C3=A9ndez?= Date: Mon, 16 May 2022 21:03:18 -0500 Subject: [PATCH 3/3] Read version from postgres -V --- .../containers/PostgreSQLContainer.java | 2 +- .../containers/PostgreSQLWaitStrategy.java | 48 +++++++++++-------- 2 files changed, 29 insertions(+), 21 deletions(-) diff --git a/modules/postgresql/src/main/java/org/testcontainers/containers/PostgreSQLContainer.java b/modules/postgresql/src/main/java/org/testcontainers/containers/PostgreSQLContainer.java index 1eadd69e3bb..32d2a2cadd2 100644 --- a/modules/postgresql/src/main/java/org/testcontainers/containers/PostgreSQLContainer.java +++ b/modules/postgresql/src/main/java/org/testcontainers/containers/PostgreSQLContainer.java @@ -46,7 +46,7 @@ public PostgreSQLContainer(final DockerImageName dockerImageName) { dockerImageName.assertCompatibleWith(DEFAULT_IMAGE_NAME); - this.waitStrategy = new PostgreSQLWaitStrategy(dockerImageName.getVersionPart()); + this.waitStrategy = new PostgreSQLWaitStrategy(); this.setCommand("postgres", "-c", FSYNC_OFF_OPTION); addExposedPort(POSTGRESQL_PORT); diff --git a/modules/postgresql/src/main/java/org/testcontainers/containers/PostgreSQLWaitStrategy.java b/modules/postgresql/src/main/java/org/testcontainers/containers/PostgreSQLWaitStrategy.java index ddd60ad7749..9b0afaebdf0 100644 --- a/modules/postgresql/src/main/java/org/testcontainers/containers/PostgreSQLWaitStrategy.java +++ b/modules/postgresql/src/main/java/org/testcontainers/containers/PostgreSQLWaitStrategy.java @@ -3,35 +3,43 @@ import org.testcontainers.containers.wait.strategy.AbstractWaitStrategy; import org.testcontainers.utility.ComparableVersion; +import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; public class PostgreSQLWaitStrategy extends AbstractWaitStrategy { - private final String version; - - public PostgreSQLWaitStrategy(String version) { - this.version = version; - } + private final Pattern pattern = Pattern.compile("(?s)(?:\\d\\S*)"); @Override protected void waitUntilReady() { - boolean isAtLeastMajorVersion94 = new ComparableVersion(this.version).isGreaterThanOrEqualTo("9.4"); - - List firstAttempt = new ArrayList<>(); - firstAttempt.add(".*PostgreSQL init process complete.*$"); - firstAttempt.add(".*database system is ready to accept connections.*$"); - - List secondAttempt = new ArrayList<>(); - if (isAtLeastMajorVersion94) { - secondAttempt.add(".*PostgreSQL Database directory appears to contain a database.*$"); + try { + String postgresVersion = this.waitStrategyTarget.execInContainer("postgres", "-V").getStdout(); + Matcher matcher = this.pattern.matcher(postgresVersion); + if (matcher.find()) { + String version = matcher.group(); + boolean isAtLeastMajorVersion94 = new ComparableVersion(version).isGreaterThanOrEqualTo("9.4"); + + List firstAttempt = new ArrayList<>(); + firstAttempt.add(".*PostgreSQL init process complete.*$"); + firstAttempt.add(".*database system is ready to accept connections.*$"); + + List secondAttempt = new ArrayList<>(); + if (isAtLeastMajorVersion94) { + secondAttempt.add(".*PostgreSQL Database directory appears to contain a database.*$"); + } + secondAttempt.add(".*database system is ready to accept connections.*$"); + + new MultiLogMessageWaitStrategy() + .withRegEx(firstAttempt) + .withRegEx(secondAttempt) + .waitUntilReady(this.waitStrategyTarget); + } + } catch (IOException | InterruptedException e) { + throw new RuntimeException(e); } - secondAttempt.add(".*database system is ready to accept connections.*$"); - - new MultiLogMessageWaitStrategy() - .withRegEx(firstAttempt) - .withRegEx(secondAttempt) - .waitUntilReady(this.waitStrategyTarget); } }