From 0f51dfa6206162baa610463ce5d56358b5812d60 Mon Sep 17 00:00:00 2001 From: Richard North Date: Thu, 29 Oct 2020 11:02:48 +0000 Subject: [PATCH 01/16] Refactor Testcontainers configuration to allow config by env var --- .../utility/TestcontainersConfiguration.java | 252 +++++++++++++----- .../TestcontainersConfigurationTest.java | 86 ++++-- 2 files changed, 258 insertions(+), 80 deletions(-) diff --git a/core/src/main/java/org/testcontainers/utility/TestcontainersConfiguration.java b/core/src/main/java/org/testcontainers/utility/TestcontainersConfiguration.java index 5b89140fd61..b3c8e88a328 100644 --- a/core/src/main/java/org/testcontainers/utility/TestcontainersConfiguration.java +++ b/core/src/main/java/org/testcontainers/utility/TestcontainersConfiguration.java @@ -1,15 +1,17 @@ package org.testcontainers.utility; import com.google.common.annotations.VisibleForTesting; -import lombok.AccessLevel; +import com.google.common.collect.ImmutableMap; import lombok.Data; import lombok.Getter; import lombok.NonNull; -import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.Synchronized; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang.exception.ExceptionUtils; +import org.jetbrains.annotations.Contract; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import org.testcontainers.UnstableAPI; import java.io.File; @@ -20,22 +22,57 @@ import java.io.OutputStream; import java.net.MalformedURLException; import java.net.URL; +import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Properties; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Stream; /** - * Provides a mechanism for fetching configuration/defaults from the classpath. + * Provides a mechanism for fetching configuration/default settings. + *

+ * Configuration may be provided in: + *

+ *

+ * Note that, if using environment variables, property names are in upper case separated by underscores, preceded by + * TESTCONTAINERS_. */ @Data @Slf4j -@RequiredArgsConstructor(access = AccessLevel.PRIVATE) public class TestcontainersConfiguration { private static String PROPERTIES_FILE_NAME = "testcontainers.properties"; - private static File ENVIRONMENT_CONFIG_FILE = new File(System.getProperty("user.home"), "." + PROPERTIES_FILE_NAME); + private static File USER_CONFIG_FILE = new File(System.getProperty("user.home"), "." + PROPERTIES_FILE_NAME); + + private static final String AMBASSADOR_IMAGE = "richnorth/ambassador"; + private static final String SOCAT_IMAGE = "alpine/socat"; + private static final String VNC_RECORDER_IMAGE = "testcontainers/vnc-recorder"; + private static final String COMPOSE_IMAGE = "docker/compose"; + private static final String ALPINE_IMAGE = "alpine"; + private static final String RYUK_IMAGE = "testcontainers/ryuk"; + private static final String KAFKA_IMAGE = "confluentinc/cp-kafka"; + private static final String PULSAR_IMAGE = "apachepulsar/pulsar"; + private static final String LOCALSTACK_IMAGE = "localstack/localstack"; + private static final String SSHD_IMAGE = "testcontainers/sshd"; + + private static final ImmutableMap CONTAINER_MAPPING = ImmutableMap.builder() + .put(DockerImageName.parse(AMBASSADOR_IMAGE), "ambassador.container.image") + .put(DockerImageName.parse(SOCAT_IMAGE), "socat.container.image") + .put(DockerImageName.parse(VNC_RECORDER_IMAGE), "vncrecorder.container.image") + .put(DockerImageName.parse(COMPOSE_IMAGE), "compose.container.image") + .put(DockerImageName.parse(ALPINE_IMAGE), "tinyimage.container.image") + .put(DockerImageName.parse(RYUK_IMAGE), "ryuk.container.image") + .put(DockerImageName.parse(KAFKA_IMAGE), "kafka.container.image") + .put(DockerImageName.parse(PULSAR_IMAGE), "pulsar.container.image") + .put(DockerImageName.parse(LOCALSTACK_IMAGE), "localstack.container.image") + .put(DockerImageName.parse(SSHD_IMAGE), "sshd.container.image") + .build(); @Getter(lazy = true) private static final TestcontainersConfiguration instance = loadConfiguration(); @@ -47,163 +84,234 @@ static AtomicReference getInstanceField() { return (AtomicReference) (Object) instance; } - @Getter(AccessLevel.NONE) - private final Properties environmentProperties; + private final Properties userProperties; + private final Properties classpathProperties; + private final Map environment; - private final Properties properties = new Properties(); - - TestcontainersConfiguration(Properties environmentProperties, Properties classpathProperties) { - this.environmentProperties = environmentProperties; - - this.properties.putAll(classpathProperties); - this.properties.putAll(environmentProperties); - } - - private DockerImageName getImage(final String key, final String defaultValue) { - return DockerImageName - .parse(properties.getProperty(key, defaultValue).trim()) - .asCompatibleSubstituteFor(defaultValue); + TestcontainersConfiguration(Properties userProperties, Properties classpathProperties, final Map environment) { + this.userProperties = userProperties; + this.classpathProperties = classpathProperties; + this.environment = environment; } @Deprecated public String getAmbassadorContainerImage() { - return getAmbassadorContainerDockerImageName().asCanonicalNameString(); - } - - @Deprecated - public DockerImageName getAmbassadorContainerDockerImageName() { - return getImage("ambassador.container.image", "richnorth/ambassador:latest"); + return getImage(AMBASSADOR_IMAGE).asCanonicalNameString(); } @Deprecated public String getSocatContainerImage() { - return getSocatDockerImageName().asCanonicalNameString(); + return getImage(SOCAT_IMAGE).asCanonicalNameString(); } public DockerImageName getSocatDockerImageName() { - return getImage("socat.container.image", "alpine/socat:latest"); + return getImage(SOCAT_IMAGE); } @Deprecated public String getVncRecordedContainerImage() { - return getVncDockerImageName().asCanonicalNameString(); + return getImage(VNC_RECORDER_IMAGE).asCanonicalNameString(); } public DockerImageName getVncDockerImageName() { - return getImage("vncrecorder.container.image", "testcontainers/vnc-recorder:1.1.0"); + return getImage(VNC_RECORDER_IMAGE); } @Deprecated public String getDockerComposeContainerImage() { - return getDockerComposeDockerImageName().asCanonicalNameString(); + return getImage(COMPOSE_IMAGE).asCanonicalNameString(); } public DockerImageName getDockerComposeDockerImageName() { - return getImage("compose.container.image", "docker/compose:1.24.1"); + return getImage(COMPOSE_IMAGE); } @Deprecated public String getTinyImage() { - return getTinyDockerImageName().asCanonicalNameString(); + return getImage(ALPINE_IMAGE).asCanonicalNameString(); } public DockerImageName getTinyDockerImageName() { - return getImage("tinyimage.container.image", "alpine:3.5"); + return getImage(ALPINE_IMAGE); } public boolean isRyukPrivileged() { - return Boolean.parseBoolean((String) properties.getOrDefault("ryuk.container.privileged", "false")); + return Boolean + .parseBoolean(getEnvVarOrProperty("ryuk.container.privileged", "false")); } @Deprecated public String getRyukImage() { - return getRyukDockerImageName().asCanonicalNameString(); + return getImage(RYUK_IMAGE).asCanonicalNameString(); } public DockerImageName getRyukDockerImageName() { - return getImage("ryuk.container.image", "testcontainers/ryuk:0.3.0"); + return getImage(RYUK_IMAGE); } @Deprecated public String getSSHdImage() { - return getSSHdDockerImageName().asCanonicalNameString(); + return getImage(SSHD_IMAGE).asCanonicalNameString(); } public DockerImageName getSSHdDockerImageName() { - return getImage("sshd.container.image", "testcontainers/sshd:1.0.0"); + return getImage(SSHD_IMAGE); } public Integer getRyukTimeout() { - return Integer.parseInt((String) properties.getOrDefault("ryuk.container.timeout", "30")); + return Integer.parseInt(getEnvVarOrProperty("ryuk.container.timeout", "30")); } @Deprecated public String getKafkaImage() { - return getKafkaDockerImageName().asCanonicalNameString(); + return getImage(KAFKA_IMAGE).asCanonicalNameString(); } public DockerImageName getKafkaDockerImageName() { - return getImage("kafka.container.image", "confluentinc/cp-kafka"); + return getImage(KAFKA_IMAGE); + } + + + @Deprecated + public String getOracleImage() { + return getEnvVarOrUserProperty("oracle.container.image", null); } @Deprecated public String getPulsarImage() { - return getPulsarDockerImageName().asCanonicalNameString(); + return getImage(PULSAR_IMAGE).asCanonicalNameString(); } public DockerImageName getPulsarDockerImageName() { - return getImage("pulsar.container.image", "apachepulsar/pulsar"); + return getImage(PULSAR_IMAGE); } @Deprecated public String getLocalStackImage() { - return getLocalstackDockerImageName().asCanonicalNameString(); + return getImage(LOCALSTACK_IMAGE).asCanonicalNameString(); } public DockerImageName getLocalstackDockerImageName() { - return getImage("localstack.container.image", "localstack/localstack"); + return getImage(LOCALSTACK_IMAGE); } + public boolean isDisableChecks() { - return Boolean.parseBoolean((String) environmentProperties.getOrDefault("checks.disable", "false")); + return Boolean.parseBoolean(getEnvVarOrUserProperty("checks.disable", "false")); } @UnstableAPI public boolean environmentSupportsReuse() { - return Boolean.parseBoolean((String) environmentProperties.getOrDefault("testcontainers.reuse.enable", "false")); + // specifically not supported as an environment variable or classpath property + return Boolean.parseBoolean(getEnvVarOrUserProperty("testcontainers.reuse.enable", "false")); } public String getDockerClientStrategyClassName() { - return (String) environmentProperties.get("docker.client.strategy"); + return getEnvVarOrUserProperty("docker.client.strategy", null); } public String getTransportType() { - return properties.getProperty("transport.type", "okhttp"); + return getEnvVarOrProperty("transport.type", "okhttp"); } public Integer getImagePullPauseTimeout() { - return Integer.parseInt((String) properties.getOrDefault("pull.pause.timeout", "30")); + return Integer.parseInt(getEnvVarOrProperty("pull.pause.timeout", "30")); } - @Synchronized + @Nullable + @Contract("_, !null, _ -> !null") + private String getConfigurable(@NotNull final String propertyName, @Nullable final String defaultValue, Properties... propertiesSources) { + String envVarName = propertyName.replaceAll("\\.", "_").toUpperCase(); + if (!envVarName.startsWith("TESTCONTAINERS_")) { + envVarName = "TESTCONTAINERS_" + envVarName; + } + + if (environment.containsKey(envVarName)) { + return environment.get(envVarName); + } + + for (final Properties properties : propertiesSources) { + if (properties.get(propertyName) != null) { + return (String) properties.get(propertyName); + } + } + + return defaultValue; + } + + /** + * Gets a configured setting from an environment variable (if present) or a configuration file property otherwise. + * The configuration file will be the .testcontainers.properties file in the user's home directory or + * a testcontainers.properties found on the classpath. + * + * @param propertyName name of configuration file property (dot-separated lower case) + * @return the found value, or null if not set + */ + @Nullable + @Contract("_, !null -> !null") + public String getEnvVarOrProperty(@NotNull final String propertyName, @Nullable final String defaultValue) { + return getConfigurable(propertyName, defaultValue, userProperties, classpathProperties); + } + + /** + * Gets a configured setting from an environment variable (if present) or a configuration file property otherwise. + * The configuration file will be the .testcontainers.properties file in the user's home directory. + * + * @param propertyName name of configuration file property (dot-separated lower case) + * @return the found value, or null if not set + */ + @Nullable + @Contract("_, !null -> !null") + public String getEnvVarOrUserProperty(@NotNull final String propertyName, @Nullable final String defaultValue) { + return getConfigurable(propertyName, defaultValue, userProperties); + } + + /** + * Gets a configured setting from a the user's configuration file. + * The configuration file will be the .testcontainers.properties file in the user's home directory. + * + * @param propertyName name of configuration file property (dot-separated lower case) + * @return the found value, or null if not set + */ + @Nullable + @Contract("_, !null -> !null") + public String getUserProperty(@NotNull final String propertyName, @Nullable final String defaultValue) { + return getConfigurable(propertyName, defaultValue); + } + + @Deprecated + public Properties getProperties() { + return Stream.of(userProperties, classpathProperties) + .reduce(new Properties(), (a, b) -> { + a.putAll(b); + return a; + }); + } + + @Deprecated public boolean updateGlobalConfig(@NonNull String prop, @NonNull String value) { + return updateUserConfig(prop, value); + } + + @Synchronized + public boolean updateUserConfig(@NonNull String prop, @NonNull String value) { try { - if (value.equals(environmentProperties.get(prop))) { + if (value.equals(userProperties.get(prop))) { return false; } - environmentProperties.setProperty(prop, value); + userProperties.setProperty(prop, value); - ENVIRONMENT_CONFIG_FILE.createNewFile(); - try (OutputStream outputStream = new FileOutputStream(ENVIRONMENT_CONFIG_FILE)) { - environmentProperties.store(outputStream, "Modified by Testcontainers"); + USER_CONFIG_FILE.createNewFile(); + try (OutputStream outputStream = new FileOutputStream(USER_CONFIG_FILE)) { + userProperties.store(outputStream, "Modified by Testcontainers"); } // Update internal state only if environment config was successfully updated - properties.setProperty(prop, value); + userProperties.setProperty(prop, value); return true; } catch (Exception e) { - log.debug("Can't store environment property {} in {}", prop, ENVIRONMENT_CONFIG_FILE); + log.debug("Can't store environment property {} in {}", prop, USER_CONFIG_FILE); return false; } } @@ -211,7 +319,7 @@ public boolean updateGlobalConfig(@NonNull String prop, @NonNull String value) { @SneakyThrows(MalformedURLException.class) private static TestcontainersConfiguration loadConfiguration() { return new TestcontainersConfiguration( - readProperties(ENVIRONMENT_CONFIG_FILE.toURI().toURL()), + readProperties(USER_CONFIG_FILE.toURI().toURL()), Stream .of( TestcontainersConfiguration.class.getClassLoader(), @@ -223,8 +331,8 @@ private static TestcontainersConfiguration loadConfiguration() { .reduce(new Properties(), (a, b) -> { a.putAll(b); return a; - }) - ); + }), + System.getenv()); } private static Properties readProperties(URL url) { @@ -239,4 +347,24 @@ private static Properties readProperties(URL url) { } return properties; } + + private DockerImageName getImage(final String defaultValue) { + return getConfiguredSubstituteImage(DockerImageName.parse(defaultValue)); + } + + DockerImageName getConfiguredSubstituteImage(DockerImageName original) { + for (final Map.Entry entry : CONTAINER_MAPPING.entrySet()) { + if (original.isCompatibleWith(entry.getKey())) { + return + Optional.ofNullable(entry.getValue()) + .map(propertyName -> getEnvVarOrProperty(propertyName, null)) + .map(String::valueOf) + .map(String::trim) + .map(DockerImageName::parse) + .orElse(original) + .asCompatibleSubstituteFor(original); + } + } + return original; + } } diff --git a/core/src/test/java/org/testcontainers/utility/TestcontainersConfigurationTest.java b/core/src/test/java/org/testcontainers/utility/TestcontainersConfigurationTest.java index ccd68fbd984..e48a2a6fdf9 100644 --- a/core/src/test/java/org/testcontainers/utility/TestcontainersConfigurationTest.java +++ b/core/src/test/java/org/testcontainers/utility/TestcontainersConfigurationTest.java @@ -1,57 +1,107 @@ package org.testcontainers.utility; -import static org.rnorth.visibleassertions.VisibleAssertions.assertEquals; -import static org.rnorth.visibleassertions.VisibleAssertions.assertFalse; -import static org.rnorth.visibleassertions.VisibleAssertions.assertTrue; +import org.junit.Before; +import org.junit.Test; +import java.util.HashMap; +import java.util.Map; import java.util.Properties; import java.util.UUID; -import org.junit.Test; + +import static org.rnorth.visibleassertions.VisibleAssertions.assertEquals; +import static org.rnorth.visibleassertions.VisibleAssertions.assertFalse; +import static org.rnorth.visibleassertions.VisibleAssertions.assertTrue; public class TestcontainersConfigurationTest { - final Properties environmentProperties = new Properties(); + private Properties userProperties; + private Properties classpathProperties; + private Map environment; - final Properties classpathProperties = new Properties(); + @Before + public void setUp() { + userProperties = new Properties(); + classpathProperties = new Properties(); + environment = new HashMap<>(); + } @Test - public void shouldReadChecksFromEnvironmentOnly() { + public void shouldNotReadChecksFromClasspathProperties() { assertFalse("checks enabled by default", newConfig().isDisableChecks()); classpathProperties.setProperty("checks.disable", "true"); assertFalse("checks are not affected by classpath properties", newConfig().isDisableChecks()); + } + + @Test + public void shouldReadChecksFromUserProperties() { + assertFalse("checks enabled by default", newConfig().isDisableChecks()); - environmentProperties.setProperty("checks.disable", "true"); - assertTrue("checks disabled", newConfig().isDisableChecks()); + userProperties.setProperty("checks.disable", "true"); + assertTrue("checks disabled via user properties", newConfig().isDisableChecks()); } @Test - public void shouldReadDockerClientStrategyFromEnvironmentOnly() { + public void shouldReadChecksFromEnvironment() { + assertFalse("checks enabled by default", newConfig().isDisableChecks()); + + userProperties.remove("checks.disable"); + environment.put("TESTCONTAINERS_CHECKS_DISABLE", "true"); + assertTrue("checks disabled via env var", newConfig().isDisableChecks()); + } + + @Test + public void shouldNotReadDockerClientStrategyFromClasspathProperties() { String currentValue = newConfig().getDockerClientStrategyClassName(); classpathProperties.setProperty("docker.client.strategy", UUID.randomUUID().toString()); assertEquals("Docker client strategy is not affected by classpath properties", currentValue, newConfig().getDockerClientStrategyClassName()); + } + + @Test + public void shouldReadDockerClientStrategyFromUserProperties() { + userProperties.setProperty("docker.client.strategy", "foo"); + assertEquals("Docker client strategy is changed by user property", "foo", newConfig().getDockerClientStrategyClassName()); + } - environmentProperties.setProperty("docker.client.strategy", "foo"); - assertEquals("Docker client strategy is changed", "foo", newConfig().getDockerClientStrategyClassName()); + @Test + public void shouldReadDockerClientStrategyFromEnvironment() { + userProperties.remove("docker.client.strategy"); + environment.put("TESTCONTAINERS_DOCKER_CLIENT_STRATEGY", "foo"); + assertEquals("Docker client strategy is changed by env var", "foo", newConfig().getDockerClientStrategyClassName()); } @Test - public void shouldReadReuseFromEnvironmentOnly() { + public void shouldNotReadReuseFromClasspathProperties() { assertFalse("no reuse by default", newConfig().environmentSupportsReuse()); classpathProperties.setProperty("testcontainers.reuse.enable", "true"); assertFalse("reuse is not affected by classpath properties", newConfig().environmentSupportsReuse()); + } - environmentProperties.setProperty("testcontainers.reuse.enable", "true"); - assertTrue("reuse enabled", newConfig().environmentSupportsReuse()); + @Test + public void shouldReadReuseFromUserProperties() { + assertFalse("no reuse by default", newConfig().environmentSupportsReuse()); + + userProperties.setProperty("testcontainers.reuse.enable", "true"); + assertTrue("reuse enabled via user property", newConfig().environmentSupportsReuse()); + } + @Test + public void shouldReadReuseFromEnvironment() { + assertFalse("no reuse by default", newConfig().environmentSupportsReuse()); - environmentProperties.setProperty("ryuk.container.image", " testcontainersofficial/ryuk:0.3.0 "); - assertEquals("trailing whitespace was not removed from image name property", "testcontainersofficial/ryuk:0.3.0",newConfig().getRyukDockerImageName().asCanonicalNameString()); + userProperties.remove("testcontainers.reuse.enable"); + environment.put("TESTCONTAINERS_REUSE_ENABLE", "true"); + assertTrue("reuse enabled via env var", newConfig().environmentSupportsReuse()); + } + @Test + public void shouldTrimImageNames() { + userProperties.setProperty("ryuk.container.image", " testcontainersofficial/ryuk:0.3.0 "); + assertEquals("trailing whitespace was not removed from image name property", "testcontainersofficial/ryuk:0.3.0",newConfig().getRyukImage()); } private TestcontainersConfiguration newConfig() { - return new TestcontainersConfiguration(environmentProperties, classpathProperties); + return new TestcontainersConfiguration(userProperties, classpathProperties, environment); } } From 950af34178f3a28a3747b0b446738e83c7b62bbf Mon Sep 17 00:00:00 2001 From: Richard North Date: Thu, 29 Oct 2020 11:53:17 +0000 Subject: [PATCH 02/16] Add Image substitution mechanism Builds upon #3021 and #3411: * adds a pluggable image substitution mechanism using ServiceLoader, enabling users to perform custom substitution/auditing of images being used by their tests * provides a default implementation that behaves similarly to legacy `TestcontainersConfiguration` approach (`testcontainers.properties`) Notes: * behaviour is similar but not quite identical to `TestcontainersConfiguration`: use of a configured custom image for, e.g. Kafka/Pulsar that does not have a tag specified causes the substitution to take effect for all usages. It seems very unlikely that people would be using a mix of the config file image overrides in some places _and_ specific images specified in code in others. * Duplication of default image names in modules vs `TestcontainersConfiguration` class is intentional: specifying image overrides in `testcontainers.properties` should be removed in the future. * ~Add log deprecation warnings when `testcontainers.properties` image overrides are used.~ Defer to a future release? --- .../testcontainers/DockerClientFactory.java | 11 +- .../containers/DockerComposeContainer.java | 5 +- .../containers/GenericContainer.java | 6 +- .../containers/PortForwardingContainer.java | 6 +- .../containers/SocatContainer.java | 8 +- .../containers/VncRecordingContainer.java | 4 +- .../DockerClientProviderStrategy.java | 2 +- .../images/RemoteDockerImage.java | 10 +- ...ConfigurationFileImageNameSubstitutor.java | 52 +++++++++ .../utility/DefaultImageNameSubstitutor.java | 43 +++++++ .../utility/ImageNameSubstitutor.java | 97 ++++++++++++++++ .../utility/ResourceReaper.java | 4 +- .../utility/TestcontainersConfiguration.java | 37 ------ .../testcontainers/utility/Versioning.java | 1 + ...estcontainers.utility.ImageNameSubstitutor | 1 + .../DockerClientFactoryTest.java | 9 +- .../java/org/testcontainers/TestImages.java | 3 +- .../utility/AuthenticatedImagePullTest.java | 26 ++--- .../DefaultImageNameSubstitutorTest.java | 38 ++++++ .../DockerImageNameCompatibilityTest.java | 11 +- .../utility/ImageNameSubstitutorTest.java | 17 +++ .../TestcontainersConfigurationTest.java | 30 +++++ docs/examples/junit4/generic/build.gradle | 3 + .../generic/ExampleImageNameSubstitutor.java | 32 +++++ .../generic/ImageNameSubstitutionTest.java | 43 +++++++ docs/features/configuration.md | 36 ++++-- docs/features/image_name_substitution.md | 109 ++++++++++++++++++ docs/features/pull_rate_limiting.md | 19 +++ .../containers/KafkaContainer.java | 5 +- .../containers/KafkaContainerTest.java | 2 +- .../localstack/LocalStackContainer.java | 5 +- .../containers/OracleContainer.java | 3 +- .../containers/PulsarContainer.java | 5 +- .../spock/SpockTestImages.groovy | 5 +- 34 files changed, 575 insertions(+), 113 deletions(-) create mode 100644 core/src/main/java/org/testcontainers/utility/ConfigurationFileImageNameSubstitutor.java create mode 100644 core/src/main/java/org/testcontainers/utility/DefaultImageNameSubstitutor.java create mode 100644 core/src/main/java/org/testcontainers/utility/ImageNameSubstitutor.java create mode 100644 core/src/main/resources/META-INF/services/org.testcontainers.utility.ImageNameSubstitutor create mode 100644 core/src/test/java/org/testcontainers/utility/DefaultImageNameSubstitutorTest.java create mode 100644 core/src/test/java/org/testcontainers/utility/ImageNameSubstitutorTest.java create mode 100644 docs/examples/junit4/generic/src/test/java/generic/ExampleImageNameSubstitutor.java create mode 100644 docs/examples/junit4/generic/src/test/java/generic/ImageNameSubstitutionTest.java create mode 100644 docs/features/image_name_substitution.md create mode 100644 docs/features/pull_rate_limiting.md diff --git a/core/src/main/java/org/testcontainers/DockerClientFactory.java b/core/src/main/java/org/testcontainers/DockerClientFactory.java index 571b18fbcce..d44cd3b5b6f 100644 --- a/core/src/main/java/org/testcontainers/DockerClientFactory.java +++ b/core/src/main/java/org/testcontainers/DockerClientFactory.java @@ -25,6 +25,8 @@ import org.testcontainers.dockerclient.TransportConfig; import org.testcontainers.images.TimeLimitedLoggedPullImageResultCallback; import org.testcontainers.utility.ComparableVersion; +import org.testcontainers.utility.DockerImageName; +import org.testcontainers.utility.ImageNameSubstitutor; import org.testcontainers.utility.MountableFile; import org.testcontainers.utility.ResourceReaper; import org.testcontainers.utility.TestcontainersConfiguration; @@ -61,7 +63,7 @@ public class DockerClientFactory { TESTCONTAINERS_SESSION_ID_LABEL, SESSION_ID ); - private static final String TINY_IMAGE = TestcontainersConfiguration.getInstance().getTinyDockerImageName().asCanonicalNameString(); + private static final DockerImageName TINY_IMAGE = DockerImageName.parse("alpine:3.5"); private static DockerClientFactory instance; // Cached client configuration @@ -343,8 +345,11 @@ public T runInsideDocker(Consumer createContainerCmdCons } private T runInsideDocker(DockerClient client, Consumer createContainerCmdConsumer, BiFunction block) { - checkAndPullImage(client, TINY_IMAGE); - CreateContainerCmd createContainerCmd = client.createContainerCmd(TINY_IMAGE) + + final String tinyImage = ImageNameSubstitutor.instance().apply(TINY_IMAGE).asCanonicalNameString(); + + checkAndPullImage(client, tinyImage); + CreateContainerCmd createContainerCmd = client.createContainerCmd(tinyImage) .withLabels(DEFAULT_LABELS); createContainerCmdConsumer.accept(createContainerCmd); String id = createContainerCmd.exec().getId(); diff --git a/core/src/main/java/org/testcontainers/containers/DockerComposeContainer.java b/core/src/main/java/org/testcontainers/containers/DockerComposeContainer.java index bd06083644d..7a67073e383 100644 --- a/core/src/main/java/org/testcontainers/containers/DockerComposeContainer.java +++ b/core/src/main/java/org/testcontainers/containers/DockerComposeContainer.java @@ -29,11 +29,11 @@ import org.testcontainers.utility.AuditLogger; import org.testcontainers.utility.Base58; import org.testcontainers.utility.CommandLine; +import org.testcontainers.utility.DockerImageName; import org.testcontainers.utility.DockerLoggerFactory; import org.testcontainers.utility.LogUtils; import org.testcontainers.utility.MountableFile; import org.testcontainers.utility.ResourceReaper; -import org.testcontainers.utility.TestcontainersConfiguration; import org.zeroturnaround.exec.InvalidExitValueException; import org.zeroturnaround.exec.ProcessExecutor; import org.zeroturnaround.exec.stream.slf4j.Slf4jStream; @@ -608,10 +608,11 @@ interface DockerCompose { class ContainerisedDockerCompose extends GenericContainer implements DockerCompose { public static final char UNIX_PATH_SEPERATOR = ':'; + public static final DockerImageName DEFAULT_IMAGE_NAME = DockerImageName.parse("docker/compose:1.24.1"); public ContainerisedDockerCompose(List composeFiles, String identifier) { - super(TestcontainersConfiguration.getInstance().getDockerComposeDockerImageName()); + super(DEFAULT_IMAGE_NAME); addEnv(ENV_PROJECT_NAME, identifier); // Map the docker compose file into the container diff --git a/core/src/main/java/org/testcontainers/containers/GenericContainer.java b/core/src/main/java/org/testcontainers/containers/GenericContainer.java index 95050d8f8ee..641096a0779 100644 --- a/core/src/main/java/org/testcontainers/containers/GenericContainer.java +++ b/core/src/main/java/org/testcontainers/containers/GenericContainer.java @@ -239,13 +239,9 @@ public GenericContainer(@NonNull final RemoteDockerImage image) { */ @Deprecated public GenericContainer() { - this(TestcontainersConfiguration.getInstance().getTinyDockerImageName().asCanonicalNameString()); + this(TestcontainersConfiguration.getInstance().getTinyImage()); } - /** - * @deprecated use {@link GenericContainer(DockerImageName)} instead - */ - @Deprecated public GenericContainer(@NonNull final String dockerImageName) { this.setDockerImageName(dockerImageName); } diff --git a/core/src/main/java/org/testcontainers/containers/PortForwardingContainer.java b/core/src/main/java/org/testcontainers/containers/PortForwardingContainer.java index e42f2681675..b3f020500a1 100644 --- a/core/src/main/java/org/testcontainers/containers/PortForwardingContainer.java +++ b/core/src/main/java/org/testcontainers/containers/PortForwardingContainer.java @@ -5,15 +5,15 @@ import lombok.AccessLevel; import lombok.Getter; import lombok.SneakyThrows; -import org.testcontainers.utility.TestcontainersConfiguration; +import org.testcontainers.utility.DockerImageName; import java.time.Duration; import java.util.AbstractMap; import java.util.Collections; +import java.util.Map.Entry; import java.util.Optional; import java.util.Set; import java.util.UUID; -import java.util.Map.Entry; import java.util.concurrent.ConcurrentHashMap; public enum PortForwardingContainer { @@ -29,7 +29,7 @@ public enum PortForwardingContainer { @SneakyThrows private Connection createSSHSession() { String password = UUID.randomUUID().toString(); - container = new GenericContainer<>(TestcontainersConfiguration.getInstance().getSSHdDockerImageName()) + container = new GenericContainer<>(DockerImageName.parse("testcontainers/sshd:1.0.0")) .withExposedPorts(22) .withEnv("PASSWORD", password) .withCommand( diff --git a/core/src/main/java/org/testcontainers/containers/SocatContainer.java b/core/src/main/java/org/testcontainers/containers/SocatContainer.java index fbca73eb29d..7592d9f4d1f 100644 --- a/core/src/main/java/org/testcontainers/containers/SocatContainer.java +++ b/core/src/main/java/org/testcontainers/containers/SocatContainer.java @@ -1,12 +1,10 @@ package org.testcontainers.containers; -import org.testcontainers.utility.Base58; -import org.testcontainers.utility.DockerImageName; -import org.testcontainers.utility.TestcontainersConfiguration; - import java.util.HashMap; import java.util.Map; import java.util.stream.Collectors; +import org.testcontainers.utility.Base58; +import org.testcontainers.utility.DockerImageName; /** * A socat container is used as a TCP proxy, enabling any TCP port of another container to be exposed @@ -17,7 +15,7 @@ public class SocatContainer extends GenericContainer { private final Map targets = new HashMap<>(); public SocatContainer() { - this(TestcontainersConfiguration.getInstance().getSocatDockerImageName()); + this(DockerImageName.parse("alpine/socat:1.7.3.4-r0")); } public SocatContainer(final DockerImageName dockerImageName) { diff --git a/core/src/main/java/org/testcontainers/containers/VncRecordingContainer.java b/core/src/main/java/org/testcontainers/containers/VncRecordingContainer.java index 8ac060c2af6..de720110a51 100644 --- a/core/src/main/java/org/testcontainers/containers/VncRecordingContainer.java +++ b/core/src/main/java/org/testcontainers/containers/VncRecordingContainer.java @@ -6,7 +6,7 @@ import lombok.ToString; import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy; -import org.testcontainers.utility.TestcontainersConfiguration; +import org.testcontainers.utility.DockerImageName; import java.io.File; import java.io.InputStream; @@ -52,7 +52,7 @@ public VncRecordingContainer(@NonNull GenericContainer targetContainer) { * Create a sidekick container and attach it to another container. The VNC output of that container will be recorded. */ public VncRecordingContainer(@NonNull Network network, @NonNull String targetNetworkAlias) throws IllegalStateException { - super(TestcontainersConfiguration.getInstance().getVncDockerImageName()); + super(DockerImageName.parse("testcontainers/vnc-recorder:1.1.0")); this.targetNetworkAlias = targetNetworkAlias; withNetwork(network); diff --git a/core/src/main/java/org/testcontainers/dockerclient/DockerClientProviderStrategy.java b/core/src/main/java/org/testcontainers/dockerclient/DockerClientProviderStrategy.java index e21c75a581d..1bd51da1263 100644 --- a/core/src/main/java/org/testcontainers/dockerclient/DockerClientProviderStrategy.java +++ b/core/src/main/java/org/testcontainers/dockerclient/DockerClientProviderStrategy.java @@ -171,7 +171,7 @@ public static DockerClientProviderStrategy getFirstValidStrategy(List imageFuture) { @@ -100,7 +101,10 @@ protected final String resolve() { } private DockerImageName getImageName() throws InterruptedException, ExecutionException { - return imageNameFuture.get(); + final DockerImageName specifiedImageName = imageNameFuture.get(); + + // Allow the image name to be substituted + return ImageNameSubstitutor.instance().apply(specifiedImageName); } @ToString.Include(name = "imageName", rank = 1) diff --git a/core/src/main/java/org/testcontainers/utility/ConfigurationFileImageNameSubstitutor.java b/core/src/main/java/org/testcontainers/utility/ConfigurationFileImageNameSubstitutor.java new file mode 100644 index 00000000000..e163f8ffff4 --- /dev/null +++ b/core/src/main/java/org/testcontainers/utility/ConfigurationFileImageNameSubstitutor.java @@ -0,0 +1,52 @@ +package org.testcontainers.utility; + +import com.google.common.annotations.VisibleForTesting; +import lombok.extern.slf4j.Slf4j; + +/** + * {@link ImageNameSubstitutor} which takes replacement image names from configuration. + * See {@link TestcontainersConfiguration} for the subset of image names which can be substituted using this mechanism. + *

+ * WARNING: this class is not intended to be public, but {@link java.util.ServiceLoader} + * requires it to be so. Public visibility DOES NOT make it part of the public API. + */ +@Slf4j +public class ConfigurationFileImageNameSubstitutor extends ImageNameSubstitutor { + + private final TestcontainersConfiguration configuration; + + public ConfigurationFileImageNameSubstitutor() { + this(TestcontainersConfiguration.getInstance()); + } + + @VisibleForTesting + ConfigurationFileImageNameSubstitutor(TestcontainersConfiguration configuration) { + this.configuration = configuration; + } + + @Override + public DockerImageName apply(final DockerImageName original) { + final DockerImageName result = configuration + .getConfiguredSubstituteImage(original) + .asCompatibleSubstituteFor(original); + + if (!result.equals(original)) { + log.warn("Image name {} was substituted by configuration to {}. This approach is deprecated and will be removed in the future", + original, + result + ); + } + + return result; + } + + @Override + protected int getPriority() { + return -2; + } + + @Override + protected String getDescription() { + return getClass().getSimpleName(); + } +} diff --git a/core/src/main/java/org/testcontainers/utility/DefaultImageNameSubstitutor.java b/core/src/main/java/org/testcontainers/utility/DefaultImageNameSubstitutor.java new file mode 100644 index 00000000000..5657a9051aa --- /dev/null +++ b/core/src/main/java/org/testcontainers/utility/DefaultImageNameSubstitutor.java @@ -0,0 +1,43 @@ +package org.testcontainers.utility; + +import com.google.common.annotations.VisibleForTesting; +import lombok.extern.slf4j.Slf4j; + +/** + * Testcontainers' default implementation of {@link ImageNameSubstitutor}. + * Delegates to {@link ConfigurationFileImageNameSubstitutor}. + *

+ * WARNING: this class is not intended to be public, but {@link java.util.ServiceLoader} + * requires it to be so. Public visibility DOES NOT make it part of the public API. + */ +@Slf4j +public class DefaultImageNameSubstitutor extends ImageNameSubstitutor { + + private final ConfigurationFileImageNameSubstitutor configurationFileImageNameSubstitutor; + + public DefaultImageNameSubstitutor() { + configurationFileImageNameSubstitutor = new ConfigurationFileImageNameSubstitutor(); + } + + @VisibleForTesting + DefaultImageNameSubstitutor( + final ConfigurationFileImageNameSubstitutor configurationFileImageNameSubstitutor + ) { + this.configurationFileImageNameSubstitutor = configurationFileImageNameSubstitutor; + } + + @Override + public DockerImageName apply(final DockerImageName original) { + return configurationFileImageNameSubstitutor.apply(original); + } + + @Override + protected int getPriority() { + return 0; + } + + @Override + protected String getDescription() { + return "DefaultImageNameSubstitutor (delegates to '" + configurationFileImageNameSubstitutor.getDescription() + "')"; + } +} diff --git a/core/src/main/java/org/testcontainers/utility/ImageNameSubstitutor.java b/core/src/main/java/org/testcontainers/utility/ImageNameSubstitutor.java new file mode 100644 index 00000000000..4179d2b5f52 --- /dev/null +++ b/core/src/main/java/org/testcontainers/utility/ImageNameSubstitutor.java @@ -0,0 +1,97 @@ +package org.testcontainers.utility; + +import com.google.common.annotations.VisibleForTesting; +import lombok.extern.slf4j.Slf4j; + +import java.util.ServiceLoader; +import java.util.function.Function; +import java.util.stream.StreamSupport; + +import static java.util.Comparator.comparingInt; + +/** + * An image name substitutor converts a Docker image name, as may be specified in code, to an alternative name. + * This is intended to provide a way to override image names, for example to enforce pulling of images from a private + * registry. + */ +@Slf4j +public abstract class ImageNameSubstitutor implements Function { + + @VisibleForTesting + static ImageNameSubstitutor instance; + + public synchronized static ImageNameSubstitutor instance() { + if (instance == null) { + final ServiceLoader serviceLoader = ServiceLoader.load(ImageNameSubstitutor.class); + + instance = StreamSupport.stream(serviceLoader.spliterator(), false) + .peek(it -> log.debug("Found ImageNameSubstitutor using ServiceLoader: {} (priority {}) ", it, it.getPriority())) + .max(comparingInt(ImageNameSubstitutor::getPriority)) + .map(ImageNameSubstitutor::wrapWithLogging) + .orElseThrow(() -> new RuntimeException("Unable to find any ImageNameSubstitutor using ServiceLoader")); + + log.info("Using ImageNameSubstitutor: {}", instance); + } + + return instance; + } + + private static ImageNameSubstitutor wrapWithLogging(final ImageNameSubstitutor wrappedInstance) { + return new LogWrappedImageNameSubstitutor(wrappedInstance); + } + + /** + * Substitute a {@link DockerImageName} for another, for example to replace a generic Docker Hub image name with a + * private registry copy of the image. + * + * @param original original name to be replaced + * @return a replacement name, or the original, as appropriate + */ + public abstract DockerImageName apply(DockerImageName original); + + /** + * Priority of this {@link ImageNameSubstitutor} compared to other instances that may be found by the service + * loader. The highest priority instance found will always be used. + * + * @return a priority + */ + protected abstract int getPriority(); + + protected abstract String getDescription(); + + /** + * Wrapper substitutor which logs which substitutions have been performed. + */ + static class LogWrappedImageNameSubstitutor extends ImageNameSubstitutor { + @VisibleForTesting + final ImageNameSubstitutor wrappedInstance; + + public LogWrappedImageNameSubstitutor(final ImageNameSubstitutor wrappedInstance) { + this.wrappedInstance = wrappedInstance; + } + + @Override + public DockerImageName apply(final DockerImageName original) { + final String className = wrappedInstance.getClass().getName(); + final DockerImageName replacementImage = wrappedInstance.apply(original); + + if (!replacementImage.equals(original)) { + log.info("Using {} as a substitute image for {} (using image substitutor: {})", replacementImage.asCanonicalNameString(), original.asCanonicalNameString(), className); + return replacementImage; + } else { + log.debug("Did not find a substitute image for {} (using image substitutor: {})", original.asCanonicalNameString(), className); + return original; + } + } + + @Override + protected int getPriority() { + return wrappedInstance.getPriority(); + } + + @Override + protected String getDescription() { + return wrappedInstance.getDescription(); + } + } +} diff --git a/core/src/main/java/org/testcontainers/utility/ResourceReaper.java b/core/src/main/java/org/testcontainers/utility/ResourceReaper.java index c37e6660654..3b0cfa1e289 100644 --- a/core/src/main/java/org/testcontainers/utility/ResourceReaper.java +++ b/core/src/main/java/org/testcontainers/utility/ResourceReaper.java @@ -72,7 +72,9 @@ private ResourceReaper() { @SneakyThrows(InterruptedException.class) public static String start(String hostIpAddress, DockerClient client) { - String ryukImage = TestcontainersConfiguration.getInstance().getRyukDockerImageName().asCanonicalNameString(); + String ryukImage = ImageNameSubstitutor.instance() + .apply(DockerImageName.parse("testcontainers/ryuk:0.3.0")) + .asCanonicalNameString(); DockerClientFactory.instance().checkAndPullImage(client, ryukImage); List binds = new ArrayList<>(); diff --git a/core/src/main/java/org/testcontainers/utility/TestcontainersConfiguration.java b/core/src/main/java/org/testcontainers/utility/TestcontainersConfiguration.java index b3c8e88a328..b685eb011bc 100644 --- a/core/src/main/java/org/testcontainers/utility/TestcontainersConfiguration.java +++ b/core/src/main/java/org/testcontainers/utility/TestcontainersConfiguration.java @@ -104,37 +104,21 @@ public String getSocatContainerImage() { return getImage(SOCAT_IMAGE).asCanonicalNameString(); } - public DockerImageName getSocatDockerImageName() { - return getImage(SOCAT_IMAGE); - } - @Deprecated public String getVncRecordedContainerImage() { return getImage(VNC_RECORDER_IMAGE).asCanonicalNameString(); } - public DockerImageName getVncDockerImageName() { - return getImage(VNC_RECORDER_IMAGE); - } - @Deprecated public String getDockerComposeContainerImage() { return getImage(COMPOSE_IMAGE).asCanonicalNameString(); } - public DockerImageName getDockerComposeDockerImageName() { - return getImage(COMPOSE_IMAGE); - } - @Deprecated public String getTinyImage() { return getImage(ALPINE_IMAGE).asCanonicalNameString(); } - public DockerImageName getTinyDockerImageName() { - return getImage(ALPINE_IMAGE); - } - public boolean isRyukPrivileged() { return Boolean .parseBoolean(getEnvVarOrProperty("ryuk.container.privileged", "false")); @@ -145,19 +129,11 @@ public String getRyukImage() { return getImage(RYUK_IMAGE).asCanonicalNameString(); } - public DockerImageName getRyukDockerImageName() { - return getImage(RYUK_IMAGE); - } - @Deprecated public String getSSHdImage() { return getImage(SSHD_IMAGE).asCanonicalNameString(); } - public DockerImageName getSSHdDockerImageName() { - return getImage(SSHD_IMAGE); - } - public Integer getRyukTimeout() { return Integer.parseInt(getEnvVarOrProperty("ryuk.container.timeout", "30")); } @@ -167,11 +143,6 @@ public String getKafkaImage() { return getImage(KAFKA_IMAGE).asCanonicalNameString(); } - public DockerImageName getKafkaDockerImageName() { - return getImage(KAFKA_IMAGE); - } - - @Deprecated public String getOracleImage() { return getEnvVarOrUserProperty("oracle.container.image", null); @@ -182,19 +153,11 @@ public String getPulsarImage() { return getImage(PULSAR_IMAGE).asCanonicalNameString(); } - public DockerImageName getPulsarDockerImageName() { - return getImage(PULSAR_IMAGE); - } - @Deprecated public String getLocalStackImage() { return getImage(LOCALSTACK_IMAGE).asCanonicalNameString(); } - public DockerImageName getLocalstackDockerImageName() { - return getImage(LOCALSTACK_IMAGE); - } - public boolean isDisableChecks() { return Boolean.parseBoolean(getEnvVarOrUserProperty("checks.disable", "false")); diff --git a/core/src/main/java/org/testcontainers/utility/Versioning.java b/core/src/main/java/org/testcontainers/utility/Versioning.java index 84dcf46274e..4b1da407440 100644 --- a/core/src/main/java/org/testcontainers/utility/Versioning.java +++ b/core/src/main/java/org/testcontainers/utility/Versioning.java @@ -46,6 +46,7 @@ public int hashCode() { @EqualsAndHashCode class TagVersioning implements Versioning { public static final String TAG_REGEX = "[\\w][\\w.\\-]{0,127}"; + static final TagVersioning LATEST = new TagVersioning("latest"); private final String tag; TagVersioning(String tag) { diff --git a/core/src/main/resources/META-INF/services/org.testcontainers.utility.ImageNameSubstitutor b/core/src/main/resources/META-INF/services/org.testcontainers.utility.ImageNameSubstitutor new file mode 100644 index 00000000000..069dc7ee710 --- /dev/null +++ b/core/src/main/resources/META-INF/services/org.testcontainers.utility.ImageNameSubstitutor @@ -0,0 +1 @@ +org.testcontainers.utility.DefaultImageNameSubstitutor diff --git a/core/src/test/java/org/testcontainers/DockerClientFactoryTest.java b/core/src/test/java/org/testcontainers/DockerClientFactoryTest.java index 030e4164f00..d484cc92eaf 100644 --- a/core/src/test/java/org/testcontainers/DockerClientFactoryTest.java +++ b/core/src/test/java/org/testcontainers/DockerClientFactoryTest.java @@ -1,5 +1,9 @@ package org.testcontainers; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.testcontainers.TestImages.TINY_IMAGE; + import com.github.dockerjava.api.DockerClient; import com.github.dockerjava.api.exception.DockerException; import com.github.dockerjava.api.exception.NotFoundException; @@ -12,9 +16,6 @@ import org.testcontainers.utility.MockTestcontainersConfigurationRule; import org.testcontainers.utility.TestcontainersConfiguration; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - /** * Test for {@link DockerClientFactory}. */ @@ -30,7 +31,7 @@ public void runCommandInsideDockerShouldNotFailIfImageDoesNotExistsLocally() { try { //remove tiny image, so it will be pulled during next command run dockFactory.client() - .removeImageCmd(TestcontainersConfiguration.getInstance().getTinyDockerImageName().asCanonicalNameString()) + .removeImageCmd(TINY_IMAGE.asCanonicalNameString()) .withForce(true).exec(); } catch (NotFoundException ignored) { // Do not fail if it's not pulled yet diff --git a/core/src/test/java/org/testcontainers/TestImages.java b/core/src/test/java/org/testcontainers/TestImages.java index 3f16e88d882..7052f7e8f9d 100644 --- a/core/src/test/java/org/testcontainers/TestImages.java +++ b/core/src/test/java/org/testcontainers/TestImages.java @@ -1,7 +1,6 @@ package org.testcontainers; import org.testcontainers.utility.DockerImageName; -import org.testcontainers.utility.TestcontainersConfiguration; public interface TestImages { DockerImageName REDIS_IMAGE = DockerImageName.parse("redis:3.0.2"); @@ -9,5 +8,5 @@ public interface TestImages { DockerImageName MONGODB_IMAGE = DockerImageName.parse("mongo:3.1.5"); DockerImageName ALPINE_IMAGE = DockerImageName.parse("alpine:3.2"); DockerImageName DOCKER_REGISTRY_IMAGE = DockerImageName.parse("registry:2.7.0"); - DockerImageName TINY_IMAGE = TestcontainersConfiguration.getInstance().getTinyDockerImageName(); + DockerImageName TINY_IMAGE = DockerImageName.parse("alpine:3.5"); } diff --git a/core/src/test/java/org/testcontainers/utility/AuthenticatedImagePullTest.java b/core/src/test/java/org/testcontainers/utility/AuthenticatedImagePullTest.java index e49ff2880df..c99770c4fbf 100644 --- a/core/src/test/java/org/testcontainers/utility/AuthenticatedImagePullTest.java +++ b/core/src/test/java/org/testcontainers/utility/AuthenticatedImagePullTest.java @@ -1,9 +1,21 @@ package org.testcontainers.utility; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.when; +import static org.rnorth.visibleassertions.VisibleAssertions.assertTrue; +import static org.testcontainers.TestImages.DOCKER_REGISTRY_IMAGE; +import static org.testcontainers.TestImages.TINY_IMAGE; + import com.github.dockerjava.api.DockerClient; import com.github.dockerjava.api.async.ResultCallback; import com.github.dockerjava.api.command.PullImageResultCallback; import com.github.dockerjava.api.model.AuthConfig; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.concurrent.TimeUnit; import org.intellij.lang.annotations.Language; import org.junit.AfterClass; import org.junit.Before; @@ -18,18 +30,6 @@ import org.testcontainers.containers.wait.strategy.HttpWaitStrategy; import org.testcontainers.images.builder.ImageFromDockerfile; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.concurrent.TimeUnit; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.when; -import static org.rnorth.visibleassertions.VisibleAssertions.assertTrue; -import static org.testcontainers.TestImages.DOCKER_REGISTRY_IMAGE; - /** * This test checks the integration between Testcontainers and an authenticated registry, but uses * a mock instance of {@link RegistryAuthLocator} - the purpose of the test is solely to ensure that @@ -165,7 +165,7 @@ private Path getLocalTempFile(String s) throws IOException { private static void putImageInRegistry() throws InterruptedException { // It doesn't matter which image we use for this test, but use one that's likely to have been pulled already - final String dummySourceImage = TestcontainersConfiguration.getInstance().getRyukDockerImageName().asCanonicalNameString(); + final String dummySourceImage = TINY_IMAGE.asCanonicalNameString(); client.pullImageCmd(dummySourceImage) .exec(new PullImageResultCallback()) diff --git a/core/src/test/java/org/testcontainers/utility/DefaultImageNameSubstitutorTest.java b/core/src/test/java/org/testcontainers/utility/DefaultImageNameSubstitutorTest.java new file mode 100644 index 00000000000..a6f7813c3d3 --- /dev/null +++ b/core/src/test/java/org/testcontainers/utility/DefaultImageNameSubstitutorTest.java @@ -0,0 +1,38 @@ +package org.testcontainers.utility; + +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.mockito.Mockito; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.eq; + +public class DefaultImageNameSubstitutorTest { + + public static final DockerImageName ORIGINAL_IMAGE = DockerImageName.parse("foo"); + public static final DockerImageName SUBSTITUTE_IMAGE = DockerImageName.parse("bar"); + private ConfigurationFileImageNameSubstitutor underTest; + + @Rule + public MockTestcontainersConfigurationRule config = new MockTestcontainersConfigurationRule(); + + @Before + public void setUp() { + underTest = new ConfigurationFileImageNameSubstitutor(TestcontainersConfiguration.getInstance()); + } + + @Test + public void testConfigurationLookup() { + Mockito + .doReturn(SUBSTITUTE_IMAGE) + .when(TestcontainersConfiguration.getInstance()) + .getConfiguredSubstituteImage(eq(ORIGINAL_IMAGE)); + + final DockerImageName substitute = underTest.apply(ORIGINAL_IMAGE); + + assertEquals("match is found", SUBSTITUTE_IMAGE, substitute); + assertTrue("compatibility is automatically set", substitute.isCompatibleWith(ORIGINAL_IMAGE)); + } +} diff --git a/core/src/test/java/org/testcontainers/utility/DockerImageNameCompatibilityTest.java b/core/src/test/java/org/testcontainers/utility/DockerImageNameCompatibilityTest.java index 9e7136ea6d3..22d67de5785 100644 --- a/core/src/test/java/org/testcontainers/utility/DockerImageNameCompatibilityTest.java +++ b/core/src/test/java/org/testcontainers/utility/DockerImageNameCompatibilityTest.java @@ -1,13 +1,13 @@ package org.testcontainers.utility; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; - import static org.hamcrest.core.StringContains.containsString; import static org.rnorth.visibleassertions.VisibleAssertions.assertFalse; import static org.rnorth.visibleassertions.VisibleAssertions.assertTrue; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + public class DockerImageNameCompatibilityTest { @@ -26,9 +26,8 @@ public void testNoTagTreatedAsWildcard() { /* foo:1.2.3 != foo:4.5.6 foo:1.2.3 ~= foo - foo:1.2.3 ~= foo:latest - The test is effectively making sure that no tag and `latest` tag are equivalent + The test is effectively making sure that 'no tag' is treated as a wildcard */ assertFalse("foo:4.5.6 != foo:1.2.3", subject.isCompatibleWith(DockerImageName.parse("foo:1.2.3"))); assertTrue("foo:4.5.6 ~= foo", subject.isCompatibleWith(DockerImageName.parse("foo"))); diff --git a/core/src/test/java/org/testcontainers/utility/ImageNameSubstitutorTest.java b/core/src/test/java/org/testcontainers/utility/ImageNameSubstitutorTest.java new file mode 100644 index 00000000000..d39e83391f9 --- /dev/null +++ b/core/src/test/java/org/testcontainers/utility/ImageNameSubstitutorTest.java @@ -0,0 +1,17 @@ +package org.testcontainers.utility; + +import org.junit.Test; +import org.testcontainers.utility.ImageNameSubstitutor.LogWrappedImageNameSubstitutor; + +import static org.junit.Assert.assertTrue; + +public class ImageNameSubstitutorTest { + + @Test + public void simpleServiceLoadingTest() { + final ImageNameSubstitutor imageNameSubstitutor = ImageNameSubstitutor.instance(); + + assertTrue(imageNameSubstitutor instanceof LogWrappedImageNameSubstitutor); + assertTrue(((LogWrappedImageNameSubstitutor) imageNameSubstitutor).wrappedInstance instanceof DefaultImageNameSubstitutor); + } +} diff --git a/core/src/test/java/org/testcontainers/utility/TestcontainersConfigurationTest.java b/core/src/test/java/org/testcontainers/utility/TestcontainersConfigurationTest.java index e48a2a6fdf9..147e5e60bd4 100644 --- a/core/src/test/java/org/testcontainers/utility/TestcontainersConfigurationTest.java +++ b/core/src/test/java/org/testcontainers/utility/TestcontainersConfigurationTest.java @@ -25,6 +25,36 @@ public void setUp() { environment = new HashMap<>(); } + @Test + public void shouldSubstituteImageNamesFromClasspathProperties() { + classpathProperties.setProperty("ryuk.container.image", "foo:version"); + assertEquals( + "an image name can be pulled from classpath properties", + DockerImageName.parse("foo:version"), + newConfig().getConfiguredSubstituteImage(DockerImageName.parse("testcontainers/ryuk:any")) + ); + } + + @Test + public void shouldSubstituteImageNamesFromUserProperties() { + userProperties.setProperty("ryuk.container.image", "foo:version"); + assertEquals( + "an image name can be pulled from user properties", + DockerImageName.parse("foo:version"), + newConfig().getConfiguredSubstituteImage(DockerImageName.parse("testcontainers/ryuk:any")) + ); + } + + @Test + public void shouldSubstituteImageNamesFromEnvironmentVariables() { + environment.put("TESTCONTAINERS_RYUK_CONTAINER_IMAGE", "foo:version"); + assertEquals( + "an image name can be pulled from an environment variable", + DockerImageName.parse("foo:version"), + newConfig().getConfiguredSubstituteImage(DockerImageName.parse("testcontainers/ryuk:any")) + ); + } + @Test public void shouldNotReadChecksFromClasspathProperties() { assertFalse("checks enabled by default", newConfig().isDisableChecks()); diff --git a/docs/examples/junit4/generic/build.gradle b/docs/examples/junit4/generic/build.gradle index 5ce878c7ba2..ff1b11f142e 100644 --- a/docs/examples/junit4/generic/build.gradle +++ b/docs/examples/junit4/generic/build.gradle @@ -4,6 +4,9 @@ dependencies { testCompile "junit:junit:4.12" testCompile project(":testcontainers") testCompile project(":selenium") + testCompile project(":mysql") + + testCompile 'mysql:mysql-connector-java:8.0.21' testCompile "org.seleniumhq.selenium:selenium-api:3.141.59" testCompile 'org.assertj:assertj-core:3.15.0' } diff --git a/docs/examples/junit4/generic/src/test/java/generic/ExampleImageNameSubstitutor.java b/docs/examples/junit4/generic/src/test/java/generic/ExampleImageNameSubstitutor.java new file mode 100644 index 00000000000..dfb5351ecfd --- /dev/null +++ b/docs/examples/junit4/generic/src/test/java/generic/ExampleImageNameSubstitutor.java @@ -0,0 +1,32 @@ +package generic; + +import org.testcontainers.utility.DockerImageName; +import org.testcontainers.utility.ImageNameSubstitutor; + +public class ExampleImageNameSubstitutor extends ImageNameSubstitutor { + + @Override + public DockerImageName apply(DockerImageName original) { + // convert the original name to something appropriate for + // our build environment + return DockerImageName.parse( + // your code goes here - silly example of capitalising + // the original name is shown + original.asCanonicalNameString().toUpperCase() + ); + } + + @Override + protected int getPriority() { + // the highest priority substitutor is used. + // Use something higher than 0, which is the priority + // of the default implementation + return 1; + } + + @Override + protected String getDescription() { + // used in logs + return "example image name substitutor"; + } +} diff --git a/docs/examples/junit4/generic/src/test/java/generic/ImageNameSubstitutionTest.java b/docs/examples/junit4/generic/src/test/java/generic/ImageNameSubstitutionTest.java new file mode 100644 index 00000000000..dcb421a7cbc --- /dev/null +++ b/docs/examples/junit4/generic/src/test/java/generic/ImageNameSubstitutionTest.java @@ -0,0 +1,43 @@ +package generic; + + +import org.junit.Ignore; +import org.junit.Test; +import org.testcontainers.containers.MySQLContainer; +import org.testcontainers.utility.DockerImageName; + +public class ImageNameSubstitutionTest { + + @Test + public void simpleExample() { + try ( + // directDockerHubReference { + // Referring directly to an image on Docker Hub (mysql:8.0.22) + final MySQLContainer mysql = new MySQLContainer<>( + DockerImageName.parse("mysql:8.0.22") + ) + + // start the container and use it for testing + // } + ) { + mysql.start(); + } + } + + @Test @Ignore + public void substitutedExample() { + try ( + // hardcodedMirror { + // Referring directly to an image on a private registry - image name will vary + final MySQLContainer mysql = new MySQLContainer<>( + DockerImageName.parse("registry.mycompany.com/mirror/mysql:8.0.22") + .asCompatibleSubstituteFor("mysql") + ) + + // start the container and use it for testing + // } + ) { + mysql.start(); + } + } +} diff --git a/docs/features/configuration.md b/docs/features/configuration.md index 8b12f8e8d54..0ae05e7d989 100644 --- a/docs/features/configuration.md +++ b/docs/features/configuration.md @@ -2,14 +2,19 @@ You can override some default properties if your environment requires that. -## Configuration file location +## Configuration locations The configuration will be loaded from multiple locations. Properties are considered in the following order: -1. `.testcontainers.properties` in user's home folder. Example locations: +1. Environment variables +2. `.testcontainers.properties` in user's home folder. Example locations: **Linux:** `/home/myuser/.testcontainers.properties` **Windows:** `C:/Users/myuser/.testcontainers.properties` **macOS:** `/Users/myuser/.testcontainers.properties` -2. `testcontainers.properties` on classpath +3. `testcontainers.properties` on classpath + +Note that when using environment variables, configuration property names should be set in upper +case with underscore separators, preceded by `TESTCONTAINERS_` - e.g. `checks.disable` becomes +`TESTCONTAINERS_CHECKS_DISABLE`. ## Disabling the startup checks > **checks.disable = [true|false]** @@ -26,18 +31,28 @@ It takes a couple of seconds, but if you want to speed up your tests, you can di ## Customizing images +!!! note + This approach is discouraged and deprecated, but is documented for completeness. + Overriding individual image names via configuration may be removed in 2021. + Testcontainers uses public Docker images to perform different actions like startup checks, VNC recording and others. Some companies disallow the usage of Docker Hub, but you can override `*.image` properties with your own images from your private registry to workaround that. +> **ryuk.container.image = testcontainersofficial/ryuk:0.3.0** +> Performs fail-safe cleanup of containers, and always required (unless [Ryuk is disabled](#disabling-ryuk)) + > **tinyimage.container.image = alpine:3.5** -> Used by Testcontainers' core +> Used to check whether images can be pulled at startup, and always required (unless [startup checks are disabled](#disabling-the-startup-checks)) + +> **sshd.container.image = testcontainers/sshd:1.0.0** +> Required if [exposing host ports to containers](./networking.md#exposing-host-ports-to-the-container) > **vncrecorder.container.image = testcontainersofficial/vnc-recorder:1.1.0** -> Used by VNC recorder in Testcontainers' Seleniun integration +> Used by VNC recorder in Testcontainers' Selenium integration -> **ambassador.container.image = richnorth/ambassador:latest** +> **socat.container.image = alpine/socat** > **compose.container.image = docker/compose:1.8.0** -> Used by Docker Compose integration +> Required if using [Docker Compose](../modules/docker_compose.md) > **kafka.container.image = confluentinc/cp-kafka** > Used by KafkaContainer @@ -45,11 +60,10 @@ Some companies disallow the usage of Docker Hub, but you can override `*.image` > **localstack.container.image = localstack/localstack** > Used by LocalStack -Another possibility is to set up a registry mirror in your environment so that all images are pulled from there and not directly from Docker Hub. -For more information, see the [official Docker documentation about "Registry as a pull through cache"](https://docs.docker.com/registry/recipes/mirror/). +> **pulsar.container.image = apachepulsar/pulsar:2.2.0** +> Used by Apache Pulsar -!!!tip - Registry mirror currently only works for Docker images with image name that has no registry specified (for example, for Docker image `mariadb:10.3.6`, it works, for Docker image `quay.io/something/else`, not). +See [Image Name Substitution](./image_name_substitution.md) for other strategies for substituting image names to pull from other registries. ## Customizing Ryuk resource reaper diff --git a/docs/features/image_name_substitution.md b/docs/features/image_name_substitution.md new file mode 100644 index 00000000000..ee24e790bb7 --- /dev/null +++ b/docs/features/image_name_substitution.md @@ -0,0 +1,109 @@ +# Image name substitution + +Testcontainers supports automatic substitution of Docker image names. + +This allows replacement of an image name specified in test code with an alternative name - for example, to replace the +name of a Docker Hub image dependency with an alternative hosted on a private image registry. + +This is advisable to avoid [Docker Hub rate limiting](./pull_rate_limiting.md), and some companies will prefer this for policy reasons. + +This page describes four approaches for image name substitution: + +* [Manual substitution](#manual-substitution) - not relying upon an automated approach +* Using an Image Name Substitutor: + * [Developing a custom function for transforming image names on the fly](#developing-a-custom-function-for-transforming-image-names-on-the-fly) + * [Overriding image names individually in configuration](#overriding-image-names-individually-in-configuration) + +It is assumed that you have already set up a private registry hosting [all the Docker images your build requires](./pull_rate_limiting.md#which-images-are-used-by-testcontainers). + + + + +## Manual substitution + +Consider this if: + +* You use only a few images and updating code is not a chore +* All developers and CI machines in your organisation have access to a common registry server +* You also use one of the automated mechanisms to substitute [the images that Testcontainers itself requires](./pull_rate_limiting.md#which-images-are-used-by-testcontainers) + +This approach simply entails modifying test code manually, e.g. changing: + +For example, you may have a test that uses the `mysql` container image from Docker Hub: + + +[Direct Docker Hub image name](../examples/junit4/generic/src/test/java/generic/ImageNameSubstitutionTest.java) inside_block:directDockerHubReference + + +to: + + +[Private registry image name](../examples/junit4/generic/src/test/java/generic/ImageNameSubstitutionTest.java) inside_block:hardcodedMirror + + + + + + + + + + +## Developing a custom function for transforming image names on the fly + +Consider this if: + +* You have complex rules about which private registry images should be used as substitutes, e.g.: + * non-deterministic mapping of names meaning that a [name prefix](#adding-a-registry-url-prefix-to-image-names-automatically) cannot be used + * rules depending upon developer identity or location +* or you wish to add audit logging of images used in the build +* or you wish to prevent accidental usage of images that are not on an approved list + +In this case, image name references in code are **unchanged**. +i.e. you would leave as-is: + + +[Unchanged direct Docker Hub image name](../examples/junit4/generic/src/test/java/generic/ImageNameSubstitutionTest.java) inside_block:directDockerHubReference + + +You can implement a custom image name substitutor by: + +* subclassing `org.testcontainers.utility.ImageNameSubstitutor` +* making sure that Testcontainers can find your custom implementation by creating a service loader file. **Do not miss this step!** + +The following is an example image substitutor implementation: + + +[Example Image Substitutor](../examples/junit4/generic/src/test/java/generic/ExampleImageNameSubstitutor.java) block:ExampleImageNameSubstitutor + + +Testcontainers can be configured to find it at runtime using the Service Loader mechanism. +To do this, create a file on the classpath at `META-INF/services/org.testcontainers.utility.ImageNameSubstitutor` +containing the full name of your custom image substitutor. + +For example: + +```text tab="src/main/resources/META-INF/services/org.testcontainers.utility.ImageNameSubstitutor" +com.mycompany.testcontainers.ExampleImageNameSubstitutor +``` + + +## Overriding image names individually in configuration + +!!! note + This approach is discouraged and deprecated, but is documented for completeness. + Overriding individual image names via configuration may be removed in 2021. + +Consider this if: + +* You have many references to image names in code and changing them is impractical, and +* None of the other options are practical for you + +In this case, image name references in code are left **unchanged**. +i.e. you would leave as-is: + + +[Unchanged direct Docker Hub image name](../examples/junit4/generic/src/test/java/generic/ImageNameSubstitutionTest.java) inside_block:directDockerHubReference + + +You can force Testcontainers to substitute in a different image [using a configuration file](./configuration.md), which allows some (but not all) container names to be substituted. diff --git a/docs/features/pull_rate_limiting.md b/docs/features/pull_rate_limiting.md new file mode 100644 index 00000000000..be266545afb --- /dev/null +++ b/docs/features/pull_rate_limiting.md @@ -0,0 +1,19 @@ +# Image Registry rate limiting + +As of November 2020 Docker Hub pulls are rate limited. +As Testcontainers uses Docker Hub for standard images, some users may hit these rate limits and should mitigate accordingly. + +Suggested mitigations are noted in [this issue](https://github.com/testcontainers/testcontainers-java/issues/3099) at present. + +## Which images are used by Testcontainers? + +As of the current version of Testcontainers ({{latest_version}}): + +* every image directly used by your tests +* images pulled by Testcontainers itself to support functionality: + * [`testcontainers/ryuk`](https://hub.docker.com/r/testcontainers/ryuk) - performs fail-safe cleanup of containers, and always required (unless [Ryuk is disabled](./configuration.md#disabling-ryuk)) + * [`alpine`](https://hub.docker.com/r/_/alpine) - used to check whether images can be pulled at startup, and always required (unless [startup checks are disabled](./configuration.md#disabling-the-startup-checks)) + * [`testcontainers/sshd`](https://hub.docker.com/r/testcontainers/sshd) - required if [exposing host ports to containers](./networking.md#exposing-host-ports-to-the-container) + * [`testcontainers/vnc-recorder`](https://hub.docker.com/r/testcontainers/vnc-recorder) - required if using [Webdriver containers](../modules/webdriver_containers.md) and using the screen recording feature + * [`docker/compose`](https://hub.docker.com/r/docker/compose) - required if using [Docker Compose](../modules/docker_compose.md) + * [`alpine/socat`](https://hub.docker.com/r/alpine/socat) - required if using [Docker Compose](../modules/docker_compose.md) diff --git a/modules/kafka/src/main/java/org/testcontainers/containers/KafkaContainer.java b/modules/kafka/src/main/java/org/testcontainers/containers/KafkaContainer.java index ae44e4803a9..e195d72f64a 100644 --- a/modules/kafka/src/main/java/org/testcontainers/containers/KafkaContainer.java +++ b/modules/kafka/src/main/java/org/testcontainers/containers/KafkaContainer.java @@ -4,7 +4,6 @@ import lombok.SneakyThrows; import org.testcontainers.images.builder.Transferable; import org.testcontainers.utility.DockerImageName; -import org.testcontainers.utility.TestcontainersConfiguration; import java.nio.charset.StandardCharsets; import java.util.stream.Collectors; @@ -36,7 +35,7 @@ public class KafkaContainer extends GenericContainer { */ @Deprecated public KafkaContainer() { - this(TestcontainersConfiguration.getInstance().getKafkaDockerImageName().withTag(DEFAULT_TAG)); + this(DEFAULT_IMAGE_NAME.withTag(DEFAULT_TAG)); } /** @@ -44,7 +43,7 @@ public KafkaContainer() { */ @Deprecated public KafkaContainer(String confluentPlatformVersion) { - this(TestcontainersConfiguration.getInstance().getKafkaDockerImageName().withTag(confluentPlatformVersion)); + this(DEFAULT_IMAGE_NAME.withTag(confluentPlatformVersion)); } public KafkaContainer(final DockerImageName dockerImageName) { diff --git a/modules/kafka/src/test/java/org/testcontainers/containers/KafkaContainerTest.java b/modules/kafka/src/test/java/org/testcontainers/containers/KafkaContainerTest.java index 08b9fb6f856..cfc2f8178bd 100644 --- a/modules/kafka/src/test/java/org/testcontainers/containers/KafkaContainerTest.java +++ b/modules/kafka/src/test/java/org/testcontainers/containers/KafkaContainerTest.java @@ -61,7 +61,7 @@ public void testUsageWithSpecificImage() throws Exception { @Test public void testUsageWithVersion() throws Exception { try ( - KafkaContainer kafka = new KafkaContainer("5.2.1") + KafkaContainer kafka = new KafkaContainer("5.5.1") ) { kafka.start(); testKafkaFunctionality(kafka.getBootstrapServers()); diff --git a/modules/localstack/src/main/java/org/testcontainers/containers/localstack/LocalStackContainer.java b/modules/localstack/src/main/java/org/testcontainers/containers/localstack/LocalStackContainer.java index 7532ebe0b3d..b8416afb1ed 100644 --- a/modules/localstack/src/main/java/org/testcontainers/containers/localstack/LocalStackContainer.java +++ b/modules/localstack/src/main/java/org/testcontainers/containers/localstack/LocalStackContainer.java @@ -14,7 +14,6 @@ import org.testcontainers.containers.wait.strategy.Wait; import org.testcontainers.utility.ComparableVersion; import org.testcontainers.utility.DockerImageName; -import org.testcontainers.utility.TestcontainersConfiguration; import java.net.InetAddress; import java.net.URI; @@ -65,7 +64,7 @@ public class LocalStackContainer extends GenericContainer { */ @Deprecated public LocalStackContainer() { - this(TestcontainersConfiguration.getInstance().getLocalstackDockerImageName().withTag(DEFAULT_TAG)); + this(DEFAULT_IMAGE_NAME.withTag(DEFAULT_TAG)); } /** @@ -73,7 +72,7 @@ public LocalStackContainer() { */ @Deprecated public LocalStackContainer(String version) { - this(TestcontainersConfiguration.getInstance().getLocalstackDockerImageName().withTag(version)); + this(DEFAULT_IMAGE_NAME.withTag(version)); } /** diff --git a/modules/oracle-xe/src/main/java/org/testcontainers/containers/OracleContainer.java b/modules/oracle-xe/src/main/java/org/testcontainers/containers/OracleContainer.java index b5700c1e84e..0a66ad6fd27 100644 --- a/modules/oracle-xe/src/main/java/org/testcontainers/containers/OracleContainer.java +++ b/modules/oracle-xe/src/main/java/org/testcontainers/containers/OracleContainer.java @@ -22,8 +22,7 @@ public class OracleContainer extends JdbcDatabaseContainer { private String password = "oracle"; private static String resolveImageName() { - String image = TestcontainersConfiguration.getInstance() - .getProperties().getProperty("oracle.container.image"); + String image = TestcontainersConfiguration.getInstance().getOracleImage(); if (image == null) { throw new IllegalStateException("An image to use for Oracle containers must be configured. " + diff --git a/modules/pulsar/src/main/java/org/testcontainers/containers/PulsarContainer.java b/modules/pulsar/src/main/java/org/testcontainers/containers/PulsarContainer.java index bceff8927d7..b3c836b56a5 100644 --- a/modules/pulsar/src/main/java/org/testcontainers/containers/PulsarContainer.java +++ b/modules/pulsar/src/main/java/org/testcontainers/containers/PulsarContainer.java @@ -3,7 +3,6 @@ import org.testcontainers.containers.wait.strategy.Wait; import org.testcontainers.containers.wait.strategy.WaitAllStrategy; import org.testcontainers.utility.DockerImageName; -import org.testcontainers.utility.TestcontainersConfiguration; /** * This container wraps Apache Pulsar running in standalone mode @@ -25,7 +24,7 @@ public class PulsarContainer extends GenericContainer { */ @Deprecated public PulsarContainer() { - this(TestcontainersConfiguration.getInstance().getPulsarDockerImageName().withTag(DEFAULT_TAG)); + this(DEFAULT_IMAGE_NAME.withTag(DEFAULT_TAG)); } /** @@ -33,7 +32,7 @@ public PulsarContainer() { */ @Deprecated public PulsarContainer(String pulsarVersion) { - this(TestcontainersConfiguration.getInstance().getPulsarDockerImageName().withTag(pulsarVersion)); + this(DEFAULT_IMAGE_NAME.withTag(pulsarVersion)); } public PulsarContainer(final DockerImageName dockerImageName) { diff --git a/modules/spock/src/test/groovy/org/testcontainers/spock/SpockTestImages.groovy b/modules/spock/src/test/groovy/org/testcontainers/spock/SpockTestImages.groovy index 526c1fe5c42..9698e7d433a 100644 --- a/modules/spock/src/test/groovy/org/testcontainers/spock/SpockTestImages.groovy +++ b/modules/spock/src/test/groovy/org/testcontainers/spock/SpockTestImages.groovy @@ -1,11 +1,10 @@ -package org.testcontainers.spock; +package org.testcontainers.spock import org.testcontainers.utility.DockerImageName -import org.testcontainers.utility.TestcontainersConfiguration; interface SpockTestImages { DockerImageName MYSQL_IMAGE = DockerImageName.parse("mysql:5.7.22") DockerImageName POSTGRES_TEST_IMAGE = DockerImageName.parse("postgres:9.6.12") DockerImageName HTTPD_IMAGE = DockerImageName.parse("httpd:2.4-alpine") - DockerImageName TINY_IMAGE = TestcontainersConfiguration.getInstance().getTinyDockerImageName() + DockerImageName TINY_IMAGE = DockerImageName.parse("alpine:3.5") } From 58179b38c687a945cbbe6263dcf1a67fc44a8dba Mon Sep 17 00:00:00 2001 From: Richard North Date: Thu, 29 Oct 2020 12:16:57 +0000 Subject: [PATCH 03/16] Add Image substitution mechanism Builds upon #3021 and #3411: * adds a pluggable image substitution mechanism using ServiceLoader, enabling users to perform custom substitution/auditing of images being used by their tests * provides a default implementation that behaves similarly to legacy `TestcontainersConfiguration` approach (`testcontainers.properties`) Notes: * behaviour is similar but not quite identical to `TestcontainersConfiguration`: use of a configured custom image for, e.g. Kafka/Pulsar that does not have a tag specified causes the substitution to take effect for all usages. It seems very unlikely that people would be using a mix of the config file image overrides in some places _and_ specific images specified in code in others. * Duplication of default image names in modules vs `TestcontainersConfiguration` class is intentional: specifying image overrides in `testcontainers.properties` should be removed in the future. * ~Add log deprecation warnings when `testcontainers.properties` image overrides are used.~ Defer to a future release? --- .../utility/DefaultImageNameSubstitutor.java | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/testcontainers/utility/DefaultImageNameSubstitutor.java b/core/src/main/java/org/testcontainers/utility/DefaultImageNameSubstitutor.java index 5657a9051aa..a0d01ff794d 100644 --- a/core/src/main/java/org/testcontainers/utility/DefaultImageNameSubstitutor.java +++ b/core/src/main/java/org/testcontainers/utility/DefaultImageNameSubstitutor.java @@ -5,7 +5,7 @@ /** * Testcontainers' default implementation of {@link ImageNameSubstitutor}. - * Delegates to {@link ConfigurationFileImageNameSubstitutor}. + * Delegates to {@link ConfigurationFileImageNameSubstitutor} followed by {@link PrefixingImageNameSubstitutor}. *

* WARNING: this class is not intended to be public, but {@link java.util.ServiceLoader} * requires it to be so. Public visibility DOES NOT make it part of the public API. @@ -14,21 +14,29 @@ public class DefaultImageNameSubstitutor extends ImageNameSubstitutor { private final ConfigurationFileImageNameSubstitutor configurationFileImageNameSubstitutor; + private final PrefixingImageNameSubstitutor prefixingImageNameSubstitutor; public DefaultImageNameSubstitutor() { configurationFileImageNameSubstitutor = new ConfigurationFileImageNameSubstitutor(); + prefixingImageNameSubstitutor = new PrefixingImageNameSubstitutor(); } @VisibleForTesting DefaultImageNameSubstitutor( - final ConfigurationFileImageNameSubstitutor configurationFileImageNameSubstitutor + final ConfigurationFileImageNameSubstitutor configurationFileImageNameSubstitutor, + final PrefixingImageNameSubstitutor prefixingImageNameSubstitutor ) { this.configurationFileImageNameSubstitutor = configurationFileImageNameSubstitutor; + this.prefixingImageNameSubstitutor = prefixingImageNameSubstitutor; } @Override public DockerImageName apply(final DockerImageName original) { - return configurationFileImageNameSubstitutor.apply(original); + return prefixingImageNameSubstitutor.apply( + configurationFileImageNameSubstitutor.apply( + original + ) + ); } @Override @@ -38,6 +46,6 @@ protected int getPriority() { @Override protected String getDescription() { - return "DefaultImageNameSubstitutor (delegates to '" + configurationFileImageNameSubstitutor.getDescription() + "')"; + return "DefaultImageNameSubstitutor (composite of '" + configurationFileImageNameSubstitutor.getDescription() + "' and '" + prefixingImageNameSubstitutor.getDescription() + "')"; } } From 658c98a4e72ed1af6567f02d467755ea9939a013 Mon Sep 17 00:00:00 2001 From: Richard North Date: Thu, 29 Oct 2020 11:53:17 +0000 Subject: [PATCH 04/16] Improve default image name substitution For many orgs, sticking a prefix on the front of image names might be enough to use a private registry. I've added a default behaviour whereby, if a particular environment variable is present, image names are automatically substituted. e.g. `TESTCONTAINERS_IMAGE_NAME_PREFIX=my.registry.com/` would transform `redis` to `my.registry.com/redis` etc. --- .../PrefixingImageNameSubstitutor.java | 53 +++++++++++++ .../PrefixingImageNameSubstitutorTest.java | 75 +++++++++++++++++++ docs/features/image_name_substitution.md | 28 +++++++ 3 files changed, 156 insertions(+) create mode 100644 core/src/main/java/org/testcontainers/utility/PrefixingImageNameSubstitutor.java create mode 100644 core/src/test/java/org/testcontainers/utility/PrefixingImageNameSubstitutorTest.java diff --git a/core/src/main/java/org/testcontainers/utility/PrefixingImageNameSubstitutor.java b/core/src/main/java/org/testcontainers/utility/PrefixingImageNameSubstitutor.java new file mode 100644 index 00000000000..6253f478a1b --- /dev/null +++ b/core/src/main/java/org/testcontainers/utility/PrefixingImageNameSubstitutor.java @@ -0,0 +1,53 @@ +package org.testcontainers.utility; + +import com.google.common.annotations.VisibleForTesting; +import lombok.NoArgsConstructor; +import org.testcontainers.UnstableAPI; + +/** + * An {@link ImageNameSubstitutor} which applies a prefix to all image names, e.g. a private registry host and path. + * The prefix may be set via an environment variable (TESTCONTAINERS_IMAGE_NAME_PREFIX) or an equivalent + * configuration file entry (see {@link TestcontainersConfiguration}). + *

+ * WARNING: this class is not intended to be public, but {@link java.util.ServiceLoader} + * requires it to be so. Public visibility DOES NOT make it part of the public API. + */ +@UnstableAPI +@NoArgsConstructor +public final class PrefixingImageNameSubstitutor extends ImageNameSubstitutor { + + @VisibleForTesting + static final String PROPERTY_KEY = "testcontainers.image.name.prefix"; + + private TestcontainersConfiguration configuration = TestcontainersConfiguration.getInstance(); + + @VisibleForTesting + PrefixingImageNameSubstitutor(final TestcontainersConfiguration configuration) { + this.configuration = configuration; + } + + @Override + public DockerImageName apply(DockerImageName original) { + final String prefix = configuration.getEnvVarOrProperty(PROPERTY_KEY, ""); + + if (prefix != null && !prefix.isEmpty()) { + if (!original.asCanonicalNameString().startsWith(prefix)) { + return DockerImageName.parse(prefix + original.asCanonicalNameString()); + } else { + return original; + } + } else { + return original; + } + } + + @Override + protected int getPriority() { + return -1; + } + + @Override + protected String getDescription() { + return getClass().getSimpleName(); + } +} diff --git a/core/src/test/java/org/testcontainers/utility/PrefixingImageNameSubstitutorTest.java b/core/src/test/java/org/testcontainers/utility/PrefixingImageNameSubstitutorTest.java new file mode 100644 index 00000000000..3e6675660cf --- /dev/null +++ b/core/src/test/java/org/testcontainers/utility/PrefixingImageNameSubstitutorTest.java @@ -0,0 +1,75 @@ +package org.testcontainers.utility; + +import org.junit.Before; +import org.junit.Test; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.rnorth.visibleassertions.VisibleAssertions.assertEquals; +import static org.testcontainers.utility.PrefixingImageNameSubstitutor.PROPERTY_KEY; + +public class PrefixingImageNameSubstitutorTest { + + private TestcontainersConfiguration mockConfiguration; + private PrefixingImageNameSubstitutor underTest; + + @Before + public void setUp() { + mockConfiguration = mock(TestcontainersConfiguration.class); + underTest = new PrefixingImageNameSubstitutor(mockConfiguration); + } + + @Test + public void testHappyPath() { + when(mockConfiguration.getEnvVarOrProperty(eq(PROPERTY_KEY), any())).thenReturn("someregistry.com/"); + + final DockerImageName result = underTest.apply(DockerImageName.parse("some/image:tag")); + + assertEquals( + "The prefix is applied", + "someregistry.com/some/image:tag", + result.asCanonicalNameString() + ); + } + + @Test + public void testNoDoublePrefixing() { + when(mockConfiguration.getEnvVarOrProperty(eq(PROPERTY_KEY), any())).thenReturn("someregistry.com/"); + + final DockerImageName result = underTest.apply(DockerImageName.parse("someregistry.com/some/image:tag")); + + assertEquals( + "The prefix is not applied if already present", + "someregistry.com/some/image:tag", + result.asCanonicalNameString() + ); + } + + @Test + public void testHandlesNullValue() { + when(mockConfiguration.getEnvVarOrProperty(eq(PROPERTY_KEY), any())).thenReturn(null); + + final DockerImageName result = underTest.apply(DockerImageName.parse("some/image:tag")); + + assertEquals( + "The prefix is not applied if the env var is not set", + "some/image:tag", + result.asCanonicalNameString() + ); + } + + @Test + public void testHandlesEmptyValue() { + when(mockConfiguration.getEnvVarOrProperty(eq(PROPERTY_KEY), any())).thenReturn(""); + + final DockerImageName result = underTest.apply(DockerImageName.parse("some/image:tag")); + + assertEquals( + "The prefix is not applied if the env var is not set", + "some/image:tag", + result.asCanonicalNameString() + ); + } +} diff --git a/docs/features/image_name_substitution.md b/docs/features/image_name_substitution.md index ee24e790bb7..9270660704a 100644 --- a/docs/features/image_name_substitution.md +++ b/docs/features/image_name_substitution.md @@ -11,6 +11,7 @@ This page describes four approaches for image name substitution: * [Manual substitution](#manual-substitution) - not relying upon an automated approach * Using an Image Name Substitutor: + * Recommended: [Adding a registry URL prefix to image names automatically](#adding-a-registry-url-prefix-to-image-names-automatically) * [Developing a custom function for transforming image names on the fly](#developing-a-custom-function-for-transforming-image-names-on-the-fly) * [Overriding image names individually in configuration](#overriding-image-names-individually-in-configuration) @@ -45,6 +46,33 @@ to: +## Adding a registry URL prefix to image names automatically + +Consider this if: + +* Developers and CI machines need to use different image names. For example, developers are able to pull images from Docker Hub, but CI machines need to pull from a private registry +* Your private registry has copies of images from Docker Hub where the names are predictable, and just adding a prefix is enough. + For example, `registry.mycompany.com/mirror/mysql:8.0.22` can be derived from the original Docker Hub image name (`mysql:8.0.22`) with a consistent prefix string: `registry.mycompany.com/mirror/` + +In this case, image name references in code are **unchanged**. +i.e. you would leave as-is: + + +[Unchanged direct Docker Hub image name](../examples/junit4/generic/src/test/java/generic/ImageNameSubstitutionTest.java) inside_block:directDockerHubReference + + +You can then configure Testcontainers to apply the prefix `registry.mycompany.com/mirror/` to every image that it tries to pull. +This can be done in one of two ways: + +* Setting an environment variable, `TESTCONTAINERS_IMAGE_NAME_PREFIX=registry.mycompany.com/mirror/` +* Via config file, setting `testcontainers.image.name.prefix=registry.mycompany.com/mirror/` in either: + * the `~/.testcontainers.properties` file in your user home directory, or + * a file named `testcontainers.properties` on the classpath + +Testcontainers will automatically apply this prefix to every image that it pulls - please verify that all [the required images](./pull_rate_limiting.md#which-images-are-used-by-testcontainers) exist in your registry. + +Note that the prefix-based substitution will skip applying a prefix if it is already set. +This is intended to help avoid obvious mistakes if image names have been partially migrated to a private image registry via changes to code. From 9bf309712a9110cfc3ee778c2a7ffacbd7a726a7 Mon Sep 17 00:00:00 2001 From: Richard North Date: Sun, 8 Nov 2020 10:53:33 +0000 Subject: [PATCH 05/16] Rework - WIP Docs update required --- .../utility/DefaultImageNameSubstitutor.java | 8 +- .../utility/DockerImageName.java | 23 +++--- .../PrefixingImageNameSubstitutor.java | 47 ++++++----- .../utility/TestcontainersConfiguration.java | 80 ------------------- .../PrefixingImageNameSubstitutorTest.java | 63 +++++++++++++-- 5 files changed, 99 insertions(+), 122 deletions(-) diff --git a/core/src/main/java/org/testcontainers/utility/DefaultImageNameSubstitutor.java b/core/src/main/java/org/testcontainers/utility/DefaultImageNameSubstitutor.java index 8df73f966f6..79dfd9de128 100644 --- a/core/src/main/java/org/testcontainers/utility/DefaultImageNameSubstitutor.java +++ b/core/src/main/java/org/testcontainers/utility/DefaultImageNameSubstitutor.java @@ -32,11 +32,9 @@ public DefaultImageNameSubstitutor() { @Override public DockerImageName apply(final DockerImageName original) { - return prefixingImageNameSubstitutor.apply( - configurationFileImageNameSubstitutor.apply( - original - ) - ); + return configurationFileImageNameSubstitutor + .andThen(prefixingImageNameSubstitutor) + .apply(original); } @Override diff --git a/core/src/main/java/org/testcontainers/utility/DockerImageName.java b/core/src/main/java/org/testcontainers/utility/DockerImageName.java index af8e5300863..cab3f09d22d 100644 --- a/core/src/main/java/org/testcontainers/utility/DockerImageName.java +++ b/core/src/main/java/org/testcontainers/utility/DockerImageName.java @@ -5,6 +5,7 @@ import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.EqualsAndHashCode; +import lombok.Getter; import lombok.With; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -24,8 +25,8 @@ public final class DockerImageName { private static final Pattern REPO_NAME = Pattern.compile(REPO_NAME_PART + "(/" + REPO_NAME_PART + ")*"); private final String rawName; - private final String registry; - private final String repo; + @With @Getter private final String registry; + @With @Getter private final String repository; @NotNull @With(AccessLevel.PRIVATE) private final Versioning versioning; @Nullable @With(AccessLevel.PRIVATE) @@ -68,13 +69,13 @@ public DockerImageName(String fullImageName) { } if (remoteName.contains("@sha256:")) { - repo = remoteName.split("@sha256:")[0]; + repository = remoteName.split("@sha256:")[0]; versioning = new Sha256Versioning(remoteName.split("@sha256:")[1]); } else if (remoteName.contains(":")) { - repo = remoteName.split(":")[0]; + repository = remoteName.split(":")[0]; versioning = new TagVersioning(remoteName.split(":")[1]); } else { - repo = remoteName; + repository = remoteName; versioning = Versioning.ANY; } @@ -110,10 +111,10 @@ public DockerImageName(String nameWithoutTag, @NotNull String version) { } if (version.startsWith("sha256:")) { - repo = remoteName; + repository = remoteName; versioning = new Sha256Versioning(version.replace("sha256:", "")); } else { - repo = remoteName; + repository = remoteName; versioning = new TagVersioning(version); } @@ -125,9 +126,9 @@ public DockerImageName(String nameWithoutTag, @NotNull String version) { */ public String getUnversionedPart() { if (!"".equals(registry)) { - return registry + "/" + repo; + return registry + "/" + repository; } else { - return repo; + return repository; } } @@ -158,8 +159,8 @@ public String toString() { public void assertValid() { //noinspection UnstableApiUsage HostAndPort.fromString(registry); // return value ignored - this throws if registry is not a valid host:port string - if (!REPO_NAME.matcher(repo).matches()) { - throw new IllegalArgumentException(repo + " is not a valid Docker image name (in " + rawName + ")"); + if (!REPO_NAME.matcher(repository).matches()) { + throw new IllegalArgumentException(repository + " is not a valid Docker image name (in " + rawName + ")"); } if (!versioning.isValid()) { throw new IllegalArgumentException(versioning + " is not a valid image versioning identifier (in " + rawName + ")"); diff --git a/core/src/main/java/org/testcontainers/utility/PrefixingImageNameSubstitutor.java b/core/src/main/java/org/testcontainers/utility/PrefixingImageNameSubstitutor.java index 6253f478a1b..bc139fadce3 100644 --- a/core/src/main/java/org/testcontainers/utility/PrefixingImageNameSubstitutor.java +++ b/core/src/main/java/org/testcontainers/utility/PrefixingImageNameSubstitutor.java @@ -2,22 +2,22 @@ import com.google.common.annotations.VisibleForTesting; import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.testcontainers.UnstableAPI; /** * An {@link ImageNameSubstitutor} which applies a prefix to all image names, e.g. a private registry host and path. * The prefix may be set via an environment variable (TESTCONTAINERS_IMAGE_NAME_PREFIX) or an equivalent * configuration file entry (see {@link TestcontainersConfiguration}). - *

- * WARNING: this class is not intended to be public, but {@link java.util.ServiceLoader} - * requires it to be so. Public visibility DOES NOT make it part of the public API. */ @UnstableAPI @NoArgsConstructor -public final class PrefixingImageNameSubstitutor extends ImageNameSubstitutor { +@Slf4j +final class PrefixingImageNameSubstitutor extends ImageNameSubstitutor { @VisibleForTesting - static final String PROPERTY_KEY = "testcontainers.image.name.prefix"; + static final String REGISTRY_PROPERTY_KEY = "hub.image.override.registry"; + static final String REPOSITORY_PREFIX_PROPERTY_KEY = "hub.image.override.repository.prefix"; private TestcontainersConfiguration configuration = TestcontainersConfiguration.getInstance(); @@ -28,22 +28,33 @@ public final class PrefixingImageNameSubstitutor extends ImageNameSubstitutor { @Override public DockerImageName apply(DockerImageName original) { - final String prefix = configuration.getEnvVarOrProperty(PROPERTY_KEY, ""); - - if (prefix != null && !prefix.isEmpty()) { - if (!original.asCanonicalNameString().startsWith(prefix)) { - return DockerImageName.parse(prefix + original.asCanonicalNameString()); - } else { - return original; - } - } else { + final String registryOverride = configuration.getEnvVarOrProperty(REGISTRY_PROPERTY_KEY, ""); + final String repositoryPrefixOrEmpty = configuration.getEnvVarOrProperty(REPOSITORY_PREFIX_PROPERTY_KEY, ""); + boolean overrideIsConfigured = !registryOverride.isEmpty(); + + if (!overrideIsConfigured) { + log.debug("No override is configured"); return original; } - } - @Override - protected int getPriority() { - return -1; + boolean isAHubImage = original.getRegistry().isEmpty() || + original.getRegistry().equals("docker.io") || + original.getRegistry().equals("registry.hub.docker.com"); + if (!isAHubImage) { + log.debug("Image {} is not a Docker Hub image - not applying registry/repository change", original); + return original; + } + + log.debug( + "Applying changes to image name {}: Changing registry part to '{}' and applying prefix '{}' to repository name part", + original, + registryOverride, + repositoryPrefixOrEmpty + ); + + return original + .withRegistry(registryOverride) + .withRepository(repositoryPrefixOrEmpty + original.getRepository()); } @Override diff --git a/core/src/main/java/org/testcontainers/utility/TestcontainersConfiguration.java b/core/src/main/java/org/testcontainers/utility/TestcontainersConfiguration.java index 7802a578cfb..98b8379b2cc 100644 --- a/core/src/main/java/org/testcontainers/utility/TestcontainersConfiguration.java +++ b/core/src/main/java/org/testcontainers/utility/TestcontainersConfiguration.java @@ -23,8 +23,6 @@ import java.net.MalformedURLException; import java.net.URL; import java.util.Map; -import java.util.Map; -import java.util.Optional; import java.util.Optional; import java.util.Properties; import java.util.concurrent.atomic.AtomicReference; @@ -214,7 +212,6 @@ private String getConfigurable(@NotNull final String propertyName, @Nullable fin * @param propertyName name of configuration file property (dot-separated lower case) * @return the found value, or null if not set */ - @Nullable @Contract("_, !null -> !null") public String getEnvVarOrProperty(@NotNull final String propertyName, @Nullable final String defaultValue) { return getConfigurable(propertyName, defaultValue, userProperties, classpathProperties); @@ -227,7 +224,6 @@ public String getEnvVarOrProperty(@NotNull final String propertyName, @Nullable * @param propertyName name of configuration file property (dot-separated lower case) * @return the found value, or null if not set */ - @Nullable @Contract("_, !null -> !null") public String getEnvVarOrUserProperty(@NotNull final String propertyName, @Nullable final String defaultValue) { return getConfigurable(propertyName, defaultValue, userProperties); @@ -240,7 +236,6 @@ public String getEnvVarOrUserProperty(@NotNull final String propertyName, @Nulla * @param propertyName name of configuration file property (dot-separated lower case) * @return the found value, or null if not set */ - @Nullable @Contract("_, !null -> !null") public String getUserProperty(@NotNull final String propertyName, @Nullable final String defaultValue) { return getConfigurable(propertyName, defaultValue); @@ -267,81 +262,6 @@ public boolean updateGlobalConfig(@NonNull String prop, @NonNull String value) { return updateUserConfig(prop, value); } - @Nullable - @Contract("_, !null, _ -> !null") - private String getConfigurable(@NotNull final String propertyName, @Nullable final String defaultValue, Properties... propertiesSources) { - String envVarName = propertyName.replaceAll("\\.", "_").toUpperCase(); - if (!envVarName.startsWith("TESTCONTAINERS_")) { - envVarName = "TESTCONTAINERS_" + envVarName; - } - - if (environment.containsKey(envVarName)) { - return environment.get(envVarName); - } - - for (final Properties properties : propertiesSources) { - if (properties.get(propertyName) != null) { - return (String) properties.get(propertyName); - } - } - - return defaultValue; - } - - /** - * Gets a configured setting from an environment variable (if present) or a configuration file property otherwise. - * The configuration file will be the .testcontainers.properties file in the user's home directory or - * a testcontainers.properties found on the classpath. - * - * @param propertyName name of configuration file property (dot-separated lower case) - * @return the found value, or null if not set - */ - @Nullable - @Contract("_, !null -> !null") - public String getEnvVarOrProperty(@NotNull final String propertyName, @Nullable final String defaultValue) { - return getConfigurable(propertyName, defaultValue, userProperties, classpathProperties); - } - - /** - * Gets a configured setting from an environment variable (if present) or a configuration file property otherwise. - * The configuration file will be the .testcontainers.properties file in the user's home directory. - * - * @param propertyName name of configuration file property (dot-separated lower case) - * @return the found value, or null if not set - */ - @Nullable - @Contract("_, !null -> !null") - public String getEnvVarOrUserProperty(@NotNull final String propertyName, @Nullable final String defaultValue) { - return getConfigurable(propertyName, defaultValue, userProperties); - } - - /** - * Gets a configured setting from a the user's configuration file. - * The configuration file will be the .testcontainers.properties file in the user's home directory. - * - * @param propertyName name of configuration file property (dot-separated lower case) - * @return the found value, or null if not set - */ - @Nullable - @Contract("_, !null -> !null") - public String getUserProperty(@NotNull final String propertyName, @Nullable final String defaultValue) { - return getConfigurable(propertyName, defaultValue); - } - - @Deprecated - public Properties getProperties() { - return Stream.of(userProperties, classpathProperties) - .reduce(new Properties(), (a, b) -> { - a.putAll(b); - return a; - }); - } - - @Deprecated - public boolean updateUserConfig(@NonNull String prop, @NonNull String value) { - return updateUserConfig(prop, value); - } - @Synchronized public boolean updateUserConfig(@NonNull String prop, @NonNull String value) { try { diff --git a/core/src/test/java/org/testcontainers/utility/PrefixingImageNameSubstitutorTest.java b/core/src/test/java/org/testcontainers/utility/PrefixingImageNameSubstitutorTest.java index 3e6675660cf..a4cb4acc823 100644 --- a/core/src/test/java/org/testcontainers/utility/PrefixingImageNameSubstitutorTest.java +++ b/core/src/test/java/org/testcontainers/utility/PrefixingImageNameSubstitutorTest.java @@ -8,7 +8,8 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.rnorth.visibleassertions.VisibleAssertions.assertEquals; -import static org.testcontainers.utility.PrefixingImageNameSubstitutor.PROPERTY_KEY; +import static org.testcontainers.utility.PrefixingImageNameSubstitutor.REGISTRY_PROPERTY_KEY; +import static org.testcontainers.utility.PrefixingImageNameSubstitutor.REPOSITORY_PREFIX_PROPERTY_KEY; public class PrefixingImageNameSubstitutorTest { @@ -23,20 +24,64 @@ public void setUp() { @Test public void testHappyPath() { - when(mockConfiguration.getEnvVarOrProperty(eq(PROPERTY_KEY), any())).thenReturn("someregistry.com/"); + when(mockConfiguration.getEnvVarOrProperty(eq(REGISTRY_PROPERTY_KEY), any())).thenReturn("someregistry.com"); + when(mockConfiguration.getEnvVarOrProperty(eq(REPOSITORY_PREFIX_PROPERTY_KEY), any())).thenReturn("our-mirror/"); final DockerImageName result = underTest.apply(DockerImageName.parse("some/image:tag")); assertEquals( "The prefix is applied", - "someregistry.com/some/image:tag", + "someregistry.com/our-mirror/some/image:tag", + result.asCanonicalNameString() + ); + } + + @Test + public void hubIoRegistryIsChanged() { + when(mockConfiguration.getEnvVarOrProperty(eq(REGISTRY_PROPERTY_KEY), any())).thenReturn("someregistry.com"); + when(mockConfiguration.getEnvVarOrProperty(eq(REPOSITORY_PREFIX_PROPERTY_KEY), any())).thenReturn("our-mirror/"); + + final DockerImageName result = underTest.apply(DockerImageName.parse("docker.io/some/image:tag")); + + assertEquals( + "The prefix is applied", + "someregistry.com/our-mirror/some/image:tag", + result.asCanonicalNameString() + ); + } + + @Test + public void hubComRegistryIsChanged() { + when(mockConfiguration.getEnvVarOrProperty(eq(REGISTRY_PROPERTY_KEY), any())).thenReturn("someregistry.com"); + when(mockConfiguration.getEnvVarOrProperty(eq(REPOSITORY_PREFIX_PROPERTY_KEY), any())).thenReturn("our-mirror/"); + + final DockerImageName result = underTest.apply(DockerImageName.parse("registry.hub.docker.com/some/image:tag")); + + assertEquals( + "The prefix is applied", + "someregistry.com/our-mirror/some/image:tag", + result.asCanonicalNameString() + ); + } + + @Test + public void thirdPartyRegistriesNotAffected() { + when(mockConfiguration.getEnvVarOrProperty(eq(REGISTRY_PROPERTY_KEY), any())).thenReturn("someregistry.com"); + when(mockConfiguration.getEnvVarOrProperty(eq(REPOSITORY_PREFIX_PROPERTY_KEY), any())).thenReturn("our-mirror/"); + + final DockerImageName result = underTest.apply(DockerImageName.parse("gcr.io/something/image:tag")); + + assertEquals( + "The prefix is not applied if a third party registry is used", + "gcr.io/something/image:tag", result.asCanonicalNameString() ); } @Test public void testNoDoublePrefixing() { - when(mockConfiguration.getEnvVarOrProperty(eq(PROPERTY_KEY), any())).thenReturn("someregistry.com/"); + when(mockConfiguration.getEnvVarOrProperty(eq(REGISTRY_PROPERTY_KEY), any())).thenReturn("someregistry.com"); + when(mockConfiguration.getEnvVarOrProperty(eq(REPOSITORY_PREFIX_PROPERTY_KEY), any())).thenReturn("our-mirror/"); final DockerImageName result = underTest.apply(DockerImageName.parse("someregistry.com/some/image:tag")); @@ -48,8 +93,9 @@ public void testNoDoublePrefixing() { } @Test - public void testHandlesNullValue() { - when(mockConfiguration.getEnvVarOrProperty(eq(PROPERTY_KEY), any())).thenReturn(null); + public void testHandlesEmptyValue() { + when(mockConfiguration.getEnvVarOrProperty(eq(REGISTRY_PROPERTY_KEY), any())).thenReturn(""); + when(mockConfiguration.getEnvVarOrProperty(eq(REPOSITORY_PREFIX_PROPERTY_KEY), any())).thenReturn(""); final DockerImageName result = underTest.apply(DockerImageName.parse("some/image:tag")); @@ -61,8 +107,9 @@ public void testHandlesNullValue() { } @Test - public void testHandlesEmptyValue() { - when(mockConfiguration.getEnvVarOrProperty(eq(PROPERTY_KEY), any())).thenReturn(""); + public void testOnlyAppliesIfRegistryOverrideSet() { + when(mockConfiguration.getEnvVarOrProperty(eq(REGISTRY_PROPERTY_KEY), any())).thenReturn(""); + when(mockConfiguration.getEnvVarOrProperty(eq(REPOSITORY_PREFIX_PROPERTY_KEY), any())).thenReturn("our-mirror/"); final DockerImageName result = underTest.apply(DockerImageName.parse("some/image:tag")); From 2176ef8e2aa8c0a62a32d505b3f55dfc1d24112a Mon Sep 17 00:00:00 2001 From: Richard North Date: Sun, 8 Nov 2020 11:35:55 +0000 Subject: [PATCH 06/16] Provisional changes - single config setting, applied to Hub image names only --- .../PrefixingImageNameSubstitutor.java | 24 ++++++------- .../PrefixingImageNameSubstitutorTest.java | 35 ++++--------------- docs/features/image_name_substitution.md | 20 +++++------ 3 files changed, 28 insertions(+), 51 deletions(-) diff --git a/core/src/main/java/org/testcontainers/utility/PrefixingImageNameSubstitutor.java b/core/src/main/java/org/testcontainers/utility/PrefixingImageNameSubstitutor.java index bc139fadce3..79992078662 100644 --- a/core/src/main/java/org/testcontainers/utility/PrefixingImageNameSubstitutor.java +++ b/core/src/main/java/org/testcontainers/utility/PrefixingImageNameSubstitutor.java @@ -7,7 +7,7 @@ /** * An {@link ImageNameSubstitutor} which applies a prefix to all image names, e.g. a private registry host and path. - * The prefix may be set via an environment variable (TESTCONTAINERS_IMAGE_NAME_PREFIX) or an equivalent + * The prefix may be set via an environment variable (TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX) or an equivalent * configuration file entry (see {@link TestcontainersConfiguration}). */ @UnstableAPI @@ -16,8 +16,7 @@ final class PrefixingImageNameSubstitutor extends ImageNameSubstitutor { @VisibleForTesting - static final String REGISTRY_PROPERTY_KEY = "hub.image.override.registry"; - static final String REPOSITORY_PREFIX_PROPERTY_KEY = "hub.image.override.repository.prefix"; + static final String PREFIX_PROPERTY_KEY = "hub.image.name.prefix"; private TestcontainersConfiguration configuration = TestcontainersConfiguration.getInstance(); @@ -28,12 +27,10 @@ final class PrefixingImageNameSubstitutor extends ImageNameSubstitutor { @Override public DockerImageName apply(DockerImageName original) { - final String registryOverride = configuration.getEnvVarOrProperty(REGISTRY_PROPERTY_KEY, ""); - final String repositoryPrefixOrEmpty = configuration.getEnvVarOrProperty(REPOSITORY_PREFIX_PROPERTY_KEY, ""); - boolean overrideIsConfigured = !registryOverride.isEmpty(); + final String configuredPrefix = configuration.getEnvVarOrProperty(PREFIX_PROPERTY_KEY, ""); - if (!overrideIsConfigured) { - log.debug("No override is configured"); + if (!!configuredPrefix.isEmpty()) { + log.debug("No prefix is configured"); return original; } @@ -46,15 +43,16 @@ public DockerImageName apply(DockerImageName original) { } log.debug( - "Applying changes to image name {}: Changing registry part to '{}' and applying prefix '{}' to repository name part", + "Applying changes to image name {}: applying prefix '{}'", original, - registryOverride, - repositoryPrefixOrEmpty + configuredPrefix ); + DockerImageName prefixAsImage = DockerImageName.parse(configuredPrefix); + return original - .withRegistry(registryOverride) - .withRepository(repositoryPrefixOrEmpty + original.getRepository()); + .withRegistry(prefixAsImage.getRegistry()) + .withRepository(prefixAsImage.getRepository() + original.getRepository()); } @Override diff --git a/core/src/test/java/org/testcontainers/utility/PrefixingImageNameSubstitutorTest.java b/core/src/test/java/org/testcontainers/utility/PrefixingImageNameSubstitutorTest.java index a4cb4acc823..1f9cfabee66 100644 --- a/core/src/test/java/org/testcontainers/utility/PrefixingImageNameSubstitutorTest.java +++ b/core/src/test/java/org/testcontainers/utility/PrefixingImageNameSubstitutorTest.java @@ -8,8 +8,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.rnorth.visibleassertions.VisibleAssertions.assertEquals; -import static org.testcontainers.utility.PrefixingImageNameSubstitutor.REGISTRY_PROPERTY_KEY; -import static org.testcontainers.utility.PrefixingImageNameSubstitutor.REPOSITORY_PREFIX_PROPERTY_KEY; +import static org.testcontainers.utility.PrefixingImageNameSubstitutor.PREFIX_PROPERTY_KEY; public class PrefixingImageNameSubstitutorTest { @@ -24,8 +23,7 @@ public void setUp() { @Test public void testHappyPath() { - when(mockConfiguration.getEnvVarOrProperty(eq(REGISTRY_PROPERTY_KEY), any())).thenReturn("someregistry.com"); - when(mockConfiguration.getEnvVarOrProperty(eq(REPOSITORY_PREFIX_PROPERTY_KEY), any())).thenReturn("our-mirror/"); + when(mockConfiguration.getEnvVarOrProperty(eq(PREFIX_PROPERTY_KEY), any())).thenReturn("someregistry.com/our-mirror/"); final DockerImageName result = underTest.apply(DockerImageName.parse("some/image:tag")); @@ -38,8 +36,7 @@ public void testHappyPath() { @Test public void hubIoRegistryIsChanged() { - when(mockConfiguration.getEnvVarOrProperty(eq(REGISTRY_PROPERTY_KEY), any())).thenReturn("someregistry.com"); - when(mockConfiguration.getEnvVarOrProperty(eq(REPOSITORY_PREFIX_PROPERTY_KEY), any())).thenReturn("our-mirror/"); + when(mockConfiguration.getEnvVarOrProperty(eq(PREFIX_PROPERTY_KEY), any())).thenReturn("someregistry.com/our-mirror/"); final DockerImageName result = underTest.apply(DockerImageName.parse("docker.io/some/image:tag")); @@ -52,8 +49,7 @@ public void hubIoRegistryIsChanged() { @Test public void hubComRegistryIsChanged() { - when(mockConfiguration.getEnvVarOrProperty(eq(REGISTRY_PROPERTY_KEY), any())).thenReturn("someregistry.com"); - when(mockConfiguration.getEnvVarOrProperty(eq(REPOSITORY_PREFIX_PROPERTY_KEY), any())).thenReturn("our-mirror/"); + when(mockConfiguration.getEnvVarOrProperty(eq(PREFIX_PROPERTY_KEY), any())).thenReturn("someregistry.com/our-mirror/"); final DockerImageName result = underTest.apply(DockerImageName.parse("registry.hub.docker.com/some/image:tag")); @@ -66,8 +62,7 @@ public void hubComRegistryIsChanged() { @Test public void thirdPartyRegistriesNotAffected() { - when(mockConfiguration.getEnvVarOrProperty(eq(REGISTRY_PROPERTY_KEY), any())).thenReturn("someregistry.com"); - when(mockConfiguration.getEnvVarOrProperty(eq(REPOSITORY_PREFIX_PROPERTY_KEY), any())).thenReturn("our-mirror/"); + when(mockConfiguration.getEnvVarOrProperty(eq(PREFIX_PROPERTY_KEY), any())).thenReturn("someregistry.com/our-mirror/"); final DockerImageName result = underTest.apply(DockerImageName.parse("gcr.io/something/image:tag")); @@ -80,8 +75,7 @@ public void thirdPartyRegistriesNotAffected() { @Test public void testNoDoublePrefixing() { - when(mockConfiguration.getEnvVarOrProperty(eq(REGISTRY_PROPERTY_KEY), any())).thenReturn("someregistry.com"); - when(mockConfiguration.getEnvVarOrProperty(eq(REPOSITORY_PREFIX_PROPERTY_KEY), any())).thenReturn("our-mirror/"); + when(mockConfiguration.getEnvVarOrProperty(eq(PREFIX_PROPERTY_KEY), any())).thenReturn("someregistry.com/our-mirror/"); final DockerImageName result = underTest.apply(DockerImageName.parse("someregistry.com/some/image:tag")); @@ -94,22 +88,7 @@ public void testNoDoublePrefixing() { @Test public void testHandlesEmptyValue() { - when(mockConfiguration.getEnvVarOrProperty(eq(REGISTRY_PROPERTY_KEY), any())).thenReturn(""); - when(mockConfiguration.getEnvVarOrProperty(eq(REPOSITORY_PREFIX_PROPERTY_KEY), any())).thenReturn(""); - - final DockerImageName result = underTest.apply(DockerImageName.parse("some/image:tag")); - - assertEquals( - "The prefix is not applied if the env var is not set", - "some/image:tag", - result.asCanonicalNameString() - ); - } - - @Test - public void testOnlyAppliesIfRegistryOverrideSet() { - when(mockConfiguration.getEnvVarOrProperty(eq(REGISTRY_PROPERTY_KEY), any())).thenReturn(""); - when(mockConfiguration.getEnvVarOrProperty(eq(REPOSITORY_PREFIX_PROPERTY_KEY), any())).thenReturn("our-mirror/"); + when(mockConfiguration.getEnvVarOrProperty(eq(PREFIX_PROPERTY_KEY), any())).thenReturn(""); final DockerImageName result = underTest.apply(DockerImageName.parse("some/image:tag")); diff --git a/docs/features/image_name_substitution.md b/docs/features/image_name_substitution.md index 923b58c8b20..d3a49afa748 100644 --- a/docs/features/image_name_substitution.md +++ b/docs/features/image_name_substitution.md @@ -45,7 +45,9 @@ to: -## Adding a registry URL prefix to image names automatically +## Automatically modifying Docker Hub image names + +Testcontainers can be configured to modify Docker Hub image names on the fly to apply a prefix string. Consider this if: @@ -60,19 +62,17 @@ i.e. you would leave as-is: [Unchanged direct Docker Hub image name](../examples/junit4/generic/src/test/java/generic/ImageNameSubstitutionTest.java) inside_block:directDockerHubReference -You can then configure Testcontainers to apply the prefix `registry.mycompany.com/mirror/` to every image that it tries to pull. +You can then configure Testcontainers to apply the prefix `registry.mycompany.com/mirror/` to every image that it tries to pull from Docker Hub. This can be done in one of two ways: -* Setting an environment variable, `TESTCONTAINERS_IMAGE_NAME_PREFIX=registry.mycompany.com/mirror/` -* Via config file, setting `testcontainers.image.name.prefix=registry.mycompany.com/mirror/` in either: +* Setting environment variables `TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX=registry.mycompany.com/mirror/` +* Via config file, setting `hub.image.name.prefix` in either: * the `~/.testcontainers.properties` file in your user home directory, or * a file named `testcontainers.properties` on the classpath + +Testcontainers will automatically apply the prefix to every image that it pulls from Docker Hub - please verify that all [the required images](./pull_rate_limiting.md#which-images-are-used-by-testcontainers) exist in your registry. -Testcontainers will automatically apply this prefix to every image that it pulls - please verify that all [the required images](./pull_rate_limiting.md#which-images-are-used-by-testcontainers) exist in your registry. - -Note that the prefix-based substitution will skip applying a prefix if it is already set. -This is intended to help avoid obvious mistakes if image names have been partially migrated to a private image registry via changes to code. - +Testcontainers will not apply the prefix to non-Hub image names. @@ -81,7 +81,7 @@ This is intended to help avoid obvious mistakes if image names have been partial Consider this if: * You have complex rules about which private registry images should be used as substitutes, e.g.: - * non-deterministic mapping of names meaning that a [name prefix](#adding-a-registry-url-prefix-to-image-names-automatically) cannot be used + * non-deterministic mapping of names meaning that a [name prefix](#automatically-modifying-docker-hub-image-names) cannot be used * rules depending upon developer identity or location * or you wish to add audit logging of images used in the build * or you wish to prevent accidental usage of images that are not on an approved list From 8565f7f325b1b8be114771804cd4d59c6038abd2 Mon Sep 17 00:00:00 2001 From: Richard North Date: Sun, 8 Nov 2020 11:38:03 +0000 Subject: [PATCH 07/16] Tidy up merge issue --- .../testcontainers/utility/DefaultImageNameSubstitutor.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/core/src/main/java/org/testcontainers/utility/DefaultImageNameSubstitutor.java b/core/src/main/java/org/testcontainers/utility/DefaultImageNameSubstitutor.java index 79dfd9de128..65492c4d53f 100644 --- a/core/src/main/java/org/testcontainers/utility/DefaultImageNameSubstitutor.java +++ b/core/src/main/java/org/testcontainers/utility/DefaultImageNameSubstitutor.java @@ -6,9 +6,6 @@ /** * Testcontainers' default implementation of {@link ImageNameSubstitutor}. * Delegates to {@link ConfigurationFileImageNameSubstitutor} followed by {@link PrefixingImageNameSubstitutor}. - *

- * WARNING: this class is not intended to be public, but {@link java.util.ServiceLoader} - * requires it to be so. Public visibility DOES NOT make it part of the public API. */ @Slf4j public class DefaultImageNameSubstitutor extends ImageNameSubstitutor { From 45ac5f4b4dfb6fe8aa99483d1e0352bbeb3d7f95 Mon Sep 17 00:00:00 2001 From: Richard North Date: Sun, 8 Nov 2020 11:38:23 +0000 Subject: [PATCH 08/16] Tidy up merge issue --- .../org/testcontainers/utility/DefaultImageNameSubstitutor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/org/testcontainers/utility/DefaultImageNameSubstitutor.java b/core/src/main/java/org/testcontainers/utility/DefaultImageNameSubstitutor.java index 65492c4d53f..840b8a77b40 100644 --- a/core/src/main/java/org/testcontainers/utility/DefaultImageNameSubstitutor.java +++ b/core/src/main/java/org/testcontainers/utility/DefaultImageNameSubstitutor.java @@ -8,7 +8,7 @@ * Delegates to {@link ConfigurationFileImageNameSubstitutor} followed by {@link PrefixingImageNameSubstitutor}. */ @Slf4j -public class DefaultImageNameSubstitutor extends ImageNameSubstitutor { +final class DefaultImageNameSubstitutor extends ImageNameSubstitutor { private final ConfigurationFileImageNameSubstitutor configurationFileImageNameSubstitutor; private final PrefixingImageNameSubstitutor prefixingImageNameSubstitutor; From 73d22916d9bbb20dbf9eff8f163772ae3787158a Mon Sep 17 00:00:00 2001 From: Richard North Date: Sun, 8 Nov 2020 11:41:51 +0000 Subject: [PATCH 09/16] Tidy up merge issue --- docs/features/image_name_substitution.md | 2 +- docs/features/pull_rate_limiting.md | 19 ------------------- 2 files changed, 1 insertion(+), 20 deletions(-) delete mode 100644 docs/features/pull_rate_limiting.md diff --git a/docs/features/image_name_substitution.md b/docs/features/image_name_substitution.md index d3a49afa748..2c7f4f0d733 100644 --- a/docs/features/image_name_substitution.md +++ b/docs/features/image_name_substitution.md @@ -70,7 +70,7 @@ This can be done in one of two ways: * the `~/.testcontainers.properties` file in your user home directory, or * a file named `testcontainers.properties` on the classpath -Testcontainers will automatically apply the prefix to every image that it pulls from Docker Hub - please verify that all [the required images](./pull_rate_limiting.md#which-images-are-used-by-testcontainers) exist in your registry. +Testcontainers will automatically apply the prefix to every image that it pulls from Docker Hub - please verify that all [the required images](../supported_docker_environment/image_registry_rate_limiting.md#which-images-are-used-by-testcontainers) exist in your registry. Testcontainers will not apply the prefix to non-Hub image names. diff --git a/docs/features/pull_rate_limiting.md b/docs/features/pull_rate_limiting.md deleted file mode 100644 index be266545afb..00000000000 --- a/docs/features/pull_rate_limiting.md +++ /dev/null @@ -1,19 +0,0 @@ -# Image Registry rate limiting - -As of November 2020 Docker Hub pulls are rate limited. -As Testcontainers uses Docker Hub for standard images, some users may hit these rate limits and should mitigate accordingly. - -Suggested mitigations are noted in [this issue](https://github.com/testcontainers/testcontainers-java/issues/3099) at present. - -## Which images are used by Testcontainers? - -As of the current version of Testcontainers ({{latest_version}}): - -* every image directly used by your tests -* images pulled by Testcontainers itself to support functionality: - * [`testcontainers/ryuk`](https://hub.docker.com/r/testcontainers/ryuk) - performs fail-safe cleanup of containers, and always required (unless [Ryuk is disabled](./configuration.md#disabling-ryuk)) - * [`alpine`](https://hub.docker.com/r/_/alpine) - used to check whether images can be pulled at startup, and always required (unless [startup checks are disabled](./configuration.md#disabling-the-startup-checks)) - * [`testcontainers/sshd`](https://hub.docker.com/r/testcontainers/sshd) - required if [exposing host ports to containers](./networking.md#exposing-host-ports-to-the-container) - * [`testcontainers/vnc-recorder`](https://hub.docker.com/r/testcontainers/vnc-recorder) - required if using [Webdriver containers](../modules/webdriver_containers.md) and using the screen recording feature - * [`docker/compose`](https://hub.docker.com/r/docker/compose) - required if using [Docker Compose](../modules/docker_compose.md) - * [`alpine/socat`](https://hub.docker.com/r/alpine/socat) - required if using [Docker Compose](../modules/docker_compose.md) From c7c0a0933d74e548df5f02f487a43a44cb1395ca Mon Sep 17 00:00:00 2001 From: Richard North Date: Sat, 21 Nov 2020 16:06:30 +0000 Subject: [PATCH 10/16] Remove excess para --- docs/features/configuration.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/features/configuration.md b/docs/features/configuration.md index db97ab2a227..b567a8f564f 100644 --- a/docs/features/configuration.md +++ b/docs/features/configuration.md @@ -72,8 +72,6 @@ Some companies disallow the usage of Docker Hub, but you can override `*.image` > **pulsar.container.image = apachepulsar/pulsar:2.2.0** > Used by Apache Pulsar -See [Image Name Substitution](./image_name_substitution.md) for other strategies for substituting image names to pull from other registries. - ## Customizing Ryuk resource reaper > **ryuk.container.image = testcontainers/ryuk:0.3.0** From 0d706f305e37e459ffaf16048ab7b9ce3e6a378a Mon Sep 17 00:00:00 2001 From: Richard North Date: Tue, 1 Dec 2020 07:31:05 +0000 Subject: [PATCH 11/16] Update core/src/main/java/org/testcontainers/utility/PrefixingImageNameSubstitutor.java Co-authored-by: Sergei Egorov --- .../testcontainers/utility/PrefixingImageNameSubstitutor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/org/testcontainers/utility/PrefixingImageNameSubstitutor.java b/core/src/main/java/org/testcontainers/utility/PrefixingImageNameSubstitutor.java index 79992078662..b0aac13d6c5 100644 --- a/core/src/main/java/org/testcontainers/utility/PrefixingImageNameSubstitutor.java +++ b/core/src/main/java/org/testcontainers/utility/PrefixingImageNameSubstitutor.java @@ -29,7 +29,7 @@ final class PrefixingImageNameSubstitutor extends ImageNameSubstitutor { public DockerImageName apply(DockerImageName original) { final String configuredPrefix = configuration.getEnvVarOrProperty(PREFIX_PROPERTY_KEY, ""); - if (!!configuredPrefix.isEmpty()) { + if (configuredPrefix.isEmpty()) { log.debug("No prefix is configured"); return original; } From 6370a63392bb1dddc64b33c587fd515c195fc3fc Mon Sep 17 00:00:00 2001 From: Richard North Date: Tue, 1 Dec 2020 20:25:56 +0000 Subject: [PATCH 12/16] Simplify Hub image detection --- .../utility/PrefixingImageNameSubstitutor.java | 6 +----- docs/features/image_name_substitution.md | 4 +++- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/org/testcontainers/utility/PrefixingImageNameSubstitutor.java b/core/src/main/java/org/testcontainers/utility/PrefixingImageNameSubstitutor.java index b0aac13d6c5..c720416deac 100644 --- a/core/src/main/java/org/testcontainers/utility/PrefixingImageNameSubstitutor.java +++ b/core/src/main/java/org/testcontainers/utility/PrefixingImageNameSubstitutor.java @@ -3,14 +3,12 @@ import com.google.common.annotations.VisibleForTesting; import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.testcontainers.UnstableAPI; /** * An {@link ImageNameSubstitutor} which applies a prefix to all image names, e.g. a private registry host and path. * The prefix may be set via an environment variable (TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX) or an equivalent * configuration file entry (see {@link TestcontainersConfiguration}). */ -@UnstableAPI @NoArgsConstructor @Slf4j final class PrefixingImageNameSubstitutor extends ImageNameSubstitutor { @@ -34,9 +32,7 @@ public DockerImageName apply(DockerImageName original) { return original; } - boolean isAHubImage = original.getRegistry().isEmpty() || - original.getRegistry().equals("docker.io") || - original.getRegistry().equals("registry.hub.docker.com"); + boolean isAHubImage = original.getRegistry().isEmpty(); if (!isAHubImage) { log.debug("Image {} is not a Docker Hub image - not applying registry/repository change", original); return original; diff --git a/docs/features/image_name_substitution.md b/docs/features/image_name_substitution.md index 2c7f4f0d733..46cf99b0ef9 100644 --- a/docs/features/image_name_substitution.md +++ b/docs/features/image_name_substitution.md @@ -72,7 +72,9 @@ This can be done in one of two ways: Testcontainers will automatically apply the prefix to every image that it pulls from Docker Hub - please verify that all [the required images](../supported_docker_environment/image_registry_rate_limiting.md#which-images-are-used-by-testcontainers) exist in your registry. -Testcontainers will not apply the prefix to non-Hub image names. +Testcontainers will not apply the prefix to: +* non-Hub image names (e.g. where another registry is set) +* Docker Hub image names where the hub registry is explicitly part of the name (i.e. anything with a `docker.io` or `registry.hub.docker.com` host part) From 714f482663db66208c156c0912eed3cdfa69214e Mon Sep 17 00:00:00 2001 From: Richard North Date: Tue, 1 Dec 2020 20:40:47 +0000 Subject: [PATCH 13/16] Update test to reflect changed hub identification logic --- .../utility/PrefixingImageNameSubstitutorTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/src/test/java/org/testcontainers/utility/PrefixingImageNameSubstitutorTest.java b/core/src/test/java/org/testcontainers/utility/PrefixingImageNameSubstitutorTest.java index 1f9cfabee66..c5bde6fdcdf 100644 --- a/core/src/test/java/org/testcontainers/utility/PrefixingImageNameSubstitutorTest.java +++ b/core/src/test/java/org/testcontainers/utility/PrefixingImageNameSubstitutorTest.java @@ -35,27 +35,27 @@ public void testHappyPath() { } @Test - public void hubIoRegistryIsChanged() { + public void hubIoRegistryIsNotChanged() { when(mockConfiguration.getEnvVarOrProperty(eq(PREFIX_PROPERTY_KEY), any())).thenReturn("someregistry.com/our-mirror/"); final DockerImageName result = underTest.apply(DockerImageName.parse("docker.io/some/image:tag")); assertEquals( "The prefix is applied", - "someregistry.com/our-mirror/some/image:tag", + "docker.io/some/image:tag", result.asCanonicalNameString() ); } @Test - public void hubComRegistryIsChanged() { + public void hubComRegistryIsNotChanged() { when(mockConfiguration.getEnvVarOrProperty(eq(PREFIX_PROPERTY_KEY), any())).thenReturn("someregistry.com/our-mirror/"); final DockerImageName result = underTest.apply(DockerImageName.parse("registry.hub.docker.com/some/image:tag")); assertEquals( "The prefix is applied", - "someregistry.com/our-mirror/some/image:tag", + "registry.hub.docker.com/some/image:tag", result.asCanonicalNameString() ); } From abdcd930383421bab2a400d42625c04cf1a35499 Mon Sep 17 00:00:00 2001 From: Richard North Date: Tue, 1 Dec 2020 20:45:37 +0000 Subject: [PATCH 14/16] Add tests that the prefix is applied literally (both for slash-present and -absent cases, the prefix is applied as-is) --- .../PrefixingImageNameSubstitutorTest.java | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/core/src/test/java/org/testcontainers/utility/PrefixingImageNameSubstitutorTest.java b/core/src/test/java/org/testcontainers/utility/PrefixingImageNameSubstitutorTest.java index c5bde6fdcdf..d2c33989deb 100644 --- a/core/src/test/java/org/testcontainers/utility/PrefixingImageNameSubstitutorTest.java +++ b/core/src/test/java/org/testcontainers/utility/PrefixingImageNameSubstitutorTest.java @@ -98,4 +98,43 @@ public void testHandlesEmptyValue() { result.asCanonicalNameString() ); } + + @Test + public void testHandlesRegistryOnlyWithTrailingSlash() { + when(mockConfiguration.getEnvVarOrProperty(eq(PREFIX_PROPERTY_KEY), any())).thenReturn("someregistry.com/"); + + final DockerImageName result = underTest.apply(DockerImageName.parse("some/image:tag")); + + assertEquals( + "The prefix is applied", + "someregistry.com/some/image:tag", + result.asCanonicalNameString() + ); + } + + @Test + public void testCombinesLiterallyForRegistryOnlyWithoutTrailingSlash() { + when(mockConfiguration.getEnvVarOrProperty(eq(PREFIX_PROPERTY_KEY), any())).thenReturn("someregistry.com"); + + final DockerImageName result = underTest.apply(DockerImageName.parse("some/image:tag")); + + assertEquals( + "The prefix is applied", + "someregistry.comsome/image:tag", // treating the prefix literally, for predictability + result.asCanonicalNameString() + ); + } + + @Test + public void testCombinesLiterallyForBothPartsWithoutTrailingSlash() { + when(mockConfiguration.getEnvVarOrProperty(eq(PREFIX_PROPERTY_KEY), any())).thenReturn("someregistry.com/our-mirror"); + + final DockerImageName result = underTest.apply(DockerImageName.parse("some/image:tag")); + + assertEquals( + "The prefix is applied", + "someregistry.com/our-mirrorsome/image:tag", // treating the prefix literally, for predictability + result.asCanonicalNameString() + ); + } } From cd63ec6081cf60ee811ad5653de1b85cd5e67285 Mon Sep 17 00:00:00 2001 From: Richard North Date: Tue, 1 Dec 2020 21:18:58 +0000 Subject: [PATCH 15/16] Remove duplicate getter (Lombok @Getter exists) --- .../main/java/org/testcontainers/utility/DockerImageName.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/core/src/main/java/org/testcontainers/utility/DockerImageName.java b/core/src/main/java/org/testcontainers/utility/DockerImageName.java index cab3f09d22d..b9d266a41ca 100644 --- a/core/src/main/java/org/testcontainers/utility/DockerImageName.java +++ b/core/src/main/java/org/testcontainers/utility/DockerImageName.java @@ -167,10 +167,6 @@ public void assertValid() { } } - public String getRegistry() { - return registry; - } - /** * @param newTag version tag for the copy to use * @return an immutable copy of this {@link DockerImageName} with the new version tag From 9984aa2eeca2b25228395b4b1d2d06530841b604 Mon Sep 17 00:00:00 2001 From: Richard North Date: Wed, 9 Dec 2020 22:46:10 +0000 Subject: [PATCH 16/16] Update docs/features/image_name_substitution.md Co-authored-by: Sergei Egorov --- docs/features/image_name_substitution.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/features/image_name_substitution.md b/docs/features/image_name_substitution.md index 46cf99b0ef9..37088473d72 100644 --- a/docs/features/image_name_substitution.md +++ b/docs/features/image_name_substitution.md @@ -73,6 +73,7 @@ This can be done in one of two ways: Testcontainers will automatically apply the prefix to every image that it pulls from Docker Hub - please verify that all [the required images](../supported_docker_environment/image_registry_rate_limiting.md#which-images-are-used-by-testcontainers) exist in your registry. Testcontainers will not apply the prefix to: + * non-Hub image names (e.g. where another registry is set) * Docker Hub image names where the hub registry is explicitly part of the name (i.e. anything with a `docker.io` or `registry.hub.docker.com` host part)