Add a way to configure an OpenSearch distribution for the Elasticsear…

…ch DevService

bom/application/pom.xml

@@ -207,6 +207,8 @@
         <apicurio-common-rest-client.version>0.1.17.Final</apicurio-common-rest-client.version> <!-- must be the version Apicurio Registry uses -->
         <testcontainers.version>1.18.3</testcontainers.version> <!-- Make sure to also update docker-java.version to match its needs -->
         <docker-java.version>3.3.0</docker-java.version> <!-- must be the version Testcontainers use -->
+        <!-- Check the compatibility matrix (https://github.com/opensearch-project/opensearch-testcontainers) before upgrading: -->
+        <opensearch-testcontainers.version>2.0.0</opensearch-testcontainers.version>
         <com.dajudge.kindcontainer>1.3.0</com.dajudge.kindcontainer>
         <aesh.version>2.7</aesh.version>
         <aesh-readline.version>2.4</aesh-readline.version>
@@ -383,6 +385,11 @@
                     </exclusion>
                 </exclusions>
             </dependency>
+            <dependency>
+                <groupId>org.opensearch</groupId>
+                <artifactId>opensearch-testcontainers</artifactId>
+                <version>${opensearch-testcontainers.version}</version>
+            </dependency>
 
             <!-- OpenTelemetry components, imported as a BOM -->
             <dependency>

docs/src/main/asciidoc/_attributes.adoc

@@ -13,6 +13,7 @@
 :gradle-version: ${gradle-wrapper.version}
 :elasticsearch-version: ${elasticsearch-server.version}
 :elasticsearch-image: ${elasticsearch.image}
+:opensearch-image: ${opensearch.image}
 :infinispan-version: ${infinispan.version}
 :infinispan-protostream-version: ${infinispan.protostream.version}
 :logstash-image: ${logstash.image}

docs/src/main/asciidoc/elasticsearch-dev-services.adoc

@@ -4,8 +4,9 @@ and pull requests should be submitted there:
 https://github.com/quarkusio/quarkus/tree/main/docs/src/main/asciidoc
 ////
 = Dev Services for Elasticsearch
-
 include::_attributes.adoc[]
+:categories: data
+:summary: Start Elasticsearch automatically in dev and test modes
 
 If any Elasticsearch-related extension is present (e.g. `quarkus-elasticsearch-rest-client` or `quarkus-hibernate-search-orm-elasticsearch`),
 Dev Services for Elasticsearch automatically starts an Elasticsearch server in dev mode and when running tests.
@@ -48,14 +49,30 @@ Note that the Elasticsearch hosts property is automatically configured with the
 
 == Configuring the image
 
-Dev Services for Elasticsearch only support Elasticsearch based images, OpenSearch is not supported at the moment.
+Dev Services for Elasticsearch support distributions based on both Elasticsearch and OpenSearch images.
+
+When using Hibernate Search, Dev Services will default to Elasticsearch or OpenSearch based on Hibernate Search configuration.
+
+Otherwise, Dev Services will default to Elasticsearch. To use OpenSearch, configure the distribution explicitly:
+[source,properties,subs="attributes"]
+----
+quarkus.elasticsearch.devservices.distribution=opensearch
+----
 
-If you need to use a different image than the default one you can configure it via:
-[source, properties]
+If you need to use a different Elasticsearch or OpenSearch image than the default one you can configure it via:
+[source,properties,subs="attributes"]
 ----
 quarkus.elasticsearch.devservices.image-name={elasticsearch-image}
 ----
 
+For exotic image names, Quarkus might be unable to infer the distribution (`elastic` or `opensearch`).
+In these cases starting the Dev Services will fail, and you will need to configure the distribution explicitly:
+[source,properties,subs="attributes"]
+----
+quarkus.elasticsearch.devservices.image-name=my-custom-image-with-no-clue-about-the-distribution:1.0
+quarkus.elasticsearch.devservices.distribution=elasticsearch
+----
+
 == Current limitations
 
 Currently, only the default backend for Hibernate Search Elasticsearch is supported, because Dev Services for Elasticsearch can only start one Elasticsearch container.

extensions/devservices/common/src/main/java/io/quarkus/devservices/common/ConfigureUtil.java

@@ -3,7 +3,8 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.UncheckedIOException;
-import java.util.Collections;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Properties;
@@ -43,7 +44,8 @@ public static String configureSharedNetwork(GenericContainer<?> container, Strin
         }
 
         String hostName = (hostNamePrefix + "-" + Base58.randomString(5)).toLowerCase(Locale.ROOT);
-        container.setNetworkAliases(Collections.singletonList(hostName));
+        // some containers might try to add their own aliases on start, so we want to keep this list modifiable:
+        container.setNetworkAliases(new ArrayList<>(List.of(hostName)));
 
         return hostName;
     }

extensions/elasticsearch-rest-client-common/deployment/pom.xml

@@ -39,6 +39,10 @@
                 </exclusion>
             </exclusions>
         </dependency>
+        <dependency>
+            <groupId>org.opensearch</groupId>
+            <artifactId>opensearch-testcontainers</artifactId>
+        </dependency>
     </dependencies>
 
     <build>

extensions/elasticsearch-rest-client-common/deployment/src/main/java/io/quarkus/elasticsearch/restclient/common/deployment/DevServicesElasticsearchProcessor.java

@@ -5,12 +5,15 @@
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import java.util.function.Supplier;
 
 import org.jboss.logging.Logger;
+import org.opensearch.testcontainers.OpensearchContainer;
+import org.testcontainers.containers.GenericContainer;
 import org.testcontainers.elasticsearch.ElasticsearchContainer;
 import org.testcontainers.utility.DockerImageName;
 
@@ -31,6 +34,7 @@
 import io.quarkus.devservices.common.ConfigureUtil;
 import io.quarkus.devservices.common.ContainerAddress;
 import io.quarkus.devservices.common.ContainerLocator;
+import io.quarkus.elasticsearch.restclient.common.deployment.ElasticsearchDevServicesBuildTimeConfig.Distribution;
 import io.quarkus.runtime.configuration.ConfigUtils;
 
 /**
@@ -49,6 +53,9 @@ public class DevServicesElasticsearchProcessor {
 
     private static final ContainerLocator elasticsearchContainerLocator = new ContainerLocator(DEV_SERVICE_LABEL,
             ELASTICSEARCH_PORT);
+    private static final Distribution DEFAULT_DISTRIBUTION = Distribution.ELASTIC;
+    private static final String DEV_SERVICE_ELASTICSEARCH = "elasticsearch";
+    private static final String DEV_SERVICE_OPENSEARCH = "opensearch";
 
     static volatile DevServicesResultBuildItem.RunningDevService devService;
     static volatile ElasticsearchDevServicesBuildTimeConfig cfg;
@@ -171,35 +178,44 @@ private DevServicesResultBuildItem.RunningDevService startElasticsearch(
             return null;
         }
 
-        // We only support ELASTIC container for now
-        if (buildItemConfig.distribution == DevservicesElasticsearchBuildItem.Distribution.OPENSEARCH) {
-            throw new BuildException("Dev Services for Elasticsearch doesn't support OpenSearch", Collections.emptyList());
-        }
-
+        Distribution resolvedDistribution = resolveDistribution(config, buildItemConfig);
+        DockerImageName resolvedImageName = resolveImageName(config, resolvedDistribution);
         // Hibernate Search Elasticsearch have a version configuration property, we need to check that it is coherent
         // with the image we are about to launch
         if (buildItemConfig.version != null) {
-            String containerTag = config.imageName.substring(config.imageName.indexOf(':') + 1);
+            String containerTag = resolvedImageName.getVersionPart();
             if (!containerTag.startsWith(buildItemConfig.version)) {
                 throw new BuildException(
-                        "Dev Services for Elasticsearch detected a version mismatch, container image is " + config.imageName
-                                + " but the configured version is " + buildItemConfig.version +
-                                ". Either configure a different image or disable Dev Services for Elasticsearch.",
+                        "Dev Services for Elasticsearch detected a version mismatch."
+                                + " Consuming extensions are configured to use version " + config.imageName
+                                + " but Dev Services are configured to use version " + buildItemConfig.version +
+                                ". Either configure the same version or disable Dev Services for Elasticsearch.",
                         Collections.emptyList());
             }
         }
 
+        if (buildItemConfig.distribution != null
+                && !buildItemConfig.distribution.equals(resolvedDistribution)) {
+            throw new BuildException(
+                    "Dev Services for Elasticsearch detected a distribution mismatch."
+                            + " Consuming extensions are configured to use distribution " + config.distribution
+                            + " but Dev Services are configured to use distribution " + buildItemConfig.distribution +
+                            ". Either configure the same distribution or disable Dev Services for Elasticsearch.",
+                    Collections.emptyList());
+        }
+
         final Optional<ContainerAddress> maybeContainerAddress = elasticsearchContainerLocator.locateContainer(
                 config.serviceName,
                 config.shared,
                 launchMode.getLaunchMode());
 
         // Starting the server
         final Supplier<DevServicesResultBuildItem.RunningDevService> defaultElasticsearchSupplier = () -> {
-            ElasticsearchContainer container = new ElasticsearchContainer(
-                    DockerImageName.parse(config.imageName)
-                            .asCompatibleSubstituteFor("docker.elastic.co/elasticsearch/elasticsearch"));
-            ConfigureUtil.configureSharedNetwork(container, "elasticsearch");
+
+            GenericContainer<?> container = resolvedDistribution.equals(Distribution.ELASTIC)
+                    ? createElasticsearchContainer(config, resolvedImageName)
+                    : createOpensearchContainer(config, resolvedImageName);
+
             if (config.serviceName != null) {
                 container.withLabel(DEV_SERVICE_LABEL, config.serviceName);
             }
@@ -208,22 +224,14 @@ private DevServicesResultBuildItem.RunningDevService startElasticsearch(
             }
             timeout.ifPresent(container::withStartupTimeout);
 
-            container.addEnv("ES_JAVA_OPTS", config.javaOpts);
-            // Disable security as else we would need to configure it correctly to avoid tons of WARNING in the log
-            container.addEnv("xpack.security.enabled", "false");
-            // Disable disk-based shard allocation thresholds:
-            // in a single-node setup they just don't make sense,
-            // and lead to problems on large disks with little space left.
-            // See https://www.elastic.co/guide/en/elasticsearch/reference/8.8/modules-cluster.html#disk-based-shard-allocation
-            container.addEnv("cluster.routing.allocation.disk.threshold_enabled", "false");
-
             container.withEnv(config.containerEnv);
 
             container.start();
             return new DevServicesResultBuildItem.RunningDevService(Feature.ELASTICSEARCH_REST_CLIENT_COMMON.getName(),
                     container.getContainerId(),
                     container::close,
-                    buildPropertiesMap(buildItemConfig, container.getHttpHostAddress()));
+                    buildPropertiesMap(buildItemConfig,
+                            container.getHost() + ":" + container.getMappedPort(ELASTICSEARCH_PORT)));
         };
 
         return maybeContainerAddress
@@ -235,6 +243,77 @@ private DevServicesResultBuildItem.RunningDevService startElasticsearch(
                 .orElseGet(defaultElasticsearchSupplier);
     }
 
+    private GenericContainer<?> createElasticsearchContainer(ElasticsearchDevServicesBuildTimeConfig config,
+            DockerImageName resolvedImageName) {
+        ElasticsearchContainer container = new ElasticsearchContainer(
+                resolvedImageName.asCompatibleSubstituteFor("docker.elastic.co/elasticsearch/elasticsearch"));
+        ConfigureUtil.configureSharedNetwork(container, DEV_SERVICE_ELASTICSEARCH);
+
+        // Disable security as else we would need to configure it correctly to avoid tons of WARNING in the log
+        container.addEnv("xpack.security.enabled", "false");
+        // Disable disk-based shard allocation thresholds:
+        // in a single-node setup they just don't make sense,
+        // and lead to problems on large disks with little space left.
+        // See https://www.elastic.co/guide/en/elasticsearch/reference/8.8/modules-cluster.html#disk-based-shard-allocation
+        container.addEnv("cluster.routing.allocation.disk.threshold_enabled", "false");
+        container.addEnv("ES_JAVA_OPTS", config.javaOpts);
+        return container;
+    }
+
+    private GenericContainer<?> createOpensearchContainer(ElasticsearchDevServicesBuildTimeConfig config,
+            DockerImageName resolvedImageName) {
+        OpensearchContainer container = new OpensearchContainer(
+                resolvedImageName.asCompatibleSubstituteFor("opensearchproject/opensearch"));
+        ConfigureUtil.configureSharedNetwork(container, DEV_SERVICE_OPENSEARCH);
+
+        container.addEnv("bootstrap.memory_lock", "true");
+        container.addEnv("plugins.index_state_management.enabled", "false");
+
+        container.addEnv("OPENSEARCH_JAVA_OPTS", config.javaOpts);
+        return container;
+    }
+
+    private DockerImageName resolveImageName(ElasticsearchDevServicesBuildTimeConfig config,
+            Distribution resolvedDistribution) {
+        return DockerImageName.parse(config.imageName.orElseGet(() -> ConfigureUtil.getDefaultImageNameFor(
+                Distribution.ELASTIC.equals(resolvedDistribution)
+                        ? DEV_SERVICE_ELASTICSEARCH
+                        : DEV_SERVICE_OPENSEARCH)));
+    }
+
+    private Distribution resolveDistribution(ElasticsearchDevServicesBuildTimeConfig config,
+            DevservicesElasticsearchBuildItemsConfiguration buildItemConfig) throws BuildException {
+        // First, let's see if it was explicitly configured:
+        if (config.distribution.isPresent()) {
+            return config.distribution.get();
+        }
+        // Now let's see if we can guess it from the image:
+        if (config.imageName.isPresent()) {
+            String imageNameRepository = DockerImageName.parse(config.imageName.get()).getRepository()
+                    .toLowerCase(Locale.ROOT);
+            if (imageNameRepository.contains(DEV_SERVICE_OPENSEARCH)) {
+                return Distribution.OPENSEARCH;
+            }
+            if (imageNameRepository.contains(DEV_SERVICE_ELASTICSEARCH)) {
+                return Distribution.ELASTIC;
+            }
+            // no luck guessing so let's ask user to be more explicit:
+            throw new BuildException(
+                    "Wasn't able to determine the distribution of the search service based on the provided image name ["
+                            + config.imageName.get()
+                            + "]. Please specify the distribution explicitly.",
+                    Collections.emptyList());
+        }
+        // Otherwise, let's see if the build item has a value available:
+        if (buildItemConfig.distribution != null) {
+            return buildItemConfig.distribution;
+        }
+        // If we didn't get an explicit distribution
+        // and no image name was provided
+        // then elastic is a default distribution:
+        return DEFAULT_DISTRIBUTION;
+    }
+
     private Map<String, String> buildPropertiesMap(DevservicesElasticsearchBuildItemsConfiguration buildItemConfig,
             String httpHosts) {
         Map<String, String> propertiesToSet = new HashMap<>();
@@ -251,7 +330,7 @@ private String displayProperties(Set<String> hostsConfigProperties) {
     private static class DevservicesElasticsearchBuildItemsConfiguration {
         private Set<String> hostsConfigProperties;
         private String version;
-        private DevservicesElasticsearchBuildItem.Distribution distribution;
+        private Distribution distribution;
 
         private DevservicesElasticsearchBuildItemsConfiguration(List<DevservicesElasticsearchBuildItem> buildItems)
                 throws BuildException {

extensions/elasticsearch-rest-client-common/deployment/src/main/java/io/quarkus/elasticsearch/restclient/common/deployment/DevservicesElasticsearchBuildItem.java

@@ -1,5 +1,7 @@
 package io.quarkus.elasticsearch.restclient.common.deployment;
 
+import static io.quarkus.elasticsearch.restclient.common.deployment.ElasticsearchDevServicesBuildTimeConfig.Distribution;
+
 import io.quarkus.builder.item.MultiBuildItem;
 
 public final class DevservicesElasticsearchBuildItem extends MultiBuildItem {
@@ -11,7 +13,7 @@ public final class DevservicesElasticsearchBuildItem extends MultiBuildItem {
     public DevservicesElasticsearchBuildItem(String hostsConfigProperty) {
         this.hostsConfigProperty = hostsConfigProperty;
         this.version = null;
-        this.distribution = Distribution.ELASTIC;
+        this.distribution = null;
     }
 
     public DevservicesElasticsearchBuildItem(String configItemName, String version, Distribution distribution) {
@@ -32,8 +34,4 @@ public Distribution getDistribution() {
         return distribution;
     }
 
-    public enum Distribution {
-        ELASTIC,
-        OPENSEARCH
-    }
 }

extensions/elasticsearch-rest-client-common/deployment/src/main/java/io/quarkus/elasticsearch/restclient/common/deployment/ElasticsearchDevServicesBuildTimeConfig.java

@@ -27,12 +27,27 @@ public class ElasticsearchDevServicesBuildTimeConfig {
     @ConfigItem
     public Optional<Integer> port;
 
+    /**
+     * Defaults to a distribution inferred from the explicitly configured `image-name` (if any),
+     * or by default to the distribution configured in depending extensions (e.g. Hibernate Search),
+     * or by default to `elastic`.
+     *
+     * @asciidoclet
+     */
+    @ConfigItem
+    public Optional<Distribution> distribution;
+
     /**
      * The Elasticsearch container image to use.
-     * Defaults to the elasticsearch image provided by Elastic.
+     * Defaults depend on the configured `distribution`:
+     *
+     * * For the `elastic` distribution: {elasticsearch-image}
+     * * For the `opensearch` distribution: {opensearch-image}
+     *
+     * @asciidoclet
      */
-    @ConfigItem(defaultValue = "docker.elastic.co/elasticsearch/elasticsearch:8.8.2")
-    public String imageName;
+    @ConfigItem
+    public Optional<String> imageName;
 
     /**
      * The value for the ES_JAVA_OPTS env variable.
@@ -84,6 +99,7 @@ public boolean equals(Object o) {
         return Objects.equals(shared, that.shared)
                 && Objects.equals(enabled, that.enabled)
                 && Objects.equals(port, that.port)
+                && Objects.equals(distribution, that.distribution)
                 && Objects.equals(imageName, that.imageName)
                 && Objects.equals(javaOpts, that.javaOpts)
                 && Objects.equals(serviceName, that.serviceName)
@@ -92,6 +108,11 @@ public boolean equals(Object o) {
 
     @Override
     public int hashCode() {
-        return Objects.hash(enabled, port, imageName, javaOpts, shared, serviceName, containerEnv);
+        return Objects.hash(enabled, port, distribution, imageName, javaOpts, shared, serviceName, containerEnv);
+    }
+
+    public enum Distribution {
+        ELASTIC,
+        OPENSEARCH
     }
 }

extensions/elasticsearch-rest-client-common/deployment/src/main/resources/elasticsearch-devservice.properties

@@ -0,0 +1 @@
+default.image=${elasticsearch.image}

extensions/elasticsearch-rest-client-common/deployment/src/main/resources/opensearch-devservice.properties

@@ -0,0 +1 @@
+default.image=${opensearch.image}