Add a way to configure an OpenSearch distribution for the Elasticsear…

…ch DevService

bom/application/pom.xml

@@ -206,6 +206,8 @@
         <apicurio-common-rest-client.version>0.1.17.Final</apicurio-common-rest-client.version> <!-- must be the version Apicurio Registry uses -->
         <testcontainers.version>1.18.3</testcontainers.version> <!-- Make sure to also update docker-java.version to match its needs -->
         <docker-java.version>3.3.0</docker-java.version> <!-- must be the version Testcontainers use -->
+        <!-- Check the compatibility matrix (https://github.com/opensearch-project/opensearch-testcontainers) before upgrading: -->
+        <opensearch-testcontainers.version>2.0.0</opensearch-testcontainers.version>
         <com.dajudge.kindcontainer>1.3.0</com.dajudge.kindcontainer>
         <aesh.version>2.7</aesh.version>
         <aesh-readline.version>2.4</aesh-readline.version>
@@ -382,6 +384,11 @@
                     </exclusion>
                 </exclusions>
             </dependency>
+            <dependency>
+                <groupId>org.opensearch</groupId>
+                <artifactId>opensearch-testcontainers</artifactId>
+                <version>${opensearch-testcontainers.version}</version>
+            </dependency>
 
             <!-- OpenTelemetry components, imported as a BOM -->
             <dependency>

docs/src/main/asciidoc/_attributes.adoc

@@ -13,6 +13,7 @@
 :gradle-version: ${gradle-wrapper.version}
 :elasticsearch-version: ${elasticsearch-server.version}
 :elasticsearch-image: ${elasticsearch.image}
+:opensearch-image: ${opensearch.image}
 :infinispan-version: ${infinispan.version}
 :infinispan-protostream-version: ${infinispan.protostream.version}
 :logstash-image: ${logstash.image}

docs/src/main/asciidoc/elasticsearch-dev-services.adoc

@@ -4,8 +4,9 @@ and pull requests should be submitted there:
 https://github.com/quarkusio/quarkus/tree/main/docs/src/main/asciidoc
 ////
 = Dev Services for Elasticsearch
-
 include::_attributes.adoc[]
+:categories: data
+:summary: Start Elasticsearch automatically in dev and test modes
 
 If any Elasticsearch-related extension is present (e.g. `quarkus-elasticsearch-rest-client` or `quarkus-hibernate-search-orm-elasticsearch`),
 Dev Services for Elasticsearch automatically starts an Elasticsearch server in dev mode and when running tests.
@@ -48,14 +49,20 @@ Note that the Elasticsearch hosts property is automatically configured with the
 
 == Configuring the image
 
-Dev Services for Elasticsearch only support Elasticsearch based images, OpenSearch is not supported at the moment.
+Dev Services for Elasticsearch support distributions based on both Elasticsearch and OpenSearch images.
 
-If you need to use a different image than the default one you can configure it via:
-[source, properties]
+If you need to use a different Elasticsearch-based image than the default one you can configure it via:
+[source,properties,subs="attributes"]
 ----
 quarkus.elasticsearch.devservices.image-name={elasticsearch-image}
 ----
 
+Alternatively, if an OpenSearch-based image should be used, the same property can be used:
+[source,properties,subs="attributes"]
+----
+quarkus.elasticsearch.devservices.image-name={opensearch-image}
+----
+
 == Current limitations
 
 Currently, only the default backend for Hibernate Search Elasticsearch is supported, because Dev Services for Elasticsearch can only start one Elasticsearch container.

extensions/devservices/common/src/main/java/io/quarkus/devservices/common/ConfigureUtil.java

@@ -3,7 +3,8 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.UncheckedIOException;
-import java.util.Collections;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Properties;
@@ -43,7 +44,8 @@ public static String configureSharedNetwork(GenericContainer<?> container, Strin
         }
 
         String hostName = (hostNamePrefix + "-" + Base58.randomString(5)).toLowerCase(Locale.ROOT);
-        container.setNetworkAliases(Collections.singletonList(hostName));
+        // some containers might try to add their own aliases on start, so we want to keep this list modifiable:
+        container.setNetworkAliases(new ArrayList<>(List.of(hostName)));
 
         return hostName;
     }

extensions/elasticsearch-rest-client-common/deployment/pom.xml

@@ -39,6 +39,10 @@
                 </exclusion>
             </exclusions>
         </dependency>
+        <dependency>
+            <groupId>org.opensearch</groupId>
+            <artifactId>opensearch-testcontainers</artifactId>
+        </dependency>
     </dependencies>
 
     <build>

extensions/elasticsearch-rest-client-common/deployment/src/main/java/io/quarkus/elasticsearch/restclient/common/deployment/DevServicesElasticsearchProcessor.java

@@ -11,6 +11,8 @@
 import java.util.function.Supplier;
 
 import org.jboss.logging.Logger;
+import org.opensearch.testcontainers.OpensearchContainer;
+import org.testcontainers.containers.GenericContainer;
 import org.testcontainers.elasticsearch.ElasticsearchContainer;
 import org.testcontainers.utility.DockerImageName;
 
@@ -31,6 +33,7 @@
 import io.quarkus.devservices.common.ConfigureUtil;
 import io.quarkus.devservices.common.ContainerAddress;
 import io.quarkus.devservices.common.ContainerLocator;
+import io.quarkus.elasticsearch.restclient.common.deployment.ElasticsearchDevServicesBuildTimeConfig.Distribution;
 import io.quarkus.runtime.configuration.ConfigUtils;
 
 /**
@@ -49,6 +52,9 @@ public class DevServicesElasticsearchProcessor {
 
     private static final ContainerLocator elasticsearchContainerLocator = new ContainerLocator(DEV_SERVICE_LABEL,
             ELASTICSEARCH_PORT);
+    private static final String DEFAULT_ELASTICSEARCH_IMAGE = "docker.elastic.co/elasticsearch/elasticsearch:8.8.2";
+    private static final String DEFAULT_OPEN_SEARCH_IMAGE = "docker.io/opensearchproject/opensearch:2.9.0";
+    private static final Distribution DEFAULT_DISTRIBUTION = Distribution.ELASTIC;
 
     static volatile DevServicesResultBuildItem.RunningDevService devService;
     static volatile ElasticsearchDevServicesBuildTimeConfig cfg;
@@ -171,15 +177,12 @@ private DevServicesResultBuildItem.RunningDevService startElasticsearch(
             return null;
         }
 
-        // We only support ELASTIC container for now
-        if (buildItemConfig.distribution == DevservicesElasticsearchBuildItem.Distribution.OPENSEARCH) {
-            throw new BuildException("Dev Services for Elasticsearch doesn't support OpenSearch", Collections.emptyList());
-        }
-
+        Distribution resolvedDistribution = resolveDistribution(config, buildItemConfig);
         // Hibernate Search Elasticsearch have a version configuration property, we need to check that it is coherent
         // with the image we are about to launch
         if (buildItemConfig.version != null) {
-            String containerTag = config.imageName.substring(config.imageName.indexOf(':') + 1);
+            String imageName = resolveImageName(config, resolvedDistribution);
+            String containerTag = DockerImageName.parse(imageName).getVersionPart();
             if (!containerTag.startsWith(buildItemConfig.version)) {
                 throw new BuildException(
                         "Dev Services for Elasticsearch detected a version mismatch, container image is " + config.imageName
@@ -189,17 +192,27 @@ private DevServicesResultBuildItem.RunningDevService startElasticsearch(
             }
         }
 
+        if (buildItemConfig.distribution != null
+                && !config.distribution.orElse(buildItemConfig.distribution).equals(buildItemConfig.distribution)) {
+            throw new BuildException(
+                    "Dev Services for Elasticsearch detected a distribution mismatch, distribution is " + config.distribution
+                            + " but the configured distribution is " + buildItemConfig.distribution +
+                            ". Either configure the same distribution or disable Dev Services for Elasticsearch.",
+                    Collections.emptyList());
+        }
+
         final Optional<ContainerAddress> maybeContainerAddress = elasticsearchContainerLocator.locateContainer(
                 config.serviceName,
                 config.shared,
                 launchMode.getLaunchMode());
 
         // Starting the server
         final Supplier<DevServicesResultBuildItem.RunningDevService> defaultElasticsearchSupplier = () -> {
-            ElasticsearchContainer container = new ElasticsearchContainer(
-                    DockerImageName.parse(config.imageName)
-                            .asCompatibleSubstituteFor("docker.elastic.co/elasticsearch/elasticsearch"));
-            ConfigureUtil.configureSharedNetwork(container, "elasticsearch");
+
+            GenericContainer<?> container = resolvedDistribution.equals(Distribution.ELASTIC)
+                    ? createElasticsearchContainer(config)
+                    : createOpensearchContainer(config);
+
             if (config.serviceName != null) {
                 container.withLabel(DEV_SERVICE_LABEL, config.serviceName);
             }
@@ -208,22 +221,13 @@ private DevServicesResultBuildItem.RunningDevService startElasticsearch(
             }
             timeout.ifPresent(container::withStartupTimeout);
 
-            container.addEnv("ES_JAVA_OPTS", config.javaOpts);
-            // Disable security as else we would need to configure it correctly to avoid tons of WARNING in the log
-            container.addEnv("xpack.security.enabled", "false");
-            // Disable disk-based shard allocation thresholds:
-            // in a single-node setup they just don't make sense,
-            // and lead to problems on large disks with little space left.
-            // See https://www.elastic.co/guide/en/elasticsearch/reference/8.8/modules-cluster.html#disk-based-shard-allocation
-            container.addEnv("cluster.routing.allocation.disk.threshold_enabled", "false");
-
             container.withEnv(config.containerEnv);
 
             container.start();
             return new DevServicesResultBuildItem.RunningDevService(Feature.ELASTICSEARCH_REST_CLIENT_COMMON.getName(),
                     container.getContainerId(),
                     container::close,
-                    buildPropertiesMap(buildItemConfig, container.getHttpHostAddress()));
+                    buildPropertiesMap(buildItemConfig, container.getHost() + ":" + container.getMappedPort(9200)));
         };
 
         return maybeContainerAddress
@@ -235,6 +239,75 @@ private DevServicesResultBuildItem.RunningDevService startElasticsearch(
                 .orElseGet(defaultElasticsearchSupplier);
     }
 
+    private GenericContainer<?> createElasticsearchContainer(ElasticsearchDevServicesBuildTimeConfig config) {
+        ElasticsearchContainer container = new ElasticsearchContainer(
+                DockerImageName.parse(config.imageName.orElse(DEFAULT_ELASTICSEARCH_IMAGE))
+                        .asCompatibleSubstituteFor("docker.elastic.co/elasticsearch/elasticsearch"));
+        ConfigureUtil.configureSharedNetwork(container, "elasticsearch");
+
+        // Disable security as else we would need to configure it correctly to avoid tons of WARNING in the log
+        container.addEnv("xpack.security.enabled", "false");
+        // Disable disk-based shard allocation thresholds:
+        // in a single-node setup they just don't make sense,
+        // and lead to problems on large disks with little space left.
+        // See https://www.elastic.co/guide/en/elasticsearch/reference/8.8/modules-cluster.html#disk-based-shard-allocation
+        container.addEnv("cluster.routing.allocation.disk.threshold_enabled", "false");
+        container.addEnv("ES_JAVA_OPTS", config.javaOpts);
+        return container;
+    }
+
+    private GenericContainer<?> createOpensearchContainer(ElasticsearchDevServicesBuildTimeConfig config) {
+        OpensearchContainer container = new OpensearchContainer(
+                DockerImageName.parse(config.imageName.orElse(DevServicesElasticsearchProcessor.DEFAULT_OPEN_SEARCH_IMAGE))
+                        .asCompatibleSubstituteFor("opensearchproject/opensearch"));
+        container.addEnv("OPENSEARCH_JAVA_OPTS", config.javaOpts);
+        container.addEnv("bootstrap.memory_lock", "true");
+        container.addEnv("plugins.index_state_management.enabled", "false");
+        ConfigureUtil.configureSharedNetwork(container, "opensearch");
+        return container;
+    }
+
+    private String resolveImageName(ElasticsearchDevServicesBuildTimeConfig config,
+            Distribution resolvedDistribution) {
+        return config.imageName.orElseGet(
+                () -> Distribution.ELASTIC.equals(resolvedDistribution)
+                        ? DEFAULT_ELASTICSEARCH_IMAGE
+                        : DEFAULT_OPEN_SEARCH_IMAGE);
+    }
+
+    private Distribution resolveDistribution(ElasticsearchDevServicesBuildTimeConfig config,
+            DevservicesElasticsearchBuildItemsConfiguration buildItemConfig) throws BuildException {
+        // If the build item has a distribution -- great:
+        if (buildItemConfig.distribution != null) {
+            return buildItemConfig.distribution;
+        }
+        // Otherwise, let's see if it was explicitly configured:
+        if (config.distribution.isPresent()) {
+            return config.distribution.get();
+        }
+        // Now let's see if we can guess it from the image:
+        if (config.imageName.isPresent()) {
+            String imageNameRepository = DockerImageName.parse(config.imageName.get()).getRepository();
+            if ("opensearchproject/opensearch".equalsIgnoreCase(imageNameRepository)) {
+                return Distribution.OPENSEARCH;
+            }
+            if ("elasticsearch/elasticsearch".equalsIgnoreCase(imageNameRepository)
+                    || "elastic/elasticsearch".equalsIgnoreCase(imageNameRepository)) {
+                return Distribution.ELASTIC;
+            }
+            // no luck guessing so let's ask user to be more explicit:
+            throw new BuildException(
+                    "Wasn't able to determine the distribution of the search service based on the provided image name ["
+                            + config.imageName.get()
+                            + "]. Please specify the distribution explicitly.",
+                    Collections.emptyList());
+        }
+        // If we didn't get an explicit distribution
+        // and no image name was provided
+        // then elastic is a default distribution:
+        return DEFAULT_DISTRIBUTION;
+    }
+
     private Map<String, String> buildPropertiesMap(DevservicesElasticsearchBuildItemsConfiguration buildItemConfig,
             String httpHosts) {
         Map<String, String> propertiesToSet = new HashMap<>();
@@ -251,7 +324,7 @@ private String displayProperties(Set<String> hostsConfigProperties) {
     private static class DevservicesElasticsearchBuildItemsConfiguration {
         private Set<String> hostsConfigProperties;
         private String version;
-        private DevservicesElasticsearchBuildItem.Distribution distribution;
+        private Distribution distribution;
 
         private DevservicesElasticsearchBuildItemsConfiguration(List<DevservicesElasticsearchBuildItem> buildItems)
                 throws BuildException {

extensions/elasticsearch-rest-client-common/deployment/src/main/java/io/quarkus/elasticsearch/restclient/common/deployment/DevservicesElasticsearchBuildItem.java

@@ -1,5 +1,7 @@
 package io.quarkus.elasticsearch.restclient.common.deployment;
 
+import static io.quarkus.elasticsearch.restclient.common.deployment.ElasticsearchDevServicesBuildTimeConfig.Distribution;
+
 import io.quarkus.builder.item.MultiBuildItem;
 
 public final class DevservicesElasticsearchBuildItem extends MultiBuildItem {
@@ -11,7 +13,7 @@ public final class DevservicesElasticsearchBuildItem extends MultiBuildItem {
     public DevservicesElasticsearchBuildItem(String hostsConfigProperty) {
         this.hostsConfigProperty = hostsConfigProperty;
         this.version = null;
-        this.distribution = Distribution.ELASTIC;
+        this.distribution = null;
     }
 
     public DevservicesElasticsearchBuildItem(String configItemName, String version, Distribution distribution) {
@@ -32,8 +34,4 @@ public Distribution getDistribution() {
         return distribution;
     }
 
-    public enum Distribution {
-        ELASTIC,
-        OPENSEARCH
-    }
 }

extensions/elasticsearch-rest-client-common/deployment/src/main/java/io/quarkus/elasticsearch/restclient/common/deployment/ElasticsearchDevServicesBuildTimeConfig.java

@@ -27,12 +27,19 @@ public class ElasticsearchDevServicesBuildTimeConfig {
     @ConfigItem
     public Optional<Integer> port;
 
+    /**
+     * The distribution of search services to use.
+     * Defaults to the Elasticsearch distribution.
+     */
+    @ConfigItem
+    public Optional<Distribution> distribution;
+
     /**
      * The Elasticsearch container image to use.
      * Defaults to the elasticsearch image provided by Elastic.
      */
-    @ConfigItem(defaultValue = "docker.elastic.co/elasticsearch/elasticsearch:8.8.2")
-    public String imageName;
+    @ConfigItem
+    public Optional<String> imageName;
 
     /**
      * The value for the ES_JAVA_OPTS env variable.
@@ -84,6 +91,7 @@ public boolean equals(Object o) {
         return Objects.equals(shared, that.shared)
                 && Objects.equals(enabled, that.enabled)
                 && Objects.equals(port, that.port)
+                && Objects.equals(distribution, that.distribution)
                 && Objects.equals(imageName, that.imageName)
                 && Objects.equals(javaOpts, that.javaOpts)
                 && Objects.equals(serviceName, that.serviceName)
@@ -92,6 +100,11 @@ public boolean equals(Object o) {
 
     @Override
     public int hashCode() {
-        return Objects.hash(enabled, port, imageName, javaOpts, shared, serviceName, containerEnv);
+        return Objects.hash(enabled, port, distribution, imageName, javaOpts, shared, serviceName, containerEnv);
+    }
+
+    public enum Distribution {
+        ELASTIC,
+        OPENSEARCH
     }
 }

extensions/hibernate-search-orm-elasticsearch/deployment/src/main/java/io/quarkus/hibernate/search/orm/elasticsearch/deployment/HibernateSearchElasticsearchProcessor.java

@@ -1,5 +1,6 @@
 package io.quarkus.hibernate.search.orm.elasticsearch.deployment;
 
+import static io.quarkus.elasticsearch.restclient.common.deployment.ElasticsearchDevServicesBuildTimeConfig.Distribution;
 import static io.quarkus.hibernate.search.orm.elasticsearch.deployment.ClassNames.INDEXED;
 import static io.quarkus.hibernate.search.orm.elasticsearch.deployment.ClassNames.PROJECTION_CONSTRUCTOR;
 import static io.quarkus.hibernate.search.orm.elasticsearch.deployment.ClassNames.ROOT_MAPPING;
@@ -402,7 +403,7 @@ DevservicesElasticsearchBuildItem devServices(HibernateSearchElasticsearchBuildT
                 "hosts");
         return new DevservicesElasticsearchBuildItem(hostsPropertyKey,
                 version.versionString(),
-                DevservicesElasticsearchBuildItem.Distribution.valueOf(version.distribution().toString().toUpperCase()));
+                Distribution.valueOf(version.distribution().toString().toUpperCase()));
     }
 
     @BuildStep(onlyIfNot = IsNormal.class)

integration-tests/hibernate-search-orm-elasticsearch/README.md

@@ -7,12 +7,12 @@ By default, the tests of this module are disabled.
 To run the tests in a standard JVM with Elasticsearch started in the JVM, you can run the following command:
 
 ```
-mvn clean install -Dtest-containers
+mvn clean install -Dtest-containers -Dstart-containers
 ```
 
 Additionally, you can generate a native image and run the tests for this native image by adding `-Dnative`:
 
 ```
-mvn clean install -Dtest-containers -Dnative
+mvn clean install -Dtest-containers -Dstart-containers -Dnative
 ```
 

integration-tests/hibernate-search-orm-opensearch/README.md

@@ -7,12 +7,12 @@ By default, the tests of this module are disabled.
 To run the tests in a standard JVM with OpenSearch started in the JVM, you can run the following command:
 
 ```
-mvn clean install -Dtest-containers
+mvn clean install -Dtest-containers -Dstart-containers
 ```
 
 Additionally, you can generate a native image and run the tests for this native image by adding `-Dnative`:
 
 ```
-mvn clean install -Dtest-containers -Dnative
+mvn clean install -Dtest-containers -Dstart-containers -Dnative
 ```