feat: add Elasticsearch module (#1574)

* chore: initialise elasticsearch module

Command: "go run . new module --image elasticsearch:8.0.0 --name elasticsearch --title Elasticsearch"

* chore: extract minimal image to constant

* chore: define ES contants

* feat: implement HTTP/HTTPS features

* chore: suppot configuring the HTTP client from container options

* chore: store container address in the settings

* chore: support setting elasticsearch user to elastic

* docs: document module

* fix: do not log props file to avoid example errors

* chore: run tests for 6.8.23

* fix: lint

* fix: proper output in example

.github/dependabot.yml

@@ -181,6 +181,17 @@ updates:
         all:
             patterns:
                 - '*'
+    - package-ecosystem: gomod
+      directory: /modules/elasticsearch
+      schedule:
+        interval: monthly
+        day: sunday
+      open-pull-requests-limit: 3
+      rebase-strategy: disabled
+      groups:
+        all:
+            patterns:
+                - '*'
     - package-ecosystem: gomod
       directory: /modules/k3s
       schedule:

.github/workflows/ci.yml

@@ -96,7 +96,7 @@ jobs:
       matrix:
         go-version: [1.20.x, 1.x]
         platform: [ubuntu-latest, macos-latest]
-        module: [artemis, clickhouse, compose, couchbase, k3s, localstack, mariadb, mongodb, mysql, nats, neo4j, postgres, pulsar, redis, redpanda, vault]
+        module: [artemis, clickhouse, compose, couchbase, elasticsearch, k3s, localstack, mariadb, mongodb, mysql, nats, neo4j, postgres, pulsar, redis, redpanda, vault]
     uses: ./.github/workflows/ci-test-go.yml
     with:
       go-version: ${{ matrix.go-version }}

.vscode/.testcontainers-go.code-workspace

@@ -57,6 +57,10 @@
             "name": "module / couchbase",
             "path": "../modules/couchbase"
         },
+        {
+            "name": "module / elasticsearch",
+            "path": "../modules/elasticsearch"
+        },
         {
             "name": "module / k3s",
             "path": "../modules/k3s"

docs/modules/elasticsearch.md

@@ -0,0 +1,101 @@
+# Elasticsearch
+
+Not available until the next release of testcontainers-go <a href="https://github.com/testcontainers/testcontainers-go"><span class="tc-version">:material-tag: main</span></a>
+
+## Introduction
+
+The Testcontainers module for Elasticsearch.
+
+## Adding this module to your project dependencies
+
+Please run the following command to add the Elasticsearch module to your Go dependencies:
+
+```
+go get github.com/testcontainers/testcontainers-go/modules/elasticsearch
+```
+
+## Usage example
+
+<!--codeinclude-->
+[Creating a Elasticsearch container](../../modules/elasticsearch/examples_test.go) inside_block:runElasticsearchContainer
+<!--/codeinclude-->
+
+## Module reference
+
+The Elasticsearch module exposes one entrypoint function to create the Elasticsearch container, and this function receives two parameters:
+
+```golang
+func RunContainer(ctx context.Context, opts ...testcontainers.ContainerCustomizer) (*ElasticsearchContainer, error)
+```
+
+- `context.Context`, the Go context.
+- `testcontainers.ContainerCustomizer`, a variadic argument for passing options.
+
+### Container Options
+
+When starting the Elasticsearch container, you can pass options in a variadic way to configure it.
+
+#### Image
+
+If you need to set a different Elasticsearch Docker image, you can use `testcontainers.WithImage` with a valid Docker image
+for Elasticsearch. E.g. `testcontainers.WithImage("docker.elastic.co/elasticsearch/elasticsearch:8.0.0")`.
+
+#### Wait Strategies
+
+If you need to set a different wait strategy for Elasticsearch, you can use `testcontainers.WithWaitStrategy` with a valid wait strategy
+for Elasticsearch.
+
+!!!info
+    The default deadline for the wait strategy is 60 seconds.
+
+At the same time, it's possible to set a wait strategy and a custom deadline with `testcontainers.WithWaitStrategyAndDeadline`.
+
+#### Docker type modifiers
+
+If you need an advanced configuration for Elasticsearch, you can leverage the following Docker type modifiers:
+
+- `testcontainers.WithConfigModifier`
+- `testcontainers.WithHostConfigModifier`
+- `testcontainers.WithEndpointSettingsModifier`
+
+Please read the [Create containers: Advanced Settings](../features/creating_container.md#advanced-settings) documentation for more information.
+
+#### Elasticsearch password
+
+If you need to set a different password to request authorization when performing HTTP requests to the container, you can use the `WithPassword` option.  By default, the username is set to `elastic`, and the password is set to `changeme`.
+
+!!!info
+    In versions of Elasticsearch prior to 8.0.0, the default password is empty.
+
+<!--codeinclude-->
+[Custom Password](../../modules/elasticsearch/examples_test.go) inside_block:usingPassword
+<!--/codeinclude-->
+
+### Configuring the access to the Elasticsearch container
+
+The Elasticsearch container exposes its settings in order to configure the client to connect to it. With those settings it's very easy to setup up our preferred way to connect to the container. We are going to show you two ways to connect to the container, using the HTTP client from the standard library, and using the Elasticsearch client.
+
+!!!info
+    The `TLS` access is only supported on Elasticsearch 8 and above, so please pay attention to how the below examples are using the `CACert` and `URL` settings.
+
+#### Using the standard library's HTTP client
+
+<!--codeinclude-->
+[Create an HTTP client](../../modules/elasticsearch/elasticsearch_test.go) inside_block:createHTTPClient
+<!--/codeinclude-->
+
+The `esContainer` instance is obtained from the `elasticsearch.RunContainer` function.
+
+In the case you configured the Elasticsearch container to set up a password, you'll need to add the `Authorization` header to the request. You can use the `SetBasicAuth` method from the HTTP request to generate the header value.
+
+<!--codeinclude-->
+[Using an authenticated client](../../modules/elasticsearch/elasticsearch_test.go) inside_block:basicAuthHeader
+<!--/codeinclude-->
+
+#### Using the Elasticsearch client
+
+First, you must install the Elasticsearch Go client, so please read their [install guide](https://www.elastic.co/guide/en/elasticsearch/client/go-api/current/installation.html) for more information.
+
+<!--codeinclude-->
+[Create an Elasticsearch client](../../modules/elasticsearch/examples_test.go) inside_block:elasticsearchClient
+<!--/codeinclude-->

internal/config/config.go

@@ -89,8 +89,6 @@ func read() Config {
 		return applyEnvironmentConfiguration(config)
 	}
 
-	fmt.Printf("Testcontainers properties file has been found: %s\n", tcProp)
-
 	return applyEnvironmentConfiguration(config)
 }
 

mkdocs.yml

@@ -64,6 +64,7 @@ nav:
         - modules/artemis.md
         - modules/clickhouse.md
         - modules/couchbase.md
+        - modules/elasticsearch.md
         - modules/k3s.md
         - modules/localstack.md
         - modules/mariadb.md

modules/elasticsearch/Makefile

@@ -0,0 +1,5 @@
+include ../../commons-test.mk
+
+.PHONY: test
+test:
+	$(MAKE) test-elasticsearch

modules/elasticsearch/elasticsearch.go

@@ -0,0 +1,211 @@
+package elasticsearch
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/testcontainers/testcontainers-go"
+	"github.com/testcontainers/testcontainers-go/wait"
+)
+
+const (
+	defaultHTTPPort     = "9200"
+	defaultTCPPort      = "9300"
+	defaultPassword     = "changeme"
+	defaultUsername     = "elastic"
+	minimalImageVersion = "7.9.2"
+)
+
+const (
+	DefaultBaseImage    = "docker.elastic.co/elasticsearch/elasticsearch"
+	DefaultBaseImageOSS = "docker.elastic.co/elasticsearch/elasticsearch-oss"
+)
+
+// ElasticsearchContainer represents the Elasticsearch container type used in the module
+type ElasticsearchContainer struct {
+	testcontainers.Container
+	Settings Options
+}
+
+// RunContainer creates an instance of the Elasticsearch container type
+func RunContainer(ctx context.Context, opts ...testcontainers.ContainerCustomizer) (*ElasticsearchContainer, error) {
+	req := testcontainers.GenericContainerRequest{
+		ContainerRequest: testcontainers.ContainerRequest{
+			Image: fmt.Sprintf("%s:%s", DefaultBaseImage, minimalImageVersion),
+			Env: map[string]string{
+				"discovery.type": "single-node",
+				"cluster.routing.allocation.disk.threshold_enabled": "false",
+			},
+			ExposedPorts: []string{
+				defaultHTTPPort + "/tcp",
+				defaultTCPPort + "/tcp",
+			},
+			// regex that
+			//   matches 8.3 JSON logging with started message and some follow up content within the message field
+			//   matches 8.0 JSON logging with no whitespace between message field and content
+			//   matches 7.x JSON logging with whitespace between message field and content
+			//   matches 6.x text logging with node name in brackets and just a 'started' message till the end of the line
+			WaitingFor: wait.ForLog(`.*("message":\s?"started(\s|")?.*|]\sstarted\n)`).AsRegexp(),
+			LifecycleHooks: []testcontainers.ContainerLifecycleHooks{
+				{
+					// the container needs a post create hook to set the default JVM options in a file
+					PostCreates: []testcontainers.ContainerHook{},
+				},
+			},
+		},
+		Started: true,
+	}
+
+	// Gather all config options (defaults and then apply provided options)
+	settings := defaultOptions()
+	for _, opt := range opts {
+		if apply, ok := opt.(Option); ok {
+			apply(settings)
+		}
+		opt.Customize(&req)
+	}
+
+	// Transfer the certificate settings to the container request
+	err := configureCertificate(settings, &req)
+	if err != nil {
+		return nil, err
+	}
+
+	// Transfer the password settings to the container request
+	err = configurePassword(settings, &req)
+	if err != nil {
+		return nil, err
+	}
+
+	if isAtLeastVersion(req.Image, 7) {
+		req.LifecycleHooks[0].PostCreates = append(req.LifecycleHooks[0].PostCreates, configureJvmOpts)
+	}
+
+	container, err := testcontainers.GenericContainer(ctx, req)
+	if err != nil {
+		return nil, err
+	}
+
+	esContainer := &ElasticsearchContainer{Container: container, Settings: *settings}
+
+	address, err := configureAddress(ctx, esContainer)
+	if err != nil {
+		return nil, err
+	}
+
+	esContainer.Settings.Address = address
+
+	return esContainer, nil
+}
+
+// configureAddress sets the address of the Elasticsearch container.
+// If the certificate is set, it will use https as protocol, otherwise http.
+func configureAddress(ctx context.Context, c *ElasticsearchContainer) (string, error) {
+	containerPort, err := c.MappedPort(ctx, defaultHTTPPort+"/tcp")
+	if err != nil {
+		return "", err
+	}
+
+	host, err := c.Host(ctx)
+	if err != nil {
+		return "", err
+	}
+
+	proto := "http"
+	if c.Settings.CACert != nil {
+		proto = "https"
+	}
+
+	return fmt.Sprintf("%s://%s:%s", proto, host, containerPort.Port()), nil
+}
+
+// configureCertificate transfers the certificate settings to the container request.
+// For that, it defines a post start hook that copies the certificate from the container to the host.
+// The certificate is only available since version 8, and will be located in a well-known location.
+func configureCertificate(settings *Options, req *testcontainers.GenericContainerRequest) error {
+	if isAtLeastVersion(req.Image, 8) {
+		// The container needs a post start hook to copy the certificate from the container to the host.
+		// This certificate is only available since version 8
+		req.LifecycleHooks[0].PostStarts = append(req.LifecycleHooks[0].PostStarts,
+			func(ctx context.Context, container testcontainers.Container) error {
+				const defaultCaCertPath = "/usr/share/elasticsearch/config/certs/http_ca.crt"
+
+				readCloser, err := container.CopyFileFromContainer(ctx, defaultCaCertPath)
+				if err != nil {
+					return err
+				}
+
+				// receive the bytes from the default location
+				certBytes, err := io.ReadAll(readCloser)
+				if err != nil {
+					return err
+				}
+
+				settings.CACert = certBytes
+
+				return nil
+			})
+	}
+
+	return nil
+}
+
+// configurePassword transfers the password settings to the container request.
+// If the password is not set, it will be set to "changeme" for Elasticsearch 8
+func configurePassword(settings *Options, req *testcontainers.GenericContainerRequest) error {
+	// set "changeme" as default password for Elasticsearch 8
+	if isAtLeastVersion(req.Image, 8) && settings.Password == "" {
+		WithPassword(defaultPassword)(settings)
+	}
+
+	if settings.Password != "" {
+		if isOSS(req.Image) {
+			return fmt.Errorf("it's not possible to activate security on Elastic OSS Image. Please switch to the default distribution.")
+		}
+
+		if _, ok := req.Env["ELASTIC_PASSWORD"]; !ok {
+			req.Env["ELASTIC_PASSWORD"] = settings.Password
+		}
+
+		// major version 8 is secure by default and does not need this to enable authentication
+		if !isAtLeastVersion(req.Image, 8) {
+			req.Env["xpack.security.enabled"] = "true"
+		}
+	}
+
+	return nil
+}
+
+// configureJvmOpts sets the default memory of the Elasticsearch instance to 2GB.
+// This functions, which is only available since version 7, is called as a post create hook
+// for the container request.
+func configureJvmOpts(ctx context.Context, container testcontainers.Container) error {
+	// Sets default memory of elasticsearch instance to 2GB
+	defaultJVMOpts := `-Xms2G
+-Xmx2G
+-Dingest.geoip.downloader.enabled.default=false
+`
+
+	tmpDir := os.TempDir()
+
+	tmpFile, err := os.CreateTemp(tmpDir, "elasticsearch-default-memory-vm.options")
+	if err != nil {
+		return err
+	}
+	defer os.Remove(tmpFile.Name()) // clean up
+
+	if _, err := tmpFile.WriteString(defaultJVMOpts); err != nil {
+		return err
+	}
+
+	// Spaces are deliberate to allow user to define additional jvm options as elasticsearch resolves option files lexicographically
+	if err := container.CopyFileToContainer(
+		ctx, tmpFile.Name(),
+		"/usr/share/elasticsearch/config/jvm.options.d/ elasticsearch-default-memory-vm.options", 0o644); err != nil {
+		return err
+	}
+
+	return nil
+}