feat: more robust Reusable containers experience

container.go

@@ -101,7 +101,7 @@ type FromDockerfile struct {
 	// BuildOptionsModifier Modifier for the build options before image build. Use it for
 	// advanced configurations while building the image. Please consider that the modifier
 	// is called after the default build options are set.
-	BuildOptionsModifier func(*types.ImageBuildOptions)
+	BuildOptionsModifier func(*types.ImageBuildOptions) `hash:"ignore"`
 }
 
 type ContainerFile struct {
@@ -140,6 +140,7 @@ type ContainerRequest struct {
 	RegistryCred            string // Deprecated: Testcontainers will detect registry credentials automatically
 	WaitingFor              wait.Strategy
 	Name                    string // for specifying container name
+	Reuse                   bool   // For reusing an existing container
 	Hostname                string
 	WorkingDir              string                                     // specify the working directory of the container
 	ExtraHosts              []string                                   // Deprecated: Use HostConfigModifier instead
@@ -160,10 +161,10 @@ type ContainerRequest struct {
 	ShmSize                 int64                                      // Amount of memory shared with the host (in bytes)
 	CapAdd                  []string                                   // Deprecated: Use HostConfigModifier instead. Add Linux capabilities
 	CapDrop                 []string                                   // Deprecated: Use HostConfigModifier instead. Drop Linux capabilities
-	ConfigModifier          func(*container.Config)                    // Modifier for the config before container creation
-	HostConfigModifier      func(*container.HostConfig)                // Modifier for the host config before container creation
-	EnpointSettingsModifier func(map[string]*network.EndpointSettings) // Modifier for the network settings before container creation
-	LifecycleHooks          []ContainerLifecycleHooks                  // define hooks to be executed during container lifecycle
+	ConfigModifier          func(*container.Config)                    `hash:"ignore"` // Modifier for the config before container creation
+	HostConfigModifier      func(*container.HostConfig)                `hash:"ignore"` // Modifier for the host config before container creation
+	EnpointSettingsModifier func(map[string]*network.EndpointSettings) `hash:"ignore"` // Modifier for the network settings before container creation
+	LifecycleHooks          []ContainerLifecycleHooks                  `hash:"ignore"` // define hooks to be executed during container lifecycle
 	LogConsumerCfg          *LogConsumerConfig                         // define the configuration for the log producer and its log consumers to follow the logs
 }
 

docker.go

@@ -1122,29 +1122,80 @@ func (p *DockerProvider) CreateContainer(ctx context.Context, req ContainerReque
 
 	req.LifecycleHooks = []ContainerLifecycleHooks{combineContainerHooks(defaultHooks, req.LifecycleHooks)}
 
-	err = req.creatingHook(ctx)
-	if err != nil {
-		return nil, err
+	if req.Reuse {
+		// Remove the SessionID label from the request, as we don't want Ryuk to control
+		// the container lifecycle in the case of reusing containers.
+		delete(req.Labels, core.LabelSessionID)
+	}
+
+	var resp container.CreateResponse
+	if req.Reuse {
+		// we must protect the reusability of the container in the case it's invoked
+		// in a parallel execution, via ParallelContainers or t.Parallel()
+		reuseContainerMx.Lock()
+		defer reuseContainerMx.Unlock()
+
+		// calculate the hash, and add the labels, just before creating the container
+		hash := req.hash()
+		req.Labels[core.LabelContainerHash] = fmt.Sprintf("%d", hash.Hash)
+		req.Labels[core.LabelCopiedFilesHash] = fmt.Sprintf("%d", hash.FilesHash)
+
+		// in the case different test programs are creating a container with the same hash,
+		// we must check if the container is already created. For that we wait up to 5 seconds
+		// for the container to be created. If the error means the container is not found, we
+		// can proceed with the creation of the container.
+		// This is needed because we need to synchronize the creation of the container across
+		// different test programs.
+		c, err := p.waitContainerCreationInTimeout(ctx, hash, 5*time.Second)
+		if err != nil && !errdefs.IsNotFound(err) {
+			// another error occurred different from not found, so we return the error
+			return nil, err
+		}
+
+		// Create a new container if the request is to reuse the container, but there is no container found by hash
+		if c != nil {
+			resp.ID = c.ID
+
+			// replace the logging messages for reused containers:
+			// we know the first lifecycle hook is the logger hook,
+			// so it's safe to replace its first message for reused containers.
+			req.LifecycleHooks[0].PreCreates[0] = func(ctx context.Context, req ContainerRequest) error {
+				Logger.Printf("🔥 Reusing container: %s", resp.ID[:12])
+				return nil
+			}
+			req.LifecycleHooks[0].PostCreates[0] = func(ctx context.Context, c Container) error {
+				Logger.Printf("🔥 Container reused: %s", resp.ID[:12])
+				return nil
+			}
+		}
 	}
 
-	resp, err := p.client.ContainerCreate(ctx, dockerInput, hostConfig, networkingConfig, platform, req.Name)
-	if err != nil {
-		return nil, fmt.Errorf("container create: %w", err)
-	}
-
-	// #248: If there is more than one network specified in the request attach newly created container to them one by one
-	if len(req.Networks) > 1 {
-		for _, n := range req.Networks[1:] {
-			nw, err := p.GetNetwork(ctx, NetworkRequest{
-				Name: n,
-			})
-			if err == nil {
-				endpointSetting := network.EndpointSettings{
-					Aliases: req.NetworkAliases[n],
-				}
-				err = p.client.NetworkConnect(ctx, nw.ID, resp.ID, &endpointSetting)
-				if err != nil {
-					return nil, fmt.Errorf("network connect: %w", err)
+	// If the container was not found by hash, create a new one
+	if resp.ID == "" {
+		err = req.creatingHook(ctx)
+		if err != nil {
+			return nil, err
+		}
+
+		resp, err = p.client.ContainerCreate(ctx, dockerInput, hostConfig, networkingConfig, platform, req.Name)
+		if err != nil {
+			return nil, fmt.Errorf("container create: %w", err)
+		}
+
+		// #248: If there is more than one network specified in the request attach newly created container to them one by one
+		if len(req.Networks) > 1 {
+			for _, n := range req.Networks[1:] {
+				nw, err := p.GetNetwork(ctx, NetworkRequest{
+					Name: n,
+				})
+				if err == nil {
+					endpointSetting := network.EndpointSettings{
+						Aliases: req.NetworkAliases[n],
+					}
+					err = p.client.NetworkConnect(ctx, nw.ID, resp.ID, &endpointSetting)
+					if err != nil {
+						return nil, fmt.Errorf("network connect: %w", err)
+					}
 				}
 			}
 		}
@@ -1194,10 +1245,35 @@ func (p *DockerProvider) findContainerByName(ctx context.Context, name string) (
 	return nil, nil
 }
 
-func (p *DockerProvider) waitContainerCreation(ctx context.Context, name string) (*types.Container, error) {
+func (p *DockerProvider) findContainerByHash(ctx context.Context, ch containerHash) (*types.Container, error) {
+	filter := filters.NewArgs(
+		filters.Arg("label", fmt.Sprintf("%s=%d", core.LabelContainerHash, ch.Hash)),
+		filters.Arg("label", fmt.Sprintf("%s=%d", core.LabelCopiedFilesHash, ch.FilesHash)),
+	)
+
+	containers, err := p.client.ContainerList(ctx, container.ListOptions{Filters: filter})
+	if err != nil {
+		return nil, err
+	}
+	defer p.Close()
+
+	if len(containers) > 0 {
+		return &containers[0], nil
+	}
+	return nil, nil
+}
+
+func (p *DockerProvider) waitContainerCreation(ctx context.Context, hash containerHash) (*types.Container, error) {
+	return p.waitContainerCreationInTimeout(ctx, hash, 5*time.Second)
+}
+
+func (p *DockerProvider) waitContainerCreationInTimeout(ctx context.Context, hash containerHash, timeout time.Duration) (*types.Container, error) {
+	exp := backoff.NewExponentialBackOff()
+	exp.MaxElapsedTime = timeout
+
 	return backoff.RetryNotifyWithData(
 		func() (*types.Container, error) {
-			c, err := p.findContainerByName(ctx, name)
+			c, err := p.findContainerByHash(ctx, hash)
 			if err != nil {
 				if !errdefs.IsNotFound(err) && isPermanentClientError(err) {
 					return nil, backoff.Permanent(err)
@@ -1206,11 +1282,11 @@ func (p *DockerProvider) waitContainerCreation(ctx context.Context, name string)
 			}
 
 			if c == nil {
-				return nil, errdefs.NotFound(fmt.Errorf("container %s not found", name))
+				return nil, errdefs.NotFound(fmt.Errorf("container %v not found", hash))
 			}
 			return c, nil
 		},
-		backoff.WithContext(backoff.NewExponentialBackOff(), ctx),
+		backoff.WithContext(exp, ctx),
 		func(err error, duration time.Duration) {
 			if errdefs.IsNotFound(err) {
 				return
@@ -1220,8 +1296,10 @@ func (p *DockerProvider) waitContainerCreation(ctx context.Context, name string)
 	)
 }
 
+// Deprecated: it will be removed in the next major release.
 func (p *DockerProvider) ReuseOrCreateContainer(ctx context.Context, req ContainerRequest) (Container, error) {
-	c, err := p.findContainerByName(ctx, req.Name)
+	hash := req.hash()
+	c, err := p.findContainerByHash(ctx, hash)
 	if err != nil {
 		return nil, err
 	}
@@ -1233,7 +1311,7 @@ func (p *DockerProvider) ReuseOrCreateContainer(ctx context.Context, req Contain
 		if !createContainerFailDueToNameConflictRegex.MatchString(err.Error()) {
 			return nil, err
 		}
-		c, err = p.waitContainerCreation(ctx, req.Name)
+		c, err = p.waitContainerCreation(ctx, hash)
 		if err != nil {
 			return nil, err
 		}

docker_test.go

@@ -2212,7 +2212,7 @@ func TestDockerProvider_waitContainerCreation_retries(t *testing.T) {
 			// give a chance to retry
 			ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
 			defer cancel()
-			_, _ = p.waitContainerCreation(ctx, "someID")
+			_, _ = p.waitContainerCreation(ctx, containerHash{})
 
 			assert.Positive(t, m.containerListCount)
 			assert.Equal(t, tt.shouldRetry, m.containerListCount > 1)

docs/features/common_functional_options.md

@@ -135,6 +135,18 @@ If you want to attach your containers to a throw-away network, you can use the `
 
 In the case you need to retrieve the network name, you can use the `Networks(ctx)` method of the `Container` interface, right after it's running, which returns a slice of strings with the names of the networks where the container is attached.
 
+#### WithReuse
+
+- Not available until the next release of testcontainers-go <a href="https://github.com/testcontainers/testcontainers-go"><span class="tc-version">:material-tag: main</span></a>
+
+If you want to reuse a container across different test executions, you can use `testcontainers.WithReuse` option. This option will keep the container running after the test execution, so it can be reused by any other test sharing the same `ContainerRequest`. As a result, the container is not terminated by Ryuk.
+
+```golang
+postgres, err = postgresModule.Run(ctx, "postgres:15-alpine", testcontainers.WithReuse())
+```
+
+Please read the [Reuse containers](/features/creating_container#reusable-container) documentation for more information.
+
 #### Docker type modifiers
 
 If you need an advanced configuration for the container, you can leverage the following Docker type modifiers:
@@ -143,7 +155,7 @@ If you need an advanced configuration for the container, you can leverage the fo
 - `testcontainers.WithHostConfigModifier`
 - `testcontainers.WithEndpointSettingsModifier`
 
-Please read the [Create containers: Advanced Settings](/features/creating_container.md#advanced-settings) documentation for more information.
+Please read the [Create containers: Advanced Settings](/features/creating_container#advanced-settings) documentation for more information.
 
 #### Customising the ContainerRequest
 

docs/features/creating_container.md

@@ -145,13 +145,35 @@ The aforementioned `GenericContainer` function and the `ContainerRequest` struct
 
 ## Reusable container
 
-With `Reuse` option you can reuse an existing container. Reusing will work only if you pass an 
-existing container name via 'req.Name' field. If the name is not in a list of existing containers, 
-the function will create a new generic container. If `Reuse` is true and `Name` is empty, you will get error.
+!!!warning
+	Reusing containers is an experimental feature, so please acknowledge you can experience some issues while using it. If you find any issue, please report it [here](https://github.com/testcontainers/testcontainers-go/issues/new?assignees=&labels=bug&projects=&template=bug_report.yml&title=%5BBug%5D%3A+).
+
+A `ReusableContainer` is a container you mark to be reused across different tests. Reusing containers works out of the box just by setting the `Reuse` field in the `ContainerRequest` to `true`.
+Internally, _Testcontainers for Go_ automatically creates a hash of the container request and adds it as a container label. Two labels are added:
+
+- `org.testcontainers.hash` - the hash of the container request.
+- `org.testcontainers.copied_files.hash` - the hash of the files copied to the container using the `Files` field in the container request.
+
+!!!info
+	Only the files copied in the container request will be checked for reuse. If you copy a file to a container after it has been created, as in the example below, the container will still be reused, because the original request has not changed. Directories added in the `Files` field are not included in the hash, to avoid performance issues calculating the hash of large directories.
+
+If there is no container with those two labels matching the hash values, _Testcontainers for Go_ creates a new container. Otherwise, it reuses the existing one.
+
+This behaviour persists across multiple test runs, as long as the container request remains the same. Ryuk the resource reaper does not terminate that container if it is marked for reuse, as it does not match the prune conditions used by Ryuk. To know more about Ryuk, please read the [Garbage Collector](/features/garbage_collector#ryuk) documentation.
+
+!!!warning
+	In the case different test programs are creating a container with the same hash, we must check if the container is already created.
+	For that _Testcontainers for Go_ waits up-to 5 seconds for the container to be created. If the container is not found,
+	the code proceedes with the creation of the container, else the container is reused.
+	This wait is needed because we need to synchronize the creation of the container across different test programs,
+	so you could find very rare situations where the container is not found in different test sessions and it is created in them.
+
+### Reuse example
+
+The following example creates an NGINX container, adds a file into it and then reuses the container again for checking the file:
 
-The following test creates an NGINX container, adds a file into it and then reuses the container again for checking the file:
 ```go
-package main
+package testcontainers_test
 
 import (
 	"context"
@@ -162,43 +184,42 @@ import (
 	"github.com/testcontainers/testcontainers-go/wait"
 )
 
-const (
-	reusableContainerName = "my_test_reusable_container"
-)
-
-func main() {
+func ExampleReusableContainer_usingACopiedFile() {
 	ctx := context.Background()
 
+	req := testcontainers.ContainerRequest{
+		Image:        "nginx:1.17.6",
+		ExposedPorts: []string{"80/tcp"},
+		WaitingFor:   wait.ForListeningPort("80/tcp"),
+		Reuse:        true, // mark the container as reusable
+	}
+
 	n1, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
-		ContainerRequest: testcontainers.ContainerRequest{
-			Image:        "nginx:1.17.6",
-			ExposedPorts: []string{"80/tcp"},
-			WaitingFor:   wait.ForListeningPort("80/tcp"),
-			Name:         reusableContainerName,
-		},
-		Started: true,
+		ContainerRequest: req,
+		Started:          true,
 	})
 	if err != nil {
 		log.Fatal(err)
 	}
-	defer n1.Terminate(ctx)
+	// not terminating the container on purpose, so that it can be reused in a different test.
+	// defer n1.Terminate(ctx)
 
-	copiedFileName := "hello_copy.sh"
-	err = n1.CopyFileToContainer(ctx, "./testdata/hello.sh", "/"+copiedFileName, 700)
+	// Let's copy a file to the container, to demonstrate that successive containers can use the same files
+	// when the container is marked for reuse.
+	bs := []byte(`#!/usr/bin/env bash
+echo "hello world" > /data/hello.txt
+echo "done"`)
 
+	copiedFileName := "hello_copy.sh"
+	err = n1.CopyToContainer(ctx, bs, "/"+copiedFileName, 700)
 	if err != nil {
 		log.Fatal(err)
 	}
 
+	// Because n2 uses the same container request, it will reuse the container created by n1.
 	n2, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
-		ContainerRequest: testcontainers.ContainerRequest{
-			Image:        "nginx:1.17.6",
-			ExposedPorts: []string{"80/tcp"},
-			WaitingFor:   wait.ForListeningPort("80/tcp"),
-			Name:         reusableContainerName,
-		},
-		Started: true,
-		Reuse:   true,
+		ContainerRequest: req,
+		Started:          true,
 	})
 	if err != nil {
 		log.Fatal(err)
@@ -208,10 +229,17 @@ func main() {
 	if err != nil {
 		log.Fatal(err)
 	}
+
+	// the file must exist in this second container, as it's reusing the first one
 	fmt.Println(c)
+
+	// Output: 0
 }
+
 ```
 
+Becuase the `Reuse` option is set to `true`, and the copied files have not changed, the container request is the same, resulting in the second container reusing the first one and the file `hello_copy.sh` being executed.
+
 ## Parallel running
 
 `testcontainers.ParallelContainers` - defines the containers that should be run in parallel mode.

docs/modules/index.md

@@ -202,9 +202,10 @@ In order to simplify the creation of the container for a given module, `Testcont
 - `testcontainers.WithAfterReadyCommand`: a function that sets the execution of a command right after the container is ready (its wait strategy is satisfied).
 - `testcontainers.WithNetwork`: a function that sets the network and the network aliases for the container request.
 - `testcontainers.WithNewNetwork`: a function that sets the network aliases for a throw-away network for the container request.
-- `testcontainers.WithConfigModifier`: a function that sets the config Docker type for the container request. Please see [Advanced Settings](../features/creating_container.md#advanced-settings) for more information.
-- `testcontainers.WithEndpointSettingsModifier`: a function that sets the endpoint settings Docker type for the container request. Please see [Advanced Settings](../features/creating_container.md#advanced-settings) for more information.
-- `testcontainers.WithHostConfigModifier`: a function that sets the host config Docker type for the container request. Please see [Advanced Settings](../features/creating_container.md#advanced-settings) for more information.
+- `testcontainers.WithReuse`: a function that marks the container to be reused. Please see [Reusing containers](../features/creating_container#reusable-container) for more information.
+- `testcontainers.WithConfigModifier`: a function that sets the config Docker type for the container request. Please see [Advanced Settings](../features/creating_container#advanced-settings) for more information.
+- `testcontainers.WithEndpointSettingsModifier`: a function that sets the endpoint settings Docker type for the container request. Please see [Advanced Settings](../features/creating_container#advanced-settings) for more information.
+- `testcontainers.WithHostConfigModifier`: a function that sets the host config Docker type for the container request. Please see [Advanced Settings](../features/creating_container#advanced-settings) for more information.
 - `testcontainers.WithWaitStrategy`: a function that sets the wait strategy for the container request, adding all the passed wait strategies to the container request, using a `testcontainers.MultiStrategy` with 60 seconds of deadline. Please see [Wait strategies](../features/wait/multi.md) for more information.
 - `testcontainers.WithWaitStrategyAndDeadline`: a function that sets the wait strategy for the container request, adding all the passed wait strategies to the container request, using a `testcontainers.MultiStrategy` with the passed deadline. Please see [Wait strategies](../features/wait/multi.md) for more information.
 - `testcontainers.CustomizeRequest`: a function that merges the default options with the ones provided by the user. Recommended for completely customizing the container request.