From 4fee0d91ca582f4401637959b7e24f7335eac463 Mon Sep 17 00:00:00 2001 From: esm Date: Sun, 21 Apr 2024 22:14:10 -0400 Subject: [PATCH 1/8] If the ryuk container has a health status, wait for a healthy container before returning. --- docker.go | 12 ++++++++++++ reaper.go | 4 ++++ 2 files changed, 16 insertions(+) diff --git a/docker.go b/docker.go index e188c25df2..6e99165716 100644 --- a/docker.go +++ b/docker.go @@ -49,6 +49,11 @@ const ( packagePath = "github.com/testcontainers/testcontainers-go" logStoppedForOutOfSyncMessage = "Stopping log consumer: Headers out of sync" + + healthStatusNone = "" // default status for a container with no healthcheck + healthStatusHealthy = "healthy" // healthy container + healthStatusStarting = "starting" // starting container + healthStatusUnhealthy = "unhealthy" // unhealthy container ) var createContainerFailDueToNameConflictRegex = regexp.MustCompile("Conflict. The container name .* is already in use by container .*") @@ -86,6 +91,8 @@ type DockerContainer struct { logProductionTimeout *time.Duration logger Logging lifecycleHooks []ContainerLifecycleHooks + + healthStatus string // container health status, will default to healthStatusNone if no healthcheck is present } // SetLogger sets the logger for the container @@ -1590,6 +1597,11 @@ func containerFromDockerResponse(ctx context.Context, response types.Container) return nil, err } + // the health status of the container, if any + if health := container.raw.State.Health; health != nil { + container.healthStatus = health.Status + } + return &container, nil } diff --git a/reaper.go b/reaper.go index 54feb90cbe..87205d0ff1 100644 --- a/reaper.go +++ b/reaper.go @@ -117,6 +117,10 @@ func lookUpReaperContainer(ctx context.Context, sessionID string) (*DockerContai return err } + if r.healthStatus != healthStatusHealthy && r.healthStatus != healthStatusNone { + return fmt.Errorf("container %s is not healthy, wanted status=%s, got status=%s", resp[0].ID[:8], healthStatusHealthy, r.healthStatus) + } + reaperContainer = r return nil From b097968b60c082a7d43670f41119d31ffc7e9f02 Mon Sep 17 00:00:00 2001 From: esm Date: Mon, 6 May 2024 20:27:06 -0400 Subject: [PATCH 2/8] Use docker/types for health status, but also check for the zero value. --- docker.go | 5 ----- reaper.go | 8 ++++++-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/docker.go b/docker.go index 6e99165716..e3999a7f62 100644 --- a/docker.go +++ b/docker.go @@ -49,11 +49,6 @@ const ( packagePath = "github.com/testcontainers/testcontainers-go" logStoppedForOutOfSyncMessage = "Stopping log consumer: Headers out of sync" - - healthStatusNone = "" // default status for a container with no healthcheck - healthStatusHealthy = "healthy" // healthy container - healthStatusStarting = "starting" // starting container - healthStatusUnhealthy = "unhealthy" // unhealthy container ) var createContainerFailDueToNameConflictRegex = regexp.MustCompile("Conflict. The container name .* is already in use by container .*") diff --git a/reaper.go b/reaper.go index 87205d0ff1..4f288ed48d 100644 --- a/reaper.go +++ b/reaper.go @@ -4,6 +4,7 @@ import ( "bufio" "context" "fmt" + "github.com/docker/docker/api/types" "math/rand" "net" "strings" @@ -117,8 +118,11 @@ func lookUpReaperContainer(ctx context.Context, sessionID string) (*DockerContai return err } - if r.healthStatus != healthStatusHealthy && r.healthStatus != healthStatusNone { - return fmt.Errorf("container %s is not healthy, wanted status=%s, got status=%s", resp[0].ID[:8], healthStatusHealthy, r.healthStatus) + // if a health status is present on the container, and the container is not healthy, error + if r.healthStatus != "" { + if r.healthStatus != types.Healthy && r.healthStatus != types.NoHealthcheck { + return fmt.Errorf("container %s is not healthy, wanted status=%s, got status=%s", resp[0].ID[:8], types.Healthy, r.healthStatus) + } } reaperContainer = r From 0c992859c9a665dc1ad3ccd734d6192bfa782d7d Mon Sep 17 00:00:00 2001 From: esm Date: Sun, 19 May 2024 15:57:30 -0400 Subject: [PATCH 3/8] Wait for the ryuk port to be available in case the container is still being started by newReaper in a separate process. A race between newReaper and lookUpReaperContainer occurs: - newReaper creates the container with a ContainerRequest.WaitingFor = wait.ForListeningPort(listeningPort) - newReaper starts the container - lookUpReaperContainer obtains the container and returns. - newReaper invokes the readiness hook wait.ForListeningPort(listeningPort). --- reaper.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/reaper.go b/reaper.go index 4f288ed48d..cd1c8ba0c9 100644 --- a/reaper.go +++ b/reaper.go @@ -170,6 +170,27 @@ func reuseOrCreateReaper(ctx context.Context, sessionID string, provider ReaperP if err != nil { return nil, err } + + Logger.Printf("⏳ Waiting for Reaper port to be ready") + + var containerJson *types.ContainerJSON + + if containerJson, err = reaperContainer.Inspect(ctx); err != nil { + return nil, fmt.Errorf("failed to inspect reaper container %s: %w", reaperContainer.ID[:8], err) + } + + if containerJson != nil && containerJson.NetworkSettings != nil { + for port := range containerJson.NetworkSettings.Ports { + err := wait.ForListeningPort(port). + WithPollInterval(100*time.Millisecond). + WaitUntilReady(ctx, reaperContainer) + if err != nil { + return nil, fmt.Errorf("failed waiting for reaper container %s port %s/%s to be ready: %w", + reaperContainer.ID[:8], port.Proto(), port.Port(), err) + } + } + } + return reaperInstance, nil } From f92db6eea5a30d9aaf0a9d8fbbed1dc9eded5a2b Mon Sep 17 00:00:00 2001 From: esm Date: Mon, 20 May 2024 20:43:06 -0400 Subject: [PATCH 4/8] Fix whitespace. --- reaper.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reaper.go b/reaper.go index cd1c8ba0c9..4370f5b5e3 100644 --- a/reaper.go +++ b/reaper.go @@ -171,7 +171,7 @@ func reuseOrCreateReaper(ctx context.Context, sessionID string, provider ReaperP return nil, err } - Logger.Printf("⏳ Waiting for Reaper port to be ready") + Logger.Printf("⏳ Waiting for Reaper port to be ready") var containerJson *types.ContainerJSON From f9d0cf5222c464bc32b93410b107fb51ecc7bd11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Mon, 10 Jun 2024 11:23:44 +0200 Subject: [PATCH 5/8] chore: simplify --- reaper.go | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/reaper.go b/reaper.go index 4370f5b5e3..56d4b4b1b1 100644 --- a/reaper.go +++ b/reaper.go @@ -4,13 +4,14 @@ import ( "bufio" "context" "fmt" - "github.com/docker/docker/api/types" "math/rand" "net" "strings" "sync" "time" + "github.com/docker/docker/api/types" + "github.com/cenkalti/backoff/v4" "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/filters" @@ -118,14 +119,16 @@ func lookUpReaperContainer(ctx context.Context, sessionID string) (*DockerContai return err } - // if a health status is present on the container, and the container is not healthy, error - if r.healthStatus != "" { - if r.healthStatus != types.Healthy && r.healthStatus != types.NoHealthcheck { - return fmt.Errorf("container %s is not healthy, wanted status=%s, got status=%s", resp[0].ID[:8], types.Healthy, r.healthStatus) - } + reaperContainer = r + + if r.healthStatus == types.Healthy || r.healthStatus == types.NoHealthcheck { + return nil } - reaperContainer = r + // if a health status is present on the container, and the container is healthy, error + if r.healthStatus != "" { + return fmt.Errorf("container %s is not healthy, wanted status=%s, got status=%s", resp[0].ID[:8], types.Healthy, r.healthStatus) + } return nil }, backoff.WithContext(exp, ctx)) From ab71f414119a1022e4d1990750cfdb884bfaa9fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Mon, 10 Jun 2024 11:24:05 +0200 Subject: [PATCH 6/8] chore: make lint --- internal/config/config_test.go | 4 ++-- reaper.go | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 15b135177f..efd2e054e6 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -278,7 +278,7 @@ func TestReadTCConfig(t *testing.T) { ``, map[string]string{ "TESTCONTAINERS_RYUK_RECONNECTION_TIMEOUT": "13s", - "TESTCONTAINERS_RYUK_CONNECTION_TIMEOUT": "12s", + "TESTCONTAINERS_RYUK_CONNECTION_TIMEOUT": "12s", }, Config{ RyukReconnectionTimeout: 13 * time.Second, @@ -291,7 +291,7 @@ func TestReadTCConfig(t *testing.T) { ryuk.reconnection.timeout=23s`, map[string]string{ "TESTCONTAINERS_RYUK_RECONNECTION_TIMEOUT": "13s", - "TESTCONTAINERS_RYUK_CONNECTION_TIMEOUT": "12s", + "TESTCONTAINERS_RYUK_CONNECTION_TIMEOUT": "12s", }, Config{ RyukReconnectionTimeout: 13 * time.Second, diff --git a/reaper.go b/reaper.go index 56d4b4b1b1..981e53f836 100644 --- a/reaper.go +++ b/reaper.go @@ -10,9 +10,8 @@ import ( "sync" "time" - "github.com/docker/docker/api/types" - "github.com/cenkalti/backoff/v4" + "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/filters" "github.com/docker/docker/errdefs" From b835925ce0c94bb5866c0d9aebc3fea0e83c2abe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Mon, 10 Jun 2024 13:20:58 +0200 Subject: [PATCH 7/8] chore: change emoji in log output when waiting --- generic_test.go | 2 +- lifecycle.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/generic_test.go b/generic_test.go index 72688876ec..c566e925c1 100644 --- a/generic_test.go +++ b/generic_test.go @@ -134,7 +134,7 @@ func TestGenericReusableContainerInSubprocess(t *testing.T) { output := createReuseContainerInSubprocess(t) // check is reuse container with WaitingFor work correctly. - require.True(t, strings.Contains(output, "🚧 Waiting for container id")) + require.True(t, strings.Contains(output, "⏳ Waiting for container id")) require.True(t, strings.Contains(output, "🔔 Container is ready")) }() } diff --git a/lifecycle.go b/lifecycle.go index 578773b9f5..f46318ce7c 100644 --- a/lifecycle.go +++ b/lifecycle.go @@ -207,7 +207,7 @@ var defaultReadinessHook = func() ContainerLifecycleHooks { // if a Wait Strategy has been specified, wait before returning if dockerContainer.WaitingFor != nil { dockerContainer.logger.Printf( - "🚧 Waiting for container id %s image: %s. Waiting for: %+v", + "⏳ Waiting for container id %s image: %s. Waiting for: %+v", dockerContainer.ID[:12], dockerContainer.Image, dockerContainer.WaitingFor, ) if err := dockerContainer.WaitingFor.WaitUntilReady(ctx, c); err != nil { From d7f1b7184d62b8a6f63d4db7d963c5d1babd80ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Mon, 10 Jun 2024 13:21:20 +0200 Subject: [PATCH 8/8] chore: move inside reuseReaper --- reaper.go | 41 +++++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/reaper.go b/reaper.go index 981e53f836..4b5fabd09c 100644 --- a/reaper.go +++ b/reaper.go @@ -173,26 +173,6 @@ func reuseOrCreateReaper(ctx context.Context, sessionID string, provider ReaperP return nil, err } - Logger.Printf("⏳ Waiting for Reaper port to be ready") - - var containerJson *types.ContainerJSON - - if containerJson, err = reaperContainer.Inspect(ctx); err != nil { - return nil, fmt.Errorf("failed to inspect reaper container %s: %w", reaperContainer.ID[:8], err) - } - - if containerJson != nil && containerJson.NetworkSettings != nil { - for port := range containerJson.NetworkSettings.Ports { - err := wait.ForListeningPort(port). - WithPollInterval(100*time.Millisecond). - WaitUntilReady(ctx, reaperContainer) - if err != nil { - return nil, fmt.Errorf("failed waiting for reaper container %s port %s/%s to be ready: %w", - reaperContainer.ID[:8], port.Proto(), port.Port(), err) - } - } - } - return reaperInstance, nil } @@ -223,6 +203,27 @@ func reuseReaperContainer(ctx context.Context, sessionID string, provider Reaper if err != nil { return nil, err } + + Logger.Printf("⏳ Waiting for Reaper port to be ready") + + var containerJson *types.ContainerJSON + + if containerJson, err = reaperContainer.Inspect(ctx); err != nil { + return nil, fmt.Errorf("failed to inspect reaper container %s: %w", reaperContainer.ID[:8], err) + } + + if containerJson != nil && containerJson.NetworkSettings != nil { + for port := range containerJson.NetworkSettings.Ports { + err := wait.ForListeningPort(port). + WithPollInterval(100*time.Millisecond). + WaitUntilReady(ctx, reaperContainer) + if err != nil { + return nil, fmt.Errorf("failed waiting for reaper container %s port %s/%s to be ready: %w", + reaperContainer.ID[:8], port.Proto(), port.Port(), err) + } + } + } + return &Reaper{ Provider: provider, SessionID: sessionID,