From 937f3b580fd5c79efa95a1e37a00b5dbd601e2f1 Mon Sep 17 00:00:00 2001 From: Romain Laurent Date: Thu, 3 Oct 2024 10:59:59 +0200 Subject: [PATCH 1/3] feat(ryuk): wait on Started log --- reaper.go | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/reaper.go b/reaper.go index c41520b5b7..6121287c64 100644 --- a/reaper.go +++ b/reaper.go @@ -204,22 +204,10 @@ func reuseReaperContainer(ctx context.Context, sessionID string, provider Reaper Logger.Printf("⏳ Waiting for Reaper port to be ready") - var containerJson *types.ContainerJSON - - if containerJson, err = reaperContainer.Inspect(ctx); err != nil { - return nil, fmt.Errorf("failed to inspect reaper container %s: %w", reaperContainer.ID[:8], err) - } - - if containerJson != nil && containerJson.NetworkSettings != nil { - for port := range containerJson.NetworkSettings.Ports { - err := wait.ForListeningPort(port). - WithPollInterval(100*time.Millisecond). - WaitUntilReady(ctx, reaperContainer) - if err != nil { - return nil, fmt.Errorf("failed waiting for reaper container %s port %s/%s to be ready: %w", - reaperContainer.ID[:8], port.Proto(), port.Port(), err) - } - } + err = wait.ForLog("Started").WaitUntilReady(ctx, reaperContainer) + if err != nil { + return nil, fmt.Errorf("failed waiting for reaper container %s to be ready: %w", + reaperContainer.ID[:8], err) } return &Reaper{ @@ -249,7 +237,7 @@ func newReaper(ctx context.Context, sessionID string, provider ReaperProvider) ( ExposedPorts: []string{string(listeningPort)}, Labels: core.DefaultLabels(sessionID), Privileged: tcConfig.RyukPrivileged, - WaitingFor: wait.ForListeningPort(listeningPort), + WaitingFor: wait.ForLog("Started"), Name: reaperContainerNameFromSessionID(sessionID), HostConfigModifier: func(hc *container.HostConfig) { hc.AutoRemove = true From 755f116cd5dd991db5212329b06aff41ef8bf652 Mon Sep 17 00:00:00 2001 From: Romain Laurent Date: Thu, 3 Oct 2024 16:26:04 +0200 Subject: [PATCH 2/3] fix(ryul): test by overriding reaper endpoint --- reaper.go | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/reaper.go b/reaper.go index 6121287c64..386c96754f 100644 --- a/reaper.go +++ b/reaper.go @@ -6,6 +6,7 @@ import ( "fmt" "math/rand" "net" + "os" "strings" "sync" "time" @@ -197,15 +198,14 @@ func reuseOrCreateReaper(ctx context.Context, sessionID string, provider ReaperP // reuseReaperContainer constructs a Reaper from an already running reaper // DockerContainer. func reuseReaperContainer(ctx context.Context, sessionID string, provider ReaperProvider, reaperContainer *DockerContainer) (*Reaper, error) { - endpoint, err := reaperContainer.PortEndpoint(ctx, "8080", "") + endpoint, err := reaperEndpoint(ctx, reaperContainer) if err != nil { return nil, err } Logger.Printf("⏳ Waiting for Reaper port to be ready") - err = wait.ForLog("Started").WaitUntilReady(ctx, reaperContainer) - if err != nil { + if err := wait.ForLog("Started!").WaitUntilReady(ctx, reaperContainer); err != nil { return nil, fmt.Errorf("failed waiting for reaper container %s to be ready: %w", reaperContainer.ID[:8], err) } @@ -218,6 +218,14 @@ func reuseReaperContainer(ctx context.Context, sessionID string, provider Reaper }, nil } +func reaperEndpoint(ctx context.Context, c Container) (string, error) { + if _, exists := os.LookupEnv("TESTCONTAINERS_RYUK_ENDPOINT_OVERRIDE_BY_NAME"); exists { + return net.JoinHostPort(reaperContainerNameFromSessionID(c.SessionID()), "8080"), nil + } + + return c.PortEndpoint(ctx, "8080", "") +} + // newReaper creates a Reaper with a sessionID to identify containers and a // provider to use. Do not call this directly, use reuseOrCreateReaper instead. func newReaper(ctx context.Context, sessionID string, provider ReaperProvider) (*Reaper, error) { @@ -237,7 +245,7 @@ func newReaper(ctx context.Context, sessionID string, provider ReaperProvider) ( ExposedPorts: []string{string(listeningPort)}, Labels: core.DefaultLabels(sessionID), Privileged: tcConfig.RyukPrivileged, - WaitingFor: wait.ForLog("Started"), + WaitingFor: wait.ForLog("Started!"), Name: reaperContainerNameFromSessionID(sessionID), HostConfigModifier: func(hc *container.HostConfig) { hc.AutoRemove = true @@ -313,7 +321,7 @@ func newReaper(ctx context.Context, sessionID string, provider ReaperProvider) ( } reaper.container = c - endpoint, err := c.PortEndpoint(ctx, "8080", "") + endpoint, err := reaperEndpoint(ctx, c) if err != nil { return nil, err } @@ -332,6 +340,7 @@ type Reaper struct { // Connect runs a goroutine which can be terminated by sending true into the returned channel func (r *Reaper) Connect() (chan bool, error) { + time.Sleep(2 * time.Second) conn, err := net.DialTimeout("tcp", r.Endpoint, 10*time.Second) if err != nil { return nil, fmt.Errorf("%w: Connecting to Ryuk on %s failed", err, r.Endpoint) From 5b0839041b4bc0c04f212b5e1136c0bcbc8bbf3b Mon Sep 17 00:00:00 2001 From: Romain Laurent Date: Thu, 3 Oct 2024 18:16:47 +0200 Subject: [PATCH 3/3] feat(ryuk): use an external ryuk --- docker.go | 66 +++++++++++++++++++++++---------------- internal/config/config.go | 11 +++++++ 2 files changed, 50 insertions(+), 27 deletions(-) diff --git a/docker.go b/docker.go index 9319c630dd..994a8de7fc 100644 --- a/docker.go +++ b/docker.go @@ -979,6 +979,36 @@ func (p *DockerProvider) BuildImage(ctx context.Context, img ImageBuildInfo) (st return buildOptions.Tags[0], nil } +func (p *DockerProvider) getTermChan(ctx context.Context, sessionID string) (chan bool, error) { + var ( + r *Reaper + err error + ) + + if p.config.RyukDisabled { + return nil, nil + } + + if p.config.RyukExternalAddress != "" { + r = &Reaper{ + Endpoint: p.config.RyukExternalAddress, + SessionID: sessionID, + } + } else { + r, err = reuseOrCreateReaper(context.WithValue(ctx, core.DockerHostContextKey, p.host), core.SessionID(), p) + if err != nil { + return nil, fmt.Errorf("%w: creating reaper failed", err) + } + } + + termSignal, err := r.Connect() + if err != nil { + return nil, fmt.Errorf("%w: connecting to reaper failed", err) + } + + return termSignal, nil +} + // CreateContainer fulfils a request for a container without starting it func (p *DockerProvider) CreateContainer(ctx context.Context, req ContainerRequest) (Container, error) { var err error @@ -1026,14 +1056,10 @@ func (p *DockerProvider) CreateContainer(ctx context.Context, req ContainerReque var termSignal chan bool // the reaper does not need to start a reaper for itself isReaperContainer := strings.HasSuffix(imageName, config.ReaperDefaultImage) - if !p.config.RyukDisabled && !isReaperContainer { - r, err := reuseOrCreateReaper(context.WithValue(ctx, core.DockerHostContextKey, p.host), core.SessionID(), p) - if err != nil { - return nil, fmt.Errorf("%w: creating reaper failed", err) - } - termSignal, err = r.Connect() + if !isReaperContainer { + termSignal, err = p.getTermChan(ctx, core.SessionID()) if err != nil { - return nil, fmt.Errorf("%w: connecting to reaper failed", err) + return nil, err } } @@ -1277,16 +1303,9 @@ func (p *DockerProvider) ReuseOrCreateContainer(ctx context.Context, req Contain sessionID := core.SessionID() - var termSignal chan bool - if !p.config.RyukDisabled { - r, err := reuseOrCreateReaper(context.WithValue(ctx, core.DockerHostContextKey, p.host), sessionID, p) - if err != nil { - return nil, fmt.Errorf("reaper: %w", err) - } - termSignal, err = r.Connect() - if err != nil { - return nil, fmt.Errorf("%w: connecting to reaper failed", err) - } + termSignal, err := p.getTermChan(ctx, sessionID) + if err != nil { + return nil, err } // default hooks include logger hook and pre-create hook @@ -1483,16 +1502,9 @@ func (p *DockerProvider) CreateNetwork(ctx context.Context, req NetworkRequest) sessionID := core.SessionID() - var termSignal chan bool - if !p.config.RyukDisabled { - r, err := reuseOrCreateReaper(context.WithValue(ctx, core.DockerHostContextKey, p.host), sessionID, p) - if err != nil { - return nil, fmt.Errorf("%w: creating network reaper failed", err) - } - termSignal, err = r.Connect() - if err != nil { - return nil, fmt.Errorf("%w: connecting to network reaper failed", err) - } + termSignal, err := p.getTermChan(ctx, sessionID) + if err != nil { + return nil, err } // add the labels that the reaper will use to terminate the network to the request diff --git a/internal/config/config.go b/internal/config/config.go index a172fa3a16..d68c4ca548 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -60,6 +60,12 @@ type Config struct { // Environment variable: TESTCONTAINERS_RYUK_DISABLED RyukDisabled bool `properties:"ryuk.disabled,default=false"` + // RyukExternalAddress is the address of the external Garbage Collector container. + // If set, Testcontainers will not attempt to start its own Ryuk container. + // + // Environment variable: TESTCONTAINERS_RYUK_EXTERNAL_ADDRESS + RyukExternalAddress string `properties:"ryuk.external.address,default="` + // RyukPrivileged is a flag to enable or disable the privileged mode for the Garbage Collector container. // Setting this to true will run the Garbage Collector container in privileged mode. // @@ -116,6 +122,11 @@ func read() Config { config.RyukDisabled = ryukDisabledEnv == "true" } + ryukExternalAddress := os.Getenv("TESTCONTAINERS_RYUK_EXTERNAL_ADDRESS") + if ryukExternalAddress != "" { + config.RyukExternalAddress = ryukExternalAddress + } + hubImageNamePrefix := os.Getenv("TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX") if hubImageNamePrefix != "" { config.HubImageNamePrefix = hubImageNamePrefix