From a768373c8f1c86c47ef043f9d9113d4b858a66fd Mon Sep 17 00:00:00 2001 From: val06 Date: Wed, 27 Nov 2024 02:41:47 +0200 Subject: [PATCH] [USM] refactored docker-compose testutil (#31470) Co-authored-by: Guy Arbitman --- .github/CODEOWNERS | 1 + .../module/impl_linux_test.go | 11 +- pkg/gpu/testutil/docker.go | 29 --- pkg/gpu/testutil/samplebins.go | 6 +- pkg/network/protocols/amqp/server.go | 10 +- pkg/network/protocols/kafka/server.go | 11 +- pkg/network/protocols/mongo/server.go | 11 +- pkg/network/protocols/mysql/server.go | 10 +- pkg/network/protocols/postgres/server.go | 11 +- pkg/network/protocols/redis/server.go | 10 +- pkg/network/protocols/testutil/pcaputils.go | 1 + pkg/network/protocols/testutil/serverutils.go | 155 --------------- .../protocols/testutil/tls_settings.go | 1 + .../protocols/tls/gotls/testutil/server.go | 11 +- pkg/network/protocols/tls/nodejs/nodejs.go | 14 +- pkg/network/usm/monitor_tls_test.go | 12 +- .../usm/sharedlibraries/testutil/testutil.go | 2 +- pkg/util/testutil/docker/config.go | 185 ++++++++++++++++++ pkg/util/testutil/docker/get.go | 47 +++++ pkg/util/testutil/docker/run.go | 108 ++++++++++ .../testutil/patternscanner.go | 6 +- 21 files changed, 437 insertions(+), 215 deletions(-) delete mode 100644 pkg/gpu/testutil/docker.go delete mode 100644 pkg/network/protocols/testutil/serverutils.go create mode 100644 pkg/util/testutil/docker/config.go create mode 100644 pkg/util/testutil/docker/get.go create mode 100644 pkg/util/testutil/docker/run.go rename pkg/{network/protocols => util}/testutil/patternscanner.go (95%) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index aea9022667a2e..e5db92be230da 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -465,6 +465,7 @@ /pkg/util/pdhutil/ @DataDog/windows-agent /pkg/util/winutil/ @DataDog/windows-agent /pkg/util/testutil/flake @DataDog/agent-devx-loops +/pkg/util/testutil/docker @DataDog/universal-service-monitoring @DataDog/ebpf-platform /pkg/util/trie @DataDog/container-integrations /pkg/languagedetection @DataDog/processes @DataDog/universal-service-monitoring /pkg/linters/ @DataDog/agent-devx-loops diff --git a/pkg/collector/corechecks/servicediscovery/module/impl_linux_test.go b/pkg/collector/corechecks/servicediscovery/module/impl_linux_test.go index bdea1f5c70c8c..bd71b5d00e6b5 100644 --- a/pkg/collector/corechecks/servicediscovery/module/impl_linux_test.go +++ b/pkg/collector/corechecks/servicediscovery/module/impl_linux_test.go @@ -46,12 +46,12 @@ import ( "github.com/DataDog/datadog-agent/pkg/collector/corechecks/servicediscovery/model" "github.com/DataDog/datadog-agent/pkg/network" "github.com/DataDog/datadog-agent/pkg/network/protocols/http/testutil" - protocolUtils "github.com/DataDog/datadog-agent/pkg/network/protocols/testutil" "github.com/DataDog/datadog-agent/pkg/network/protocols/tls/nodejs" fileopener "github.com/DataDog/datadog-agent/pkg/network/usm/sharedlibraries/testutil" usmtestutil "github.com/DataDog/datadog-agent/pkg/network/usm/testutil" "github.com/DataDog/datadog-agent/pkg/util/fxutil" "github.com/DataDog/datadog-agent/pkg/util/kernel" + dockerutils "github.com/DataDog/datadog-agent/pkg/util/testutil/docker" ) func setupDiscoveryModule(t *testing.T) string { @@ -772,10 +772,13 @@ func TestDocker(t *testing.T) { url := setupDiscoveryModule(t) dir, _ := testutil.CurDir() - err := protocolUtils.RunDockerServer(t, "foo-server", - dir+"/testdata/docker-compose.yml", []string{}, + dockerCfg := dockerutils.NewComposeConfig("foo-server", + dockerutils.DefaultTimeout, + dockerutils.DefaultRetries, regexp.MustCompile("Serving.*"), - protocolUtils.DefaultTimeout, 3) + dockerutils.EmptyEnv, + filepath.Join(dir, "testdata", "docker-compose.yml")) + err := dockerutils.Run(t, dockerCfg) require.NoError(t, err) proc, err := procfs.NewDefaultFS() diff --git a/pkg/gpu/testutil/docker.go b/pkg/gpu/testutil/docker.go deleted file mode 100644 index 5e17f9cc94af5..0000000000000 --- a/pkg/gpu/testutil/docker.go +++ /dev/null @@ -1,29 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2024-present Datadog, Inc. - -//go:build test - -// Package testutil holds different utilities and stubs for testing -package testutil - -import ( - "bytes" - "fmt" - "os/exec" - "strings" -) - -// GetDockerContainerID returns the ID of a docker container. -func GetDockerContainerID(dockerName string) (string, error) { - // Ensuring no previous instances exists. - c := exec.Command("docker", "inspect", "-f", "{{.Id}}", dockerName) - var stdout, stderr bytes.Buffer - c.Stdout = &stdout - c.Stderr = &stderr - if err := c.Run(); err != nil { - return "", fmt.Errorf("failed to get %s ID: %s", dockerName, stderr.String()) - } - return strings.TrimSpace(stdout.String()), nil -} diff --git a/pkg/gpu/testutil/samplebins.go b/pkg/gpu/testutil/samplebins.go index a8d74ba47992f..23ab544162e91 100644 --- a/pkg/gpu/testutil/samplebins.go +++ b/pkg/gpu/testutil/samplebins.go @@ -21,9 +21,9 @@ import ( "github.com/stretchr/testify/require" "github.com/DataDog/datadog-agent/pkg/network/protocols/http/testutil" - prototestutil "github.com/DataDog/datadog-agent/pkg/network/protocols/testutil" "github.com/DataDog/datadog-agent/pkg/security/utils" "github.com/DataDog/datadog-agent/pkg/util/log" + dockerutils "github.com/DataDog/datadog-agent/pkg/util/testutil/docker" ) // SampleName represents the name of the sample binary. @@ -177,12 +177,12 @@ func RunSampleInDockerWithArgs(t *testing.T, name SampleName, image DockerImage, var err error // The docker container might take a bit to start, so we retry until we get the PID require.EventuallyWithT(t, func(c *assert.CollectT) { - dockerPID, err = prototestutil.GetDockerPID(containerName) + dockerPID, err = dockerutils.GetMainPID(containerName) assert.NoError(c, err) }, 1*time.Second, 100*time.Millisecond, "failed to get docker PID") require.EventuallyWithT(t, func(c *assert.CollectT) { - dockerContainerID, err = GetDockerContainerID(containerName) + dockerContainerID, err = dockerutils.GetContainerID(containerName) assert.NoError(c, err) }, 1*time.Second, 100*time.Millisecond, "failed to get docker container ID") diff --git a/pkg/network/protocols/amqp/server.go b/pkg/network/protocols/amqp/server.go index ff01e2df4e56c..b6d5e14338e4a 100644 --- a/pkg/network/protocols/amqp/server.go +++ b/pkg/network/protocols/amqp/server.go @@ -16,6 +16,7 @@ import ( httpUtils "github.com/DataDog/datadog-agent/pkg/network/protocols/http/testutil" protocolsUtils "github.com/DataDog/datadog-agent/pkg/network/protocols/testutil" + dockerutils "github.com/DataDog/datadog-agent/pkg/util/testutil/docker" ) const ( @@ -44,7 +45,14 @@ func RunServer(t testing.TB, serverAddr, serverPort string, enableTLS bool) erro startupRegexp := startupRegexpGenerators[enableTLS](t, serverPort) dir, _ := httpUtils.CurDir() - return protocolsUtils.RunDockerServer(t, "amqp", dir+"/testdata/docker-compose.yml", env, startupRegexp, protocolsUtils.DefaultTimeout, 3) + + dockerCfg := dockerutils.NewComposeConfig("amqp", + dockerutils.DefaultTimeout, + dockerutils.DefaultRetries, + startupRegexp, + env, + filepath.Join(dir, "testdata", "docker-compose.yml")) + return dockerutils.Run(t, dockerCfg) } // getServerEnv returns the environment to configure the amqp server diff --git a/pkg/network/protocols/kafka/server.go b/pkg/network/protocols/kafka/server.go index 3a29851fcffcd..b1d892d985814 100644 --- a/pkg/network/protocols/kafka/server.go +++ b/pkg/network/protocols/kafka/server.go @@ -12,10 +12,9 @@ import ( "path/filepath" "regexp" "testing" - "time" "github.com/DataDog/datadog-agent/pkg/network/protocols/http/testutil" - protocolsUtils "github.com/DataDog/datadog-agent/pkg/network/protocols/testutil" + dockerutils "github.com/DataDog/datadog-agent/pkg/util/testutil/docker" ) // RunServer runs a kafka server in a docker container @@ -41,5 +40,11 @@ func RunServer(t testing.TB, serverAddr, serverPort string) error { return err } - return protocolsUtils.RunDockerServer(t, "kafka", dir+"/testdata/docker-compose.yml", env, regexp.MustCompile(`.*started \(kafka.server.KafkaServer\).*`), 1*time.Minute, 3) + dockerCfg := dockerutils.NewComposeConfig("kafka", + dockerutils.DefaultTimeout, + dockerutils.DefaultRetries, + regexp.MustCompile(`.*started \(kafka.server.KafkaServer\).*`), + env, + filepath.Join(dir, "testdata", "docker-compose.yml")) + return dockerutils.Run(t, dockerCfg) } diff --git a/pkg/network/protocols/mongo/server.go b/pkg/network/protocols/mongo/server.go index 70d31a67d8fa1..3abf2f69300a2 100644 --- a/pkg/network/protocols/mongo/server.go +++ b/pkg/network/protocols/mongo/server.go @@ -7,11 +7,12 @@ package mongo import ( "fmt" + "path/filepath" "regexp" "testing" "github.com/DataDog/datadog-agent/pkg/network/protocols/http/testutil" - protocolsUtils "github.com/DataDog/datadog-agent/pkg/network/protocols/testutil" + dockerutils "github.com/DataDog/datadog-agent/pkg/util/testutil/docker" ) const ( @@ -30,5 +31,11 @@ func RunServer(t testing.TB, serverAddress, serverPort string) error { "MONGO_PASSWORD=" + Pass, } dir, _ := testutil.CurDir() - return protocolsUtils.RunDockerServer(t, "mongo", dir+"/testdata/docker-compose.yml", env, regexp.MustCompile(fmt.Sprintf(".*Waiting for connections.*port.*:%s.*", serverPort)), protocolsUtils.DefaultTimeout, 3) + dockerCfg := dockerutils.NewComposeConfig("mongo", + dockerutils.DefaultTimeout, + dockerutils.DefaultRetries, + regexp.MustCompile(fmt.Sprintf(".*Waiting for connections.*port.*:%s.*", serverPort)), + env, + filepath.Join(dir, "testdata", "docker-compose.yml")) + return dockerutils.Run(t, dockerCfg) } diff --git a/pkg/network/protocols/mysql/server.go b/pkg/network/protocols/mysql/server.go index ac17a230353d9..2ead1b2cbe9b5 100644 --- a/pkg/network/protocols/mysql/server.go +++ b/pkg/network/protocols/mysql/server.go @@ -14,7 +14,7 @@ import ( "github.com/stretchr/testify/require" "github.com/DataDog/datadog-agent/pkg/network/protocols/http/testutil" - protocolsUtils "github.com/DataDog/datadog-agent/pkg/network/protocols/testutil" + dockerutils "github.com/DataDog/datadog-agent/pkg/util/testutil/docker" ) const ( @@ -45,5 +45,11 @@ func RunServer(t testing.TB, serverAddr, serverPort string, withTLS bool) error env = append(env, "MYSQL_TLS_ARGS=--require-secure-transport --ssl-cert=/mysql-test/cert.pem.0 --ssl-key=/mysql-test/server.key") } - return protocolsUtils.RunDockerServer(t, "MYSQL", dir+"/testdata/docker-compose.yml", env, regexp.MustCompile(fmt.Sprintf(".*ready for connections.*port: %s.*", serverPort)), protocolsUtils.DefaultTimeout, 3) + dockerCfg := dockerutils.NewComposeConfig("MYSQL", + dockerutils.DefaultTimeout, + dockerutils.DefaultRetries, + regexp.MustCompile(fmt.Sprintf(".*ready for connections.*port: %s.*", serverPort)), + env, + filepath.Join(dir, "testdata", "docker-compose.yml")) + return dockerutils.Run(t, dockerCfg) } diff --git a/pkg/network/protocols/postgres/server.go b/pkg/network/protocols/postgres/server.go index a4366bc661205..8b95627347c16 100644 --- a/pkg/network/protocols/postgres/server.go +++ b/pkg/network/protocols/postgres/server.go @@ -19,7 +19,7 @@ import ( "github.com/stretchr/testify/require" "github.com/DataDog/datadog-agent/pkg/network/protocols/http/testutil" - protocolsUtils "github.com/DataDog/datadog-agent/pkg/network/protocols/testutil" + dockerutils "github.com/DataDog/datadog-agent/pkg/util/testutil/docker" ) // RunServer runs a postgres server in a docker container @@ -48,8 +48,13 @@ func RunServer(t testing.TB, serverAddr, serverPort string, enableTLS bool) erro "ENCRYPTION_MODE=" + encryptionMode, "TESTDIR=" + testDataDir, } - - return protocolsUtils.RunDockerServer(t, "postgres", filepath.Join(testDataDir, "docker-compose.yml"), env, regexp.MustCompile(fmt.Sprintf(".*listening on IPv4 address \"0.0.0.0\", port %s", serverPort)), protocolsUtils.DefaultTimeout, 3) + dockerCfg := dockerutils.NewComposeConfig("postgres", + dockerutils.DefaultTimeout, + dockerutils.DefaultRetries, + regexp.MustCompile(fmt.Sprintf(".*listening on IPv4 address \"0.0.0.0\", port %s", serverPort)), + env, + filepath.Join(testDataDir, "docker-compose.yml")) + return dockerutils.Run(t, dockerCfg) } // copyFile copies a file from src to dst diff --git a/pkg/network/protocols/redis/server.go b/pkg/network/protocols/redis/server.go index 30f2500b616ce..4adf5191dc8bb 100644 --- a/pkg/network/protocols/redis/server.go +++ b/pkg/network/protocols/redis/server.go @@ -18,7 +18,7 @@ import ( "github.com/stretchr/testify/require" "github.com/DataDog/datadog-agent/pkg/network/protocols/http/testutil" - protocolsUtils "github.com/DataDog/datadog-agent/pkg/network/protocols/testutil" + dockerutils "github.com/DataDog/datadog-agent/pkg/util/testutil/docker" ) // RunServer runs a Redis server in a docker container @@ -42,5 +42,11 @@ func RunServer(t testing.TB, serverAddr, serverPort string, enableTLS bool) erro env = append(env, args) } - return protocolsUtils.RunDockerServer(t, "redis", dir+"/testdata/docker-compose.yml", env, regexp.MustCompile(".*Ready to accept connections"), protocolsUtils.DefaultTimeout, 3) + dockerCfg := dockerutils.NewComposeConfig("redis", + dockerutils.DefaultTimeout, + dockerutils.DefaultRetries, + regexp.MustCompile(".*Ready to accept connections"), + env, + filepath.Join(dir, "testdata", "docker-compose.yml")) + return dockerutils.Run(t, dockerCfg) } diff --git a/pkg/network/protocols/testutil/pcaputils.go b/pkg/network/protocols/testutil/pcaputils.go index 582d2fe65fd40..ede05ecd2ad67 100644 --- a/pkg/network/protocols/testutil/pcaputils.go +++ b/pkg/network/protocols/testutil/pcaputils.go @@ -5,6 +5,7 @@ //go:build linux +// Package testutil provides general utilities for protocols UTs. package testutil import ( diff --git a/pkg/network/protocols/testutil/serverutils.go b/pkg/network/protocols/testutil/serverutils.go deleted file mode 100644 index 1bced2d2d7a37..0000000000000 --- a/pkg/network/protocols/testutil/serverutils.go +++ /dev/null @@ -1,155 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2016-present Datadog, Inc. - -package testutil - -import ( - "bytes" - "context" - "fmt" - "os/exec" - "regexp" - "strconv" - "strings" - "testing" - "time" - - "github.com/stretchr/testify/require" -) - -const ( - // DefaultTimeout is the default timeout for running a server. - DefaultTimeout = time.Minute -) - -// GetDockerPID returns the PID of a docker container. -func GetDockerPID(dockerName string) (int64, error) { - // Ensuring no previous instances exists. - c := exec.Command("docker", "inspect", "-f", "{{.State.Pid}}", dockerName) - var stdout, stderr bytes.Buffer - c.Stdout = &stdout - c.Stderr = &stderr - if err := c.Run(); err != nil { - return 0, fmt.Errorf("failed to get %s pid: %s", dockerName, stderr.String()) - } - pid, err := strconv.ParseInt(strings.TrimSpace(stdout.String()), 10, 64) - if pid == 0 { - return 0, fmt.Errorf("failed to retrieve %s pid, container is not running", dockerName) - } - return pid, err -} - -// RunDockerServer is a template for running a protocols server in a docker. -// - serverName is a friendly name of the server we are setting (AMQP, mongo, etc.). -// - dockerPath is the path for the docker-compose. -// - env is any environment variable required for running the server. -// - serverStartRegex is a regex to be matched on the server logs to ensure it started correctly. -func RunDockerServer(t testing.TB, serverName, dockerPath string, env []string, serverStartRegex *regexp.Regexp, timeout time.Duration, retryCount int) error { - var err error - for i := 0; i < retryCount; i++ { - err = runDockerServer(t, serverName, dockerPath, env, serverStartRegex, timeout) - if err == nil { - return nil - } - t.Logf("failed to start %s server, retrying: %v", serverName, err) - time.Sleep(5 * time.Second) - } - return err -} - -func runDockerServer(t testing.TB, serverName, dockerPath string, env []string, serverStartRegex *regexp.Regexp, timeout time.Duration) error { - t.Helper() - // Ensuring the following command won't block for ever - timedContext, cancel := context.WithTimeout(context.Background(), timeout) - t.Cleanup(cancel) - // Ensuring no previous instances exists. - c := exec.CommandContext(timedContext, "docker-compose", "-f", dockerPath, "down", "--remove-orphans", "--volumes") - c.Env = append(c.Env, env...) - _ = c.Run() - cancel() - - ctx, cancel := context.WithCancel(context.Background()) - t.Cleanup(cancel) - - cmd := exec.CommandContext(ctx, "docker-compose", "-f", dockerPath, "up", "--remove-orphans", "-V") - patternScanner := NewScanner(serverStartRegex, make(chan struct{}, 1)) - - cmd.Stdout = patternScanner - cmd.Stderr = patternScanner - cmd.Env = append(cmd.Env, env...) - err := cmd.Start() - require.NoErrorf(t, err, "could not start %s with docker-compose", serverName) - - t.Cleanup(func() { - cancel() - _ = cmd.Wait() - timedContext, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - - c := exec.CommandContext(timedContext, "docker-compose", "-f", dockerPath, "down", "--remove-orphans", "--volumes") - c.Env = append(c.Env, env...) - _ = c.Run() // We need to wait for the command to finish so that the docker containers get cleaned up properly before another docker-compose up call - }) - - for { - select { - case <-ctx.Done(): - if err := ctx.Err(); err != nil { - patternScanner.PrintLogs(t) - return fmt.Errorf("failed to start %s pid %d server: %s", serverName, cmd.Process.Pid, err) - } - case <-patternScanner.DoneChan: - t.Logf("%s server pid (docker) %d is ready", serverName, cmd.Process.Pid) - - return nil - case <-time.After(timeout): - patternScanner.PrintLogs(t) - // please don't use t.Fatalf() here as we could test if it failed later - return fmt.Errorf("failed to start %s server pid %d: timed out after %s", serverName, cmd.Process.Pid, timeout.String()) - } - } -} - -// RunHostServer is a template for running a command on the Host. -// - command is the path for the command to execute. -// - env is any environment variable required for running the server. -// - serverStartRegex is a regex to be matched on the server logs to ensure it started correctly. -// return true on success -func RunHostServer(t *testing.T, command []string, env []string, serverStartRegex *regexp.Regexp) bool { - if len(command) < 1 { - t.Fatalf("command not set %v host server", command) - } - t.Helper() - ctx, cancel := context.WithTimeout(context.Background(), time.Minute) - defer cancel() - - cmd := exec.CommandContext(ctx, command[0], command[1:]...) - serverName := cmd.String() - patternScanner := NewScanner(serverStartRegex, make(chan struct{}, 1)) - - cmd.Stdout = patternScanner - cmd.Stderr = patternScanner - cmd.Env = append(cmd.Env, env...) - err := cmd.Start() - require.NoErrorf(t, err, "could not start %s on host", serverName) - t.Cleanup(func() { - _ = cmd.Wait() - }) - - for { - select { - case <-ctx.Done(): - if err := ctx.Err(); err != nil { - patternScanner.PrintLogs(t) - t.Errorf("failed to start %s pid %d server: %s", serverName, cmd.Process.Pid, err) - } - return false - case <-patternScanner.DoneChan: - t.Logf("%s host server pid %d is ready", serverName, cmd.Process.Pid) - patternScanner.PrintLogs(t) - return true - } - } -} diff --git a/pkg/network/protocols/testutil/tls_settings.go b/pkg/network/protocols/testutil/tls_settings.go index c6f537883a98f..c2ffb0771b543 100644 --- a/pkg/network/protocols/testutil/tls_settings.go +++ b/pkg/network/protocols/testutil/tls_settings.go @@ -3,6 +3,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. +// Package testutil provides general utilities for protocols UTs. package testutil // Constants to represent whether the connection should be encrypted with TLSEnabled. diff --git a/pkg/network/protocols/tls/gotls/testutil/server.go b/pkg/network/protocols/tls/gotls/testutil/server.go index 866878dcc9c86..9ae6e47339709 100644 --- a/pkg/network/protocols/tls/gotls/testutil/server.go +++ b/pkg/network/protocols/tls/gotls/testutil/server.go @@ -10,7 +10,7 @@ import ( "testing" "github.com/DataDog/datadog-agent/pkg/network/protocols/http/testutil" - protocolsUtils "github.com/DataDog/datadog-agent/pkg/network/protocols/testutil" + dockerutils "github.com/DataDog/datadog-agent/pkg/util/testutil/docker" ) // RunServer runs a go-httpbin server in a docker container. @@ -21,5 +21,12 @@ func RunServer(t testing.TB, serverPort string) error { t.Helper() dir, _ := testutil.CurDir() - return protocolsUtils.RunDockerServer(t, "https-gotls", dir+"/../testdata/docker-compose.yml", env, regexp.MustCompile("go-httpbin listening on https://0.0.0.0:8080"), protocolsUtils.DefaultTimeout, 3) + dockerCfg := dockerutils.NewComposeConfig("https-gotls", + dockerutils.DefaultTimeout, + dockerutils.DefaultRetries, + regexp.MustCompile("go-httpbin listening on https://0.0.0.0:8080"), + env, + dir+"/../testdata/docker-compose.yml") + return dockerutils.Run(t, dockerCfg) + } diff --git a/pkg/network/protocols/tls/nodejs/nodejs.go b/pkg/network/protocols/tls/nodejs/nodejs.go index 09ee5ecebf497..c4b8b3682a25c 100644 --- a/pkg/network/protocols/tls/nodejs/nodejs.go +++ b/pkg/network/protocols/tls/nodejs/nodejs.go @@ -11,11 +11,12 @@ package nodejs import ( "io" "os" + "path" "regexp" "testing" "github.com/DataDog/datadog-agent/pkg/network/protocols/http/testutil" - protocolsUtils "github.com/DataDog/datadog-agent/pkg/network/protocols/testutil" + dockerutils "github.com/DataDog/datadog-agent/pkg/util/testutil/docker" ) func copyFile(src, dst string) error { @@ -61,10 +62,17 @@ func RunServerNodeJS(t *testing.T, key, cert, serverPort string) error { "CERTS_DIR=/v/certs", "TESTDIR=" + dir + "/testdata", } - return protocolsUtils.RunDockerServer(t, "nodejs-server", dir+"/testdata/docker-compose.yml", env, regexp.MustCompile("Server running at https.*"), protocolsUtils.DefaultTimeout, 3) + + dockerCfg := dockerutils.NewComposeConfig("nodejs-server", + dockerutils.DefaultTimeout, + dockerutils.DefaultRetries, + regexp.MustCompile("Server running at https.*"), + env, + path.Join(dir, "testdata", "docker-compose.yml")) + return dockerutils.Run(t, dockerCfg) } // GetNodeJSDockerPID returns the PID of the nodejs docker container. func GetNodeJSDockerPID() (int64, error) { - return protocolsUtils.GetDockerPID("node-node-1") + return dockerutils.GetMainPID("node-node-1") } diff --git a/pkg/network/usm/monitor_tls_test.go b/pkg/network/usm/monitor_tls_test.go index 4a0776f259dbd..daadd6ca3e3a2 100644 --- a/pkg/network/usm/monitor_tls_test.go +++ b/pkg/network/usm/monitor_tls_test.go @@ -38,7 +38,6 @@ import ( "github.com/DataDog/datadog-agent/pkg/network/protocols/http" "github.com/DataDog/datadog-agent/pkg/network/protocols/http/testutil" "github.com/DataDog/datadog-agent/pkg/network/protocols/http2" - protocolsUtils "github.com/DataDog/datadog-agent/pkg/network/protocols/testutil" gotlstestutil "github.com/DataDog/datadog-agent/pkg/network/protocols/tls/gotls/testutil" "github.com/DataDog/datadog-agent/pkg/network/protocols/tls/nodejs" usmconfig "github.com/DataDog/datadog-agent/pkg/network/usm/config" @@ -46,6 +45,7 @@ import ( usmtestutil "github.com/DataDog/datadog-agent/pkg/network/usm/testutil" "github.com/DataDog/datadog-agent/pkg/network/usm/utils" procmontestutil "github.com/DataDog/datadog-agent/pkg/process/monitor/testutil" + dockerutils "github.com/DataDog/datadog-agent/pkg/util/testutil/docker" ) type tlsSuite struct { @@ -111,8 +111,14 @@ func (s *tlsSuite) TestHTTPSViaLibraryIntegration() { require.NoError(t, err) dir = path.Join(dir, "testdata", "musl") - protocolsUtils.RunDockerServer(t, "musl-alpine", path.Join(dir, "/docker-compose.yml"), - nil, regexp.MustCompile("started"), protocolsUtils.DefaultTimeout, 3) + dockerCfg := dockerutils.NewComposeConfig("musl-alpine", + dockerutils.DefaultTimeout, + dockerutils.DefaultRetries, + regexp.MustCompile("started"), + dockerutils.EmptyEnv, + path.Join(dir, "/docker-compose.yml")) + err = dockerutils.Run(t, dockerCfg) + require.NoError(t, err) rawout, err := exec.Command("docker", "inspect", "-f", "{{.State.Pid}}", "musl-alpine-1").Output() require.NoError(t, err) diff --git a/pkg/network/usm/sharedlibraries/testutil/testutil.go b/pkg/network/usm/sharedlibraries/testutil/testutil.go index 80f6832cd2f63..be937a68d330a 100644 --- a/pkg/network/usm/sharedlibraries/testutil/testutil.go +++ b/pkg/network/usm/sharedlibraries/testutil/testutil.go @@ -19,9 +19,9 @@ import ( "github.com/stretchr/testify/require" "github.com/DataDog/datadog-agent/pkg/network/protocols/http/testutil" - protocolstestutil "github.com/DataDog/datadog-agent/pkg/network/protocols/testutil" usmtestutil "github.com/DataDog/datadog-agent/pkg/network/usm/testutil" "github.com/DataDog/datadog-agent/pkg/util/log" + protocolstestutil "github.com/DataDog/datadog-agent/pkg/util/testutil" ) // mutex protecting build process diff --git a/pkg/util/testutil/docker/config.go b/pkg/util/testutil/docker/config.go new file mode 100644 index 0000000000000..c7b34281a979e --- /dev/null +++ b/pkg/util/testutil/docker/config.go @@ -0,0 +1,185 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024-present Datadog, Inc. + +//go:build test + +package docker + +import ( + "fmt" + "regexp" + "time" +) + +const ( + // DefaultTimeout is the default timeout for running a server. + DefaultTimeout = time.Minute + + // DefaultRetries is the default number of retries for starting a container/s. + DefaultRetries = 3 +) + +// EmptyEnv is a sugar syntax for empty environment variables +var EmptyEnv []string + +type commandType string + +const ( + dockerCommand commandType = "docker" + // we are using old v1 docker-compose command because our CI doesn't support docker cli v2 yet + composeCommand commandType = "docker-compose" + runCommand commandType = "run" + removeCommand commandType = "rm" +) + +type subCommandType int + +const ( + start = iota + kill +) + +// Compile-time interface compliance check +var _ LifecycleConfig = (*runConfig)(nil) +var _ LifecycleConfig = (*composeConfig)(nil) + +// LifecycleConfig is an interface for the common configuration of a container lifecycle. +type LifecycleConfig interface { + Timeout() time.Duration + Retries() int + LogPattern() *regexp.Regexp + Env() []string + Name() string + command() string + commandArgs(t subCommandType) []string +} + +// Timeout returns the timeout to be used when running a container/s +func (b baseConfig) Timeout() time.Duration { + return b.timeout +} + +// Retries returns the number of retries to be used when trying to start the container/s +func (b baseConfig) Retries() int { + return b.retries +} + +// LogPattern returns the regex pattern to match logs for readiness +func (b baseConfig) LogPattern() *regexp.Regexp { + return b.logPattern +} + +// Env returns the environment variables to set for the container/s +func (b baseConfig) Env() []string { + return b.env +} + +// Name returns the name of the docker container or a friendly name for the docker-compose setup +func (b baseConfig) Name() string { + return b.name +} + +// baseConfig contains shared configurations for both Docker and Docker Compose. +type baseConfig struct { + name string // Container name for docker or an alias for docker-compose + timeout time.Duration // Timeout for the entire operation. + retries int // Number of retries for starting. + logPattern *regexp.Regexp // Regex pattern to match logs for readiness. + env []string // Environment variables to set. +} + +// runConfig contains specific configurations for Docker containers, embedding BaseConfig. +type runConfig struct { + baseConfig // Embed general configuration. + ImageName string // Docker image to use. + Binary string // Binary to run inside the container. + BinaryArgs []string // Arguments for the binary. + Mounts map[string]string // Mounts (host path -> container path). +} + +func (r runConfig) command() string { + return string(dockerCommand) +} + +func (r runConfig) commandArgs(t subCommandType) []string { + var args []string + switch t { + case start: + // we want to remove the container after usage, as it is a temporary container for a particular test + args = []string{string(runCommand), "--rm"} + + // Add mounts + for hostPath, containerPath := range r.Mounts { + args = append(args, "-v", fmt.Sprintf("%s:%s", hostPath, containerPath)) + } + + // Pass environment variables to the container as docker args + for _, env := range r.Env() { + args = append(args, "-e", env) + } + + //append container name and container image name + args = append(args, "--name", r.Name(), r.ImageName) + + //provide main binary and binary arguments to run inside the docker container + args = append(args, r.Binary) + args = append(args, r.BinaryArgs...) + case kill: + args = []string{string(removeCommand), "-f", r.Name(), "--volumes"} + } + return args +} + +// composeConfig contains specific configurations for Docker Compose, embedding BaseConfig. +type composeConfig struct { + baseConfig // Embed general configuration. + File string // Path to the docker-compose file. +} + +func (c composeConfig) command() string { + return string(composeCommand) +} + +func (c composeConfig) commandArgs(t subCommandType) []string { + switch t { + case start: + return []string{"-f", c.File, "up", "--remove-orphans", "-V"} + case kill: + return []string{"-f", c.File, "down", "--remove-orphans", "--volumes"} + default: + return nil + } +} + +// NewRunConfig creates a new runConfig instance for a single docker container. +func NewRunConfig(name string, timeout time.Duration, retries int, logPattern *regexp.Regexp, env []string, imageName, binary string, binaryArgs []string, mounts map[string]string) LifecycleConfig { + return runConfig{ + baseConfig: baseConfig{ + timeout: timeout, + retries: retries, + logPattern: logPattern, + env: env, + name: name, + }, + ImageName: imageName, + Binary: binary, + BinaryArgs: binaryArgs, + Mounts: mounts, + } +} + +// NewComposeConfig creates a new composeConfig instance for the docker-compose. +func NewComposeConfig(name string, timeout time.Duration, retries int, logPattern *regexp.Regexp, env []string, file string) LifecycleConfig { + return composeConfig{ + baseConfig: baseConfig{ + timeout: timeout, + retries: retries, + logPattern: logPattern, + env: env, + name: name, + }, + File: file, + } +} diff --git a/pkg/util/testutil/docker/get.go b/pkg/util/testutil/docker/get.go new file mode 100644 index 0000000000000..eb4ea504ab892 --- /dev/null +++ b/pkg/util/testutil/docker/get.go @@ -0,0 +1,47 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024-present Datadog, Inc. + +//go:build test + +// Package docker provides API to manage docker/docker-compose lifecycle in UTs +package docker + +import ( + "bytes" + "fmt" + "os/exec" + "strconv" + "strings" +) + +// GetMainPID returns the PID of the main process in the docker container. +func GetMainPID(dockerName string) (int64, error) { + // Ensuring no previous instances exists. + c := exec.Command("docker", "inspect", "-f", "{{.State.Pid}}", dockerName) + var stdout, stderr bytes.Buffer + c.Stdout = &stdout + c.Stderr = &stderr + if err := c.Run(); err != nil { + return 0, fmt.Errorf("failed to get %s pid: %s", dockerName, stderr.String()) + } + pid, err := strconv.ParseInt(strings.TrimSpace(stdout.String()), 10, 64) + if pid == 0 { + return 0, fmt.Errorf("failed to retrieve %s pid, container is not running", dockerName) + } + return pid, err +} + +// GetContainerID returns the ID of a docker container. +func GetContainerID(dockerName string) (string, error) { + // Ensuring no previous instances exists. + c := exec.Command("docker", "inspect", "-f", "{{.Id}}", dockerName) + var stdout, stderr bytes.Buffer + c.Stdout = &stdout + c.Stderr = &stderr + if err := c.Run(); err != nil { + return "", fmt.Errorf("failed to get %s ID: %s", dockerName, stderr.String()) + } + return strings.TrimSpace(stdout.String()), nil +} diff --git a/pkg/util/testutil/docker/run.go b/pkg/util/testutil/docker/run.go new file mode 100644 index 0000000000000..2f3049858675a --- /dev/null +++ b/pkg/util/testutil/docker/run.go @@ -0,0 +1,108 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024-present Datadog, Inc. + +//go:build test + +package docker + +import ( + "context" + "fmt" + "os/exec" + "testing" + "time" + + "github.com/DataDog/datadog-agent/pkg/util/testutil" +) + +// Run starts the container/s and ensures their successful invocation +// LifecycleConfig is an interface that abstracts the configuration of the container/s +// Use NewRunConfig to run a single docker container or NewComposeConfig to spin docker-compose +// This method is using testing.TB interface to handle cleanup and logging during UTs execution +func Run(t testing.TB, cfg LifecycleConfig) error { + var err error + var ctx context.Context + for i := 0; i < cfg.Retries(); i++ { + t.Helper() + // Ensuring no previous instances exists. + killPreviousInstances(cfg) + + scanner := testutil.NewScanner(cfg.LogPattern(), make(chan struct{}, 1)) + // attempt to start the container/s + ctx, err = run(t, cfg, scanner) + if err != nil { + t.Logf("could not start %s: %v", cfg.Name(), err) + //this iteration failed, retry + continue + } + + //check container logs for successful start + if err = checkReadiness(ctx, cfg, scanner); err == nil { + // target container/s started successfully, we can stop the retries loop and finish here + t.Logf("%s command succeeded. %s container is running", cfg.command(), cfg.Name()) + return nil + } + t.Logf("[Attempt #%v] failed to start %s server: %v", i+1, cfg.Name(), err) + scanner.PrintLogs(t) + time.Sleep(5 * time.Second) + } + return err +} + +// we do best-effort to kill previous instances, hence ignoring any errors +func killPreviousInstances(cfg LifecycleConfig) { + // Ensuring the following command won't block forever + timedContext, cancel := context.WithTimeout(context.Background(), cfg.Timeout()) + defer cancel() + args := cfg.commandArgs(kill) + + // Ensuring no previous instances exists. + c := exec.CommandContext(timedContext, cfg.command(), args...) + c.Env = append(c.Env, cfg.Env()...) + + // run synchronously to ensure all instances are killed + _ = c.Run() +} + +func run(t testing.TB, cfg LifecycleConfig, scanner *testutil.PatternScanner) (context.Context, error) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + args := cfg.commandArgs(start) + + //prepare the command + cmd := exec.CommandContext(ctx, cfg.command(), args...) + cmd.Env = append(cmd.Env, cfg.Env()...) + cmd.Stdout = scanner + cmd.Stderr = scanner + + // run asynchronously and don't wait for the command to finish + if err := cmd.Start(); err != nil { + return nil, err + } + //register cleanup function to kill the instances upon finishing the test + t.Cleanup(func() { + cancel() + _ = cmd.Wait() + killPreviousInstances(cfg) + }) + + return ctx, nil +} + +func checkReadiness(ctx context.Context, cfg LifecycleConfig, scanner *testutil.PatternScanner) error { + for { + select { + case <-ctx.Done(): + if err := ctx.Err(); err != nil { + return fmt.Errorf("failed to start the container %s due to: %w", cfg.Name(), err) + } + case <-scanner.DoneChan: + return nil + case <-time.After(cfg.Timeout()): + return fmt.Errorf("failed to start the container %s, reached timeout of %v", cfg.Name(), cfg.Timeout()) + } + } +} diff --git a/pkg/network/protocols/testutil/patternscanner.go b/pkg/util/testutil/patternscanner.go similarity index 95% rename from pkg/network/protocols/testutil/patternscanner.go rename to pkg/util/testutil/patternscanner.go index ef95e75b24c1a..a5b7d685e4f1d 100644 --- a/pkg/network/protocols/testutil/patternscanner.go +++ b/pkg/util/testutil/patternscanner.go @@ -3,7 +3,9 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -// Package testutil provides utilities for testing the network package. +//go:build test + +// Package testutil provides general test utilities package testutil import ( @@ -72,5 +74,5 @@ func (ps *PatternScanner) Write(p []byte) (n int, err error) { // PrintLogs writes the captured logs into the test logger. func (ps *PatternScanner) PrintLogs(t testing.TB) { - t.Log(strings.Join(ps.buffers, "")) + t.Log(strings.Join(ps.buffers, "\n")) }