diff --git a/systemtest/containers.go b/systemtest/containers.go index 161444c4a3a..d50f0d720b6 100644 --- a/systemtest/containers.go +++ b/systemtest/containers.go @@ -18,6 +18,8 @@ package systemtest import ( + "archive/tar" + "bytes" "context" "encoding/json" "errors" @@ -29,8 +31,8 @@ import ( "net/url" "os" "os/exec" - "path" "path/filepath" + "runtime" "strings" "time" @@ -42,6 +44,7 @@ import ( "github.com/testcontainers/testcontainers-go/wait" "golang.org/x/sync/errgroup" + "github.com/elastic/apm-server/systemtest/apmservertest" "github.com/elastic/apm-server/systemtest/estest" "github.com/elastic/go-elasticsearch/v7" ) @@ -281,6 +284,11 @@ func NewUnstartedElasticAgentContainer() (*ElasticAgentContainer, error) { Scheme: "https", Host: net.JoinHostPort(fleetServerIPAddress, fleetServerPort), } + containerCACertPath := "/etc/pki/tls/certs/fleet-ca.pem" + hostCACertPath, err := filepath.Abs("../testing/docker/fleet-server/ca.pem") + if err != nil { + return nil, err + } // Use the same stack version as used for fleet-server. agentImageVersion := fleetServerContainer.Image[strings.LastIndex(fleetServerContainer.Image, ":")+1:] @@ -292,43 +300,34 @@ func NewUnstartedElasticAgentContainer() (*ElasticAgentContainer, error) { if err != nil { return nil, err } - agentVCSRef := agentImageDetails.Config.Labels["org.label-schema.vcs-ref"] - agentDataHashDir := path.Join("/usr/share/elastic-agent/data", "elastic-agent-"+agentVCSRef[:6]) - agentInstallDir := path.Join(agentDataHashDir, "install") + stackVersion := agentImageDetails.Config.Labels["org.label-schema.version"] + + // Build a custom elastic-agent image with a locally built apm-server binary injected. + agentImage, err = buildElasticAgentImage(context.Background(), docker, stackVersion, agentImageVersion) + if err != nil { + return nil, err + } req := testcontainers.ContainerRequest{ Image: agentImage, AutoRemove: true, Networks: networks, + BindMounts: map[string]string{hostCACertPath: containerCACertPath}, Env: map[string]string{ - // NOTE(axw) because we bind-mount the apm-server artifacts in, they end up owned by the - // current user rather than root. Disable Beats's strict permission checks to avoid resulting - // complaints, as they're irrelevant to these system tests. - "BEAT_STRICT_PERMS": "false", + "FLEET_URL": fleetServerURL.String(), + "FLEET_CA": containerCACertPath, }, } return &ElasticAgentContainer{ - request: req, - installDir: agentInstallDir, - fleetServerURL: fleetServerURL.String(), - StackVersion: agentImageVersion, - BindMountInstall: make(map[string]string), + request: req, + StackVersion: agentImageVersion, }, nil } // ElasticAgentContainer represents an ephemeral Elastic Agent container. type ElasticAgentContainer struct { - container testcontainers.Container - request testcontainers.ContainerRequest - fleetServerURL string - - // installDir holds the location of the "install" directory inside - // the Elastic Agent container. - // - // This will be set when the ElasticAgentContainer object is created, - // and can be used to anticipate the location into which artifacts - // can be bind-mounted. - installDir string + container testcontainers.Container + request testcontainers.ContainerRequest // StackVersion holds the stack version of the container image, // e.g. 8.0.0-SNAPSHOT. @@ -344,11 +343,6 @@ type ElasticAgentContainer struct { // by exposed port. This will be populated by Start. Addrs map[string]string - // BindMountInstall holds a map of files to bind mount into the - // container, mapping from the host location to target paths relative - // to the install directory in the container. - BindMountInstall map[string]string - // FleetEnrollmentToken holds an optional Fleet enrollment token to // use for enrolling the agent with Fleet. The agent will only enroll // if this is specified. @@ -366,27 +360,12 @@ func (c *ElasticAgentContainer) Start() error { defer cancel() // Update request from user-definable fields. - c.request.Env["FLEET_URL"] = c.fleetServerURL if c.FleetEnrollmentToken != "" { c.request.Env["FLEET_ENROLL"] = "1" c.request.Env["FLEET_ENROLLMENT_TOKEN"] = c.FleetEnrollmentToken } - c.request.ExposedPorts = c.ExposedPorts c.request.WaitingFor = c.WaitingFor - c.request.BindMounts = map[string]string{} - for source, target := range c.BindMountInstall { - c.request.BindMounts[source] = path.Join(c.installDir, target) - } - - // Inject CA certificate for verifying fleet-server. - containerCACertPath := "/etc/pki/tls/certs/fleet-ca.pem" - hostCACertPath, err := filepath.Abs("../testing/docker/fleet-server/ca.pem") - if err != nil { - return err - } - c.request.BindMounts[hostCACertPath] = containerCACertPath - c.request.Env["FLEET_CA"] = containerCACertPath container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ ContainerRequest: c.request, @@ -457,3 +436,91 @@ func matchFleetServerAPIStatusHealthy(r io.Reader) bool { } return status.Status == "HEALTHY" } + +// buildElasticAgentImage builds a Docker image from the published image with a locally built apm-server injected. +func buildElasticAgentImage(ctx context.Context, docker *client.Client, stackVersion, imageVersion string) (string, error) { + imageName := fmt.Sprintf("elastic-agent-systemtest:%s", imageVersion) + log.Printf("Building image %s...", imageName) + + // Build apm-server, and copy it into the elastic-agent container's "install" directory. + // This bypasses downloading the artifact. + arch := runtime.GOARCH + if arch == "amd64" { + arch = "x86_64" + } + apmServerInstallDir := fmt.Sprintf("./state/data/install/apm-server-%s-linux-%s", stackVersion, arch) + apmServerBinary, err := apmservertest.BuildServerBinary("linux") + if err != nil { + return "", err + } + + // Binaries to copy from disk into the build context. + binaries := map[string]string{ + "apm-server": apmServerBinary, + } + + // Generate Dockerfile contents. + var dockerfile bytes.Buffer + fmt.Fprintf(&dockerfile, "FROM docker.elastic.co/beats/elastic-agent:%s\n", imageVersion) + fmt.Fprintf(&dockerfile, "COPY --chown=elastic-agent:elastic-agent apm-server apm-server.yml %s/\n", apmServerInstallDir) + + // Files to generate in the build context. + generatedFiles := map[string][]byte{ + "Dockerfile": dockerfile.Bytes(), + "apm-server.yml": []byte(""), + } + + var buildContext bytes.Buffer + tarw := tar.NewWriter(&buildContext) + for name, path := range binaries { + f, err := os.Open(path) + if err != nil { + return "", err + } + defer f.Close() + info, err := f.Stat() + if err != nil { + return "", err + } + if err := tarw.WriteHeader(&tar.Header{ + Name: name, + Size: info.Size(), + Mode: 0755, + Uname: "elastic-agent", + Gname: "elastic-agent", + }); err != nil { + return "", err + } + if _, err := io.Copy(tarw, f); err != nil { + return "", err + } + } + for name, content := range generatedFiles { + if err := tarw.WriteHeader(&tar.Header{ + Name: name, + Size: int64(len(content)), + Mode: 0644, + Uname: "elastic-agent", + Gname: "elastic-agent", + }); err != nil { + return "", err + } + if _, err := tarw.Write(content); err != nil { + return "", err + } + } + if err := tarw.Close(); err != nil { + return "", err + } + + resp, err := docker.ImageBuild(ctx, &buildContext, types.ImageBuildOptions{Tags: []string{imageName}}) + if err != nil { + return "", err + } + defer resp.Body.Close() + if _, err := io.Copy(ioutil.Discard, resp.Body); err != nil { + return "", err + } + log.Printf("Built image %s", imageName) + return imageName, nil +} diff --git a/systemtest/fleet_test.go b/systemtest/fleet_test.go index aad92b43574..761bccaad3c 100644 --- a/systemtest/fleet_test.go +++ b/systemtest/fleet_test.go @@ -19,12 +19,8 @@ package systemtest_test import ( "context" - "fmt" "io/ioutil" "net/url" - "path" - "path/filepath" - "runtime" "testing" "time" @@ -35,7 +31,6 @@ import ( "go.elastic.co/apm/transport" "github.com/elastic/apm-server/systemtest" - "github.com/elastic/apm-server/systemtest/apmservertest" "github.com/elastic/apm-server/systemtest/fleettest" ) @@ -80,24 +75,6 @@ func TestFleetIntegration(t *testing.T) { } }() - // Build apm-server, and bind-mount it into the elastic-agent container's "install" - // directory. This bypasses downloading the artifact. - arch := runtime.GOARCH - if arch == "amd64" { - arch = "x86_64" - } - apmServerArtifactName := fmt.Sprintf("apm-server-%s-linux-%s", agent.StackVersion, arch) - - // Bind-mount the apm-server binary and apm-server.yml into the container's - // "install" directory. This causes elastic-agent to skip installing the - // artifact. - apmServerBinary, err := apmservertest.BuildServerBinary("linux") - require.NoError(t, err) - agent.BindMountInstall[apmServerBinary] = path.Join(apmServerArtifactName, "apm-server") - apmServerConfigFile, err := filepath.Abs("../apm-server.yml") - require.NoError(t, err) - agent.BindMountInstall[apmServerConfigFile] = path.Join(apmServerArtifactName, "apm-server.yml") - // Start elastic-agent with port 8200 exposed, and wait for the server to service // healthcheck requests to port 8200. agent.ExposedPorts = []string{"8200"}