[7.x] systemtest: fix apm-server binary injection (backport #5440)

systemtest/containers.go

@@ -18,6 +18,8 @@
 package systemtest
 
 import (
+	"archive/tar"
+	"bytes"
 	"context"
 	"encoding/json"
 	"errors"
@@ -29,8 +31,8 @@ import (
 	"net/url"
 	"os"
 	"os/exec"
-	"path"
 	"path/filepath"
+	"runtime"
 	"strings"
 	"time"
 
@@ -42,6 +44,7 @@ import (
 	"github.com/testcontainers/testcontainers-go/wait"
 	"golang.org/x/sync/errgroup"
 
+	"github.com/elastic/apm-server/systemtest/apmservertest"
 	"github.com/elastic/apm-server/systemtest/estest"
 	"github.com/elastic/go-elasticsearch/v7"
 )
@@ -281,6 +284,11 @@ func NewUnstartedElasticAgentContainer() (*ElasticAgentContainer, error) {
 		Scheme: "https",
 		Host:   net.JoinHostPort(fleetServerIPAddress, fleetServerPort),
 	}
+	containerCACertPath := "/etc/pki/tls/certs/fleet-ca.pem"
+	hostCACertPath, err := filepath.Abs("../testing/docker/fleet-server/ca.pem")
+	if err != nil {
+		return nil, err
+	}
 
 	// Use the same stack version as used for fleet-server.
 	agentImageVersion := fleetServerContainer.Image[strings.LastIndex(fleetServerContainer.Image, ":")+1:]
@@ -292,43 +300,34 @@ func NewUnstartedElasticAgentContainer() (*ElasticAgentContainer, error) {
 	if err != nil {
 		return nil, err
 	}
-	agentVCSRef := agentImageDetails.Config.Labels["org.label-schema.vcs-ref"]
-	agentDataHashDir := path.Join("/usr/share/elastic-agent/data", "elastic-agent-"+agentVCSRef[:6])
-	agentInstallDir := path.Join(agentDataHashDir, "install")
+	stackVersion := agentImageDetails.Config.Labels["org.label-schema.version"]
+
+	// Build a custom elastic-agent image with a locally built apm-server binary injected.
+	agentImage, err = buildElasticAgentImage(context.Background(), docker, stackVersion, agentImageVersion)
+	if err != nil {
+		return nil, err
+	}
 
 	req := testcontainers.ContainerRequest{
 		Image:      agentImage,
 		AutoRemove: true,
 		Networks:   networks,
+		BindMounts: map[string]string{hostCACertPath: containerCACertPath},
 		Env: map[string]string{
-			// NOTE(axw) because we bind-mount the apm-server artifacts in, they end up owned by the
-			// current user rather than root. Disable Beats's strict permission checks to avoid resulting
-			// complaints, as they're irrelevant to these system tests.
-			"BEAT_STRICT_PERMS": "false",
+			"FLEET_URL": fleetServerURL.String(),
+			"FLEET_CA":  containerCACertPath,
 		},
 	}
 	return &ElasticAgentContainer{
-		request:          req,
-		installDir:       agentInstallDir,
-		fleetServerURL:   fleetServerURL.String(),
-		StackVersion:     agentImageVersion,
-		BindMountInstall: make(map[string]string),
+		request:      req,
+		StackVersion: agentImageVersion,
 	}, nil
 }
 
 // ElasticAgentContainer represents an ephemeral Elastic Agent container.
 type ElasticAgentContainer struct {
-	container      testcontainers.Container
-	request        testcontainers.ContainerRequest
-	fleetServerURL string
-
-	// installDir holds the location of the "install" directory inside
-	// the Elastic Agent container.
-	//
-	// This will be set when the ElasticAgentContainer object is created,
-	// and can be used to anticipate the location into which artifacts
-	// can be bind-mounted.
-	installDir string
+	container testcontainers.Container
+	request   testcontainers.ContainerRequest
 
 	// StackVersion holds the stack version of the container image,
 	// e.g. 8.0.0-SNAPSHOT.
@@ -344,11 +343,6 @@ type ElasticAgentContainer struct {
 	// by exposed port. This will be populated by Start.
 	Addrs map[string]string
 
-	// BindMountInstall holds a map of files to bind mount into the
-	// container, mapping from the host location to target paths relative
-	// to the install directory in the container.
-	BindMountInstall map[string]string
-
 	// FleetEnrollmentToken holds an optional Fleet enrollment token to
 	// use for enrolling the agent with Fleet. The agent will only enroll
 	// if this is specified.
@@ -366,27 +360,12 @@ func (c *ElasticAgentContainer) Start() error {
 	defer cancel()
 
 	// Update request from user-definable fields.
-	c.request.Env["FLEET_URL"] = c.fleetServerURL
 	if c.FleetEnrollmentToken != "" {
 		c.request.Env["FLEET_ENROLL"] = "1"
 		c.request.Env["FLEET_ENROLLMENT_TOKEN"] = c.FleetEnrollmentToken
 	}
-
 	c.request.ExposedPorts = c.ExposedPorts
 	c.request.WaitingFor = c.WaitingFor
-	c.request.BindMounts = map[string]string{}
-	for source, target := range c.BindMountInstall {
-		c.request.BindMounts[source] = path.Join(c.installDir, target)
-	}
-
-	// Inject CA certificate for verifying fleet-server.
-	containerCACertPath := "/etc/pki/tls/certs/fleet-ca.pem"
-	hostCACertPath, err := filepath.Abs("../testing/docker/fleet-server/ca.pem")
-	if err != nil {
-		return err
-	}
-	c.request.BindMounts[hostCACertPath] = containerCACertPath
-	c.request.Env["FLEET_CA"] = containerCACertPath
 
 	container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
 		ContainerRequest: c.request,
@@ -457,3 +436,91 @@ func matchFleetServerAPIStatusHealthy(r io.Reader) bool {
 	}
 	return status.Status == "HEALTHY"
 }
+
+// buildElasticAgentImage builds a Docker image from the published image with a locally built apm-server injected.
+func buildElasticAgentImage(ctx context.Context, docker *client.Client, stackVersion, imageVersion string) (string, error) {
+	imageName := fmt.Sprintf("elastic-agent-systemtest:%s", imageVersion)
+	log.Printf("Building image %s...", imageName)
+
+	// Build apm-server, and copy it into the elastic-agent container's "install" directory.
+	// This bypasses downloading the artifact.
+	arch := runtime.GOARCH
+	if arch == "amd64" {
+		arch = "x86_64"
+	}
+	apmServerInstallDir := fmt.Sprintf("./state/data/install/apm-server-%s-linux-%s", stackVersion, arch)
+	apmServerBinary, err := apmservertest.BuildServerBinary("linux")
+	if err != nil {
+		return "", err
+	}
+
+	// Binaries to copy from disk into the build context.
+	binaries := map[string]string{
+		"apm-server": apmServerBinary,
+	}
+
+	// Generate Dockerfile contents.
+	var dockerfile bytes.Buffer
+	fmt.Fprintf(&dockerfile, "FROM docker.elastic.co/beats/elastic-agent:%s\n", imageVersion)
+	fmt.Fprintf(&dockerfile, "COPY --chown=elastic-agent:elastic-agent apm-server apm-server.yml %s/\n", apmServerInstallDir)
+
+	// Files to generate in the build context.
+	generatedFiles := map[string][]byte{
+		"Dockerfile":     dockerfile.Bytes(),
+		"apm-server.yml": []byte(""),
+	}
+
+	var buildContext bytes.Buffer
+	tarw := tar.NewWriter(&buildContext)
+	for name, path := range binaries {
+		f, err := os.Open(path)
+		if err != nil {
+			return "", err
+		}
+		defer f.Close()
+		info, err := f.Stat()
+		if err != nil {
+			return "", err
+		}
+		if err := tarw.WriteHeader(&tar.Header{
+			Name:  name,
+			Size:  info.Size(),
+			Mode:  0755,
+			Uname: "elastic-agent",
+			Gname: "elastic-agent",
+		}); err != nil {
+			return "", err
+		}
+		if _, err := io.Copy(tarw, f); err != nil {
+			return "", err
+		}
+	}
+	for name, content := range generatedFiles {
+		if err := tarw.WriteHeader(&tar.Header{
+			Name:  name,
+			Size:  int64(len(content)),
+			Mode:  0644,
+			Uname: "elastic-agent",
+			Gname: "elastic-agent",
+		}); err != nil {
+			return "", err
+		}
+		if _, err := tarw.Write(content); err != nil {
+			return "", err
+		}
+	}
+	if err := tarw.Close(); err != nil {
+		return "", err
+	}
+
+	resp, err := docker.ImageBuild(ctx, &buildContext, types.ImageBuildOptions{Tags: []string{imageName}})
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+	if _, err := io.Copy(ioutil.Discard, resp.Body); err != nil {
+		return "", err
+	}
+	log.Printf("Built image %s", imageName)
+	return imageName, nil
+}

systemtest/fleet_test.go

@@ -19,12 +19,8 @@ package systemtest_test
 
 import (
 	"context"
-	"fmt"
 	"io/ioutil"
 	"net/url"
-	"path"
-	"path/filepath"
-	"runtime"
 	"testing"
 	"time"
 
@@ -35,7 +31,6 @@ import (
 	"go.elastic.co/apm/transport"
 
 	"github.com/elastic/apm-server/systemtest"
-	"github.com/elastic/apm-server/systemtest/apmservertest"
 	"github.com/elastic/apm-server/systemtest/fleettest"
 )
 
@@ -80,24 +75,6 @@ func TestFleetIntegration(t *testing.T) {
 		}
 	}()
 
-	// Build apm-server, and bind-mount it into the elastic-agent container's "install"
-	// directory. This bypasses downloading the artifact.
-	arch := runtime.GOARCH
-	if arch == "amd64" {
-		arch = "x86_64"
-	}
-	apmServerArtifactName := fmt.Sprintf("apm-server-%s-linux-%s", agent.StackVersion, arch)
-
-	// Bind-mount the apm-server binary and apm-server.yml into the container's
-	// "install" directory. This causes elastic-agent to skip installing the
-	// artifact.
-	apmServerBinary, err := apmservertest.BuildServerBinary("linux")
-	require.NoError(t, err)
-	agent.BindMountInstall[apmServerBinary] = path.Join(apmServerArtifactName, "apm-server")
-	apmServerConfigFile, err := filepath.Abs("../apm-server.yml")
-	require.NoError(t, err)
-	agent.BindMountInstall[apmServerConfigFile] = path.Join(apmServerArtifactName, "apm-server.yml")
-
 	// Start elastic-agent with port 8200 exposed, and wait for the server to service
 	// healthcheck requests to port 8200.
 	agent.ExposedPorts = []string{"8200"}