[Heartbeat] Build elastic agent with synthetics support / offline mode (

#27052)

Creates a new offline tagged docker image that includes the dependencies required for synthetics. Building elastic-agent docker images will now create additional -offline tagged images with these extras. We may need to do additional work to ensure that these additional images are incorporated into the release process and onto the public website.

These changes also set the ELASTIC_SYNTHETICS_CAPABLE env flag, as done in heartbeat, to enable synthetics features in the docker environment. This dovetails with the work @dominiqueclarke is doing in elastic/integrations#1064 and elsewhere

Fixes #22932

CHANGELOG.next.asciidoc

@@ -498,6 +498,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add proxy support for AWS functions. {pull}26832[26832]
 - Added policies to the elasticsearch output for non indexible events {pull}26952[26952]
 - Add sha256 digests to RPM packages. {issue}23670[23670]
+- Add new 'offline' docker image for Elastic Agent. {pull}27052[27052]
 
 *Auditbeat*
 

dev-tools/mage/dockerbuilder.go

@@ -66,25 +66,29 @@ func (b *dockerBuilder) Build() error {
 		return err
 	}
 
-	if err := b.prepareBuild(); err != nil {
-		return errors.Wrap(err, "failed to prepare build")
-	}
+	// We always have at least one default variant
+	variants := append([]string{""}, b.PackageSpec.Variants...)
+	for _, variant := range variants {
+		if err := b.prepareBuild(variant); err != nil {
+			return errors.Wrap(err, "failed to prepare build")
+		}
 
-	tries := 3
-	tag, err := b.dockerBuild()
-	for err != nil && tries != 0 {
-		fmt.Println(">> Building docker images again (after 10 seconds)")
-		// This sleep is to avoid hitting the docker build issues when resources are not available.
-		time.Sleep(10)
-		tag, err = b.dockerBuild()
-		tries -= 1
-	}
-	if err != nil {
-		return errors.Wrap(err, "failed to build docker")
-	}
+		tag, err := b.dockerBuild(variant)
+		tries := 3
+		for err != nil && tries != 0 {
+			fmt.Println(">> Building docker images again (after 10 s)")
+			// This sleep is to avoid hitting the docker build issues when resources are not available.
+			time.Sleep(time.Second * 10)
+			tag, err = b.dockerBuild(variant)
+			tries -= 1
+		}
+		if err != nil {
+			return errors.Wrap(err, "failed to build docker")
+		}
 
-	if err := b.dockerSave(tag); err != nil {
-		return errors.Wrap(err, "failed to save docker as artifact")
+		if err := b.dockerSave(tag); err != nil {
+			return errors.Wrap(err, "failed to save docker as artifact")
+		}
 	}
 
 	return nil
@@ -120,7 +124,7 @@ func (b *dockerBuilder) copyFiles() error {
 	return nil
 }
 
-func (b *dockerBuilder) prepareBuild() error {
+func (b *dockerBuilder) prepareBuild(variant string) error {
 	elasticBeatsDir, err := ElasticBeatsDir()
 	if err != nil {
 		return err
@@ -130,6 +134,7 @@ func (b *dockerBuilder) prepareBuild() error {
 	data := map[string]interface{}{
 		"ExposePorts": b.exposePorts(),
 		"ModulesDirs": b.modulesDirs(),
+		"Variant":     variant,
 	}
 
 	err = filepath.Walk(templatesDir, func(path string, info os.FileInfo, _ error) error {
@@ -189,8 +194,11 @@ func (b *dockerBuilder) expandDockerfile(templatesDir string, data map[string]in
 	return nil
 }
 
-func (b *dockerBuilder) dockerBuild() (string, error) {
+func (b *dockerBuilder) dockerBuild(variant string) (string, error) {
 	tag := fmt.Sprintf("%s:%s", b.imageName, b.Version)
+	if variant != "" {
+		tag = fmt.Sprintf("%s-%s", tag, variant)
+	}
 	if b.Snapshot {
 		tag = tag + "-SNAPSHOT"
 	}

dev-tools/mage/pkgtypes.go

@@ -90,6 +90,7 @@ type PackageSpec struct {
 	Files             map[string]PackageFile `yaml:"files"`
 	OutputFile        string                 `yaml:"output_file,omitempty"` // Optional
 	ExtraVars         map[string]string      `yaml:"extra_vars,omitempty"`  // Optional
+	Variants          []string               `yaml:"variants"`              // Optional
 
 	evalContext            map[string]interface{}
 	packageDir             string

dev-tools/packaging/packages.yml

@@ -1000,6 +1000,9 @@ specs:
         <<: *agent_docker_spec
         <<: *elastic_docker_spec
         <<: *elastic_license_for_binaries
+        # This image gets an 'offline' variant for synthetics and other large
+        # packages too big to fit in the main image
+        variants: ["offline"]
         files:
           '{{.BeatName}}{{.BinaryExt}}':
             source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
@@ -1023,6 +1026,9 @@ specs:
         <<: *agent_docker_arm_spec
         <<: *elastic_docker_spec
         <<: *elastic_license_for_binaries
+        # This image gets an 'offline' variant for synthetics and other large
+        # packages too big to fit in the main image
+        variants: ["offline"]
         files:
           '{{.BeatName}}{{.BinaryExt}}':
             source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}

dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl

@@ -27,6 +27,10 @@ RUN mkdir -p {{ $beatHome }}/data {{ $beatHome }}/data/elastic-agent-{{ commit_s
 
 FROM {{ .from }}
 
+# Contains the elastic agent image variant, an empty string for the standard variant
+# or "offline" for the offline one.
+ENV ELASTIC_AGENT_IMAGE_VARIANT={{.Variant}}
+
 {{- if contains .from "ubi-minimal" }}
 RUN for iter in {1..10}; do microdnf update -y && microdnf install -y shadow-utils jq &&  microdnf clean all && exit_code=0 && break || exit_code=$? && echo "microdnf error: retry $iter in 10s" && sleep 10; done; (exit $exit_code)
 {{- else }}
@@ -37,10 +41,31 @@ RUN case $(arch) in aarch64) YUM_FLAGS="-x bind-license";; esac; \
         yum install -y epel-release &&  \
         yum update -y $YUM_FLAGS && \
         yum install -y jq && \
+
+        yum clean all && \
+        exit_code=0 && break || exit_code=$? && echo "yum error: retry $iter in 10s" && sleep 10; \
+    done; \
+    (exit $exit_code)
+{{- end }}
+
+{{- if (and (eq .Variant "offline") (not (contains .from "ubi-minimal"))) }}
+RUN for iter in {1..10}; do \
+        yum -y install atk cups gtk gdk xrandr pango libXcomposite libXcursor libXdamage \
+        libXext libXi libXtst cups-libs libXScrnSaver libXrandr GConf2 \
+        alsa-lib atk gtk3 ipa-gothic-fonts xorg-x11-fonts-100dpi xorg-x11-fonts-75dpi xorg-x11-utils \
+        xorg-x11-fonts-cyrillic xorg-x11-fonts-Type1 xorg-x11-fonts-misc \
         yum clean all && \
         exit_code=0 && break || exit_code=$? && echo "yum error: retry $iter in 10s" && sleep 10; \
     done; \
     (exit $exit_code)
+ENV NODE_PATH={{ $beatHome }}/.node
+RUN echo \
+    $NODE_PATH \
+    {{ $beatHome }}/.config \
+    {{ $beatHome }}/.synthetics \
+    {{ $beatHome }}/.npm \
+    {{ $beatHome }}/.cache \
+    | xargs -IDIR sh -c 'mkdir -p DIR && chmod 0770 DIR'
 {{- end }}
 
 LABEL \
@@ -112,9 +137,42 @@ COPY --from=home {{ $beatHome }}/NOTICE.txt /licenses
 {{- if ne .user "root" }}
 RUN groupadd --gid 1000 {{ .BeatName }}
 RUN useradd -M --uid 1000 --gid 1000 --groups 0 --home {{ $beatHome }} {{ .user }}
+{{- if (and (eq .Variant "offline") (not (contains .from "ubi-minimal")))  }}
+RUN chown {{ .user }} $NODE_PATH
+{{- end }}
 {{- end }}
 USER {{ .user }}
 
+{{- if (and (eq .Variant "offline") (not (contains .from "ubi-minimal")))  }}
+# Setup synthetics env vars
+ENV ELASTIC_SYNTHETICS_CAPABLE=true
+ENV SUITES_DIR={{ $beatHome }}/suites
+ENV NODE_VERSION=12.22.3
+ENV PATH="$NODE_PATH/node/bin:$PATH"
+# Install the latest version of @elastic/synthetics forcefully ignoring the previously
+# cached node_modules, heartbeat then calls the global executable to run test suites
+# Setup node
+RUN cd {{$beatHome}}/.node \
+  && NODE_DOWNLOAD_URL="" \
+  && case "$(arch)" in \
+       x86_64) \
+           NODE_DOWNLOAD_URL=https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-x64.tar.xz \
+           ;; \
+       aarch64) \
+           NODE_DOWNLOAD_URL=https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-arm64.tar.xz \
+           ;; \
+       *) \
+           echo >&2 ; echo >&2 "Unsupported architecture \$(arch)" ; echo >&2 ; exit 1 ; \
+           ;; \
+     esac \
+  && mkdir -p node \
+  && curl ${NODE_DOWNLOAD_URL} | tar -xJ --strip 1 -C node \
+  && chmod ug+rwX -R $NODE_PATH \
+  && npm i -g -f @elastic/synthetics && chmod ug+rwX -R $NODE_PATH
+{{- end }}
+
+
+
 {{- range $i, $port := .ExposePorts }}
 EXPOSE {{ $port }}
 {{- end }}

x-pack/elastic-agent/README.md

@@ -16,3 +16,7 @@ If you are in the 7.13 branch, this will create the `docker.elastic.co/beats/ela
 ```
 elastic-package stack up --version=7.13.0-SNAPSHOT -v
 ```
+
+Please note that the docker container is built in both standard and 'offline' variants.
+The 'offline' variant contains extra files, like the chromium browser, that are too large
+for the standard variant.

x-pack/heartbeat/monitors/browser/synthexec/enrich.go

@@ -122,9 +122,9 @@ func (je *journeyEnricher) enrichSynthEvent(event *beat.Event, se *SynthEvent) e
 	case "step/screenshot_ref":
 		fallthrough
 	case "screenshot/block":
-		add_data_stream_index.SetEventDataset(event, "browser_screenshot")
+		add_data_stream_index.SetEventDataset(event, "browser.screenshot")
 	case "journey/network_info":
-		add_data_stream_index.SetEventDataset(event, "browser_network")
+		add_data_stream_index.SetEventDataset(event, "browser.network")
 	}
 
 	if se.Id != "" {

x-pack/heartbeat/monitors/browser/synthexec/enrich_test.go

@@ -151,7 +151,7 @@ func TestEnrichSynthEvent(t *testing.T) {
 			&SynthEvent{Type: "step/screenshot"},
 			false,
 			func(t *testing.T, e *beat.Event, je *journeyEnricher) {
-				require.Equal(t, "browser_screenshot", e.Meta[add_data_stream_index.FieldMetaCustomDataset])
+				require.Equal(t, "browser.screenshot", e.Meta[add_data_stream_index.FieldMetaCustomDataset])
 			},
 		},
 		{
@@ -160,7 +160,7 @@ func TestEnrichSynthEvent(t *testing.T) {
 			&SynthEvent{Type: "step/screenshot_ref"},
 			false,
 			func(t *testing.T, e *beat.Event, je *journeyEnricher) {
-				require.Equal(t, "browser_screenshot", e.Meta[add_data_stream_index.FieldMetaCustomDataset])
+				require.Equal(t, "browser.screenshot", e.Meta[add_data_stream_index.FieldMetaCustomDataset])
 			},
 		},
 		{
@@ -171,7 +171,7 @@ func TestEnrichSynthEvent(t *testing.T) {
 			func(t *testing.T, e *beat.Event, je *journeyEnricher) {
 				require.Equal(t, "my_id", e.Meta["_id"])
 				require.Equal(t, events.OpTypeCreate, e.Meta[events.FieldMetaOpType])
-				require.Equal(t, "browser_screenshot", e.Meta[add_data_stream_index.FieldMetaCustomDataset])
+				require.Equal(t, "browser.screenshot", e.Meta[add_data_stream_index.FieldMetaCustomDataset])
 			},
 		},
 		{
@@ -180,7 +180,7 @@ func TestEnrichSynthEvent(t *testing.T) {
 			&SynthEvent{Type: "journey/network_info"},
 			false,
 			func(t *testing.T, e *beat.Event, je *journeyEnricher) {
-				require.Equal(t, "browser_network", e.Meta[add_data_stream_index.FieldMetaCustomDataset])
+				require.Equal(t, "browser.network", e.Meta[add_data_stream_index.FieldMetaCustomDataset])
 			},
 		},
 	}

x-pack/osquerybeat/beater/config_plugin_test.go

@@ -12,11 +12,12 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/google/go-cmp/cmp"
+
 	"github.com/elastic/beats/v7/libbeat/logp"
 	"github.com/elastic/beats/v7/x-pack/osquerybeat/internal/config"
 	"github.com/elastic/beats/v7/x-pack/osquerybeat/internal/ecs"
 	"github.com/elastic/beats/v7/x-pack/osquerybeat/internal/testutil"
-	"github.com/google/go-cmp/cmp"
 )
 
 func renderFullConfigJSON(inputs []config.InputConfig) (string, error) {