diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 652cc13351f..813a48fcacd 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -290,6 +290,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add proxy support for AWS functions. {pull}26832[26832] - Added policies to the elasticsearch output for non indexible events {pull}26952[26952] - Add sha256 digests to RPM packages. {issue}23670[23670] +- Add new 'offline' docker image for Elastic Agent. {pull}27052[27052] *Auditbeat* diff --git a/dev-tools/mage/dockerbuilder.go b/dev-tools/mage/dockerbuilder.go index d02abad2c57..986209609d2 100644 --- a/dev-tools/mage/dockerbuilder.go +++ b/dev-tools/mage/dockerbuilder.go @@ -66,25 +66,29 @@ func (b *dockerBuilder) Build() error { return err } - if err := b.prepareBuild(); err != nil { - return errors.Wrap(err, "failed to prepare build") - } + // We always have at least one default variant + variants := append([]string{""}, b.PackageSpec.Variants...) + for _, variant := range variants { + if err := b.prepareBuild(variant); err != nil { + return errors.Wrap(err, "failed to prepare build") + } - tries := 3 - tag, err := b.dockerBuild() - for err != nil && tries != 0 { - fmt.Println(">> Building docker images again (after 10 seconds)") - // This sleep is to avoid hitting the docker build issues when resources are not available. - time.Sleep(10) - tag, err = b.dockerBuild() - tries -= 1 - } - if err != nil { - return errors.Wrap(err, "failed to build docker") - } + tag, err := b.dockerBuild(variant) + tries := 3 + for err != nil && tries != 0 { + fmt.Println(">> Building docker images again (after 10 s)") + // This sleep is to avoid hitting the docker build issues when resources are not available. + time.Sleep(time.Second * 10) + tag, err = b.dockerBuild(variant) + tries -= 1 + } + if err != nil { + return errors.Wrap(err, "failed to build docker") + } - if err := b.dockerSave(tag); err != nil { - return errors.Wrap(err, "failed to save docker as artifact") + if err := b.dockerSave(tag); err != nil { + return errors.Wrap(err, "failed to save docker as artifact") + } } return nil @@ -120,7 +124,7 @@ func (b *dockerBuilder) copyFiles() error { return nil } -func (b *dockerBuilder) prepareBuild() error { +func (b *dockerBuilder) prepareBuild(variant string) error { elasticBeatsDir, err := ElasticBeatsDir() if err != nil { return err @@ -130,6 +134,7 @@ func (b *dockerBuilder) prepareBuild() error { data := map[string]interface{}{ "ExposePorts": b.exposePorts(), "ModulesDirs": b.modulesDirs(), + "Variant": variant, } err = filepath.Walk(templatesDir, func(path string, info os.FileInfo, _ error) error { @@ -189,8 +194,11 @@ func (b *dockerBuilder) expandDockerfile(templatesDir string, data map[string]in return nil } -func (b *dockerBuilder) dockerBuild() (string, error) { +func (b *dockerBuilder) dockerBuild(variant string) (string, error) { tag := fmt.Sprintf("%s:%s", b.imageName, b.Version) + if variant != "" { + tag = fmt.Sprintf("%s-%s", tag, variant) + } if b.Snapshot { tag = tag + "-SNAPSHOT" } diff --git a/dev-tools/mage/pkgtypes.go b/dev-tools/mage/pkgtypes.go index c2c454c873d..9a8ebc6d2ad 100644 --- a/dev-tools/mage/pkgtypes.go +++ b/dev-tools/mage/pkgtypes.go @@ -90,6 +90,7 @@ type PackageSpec struct { Files map[string]PackageFile `yaml:"files"` OutputFile string `yaml:"output_file,omitempty"` // Optional ExtraVars map[string]string `yaml:"extra_vars,omitempty"` // Optional + Variants []string `yaml:"variants"` // Optional evalContext map[string]interface{} packageDir string diff --git a/dev-tools/packaging/packages.yml b/dev-tools/packaging/packages.yml index ba657888cce..e807f582ceb 100644 --- a/dev-tools/packaging/packages.yml +++ b/dev-tools/packaging/packages.yml @@ -1001,6 +1001,9 @@ specs: <<: *agent_docker_spec <<: *elastic_docker_spec <<: *elastic_license_for_binaries + # This image gets an 'offline' variant for synthetics and other large + # packages too big to fit in the main image + variants: ["offline"] files: '{{.BeatName}}{{.BinaryExt}}': source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} @@ -1024,6 +1027,9 @@ specs: <<: *agent_docker_arm_spec <<: *elastic_docker_spec <<: *elastic_license_for_binaries + # This image gets an 'offline' variant for synthetics and other large + # packages too big to fit in the main image + variants: ["offline"] files: '{{.BeatName}}{{.BinaryExt}}': source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} diff --git a/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl b/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl index a2408ba30c8..a31565c28b3 100644 --- a/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl +++ b/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl @@ -27,6 +27,10 @@ RUN mkdir -p {{ $beatHome }}/data {{ $beatHome }}/data/elastic-agent-{{ commit_s FROM {{ .from }} +# Contains the elastic agent image variant, an empty string for the standard variant +# or "offline" for the offline one. +ENV ELASTIC_AGENT_IMAGE_VARIANT={{.Variant}} + {{- if contains .from "ubi-minimal" }} RUN for iter in {1..10}; do microdnf update -y && microdnf install -y shadow-utils jq && microdnf clean all && exit_code=0 && break || exit_code=$? && echo "microdnf error: retry $iter in 10s" && sleep 10; done; (exit $exit_code) {{- else }} @@ -37,10 +41,31 @@ RUN case $(arch) in aarch64) YUM_FLAGS="-x bind-license";; esac; \ yum install -y epel-release && \ yum update -y $YUM_FLAGS && \ yum install -y jq && \ + + yum clean all && \ + exit_code=0 && break || exit_code=$? && echo "yum error: retry $iter in 10s" && sleep 10; \ + done; \ + (exit $exit_code) +{{- end }} + +{{- if (and (eq .Variant "offline") (not (contains .from "ubi-minimal"))) }} +RUN for iter in {1..10}; do \ + yum -y install atk cups gtk gdk xrandr pango libXcomposite libXcursor libXdamage \ + libXext libXi libXtst cups-libs libXScrnSaver libXrandr GConf2 \ + alsa-lib atk gtk3 ipa-gothic-fonts xorg-x11-fonts-100dpi xorg-x11-fonts-75dpi xorg-x11-utils \ + xorg-x11-fonts-cyrillic xorg-x11-fonts-Type1 xorg-x11-fonts-misc \ yum clean all && \ exit_code=0 && break || exit_code=$? && echo "yum error: retry $iter in 10s" && sleep 10; \ done; \ (exit $exit_code) +ENV NODE_PATH={{ $beatHome }}/.node +RUN echo \ + $NODE_PATH \ + {{ $beatHome }}/.config \ + {{ $beatHome }}/.synthetics \ + {{ $beatHome }}/.npm \ + {{ $beatHome }}/.cache \ + | xargs -IDIR sh -c 'mkdir -p DIR && chmod 0770 DIR' {{- end }} LABEL \ @@ -112,9 +137,42 @@ COPY --from=home {{ $beatHome }}/NOTICE.txt /licenses {{- if ne .user "root" }} RUN groupadd --gid 1000 {{ .BeatName }} RUN useradd -M --uid 1000 --gid 1000 --groups 0 --home {{ $beatHome }} {{ .user }} +{{- if (and (eq .Variant "offline") (not (contains .from "ubi-minimal"))) }} +RUN chown {{ .user }} $NODE_PATH +{{- end }} {{- end }} USER {{ .user }} +{{- if (and (eq .Variant "offline") (not (contains .from "ubi-minimal"))) }} +# Setup synthetics env vars +ENV ELASTIC_SYNTHETICS_CAPABLE=true +ENV SUITES_DIR={{ $beatHome }}/suites +ENV NODE_VERSION=12.22.3 +ENV PATH="$NODE_PATH/node/bin:$PATH" +# Install the latest version of @elastic/synthetics forcefully ignoring the previously +# cached node_modules, heartbeat then calls the global executable to run test suites +# Setup node +RUN cd {{$beatHome}}/.node \ + && NODE_DOWNLOAD_URL="" \ + && case "$(arch)" in \ + x86_64) \ + NODE_DOWNLOAD_URL=https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-x64.tar.xz \ + ;; \ + aarch64) \ + NODE_DOWNLOAD_URL=https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-arm64.tar.xz \ + ;; \ + *) \ + echo >&2 ; echo >&2 "Unsupported architecture \$(arch)" ; echo >&2 ; exit 1 ; \ + ;; \ + esac \ + && mkdir -p node \ + && curl ${NODE_DOWNLOAD_URL} | tar -xJ --strip 1 -C node \ + && chmod ug+rwX -R $NODE_PATH \ + && npm i -g -f @elastic/synthetics && chmod ug+rwX -R $NODE_PATH +{{- end }} + + + {{- range $i, $port := .ExposePorts }} EXPOSE {{ $port }} {{- end }} diff --git a/x-pack/elastic-agent/README.md b/x-pack/elastic-agent/README.md index 656892c16f6..1483a206475 100644 --- a/x-pack/elastic-agent/README.md +++ b/x-pack/elastic-agent/README.md @@ -16,3 +16,7 @@ If you are in the 7.13 branch, this will create the `docker.elastic.co/beats/ela ``` elastic-package stack up --version=7.13.0-SNAPSHOT -v ``` + +Please note that the docker container is built in both standard and 'offline' variants. +The 'offline' variant contains extra files, like the chromium browser, that are too large +for the standard variant. \ No newline at end of file diff --git a/x-pack/heartbeat/monitors/browser/synthexec/enrich.go b/x-pack/heartbeat/monitors/browser/synthexec/enrich.go index a0b6758644f..a842dd28794 100644 --- a/x-pack/heartbeat/monitors/browser/synthexec/enrich.go +++ b/x-pack/heartbeat/monitors/browser/synthexec/enrich.go @@ -122,9 +122,9 @@ func (je *journeyEnricher) enrichSynthEvent(event *beat.Event, se *SynthEvent) e case "step/screenshot_ref": fallthrough case "screenshot/block": - add_data_stream_index.SetEventDataset(event, "browser_screenshot") + add_data_stream_index.SetEventDataset(event, "browser.screenshot") case "journey/network_info": - add_data_stream_index.SetEventDataset(event, "browser_network") + add_data_stream_index.SetEventDataset(event, "browser.network") } if se.Id != "" { diff --git a/x-pack/heartbeat/monitors/browser/synthexec/enrich_test.go b/x-pack/heartbeat/monitors/browser/synthexec/enrich_test.go index 83c5af37b28..8addfece0f4 100644 --- a/x-pack/heartbeat/monitors/browser/synthexec/enrich_test.go +++ b/x-pack/heartbeat/monitors/browser/synthexec/enrich_test.go @@ -151,7 +151,7 @@ func TestEnrichSynthEvent(t *testing.T) { &SynthEvent{Type: "step/screenshot"}, false, func(t *testing.T, e *beat.Event, je *journeyEnricher) { - require.Equal(t, "browser_screenshot", e.Meta[add_data_stream_index.FieldMetaCustomDataset]) + require.Equal(t, "browser.screenshot", e.Meta[add_data_stream_index.FieldMetaCustomDataset]) }, }, { @@ -160,7 +160,7 @@ func TestEnrichSynthEvent(t *testing.T) { &SynthEvent{Type: "step/screenshot_ref"}, false, func(t *testing.T, e *beat.Event, je *journeyEnricher) { - require.Equal(t, "browser_screenshot", e.Meta[add_data_stream_index.FieldMetaCustomDataset]) + require.Equal(t, "browser.screenshot", e.Meta[add_data_stream_index.FieldMetaCustomDataset]) }, }, { @@ -171,7 +171,7 @@ func TestEnrichSynthEvent(t *testing.T) { func(t *testing.T, e *beat.Event, je *journeyEnricher) { require.Equal(t, "my_id", e.Meta["_id"]) require.Equal(t, events.OpTypeCreate, e.Meta[events.FieldMetaOpType]) - require.Equal(t, "browser_screenshot", e.Meta[add_data_stream_index.FieldMetaCustomDataset]) + require.Equal(t, "browser.screenshot", e.Meta[add_data_stream_index.FieldMetaCustomDataset]) }, }, { @@ -180,7 +180,7 @@ func TestEnrichSynthEvent(t *testing.T) { &SynthEvent{Type: "journey/network_info"}, false, func(t *testing.T, e *beat.Event, je *journeyEnricher) { - require.Equal(t, "browser_network", e.Meta[add_data_stream_index.FieldMetaCustomDataset]) + require.Equal(t, "browser.network", e.Meta[add_data_stream_index.FieldMetaCustomDataset]) }, }, } diff --git a/x-pack/osquerybeat/beater/config_plugin_test.go b/x-pack/osquerybeat/beater/config_plugin_test.go index 9a9783e639d..73ca62e762e 100644 --- a/x-pack/osquerybeat/beater/config_plugin_test.go +++ b/x-pack/osquerybeat/beater/config_plugin_test.go @@ -12,11 +12,12 @@ import ( "strings" "testing" + "github.com/google/go-cmp/cmp" + "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/x-pack/osquerybeat/internal/config" "github.com/elastic/beats/v7/x-pack/osquerybeat/internal/ecs" "github.com/elastic/beats/v7/x-pack/osquerybeat/internal/testutil" - "github.com/google/go-cmp/cmp" ) func renderFullConfigJSON(inputs []config.InputConfig) (string, error) {