diff --git a/CHANGELOG-developer.asciidoc b/CHANGELOG-developer.asciidoc index 5e07694adb8..fade083a701 100644 --- a/CHANGELOG-developer.asciidoc +++ b/CHANGELOG-developer.asciidoc @@ -61,3 +61,4 @@ The list below covers the major changes between 6.3.0 and master only. - Add `mage.KibanaDashboards` for collecting Kibana dashboards and generating index patterns. {pull}8615[8615] - Allow to disable config resolver using the `Settings.DisableConfigResolver` field when initializing libbeat. {pull}8769[8769] - Add `mage.AddPlatforms` to allow to specify dependent platforms when building a beat. {pull}8889[8889] +- Add docker image building to `mage.Package`. {pull}8898[8898] diff --git a/auditbeat/_meta/beat.docker.yml b/auditbeat/_meta/beat.docker.yml new file mode 100644 index 00000000000..1b50298c6af --- /dev/null +++ b/auditbeat/_meta/beat.docker.yml @@ -0,0 +1,14 @@ +auditbeat.modules: + +- module: auditd + audit_rules: | + -w /etc/passwd -p wa -k identity + -a always,exit -F arch=b32 -S open,creat,truncate,ftruncate,openat,open_by_handle_at -F exit=-EPERM -k access + +- module: file_integrity + paths: + - /bin + - /usr/bin + - /sbin + - /usr/sbin + - /etc diff --git a/auditbeat/auditbeat.docker.yml b/auditbeat/auditbeat.docker.yml new file mode 100644 index 00000000000..3178297b6f7 --- /dev/null +++ b/auditbeat/auditbeat.docker.yml @@ -0,0 +1,21 @@ +auditbeat.modules: + +- module: auditd + audit_rules: | + -w /etc/passwd -p wa -k identity + -a always,exit -F arch=b32 -S open,creat,truncate,ftruncate,openat,open_by_handle_at -F exit=-EPERM -k access + +- module: file_integrity + paths: + - /bin + - /usr/bin + - /sbin + - /usr/sbin + - /etc +processors: +- add_cloud_metadata: ~ + +output.elasticsearch: + hosts: '${ELASTICSEARCH_HOSTS:elasticsearch:9200}' + username: '${ELASTICSEARCH_USERNAME:}' + password: '${ELASTICSEARCH_PASSWORD:}' diff --git a/auditbeat/magefile.go b/auditbeat/magefile.go index d92682d671c..2f090d2eaa5 100644 --- a/auditbeat/magefile.go +++ b/auditbeat/magefile.go @@ -207,6 +207,8 @@ func customizePackaging() { case mage.Deb, mage.RPM, mage.DMG: args.Spec.ReplaceFile("/etc/{{.BeatName}}/{{.BeatName}}.yml", shortConfig) args.Spec.ReplaceFile("/etc/{{.BeatName}}/{{.BeatName}}.reference.yml", referenceConfig) + case mage.Docker: + args.Spec.ExtraVar("user", "root") default: panic(errors.Errorf("unhandled package type: %v", pkgType)) } diff --git a/dev-tools/mage/dockerbuilder.go b/dev-tools/mage/dockerbuilder.go new file mode 100644 index 00000000000..44791010217 --- /dev/null +++ b/dev-tools/mage/dockerbuilder.go @@ -0,0 +1,192 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mage + +import ( + "bytes" + "compress/gzip" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/magefile/mage/sh" + "github.com/pkg/errors" +) + +type dockerBuilder struct { + PackageSpec + + buildDir string + beatDir string +} + +func newDockerBuilder(spec PackageSpec) (*dockerBuilder, error) { + buildDir := filepath.Join(spec.packageDir, "docker-build") + beatDir := filepath.Join(buildDir, "beat") + + return &dockerBuilder{ + PackageSpec: spec, + buildDir: buildDir, + beatDir: beatDir, + }, nil +} + +func (b *dockerBuilder) Build() error { + if err := os.RemoveAll(b.buildDir); err != nil { + return errors.Wrapf(err, "failed to clean existing build directory %s", b.buildDir) + } + + if err := b.copyFiles(); err != nil { + return err + } + + if err := b.prepareBuild(); err != nil { + return errors.Wrap(err, "failed to prepare build") + } + + tag, err := b.dockerBuild() + if err != nil { + return errors.Wrap(err, "failed to build docker") + } + + if err := b.dockerSave(tag); err != nil { + return errors.Wrap(err, "failed to save docker as artifact") + } + + return nil +} + +func (b *dockerBuilder) modulesDirs() []string { + var modulesd []string + for _, f := range b.Files { + if f.Modules { + modulesd = append(modulesd, f.Target) + } + } + return modulesd +} + +func (b *dockerBuilder) exposePorts() []string { + if ports, _ := b.ExtraVars["expose_ports"]; ports != "" { + return strings.Split(ports, ",") + } + return nil +} + +func (b *dockerBuilder) copyFiles() error { + for _, f := range b.Files { + target := filepath.Join(b.beatDir, f.Target) + if err := Copy(f.Source, target); err != nil { + return errors.Wrapf(err, "failed to copy from %s to %s", f.Source, target) + } + } + return nil +} + +func (b *dockerBuilder) prepareBuild() error { + elasticBeatsDir, err := ElasticBeatsDir() + if err != nil { + return err + } + templatesDir := filepath.Join(elasticBeatsDir, "dev-tools/packaging/templates/docker") + + data := map[string]interface{}{ + "ExposePorts": b.exposePorts(), + "ModulesDirs": b.modulesDirs(), + } + + return filepath.Walk(templatesDir, func(path string, info os.FileInfo, _ error) error { + if !info.IsDir() { + target := strings.TrimSuffix( + filepath.Join(b.buildDir, filepath.Base(path)), + ".tmpl", + ) + + err = b.ExpandFile(path, target, data) + if err != nil { + return errors.Wrapf(err, "expanding template '%s' to '%s'", path, target) + } + } + return nil + }) +} + +func (b *dockerBuilder) dockerBuild() (string, error) { + tag := fmt.Sprintf("%s:%s", b.Name, b.Version) + if b.Snapshot { + tag = tag + "-SNAPSHOT" + } + + if repository, _ := b.ExtraVars["repository"]; repository != "" { + tag = fmt.Sprintf("%s/%s", repository, tag) + } + return tag, sh.Run("docker", "build", "-t", tag, b.buildDir) +} + +func (b *dockerBuilder) dockerSave(tag string) error { + // Save the container as artifact + outputFile := b.OutputFile + if outputFile == "" { + outputTar, err := b.Expand(defaultBinaryName + ".docker.tar.gz") + if err != nil { + return err + } + outputFile = filepath.Join(distributionsDir, outputTar) + } + var stderr bytes.Buffer + cmd := exec.Command("docker", "save", tag) + cmd.Stderr = &stderr + stdout, err := cmd.StdoutPipe() + if err != nil { + return err + } + if err = cmd.Start(); err != nil { + return err + } + + err = func() error { + f, err := os.Create(outputFile) + if err != nil { + return err + } + defer f.Close() + + w := gzip.NewWriter(f) + defer w.Close() + + _, err = io.Copy(w, stdout) + if err != nil { + return err + } + return nil + }() + if err != nil { + return err + } + + if err = cmd.Wait(); err != nil { + if errmsg := strings.TrimSpace(stderr.String()); errmsg != "" { + err = errors.Wrap(errors.New(errmsg), err.Error()) + } + return err + } + return errors.Wrap(CreateSHA512File(outputFile), "failed to create .sha512 file") +} diff --git a/dev-tools/mage/pkgtypes.go b/dev-tools/mage/pkgtypes.go index 736ddebc8db..a283828b4fc 100644 --- a/dev-tools/mage/pkgtypes.go +++ b/dev-tools/mage/pkgtypes.go @@ -61,6 +61,7 @@ const ( Zip TarGz DMG + Docker ) // OSPackageArgs define a set of package types to build for an operating @@ -103,6 +104,7 @@ type PackageFile struct { Target string `yaml:"target,omitempty"` // Target location in package. Relative paths are added to a package specific directory (e.g. metricbeat-7.0.0-linux-x86_64). Mode os.FileMode `yaml:"mode,omitempty"` // Target mode for file. Does not apply when source is a directory. Config bool `yaml:"config"` // Mark file as config in the package (deb and rpm only). + Modules bool `yaml:"modules"` // Mark directory as directory with modules. Dep func(PackageSpec) error `yaml:"-" hash:"-" json:"-"` // Dependency to invoke during Evaluate. } @@ -165,6 +167,9 @@ var OSArchNames = map[string]map[PackageType]map[string]string{ "ppc64le": "ppc64le", "s390x": "s390x", }, + Docker: map[string]string{ + "amd64": "amd64", + }, }, } @@ -204,6 +209,8 @@ func (typ PackageType) String() string { return "tar.gz" case DMG: return "dmg" + case Docker: + return "docker" default: return "invalid" } @@ -227,6 +234,8 @@ func (typ *PackageType) UnmarshalText(text []byte) error { *typ = Zip case "dmg": *typ = DMG + case "docker": + *typ = Docker default: return errors.Errorf("unknown package type: %v", string(text)) } @@ -256,6 +265,8 @@ func (typ PackageType) Build(spec PackageSpec) error { return PackageTarGz(spec) case DMG: return PackageDMG(spec) + case Docker: + return PackageDocker(spec) default: return errors.Errorf("unknown package type: %v", typ) } @@ -282,6 +293,14 @@ func (s PackageSpec) ReplaceFile(target string, file PackageFile) { s.Files[target] = file } +// ExtraVar adds or replaces a variable to `extra_vars` in package specs. +func (s *PackageSpec) ExtraVar(key, value string) { + if s.ExtraVars == nil { + s.ExtraVars = make(map[string]string) + } + s.ExtraVars[key] = value +} + // Expand expands a templated string using data from the spec. func (s PackageSpec) Expand(in string, args ...map[string]interface{}) (string, error) { return expandTemplate("inline", in, FuncMap, @@ -823,3 +842,16 @@ func PackageDMG(spec PackageSpec) error { return b.Build() } + +// PackageDocker packages the Beat into a docker image. +func PackageDocker(spec PackageSpec) error { + if err := HaveDocker(); err != nil { + return errors.Errorf("docker daemon required to build images: %s", err) + } + + b, err := newDockerBuilder(spec) + if err != nil { + return err + } + return b.Build() +} diff --git a/dev-tools/packaging/package_test.go b/dev-tools/packaging/package_test.go index a19ec9b6fa2..a2ea573be9f 100644 --- a/dev-tools/packaging/package_test.go +++ b/dev-tools/packaging/package_test.go @@ -25,9 +25,11 @@ import ( "archive/zip" "bytes" "compress/gzip" + "encoding/json" "flag" "fmt" "io" + "io/ioutil" "os" "path/filepath" "regexp" @@ -78,7 +80,8 @@ func TestDeb(t *testing.T) { } func TestTar(t *testing.T) { - tars := getFiles(t, regexp.MustCompile(`\.tar\.gz$`)) + // Regexp matches *-arch.tar.gz, but not *-arch.docker.tar.gz + tars := getFiles(t, regexp.MustCompile(`-\w+\.tar\.gz$`)) for _, tar := range tars { checkTar(t, tar) } @@ -91,6 +94,13 @@ func TestZip(t *testing.T) { } } +func TestDocker(t *testing.T) { + dockers := getFiles(t, regexp.MustCompile(`\.docker\.tar\.gz$`)) + for _, docker := range dockers { + checkDocker(t, docker) + } +} + // Sub-tests func checkRPM(t *testing.T, file string) { @@ -159,6 +169,18 @@ func checkZip(t *testing.T, file string) { checkModulesPermissions(t, p) } +func checkDocker(t *testing.T, file string) { + p, info, err := readDocker(file) + if err != nil { + t.Error(err) + return + } + + checkDockerEntryPoint(t, p, info) + checkModulesPresent(t, "", p) + checkModulesDPresent(t, "", p) +} + // Verify that the main configuration file is installed with a 0600 file mode. func checkConfigPermissions(t *testing.T, p *packageFile) { t.Run(p.Name+" config file permissions", func(t *testing.T) { @@ -311,6 +333,30 @@ func checkSystemdUnitPermissions(t *testing.T, p *packageFile) { }) } +func checkDockerEntryPoint(t *testing.T, p *packageFile, info *dockerInfo) { + expectedMode := os.FileMode(0755) + + t.Run(fmt.Sprintf("%s entrypoint", p.Name), func(t *testing.T) { + if len(info.Config.Entrypoint) == 0 { + t.Fatal("no entrypoint") + } + + entrypoint := info.Config.Entrypoint[0] + if strings.HasPrefix(entrypoint, "/") { + entrypoint := strings.TrimPrefix(entrypoint, "/") + entry, found := p.Contents[entrypoint] + if !found { + t.Fatalf("%s entrypoint not found in docker", entrypoint) + } + if mode := entry.Mode.Perm(); mode != expectedMode { + t.Fatalf("%s entrypoint mode is %s, expected: %s", entrypoint, mode, expectedMode) + } + } else { + t.Fatal("TODO: check if binary is in $PATH") + } + }) +} + // Helpers type packageFile struct { @@ -457,3 +503,128 @@ func readZip(zipFile string) (*packageFile, error) { return p, nil } + +func readDocker(dockerFile string) (*packageFile, *dockerInfo, error) { + file, err := os.Open(dockerFile) + if err != nil { + return nil, nil, err + } + defer file.Close() + + var manifest *dockerManifest + var info *dockerInfo + layers := make(map[string]*packageFile) + + gzipReader, err := gzip.NewReader(file) + if err != nil { + return nil, nil, err + } + defer gzipReader.Close() + + tarReader := tar.NewReader(gzipReader) + for { + header, err := tarReader.Next() + if err != nil { + if err == io.EOF { + break + } + return nil, nil, err + } + + switch { + case header.Name == "manifest.json": + manifest, err = readDockerManifest(tarReader) + if err != nil { + return nil, nil, err + } + case strings.HasSuffix(header.Name, ".json") && header.Name != "manifest.json": + info, err = readDockerInfo(tarReader) + if err != nil { + return nil, nil, err + } + case strings.HasSuffix(header.Name, "/layer.tar"): + layer, err := readTarContents(header.Name, tarReader) + if err != nil { + return nil, nil, err + } + layers[filepath.Dir(header.Name)] = layer + } + } + + if len(info.Config.Entrypoint) == 0 { + return nil, nil, fmt.Errorf("no entrypoint") + } + + workingDir := info.Config.WorkingDir + entrypoint := info.Config.Entrypoint[0] + + // Read layers in order and for each file keep only the entry seen in the later layer + p := &packageFile{Name: filepath.Base(dockerFile), Contents: map[string]packageEntry{}} + for _, layer := range manifest.Layers { + layerID := filepath.Dir(layer) + layerFile, found := layers[layerID] + if !found { + return nil, nil, fmt.Errorf("layer not found: %s", layerID) + } + for name, entry := range layerFile.Contents { + // Check only files in working dir and entrypoint + if strings.HasPrefix("/"+name, workingDir) || "/"+name == entrypoint { + p.Contents[name] = entry + } + } + } + + if len(p.Contents) == 0 { + return nil, nil, fmt.Errorf("no files found in docker working directory (%s)", info.Config.WorkingDir) + } + + return p, info, nil +} + +type dockerManifest struct { + Config string + RepoTags []string + Layers []string +} + +func readDockerManifest(r io.Reader) (*dockerManifest, error) { + data, err := ioutil.ReadAll(r) + if err != nil { + return nil, err + } + + var manifests []*dockerManifest + err = json.Unmarshal(data, &manifests) + if err != nil { + return nil, err + + } + + if len(manifests) != 1 { + return nil, fmt.Errorf("one and only one manifest expected, %d found", len(manifests)) + } + + return manifests[0], nil +} + +type dockerInfo struct { + Config struct { + WorkingDir string + Entrypoint []string + } `json:"config"` +} + +func readDockerInfo(r io.Reader) (*dockerInfo, error) { + data, err := ioutil.ReadAll(r) + if err != nil { + return nil, err + } + + var info dockerInfo + err = json.Unmarshal(data, &info) + if err != nil { + return nil, err + } + + return &info, nil +} diff --git a/dev-tools/packaging/packages.yml b/dev-tools/packaging/packages.yml index 7d70f31e522..6ffbe155718 100644 --- a/dev-tools/packaging/packages.yml +++ b/dev-tools/packaging/packages.yml @@ -157,6 +157,22 @@ shared: template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/windows/uninstall-service.ps1.tmpl' mode: 0755 + - &docker_spec + <<: *binary_spec + extra_vars: + from: 'centos:7' + user: '{{ .BeatName }}' + linux_capabilities: '' + files: + '{{.BeatName}}.yml': + source: '{{.BeatName}}.docker.yml' + mode: 0600 + config: true + + - &elastic_docker_spec + extra_vars: + repository: 'docker.elastic.co/beats' + # # License modifiers for Apache 2.0 # @@ -229,6 +245,11 @@ specs: spec: <<: *deb_rpm_spec + - os: linux + types: [docker] + spec: + <<: *docker_spec + elastic_beat_without_xpack: ### # OSS Packages @@ -268,6 +289,14 @@ specs: <<: *apache_license_for_deb_rpm name: '{{.BeatName}}-oss' + - os: linux + types: [docker] + spec: + <<: *docker_spec + <<: *elastic_docker_spec + <<: *apache_license_for_binaries + name: '{{.BeatName}}-oss' + ### # Elastic Licensed Packages ### @@ -301,6 +330,13 @@ specs: <<: *deb_rpm_spec <<: *elastic_license_for_deb_rpm + - os: linux + types: [docker] + spec: + <<: *docker_spec + <<: *elastic_docker_spec + <<: *elastic_license_for_binaries + # Official Beats elastic_beat: ### @@ -341,6 +377,14 @@ specs: <<: *apache_license_for_deb_rpm name: '{{.BeatName}}-oss' + - os: linux + types: [docker] + spec: + <<: *docker_spec + <<: *elastic_docker_spec + <<: *apache_license_for_binaries + name: '{{.BeatName}}-oss' + ### # Elastic Licensed Packages ### @@ -385,3 +429,13 @@ specs: files: /usr/share/{{.BeatName}}/bin/{{.BeatName}}{{.BinaryExt}}: source: ../x-pack/{{.BeatName}}/build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} + + - os: linux + types: [docker] + spec: + <<: *docker_spec + <<: *elastic_docker_spec + <<: *elastic_license_for_binaries + files: + '{{.BeatName}}{{.BinaryExt}}': + source: ../x-pack/{{.BeatName}}/build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} diff --git a/dev-tools/packaging/templates/docker/Dockerfile.tmpl b/dev-tools/packaging/templates/docker/Dockerfile.tmpl new file mode 100644 index 00000000000..78944029e15 --- /dev/null +++ b/dev-tools/packaging/templates/docker/Dockerfile.tmpl @@ -0,0 +1,51 @@ +{{- $beatHome := printf "%s/%s" "/usr/share" .BeatName }} +{{- $beatBinary := printf "%s/%s" $beatHome .BeatName }} +{{- $repoInfo := repo }} + +FROM {{ .from }} + +LABEL \ + org.label-schema.schema-version="1.0" \ + org.label-schema.vendor="{{ .BeatVendor }}" \ + org.label-schema.name="{{ .BeatName }}" \ + org.label-schema.version="{{ beat_version }}" \ + org.label-schema.url="{{ .BeatURL }}" \ + org.label-schema.vcs-url="{{ $repoInfo.RootImportPath }}" \ + org.label-schema.vcs-ref="{{ commit }}" \ + license="{{ .License }}" \ + description="{{ .BeatDescription }}" + +ENV ELASTIC_CONTAINER "true" +ENV PATH={{ $beatHome }}:$PATH + +COPY beat {{ $beatHome }} +COPY docker-entrypoint /usr/local/bin/docker-entrypoint +RUN chmod 755 /usr/local/bin/docker-entrypoint + +RUN groupadd --gid 1000 {{ .BeatName }} + +RUN mkdir {{ $beatHome }}/data {{ $beatHome }}/logs && \ + chown -R root:{{ .BeatName }} {{ $beatHome }} && \ + find {{ $beatHome }} -type d -exec chmod 0750 {} \; && \ + find {{ $beatHome }} -type f -exec chmod 0640 {} \; && \ + chmod 0750 {{ $beatBinary }} && \ +{{- if .linux_capabilities }} + setcap {{ .linux_capabilities }} {{ $beatBinary }} && \ +{{- end }} +{{- range $i, $modulesd := .ModulesDirs }} + chmod 0770 {{ $beatHome}}/{{ $modulesd }} && \ +{{- end }} + chmod 0770 {{ $beatHome }}/data {{ $beatHome }}/logs + +{{- if ne .user "root" }} +RUN useradd -M --uid 1000 --gid 1000 --home {{ $beatHome }} {{ .user }} +{{- end }} +USER {{ .user }} + +{{- range $i, $port := .ExposePorts }} +EXPOSE {{ $port }} +{{- end }} + +WORKDIR {{ $beatHome }} +ENTRYPOINT ["/usr/local/bin/docker-entrypoint"] +CMD ["-e"] diff --git a/dev-tools/packaging/templates/docker/docker-entrypoint.tmpl b/dev-tools/packaging/templates/docker/docker-entrypoint.tmpl new file mode 100644 index 00000000000..f073e21e318 --- /dev/null +++ b/dev-tools/packaging/templates/docker/docker-entrypoint.tmpl @@ -0,0 +1,25 @@ +#!/bin/bash + +set -euo pipefail + +# Check if the the user has invoked the image with flags. +# eg. "{{ .BeatName }} -c {{ .BeatName }}.yml" +if [[ -z $1 ]] || [[ ${1:0:1} == '-' ]] ; then + exec {{ .BeatName }} "$@" +else + # They may be looking for a Beat subcommand, like "{{ .BeatName }} setup". + subcommands=$({{ .BeatName }} help \ + | awk 'BEGIN {RS=""; FS="\n"} /Available Commands:/' \ + | awk '/^\s+/ {print $1}') + + # If we _did_ get a subcommand, pass it to {{ .BeatName }}. + for subcommand in $subcommands; do + if [[ $1 == $subcommand ]]; then + exec {{ .BeatName }} "$@" + fi + done +fi + +# If neither of those worked, then they have specified the binary they want, so +# just do exactly as they say. +exec "$@" diff --git a/filebeat/_meta/beat.docker.yml b/filebeat/_meta/beat.docker.yml new file mode 100644 index 00000000000..756c2df5217 --- /dev/null +++ b/filebeat/_meta/beat.docker.yml @@ -0,0 +1,5 @@ +filebeat.config: + modules: + path: ${path.config}/modules.d/*.yml + reload.enabled: false + diff --git a/filebeat/filebeat.docker.yml b/filebeat/filebeat.docker.yml new file mode 100644 index 00000000000..99cf52e1cb6 --- /dev/null +++ b/filebeat/filebeat.docker.yml @@ -0,0 +1,12 @@ +filebeat.config: + modules: + path: ${path.config}/modules.d/*.yml + reload.enabled: false + +processors: +- add_cloud_metadata: ~ + +output.elasticsearch: + hosts: '${ELASTICSEARCH_HOSTS:elasticsearch:9200}' + username: '${ELASTICSEARCH_USERNAME:}' + password: '${ELASTICSEARCH_PASSWORD:}' diff --git a/filebeat/magefile.go b/filebeat/magefile.go index 3b42f3d6435..59678382266 100644 --- a/filebeat/magefile.go +++ b/filebeat/magefile.go @@ -173,34 +173,40 @@ func customizePackaging() { modulesDTarget = "modules.d" modulesD = mage.PackageFile{ - Mode: 0644, - Source: "modules.d", - Config: true, + Mode: 0644, + Source: "modules.d", + Config: true, + Modules: true, } modulesDXPack = mage.PackageFile{ - Mode: 0644, - Source: dirModulesDGeneratedXPack, - Config: true, + Mode: 0644, + Source: dirModulesDGeneratedXPack, + Config: true, + Modules: true, } ) for _, args := range mage.Packages { mods := module modsD := modulesD + pkgType := args.Types[0] if args.Spec.License == "Elastic License" { mods = moduleXPack modsD = modulesDXPack replacePackageFileSource(args, map[string]string{ "fields.yml": "../x-pack/{{.BeatName}}/fields.yml", - "{{.BeatName}}.yml": "../x-pack/{{.BeatName}}/{{.BeatName}}.yml", "{{.BeatName}}.reference.yml": "../x-pack/{{.BeatName}}/{{.BeatName}}.reference.yml", "_meta/kibana.generated": "../x-pack/{{.BeatName}}/build/kibana", }) + if pkgType != mage.Docker { + replacePackageFileSource(args, map[string]string{ + "{{.BeatName}}.yml": "../x-pack/{{.BeatName}}/{{.BeatName}}.yml", + }) + } } - pkgType := args.Types[0] switch pkgType { - case mage.TarGz, mage.Zip: + case mage.TarGz, mage.Zip, mage.Docker: args.Spec.Files[moduleTarget] = mods args.Spec.Files[modulesDTarget] = modsD case mage.Deb, mage.RPM: diff --git a/generator/beat/{beat}/_meta/beat.docker.yml b/generator/beat/{beat}/_meta/beat.docker.yml new file mode 100644 index 00000000000..665509ac566 --- /dev/null +++ b/generator/beat/{beat}/_meta/beat.docker.yml @@ -0,0 +1,2 @@ +{beat}: + period: 1s diff --git a/heartbeat/_meta/beat.docker.yml b/heartbeat/_meta/beat.docker.yml new file mode 100644 index 00000000000..496602310d1 --- /dev/null +++ b/heartbeat/_meta/beat.docker.yml @@ -0,0 +1,13 @@ +heartbeat.monitors: +- type: http + schedule: '@every 5s' + urls: + - http://elasticsearch:9200 + - http://kibana:5601 + +- type: icmp + schedule: '@every 5s' + hosts: + - elasticsearch + - kibana + diff --git a/heartbeat/heartbeat.docker.yml b/heartbeat/heartbeat.docker.yml new file mode 100644 index 00000000000..9a344a539e1 --- /dev/null +++ b/heartbeat/heartbeat.docker.yml @@ -0,0 +1,20 @@ +heartbeat.monitors: +- type: http + schedule: '@every 5s' + urls: + - http://elasticsearch:9200 + - http://kibana:5601 + +- type: icmp + schedule: '@every 5s' + hosts: + - elasticsearch + - kibana + +processors: +- add_cloud_metadata: ~ + +output.elasticsearch: + hosts: '${ELASTICSEARCH_HOSTS:elasticsearch:9200}' + username: '${ELASTICSEARCH_USERNAME:}' + password: '${ELASTICSEARCH_PASSWORD:}' diff --git a/heartbeat/magefile.go b/heartbeat/magefile.go index 3c751e3effb..6b5fc419f40 100644 --- a/heartbeat/magefile.go +++ b/heartbeat/magefile.go @@ -80,6 +80,8 @@ func Package() { defer func() { fmt.Println("package ran for", time.Since(start)) }() mage.UseElasticBeatPackaging() + customizePackaging() + mg.Deps(Update) mg.Deps(CrossBuild, CrossBuildXPack, CrossBuildGoDaemon) mg.SerialDeps(mage.Package, TestPackages) @@ -113,3 +115,13 @@ func GoTestUnit(ctx context.Context) error { func GoTestIntegration(ctx context.Context) error { return mage.GoTest(ctx, mage.DefaultGoTestIntegrationArgs()) } + +func customizePackaging() { + for _, args := range mage.Packages { + pkgType := args.Types[0] + switch pkgType { + case mage.Docker: + args.Spec.ExtraVar("linux_capabilities", "cap_net_raw=eip") + } + } +} diff --git a/journalbeat/_meta/beat.docker.yml b/journalbeat/_meta/beat.docker.yml new file mode 100644 index 00000000000..2c87c0346b6 --- /dev/null +++ b/journalbeat/_meta/beat.docker.yml @@ -0,0 +1,4 @@ +journalbeat.inputs: +- paths: [] + seek: tail + diff --git a/journalbeat/journalbeat.docker.yml b/journalbeat/journalbeat.docker.yml new file mode 100644 index 00000000000..aa709469a72 --- /dev/null +++ b/journalbeat/journalbeat.docker.yml @@ -0,0 +1,11 @@ +journalbeat.inputs: +- paths: [] + seek: tail + +processors: +- add_cloud_metadata: ~ + +output.elasticsearch: + hosts: '${ELASTICSEARCH_HOSTS:elasticsearch:9200}' + username: '${ELASTICSEARCH_USERNAME:}' + password: '${ELASTICSEARCH_PASSWORD:}' diff --git a/libbeat/_meta/config.docker.yml b/libbeat/_meta/config.docker.yml new file mode 100644 index 00000000000..241ec990672 --- /dev/null +++ b/libbeat/_meta/config.docker.yml @@ -0,0 +1,7 @@ +processors: +- add_cloud_metadata: ~ + +output.elasticsearch: + hosts: '${ELASTICSEARCH_HOSTS:elasticsearch:9200}' + username: '${ELASTICSEARCH_USERNAME:}' + password: '${ELASTICSEARCH_PASSWORD:}' diff --git a/libbeat/scripts/Makefile b/libbeat/scripts/Makefile index 599c87c171c..2ccc1edaa1f 100755 --- a/libbeat/scripts/Makefile +++ b/libbeat/scripts/Makefile @@ -325,8 +325,10 @@ update: python-env fields collect @mkdir -p _meta @# Update config files. @cat _meta/beat.yml ${ES_BEATS}/libbeat/_meta/config.yml | sed -e "s/beatname/${BEAT_NAME}/g;s/beat-index-prefix/${BEAT_INDEX_PREFIX}/g" > ${BEAT_NAME}.yml - @chmod 0640 ${BEAT_NAME}.yml - +ifneq (,$(wildcard _meta/beat.docker.yml)) + @cat _meta/beat.docker.yml ${ES_BEATS}/libbeat/_meta/config.docker.yml > ${BEAT_NAME}.docker.yml + @chmod 0640 ${BEAT_NAME}.yml ${BEAT_NAME}.docker.yml +endif @# Update reference config files. ifeq ($(BEAT_REF_YAML),true) diff --git a/metricbeat/_meta/beat.docker.yml b/metricbeat/_meta/beat.docker.yml new file mode 100644 index 00000000000..16b19a866dd --- /dev/null +++ b/metricbeat/_meta/beat.docker.yml @@ -0,0 +1,4 @@ +metricbeat.config.modules: + path: ${path.config}/modules.d/*.yml + reload.enabled: false + diff --git a/metricbeat/magefile.go b/metricbeat/magefile.go index 047f97e04bc..882be9a6f7b 100644 --- a/metricbeat/magefile.go +++ b/metricbeat/magefile.go @@ -130,14 +130,16 @@ func customizePackaging() { unixModulesDir = "/etc/{{.BeatName}}/modules.d" modulesDir = mage.PackageFile{ - Mode: 0644, - Source: "modules.d", - Config: true, + Mode: 0644, + Source: "modules.d", + Config: true, + Modules: true, } windowsModulesDir = mage.PackageFile{ - Mode: 0644, - Source: "{{.PackageDir}}/modules.d", - Config: true, + Mode: 0644, + Source: "{{.PackageDir}}/modules.d", + Config: true, + Modules: true, Dep: func(spec mage.PackageSpec) error { if err := mage.Copy("modules.d", spec.MustExpand("{{.PackageDir}}/modules.d")); err != nil { return errors.Wrap(err, "failed to copy modules.d dir") @@ -173,7 +175,7 @@ func customizePackaging() { default: pkgType := args.Types[0] switch pkgType { - case mage.TarGz, mage.Zip: + case mage.TarGz, mage.Zip, mage.Docker: args.Spec.Files[archiveModulesDir] = modulesDir case mage.Deb, mage.RPM, mage.DMG: args.Spec.Files[unixModulesDir] = modulesDir diff --git a/metricbeat/metricbeat.docker.yml b/metricbeat/metricbeat.docker.yml new file mode 100644 index 00000000000..982018eefc7 --- /dev/null +++ b/metricbeat/metricbeat.docker.yml @@ -0,0 +1,11 @@ +metricbeat.config.modules: + path: ${path.config}/modules.d/*.yml + reload.enabled: false + +processors: +- add_cloud_metadata: ~ + +output.elasticsearch: + hosts: '${ELASTICSEARCH_HOSTS:elasticsearch:9200}' + username: '${ELASTICSEARCH_USERNAME:}' + password: '${ELASTICSEARCH_PASSWORD:}' diff --git a/packetbeat/_meta/beat.docker.yml b/packetbeat/_meta/beat.docker.yml new file mode 100644 index 00000000000..4a7b12464ee --- /dev/null +++ b/packetbeat/_meta/beat.docker.yml @@ -0,0 +1,35 @@ +packetbeat.interfaces.device: any + +packetbeat.flows: + timeout: 30s + period: 10s + +packetbeat.protocols.dns: + ports: [53] + include_authorities: true + include_additionals: true + +packetbeat.protocols.http: + ports: [80, 5601, 9200, 8080, 8081, 5000, 8002] + +packetbeat.protocols.memcache: + ports: [11211] + +packetbeat.protocols.mysql: + ports: [3306] + +packetbeat.protocols.pgsql: + ports: [5432] + +packetbeat.protocols.redis: + ports: [6379] + +packetbeat.protocols.thrift: + ports: [9090] + +packetbeat.protocols.mongodb: + ports: [27017] + +packetbeat.protocols.cassandra: + ports: [9042] + diff --git a/packetbeat/magefile.go b/packetbeat/magefile.go index 0ac9bd19b26..4fc507e7555 100644 --- a/packetbeat/magefile.go +++ b/packetbeat/magefile.go @@ -447,5 +447,11 @@ func customizePackaging() { args.Spec.ReplaceFile("{{.BeatName}}.yml", configYml) args.Spec.ReplaceFile("{{.BeatName}}.reference.yml", referenceConfigYml) } + + pkgType := args.Types[0] + switch pkgType { + case mage.Docker: + args.Spec.ExtraVar("linux_capabilities", "cap_net_raw,cap_net_admin=eip") + } } } diff --git a/packetbeat/packetbeat.docker.yml b/packetbeat/packetbeat.docker.yml new file mode 100644 index 00000000000..b475f516f49 --- /dev/null +++ b/packetbeat/packetbeat.docker.yml @@ -0,0 +1,42 @@ +packetbeat.interfaces.device: any + +packetbeat.flows: + timeout: 30s + period: 10s + +packetbeat.protocols.dns: + ports: [53] + include_authorities: true + include_additionals: true + +packetbeat.protocols.http: + ports: [80, 5601, 9200, 8080, 8081, 5000, 8002] + +packetbeat.protocols.memcache: + ports: [11211] + +packetbeat.protocols.mysql: + ports: [3306] + +packetbeat.protocols.pgsql: + ports: [5432] + +packetbeat.protocols.redis: + ports: [6379] + +packetbeat.protocols.thrift: + ports: [9090] + +packetbeat.protocols.mongodb: + ports: [27017] + +packetbeat.protocols.cassandra: + ports: [9042] + +processors: +- add_cloud_metadata: ~ + +output.elasticsearch: + hosts: '${ELASTICSEARCH_HOSTS:elasticsearch:9200}' + username: '${ELASTICSEARCH_USERNAME:}' + password: '${ELASTICSEARCH_PASSWORD:}'