Cherry-pick #8898 to 6.x: Build docker images when building packages

CHANGELOG-developer.asciidoc

@@ -61,3 +61,4 @@ The list below covers the major changes between 6.3.0 and master only.
 - Add `mage.KibanaDashboards` for collecting Kibana dashboards and generating index patterns. {pull}8615[8615]
 - Allow to disable config resolver using the `Settings.DisableConfigResolver` field when initializing libbeat. {pull}8769[8769]
 - Add `mage.AddPlatforms` to allow to specify dependent platforms when building a beat. {pull}8889[8889]
+- Add docker image building to `mage.Package`. {pull}8898[8898]

auditbeat/_meta/beat.docker.yml

@@ -0,0 +1,14 @@
+auditbeat.modules:
+
+- module: auditd
+  audit_rules: |
+    -w /etc/passwd -p wa -k identity
+    -a always,exit -F arch=b32 -S open,creat,truncate,ftruncate,openat,open_by_handle_at -F exit=-EPERM -k access
+
+- module: file_integrity
+  paths:
+    - /bin
+    - /usr/bin
+    - /sbin
+    - /usr/sbin
+    - /etc

auditbeat/auditbeat.docker.yml

@@ -0,0 +1,21 @@
+auditbeat.modules:
+
+- module: auditd
+  audit_rules: |
+    -w /etc/passwd -p wa -k identity
+    -a always,exit -F arch=b32 -S open,creat,truncate,ftruncate,openat,open_by_handle_at -F exit=-EPERM -k access
+
+- module: file_integrity
+  paths:
+    - /bin
+    - /usr/bin
+    - /sbin
+    - /usr/sbin
+    - /etc
+processors:
+- add_cloud_metadata: ~
+
+output.elasticsearch:
+  hosts: '${ELASTICSEARCH_HOSTS:elasticsearch:9200}'
+  username: '${ELASTICSEARCH_USERNAME:}'
+  password: '${ELASTICSEARCH_PASSWORD:}'

auditbeat/magefile.go

@@ -207,6 +207,8 @@ func customizePackaging() {
 		case mage.Deb, mage.RPM, mage.DMG:
 			args.Spec.ReplaceFile("/etc/{{.BeatName}}/{{.BeatName}}.yml", shortConfig)
 			args.Spec.ReplaceFile("/etc/{{.BeatName}}/{{.BeatName}}.reference.yml", referenceConfig)
+		case mage.Docker:
+			args.Spec.ExtraVar("user", "root")
 		default:
 			panic(errors.Errorf("unhandled package type: %v", pkgType))
 		}

dev-tools/mage/dockerbuilder.go

@@ -0,0 +1,192 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package mage
+
+import (
+	"bytes"
+	"compress/gzip"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"github.com/magefile/mage/sh"
+	"github.com/pkg/errors"
+)
+
+type dockerBuilder struct {
+	PackageSpec
+
+	buildDir string
+	beatDir  string
+}
+
+func newDockerBuilder(spec PackageSpec) (*dockerBuilder, error) {
+	buildDir := filepath.Join(spec.packageDir, "docker-build")
+	beatDir := filepath.Join(buildDir, "beat")
+
+	return &dockerBuilder{
+		PackageSpec: spec,
+		buildDir:    buildDir,
+		beatDir:     beatDir,
+	}, nil
+}
+
+func (b *dockerBuilder) Build() error {
+	if err := os.RemoveAll(b.buildDir); err != nil {
+		return errors.Wrapf(err, "failed to clean existing build directory %s", b.buildDir)
+	}
+
+	if err := b.copyFiles(); err != nil {
+		return err
+	}
+
+	if err := b.prepareBuild(); err != nil {
+		return errors.Wrap(err, "failed to prepare build")
+	}
+
+	tag, err := b.dockerBuild()
+	if err != nil {
+		return errors.Wrap(err, "failed to build docker")
+	}
+
+	if err := b.dockerSave(tag); err != nil {
+		return errors.Wrap(err, "failed to save docker as artifact")
+	}
+
+	return nil
+}
+
+func (b *dockerBuilder) modulesDirs() []string {
+	var modulesd []string
+	for _, f := range b.Files {
+		if f.Modules {
+			modulesd = append(modulesd, f.Target)
+		}
+	}
+	return modulesd
+}
+
+func (b *dockerBuilder) exposePorts() []string {
+	if ports, _ := b.ExtraVars["expose_ports"]; ports != "" {
+		return strings.Split(ports, ",")
+	}
+	return nil
+}
+
+func (b *dockerBuilder) copyFiles() error {
+	for _, f := range b.Files {
+		target := filepath.Join(b.beatDir, f.Target)
+		if err := Copy(f.Source, target); err != nil {
+			return errors.Wrapf(err, "failed to copy from %s to %s", f.Source, target)
+		}
+	}
+	return nil
+}
+
+func (b *dockerBuilder) prepareBuild() error {
+	elasticBeatsDir, err := ElasticBeatsDir()
+	if err != nil {
+		return err
+	}
+	templatesDir := filepath.Join(elasticBeatsDir, "dev-tools/packaging/templates/docker")
+
+	data := map[string]interface{}{
+		"ExposePorts": b.exposePorts(),
+		"ModulesDirs": b.modulesDirs(),
+	}
+
+	return filepath.Walk(templatesDir, func(path string, info os.FileInfo, _ error) error {
+		if !info.IsDir() {
+			target := strings.TrimSuffix(
+				filepath.Join(b.buildDir, filepath.Base(path)),
+				".tmpl",
+			)
+
+			err = b.ExpandFile(path, target, data)
+			if err != nil {
+				return errors.Wrapf(err, "expanding template '%s' to '%s'", path, target)
+			}
+		}
+		return nil
+	})
+}
+
+func (b *dockerBuilder) dockerBuild() (string, error) {
+	tag := fmt.Sprintf("%s:%s", b.Name, b.Version)
+	if b.Snapshot {
+		tag = tag + "-SNAPSHOT"
+	}
+
+	if repository, _ := b.ExtraVars["repository"]; repository != "" {
+		tag = fmt.Sprintf("%s/%s", repository, tag)
+	}
+	return tag, sh.Run("docker", "build", "-t", tag, b.buildDir)
+}
+
+func (b *dockerBuilder) dockerSave(tag string) error {
+	// Save the container as artifact
+	outputFile := b.OutputFile
+	if outputFile == "" {
+		outputTar, err := b.Expand(defaultBinaryName + ".docker.tar.gz")
+		if err != nil {
+			return err
+		}
+		outputFile = filepath.Join(distributionsDir, outputTar)
+	}
+	var stderr bytes.Buffer
+	cmd := exec.Command("docker", "save", tag)
+	cmd.Stderr = &stderr
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return err
+	}
+	if err = cmd.Start(); err != nil {
+		return err
+	}
+
+	err = func() error {
+		f, err := os.Create(outputFile)
+		if err != nil {
+			return err
+		}
+		defer f.Close()
+
+		w := gzip.NewWriter(f)
+		defer w.Close()
+
+		_, err = io.Copy(w, stdout)
+		if err != nil {
+			return err
+		}
+		return nil
+	}()
+	if err != nil {
+		return err
+	}
+
+	if err = cmd.Wait(); err != nil {
+		if errmsg := strings.TrimSpace(stderr.String()); errmsg != "" {
+			err = errors.Wrap(errors.New(errmsg), err.Error())
+		}
+		return err
+	}
+	return errors.Wrap(CreateSHA512File(outputFile), "failed to create .sha512 file")
+}

dev-tools/mage/pkgtypes.go

@@ -61,6 +61,7 @@ const (
 	Zip
 	TarGz
 	DMG
+	Docker
 )
 
 // OSPackageArgs define a set of package types to build for an operating
@@ -103,6 +104,7 @@ type PackageFile struct {
 	Target   string                  `yaml:"target,omitempty"`    // Target location in package. Relative paths are added to a package specific directory (e.g. metricbeat-7.0.0-linux-x86_64).
 	Mode     os.FileMode             `yaml:"mode,omitempty"`      // Target mode for file. Does not apply when source is a directory.
 	Config   bool                    `yaml:"config"`              // Mark file as config in the package (deb and rpm only).
+	Modules  bool                    `yaml:"modules"`             // Mark directory as directory with modules.
 	Dep      func(PackageSpec) error `yaml:"-" hash:"-" json:"-"` // Dependency to invoke during Evaluate.
 }
 
@@ -165,6 +167,9 @@ var OSArchNames = map[string]map[PackageType]map[string]string{
 			"ppc64le":  "ppc64le",
 			"s390x":    "s390x",
 		},
+		Docker: map[string]string{
+			"amd64": "amd64",
+		},
 	},
 }
 
@@ -204,6 +209,8 @@ func (typ PackageType) String() string {
 		return "tar.gz"
 	case DMG:
 		return "dmg"
+	case Docker:
+		return "docker"
 	default:
 		return "invalid"
 	}
@@ -227,6 +234,8 @@ func (typ *PackageType) UnmarshalText(text []byte) error {
 		*typ = Zip
 	case "dmg":
 		*typ = DMG
+	case "docker":
+		*typ = Docker
 	default:
 		return errors.Errorf("unknown package type: %v", string(text))
 	}
@@ -256,6 +265,8 @@ func (typ PackageType) Build(spec PackageSpec) error {
 		return PackageTarGz(spec)
 	case DMG:
 		return PackageDMG(spec)
+	case Docker:
+		return PackageDocker(spec)
 	default:
 		return errors.Errorf("unknown package type: %v", typ)
 	}
@@ -282,6 +293,14 @@ func (s PackageSpec) ReplaceFile(target string, file PackageFile) {
 	s.Files[target] = file
 }
 
+// ExtraVar adds or replaces a variable to `extra_vars` in package specs.
+func (s *PackageSpec) ExtraVar(key, value string) {
+	if s.ExtraVars == nil {
+		s.ExtraVars = make(map[string]string)
+	}
+	s.ExtraVars[key] = value
+}
+
 // Expand expands a templated string using data from the spec.
 func (s PackageSpec) Expand(in string, args ...map[string]interface{}) (string, error) {
 	return expandTemplate("inline", in, FuncMap,
@@ -823,3 +842,16 @@ func PackageDMG(spec PackageSpec) error {
 
 	return b.Build()
 }
+
+// PackageDocker packages the Beat into a docker image.
+func PackageDocker(spec PackageSpec) error {
+	if err := HaveDocker(); err != nil {
+		return errors.Errorf("docker daemon required to build images: %s", err)
+	}
+
+	b, err := newDockerBuilder(spec)
+	if err != nil {
+		return err
+	}
+	return b.Build()
+}