From fe30916bd1092ff3acab6cb8e610eb51197e4807 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Mon, 20 Jul 2020 09:03:06 +0200 Subject: [PATCH] [Ingest Manager] Allow using drop path for agent build (#20019) * use drop path * changelog --- dev-tools/mage/pkg.go | 4 ++ dev-tools/packaging/packages.yml | 32 +++++------ x-pack/elastic-agent/CHANGELOG.asciidoc | 1 + x-pack/elastic-agent/magefile.go | 70 ++++++++++++++++++++----- 4 files changed, 78 insertions(+), 29 deletions(-) diff --git a/dev-tools/mage/pkg.go b/dev-tools/mage/pkg.go index 7a9f4926bb4..4ecdec89d39 100644 --- a/dev-tools/mage/pkg.go +++ b/dev-tools/mage/pkg.go @@ -20,6 +20,7 @@ package mage import ( "fmt" "log" + "os" "runtime" "strconv" @@ -71,6 +72,8 @@ func Package() error { continue } + agentPackageDrop, _ := os.LookupEnv("AGENT_DROP_PATH") + spec := pkg.Spec.Clone() spec.OS = target.GOOS() spec.Arch = packageArch @@ -83,6 +86,7 @@ func Package() error { "AgentArchName": agentPackageArch, "PackageType": pkgType.String(), "BinaryExt": binaryExtension(target.GOOS()), + "AgentDropPath": agentPackageDrop, } spec.packageDir, err = pkgType.PackagingDir(packageStagingDir, target, spec) diff --git a/dev-tools/packaging/packages.yml b/dev-tools/packaging/packages.yml index 8b113e91c4a..531d388bc99 100644 --- a/dev-tools/packaging/packages.yml +++ b/dev-tools/packaging/packages.yml @@ -55,16 +55,16 @@ shared: template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/{{.PackageType}}/init.sh.tmpl' mode: 0755 /var/lib/{{.BeatName}}/downloads/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz: - source: '{{ elastic_beats_dir }}/x-pack/filebeat/build/distributions/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz' + source: '{{.AgentDropPath}}/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz' mode: 0644 /var/lib/{{.BeatName}}/downloads/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz: - source: '{{ elastic_beats_dir }}/x-pack/metricbeat/build/distributions/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz' + source: '{{.AgentDropPath}}/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz' mode: 0644 /var/lib/{{.BeatName}}/downloads/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512: - source: '{{ elastic_beats_dir }}/x-pack/filebeat/build/distributions/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512' + source: '{{.AgentDropPath}}/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512' mode: 0644 /var/lib/{{.BeatName}}/downloads/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512: - source: '{{ elastic_beats_dir }}/x-pack/metricbeat/build/distributions/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512' + source: '{{.AgentDropPath}}/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512' mode: 0644 @@ -104,16 +104,16 @@ shared: mode: 0600 config: true /etc/{{.BeatName}}/data/downloads/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz: - source: '{{ elastic_beats_dir }}/x-pack/filebeat/build/distributions/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz' + source: '{{.AgentDropPath}}/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz' mode: 0644 /etc/{{.BeatName}}/data/downloads/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz: - source: '{{ elastic_beats_dir }}/x-pack/metricbeat/build/distributions/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz' + source: '{{.AgentDropPath}}/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz' mode: 0644 /etc/{{.BeatName}}/data/downloads/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512: - source: '{{ elastic_beats_dir }}/x-pack/filebeat/build/distributions/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512' + source: '{{.AgentDropPath}}/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512' mode: 0644 /etc/{{.BeatName}}/data/downloads/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512: - source: '{{ elastic_beats_dir }}/x-pack/metricbeat/build/distributions/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512' + source: '{{.AgentDropPath}}/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512' mode: 0644 - &agent_binary_files @@ -144,17 +144,17 @@ shared: files: <<: *agent_binary_files 'data/downloads/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz': - source: '{{ elastic_beats_dir }}/x-pack/filebeat/build/distributions/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz' + source: '{{.AgentDropPath}}/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz' mode: 0644 'data/downloads/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz': - source: '{{ elastic_beats_dir }}/x-pack/metricbeat/build/distributions/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz' + source: '{{.AgentDropPath}}/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz' mode: 0644 <<: *agent_binary_files 'data/downloads/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512': - source: '{{ elastic_beats_dir }}/x-pack/filebeat/build/distributions/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512' + source: '{{.AgentDropPath}}/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512' mode: 0644 'data/downloads/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512': - source: '{{ elastic_beats_dir }}/x-pack/metricbeat/build/distributions/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512' + source: '{{.AgentDropPath}}/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512' mode: 0644 # Binary package spec (zip for windows) for community beats. @@ -169,16 +169,16 @@ shared: template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/windows/uninstall-service.ps1.tmpl' mode: 0755 'data/downloads/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip': - source: '{{ elastic_beats_dir }}/x-pack/filebeat/build/distributions/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip' + source: '{{.AgentDropPath}}/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip' mode: 0644 'data/downloads/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip': - source: '{{ elastic_beats_dir }}/x-pack/metricbeat/build/distributions/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip' + source: '{{.AgentDropPath}}/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip' mode: 0644 'data/downloads/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.sha512': - source: '{{ elastic_beats_dir }}/x-pack/filebeat/build/distributions/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.sha512' + source: '{{.AgentDropPath}}/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.sha512' mode: 0644 'data/downloads/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.sha512': - source: '{{ elastic_beats_dir }}/x-pack/metricbeat/build/distributions/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.sha512' + source: '{{.AgentDropPath}}/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.sha512' mode: 0644 - &agent_docker_spec diff --git a/x-pack/elastic-agent/CHANGELOG.asciidoc b/x-pack/elastic-agent/CHANGELOG.asciidoc index ba8edd9f49f..9b3ab9b2ac3 100644 --- a/x-pack/elastic-agent/CHANGELOG.asciidoc +++ b/x-pack/elastic-agent/CHANGELOG.asciidoc @@ -89,4 +89,5 @@ - Agent now sends its own logs to elasticsearch {pull}19811[19811] - Add --insecure option to enroll command {pull}19900[19900] - Will retry to enroll if the server return a 429. {pull}19918[19811] +- Allow to specify what artifacts to embed at build times {pull}20019[20019] - Add --staging option to enroll command {pull}20026[20026] diff --git a/x-pack/elastic-agent/magefile.go b/x-pack/elastic-agent/magefile.go index 613490b1241..e17f8a68eab 100644 --- a/x-pack/elastic-agent/magefile.go +++ b/x-pack/elastic-agent/magefile.go @@ -40,6 +40,7 @@ const ( metaDir = "_meta" snapshotEnv = "SNAPSHOT" configFile = "elastic-agent.yml" + agentDropPath = "AGENT_DROP_PATH" ) // Aliases for commands required by master makefile @@ -495,26 +496,52 @@ func packageAgent(requiredPackages []string, packagingFn func()) { version = release.Version() } - packedBeats := []string{"filebeat", "metricbeat"} - - for _, b := range packedBeats { - pwd, err := filepath.Abs(filepath.Join("..", b)) + // build deps only when drop is not provided + if dropPathEnv, found := os.LookupEnv(agentDropPath); !found || len(dropPathEnv) == 0 { + // prepare new drop + dropPath := filepath.Join("build", "distributions", "elastic-agent-drop") + dropPath, err := filepath.Abs(dropPath) if err != nil { panic(err) } - if requiredPackagesPresent(pwd, b, version, requiredPackages) { - continue + if err := os.MkdirAll(dropPath, 0755); err != nil { + panic(err) } - cmd := exec.Command("mage", "package") - cmd.Dir = pwd - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - cmd.Env = append(os.Environ(), fmt.Sprintf("PWD=%s", pwd), "AGENT_PACKAGING=on") + os.Setenv(agentDropPath, dropPath) - if err := cmd.Run(); err != nil { - panic(err) + // cleanup after build + defer os.RemoveAll(dropPath) + defer os.Unsetenv(agentDropPath) + + packedBeats := []string{"filebeat", "metricbeat"} + + for _, b := range packedBeats { + pwd, err := filepath.Abs(filepath.Join("..", b)) + if err != nil { + panic(err) + } + + if requiredPackagesPresent(pwd, b, version, requiredPackages) { + continue + } + + cmd := exec.Command("mage", "package") + cmd.Dir = pwd + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Env = append(os.Environ(), fmt.Sprintf("PWD=%s", pwd), "AGENT_PACKAGING=on") + + if err := cmd.Run(); err != nil { + panic(err) + } + + // copy to new drop + sourcePath := filepath.Join(pwd, "build", "distributions") + if err := copyAll(sourcePath, dropPath); err != nil { + panic(err) + } } } @@ -526,6 +553,23 @@ func packageAgent(requiredPackages []string, packagingFn func()) { mg.SerialDeps(devtools.Package) } +func copyAll(from, to string) error { + return filepath.Walk(from, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if info.IsDir() { + return nil + } + + targetFile := filepath.Join(to, info.Name()) + + // overwrites with current build + return sh.Copy(targetFile, path) + }) +} + func dockerTag() string { const commitLen = 7 tagBase := "elastic-agent"