Cherry-pick elastic#19480 to 7.x: Agent includes pgp file (elastic#21222

)

Cherry-pick elastic#19480 to 7.x:  Agent includes pgp file  (elastic#21222)

dev-tools/mage/crossbuild.go

@@ -267,6 +267,7 @@ func (b GolangCrossBuilder) Build() error {
 		"--env", "MAGEFILE_VERBOSE="+verbose,
 		"--env", "MAGEFILE_TIMEOUT="+EnvOr("MAGEFILE_TIMEOUT", ""),
 		"--env", fmt.Sprintf("SNAPSHOT=%v", Snapshot),
+		"--env", fmt.Sprintf("DEV=%v", DevBuild),
 		"-v", repoInfo.RootDir+":"+mountPoint,
 		"-w", workDir,
 		image,

dev-tools/mage/settings.go

@@ -77,6 +77,7 @@ var (
 	BeatProjectType ProjectType
 
 	Snapshot bool
+	DevBuild bool
 
 	versionQualified bool
 	versionQualifier string
@@ -117,6 +118,11 @@ func init() {
 		panic(errors.Wrap(err, "failed to parse SNAPSHOT env value"))
 	}
 
+	DevBuild, err = strconv.ParseBool(EnvOr("DEV", "false"))
+	if err != nil {
+		panic(errors.Wrap(err, "failed to parse DEV env value"))
+	}
+
 	versionQualifier, versionQualified = os.LookupEnv("VERSION_QUALIFIER")
 }
 
@@ -165,6 +171,7 @@ func varMap(args ...map[string]interface{}) map[string]interface{} {
 		"BeatURL":         BeatURL,
 		"BeatUser":        BeatUser,
 		"Snapshot":        Snapshot,
+		"DEV":             DevBuild,
 		"Qualifier":       versionQualifier,
 	}
 

x-pack/elastic-agent/CHANGELOG.next.asciidoc

@@ -10,6 +10,8 @@
 - Docker container is not run as root by default. {pull}21213[21213]
 
 ==== Bugfixes
+- Copy Action store on upgrade {pull}21298[21298]
+- Include inputs in action store actions {pull}21298[21298]
 
 ==== New features
 

x-pack/elastic-agent/GPG-KEY-elasticsearch

@@ -0,0 +1,31 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: GnuPG v2.0.14 (GNU/Linux)
+
+mQENBFI3HsoBCADXDtbNJnxbPqB1vDNtCsqhe49vFYsZN9IOZsZXgp7aHjh6CJBD
+A+bGFOwyhbd7at35jQjWAw1O3cfYsKAmFy+Ar3LHCMkV3oZspJACTIgCrwnkic/9
+CUliQe324qvObU2QRtP4Fl0zWcfb/S8UYzWXWIFuJqMvE9MaRY1bwUBvzoqavLGZ
+j3SF1SPO+TB5QrHkrQHBsmX+Jda6d4Ylt8/t6CvMwgQNlrlzIO9WT+YN6zS+sqHd
+1YK/aY5qhoLNhp9G/HxhcSVCkLq8SStj1ZZ1S9juBPoXV1ZWNbxFNGwOh/NYGldD
+2kmBf3YgCqeLzHahsAEpvAm8TBa7Q9W21C8vABEBAAG0RUVsYXN0aWNzZWFyY2gg
+KEVsYXN0aWNzZWFyY2ggU2lnbmluZyBLZXkpIDxkZXZfb3BzQGVsYXN0aWNzZWFy
+Y2gub3JnPokBOAQTAQIAIgUCUjceygIbAwYLCQgHAwIGFQgCCQoLBBYCAwECHgEC
+F4AACgkQ0n1mbNiOQrRzjAgAlTUQ1mgo3nK6BGXbj4XAJvuZDG0HILiUt+pPnz75
+nsf0NWhqR4yGFlmpuctgCmTD+HzYtV9fp9qW/bwVuJCNtKXk3sdzYABY+Yl0Cez/
+7C2GuGCOlbn0luCNT9BxJnh4mC9h/cKI3y5jvZ7wavwe41teqG14V+EoFSn3NPKm
+TxcDTFrV7SmVPxCBcQze00cJhprKxkuZMPPVqpBS+JfDQtzUQD/LSFfhHj9eD+Xe
+8d7sw+XvxB2aN4gnTlRzjL1nTRp0h2/IOGkqYfIG9rWmSLNlxhB2t+c0RsjdGM4/
+eRlPWylFbVMc5pmDpItrkWSnzBfkmXL3vO2X3WvwmSFiQbkBDQRSNx7KAQgA5JUl
+zcMW5/cuyZR8alSacKqhSbvoSqqbzHKcUQZmlzNMKGTABFG1yRx9r+wa/fvqP6OT
+RzRDvVS/cycws8YX7Ddum7x8uI95b9ye1/Xy5noPEm8cD+hplnpU+PBQZJ5XJ2I+
+1l9Nixx47wPGXeClLqcdn0ayd+v+Rwf3/XUJrvccG2YZUiQ4jWZkoxsA07xx7Bj+
+Lt8/FKG7sHRFvePFU0ZS6JFx9GJqjSBbHRRkam+4emW3uWgVfZxuwcUCn1ayNgRt
+KiFv9jQrg2TIWEvzYx9tywTCxc+FFMWAlbCzi+m4WD+QUWWfDQ009U/WM0ks0Kww
+EwSk/UDuToxGnKU2dQARAQABiQEfBBgBAgAJBQJSNx7KAhsMAAoJENJ9ZmzYjkK0
+c3MIAIE9hAR20mqJWLcsxLtrRs6uNF1VrpB+4n/55QU7oxA1iVBO6IFu4qgsF12J
+TavnJ5MLaETlggXY+zDef9syTPXoQctpzcaNVDmedwo1SiL03uMoblOvWpMR/Y0j
+6rm7IgrMWUDXDPvoPGjMl2q1iTeyHkMZEyUJ8SKsaHh4jV9wp9KmC8C+9CwMukL7
+vM5w8cgvJoAwsp3Fn59AxWthN3XJYcnMfStkIuWgR7U2r+a210W6vnUxU4oN0PmM
+cursYPyeV0NX/KQeUeNMwGTFB6QHS/anRaGQewijkrYYoTNtfllxIu9XYmiBERQ/
+qPDlGRlOgVTd9xUfHFkzB52c70E=
+=92oX
+-----END PGP PUBLIC KEY BLOCK-----

x-pack/elastic-agent/magefile.go

@@ -14,6 +14,7 @@ import (
 	"os/exec"
 	"path/filepath"
 	"runtime"
+	"strconv"
 	"strings"
 	"time"
 
@@ -40,6 +41,7 @@ const (
 	buildDir       = "build"
 	metaDir        = "_meta"
 	snapshotEnv    = "SNAPSHOT"
+	devEnv         = "DEV"
 	configFile     = "elastic-agent.yml"
 	agentDropPath  = "AGENT_DROP_PATH"
 )
@@ -345,7 +347,7 @@ func commitID() string {
 
 // Update is an alias for executing control protocol, configs, and specs.
 func Update() {
-	mg.SerialDeps(Config, BuildSpec, BuildFleetCfg)
+	mg.SerialDeps(Config, BuildSpec, BuildPGP, BuildFleetCfg)
 }
 
 // CrossBuild cross-builds the beat for all target platforms.
@@ -370,7 +372,7 @@ func ControlProto() error {
 
 // BuildSpec make sure that all the suppported program spec are built into the binary.
 func BuildSpec() error {
-	// go run x-pack/agent/dev-tools/cmd/buildspec/buildspec.go --in x-pack/agent/spec/*.yml --out x-pack/agent/pkg/agent/program/supported.go
+	// go run x-pack/elastic-agent/dev-tools/cmd/buildspec/buildspec.go --in x-pack/agent/spec/*.yml --out x-pack/elastic-agent/pkg/agent/program/supported.go
 	goF := filepath.Join("dev-tools", "cmd", "buildspec", "buildspec.go")
 	in := filepath.Join("spec", "*.yml")
 	out := filepath.Join("pkg", "agent", "program", "supported.go")
@@ -379,6 +381,16 @@ func BuildSpec() error {
 	return RunGo("run", goF, "--in", in, "--out", out)
 }
 
+func BuildPGP() error {
+	// go run x-pack/elastic-agent/dev-tools/cmd/buildpgp/build_pgp.go --in x-pack/agent/spec/GPG-KEY-elasticsearch --out x-pack/elastic-agent/pkg/release/pgp.go
+	goF := filepath.Join("dev-tools", "cmd", "buildpgp", "build_pgp.go")
+	in := "GPG-KEY-elasticsearch"
+	out := filepath.Join("pkg", "release", "pgp.go")
+
+	fmt.Printf(">> BuildPGP from %s to %s\n", in, out)
+	return RunGo("run", goF, "--in", in, "--out", out)
+}
+
 func configYML() error {
 	return devtools.Config(devtools.AllConfigTypes, ConfigFileParams(), ".")
 }
@@ -595,6 +607,12 @@ func buildVars() map[string]string {
 	isSnapshot, _ := os.LookupEnv(snapshotEnv)
 	vars["github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release.snapshot"] = isSnapshot
 
+	if isDevFlag, devFound := os.LookupEnv(devEnv); devFound {
+		if isDev, err := strconv.ParseBool(isDevFlag); err == nil && isDev {
+			vars["github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release.allowEmptyPgp"] = "true"
+		}
+	}
+
 	return vars
 }
 

x-pack/elastic-agent/pkg/agent/application/config.go

@@ -11,6 +11,7 @@ import (
 
 	"gopkg.in/yaml.v2"
 
+	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/configuration"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config"
@@ -54,7 +55,10 @@ func LoadConfigFromFile(path string) (*config.Config, error) {
 //
 // This must be used to load the Agent configuration, so that variables defined in the inputs are not
 // parsed by go-ucfg. Variables from the inputs should be parsed by the transpiler.
-func LoadConfig(m map[string]interface{}) (*config.Config, error) {
+func LoadConfig(in map[string]interface{}) (*config.Config, error) {
+	// make copy of a map so we dont affect a caller
+	m := common.MapStr(in).Clone()
+
 	inputs, ok := m["inputs"]
 	if ok {
 		// remove the inputs

x-pack/elastic-agent/pkg/agent/application/stream.go

@@ -19,6 +19,7 @@ import (
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/monitoring"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/server"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/state"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release"
 )
 
 type operatorStream struct {
@@ -56,7 +57,8 @@ func streamFactory(ctx context.Context, cfg *configuration.SettingsConfig, srv *
 
 func newOperator(ctx context.Context, log *logger.Logger, id routingKey, config *configuration.SettingsConfig, srv *server.Server, r state.Reporter, m monitoring.Monitor) (*operation.Operator, error) {
 	fetcher := downloader.NewDownloader(log, config.DownloadConfig)
-	verifier, err := downloader.NewVerifier(log, config.DownloadConfig)
+	allowEmptyPgp, pgp := release.PGP()
+	verifier, err := downloader.NewVerifier(log, config.DownloadConfig, allowEmptyPgp, pgp)
 	if err != nil {
 		return nil, errors.New(err, "initiating verifier")
 	}

x-pack/elastic-agent/pkg/agent/application/upgrade/step_download.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors"
 	downloader "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/artifact/download/localremote"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release"
 )
 
 func (u *Upgrader) downloadArtifact(ctx context.Context, version, sourceURI string) (string, error) {
@@ -18,7 +19,8 @@ func (u *Upgrader) downloadArtifact(ctx context.Context, version, sourceURI stri
 		settings.SourceURI = sourceURI
 	}
 
-	verifier, err := downloader.NewVerifier(u.log, &settings)
+	allowEmptyPgp, pgp := release.PGP()
+	verifier, err := downloader.NewVerifier(u.log, &settings, allowEmptyPgp, pgp)
 	if err != nil {
 		return "", errors.New(err, "initiating verifier")
 	}

x-pack/elastic-agent/pkg/agent/application/upgrade/upgrade.go

@@ -14,6 +14,7 @@ import (
 
 	"gopkg.in/yaml.v2"
 
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/info"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/paths"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/artifact"
@@ -78,6 +79,10 @@ func (u *Upgrader) Upgrade(ctx context.Context, a *fleetapi.ActionUpgrade) error
 		return errors.New("upgrading to same version")
 	}
 
+	if err := copyActionStore(newHash); err != nil {
+		return errors.New(err, "failed to copy action store")
+	}
+
 	if err := u.changeSymlink(ctx, newHash); err != nil {
 		rollbackInstall(newHash)
 		return err
@@ -137,3 +142,21 @@ func isSubdir(base, target string) (bool, error) {
 func rollbackInstall(hash string) {
 	os.RemoveAll(filepath.Join(paths.Data(), fmt.Sprintf("%s-%s", agentName, hash)))
 }
+
+func copyActionStore(newHash string) error {
+	currentActionStorePath := info.AgentActionStoreFile()
+
+	newHome := filepath.Join(filepath.Dir(paths.Home()), fmt.Sprintf("%s-%s", agentName, newHash))
+	newActionStorePath := filepath.Join(newHome, filepath.Base(currentActionStorePath))
+
+	currentActionStore, err := ioutil.ReadFile(currentActionStorePath)
+	if os.IsNotExist(err) {
+		// nothing to copy
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+
+	return ioutil.WriteFile(newActionStorePath, currentActionStore, 0600)
+}

x-pack/elastic-agent/pkg/artifact/config.go

@@ -30,10 +30,6 @@ type Config struct {
 	// Timeout: timeout for downloading package
 	Timeout time.Duration `json:"timeout" config:"timeout"`
 
-	// PgpFile: filepath to a public key used for verifying downloaded artifacts
-	// if not file is present elastic-agent will try to load public key from elastic.co website.
-	PgpFile string `json:"pgpfile" config:"pgpfile"`
-
 	// InstallPath: path to the directory containing installed packages
 	InstallPath string `yaml:"installPath" config:"install_path"`
 
@@ -48,12 +44,10 @@ type Config struct {
 // DefaultConfig creates a config with pre-set default values.
 func DefaultConfig() *Config {
 	homePath := paths.Home()
-	dataPath := paths.Data()
 	return &Config{
 		SourceURI:       "https://artifacts.elastic.co/downloads/",
 		TargetDirectory: filepath.Join(homePath, "downloads"),
 		Timeout:         30 * time.Second,
-		PgpFile:         filepath.Join(dataPath, "elastic.pgp"),
 		InstallPath:     filepath.Join(homePath, "install"),
 	}
 }