Test preconfigured agent policies and output (#1680)

(cherry picked from commit 56ce765)

.ci/.e2e-tests-for-elastic-agent.yaml

@@ -39,3 +39,6 @@ SUITES:
       - name: "Beats Background Processes"
         pullRequestFilter: " && ~debian"
         tags: "running_on_beats"
+      - name: "Preconfigured Policies"
+        pullRequestFilter: " && ~debian"
+        tags: "preconfigured_policies"

.ci/.e2e-tests.yaml

@@ -54,6 +54,9 @@ SUITES:
       - name: "Beats Background Processes"
         pullRequestFilter: " && ~debian"
         tags: "running_on_beats"
+      - name: "Preconfigured Policies"
+        pullRequestFilter: " && ~debian"
+        tags: "preconfigured_policies"
   - suite: "kubernetes-autodiscover"
     platforms:
       - labels: "ubuntu-18.04"

cli/config/compose/profiles/fleet/preconfigured-policies/kibana.config.yml

@@ -4,7 +4,7 @@ server.host: "0.0.0.0"
 
 telemetry.enabled: false
 
-elasticsearch.hosts: [ "http://elasticsearch:9200" ]
+elasticsearch.hosts: ["http://elasticsearch:9200"]
 elasticsearch.username: elastic
 elasticsearch.password: changeme
 xpack.monitoring.ui.container.elasticsearch.enabled: true
@@ -13,8 +13,22 @@ xpack.fleet.enabled: true
 xpack.fleet.registryUrl: https://epr-staging.elastic.co
 xpack.fleet.agents.enabled: true
 xpack.fleet.agents.elasticsearch.host: http://elasticsearch:9200
-xpack.fleet.agents.fleet_server.hosts:
-  ["http://fleet-server:8220"]
+xpack.fleet.agents.fleet_server.hosts: ["http://fleet-server:8220"]
 
 xpack.encryptedSavedObjects.encryptionKey: "12345678901234567890123456789012"
 xpack.fleet.agents.tlsCheckDisabled: true
+xpack.fleet.outputs:
+  - name: "Test output"
+    type: "elasticsearch"
+    id: "output-123"
+    hosts: ["http://elasticsearch:9200"]
+xpack.fleet.agentPolicies:
+  - name: Test preconfigured policy
+    id: "test-preconfigured"
+    data_output_id: output-123
+    monitoring_output_id: output-123
+    monitoring_enabled: ["logs", "metrics"]
+    package_policies:
+      - package:
+          name: system
+        name: sytem-1

e2e/_suites/fleet/features/preconfigured_policies.feature

@@ -1,37 +1,23 @@
-@policies
+@preconfigured_policies
 Feature: Preconfigured Policies
   Scenarios for Preconfigured Policies
 
-Scenario Outline: Example using Kibana with custom policy
+Scenario Outline: Enrolling an agent in a preconfigured policy
   Given kibana uses "preconfigured-policies" profile
-    And a "<os>" agent is deployed to Fleet with "tar" installer
+  And agent uses enrollment token from "Test preconfigured policy" policy
+  And a "<os>" agent is deployed to Fleet with "tar" installer
   When the "elastic-agent" process is in the "started" state on the host
   Then the agent is listed in Fleet as "online"
-    And system package dashboards are listed in Fleet
+  And the agent is enrolled into "Test preconfigured policy" policy
 
-@centos
-Examples: Centos
-| os     |
-| centos |
+# Seems to be flaky https://github.com/elastic/e2e-testing/issues/1516
+# @centos
+# Examples: Centos
+# | os     |
+# | centos |
 
 @debian
 Examples: Debian
 | os     |
 | debian |
 
-Scenario Outline: Example using Kibana with default config
-  Given kibana uses "default" profile
-    And a "<os>" agent is deployed to Fleet with "tar" installer
-  When the "elastic-agent" process is in the "started" state on the host
-  Then the agent is listed in Fleet as "online"
-    And system package dashboards are listed in Fleet
-
-@centos
-Examples: Centos
-| os     |
-| centos |
-
-@debian
-Examples: Debian
-| os     |
-| debian |

e2e/_suites/fleet/fleet.go

@@ -42,6 +42,7 @@ var deployedAgentsCount = 0
 // FleetTestSuite represents the scenarios for Fleet-mode
 type FleetTestSuite struct {
 	// integrations
+	KibanaProfile       string
 	StandAlone          bool
 	CurrentToken        string // current enrollment token
 	CurrentTokenID      string // current enrollment tokenID
@@ -64,7 +65,11 @@ type FleetTestSuite struct {
 
 // afterScenario destroys the state created by a scenario
 func (fts *FleetTestSuite) afterScenario() {
-	defer func() { deployedAgentsCount = 0 }()
+	defer func() {
+		// Reset Kibana Profile to default
+		fts.KibanaProfile = ""
+		deployedAgentsCount = 0
+	}()
 
 	span := tx.StartSpan("Clean up", "test.scenario.clean", nil)
 	fts.currentContext = apm.ContextWithSpan(context.Background(), span)
@@ -108,7 +113,8 @@ func (fts *FleetTestSuite) afterScenario() {
 		}
 	}
 
-	_ = fts.deployer.Remove(fts.currentContext, deploy.NewServiceRequest(common.FleetProfileName), []deploy.ServiceRequest{deploy.NewServiceRequest(serviceName)}, common.ProfileEnv)
+	env := fts.getProfileEnv()
+	_ = fts.deployer.Remove(fts.currentContext, deploy.NewServiceRequest(common.FleetProfileName), []deploy.ServiceRequest{deploy.NewServiceRequest(serviceName)}, env)
 
 	err := fts.kibanaClient.DeleteEnrollmentAPIKey(fts.currentContext, fts.CurrentTokenID)
 	if err != nil {
@@ -148,6 +154,7 @@ func (fts *FleetTestSuite) beforeScenario() {
 
 func (fts *FleetTestSuite) contributeSteps(s *godog.ScenarioContext) {
 	s.Step(`^kibana uses "([^"]*)" profile$`, fts.kibanaUsesProfile)
+	s.Step(`^agent uses enrollment token from "([^"]*)" policy$`, fts.agentUsesPolicy)
 	s.Step(`^a "([^"]*)" agent is deployed to Fleet$`, fts.anAgentIsDeployedToFleet)
 	s.Step(`^a "([^"]*)" agent is deployed to Fleet on top of "([^"]*)"$`, fts.anAgentIsDeployedToFleetOnTopOfBeat)
 	s.Step(`^a "([^"]*)" agent is deployed to Fleet with "([^"]*)" installer$`, fts.anAgentIsDeployedToFleetWithInstaller)
@@ -165,6 +172,7 @@ func (fts *FleetTestSuite) contributeSteps(s *godog.ScenarioContext) {
 	s.Step(`^the file system Agent folder is empty$`, fts.theFileSystemAgentFolderIsEmpty)
 	s.Step(`^certs are installed$`, fts.installCerts)
 	s.Step(`^a Linux data stream exists with some data$`, fts.checkDataStream)
+	s.Step(`^the agent is enrolled into "([^"]*)" policy$`, fts.agentRunPolicy)
 
 	// endpoint steps
 	s.Step(`^the "([^"]*)" integration is "([^"]*)" in the policy$`, fts.theIntegrationIsOperatedInThePolicy)
@@ -337,6 +345,47 @@ func (fts *FleetTestSuite) agentInVersion(version string) error {
 	return backoff.Retry(agentInVersionFn, exp)
 }
 
+func (fts *FleetTestSuite) agentRunPolicy(policyName string) error {
+	agentRunPolicyFn := func() error {
+		agentService := deploy.NewServiceRequest(common.ElasticAgentServiceName)
+		manifest, _ := fts.deployer.Inspect(fts.currentContext, agentService)
+
+		policies, err := fts.kibanaClient.ListPolicies(fts.currentContext)
+		if err != nil {
+			return err
+		}
+
+		var policy *kibana.Policy
+		for _, p := range policies {
+			if policyName == p.Name {
+				policy = &p
+				break
+			}
+		}
+
+		if policy == nil {
+			return fmt.Errorf("Policy not found '%s'", policyName)
+		}
+
+		agent, err := fts.kibanaClient.GetAgentByHostname(fts.currentContext, manifest.Hostname)
+		if err != nil {
+			return err
+		}
+
+		if agent.PolicyID != policy.ID {
+			log.Errorf("FOUND %s %s", agent.PolicyID, policy.ID)
+			return fmt.Errorf("Agent not running the correct policy (running '%s' instead of '%s')", agent.PolicyID, policy.ID)
+		}
+
+		return nil
+	}
+	maxTimeout := time.Duration(utils.TimeoutFactor) * time.Minute * 2
+	exp := utils.GetExponentialBackOff(maxTimeout)
+
+	return backoff.Retry(agentRunPolicyFn, exp)
+
+}
+
 // this step infers the installer type from the underlying OS image
 // supported images: centos and debian
 func (fts *FleetTestSuite) anAgentIsDeployedToFleet(image string) error {
@@ -397,6 +446,7 @@ func (fts *FleetTestSuite) anAgentIsDeployedToFleetWithInstallerAndFleetServer(i
 
 	// Grab a new enrollment key for new agent
 	enrollmentKey, err := fts.kibanaClient.CreateEnrollmentAPIKey(fts.currentContext, fts.Policy)
+
 	if err != nil {
 		return err
 	}
@@ -411,8 +461,8 @@ func (fts *FleetTestSuite) anAgentIsDeployedToFleetWithInstallerAndFleetServer(i
 	services := []deploy.ServiceRequest{
 		agentService,
 	}
-
-	err = fts.deployer.Add(fts.currentContext, deploy.NewServiceRequest(common.FleetProfileName), services, common.ProfileEnv)
+	env := fts.getProfileEnv()
+	err = fts.deployer.Add(fts.currentContext, deploy.NewServiceRequest(common.FleetProfileName), services, env)
 	if err != nil {
 		return err
 	}
@@ -500,6 +550,15 @@ func bootstrapFleet(ctx context.Context, env map[string]string) error {
 				"env":   env,
 			}).Fatal("Unable to create kibana client")
 		}
+
+		err = kibanaClient.RecreateFleet(ctx)
+		if err != nil {
+			log.WithFields(log.Fields{
+				"error": err,
+				"env":   env,
+			}).Fatal("Fleet could not be recreated")
+		}
+
 		err = kibanaClient.WaitForFleet(ctx)
 		if err != nil {
 			log.WithFields(log.Fields{
@@ -515,15 +574,53 @@ func bootstrapFleet(ctx context.Context, env map[string]string) error {
 // is executed before any other in the test scenario. It will configure the Kibana profile to be used
 // in the scenario, changing the configuration file to be used.
 func (fts *FleetTestSuite) kibanaUsesProfile(profile string) error {
-	// copy the current profile environment, overriding the kibana profile with the one passed in the step
+	fts.KibanaProfile = profile
+
+	env := fts.getProfileEnv()
+
+	return bootstrapFleet(context.Background(), env)
+}
+
+func (fts *FleetTestSuite) getProfileEnv() map[string]string {
+
 	env := map[string]string{}
+
 	for k, v := range common.ProfileEnv {
 		env[k] = v
 	}
 
-	env["kibanaProfile"] = profile
+	if fts.KibanaProfile != "" {
+		env["kibanaProfile"] = fts.KibanaProfile
+	}
+
+	return env
+}
 
-	return bootstrapFleet(context.Background(), env)
+func (fts *FleetTestSuite) agentUsesPolicy(policyName string) error {
+	agentUsesPolicyFn := func() error {
+		policies, err := fts.kibanaClient.ListPolicies(fts.currentContext)
+		if err != nil {
+			return err
+		}
+
+		for _, p := range policies {
+			if policyName == p.Name {
+
+				fts.Policy = p
+				break
+			}
+		}
+
+		if fts.Policy.Name != policyName {
+			return fmt.Errorf("Policy not found '%s'", policyName)
+		}
+
+		return nil
+	}
+	maxTimeout := time.Duration(utils.TimeoutFactor) * time.Minute * 2
+	exp := utils.GetExponentialBackOff(maxTimeout)
+
+	return backoff.Retry(agentUsesPolicyFn, exp)
 }
 
 func (fts *FleetTestSuite) setup() error {
@@ -1069,8 +1166,8 @@ func (fts *FleetTestSuite) anAttemptToEnrollANewAgentFails() error {
 	services := []deploy.ServiceRequest{
 		agentService,
 	}
-
-	err := fts.deployer.Add(fts.currentContext, deploy.NewServiceRequest(common.FleetProfileName), services, common.ProfileEnv)
+	env := fts.getProfileEnv()
+	err := fts.deployer.Add(fts.currentContext, deploy.NewServiceRequest(common.FleetProfileName), services, env)
 	if err != nil {
 		return err
 	}