From 30caca26e27968aebc0bdd0919d1992c0ca0d0fa Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Mon, 18 Oct 2021 20:07:45 -0400 Subject: [PATCH 1/7] Test preconfiguration --- .../preconfigured-policies/kibana.config.yml | 20 ++++- .../features/preconfigured_policies.feature | 30 ++----- e2e/_suites/fleet/fleet.go | 78 +++++++++++++++++++ 3 files changed, 100 insertions(+), 28 deletions(-) diff --git a/cli/config/compose/profiles/fleet/preconfigured-policies/kibana.config.yml b/cli/config/compose/profiles/fleet/preconfigured-policies/kibana.config.yml index 3d97786aa4..5f2c3f539f 100644 --- a/cli/config/compose/profiles/fleet/preconfigured-policies/kibana.config.yml +++ b/cli/config/compose/profiles/fleet/preconfigured-policies/kibana.config.yml @@ -4,7 +4,7 @@ server.host: "0.0.0.0" telemetry.enabled: false -elasticsearch.hosts: [ "http://elasticsearch:9200" ] +elasticsearch.hosts: ["http://elasticsearch:9200"] elasticsearch.username: elastic elasticsearch.password: changeme xpack.monitoring.ui.container.elasticsearch.enabled: true @@ -13,8 +13,22 @@ xpack.fleet.enabled: true xpack.fleet.registryUrl: https://epr-staging.elastic.co xpack.fleet.agents.enabled: true xpack.fleet.agents.elasticsearch.host: http://elasticsearch:9200 -xpack.fleet.agents.fleet_server.hosts: - ["http://fleet-server:8220"] +xpack.fleet.agents.fleet_server.hosts: ["http://fleet-server:8220"] xpack.encryptedSavedObjects.encryptionKey: "12345678901234567890123456789012" xpack.fleet.agents.tlsCheckDisabled: true +xpack.fleet.outputs: + - name: "Test output" + type: "elasticsearch" + id: "output-123" + hosts: ["http://elasticsearch:9200"] +xpack.fleet.agentPolicies: + - name: Test preconfigured policy + id: "test-preconfigured" + data_output_id: output-123 + monitoring_output_id: output-123 + monitoring_enabled: ["logs", "metrics"] + package_policies: + - package: + name: system + name: sytem-1 diff --git a/e2e/_suites/fleet/features/preconfigured_policies.feature b/e2e/_suites/fleet/features/preconfigured_policies.feature index 34ab597e58..4f1b9f28c6 100644 --- a/e2e/_suites/fleet/features/preconfigured_policies.feature +++ b/e2e/_suites/fleet/features/preconfigured_policies.feature @@ -2,34 +2,14 @@ Feature: Preconfigured Policies Scenarios for Preconfigured Policies -Scenario Outline: Example using Kibana with custom policy +Scenario Outline: Enrolling an agent in a preconfigured policy Given kibana uses "preconfigured-policies" profile - And a "" agent is deployed to Fleet with "tar" installer + And agent uses "Test preconfigured policy" policy + And a "" agent is deployed to Fleet with "tar" installer When the "elastic-agent" process is in the "started" state on the host Then the agent is listed in Fleet as "online" - And system package dashboards are listed in Fleet - -@centos -Examples: Centos -| os | -| centos | - -@debian -Examples: Debian -| os | -| debian | - -Scenario Outline: Example using Kibana with default config - Given kibana uses "default" profile - And a "" agent is deployed to Fleet with "tar" installer - When the "elastic-agent" process is in the "started" state on the host - Then the agent is listed in Fleet as "online" - And system package dashboards are listed in Fleet - -@centos -Examples: Centos -| os | -| centos | + And the agent run the "Test preconfigured policy" policy + @debian Examples: Debian diff --git a/e2e/_suites/fleet/fleet.go b/e2e/_suites/fleet/fleet.go index 51d95a3c51..a1024c3954 100644 --- a/e2e/_suites/fleet/fleet.go +++ b/e2e/_suites/fleet/fleet.go @@ -148,6 +148,7 @@ func (fts *FleetTestSuite) beforeScenario() { func (fts *FleetTestSuite) contributeSteps(s *godog.ScenarioContext) { s.Step(`^kibana uses "([^"]*)" profile$`, fts.kibanaUsesProfile) + s.Step(`^agent uses "([^"]*)" policy$`, fts.agentUsesPolicy) s.Step(`^a "([^"]*)" agent is deployed to Fleet$`, fts.anAgentIsDeployedToFleet) s.Step(`^a "([^"]*)" agent is deployed to Fleet on top of "([^"]*)"$`, fts.anAgentIsDeployedToFleetOnTopOfBeat) s.Step(`^a "([^"]*)" agent is deployed to Fleet with "([^"]*)" installer$`, fts.anAgentIsDeployedToFleetWithInstaller) @@ -165,6 +166,7 @@ func (fts *FleetTestSuite) contributeSteps(s *godog.ScenarioContext) { s.Step(`^the file system Agent folder is empty$`, fts.theFileSystemAgentFolderIsEmpty) s.Step(`^certs are installed$`, fts.installCerts) s.Step(`^a Linux data stream exists with some data$`, fts.checkDataStream) + s.Step(`^the agent run the "([^"]*)" policy$`, fts.agentRunPolicy) // endpoint steps s.Step(`^the "([^"]*)" integration is "([^"]*)" in the policy$`, fts.theIntegrationIsOperatedInThePolicy) @@ -337,6 +339,47 @@ func (fts *FleetTestSuite) agentInVersion(version string) error { return backoff.Retry(agentInVersionFn, exp) } +func (fts *FleetTestSuite) agentRunPolicy(policyName string) error { + agentRunPolicyFn := func() error { + agentService := deploy.NewServiceRequest(common.ElasticAgentServiceName) + manifest, _ := fts.deployer.Inspect(fts.currentContext, agentService) + + policies, err := fts.kibanaClient.ListPolicies(fts.currentContext) + if err != nil { + return err + } + + var policy *kibana.Policy + for _, p := range policies { + if policyName == p.Name { + log.Info("FOUND POLICY") + policy = &p + } + } + + if policy == nil { + return fmt.Errorf("Policy not found '%s'", policyName) + } + + agent, err := fts.kibanaClient.GetAgentByHostname(fts.currentContext, manifest.Hostname) + if err != nil { + return err + } + + if agent.PolicyID != policy.ID { + log.Errorf("FOUND %s %s", agent.PolicyID, policy.ID) + return fmt.Errorf("Agent not running the correct policy (running '%s' instead of '%s')", agent.PolicyID, policy.ID) + } + + return nil + } + maxTimeout := time.Duration(utils.TimeoutFactor) * time.Minute * 2 + exp := utils.GetExponentialBackOff(maxTimeout) + + return backoff.Retry(agentRunPolicyFn, exp) + +} + // this step infers the installer type from the underlying OS image // supported images: centos and debian func (fts *FleetTestSuite) anAgentIsDeployedToFleet(image string) error { @@ -397,6 +440,7 @@ func (fts *FleetTestSuite) anAgentIsDeployedToFleetWithInstallerAndFleetServer(i // Grab a new enrollment key for new agent enrollmentKey, err := fts.kibanaClient.CreateEnrollmentAPIKey(fts.currentContext, fts.Policy) + if err != nil { return err } @@ -500,6 +544,15 @@ func bootstrapFleet(ctx context.Context, env map[string]string) error { "env": env, }).Fatal("Unable to create kibana client") } + + err = kibanaClient.RecreateFleet(ctx) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + "env": env, + }).Fatal("Fleet could not be recreated") + } + err = kibanaClient.WaitForFleet(ctx) if err != nil { log.WithFields(log.Fields{ @@ -525,6 +578,31 @@ func (fts *FleetTestSuite) kibanaUsesProfile(profile string) error { return bootstrapFleet(context.Background(), env) } +func (fts *FleetTestSuite) agentUsesPolicy(policyName string) error { + agentUsesPolicyFn := func() error { + policies, err := fts.kibanaClient.ListPolicies(fts.currentContext) + if err != nil { + return err + } + + for _, p := range policies { + if policyName == p.Name { + + fts.Policy = p + } + } + + if fts.Policy.Name != policyName { + return fmt.Errorf("Policy not found '%s'", policyName) + } + + return nil + } + maxTimeout := time.Duration(utils.TimeoutFactor) * time.Minute * 2 + exp := utils.GetExponentialBackOff(maxTimeout) + + return backoff.Retry(agentUsesPolicyFn, exp) +} func (fts *FleetTestSuite) setup() error { log.Trace("Creating Fleet setup") From acbcbf1f48e02c5a1ddf945aa291b061a239939d Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Tue, 19 Oct 2021 10:27:06 -0400 Subject: [PATCH 2/7] Add to CI --- .ci/.e2e-tests.yaml | 3 +++ e2e/_suites/fleet/features/preconfigured_policies.feature | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.ci/.e2e-tests.yaml b/.ci/.e2e-tests.yaml index a15a67b3c6..5baf95fed2 100644 --- a/.ci/.e2e-tests.yaml +++ b/.ci/.e2e-tests.yaml @@ -54,6 +54,9 @@ SUITES: - name: "Beats Background Processes" pullRequestFilter: " && ~debian" tags: "running_on_beats" + - name: "Preconfigured Policies" + pullRequestFilter: " && ~debian" + tags: "preconfigured_policies" - suite: "kubernetes-autodiscover" platforms: - labels: "ubuntu-18.04" diff --git a/e2e/_suites/fleet/features/preconfigured_policies.feature b/e2e/_suites/fleet/features/preconfigured_policies.feature index 4f1b9f28c6..704ef727a3 100644 --- a/e2e/_suites/fleet/features/preconfigured_policies.feature +++ b/e2e/_suites/fleet/features/preconfigured_policies.feature @@ -1,4 +1,4 @@ -@policies +@preconfigured_policies Feature: Preconfigured Policies Scenarios for Preconfigured Policies From b4c5fa1dae95acf115f3529d61a902bd0fc97182 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Tue, 19 Oct 2021 11:19:42 -0400 Subject: [PATCH 3/7] add to CI take 2 --- .ci/.e2e-tests-for-elastic-agent.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.ci/.e2e-tests-for-elastic-agent.yaml b/.ci/.e2e-tests-for-elastic-agent.yaml index f7ca98a89f..627adbc22e 100644 --- a/.ci/.e2e-tests-for-elastic-agent.yaml +++ b/.ci/.e2e-tests-for-elastic-agent.yaml @@ -39,3 +39,6 @@ SUITES: - name: "Beats Background Processes" pullRequestFilter: " && ~debian" tags: "running_on_beats" + - name: "Preconfigured Policies" + pullRequestFilter: " && ~debian" + tags: "preconfigured_policies" From ecc07cc6f5fb5b58b344f134fef8a0e012c0cef4 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Tue, 19 Oct 2021 12:58:11 -0400 Subject: [PATCH 4/7] Fix tests --- e2e/_suites/fleet/fleet.go | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/e2e/_suites/fleet/fleet.go b/e2e/_suites/fleet/fleet.go index a1024c3954..90bd68e0d0 100644 --- a/e2e/_suites/fleet/fleet.go +++ b/e2e/_suites/fleet/fleet.go @@ -42,6 +42,7 @@ var deployedAgentsCount = 0 // FleetTestSuite represents the scenarios for Fleet-mode type FleetTestSuite struct { // integrations + KibanaProfile string StandAlone bool CurrentToken string // current enrollment token CurrentTokenID string // current enrollment tokenID @@ -66,6 +67,9 @@ type FleetTestSuite struct { func (fts *FleetTestSuite) afterScenario() { defer func() { deployedAgentsCount = 0 }() + // Reset Kibana Profile to default + fts.KibanaProfile = "" + span := tx.StartSpan("Clean up", "test.scenario.clean", nil) fts.currentContext = apm.ContextWithSpan(context.Background(), span) defer span.End() @@ -108,7 +112,8 @@ func (fts *FleetTestSuite) afterScenario() { } } - _ = fts.deployer.Remove(fts.currentContext, deploy.NewServiceRequest(common.FleetProfileName), []deploy.ServiceRequest{deploy.NewServiceRequest(serviceName)}, common.ProfileEnv) + env := fts.getProfileEnv() + _ = fts.deployer.Remove(fts.currentContext, deploy.NewServiceRequest(common.FleetProfileName), []deploy.ServiceRequest{deploy.NewServiceRequest(serviceName)}, env) err := fts.kibanaClient.DeleteEnrollmentAPIKey(fts.currentContext, fts.CurrentTokenID) if err != nil { @@ -352,8 +357,8 @@ func (fts *FleetTestSuite) agentRunPolicy(policyName string) error { var policy *kibana.Policy for _, p := range policies { if policyName == p.Name { - log.Info("FOUND POLICY") policy = &p + break } } @@ -455,8 +460,8 @@ func (fts *FleetTestSuite) anAgentIsDeployedToFleetWithInstallerAndFleetServer(i services := []deploy.ServiceRequest{ agentService, } - - err = fts.deployer.Add(fts.currentContext, deploy.NewServiceRequest(common.FleetProfileName), services, common.ProfileEnv) + env := fts.getProfileEnv() + err = fts.deployer.Add(fts.currentContext, deploy.NewServiceRequest(common.FleetProfileName), services, env) if err != nil { return err } @@ -568,16 +573,28 @@ func bootstrapFleet(ctx context.Context, env map[string]string) error { // is executed before any other in the test scenario. It will configure the Kibana profile to be used // in the scenario, changing the configuration file to be used. func (fts *FleetTestSuite) kibanaUsesProfile(profile string) error { - // copy the current profile environment, overriding the kibana profile with the one passed in the step + fts.KibanaProfile = profile + + env := fts.getProfileEnv() + + return bootstrapFleet(context.Background(), env) +} + +func (fts *FleetTestSuite) getProfileEnv() map[string]string { + env := map[string]string{} + for k, v := range common.ProfileEnv { env[k] = v } - env["kibanaProfile"] = profile + if fts.KibanaProfile != "" { + env["kibanaProfile"] = fts.KibanaProfile + } - return bootstrapFleet(context.Background(), env) + return env } + func (fts *FleetTestSuite) agentUsesPolicy(policyName string) error { agentUsesPolicyFn := func() error { policies, err := fts.kibanaClient.ListPolicies(fts.currentContext) @@ -589,6 +606,7 @@ func (fts *FleetTestSuite) agentUsesPolicy(policyName string) error { if policyName == p.Name { fts.Policy = p + break } } @@ -1147,8 +1165,8 @@ func (fts *FleetTestSuite) anAttemptToEnrollANewAgentFails() error { services := []deploy.ServiceRequest{ agentService, } - - err := fts.deployer.Add(fts.currentContext, deploy.NewServiceRequest(common.FleetProfileName), services, common.ProfileEnv) + env := fts.getProfileEnv() + err := fts.deployer.Add(fts.currentContext, deploy.NewServiceRequest(common.FleetProfileName), services, env) if err != nil { return err } From d343cbaf275356d080378d36948812b6f60a0861 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Wed, 20 Oct 2021 09:10:12 -0400 Subject: [PATCH 5/7] Fix after code review --- .../fleet/features/preconfigured_policies.feature | 9 +++++++-- e2e/_suites/fleet/fleet.go | 13 +++++++------ 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/e2e/_suites/fleet/features/preconfigured_policies.feature b/e2e/_suites/fleet/features/preconfigured_policies.feature index 704ef727a3..5890180d0d 100644 --- a/e2e/_suites/fleet/features/preconfigured_policies.feature +++ b/e2e/_suites/fleet/features/preconfigured_policies.feature @@ -4,14 +4,19 @@ Feature: Preconfigured Policies Scenario Outline: Enrolling an agent in a preconfigured policy Given kibana uses "preconfigured-policies" profile - And agent uses "Test preconfigured policy" policy + And agent uses enrollment token from "Test preconfigured policy" policy And a "" agent is deployed to Fleet with "tar" installer When the "elastic-agent" process is in the "started" state on the host Then the agent is listed in Fleet as "online" - And the agent run the "Test preconfigured policy" policy + And the agent is enrolled into "Test preconfigured policy" policy @debian Examples: Debian | os | | debian | + +@centos +Examples: Centos +| os | +| centos | \ No newline at end of file diff --git a/e2e/_suites/fleet/fleet.go b/e2e/_suites/fleet/fleet.go index 90bd68e0d0..666a7a176a 100644 --- a/e2e/_suites/fleet/fleet.go +++ b/e2e/_suites/fleet/fleet.go @@ -65,10 +65,11 @@ type FleetTestSuite struct { // afterScenario destroys the state created by a scenario func (fts *FleetTestSuite) afterScenario() { - defer func() { deployedAgentsCount = 0 }() - - // Reset Kibana Profile to default - fts.KibanaProfile = "" + defer func() { + // Reset Kibana Profile to default + fts.KibanaProfile = "" + deployedAgentsCount = 0 + }() span := tx.StartSpan("Clean up", "test.scenario.clean", nil) fts.currentContext = apm.ContextWithSpan(context.Background(), span) @@ -153,7 +154,7 @@ func (fts *FleetTestSuite) beforeScenario() { func (fts *FleetTestSuite) contributeSteps(s *godog.ScenarioContext) { s.Step(`^kibana uses "([^"]*)" profile$`, fts.kibanaUsesProfile) - s.Step(`^agent uses "([^"]*)" policy$`, fts.agentUsesPolicy) + s.Step(`^agent uses enrollment token from "([^"]*)" policy$`, fts.agentUsesPolicy) s.Step(`^a "([^"]*)" agent is deployed to Fleet$`, fts.anAgentIsDeployedToFleet) s.Step(`^a "([^"]*)" agent is deployed to Fleet on top of "([^"]*)"$`, fts.anAgentIsDeployedToFleetOnTopOfBeat) s.Step(`^a "([^"]*)" agent is deployed to Fleet with "([^"]*)" installer$`, fts.anAgentIsDeployedToFleetWithInstaller) @@ -171,7 +172,7 @@ func (fts *FleetTestSuite) contributeSteps(s *godog.ScenarioContext) { s.Step(`^the file system Agent folder is empty$`, fts.theFileSystemAgentFolderIsEmpty) s.Step(`^certs are installed$`, fts.installCerts) s.Step(`^a Linux data stream exists with some data$`, fts.checkDataStream) - s.Step(`^the agent run the "([^"]*)" policy$`, fts.agentRunPolicy) + s.Step(`^the agent is enrolled into "([^"]*)" policy$`, fts.agentRunPolicy) // endpoint steps s.Step(`^the "([^"]*)" integration is "([^"]*)" in the policy$`, fts.theIntegrationIsOperatedInThePolicy) From a3fe56c54b19c4c55603f8af80ce11c2860a0b70 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Wed, 20 Oct 2021 10:49:33 -0400 Subject: [PATCH 6/7] Update e2e/_suites/fleet/features/preconfigured_policies.feature MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Manuel de la Peña --- e2e/_suites/fleet/features/preconfigured_policies.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/_suites/fleet/features/preconfigured_policies.feature b/e2e/_suites/fleet/features/preconfigured_policies.feature index 5890180d0d..a9dc20a72d 100644 --- a/e2e/_suites/fleet/features/preconfigured_policies.feature +++ b/e2e/_suites/fleet/features/preconfigured_policies.feature @@ -19,4 +19,4 @@ Examples: Debian @centos Examples: Centos | os | -| centos | \ No newline at end of file +| centos | From 93371c9a223bf56e96c7f084c447148d2c0e45a9 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Wed, 20 Oct 2021 14:04:54 -0400 Subject: [PATCH 7/7] comment centos --- .../fleet/features/preconfigured_policies.feature | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/e2e/_suites/fleet/features/preconfigured_policies.feature b/e2e/_suites/fleet/features/preconfigured_policies.feature index a9dc20a72d..8c75ec8501 100644 --- a/e2e/_suites/fleet/features/preconfigured_policies.feature +++ b/e2e/_suites/fleet/features/preconfigured_policies.feature @@ -9,14 +9,15 @@ Scenario Outline: Enrolling an agent in a preconfigured policy When the "elastic-agent" process is in the "started" state on the host Then the agent is listed in Fleet as "online" And the agent is enrolled into "Test preconfigured policy" policy - + +# Seems to be flaky https://github.com/elastic/e2e-testing/issues/1516 +# @centos +# Examples: Centos +# | os | +# | centos | @debian Examples: Debian | os | | debian | -@centos -Examples: Centos -| os | -| centos |