diff --git a/.ci/.e2e-tests.yaml b/.ci/.e2e-tests.yaml index aa350a4a49..7a29411b23 100644 --- a/.ci/.e2e-tests.yaml +++ b/.ci/.e2e-tests.yaml @@ -17,6 +17,9 @@ SUITES: - name: "Fleet" pullRequestFilter: " && ~debian" tags: "fleet_mode_agent" + - name: "Fleet Server" + pullRequestFilter: " && ~debian" + tags: "fleet_server" - name: "Endpoint Integration" pullRequestFilter: " && ~debian" tags: "agent_endpoint_integration" diff --git a/cli/config/compose/profiles/fleet/configurations/kibana.config.yml b/cli/config/compose/profiles/fleet/configurations/kibana.config.yml index e0fd646242..e9a660a62d 100644 --- a/cli/config/compose/profiles/fleet/configurations/kibana.config.yml +++ b/cli/config/compose/profiles/fleet/configurations/kibana.config.yml @@ -15,5 +15,6 @@ xpack.fleet.enabled: true xpack.fleet.registryUrl: http://package-registry:8080 xpack.fleet.agents.enabled: true xpack.fleet.agents.elasticsearch.host: http://elasticsearch:9200 +xpack.fleet.agents.fleetServerEnabled: true xpack.fleet.agents.kibana.host: http://kibana:5601 xpack.fleet.agents.tlsCheckDisabled: true diff --git a/cli/docker/docker.go b/cli/docker/docker.go index 999b633fa8..8f7236b778 100644 --- a/cli/docker/docker.go +++ b/cli/docker/docker.go @@ -27,6 +27,11 @@ const OPNetworkName = "elastic-dev-network" // ExecCommandIntoContainer executes a command, as a user, into a container func ExecCommandIntoContainer(ctx context.Context, containerName string, user string, cmd []string) (string, error) { + return ExecCommandIntoContainerWithEnv(ctx, containerName, user, cmd, []string{}) +} + +// ExecCommandIntoContainerWithEnv executes a command, as a user, with env, into a container +func ExecCommandIntoContainerWithEnv(ctx context.Context, containerName string, user string, cmd []string, env []string) (string, error) { dockerClient := getDockerClient() detach := false @@ -36,6 +41,7 @@ func ExecCommandIntoContainer(ctx context.Context, containerName string, user st "container": containerName, "command": cmd, "detach": detach, + "env": env, "tty": tty, }).Trace("Creating command to be executed in container") @@ -48,12 +54,14 @@ func ExecCommandIntoContainer(ctx context.Context, containerName string, user st AttachStdout: true, Detach: detach, Cmd: cmd, + Env: env, }) if err != nil { log.WithFields(log.Fields{ "container": containerName, "command": cmd, + "env": env, "error": err, "detach": detach, "tty": tty, @@ -65,6 +73,7 @@ func ExecCommandIntoContainer(ctx context.Context, containerName string, user st "container": containerName, "command": cmd, "detach": detach, + "env": env, "tty": tty, }).Trace("Command to be executed in container created") @@ -77,6 +86,7 @@ func ExecCommandIntoContainer(ctx context.Context, containerName string, user st "container": containerName, "command": cmd, "detach": detach, + "env": env, "error": err, "tty": tty, }).Error("Could not execute command in container") @@ -91,6 +101,7 @@ func ExecCommandIntoContainer(ctx context.Context, containerName string, user st "container": containerName, "command": cmd, "detach": detach, + "env": env, "error": err, "tty": tty, }).Error("Could not parse command output from container") @@ -102,6 +113,7 @@ func ExecCommandIntoContainer(ctx context.Context, containerName string, user st "container": containerName, "command": cmd, "detach": detach, + "env": env, "tty": tty, }).Trace("Command sucessfully executed in container") diff --git a/e2e/_suites/fleet/features/fleet_server.feature b/e2e/_suites/fleet/features/fleet_server.feature new file mode 100644 index 0000000000..ff863764fd --- /dev/null +++ b/e2e/_suites/fleet/features/fleet_server.feature @@ -0,0 +1,19 @@ +@fleet_server +Feature: Fleet Server + Scenarios for Fleet Server, where an Elasticseach and a Kibana instances are already provisioned, + so that the Agent is able to communicate with them + +@start-fleet-server +Scenario Outline: Deploying an Elastic Agent that starts Fleet Server + When a "" agent is deployed to Fleet with "tar" installer in fleet-server mode + Then the agent is listed in Fleet as "online" + +@centos +Examples: Centos +| os | +| centos | + +@debian +Examples: Debian +| os | +| debian | diff --git a/e2e/_suites/fleet/fleet.go b/e2e/_suites/fleet/fleet.go index e3344c12a8..eb9565f972 100644 --- a/e2e/_suites/fleet/fleet.go +++ b/e2e/_suites/fleet/fleet.go @@ -123,7 +123,7 @@ func (fts *FleetTestSuite) beforeScenario() { fts.Version = agentVersion // create policy with system monitoring enabled - defaultPolicy, err := getAgentDefaultPolicy() + defaultPolicy, err := getAgentDefaultPolicy("is_default") if err != nil { log.WithFields(log.Fields{ "err": err, @@ -161,6 +161,9 @@ func (fts *FleetTestSuite) contributeSteps(s *godog.ScenarioContext) { s.Step(`^the policy response will be shown in the Security App$`, fts.thePolicyResponseWillBeShownInTheSecurityApp) s.Step(`^the policy is updated to have "([^"]*)" in "([^"]*)" mode$`, fts.thePolicyIsUpdatedToHaveMode) s.Step(`^the policy will reflect the change in the Security App$`, fts.thePolicyWillReflectTheChangeInTheSecurityApp) + + // fleet server steps + s.Step(`^a "([^"]*)" agent is deployed to Fleet with "([^"]*)" installer in fleet-server mode$`, fts.anAgentIsDeployedToFleetWithInstallerInFleetMode) } func (fts *FleetTestSuite) anStaleAgentIsDeployedToFleetWithInstaller(image, version, installerType string) error { @@ -291,10 +294,15 @@ func (fts *FleetTestSuite) agentInVersion(version string) error { // supported installers: tar, systemd func (fts *FleetTestSuite) anAgentIsDeployedToFleetWithInstaller(image string, installerType string) error { + return fts.anAgentIsDeployedToFleetWithInstallerAndFleetServer(image, installerType, false) +} + +func (fts *FleetTestSuite) anAgentIsDeployedToFleetWithInstallerAndFleetServer(image string, installerType string, bootstrapFleetServer bool) error { log.WithFields(log.Fields{ - "image": image, - "installer": installerType, - }).Trace("Deploying an agent to Fleet with base image") + "bootstrapFleetServer": bootstrapFleetServer, + "image": image, + "installer": installerType, + }).Trace("Deploying an agent to Fleet with base image and fleet server") fts.Image = image fts.InstallerType = installerType @@ -316,7 +324,8 @@ func (fts *FleetTestSuite) anAgentIsDeployedToFleetWithInstaller(image string, i fts.CurrentToken = tokenJSONObject.Path("api_key").Data().(string) fts.CurrentTokenID = tokenJSONObject.Path("id").Data().(string) - err = deployAgentToFleet(installer, containerName, fts.CurrentToken) + var fleetConfig *FleetConfig + fleetConfig, err = deployAgentToFleet(installer, containerName, fts.CurrentToken, bootstrapFleetServer) fts.Cleanup = true if err != nil { return err @@ -324,7 +333,7 @@ func (fts *FleetTestSuite) anAgentIsDeployedToFleetWithInstaller(image string, i // the installation process for TAR includes the enrollment if installer.installerType != "tar" { - err = installer.EnrollFn(fts.CurrentToken) + err = installer.EnrollFn(fleetConfig) if err != nil { return err } @@ -452,10 +461,10 @@ func theAgentIsListedInFleetWithStatus(desiredStatus, hostname string) error { "status": desiredStatus, }).Info("The Agent is not present in Fleet, as expected") return nil - } else if desiredStatus == "online" { - retryCount++ - return fmt.Errorf("The agent is not present in Fleet, but it should") } + + retryCount++ + return fmt.Errorf("The agent is not present in Fleet in the '%s' status, but it should", desiredStatus) } isAgentInStatus, err := isAgentInStatus(agentID, desiredStatus) @@ -618,7 +627,13 @@ func (fts *FleetTestSuite) theAgentIsReenrolledOnTheHost() error { installer := fts.getInstaller() - err := installer.EnrollFn(fts.CurrentToken) + // a restart does not need to bootstrap the Fleet Server again + cfg, err := NewFleetConfig(fts.CurrentToken, false, false) + if err != nil { + return err + } + + err = installer.EnrollFn(cfg) if err != nil { return err } @@ -663,7 +678,7 @@ func (fts *FleetTestSuite) thePolicyShowsTheDatasourceAdded(packageName string) fts.Integration = integration configurationIsPresentFn := func() error { - defaultPolicy, err := getAgentDefaultPolicy() + defaultPolicy, err := getAgentDefaultPolicy("is_default") if err != nil { log.WithFields(log.Fields{ "error": err, @@ -1033,14 +1048,14 @@ func (fts *FleetTestSuite) anAttemptToEnrollANewAgentFails() error { containerName := fmt.Sprintf("%s_%s_%s_%d", profile, fts.Image+"-systemd", ElasticAgentServiceName, 2) // name of the new container - err := deployAgentToFleet(installer, containerName, fts.CurrentToken) + fleetConfig, err := deployAgentToFleet(installer, containerName, fts.CurrentToken, false) // the installation process for TAR includes the enrollment if installer.installerType != "tar" { if err != nil { return err } - err = installer.EnrollFn(fts.CurrentToken) + err = installer.EnrollFn(fleetConfig) if err == nil { err = fmt.Errorf("The agent was enrolled although the token was previously revoked") @@ -1334,7 +1349,7 @@ func createFleetToken(name string, policyID string) (*gabs.Container, error) { return tokenItem, nil } -func deployAgentToFleet(installer ElasticAgentInstaller, containerName string, token string) error { +func deployAgentToFleet(installer ElasticAgentInstaller, containerName string, token string, bootstrapFleetServer bool) (*FleetConfig, error) { profile := installer.profile // name of the runtime dependencies compose file service := installer.service // name of the service serviceTag := installer.tag // docker tag of the service @@ -1357,24 +1372,31 @@ func deployAgentToFleet(installer ElasticAgentInstaller, containerName string, t "service": service, "tag": serviceTag, }).Error("Could not run the target box") - return err + return nil, err } err = installer.PreInstallFn() if err != nil { - return err + return nil, err + } + + cfg, cfgError := NewFleetConfig(token, bootstrapFleetServer, false) + if cfgError != nil { + return nil, cfgError } - err = installer.InstallFn(containerName, token) + err = installer.InstallFn(cfg) if err != nil { - return err + return nil, err } - return installer.PostInstallFn() + return cfg, installer.PostInstallFn() } -// getAgentDefaultPolicy sends a GET request to Fleet for the existing default policy -func getAgentDefaultPolicy() (*gabs.Container, error) { +// getAgentDefaultPolicy sends a GET request to Fleet for the existing default policy, using the +// "defaultPolicyFieldName" passed as parameter as field to be used to find the policy in list +// of fleet policies +func getAgentDefaultPolicy(defaultPolicyFieldName string) (*gabs.Container, error) { r := createDefaultHTTPRequest(ingestManagerAgentPoliciesURL) body, err := curl.Get(r) if err != nil { @@ -1402,10 +1424,21 @@ func getAgentDefaultPolicy() (*gabs.Container, error) { "count": len(policies.Children()), }).Trace("Fleet policies retrieved") - // TODO: perform a strong check to capture default policy - defaultPolicy := policies.Index(0) + for _, policy := range policies.Children() { + if !policy.Exists(defaultPolicyFieldName) { + continue + } + + if policy.Path(defaultPolicyFieldName).Data().(bool) { + log.WithFields(log.Fields{ + "field": defaultPolicyFieldName, + "policy": policy, + }).Trace("Default Policy was found") + return policy, nil + } + } - return defaultPolicy, nil + return nil, fmt.Errorf("Default policy was not found with '%s' field equals to 'true'", defaultPolicyFieldName) } func getAgentEvents(applicationName string, agentID string, packagePolicyID string, updatedAt string) error { @@ -1581,6 +1614,11 @@ func isAgentInStatus(agentID string, desiredStatus string) (bool, error) { jsonResponse, err := gabs.ParseJSON([]byte(body)) + log.WithFields(log.Fields{ + "agentID": agentID, + "desiredStatus": desiredStatus, + }).Info(jsonResponse) + agentStatus := jsonResponse.Path("item.status").Data().(string) return (strings.ToLower(agentStatus) == strings.ToLower(desiredStatus)), nil diff --git a/e2e/_suites/fleet/fleet_server.go b/e2e/_suites/fleet/fleet_server.go new file mode 100644 index 0000000000..b207474410 --- /dev/null +++ b/e2e/_suites/fleet/fleet_server.go @@ -0,0 +1,110 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package main + +import ( + "fmt" + + log "github.com/sirupsen/logrus" +) + +// FleetConfig represents the configuration for Fleet Server when building the enrollment command +type FleetConfig struct { + EnrollmentToken string + ElasticsearchPort int + ElasticsearchURI string + ElasticsearchCredentials string + KibanaPort int + KibanaURI string + // server + BootstrapFleetServer bool + ServerPolicyID string +} + +// NewFleetConfig builds a new configuration for the fleet agent, defaulting ES credentials, URI and port. +// If the 'bootstrappFleetServer' flag is true, the it will create the config for the initial fleet server +// used to bootstrap Fleet Server +// If the 'fleetServerMode' flag is true, the it will create the config for an agent using an existing Fleet +// Server to connect to Fleet. It will also retrieve the default policy ID for fleet server +func NewFleetConfig(token string, bootstrapFleetServer bool, fleetServerMode bool) (*FleetConfig, error) { + cfg := &FleetConfig{ + BootstrapFleetServer: bootstrapFleetServer, + EnrollmentToken: token, + ElasticsearchCredentials: "elastic:changeme", + ElasticsearchPort: 9200, + ElasticsearchURI: "elasticsearch", + KibanaPort: 5601, + KibanaURI: "kibana", + } + + if fleetServerMode { + defaultFleetServerPolicy, err := getAgentDefaultPolicy("is_default_fleet_server") + if err != nil { + return nil, err + } + + cfg.ServerPolicyID = defaultFleetServerPolicy.Path("id").Data().(string) + + log.WithFields(log.Fields{ + "elasticsearch": cfg.ElasticsearchURI, + "elasticsearchPort": cfg.ElasticsearchPort, + "policyID": cfg.ServerPolicyID, + "token": cfg.EnrollmentToken, + }).Debug("Fleet Server config created") + } + + return cfg, nil +} + +func (cfg FleetConfig) flags() []string { + if cfg.BootstrapFleetServer { + // TO-DO: remove all code to calculate the fleet-server policy, because it's inferred by the fleet-server + return []string{ + "--force", + "--fleet-server-es", fmt.Sprintf("http://%s@%s:%d", cfg.ElasticsearchCredentials, cfg.ElasticsearchURI, cfg.ElasticsearchPort), + } + } + + /* + // agent using an already bootstrapped fleet-server + fleetServerHost := "https://hostname_of_the_bootstrapped_fleet_server:8220" + return []string{ + "-e", "-v", "--force", "--insecure", + // ensure the enrollment belongs to the default policy + "--enrollment-token=" + cfg.EnrollmentToken, + "--url", fleetServerHost, + } + */ + + baseFlags := []string{"-e", "-v", "--force", "--insecure", "--enrollment-token=" + cfg.EnrollmentToken} + + if cfg.ServerPolicyID != "" { + baseFlags = append(baseFlags, "--fleet-server-insecure-http", "--fleet-server", fmt.Sprintf("http://%s@%s:%d", cfg.ElasticsearchCredentials, cfg.ElasticsearchURI, cfg.ElasticsearchPort), "--fleet-server-host=http://0.0.0.0", "--fleet-server-policy", cfg.ServerPolicyID) + } + + return append(baseFlags, "--kibana-url", fmt.Sprintf("http://%s@%s:%d", cfg.ElasticsearchCredentials, cfg.KibanaURI, cfg.KibanaPort)) +} + +func (fts *FleetTestSuite) anAgentIsDeployedToFleetWithInstallerInFleetMode(image string, installerType string) error { + fts.ElasticAgentStopped = true + return fts.anAgentIsDeployedToFleetWithInstallerAndFleetServer(image, installerType, true) +} + +// bootstrapFleetServer runs a command for the elastic-agent +func bootstrapFleetServer(profile string, image string, service string, binary string, cfg *FleetConfig) error { + log.Debug("Bootstrapping Fleet Server") + + args := []string{ + "-f", "--fleet-server-insecure-http", + "--fleet-server", fmt.Sprintf("http://%s@%s:%d", cfg.ElasticsearchCredentials, cfg.ElasticsearchURI, cfg.ElasticsearchPort), + } + + err := runElasticAgentCommand(profile, image, service, binary, "install", args) + if err != nil { + return fmt.Errorf("Failed to install the agent with subcommand: %v", err) + } + + return nil +} diff --git a/e2e/_suites/fleet/installers.go b/e2e/_suites/fleet/installers.go index b1d443204e..894553ff43 100644 --- a/e2e/_suites/fleet/installers.go +++ b/e2e/_suites/fleet/installers.go @@ -17,7 +17,7 @@ import ( // InstallerPackage represents the operations that can be performed by an installer package type type InstallerPackage interface { - Install(containerName string, token string) error + Install(cfg *FleetConfig) error InstallCerts() error PrintLogs(containerName string) error Postinstall() error @@ -119,7 +119,7 @@ func NewDEBPackage(binaryName string, profile string, image string, service stri } // Install installs a DEB package -func (i *DEBPackage) Install(containerName string, token string) error { +func (i *DEBPackage) Install(cfg *FleetConfig) error { return i.extractPackage([]string{"apt", "install", "/" + i.binaryName, "-y"}) } @@ -182,7 +182,7 @@ func NewDockerPackage(binaryName string, profile string, image string, service s } // Install installs a Docker package -func (i *DockerPackage) Install(containerName string, token string) error { +func (i *DockerPackage) Install(cfg *FleetConfig) error { log.Trace("No install commands for Docker packages") return nil } @@ -264,7 +264,7 @@ func NewRPMPackage(binaryName string, profile string, image string, service stri } // Install installs a RPM package -func (i *RPMPackage) Install(containerName string, token string) error { +func (i *RPMPackage) Install(cfg *FleetConfig) error { return i.extractPackage([]string{"yum", "localinstall", "/" + i.binaryName, "-y"}) } @@ -326,10 +326,11 @@ func NewTARPackage(binaryName string, profile string, image string, service stri } // Install installs a TAR package -func (i *TARPackage) Install(containerName string, token string) error { +func (i *TARPackage) Install(cfg *FleetConfig) error { // install the elastic-agent to /usr/bin/elastic-agent using command binary := fmt.Sprintf("/elastic-agent/%s", i.artifact) - args := []string{"--force", "--insecure", "--enrollment-token=" + token, "--kibana-url", "http://kibana:5601"} + + args := cfg.flags() err := runElasticAgentCommand(i.profile, i.image, i.service, binary, "install", args) if err != nil { @@ -371,8 +372,8 @@ func (i *TARPackage) Preinstall() error { // simplify layout cmds := [][]string{ - []string{"rm", "-fr", "/elastic-agent"}, - []string{"mv", fmt.Sprintf("/%s-%s-%s-%s", i.artifact, i.version, i.OS, i.arch), "/elastic-agent"}, + {"rm", "-fr", "/elastic-agent"}, + {"mv", fmt.Sprintf("/%s-%s-%s-%s", i.artifact, i.version, i.OS, i.arch), "/elastic-agent"}, } for _, cmd := range cmds { err = steps.ExecCommandInService(i.profile, i.image, i.service, cmd, profileEnv, false) diff --git a/e2e/_suites/fleet/services.go b/e2e/_suites/fleet/services.go index 950be4f08c..a0ada0e9e5 100644 --- a/e2e/_suites/fleet/services.go +++ b/e2e/_suites/fleet/services.go @@ -22,10 +22,10 @@ type ElasticAgentInstaller struct { artifactOS string // OS of the artifact artifactVersion string // version of the artifact binaryPath string // the local path where the agent for the binary is located - EnrollFn func(token string) error + EnrollFn func(cfg *FleetConfig) error image string // docker image installerType string - InstallFn func(containerName string, token string) error + InstallFn func(cfg *FleetConfig) error InstallCertsFn func() error name string // the name for the binary processName string // name of the elastic-agent process @@ -59,17 +59,23 @@ func (i *ElasticAgentInstaller) listElasticAgentWorkingDirContent(containerName return content, nil } -func buildEnrollmentFlags(token string) []string { - return []string{"--kibana-url=http://kibana:5601", "--enrollment-token=" + token, "-f", "--insecure"} -} - // runElasticAgentCommand runs a command for the elastic-agent func runElasticAgentCommand(profile string, image string, service string, process string, command string, arguments []string) error { + return runElasticAgentCommandWithEnv(profile, image, service, process, command, arguments, map[string]string{}) +} + +// runElasticAgentCommandWithEnv runs a command with env for the elastic-agent +func runElasticAgentCommandWithEnv(profile string, image string, service string, process string, command string, arguments []string, env map[string]string) error { cmds := []string{ process, command, } cmds = append(cmds, arguments...) + // append passed env to profile env + for k, v := range env { + profileEnv[k] = v + } + err := steps.ExecCommandInService(profile, image, service, cmds, profileEnv, false) if err != nil { log.WithFields(log.Fields{ @@ -171,8 +177,8 @@ func newCentosInstaller(image string, tag string, version string) (ElasticAgentI return ElasticAgentInstaller{}, err } - enrollFn := func(token string) error { - return runElasticAgentCommand(profile, image, service, ElasticAgentProcessName, "enroll", buildEnrollmentFlags(token)) + enrollFn := func(cfg *FleetConfig) error { + return runElasticAgentCommand(profile, image, service, ElasticAgentProcessName, "enroll", cfg.flags()) } workingDir := "/var/lib/elastic-agent" @@ -237,8 +243,8 @@ func newDebianInstaller(image string, tag string, version string) (ElasticAgentI return ElasticAgentInstaller{}, err } - enrollFn := func(token string) error { - return runElasticAgentCommand(profile, image, service, ElasticAgentProcessName, "enroll", buildEnrollmentFlags(token)) + enrollFn := func(cfg *FleetConfig) error { + return runElasticAgentCommand(profile, image, service, ElasticAgentProcessName, "enroll", cfg.flags()) } workingDir := "/var/lib/elastic-agent" @@ -320,7 +326,7 @@ func newDockerInstaller(ubi8 bool, version string) (ElasticAgentInstaller, error logFileName := "elastic-agent-json.log" logFile := logsDir + "/" + logFileName - enrollFn := func(token string) error { + enrollFn := func(cfg *FleetConfig) error { return nil } @@ -389,8 +395,8 @@ func newTarInstaller(image string, tag string, version string) (ElasticAgentInst logFileName := "elastic-agent-json.log" logFile := logsDir + "/" + logFileName - enrollFn := func(token string) error { - return runElasticAgentCommand(profile, dockerImage, service, ElasticAgentProcessName, "enroll", buildEnrollmentFlags(token)) + enrollFn := func(cfg *FleetConfig) error { + return runElasticAgentCommand(profile, dockerImage, service, ElasticAgentProcessName, "enroll", cfg.flags()) } //