feat: support flavours in services, specially in the elastic-agent

cli/cmd/deploy.go

@@ -8,7 +8,7 @@ import (
 	"context"
 
 	"github.com/elastic/e2e-testing/cli/config"
-	"github.com/elastic/e2e-testing/internal/compose"
+	"github.com/elastic/e2e-testing/internal/deploy"
 	log "github.com/sirupsen/logrus"
 
 	"github.com/spf13/cobra"
@@ -63,12 +63,16 @@ func buildDeployServiceCommand(srv string) *cobra.Command {
 		Short: `Deploys a ` + srv + ` service`,
 		Long:  `Deploys a ` + srv + ` service, adding it to a running profile, identified by its name`,
 		Run: func(cmd *cobra.Command, args []string) {
-			serviceManager := compose.NewServiceManager()
+			serviceManager := deploy.NewServiceManager()
 
 			env := map[string]string{}
 			env = config.PutServiceEnvironment(env, srv, versionToRun)
 
-			err := serviceManager.AddServicesToCompose(context.Background(), deployToProfile, []string{srv}, env)
+			err := serviceManager.AddServicesToCompose(
+				context.Background(),
+				deploy.NewServiceRequest(deployToProfile),
+				[]deploy.ServiceRequest{deploy.NewServiceRequest(srv)},
+				env)
 			if err != nil {
 				log.WithFields(log.Fields{
 					"profile":  deployToProfile,
@@ -85,12 +89,16 @@ func buildUndeployServiceCommand(srv string) *cobra.Command {
 		Short: `Undeploys a ` + srv + ` service`,
 		Long:  `Undeploys a ` + srv + ` service, removing it from a running profile, identified by its name`,
 		Run: func(cmd *cobra.Command, args []string) {
-			serviceManager := compose.NewServiceManager()
+			serviceManager := deploy.NewServiceManager()
 
 			env := map[string]string{}
 			env = config.PutServiceEnvironment(env, srv, versionToRun)
 
-			err := serviceManager.RemoveServicesFromCompose(context.Background(), deployToProfile, []string{srv}, env)
+			err := serviceManager.RemoveServicesFromCompose(
+				context.Background(),
+				deploy.NewServiceRequest(deployToProfile),
+				[]deploy.ServiceRequest{deploy.NewServiceRequest(srv)},
+				env)
 			if err != nil {
 				log.WithFields(log.Fields{
 					"profile":  deployToProfile,

cli/cmd/run.go

@@ -10,7 +10,7 @@ import (
 	"strings"
 
 	"github.com/elastic/e2e-testing/cli/config"
-	"github.com/elastic/e2e-testing/internal/compose"
+	"github.com/elastic/e2e-testing/internal/deploy"
 	log "github.com/sirupsen/logrus"
 
 	"github.com/spf13/cobra"
@@ -64,7 +64,7 @@ func buildRunServiceCommand(srv string) *cobra.Command {
 		Short: `Runs a ` + srv + ` service`,
 		Long:  `Runs a ` + srv + ` service, spinning up a Docker container for it and exposing its internal configuration so that you are able to connect to it in an easy manner`,
 		Run: func(cmd *cobra.Command, args []string) {
-			serviceManager := compose.NewServiceManager()
+			serviceManager := deploy.NewServiceManager()
 
 			env := config.PutServiceEnvironment(map[string]string{}, srv, versionToRun)
 
@@ -76,7 +76,8 @@ func buildRunServiceCommand(srv string) *cobra.Command {
 				env[k] = v
 			}
 
-			err := serviceManager.RunCompose(context.Background(), false, []string{srv}, env)
+			err := serviceManager.RunCompose(
+				context.Background(), false, []deploy.ServiceRequest{deploy.NewServiceRequest(srv)}, env)
 			if err != nil {
 				log.WithFields(log.Fields{
 					"service": srv,
@@ -96,7 +97,7 @@ Example:
   go run main.go run profile fleet -s elastic-agent:8.0.0-SNAPSHOT
 `,
 		Run: func(cmd *cobra.Command, args []string) {
-			serviceManager := compose.NewServiceManager()
+			serviceManager := deploy.NewServiceManager()
 
 			env := map[string]string{
 				"profileVersion": versionToRun,
@@ -110,14 +111,15 @@ Example:
 				env[k] = v
 			}
 
-			err := serviceManager.RunCompose(context.Background(), true, []string{key}, env)
+			err := serviceManager.RunCompose(
+				context.Background(), true, []deploy.ServiceRequest{deploy.NewServiceRequest(key)}, env)
 			if err != nil {
 				log.WithFields(log.Fields{
 					"profile": key,
 				}).Error("Could not run the profile.")
 			}
 
-			composeNames := []string{}
+			composeNames := []deploy.ServiceRequest{}
 			if len(servicesToRun) > 0 {
 				for _, srv := range servicesToRun {
 					arr := strings.Split(srv, ":")
@@ -137,10 +139,10 @@ Example:
 					}).Trace("Adding service")
 
 					env = config.PutServiceEnvironment(env, image, tag)
-					composeNames = append(composeNames, image)
+					composeNames = append(composeNames, deploy.NewServiceRequest(image))
 				}
 
-				err = serviceManager.AddServicesToCompose(context.Background(), key, composeNames, env)
+				err = serviceManager.AddServicesToCompose(context.Background(), deploy.NewServiceRequest(key), composeNames, env)
 				if err != nil {
 					log.WithFields(log.Fields{
 						"profile":  key,

cli/cmd/stop.go

@@ -8,7 +8,7 @@ import (
 	"context"
 
 	"github.com/elastic/e2e-testing/cli/config"
-	"github.com/elastic/e2e-testing/internal/compose"
+	"github.com/elastic/e2e-testing/internal/deploy"
 	log "github.com/sirupsen/logrus"
 
 	"github.com/spf13/cobra"
@@ -55,9 +55,10 @@ func buildStopServiceCommand(srv string) *cobra.Command {
 		Short: `Stops a ` + srv + ` service`,
 		Long:  `Stops a ` + srv + ` service, stoppping its Docker container`,
 		Run: func(cmd *cobra.Command, args []string) {
-			serviceManager := compose.NewServiceManager()
+			serviceManager := deploy.NewServiceManager()
 
-			err := serviceManager.StopCompose(context.Background(), false, []string{srv})
+			err := serviceManager.StopCompose(
+				context.Background(), false, []deploy.ServiceRequest{deploy.NewServiceRequest(srv)})
 			if err != nil {
 				log.WithFields(log.Fields{
 					"service": srv,
@@ -73,9 +74,10 @@ func buildStopProfileCommand(key string, profile config.Profile) *cobra.Command
 		Short: `Stops the ` + profile.Name + ` profile`,
 		Long:  `Stops the ` + profile.Name + ` profile, stopping the Services that compound it`,
 		Run: func(cmd *cobra.Command, args []string) {
-			serviceManager := compose.NewServiceManager()
+			serviceManager := deploy.NewServiceManager()
 
-			err := serviceManager.StopCompose(context.Background(), true, []string{key})
+			err := serviceManager.StopCompose(
+				context.Background(), true, []deploy.ServiceRequest{deploy.NewServiceRequest(key)})
 			if err != nil {
 				log.WithFields(log.Fields{
 					"profile": key,

cli/config/compose/services/centos-systemd/docker-compose.yml → cli/config/compose/services/elastic-agent/centos/docker-compose.yml

@@ -1,6 +1,6 @@
 version: '2.4'
 services:
-  centos-systemd:
+  elastic-agent:
     image: centos/systemd:${centos_systemdTag:-latest} 
     container_name: ${centos_systemdContainerName}
     entrypoint: "/usr/sbin/init"

cli/config/compose/services/debian-systemd/docker-compose.yml → cli/config/compose/services/elastic-agent/debian/docker-compose.yml

@@ -1,6 +1,6 @@
 version: '2.4'
 services:
-  debian-systemd:
+  elastic-agent:
     image: alehaa/debian-systemd:${debian_systemdTag:-stretch}
     container_name: ${debian_systemdContainerName} 
     entrypoint: "/sbin/init"

cli/config/config.go

@@ -73,16 +73,14 @@ func FileExists(configFile string) (bool, error) {
 
 // GetComposeFile returns the path of the compose file, looking up the
 // tool's workdir
-func GetComposeFile(isProfile bool, composeName string, composeFileName ...string) (string, error) {
-	if isProfile || composeFileName == nil || composeFileName[0] == "" {
-		composeFileName = []string{"docker-compose.yml"}
-	}
+func GetComposeFile(isProfile bool, composeName string) (string, error) {
+	composeFileName := "docker-compose.yml"
 	serviceType := "services"
 	if isProfile {
 		serviceType = "profiles"
 	}
 
-	composeFilePath := path.Join(Op.Workspace, "compose", serviceType, composeName, composeFileName[0])
+	composeFilePath := path.Join(Op.Workspace, "compose", serviceType, composeName, composeFileName)
 	found, err := io.Exists(composeFilePath)
 	if found && err == nil {
 		log.WithFields(log.Fields{

e2e/_suites/fleet/features/backend_processes.feature

@@ -21,7 +21,7 @@ Examples: Debian
 
 @enroll
 Scenario Outline: Deploying the <os> agent with enroll and then run on rpm and deb
-  Given a "<os>" agent is deployed to Fleet with "systemd" installer
+  Given a "<os>" agent is deployed to Fleet
   When the "elastic-agent" process is in the "started" state on the host
   Then there are "2" instances of the "filebeat" process in the "started" state
     And there are "2" instances of the "metricbeat" process in the "started" state

e2e/_suites/fleet/features/fleet_mode_agent.feature

@@ -21,7 +21,7 @@ Examples: Debian
 
 @enroll
 Scenario Outline: Deploying the <os> agent with enroll and then run on rpm and deb
-  Given a "<os>" agent is deployed to Fleet with "systemd" installer
+  Given a "<os>" agent is deployed to Fleet
   When the "elastic-agent" process is in the "started" state on the host
   Then the agent is listed in Fleet as "online"
     And system package dashboards are listed in Fleet

e2e/_suites/fleet/fleet.go

@@ -15,9 +15,7 @@ import (
 	"github.com/cenkalti/backoff/v4"
 	"github.com/cucumber/godog"
 	"github.com/elastic/e2e-testing/internal/common"
-	"github.com/elastic/e2e-testing/internal/compose"
 	"github.com/elastic/e2e-testing/internal/deploy"
-	"github.com/elastic/e2e-testing/internal/docker"
 	"github.com/elastic/e2e-testing/internal/elasticsearch"
 	"github.com/elastic/e2e-testing/internal/installer"
 	"github.com/elastic/e2e-testing/internal/kibana"
@@ -90,7 +88,14 @@ func (fts *FleetTestSuite) afterScenario() {
 
 	developerMode := shell.GetEnvBool("DEVELOPER_MODE")
 	if !developerMode {
-		_ = fts.deployer.Remove([]string{common.FleetProfileName, serviceName}, common.ProfileEnv)
+		agentInstaller := fts.getInstaller()
+
+		_ = fts.deployer.Remove(
+			[]deploy.ServiceRequest{
+				deploy.NewServiceRequest(common.FleetProfileName),
+				deploy.NewServiceRequest(serviceName).WithFlavour(agentInstaller.Image),
+			},
+			common.ProfileEnv)
 	} else {
 		log.WithField("service", serviceName).Info("Because we are running in development mode, the service won't be stopped")
 	}
@@ -131,6 +136,7 @@ func (fts *FleetTestSuite) beforeScenario() {
 }
 
 func (fts *FleetTestSuite) contributeSteps(s *godog.ScenarioContext) {
+	s.Step(`^a "([^"]*)" agent is deployed to Fleet$`, fts.anAgentIsDeployedToFleet)
 	s.Step(`^a "([^"]*)" agent is deployed to Fleet with "([^"]*)" installer$`, fts.anAgentIsDeployedToFleetWithInstaller)
 	s.Step(`^a "([^"]*)" agent "([^"]*)" is deployed to Fleet with "([^"]*)" installer$`, fts.anStaleAgentIsDeployedToFleetWithInstaller)
 	s.Step(`^agent is in version "([^"]*)"$`, fts.agentInVersion)
@@ -306,7 +312,18 @@ func (fts *FleetTestSuite) agentInVersion(version string) error {
 	return backoff.Retry(agentInVersionFn, exp)
 }
 
-// supported installers: tar, systemd
+// this step infers the installer type from the underlying OS image
+// supported images: centos and debian
+func (fts *FleetTestSuite) anAgentIsDeployedToFleet(image string) error {
+	installerType := "rpm"
+	if image == "debian" {
+		installerType = "deb"
+	}
+
+	return fts.anAgentIsDeployedToFleetWithInstallerAndFleetServer(image, installerType)
+}
+
+// supported installers: tar, rpm, deb
 func (fts *FleetTestSuite) anAgentIsDeployedToFleetWithInstaller(image string, installerType string) error {
 	return fts.anAgentIsDeployedToFleetWithInstallerAndFleetServer(image, installerType)
 }
@@ -348,7 +365,7 @@ func (fts *FleetTestSuite) anAgentIsDeployedToFleetWithInstallerAndFleetServer(i
 	}
 
 	// get container hostname once
-	hostname, err := docker.GetContainerHostname(containerName)
+	hostname, err := deploy.GetContainerHostname(containerName)
 	if err != nil {
 		return err
 	}
@@ -361,7 +378,7 @@ func (fts *FleetTestSuite) anAgentIsDeployedToFleetWithInstallerAndFleetServer(i
 // we are using the Docker client instead of docker-compose because it does not support
 // returning the output of a command: it simply returns error level
 func (fts *FleetTestSuite) getContainerName(i installer.ElasticAgentInstaller, index int) string {
-	return fmt.Sprintf("%s_%s_%s_%d", i.Profile, i.Image, common.ElasticAgentServiceName, index)
+	return fmt.Sprintf("%s_%s_%d", i.Profile, common.ElasticAgentServiceName, index)
 }
 
 // getServiceName returns the current service name, the one defined at the docker compose
@@ -391,17 +408,20 @@ func (fts *FleetTestSuite) processStateChangedOnTheHost(process string, state st
 
 	serviceName := agentInstaller.Service // name of the service
 
+	profileService := deploy.NewServiceRequest(profile)
+	imageService := deploy.NewServiceRequest(common.ElasticAgentServiceName).WithFlavour(agentInstaller.Image)
+
 	if state == "started" {
-		return installer.SystemctlRun(profile, agentInstaller.Image, serviceName, "start")
+		return installer.SystemctlRun(profileService, imageService, serviceName, "start")
 	} else if state == "restarted" {
-		err := installer.SystemctlRun(profile, agentInstaller.Image, serviceName, "stop")
+		err := installer.SystemctlRun(profileService, imageService, serviceName, "stop")
 		if err != nil {
 			return err
 		}
 
 		utils.Sleep(time.Duration(utils.TimeoutFactor) * 10 * time.Second)
 
-		err = installer.SystemctlRun(profile, agentInstaller.Image, serviceName, "start")
+		err = installer.SystemctlRun(profileService, imageService, serviceName, "start")
 		if err != nil {
 			return err
 		}
@@ -427,7 +447,7 @@ func (fts *FleetTestSuite) processStateChangedOnTheHost(process string, state st
 		"process": process,
 	}).Trace("Stopping process on the service")
 
-	err := installer.SystemctlRun(profile, agentInstaller.Image, serviceName, "stop")
+	err := installer.SystemctlRun(profileService, imageService, serviceName, "stop")
 	if err != nil {
 		log.WithFields(log.Fields{
 			"action":  state,
@@ -1136,7 +1156,10 @@ func deployAgentToFleet(agentInstaller installer.ElasticAgentInstaller, deployer
 	// we are setting the container name because Centos service could be reused by any other test suite
 	common.ProfileEnv[envVarsPrefix+"ContainerName"] = containerName
 
-	services := []string{profile, service}
+	services := []deploy.ServiceRequest{
+		deploy.NewServiceRequest(profile),
+		deploy.NewServiceRequest(common.ElasticAgentServiceName).WithFlavour(agentInstaller.Image),
+	}
 	err := deployer.Add(services, common.ProfileEnv)
 	if err != nil {
 		log.WithFields(log.Fields{
@@ -1150,7 +1173,7 @@ func deployAgentToFleet(agentInstaller installer.ElasticAgentInstaller, deployer
 	targetFile := "/"
 
 	// copy downloaded agent to the root dir of the container
-	err = docker.CopyFileToContainer(context.Background(), containerName, agentInstaller.BinaryPath, targetFile, isTar)
+	err = deploy.CopyFileToContainer(context.Background(), containerName, agentInstaller.BinaryPath, targetFile, isTar)
 	if err != nil {
 		return nil, err
 	}
@@ -1217,16 +1240,17 @@ func inputs(integration string) []kibana.Input {
 }
 
 func (fts *FleetTestSuite) getContainerLogs() error {
-	serviceManager := compose.NewServiceManager()
+	serviceManager := deploy.NewServiceManager()
 
-	profile := common.FleetProfileName
+	agentInstaller := fts.getInstaller()
+	profile := deploy.NewServiceRequest(common.FleetProfileName)
 	serviceName := common.ElasticAgentServiceName
 
-	composes := []string{
-		profile,     // profile name
-		serviceName, // agent service
+	services := []deploy.ServiceRequest{
+		profile, // profile name
+		deploy.NewServiceRequest(serviceName).WithFlavour(agentInstaller.Image), // agent service
 	}
-	err := serviceManager.RunCommand(profile, composes, []string{"logs", serviceName}, common.ProfileEnv)
+	err := serviceManager.RunCommand(profile, services, []string{"logs", serviceName}, common.ProfileEnv)
 	if err != nil {
 		log.WithFields(log.Fields{
 			"error":   err,

e2e/_suites/fleet/ingest_manager_test.go

@@ -13,7 +13,6 @@ import (
 	"github.com/elastic/e2e-testing/cli/config"
 	"github.com/elastic/e2e-testing/internal/common"
 	"github.com/elastic/e2e-testing/internal/deploy"
-	"github.com/elastic/e2e-testing/internal/docker"
 	"github.com/elastic/e2e-testing/internal/installer"
 	"github.com/elastic/e2e-testing/internal/kibana"
 	"github.com/elastic/e2e-testing/internal/shell"
@@ -100,7 +99,7 @@ func InitializeIngestManagerTestSuite(ctx *godog.TestSuiteContext) {
 				"docker.elastic.co/observability-ci/kibana:" + common.KibanaVersion,
 				"docker.elastic.co/observability-ci/kibana-ubi8:" + common.KibanaVersion,
 			}
-			docker.PullImages(images)
+			deploy.PullImages(images)
 		}
 
 		deployer := deploy.New(common.Provider)