feat: support flavours in services, specially in the elastic-agent (#…

…1162)

* chore: move compose to deploy package

* feat: use a ServiceRequest when adding services

* feat: add service flavour support

* chore: remove unused centos/debian services

* fixup: add service flavour

* chore: move docker client to the deploy package

We will need another abstraction to represent the Docker client operations,
as it's clear what is a deployment and what is an operation in the deployment.
Maybe a Client struct for each provider will help out in differenciate it

* chore: use ServiceRequest everywhere

* chore: run agent commands with a ServiceRequest

* chore: use ServiceRequest in metricbeat test suite

* chore: pass flavours to installers

* chore: add a step to install the agent for the underlying OS

* chore: always add flavour

* fix: use installer for fleet_mode when removing services at the end of the scenario

* fix: update broken references in metricbeat test suite

* fix: update broken references in helm test suite

* fix: standalone does not have an installer

* fix: use service instead of image to get a service request for the agent

* feat: support for scaling services in compose

* fix: run second agent using compose scale option

* fix: update kibana's default Docker namespace

cli/cmd/deploy.go

@@ -8,7 +8,7 @@ import (
 	"context"
 
 	"github.com/elastic/e2e-testing/cli/config"
-	"github.com/elastic/e2e-testing/internal/compose"
+	"github.com/elastic/e2e-testing/internal/deploy"
 	log "github.com/sirupsen/logrus"
 
 	"github.com/spf13/cobra"
@@ -63,12 +63,16 @@ func buildDeployServiceCommand(srv string) *cobra.Command {
 		Short: `Deploys a ` + srv + ` service`,
 		Long:  `Deploys a ` + srv + ` service, adding it to a running profile, identified by its name`,
 		Run: func(cmd *cobra.Command, args []string) {
-			serviceManager := compose.NewServiceManager()
+			serviceManager := deploy.NewServiceManager()
 
 			env := map[string]string{}
 			env = config.PutServiceEnvironment(env, srv, versionToRun)
 
-			err := serviceManager.AddServicesToCompose(context.Background(), deployToProfile, []string{srv}, env)
+			err := serviceManager.AddServicesToCompose(
+				context.Background(),
+				deploy.NewServiceRequest(deployToProfile),
+				[]deploy.ServiceRequest{deploy.NewServiceRequest(srv)},
+				env)
 			if err != nil {
 				log.WithFields(log.Fields{
 					"profile":  deployToProfile,
@@ -85,12 +89,16 @@ func buildUndeployServiceCommand(srv string) *cobra.Command {
 		Short: `Undeploys a ` + srv + ` service`,
 		Long:  `Undeploys a ` + srv + ` service, removing it from a running profile, identified by its name`,
 		Run: func(cmd *cobra.Command, args []string) {
-			serviceManager := compose.NewServiceManager()
+			serviceManager := deploy.NewServiceManager()
 
 			env := map[string]string{}
 			env = config.PutServiceEnvironment(env, srv, versionToRun)
 
-			err := serviceManager.RemoveServicesFromCompose(context.Background(), deployToProfile, []string{srv}, env)
+			err := serviceManager.RemoveServicesFromCompose(
+				context.Background(),
+				deploy.NewServiceRequest(deployToProfile),
+				[]deploy.ServiceRequest{deploy.NewServiceRequest(srv)},
+				env)
 			if err != nil {
 				log.WithFields(log.Fields{
 					"profile":  deployToProfile,

cli/cmd/run.go

@@ -10,7 +10,7 @@ import (
 	"strings"
 
 	"github.com/elastic/e2e-testing/cli/config"
-	"github.com/elastic/e2e-testing/internal/compose"
+	"github.com/elastic/e2e-testing/internal/deploy"
 	log "github.com/sirupsen/logrus"
 
 	"github.com/spf13/cobra"
@@ -64,7 +64,7 @@ func buildRunServiceCommand(srv string) *cobra.Command {
 		Short: `Runs a ` + srv + ` service`,
 		Long:  `Runs a ` + srv + ` service, spinning up a Docker container for it and exposing its internal configuration so that you are able to connect to it in an easy manner`,
 		Run: func(cmd *cobra.Command, args []string) {
-			serviceManager := compose.NewServiceManager()
+			serviceManager := deploy.NewServiceManager()
 
 			env := config.PutServiceEnvironment(map[string]string{}, srv, versionToRun)
 
@@ -76,7 +76,8 @@ func buildRunServiceCommand(srv string) *cobra.Command {
 				env[k] = v
 			}
 
-			err := serviceManager.RunCompose(context.Background(), false, []string{srv}, env)
+			err := serviceManager.RunCompose(
+				context.Background(), false, []deploy.ServiceRequest{deploy.NewServiceRequest(srv)}, env)
 			if err != nil {
 				log.WithFields(log.Fields{
 					"service": srv,
@@ -96,7 +97,7 @@ Example:
   go run main.go run profile fleet -s elastic-agent:8.0.0-SNAPSHOT
 `,
 		Run: func(cmd *cobra.Command, args []string) {
-			serviceManager := compose.NewServiceManager()
+			serviceManager := deploy.NewServiceManager()
 
 			env := map[string]string{
 				"profileVersion": versionToRun,
@@ -110,14 +111,15 @@ Example:
 				env[k] = v
 			}
 
-			err := serviceManager.RunCompose(context.Background(), true, []string{key}, env)
+			err := serviceManager.RunCompose(
+				context.Background(), true, []deploy.ServiceRequest{deploy.NewServiceRequest(key)}, env)
 			if err != nil {
 				log.WithFields(log.Fields{
 					"profile": key,
 				}).Error("Could not run the profile.")
 			}
 
-			composeNames := []string{}
+			composeNames := []deploy.ServiceRequest{}
 			if len(servicesToRun) > 0 {
 				for _, srv := range servicesToRun {
 					arr := strings.Split(srv, ":")
@@ -137,10 +139,10 @@ Example:
 					}).Trace("Adding service")
 
 					env = config.PutServiceEnvironment(env, image, tag)
-					composeNames = append(composeNames, image)
+					composeNames = append(composeNames, deploy.NewServiceRequest(image))
 				}
 
-				err = serviceManager.AddServicesToCompose(context.Background(), key, composeNames, env)
+				err = serviceManager.AddServicesToCompose(context.Background(), deploy.NewServiceRequest(key), composeNames, env)
 				if err != nil {
 					log.WithFields(log.Fields{
 						"profile":  key,

cli/cmd/stop.go

@@ -8,7 +8,7 @@ import (
 	"context"
 
 	"github.com/elastic/e2e-testing/cli/config"
-	"github.com/elastic/e2e-testing/internal/compose"
+	"github.com/elastic/e2e-testing/internal/deploy"
 	log "github.com/sirupsen/logrus"
 
 	"github.com/spf13/cobra"
@@ -55,9 +55,10 @@ func buildStopServiceCommand(srv string) *cobra.Command {
 		Short: `Stops a ` + srv + ` service`,
 		Long:  `Stops a ` + srv + ` service, stoppping its Docker container`,
 		Run: func(cmd *cobra.Command, args []string) {
-			serviceManager := compose.NewServiceManager()
+			serviceManager := deploy.NewServiceManager()
 
-			err := serviceManager.StopCompose(context.Background(), false, []string{srv})
+			err := serviceManager.StopCompose(
+				context.Background(), false, []deploy.ServiceRequest{deploy.NewServiceRequest(srv)})
 			if err != nil {
 				log.WithFields(log.Fields{
 					"service": srv,
@@ -73,9 +74,10 @@ func buildStopProfileCommand(key string, profile config.Profile) *cobra.Command
 		Short: `Stops the ` + profile.Name + ` profile`,
 		Long:  `Stops the ` + profile.Name + ` profile, stopping the Services that compound it`,
 		Run: func(cmd *cobra.Command, args []string) {
-			serviceManager := compose.NewServiceManager()
+			serviceManager := deploy.NewServiceManager()
 
-			err := serviceManager.StopCompose(context.Background(), true, []string{key})
+			err := serviceManager.StopCompose(
+				context.Background(), true, []deploy.ServiceRequest{deploy.NewServiceRequest(key)})
 			if err != nil {
 				log.WithFields(log.Fields{
 					"profile": key,

cli/config/compose/profiles/fleet/docker-compose.yml

@@ -29,7 +29,7 @@ services:
       test: "curl -f http://localhost:5601/login | grep kbn-injected-metadata 2>&1 >/dev/null"
       retries: 600
       interval: 1s
-    image: "docker.elastic.co/${kibanaDockerNamespace:-beats}/kibana:${kibanaVersion:-8.0.0-SNAPSHOT}"
+    image: "docker.elastic.co/${kibanaDockerNamespace:-kibana}/kibana:${kibanaVersion:-8.0.0-SNAPSHOT}"
     ports:
       - "5601:5601"
     volumes:

cli/config/compose/services/centos-systemd/docker-compose.yml → cli/config/compose/services/elastic-agent/centos/docker-compose.yml

@@ -1,6 +1,6 @@
 version: '2.4'
 services:
-  centos-systemd:
+  elastic-agent:
     image: centos/systemd:${centos_systemdTag:-latest} 
     container_name: ${centos_systemdContainerName}
     entrypoint: "/usr/sbin/init"

cli/config/compose/services/debian-systemd/docker-compose.yml → cli/config/compose/services/elastic-agent/debian/docker-compose.yml

@@ -1,6 +1,6 @@
 version: '2.4'
 services:
-  debian-systemd:
+  elastic-agent:
     image: alehaa/debian-systemd:${debian_systemdTag:-stretch}
     container_name: ${debian_systemdContainerName} 
     entrypoint: "/sbin/init"

cli/config/config.go

@@ -73,16 +73,14 @@ func FileExists(configFile string) (bool, error) {
 
 // GetComposeFile returns the path of the compose file, looking up the
 // tool's workdir
-func GetComposeFile(isProfile bool, composeName string, composeFileName ...string) (string, error) {
-	if isProfile || composeFileName == nil || composeFileName[0] == "" {
-		composeFileName = []string{"docker-compose.yml"}
-	}
+func GetComposeFile(isProfile bool, composeName string) (string, error) {
+	composeFileName := "docker-compose.yml"
 	serviceType := "services"
 	if isProfile {
 		serviceType = "profiles"
 	}
 
-	composeFilePath := path.Join(Op.Workspace, "compose", serviceType, composeName, composeFileName[0])
+	composeFilePath := path.Join(Op.Workspace, "compose", serviceType, composeName, composeFileName)
 	found, err := io.Exists(composeFilePath)
 	if found && err == nil {
 		log.WithFields(log.Fields{

e2e/_suites/fleet/features/backend_processes.feature

@@ -21,7 +21,7 @@ Examples: Debian
 
 @enroll
 Scenario Outline: Deploying the <os> agent with enroll and then run on rpm and deb
-  Given a "<os>" agent is deployed to Fleet with "systemd" installer
+  Given a "<os>" agent is deployed to Fleet
   When the "elastic-agent" process is in the "started" state on the host
   Then there are "2" instances of the "filebeat" process in the "started" state
     And there are "2" instances of the "metricbeat" process in the "started" state

e2e/_suites/fleet/features/fleet_mode_agent.feature

@@ -21,7 +21,7 @@ Examples: Debian
 
 @enroll
 Scenario Outline: Deploying the <os> agent with enroll and then run on rpm and deb
-  Given a "<os>" agent is deployed to Fleet with "systemd" installer
+  Given a "<os>" agent is deployed to Fleet
   When the "elastic-agent" process is in the "started" state on the host
   Then the agent is listed in Fleet as "online"
     And system package dashboards are listed in Fleet