v2 refactor (#1008)

* Move kibana into internals, update fleet test suite
* migrate docker-compose related code to internal layout
* move docker related code to internal layout
* move git related code to internal layout
* move common attributes into internal common file system layout
* move elasticsearch specifics into its own filesystem layout
* move installer based code to internal layout
* move shell related code to internal layout
* move sanitizer code to internal layout
* move io related code to internal layout
* move utils into internal layout
* Update package integration querying/altering
* move curl to internal layout
* move helm to internal layout
* move kubectl into internal layout
* move state internal filesystem
* cleanup config in stand-alone
* remove unused files
* Uniquify the stand-alone step for checking agent status (#993)

There were 2 steps identical in both the stand-alone and fleet test suites.
Running the stand-alone test suite was picking up the step from the fleet test
suite and trying to reference the FleetTestSuite structure which did not hold
any of the agent information (like the hostname) for the stand alone tests.

This fixes it so that the standalone test step is being referenced in the
correct test suite.

* Remove the agent config file parameters for stand alone (#983)
* Update helm/metricbeat tests to use new layout
* Fix policy endpoint update
* fix panic on helm init
* Fix step reference as this being merged seperately
* Update function call to correct standalone step
* Fix merge conflict
* update ProfileEnv query/set for KibanaVersion
* More fixes to agent endpoint security checks
* update backend feature to call out endpoint in step
* use common.TimeoutFactor in docker checkprocess state
* Update adding endpoint integration
* enable features for fleet server
* not necessary to enroll after install
* wait for filebeat/metricbeat before restarts
* clear out fts.CurrentToken during beforeScenario
* attach system integration on deploy
* enroll if rpm
* dont store fleet policy
* update kibana config for latest fleet server
* Update e2e/_suites/fleet/fleet.go
* Update e2e/_suites/fleet/fleet.go
* Update e2e/_suites/fleet/fleet.go
* Update .pre-commit-config.yaml
* Update e2e/Makefile
* rename apt -> deb for installer type
* execute docker start/stop with timeout between
* fixes fleet_server scenario
* Utilize fleet server in all tests
* Fix enrollment url for fleet server
* Query elasticsearch logs for endpoint security event changes
* Increase search result size for ES
* Fix issue with fleet server restarting continuously
* unpin kibana pr now that most major breakage is resolved
* force unenroll
* for new fleet bootstrap on re-enrollment
* Fix unenrollment
* Add timeout safeguard to elastic-agent execution

In some cases such as attempting to re-enroll with a revoked token, the
elastic-agent will retry indefinitely. This fix adds a safeguard utilizing
'timeout' command prepended to the elastic-agent command so that it will timeout
after TimeoutFactor

Signed-off-by: Adam Stokes <[email protected]>
Co-authored-by: Manuel de la Peña <[email protected]>
(cherry picked from commit 5f59670)

# Conflicts:
#	e2e/Makefile
#	e2e/_suites/fleet/ingest_manager_test.go
#	e2e/_suites/fleet/stand-alone.go
#	e2e/_suites/fleet/world.go

cli/cmd/deploy.go

@@ -8,7 +8,7 @@ import (
 	"context"
 
 	"github.com/elastic/e2e-testing/cli/config"
-	"github.com/elastic/e2e-testing/cli/services"
+	"github.com/elastic/e2e-testing/internal/compose"
 	log "github.com/sirupsen/logrus"
 
 	"github.com/spf13/cobra"
@@ -63,7 +63,7 @@ func buildDeployServiceCommand(srv string) *cobra.Command {
 		Short: `Deploys a ` + srv + ` service`,
 		Long:  `Deploys a ` + srv + ` service, adding it to a running profile, identified by its name`,
 		Run: func(cmd *cobra.Command, args []string) {
-			serviceManager := services.NewServiceManager()
+			serviceManager := compose.NewServiceManager()
 
 			env := map[string]string{}
 			env = config.PutServiceEnvironment(env, srv, versionToRun)
@@ -85,7 +85,7 @@ func buildUndeployServiceCommand(srv string) *cobra.Command {
 		Short: `Undeploys a ` + srv + ` service`,
 		Long:  `Undeploys a ` + srv + ` service, removing it from a running profile, identified by its name`,
 		Run: func(cmd *cobra.Command, args []string) {
-			serviceManager := services.NewServiceManager()
+			serviceManager := compose.NewServiceManager()
 
 			env := map[string]string{}
 			env = config.PutServiceEnvironment(env, srv, versionToRun)

cli/cmd/run.go

@@ -9,7 +9,7 @@ import (
 	"strings"
 
 	"github.com/elastic/e2e-testing/cli/config"
-	"github.com/elastic/e2e-testing/cli/services"
+	"github.com/elastic/e2e-testing/internal/compose"
 	log "github.com/sirupsen/logrus"
 
 	"github.com/spf13/cobra"
@@ -63,7 +63,7 @@ func buildRunServiceCommand(srv string) *cobra.Command {
 		Short: `Runs a ` + srv + ` service`,
 		Long:  `Runs a ` + srv + ` service, spinning up a Docker container for it and exposing its internal configuration so that you are able to connect to it in an easy manner`,
 		Run: func(cmd *cobra.Command, args []string) {
-			serviceManager := services.NewServiceManager()
+			serviceManager := compose.NewServiceManager()
 
 			env := config.PutServiceEnvironment(map[string]string{}, srv, versionToRun)
 
@@ -91,7 +91,7 @@ func buildRunProfileCommand(key string, profile config.Profile) *cobra.Command {
 		Short: `Runs the ` + profile.Name + ` profile`,
 		Long:  `Runs the ` + profile.Name + ` profile, spinning up the Services that compound it`,
 		Run: func(cmd *cobra.Command, args []string) {
-			serviceManager := services.NewServiceManager()
+			serviceManager := compose.NewServiceManager()
 
 			env := map[string]string{
 				"profileVersion": versionToRun,

cli/cmd/stop.go

@@ -8,7 +8,7 @@ import (
 	"context"
 
 	"github.com/elastic/e2e-testing/cli/config"
-	"github.com/elastic/e2e-testing/cli/services"
+	"github.com/elastic/e2e-testing/internal/compose"
 	log "github.com/sirupsen/logrus"
 
 	"github.com/spf13/cobra"
@@ -55,7 +55,7 @@ func buildStopServiceCommand(srv string) *cobra.Command {
 		Short: `Stops a ` + srv + ` service`,
 		Long:  `Stops a ` + srv + ` service, stoppping its Docker container`,
 		Run: func(cmd *cobra.Command, args []string) {
-			serviceManager := services.NewServiceManager()
+			serviceManager := compose.NewServiceManager()
 
 			err := serviceManager.StopCompose(context.Background(), false, []string{srv})
 			if err != nil {
@@ -73,7 +73,7 @@ func buildStopProfileCommand(key string, profile config.Profile) *cobra.Command
 		Short: `Stops the ` + profile.Name + ` profile`,
 		Long:  `Stops the ` + profile.Name + ` profile, stopping the Services that compound it`,
 		Run: func(cmd *cobra.Command, args []string) {
-			serviceManager := services.NewServiceManager()
+			serviceManager := compose.NewServiceManager()
 
 			err := serviceManager.StopCompose(context.Background(), true, []string{key})
 			if err != nil {

cli/cmd/sync.go

@@ -13,9 +13,9 @@ import (
 	"strings"
 
 	"github.com/elastic/e2e-testing/cli/config"
-	git "github.com/elastic/e2e-testing/cli/internal"
-	io "github.com/elastic/e2e-testing/cli/internal"
-	"github.com/elastic/e2e-testing/cli/services"
+	git "github.com/elastic/e2e-testing/internal/git"
+	io "github.com/elastic/e2e-testing/internal/io"
+	"github.com/elastic/e2e-testing/internal/sanitizer"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 
@@ -197,7 +197,7 @@ func copyIntegrationsComposeFiles(beats git.Project, pattern string, target stri
 }
 
 type service interface{}
-type compose struct {
+type composeFile struct {
 	Version  string             `yaml:"version"`
 	Services map[string]service `yaml:"services"`
 }
@@ -212,7 +212,7 @@ func sanitizeComposeFile(composeFilePath string, targetFilePath string) error {
 		return err
 	}
 
-	c := compose{}
+	c := composeFile{}
 	err = yaml.Unmarshal(bytes, &c)
 	if err != nil {
 		log.WithFields(log.Fields{
@@ -276,7 +276,7 @@ func sanitizeConfigurationFile(serviceName string, configFilePath string) error
 	// prepend modules header
 	content = "metricbeat.modules:\n" + content
 
-	serviceSanitizer := services.GetConfigSanitizer(serviceName)
+	serviceSanitizer := sanitizer.GetConfigSanitizer(serviceName)
 	content = serviceSanitizer.Sanitize(content)
 
 	log.WithFields(log.Fields{

cli/cmd/sync_test.go

@@ -6,7 +6,7 @@ import (
 	"testing"
 
 	"github.com/Flaque/filet"
-	io "github.com/elastic/e2e-testing/cli/internal"
+	"github.com/elastic/e2e-testing/internal/io"
 	"github.com/stretchr/testify/assert"
 	"gopkg.in/yaml.v2"
 )
@@ -28,7 +28,7 @@ func TestSanitizeComposeFile_Multiple(t *testing.T) {
 	bytes, err := io.ReadFile(target)
 	assert.Nil(t, err)
 
-	c := compose{}
+	c := composeFile{}
 	err = yaml.Unmarshal(bytes, &c)
 	assert.Nil(t, err)
 
@@ -73,7 +73,7 @@ func TestSanitizeComposeFile_Single(t *testing.T) {
 	bytes, err := io.ReadFile(target)
 	assert.Nil(t, err)
 
-	c := compose{}
+	c := composeFile{}
 	err = yaml.Unmarshal(bytes, &c)
 	assert.Nil(t, err)
 

cli/config/config.go

@@ -11,8 +11,8 @@ import (
 	"path/filepath"
 	"strings"
 
-	io "github.com/elastic/e2e-testing/cli/internal"
-	shell "github.com/elastic/e2e-testing/cli/shell"
+	io "github.com/elastic/e2e-testing/internal/io"
+	shell "github.com/elastic/e2e-testing/internal/shell"
 
 	packr "github.com/gobuffalo/packr/v2"
 	homedir "github.com/mitchellh/go-homedir"

cli/config/config_test.go

@@ -11,7 +11,7 @@ import (
 	"strings"
 	"testing"
 
-	io "github.com/elastic/e2e-testing/cli/internal"
+	"github.com/elastic/e2e-testing/internal/io"
 
 	"github.com/Flaque/filet"
 	"github.com/sirupsen/logrus"