refactor testing/integration and pkg/testing (#3378)

* pkg/testing/tools
  - split the 'tools' into smaller packages with well defined scope: 'estools', 'fleettools' and 'check'
  - adjust method names for consistency

* testing/integration:
  - make diagnostics to require to be run on a remote host
  - add more context to errors
  - reorder functions on endpoint_security_test so tests are the first functions and when the test fails because the 'Endpoint' directory isn't removed, it now prints the content of the directory

(cherry picked from commit 2201b6d)

# Conflicts:
#	pkg/testing/tools/cmd.go
#	pkg/testing/tools/fleettools/fleet.go
#	pkg/testing/tools/tools.go
#	testing/integration/upgrade_fleet_test.go

pkg/component/runtime/manager_test.go

@@ -1042,52 +1042,59 @@ func TestManager_FakeInput_GoodUnitToBad(t *testing.T) {
 				t.Logf("component state changed: %+v", state)
 				if state.State == client.UnitStateFailed {
 					subErrCh <- fmt.Errorf("component failed: %s", state.Message)
+					return
+				}
+
+				unit, ok := state.Units[ComponentUnitKey{UnitType: client.UnitTypeInput, UnitID: "good-input"}]
+				if !ok {
+					subErrCh <- errors.New("unit missing: good-input")
+					return
+				}
+				if unitGood {
+					switch unit.State {
+					case client.UnitStateFailed:
+						subErrCh <- fmt.Errorf("unit failed: %s", unit.Message)
+						return
+					case client.UnitStateHealthy:
+						// good unit it; now make it bad
+						t.Logf("marking good-input as having a hard-error for config")
+						updatedComp := comp
+						updatedComp.Units = make([]component.Unit, len(comp.Units))
+						copy(updatedComp.Units, comp.Units)
+						updatedComp.Units[1] = component.Unit{
+							ID:   "good-input",
+							Type: client.UnitTypeInput,
+							Err:  errors.New("hard-error for config"),
+						}
+						unitGood = false
+						err := m.Update(component.Model{Components: []component.Component{updatedComp}})
+						if err != nil {
+							subErrCh <- err
+						}
+					case client.UnitStateStarting:
+						// acceptable
+					default:
+						// unknown state that should not have occurred
+						subErrCh <- fmt.Errorf("unit reported unexpected state: %v", unit.State)
+					}
 				} else {
-					unit, ok := state.Units[ComponentUnitKey{UnitType: client.UnitTypeInput, UnitID: "good-input"}]
-					if ok {
-						if unitGood {
-							if unit.State == client.UnitStateFailed {
-								subErrCh <- fmt.Errorf("unit failed: %s", unit.Message)
-							} else if unit.State == client.UnitStateHealthy {
-								// good unit it; now make it bad
-								t.Logf("marking good-input as having a hard-error for config")
-								updatedComp := comp
-								updatedComp.Units = make([]component.Unit, len(comp.Units))
-								copy(updatedComp.Units, comp.Units)
-								updatedComp.Units[1] = component.Unit{
-									ID:   "good-input",
-									Type: client.UnitTypeInput,
-									Err:  errors.New("hard-error for config"),
-								}
-								unitGood = false
-								err := m.Update(component.Model{Components: []component.Component{updatedComp}})
-								if err != nil {
-									subErrCh <- err
-								}
-							} else if unit.State == client.UnitStateStarting {
-								// acceptable
-							} else {
-								// unknown state that should not have occurred
-								subErrCh <- fmt.Errorf("unit reported unexpected state: %v", unit.State)
-							}
-						} else {
-							if unit.State == client.UnitStateFailed {
-								// went to failed; stop whole component
-								err := m.Update(component.Model{Components: []component.Component{}})
-								if err != nil {
-									subErrCh <- err
-								}
-							} else if unit.State == client.UnitStateStopped {
-								// unit was stopped
-								subErrCh <- nil
-							} else {
-								subErrCh <- errors.New("good-input unit should be either failed or stopped")
-							}
+					switch unit.State {
+					case client.UnitStateFailed:
+						// went to failed; stop whole component
+						err := m.Update(component.Model{Components: []component.Component{}})
+						if err != nil {
+							subErrCh <- err
 						}
-					} else {
-						subErrCh <- errors.New("unit missing: good-input")
+					case client.UnitStateStopped:
+						// unit was stopped
+						subErrCh <- nil
+					default:
+						subErrCh <- fmt.Errorf(
+							"good-input unit should be either failed or stopped, but it's '%s'",
+							unit.State)
 					}
 				}
+
 			}
 		}
 	}()

pkg/testing/fixture_install.go

@@ -83,8 +83,12 @@ func (i InstallOpts) toCmdArgs() []string {
 	return args
 }
 
-// Install installs the prepared Elastic Agent binary and returns:
-//   - the combined output of stdout and stderr
+// Install installs the prepared Elastic Agent binary and registers a t.Cleanup
+// function to uninstall the agent if it hasn't been uninstalled. It also takes
+// care of collecting a diagnostics when AGENT_COLLECT_DIAG=true or the test
+// has failed.
+// It returns:
+//   - the combined output of Install command stdout and stderr
 //   - an error if any.
 func (f *Fixture) Install(ctx context.Context, installOpts *InstallOpts, opts ...process.CmdOption) ([]byte, error) {
 	f.t.Logf("[test %s] Inside fixture install function", f.t.Name())
@@ -133,12 +137,12 @@ func (f *Fixture) Install(ctx context.Context, installOpts *InstallOpts, opts ..
 
 		// environment variable AGENT_KEEP_INSTALLED=true will skip the uninstall
 		// useful to debug the issue with the Elastic Agent
-		if f.t.Failed() && keepInstalled() {
+		if f.t.Failed() && keepInstalledFlag() {
 			f.t.Logf("skipping uninstall; test failed and AGENT_KEEP_INSTALLED=true")
 			return
 		}
 
-		if keepInstalled() {
+		if keepInstalledFlag() {
 			f.t.Logf("ignoring AGENT_KEEP_INSTALLED=true as test succeeded, " +
 				"keeping the agent installed will jeopardise other tests")
 		}
@@ -303,7 +307,7 @@ func collectDiagFlag() bool {
 	return v
 }
 
-func keepInstalled() bool {
+func keepInstalledFlag() bool {
 	// failure reports false (ignore error)
 	v, _ := strconv.ParseBool(os.Getenv("AGENT_KEEP_INSTALLED"))
 	return v

pkg/testing/tools/artifacts_api.go

@@ -135,7 +135,7 @@ type ArtifactAPIClient struct {
 	url string
 }
 
-// Creates a new Artifact API client
+// NewArtifactAPIClient creates a new Artifact API client
 func NewArtifactAPIClient(opts ...ArtifactAPIClientOpt) *ArtifactAPIClient {
 	c := &ArtifactAPIClient{
 		url: defaultArtifactAPIURL,
@@ -204,7 +204,7 @@ func (aac ArtifactAPIClient) GetBuildDetails(ctx context.Context, version string
 	return checkResponseAndUnmarshal[BuildDetails](resp)
 }
 
-func (aac *ArtifactAPIClient) composeURL(relativePath string) (string, error) {
+func (aac ArtifactAPIClient) composeURL(relativePath string) (string, error) {
 	joinedURL, err := url.JoinPath(aac.url, relativePath)
 	if err != nil {
 		return "", fmt.Errorf("composing URL with %q %q: %w", aac.url, relativePath, err)
@@ -213,7 +213,7 @@ func (aac *ArtifactAPIClient) composeURL(relativePath string) (string, error) {
 	return joinedURL, nil
 }
 
-func (aac *ArtifactAPIClient) createAndPerformRequest(ctx context.Context, URL string) (*http.Response, error) {
+func (aac ArtifactAPIClient) createAndPerformRequest(ctx context.Context, URL string) (*http.Response, error) {
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, URL, nil)
 	if err != nil {
 		err = fmt.Errorf("composing request: %w", err)

pkg/testing/tools/check/check.go

@@ -11,14 +11,16 @@ import (
 
 	"github.com/stretchr/testify/assert"
 
+	"github.com/elastic/elastic-agent-libs/kibana"
 	"github.com/elastic/elastic-agent/pkg/control/v2/cproto"
 	integrationtest "github.com/elastic/elastic-agent/pkg/testing"
+	"github.com/elastic/elastic-agent/pkg/testing/tools/fleettools"
 )
 
 // ConnectedToFleet checks if the agent defined in the fixture is connected to
 // Fleet Server. It uses assert.Eventually and if it fails the last error will
 // be printed. It returns if the agent is connected to Fleet Server or not.
-func ConnectedToFleet(t *testing.T, fixture *integrationtest.Fixture) bool {
+func ConnectedToFleet(t *testing.T, fixture *integrationtest.Fixture, timeout time.Duration) bool {
 	t.Helper()
 
 	var err error
@@ -28,7 +30,7 @@ func ConnectedToFleet(t *testing.T, fixture *integrationtest.Fixture) bool {
 		return agentStatus.FleetState == int(cproto.State_HEALTHY)
 	}
 
-	connected := assert.Eventually(t, assertFn, 5*time.Minute, 5*time.Second,
+	connected := assert.Eventually(t, assertFn, timeout, 5*time.Second,
 		"want fleet state %s, got %s. agent status: %v",
 		cproto.State_HEALTHY, cproto.State(agentStatus.FleetState), agentStatus)
 
@@ -39,3 +41,26 @@ func ConnectedToFleet(t *testing.T, fixture *integrationtest.Fixture) bool {
 
 	return connected
 }
+
+// FleetAgentStatus returns a niladic function that returns true if the agent
+// has reached expectedStatus; false otherwise. The returned function is intended
+// for use with assert.Eventually or require.Eventually.
+func FleetAgentStatus(t *testing.T,
+	client *kibana.Client,
+	policyID,
+	expectedStatus string) func() bool {
+	return func() bool {
+		currentStatus, err := fleettools.GetAgentStatus(client, policyID)
+		if err != nil {
+			t.Errorf("unable to determine agent status: %s", err.Error())
+			return false
+		}
+
+		if currentStatus == expectedStatus {
+			return true
+		}
+
+		t.Logf("Agent fleet status: %s", currentStatus)
+		return false
+	}
+}

pkg/testing/tools/elasticsearch.go → pkg/testing/tools/estools/elasticsearch.go

@@ -2,7 +2,7 @@
 // or more contributor license agreements. Licensed under the Elastic License;
 // you may not use this file except in compliance with the Elastic License.
 
-package tools
+package estools
 
 import (
 	"bytes"

pkg/testing/tools/agents.go → pkg/testing/tools/fleettools/fleet.go

@@ -2,21 +2,15 @@
 // or more contributor license agreements. Licensed under the Elastic License;
 // you may not use this file except in compliance with the Elastic License.
 
-package tools
+package fleettools
 
 import (
 	"context"
 	"errors"
 	"fmt"
 	"os"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/require"
 
 	"github.com/elastic/elastic-agent-libs/kibana"
-	"github.com/elastic/elastic-agent/pkg/control/v2/client"
-	"github.com/elastic/elastic-agent/pkg/control/v2/cproto"
 )
 
 // GetAgentByPolicyIDAndHostnameFromList get an agent by the local_metadata.host.name property, reading from the agents list
@@ -47,6 +41,14 @@ func GetAgentByPolicyIDAndHostnameFromList(client *kibana.Client, policyID, host
 	return hostnameAgents[0], nil
 }
 
+func GetAgentIDByHostname(client *kibana.Client, policyID, hostname string) (string, error) {
+	agent, err := GetAgentByPolicyIDAndHostnameFromList(client, policyID, hostname)
+	if err != nil {
+		return "", err
+	}
+	return agent.Agent.ID, nil
+}
+
 func GetAgentStatus(client *kibana.Client, policyID string) (string, error) {
 	hostname, err := os.Hostname()
 	if err != nil {
@@ -97,6 +99,7 @@ func UnEnrollAgent(client *kibana.Client, policyID string) error {
 	return nil
 }
 
+<<<<<<< HEAD:pkg/testing/tools/agents.go
 func GetAgentIDByHostname(client *kibana.Client, policyID, hostname string) (string, error) {
 	agent, err := GetAgentByPolicyIDAndHostnameFromList(client, policyID, hostname)
 	if err != nil {
@@ -106,6 +109,10 @@ func GetAgentIDByHostname(client *kibana.Client, policyID, hostname string) (str
 }
 
 func UpgradeAgent(client *kibana.Client, policyID, version string) error {
+=======
+func UpgradeAgent(client *kibana.Client, policyID, version string, force bool) error {
+	// TODO: fix me: this does not work if FQDN is enabled
+>>>>>>> 2201b6d549 (refactor testing/integration and pkg/testing (#3378)):pkg/testing/tools/fleettools/fleet.go
 	hostname, err := os.Hostname()
 	if err != nil {
 		return err
@@ -127,7 +134,7 @@ func UpgradeAgent(client *kibana.Client, policyID, version string) error {
 	return nil
 }
 
-func GetDefaultFleetServerURL(client *kibana.Client) (string, error) {
+func DefaultURL(client *kibana.Client) (string, error) {
 	req := kibana.ListFleetServerHostsRequest{}
 	resp, err := client.ListFleetServerHosts(context.Background(), req)
 	if err != nil {
@@ -143,23 +150,5 @@ func GetDefaultFleetServerURL(client *kibana.Client) (string, error) {
 		}
 	}
 
-	return "", errors.New("unable to determine default fleet server host")
-}
-
-func WaitForAgent(ctx context.Context, t *testing.T, c client.Client) {
-	require.Eventually(t, func() bool {
-		err := c.Connect(ctx)
-		if err != nil {
-			t.Logf("connecting client to agent: %v", err)
-			return false
-		}
-		defer c.Disconnect()
-		state, err := c.State(ctx)
-		if err != nil {
-			t.Logf("error getting the agent state: %v", err)
-			return false
-		}
-		t.Logf("agent state: %+v", state)
-		return state.State == cproto.State_HEALTHY
-	}, 2*time.Minute, 10*time.Second, "Agent never became healthy")
+	return "", errors.New("unable to determine default fleet server URL")
 }