feat: instrument Helm charts test suite (elastic#858) (elastic#860)

* chore: add a span for waiting for elasticsearch

* chore: add a span for waiting for Kibana

* chore: add spans for Running/Stopping docker compose

* chore: log file name when loading a docker image from TAR

* chore: extract init APM dependencies to a steps package

* fix: enrich log for local APM services

* chore: enrich log for starting compose files

* fix: consistent span type for compose services

* chore: do not initialise helm test suite with Go init

* chore: pass context to shell.Execute

* feat: instrument helm charts test suite

* fix: use passed context

cli/cmd/run.go

@@ -64,7 +64,7 @@ func buildRunServiceCommand(srv string) *cobra.Command {
 
 			env := config.PutServiceEnvironment(map[string]string{}, srv, versionToRun)
 
-			err := serviceManager.RunCompose(false, []string{srv}, env)
+			err := serviceManager.RunCompose(context.Background(), false, []string{srv}, env)
 			if err != nil {
 				log.WithFields(log.Fields{
 					"service": srv,
@@ -86,7 +86,7 @@ func buildRunProfileCommand(key string, profile config.Profile) *cobra.Command {
 				"profileVersion": versionToRun,
 			}
 
-			err := serviceManager.RunCompose(true, []string{key}, env)
+			err := serviceManager.RunCompose(context.Background(), true, []string{key}, env)
 			if err != nil {
 				log.WithFields(log.Fields{
 					"profile": key,

cli/cmd/stop.go

@@ -5,6 +5,8 @@
 package cmd
 
 import (
+	"context"
+
 	"github.com/elastic/e2e-testing/cli/config"
 	"github.com/elastic/e2e-testing/cli/services"
 	log "github.com/sirupsen/logrus"
@@ -55,7 +57,7 @@ func buildStopServiceCommand(srv string) *cobra.Command {
 		Run: func(cmd *cobra.Command, args []string) {
 			serviceManager := services.NewServiceManager()
 
-			err := serviceManager.StopCompose(false, []string{srv})
+			err := serviceManager.StopCompose(context.Background(), false, []string{srv})
 			if err != nil {
 				log.WithFields(log.Fields{
 					"service": srv,
@@ -73,7 +75,7 @@ func buildStopProfileCommand(key string, profile config.Profile) *cobra.Command
 		Run: func(cmd *cobra.Command, args []string) {
 			serviceManager := services.NewServiceManager()
 
-			err := serviceManager.StopCompose(true, []string{key})
+			err := serviceManager.StopCompose(context.Background(), true, []string{key})
 			if err != nil {
 				log.WithFields(log.Fields{
 					"profile": key,

cli/config/compose/profiles/helm/docker-compose.yml

@@ -0,0 +1,14 @@
+version: '2.4'
+services:
+  elasticsearch:
+    environment:
+      - bootstrap.memory_lock=true
+      - discovery.type=single-node
+      - ES_JAVA_OPTS=-Xms512m -Xmx512m
+      - xpack.security.enabled=false
+      - xpack.monitoring.collection.enabled=true
+      - ELASTIC_USERNAME=elastic
+      - ELASTIC_PASSWORD=changeme
+    image: "docker.elastic.co/elasticsearch/elasticsearch:${stackVersion:-8.0.0-SNAPSHOT}"
+    ports:
+      - "9200:9200"

cli/docker/docker.go

@@ -201,6 +201,7 @@ func LoadImage(imagePath string) error {
 	}
 
 	log.WithFields(log.Fields{
+		"image":    fileNamePath,
 		"response": imageLoadResponse,
 	}).Debug("Docker image loaded successfully")
 	return nil

cli/services/helm.go

@@ -5,18 +5,20 @@
 package services
 
 import (
+	"context"
 	"errors"
 	"strings"
 
 	"github.com/elastic/e2e-testing/cli/shell"
 	log "github.com/sirupsen/logrus"
+	"go.elastic.co/apm"
 )
 
 // HelmManager defines the operations for Helm
 type HelmManager interface {
-	AddRepo(repo string, URL string) error
-	DeleteChart(chart string) error
-	InstallChart(name string, chart string, version string, flags []string) error
+	AddRepo(ctx context.Context, repo string, URL string) error
+	DeleteChart(ctx context.Context, chart string) error
+	InstallChart(ctx context.Context, name string, chart string, version string, flags []string) error
 }
 
 // HelmFactory returns oone of the Helm supported versions, or an error
@@ -39,13 +41,18 @@ type helm3X struct {
 	helm
 }
 
-func (h *helm3X) AddRepo(repo string, url string) error {
+func (h *helm3X) AddRepo(ctx context.Context, repo string, url string) error {
+	span, _ := apm.StartSpanOptions(ctx, "Adding Helm repository", "helm.repo.add", apm.SpanOptions{
+		Parent: apm.SpanFromContext(ctx).TraceContext(),
+	})
+	defer span.End()
+
 	// use chart as application name
 	args := []string{
 		"repo", "add", repo, url,
 	}
 
-	output, err := helmExecute(args...)
+	output, err := helmExecute(ctx, args...)
 	if err != nil {
 		return err
 	}
@@ -58,12 +65,17 @@ func (h *helm3X) AddRepo(repo string, url string) error {
 	return nil
 }
 
-func (h *helm3X) DeleteChart(chart string) error {
+func (h *helm3X) DeleteChart(ctx context.Context, chart string) error {
+	span, _ := apm.StartSpanOptions(ctx, "Deleting Helm chart", "helm.chart.delete", apm.SpanOptions{
+		Parent: apm.SpanFromContext(ctx).TraceContext(),
+	})
+	defer span.End()
+
 	args := []string{
 		"delete", chart,
 	}
 
-	output, err := helmExecute(args...)
+	output, err := helmExecute(ctx, args...)
 	if err != nil {
 		return err
 	}
@@ -75,13 +87,18 @@ func (h *helm3X) DeleteChart(chart string) error {
 	return nil
 }
 
-func (h *helm3X) InstallChart(name string, chart string, version string, flags []string) error {
+func (h *helm3X) InstallChart(ctx context.Context, name string, chart string, version string, flags []string) error {
+	span, _ := apm.StartSpanOptions(ctx, "Installing Helm chart", "helm.chart.install", apm.SpanOptions{
+		Parent: apm.SpanFromContext(ctx).TraceContext(),
+	})
+	defer span.End()
+
 	args := []string{
 		"install", name, chart, "--version", version,
 	}
 	args = append(args, flags...)
 
-	output, err := helmExecute(args...)
+	output, err := helmExecute(ctx, args...)
 	if err != nil {
 		return err
 	}
@@ -95,8 +112,8 @@ func (h *helm3X) InstallChart(name string, chart string, version string, flags [
 	return nil
 }
 
-func helmExecute(args ...string) (string, error) {
-	output, err := shell.Execute(".", "helm", args...)
+func helmExecute(ctx context.Context, args ...string) (string, error) {
+	output, err := shell.Execute(ctx, ".", "helm", args...)
 	if err != nil {
 		return "", err
 	}

cli/services/kibana.go

@@ -5,13 +5,15 @@
 package services
 
 import (
+	"context"
 	"fmt"
 	"strings"
 	"time"
 
 	backoff "github.com/cenkalti/backoff/v4"
 	curl "github.com/elastic/e2e-testing/cli/shell"
 	log "github.com/sirupsen/logrus"
+	"go.elastic.co/apm"
 )
 
 // KibanaBaseURL All URLs running on localhost as Kibana is expected to be exposed there
@@ -238,7 +240,7 @@ func (k *KibanaClient) UpdateIntegrationPackageConfig(packageConfigID string, pa
 
 // WaitForKibana waits for kibana running in localhost:5601 to be healthy, returning false
 // if kibana does not get healthy status in a defined number of minutes.
-func (k *KibanaClient) WaitForKibana(maxTimeoutMinutes time.Duration) (bool, error) {
+func (k *KibanaClient) WaitForKibana(ctx context.Context, maxTimeoutMinutes time.Duration) (bool, error) {
 	k.withURL("/status")
 
 	var (
@@ -259,6 +261,11 @@ func (k *KibanaClient) WaitForKibana(maxTimeoutMinutes time.Duration) (bool, err
 	retryCount := 1
 
 	kibanaStatus := func() error {
+		span, _ := apm.StartSpanOptions(ctx, "Health", "kibana.health", apm.SpanOptions{
+			Parent: apm.SpanFromContext(ctx).TraceContext(),
+		})
+		defer span.End()
+
 		r := curl.HTTPRequest{
 			BasicAuthUser:     "elastic",
 			BasicAuthPassword: "changeme",

cli/services/kubectl.go

@@ -5,10 +5,12 @@
 package services
 
 import (
+	"context"
 	"encoding/json"
 	"strings"
 
 	"github.com/elastic/e2e-testing/cli/shell"
+	"go.elastic.co/apm"
 )
 
 // Resource hide the real type of the enum
@@ -39,8 +41,13 @@ type Kubectl struct {
 }
 
 // GetStringResourcesBySelector Use kubectl to get a resource identified by a selector, return the resource in a string.
-func (k *Kubectl) GetStringResourcesBySelector(resourceType, selector string) (string, error) {
-	output, err := k.Run("get", resourceType, "--selector", selector, "-o", "json")
+func (k *Kubectl) GetStringResourcesBySelector(ctx context.Context, resourceType, selector string) (string, error) {
+	span, _ := apm.StartSpanOptions(ctx, "Getting String resource by selector", "kubectl.stringResources.getBySelector", apm.SpanOptions{
+		Parent: apm.SpanFromContext(ctx).TraceContext(),
+	})
+	defer span.End()
+
+	output, err := k.Run(ctx, "get", resourceType, "--selector", selector, "-o", "json")
 	if err != nil {
 		return "", err
 	}
@@ -49,8 +56,13 @@ func (k *Kubectl) GetStringResourcesBySelector(resourceType, selector string) (s
 }
 
 // GetResourcesBySelector Use kubectl to get a resource identified by a selector, return the resource in a map[string]interface{}.
-func (k *Kubectl) GetResourcesBySelector(resourceType, selector string) (map[string]interface{}, error) {
-	output, err := k.Run("get", resourceType, "--selector", selector, "-o", "json")
+func (k *Kubectl) GetResourcesBySelector(ctx context.Context, resourceType, selector string) (map[string]interface{}, error) {
+	span, _ := apm.StartSpanOptions(ctx, "Getting resource by selector", "kubectl.resources.getBySelector", apm.SpanOptions{
+		Parent: apm.SpanFromContext(ctx).TraceContext(),
+	})
+	defer span.End()
+
+	output, err := k.Run(ctx, "get", resourceType, "--selector", selector, "-o", "json")
 	if err != nil {
 		return nil, err
 	}
@@ -59,22 +71,32 @@ func (k *Kubectl) GetResourcesBySelector(resourceType, selector string) (map[str
 }
 
 // GetResourceJSONPath use kubectl to get a resource by type and name, and a jsonpath, and return the definition of the resource in JSON.
-func (k *Kubectl) GetResourceJSONPath(resourceType, resource, jsonPath string) (string, error) {
-	output, err := k.Run("get", resourceType, resource, "-o", "jsonpath='"+jsonPath+"'")
+func (k *Kubectl) GetResourceJSONPath(ctx context.Context, resourceType, resource, jsonPath string) (string, error) {
+	span, _ := apm.StartSpanOptions(ctx, "Getting resource by JSON path", "kubectl.resources.getByJSONPath", apm.SpanOptions{
+		Parent: apm.SpanFromContext(ctx).TraceContext(),
+	})
+	defer span.End()
+
+	output, err := k.Run(ctx, "get", resourceType, resource, "-o", "jsonpath='"+jsonPath+"'")
 	if err != nil {
 		return "{}", err
 	}
 	return output, nil
 }
 
 // GetResourceSelector use kubectl to get the selector for a resource identified by type and name.
-func (k *Kubectl) GetResourceSelector(resourceType, resource string) (string, error) {
-	output, err := k.GetResourceJSONPath(resourceType, resource, "{.metadata.selfLink}")
+func (k *Kubectl) GetResourceSelector(ctx context.Context, resourceType, resource string) (string, error) {
+	span, _ := apm.StartSpanOptions(ctx, "Getting resource selector", "kubectl.resources.getSelector", apm.SpanOptions{
+		Parent: apm.SpanFromContext(ctx).TraceContext(),
+	})
+	defer span.End()
+
+	output, err := k.GetResourceJSONPath(ctx, resourceType, resource, "{.metadata.selfLink}")
 	if err != nil {
 		return "", err
 	}
 
-	output, err = k.Run("get", "--raw", strings.ReplaceAll(output, "'", "")+"/scale")
+	output, err = k.Run(ctx, "get", "--raw", strings.ReplaceAll(output, "'", "")+"/scale")
 	if err != nil {
 		return "", err
 	}
@@ -89,8 +111,8 @@ func (k *Kubectl) GetResourceSelector(resourceType, resource string) (string, er
 }
 
 // Run a kubectl command and return the output
-func (k *Kubectl) Run(args ...string) (string, error) {
-	return shell.Execute(".", "kubectl", args...)
+func (k *Kubectl) Run(ctx context.Context, args ...string) (string, error) {
+	return shell.Execute(ctx, ".", "kubectl", args...)
 }
 
 // jsonToObj Converts a JSON string to a map[string]interface{}.

cli/services/manager.go

@@ -22,8 +22,8 @@ type ServiceManager interface {
 	AddServicesToCompose(ctx context.Context, profile string, composeNames []string, env map[string]string) error
 	RemoveServicesFromCompose(ctx context.Context, profile string, composeNames []string, env map[string]string) error
 	RunCommand(profile string, composeNames []string, composeArgs []string, env map[string]string) error
-	RunCompose(isProfile bool, composeNames []string, env map[string]string) error
-	StopCompose(isProfile bool, composeNames []string) error
+	RunCompose(ctx context.Context, isProfile bool, composeNames []string, env map[string]string) error
+	StopCompose(ctx context.Context, isProfile bool, composeNames []string) error
 }
 
 // DockerServiceManager implementation of the service manager interface
@@ -37,7 +37,7 @@ func NewServiceManager() ServiceManager {
 
 // AddServicesToCompose adds services to a running docker compose
 func (sm *DockerServiceManager) AddServicesToCompose(ctx context.Context, profile string, composeNames []string, env map[string]string) error {
-	span, _ := apm.StartSpanOptions(ctx, "Add services to Docker Compose", "docker-compose.service.add", apm.SpanOptions{
+	span, _ := apm.StartSpanOptions(ctx, "Add services to Docker Compose", "docker-compose.services.add", apm.SpanOptions{
 		Parent: apm.SpanFromContext(ctx).TraceContext(),
 	})
 	defer span.End()
@@ -65,7 +65,7 @@ func (sm *DockerServiceManager) AddServicesToCompose(ctx context.Context, profil
 
 // RemoveServicesFromCompose removes services from a running docker compose
 func (sm *DockerServiceManager) RemoveServicesFromCompose(ctx context.Context, profile string, composeNames []string, env map[string]string) error {
-	span, _ := apm.StartSpanOptions(ctx, "Remove services from Docker Compose", "docker-compose.service.remove", apm.SpanOptions{
+	span, _ := apm.StartSpanOptions(ctx, "Remove services from Docker Compose", "docker-compose.services.remove", apm.SpanOptions{
 		Parent: apm.SpanFromContext(ctx).TraceContext(),
 	})
 	defer span.End()
@@ -111,12 +111,22 @@ func (sm *DockerServiceManager) RunCommand(profile string, composeNames []string
 }
 
 // RunCompose runs a docker compose by its name
-func (sm *DockerServiceManager) RunCompose(isProfile bool, composeNames []string, env map[string]string) error {
+func (sm *DockerServiceManager) RunCompose(ctx context.Context, isProfile bool, composeNames []string, env map[string]string) error {
+	span, _ := apm.StartSpanOptions(ctx, "Starting Docker Compose files", "docker-compose.services.up", apm.SpanOptions{
+		Parent: apm.SpanFromContext(ctx).TraceContext(),
+	})
+	defer span.End()
+
 	return executeCompose(sm, isProfile, composeNames, []string{"up", "-d"}, env)
 }
 
 // StopCompose stops a docker compose by its name
-func (sm *DockerServiceManager) StopCompose(isProfile bool, composeNames []string) error {
+func (sm *DockerServiceManager) StopCompose(ctx context.Context, isProfile bool, composeNames []string) error {
+	span, _ := apm.StartSpanOptions(ctx, "Stopping Docker Compose files", "docker-compose.services.down", apm.SpanOptions{
+		Parent: apm.SpanFromContext(ctx).TraceContext(),
+	})
+	defer span.End()
+
 	composeFilePaths := make([]string, len(composeNames))
 	for i, composeName := range composeNames {
 		b := isProfile

cli/shell/shell.go

@@ -6,12 +6,14 @@ package shell
 
 import (
 	"bytes"
+	"context"
 	"os"
 	"os/exec"
 	"strconv"
 	"strings"
 
 	log "github.com/sirupsen/logrus"
+	"go.elastic.co/apm"
 )
 
 // CheckInstalledSoftware checks that the required software is present
@@ -30,7 +32,12 @@ func CheckInstalledSoftware(binaries []string) {
 // - workspace: represents the location where to execute the command
 // - command: represents the name of the binary to execute
 // - args: represents the arguments to be passed to the command
-func Execute(workspace string, command string, args ...string) (string, error) {
+func Execute(ctx context.Context, workspace string, command string, args ...string) (string, error) {
+	span, _ := apm.StartSpanOptions(ctx, "Executing shell command", "shell.command.execute", apm.SpanOptions{
+		Parent: apm.SpanFromContext(ctx).TraceContext(),
+	})
+	defer span.End()
+
 	log.WithFields(log.Fields{
 		"command": command,
 		"args":    args,