Fix downloading agent manifest from upstream

internal/configuration/locations/locations.go

@@ -23,8 +23,7 @@ const (
 
 	fieldsCachedDir = "cache/fields"
 
-	kubernetesDeployerElasticAgentYmlFile = "elastic-agent.yml"
-	terraformDeployerYmlFile              = "terraform-deployer.yml"
+	terraformDeployerYmlFile = "terraform-deployer.yml"
 )
 
 var (
@@ -84,11 +83,6 @@ func (loc LocationManager) KubernetesDeployerDir() string {
 	return filepath.Join(loc.stackPath, kubernetesDeployerDir)
 }
 
-// KubernetesDeployerAgentYml returns the Kubernetes Deployer Elastic Agent yml
-func (loc LocationManager) KubernetesDeployerAgentYml() string {
-	return filepath.Join(loc.stackPath, kubernetesDeployerDir, kubernetesDeployerElasticAgentYmlFile)
-}
-
 // TerraformDeployerDir returns the Terraform Directory
 func (loc LocationManager) TerraformDeployerDir() string {
 	return filepath.Join(loc.stackPath, terraformDeployerDir)

internal/install/install.go

@@ -8,7 +8,6 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
-	"strings"
 	"time"
 
 	"github.com/pkg/errors"
@@ -60,11 +59,6 @@ func EnsureInstalled() error {
 		return errors.Wrap(err, "writing stack resources failed")
 	}
 
-	err = writeKubernetesDeployerResources(elasticPackagePath)
-	if err != nil {
-		return errors.Wrap(err, "writing Kubernetes deployer resources failed")
-	}
-
 	err = writeTerraformDeployerResources(elasticPackagePath)
 	if err != nil {
 		return errors.Wrap(err, "writing Terraform deployer resources failed")
@@ -172,26 +166,6 @@ func writeStackResources(elasticPackagePath *locations.LocationManager) error {
 
 }
 
-func writeKubernetesDeployerResources(elasticPackagePath *locations.LocationManager) error {
-	err := os.MkdirAll(elasticPackagePath.KubernetesDeployerDir(), 0755)
-	if err != nil {
-		return errors.Wrapf(err, "creating directory failed (path: %s)", elasticPackagePath.KubernetesDeployerDir())
-	}
-
-	appConfig, err := Configuration()
-	if err != nil {
-		return errors.Wrap(err, "can't read application configuration")
-	}
-
-	err = writeStaticResource(err, elasticPackagePath.KubernetesDeployerAgentYml(),
-		strings.ReplaceAll(kubernetesDeployerElasticAgentYml, "{{ ELASTIC_AGENT_IMAGE_REF }}",
-			appConfig.DefaultStackImageRefs().ElasticAgent))
-	if err != nil {
-		return errors.Wrap(err, "writing static resource failed")
-	}
-	return nil
-}
-
 func writeTerraformDeployerResources(elasticPackagePath *locations.LocationManager) error {
 	terraformDeployer := elasticPackagePath.TerraformDeployerDir()
 	err := os.MkdirAll(terraformDeployer, 0755)

internal/kubectl/kubectl.go

@@ -49,3 +49,21 @@ func modifyKubernetesResources(action string, definitionPaths ...string) ([]byte
 	}
 	return output, nil
 }
+
+// applyKubernetesResourcesStdin applies a Kubernetes manifest provided as stdin.
+// It returns the resources created as output and an error
+func applyKubernetesResourcesStdin(input []byte) ([]byte, error) {
+	// create kubectl apply command
+	kubectlCmd := exec.Command("kubectl", "apply", "-f", "-", "-o", "yaml")
+	//Stdin of kubectl command is the manifest provided
+	kubectlCmd.Stdin = bytes.NewReader(input)
+	errOutput := new(bytes.Buffer)
+	kubectlCmd.Stderr = errOutput
+
+	logger.Debugf("run command: %s", kubectlCmd)
+	output, err := kubectlCmd.Output()
+	if err != nil {
+		return nil, errors.Wrapf(err, "kubectl apply failed (stderr=%q)", errOutput.String())
+	}
+	return output, nil
+}

internal/kubectl/kubectl_apply.go

@@ -84,6 +84,22 @@ func Apply(definitionPaths ...string) error {
 	return nil
 }
 
+// ApplyStdin function adds resources to the Kubernetes cluster based on provided stdin.
+func ApplyStdin(input []byte) error {
+	logger.Debugf("Apply Kubernetes stdin")
+	out, err := applyKubernetesResourcesStdin(input)
+	if err != nil {
+		return errors.Wrap(err, "can't modify Kubernetes resources (apply stdin)")
+	}
+
+	logger.Debugf("Handle \"apply\" command output")
+	err = handleApplyCommandOutput(out)
+	if err != nil {
+		return errors.Wrap(err, "can't handle command output")
+	}
+	return nil
+}
+
 func handleApplyCommandOutput(out []byte) error {
 	logger.Debugf("Extract resources from command output")
 	resources, err := extractResources(out)

internal/testrunner/runners/system/servicedeployer/kubernetes.go

@@ -5,18 +5,25 @@
 package servicedeployer
 
 import (
+	"fmt"
+	"io"
+	"net/http"
 	"os"
 	"path/filepath"
+	"regexp"
 	"strings"
+	"time"
 
 	"github.com/pkg/errors"
 
-	"github.com/elastic/elastic-package/internal/configuration/locations"
+	"github.com/elastic/elastic-package/internal/install"
 	"github.com/elastic/elastic-package/internal/kind"
 	"github.com/elastic/elastic-package/internal/kubectl"
 	"github.com/elastic/elastic-package/internal/logger"
 )
 
+const elasticAgentManagedYamlURL = "https://raw.githubusercontent.com/elastic/beats/7.x/deploy/kubernetes/elastic-agent-managed-kubernetes.yaml"
+
 // KubernetesServiceDeployer is responsible for deploying resources in the Kubernetes cluster.
 type KubernetesServiceDeployer struct {
 	definitionsDir string
@@ -144,14 +151,87 @@ func findKubernetesDefinitions(definitionsDir string) ([]string, error) {
 func installElasticAgentInCluster() error {
 	logger.Debug("install Elastic Agent in the Kubernetes cluster")
 
-	locationManager, err := locations.NewLocationManager()
+	elasticAgentManagedYaml, err := getElasticAgentYAML()
 	if err != nil {
-		return errors.Wrap(err, "can't locate Kubernetes file for Elastic Agent in ")
+		return errors.Wrap(err, "can't retrieve Kubernetes file for Elastic Agent")
 	}
 
-	err = kubectl.Apply(locationManager.KubernetesDeployerAgentYml())
+	err = kubectl.ApplyStdin(elasticAgentManagedYaml)
 	if err != nil {
 		return errors.Wrap(err, "can't install Elastic-Agent in Kubernetes cluster")
 	}
 	return nil
 }
+
+// downloadElasticAgentManagedYAML will download a url from a path and return the response body.
+func downloadElasticAgentManagedYAML(url string) ([]byte, error) {
+	// Get the data
+	resp, err := http.Get(url)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to get file from URL %s", url)
+	}
+	defer resp.Body.Close()
+
+	b, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to read response body")
+	}
+
+	logger.Debugf("status code when downloading elastic-agent-managed-kubernetes.yaml is %d", resp.StatusCode)
+	if resp.StatusCode != 200 {
+		return nil, fmt.Errorf("downloading failed due to status code %d, resp body: %s", resp.StatusCode, string(b))
+	}
+	return b, nil
+}
+
+// getElasticAgentYAML retrieves elastic-agent-managed.yaml from upstream and modifies the file as needed
+// to run locally.
+func getElasticAgentYAML() ([]byte, error) {
+	appConfig, err := install.Configuration()
+	if err != nil {
+		return nil, errors.Wrap(err, "can't read application configuration")
+	}
+
+	logger.Debugf("downloading elastic-agent-managed-kubernetes.yaml from %s", elasticAgentManagedYamlURL)
+	// retry downloading elastic agent manifest for 5 times (sleep 10 seconds between each try) in case of error
+	elasticAgentManagedYaml, err := retryDownloadElasticAgentManagedYAML(elasticAgentManagedYamlURL, 5, 10,
+		downloadElasticAgentManagedYAML)
+	if err != nil {
+		return nil, errors.Wrapf(err, "downloading failed for file from source  %s", elasticAgentManagedYamlURL)
+	}
+
+	// Set regex to match fleet url from yaml file
+	fleetURLRegex := regexp.MustCompile("http(s){0,1}:\\/\\/fleet-server:(\\d+)")
+	// Replace fleet url
+	elasticAgentManagedYaml = fleetURLRegex.ReplaceAll(elasticAgentManagedYaml, []byte("http://fleet-server:8220"))
+
+	// Set regex to match image name from yaml file
+	imageRegex := regexp.MustCompile("docker.elastic.co/beats/elastic-agent:\\d.+")
+	// Replace image name
+	elasticAgentManagedYaml = imageRegex.ReplaceAll(elasticAgentManagedYaml, []byte(appConfig.DefaultStackImageRefs().ElasticAgent))
+
+	return elasticAgentManagedYaml, nil
+}
+
+// retryDownloadElasticAgentManagedYAML retries downloading elastic agent managed manifest for x attempts
+// until there is no error and bytes of the file are more than 2000.
+func retryDownloadElasticAgentManagedYAML(url string, attempts int, sleep time.Duration, f func(string) ([]byte, error)) (
+	elasticAgentManagedYaml []byte, err error) {
+	for i := 0; i < attempts; i++ {
+		if i > 0 {
+			logger.Debugf("retrying download attempt %d", i+1)
+			time.Sleep(sleep * time.Second)
+		}
+		elasticAgentManagedYaml, err = f(url)
+		if err == nil {
+			logger.Debugf("downloaded %d bytes", len(elasticAgentManagedYaml))
+			if len(elasticAgentManagedYaml) > 2000 {
+				return elasticAgentManagedYaml, nil
+			}
+			err = fmt.Errorf("bytes downloaded should be more than 2000 but where: %d", len(elasticAgentManagedYaml))
+			logger.Debugf("failed because %s", err)
+		}
+	}
+	return nil,
+		errors.Wrapf(err, "failed after %d unsuccessful attempts of downloading elastic-agent-managed-kubernetes.yaml", attempts)
+}

scripts/test-check-packages.sh

@@ -10,7 +10,7 @@ cleanup() {
 
   # Dump kubectl details
   kubectl describe pods --all-namespaces > build/kubectl-dump.txt
-  kubectl logs -l app=kind-fleet-agent-clusterscope -n kube-system >> build/kubectl-dump.txt
+  kubectl logs -l app=elastic-agent -n kube-system >> build/kubectl-dump.txt
 
   # Take down the kind cluster
   kind delete cluster