From 996ed04d5dc5b1426a55e40f441c92db9f990ba5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Wed, 12 May 2021 15:59:03 +0200 Subject: [PATCH] feat: simplify the initialisation of versions (#1159) * chore: use fixed version in shell scripts * chore: move retry to utils We could move it to its own package, but at this moment it's very small * chore: initialise stackVesion at one single place * chore: initialise agent version base at one single place * chore: initialise agent version at one single place * chore: reduce the number of requests to Elastic's artifacts endpoint * chore: rename AgentVersionBase variable to BeatVersionBase * chore: rename AgentVersion variable to BeatVersion * chore: use Beat version in metricbeat test suite * chore: check if the version must use the fallback after coming from a Git SHA --- .ci/scripts/clean-docker.sh | 2 +- .ci/scripts/fleet-test.sh | 4 +- .ci/scripts/functional-test.sh | 10 +-- .ci/scripts/metricbeat-test.sh | 10 +-- .stack-version | 1 + e2e/_suites/fleet/fleet.go | 52 +++++++------- e2e/_suites/fleet/ingest_manager_test.go | 42 ++---------- e2e/_suites/fleet/stand-alone.go | 8 +-- e2e/_suites/fleet/world.go | 7 +- e2e/_suites/helm/helm_charts_test.go | 24 ++----- .../autodiscover_test.go | 7 +- e2e/_suites/metricbeat/metricbeat_test.go | 68 +++++-------------- internal/common/defaults.go | 57 ++++++++++++++-- internal/docker/docker.go | 4 +- internal/elasticsearch/client.go | 8 +-- internal/installer/deb.go | 2 +- internal/installer/docker.go | 6 +- internal/installer/elasticagent.go | 4 +- internal/installer/rpm.go | 2 +- internal/installer/tar.go | 4 +- internal/kibana/server.go | 14 ++-- internal/kubernetes/kubernetes.go | 4 +- internal/{common => utils}/retry.go | 2 +- internal/utils/utils.go | 28 ++++++-- 24 files changed, 179 insertions(+), 191 deletions(-) create mode 100644 .stack-version rename internal/{common => utils}/retry.go (98%) diff --git a/.ci/scripts/clean-docker.sh b/.ci/scripts/clean-docker.sh index ef2a271355..bc3fc0f29c 100755 --- a/.ci/scripts/clean-docker.sh +++ b/.ci/scripts/clean-docker.sh @@ -9,7 +9,7 @@ set -euxo pipefail # Build and test the app using the install and test make goals. # -readonly VERSION="7.13.0-SNAPSHOT" +readonly VERSION="$(cat $(pwd)/.stack-version)" main() { # refresh docker images diff --git a/.ci/scripts/fleet-test.sh b/.ci/scripts/fleet-test.sh index 30ec8f59ed..edaa021877 100755 --- a/.ci/scripts/fleet-test.sh +++ b/.ci/scripts/fleet-test.sh @@ -9,10 +9,10 @@ set -euxo pipefail # Run the functional tests for fleets using the functional-test wrapper # # Parameters: -# - STACK_VERSION - that's the version of the stack to be tested. Default '7.13.0-SNAPSHOT'. +# - STACK_VERSION - that's the version of the stack to be tested. Default is stored in '.stack-version'. # -STACK_VERSION=${1:-'7.13.0-SNAPSHOT'} +STACK_VERSION=${1:-"$(cat $(pwd)/.stack-version)"} SUITE='fleet' # Exclude the nightly tests in the CI. diff --git a/.ci/scripts/functional-test.sh b/.ci/scripts/functional-test.sh index e9117fa6ce..a0e2edd791 100755 --- a/.ci/scripts/functional-test.sh +++ b/.ci/scripts/functional-test.sh @@ -12,14 +12,16 @@ set -euxo pipefail # Parameters: # - SUITE - that's the suite to be tested. Default '' which means all of them. # - TAGS - that's the tags to be tested. Default '' which means all of them. -# - STACK_VERSION - that's the version of the stack to be tested. Default '7.13.0-SNAPSHOT'. -# - BEAT_VERSION - that's the version of the metricbeat to be tested. Default '7.13.0-SNAPSHOT'. +# - STACK_VERSION - that's the version of the stack to be tested. Default is stored in '.stack-version'. +# - BEAT_VERSION - that's the version of the metricbeat to be tested. Default is stored in '.stack-version'. # +BASE_VERSION="$(cat $(pwd)/.stack-version)" + SUITE=${1:-''} TAGS=${2:-''} -STACK_VERSION=${3:-'7.13.0-SNAPSHOT'} -BEAT_VERSION=${4:-'7.13.0-SNAPSHOT'} +STACK_VERSION=${3:-"${BASE_VERSION}"} +BEAT_VERSION=${4:-"${BASE_VERSION}"} ## Install the required dependencies for the given SUITE .ci/scripts/install-test-dependencies.sh "${SUITE}" diff --git a/.ci/scripts/metricbeat-test.sh b/.ci/scripts/metricbeat-test.sh index 7e78fa2b97..460d649e63 100755 --- a/.ci/scripts/metricbeat-test.sh +++ b/.ci/scripts/metricbeat-test.sh @@ -9,12 +9,14 @@ set -euxo pipefail # Run the functional tests for metricbeat using the functional-test wrapper # # Parameters: -# - STACK_VERSION - that's the version of the stack to be tested. Default '7.13.0-SNAPSHOT'. -# - BEAT_VERSION - that's the version of the metricbeat to be tested. Default '7.13.0-SNAPSHOT'. +# - STACK_VERSION - that's the version of the stack to be tested. Default is stored in '.stack-version'. +# - BEAT_VERSION - that's the version of the metricbeat to be tested. Default is stored in '.stack-version'. # -STACK_VERSION=${1:-'7.13.0-SNAPSHOT'} -BEAT_VERSION=${2:-'7.13.0-SNAPSHOT'} +BASE_VERSION="$(cat $(pwd)/.stack-version)" + +STACK_VERSION=${1:-"${BASE_VERSION}"} +BEAT_VERSION=${2:-"${BASE_VERSION}"} SUITE='metricbeat' .ci/scripts/functional-test.sh "${SUITE}" "" "${STACK_VERSION}" "${BEAT_VERSION}" diff --git a/.stack-version b/.stack-version new file mode 100644 index 0000000000..96cfbb19ae --- /dev/null +++ b/.stack-version @@ -0,0 +1 @@ +7.13.0-SNAPSHOT diff --git a/e2e/_suites/fleet/fleet.go b/e2e/_suites/fleet/fleet.go index 3222ca7416..ee4e1bc66a 100644 --- a/e2e/_suites/fleet/fleet.go +++ b/e2e/_suites/fleet/fleet.go @@ -118,7 +118,7 @@ func (fts *FleetTestSuite) beforeScenario() { fts.StandAlone = false fts.ElasticAgentStopped = false - fts.Version = common.AgentVersion + fts.Version = common.BeatVersion policy, err := fts.kibanaClient.GetDefaultPolicy(false) if err != nil { @@ -183,8 +183,8 @@ func (fts *FleetTestSuite) theStandaloneAgentIsListedInFleetWithStatus(desiredSt return theAgentIsListedInFleetWithStatus(desiredStatus, hostname) } - maxTimeout := time.Duration(common.TimeoutFactor) * time.Minute * 2 - exp := common.GetExponentialBackOff(maxTimeout) + maxTimeout := time.Duration(utils.TimeoutFactor) * time.Minute * 2 + exp := utils.GetExponentialBackOff(maxTimeout) err := backoff.Retry(waitForAgents, exp) if err != nil { @@ -218,7 +218,7 @@ func (fts *FleetTestSuite) anStaleAgentIsDeployedToFleetWithInstaller(image, ver case "stale": version = common.AgentStaleVersion case "latest": - version = common.AgentVersion + version = common.BeatVersion default: version = common.AgentStaleVersion } @@ -240,7 +240,7 @@ func (fts *FleetTestSuite) installCerts() error { log.WithFields(log.Fields{ "installer": agentInstaller, "version": fts.Version, - "agentVersion": common.AgentVersion, + "agentVersion": common.BeatVersion, "agentStaleVersion": common.AgentStaleVersion, }).Error("No installer found") return errors.New("no installer found") @@ -249,7 +249,7 @@ func (fts *FleetTestSuite) installCerts() error { err := agentInstaller.InstallCertsFn() if err != nil { log.WithFields(log.Fields{ - "agentVersion": common.AgentVersion, + "agentVersion": common.BeatVersion, "agentStaleVersion": common.AgentStaleVersion, "error": err, "installer": agentInstaller, @@ -266,9 +266,9 @@ func (fts *FleetTestSuite) anAgentIsUpgraded(desiredVersion string) error { case "stale": desiredVersion = common.AgentStaleVersion case "latest": - desiredVersion = common.AgentVersion + desiredVersion = common.BeatVersion default: - desiredVersion = common.AgentVersion + desiredVersion = common.BeatVersion } return fts.kibanaClient.UpgradeAgent(fts.Hostname, desiredVersion) @@ -279,7 +279,7 @@ func (fts *FleetTestSuite) agentInVersion(version string) error { case "stale": version = common.AgentStaleVersion case "latest": - version = common.AgentVersion + version = common.BeatVersion } agentInVersionFn := func() error { @@ -300,8 +300,8 @@ func (fts *FleetTestSuite) agentInVersion(version string) error { return nil } - maxTimeout := time.Duration(common.TimeoutFactor) * time.Minute * 2 - exp := common.GetExponentialBackOff(maxTimeout) + maxTimeout := time.Duration(utils.TimeoutFactor) * time.Minute * 2 + exp := utils.GetExponentialBackOff(maxTimeout) return backoff.Retry(agentInVersionFn, exp) } @@ -399,7 +399,7 @@ func (fts *FleetTestSuite) processStateChangedOnTheHost(process string, state st return err } - utils.Sleep(time.Duration(common.TimeoutFactor) * 10 * time.Second) + utils.Sleep(time.Duration(utils.TimeoutFactor) * 10 * time.Second) err = installer.SystemctlRun(profile, agentInstaller.Image, serviceName, "start") if err != nil { @@ -441,7 +441,7 @@ func (fts *FleetTestSuite) processStateChangedOnTheHost(process string, state st containerName := fts.getContainerName(agentInstaller, 1) - return CheckProcessState(fts.deployer, containerName, process, "stopped", 1, common.TimeoutFactor) + return CheckProcessState(fts.deployer, containerName, process, "stopped", 1, utils.TimeoutFactor) } func (fts *FleetTestSuite) setup() error { @@ -466,10 +466,10 @@ func theAgentIsListedInFleetWithStatus(desiredStatus string, hostname string) er if err != nil { return err } - maxTimeout := time.Duration(common.TimeoutFactor) * time.Minute * 2 + maxTimeout := time.Duration(utils.TimeoutFactor) * time.Minute * 2 retryCount := 1 - exp := common.GetExponentialBackOff(maxTimeout) + exp := utils.GetExponentialBackOff(maxTimeout) agentOnlineFn := func() error { agentID, err := kibanaClient.GetAgentIDByHostname(hostname) @@ -563,7 +563,7 @@ func (fts *FleetTestSuite) theHostIsRestarted() error { }).Error("Could not stop the service") } - utils.Sleep(time.Duration(common.TimeoutFactor) * 10 * time.Second) + utils.Sleep(time.Duration(utils.TimeoutFactor) * 10 * time.Second) _, err = shell.Execute(context.Background(), ".", "docker", "start", containerName) if err != nil { @@ -586,10 +586,10 @@ func (fts *FleetTestSuite) systemPackageDashboardsAreListedInFleet() error { log.Trace("Checking system Package dashboards in Fleet") dataStreamsCount := 0 - maxTimeout := time.Duration(common.TimeoutFactor) * time.Minute + maxTimeout := time.Duration(utils.TimeoutFactor) * time.Minute retryCount := 1 - exp := common.GetExponentialBackOff(maxTimeout) + exp := utils.GetExponentialBackOff(maxTimeout) countDataStreamsFn := func() error { dataStreams, err := fts.kibanaClient.GetDataStreams() @@ -730,10 +730,10 @@ func theIntegrationIsOperatedInThePolicy(client *kibana.Client, policy kibana.Po func (fts *FleetTestSuite) theHostNameIsNotShownInTheAdminViewInTheSecurityApp() error { log.Trace("Checking if the hostname is not shown in the Administration view in the Security App") - maxTimeout := time.Duration(common.TimeoutFactor) * time.Minute + maxTimeout := time.Duration(utils.TimeoutFactor) * time.Minute retryCount := 1 - exp := common.GetExponentialBackOff(maxTimeout) + exp := utils.GetExponentialBackOff(maxTimeout) agentListedInSecurityFn := func() error { host, err := fts.kibanaClient.IsAgentListedInSecurityApp(fts.Hostname) @@ -770,10 +770,10 @@ func (fts *FleetTestSuite) theHostNameIsNotShownInTheAdminViewInTheSecurityApp() func (fts *FleetTestSuite) theHostNameIsShownInTheAdminViewInTheSecurityApp(status string) error { log.Trace("Checking if the hostname is shown in the Admin view in the Security App") - maxTimeout := time.Duration(common.TimeoutFactor) * time.Minute + maxTimeout := time.Duration(utils.TimeoutFactor) * time.Minute retryCount := 1 - exp := common.GetExponentialBackOff(maxTimeout) + exp := utils.GetExponentialBackOff(maxTimeout) agentListedInSecurityFn := func() error { matches, err := fts.kibanaClient.IsAgentListedInSecurityAppWithStatus(fts.Hostname, status) @@ -825,10 +825,10 @@ func (fts *FleetTestSuite) thePolicyResponseWillBeShownInTheSecurityApp() error return err } - maxTimeout := time.Duration(common.TimeoutFactor) * time.Minute + maxTimeout := time.Duration(utils.TimeoutFactor) * time.Minute retryCount := 1 - exp := common.GetExponentialBackOff(maxTimeout) + exp := utils.GetExponentialBackOff(maxTimeout) getEventsFn := func() error { listed, err := fts.kibanaClient.IsPolicyResponseListedInSecurityApp(agentID) @@ -924,10 +924,10 @@ func (fts *FleetTestSuite) thePolicyWillReflectTheChangeInTheSecurityApp() error return err } - maxTimeout := time.Duration(common.TimeoutFactor) * time.Minute * 2 + maxTimeout := time.Duration(utils.TimeoutFactor) * time.Minute * 2 retryCount := 1 - exp := common.GetExponentialBackOff(maxTimeout) + exp := utils.GetExponentialBackOff(maxTimeout) getEventsFn := func() error { err := fts.kibanaClient.GetAgentEvents("endpoint-security", agentID, pkgPolicy.ID, fts.PolicyUpdatedAt) diff --git a/e2e/_suites/fleet/ingest_manager_test.go b/e2e/_suites/fleet/ingest_manager_test.go index 806e203a5c..cbfebbb38c 100644 --- a/e2e/_suites/fleet/ingest_manager_test.go +++ b/e2e/_suites/fleet/ingest_manager_test.go @@ -38,37 +38,7 @@ func setUpSuite() { log.Info("Running in Developer mode 💻: runtime dependencies between different test runs will be reused to speed up dev cycle") } - // check if base version is an alias - v, err := utils.GetElasticArtifactVersion(common.AgentVersionBase) - if err != nil { - log.WithFields(log.Fields{ - "error": err, - "version": common.AgentVersionBase, - }).Fatal("Failed to get agent base version, aborting") - } - common.AgentVersionBase = v - - common.AgentVersion = shell.GetEnv("BEAT_VERSION", common.AgentVersionBase) - - // check if version is an alias - v, err = utils.GetElasticArtifactVersion(common.AgentVersion) - if err != nil { - log.WithFields(log.Fields{ - "error": err, - "version": common.AgentVersion, - }).Fatal("Failed to get agent version, aborting") - } - common.AgentVersion = v - - common.StackVersion = shell.GetEnv("STACK_VERSION", common.StackVersion) - v, err = utils.GetElasticArtifactVersion(common.StackVersion) - if err != nil { - log.WithFields(log.Fields{ - "error": err, - "version": common.StackVersion, - }).Fatal("Failed to get stack version, aborting") - } - common.StackVersion = v + common.InitVersions() common.KibanaVersion = shell.GetEnv("KIBANA_VERSION", "") if common.KibanaVersion == "" { @@ -119,12 +89,12 @@ func InitializeIngestManagerTestSuite(ctx *godog.TestSuiteContext) { if !shell.GetEnvBool("SKIP_PULL") { images := []string{ - "docker.elastic.co/beats/elastic-agent:" + common.AgentVersion, - "docker.elastic.co/beats/elastic-agent-ubi8:" + common.AgentVersion, + "docker.elastic.co/beats/elastic-agent:" + common.BeatVersion, + "docker.elastic.co/beats/elastic-agent-ubi8:" + common.BeatVersion, "docker.elastic.co/elasticsearch/elasticsearch:" + common.StackVersion, "docker.elastic.co/kibana/kibana:" + common.KibanaVersion, - "docker.elastic.co/observability-ci/elastic-agent:" + common.AgentVersion, - "docker.elastic.co/observability-ci/elastic-agent-ubi8:" + common.AgentVersion, + "docker.elastic.co/observability-ci/elastic-agent:" + common.BeatVersion, + "docker.elastic.co/observability-ci/elastic-agent-ubi8:" + common.BeatVersion, "docker.elastic.co/observability-ci/elasticsearch:" + common.StackVersion, "docker.elastic.co/observability-ci/elasticsearch-ubi8:" + common.StackVersion, "docker.elastic.co/observability-ci/kibana:" + common.KibanaVersion, @@ -146,7 +116,7 @@ func InitializeIngestManagerTestSuite(ctx *godog.TestSuiteContext) { return nil }) - imts.Fleet.Version = common.AgentVersionBase + imts.Fleet.Version = common.BeatVersionBase imts.Fleet.RuntimeDependenciesStartDate = time.Now().UTC() }) diff --git a/e2e/_suites/fleet/stand-alone.go b/e2e/_suites/fleet/stand-alone.go index 87933670b1..568660f752 100644 --- a/e2e/_suites/fleet/stand-alone.go +++ b/e2e/_suites/fleet/stand-alone.go @@ -48,7 +48,7 @@ func (fts *FleetTestSuite) aStandaloneAgentIsDeployedWithFleetServerModeOnCloud( } func (fts *FleetTestSuite) thereIsNewDataInTheIndexFromAgent() error { - maxTimeout := time.Duration(common.TimeoutFactor) * time.Minute * 2 + maxTimeout := time.Duration(utils.TimeoutFactor) * time.Minute * 2 minimumHitsCount := 50 result, err := searchAgentData(fts.Hostname, fts.RuntimeDependenciesStartDate, minimumHitsCount, maxTimeout) @@ -94,7 +94,7 @@ func (fts *FleetTestSuite) startStandAloneAgent(image string, composeFilename st fts.StandAlone = true log.Trace("Deploying an agent to Fleet") - dockerImageTag := common.AgentVersion + dockerImageTag := common.BeatVersion useCISnapshots := shell.GetEnvBool("BEATS_USE_CI_SNAPSHOTS") beatsLocalPath := shell.GetEnv("BEATS_LOCAL_PATH", "") @@ -102,7 +102,7 @@ func (fts *FleetTestSuite) startStandAloneAgent(image string, composeFilename st // load the docker images that were already: // a. downloaded from the GCP bucket // b. fetched from the local beats binaries - dockerInstaller := installer.GetElasticAgentInstaller("docker", image, common.AgentVersion) + dockerInstaller := installer.GetElasticAgentInstaller("docker", image, common.BeatVersion) dockerInstaller.PreInstallFn() @@ -159,7 +159,7 @@ func (fts *FleetTestSuite) thePolicyShowsTheDatasourceAdded(packageName string) maxTimeout := time.Minute retryCount := 1 - exp := common.GetExponentialBackOff(maxTimeout) + exp := utils.GetExponentialBackOff(maxTimeout) configurationIsPresentFn := func() error { packagePolicy, err := fts.kibanaClient.GetIntegrationFromAgentPolicy(packageName, fts.Policy) diff --git a/e2e/_suites/fleet/world.go b/e2e/_suites/fleet/world.go index 716faf2939..5737f84430 100644 --- a/e2e/_suites/fleet/world.go +++ b/e2e/_suites/fleet/world.go @@ -13,6 +13,7 @@ import ( "github.com/cenkalti/backoff/v4" "github.com/elastic/e2e-testing/internal/common" "github.com/elastic/e2e-testing/internal/deploy" + "github.com/elastic/e2e-testing/internal/utils" log "github.com/sirupsen/logrus" ) @@ -42,14 +43,14 @@ func (imts *IngestManagerTestSuite) thereAreInstancesOfTheProcessInTheState(ocur return err } - return CheckProcessState(imts.Fleet.deployer, containerName, process, state, count, common.TimeoutFactor) + return CheckProcessState(imts.Fleet.deployer, containerName, process, state, count, utils.TimeoutFactor) } // CheckProcessState checks if a process is in the desired state in a container // name of the container for the service: // we are using the underlying deployer to run the commands in the container/service func CheckProcessState(deployer deploy.Deployment, service string, process string, state string, occurrences int, timeoutFactor int) error { - timeout := time.Duration(common.TimeoutFactor) * time.Minute + timeout := time.Duration(utils.TimeoutFactor) * time.Minute err := waitForProcess(deployer, service, process, state, occurrences, timeout) if err != nil { @@ -76,7 +77,7 @@ func CheckProcessState(deployer deploy.Deployment, service string, process strin // waitForProcess polls a container executing "ps" command until the process is in the desired state (present or not), // or a timeout happens func waitForProcess(deployer deploy.Deployment, service string, process string, desiredState string, ocurrences int, maxTimeout time.Duration) error { - exp := common.GetExponentialBackOff(maxTimeout) + exp := utils.GetExponentialBackOff(maxTimeout) mustBePresent := false if desiredState == "started" { diff --git a/e2e/_suites/helm/helm_charts_test.go b/e2e/_suites/helm/helm_charts_test.go index 05ebbb1ace..9f41d649d3 100644 --- a/e2e/_suites/helm/helm_charts_test.go +++ b/e2e/_suites/helm/helm_charts_test.go @@ -49,10 +49,6 @@ var helmChartVersion = "7.11.2" // kubernetesVersion represents the default version used for Kubernetes var kubernetesVersion = "1.18.2" -// stackVersion is the version of the stack to use -// It can be overriden by STACK_VERSION env var -var stackVersion = "7.13.0-SNAPSHOT" - var testSuite HelmChartTestSuite var tx *apm.Transaction @@ -77,15 +73,7 @@ func setupSuite() { helmChartVersion = shell.GetEnv("HELM_CHART_VERSION", helmChartVersion) kubernetesVersion = shell.GetEnv("KUBERNETES_VERSION", kubernetesVersion) - stackVersion = shell.GetEnv("STACK_VERSION", stackVersion) - v, err := utils.GetElasticArtifactVersion(stackVersion) - if err != nil { - log.WithFields(log.Fields{ - "error": err, - "version": stackVersion, - }).Fatal("Failed to get stack version, aborting") - } - stackVersion = v + common.InitVersions() h, err := helm.Factory(helmVersion) if err != nil { @@ -182,9 +170,9 @@ func (ts *HelmChartTestSuite) aResourceWillExposePods(resourceType string) error return err } - maxTimeout := time.Duration(common.TimeoutFactor) * time.Minute + maxTimeout := time.Duration(utils.TimeoutFactor) * time.Minute - exp := common.GetExponentialBackOff(maxTimeout) + exp := utils.GetExponentialBackOff(maxTimeout) retryCount := 1 checkEndpointsFn := func() error { @@ -407,7 +395,7 @@ func (ts *HelmChartTestSuite) install(ctx context.Context, chart string) error { "chart": ts.Name, }).Info("Rancher Local Path Provisioner and local-path storage class for Elasticsearch volumes installed") - maxTimeout := common.TimeoutFactor * 100 + maxTimeout := utils.TimeoutFactor * 100 log.Debug("Applying workaround to use Rancher's local-path storage class for Elasticsearch volumes") flags = []string{"--wait", fmt.Sprintf("--timeout=%ds", maxTimeout), "--values", "https://raw.githubusercontent.com/elastic/helm-charts/master/elasticsearch/examples/kubernetes-kind/values.yaml"} @@ -675,7 +663,7 @@ func InitializeHelmChartTestSuite(ctx *godog.TestSuiteContext) { serviceManager := compose.NewServiceManager() env := map[string]string{ - "stackVersion": stackVersion, + "stackVersion": common.StackVersion, } err := serviceManager.RunCompose(suiteContext, true, []string{"helm"}, env) @@ -684,7 +672,7 @@ func InitializeHelmChartTestSuite(ctx *godog.TestSuiteContext) { "profile": "metricbeat", }).Warn("Could not run the profile.") } - steps.AddAPMServicesForInstrumentation(suiteContext, "helm", stackVersion, true, env) + steps.AddAPMServicesForInstrumentation(suiteContext, "helm", common.StackVersion, true, env) } err := testSuite.createCluster(suiteContext, testSuite.KubernetesVersion) diff --git a/e2e/_suites/kubernetes-autodiscover/autodiscover_test.go b/e2e/_suites/kubernetes-autodiscover/autodiscover_test.go index d420ce160d..f85d4219fb 100644 --- a/e2e/_suites/kubernetes-autodiscover/autodiscover_test.go +++ b/e2e/_suites/kubernetes-autodiscover/autodiscover_test.go @@ -23,7 +23,6 @@ import ( log "github.com/sirupsen/logrus" "github.com/elastic/e2e-testing/cli/config" - "github.com/elastic/e2e-testing/internal/common" "github.com/elastic/e2e-testing/internal/docker" "github.com/elastic/e2e-testing/internal/kubernetes" "github.com/elastic/e2e-testing/internal/shell" @@ -123,7 +122,7 @@ func (m *podsManager) configureDockerImage(podName string) error { // this method will detect if the GITHUB_CHECK_SHA1 variable is set artifactName := utils.BuildArtifactName(podName, beatVersion, defaultBeatVersion, "linux", "amd64", "tar.gz", true) - imagePath, err := utils.FetchBeatsBinary(artifactName, podName, beatVersion, defaultBeatVersion, common.TimeoutFactor, true) + imagePath, err := utils.FetchBeatsBinary(artifactName, podName, beatVersion, defaultBeatVersion, utils.TimeoutFactor, true) if err != nil { return err } @@ -464,8 +463,8 @@ func InitializeTestSuite(ctx *godog.TestSuiteContext) { // init logger config.Init() - defaultEventsWaitTimeout = defaultEventsWaitTimeout * time.Duration(common.TimeoutFactor) - defaultDeployWaitTimeout = defaultDeployWaitTimeout * time.Duration(common.TimeoutFactor) + defaultEventsWaitTimeout = defaultEventsWaitTimeout * time.Duration(utils.TimeoutFactor) + defaultDeployWaitTimeout = defaultDeployWaitTimeout * time.Duration(utils.TimeoutFactor) err := cluster.Initialize(suiteContext, "testdata/kind.yml") if err != nil { diff --git a/e2e/_suites/metricbeat/metricbeat_test.go b/e2e/_suites/metricbeat/metricbeat_test.go index a8c5849f21..fa99b751f9 100644 --- a/e2e/_suites/metricbeat/metricbeat_test.go +++ b/e2e/_suites/metricbeat/metricbeat_test.go @@ -34,18 +34,8 @@ var developerMode = false var elasticAPMActive = false -var metricbeatVersionBase = "7.13.0-SNAPSHOT" - -// metricbeatVersion is the version of the metricbeat to use -// It can be overriden by BEAT_VERSION env var -var metricbeatVersion = metricbeatVersionBase - var serviceManager compose.ServiceManager -// stackVersion is the version of the stack to use -// It can be overriden by STACK_VERSION env var -var stackVersion = metricbeatVersionBase - var testSuite MetricbeatTestSuite var tx *apm.Transaction @@ -66,27 +56,7 @@ func setupSuite() { }).Info("Current execution will be instrumented 🛠") } - // check if base version is an alias - v, err := utils.GetElasticArtifactVersion(metricbeatVersionBase) - if err != nil { - log.WithFields(log.Fields{ - "error": err, - "version": metricbeatVersionBase, - }).Fatal("Failed to get metricbeat base version, aborting") - } - metricbeatVersionBase = v - - metricbeatVersion = shell.GetEnv("BEAT_VERSION", metricbeatVersionBase) - - stackVersion = shell.GetEnv("STACK_VERSION", stackVersion) - v, err = utils.GetElasticArtifactVersion(stackVersion) - if err != nil { - log.WithFields(log.Fields{ - "error": err, - "version": stackVersion, - }).Fatal("Failed to get stack version, aborting") - } - stackVersion = v + common.InitVersions() serviceManager = compose.NewServiceManager() @@ -167,7 +137,7 @@ func (mts *MetricbeatTestSuite) CleanUp() error { defer fn(context.Background()) env := map[string]string{ - "stackVersion": stackVersion, + "stackVersion": common.StackVersion, } services := []string{"metricbeat"} @@ -262,7 +232,7 @@ func InitializeMetricbeatTestSuite(ctx *godog.TestSuiteContext) { serviceManager := compose.NewServiceManager() env := map[string]string{ - "stackVersion": stackVersion, + "stackVersion": common.StackVersion, } err := serviceManager.RunCompose(suiteContext, true, []string{"metricbeat"}, env) @@ -272,7 +242,7 @@ func InitializeMetricbeatTestSuite(ctx *godog.TestSuiteContext) { }).Fatal("Could not run the profile.") } - minutesToBeHealthy := time.Duration(common.TimeoutFactor) * time.Minute + minutesToBeHealthy := time.Duration(utils.TimeoutFactor) * time.Minute healthy, err := elasticsearch.WaitForElasticsearch(suiteContext, minutesToBeHealthy) if !healthy { log.WithFields(log.Fields{ @@ -283,7 +253,7 @@ func InitializeMetricbeatTestSuite(ctx *godog.TestSuiteContext) { elasticAPMEnvironment := shell.GetEnv("ELASTIC_APM_ENVIRONMENT", "ci") if elasticAPMActive && elasticAPMEnvironment == "local" { - steps.AddAPMServicesForInstrumentation(suiteContext, "metricbeat", stackVersion, true, env) + steps.AddAPMServicesForInstrumentation(suiteContext, "metricbeat", common.StackVersion, true, env) } }) @@ -320,7 +290,7 @@ func (mts *MetricbeatTestSuite) installedAndConfiguredForModule(serviceType stri serviceType = strings.ToLower(serviceType) // at this point we have everything to define the index name - mts.Version = metricbeatVersion + mts.Version = common.BeatVersion mts.setIndexName() mts.ServiceType = serviceType @@ -373,16 +343,10 @@ func (mts *MetricbeatTestSuite) installedAndConfiguredForVariantModule(serviceVa } func (mts *MetricbeatTestSuite) installedUsingConfiguration(configuration string) error { - // restore initial state - metricbeatVersionBackup := metricbeatVersion - defer func() { metricbeatVersion = metricbeatVersionBackup }() - // at this point we have everything to define the index name - mts.Version = metricbeatVersion + mts.Version = common.BeatVersion mts.setIndexName() - metricbeatVersion = utils.CheckPRVersion(metricbeatVersion, metricbeatVersionBase) - configurationFilePath, err := steps.FetchBeatConfiguration(false, "metricbeat", configuration+".yml") if err != nil { return err @@ -407,9 +371,9 @@ func (mts *MetricbeatTestSuite) runMetricbeatService() error { useCISnapshots := shell.GetEnvBool("BEATS_USE_CI_SNAPSHOTS") beatsLocalPath := shell.GetEnv("BEATS_LOCAL_PATH", "") if useCISnapshots || beatsLocalPath != "" { - artifactName := utils.BuildArtifactName("metricbeat", mts.Version, metricbeatVersionBase, "linux", "amd64", "tar.gz", true) + artifactName := utils.BuildArtifactName("metricbeat", mts.Version, common.BeatVersionBase, "linux", "amd64", "tar.gz", true) - imagePath, err := utils.FetchBeatsBinary(artifactName, "metricbeat", mts.Version, metricbeatVersionBase, common.TimeoutFactor, true) + imagePath, err := utils.FetchBeatsBinary(artifactName, "metricbeat", mts.Version, common.BeatVersionBase, utils.TimeoutFactor, true) if err != nil { return err } @@ -422,7 +386,7 @@ func (mts *MetricbeatTestSuite) runMetricbeatService() error { mts.Version = mts.Version + "-amd64" err = docker.TagImage( - "docker.elastic.co/beats/metricbeat:"+metricbeatVersionBase, + "docker.elastic.co/beats/metricbeat:"+common.BeatVersionBase, "docker.elastic.co/observability-ci/metricbeat:"+mts.Version, ) if err != nil { @@ -431,7 +395,7 @@ func (mts *MetricbeatTestSuite) runMetricbeatService() error { } // this is needed because, in general, the target service (apache, mysql, redis) does not have a healthcheck - waitForService := time.Duration(common.TimeoutFactor) * 10 * time.Second + waitForService := time.Duration(utils.TimeoutFactor) * 10 * time.Second if mts.ServiceName == "ceph" { // see https://github.com/elastic/beats/blob/ef6274d0d1e36308a333cbed69846a1bd63528ae/metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree_integration_test.go#L35 // Ceph service needs more time to start up @@ -453,7 +417,7 @@ func (mts *MetricbeatTestSuite) runMetricbeatService() error { "logLevel": logLevel, "metricbeatConfigFile": mts.configurationFile, "metricbeatTag": mts.Version, - "stackVersion": stackVersion, + "stackVersion": common.StackVersion, mts.ServiceName + "Tag": mts.ServiceVersion, "serviceName": mts.ServiceName, } @@ -517,7 +481,7 @@ func (mts *MetricbeatTestSuite) serviceIsRunningForMetricbeat(serviceType string serviceType = strings.ToLower(serviceType) env := map[string]string{ - "stackVersion": stackVersion, + "stackVersion": common.StackVersion, } env = config.PutServiceEnvironment(env, serviceType, serviceVersion) @@ -542,7 +506,7 @@ func (mts *MetricbeatTestSuite) serviceVariantIsRunningForMetricbeat( serviceType = strings.ToLower(serviceType) env := map[string]string{ - "stackVersion": stackVersion, + "stackVersion": common.StackVersion, } env = config.PutServiceVariantEnvironment(env, serviceType, serviceVariant, serviceVersion) @@ -577,7 +541,7 @@ func (mts *MetricbeatTestSuite) thereAreEventsInTheIndex() error { } minimumHitsCount := 5 - maxTimeout := time.Duration(common.TimeoutFactor) * time.Minute + maxTimeout := time.Duration(utils.TimeoutFactor) * time.Minute result, err := elasticsearch.WaitForNumberOfHits(mts.currentContext, mts.getIndexName(), esQuery, minimumHitsCount, maxTimeout) if err != nil { @@ -613,7 +577,7 @@ func (mts *MetricbeatTestSuite) thereAreNoErrorsInTheIndex() error { } minimumHitsCount := 5 - maxTimeout := time.Duration(common.TimeoutFactor) * time.Minute + maxTimeout := time.Duration(utils.TimeoutFactor) * time.Minute result, err := elasticsearch.WaitForNumberOfHits(mts.currentContext, mts.getIndexName(), esQuery, minimumHitsCount, maxTimeout) if err != nil { diff --git a/internal/common/defaults.go b/internal/common/defaults.go index 160e8987b7..54ec149a6a 100644 --- a/internal/common/defaults.go +++ b/internal/common/defaults.go @@ -4,6 +4,12 @@ package common +import ( + "github.com/elastic/e2e-testing/internal/shell" + "github.com/elastic/e2e-testing/internal/utils" + log "github.com/sirupsen/logrus" +) + // ElasticAgentProcessName the name of the process for the Elastic Agent const ElasticAgentProcessName = "elastic-agent" @@ -22,12 +28,12 @@ const FleetProfileName = "fleet" // FleetServerAgentServiceName the name of the service for the Elastic Agent const FleetServerAgentServiceName = "fleet-server" -// AgentVersionBase is the base version of the agent to use -var AgentVersionBase = "7.13.0-SNAPSHOT" +// BeatVersionBase is the base version of the Beat to use +var BeatVersionBase = "7.13.0-SNAPSHOT" -// AgentVersion is the version of the agent to use +// BeatVersion is the version of the Beat to use // It can be overriden by BEAT_VERSION env var -var AgentVersion = AgentVersionBase +var BeatVersion = BeatVersionBase // AgentStaleVersion is the version of the agent to use as a base during upgrade // It can be overriden by ELASTIC_AGENT_STALE_VERSION env var. Using latest GA as a default. @@ -35,11 +41,11 @@ var AgentStaleVersion = "7.12-SNAPSHOT" // StackVersion is the version of the stack to use // It can be overriden by STACK_VERSION env var -var StackVersion = AgentVersionBase +var StackVersion = BeatVersionBase // KibanaVersion is the version of kibana to use // It can be override by KIBANA_VERSION -var KibanaVersion = AgentVersionBase +var KibanaVersion = BeatVersionBase // ProfileEnv is the environment to be applied to any execution // affecting the runtime dependencies (or profile) @@ -47,3 +53,42 @@ var ProfileEnv map[string]string // Provider is the deployment provider used, currently docker is supported var Provider = "docker" + +// InitVersions initialise default versions. We do not want to do it in the init phase +// supporting lazy-loading the versions when needed. Basically, the CLI part does not +// need to load them +func InitVersions() { + v, err := utils.GetElasticArtifactVersion(BeatVersionBase) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + "version": BeatVersionBase, + }).Fatal("Failed to get Beat base version, aborting") + } + BeatVersionBase = v + + BeatVersion = shell.GetEnv("BEAT_VERSION", BeatVersionBase) + + // check if version is an alias + v, err = utils.GetElasticArtifactVersion(BeatVersion) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + "version": BeatVersion, + }).Fatal("Failed to get Beat version, aborting") + } + BeatVersion = v + + // detects if the BeatVersion is set by the GITHUB_CHECK_SHA1 variable + BeatVersion = utils.CheckPRVersion(BeatVersion, BeatVersionBase) + + StackVersion = shell.GetEnv("STACK_VERSION", BeatVersionBase) + v, err = utils.GetElasticArtifactVersion(StackVersion) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + "version": StackVersion, + }).Fatal("Failed to get stack version, aborting") + } + StackVersion = v +} diff --git a/internal/docker/docker.go b/internal/docker/docker.go index e7aa629597..ea795961e5 100644 --- a/internal/docker/docker.go +++ b/internal/docker/docker.go @@ -23,7 +23,7 @@ import ( "github.com/docker/docker/api/types/filters" "github.com/docker/docker/client" "github.com/docker/docker/pkg/stdcopy" - "github.com/elastic/e2e-testing/internal/common" + "github.com/elastic/e2e-testing/internal/utils" log "github.com/sirupsen/logrus" ) @@ -385,7 +385,7 @@ func TagImage(src string, target string) error { dockerClient := getDockerClient() maxTimeout := 15 * time.Second - exp := common.GetExponentialBackOff(maxTimeout) + exp := utils.GetExponentialBackOff(maxTimeout) retryCount := 0 tagImageFn := func() error { diff --git a/internal/elasticsearch/client.go b/internal/elasticsearch/client.go index c8b32e48f8..4de42d8a8e 100644 --- a/internal/elasticsearch/client.go +++ b/internal/elasticsearch/client.go @@ -13,9 +13,9 @@ import ( "time" backoff "github.com/cenkalti/backoff/v4" - "github.com/elastic/e2e-testing/internal/common" curl "github.com/elastic/e2e-testing/internal/curl" "github.com/elastic/e2e-testing/internal/shell" + "github.com/elastic/e2e-testing/internal/utils" es "github.com/elastic/go-elasticsearch/v8" log "github.com/sirupsen/logrus" "go.elastic.co/apm" @@ -202,7 +202,7 @@ func WaitForElasticsearch(ctx context.Context, maxTimeoutMinutes time.Duration) // WaitForElasticsearchFromHostPort waits for an elasticsearch running in a host:port to be healthy, returning false // if elasticsearch does not get healthy status in a defined number of minutes. func WaitForElasticsearchFromHostPort(ctx context.Context, host string, port int, maxTimeoutMinutes time.Duration) (bool, error) { - exp := common.GetExponentialBackOff(maxTimeoutMinutes) + exp := utils.GetExponentialBackOff(maxTimeoutMinutes) retryCount := 1 @@ -251,7 +251,7 @@ func WaitForElasticsearchFromHostPort(ctx context.Context, host string, port int // WaitForIndices waits for the elasticsearch indices to return the list of indices. func WaitForIndices() (string, error) { - exp := common.GetExponentialBackOff(60 * time.Second) + exp := utils.GetExponentialBackOff(60 * time.Second) retryCount := 1 body := "" @@ -294,7 +294,7 @@ func WaitForIndices() (string, error) { // WaitForNumberOfHits waits for an elasticsearch query to return more than a number of hits, // returning false if the query does not reach that number in a defined number of time. func WaitForNumberOfHits(ctx context.Context, indexName string, query map[string]interface{}, desiredHits int, maxTimeout time.Duration) (SearchResult, error) { - exp := common.GetExponentialBackOff(maxTimeout) + exp := utils.GetExponentialBackOff(maxTimeout) retryCount := 1 result := SearchResult{} diff --git a/internal/installer/deb.go b/internal/installer/deb.go index 5ec4c07c2b..2c708acbc1 100644 --- a/internal/installer/deb.go +++ b/internal/installer/deb.go @@ -77,7 +77,7 @@ func newDebianInstaller(image string, tag string, version string) (ElasticAgentI arch := "amd64" extension := "deb" - binaryName := utils.BuildArtifactName(artifact, version, common.AgentVersionBase, os, arch, extension, false) + binaryName := utils.BuildArtifactName(artifact, version, common.BeatVersionBase, os, arch, extension, false) binaryPath, err := downloadAgentBinary(binaryName, artifact, version) if err != nil { log.WithFields(log.Fields{ diff --git a/internal/installer/docker.go b/internal/installer/docker.go index 1877dde9e6..3906a10d15 100644 --- a/internal/installer/docker.go +++ b/internal/installer/docker.go @@ -62,7 +62,7 @@ func (i *DockerPackage) Preinstall() error { // we need to tag the loaded image because its tag relates to the target branch return docker.TagImage( - "docker.elastic.co/beats/"+i.artifact+":"+common.AgentVersionBase, + "docker.elastic.co/beats/"+i.artifact+":"+common.BeatVersionBase, "docker.elastic.co/observability-ci/"+i.artifact+":"+i.originalVersion+"-amd64", ) } @@ -99,7 +99,7 @@ func (i *DockerPackage) WithOS(OS string) *DockerPackage { // WithVersion sets the version func (i *DockerPackage) WithVersion(version string) *DockerPackage { - i.version = utils.CheckPRVersion(version, common.AgentVersionBase) // sanitize version + i.version = utils.CheckPRVersion(version, common.BeatVersionBase) // sanitize version i.originalVersion = version return i } @@ -123,7 +123,7 @@ func newDockerInstaller(ubi8 bool, version string) (ElasticAgentInstaller, error arch := "amd64" extension := "tar.gz" - binaryName := utils.BuildArtifactName(artifactName, version, common.AgentVersionBase, os, arch, extension, true) + binaryName := utils.BuildArtifactName(artifactName, version, common.BeatVersionBase, os, arch, extension, true) binaryPath, err := downloadAgentBinary(binaryName, artifact, version) if err != nil { log.WithFields(log.Fields{ diff --git a/internal/installer/elasticagent.go b/internal/installer/elasticagent.go index 347af4d3dc..45213be826 100644 --- a/internal/installer/elasticagent.go +++ b/internal/installer/elasticagent.go @@ -64,7 +64,7 @@ func (i *ElasticAgentInstaller) ListElasticAgentWorkingDirContent(containerName // runElasticAgentCommandEnv runs a command for the elastic-agent func runElasticAgentCommandEnv(profile string, image string, service string, process string, command string, arguments []string, env map[string]string) error { cmds := []string{ - "timeout", fmt.Sprintf("%dm", common.TimeoutFactor), process, command, + "timeout", fmt.Sprintf("%dm", utils.TimeoutFactor), process, command, } cmds = append(cmds, arguments...) @@ -97,7 +97,7 @@ func runElasticAgentCommandEnv(profile string, image string, service string, pro // Else, if the environment variable BEATS_USE_CI_SNAPSHOTS is set, then the artifact // to be downloaded will be defined by the latest snapshot produced by the Beats CI. func downloadAgentBinary(artifactName string, artifact string, version string) (string, error) { - imagePath, err := utils.FetchBeatsBinary(artifactName, artifact, version, common.AgentVersionBase, common.TimeoutFactor, true) + imagePath, err := utils.FetchBeatsBinary(artifactName, artifact, version, common.BeatVersionBase, utils.TimeoutFactor, true) if err != nil { return "", err } diff --git a/internal/installer/rpm.go b/internal/installer/rpm.go index acc7c554fb..b80c723c18 100644 --- a/internal/installer/rpm.go +++ b/internal/installer/rpm.go @@ -81,7 +81,7 @@ func newCentosInstaller(image string, tag string, version string) (ElasticAgentI arch := "x86_64" extension := "rpm" - binaryName := utils.BuildArtifactName(artifact, version, common.AgentVersionBase, os, arch, extension, false) + binaryName := utils.BuildArtifactName(artifact, version, common.BeatVersionBase, os, arch, extension, false) binaryPath, err := downloadAgentBinary(binaryName, artifact, version) if err != nil { log.WithFields(log.Fields{ diff --git a/internal/installer/tar.go b/internal/installer/tar.go index 8d9257ad4c..b98f5c04ba 100644 --- a/internal/installer/tar.go +++ b/internal/installer/tar.go @@ -152,7 +152,7 @@ func newTarInstaller(image string, tag string, version string) (ElasticAgentInst arch := "x86_64" extension := "tar.gz" - binaryName := utils.BuildArtifactName(artifact, version, common.AgentVersionBase, os, arch, extension, false) + binaryName := utils.BuildArtifactName(artifact, version, common.BeatVersionBase, os, arch, extension, false) binaryPath, err := downloadAgentBinary(binaryName, artifact, version) if err != nil { log.WithFields(log.Fields{ @@ -184,7 +184,7 @@ func newTarInstaller(image string, tag string, version string) (ElasticAgentInst WithArtifact(artifact). WithOS(os). WithOSFlavour(image). - WithVersion(utils.CheckPRVersion(version, common.AgentVersionBase)) // sanitize version + WithVersion(utils.CheckPRVersion(version, common.BeatVersionBase)) // sanitize version return ElasticAgentInstaller{ artifactArch: arch, diff --git a/internal/kibana/server.go b/internal/kibana/server.go index 9983a42bda..0ff3ae8f91 100644 --- a/internal/kibana/server.go +++ b/internal/kibana/server.go @@ -12,7 +12,7 @@ import ( "github.com/Jeffail/gabs/v2" "github.com/cenkalti/backoff/v4" - "github.com/elastic/e2e-testing/internal/common" + "github.com/elastic/e2e-testing/internal/utils" "github.com/pkg/errors" log "github.com/sirupsen/logrus" "go.elastic.co/apm" @@ -193,8 +193,8 @@ func (c *Client) RecreateFleet() error { }).Info("Fleet setup done") return nil } - maxTimeout := time.Duration(common.TimeoutFactor) * time.Minute * 2 - exp := common.GetExponentialBackOff(maxTimeout) + maxTimeout := time.Duration(utils.TimeoutFactor) * time.Minute * 2 + exp := utils.GetExponentialBackOff(maxTimeout) err := backoff.Retry(waitForFleet, exp) if err != nil { @@ -244,8 +244,8 @@ func (c *Client) WaitForFleet() error { log.Info("Fleet setup complete") return nil } - maxTimeout := time.Duration(common.TimeoutFactor) * time.Minute * 2 - exp := common.GetExponentialBackOff(maxTimeout) + maxTimeout := time.Duration(utils.TimeoutFactor) * time.Minute * 2 + exp := utils.GetExponentialBackOff(maxTimeout) err := backoff.Retry(waitForFleet, exp) if err != nil { @@ -257,8 +257,8 @@ func (c *Client) WaitForFleet() error { // WaitForReady waits for Kibana to be healthy and accept connections func (c *Client) WaitForReady(maxTimeoutMinutes time.Duration) (bool, error) { - maxTimeout := time.Duration(common.TimeoutFactor) * time.Minute * 2 - exp := common.GetExponentialBackOff(maxTimeout) + maxTimeout := time.Duration(utils.TimeoutFactor) * time.Minute * 2 + exp := utils.GetExponentialBackOff(maxTimeout) ctx := context.Background() diff --git a/internal/kubernetes/kubernetes.go b/internal/kubernetes/kubernetes.go index a050d6af05..dbbf5fa65c 100644 --- a/internal/kubernetes/kubernetes.go +++ b/internal/kubernetes/kubernetes.go @@ -17,8 +17,8 @@ import ( "github.com/google/uuid" log "github.com/sirupsen/logrus" - "github.com/elastic/e2e-testing/internal/common" "github.com/elastic/e2e-testing/internal/shell" + "github.com/elastic/e2e-testing/internal/utils" ) // Control struct for k8s cluster @@ -67,7 +67,7 @@ func (c Control) createNamespace(ctx context.Context, namespace string) error { // Wait for default account to be available, if not it is not possible to // deploy pods in this namespace. timeout := 60 * time.Second - exp := backoff.WithContext(common.GetExponentialBackOff(timeout), ctx) + exp := backoff.WithContext(utils.GetExponentialBackOff(timeout), ctx) return backoff.Retry(func() error { _, err := c.Run(ctx, "get", "serviceaccount", "default") if err != nil { diff --git a/internal/common/retry.go b/internal/utils/retry.go similarity index 98% rename from internal/common/retry.go rename to internal/utils/retry.go index 2c221bd61f..c1ce828388 100644 --- a/internal/common/retry.go +++ b/internal/utils/retry.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package common +package utils import ( "time" diff --git a/internal/utils/utils.go b/internal/utils/utils.go index 3059381535..ce942d2ac7 100644 --- a/internal/utils/utils.go +++ b/internal/utils/utils.go @@ -18,7 +18,6 @@ import ( "github.com/Jeffail/gabs/v2" backoff "github.com/cenkalti/backoff/v4" - "github.com/elastic/e2e-testing/internal/common" curl "github.com/elastic/e2e-testing/internal/curl" internalio "github.com/elastic/e2e-testing/internal/io" "github.com/elastic/e2e-testing/internal/shell" @@ -26,6 +25,11 @@ import ( log "github.com/sirupsen/logrus" ) +// to avoid fetching the same Elastic artifacts version, we are adding this map to cache the version of the Elastic artifacts, +// using as key the URL of the version. If another request is trying to fetch the same URL, it will return the string version +// of the already requested one. +var elasticVersionsCache = map[string]string{} + // to avoid downloading the same artifacts, we are adding this map to cache the URL of the downloaded binaries, using as key // the URL of the artifact. If another installer is trying to download the same URL, it will return the location of the // already downloaded artifact. @@ -187,7 +191,17 @@ func getGCPBucketCoordinates(fileName string, artifact string, version string, f // If the version is a PR, then it will return the version without checking the artifacts API // i.e. GetElasticArtifactVersion("$VERSION") func GetElasticArtifactVersion(version string) (string, error) { - exp := common.GetExponentialBackOff(time.Minute) + cacheKey := fmt.Sprintf("https://artifacts-api.elastic.co/v1/versions/%s/?x-elastic-no-kpi=true", version) + + if val, ok := elasticVersionsCache[cacheKey]; ok { + log.WithFields(log.Fields{ + "URL": cacheKey, + "version": val, + }).Debug("Retrieving version from local cache") + return val, nil + } + + exp := GetExponentialBackOff(time.Minute) retryCount := 1 @@ -195,7 +209,7 @@ func GetElasticArtifactVersion(version string) (string, error) { apiStatus := func() error { r := curl.HTTPRequest{ - URL: fmt.Sprintf("https://artifacts-api.elastic.co/v1/versions/%s/?x-elastic-no-kpi=true", version), + URL: cacheKey, } response, err := curl.Get(r) @@ -247,6 +261,8 @@ func GetElasticArtifactVersion(version string) (string, error) { "version": latestVersion, }).Debug("Latest version for current version obtained") + elasticVersionsCache[cacheKey] = latestVersion + return latestVersion, nil } @@ -256,7 +272,7 @@ func GetElasticArtifactVersion(version string) (string, error) { // i.e. GetElasticArtifactURL("elastic-agent-$VERSION-x86_64.rpm", "elastic-agent","$VERSION") // i.e. GetElasticArtifactURL("elastic-agent-$VERSION-linux-amd64.tar.gz", "elastic-agent","$VERSION") func GetElasticArtifactURL(artifactName string, artifact string, version string) (string, error) { - exp := common.GetExponentialBackOff(time.Minute) + exp := GetExponentialBackOff(time.Minute) retryCount := 1 @@ -320,7 +336,7 @@ func GetElasticArtifactURL(artifactName string, artifact string, version string) // GetObjectURLFromBucket extracts the media URL for the desired artifact from the // Google Cloud Storage bucket used by the CI to push snapshots func GetObjectURLFromBucket(bucket string, prefix string, object string, maxtimeout time.Duration) (string, error) { - exp := common.GetExponentialBackOff(maxtimeout) + exp := GetExponentialBackOff(maxtimeout) retryCount := 1 @@ -447,7 +463,7 @@ func DownloadFile(url string) (string, error) { filepath := tempFile.Name() - exp := common.GetExponentialBackOff(3) + exp := GetExponentialBackOff(3) retryCount := 1 var fileReader io.ReadCloser