[chore] Fix E2E autoscale test for OpenShift (open-telemetry#1365)

* Improve the reliability of the autoscale E2E test

Signed-off-by: Israel Blancas <[email protected]>

* Revert change

Signed-off-by: Israel Blancas <[email protected]>

---------

Signed-off-by: Israel Blancas <[email protected]>

tests/e2e/autoscale/00-assert.yaml

@@ -5,28 +5,9 @@ metadata:
 status:
   readyReplicas: 1
 ---
-apiVersion: autoscaling/v1
-kind: HorizontalPodAutoscaler
-metadata:
-  name: simplest-collector
-spec:
-  minReplicas: 1
-  maxReplicas: 2
-# This is not neccesarily exact.  We really just want to wait until this is no longer <unknown>
-status:
-  currentCPUUtilizationPercentage: 20
----
-apiVersion: autoscaling/v2beta2
-kind: HorizontalPodAutoscaler
+apiVersion: apps/v1
+kind: Deployment
 metadata:
   name: simplest-set-utilization-collector
-spec:
-  minReplicas: 1
-  maxReplicas: 2
-  metrics:
-  - type: Resource
-    resource:
-      name: cpu
-      target:
-        type: Utilization
-        averageUtilization: 50
+status:
+  readyReplicas: 1

tests/e2e/autoscale/00-install.yaml

@@ -1,5 +1,6 @@
-# This creates two different deployments. The first one will be used to see if we scale properly.  (Note that we are
-# only scaling up to 2 because of limitations of KUTTL). The second is to check the targetCPUUtilization option.
+# This creates two different deployments:
+#   * The first one will be used to see if we scale properly
+#   * The second is to check the targetCPUUtilization option
 #
 apiVersion: opentelemetry.io/v1alpha1
 kind: OpenTelemetryCollector

tests/e2e/autoscale/01-check-simplest-collector-hpa.yaml

@@ -0,0 +1,4 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: go run ./wait-until-hpa-ready.go --hpa simplest-collector

tests/e2e/autoscale/02-check-simplest-set-utilization-collector-hpa.yaml

@@ -0,0 +1,4 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: go run ./wait-until-hpa-ready.go --hpa simplest-set-utilization-collector

tests/e2e/autoscale/01-assert.yaml → tests/e2e/autoscale/03-assert.yaml

@@ -1,17 +1,6 @@
 # Wait until tracegen has completed and the simplest deployment has scaled up to 2
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: tracegen
-status:
-  conditions:
-    - status: "True"
-      type: Complete
-
----
 apiVersion: opentelemetry.io/v1alpha1
 kind: OpenTelemetryCollector
-
 metadata:
   name: simplest
 status:

tests/e2e/autoscale/01-install.yaml → tests/e2e/autoscale/03-install.yaml

@@ -13,6 +13,8 @@ spec:
           args:
             - -otlp-endpoint=simplest-collector-headless:4317
             - -otlp-insecure
+            # High duration to ensure the trace creation doesn't stop.
+            # It'll be stopped in step 4
             - -duration=1m
             - -workers=20
       restartPolicy: Never

tests/e2e/autoscale/04-delete.yaml

@@ -0,0 +1,7 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+delete:
+- apiVersion: batch/v1
+  kind: Job
+  metadata:
+    name: tracegen

tests/e2e/autoscale/02-assert.yaml → tests/e2e/autoscale/05-assert.yaml

@@ -1,7 +1,6 @@
 # Wait for the collector to scale back down to 1
 apiVersion: opentelemetry.io/v1alpha1
 kind: OpenTelemetryCollector
-
 metadata:
   name: simplest
 status:

tests/e2e/autoscale/wait-until-hpa-ready.go

@@ -0,0 +1,109 @@
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/spf13/pflag"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/client-go/util/homedir"
+)
+
+func main() {
+	var hpaName string
+	var timeout int
+	var kubeconfigPath string
+
+	defaultKubeconfigPath := filepath.Join(homedir.HomeDir(), ".kube", "config")
+
+	pflag.IntVar(&timeout, "timeout", 600, "The timeout for the check.")
+	pflag.StringVar(&hpaName, "hpa", "", "HPA to check")
+	pflag.StringVar(&kubeconfigPath, "kubeconfig-path", defaultKubeconfigPath, "Absolute path to the KubeconfigPath file")
+	pflag.Parse()
+
+	if len(hpaName) == 0 {
+		fmt.Println("hpa flag is mandatory")
+		os.Exit(1)
+	}
+
+	config, err := clientcmd.BuildConfigFromFlags("", kubeconfigPath)
+	if err != nil {
+		fmt.Printf("Error reading the kubeconfig: %s\n", err)
+		os.Exit(1)
+	}
+
+	client, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+
+	namespace, err := client.CoreV1().Namespaces().Get(context.Background(), os.Getenv("NAMESPACE"), metav1.GetOptions{})
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+
+	hpaClientV2 := client.AutoscalingV2().HorizontalPodAutoscalers(namespace.Name)
+	hpaClientV1 := client.AutoscalingV1().HorizontalPodAutoscalers(namespace.Name)
+
+	pollInterval := time.Second
+
+	// Search in v2 and v1 for an HPA with the given name
+	err = wait.Poll(pollInterval, 0, func() (done bool, err error) {
+		hpav2, err := hpaClientV2.Get(
+			context.Background(),
+			hpaName,
+			metav1.GetOptions{},
+		)
+		if err != nil {
+			hpav1, err := hpaClientV1.Get(
+				context.Background(),
+				hpaName,
+				metav1.GetOptions{},
+			)
+			if err != nil {
+				fmt.Printf("HPA %s not found\n", hpaName)
+				return false, nil
+			}
+
+			if hpav1.Status.CurrentCPUUtilizationPercentage == nil {
+				fmt.Printf("Current metrics are not set yet for HPA %s\n", hpaName)
+				return false, nil
+			}
+			return true, nil
+		}
+
+		if hpav2.Status.CurrentMetrics == nil {
+			fmt.Printf("Current metrics are not set yet for HPA %s\n", hpaName)
+			return false, nil
+		}
+		return true, nil
+	})
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+
+	fmt.Printf("%s is ready!\n", hpaName)
+}