test: Self-monitor webhook E2E test (#932)

.github/workflows/pr-integration.yml

@@ -200,7 +200,7 @@ jobs:
         with:
           failure: failure()
 
-  e2e-self-monitor:
+  e2e-self-monitor-traces:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repo
@@ -212,7 +212,27 @@ jobs:
           github-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Run tests
-        run:  bin/ginkgo run --tags e2e --label-filter="self-mon" test/e2e
+        run:  bin/ginkgo run --tags e2e --label-filter="self-mon-traces" test/e2e
+
+      - name: Finalize Test
+        uses: "./.github/template/finalize-test"
+        if: success() || failure()
+        with:
+          failure: failure()
+
+  e2e-self-monitor-metrics:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+
+      - name: Prepare Test
+        uses: "./.github/template/prepare-test"
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Run tests
+        run: bin/ginkgo run --tags e2e --label-filter="self-mon-metrics" test/e2e
 
       - name: Finalize Test
         uses: "./.github/template/finalize-test"

.golangci.yaml

@@ -78,6 +78,8 @@ linters-settings:
         alias: logpipelinevalidationmocks
       - pkg: github.com/prometheus/client_golang/api/prometheus/v1
         alias: promv1
+      - pkg: github.com/prometheus/client_model/go
+        alias: prommodel
       - pkg: go.opentelemetry.io/otel/sdk/trace
         alias: tracesdk
       - pkg: istio.io/api/security/v1beta1

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/onsi/ginkgo/v2 v2.17.1
 	github.com/onsi/gomega v1.31.1
 	github.com/prometheus/client_golang v1.19.0
+	github.com/prometheus/client_model v0.6.0
 	github.com/prometheus/common v0.50.0
 	github.com/stretchr/testify v1.9.0
 	go.opentelemetry.io/collector/pdata v1.3.0
@@ -63,7 +64,6 @@ require (
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
-	github.com/prometheus/client_model v0.6.0 // indirect
 	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/stretchr/objx v0.5.2 // indirect

test/e2e/manager_test.go

@@ -78,19 +78,15 @@ var _ = Describe("Telemetry Manager", func() {
 
 		It("Should have a metrics service", Label("telemetry"), func() {
 			var service corev1.Service
-			key := types.NamespacedName{
-				Name:      "telemetry-manager-metrics",
-				Namespace: kitkyma.SystemNamespaceName,
-			}
-			err := k8sClient.Get(ctx, key, &service)
+			err := k8sClient.Get(ctx, kitkyma.TelemetryManagerMetricsServiceName, &service)
 			Expect(err).NotTo(HaveOccurred())
 
 			Expect(service.Annotations).Should(HaveKeyWithValue("prometheus.io/scrape", "true"))
 			Expect(service.Annotations).Should(HaveKeyWithValue("prometheus.io/port", "8080"))
 
 			Eventually(func() []corev1.EndpointAddress {
 				var endpoints corev1.Endpoints
-				err := k8sClient.Get(ctx, key, &endpoints)
+				err := k8sClient.Get(ctx, kitkyma.TelemetryManagerMetricsServiceName, &endpoints)
 				Expect(err).NotTo(HaveOccurred())
 				return endpoints.Subsets[0].Addresses
 			}, periodic.EventuallyTimeout, periodic.DefaultInterval).ShouldNot(BeEmpty())

test/e2e/metrics_namespace_selector_test.go

@@ -59,9 +59,9 @@ var _ = Describe("Metrics Namespace Selector", Label("metrics"), func() {
 		objs = append(objs, pipelineExcludeApp1Ns.K8sObject())
 
 		objs = append(objs,
-			telemetrygen.New(app1Ns).K8sObject(),
-			telemetrygen.New(app2Ns).K8sObject(),
-			telemetrygen.New(kitkyma.SystemNamespaceName).K8sObject(),
+			telemetrygen.New(app1Ns, telemetrygen.SignalTypeMetrics).K8sObject(),
+			telemetrygen.New(app2Ns, telemetrygen.SignalTypeMetrics).K8sObject(),
+			telemetrygen.New(kitkyma.SystemNamespaceName, telemetrygen.SignalTypeMetrics).K8sObject(),
 
 			prommetricgen.New(app1Ns).Pod().WithPrometheusAnnotations(prommetricgen.SchemeHTTP).K8sObject(),
 			prommetricgen.New(app2Ns).Pod().WithPrometheusAnnotations(prommetricgen.SchemeHTTP).K8sObject(),

test/e2e/metrics_otlp_input_test.go

@@ -36,7 +36,7 @@ var _ = Describe("Metrics OTLP Input", Label("metrics"), func() {
 			OtlpInput(false)
 		objs = append(objs, pipelineWithoutOTLP.K8sObject())
 
-		objs = append(objs, telemetrygen.New(appNs).K8sObject())
+		objs = append(objs, telemetrygen.New(appNs, telemetrygen.SignalTypeMetrics).K8sObject())
 		return objs
 	}
 

test/e2e/self_monitor_basic_metrics_test.go → test/e2e/metrics_self_monitor_basic_test.go

@@ -10,19 +10,22 @@ import (
 	. "github.com/onsi/gomega"
 	corev1 "k8s.io/api/core/v1"
 	networkingv1 "k8s.io/api/networking/v1"
+	"k8s.io/apimachinery/pkg/api/meta"
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
+	telemetryv1alpha1 "github.com/kyma-project/telemetry-manager/apis/telemetry/v1alpha1"
+	"github.com/kyma-project/telemetry-manager/internal/conditions"
 	"github.com/kyma-project/telemetry-manager/internal/otelcollector/ports"
 	kitk8s "github.com/kyma-project/telemetry-manager/test/testkit/k8s"
 	kitkyma "github.com/kyma-project/telemetry-manager/test/testkit/kyma"
 	"github.com/kyma-project/telemetry-manager/test/testkit/mocks/backend"
-	kitmetrics "github.com/kyma-project/telemetry-manager/test/testkit/otel/metrics"
+	"github.com/kyma-project/telemetry-manager/test/testkit/mocks/telemetrygen"
 	"github.com/kyma-project/telemetry-manager/test/testkit/periodic"
 	"github.com/kyma-project/telemetry-manager/test/testkit/verifiers"
 )
 
-var _ = Describe("Telemetry Self Monitor", Label("self-mon"), Ordered, func() {
+var _ = Describe("Metrics Self Monitor", Label("self-mon-metrics"), Ordered, func() {
 	const (
 		mockBackendName = "metrics-receiver-selfmon"
 		mockNs          = "metrics-basic-selfmon-test"
@@ -38,20 +41,22 @@ var _ = Describe("Telemetry Self Monitor", Label("self-mon"), Ordered, func() {
 
 		objs = append(objs, kitk8s.NewNamespace(mockNs).K8sObject())
 
-		mockBackend := backend.New(mockBackendName, mockNs, backend.SignalTypeMetrics, backend.WithPersistentHostSecret(isOperational()))
+		mockBackend := backend.New(mockBackendName, mockNs, backend.SignalTypeMetrics)
 		objs = append(objs, mockBackend.K8sObjects()...)
 		telemetryExportURL = mockBackend.TelemetryExportURL(proxyClient)
 
 		pipeline := kitk8s.NewMetricPipelineV1Alpha1(fmt.Sprintf("%s-pipeline", mockBackendName)).
-			WithOutputEndpointFromSecret(mockBackend.HostSecretRefV1Alpha1()).
-			RuntimeInput(true)
+			WithOutputEndpointFromSecret(mockBackend.HostSecretRefV1Alpha1())
 		pipelineName = pipeline.Name()
-		objs = append(objs, pipeline.K8sObject())
+		objs = append(objs,
+			telemetrygen.New(kitkyma.DefaultNamespaceName, telemetrygen.SignalTypeMetrics).K8sObject(),
+			pipeline.K8sObject(),
+		)
 
 		return objs
 	}
 
-	Context("When a metrics pipeline exists", Ordered, func() {
+	Context("When a metric pipeline exists", Ordered, func() {
 		BeforeAll(func() {
 			k8sObjects := makeResources()
 
@@ -60,9 +65,11 @@ var _ = Describe("Telemetry Self Monitor", Label("self-mon"), Ordered, func() {
 			})
 			Expect(kitk8s.CreateObjects(ctx, k8sClient, k8sObjects...)).Should(Succeed())
 		})
+
 		It("Should have a running self-monitor", func() {
 			verifiers.DeploymentShouldBeReady(ctx, k8sClient, kitkyma.SelfMonitorName)
 		})
+
 		It("Should have a network policy deployed", func() {
 			var networkPolicy networkingv1.NetworkPolicy
 			Expect(k8sClient.Get(ctx, kitkyma.SelfMonitorNetworkPolicy, &networkPolicy)).To(Succeed())
@@ -86,27 +93,31 @@ var _ = Describe("Telemetry Self Monitor", Label("self-mon"), Ordered, func() {
 			Expect(k8sClient.Get(ctx, kitkyma.SelfMonitorName, &service)).To(Succeed())
 		})
 
-		It("Should have a metrics backend running", Label(operationalTest), func() {
+		It("Should have a metrics backend running", func() {
 			verifiers.DeploymentShouldBeReady(ctx, k8sClient, types.NamespacedName{Name: mockBackendName, Namespace: mockNs})
 		})
 
-		It("Should have a running pipeline", Label(operationalTest), func() {
+		It("Should have a running pipeline", func() {
 			verifiers.MetricPipelineShouldBeHealthy(ctx, k8sClient, pipelineName)
 		})
 
-		It("Should verify end-to-end metric delivery", Label(operationalTest), func() {
-			gatewayPushURL := proxyClient.ProxyURLForService(kitkyma.SystemNamespaceName, "telemetry-otlp-metrics", "v1/metrics/", ports.OTLPHTTP)
-			gauges := kitmetrics.MakeAndSendGaugeMetrics(proxyClient, gatewayPushURL)
-			verifiers.MetricsShouldBeDelivered(proxyClient, telemetryExportURL, gauges)
+		It("Should deliver telemetrygen metrics", func() {
+			verifiers.MetricsFromNamespaceShouldBeDelivered(proxyClient, telemetryExportURL, kitkyma.DefaultNamespaceName, telemetrygen.MetricNames)
 		})
 
-		It("Should be able to get metric gateway metrics endpoint", Label(operationalTest), func() {
-			gatewayMetricsURL := proxyClient.ProxyURLForService(kitkyma.MetricGatewayMetrics.Namespace, kitkyma.MetricGatewayMetrics.Name, "metrics", ports.Metrics)
-			verifiers.ShouldExposeCollectorMetrics(proxyClient, gatewayMetricsURL)
+		It("Should have TypeFlowHealthy condition set to True", func() {
+			//TODO: add the conditions.TypeFlowHealthy check to verifiers.MetricPipelineShouldBeHealthy after self monitor is released
+			Eventually(func(g Gomega) {
+				var pipeline telemetryv1alpha1.MetricPipeline
+				key := types.NamespacedName{Name: pipelineName}
+				g.Expect(k8sClient.Get(ctx, key, &pipeline)).To(Succeed())
+				g.Expect(meta.IsStatusConditionTrue(pipeline.Status.Conditions, conditions.TypeFlowHealthy)).To(BeTrue())
+			}, periodic.EventuallyTimeout, periodic.DefaultInterval).Should(Succeed())
 		})
 
-		It("The telemetryFlowHealthy condition should be true", func() {
-			verifiers.MetricPipelineTelemetryHealthFlowIsHealthy(ctx, k8sClient, pipelineName)
+		It("Should ensure that the self-monitor webhook has been called", func() {
+			// Pushing metrics to the metric gateway triggers an alert, which in turn makes the self-monitor call the webhook
+			verifiers.SelfMonitorWebhookShouldHaveBeenCalled(proxyClient)
 		})
 	})
 })

test/e2e/self_monitor_basic_trace_test.go → test/e2e/traces_self_monitor_basic_test.go

@@ -10,18 +10,22 @@ import (
 	. "github.com/onsi/gomega"
 	corev1 "k8s.io/api/core/v1"
 	networkingv1 "k8s.io/api/networking/v1"
+	"k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
+	telemetryv1alpha1 "github.com/kyma-project/telemetry-manager/apis/telemetry/v1alpha1"
+	"github.com/kyma-project/telemetry-manager/internal/conditions"
 	"github.com/kyma-project/telemetry-manager/internal/otelcollector/ports"
 	kitk8s "github.com/kyma-project/telemetry-manager/test/testkit/k8s"
 	kitkyma "github.com/kyma-project/telemetry-manager/test/testkit/kyma"
 	"github.com/kyma-project/telemetry-manager/test/testkit/mocks/backend"
-	kittraces "github.com/kyma-project/telemetry-manager/test/testkit/otel/traces"
+	"github.com/kyma-project/telemetry-manager/test/testkit/mocks/telemetrygen"
 	"github.com/kyma-project/telemetry-manager/test/testkit/periodic"
 	"github.com/kyma-project/telemetry-manager/test/testkit/verifiers"
 )
 
-var _ = Describe("Telemetry Self Monitor", Label("self-mon"), Ordered, func() {
+var _ = Describe("Traces Self Monitor", Label("self-mon-traces"), Ordered, func() {
 	const (
 		mockBackendName = "traces-receiver-selfmon"
 		mockNs          = "traces-basic-selfmon-test"
@@ -37,15 +41,17 @@ var _ = Describe("Telemetry Self Monitor", Label("self-mon"), Ordered, func() {
 
 		objs = append(objs, kitk8s.NewNamespace(mockNs).K8sObject())
 
-		mockBackend := backend.New(mockBackendName, mockNs, backend.SignalTypeTraces, backend.WithPersistentHostSecret(isOperational()))
+		mockBackend := backend.New(mockBackendName, mockNs, backend.SignalTypeTraces)
 		objs = append(objs, mockBackend.K8sObjects()...)
 		telemetryExportURL = mockBackend.TelemetryExportURL(proxyClient)
 
 		pipeline := kitk8s.NewTracePipelineV1Alpha1(fmt.Sprintf("%s-pipeline", mockBackend.Name())).
-			WithOutputEndpointFromSecret(mockBackend.HostSecretRefV1Alpha1()).
-			Persistent(isOperational())
+			WithOutputEndpointFromSecret(mockBackend.HostSecretRefV1Alpha1())
 		pipelineName = pipeline.Name()
-		objs = append(objs, pipeline.K8sObject())
+		objs = append(objs,
+			pipeline.K8sObject(),
+			telemetrygen.New(kitkyma.DefaultNamespaceName, telemetrygen.SignalTypeTraces).K8sObject(),
+		)
 
 		return objs
 	}
@@ -59,9 +65,11 @@ var _ = Describe("Telemetry Self Monitor", Label("self-mon"), Ordered, func() {
 			})
 			Expect(kitk8s.CreateObjects(ctx, k8sClient, k8sObjects...)).Should(Succeed())
 		})
+
 		It("Should have a running self-monitor", func() {
 			verifiers.DeploymentShouldBeReady(ctx, k8sClient, kitkyma.SelfMonitorName)
 		})
+
 		It("Should have a network policy deployed", func() {
 			var networkPolicy networkingv1.NetworkPolicy
 			Expect(k8sClient.Get(ctx, kitkyma.SelfMonitorNetworkPolicy, &networkPolicy)).To(Succeed())
@@ -85,23 +93,27 @@ var _ = Describe("Telemetry Self Monitor", Label("self-mon"), Ordered, func() {
 			Expect(k8sClient.Get(ctx, kitkyma.SelfMonitorName, &service)).To(Succeed())
 		})
 
-		It("Should have a running pipeline", Label(operationalTest), func() {
+		It("Should have a running pipeline", func() {
 			verifiers.TracePipelineShouldBeHealthy(ctx, k8sClient, pipelineName)
 		})
 
-		It("Should verify end-to-end trace delivery", Label(operationalTest), func() {
-			gatewayPushURL := proxyClient.ProxyURLForService(kitkyma.SystemNamespaceName, "telemetry-otlp-traces", "v1/traces/", ports.OTLPHTTP)
-			traceID, spanIDs, attrs := kittraces.MakeAndSendTraces(proxyClient, gatewayPushURL)
-			verifiers.TracesShouldBeDelivered(proxyClient, telemetryExportURL, traceID, spanIDs, attrs)
+		It("Should deliver telemetrygen traces", func() {
+			verifiers.TracesFromNamespaceShouldBeDelivered(proxyClient, telemetryExportURL, kitkyma.DefaultNamespaceName)
 		})
 
-		It("Should be able to get trace gateway metrics endpoint", Label(operationalTest), func() {
-			gatewayMetricsURL := proxyClient.ProxyURLForService(kitkyma.TraceGatewayMetrics.Namespace, kitkyma.TraceGatewayMetrics.Name, "metrics", ports.Metrics)
-			verifiers.ShouldExposeCollectorMetrics(proxyClient, gatewayMetricsURL)
+		It("The telemetryFlowHealthy condition should be true", func() {
+			//TODO: add the conditions.TypeFlowHealthy check to verifiers.TracePipelineShouldBeHealthy after self monitor is released
+			Eventually(func(g Gomega) {
+				var pipeline telemetryv1alpha1.TracePipeline
+				key := types.NamespacedName{Name: pipelineName}
+				g.Expect(k8sClient.Get(ctx, key, &pipeline)).To(Succeed())
+				g.Expect(meta.IsStatusConditionTrue(pipeline.Status.Conditions, conditions.TypeFlowHealthy)).To(BeTrue())
+			}, periodic.EventuallyTimeout, periodic.DefaultInterval).Should(Succeed())
 		})
 
-		It("The telemetryFlowHealthy condition should be true", func() {
-			verifiers.TracePipelineTelemetryHealthFlowIsHealthy(ctx, k8sClient, pipelineName)
+		It("Should ensure that the self-monitor webhook has been called", func() {
+			// Pushing traces to the trace gateway triggers an alert, which in turn makes the self-monitor call the webhook
+			verifiers.SelfMonitorWebhookShouldHaveBeenCalled(proxyClient)
 		})
 	})
 })

test/testkit/kyma/common_names.go

@@ -10,6 +10,8 @@ const (
 	KubeNamespace            = "kube-system"
 	IstioSystemNamespaceName = "istio-system"
 
+	TelemetryManagerMetricsPort = 8080
+
 	MetricGatewayBaseName = "telemetry-metric-gateway"
 	MetricAgentBaseName   = "telemetry-metric-agent"
 	TraceGatewayBaseName  = "telemetry-trace-collector"
@@ -18,7 +20,8 @@ const (
 )
 
 var (
-	TelemetryOperatorWebhookServiceName = types.NamespacedName{Name: "telemetry-manager-webhook", Namespace: SystemNamespaceName}
+	TelemetryManagerMetricsServiceName = types.NamespacedName{Name: "telemetry-manager-metrics", Namespace: SystemNamespaceName}
+	TelemetryManagerWebhookServiceName = types.NamespacedName{Name: "telemetry-manager-webhook", Namespace: SystemNamespaceName}
 
 	MetricGatewayName          = types.NamespacedName{Name: MetricGatewayBaseName, Namespace: SystemNamespaceName}
 	MetricGatewayMetrics       = types.NamespacedName{Name: MetricGatewayBaseName + "-metrics", Namespace: SystemNamespaceName}