From 3c64d305addd729ff915db7034345fe64368ce32 Mon Sep 17 00:00:00 2001 From: Stanislav Khalash Date: Thu, 28 Mar 2024 11:40:49 +0100 Subject: [PATCH] test: Self-monitor webhook E2E test (#932) --- .github/workflows/pr-integration.yml | 24 +++++++- .golangci.yaml | 2 + go.mod | 2 +- test/e2e/manager_test.go | 8 +-- test/e2e/metrics_namespace_selector_test.go | 6 +- test/e2e/metrics_otlp_input_test.go | 2 +- ....go => metrics_self_monitor_basic_test.go} | 47 ++++++++------ ...t.go => traces_self_monitor_basic_test.go} | 44 ++++++++----- test/testkit/kyma/common_names.go | 5 +- .../matchers/prometheus/prometheus_matcher.go | 61 ++++++++++++++++--- .../prometheus/prometheus_matcher_test.go | 47 ++++++++++++-- .../mocks/telemetrygen/telemetrygen.go | 4 +- test/testkit/verifiers/metrics.go | 9 --- test/testkit/verifiers/monitoring.go | 3 +- test/testkit/verifiers/self_monitor.go | 37 +++++++++++ test/testkit/verifiers/telemetry.go | 2 +- test/testkit/verifiers/traces.go | 24 +++++--- 17 files changed, 243 insertions(+), 84 deletions(-) rename test/e2e/{self_monitor_basic_metrics_test.go => metrics_self_monitor_basic_test.go} (64%) rename test/e2e/{self_monitor_basic_trace_test.go => traces_self_monitor_basic_test.go} (66%) create mode 100644 test/testkit/verifiers/self_monitor.go diff --git a/.github/workflows/pr-integration.yml b/.github/workflows/pr-integration.yml index 19b7c8cc9..8e4b042e1 100644 --- a/.github/workflows/pr-integration.yml +++ b/.github/workflows/pr-integration.yml @@ -200,7 +200,7 @@ jobs: with: failure: failure() - e2e-self-monitor: + e2e-self-monitor-traces: runs-on: ubuntu-latest steps: - name: Checkout repo @@ -212,7 +212,27 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Run tests - run: bin/ginkgo run --tags e2e --label-filter="self-mon" test/e2e + run: bin/ginkgo run --tags e2e --label-filter="self-mon-traces" test/e2e + + - name: Finalize Test + uses: "./.github/template/finalize-test" + if: success() || failure() + with: + failure: failure() + + e2e-self-monitor-metrics: + runs-on: ubuntu-latest + steps: + - name: Checkout repo + uses: actions/checkout@v4 + + - name: Prepare Test + uses: "./.github/template/prepare-test" + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Run tests + run: bin/ginkgo run --tags e2e --label-filter="self-mon-metrics" test/e2e - name: Finalize Test uses: "./.github/template/finalize-test" diff --git a/.golangci.yaml b/.golangci.yaml index b428437a8..3b0ca0ed4 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -78,6 +78,8 @@ linters-settings: alias: logpipelinevalidationmocks - pkg: github.com/prometheus/client_golang/api/prometheus/v1 alias: promv1 + - pkg: github.com/prometheus/client_model/go + alias: prommodel - pkg: go.opentelemetry.io/otel/sdk/trace alias: tracesdk - pkg: istio.io/api/security/v1beta1 diff --git a/go.mod b/go.mod index 40e7aedf4..77c23c836 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/onsi/ginkgo/v2 v2.17.1 github.com/onsi/gomega v1.31.1 github.com/prometheus/client_golang v1.19.0 + github.com/prometheus/client_model v0.6.0 github.com/prometheus/common v0.50.0 github.com/stretchr/testify v1.9.0 go.opentelemetry.io/collector/pdata v1.3.0 @@ -63,7 +64,6 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/prometheus/client_model v0.6.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/objx v0.5.2 // indirect diff --git a/test/e2e/manager_test.go b/test/e2e/manager_test.go index ec650c136..b826caa96 100644 --- a/test/e2e/manager_test.go +++ b/test/e2e/manager_test.go @@ -78,11 +78,7 @@ var _ = Describe("Telemetry Manager", func() { It("Should have a metrics service", Label("telemetry"), func() { var service corev1.Service - key := types.NamespacedName{ - Name: "telemetry-manager-metrics", - Namespace: kitkyma.SystemNamespaceName, - } - err := k8sClient.Get(ctx, key, &service) + err := k8sClient.Get(ctx, kitkyma.TelemetryManagerMetricsServiceName, &service) Expect(err).NotTo(HaveOccurred()) Expect(service.Annotations).Should(HaveKeyWithValue("prometheus.io/scrape", "true")) @@ -90,7 +86,7 @@ var _ = Describe("Telemetry Manager", func() { Eventually(func() []corev1.EndpointAddress { var endpoints corev1.Endpoints - err := k8sClient.Get(ctx, key, &endpoints) + err := k8sClient.Get(ctx, kitkyma.TelemetryManagerMetricsServiceName, &endpoints) Expect(err).NotTo(HaveOccurred()) return endpoints.Subsets[0].Addresses }, periodic.EventuallyTimeout, periodic.DefaultInterval).ShouldNot(BeEmpty()) diff --git a/test/e2e/metrics_namespace_selector_test.go b/test/e2e/metrics_namespace_selector_test.go index 71b24968e..91528bf85 100644 --- a/test/e2e/metrics_namespace_selector_test.go +++ b/test/e2e/metrics_namespace_selector_test.go @@ -59,9 +59,9 @@ var _ = Describe("Metrics Namespace Selector", Label("metrics"), func() { objs = append(objs, pipelineExcludeApp1Ns.K8sObject()) objs = append(objs, - telemetrygen.New(app1Ns).K8sObject(), - telemetrygen.New(app2Ns).K8sObject(), - telemetrygen.New(kitkyma.SystemNamespaceName).K8sObject(), + telemetrygen.New(app1Ns, telemetrygen.SignalTypeMetrics).K8sObject(), + telemetrygen.New(app2Ns, telemetrygen.SignalTypeMetrics).K8sObject(), + telemetrygen.New(kitkyma.SystemNamespaceName, telemetrygen.SignalTypeMetrics).K8sObject(), prommetricgen.New(app1Ns).Pod().WithPrometheusAnnotations(prommetricgen.SchemeHTTP).K8sObject(), prommetricgen.New(app2Ns).Pod().WithPrometheusAnnotations(prommetricgen.SchemeHTTP).K8sObject(), diff --git a/test/e2e/metrics_otlp_input_test.go b/test/e2e/metrics_otlp_input_test.go index 65a852355..b167c51b8 100644 --- a/test/e2e/metrics_otlp_input_test.go +++ b/test/e2e/metrics_otlp_input_test.go @@ -36,7 +36,7 @@ var _ = Describe("Metrics OTLP Input", Label("metrics"), func() { OtlpInput(false) objs = append(objs, pipelineWithoutOTLP.K8sObject()) - objs = append(objs, telemetrygen.New(appNs).K8sObject()) + objs = append(objs, telemetrygen.New(appNs, telemetrygen.SignalTypeMetrics).K8sObject()) return objs } diff --git a/test/e2e/self_monitor_basic_metrics_test.go b/test/e2e/metrics_self_monitor_basic_test.go similarity index 64% rename from test/e2e/self_monitor_basic_metrics_test.go rename to test/e2e/metrics_self_monitor_basic_test.go index a61e8d601..3b1f103ca 100644 --- a/test/e2e/self_monitor_basic_metrics_test.go +++ b/test/e2e/metrics_self_monitor_basic_test.go @@ -10,19 +10,22 @@ import ( . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" + "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" + telemetryv1alpha1 "github.com/kyma-project/telemetry-manager/apis/telemetry/v1alpha1" + "github.com/kyma-project/telemetry-manager/internal/conditions" "github.com/kyma-project/telemetry-manager/internal/otelcollector/ports" kitk8s "github.com/kyma-project/telemetry-manager/test/testkit/k8s" kitkyma "github.com/kyma-project/telemetry-manager/test/testkit/kyma" "github.com/kyma-project/telemetry-manager/test/testkit/mocks/backend" - kitmetrics "github.com/kyma-project/telemetry-manager/test/testkit/otel/metrics" + "github.com/kyma-project/telemetry-manager/test/testkit/mocks/telemetrygen" "github.com/kyma-project/telemetry-manager/test/testkit/periodic" "github.com/kyma-project/telemetry-manager/test/testkit/verifiers" ) -var _ = Describe("Telemetry Self Monitor", Label("self-mon"), Ordered, func() { +var _ = Describe("Metrics Self Monitor", Label("self-mon-metrics"), Ordered, func() { const ( mockBackendName = "metrics-receiver-selfmon" mockNs = "metrics-basic-selfmon-test" @@ -38,20 +41,22 @@ var _ = Describe("Telemetry Self Monitor", Label("self-mon"), Ordered, func() { objs = append(objs, kitk8s.NewNamespace(mockNs).K8sObject()) - mockBackend := backend.New(mockBackendName, mockNs, backend.SignalTypeMetrics, backend.WithPersistentHostSecret(isOperational())) + mockBackend := backend.New(mockBackendName, mockNs, backend.SignalTypeMetrics) objs = append(objs, mockBackend.K8sObjects()...) telemetryExportURL = mockBackend.TelemetryExportURL(proxyClient) pipeline := kitk8s.NewMetricPipelineV1Alpha1(fmt.Sprintf("%s-pipeline", mockBackendName)). - WithOutputEndpointFromSecret(mockBackend.HostSecretRefV1Alpha1()). - RuntimeInput(true) + WithOutputEndpointFromSecret(mockBackend.HostSecretRefV1Alpha1()) pipelineName = pipeline.Name() - objs = append(objs, pipeline.K8sObject()) + objs = append(objs, + telemetrygen.New(kitkyma.DefaultNamespaceName, telemetrygen.SignalTypeMetrics).K8sObject(), + pipeline.K8sObject(), + ) return objs } - Context("When a metrics pipeline exists", Ordered, func() { + Context("When a metric pipeline exists", Ordered, func() { BeforeAll(func() { k8sObjects := makeResources() @@ -60,9 +65,11 @@ var _ = Describe("Telemetry Self Monitor", Label("self-mon"), Ordered, func() { }) Expect(kitk8s.CreateObjects(ctx, k8sClient, k8sObjects...)).Should(Succeed()) }) + It("Should have a running self-monitor", func() { verifiers.DeploymentShouldBeReady(ctx, k8sClient, kitkyma.SelfMonitorName) }) + It("Should have a network policy deployed", func() { var networkPolicy networkingv1.NetworkPolicy Expect(k8sClient.Get(ctx, kitkyma.SelfMonitorNetworkPolicy, &networkPolicy)).To(Succeed()) @@ -86,27 +93,31 @@ var _ = Describe("Telemetry Self Monitor", Label("self-mon"), Ordered, func() { Expect(k8sClient.Get(ctx, kitkyma.SelfMonitorName, &service)).To(Succeed()) }) - It("Should have a metrics backend running", Label(operationalTest), func() { + It("Should have a metrics backend running", func() { verifiers.DeploymentShouldBeReady(ctx, k8sClient, types.NamespacedName{Name: mockBackendName, Namespace: mockNs}) }) - It("Should have a running pipeline", Label(operationalTest), func() { + It("Should have a running pipeline", func() { verifiers.MetricPipelineShouldBeHealthy(ctx, k8sClient, pipelineName) }) - It("Should verify end-to-end metric delivery", Label(operationalTest), func() { - gatewayPushURL := proxyClient.ProxyURLForService(kitkyma.SystemNamespaceName, "telemetry-otlp-metrics", "v1/metrics/", ports.OTLPHTTP) - gauges := kitmetrics.MakeAndSendGaugeMetrics(proxyClient, gatewayPushURL) - verifiers.MetricsShouldBeDelivered(proxyClient, telemetryExportURL, gauges) + It("Should deliver telemetrygen metrics", func() { + verifiers.MetricsFromNamespaceShouldBeDelivered(proxyClient, telemetryExportURL, kitkyma.DefaultNamespaceName, telemetrygen.MetricNames) }) - It("Should be able to get metric gateway metrics endpoint", Label(operationalTest), func() { - gatewayMetricsURL := proxyClient.ProxyURLForService(kitkyma.MetricGatewayMetrics.Namespace, kitkyma.MetricGatewayMetrics.Name, "metrics", ports.Metrics) - verifiers.ShouldExposeCollectorMetrics(proxyClient, gatewayMetricsURL) + It("Should have TypeFlowHealthy condition set to True", func() { + //TODO: add the conditions.TypeFlowHealthy check to verifiers.MetricPipelineShouldBeHealthy after self monitor is released + Eventually(func(g Gomega) { + var pipeline telemetryv1alpha1.MetricPipeline + key := types.NamespacedName{Name: pipelineName} + g.Expect(k8sClient.Get(ctx, key, &pipeline)).To(Succeed()) + g.Expect(meta.IsStatusConditionTrue(pipeline.Status.Conditions, conditions.TypeFlowHealthy)).To(BeTrue()) + }, periodic.EventuallyTimeout, periodic.DefaultInterval).Should(Succeed()) }) - It("The telemetryFlowHealthy condition should be true", func() { - verifiers.MetricPipelineTelemetryHealthFlowIsHealthy(ctx, k8sClient, pipelineName) + It("Should ensure that the self-monitor webhook has been called", func() { + // Pushing metrics to the metric gateway triggers an alert, which in turn makes the self-monitor call the webhook + verifiers.SelfMonitorWebhookShouldHaveBeenCalled(proxyClient) }) }) }) diff --git a/test/e2e/self_monitor_basic_trace_test.go b/test/e2e/traces_self_monitor_basic_test.go similarity index 66% rename from test/e2e/self_monitor_basic_trace_test.go rename to test/e2e/traces_self_monitor_basic_test.go index 2192c8aea..4149eaf7a 100644 --- a/test/e2e/self_monitor_basic_trace_test.go +++ b/test/e2e/traces_self_monitor_basic_test.go @@ -10,18 +10,22 @@ import ( . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" + telemetryv1alpha1 "github.com/kyma-project/telemetry-manager/apis/telemetry/v1alpha1" + "github.com/kyma-project/telemetry-manager/internal/conditions" "github.com/kyma-project/telemetry-manager/internal/otelcollector/ports" kitk8s "github.com/kyma-project/telemetry-manager/test/testkit/k8s" kitkyma "github.com/kyma-project/telemetry-manager/test/testkit/kyma" "github.com/kyma-project/telemetry-manager/test/testkit/mocks/backend" - kittraces "github.com/kyma-project/telemetry-manager/test/testkit/otel/traces" + "github.com/kyma-project/telemetry-manager/test/testkit/mocks/telemetrygen" "github.com/kyma-project/telemetry-manager/test/testkit/periodic" "github.com/kyma-project/telemetry-manager/test/testkit/verifiers" ) -var _ = Describe("Telemetry Self Monitor", Label("self-mon"), Ordered, func() { +var _ = Describe("Traces Self Monitor", Label("self-mon-traces"), Ordered, func() { const ( mockBackendName = "traces-receiver-selfmon" mockNs = "traces-basic-selfmon-test" @@ -37,15 +41,17 @@ var _ = Describe("Telemetry Self Monitor", Label("self-mon"), Ordered, func() { objs = append(objs, kitk8s.NewNamespace(mockNs).K8sObject()) - mockBackend := backend.New(mockBackendName, mockNs, backend.SignalTypeTraces, backend.WithPersistentHostSecret(isOperational())) + mockBackend := backend.New(mockBackendName, mockNs, backend.SignalTypeTraces) objs = append(objs, mockBackend.K8sObjects()...) telemetryExportURL = mockBackend.TelemetryExportURL(proxyClient) pipeline := kitk8s.NewTracePipelineV1Alpha1(fmt.Sprintf("%s-pipeline", mockBackend.Name())). - WithOutputEndpointFromSecret(mockBackend.HostSecretRefV1Alpha1()). - Persistent(isOperational()) + WithOutputEndpointFromSecret(mockBackend.HostSecretRefV1Alpha1()) pipelineName = pipeline.Name() - objs = append(objs, pipeline.K8sObject()) + objs = append(objs, + pipeline.K8sObject(), + telemetrygen.New(kitkyma.DefaultNamespaceName, telemetrygen.SignalTypeTraces).K8sObject(), + ) return objs } @@ -59,9 +65,11 @@ var _ = Describe("Telemetry Self Monitor", Label("self-mon"), Ordered, func() { }) Expect(kitk8s.CreateObjects(ctx, k8sClient, k8sObjects...)).Should(Succeed()) }) + It("Should have a running self-monitor", func() { verifiers.DeploymentShouldBeReady(ctx, k8sClient, kitkyma.SelfMonitorName) }) + It("Should have a network policy deployed", func() { var networkPolicy networkingv1.NetworkPolicy Expect(k8sClient.Get(ctx, kitkyma.SelfMonitorNetworkPolicy, &networkPolicy)).To(Succeed()) @@ -85,23 +93,27 @@ var _ = Describe("Telemetry Self Monitor", Label("self-mon"), Ordered, func() { Expect(k8sClient.Get(ctx, kitkyma.SelfMonitorName, &service)).To(Succeed()) }) - It("Should have a running pipeline", Label(operationalTest), func() { + It("Should have a running pipeline", func() { verifiers.TracePipelineShouldBeHealthy(ctx, k8sClient, pipelineName) }) - It("Should verify end-to-end trace delivery", Label(operationalTest), func() { - gatewayPushURL := proxyClient.ProxyURLForService(kitkyma.SystemNamespaceName, "telemetry-otlp-traces", "v1/traces/", ports.OTLPHTTP) - traceID, spanIDs, attrs := kittraces.MakeAndSendTraces(proxyClient, gatewayPushURL) - verifiers.TracesShouldBeDelivered(proxyClient, telemetryExportURL, traceID, spanIDs, attrs) + It("Should deliver telemetrygen traces", func() { + verifiers.TracesFromNamespaceShouldBeDelivered(proxyClient, telemetryExportURL, kitkyma.DefaultNamespaceName) }) - It("Should be able to get trace gateway metrics endpoint", Label(operationalTest), func() { - gatewayMetricsURL := proxyClient.ProxyURLForService(kitkyma.TraceGatewayMetrics.Namespace, kitkyma.TraceGatewayMetrics.Name, "metrics", ports.Metrics) - verifiers.ShouldExposeCollectorMetrics(proxyClient, gatewayMetricsURL) + It("The telemetryFlowHealthy condition should be true", func() { + //TODO: add the conditions.TypeFlowHealthy check to verifiers.TracePipelineShouldBeHealthy after self monitor is released + Eventually(func(g Gomega) { + var pipeline telemetryv1alpha1.TracePipeline + key := types.NamespacedName{Name: pipelineName} + g.Expect(k8sClient.Get(ctx, key, &pipeline)).To(Succeed()) + g.Expect(meta.IsStatusConditionTrue(pipeline.Status.Conditions, conditions.TypeFlowHealthy)).To(BeTrue()) + }, periodic.EventuallyTimeout, periodic.DefaultInterval).Should(Succeed()) }) - It("The telemetryFlowHealthy condition should be true", func() { - verifiers.TracePipelineTelemetryHealthFlowIsHealthy(ctx, k8sClient, pipelineName) + It("Should ensure that the self-monitor webhook has been called", func() { + // Pushing traces to the trace gateway triggers an alert, which in turn makes the self-monitor call the webhook + verifiers.SelfMonitorWebhookShouldHaveBeenCalled(proxyClient) }) }) }) diff --git a/test/testkit/kyma/common_names.go b/test/testkit/kyma/common_names.go index 088345585..57fd2ac8a 100644 --- a/test/testkit/kyma/common_names.go +++ b/test/testkit/kyma/common_names.go @@ -10,6 +10,8 @@ const ( KubeNamespace = "kube-system" IstioSystemNamespaceName = "istio-system" + TelemetryManagerMetricsPort = 8080 + MetricGatewayBaseName = "telemetry-metric-gateway" MetricAgentBaseName = "telemetry-metric-agent" TraceGatewayBaseName = "telemetry-trace-collector" @@ -18,7 +20,8 @@ const ( ) var ( - TelemetryOperatorWebhookServiceName = types.NamespacedName{Name: "telemetry-manager-webhook", Namespace: SystemNamespaceName} + TelemetryManagerMetricsServiceName = types.NamespacedName{Name: "telemetry-manager-metrics", Namespace: SystemNamespaceName} + TelemetryManagerWebhookServiceName = types.NamespacedName{Name: "telemetry-manager-webhook", Namespace: SystemNamespaceName} MetricGatewayName = types.NamespacedName{Name: MetricGatewayBaseName, Namespace: SystemNamespaceName} MetricGatewayMetrics = types.NamespacedName{Name: MetricGatewayBaseName + "-metrics", Namespace: SystemNamespaceName} diff --git a/test/testkit/matchers/prometheus/prometheus_matcher.go b/test/testkit/matchers/prometheus/prometheus_matcher.go index b7cc41ab2..3fa54f374 100644 --- a/test/testkit/matchers/prometheus/prometheus_matcher.go +++ b/test/testkit/matchers/prometheus/prometheus_matcher.go @@ -5,18 +5,63 @@ import ( "github.com/onsi/gomega" "github.com/onsi/gomega/types" + prommodel "github.com/prometheus/client_model/go" "github.com/prometheus/common/expfmt" ) -func ContainPrometheusMetric(metricName string) types.GomegaMatcher { - return gomega.WithTransform(func(responseBody []byte) (bool, error) { +func WithMetricFamilies(matcher types.GomegaMatcher) types.GomegaMatcher { + return gomega.WithTransform(func(responseBody []byte) ([]*prommodel.MetricFamily, error) { var parser expfmt.TextParser - mf, err := parser.TextToMetricFamilies(bytes.NewReader(responseBody)) + mfs, _ := parser.TextToMetricFamilies(bytes.NewReader(responseBody)) //nolint:errcheck // ignore duplicate metrics parsing error and try extract metric + var result []*prommodel.MetricFamily + for _, mf := range mfs { + result = append(result, mf) + } + return result, nil + }, matcher) +} + +func ContainMetricFamily(matcher types.GomegaMatcher) types.GomegaMatcher { + return WithMetricFamilies(gomega.ContainElement(matcher)) +} + +func WithName(matcher types.GomegaMatcher) types.GomegaMatcher { + return gomega.WithTransform(func(mf *prommodel.MetricFamily) string { + return mf.GetName() + }, matcher) +} + +func WithMetrics(matcher types.GomegaMatcher) types.GomegaMatcher { + return gomega.WithTransform(func(mf *prommodel.MetricFamily) []*prommodel.Metric { + return mf.GetMetric() + }, matcher) +} + +func ContainMetric(matcher types.GomegaMatcher) types.GomegaMatcher { + return WithMetrics(gomega.ContainElement(matcher)) +} + +func WithValue(matcher types.GomegaMatcher) types.GomegaMatcher { + return gomega.WithTransform(func(m *prommodel.Metric) (float64, error) { + if m.Gauge != nil { + return m.Gauge.GetValue(), nil + } + if m.Counter != nil { + return m.Counter.GetValue(), nil + } + if m.Untyped != nil { + return m.Untyped.GetValue(), nil + } + return 0, nil + }, matcher) +} - if err != nil { - // ignore duplicate metrics parsing error and try extract metric - return mf[metricName] != nil, nil +func WithLabels(matcher types.GomegaMatcher) types.GomegaMatcher { + return gomega.WithTransform(func(m *prommodel.Metric) (map[string]string, error) { + labels := make(map[string]string) + for _, l := range m.Label { + labels[l.GetName()] = l.GetValue() } - return mf[metricName] != nil, nil - }, gomega.BeTrue()) + return labels, nil + }, matcher) } diff --git a/test/testkit/matchers/prometheus/prometheus_matcher_test.go b/test/testkit/matchers/prometheus/prometheus_matcher_test.go index ab720815c..ee5cec712 100644 --- a/test/testkit/matchers/prometheus/prometheus_matcher_test.go +++ b/test/testkit/matchers/prometheus/prometheus_matcher_test.go @@ -5,10 +5,10 @@ import ( . "github.com/onsi/gomega" ) -var _ = Describe("ContainPrometheusMetric", Label("metrics"), func() { +var _ = Describe("ContainMetric", Label("metrics"), func() { Context("with nil input", func() { It("should fail", func() { - success, err := ContainPrometheusMetric("foo_metric").Match(nil) + success, err := ContainMetricFamily(WithName(Equal("foo_metric"))).Match(nil) Expect(err).Should(HaveOccurred()) Expect(success).Should(BeFalse()) }) @@ -16,7 +16,7 @@ var _ = Describe("ContainPrometheusMetric", Label("metrics"), func() { Context("with empty input", func() { It("should fail", func() { - success, err := ContainPrometheusMetric("foo_metric").Match([]byte{}) + success, err := ContainMetricFamily(WithName(Equal("foo_metric"))).Match([]byte{}) Expect(err).ShouldNot(HaveOccurred()) Expect(success).Should(BeFalse()) }) @@ -24,7 +24,7 @@ var _ = Describe("ContainPrometheusMetric", Label("metrics"), func() { Context("with invalid input", func() { It("should fail", func() { - success, err := ContainPrometheusMetric("foo_metric").Match([]byte{1, 2, 3}) + success, err := ContainMetricFamily(WithName(Equal("foo_metric"))).Match([]byte{1, 2, 3}) Expect(err).ShouldNot(HaveOccurred()) Expect(success).Should(BeFalse()) }) @@ -39,7 +39,44 @@ fluentbit_uptime{hostname="telemetry-fluent-bit-dglkf"} 5489 # HELP fluentbit_input_bytes_total Number of input bytes. # TYPE fluentbit_input_bytes_total counter fluentbit_input_bytes_total{name="tele-tail"} 5217998` - Expect([]byte(fileBytes)).Should(ContainPrometheusMetric("fluentbit_uptime")) + Expect([]byte(fileBytes)).Should(ContainMetricFamily(WithName(Equal("fluentbit_uptime")))) }) }) }) + +var _ = Describe("WithLabels", func() { + It("should apply matcher", func() { + fileBytes := ` +# HELP fluentbit_uptime Number of seconds that Fluent Bit has been running. +# TYPE fluentbit_uptime counter +fluentbit_uptime{hostname="telemetry-fluent-bit-dglkf"} 5489 +# HELP fluentbit_input_bytes_total Number of input bytes. +# TYPE fluentbit_input_bytes_total counter +fluentbit_input_bytes_total{name="tele-tail"} 5000 +` + Expect([]byte(fileBytes)).Should(ContainMetricFamily(SatisfyAll( + WithName(Equal("fluentbit_input_bytes_total")), + ContainMetric(WithLabels(HaveKeyWithValue("name", "tele-tail"))), + ))) + }) +}) + +var _ = Describe("WithValue", func() { + It("should apply matcher", func() { + fileBytes := ` +# HELP fluentbit_uptime Number of seconds that Fluent Bit has been running. +# TYPE fluentbit_uptime counter +fluentbit_uptime{hostname="telemetry-fluent-bit-dglkf"} 5489 +# HELP fluentbit_input_bytes_total Number of input bytes. +# TYPE fluentbit_input_bytes_total counter +fluentbit_input_bytes_total{name="tele-tail"} 5000 +` + Expect([]byte(fileBytes)).Should(ContainMetricFamily(SatisfyAll( + WithName(Equal("fluentbit_input_bytes_total")), + ContainMetric(SatisfyAll( + WithLabels(HaveKeyWithValue("name", "tele-tail")), + WithValue(BeNumerically(">=", 0)), + )), + ))) + }) +}) diff --git a/test/testkit/mocks/telemetrygen/telemetrygen.go b/test/testkit/mocks/telemetrygen/telemetrygen.go index 290255b0b..779f58710 100644 --- a/test/testkit/mocks/telemetrygen/telemetrygen.go +++ b/test/testkit/mocks/telemetrygen/telemetrygen.go @@ -34,8 +34,8 @@ const ( SignalTypeMetrics = "metrics" ) -func New(namespace string) *kitk8s.Pod { - return kitk8s.NewPod("telemetrygen", namespace).WithPodSpec(PodSpec(SignalTypeMetrics, "")) +func New(namespace string, signalType SignalType) *kitk8s.Pod { + return kitk8s.NewPod("telemetrygen", namespace).WithPodSpec(PodSpec(signalType, "")) } func PodSpec(signalType SignalType, serviceNameAttrValue string) corev1.PodSpec { diff --git a/test/testkit/verifiers/metrics.go b/test/testkit/verifiers/metrics.go index f2709e875..ef2ef8f87 100644 --- a/test/testkit/verifiers/metrics.go +++ b/test/testkit/verifiers/metrics.go @@ -93,12 +93,3 @@ func MetricsFromNamespaceShouldNotBeDelivered(proxyClient *apiserverproxy.Client g.Expect(err).NotTo(HaveOccurred()) }, periodic.TelemetryConsistentlyTimeout, periodic.TelemetryInterval).Should(Succeed()) } - -func MetricPipelineTelemetryHealthFlowIsHealthy(ctx context.Context, k8sClient client.Client, pipelineName string) { - Eventually(func(g Gomega) { - var pipeline telemetryv1alpha1.MetricPipeline - key := types.NamespacedName{Name: pipelineName} - g.Expect(k8sClient.Get(ctx, key, &pipeline)).To(Succeed()) - g.Expect(meta.IsStatusConditionTrue(pipeline.Status.Conditions, conditions.TypeFlowHealthy)).To(BeTrue()) - }, periodic.EventuallyTimeout, periodic.DefaultInterval).Should(Succeed()) -} diff --git a/test/testkit/verifiers/monitoring.go b/test/testkit/verifiers/monitoring.go index 24f23e7fb..d0b42dcf1 100644 --- a/test/testkit/verifiers/monitoring.go +++ b/test/testkit/verifiers/monitoring.go @@ -16,8 +16,7 @@ func ShouldExposeCollectorMetrics(proxyClient *apiserverproxy.Client, metricsURL g.Expect(err).NotTo(HaveOccurred()) g.Expect(resp).To(HaveHTTPStatus(http.StatusOK)) - //Take otelcol_process_uptime metric as an example - g.Expect(resp).To(HaveHTTPBody(ContainPrometheusMetric("otelcol_process_uptime"))) + g.Expect(resp).To(HaveHTTPBody(ContainMetricFamily(WithName(ContainSubstring("otelcol"))))) err = resp.Body.Close() g.Expect(err).NotTo(HaveOccurred()) diff --git a/test/testkit/verifiers/self_monitor.go b/test/testkit/verifiers/self_monitor.go new file mode 100644 index 000000000..442769908 --- /dev/null +++ b/test/testkit/verifiers/self_monitor.go @@ -0,0 +1,37 @@ +package verifiers + +import ( + "net/http" + "time" + + . "github.com/onsi/gomega" + + "github.com/kyma-project/telemetry-manager/test/testkit/apiserverproxy" + kitkyma "github.com/kyma-project/telemetry-manager/test/testkit/kyma" + . "github.com/kyma-project/telemetry-manager/test/testkit/matchers/prometheus" + "github.com/kyma-project/telemetry-manager/test/testkit/periodic" +) + +func SelfMonitorWebhookShouldHaveBeenCalled(proxyClient *apiserverproxy.Client) { + Eventually(func(g Gomega) { + telemetryManagerMetricsURL := proxyClient.ProxyURLForService( + kitkyma.TelemetryManagerMetricsServiceName.Namespace, + kitkyma.TelemetryManagerMetricsServiceName.Name, + "metrics", + kitkyma.TelemetryManagerMetricsPort) + resp, err := proxyClient.Get(telemetryManagerMetricsURL) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(resp).To(HaveHTTPStatus(http.StatusOK)) + + g.Expect(resp).To(HaveHTTPBody(ContainMetricFamily(SatisfyAll( + WithName(Equal("controller_runtime_webhook_requests_total")), + ContainMetric(SatisfyAll( + WithLabels(HaveKeyWithValue("webhook", "/api/v2/alerts")), + WithValue(BeNumerically(">", 0)), + )), + )))) + + err = resp.Body.Close() + g.Expect(err).NotTo(HaveOccurred()) + }, periodic.EventuallyTimeout, 5*time.Second).Should(Succeed()) +} diff --git a/test/testkit/verifiers/telemetry.go b/test/testkit/verifiers/telemetry.go index a5ce8b15d..07bb380c7 100644 --- a/test/testkit/verifiers/telemetry.go +++ b/test/testkit/verifiers/telemetry.go @@ -16,7 +16,7 @@ import ( func WebhookShouldBeHealthy(ctx context.Context, k8sClient client.Client) { Eventually(func(g Gomega) { var endpoints corev1.Endpoints - g.Expect(k8sClient.Get(ctx, kitkyma.TelemetryOperatorWebhookServiceName, &endpoints)).To(Succeed()) + g.Expect(k8sClient.Get(ctx, kitkyma.TelemetryManagerWebhookServiceName, &endpoints)).To(Succeed()) g.Expect(endpoints.Subsets).NotTo(BeEmpty()) for _, subset := range endpoints.Subsets { g.Expect(subset.Addresses).NotTo(BeEmpty()) diff --git a/test/testkit/verifiers/traces.go b/test/testkit/verifiers/traces.go index b9254fe64..1ea2a3689 100644 --- a/test/testkit/verifiers/traces.go +++ b/test/testkit/verifiers/traces.go @@ -77,6 +77,21 @@ func TracesShouldBeDelivered(proxyClient *apiserverproxy.Client, telemetryExport }, periodic.EventuallyTimeout, periodic.TelemetryInterval).Should(Succeed()) } +func TracesFromNamespaceShouldBeDelivered(proxyClient *apiserverproxy.Client, telemetryExportURL, namespace string) { + Eventually(func(g Gomega) { + resp, err := proxyClient.Get(telemetryExportURL) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(resp).To(HaveHTTPStatus(http.StatusOK)) + g.Expect(resp).To(HaveHTTPBody( + ContainTd(SatisfyAll( + ContainResourceAttrs(HaveKeyWithValue("k8s.namespace.name", namespace)), + )), + )) + err = resp.Body.Close() + g.Expect(err).NotTo(HaveOccurred()) + }, periodic.TelemetryEventuallyTimeout, periodic.TelemetryInterval).Should(Succeed()) +} + func TracesShouldNotBePresent(proxyClient *apiserverproxy.Client, telemetryExportURL string, traceID pcommon.TraceID) { Consistently(func(g Gomega) { resp, err := proxyClient.Get(telemetryExportURL) @@ -89,12 +104,3 @@ func TracesShouldNotBePresent(proxyClient *apiserverproxy.Client, telemetryExpor g.Expect(err).NotTo(HaveOccurred()) }, periodic.ConsistentlyTimeout, periodic.TelemetryInterval).Should(Succeed()) } - -func TracePipelineTelemetryHealthFlowIsHealthy(ctx context.Context, k8sClient client.Client, pipelineName string) { - Eventually(func(g Gomega) { - var pipeline telemetryv1alpha1.TracePipeline - key := types.NamespacedName{Name: pipelineName} - g.Expect(k8sClient.Get(ctx, key, &pipeline)).To(Succeed()) - g.Expect(meta.IsStatusConditionTrue(pipeline.Status.Conditions, conditions.TypeFlowHealthy)).To(BeTrue()) - }, periodic.EventuallyTimeout, periodic.DefaultInterval).Should(Succeed()) -}