diff --git a/internal/otelcollector/config/trace/gateway/drop_noisy_spans.go b/internal/otelcollector/config/trace/gateway/drop_noisy_spans.go index bc648388c..5c2bcb283 100644 --- a/internal/otelcollector/config/trace/gateway/drop_noisy_spans.go +++ b/internal/otelcollector/config/trace/gateway/drop_noisy_spans.go @@ -15,22 +15,22 @@ var ( urlIsTelemetryTraceService = urlMatches("http(s)?:\\\\/\\\\/telemetry-otlp-traces\\\\.kyma-system(\\\\..*)?:(4317|4318).*") urlIsTelemetryMetricService = urlMatches("http(s)?:\\\\/\\\\/telemetry-otlp-metrics\\\\.kyma-system(\\\\..*)?:(4317|4318).*") - operationIsIngress = ottlexpr.JoinWithOr(spanAttributeEquals("OperationName", "Ingress"), attributeMatches("name", "ingress.*")) - operationIsEgress = ottlexpr.JoinWithOr(spanAttributeEquals("OperationName", "Egress"), attributeMatches("name", "egress.*")) + operationIsInbound = spanAttributeMatches("upstream_cluster.name", "inbound|.+") + operationIsOutbound = spanAttributeMatches("upstream_cluster.name", "outbound|.+") toFromTelemetryFluentBit = ottlexpr.JoinWithAnd(componentIsProxy, namespacesIsKymaSystem, istioCanonicalNameEquals("telemetry-fluent-bit")) toFromTelemetryTraceGateway = ottlexpr.JoinWithAnd(componentIsProxy, namespacesIsKymaSystem, istioCanonicalNameEquals("telemetry-trace-gateway")) toFromTelemetryMetricGateway = ottlexpr.JoinWithAnd(componentIsProxy, namespacesIsKymaSystem, istioCanonicalNameEquals("telemetry-metric-gateway")) toFromTelemetryMetricAgent = ottlexpr.JoinWithAnd(componentIsProxy, namespacesIsKymaSystem, istioCanonicalNameEquals("telemetry-metric-agent")) - toIstioGatewayWithHealthz = ottlexpr.JoinWithAnd(componentIsProxy, namespacesIsIstioSystem, methodIsGet, operationIsEgress, istioCanonicalNameEquals("istio-ingressgateway"), urlIsIstioHealthz) + toIstioGatewayWithHealthz = ottlexpr.JoinWithAnd(componentIsProxy, namespacesIsIstioSystem, methodIsGet, operationIsOutbound, istioCanonicalNameEquals("istio-ingressgateway"), urlIsIstioHealthz) - toTelemetryTraceService = ottlexpr.JoinWithAnd(componentIsProxy, methodIsPost, operationIsEgress, urlIsTelemetryTraceService) - toTelemetryMetricService = ottlexpr.JoinWithAnd(componentIsProxy, methodIsPost, operationIsEgress, urlIsTelemetryMetricService) + toTelemetryTraceService = ottlexpr.JoinWithAnd(componentIsProxy, methodIsPost, operationIsOutbound, urlIsTelemetryTraceService) + toTelemetryMetricService = ottlexpr.JoinWithAnd(componentIsProxy, methodIsPost, operationIsOutbound, urlIsTelemetryMetricService) //TODO: should be system namespaces after solving https://github.com/kyma-project/telemetry-manager/issues/380 - fromVMScrapeAgent = ottlexpr.JoinWithAnd(componentIsProxy, methodIsGet, operationIsIngress, userAgentMatches("vm_promscrape")) - fromTelemetryMetricAgent = ottlexpr.JoinWithAnd(componentIsProxy, methodIsGet, operationIsIngress, userAgentMatches("kyma-otelcol\\\\/.*")) + fromVMScrapeAgent = ottlexpr.JoinWithAnd(componentIsProxy, methodIsGet, operationIsInbound, userAgentMatches("vm_promscrape")) + fromTelemetryMetricAgent = ottlexpr.JoinWithAnd(componentIsProxy, methodIsGet, operationIsInbound, userAgentMatches("kyma-otelcol\\\\/.*")) ) func makeDropNoisySpansConfig() FilterProcessor { diff --git a/internal/otelcollector/config/trace/gateway/testdata/config.yaml b/internal/otelcollector/config/trace/gateway/testdata/config.yaml index 571297abf..c9d0ce5e9 100644 --- a/internal/otelcollector/config/trace/gateway/testdata/config.yaml +++ b/internal/otelcollector/config/trace/gateway/testdata/config.yaml @@ -89,11 +89,11 @@ processors: - attributes["component"] == "proxy" and resource.attributes["k8s.namespace.name"] == "kyma-system" and attributes["istio.canonical_service"] == "telemetry-trace-gateway" - attributes["component"] == "proxy" and resource.attributes["k8s.namespace.name"] == "kyma-system" and attributes["istio.canonical_service"] == "telemetry-metric-gateway" - attributes["component"] == "proxy" and resource.attributes["k8s.namespace.name"] == "kyma-system" and attributes["istio.canonical_service"] == "telemetry-metric-agent" - - attributes["component"] == "proxy" and resource.attributes["k8s.namespace.name"] == "istio-system" and attributes["http.method"] == "GET" and (attributes["OperationName"] == "Egress" or IsMatch(name, "egress.*") == true) and attributes["istio.canonical_service"] == "istio-ingressgateway" and IsMatch(attributes["http.url"], "https:\\/\\/healthz\\..+\\/healthz\\/ready") == true - - attributes["component"] == "proxy" and attributes["http.method"] == "POST" and (attributes["OperationName"] == "Egress" or IsMatch(name, "egress.*") == true) and IsMatch(attributes["http.url"], "http(s)?:\\/\\/telemetry-otlp-traces\\.kyma-system(\\..*)?:(4317|4318).*") == true - - attributes["component"] == "proxy" and attributes["http.method"] == "POST" and (attributes["OperationName"] == "Egress" or IsMatch(name, "egress.*") == true) and IsMatch(attributes["http.url"], "http(s)?:\\/\\/telemetry-otlp-metrics\\.kyma-system(\\..*)?:(4317|4318).*") == true - - attributes["component"] == "proxy" and attributes["http.method"] == "GET" and (attributes["OperationName"] == "Ingress" or IsMatch(name, "ingress.*") == true) and IsMatch(attributes["user_agent"], "vm_promscrape") == true - - attributes["component"] == "proxy" and attributes["http.method"] == "GET" and (attributes["OperationName"] == "Ingress" or IsMatch(name, "ingress.*") == true) and IsMatch(attributes["user_agent"], "kyma-otelcol\\/.*") == true + - attributes["component"] == "proxy" and resource.attributes["k8s.namespace.name"] == "istio-system" and attributes["http.method"] == "GET" and IsMatch(attributes["upstream_cluster.name"], "outbound|.+") == true and attributes["istio.canonical_service"] == "istio-ingressgateway" and IsMatch(attributes["http.url"], "https:\\/\\/healthz\\..+\\/healthz\\/ready") == true + - attributes["component"] == "proxy" and attributes["http.method"] == "POST" and IsMatch(attributes["upstream_cluster.name"], "outbound|.+") == true and IsMatch(attributes["http.url"], "http(s)?:\\/\\/telemetry-otlp-traces\\.kyma-system(\\..*)?:(4317|4318).*") == true + - attributes["component"] == "proxy" and attributes["http.method"] == "POST" and IsMatch(attributes["upstream_cluster.name"], "outbound|.+") == true and IsMatch(attributes["http.url"], "http(s)?:\\/\\/telemetry-otlp-metrics\\.kyma-system(\\..*)?:(4317|4318).*") == true + - attributes["component"] == "proxy" and attributes["http.method"] == "GET" and IsMatch(attributes["upstream_cluster.name"], "inbound|.+") == true and IsMatch(attributes["user_agent"], "vm_promscrape") == true + - attributes["component"] == "proxy" and attributes["http.method"] == "GET" and IsMatch(attributes["upstream_cluster.name"], "inbound|.+") == true and IsMatch(attributes["user_agent"], "kyma-otelcol\\/.*") == true transform/resolve-service-name: error_mode: ignore trace_statements: diff --git a/test/e2e/traces_noisy_span_filter_test.go b/test/e2e/traces_noisy_span_filter_test.go index 8c0958b75..129f86436 100644 --- a/test/e2e/traces_noisy_span_filter_test.go +++ b/test/e2e/traces_noisy_span_filter_test.go @@ -72,7 +72,7 @@ var _ = Describe(suite.ID(), Label(suite.LabelTraces), func() { vmaScrapeSpansGen := telemetrygen.NewPod(vmaScrapeSpansNs, telemetrygen.SignalTypeTraces, telemetrygen.WithTelemetryAttribute("http.method", "GET"), telemetrygen.WithTelemetryAttribute("component", "proxy"), - telemetrygen.WithTelemetryAttribute("OperationName", "Ingress"), + telemetrygen.WithTelemetryAttribute("upstream_cluster.name", "inbound|80|http://some-url"), telemetrygen.WithTelemetryAttribute("user_agent", "vm_promscrape"), ).K8sObject() healthzSpansGen := telemetrygen.NewPod(healthzSpansNs, telemetrygen.SignalTypeTraces, @@ -80,7 +80,7 @@ var _ = Describe(suite.ID(), Label(suite.LabelTraces), func() { telemetrygen.WithTelemetryAttribute("http.method", "GET"), telemetrygen.WithTelemetryAttribute("component", "proxy"), telemetrygen.WithTelemetryAttribute("istio.canonical_service", "istio-ingressgateway"), - telemetrygen.WithTelemetryAttribute("OperationName", "Egress"), + telemetrygen.WithTelemetryAttribute("upstream_cluster.name", "outbound|80||http://some-url"), telemetrygen.WithTelemetryAttribute("http.url", "https://healthz.some-url/healthz/ready"), telemetrygen.WithResourceAttribute("k8s.namespace.name", "istio-system"), ).K8sObject() @@ -92,7 +92,7 @@ var _ = Describe(suite.ID(), Label(suite.LabelTraces), func() { metricAgentScrapeSpansGen := telemetrygen.NewPod(metricAgentScrapeSpansNs, telemetrygen.SignalTypeTraces, telemetrygen.WithTelemetryAttribute("http.method", "GET"), telemetrygen.WithTelemetryAttribute("component", "proxy"), - telemetrygen.WithTelemetryAttribute("OperationName", "Ingress"), + telemetrygen.WithTelemetryAttribute("upstream_cluster.name", "inbound||"), telemetrygen.WithTelemetryAttribute("user_agent", "kyma-otelcol/0.1.0"), ).K8sObject() metricAgentSpansGen := telemetrygen.NewPod(metricAgentSpansNs, telemetrygen.SignalTypeTraces, @@ -108,7 +108,7 @@ var _ = Describe(suite.ID(), Label(suite.LabelTraces), func() { metricServiceSpansGen := telemetrygen.NewPod(metricServiceSpansNs, telemetrygen.SignalTypeTraces, telemetrygen.WithTelemetryAttribute("http.method", "POST"), telemetrygen.WithTelemetryAttribute("component", "proxy"), - telemetrygen.WithTelemetryAttribute("OperationName", "Egress"), + telemetrygen.WithTelemetryAttribute("upstream_cluster.name", "outbound||"), telemetrygen.WithTelemetryAttribute("http.url", "http://telemetry-otlp-metrics.kyma-system:4317"), telemetrygen.WithResourceAttribute("k8s.namespace.name", kitkyma.SystemNamespaceName), ).K8sObject() @@ -120,7 +120,7 @@ var _ = Describe(suite.ID(), Label(suite.LabelTraces), func() { traceServiceSpansGen := telemetrygen.NewPod(traceServiceSpansNs, telemetrygen.SignalTypeTraces, telemetrygen.WithTelemetryAttribute("http.method", "POST"), telemetrygen.WithTelemetryAttribute("component", "proxy"), - telemetrygen.WithTelemetryAttribute("OperationName", "Egress"), + telemetrygen.WithTelemetryAttribute("upstream_cluster.name", "outbound|80||http://some-url"), telemetrygen.WithTelemetryAttribute("http.url", "http://telemetry-otlp-traces.kyma-system:4317"), telemetrygen.WithResourceAttribute("k8s.namespace.name", kitkyma.SystemNamespaceName), ).K8sObject() diff --git a/test/integration/istio/traces_test.go b/test/integration/istio/traces_test.go index e5aa13fd4..06d00e4bc 100644 --- a/test/integration/istio/traces_test.go +++ b/test/integration/istio/traces_test.go @@ -158,6 +158,10 @@ var _ = Describe(suite.ID(), Label(suite.LabelIntegration), Ordered, func() { verifyCustomAppSpans(backendExportURL, appName, appNs) verifyCustomAppSpans(istiofiedBackendExportURL, appName, appNs) }) + It("Should have no noisy spans of communication to telemetry-otlp-traces endpoint", func() { + verifyNoIstioNoiseSpans(backendExportURL, istiofiedAppNs) + verifyNoIstioNoiseSpans(istiofiedBackendExportURL, istiofiedAppNs) + }) }) }) @@ -197,6 +201,21 @@ func verifyIstioSpans(backendURL, namespace string) { }, periodic.TelemetryEventuallyTimeout, periodic.TelemetryInterval).Should(Succeed()) } +func verifyNoIstioNoiseSpans(backendURL, namespace string) { + Eventually(func(g Gomega) { + resp, err := proxyClient.Get(backendURL) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(resp).To(HaveHTTPStatus(http.StatusOK)) + + g.Expect(resp).NotTo(HaveHTTPBody(HaveFlatTraces(ContainElement(SatisfyAll( + // Identify istio-proxy traces by component=proxy attribute + HaveSpanAttributes(HaveKeyWithValue("component", "proxy")), + // All calls to telemetry-otlp-traces should be dropped + HaveSpanAttributes(HaveKeyWithValue("http.url", "http://telemetry-otlp-traces.kyma-system.svc.cluster.local:4318/v1/traces")), + ))))) + }, periodic.TelemetryEventuallyTimeout, periodic.TelemetryInterval).Should(Succeed()) +} + func verifyCustomIstiofiedAppSpans(backendURL, name, namespace string) { Eventually(func(g Gomega) { resp, err := proxyClient.Get(backendURL)