Skip to content

Commit

Permalink
fix: Adjusted noisy trace filter to new istio reality (#1612)
Browse files Browse the repository at this point in the history
  • Loading branch information
@a-thaler
a-thaler authored Nov 18, 2024
1 parent b712c0a commit 8ca254f
Show file tree
Hide file tree
Showing 4 changed files with 36 additions and 17 deletions.
14 changes: 7 additions & 7 deletions internal/otelcollector/config/trace/gateway/drop_noisy_spans.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,22 +15,22 @@ var (
urlIsTelemetryTraceService = urlMatches("http(s)?:\\\\/\\\\/telemetry-otlp-traces\\\\.kyma-system(\\\\..*)?:(4317|4318).*")
urlIsTelemetryMetricService = urlMatches("http(s)?:\\\\/\\\\/telemetry-otlp-metrics\\\\.kyma-system(\\\\..*)?:(4317|4318).*")

operationIsIngress = ottlexpr.JoinWithOr(spanAttributeEquals("OperationName", "Ingress"), attributeMatches("name", "ingress.*"))
operationIsEgress = ottlexpr.JoinWithOr(spanAttributeEquals("OperationName", "Egress"), attributeMatches("name", "egress.*"))
operationIsInbound = spanAttributeMatches("upstream_cluster.name", "inbound|.+")
operationIsOutbound = spanAttributeMatches("upstream_cluster.name", "outbound|.+")

toFromTelemetryFluentBit = ottlexpr.JoinWithAnd(componentIsProxy, namespacesIsKymaSystem, istioCanonicalNameEquals("telemetry-fluent-bit"))
toFromTelemetryTraceGateway = ottlexpr.JoinWithAnd(componentIsProxy, namespacesIsKymaSystem, istioCanonicalNameEquals("telemetry-trace-gateway"))
toFromTelemetryMetricGateway = ottlexpr.JoinWithAnd(componentIsProxy, namespacesIsKymaSystem, istioCanonicalNameEquals("telemetry-metric-gateway"))
toFromTelemetryMetricAgent = ottlexpr.JoinWithAnd(componentIsProxy, namespacesIsKymaSystem, istioCanonicalNameEquals("telemetry-metric-agent"))

toIstioGatewayWithHealthz = ottlexpr.JoinWithAnd(componentIsProxy, namespacesIsIstioSystem, methodIsGet, operationIsEgress, istioCanonicalNameEquals("istio-ingressgateway"), urlIsIstioHealthz)
toIstioGatewayWithHealthz = ottlexpr.JoinWithAnd(componentIsProxy, namespacesIsIstioSystem, methodIsGet, operationIsOutbound, istioCanonicalNameEquals("istio-ingressgateway"), urlIsIstioHealthz)

toTelemetryTraceService = ottlexpr.JoinWithAnd(componentIsProxy, methodIsPost, operationIsEgress, urlIsTelemetryTraceService)
toTelemetryMetricService = ottlexpr.JoinWithAnd(componentIsProxy, methodIsPost, operationIsEgress, urlIsTelemetryMetricService)
toTelemetryTraceService = ottlexpr.JoinWithAnd(componentIsProxy, methodIsPost, operationIsOutbound, urlIsTelemetryTraceService)
toTelemetryMetricService = ottlexpr.JoinWithAnd(componentIsProxy, methodIsPost, operationIsOutbound, urlIsTelemetryMetricService)

//TODO: should be system namespaces after solving https://github.com/kyma-project/telemetry-manager/issues/380
fromVMScrapeAgent = ottlexpr.JoinWithAnd(componentIsProxy, methodIsGet, operationIsIngress, userAgentMatches("vm_promscrape"))
fromTelemetryMetricAgent = ottlexpr.JoinWithAnd(componentIsProxy, methodIsGet, operationIsIngress, userAgentMatches("kyma-otelcol\\\\/.*"))
fromVMScrapeAgent = ottlexpr.JoinWithAnd(componentIsProxy, methodIsGet, operationIsInbound, userAgentMatches("vm_promscrape"))
fromTelemetryMetricAgent = ottlexpr.JoinWithAnd(componentIsProxy, methodIsGet, operationIsInbound, userAgentMatches("kyma-otelcol\\\\/.*"))
)

func makeDropNoisySpansConfig() FilterProcessor {
Expand Down
10 changes: 5 additions & 5 deletions internal/otelcollector/config/trace/gateway/testdata/config.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,11 +89,11 @@ processors:
- attributes["component"] == "proxy" and resource.attributes["k8s.namespace.name"] == "kyma-system" and attributes["istio.canonical_service"] == "telemetry-trace-gateway"
- attributes["component"] == "proxy" and resource.attributes["k8s.namespace.name"] == "kyma-system" and attributes["istio.canonical_service"] == "telemetry-metric-gateway"
- attributes["component"] == "proxy" and resource.attributes["k8s.namespace.name"] == "kyma-system" and attributes["istio.canonical_service"] == "telemetry-metric-agent"
- attributes["component"] == "proxy" and resource.attributes["k8s.namespace.name"] == "istio-system" and attributes["http.method"] == "GET" and (attributes["OperationName"] == "Egress" or IsMatch(name, "egress.*") == true) and attributes["istio.canonical_service"] == "istio-ingressgateway" and IsMatch(attributes["http.url"], "https:\\/\\/healthz\\..+\\/healthz\\/ready") == true
- attributes["component"] == "proxy" and attributes["http.method"] == "POST" and (attributes["OperationName"] == "Egress" or IsMatch(name, "egress.*") == true) and IsMatch(attributes["http.url"], "http(s)?:\\/\\/telemetry-otlp-traces\\.kyma-system(\\..*)?:(4317|4318).*") == true
- attributes["component"] == "proxy" and attributes["http.method"] == "POST" and (attributes["OperationName"] == "Egress" or IsMatch(name, "egress.*") == true) and IsMatch(attributes["http.url"], "http(s)?:\\/\\/telemetry-otlp-metrics\\.kyma-system(\\..*)?:(4317|4318).*") == true
- attributes["component"] == "proxy" and attributes["http.method"] == "GET" and (attributes["OperationName"] == "Ingress" or IsMatch(name, "ingress.*") == true) and IsMatch(attributes["user_agent"], "vm_promscrape") == true
- attributes["component"] == "proxy" and attributes["http.method"] == "GET" and (attributes["OperationName"] == "Ingress" or IsMatch(name, "ingress.*") == true) and IsMatch(attributes["user_agent"], "kyma-otelcol\\/.*") == true
- attributes["component"] == "proxy" and resource.attributes["k8s.namespace.name"] == "istio-system" and attributes["http.method"] == "GET" and IsMatch(attributes["upstream_cluster.name"], "outbound|.+") == true and attributes["istio.canonical_service"] == "istio-ingressgateway" and IsMatch(attributes["http.url"], "https:\\/\\/healthz\\..+\\/healthz\\/ready") == true
- attributes["component"] == "proxy" and attributes["http.method"] == "POST" and IsMatch(attributes["upstream_cluster.name"], "outbound|.+") == true and IsMatch(attributes["http.url"], "http(s)?:\\/\\/telemetry-otlp-traces\\.kyma-system(\\..*)?:(4317|4318).*") == true
- attributes["component"] == "proxy" and attributes["http.method"] == "POST" and IsMatch(attributes["upstream_cluster.name"], "outbound|.+") == true and IsMatch(attributes["http.url"], "http(s)?:\\/\\/telemetry-otlp-metrics\\.kyma-system(\\..*)?:(4317|4318).*") == true
- attributes["component"] == "proxy" and attributes["http.method"] == "GET" and IsMatch(attributes["upstream_cluster.name"], "inbound|.+") == true and IsMatch(attributes["user_agent"], "vm_promscrape") == true
- attributes["component"] == "proxy" and attributes["http.method"] == "GET" and IsMatch(attributes["upstream_cluster.name"], "inbound|.+") == true and IsMatch(attributes["user_agent"], "kyma-otelcol\\/.*") == true
transform/resolve-service-name:
error_mode: ignore
trace_statements:
Expand Down
10 changes: 5 additions & 5 deletions test/e2e/traces_noisy_span_filter_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,15 +72,15 @@ var _ = Describe(suite.ID(), Label(suite.LabelTraces), func() {
vmaScrapeSpansGen := telemetrygen.NewPod(vmaScrapeSpansNs, telemetrygen.SignalTypeTraces,
telemetrygen.WithTelemetryAttribute("http.method", "GET"),
telemetrygen.WithTelemetryAttribute("component", "proxy"),
telemetrygen.WithTelemetryAttribute("OperationName", "Ingress"),
telemetrygen.WithTelemetryAttribute("upstream_cluster.name", "inbound|80|http://some-url"),
telemetrygen.WithTelemetryAttribute("user_agent", "vm_promscrape"),
).K8sObject()
healthzSpansGen := telemetrygen.NewPod(healthzSpansNs, telemetrygen.SignalTypeTraces,
telemetrygen.WithResourceAttribute("k8s.namespace.name", kitkyma.IstioSystemNamespaceName),
telemetrygen.WithTelemetryAttribute("http.method", "GET"),
telemetrygen.WithTelemetryAttribute("component", "proxy"),
telemetrygen.WithTelemetryAttribute("istio.canonical_service", "istio-ingressgateway"),
telemetrygen.WithTelemetryAttribute("OperationName", "Egress"),
telemetrygen.WithTelemetryAttribute("upstream_cluster.name", "outbound|80||http://some-url"),
telemetrygen.WithTelemetryAttribute("http.url", "https://healthz.some-url/healthz/ready"),
telemetrygen.WithResourceAttribute("k8s.namespace.name", "istio-system"),
).K8sObject()
Expand All @@ -92,7 +92,7 @@ var _ = Describe(suite.ID(), Label(suite.LabelTraces), func() {
metricAgentScrapeSpansGen := telemetrygen.NewPod(metricAgentScrapeSpansNs, telemetrygen.SignalTypeTraces,
telemetrygen.WithTelemetryAttribute("http.method", "GET"),
telemetrygen.WithTelemetryAttribute("component", "proxy"),
telemetrygen.WithTelemetryAttribute("OperationName", "Ingress"),
telemetrygen.WithTelemetryAttribute("upstream_cluster.name", "inbound||"),
telemetrygen.WithTelemetryAttribute("user_agent", "kyma-otelcol/0.1.0"),
).K8sObject()
metricAgentSpansGen := telemetrygen.NewPod(metricAgentSpansNs, telemetrygen.SignalTypeTraces,
Expand All @@ -108,7 +108,7 @@ var _ = Describe(suite.ID(), Label(suite.LabelTraces), func() {
metricServiceSpansGen := telemetrygen.NewPod(metricServiceSpansNs, telemetrygen.SignalTypeTraces,
telemetrygen.WithTelemetryAttribute("http.method", "POST"),
telemetrygen.WithTelemetryAttribute("component", "proxy"),
telemetrygen.WithTelemetryAttribute("OperationName", "Egress"),
telemetrygen.WithTelemetryAttribute("upstream_cluster.name", "outbound||"),
telemetrygen.WithTelemetryAttribute("http.url", "http://telemetry-otlp-metrics.kyma-system:4317"),
telemetrygen.WithResourceAttribute("k8s.namespace.name", kitkyma.SystemNamespaceName),
).K8sObject()
Expand All @@ -120,7 +120,7 @@ var _ = Describe(suite.ID(), Label(suite.LabelTraces), func() {
traceServiceSpansGen := telemetrygen.NewPod(traceServiceSpansNs, telemetrygen.SignalTypeTraces,
telemetrygen.WithTelemetryAttribute("http.method", "POST"),
telemetrygen.WithTelemetryAttribute("component", "proxy"),
telemetrygen.WithTelemetryAttribute("OperationName", "Egress"),
telemetrygen.WithTelemetryAttribute("upstream_cluster.name", "outbound|80||http://some-url"),
telemetrygen.WithTelemetryAttribute("http.url", "http://telemetry-otlp-traces.kyma-system:4317"),
telemetrygen.WithResourceAttribute("k8s.namespace.name", kitkyma.SystemNamespaceName),
).K8sObject()
Expand Down
19 changes: 19 additions & 0 deletions test/integration/istio/traces_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,10 @@ var _ = Describe(suite.ID(), Label(suite.LabelIntegration), Ordered, func() {
verifyCustomAppSpans(backendExportURL, appName, appNs)
verifyCustomAppSpans(istiofiedBackendExportURL, appName, appNs)
})
It("Should have no noisy spans of communication to telemetry-otlp-traces endpoint", func() {
verifyNoIstioNoiseSpans(backendExportURL, istiofiedAppNs)
verifyNoIstioNoiseSpans(istiofiedBackendExportURL, istiofiedAppNs)
})
})
})

Expand Down Expand Up @@ -197,6 +201,21 @@ func verifyIstioSpans(backendURL, namespace string) {
}, periodic.TelemetryEventuallyTimeout, periodic.TelemetryInterval).Should(Succeed())
}

func verifyNoIstioNoiseSpans(backendURL, namespace string) {
Eventually(func(g Gomega) {
resp, err := proxyClient.Get(backendURL)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(resp).To(HaveHTTPStatus(http.StatusOK))

g.Expect(resp).NotTo(HaveHTTPBody(HaveFlatTraces(ContainElement(SatisfyAll(
// Identify istio-proxy traces by component=proxy attribute
HaveSpanAttributes(HaveKeyWithValue("component", "proxy")),
// All calls to telemetry-otlp-traces should be dropped
HaveSpanAttributes(HaveKeyWithValue("http.url", "http://telemetry-otlp-traces.kyma-system.svc.cluster.local:4318/v1/traces")),
)))))
}, periodic.TelemetryEventuallyTimeout, periodic.TelemetryInterval).Should(Succeed())
}

func verifyCustomIstiofiedAppSpans(backendURL, name, namespace string) {
Eventually(func(g Gomega) {
resp, err := proxyClient.Get(backendURL)
Expand Down

0 comments on commit 8ca254f

Please sign in to comment.