From d59eb447aa648c389839728d9f053a11237ccc67 Mon Sep 17 00:00:00 2001 From: j-rufino <91056982+j-rufino@users.noreply.github.com> Date: Sun, 26 Sep 2021 04:48:01 +0200 Subject: [PATCH] ddtrace/tracer: introduce priorities PriorityRuleSamplerKeep (3) and PriorityRuleSamplerReject (-3) for rulesSampler choices. (#1012) So far there were 3 mechanisms to control sampling decisions in the agent, using the following priority values: * Sampling based on rates set by feedback loop between Agent and Tracer using priorities 1 (accept) and 0 (reject) * Sampling based on sampling rate rules set on the tracer using priorities 1 (accept) and 0 (reject) * Sampling based on explicit manual (programatic) user choice using priorities 2 (accept) and -2 (reject) Priorities 0 and 1 being used by both the dynamic agent-tracer feedback loop and by the configuration rules sampling, has two problems, that we intend to resolve by using separate priority values (-3 and 3) when the sampling decision is based on config rules. * Problem 1: The agent will always reject spans/traces with negative priority, but it might decide to keep some with priority 0. This behaviour is not intuitive for users that intend to reject a specific % of spans/traces. * Problem 2: In the backend it's not possible to differentiate and produce stats on which spans/traces were sampled based on one mechanism vs the other. Backwards compatibility The behaviour of the agent is to reject spans with negative priority and accept those with priority > 0 and so no corresponding changes on the agent are required for this to be the behaviour. --- ddtrace/ext/priority.go | 6 ++++++ ddtrace/tracer/sampler.go | 6 +++--- ddtrace/tracer/sampler_test.go | 4 ++-- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/ddtrace/ext/priority.go b/ddtrace/ext/priority.go index 7dc2cd7a88..3f48c664b0 100644 --- a/ddtrace/ext/priority.go +++ b/ddtrace/ext/priority.go @@ -9,6 +9,9 @@ package ext // In a distributed context, it should be set before any context propagation (fork, RPC calls) to be effective. const ( + // PriorityRuleSamplerReject specifies that the rule sampler has decided that this trace should be rejected. + PriorityRuleSamplerReject = -3 + // PriorityUserReject informs the backend that a trace should be rejected and not stored. // This should be used by user code overriding default priority. PriorityUserReject = -1 @@ -24,4 +27,7 @@ const ( // PriorityUserKeep informs the backend that a trace should be kept and not stored. // This should be used by user code overriding default priority. PriorityUserKeep = 2 + + // PriorityRuleSamplerKeep specifies that the rule sampler has decided that this trace should be kept. + PriorityRuleSamplerKeep = 3 ) diff --git a/ddtrace/tracer/sampler.go b/ddtrace/tracer/sampler.go index b71306c020..5377a94a39 100644 --- a/ddtrace/tracer/sampler.go +++ b/ddtrace/tracer/sampler.go @@ -311,15 +311,15 @@ func (rs *rulesSampler) apply(span *span) bool { func (rs *rulesSampler) applyRate(span *span, rate float64, now time.Time) { span.SetTag(keyRulesSamplerAppliedRate, rate) if !sampledByRate(span.TraceID, rate) { - span.SetTag(ext.SamplingPriority, ext.PriorityAutoReject) + span.SetTag(ext.SamplingPriority, ext.PriorityRuleSamplerReject) return } sampled, rate := rs.limiter.allowOne(now) if sampled { - span.SetTag(ext.SamplingPriority, ext.PriorityAutoKeep) + span.SetTag(ext.SamplingPriority, ext.PriorityRuleSamplerKeep) } else { - span.SetTag(ext.SamplingPriority, ext.PriorityAutoReject) + span.SetTag(ext.SamplingPriority, ext.PriorityRuleSamplerReject) } span.SetTag(keyRulesSamplerLimiterRate, rate) } diff --git a/ddtrace/tracer/sampler_test.go b/ddtrace/tracer/sampler_test.go index 726c056cc3..7160095a63 100644 --- a/ddtrace/tracer/sampler_test.go +++ b/ddtrace/tracer/sampler_test.go @@ -425,12 +425,12 @@ func TestRulesSamplerInternals(t *testing.T) { // first span kept, second dropped span := makeSpanAt("http.request", "test-service", now) rs.applyRate(span, 1.0, now) - assert.EqualValues(ext.PriorityAutoKeep, span.Metrics[keySamplingPriority]) + assert.EqualValues(ext.PriorityRuleSamplerKeep, span.Metrics[keySamplingPriority]) assert.Equal(1.0, span.Metrics["_dd.rule_psr"]) assert.Equal(1.0, span.Metrics["_dd.limit_psr"]) span = makeSpanAt("http.request", "test-service", now) rs.applyRate(span, 1.0, now) - assert.EqualValues(ext.PriorityAutoReject, span.Metrics[keySamplingPriority]) + assert.EqualValues(ext.PriorityRuleSamplerReject, span.Metrics[keySamplingPriority]) assert.Equal(1.0, span.Metrics["_dd.rule_psr"]) assert.Equal(0.75, span.Metrics["_dd.limit_psr"]) })