ddtrace/tracer: introduce priorities PriorityRuleSamplerKeep (3) and …

…PriorityRuleSamplerReject (-3) for rulesSampler choices. (#1012)

So far there were 3 mechanisms to control sampling decisions in the agent,
using the following priority values:

* Sampling based on rates set by feedback loop between Agent and Tracer
  using priorities 1 (accept) and 0 (reject)
* Sampling based on sampling rate rules set on the tracer using priorities
  1 (accept) and 0 (reject)
* Sampling based on explicit manual (programatic) user choice using
  priorities 2 (accept) and -2 (reject)

Priorities 0 and 1 being used by both the dynamic agent-tracer feedback
loop and by the configuration rules sampling, has two problems, that we
intend to resolve by using separate priority values (-3 and 3) when the
sampling decision is based on config rules.

* Problem 1: The agent will always reject spans/traces with negative
  priority, but it might decide to keep some with priority 0. This
  behaviour is not intuitive for users that intend to reject a specific
  % of spans/traces.
* Problem 2: In the backend it's not possible to differentiate and produce
  stats on which spans/traces were sampled based on one mechanism vs the
  other.

Backwards compatibility
The behaviour of the agent is to reject spans with negative priority and
accept those with priority > 0 and so no corresponding changes on the agent
are required for this to be the behaviour.

ddtrace/ext/priority.go

@@ -9,6 +9,9 @@ package ext
 // In a distributed context, it should be set before any context propagation (fork, RPC calls) to be effective.
 
 const (
+	// PriorityRuleSamplerReject specifies that the rule sampler has decided that this trace should be rejected.
+	PriorityRuleSamplerReject = -3
+
 	// PriorityUserReject informs the backend that a trace should be rejected and not stored.
 	// This should be used by user code overriding default priority.
 	PriorityUserReject = -1
@@ -24,4 +27,7 @@ const (
 	// PriorityUserKeep informs the backend that a trace should be kept and not stored.
 	// This should be used by user code overriding default priority.
 	PriorityUserKeep = 2
+
+	// PriorityRuleSamplerKeep specifies that the rule sampler has decided that this trace should be kept.
+	PriorityRuleSamplerKeep = 3
 )

ddtrace/tracer/sampler.go

@@ -311,15 +311,15 @@ func (rs *rulesSampler) apply(span *span) bool {
 func (rs *rulesSampler) applyRate(span *span, rate float64, now time.Time) {
 	span.SetTag(keyRulesSamplerAppliedRate, rate)
 	if !sampledByRate(span.TraceID, rate) {
-		span.SetTag(ext.SamplingPriority, ext.PriorityAutoReject)
+		span.SetTag(ext.SamplingPriority, ext.PriorityRuleSamplerReject)
 		return
 	}
 
 	sampled, rate := rs.limiter.allowOne(now)
 	if sampled {
-		span.SetTag(ext.SamplingPriority, ext.PriorityAutoKeep)
+		span.SetTag(ext.SamplingPriority, ext.PriorityRuleSamplerKeep)
 	} else {
-		span.SetTag(ext.SamplingPriority, ext.PriorityAutoReject)
+		span.SetTag(ext.SamplingPriority, ext.PriorityRuleSamplerReject)
 	}
 	span.SetTag(keyRulesSamplerLimiterRate, rate)
 }

ddtrace/tracer/sampler_test.go

@@ -425,12 +425,12 @@ func TestRulesSamplerInternals(t *testing.T) {
 		// first span kept, second dropped
 		span := makeSpanAt("http.request", "test-service", now)
 		rs.applyRate(span, 1.0, now)
-		assert.EqualValues(ext.PriorityAutoKeep, span.Metrics[keySamplingPriority])
+		assert.EqualValues(ext.PriorityRuleSamplerKeep, span.Metrics[keySamplingPriority])
 		assert.Equal(1.0, span.Metrics["_dd.rule_psr"])
 		assert.Equal(1.0, span.Metrics["_dd.limit_psr"])
 		span = makeSpanAt("http.request", "test-service", now)
 		rs.applyRate(span, 1.0, now)
-		assert.EqualValues(ext.PriorityAutoReject, span.Metrics[keySamplingPriority])
+		assert.EqualValues(ext.PriorityRuleSamplerReject, span.Metrics[keySamplingPriority])
 		assert.Equal(1.0, span.Metrics["_dd.rule_psr"])
 		assert.Equal(0.75, span.Metrics["_dd.limit_psr"])
 	})