diff --git a/lib/datadog/tracing/configuration/dynamic.rb b/lib/datadog/tracing/configuration/dynamic.rb index 4c75e03dd6d..a87e6b10d25 100644 --- a/lib/datadog/tracing/configuration/dynamic.rb +++ b/lib/datadog/tracing/configuration/dynamic.rb @@ -53,8 +53,46 @@ def configuration_object end end + # Dynamic configuration for `DD_TRACE_SAMPLING_RULES`. + class TracingSamplingRules < SimpleOption + def initialize + super('tracing_sampling_rules', 'DD_TRACE_SAMPLING_RULES', :rules) + end + + # Ensures sampler is rebuilt and new configuration is applied + def call(tracing_sampling_rules) + # Modify the remote configuration value that it matches the + # local environment variable it configures. + if tracing_sampling_rules + tracing_sampling_rules.each do |rule| + next unless (tags = rule['tags']) + + # Tag maps come in as arrays of 'key' and `value_glob`. + # We need to convert them into a hash for local use. + tag_array = tags.map! do |tag| + [tag['key'], tag['value_glob']] + end + + rule['tags'] = tag_array.to_h + end + + # The configuration is stored as JSON, so we need to convert it back + tracing_sampling_rules = tracing_sampling_rules.to_json + end + + super(tracing_sampling_rules) + Datadog.send(:components).reconfigure_live_sampler + end + + protected + + def configuration_object + Datadog.configuration.tracing.sampling + end + end + # List of all tracing dynamic configuration options supported. - OPTIONS = [LogInjectionEnabled, TracingHeaderTags, TracingSamplingRate].map do |option_class| + OPTIONS = [LogInjectionEnabled, TracingHeaderTags, TracingSamplingRate, TracingSamplingRules].map do |option_class| option = option_class.new [option.name, option.env_var, option] end diff --git a/lib/datadog/tracing/configuration/settings.rb b/lib/datadog/tracing/configuration/settings.rb index ebf14f5473c..798ecb87832 100644 --- a/lib/datadog/tracing/configuration/settings.rb +++ b/lib/datadog/tracing/configuration/settings.rb @@ -327,6 +327,7 @@ def self.extended(base) # @return [String,nil] # @public_api option :rules do |o| + o.type :string, nilable: true o.default { ENV.fetch(Configuration::Ext::Sampling::ENV_RULES, nil) } end diff --git a/lib/datadog/tracing/remote.rb b/lib/datadog/tracing/remote.rb index 9fdbe4e8e58..ebef54cbf31 100644 --- a/lib/datadog/tracing/remote.rb +++ b/lib/datadog/tracing/remote.rb @@ -12,12 +12,16 @@ class ReadError < StandardError; end class << self PRODUCT = 'APM_TRACING' + CAPABILITIES = [ + 1 << 29 # APM_TRACING_SAMPLE_RULES: Dynamic trace sampling rules configuration + ].freeze + def products [PRODUCT] end def capabilities - [] # No capabilities advertised + CAPABILITIES end def process_config(config, content) diff --git a/lib/datadog/tracing/sampling/ext.rb b/lib/datadog/tracing/sampling/ext.rb index 5d657cb8d27..c51b9b437b6 100644 --- a/lib/datadog/tracing/sampling/ext.rb +++ b/lib/datadog/tracing/sampling/ext.rb @@ -40,7 +40,7 @@ module Decision DEFAULT = '-0' # The sampling rate received in the agent's http response. AGENT_RATE = '-1' - # Sampling rule or sampling rate based on tracer config. + # Locally configured rule. TRACE_SAMPLING_RULE = '-3' # User directly sets sampling priority via {Tracing.reject!} or {Tracing.keep!}, # or by a custom sampler implementation. @@ -49,6 +49,10 @@ module Decision ASM = '-5' # Single Span Sampled. SPAN_SAMPLING_RATE = '-8' + # Dynamically configured rule, explicitly created by the user. + REMOTE_USER_RULE = '-10' + # Dynamically configured rule, automatically generated by Datadog. + REMOTE_DYNAMIC_RULE = '-11' end end end diff --git a/lib/datadog/tracing/sampling/rule.rb b/lib/datadog/tracing/sampling/rule.rb index a90916ebece..cbc8c128c8f 100644 --- a/lib/datadog/tracing/sampling/rule.rb +++ b/lib/datadog/tracing/sampling/rule.rb @@ -9,13 +9,18 @@ module Sampling # apply in case of a positive match. # @public_api class Rule - attr_reader :matcher, :sampler + PROVENANCE_LOCAL = :local + PROVENANCE_REMOTE_USER = :customer + PROVENANCE_REMOTE_DYNAMIC = :dynamic + + attr_reader :matcher, :sampler, :provenance # @param [Matcher] matcher A matcher to verify trace conformity against # @param [Sampler] sampler A sampler to be consulted on a positive match - def initialize(matcher, sampler) + def initialize(matcher, sampler, provenance) @matcher = matcher @sampler = sampler + @provenance = provenance end # Evaluates if the provided `trace` conforms to the `matcher`. @@ -56,7 +61,9 @@ class SimpleRule < Rule # @param sample_rate [Float] Sampling rate between +[0,1]+ def initialize( name: SimpleMatcher::MATCH_ALL, service: SimpleMatcher::MATCH_ALL, - resource: SimpleMatcher::MATCH_ALL, tags: {}, sample_rate: 1.0 + resource: SimpleMatcher::MATCH_ALL, tags: {}, + provenance: Rule::PROVENANCE_LOCAL, + sample_rate: 1.0 ) # We want to allow 0.0 to drop all traces, but {Datadog::Tracing::Sampling::RateSampler} # considers 0.0 an invalid rate and falls back to 100% sampling. @@ -69,7 +76,7 @@ def initialize( sampler = RateSampler.new sampler.sample_rate = sample_rate - super(SimpleMatcher.new(name: name, service: service, resource: resource, tags: tags), sampler) + super(SimpleMatcher.new(name: name, service: service, resource: resource, tags: tags), sampler, provenance) end end end diff --git a/lib/datadog/tracing/sampling/rule_sampler.rb b/lib/datadog/tracing/sampling/rule_sampler.rb index 3b867b1de8c..ddf9c58799c 100644 --- a/lib/datadog/tracing/sampling/rule_sampler.rb +++ b/lib/datadog/tracing/sampling/rule_sampler.rb @@ -64,6 +64,12 @@ def self.parse(rules, rate_limit, default_sample_rate) resource: rule['resource'], tags: rule['tags'], sample_rate: sample_rate, + provenance: if (provenance = rule['provenance']) + # `Rule::PROVENANCE_*` values are symbols, so convert strings to match + provenance.to_sym + else + Rule::PROVENANCE_LOCAL + end, } Core::BackportFrom24.hash_compact!(kwargs) @@ -127,7 +133,17 @@ def sample_trace(trace) rate_limiter.allow?(1).tap do |allowed| set_priority(trace, allowed) set_limiter_metrics(trace, rate_limiter.effective_rate) - trace.set_tag(Tracing::Metadata::Ext::Distributed::TAG_DECISION_MAKER, Ext::Decision::TRACE_SAMPLING_RULE) + + provenance = case rule.provenance + when Rule::PROVENANCE_REMOTE_USER + Ext::Decision::REMOTE_USER_RULE + when Rule::PROVENANCE_REMOTE_DYNAMIC + Ext::Decision::REMOTE_DYNAMIC_RULE + else + Ext::Decision::TRACE_SAMPLING_RULE + end + + trace.set_tag(Tracing::Metadata::Ext::Distributed::TAG_DECISION_MAKER, provenance) end rescue StandardError => e Datadog.logger.error( diff --git a/sig/datadog/core/configuration/options.rbs b/sig/datadog/core/configuration/options.rbs index 020abfbe34b..97b4a455c71 100644 --- a/sig/datadog/core/configuration/options.rbs +++ b/sig/datadog/core/configuration/options.rbs @@ -19,7 +19,7 @@ module Datadog module InstanceMethods def options: () -> untyped - def set_option: (untyped name, untyped value) -> untyped + def set_option: (untyped name, untyped value, precedence: Option::Precedence::Value) -> untyped def get_option: (untyped name) -> untyped @@ -27,6 +27,8 @@ module Datadog def option_defined?: (untyped name) -> untyped + def unset_option: (untyped name, precedence: Option::Precedence::Value) -> void + def using_default?: (Symbol option) -> bool def options_hash: () -> untyped diff --git a/sig/datadog/tracing/configuration/dynamic.rbs b/sig/datadog/tracing/configuration/dynamic.rbs index 14d2644db52..979c4e04935 100644 --- a/sig/datadog/tracing/configuration/dynamic.rbs +++ b/sig/datadog/tracing/configuration/dynamic.rbs @@ -14,7 +14,13 @@ module Datadog def initialize: () -> void def call: (untyped tracing_sampling_rate) -> untyped - def configuration_object: () -> untyped + def configuration_object: () -> Core::Configuration::Options::InstanceMethods + end + + class TracingSamplingRules < SimpleOption + def initialize: () -> void + def call: (Array[Hash[String, untyped]] tracing_sampling_rules) -> void + def configuration_object: () -> Core::Configuration::Options::InstanceMethods end # Correct type is `OPTIONS: Array[[String, String, Option]]` diff --git a/sig/datadog/tracing/configuration/dynamic/option.rbs b/sig/datadog/tracing/configuration/dynamic/option.rbs index 16e6388b3b8..3fd62a94028 100644 --- a/sig/datadog/tracing/configuration/dynamic/option.rbs +++ b/sig/datadog/tracing/configuration/dynamic/option.rbs @@ -14,7 +14,7 @@ module Datadog def initialize: (String name, String env_var, Symbol setting_key) -> void def call: (Object value) -> void - def configuration_object: () -> untyped + def configuration_object: () -> Core::Configuration::Options::InstanceMethods end end end diff --git a/sig/datadog/tracing/remote.rbs b/sig/datadog/tracing/remote.rbs index 2ed341684a6..d1d7bcd8235 100644 --- a/sig/datadog/tracing/remote.rbs +++ b/sig/datadog/tracing/remote.rbs @@ -5,6 +5,7 @@ module Datadog end PRODUCT: "APM_TRACING" + CAPABILITIES: Array[Integer] def self.products: () -> ::Array[String] diff --git a/sig/datadog/tracing/sampling/ext.rbs b/sig/datadog/tracing/sampling/ext.rbs index f63a0452e13..1f214187fce 100644 --- a/sig/datadog/tracing/sampling/ext.rbs +++ b/sig/datadog/tracing/sampling/ext.rbs @@ -16,6 +16,8 @@ module Datadog module Decision DEFAULT: ::String AGENT_RATE: ::String + REMOTE_DYNAMIC_RULE: ::String + REMOTE_USER_RULE: ::String TRACE_SAMPLING_RULE: ::String MANUAL: ::String ASM: ::String diff --git a/sig/datadog/tracing/sampling/rule_sampler.rbs b/sig/datadog/tracing/sampling/rule_sampler.rbs index 82d38accf2a..59c13dfc132 100644 --- a/sig/datadog/tracing/sampling/rule_sampler.rbs +++ b/sig/datadog/tracing/sampling/rule_sampler.rbs @@ -13,6 +13,8 @@ module Datadog def sample!: (untyped trace) -> untyped def update: (*untyped args, **untyped kwargs) -> (false | untyped) + def self.parse: (Array[Hash[Symbol, untyped]] rules, Float rate_limit, Float default_sample_rate) -> RuleSampler? + private def sample_trace: (untyped trace) { (untyped) -> untyped } -> untyped diff --git a/spec/datadog/core/remote/client/capabilities_spec.rb b/spec/datadog/core/remote/client/capabilities_spec.rb index 3b6b6fcb368..3570e642561 100644 --- a/spec/datadog/core/remote/client/capabilities_spec.rb +++ b/spec/datadog/core/remote/client/capabilities_spec.rb @@ -33,8 +33,8 @@ end describe '#base64_capabilities' do - it 'returns an empty string' do - expect(capabilities.base64_capabilities).to eq('') + it 'matches tracing capabilities only' do + expect(capabilities.base64_capabilities).to eq('IAAAAA==') end end end diff --git a/spec/datadog/tracing/configuration/dynamic_spec.rb b/spec/datadog/tracing/configuration/dynamic_spec.rb index 8a44fdcb487..0546479c7b9 100644 --- a/spec/datadog/tracing/configuration/dynamic_spec.rb +++ b/spec/datadog/tracing/configuration/dynamic_spec.rb @@ -79,3 +79,36 @@ option.call(new_value) end end + +RSpec.describe Datadog::Tracing::Configuration::Dynamic::TracingSamplingRules do + let(:old_value) { nil } + + include_examples 'tracing dynamic simple option', + name: 'tracing_sampling_rules', + env_var: 'DD_TRACE_SAMPLING_RULES', + config_key: :rules, + value: RSpec::Matchers::BuiltIn::Match.new(->(rules) { rules == '[{"sample_rate":1}]' }), + config_object: Datadog.configuration.tracing.sampling do + let(:new_value) { [{ sample_rate: 1 }] } + end + + context 'with tags' do + include_examples 'tracing dynamic simple option', + name: 'tracing_sampling_rules', + env_var: 'DD_TRACE_SAMPLING_RULES', + config_key: :rules, + value: RSpec::Matchers::BuiltIn::Match.new( + lambda do |rules| + rules == '[{"sample_rate":1,"tags":[{"key":"k","value_glob":"v"}]}]' + end + ), + config_object: Datadog.configuration.tracing.sampling do + let(:new_value) { [{ sample_rate: 1, tags: [{ key: 'k', value_glob: 'v' }] }] } + end + end + + it 'reconfigures the live sampler' do + expect(Datadog.send(:components)).to receive(:reconfigure_live_sampler) + option.call(new_value) + end +end diff --git a/spec/datadog/tracing/remote_spec.rb b/spec/datadog/tracing/remote_spec.rb index c31e8cc36af..00d57036f39 100644 --- a/spec/datadog/tracing/remote_spec.rb +++ b/spec/datadog/tracing/remote_spec.rb @@ -8,8 +8,8 @@ expect(remote.products).to contain_exactly('APM_TRACING') end - it 'declares no capabilities' do - expect(remote.capabilities).to be_empty + it 'declares rule sampling capabilities' do + expect(remote.capabilities).to eq([1 << 29]) end it 'declares matches that match APM_TRACING' do @@ -46,7 +46,8 @@ .with(contain_exactly( ['DD_LOGS_INJECTION', nil], ['DD_TRACE_HEADER_TAGS', nil], - ['DD_TRACE_SAMPLE_RATE', nil] + ['DD_TRACE_SAMPLE_RATE', nil], + ['DD_TRACE_SAMPLING_RULES', nil], )) process_config @@ -64,7 +65,8 @@ .with(contain_exactly( ['DD_LOGS_INJECTION', false], ['DD_TRACE_HEADER_TAGS', nil], - ['DD_TRACE_SAMPLE_RATE', nil] + ['DD_TRACE_SAMPLE_RATE', nil], + ['DD_TRACE_SAMPLING_RULES', nil], )) process_config diff --git a/spec/datadog/tracing/sampling/rule_sampler_spec.rb b/spec/datadog/tracing/sampling/rule_sampler_spec.rb index a55e5b0eeb2..1aeb8d3d166 100644 --- a/spec/datadog/tracing/sampling/rule_sampler_spec.rb +++ b/spec/datadog/tracing/sampling/rule_sampler_spec.rb @@ -100,6 +100,9 @@ it 'parses as a match any' do expect(actual_rule.matcher.name).to eq(Datadog::Tracing::Sampling::SimpleMatcher::MATCH_ALL) expect(actual_rule.matcher.service).to eq(Datadog::Tracing::Sampling::SimpleMatcher::MATCH_ALL) + expect(actual_rule.matcher.resource).to eq(Datadog::Tracing::Sampling::SimpleMatcher::MATCH_ALL) + expect(actual_rule.matcher.tags).to eq({}) + expect(actual_rule.provenance).to eq(Datadog::Tracing::Sampling::Rule::PROVENANCE_LOCAL) expect(actual_rule.sampler.sample_rate).to eq(0.1) end @@ -149,6 +152,23 @@ expect(actual_rules[1].sampler.sample_rate).to eq(0.2) end end + + context 'with provenance' do + context 'from customer' do + let(:rule) { { sample_rate: 1, provenance: 'customer' } } + it 'parses the provenance' do + expect(actual_rule.provenance).to eq(Datadog::Tracing::Sampling::Rule::PROVENANCE_REMOTE_USER) + end + end + + context 'from dynamic configuration' do + let(:rule) { { sample_rate: 1, provenance: 'dynamic' } } + + it 'parses the provenance' do + expect(actual_rule.provenance).to eq(Datadog::Tracing::Sampling::Rule::PROVENANCE_REMOTE_DYNAMIC) + end + end + end end context 'with a non-float sample_rate' do @@ -192,11 +212,13 @@ let(:rules) { [rule] } let(:rule) { instance_double(Datadog::Tracing::Sampling::Rule) } let(:sample_rate) { 0.8 } + let(:provenance) { :local } before do allow(rule).to receive(:match?).with(trace).and_return(true) allow(rule).to receive(:sample?).with(trace).and_return(sampled) allow(rule).to receive(:sample_rate).with(trace).and_return(sample_rate) + allow(rule).to receive(:provenance).and_return(provenance) end end @@ -244,6 +266,26 @@ let(:expected_sampled) { true } let(:sampling_priority) { 2 } end + + context 'when the rule is from a remote user' do + let(:provenance) { :customer } + + it_behaves_like 'a sampled! trace' do + let(:expected_sampled) { true } + let(:sampling_priority) { 2 } + let(:sampling_decision) { '-10' } + end + end + + context 'when the rule is dynamically configured' do + let(:provenance) { :dynamic } + + it_behaves_like 'a sampled! trace' do + let(:expected_sampled) { true } + let(:sampling_priority) { 2 } + let(:sampling_decision) { '-11' } + end + end end context 'and rate limited' do diff --git a/spec/datadog/tracing/sampling/rule_spec.rb b/spec/datadog/tracing/sampling/rule_spec.rb index 01a5b461295..7abbd46a86e 100644 --- a/spec/datadog/tracing/sampling/rule_spec.rb +++ b/spec/datadog/tracing/sampling/rule_spec.rb @@ -19,9 +19,10 @@ let(:trace_resource) { 'test-resource' } let(:trace_tags) { {} } - let(:rule) { described_class.new(matcher, sampler) } + let(:rule) { described_class.new(matcher, sampler, provenance) } let(:matcher) { instance_double(Datadog::Tracing::Sampling::Matcher) } let(:sampler) { instance_double(Datadog::Tracing::Sampling::Sampler) } + let(:provenance) { double('provenance') } describe '#match?' do subject(:match) { rule.match?(trace_op) }