From cbd25a7f95c1756fc7a8190f385000ea4ccde934 Mon Sep 17 00:00:00 2001 From: David Elner Date: Fri, 28 Dec 2018 11:20:49 -0500 Subject: [PATCH 01/13] Added: TAG_EVENT_SAMPLE_RATE constant. --- lib/ddtrace/ext/priority.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/ddtrace/ext/priority.rb b/lib/ddtrace/ext/priority.rb index 711b597c296..781f015bf4b 100644 --- a/lib/ddtrace/ext/priority.rb +++ b/lib/ddtrace/ext/priority.rb @@ -3,6 +3,8 @@ module Ext # Priority is a hint given to the backend so that it knows which traces to reject or kept. # In a distributed context, it should be set before any context propagation (fork, RPC calls) to be effective. module Priority + # Tag for span sample rate; used by agent to determine whether event is emitted. + TAG_EVENT_SAMPLE_RATE = '_dd1.sr.eausr'.freeze # Use this to explicitely inform the backend that a trace should be rejected and not stored. USER_REJECT = -1 # Used by the builtin sampler to inform the backend that a trace should be rejected and not stored. From fc14bd04469fc94a17f1a2168ef61848049b1439 Mon Sep 17 00:00:00 2001 From: David Elner Date: Fri, 28 Dec 2018 11:21:22 -0500 Subject: [PATCH 02/13] Added: Configuration::Settings :event_sample_rate option. --- lib/ddtrace/contrib/configuration/settings.rb | 1 + .../contrib/configuration/settings_spec.rb | 20 +++++++++++++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/lib/ddtrace/contrib/configuration/settings.rb b/lib/ddtrace/contrib/configuration/settings.rb index c0973e5dc82..d830c7dd0c3 100644 --- a/lib/ddtrace/contrib/configuration/settings.rb +++ b/lib/ddtrace/contrib/configuration/settings.rb @@ -9,6 +9,7 @@ class Settings option :service_name option :tracer, default: Datadog.tracer + option :event_sample_rate def initialize(options = {}) configure(options) diff --git a/spec/ddtrace/contrib/configuration/settings_spec.rb b/spec/ddtrace/contrib/configuration/settings_spec.rb index cf80e41aeb7..20581b48d13 100644 --- a/spec/ddtrace/contrib/configuration/settings_spec.rb +++ b/spec/ddtrace/contrib/configuration/settings_spec.rb @@ -9,7 +9,23 @@ describe '#options' do subject(:options) { settings.options } - it { is_expected.to include(:service_name) } - it { is_expected.to include(:tracer) } + + describe ':service_name' do + subject(:option) { options[:service_name] } + it { expect(options).to include(:service_name) } + it { expect(option.get).to be nil } + end + + describe ':tracer' do + subject(:option) { options[:tracer] } + it { expect(options).to include(:tracer) } + it { expect(option.get).to be Datadog.tracer } + end + + describe ':event_sample_rate' do + subject(:option) { options[:event_sample_rate] } + it { expect(options).to include(:event_sample_rate) } + it { expect(option.get).to be nil } + end end end From 27f00515b57387d740bad714ec3cd7f05d54cf52 Mon Sep 17 00:00:00 2001 From: David Elner Date: Fri, 28 Dec 2018 11:54:52 -0500 Subject: [PATCH 03/13] Added: Datadog::Contrib::Sampling module. --- Rakefile | 2 +- lib/ddtrace/contrib/sampling.rb | 14 ++++++++++++ spec/ddtrace/contrib/sampling_spec.rb | 33 +++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 lib/ddtrace/contrib/sampling.rb create mode 100644 spec/ddtrace/contrib/sampling_spec.rb diff --git a/Rakefile b/Rakefile index 344c9fc9c95..676f354c1bb 100644 --- a/Rakefile +++ b/Rakefile @@ -46,7 +46,7 @@ namespace :spec do end RSpec::Core::RakeTask.new(:contrib) do |t| - t.pattern = 'spec/**/contrib/{configurable,integration,patchable,patcher,registerable,configuration/*}_spec.rb' + t.pattern = 'spec/**/contrib/{configurable,integration,patchable,patcher,registerable,sampling,configuration/*}_spec.rb' end [ diff --git a/lib/ddtrace/contrib/sampling.rb b/lib/ddtrace/contrib/sampling.rb new file mode 100644 index 00000000000..e61ee55909d --- /dev/null +++ b/lib/ddtrace/contrib/sampling.rb @@ -0,0 +1,14 @@ +require 'ddtrace/ext/priority' + +module Datadog + module Contrib + # Defines sampling behavior for integrations + module Sampling + module_function + + def set_event_sample_rate(span, sample_rate) + span.set_metric(Datadog::Ext::Priority::TAG_EVENT_SAMPLE_RATE, sample_rate) unless sample_rate.nil? + end + end + end +end diff --git a/spec/ddtrace/contrib/sampling_spec.rb b/spec/ddtrace/contrib/sampling_spec.rb new file mode 100644 index 00000000000..80d6068c4ce --- /dev/null +++ b/spec/ddtrace/contrib/sampling_spec.rb @@ -0,0 +1,33 @@ +require 'spec_helper' + +require 'ddtrace/contrib/sampling' + +RSpec.describe Datadog::Contrib::Sampling do + describe '::set_event_sample_rate' do + subject(:set_event_sample_rate) { described_class.set_event_sample_rate(span, sample_rate) } + let(:span) { instance_double(Datadog::Span) } + + context 'when sample rate is nil' do + let(:sample_rate) { nil } + + it 'does not set the tag' do + expect(span).to_not receive(:set_metric) + set_event_sample_rate + end + end + + context 'when a sample rate is given' do + let(:sample_rate) { 0.5 } + + it 'sets the tag' do + expect(span).to receive(:set_metric) + .with( + Datadog::Ext::Priority::TAG_EVENT_SAMPLE_RATE, + sample_rate + ) + + set_event_sample_rate + end + end + end +end From 15de3ff9d13c32d10b8de9cc81c267f4f855ef67 Mon Sep 17 00:00:00 2001 From: David Elner Date: Wed, 2 Jan 2019 12:42:41 -0500 Subject: [PATCH 04/13] Added: event_sample_rate shared examples. --- spec/ddtrace/contrib/sampling_examples.rb | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 spec/ddtrace/contrib/sampling_examples.rb diff --git a/spec/ddtrace/contrib/sampling_examples.rb b/spec/ddtrace/contrib/sampling_examples.rb new file mode 100644 index 00000000000..e8a96e4972a --- /dev/null +++ b/spec/ddtrace/contrib/sampling_examples.rb @@ -0,0 +1,18 @@ +require 'ddtrace/ext/priority' + +RSpec.shared_examples_for 'event sample rate' do + context 'when not configured' do + it 'is not included in the tags' do + expect(span.get_metric(Datadog::Ext::Priority::TAG_EVENT_SAMPLE_RATE)).to be nil + end + end + + context 'when set' do + let(:configuration_options) { super().merge(event_sample_rate: event_sample_rate) } + let(:event_sample_rate) { 0.5 } + + it 'is included in the tags' do + expect(span.get_metric(Datadog::Ext::Priority::TAG_EVENT_SAMPLE_RATE)).to eq(event_sample_rate) + end + end +end From 8a1d89a2520dd806b0fdad54c2188e7a033d90b8 Mon Sep 17 00:00:00 2001 From: David Elner Date: Fri, 28 Dec 2018 11:55:52 -0500 Subject: [PATCH 05/13] Added: event_sample_rate tag to DelayedJob integration. --- lib/ddtrace/contrib/delayed_job/plugin.rb | 2 ++ spec/ddtrace/contrib/delayed_job/plugin_spec.rb | 10 ++++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/ddtrace/contrib/delayed_job/plugin.rb b/lib/ddtrace/contrib/delayed_job/plugin.rb index bef91e6f7b6..fdcbc8211de 100644 --- a/lib/ddtrace/contrib/delayed_job/plugin.rb +++ b/lib/ddtrace/contrib/delayed_job/plugin.rb @@ -1,4 +1,5 @@ require 'delayed/plugin' +require 'ddtrace/contrib/sampling' require 'ddtrace/contrib/delayed_job/ext' module Datadog @@ -18,6 +19,7 @@ def self.instrument(job, &block) end tracer.trace(Ext::SPAN_JOB, service: configuration[:service_name], resource: job_name) do |span| + Contrib::Sampling.set_event_sample_rate(span, configuration[:event_sample_rate]) span.set_tag(Ext::TAG_ID, job.id) span.set_tag(Ext::TAG_QUEUE, job.queue) if job.queue span.set_tag(Ext::TAG_PRIORITY, job.priority) diff --git a/spec/ddtrace/contrib/delayed_job/plugin_spec.rb b/spec/ddtrace/contrib/delayed_job/plugin_spec.rb index 1157c02bb07..530894d7db5 100644 --- a/spec/ddtrace/contrib/delayed_job/plugin_spec.rb +++ b/spec/ddtrace/contrib/delayed_job/plugin_spec.rb @@ -1,4 +1,6 @@ require 'spec_helper' +require 'ddtrace/contrib/sampling_examples' + require 'active_record' require 'delayed_job' require 'delayed_job_active_record' @@ -25,13 +27,15 @@ def job_data end let(:tracer) { get_test_tracer } + let(:configuration_options) { { tracer: tracer } } before do - Datadog.configure { |c| c.use :delayed_job, tracer: tracer } - + Datadog.configure { |c| c.use :delayed_job, configuration_options } Delayed::Worker.delay_jobs = false end + after(:each) { Datadog.registry[:delayed_job].reset_configuration! } + describe 'instrumenting worker execution' do let(:worker) { double(:worker, name: 'worker') } before do @@ -100,6 +104,8 @@ def job_data expect(span.get_tag('delayed_job.attempts')).to eq('0') end + it_behaves_like 'event sample rate' + context 'when queue name is set' do let(:queue_name) { 'queue_name' } let(:job_params) { { queue: queue_name } } From 371e551def70dca8583f4a9100a9db5845175cc2 Mon Sep 17 00:00:00 2001 From: David Elner Date: Fri, 28 Dec 2018 16:07:43 -0500 Subject: [PATCH 06/13] Added: event_sample_rate tag to Racecar integration. --- lib/ddtrace/contrib/racecar/event.rb | 2 ++ spec/ddtrace/contrib/racecar/patcher_spec.rb | 26 ++++++++++++++------ 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/lib/ddtrace/contrib/racecar/event.rb b/lib/ddtrace/contrib/racecar/event.rb index 35d8bd4cf67..8110353ed0f 100644 --- a/lib/ddtrace/contrib/racecar/event.rb +++ b/lib/ddtrace/contrib/racecar/event.rb @@ -1,3 +1,4 @@ +require 'ddtrace/contrib/sampling' require 'ddtrace/contrib/active_support/notifications/event' require 'ddtrace/contrib/racecar/ext' @@ -36,6 +37,7 @@ def process(span, event, _id, payload) span.service = configuration[:service_name] span.resource = payload[:consumer_class] + Contrib::Sampling.set_event_sample_rate(span, configuration[:event_sample_rate]) span.set_tag(Ext::TAG_TOPIC, payload[:topic]) span.set_tag(Ext::TAG_CONSUMER, payload[:consumer_class]) span.set_tag(Ext::TAG_PARTITION, payload[:partition]) diff --git a/spec/ddtrace/contrib/racecar/patcher_spec.rb b/spec/ddtrace/contrib/racecar/patcher_spec.rb index e1a039e17a5..4ee5c04ab25 100644 --- a/spec/ddtrace/contrib/racecar/patcher_spec.rb +++ b/spec/ddtrace/contrib/racecar/patcher_spec.rb @@ -1,4 +1,5 @@ require 'spec_helper' +require 'ddtrace/contrib/sampling_examples' require 'racecar' require 'racecar/cli' @@ -6,6 +7,7 @@ require 'ddtrace' RSpec.describe 'Racecar patcher' do let(:tracer) { get_test_tracer } + let(:configuration_options) { { tracer: tracer } } def all_spans tracer.writer.spans(:keep) @@ -13,10 +15,12 @@ def all_spans before(:each) do Datadog.configure do |c| - c.use :racecar, tracer: tracer + c.use :racecar, configuration_options end end + after(:each) { Datadog.registry[:racecar].reset_configuration! } + describe 'for single message processing' do let(:topic) { 'dd_trace_test_dummy' } let(:consumer) { 'DummyConsumer' } @@ -31,7 +35,7 @@ def all_spans } end - let(:racecar_span) do + let(:span) do all_spans.select { |s| s.name == Datadog::Contrib::Racecar::Ext::SPAN_MESSAGE }.first end @@ -39,7 +43,7 @@ def all_spans it 'is expected to send a span' do ActiveSupport::Notifications.instrument('process_message.racecar', payload) - racecar_span.tap do |span| + span.tap do |span| expect(span).to_not be nil expect(span.service).to eq('racecar') expect(span.name).to eq('racecar.message') @@ -67,7 +71,7 @@ def all_spans nil end - racecar_span.tap do |span| + span.tap do |span| expect(span).to_not be nil expect(span.service).to eq('racecar') expect(span.name).to eq('racecar.message') @@ -81,6 +85,10 @@ def all_spans end end end + + it_behaves_like 'event sample rate' do + before { ActiveSupport::Notifications.instrument('process_message.racecar', payload) } + end end describe 'for batch message processing' do @@ -99,7 +107,7 @@ def all_spans } end - let(:racecar_span) do + let(:span) do all_spans.select { |s| s.name == Datadog::Contrib::Racecar::Ext::SPAN_BATCH }.first end @@ -107,7 +115,7 @@ def all_spans it 'is expected to send a span' do ActiveSupport::Notifications.instrument('process_batch.racecar', payload) - racecar_span.tap do |span| + span.tap do |span| expect(span).to_not be nil expect(span.service).to eq('racecar') expect(span.name).to eq('racecar.batch') @@ -135,7 +143,7 @@ def all_spans nil end - racecar_span.tap do |span| + span.tap do |span| expect(span).to_not be nil expect(span.service).to eq('racecar') expect(span.name).to eq('racecar.batch') @@ -150,5 +158,9 @@ def all_spans end end end + + it_behaves_like 'event sample rate' do + before { ActiveSupport::Notifications.instrument('process_batch.racecar', payload) } + end end end From f9625739c13c66c3a7100d6bcf5d9ba0dd638e6c Mon Sep 17 00:00:00 2001 From: David Elner Date: Fri, 28 Dec 2018 16:23:08 -0500 Subject: [PATCH 07/13] Added: event_sample_rate tag to Rack integration. --- lib/ddtrace/contrib/rack/middlewares.rb | 30 ++- .../contrib/rack/configuration_spec.rb | 187 ++++++++++++++++++ .../contrib/rack/request_queuing_spec.rb | 179 ----------------- 3 files changed, 208 insertions(+), 188 deletions(-) create mode 100644 spec/ddtrace/contrib/rack/configuration_spec.rb delete mode 100644 spec/ddtrace/contrib/rack/request_queuing_spec.rb diff --git a/lib/ddtrace/contrib/rack/middlewares.rb b/lib/ddtrace/contrib/rack/middlewares.rb index aa6620f74d1..cb2dca46fcf 100644 --- a/lib/ddtrace/contrib/rack/middlewares.rb +++ b/lib/ddtrace/contrib/rack/middlewares.rb @@ -1,6 +1,7 @@ require 'ddtrace/ext/app_types' require 'ddtrace/ext/http' require 'ddtrace/propagation/http_propagator' +require 'ddtrace/contrib/sampling' require 'ddtrace/contrib/rack/ext' require 'ddtrace/contrib/rack/request_queue' @@ -25,7 +26,7 @@ def initialize(app) end def compute_queue_time(env, tracer) - return unless Datadog.configuration[:rack][:request_queuing] + return unless configuration[:request_queuing] # parse the request queue time request_start = Datadog::Contrib::Rack::QueueTime.get_request_start(env) @@ -34,25 +35,25 @@ def compute_queue_time(env, tracer) tracer.trace( Ext::SPAN_HTTP_SERVER_QUEUE, start_time: request_start, - service: Datadog.configuration[:rack][:web_service_name] + service: configuration[:web_service_name] ) end def call(env) # retrieve integration settings - tracer = Datadog.configuration[:rack][:tracer] + tracer = configuration[:tracer] # [experimental] create a root Span to keep track of frontend web servers # (i.e. Apache, nginx) if the header is properly set frontend_span = compute_queue_time(env, tracer) trace_options = { - service: Datadog.configuration[:rack][:service_name], + service: configuration[:service_name], resource: nil, span_type: Datadog::Ext::HTTP::TYPE } - if Datadog.configuration[:rack][:distributed_tracing] + if configuration[:distributed_tracing] context = HTTPPropagator.extract(env) tracer.provider.context = context if context.trace_id end @@ -110,7 +111,7 @@ def call(env) end def resource_name_for(env, status) - if Datadog.configuration[:rack][:middleware_names] && env['RESPONSE_MIDDLEWARE'] + if configuration[:middleware_names] && env['RESPONSE_MIDDLEWARE'] "#{env['RESPONSE_MIDDLEWARE']}##{env['REQUEST_METHOD']}" else "#{env['REQUEST_METHOD']} #{status}".strip @@ -134,13 +135,19 @@ def set_request_tags!(request_span, env, status, headers, response, original_env response_headers = parse_response_headers(headers || {}) request_span.resource ||= resource_name_for(env, status) + + # Set event sample rate, if available. + Contrib::Sampling.set_event_sample_rate(request_span, configuration[:event_sample_rate]) + if request_span.get_tag(Datadog::Ext::HTTP::METHOD).nil? request_span.set_tag(Datadog::Ext::HTTP::METHOD, env['REQUEST_METHOD']) end + if request_span.get_tag(Datadog::Ext::HTTP::URL).nil? - options = Datadog.configuration[:rack][:quantize] + options = configuration[:quantize] request_span.set_tag(Datadog::Ext::HTTP::URL, Datadog::Quantization::HTTP.url(url, options)) end + if request_span.get_tag(Datadog::Ext::HTTP::BASE_URL).nil? request_obj = ::Rack::Request.new(env) @@ -153,6 +160,7 @@ def set_request_tags!(request_span, env, status, headers, response, original_env request_span.set_tag(Datadog::Ext::HTTP::BASE_URL, base_url) end + if request_span.get_tag(Datadog::Ext::HTTP::STATUS_CODE).nil? && status request_span.set_tag(Datadog::Ext::HTTP::STATUS_CODE, status) end @@ -182,6 +190,10 @@ def set_request_tags!(request_span, env, status, headers, response, original_env If you need the Rack request span, try using `Datadog.tracer.active_span`. This key will be removed in version 1.0).freeze + def configuration + Datadog.configuration[:rack] + end + def add_deprecation_warnings(env) env.instance_eval do unless instance_variable_defined?(:@patched_with_datadog_warnings) @@ -221,7 +233,7 @@ def without_datadog_warnings def parse_request_headers(env) {}.tap do |result| - whitelist = Datadog.configuration[:rack][:headers][:request] || [] + whitelist = configuration[:headers][:request] || [] whitelist.each do |header| rack_header = header_to_rack_header(header) if env.key?(rack_header) @@ -233,7 +245,7 @@ def parse_request_headers(env) def parse_response_headers(headers) {}.tap do |result| - whitelist = Datadog.configuration[:rack][:headers][:response] || [] + whitelist = configuration[:headers][:response] || [] whitelist.each do |header| if headers.key?(header) result[Datadog::Ext::HTTP::ResponseHeaders.to_tag(header)] = headers[header] diff --git a/spec/ddtrace/contrib/rack/configuration_spec.rb b/spec/ddtrace/contrib/rack/configuration_spec.rb new file mode 100644 index 00000000000..acfbadea959 --- /dev/null +++ b/spec/ddtrace/contrib/rack/configuration_spec.rb @@ -0,0 +1,187 @@ +require 'spec_helper' +require 'ddtrace/contrib/sampling_examples' +require 'rack/test' + +require 'rack' +require 'ddtrace' +require 'ddtrace/contrib/rack/middlewares' + +RSpec.describe 'Rack integration configuration' do + include Rack::Test::Methods + + let(:tracer) { get_test_tracer } + let(:configuration_options) { { tracer: tracer } } + + let(:spans) { tracer.writer.spans } + let(:span) { spans.first } + + before(:each) do + Datadog.configure do |c| + c.use :rack, configuration_options + end + end + + after(:each) { Datadog.registry[:rack].reset_configuration! } + + shared_context 'an incoming HTTP request' do + subject(:response) { get '/' } + + let(:app) do + Rack::Builder.new do + use Datadog::Contrib::Rack::TraceMiddleware + + map '/' do + run(proc { |_env| [200, { 'Content-Type' => 'text/html' }, 'OK'] }) + end + end.to_app + end + end + + it_behaves_like 'event sample rate' do + include_context 'an incoming HTTP request' + before { is_expected.to be_ok } + end + + describe 'request queueing' do + shared_context 'queue header' do + let(:queue_value) { "t=#{queue_time}" } + let(:queue_time) { (Time.now.utc - 5).to_i } + + before(:each) do + header queue_header, queue_value + end + end + + shared_context 'no queue header' do + let(:queue_header) { nil } + let(:queue_value) { nil } + end + + shared_examples_for 'a Rack request with queuing' do + let(:queue_span) { spans.first } + let(:rack_span) { spans.last } + + it 'produces a queued Rack trace' do + is_expected.to be_ok + expect(spans).to have(2).items + + expect(queue_span.name).to eq('http_server.queue') + expect(queue_span.service).to eq(Datadog.configuration[:rack][:web_service_name]) + expect(queue_span.start_time.to_i).to eq(queue_time) + + expect(rack_span.name).to eq('rack.request') + expect(rack_span.span_type).to eq('http') + expect(rack_span.service).to eq(Datadog.configuration[:rack][:service_name]) + expect(rack_span.resource).to eq('GET 200') + expect(rack_span.get_tag('http.method')).to eq('GET') + expect(rack_span.get_tag('http.status_code')).to eq('200') + expect(rack_span.get_tag('http.url')).to eq('/') + expect(rack_span.status).to eq(0) + + expect(queue_span.span_id).to eq(rack_span.parent_id) + end + end + + shared_examples_for 'a Rack request without queuing' do + it 'produces a non-queued Rack trace' do + is_expected.to be_ok + expect(spans).to have(1).items + + expect(span).to_not be nil + expect(span.name).to eq('rack.request') + expect(span.span_type).to eq('http') + expect(span.service).to eq(Datadog.configuration[:rack][:service_name]) + expect(span.resource).to eq('GET 200') + expect(span.get_tag('http.method')).to eq('GET') + expect(span.get_tag('http.status_code')).to eq('200') + expect(span.get_tag('http.url')).to eq('/') + expect(span.status).to eq(0) + + expect(span.parent_id).to eq(0) + end + end + + context 'when enabled' do + let(:configuration_options) { super().merge(request_queuing: true) } + + context 'and a request is received' do + include_context 'an incoming HTTP request' + + context 'with X-Request-Start header' do + include_context 'queue header' do + let(:queue_header) { 'X-Request-Start' } + end + + it_behaves_like 'a Rack request with queuing' + + context 'given a custom web service name' do + let(:configuration_options) { super().merge(web_service_name: web_service_name) } + let(:web_service_name) { 'nginx' } + + it_behaves_like 'a Rack request with queuing' do + it 'sets the custom service name' do + is_expected.to be_ok + expect(queue_span.service).to eq(web_service_name) + end + end + end + end + + context 'with X-Queue-Start header' do + include_context 'queue header' do + let(:queue_header) { 'X-Queue-Start' } + end + + it_behaves_like 'a Rack request with queuing' + end + + # Ensure a queuing Span is NOT created if there is a clock skew + # where the starting time is greater than current host Time.now + context 'with a skewed queue header' do + before(:each) { header 'X-Request-Start', (Time.now.utc + 5).to_i } + it_behaves_like 'a Rack request without queuing' + end + + # Ensure a queuing Span is NOT created if the header is wrong + context 'with a invalid queue header' do + before(:each) { header 'X-Request-Start', 'foobar' } + it_behaves_like 'a Rack request without queuing' + end + + context 'without queue header' do + include_context 'no queue header' + it_behaves_like 'a Rack request without queuing' + end + end + end + + context 'when disabled' do + let(:configuration_options) { super().merge(request_queuing: false) } + + context 'and a request is received' do + include_context 'an incoming HTTP request' + + context 'with X-Request-Start header' do + include_context 'queue header' do + let(:queue_header) { 'X-Request-Start' } + end + + it_behaves_like 'a Rack request without queuing' + end + + context 'with X-Queue-Start header' do + include_context 'queue header' do + let(:queue_header) { 'X-Queue-Start' } + end + + it_behaves_like 'a Rack request without queuing' + end + + context 'without queue header' do + include_context 'no queue header' + it_behaves_like 'a Rack request without queuing' + end + end + end + end +end diff --git a/spec/ddtrace/contrib/rack/request_queuing_spec.rb b/spec/ddtrace/contrib/rack/request_queuing_spec.rb deleted file mode 100644 index 4fe5aa06d7c..00000000000 --- a/spec/ddtrace/contrib/rack/request_queuing_spec.rb +++ /dev/null @@ -1,179 +0,0 @@ -require 'spec_helper' -require 'rack/test' - -require 'rack' -require 'ddtrace' -require 'ddtrace/contrib/rack/middlewares' - -RSpec.describe 'Rack integration request queuing' do - include Rack::Test::Methods - - let(:tracer) { get_test_tracer } - let(:rack_options) { { tracer: tracer } } - - let(:spans) { tracer.writer.spans } - let(:span) { spans.first } - - before(:each) do - Datadog.configure do |c| - c.use :rack, rack_options - end - end - - after(:each) { Datadog.registry[:rack].reset_configuration! } - - shared_context 'an incoming HTTP request' do - subject(:response) { get '/' } - - let(:app) do - Rack::Builder.new do - use Datadog::Contrib::Rack::TraceMiddleware - - map '/' do - run(proc { |_env| [200, { 'Content-Type' => 'text/html' }, 'OK'] }) - end - end.to_app - end - end - - shared_context 'queue header' do - let(:queue_value) { "t=#{queue_time}" } - let(:queue_time) { (Time.now.utc - 5).to_i } - - before(:each) do - header queue_header, queue_value - end - end - - shared_context 'no queue header' do - let(:queue_header) { nil } - let(:queue_value) { nil } - end - - shared_examples_for 'a Rack request with queuing' do - let(:queue_span) { spans.first } - let(:rack_span) { spans.last } - - it 'produces a queued Rack trace' do - is_expected.to be_ok - expect(spans).to have(2).items - - expect(queue_span.name).to eq('http_server.queue') - expect(queue_span.service).to eq(Datadog.configuration[:rack][:web_service_name]) - expect(queue_span.start_time.to_i).to eq(queue_time) - - expect(rack_span.name).to eq('rack.request') - expect(rack_span.span_type).to eq('http') - expect(rack_span.service).to eq(Datadog.configuration[:rack][:service_name]) - expect(rack_span.resource).to eq('GET 200') - expect(rack_span.get_tag('http.method')).to eq('GET') - expect(rack_span.get_tag('http.status_code')).to eq('200') - expect(rack_span.get_tag('http.url')).to eq('/') - expect(rack_span.status).to eq(0) - - expect(queue_span.span_id).to eq(rack_span.parent_id) - end - end - - shared_examples_for 'a Rack request without queuing' do - it 'produces a non-queued Rack trace' do - is_expected.to be_ok - expect(spans).to have(1).items - - expect(span).to_not be nil - expect(span.name).to eq('rack.request') - expect(span.span_type).to eq('http') - expect(span.service).to eq(Datadog.configuration[:rack][:service_name]) - expect(span.resource).to eq('GET 200') - expect(span.get_tag('http.method')).to eq('GET') - expect(span.get_tag('http.status_code')).to eq('200') - expect(span.get_tag('http.url')).to eq('/') - expect(span.status).to eq(0) - - expect(span.parent_id).to eq(0) - end - end - - context 'when enabled' do - let(:rack_options) { super().merge(request_queuing: true) } - - context 'and a request is received' do - include_context 'an incoming HTTP request' - - context 'with X-Request-Start header' do - include_context 'queue header' do - let(:queue_header) { 'X-Request-Start' } - end - - it_behaves_like 'a Rack request with queuing' - - context 'given a custom web service name' do - let(:rack_options) { super().merge(web_service_name: web_service_name) } - let(:web_service_name) { 'nginx' } - - it_behaves_like 'a Rack request with queuing' do - it 'sets the custom service name' do - is_expected.to be_ok - expect(queue_span.service).to eq(web_service_name) - end - end - end - end - - context 'with X-Queue-Start header' do - include_context 'queue header' do - let(:queue_header) { 'X-Queue-Start' } - end - - it_behaves_like 'a Rack request with queuing' - end - - # Ensure a queuing Span is NOT created if there is a clock skew - # where the starting time is greater than current host Time.now - context 'with a skewed queue header' do - before(:each) { header 'X-Request-Start', (Time.now.utc + 5).to_i } - it_behaves_like 'a Rack request without queuing' - end - - # Ensure a queuing Span is NOT created if the header is wrong - context 'with a invalid queue header' do - before(:each) { header 'X-Request-Start', 'foobar' } - it_behaves_like 'a Rack request without queuing' - end - - context 'without queue header' do - include_context 'no queue header' - it_behaves_like 'a Rack request without queuing' - end - end - end - - context 'when disabled' do - let(:rack_options) { super().merge(request_queuing: false) } - - context 'and a request is received' do - include_context 'an incoming HTTP request' - - context 'with X-Request-Start header' do - include_context 'queue header' do - let(:queue_header) { 'X-Request-Start' } - end - - it_behaves_like 'a Rack request without queuing' - end - - context 'with X-Queue-Start header' do - include_context 'queue header' do - let(:queue_header) { 'X-Queue-Start' } - end - - it_behaves_like 'a Rack request without queuing' - end - - context 'without queue header' do - include_context 'no queue header' - it_behaves_like 'a Rack request without queuing' - end - end - end -end From 110b69b59428fdcbbc43585c43b6c50871bc8b3b Mon Sep 17 00:00:00 2001 From: David Elner Date: Fri, 28 Dec 2018 16:34:03 -0500 Subject: [PATCH 08/13] Added: event_sample_rate tag to Rake integration. --- lib/ddtrace/contrib/rake/instrumentation.rb | 2 ++ spec/ddtrace/contrib/rake/instrumentation_spec.rb | 13 ++++++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/lib/ddtrace/contrib/rake/instrumentation.rb b/lib/ddtrace/contrib/rake/instrumentation.rb index d2d109baa1c..f1f81961f69 100644 --- a/lib/ddtrace/contrib/rake/instrumentation.rb +++ b/lib/ddtrace/contrib/rake/instrumentation.rb @@ -1,3 +1,4 @@ +require 'ddtrace/contrib/sampling' require 'ddtrace/contrib/rake/ext' module Datadog @@ -41,6 +42,7 @@ def shutdown_tracer! def annotate_invoke!(span, args) span.resource = name + Contrib::Sampling.set_event_sample_rate(span, configuration[:event_sample_rate]) span.set_tag(Ext::TAG_TASK_ARG_NAMES, arg_names) span.set_tag(Ext::TAG_INVOKE_ARGS, quantize_args(args)) unless args.nil? rescue StandardError => e diff --git a/spec/ddtrace/contrib/rake/instrumentation_spec.rb b/spec/ddtrace/contrib/rake/instrumentation_spec.rb index ab9c8e3893f..14d24f96bb1 100644 --- a/spec/ddtrace/contrib/rake/instrumentation_spec.rb +++ b/spec/ddtrace/contrib/rake/instrumentation_spec.rb @@ -1,4 +1,5 @@ require 'spec_helper' +require 'ddtrace/contrib/sampling_examples' require 'securerandom' require 'rake' @@ -25,10 +26,11 @@ end after(:each) do + # Reset configuration to defaults + Datadog.registry[:rake].reset_configuration! + # We don't want instrumentation enabled during the rest of the test suite... - Datadog.configure do |c| - c.use :rake, enabled: false - end + Datadog.configure { |c| c.use :rake, enabled: false } end def reset_task!(task_name) @@ -87,6 +89,10 @@ def reset_task!(task_name) expect(invoke_span.resource).to eq(task_name.to_s) expect(invoke_span.parent_id).to eq(0) end + + it_behaves_like 'event sample rate' do + let(:span) { invoke_span } + end end describe '\'rake.execute\' span' do @@ -94,6 +100,7 @@ def reset_task!(task_name) expect(execute_span.name).to eq(Datadog::Contrib::Rake::Ext::SPAN_EXECUTE) expect(execute_span.resource).to eq(task_name.to_s) expect(execute_span.parent_id).to eq(invoke_span.span_id) + expect(execute_span.get_tag(Datadog::Ext::Priority::TAG_EVENT_SAMPLE_RATE)).to be nil end end end From 8ee97cc2564ea3840e8c5f0b4c5e35cee5ee02f6 Mon Sep 17 00:00:00 2001 From: David Elner Date: Fri, 28 Dec 2018 17:17:55 -0500 Subject: [PATCH 09/13] Changed: Resque to only use Pin for forking, not configuration. --- lib/ddtrace/contrib/resque/patcher.rb | 10 ---- lib/ddtrace/contrib/resque/resque_job.rb | 53 +++++++++++++------ .../contrib/resque/instrumentation_spec.rb | 10 ++-- 3 files changed, 43 insertions(+), 30 deletions(-) diff --git a/lib/ddtrace/contrib/resque/patcher.rb b/lib/ddtrace/contrib/resque/patcher.rb index 3a0d8bb51c9..ebab2584d9f 100644 --- a/lib/ddtrace/contrib/resque/patcher.rb +++ b/lib/ddtrace/contrib/resque/patcher.rb @@ -19,7 +19,6 @@ def patch do_once(:resque) do begin require_relative 'resque_job' - add_pin get_option(:workers).each { |worker| worker.extend(ResqueJob) } rescue StandardError => e Datadog::Tracer.log.error("Unable to apply Resque integration: #{e}") @@ -27,15 +26,6 @@ def patch end end - def add_pin - Pin.new( - get_option(:service_name), - app: Ext::APP, - app_type: Datadog::Ext::AppTypes::WORKER, - tracer: get_option(:tracer) - ).onto(::Resque) - end - def get_option(option) Datadog.configuration[:resque].get_option(option) end diff --git a/lib/ddtrace/contrib/resque/resque_job.rb b/lib/ddtrace/contrib/resque/resque_job.rb index 6d05f36f399..9b9f39dd117 100644 --- a/lib/ddtrace/contrib/resque/resque_job.rb +++ b/lib/ddtrace/contrib/resque/resque_job.rb @@ -9,13 +9,12 @@ module Resque # Uses Resque job hooks to create traces module ResqueJob def around_perform(*_) - pin = Pin.get_from(::Resque) - return yield unless pin && pin.tracer - pin.tracer.trace(Ext::SPAN_JOB, service: pin.service) do |span| + return yield unless datadog_configuration && tracer + + tracer.trace(Ext::SPAN_JOB, span_options) do |span| span.resource = name - span.span_type = pin.app_type + span.span_type = Datadog::Ext::AppTypes::WORKER yield - span.service = pin.service end end @@ -28,8 +27,27 @@ def on_failure_shutdown_tracer(*_) end def shutdown_tracer_when_forked! - pin = Datadog::Pin.get_from(Resque) - pin.tracer.shutdown! if pin && pin.tracer && pin.config && pin.config[:forked] + tracer.shutdown! if forked? + end + + private + + def forked? + pin = Datadog::Pin.get_from(::Resque) + return false unless pin + pin.config[:forked] == true + end + + def span_options + { service: datadog_configuration[:service_name] } + end + + def tracer + datadog_configuration.tracer + end + + def datadog_configuration + Datadog.configuration[:resque] end end end @@ -37,12 +55,17 @@ def shutdown_tracer_when_forked! end Resque.after_fork do - # get the current tracer - pin = Datadog::Pin.get_from(Resque) - next unless pin && pin.tracer - pin.config ||= {} - pin.config[:forked] = true - - # clean the state so no CoW happens - pin.tracer.provider.context = nil + configuration = Datadog.configuration[:resque] + next if configuration.nil? + + # Add a pin, marking the job as forked. + # Used to trigger shutdown in forks for performance reasons. + Datadog::Pin.new( + configuration[:service_name], + config: { forked: true } + ).onto(::Resque) + + # Clean the state so no CoW happens + next if configuration[:tracer].nil? + configuration[:tracer].provider.context = nil end diff --git a/spec/ddtrace/contrib/resque/instrumentation_spec.rb b/spec/ddtrace/contrib/resque/instrumentation_spec.rb index 39ff4f0319a..00d7b81c24c 100644 --- a/spec/ddtrace/contrib/resque/instrumentation_spec.rb +++ b/spec/ddtrace/contrib/resque/instrumentation_spec.rb @@ -7,7 +7,6 @@ include_context 'Resque job' let(:tracer) { get_test_tracer } - let(:pin) { ::Resque.datadog_pin } let(:spans) { tracer.writer.spans } let(:span) { spans.first } @@ -15,6 +14,8 @@ let(:host) { ENV.fetch('TEST_REDIS_HOST', '127.0.0.1') } let(:port) { ENV.fetch('TEST_REDIS_PORT', 6379) } + let(:configuration_options) { { tracer: tracer } } + before(:each) do # Setup Resque to use Redis ::Resque.redis = url @@ -22,13 +23,12 @@ # Patch Resque Datadog.configure do |c| - c.use :resque + c.use :resque, configuration_options end - - # Update the Resque pin with the tracer - pin.tracer = tracer end + after(:each) { Datadog.registry[:resque].reset_configuration! } + shared_examples 'job execution tracing' do context 'that succeeds' do before(:each) { perform_job(job_class) } From 507483164b9d951117b1f82cfb58896d00f24ad7 Mon Sep 17 00:00:00 2001 From: David Elner Date: Fri, 28 Dec 2018 17:37:03 -0500 Subject: [PATCH 10/13] Added: event_sample_rate tag to Resque integration. --- lib/ddtrace/contrib/resque/resque_job.rb | 2 ++ spec/ddtrace/contrib/resque/instrumentation_spec.rb | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/lib/ddtrace/contrib/resque/resque_job.rb b/lib/ddtrace/contrib/resque/resque_job.rb index 9b9f39dd117..f0d05bd70c6 100644 --- a/lib/ddtrace/contrib/resque/resque_job.rb +++ b/lib/ddtrace/contrib/resque/resque_job.rb @@ -1,5 +1,6 @@ require 'ddtrace/ext/app_types' require 'ddtrace/sync_writer' +require 'ddtrace/contrib/sampling' require 'ddtrace/contrib/sidekiq/ext' require 'resque' @@ -14,6 +15,7 @@ def around_perform(*_) tracer.trace(Ext::SPAN_JOB, span_options) do |span| span.resource = name span.span_type = Datadog::Ext::AppTypes::WORKER + Contrib::Sampling.set_event_sample_rate(span, datadog_configuration[:event_sample_rate]) yield end end diff --git a/spec/ddtrace/contrib/resque/instrumentation_spec.rb b/spec/ddtrace/contrib/resque/instrumentation_spec.rb index 00d7b81c24c..70c9768b2a1 100644 --- a/spec/ddtrace/contrib/resque/instrumentation_spec.rb +++ b/spec/ddtrace/contrib/resque/instrumentation_spec.rb @@ -1,4 +1,5 @@ require 'spec_helper' +require 'ddtrace/contrib/sampling_examples' require_relative 'job' require 'ddtrace' @@ -42,6 +43,8 @@ expect(span.service).to eq('resque') expect(span.status).to_not eq(Datadog::Ext::Errors::STATUS) end + + it_behaves_like 'event sample rate' end context 'that fails' do @@ -71,6 +74,8 @@ expect(span.status).to eq(Datadog::Ext::Errors::STATUS) expect(span.get_tag(Datadog::Ext::Errors::TYPE)).to eq(error_class_name) end + + it_behaves_like 'event sample rate' end end From 2776fa614d0a9639b78c5528211801f9a9c3186c Mon Sep 17 00:00:00 2001 From: David Elner Date: Wed, 2 Jan 2019 12:44:44 -0500 Subject: [PATCH 11/13] Added: event_sample_rate tag to Shoryuken integration. --- lib/ddtrace/contrib/shoryuken/tracer.rb | 13 ++++++++++--- spec/ddtrace/contrib/shoryuken/tracer_spec.rb | 14 ++++++++++++-- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/lib/ddtrace/contrib/shoryuken/tracer.rb b/lib/ddtrace/contrib/shoryuken/tracer.rb index 15796d36c7f..192924d6c21 100644 --- a/lib/ddtrace/contrib/shoryuken/tracer.rb +++ b/lib/ddtrace/contrib/shoryuken/tracer.rb @@ -1,17 +1,20 @@ +require 'ddtrace/contrib/sampling' + module Datadog module Contrib module Shoryuken # Tracer is a Shoryuken server-side middleware which traces executed jobs class Tracer def initialize(options = {}) - @tracer = options[:tracer] || Datadog.configuration[:shoryuken][:tracer] - @shoryuken_service = options[:service_name] || Datadog.configuration[:shoryuken][:service_name] + @tracer = options[:tracer] || configuration[:tracer] + @shoryuken_service = options[:service_name] || configuration[:service_name] set_service_info(@shoryuken_service) end def call(worker_instance, queue, sqs_msg, body) @tracer.trace(Ext::SPAN_JOB, service: @shoryuken_service, span_type: Datadog::Ext::AppTypes::WORKER) do |span| span.resource = worker_instance.class.name + Contrib::Sampling.set_event_sample_rate(span, configuration[:event_sample_rate]) span.set_tag(Ext::TAG_JOB_ID, sqs_msg.message_id) span.set_tag(Ext::TAG_JOB_QUEUE, queue) span.set_tag(Ext::TAG_JOB_ATTRIBUTES, sqs_msg.attributes) if sqs_msg.respond_to?(:attributes) @@ -23,8 +26,12 @@ def call(worker_instance, queue, sqs_msg, body) private + def configuration + Datadog.configuration[:shoryuken] + end + def set_service_info(service) - return if @tracer.services[service] + return if @tracer.nil? || @tracer.services[service] @tracer.set_service_info( service, Ext::APP, diff --git a/spec/ddtrace/contrib/shoryuken/tracer_spec.rb b/spec/ddtrace/contrib/shoryuken/tracer_spec.rb index afa15f5c60b..19daa4cde77 100644 --- a/spec/ddtrace/contrib/shoryuken/tracer_spec.rb +++ b/spec/ddtrace/contrib/shoryuken/tracer_spec.rb @@ -1,10 +1,12 @@ require 'spec_helper' +require 'ddtrace/contrib/sampling_examples' + require 'ddtrace' require 'shoryuken' RSpec.describe Datadog::Contrib::Shoryuken::Tracer do let(:tracer) { get_test_tracer } - let(:options) { { tracer: tracer } } + let(:configuration_options) { { tracer: tracer } } let(:spans) { tracer.writer.spans } let(:span) { spans.first } @@ -12,10 +14,12 @@ Shoryuken.worker_executor = Shoryuken::Worker::InlineExecutor Datadog.configure do |c| - c.use :shoryuken, options + c.use :shoryuken, configuration_options end end + after { Datadog.registry[:shoryuken].reset_configuration! } + context 'when a Shoryuken::Worker class' do subject(:worker_class) do qn = queue_name @@ -38,6 +42,12 @@ def perform(sqs_msg, body); end .and_call_original end + # TODO: These expectations do not work because Shoryuken doesn't run middleware in tests + # https://github.com/phstc/shoryuken/issues/541 + # it_behaves_like 'event sample rate' do + # before { perform_async } + # end + it do expect { perform_async }.to_not raise_error # TODO: These expectations do not work because Shoryuken doesn't run middleware in tests From 05a7831f381a82649e03203ce661e5d0b97824ce Mon Sep 17 00:00:00 2001 From: David Elner Date: Wed, 2 Jan 2019 13:25:07 -0500 Subject: [PATCH 12/13] Added: event_sample_rate tag to Sidekiq integration. --- lib/ddtrace/contrib/sidekiq/client_tracer.rb | 16 +++++++++++----- lib/ddtrace/contrib/sidekiq/server_tracer.rb | 8 +++++++- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/lib/ddtrace/contrib/sidekiq/client_tracer.rb b/lib/ddtrace/contrib/sidekiq/client_tracer.rb index 0c6954abeda..0c9535ac6b4 100644 --- a/lib/ddtrace/contrib/sidekiq/client_tracer.rb +++ b/lib/ddtrace/contrib/sidekiq/client_tracer.rb @@ -1,4 +1,5 @@ require 'ddtrace/contrib/sidekiq/tracing' +require 'ddtrace/contrib/sampling' module Datadog module Contrib @@ -9,19 +10,18 @@ class ClientTracer def initialize(options = {}) super - @sidekiq_service = options[:client_service_name] || Datadog.configuration[:sidekiq][:client_service_name] + @sidekiq_service = options[:client_service_name] || configuration[:client_service_name] + set_service_info(@sidekiq_service) end # Client middleware arguments are documented here: # https://github.com/mperham/sidekiq/wiki/Middleware#client-middleware def call(worker_class, job, queue, redis_pool) - service = @sidekiq_service - set_service_info(service) - resource = job_resource(job) - @tracer.trace(Ext::SPAN_PUSH, service: service) do |span| + @tracer.trace(Ext::SPAN_PUSH, service: @sidekiq_service) do |span| span.resource = resource + Contrib::Sampling.set_event_sample_rate(span, configuration[:event_sample_rate]) span.set_tag(Ext::TAG_JOB_ID, job['jid']) span.set_tag(Ext::TAG_JOB_QUEUE, job['queue']) span.set_tag(Ext::TAG_JOB_WRAPPER, job['class']) if job['wrapped'] @@ -29,6 +29,12 @@ def call(worker_class, job, queue, redis_pool) yield end end + + private + + def configuration + Datadog.configuration[:sidekiq] + end end end end diff --git a/lib/ddtrace/contrib/sidekiq/server_tracer.rb b/lib/ddtrace/contrib/sidekiq/server_tracer.rb index f85108c6f22..cfb200e0b29 100644 --- a/lib/ddtrace/contrib/sidekiq/server_tracer.rb +++ b/lib/ddtrace/contrib/sidekiq/server_tracer.rb @@ -1,4 +1,5 @@ require 'ddtrace/contrib/sidekiq/tracing' +require 'ddtrace/contrib/sampling' module Datadog module Contrib @@ -9,7 +10,7 @@ class ServerTracer def initialize(options = {}) super - @sidekiq_service = options[:service_name] || Datadog.configuration[:sidekiq][:service_name] + @sidekiq_service = options[:service_name] || configuration[:service_name] end def call(worker, job, queue) @@ -20,6 +21,7 @@ def call(worker, job, queue) @tracer.trace(Ext::SPAN_JOB, service: service, span_type: Datadog::Ext::AppTypes::WORKER) do |span| span.resource = resource + Contrib::Sampling.set_event_sample_rate(span, configuration[:event_sample_rate]) span.set_tag(Ext::TAG_JOB_ID, job['jid']) span.set_tag(Ext::TAG_JOB_RETRY, job['retry']) span.set_tag(Ext::TAG_JOB_QUEUE, job['queue']) @@ -32,6 +34,10 @@ def call(worker, job, queue) private + def configuration + Datadog.configuration[:sidekiq] + end + def service_from_worker_config(resource) # Try to get the Ruby class from the resource name. worker_klass = begin From f520e88824117944aa4fc809ad6f15097d8df553 Mon Sep 17 00:00:00 2001 From: David Elner Date: Wed, 2 Jan 2019 13:30:31 -0500 Subject: [PATCH 13/13] Added: event_sample_rate tag to SuckerPunch integration. --- lib/ddtrace/contrib/sucker_punch/instrumentation.rb | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/ddtrace/contrib/sucker_punch/instrumentation.rb b/lib/ddtrace/contrib/sucker_punch/instrumentation.rb index 67d238fb0a8..f8a54e41edf 100644 --- a/lib/ddtrace/contrib/sucker_punch/instrumentation.rb +++ b/lib/ddtrace/contrib/sucker_punch/instrumentation.rb @@ -1,4 +1,5 @@ require 'sucker_punch' +require 'ddtrace/contrib/sampling' require 'ddtrace/contrib/sucker_punch/ext' module Datadog @@ -18,6 +19,7 @@ def __run_perform(*args) __with_instrumentation(Ext::SPAN_PERFORM) do |span| span.resource = "PROCESS #{self}" + Contrib::Sampling.set_event_sample_rate(span, datadog_configuration[:event_sample_rate]) __run_perform_without_datadog(*args) end rescue => e @@ -43,6 +45,10 @@ def perform_in(interval, *args) private + def datadog_configuration + Datadog.configuration[:sucker_punch] + end + def __with_instrumentation(name) pin = Datadog::Pin.get_from(::SuckerPunch)