Tracing Dynamic Configuration

CHANGELOG.md

@@ -2,6 +2,9 @@
 
 ## [Unreleased]
 
+### Added
+* Runtime Metrics: YJIT statistics on Ruby 3.2 (when YJIT is enabled) ([#2711][])
+
 ## [1.12.1] - 2023-06-14
 
 ### Added

lib/datadog/core/configuration/components.rb

@@ -66,6 +66,8 @@ def build_telemetry(settings, agent_settings, logger)
           end
         end
 
+        include Datadog::Tracing::Component::InstanceMethods
+
         attr_reader \
           :health_metrics,
           :logger,
@@ -108,6 +110,9 @@ def startup!(settings)
           else
             @logger.debug('Profiling is disabled')
           end
+
+          # Start remote configuration worker
+          @remote.start if @remote
         end
 
         # Shuts down all the components in use.

lib/datadog/core/configuration/option.rb

@@ -131,6 +131,8 @@ def reset
 
           # Reset back to the lowest precedence, to allow all `set`s to succeed right after a reset.
           @precedence_set = Precedence::DEFAULT
+          # Reset all stored values
+          @value_per_precedence = Hash.new(UNSET)
         end
 
         def default_value
@@ -244,7 +246,10 @@ def internal_set(value, precedence)
           (@value = context_exec(validate_type(value), old_value, &definition.setter)).tap do |v|
             @is_set = true
             @precedence_set = precedence
-            @value_per_precedence[precedence] = v
+            # Store original value to ensure we can always safely call `#internal_set`
+            # when restoring a value from `@value_per_precedence`, and we are only running `definition.setter`
+            # on the original value, not on a valud that has already been processed by `definition.setter`.
+            @value_per_precedence[precedence] = value
             context_exec(v, old_value, &definition.on_set) if definition.on_set
           end
         end

lib/datadog/core/remote/client/capabilities.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative '../../../appsec/remote'
+require_relative '../../../tracing/remote'
 
 module Datadog
   module Core
@@ -28,6 +29,10 @@ def register(settings)
               register_products(Datadog::AppSec::Remote.products)
               register_receivers(Datadog::AppSec::Remote.receivers)
             end
+
+            register_capabilities(Datadog::Tracing::Remote.capabilities)
+            register_products(Datadog::Tracing::Remote.products)
+            register_receivers(Datadog::Tracing::Remote.receivers)
           end
 
           def register_capabilities(capabilities)

lib/datadog/core/remote/component.rb

@@ -62,8 +62,21 @@ def initialize(settings, capabilities, agent_settings)
           end
         end
 
-        def barrier(kind)
+        # Starts the Remote Configuration worker without waiting for first run
+        def start
           @worker.start
+        end
+
+        # Is the Remote Configuration worker running?
+        def started?
+          @worker.started?
+        end
+
+        # If the worker is not initialized, initialize it.
+        #
+        # Then, waits for one client sync to be executed if `kind` is `:once`.
+        def barrier(kind)
+          start
 
           case kind
           when :once
@@ -130,26 +143,15 @@ def lift
         end
 
         class << self
+          # Because the agent might not be available yet, we can't perform agent-specific checks yet, as they
+          # would prevent remote configuration from ever running.
+          #
+          # Those checks are instead performed inside the worker loop.
+          # This allows users to upgrade their agent while keeping their application running.
           def build(settings, agent_settings)
             return unless settings.remote.enabled
 
-            capabilities = Client::Capabilities.new(settings)
-
-            return if capabilities.products.empty?
-
-            negotiation = Negotiation.new(settings, agent_settings)
-
-            unless negotiation.endpoint?('/v0.7/config')
-              Datadog.logger.error do
-                'endpoint unavailable: disabling remote configuration for this process.'
-              end
-
-              return
-            end
-
-            Datadog.logger.debug { 'agent reachable and reports remote configuration endpoint' }
-
-            new(settings, capabilities, agent_settings)
+            new(settings, Client::Capabilities.new(settings), agent_settings)
           end
         end
       end

lib/datadog/core/remote/configuration/content.rb

@@ -18,12 +18,14 @@ def parse(hash)
             end
           end
 
-          attr_reader :path, :data, :hashes
+          attr_reader :path, :data, :hashes, :apply_state, :apply_error
           attr_accessor :version
 
           def initialize(path:, data:)
             @path = path
             @data = data
+            @apply_state = ApplyState::UNACKNOWLEDGED
+            @apply_error = nil
             @hashes = {}
             @version = 0
           end
@@ -36,6 +38,31 @@ def length
             @length ||= @data.size
           end
 
+          # Sets this configuration as successfully applied.
+          def applied
+            @apply_state = ApplyState::ACKNOWLEDGED
+            @apply_error = nil
+          end
+
+          # Sets this configuration as not successfully applied, with
+          # a message describing the error.
+          def errored(error_message)
+            @apply_state = ApplyState::ERROR
+            @apply_error = error_message
+          end
+
+          module ApplyState
+            # Default state of configurations.
+            # Set until the component consuming the configuration has acknowledged it was applied.
+            UNACKNOWLEDGED = 1
+
+            # Set when the configuration has been successfully applied.
+            ACKNOWLEDGED = 2
+
+            # Set when the configuration has been unsuccessfully applied.
+            ERROR = 3
+          end
+
           private
 
           def compute_and_store_hash(type)

lib/datadog/core/remote/configuration/repository.rb

@@ -94,7 +94,9 @@ def contents_to_config_states(contents)
                 {
                   id: content.path.config_id,
                   version: content.version,
-                  product: content.path.product
+                  product: content.path.product,
+                  apply_state: content.apply_state,
+                  apply_error: content.apply_error,
                 }
               end
             end

lib/datadog/core/telemetry/client.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative 'emitter'
 require_relative 'heartbeat'
 require_relative '../utils/forking'
@@ -65,6 +67,13 @@ def integrations_change!
           @emitter.request(:'app-integrations-change')
         end
 
+        # Report configuration changes caused by Remote Configuration.
+        def client_configuration_change!(changes)
+          return if !@enabled || forked?
+
+          @emitter.request('app-client-configuration-change', data: { changes: changes, origin: 'remote_config' })
+        end
+
         private
 
         def heartbeat!

lib/datadog/core/telemetry/emitter.rb

@@ -21,11 +21,13 @@ def initialize(http_transport: Datadog::Core::Telemetry::Http::Transport.new)
 
         # Retrieves and emits a TelemetryRequest object based on the request type specified
         # @param request_type [String] the type of telemetry request to collect data for
-        def request(request_type)
+        # @param data [Object] arbitrary object to be passed to the respective `request_type` handler
+        def request(request_type, data: nil)
           begin
             request = Datadog::Core::Telemetry::Event.new.telemetry_request(
               request_type: request_type,
-              seq_id: self.class.sequence.next
+              seq_id: self.class.sequence.next,
+              data: data,
             ).to_h
             @http_transport.request(request_type: request_type.to_s, payload: request.to_json)
           rescue StandardError => e

lib/datadog/core/telemetry/event.rb

@@ -1,6 +1,9 @@
+# frozen_string_literal: true
+
 require_relative 'collector'
 require_relative 'v1/app_event'
 require_relative 'v1/telemetry_request'
+require_relative 'v2/app_client_configuration_change'
 
 module Datadog
   module Core
@@ -9,7 +12,7 @@ module Telemetry
       class Event
         include Telemetry::Collector
 
-        API_VERSION = 'v1'.freeze
+        API_VERSION = 'v1'
 
         attr_reader \
           :api_version
@@ -21,12 +24,13 @@ def initialize
         # Forms a TelemetryRequest object based on the event request_type
         # @param request_type [String] the type of telemetry request to collect data for
         # @param seq_id [Integer] the ID of the request; incremented each time a telemetry request is sent to the API
-        def telemetry_request(request_type:, seq_id:)
+        # @param data [Object] arbitrary object to be passed to the respective `request_type` handler
+        def telemetry_request(request_type:, seq_id:, data: nil)
           Telemetry::V1::TelemetryRequest.new(
             api_version: @api_version,
             application: application,
             host: host,
-            payload: payload(request_type),
+            payload: payload(request_type, data),
             request_type: request_type,
             runtime_id: runtime_id,
             seq_id: seq_id,
@@ -36,14 +40,16 @@ def telemetry_request(request_type:, seq_id:)
 
         private
 
-        def payload(request_type)
+        def payload(request_type, data)
           case request_type
           when :'app-started'
             app_started
           when :'app-closing', :'app-heartbeat'
             {}
           when :'app-integrations-change'
             app_integrations_change
+          when 'app-client-configuration-change'
+            app_client_configuration_change(data)
           else
             raise ArgumentError, "Request type invalid, received request_type: #{@request_type}"
           end
@@ -61,6 +67,15 @@ def app_started
         def app_integrations_change
           Telemetry::V1::AppEvent.new(integrations: integrations)
         end
+
+        # DEV: During the transition from V1 to V2, the backend accepts many V2
+        # DEV: payloads through the V1 transport protocol.
+        # DEV: The `app-client-configuration-change` payload is one of them.
+        # DEV: Once V2 is fully implemented, `Telemetry::V2::AppClientConfigurationChange`
+        # DEV: should be reusable without major modifications.
+        def app_client_configuration_change(data)
+          Telemetry::V2::AppClientConfigurationChange.new(data[:changes], origin: data[:origin])
+        end
       end
     end
   end

lib/datadog/core/telemetry/v2/app_client_configuration_change.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require_relative 'request'
+
+module Datadog
+  module Core
+    module Telemetry
+      module V2
+        # Telemetry 'app-client-configuration-change' event.
+        # This request should contain client library configuration that have changes since the app-started event.
+        class AppClientConfigurationChange < Request
+          def initialize(configuration_changes, origin: 'unknown')
+            super('app-client-configuration-change')
+
+            @configuration_changes = configuration_changes
+            @origin = origin
+          end
+
+          # @see [Request#to_h]
+          def to_h
+            super.merge(payload: payload)
+          end
+
+          private
+
+          def payload
+            {
+              configuration: @configuration_changes.map do |name, value|
+                {
+                  name: name,
+                  value: value,
+                  origin: @origin,
+                }
+              end
+            }
+          end
+        end
+      end
+    end
+  end
+end

lib/datadog/core/telemetry/v2/request.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    module Telemetry
+      module V2
+        # Base request object for Telemetry V2.
+        #
+        # `#to_h` is the main API, which returns a Ruby
+        # Hash that will be serialized as JSON.
+        class Request
+          # @param [String] request_type the Telemetry request type, which dictates how the Hash payload should be processed
+          def initialize(request_type)
+            @request_type = request_type
+          end
+
+          # Converts this request to a Hash that will
+          # be serialized as JSON.
+          # @return [Hash]
+          def to_h
+            {
+              request_type: @request_type
+            }
+          end
+        end
+      end
+    end
+  end
+end

lib/datadog/core/transport/http/config.rb

@@ -126,6 +126,16 @@ def initialize(http_response, options = {}) # rubocop:disable Metrics/AbcSize,Me
               end.freeze
             end
 
+            def inspect
+              "#{super}, #{
+                {
+                  roots: @roots,
+                  targets: @targets,
+                  target_files: @target_files,
+                  client_configs: @client_configs,
+                }}"
+            end
+
             # When an expected key is missing
             class KeyError < StandardError
               def initialize(key)