Merge branch 'master' into sarahchen6/add-ruby-3.4-support

.circleci/config.yml

@@ -107,7 +107,7 @@ test_containers:
     image: memcached:1.5-alpine
   - &memcached_port 11211
   - &container_testagent
-    image: ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.12.0
+    image: ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.18.0
     name: testagent
     environment:
       - LOG_LEVEL=DEBUG

.github/PULL_REQUEST_TEMPLATE.md

@@ -5,6 +5,9 @@ for guidance on how to set up your development environment,
 run the test suite, write new integrations, and more.
 -->
 
+**Change log entry**
+<!-- If this is a customer-visible change, a brief summary to be placed into the change log. -->
+
 **What does this PR do?**
 <!-- A brief description of the change being made with this pull request. -->
 

.github/workflows/nix.yml

@@ -0,0 +1,55 @@
+name: Test Nix
+
+on:
+  push:
+    branches:
+      - "**"
+
+jobs:
+  test:
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+          - os: darwin
+            cpu: x86_64
+            base: macos-13 # always x86_64-darwin
+          - os: darwin
+            cpu: arm64
+            base: macos-14 # always arm64-darwin
+          - os: linux
+            cpu: x86_64
+            base: ubuntu-24.04 # always x86_64-linux-gnu
+          - os: linux
+            cpu: aarch64
+            base: arm-4core-linux-ubuntu24.04 # always aarch64-linux-gnu
+        nix:
+          - 24.05
+
+    name: Test Nix (${{ matrix.platform.cpu }}-${{ matrix.platform.os }}, ${{ matrix.nix }})
+    runs-on: ${{ matrix.platform.base }}
+
+    permissions:
+      contents: read
+      id-token: write
+
+    env:
+      SKIP_SIMPLECOV: 1
+      DD_INSTRUMENTATION_TELEMETRY_ENABLED: false
+      DD_REMOTE_CONFIGURATION_ENABLED: false
+
+    steps:
+      - name: Check CPU arch
+        run: |
+          test "$(uname -m)" = "${{ matrix.platform.cpu }}"
+      - uses: actions/checkout@v4
+      - uses: DeterminateSystems/nix-installer-action@main
+      - uses: DeterminateSystems/magic-nix-cache-action@main
+      - name: Print ruby version
+        run: |
+          nix develop --command which ruby
+          nix develop --command ruby --version
+      - name: Bundle install
+        run: nix develop --command bundle install
+      - name: Run spec:main
+        run: nix develop --command bundle exec rake spec:main

.github/workflows/system-tests.yml

@@ -12,8 +12,8 @@ env:
   REGISTRY: ghcr.io
   REPO: ghcr.io/datadog/dd-trace-rb
   ST_REF: main
-  FORCE_TESTS: -F tests/appsec/waf/test_addresses.py::Test_GraphQL -F tests/appsec/test_blocking_addresses.py::Test_BlockingGraphqlResolvers
-  FORCE_TESTS_SCENARIO: GRAPHQL_APPSEC
+  FORCE_TESTS: -F tests/appsec/test_asm_standalone.py
+  FORCE_TESTS_SCENARIO: APPSEC_STANDALONE
 
 jobs:
   build-harness:
@@ -199,6 +199,7 @@ jobs:
           - APPSEC_DISABLED
           - APPSEC_BLOCKING_FULL_DENYLIST
           - APPSEC_REQUEST_BLOCKING
+          - APPSEC_STANDALONE
         include:
           - library: ruby
             app: rack

.github/workflows/test-yjit.yaml

@@ -12,10 +12,20 @@ jobs:
           - '3.3'
           - '3.4'
           # ADD NEW RUBIES HERE
-    name: Test (${{ matrix.os }}, ${{ matrix.ruby }})
+        rubyopt:
+          - '--yjit'
+          - '--yjit --yjit-stats=quiet'
+        exclude:
+          # NOTE: `--yjit-stats=quiet` is only present in 3.3+.
+          #   We've had `--yjit-stats` since 3.0, but
+          #   1) it dumps stat info to stdout, and
+          #   2) none of our <3.3 tests require the additional stats
+          - ruby: '3.2'
+            rubyopt: '--yjit --yjit-stats=quiet'
+    name: Test YJIT (${{ matrix.os }}, ${{ matrix.ruby }} ${{ matrix.rubyopt }})
     runs-on: ${{ matrix.os }}
     env:
-      RUBYOPT: "--yjit"
+      RUBYOPT: ${{ matrix.rubyopt }}
       SKIP_SIMPLECOV: 1
       DD_INSTRUMENTATION_TELEMETRY_ENABLED: false
       DD_REMOTE_CONFIGURATION_ENABLED: false

.gitignore

@@ -38,7 +38,8 @@ ext/**/skipped_reason.txt
 # unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
 .rvmrc
 # Ignore local variables
-.envrc
+/.envrc
+/.direnv
 
 # lock files
 Gemfile.lock

.gitlab/install_datadog_deps.rb

@@ -70,7 +70,7 @@
 }
 
 [
-  'debase-ruby_core_source',
+  'datadog-ruby_core_source',
   'ffi',
   'libddwaf',
   'msgpack',

CHANGELOG.md

@@ -2,6 +2,33 @@
 
 ## [Unreleased]
 
+### Added
+
+* AppSec: Add Experimental Standalone AppSec Threats billing ([#3965][])
+
+## [2.4.0] - 2024-10-11
+
+### Added
+
+* Core: Allow changing sampling rate for customer defined tags and resources ([#3956][])
+* Profiling: Add GVL profiling for Ruby 3.2+ as a preview feature ([#3929][])
+* Profiling: Otel: Add preview support for correlating profiling with otel ruby gem ([#3984][])
+* Tracing: AppSec: Add http.route tag to Rails, Grape, and Sinatra integrations ([#3849][])
+* Tracing: Add capabilities to remote config: tracing sample rate, tracing logs injection, tracing http header tags ([#3888][])
+* AppSec: Add a force disable of AppSec feature when using Ruby >= 3.3 with old FFI gem version ([#3969][])
+
+### Changed
+
+* AppSec: Improve PII compliance ([#3857][])
+* AppSec: Integrations: Improve accuracy of login tracking for Devise ([#3867][])
+* Crashtracking feature is now disabled by default ([#3970][])
+
+### Fixed
+
+* AppSec: Integrations: Fix GraphQL instrumentation for query fragments ([#3887][])
+* Bug: Profiling: Fix (small) memory leak in profiler when forking ([#3852][])
+* Tracing: Integrations: Fix GraphQL integration reconfiguration ([#3859][])
+
 ## [2.3.0] - 2024-08-22
 
 ### Added
@@ -2962,7 +2989,8 @@ Release notes: https://github.com/DataDog/dd-trace-rb/releases/tag/v0.3.1
 Git diff: https://github.com/DataDog/dd-trace-rb/compare/v0.3.0...v0.3.1
 
 
-[Unreleased]: https://github.com/DataDog/dd-trace-rb/compare/v2.3.0...master
+[Unreleased]: https://github.com/DataDog/dd-trace-rb/compare/v2.4.0...master
+[2.4.0]: https://github.com/DataDog/dd-trace-rb/compare/v2.3.0...v2.4.0
 [2.3.0]: https://github.com/DataDog/dd-trace-rb/compare/v2.2.0...v2.3.0
 [2.2.0]: https://github.com/DataDog/dd-trace-rb/compare/v2.1.0...v2.2.0
 [2.1.0]: https://github.com/DataDog/dd-trace-rb/compare/v2.0.0...v2.1.0
@@ -4383,6 +4411,18 @@ Git diff: https://github.com/DataDog/dd-trace-rb/compare/v0.3.0...v0.3.1
 [#3837]: https://github.com/DataDog/dd-trace-rb/issues/3837
 [#3839]: https://github.com/DataDog/dd-trace-rb/issues/3839
 [#3841]: https://github.com/DataDog/dd-trace-rb/issues/3841
+[#3849]: https://github.com/DataDog/dd-trace-rb/issues/3849
+[#3852]: https://github.com/DataDog/dd-trace-rb/issues/3852
+[#3857]: https://github.com/DataDog/dd-trace-rb/issues/3857
+[#3859]: https://github.com/DataDog/dd-trace-rb/issues/3859
+[#3867]: https://github.com/DataDog/dd-trace-rb/issues/3867
+[#3887]: https://github.com/DataDog/dd-trace-rb/issues/3887
+[#3888]: https://github.com/DataDog/dd-trace-rb/issues/3888
+[#3929]: https://github.com/DataDog/dd-trace-rb/issues/3929
+[#3956]: https://github.com/DataDog/dd-trace-rb/issues/3956
+[#3969]: https://github.com/DataDog/dd-trace-rb/issues/3969
+[#3970]: https://github.com/DataDog/dd-trace-rb/issues/3970
+[#3984]: https://github.com/DataDog/dd-trace-rb/issues/3984
 [@AdrianLC]: https://github.com/AdrianLC
 [@Azure7111]: https://github.com/Azure7111
 [@BabyGroot]: https://github.com/BabyGroot

Gemfile

@@ -73,10 +73,8 @@ gem 'dogstatsd-ruby', '>= 3.3.0', '!= 5.0.0', '!= 5.0.1', '!= 5.1.0'
 if RUBY_PLATFORM != 'java'
   if RUBY_VERSION >= '2.7.0' # Bundler 1.x fails to find that versions >= 3.8.0 are not compatible because of binary gems
     gem 'google-protobuf', ['~> 3.0', '!= 3.7.0', '!= 3.7.1']
-  elsif RUBY_VERSION >= '2.3.0'
-    gem 'google-protobuf', ['~> 3.0', '!= 3.7.0', '!= 3.7.1', '< 3.19.2']
   else
-    gem 'google-protobuf', ['~> 3.0', '!= 3.7.0', '!= 3.7.1', '< 3.8.0']
+    gem 'google-protobuf', ['~> 3.0', '!= 3.7.0', '!= 3.7.1', '< 3.19.2']
   end
 end
 

Matrixfile

@@ -14,8 +14,9 @@
     '' => '✅ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ✅ jruby'
   },
   'profiling:main' => {
-    'opentelemetry' => '❌ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ❌ jruby',
-    '' => '✅ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ✅ jruby'
+    'opentelemetry'      => '❌ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ❌ jruby',
+    'opentelemetry_otlp' => '❌ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ❌ jruby',
+    ''                   => '✅ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ✅ jruby',
   },
   'profiling:ractors' => {
     '' => '❌ 2.5 / ❌ 2.6 / ❌ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ✅ jruby'
@@ -286,6 +287,8 @@
     'graphql-1.13' => '❌ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ✅ jruby',
   }
 }.each_with_object({}) do |(tasks, spec_metadata), hash|
+  # Explode arrays of task names into individual tasks
+  # e.g. ['rails', 'railsdisableenv'] => {'...'} becomes 'rails7' => {'...'}, 'railsdisableenv7' => {'...'}
   Array(tasks).each do |task|
     hash[task] = spec_metadata
   end

Steepfile

@@ -70,7 +70,6 @@ target :datadog do
   ignore 'lib/datadog/core/metrics/metric.rb'
   ignore 'lib/datadog/core/metrics/options.rb'
   ignore 'lib/datadog/core/pin.rb'
-  ignore 'lib/datadog/core/rate_limiter.rb'
   # steep fails in this file due to https://github.com/soutaro/steep/issues/1231
   ignore 'lib/datadog/core/remote/tie.rb'
   ignore 'lib/datadog/core/runtime/ext.rb'

appraisal/ruby-2.6.rb

@@ -232,6 +232,11 @@
   gem 'opentelemetry-sdk', '~> 1.1'
 end
 
+appraise 'opentelemetry_otlp' do
+  gem 'opentelemetry-sdk', '~> 1.1'
+  gem 'opentelemetry-exporter-otlp'
+end
+
 [3, 4, 5].each do |n|
   appraise "redis-#{n}" do
     gem 'redis', "~> #{n}"

appraisal/ruby-2.7.rb

@@ -235,6 +235,11 @@
   gem 'opentelemetry-sdk', '~> 1.1'
 end
 
+appraise 'opentelemetry_otlp' do
+  gem 'opentelemetry-sdk', '~> 1.1'
+  gem 'opentelemetry-exporter-otlp'
+end
+
 [3, 4, 5].each do |n|
   appraise "redis-#{n}" do
     gem 'redis', "~> #{n}"

appraisal/ruby-3.0.rb

@@ -162,6 +162,11 @@
   gem 'opentelemetry-sdk', '~> 1.1'
 end
 
+appraise 'opentelemetry_otlp' do
+  gem 'opentelemetry-sdk', '~> 1.1'
+  gem 'opentelemetry-exporter-otlp'
+end
+
 appraise 'contrib-old' do
   gem 'dalli', '< 3.0.0'
   gem 'presto-client', '>= 0.5.14' # Renamed to trino-client in >= 1.0

appraisal/ruby-3.1.rb

@@ -162,6 +162,11 @@
   gem 'opentelemetry-sdk', '~> 1.1'
 end
 
+appraise 'opentelemetry_otlp' do
+  gem 'opentelemetry-sdk', '~> 1.1'
+  gem 'opentelemetry-exporter-otlp'
+end
+
 appraise 'contrib-old' do
   gem 'dalli', '< 3.0.0'
   gem 'presto-client', '>= 0.5.14' # Renamed to trino-client in >= 1.0

appraisal/ruby-3.2.rb

@@ -162,6 +162,11 @@
   gem 'opentelemetry-sdk', '~> 1.1'
 end
 
+appraise 'opentelemetry_otlp' do
+  gem 'opentelemetry-sdk', '~> 1.1'
+  gem 'opentelemetry-exporter-otlp'
+end
+
 appraise 'contrib-old' do
   gem 'dalli', '< 3.0.0'
   gem 'presto-client', '>= 0.5.14' # Renamed to trino-client in >= 1.0

appraisal/ruby-3.3.rb

@@ -162,6 +162,11 @@
   gem 'opentelemetry-sdk', '~> 1.1'
 end
 
+appraise 'opentelemetry_otlp' do
+  gem 'opentelemetry-sdk', '~> 1.1'
+  gem 'opentelemetry-exporter-otlp'
+end
+
 appraise 'contrib-old' do
   gem 'dalli', '< 3.0.0'
   gem 'presto-client', '>= 0.5.14' # Renamed to trino-client in >= 1.0

appraisal/ruby-3.4.rb

@@ -163,6 +163,11 @@
   gem 'opentelemetry-sdk', '~> 1.1'
 end
 
+appraise 'opentelemetry_otlp' do
+  gem 'opentelemetry-sdk', '~> 1.1'
+  gem 'opentelemetry-exporter-otlp'
+end
+
 appraise 'contrib-old' do
   gem 'dalli', '< 3.0.0'
   gem 'presto-client', '>= 0.5.14' # Renamed to trino-client in >= 1.0