Merge branch 'master' into alexjf/prof-8543-ruby-heap-profiling-poc

.github/PULL_REQUEST_TEMPLATE.md

@@ -5,6 +5,17 @@ for guidance on how to set up your development environment,
 run the test suite, write new integrations, and more.
 -->
 
+**2.0 Upgrade Guide notes**
+<!--
+(If this PR is for 1.x, please delete this section)
+A public facing description that will go into the [upgrade guide](https://github.com/DataDog/dd-trace-rb/blob/master/docs/UpgradeGuide.md) for this change.
+We already know what the PR does (from the PR title),
+but users should know either:
+* A migration path for this change. In other words, how to continue using their application without behavior changes
+in 2.0 (e.g. environment variable 'X' renamed to 'Y').
+* That there's no alternative; we completely dropped support for this feature.
+-->
+
 **What does this PR do?**
 <!-- A brief description of the change being made with this pull request. -->
 

.github/workflows/system-tests.yml

@@ -170,6 +170,9 @@ jobs:
           - library: ruby
             app: rails70
             scenario: APPSEC_AUTO_EVENTS_EXTENDED
+          - library: ruby
+            app: rails70
+            scenario: APPSEC_API_SECURITY
           - library: ruby
             app: rack
             scenario: APPSEC_RULES_MONITORING_WITH_ERRORS

.gitlab/benchmarks.yml

@@ -128,16 +128,6 @@ candidate-tracer-appsec-with-api-security:
     DD_EXPERIMENTAL_API_SECURITY_ENABLED: "true"
     DD_API_SECURITY_REQUEST_SAMPLE_RATE: "1.0"
 
-candidate-tracer-appsec-with-api-security-without-response-body:
-  extends: .benchmarks
-  variables:
-    DD_BENCHMARKS_CONFIGURATION: "tracing-and-appsec-with-api-security-without-response-body"
-    DD_PROFILING_ENABLED: "false"
-    DD_APPSEC_ENABLED: "true"
-    DD_EXPERIMENTAL_API_SECURITY_ENABLED: "true"
-    DD_API_SECURITY_REQUEST_SAMPLE_RATE: "1.0"
-    DD_API_SECURITY_PARSE_RESPONSE_BODY: "false"
-
 # -----------------------------------------------------
 # Microbenchmarks that report to statsd
 # -----------------------------------------------------

CHANGELOG.md

@@ -2,6 +2,44 @@
 
 ## [Unreleased]
 
+## [1.16.2] - 2023-11-10
+
+This release reverts a change to appsec response body parsing that was introduced in [1.16.0 ](https://github.com/DataDog/dd-trace-rb/releases/tag/v1.16.0) that may cause memory leaks.
+
+### Fixed
+*  Appsec: [Revert parse response body fix introduced in 1.16.0](https://github.com/DataDog/dd-trace-rb/pull/3153) ([#3252][])
+
+## [1.16.1] - 2023-11-08
+
+### Fixed
+
+* Tracing: Fix `concurrent-ruby` future propagation without `active_trace` ([#3242][])
+* Tracing: Fix host injection error handling  ([#3240][])
+
+## [1.16.0] - 2023-11-03
+
+**This release includes a security change for the Tracing Redis integration:**
+
+Currently, the Datadog Agent removes command arguments from the resource name. However there are cases, like Redis compressed keys, where this obfuscation cannot correctly remove command arguments. To safeguard that situation, the resource name set by the tracer will only be the command (e.g. `SET`) with no arguments. To retain the previous behavior and keep arguments in the span resource, with the potential risk of some command arguments not being fully obfuscated, set ``DD_REDIS_COMMAND_ARGS=true`` or set the option `c.instrument :redis, command_args: true`.
+
+### Added
+
+* Tracing: Propagate trace through `Concurrent::Promises.future` ([#1522][])
+* Core: Name `Datadog::Core::Remote::Worker` thread ([#3207][])
+
+### Changed
+
+* Tracing: Redis - Omit command arguments from span.resource by default ([#3235][])
+* Ci-app: Bump `datadog-ci` dependency from 0.2.0 to 0.3.0 ([#3223][])
+
+### Fixed
+
+* Appsec: ASM parse response body  ([#3153][])
+* Appsec: ASM make sure to append content type and length information ([#3204][])
+* Appsec: Make sure function that checks content-type header value accepts nil content-type header value ([#3234][])
+* Profiling: Shut down profiler if any components failed ([#3197][])
+* Tracing: Fix `ActiveSupport` instrumentation of custom cache stores ([#3206][])
+
 ## [1.15.0] - 2023-10-09
 
 ### Highlights
@@ -2595,7 +2633,11 @@ Release notes: https://github.com/DataDog/dd-trace-rb/releases/tag/v0.3.1
 
 Git diff: https://github.com/DataDog/dd-trace-rb/compare/v0.3.0...v0.3.1
 
-[Unreleased]: https://github.com/DataDog/dd-trace-rb/compare/v1.15.0...master
+
+[Unreleased]: https://github.com/DataDog/dd-trace-rb/compare/v1.16.2...master
+[1.16.2]: https://github.com/DataDog/dd-trace-rb/compare/v1.16.1...v1.16.2
+[1.16.1]: https://github.com/DataDog/dd-trace-rb/compare/v1.16.0...v1.16.1
+[1.16.0]: https://github.com/DataDog/dd-trace-rb/compare/v1.15.0...v1.16.0
 [1.15.0]: https://github.com/DataDog/dd-trace-rb/compare/v1.14.0...v1.15.0
 [1.14.0]: https://github.com/DataDog/dd-trace-rb/compare/v1.13.1...1.14.0
 [1.13.1]: https://github.com/DataDog/dd-trace-rb/compare/v1.13.0...1.13.1
@@ -3333,6 +3375,7 @@ Git diff: https://github.com/DataDog/dd-trace-rb/compare/v0.3.0...v0.3.1
 [#1509]: https://github.com/DataDog/dd-trace-rb/issues/1509
 [#1510]: https://github.com/DataDog/dd-trace-rb/issues/1510
 [#1511]: https://github.com/DataDog/dd-trace-rb/issues/1511
+[#1522]: https://github.com/DataDog/dd-trace-rb/issues/1522
 [#1523]: https://github.com/DataDog/dd-trace-rb/issues/1523
 [#1524]: https://github.com/DataDog/dd-trace-rb/issues/1524
 [#1529]: https://github.com/DataDog/dd-trace-rb/issues/1529
@@ -3781,6 +3824,7 @@ Git diff: https://github.com/DataDog/dd-trace-rb/compare/v0.3.0...v0.3.1
 [#3148]: https://github.com/DataDog/dd-trace-rb/issues/3148
 [#3150]: https://github.com/DataDog/dd-trace-rb/issues/3150
 [#3152]: https://github.com/DataDog/dd-trace-rb/issues/3152
+[#3153]: https://github.com/DataDog/dd-trace-rb/issues/3153
 [#3158]: https://github.com/DataDog/dd-trace-rb/issues/3158
 [#3162]: https://github.com/DataDog/dd-trace-rb/issues/3162
 [#3163]: https://github.com/DataDog/dd-trace-rb/issues/3163
@@ -3796,6 +3840,16 @@ Git diff: https://github.com/DataDog/dd-trace-rb/compare/v0.3.0...v0.3.1
 [#3188]: https://github.com/DataDog/dd-trace-rb/issues/3188
 [#3189]: https://github.com/DataDog/dd-trace-rb/issues/3189
 [#3190]: https://github.com/DataDog/dd-trace-rb/issues/3190
+[#3197]: https://github.com/DataDog/dd-trace-rb/issues/3197
+[#3204]: https://github.com/DataDog/dd-trace-rb/issues/3204
+[#3206]: https://github.com/DataDog/dd-trace-rb/issues/3206
+[#3207]: https://github.com/DataDog/dd-trace-rb/issues/3207
+[#3223]: https://github.com/DataDog/dd-trace-rb/issues/3223
+[#3234]: https://github.com/DataDog/dd-trace-rb/issues/3234
+[#3235]: https://github.com/DataDog/dd-trace-rb/issues/3235
+[#3240]: https://github.com/DataDog/dd-trace-rb/issues/3240
+[#3242]: https://github.com/DataDog/dd-trace-rb/issues/3242
+[#3252]: https://github.com/DataDog/dd-trace-rb/issues/3252
 [@AdrianLC]: https://github.com/AdrianLC
 [@Azure7111]: https://github.com/Azure7111
 [@BabyGroot]: https://github.com/BabyGroot
@@ -3947,4 +4001,4 @@ Git diff: https://github.com/DataDog/dd-trace-rb/compare/v0.3.0...v0.3.1
 [@y-yagi]: https://github.com/y-yagi
 [@yujideveloper]: https://github.com/yujideveloper
 [@yukimurasawa]: https://github.com/yukimurasawa
-[@zachmccormick]: https://github.com/zachmccormick
+[@zachmccormick]: https://github.com/zachmccormick

Rakefile

@@ -65,8 +65,8 @@ TEST_METADATA = {
     'relational_db' => '✅ 2.1 / ✅ 2.2 / ✅ 2.3 / ✅ 2.4 / ✅ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ jruby'
   },
   'elasticsearch' => {
-    'http'        => '✅ 2.1 / ✅ 2.2 / ❌ 2.3 / ❌ 2.4 / ✅ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ jruby',
-    'contrib-old' => '❌ 2.1 / ❌ 2.2 / ✅ 2.3 / ✅ 2.4 / ✅ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ jruby'
+    'elasticsearch-7' => '✅ 2.1 / ✅ 2.2 / ✅ 2.3 / ✅ 2.4 / ✅ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ jruby',
+    'elasticsearch-8' => '❌ 2.1 / ❌ 2.2 / ❌ 2.3 / ❌ 2.4 / ✅ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ jruby'
   },
   'ethon' => {
     'http' => '✅ 2.1 / ✅ 2.2 / ✅ 2.3 / ✅ 2.4 / ✅ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ jruby'
@@ -602,58 +602,42 @@ namespace :changelog do
   end
 end
 
-native_exts = []
-
-native_exts << Rake::ExtensionTask.new("ddtrace_profiling_native_extension.#{RUBY_VERSION}_#{RUBY_PLATFORM}") do |ext|
-  ext.ext_dir = 'ext/ddtrace_profiling_native_extension'
-end
-
-native_exts << Rake::ExtensionTask.new("ddtrace_profiling_loader.#{RUBY_VERSION}_#{RUBY_PLATFORM}") do |ext|
-  ext.ext_dir = 'ext/ddtrace_profiling_loader'
-end
-
-NATIVE_CLEAN = ::Rake::FileList['ext/**/*.o', 'ext/**/*.bundle']
-namespace :native_dev do
-  def generate_ext_makefile(ext)
-    cmd = ext.make_makefile_cmd(Dir.pwd, ext.ext_dir, "#{ext.ext_dir}/#{ext.config_script}", nil)
-    puts cmd
-    chdir ext.ext_dir do
-      sh(*cmd)
-    end
-  end
+NATIVE_EXTS = [
+  Rake::ExtensionTask.new("ddtrace_profiling_native_extension.#{RUBY_VERSION}_#{RUBY_PLATFORM}") do |ext|
+    ext.ext_dir = 'ext/ddtrace_profiling_native_extension'
+  end,
 
-  def generate_ext_compile_commands(ext)
-    chdir ext.ext_dir do
-      sh('make clean; bear -- make; make clean')
-    end
+  Rake::ExtensionTask.new("ddtrace_profiling_loader.#{RUBY_VERSION}_#{RUBY_PLATFORM}") do |ext|
+    ext.ext_dir = 'ext/ddtrace_profiling_loader'
   end
+].freeze
 
-  compile_commands_tasks = native_exts.map do |ext|
-    makefile_task = file "#{ext.ext_dir}/Makefile" do
-      generate_ext_makefile(ext)
-    end
-
-    NATIVE_CLEAN << makefile_task.name
-    NATIVE_CLEAN << "#{ext.ext_dir}/extconf.h"
-    NATIVE_CLEAN << "#{ext.ext_dir}/mkmf.log"
-
-    compile_commands_task = file "#{ext.ext_dir}/compile_commands.json" => makefile_task do
-      generate_ext_compile_commands(ext)
+NATIVE_CLEAN = ::Rake::FileList[]
+namespace :native_dev do
+  compile_commands_tasks = NATIVE_EXTS.map do |ext|
+    tmp_dir_dd_native_dev = "#{ext.tmp_dir}/dd_native_dev"
+    directory tmp_dir_dd_native_dev
+    NATIVE_CLEAN << tmp_dir_dd_native_dev
+
+    compile_commands_task = file "#{ext.ext_dir}/compile_commands.json" => [tmp_dir_dd_native_dev] do |t|
+      puts "Generating #{t.name}"
+      root_dir = Dir.pwd
+      cmd = ext.make_makefile_cmd(root_dir, tmp_dir_dd_native_dev, "#{ext.ext_dir}/#{ext.config_script}", nil)
+      abs_ext_dir = (Pathname.new(root_dir) + ext.ext_dir).realpath
+      chdir tmp_dir_dd_native_dev do
+        sh(*cmd)
+        sh('make clean; bear -- make; make clean')
+        cp('compile_commands.json', "#{abs_ext_dir}/compile_commands.json")
+      end
     end
 
     NATIVE_CLEAN << compile_commands_task.name
+
     compile_commands_task
   end
 
   desc 'Setup dev environment for native extensions.'
-  task setup: compile_commands_tasks do
-    puts 'Setting up native dev env'
-  end
-
-  desc 'Remove files generated for dev setup.'
-  task :clean do
-    Rake::Cleaner.cleanup_files(NATIVE_CLEAN)
-  end
+  task setup: compile_commands_tasks
 
   CLEAN.concat(NATIVE_CLEAN)
 end

appraisal/jruby-9.2.rb

@@ -173,7 +173,6 @@
 end
 
 appraise 'http' do
-  gem 'elasticsearch'
   # Workaround bundle of JRuby/ethon issues:
   # * ethon 0.15.0 is incompatible with most JRuby 9.2 versions (fixed in 9.2.20.0),
   #   see https://github.com/typhoeus/ethon/issues/205
@@ -198,6 +197,12 @@
   end
 end
 
+[7, 8].each do |n|
+  appraise "elasticsearch-#{n}" do
+    gem 'elasticsearch', "~> #{n}"
+  end
+end
+
 appraise 'relational_db' do
   gem 'activerecord', '~> 5'
   gem 'delayed_job'
@@ -265,7 +270,6 @@
 
 appraise 'contrib-old' do
   gem 'dalli', '< 3.0.0'
-  gem 'elasticsearch', '< 8.0.0' # Dependency elasticsearch-transport renamed to elastic-transport in >= 8.0
   gem 'faraday', '0.17'
   gem 'graphql', '~> 1.12.0', '< 2.0' # TODO: Support graphql 1.13.x
   gem 'presto-client', '>= 0.5.14' # Renamed to trino-client in >= 1.0

appraisal/jruby-9.3.rb

@@ -153,7 +153,6 @@
 end
 
 appraise 'http' do
-  gem 'elasticsearch'
   gem 'ethon'
   gem 'excon'
   gem 'faraday'
@@ -170,6 +169,12 @@
   end
 end
 
+[7, 8].each do |n|
+  appraise "elasticsearch-#{n}" do
+    gem 'elasticsearch', "~> #{n}"
+  end
+end
+
 appraise 'relational_db' do
   gem 'activerecord', '~> 6.0.0'
   gem 'delayed_job'
@@ -236,7 +241,6 @@
 
 appraise 'contrib-old' do
   gem 'dalli', '< 3.0.0'
-  gem 'elasticsearch', '< 8.0.0' # Dependency elasticsearch-transport renamed to elastic-transport in >= 8.0
   gem 'faraday', '0.17'
   gem 'graphql', '~> 1.12.0', '< 2.0' # TODO: Support graphql 1.13.x
   gem 'presto-client', '>= 0.5.14' # Renamed to trino-client in >= 1.0

appraisal/jruby-9.4.rb

@@ -59,7 +59,6 @@
 end
 
 appraise 'http' do
-  gem 'elasticsearch'
   gem 'ethon'
   gem 'excon'
   gem 'faraday'
@@ -76,6 +75,12 @@
   end
 end
 
+[7, 8].each do |n|
+  appraise "elasticsearch-#{n}" do
+    gem 'elasticsearch', "~> #{n}"
+  end
+end
+
 appraise 'relational_db' do
   gem 'activerecord', '~> 7'
   gem 'delayed_job'
@@ -141,7 +146,6 @@
 
 appraise 'contrib-old' do
   gem 'dalli', '< 3.0.0'
-  gem 'elasticsearch', '< 8.0.0' # Dependency elasticsearch-transport renamed to elastic-transport in >= 8.0
   gem 'graphql', '~> 1.12.0', '< 2.0' # TODO: Support graphql 1.13.x
   gem 'presto-client', '>= 0.5.14' # Renamed to trino-client in >= 1.0
 

appraisal/ruby-2.1.rb

@@ -68,7 +68,6 @@
 end
 
 appraise 'http' do
-  gem 'elasticsearch'
   gem 'faraday'
   gem 'multipart-post', '~> 2.1.1' # Compatible with faraday 0.x
   gem 'ethon'
@@ -79,6 +78,11 @@
   gem 'typhoeus'
 end
 
+appraise 'elasticsearch-7' do
+  gem 'elasticsearch', '~> 7'
+  gem 'multipart-post', '~> 2.1.1' # Compatible with faraday 0.x
+end
+
 appraise 'relational_db' do
   gem 'activerecord', '3.2.22.5'
   gem 'activerecord-mysql-adapter'

appraisal/ruby-2.2.rb

@@ -130,7 +130,6 @@
 end
 
 appraise 'http' do
-  gem 'elasticsearch'
   gem 'ethon'
   gem 'excon'
   gem 'faraday'
@@ -141,6 +140,11 @@
   gem 'typhoeus'
 end
 
+appraise 'elasticsearch-7' do
+  gem 'elasticsearch', '~> 7'
+  gem 'multipart-post', '~> 2.1.1' # Compatible with faraday 0.x
+end
+
 appraise 'relational_db' do
   gem 'activerecord', '~> 5'
   gem 'delayed_job'

appraisal/ruby-2.3.rb

@@ -150,6 +150,10 @@
   gem 'typhoeus'
 end
 
+appraise 'elasticsearch-7' do
+  gem 'elasticsearch', '~> 7'
+end
+
 appraise 'relational_db' do
   gem 'activerecord', '~> 5'
   gem 'delayed_job'
@@ -220,7 +224,6 @@
 end
 
 appraise 'contrib-old' do
-  gem 'elasticsearch', '< 8.0.0' # Dependency elasticsearch-transport renamed to elastic-transport in >= 8.0
   gem 'faraday', '0.17'
   gem 'presto-client', '>= 0.5.14' # Renamed to trino-client in >= 1.0
 end