Upgrade to OpenSSL 3 in Agent 7, upgrade Python 3 to 3.9.17

.gitlab-ci.yml

@@ -145,12 +145,12 @@ variables:
   # Build images versions
   # To use images from datadog-agent-buildimages dev branches, set the corresponding
   # SUFFIX variable to _test_only
-  DATADOG_AGENT_BUILDIMAGES_SUFFIX: ""
-  DATADOG_AGENT_BUILDIMAGES: v16026304-782441d
-  DATADOG_AGENT_WINBUILDIMAGES_SUFFIX: ""
-  DATADOG_AGENT_WINBUILDIMAGES: v16026304-782441d
-  DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX: ""
-  DATADOG_AGENT_ARMBUILDIMAGES: v16026304-782441d
+  DATADOG_AGENT_BUILDIMAGES_SUFFIX: "_test_only"
+  DATADOG_AGENT_BUILDIMAGES: v16547699-65f0518
+  DATADOG_AGENT_WINBUILDIMAGES_SUFFIX: "_test_only"
+  DATADOG_AGENT_WINBUILDIMAGES: v16547699-65f0518
+  DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX: "_test_only"
+  DATADOG_AGENT_ARMBUILDIMAGES: v16547699-65f0518
   DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX: ""
   DATADOG_AGENT_SYSPROBE_BUILDIMAGES: v16026304-782441d
   DATADOG_AGENT_NIKOS_BUILDIMAGES_SUFFIX: ""

omnibus/config/projects/agent.rb

@@ -192,7 +192,7 @@
     "#{install_dir}\\bin\\agent\\ddtray.exe",
     "#{install_dir}\\embedded3\\python.exe",
     "#{install_dir}\\embedded3\\\\python3.dll",
-    "#{install_dir}\\embedded3\\\\python38.dll",
+    "#{install_dir}\\embedded3\\\\python39.dll",
     "#{install_dir}\\embedded3\\\\pythonw.exe"
   ]
   if with_python_runtime? '2'

omnibus/config/software/datadog-agent-finalize.rb

@@ -78,7 +78,7 @@
                 link "#{install_dir}/embedded/bin/python3", "#{install_dir}/embedded/bin/python"
 
                 delete "#{install_dir}/embedded/bin/2to3"
-                link "#{install_dir}/embedded/bin/2to3-3.8", "#{install_dir}/embedded/bin/2to3"
+                link "#{install_dir}/embedded/bin/2to3-3.9", "#{install_dir}/embedded/bin/2to3"
             end
         end
 

omnibus/config/software/datadog-agent-integrations-py2.rb

@@ -349,7 +349,7 @@
     cache_bucket = ENV.fetch('INTEGRATION_WHEELS_CACHE_BUCKET', '')
     cache_branch = `cd .. && inv release.get-release-json-value base_branch`.strip
     # On windows, `aws` actually executes Ruby's AWS SDK, but we want the Python one
-    awscli = if windows? then '"c:\Program files\python38\scripts\aws"' else 'aws' end
+    awscli = if windows? then '"c:\Program files\python39\scripts\aws"' else 'aws' end
     if cache_bucket != ''
       mkdir cached_wheels_dir
       command "inv -e agent.get-integrations-from-cache " \

omnibus/config/software/datadog-agent-integrations-py3.rb

@@ -42,10 +42,10 @@
 end
 
 relative_path 'integrations-core'
-whitelist_file "embedded/lib/python3.8/site-packages/.libsaerospike"
-whitelist_file "embedded/lib/python3.8/site-packages/aerospike.libs"
-whitelist_file "embedded/lib/python3.8/site-packages/psycopg2"
-whitelist_file "embedded/lib/python3.8/site-packages/pymqi"
+whitelist_file "embedded/lib/python3.9/site-packages/.libsaerospike"
+whitelist_file "embedded/lib/python3.9/site-packages/aerospike.libs"
+whitelist_file "embedded/lib/python3.9/site-packages/psycopg2"
+whitelist_file "embedded/lib/python3.9/site-packages/pymqi"
 
 source git: 'https://github.com/DataDog/integrations-core.git'
 
@@ -358,7 +358,7 @@
     cache_bucket = ENV.fetch('INTEGRATION_WHEELS_CACHE_BUCKET', '')
     cache_branch = `cd .. && inv release.get-release-json-value base_branch`.strip
     # On windows, `aws` actually executes Ruby's AWS SDK, but we want the Python one
-    awscli = if windows? then '"c:\Program files\python38\scripts\aws"' else 'aws' end
+    awscli = if windows? then '"c:\Program files\python39\scripts\aws"' else 'aws' end
     if cache_bucket != ''
       mkdir cached_wheels_dir
       command "inv -e agent.get-integrations-from-cache " \
@@ -481,7 +481,7 @@
       if windows?
         patch :source => "remove-maxfile-maxpath-psutil.patch", :target => "#{python_3_embedded}/Lib/site-packages/psutil/__init__.py"
       else
-        patch :source => "remove-maxfile-maxpath-psutil.patch", :target => "#{install_dir}/embedded/lib/python3.8/site-packages/psutil/__init__.py"
+        patch :source => "remove-maxfile-maxpath-psutil.patch", :target => "#{install_dir}/embedded/lib/python3.9/site-packages/psutil/__init__.py"
       end
 
       # Run pip check to make sure the agent's python environment is clean, all the dependencies are compatible

omnibus/config/software/libkrb5.rb

@@ -1,5 +1,12 @@
 name "libkrb5"
-default_version "1.18.3"
+default_version "1.20.1"
+
+dependency ENV["OMNIBUS_OPENSSL_SOFTWARE"] || "openssl"
+
+version "1.20.1" do
+  source url: "https://kerberos.org/dist/krb5/1.20/krb5-1.20.1.tar.gz"
+  source sha256: "704aed49b19eb5a7178b34b2873620ec299db08752d6a8574f95d41879ab8851"
+end
 
 version "1.18.3" do
   source url: "https://kerberos.org/dist/krb5/1.18/krb5-1.18.3.tar.gz"

omnibus/config/software/python3.rb

@@ -1,21 +1,21 @@
 name "python3"
 
 if ohai["platform"] != "windows"
-  default_version "3.8.16"
+  default_version "3.9.16"
 
   dependency "libxcrypt"
   dependency "libffi"
   dependency "ncurses"
   dependency "zlib"
-  dependency "openssl"
+  dependency ENV["OMNIBUS_OPENSSL_SOFTWARE"] || "openssl"
   dependency "pkg-config"
   dependency "bzip2"
   dependency "libsqlite3"
   dependency "liblzma"
   dependency "libyaml"
 
   source :url => "https://python.org/ftp/python/#{version}/Python-#{version}.tgz",
-         :sha256 => "71ca9d935637ed2feb59e90a368361dc91eca472a90acb1d344a2e8178ccaf10"
+         :sha256 => "1ad539e9dbd2b42df714b69726e0693bc6b9d2d2c8e91c2e43204026605140c5"
 
   relative_path "Python-#{version}"
 
@@ -57,7 +57,7 @@
     command python_configure.join(" "), :env => env
     command "make -j #{workers}", :env => env
     command "make install", :env => env
-    delete "#{install_dir}/embedded/lib/python3.8/test"
+    delete "#{install_dir}/embedded/lib/python3.9/test"
 
     # There exists no configure flag to tell Python to not compile readline support :(
     major, minor, bugfix = version.split(".")
@@ -68,7 +68,7 @@
   end
 
 else
-  default_version "3.8.16-2609a9b"
+  default_version "3.9.16-b7f54e0"
   dependency "vc_redist_14"
 
   if windows_arch_i386?
@@ -80,7 +80,7 @@
 
     # note that startring with 3.7.3 on Windows, the zip should be created without the built-in pip
     source :url => "https://dd-agent-omnibus.s3.amazonaws.com/python-windows-#{version}-x64.zip",
-         :sha256 => "E93C7A7290F422FDC09131B01DCE1F9FD94DC5273F26149FCDF8CC6B26454DE1".downcase
+           :sha256 => "DFF249E438372194588594DEC8053B19B7A472A8C880D36FFAA1E0CC5E914BAA".downcase
 
   end
   vcrt140_root = "#{Omnibus::Config.source_dir()}/vc_redist_140/expanded"

omnibus/config/software/xmlsec.rb

@@ -23,10 +23,10 @@
 
 dependency "libxml2"
 dependency "libxslt"
-dependency "openssl"
 dependency "libtool"
 dependency "libgcrypt"
 dependency "gnutls"
+dependency ENV["OMNIBUS_OPENSSL_SOFTWARE"] || "openssl"
 
 version("1.2.37") { source sha256: "5f8dfbcb6d1e56bddd0b5ec2e00a3d0ca5342a9f57c24dffde5c796b2be2871c" }
 

release.json

@@ -6,23 +6,23 @@
     },
     "nightly": {
         "INTEGRATIONS_CORE_VERSION": "master",
-        "OMNIBUS_SOFTWARE_VERSION": "master",
+        "OMNIBUS_SOFTWARE_VERSION": "slavek.kabrda/openssl3",
         "OMNIBUS_RUBY_VERSION": "datadog-5.5.0",
         "JMXFETCH_VERSION": "0.47.9",
         "JMXFETCH_HASH": "fb5c3fc2fb42db1ee5c3a7c6187b316d79d69c50b967db0b7ffde590622d0395",
-        "MACOS_BUILD_VERSION": "master",
+        "MACOS_BUILD_VERSION": "slavek.kabrda/upgrade-python-3.9",
         "WINDOWS_DDNPM_DRIVER": "release-signed",
         "WINDOWS_DDNPM_VERSION": "2.4.1",
         "WINDOWS_DDNPM_SHASUM": "f12af44306eac3ea15828fd12c24d44ae519692a94a0f1f5d4fa868c3e596b07",
         "SECURITY_AGENT_POLICIES_VERSION": "master"
     },
     "nightly-a7": {
         "INTEGRATIONS_CORE_VERSION": "master",
-        "OMNIBUS_SOFTWARE_VERSION": "master",
+        "OMNIBUS_SOFTWARE_VERSION": "slavek.kabrda/openssl3",
         "OMNIBUS_RUBY_VERSION": "datadog-5.5.0",
         "JMXFETCH_VERSION": "0.47.9",
         "JMXFETCH_HASH": "fb5c3fc2fb42db1ee5c3a7c6187b316d79d69c50b967db0b7ffde590622d0395",
-        "MACOS_BUILD_VERSION": "master",
+        "MACOS_BUILD_VERSION": "slavek.kabrda/upgrade-python-3.9",
         "WINDOWS_DDNPM_DRIVER": "release-signed",
         "WINDOWS_DDNPM_VERSION": "2.4.1",
         "WINDOWS_DDNPM_SHASUM": "f12af44306eac3ea15828fd12c24d44ae519692a94a0f1f5d4fa868c3e596b07",

releasenotes/notes/openssl3-python39-274b0b0153ee32e8.yaml

@@ -0,0 +1,12 @@
+# Each section from every release note are combined when the
+# CHANGELOG.rst is rendered. So the text needs to be worded so that
+# it does not depend on any information only available in another
+# section. This may mean repeating some details, but each section
+# must be readable independently of the other.
+#
+# Each section note must be formatted as reStructuredText.
+---
+upgrade:
+  - |
+    Embedded Python 3 interpreter is upgraded to 3.9.16 in both Agent 6 and
+    Agent 7. Embedded OpenSSL is upgraded to 3.0.9 in Agent 7.

tasks/agent.py

@@ -455,6 +455,9 @@ def get_omnibus_env(
     if go_mod_cache:
         env['OMNIBUS_GOMODCACHE'] = go_mod_cache
 
+    if int(major_version) > 6:
+        env['OMNIBUS_OPENSSL_SOFTWARE'] = 'openssl3'
+
     integrations_core_version = os.environ.get('INTEGRATIONS_CORE_VERSION')
     # Only overrides the env var if the value is a non-empty string.
     if integrations_core_version:

test/kitchen/test/integration/common/rspec_datadog/spec_helper.rb

@@ -492,6 +492,11 @@ def is_ng_installer()
       expect(is_signed).to be_truthy
 
       program_files = safe_program_files
+      # The glob in the bottom makes sure we add the full Python version dll, e.g.
+      # python39.dll or python310.dll. Thanks to this, we don't have to fix this test case
+      # manually when we upgrade the Python version. Additionally, some scenarios like
+      # win-upgrade-rollback might test two Agents with two different Python versions,
+      # so hardcoding just one dll wouldn't work in these cases.
       verify_signature_files = [
         # TODO: Uncomment this when we start shipping the security agent on Windows
         # "#{program_files}\\DataDog\\Datadog Agent\\bin\\agent\\security-agent.exe",
@@ -503,8 +508,7 @@ def is_ng_installer()
         "#{program_files}\\DataDog\\Datadog Agent\\embedded3\\python.exe",
         "#{program_files}\\DataDog\\Datadog Agent\\embedded3\\pythonw.exe",
         "#{program_files}\\DataDog\\Datadog Agent\\embedded3\\python3.dll",
-        "#{program_files}\\DataDog\\Datadog Agent\\embedded3\\python38.dll"
-      ]
+      ] + Dir.glob("#{program_files}\\DataDog\\Datadog Agent\\embedded3\\python3?*.dll")
       libdatadog_agent_two = "#{program_files}\\DataDog\\Datadog Agent\\bin\\libdatadog-agent-two.dll"
       if File.file?(libdatadog_agent_two)
         verify_signature_files += [

tools/windows/DatadogAgentInstaller/WixSetup/Datadog/AgentBinaries.cs

@@ -25,7 +25,7 @@ public AgentBinaries(string binSource, string installerSource)
             {
                 $@"{installerSource}\embedded3\python.exe",
                 $@"{installerSource}\embedded3\python3.dll",
-                $@"{installerSource}\embedded3\python38.dll",
+                $@"{installerSource}\embedded3\python39.dll",
                 $@"{installerSource}\embedded3\pythonw.exe"
             };
             PythonTwoBinaries = new[]