Merge branch 'main' into florentclarret/ruff

.circleci/config.yml

@@ -119,18 +119,6 @@ jobs:
           name: run integration tests
           command: inv -e integration-tests --race --remote-docker
 
-  release_note:
-    <<: *job_template
-    steps:
-      - restore_cache: *restore_source
-      - restore_cache: *restore_deps
-      - run:
-          command: pip install -r tasks/libs/requirements-github.txt
-          name: Install additional requirements
-      - run:
-          command: inv -e lint-releasenote
-          name: run PR check for release note
-
   go_linting:
     <<: *job_template
     resource_class: xlarge
@@ -187,20 +175,6 @@ jobs:
           #SC2059: Don't use variables in the printf format string. Use printf "..%s.." "$foo".
           command: shellcheck --severity=info -e SC2059 -e SC2028 --shell=bash ./cmd/**/*.sh ./omnibus/package-scripts/*/*
 
-  python_linting:
-    <<: *job_template
-    steps:
-      - restore_cache: *restore_source
-      - restore_cache: *restore_deps
-      - run:
-          name: setting env vars for click
-          command: |
-            echo 'export LC_ALL="C.UTF-8"' >> $BASH_ENV
-            echo 'export LANG="C.UTF-8"' >> $BASH_ENV
-      - run:
-          name: lint python files
-          command: inv -e linter.python
-
   component_linting:
     <<: *job_template
     steps:
@@ -280,13 +254,6 @@ workflows:
       - integration_tests:
           requires:
             - dependencies
-      - release_note:
-          filters:
-            branches:
-              ignore:
-                - main
-          requires:
-            - dependencies
       - go_linting:
           requires:
             - dependencies
@@ -302,9 +269,6 @@ workflows:
       - shell_linting:
           requires:
             - dependencies
-      - python_linting:
-          requires:
-            - dependencies
       - component_linting:
           requires:
             - dependencies

.github/CODEOWNERS

@@ -523,6 +523,7 @@
 /tasks/components.py                    @DataDog/agent-shared-components
 /tasks/components_templates             @DataDog/agent-shared-components
 /tasks/updater.py                       @DataDog/fleet
+/tasks/libs/omnibus_cache.py            @DataDog/agent-build-and-releases
 /tasks/installer.py                     @DataDog/fleet
 /test/                                  @DataDog/agent-developer-tools
 /test/benchmarks/                       @DataDog/agent-metrics-logs

.github/workflows/label-analysis.yml

@@ -20,10 +20,37 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
-      - name: Install Python dependencies
-        run: pip install -r tasks/requirements.txt
+      - name: Setup python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.11
+          cache: 'pip'
+          cache-dependency-path: '**/requirements*.txt'
+      - name: Install dependencies
+        run: pip install -r requirements.txt -r tasks/requirements.txt
       - name: Auto assign team label
         run: inv -e github.assign-team-label --pr-id='${{ github.event.pull_request.number }}'
+  release-note-check:
+    if: github.triggering_actor != 'dd-devflow[bot]'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Setup python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.11
+          cache: 'pip'
+          cache-dependency-path: '**/requirements*.txt'
+      - name: Install dependencies
+        run: pip install -r requirements.txt -r tasks/requirements.txt
+      - name: Check release note
+        env:
+          BRANCH_NAME: ${{ github.head_ref }}
+          PR_ID: ${{ github.event.pull_request.number }}
+        run: inv -e linter.releasenote
   fetch-labels:
     needs: assign-team-label
     if: github.triggering_actor != 'dd-devflow[bot]'
@@ -37,7 +64,7 @@ jobs:
           labels="$(gh pr view '${{ github.event.pull_request.number }}' --json labels --jq '[.labels[].name] | (join(" "))')"
           echo "Fetched labels for PR ${{github.event.number}}: $labels"
           echo "LABELS=$labels" >> "$GITHUB_OUTPUT"
-  team-label:
+  team-label-check:
     needs: fetch-labels
     if: github.triggering_actor != 'dd-devflow[bot]'
     runs-on: ubuntu-latest
@@ -58,7 +85,7 @@ jobs:
           exit 1
         env:
           LABELS: ${{ needs.fetch-labels.outputs.LABELS}}
-  skip-qa:
+  skip-qa-check:
     needs: fetch-labels
     if: github.triggering_actor != 'dd-devflow[bot]'
     runs-on: ubuntu-latest

.gitlab-ci.yml

@@ -149,6 +149,7 @@ variables:
   ## build to succeed with S3 caching disabled.
   S3_OMNIBUS_CACHE_BUCKET: dd-ci-datadog-agent-omnibus-cache-build-stable
   USE_S3_CACHING: --omnibus-s3-cache
+  OMNIBUS_GIT_CACHE_DIR: /tmp/omnibus-git-cache
   ## comment out the line below to disable integration wheels cache
   INTEGRATION_WHEELS_CACHE_BUCKET: dd-agent-omnibus
   S3_DD_AGENT_OMNIBUS_LLVM_URI: s3://dd-agent-omnibus/llvm

.gitlab/JOBOWNERS

@@ -132,7 +132,7 @@ new-e2e-windows-agent*            @DataDog/windows-agent
 new-e2e-orchestrator*             @DataDog/container-app
 e2e_pre_test*                     @DataDog/agent-developer-tools
 new-e2e-remote-config*            @DataDog/remote-config
-new-e2e-updater*                  @DataDog/fleet
+new-e2e-installer*                @DataDog/fleet
 
 # Kernel matrix testing
 upload_dependencies*              @DataDog/ebpf-platform

.gitlab/kitchen_testing/ubuntu.yml

@@ -39,3 +39,4 @@ kitchen_ubuntu_process_agent-a7:
   extends:
     - .kitchen_scenario_ubuntu_a7_x64
     - .kitchen_test_process_agent
+  allow_failure: true

.gitlab/package_build/installer.yml

@@ -21,7 +21,7 @@
     - chmod 0744 /tmp/system-probe/clang-bpf /tmp/system-probe/llc-bpf
     # NOTE: for now, we consider "ociru" to be a "redhat_target" in omnibus/lib/ostools.rb
     # if we ever start building on a different platform, that might need to change
-    - inv -e omnibus.build --release-version "$RELEASE_VERSION" --major-version "$AGENT_MAJOR_VERSION" --python-runtimes "$PYTHON_RUNTIMES" --base-dir $OMNIBUS_BASE_DIR  ${USE_S3_CACHING} --skip-deps --go-mod-cache="$GOPATH/pkg/mod" --system-probe-bin=/tmp/system-probe --host-distribution=ociru
+    - inv -e omnibus.build --release-version "$RELEASE_VERSION" --major-version "$AGENT_MAJOR_VERSION" --python-runtimes "$PYTHON_RUNTIMES" --base-dir $OMNIBUS_BASE_DIR  ${USE_S3_CACHING} --skip-deps --go-mod-cache="$GOPATH/pkg/mod" --system-probe-bin=/tmp/system-probe --host-distribution=ociru --install-directory="$INSTALL_DIR"
     - ls -la $OMNIBUS_PACKAGE_DIR
     - !reference [.upload_sbom_artifacts]
   variables:
@@ -94,7 +94,7 @@ datadog-agent-oci-arm64-a7:
     - rm -rf $OMNIBUS_PACKAGE_DIR/*
     # Artifacts and cache must live within project directory but we run omnibus in a neutral directory.
     # Thus, we move the artifacts at the end in a gitlab-friendly dir.
-    - inv -e omnibus.build --release-version "$RELEASE_VERSION" --base-dir $OMNIBUS_BASE_DIR ${USE_S3_CACHING} --skip-deps --go-mod-cache="$GOPATH/pkg/mod" --target-project="installer"
+    - inv -e omnibus.build --release-version "$RELEASE_VERSION" --base-dir $OMNIBUS_BASE_DIR ${USE_S3_CACHING} --skip-deps --go-mod-cache="$GOPATH/pkg/mod" --target-project="installer" ${INSTALL_DIR_PARAM}
     - ls -la $OMNIBUS_PACKAGE_DIR
     - $S3_CP_CMD $OMNIBUS_PACKAGE_DIR/datadog-installer-*-${PACKAGE_ARCH}.tar.xz $S3_ARTIFACTS_URI/$DESTINATION_FILE
     - !reference [.upload_sbom_artifacts]
@@ -143,6 +143,8 @@ installer-amd64-oci:
   before_script:
     - source /root/.bashrc
     - export INSTALL_DIR=/opt/datadog-packages/datadog-installer/$(inv agent.version -u)-1
+    - export INSTALL_DIR_PARAM="--install-directory=$INSTALL_DIR"
+
 
 installer-arm64-oci:
   extends: installer-arm64
@@ -151,3 +153,4 @@ installer-arm64-oci:
   before_script:
     - source /root/.bashrc
     - export INSTALL_DIR=/opt/datadog-packages/datadog-installer/$(inv agent.version -u)-1
+    - export INSTALL_DIR_PARAM="--install-directory=$INSTALL_DIR"

.gitlab/package_build/windows.yml

@@ -21,6 +21,7 @@
       -e CI_JOB_NAME_SLUG=${CI_JOB_NAME_SLUG}
       -e CI_COMMIT_REF_NAME=${CI_COMMIT_REF_NAME}
       -e OMNIBUS_TARGET=${OMNIBUS_TARGET}
+      -e OMNIBUS_GIT_CACHE_DIR="C:\TEMP\omnibus-git-cache"
       -e WINDOWS_BUILDER=true
       -e RELEASE_VERSION="$RELEASE_VERSION"
       -e MAJOR_VERSION="$AGENT_MAJOR_VERSION"

.gitlab/source_test/linux.yml

@@ -148,6 +148,14 @@ lint_flavor_heroku_linux-x64:
   variables:
     FLAVORS: '--flavors heroku'
 
+lint_python:
+  stage: source_test
+  image: gcr.io/datadoghq/agent-circleci-runner:v31988376-bfbb3afb
+  tags: ["arch:amd64"]
+  needs: []
+  script:
+    - inv -e linter.python
+
 # Exclude systemd because it cannot succeed on Centos 6: the image doesn't have the shared object required by
 # https://github.com/coreos/go-systemd/blob/c8cc474ba8655dfbdb0ac7fcc09b7faf5b643caf/sdjournal/functions.go#L46
 # This is OK because the test on systemd still runs on the debian image above

comp/core/tagger/component_mock.go

@@ -8,7 +8,7 @@
 
 package tagger
 
-// Mock implements mock-specific methods for the inventorychecks component.
+// Mock implements mock-specific methods for the tagger component.
 type Mock interface {
 	Component
 

omnibus/config/software/datadog-agent-finalize.rb

@@ -14,6 +14,8 @@
 
 skip_transitive_dependency_licensing true
 
+always_build true
+
 build do
     license :project_license
 

omnibus/config/software/datadog-agent-integrations-py2.rb

@@ -21,6 +21,8 @@
 
 source git: 'https://github.com/DataDog/integrations-core.git'
 
+always_build true
+
 integrations_core_version = ENV['INTEGRATIONS_CORE_VERSION']
 if integrations_core_version.nil? || integrations_core_version.empty?
   integrations_core_version = 'master'

omnibus/config/software/datadog-agent-integrations-py3.rb

@@ -21,6 +21,8 @@
 
 source git: 'https://github.com/DataDog/integrations-core.git'
 
+always_build true
+
 integrations_core_version = ENV['INTEGRATIONS_CORE_VERSION']
 if integrations_core_version.nil? || integrations_core_version.empty?
   integrations_core_version = 'master'

omnibus/config/software/datadog-agent.rb

@@ -18,6 +18,8 @@
 source path: '..'
 relative_path 'src/github.com/DataDog/datadog-agent'
 
+always_build true
+
 build do
   license :project_license
 

omnibus/config/software/datadog-security-agent-policies.rb

@@ -20,6 +20,8 @@
 end
 default_version policies_version
 
+always_build true
+
 build do
   license "Apache-2.0"
   license_file "./LICENSE"

omnibus/config/software/system-probe.rb

@@ -8,6 +8,8 @@
 source path: '..'
 relative_path 'src/github.com/DataDog/datadog-agent'
 
+always_build true
+
 build do
   license :project_license
 

omnibus/omnibus.rb

@@ -37,4 +37,10 @@
     s3_instance_profile true
   end
 end
-use_git_caching false
+
+if not ENV.has_key?("OMNIBUS_GIT_CACHE_DIR")
+  use_git_caching false
+else
+  use_git_caching true
+  git_cache_dir ENV["OMNIBUS_GIT_CACHE_DIR"]
+end

pkg/api/util/ipc_endpoint.go

@@ -59,8 +59,11 @@ func (end *IPCEndpoint) DoGet(options ...GetOption) ([]byte, error) {
 		if errStr, found := errMap["error"]; found {
 			return nil, errors.New(errStr)
 		}
-
-		return nil, fmt.Errorf("could not reach agent: %v\nMake sure the agent is running before requesting the runtime configuration and contact support if you continue having issues", err)
+		netErr := new(net.OpError)
+		if errors.As(err, &netErr) {
+			return nil, fmt.Errorf("could not reach agent: %v\nMake sure the agent is running before requesting the runtime configuration and contact support if you continue having issues", err)
+		}
+		return nil, err
 	}
 	return res, err
 }

pkg/installer/service/apm_inject.go

@@ -14,8 +14,10 @@ import (
 	"fmt"
 	"os"
 	"path"
+	"path/filepath"
 	"strings"
 
+	"github.com/DataDog/datadog-agent/pkg/config/setup"
 	"github.com/DataDog/datadog-agent/pkg/util/log"
 	"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer"
 )
@@ -137,7 +139,7 @@ func (a *apmInjectorInstaller) setLDPreloadConfig(ctx context.Context) error {
 	if stat != nil {
 		perms = stat.Mode()
 	}
-	err = os.WriteFile("/tmp/ld.so.preload.tmp", newLdSoPreload, perms)
+	err = os.WriteFile(filepath.Join(setup.InstallPath, "run", "ld.so.preload.tmp"), newLdSoPreload, perms)
 	if err != nil {
 		return err
 	}
@@ -190,7 +192,7 @@ func (a *apmInjectorInstaller) deleteLDPreloadConfig(ctx context.Context) error
 	if stat != nil {
 		perms = stat.Mode()
 	}
-	err = os.WriteFile("/tmp/ld.so.preload.tmp", newLdSoPreload, perms)
+	err = os.WriteFile(filepath.Join(setup.InstallPath, "run", "ld.so.preload.tmp"), newLdSoPreload, perms)
 	if err != nil {
 		return err
 	}

pkg/installer/service/docker.go

@@ -16,13 +16,14 @@ import (
 	"os/exec"
 	"path"
 
+	"github.com/DataDog/datadog-agent/pkg/config/setup"
 	"github.com/DataDog/datadog-agent/pkg/util/log"
 )
 
 type dockerDaemonConfig map[string]interface{}
 
-const (
-	tmpDockerDaemonPath = "/tmp/daemon.json.tmp"
+var (
+	tmpDockerDaemonPath = path.Join(setup.InstallPath, "run", "daemon.json.tmp")
 	dockerDaemonPath    = "/etc/docker/daemon.json"
 )
 
@@ -83,9 +84,6 @@ func (a *apmInjectorInstaller) setDockerConfigContent(previousContent []byte) ([
 		}
 	}
 
-	if _, ok := dockerConfig["default-runtime"]; ok {
-		dockerConfig["default-runtime-backup"] = dockerConfig["default-runtime"]
-	}
 	dockerConfig["default-runtime"] = "dd-shim"
 	runtimes, ok := dockerConfig["runtimes"].(map[string]interface{})
 	if !ok {
@@ -153,10 +151,7 @@ func (a *apmInjectorInstaller) deleteDockerConfigContent(previousContent []byte)
 		}
 	}
 
-	if _, ok := dockerConfig["default-runtime-backup"]; ok {
-		dockerConfig["default-runtime"] = dockerConfig["default-runtime-backup"]
-		delete(dockerConfig, "default-runtime-backup")
-	} else {
+	if defaultRuntime, ok := dockerConfig["default-runtime"].(string); ok && defaultRuntime == "dd-shim" || !ok {
 		dockerConfig["default-runtime"] = "runc"
 	}
 	runtimes, ok := dockerConfig["runtimes"].(map[string]interface{})

pkg/installer/service/docker_test.go

@@ -53,7 +53,6 @@ func TestSetDockerConfig(t *testing.T) {
     }
 }`: `{
     "default-runtime": "dd-shim",
-    "default-runtime-backup": "containerd",
     "runtimes": {
         "containerd": {
             "path": "/usr/bin/containerd"
@@ -111,8 +110,7 @@ func TestRemoveDockerConfig(t *testing.T) {
 }`,
 		// File had already overridden the default runtime
 		`{
-    "default-runtime": "dd-shim",
-	"default-runtime-backup": "containerd",
+    "default-runtime": "containerd",
     "runtimes": {
         "containerd": {
             "path": "/usr/bin/containerd"

pkg/installer/service/helper/main.go

@@ -134,11 +134,11 @@ func buildCommand(inputCommand privilegeCommand) (*exec.Cmd, error) {
 	case "create-docker-dir":
 		return exec.Command("mkdir", "-p", "/etc/docker"), nil
 	case "replace-docker":
-		return exec.Command("mv", "/tmp/daemon.json.tmp", "/etc/docker/daemon.json"), nil
+		return exec.Command("mv", filepath.Join(installPath, "run", "daemon.json.tmp"), "/etc/docker/daemon.json"), nil
 	case "restart-docker":
 		return exec.Command("systemctl", "restart", "docker"), nil
 	case "replace-ld-preload":
-		return exec.Command("mv", "/tmp/ld.so.preload.tmp", "/etc/ld.so.preload"), nil
+		return exec.Command("mv", filepath.Join(installPath, "run", "ld.so.preload.tmp"), "/etc/ld.so.preload"), nil
 	case "add-installer-to-agent-group":
 		return exec.Command("usermod", "-aG", "dd-agent", "dd-installer"), nil
 	default: