Merge branch 'main' into daniel.lavie/USMON-830-kafka-fetch-latency

.gitlab/container_build/docker_linux.yml

@@ -1,8 +1,6 @@
 ---
 .docker_build_job_definition:
   stage: container_build
-  variables:
-    CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true"
   script:
     - aws s3 sync --only-show-errors $S3_ARTIFACTS_URI $BUILD_CONTEXT
     - TAG_SUFFIX=${TAG_SUFFIX:-}

.gitlab/container_build/fakeintake.yml

@@ -9,7 +9,6 @@ docker_build_fakeintake:
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/docker:20.10-py3
   tags: ["arch:amd64"]
   variables:
-    CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true"
     TARGET: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent/fakeintake:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}
     DOCKERFILE: test/fakeintake/Dockerfile
     PLATFORMS: linux/amd64,linux/arm64

.gitlab/kitchen_testing/new-e2e_testing/windows.yml

@@ -52,6 +52,8 @@
       - E2E_MSI_TEST: TestSubServicesOpts/all-subservices
       - E2E_MSI_TEST: TestSubServicesOpts/no-subservices
       - E2E_MSI_TEST: TestInstallAltDir
+      - E2E_MSI_TEST: TestInstallExistingAltDir
+      - E2E_MSI_TEST: TestInstallAltDirAndCorruptForUninstall
       - E2E_MSI_TEST: TestInstallFail
 
 .new-e2e_windows_installer_v7_tests:
@@ -73,6 +75,8 @@
       - E2E_MSI_TEST: TestSubServicesOpts/all-subservices
       - E2E_MSI_TEST: TestSubServicesOpts/no-subservices
       - E2E_MSI_TEST: TestInstallAltDir
+      - E2E_MSI_TEST: TestInstallExistingAltDir
+      - E2E_MSI_TEST: TestInstallAltDirAndCorruptForUninstall
       - E2E_MSI_TEST: TestInstallFail
       # These tests are v7 only
       - E2E_MSI_TEST: TestNPMUpgradeToNPM

.golangci.yml

@@ -63,18 +63,19 @@ issues:
 linters:
   disable-all: true
   enable:
-    - unconvert   # Remove unnecessary type conversions
-    - unused      # Checks Go code for unused constants, variables, functions and types
-    - ineffassign # Detects when assignments to existing variables are not used
-    - misspell    # Finds commonly misspelled English words in comments
-    - gofmt       # Gofmt checks whether code was gofmt-ed
-    - revive      # Revive is a replacement for golint, a coding style checker
-    - errcheck    # errcheck is a program for checking for unchecked errors in go programs.
-    - staticcheck # staticcheck is a go vet on steroids, applying a ton of static analysis checks
-    - govet       # Vet examines Go source code and reports suspicious constructs, such as Printf calls whose arguments do not align with the format string
-    - depguard    # Depguard is useful for preventing specific packages from being used
-    - bodyclose   # checks whether HTTP response body is closed successfully
-    - gosimple    # Linter for Go source code that specializes in simplifying code.
+    - unconvert                 # Remove unnecessary type conversions
+    - unused                    # Checks Go code for unused constants, variables, functions and types
+    - ineffassign               # Detects when assignments to existing variables are not used
+    - misspell                  # Finds commonly misspelled English words in comments
+    - gofmt                     # Gofmt checks whether code was gofmt-ed
+    - revive                    # Revive is a replacement for golint, a coding style checker
+    - errcheck                  # errcheck is a program for checking for unchecked errors in go programs.
+    - staticcheck               # staticcheck is a go vet on steroids, applying a ton of static analysis checks
+    - govet                     # Vet examines Go source code and reports suspicious constructs, such as Printf calls whose arguments do not align with the format string
+    - depguard                  # Depguard is useful for preventing specific packages from being used
+    - bodyclose                 # checks whether HTTP response body is closed successfully
+    - gosimple                  # Linter for Go source code that specializes in simplifying code.
+    - gocheckcompilerdirectives # Checks Go compiler directives syntax
 
 linters-settings:
   depguard:
@@ -149,6 +150,8 @@ linters-settings:
       - name: unexported-return
       - name: unreachable-code
       - name: unused-parameter
+        arguments:
+          - allowRegex: "^_"
       - name: var-declaration
       - name: var-naming
       # non-default rules:

.run/Publish Docker.run.xml

@@ -0,0 +1,22 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="Publish Docker" type="ShConfigurationType" folderName="docker">
+    <option name="SCRIPT_TEXT" value="" />
+    <option name="INDEPENDENT_SCRIPT_PATH" value="true" />
+    <option name="SCRIPT_PATH" value="$PROJECT_DIR$/.run/bash_runner.sh" />
+    <option name="SCRIPT_OPTIONS" value="" />
+    <option name="INDEPENDENT_SCRIPT_WORKING_DIRECTORY" value="true" />
+    <option name="SCRIPT_WORKING_DIRECTORY" value="$PROJECT_DIR$" />
+    <option name="INDEPENDENT_INTERPRETER_PATH" value="true" />
+    <option name="INTERPRETER_PATH" value="/bin/bash" />
+    <option name="INTERPRETER_OPTIONS" value="" />
+    <option name="EXECUTE_IN_TERMINAL" value="true" />
+    <option name="EXECUTE_SCRIPT_FILE" value="true" />
+    <envs>
+      <env name="SCRIPT_TO_RUN" value=".run/deploy_image.sh" />
+      <env name="IMAGE_NAME" value="&lt;your-name&gt;" />
+      <env name="IMAGE_VERSION" value="&lt;version&gt;" />
+      <env name="INCLUDE_AWS_ECR_LOGIN_PASSWORD" value="true" />
+    </envs>
+    <method v="2" />
+  </configuration>
+</component>

.run/bash_runner.sh

@@ -15,6 +15,11 @@ if [[ -z $SCRIPT_TO_RUN ]]; then
   exit
 fi
 
+if [ "$INCLUDE_AWS_ECR_LOGIN_PASSWORD" = true ]; then
+    # Get AWS ECR login password
+    AWS_ECR_LOGIN_PASSWORD=$(aws-vault exec sso-sandbox-account-admin -- aws ecr get-login-password --region us-east-1)
+fi
+
 if [[ -z $REMOTE_MACHINE_IP ]]; then
   echo "REMOTE_MACHINE_IP environment variable was not set, assuming local configuration"
   source "${SCRIPT_TO_RUN}"
@@ -50,6 +55,11 @@ else
 
   # Finally create the environment variable to inject list in the format that works with sh `ssh` command
   env_variables_to_inject=$(echo "$env_vars" | grep -v -w "${remote_env_array_as_grep_patterns[@]}" | tr '\n' ' ')
+
+  if [ "$INCLUDE_AWS_ECR_LOGIN_PASSWORD" = true ]; then
+      env_variables_to_inject+="AWS_ECR_LOGIN_PASSWORD=${AWS_ECR_LOGIN_PASSWORD} "
+  fi
+
   # shellcheck disable=SC2002
   cat "${SCRIPT_TO_RUN}" | ssh -tt "vagrant@$REMOTE_MACHINE_IP" \
   "export $env_variables_to_inject;cd ${DD_AGENT_ROOT_DIR};bash --login"

.run/deploy_image.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/bash -ex
+
+# This script logs into the Docker registry, builds a Docker image, and pushes it to Amazon ECR.
+
+# Ensure AWS_ECR_LOGIN_PASSWORD is set
+if [[ -z $AWS_ECR_LOGIN_PASSWORD ]]; then
+  echo "AWS_ECR_LOGIN_PASSWORD environment variable is not set"
+  exit 1
+fi
+
+# Ensure IMAGE_NAME and IMAGE_VERSION are set
+if [[ -z $IMAGE_NAME || -z $IMAGE_VERSION ]]; then
+  echo "IMAGE_NAME and IMAGE_VERSION environment variables must be set"
+  exit 1
+fi
+
+# Docker registry and image details
+AWS_ACCOUNT_ID="601427279990"
+REGION="us-east-1"
+REPOSITORY_NAME="usm-agent"
+
+# Login to Amazon ECR
+echo $AWS_ECR_LOGIN_PASSWORD | docker login --username AWS --password-stdin "$AWS_ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com"
+
+# Build and push the Docker image
+inv -e process-agent.build-dev-image \
+  --image "$AWS_ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com/$REPOSITORY_NAME/$IMAGE_NAME:$IMAGE_VERSION" \
+  --base-image datadog/agent-dev:master-py3 --push

LICENSE-3rdparty.csv

@@ -2569,8 +2569,8 @@ core,go4.org/intern,BSD-3-Clause,"Copyright (c) 2020, Brad Fitzpatrick"
 core,go4.org/mem,Apache-2.0,Copyright 2020 The Go4 AUTHORS
 core,go4.org/netipx,BSD-3-Clause,Alex Willmer <[email protected]> | Copyright (c) 2020 The Inet.af AUTHORS. All rights reserved | Matt Layher <[email protected]> | Tailscale Inc. | Tobias Klauser <[email protected]>
 core,go4.org/unsafe/assume-no-moving-gc,BSD-3-Clause,"Copyright (c) 2020, Brad Fitzpatrick"
-core,golang.org/x/arch/arm64/arm64asm,BSD-3-Clause,Copyright (c) 2015 The Go Authors. All rights reserved
-core,golang.org/x/arch/x86/x86asm,BSD-3-Clause,Copyright (c) 2015 The Go Authors. All rights reserved
+core,golang.org/x/arch/arm64/arm64asm,BSD-3-Clause,Copyright 2015 The Go Authors
+core,golang.org/x/arch/x86/x86asm,BSD-3-Clause,Copyright 2015 The Go Authors
 core,golang.org/x/crypto/argon2,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved
 core,golang.org/x/crypto/bcrypt,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved
 core,golang.org/x/crypto/blake2b,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved
@@ -2646,9 +2646,9 @@ core,golang.org/x/oauth2/google/internal/stsexchange,BSD-3-Clause,Copyright (c)
 core,golang.org/x/oauth2/internal,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved
 core,golang.org/x/oauth2/jws,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved
 core,golang.org/x/oauth2/jwt,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved
-core,golang.org/x/sync/errgroup,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved
-core,golang.org/x/sync/semaphore,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved
-core,golang.org/x/sync/singleflight,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved
+core,golang.org/x/sync/errgroup,BSD-3-Clause,Copyright 2009 The Go Authors
+core,golang.org/x/sync/semaphore,BSD-3-Clause,Copyright 2009 The Go Authors
+core,golang.org/x/sync/singleflight,BSD-3-Clause,Copyright 2009 The Go Authors
 core,golang.org/x/sys/cpu,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved
 core,golang.org/x/sys/execabs,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved
 core,golang.org/x/sys/plan9,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved
@@ -2691,7 +2691,7 @@ core,golang.org/x/text/transform,BSD-3-Clause,Copyright (c) 2009 The Go Authors.
 core,golang.org/x/text/unicode/bidi,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved
 core,golang.org/x/text/unicode/norm,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved
 core,golang.org/x/text/width,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved
-core,golang.org/x/time/rate,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved
+core,golang.org/x/time/rate,BSD-3-Clause,Copyright 2009 The Go Authors
 core,golang/go,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved.
 core,gomodules.xyz/jsonpatch/v2,Apache-2.0,Copyright (c) 2015 The Authors
 core,gonum.org/v1/gonum/blas,BSD-3-Clause,Alexander Egurnov <[email protected]> | Andrei Blinnikov <[email protected]> | Andrew Brampton <[email protected]> | Bailey Lissington <[email protected]> | Bill Gray <[email protected]> | Bill Noon <[email protected]> | Brendan Tracey <[email protected]> | Brent Pedersen <[email protected]> | Bulat Khasanov <[email protected]> | Chad Kunde <[email protected]> | Chan Kwan Yin <[email protected]> | Chih-Wei Chang <[email protected]> | Chong-Yeol Nah <[email protected]> | Chris Tessum <[email protected]> | Christophe Meessen <[email protected]> | Christopher Waldon <[email protected]> | Clayton Northey <[email protected]> | Copyright ©2013 The Gonum Authors. All rights reserved | Dan Kortschak <[email protected]> <[email protected]> | Dan Lorenc <[email protected]> | Daniel Fireman <[email protected]> | Dario Heinisch <[email protected]> | David Kleiven <[email protected]> | David Samborski <[email protected]> | Davor Kapsa <[email protected]> | DeepMind Technologies | Delaney Gillilan <[email protected]> | Dezmond Goff <[email protected]> | Dong-hee Na <[email protected]> | Dustin Spicuzza <[email protected]> | Egon Elbre <[email protected]> | Ekaterina Efimova <[email protected]> | Eng Zer Jun <[email protected]> | Ethan Burns <[email protected]> | Ethan Reesor <[email protected]> | Evert Lammerts <[email protected]> | Evgeny Savinov <[email protected]> | Fabian Wickborn <[email protected]> | Facundo Gaich <[email protected]> | Fazlul Shahriar <[email protected]> | Francesc Campoy <[email protected]> | Google Inc | Gustaf Johansson <[email protected]> | Hossein Zolfi <[email protected]> | Iakov Davydov <[email protected]> | Igor Mikushkin <[email protected]> | Iskander Sharipov <[email protected]> | Jalem Raj Rohit <[email protected]> | James Bell <[email protected]> | James Bowman <[email protected]> | James Holmes <[email protected]> | Janne Snabb <[email protected]> | Jeremy Atkinson <[email protected]> | Jes Cok <[email protected]> | Jinesi Yelizati <[email protected]> | Jon Richards <[email protected]> | Jonas Kahler <[email protected]> | Jonas Schulze <[email protected]> | Jonathan Bluett-Duncan <[email protected]> | Jonathan J Lawlor <[email protected]> | Jonathan Reiter <[email protected]> | Jonathan Schroeder <[email protected]> | Joost van Amersfoort <[email protected]> | Jordan Stoker <[email protected]> | Joseph Watson <[email protected]> | Josh Wilson <[email protected]> | Julien Roland <[email protected]> | Kai Trukenmüller <[email protected]> | Kent English <[email protected]> | Kevin C. Zimmerman <[email protected]> | Kirill Motkov <[email protected]> | Konstantin Shaposhnikov <[email protected]> | Leonid Kneller <[email protected]> | Lyron Winderbaum <[email protected]> <[email protected]> <[email protected]> | Marco Leogrande <[email protected]> | Mark Canning <[email protected]> | Mark Skilbeck <[email protected]> | Martin Diz <[email protected]> | Matthew Connelly <[email protected]> | Matthieu Di Mercurio <[email protected]> | Max Halford <[email protected]> | Maxim Sergeev <[email protected]> | Microsoft Corporation | MinJae Kwon <[email protected]> | Nathan Edwards <[email protected]> | Nick Potts <[email protected]> | Nils Wogatzky <[email protected]> | Olivier Wulveryck <[email protected]> | Or Rikon <[email protected]> | Patricio Whittingslow <[email protected]> | Patrick DeVivo <[email protected]> | Pontus Melke <[email protected]> | Renee French | Rishi Desai <[email protected]> | Robin Eklind <[email protected]> | Roger Welin <[email protected]> | Roman Werpachowski <[email protected]> | Rondall Jones <[email protected]> | Sam Zaydel <[email protected]> | Samuel Kelemen <[email protected]> | Saran Ahluwalia <[email protected]> | Scott Holden <[email protected]> | Scott Kiesel <[email protected]> | Sebastien Binet <[email protected]> | Shawn Smith <[email protected]> | Sintela Ltd | Spencer Lyon <[email protected]> | Steve McCoy <[email protected]> | Taesu Pyo <[email protected]> | Takeshi Yoneda <[email protected]> | Tamir Hyman <[email protected]> | The University of Adelaide | The University of Minnesota | The University of Washington | Thomas Berg <[email protected]> | Tobin Harding <[email protected]> | Valentin Deleplace <[email protected]> | Vincent Thiery <[email protected]> | Vladimír Chalupecký <[email protected]> | Will Tekulve <[email protected]> | Yasuhiro Matsumoto <[email protected]> | Yevgeniy Vahlis <[email protected]> | Yucheng Zhu <[email protected]> | Yunomi <[email protected]> | Zoe Juozapaitis | antichris <[email protected]> | source{d} <[email protected]>

cmd/system-probe/modules/traceroute_linux.go

@@ -3,8 +3,6 @@
 // This product includes software developed at Datadog (https://www.datadoghq.com/).
 // Copyright 2016-present Datadog, Inc.
 
-// go:build linux
-
 package modules
 
 import (

cmd/system-probe/modules/traceroute_windows.go

@@ -3,8 +3,6 @@
 // This product includes software developed at Datadog (https://www.datadoghq.com/).
 // Copyright 2016-present Datadog, Inc.
 
-// go:build windows
-
 package modules
 
 import (

cmd/system-probe/subcommands/run/command.go

@@ -114,7 +114,7 @@ func Commands(globalParams *command.GlobalParams) []*cobra.Command {
 
 					return settings.Params{
 						Settings: map[string]settings.RuntimeSetting{
-							"log_level":                       &commonsettings.LogLevelRuntimeSetting{ConfigKey: configPrefix + "log_level"},
+							"log_level":                       commonsettings.NewLogLevelRuntimeSetting(),
 							"runtime_mutex_profile_fraction":  &commonsettings.RuntimeMutexProfileFraction{ConfigPrefix: configPrefix},
 							"runtime_block_profile_rate":      &commonsettings.RuntimeBlockProfileRate{ConfigPrefix: configPrefix},
 							"internal_profiling_goroutines":   profilingGoRoutines,
@@ -270,7 +270,7 @@ func runSystemProbe(ctxChan <-chan context.Context, errChan chan error) error {
 
 			return settings.Params{
 				Settings: map[string]settings.RuntimeSetting{
-					"log_level":                       &commonsettings.LogLevelRuntimeSetting{ConfigKey: configPrefix + "log_level"},
+					"log_level":                       commonsettings.NewLogLevelRuntimeSetting(),
 					"runtime_mutex_profile_fraction":  &commonsettings.RuntimeMutexProfileFraction{ConfigPrefix: configPrefix},
 					"runtime_block_profile_rate":      &commonsettings.RuntimeBlockProfileRate{ConfigPrefix: configPrefix},
 					"internal_profiling_goroutines":   profilingGoRoutines,