diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 38685aaf29908..5b025af7ce4bc 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -166,7 +166,6 @@ /cmd/agent/dist/conf.d/snmp.d/ @DataDog/network-device-monitoring /cmd/agent/dist/conf.d/win32_event_log.d/ @DataDog/windows-agent /cmd/agent/install*.sh @DataDog/agent-build-and-releases -/cmd/agentless-scanner/ @DataDog/agent-cspm /cmd/cluster-agent/ @DataDog/container-platform /cmd/cluster-agent/commands/ @DataDog/container-platform /cmd/cluster-agent-cloudfoundry/ @DataDog/platform-integrations diff --git a/.github/workflows/docs-dev.yml b/.github/workflows/docs-dev.yml index 0bd8a1ee8afc8..96efef946c7a3 100644 --- a/.github/workflows/docs-dev.yml +++ b/.github/workflows/docs-dev.yml @@ -6,11 +6,13 @@ on: - main paths: - docs/** + - .github/workflows/docs-dev.yml pull_request: branches: - main paths: - docs/** + - .github/workflows/docs-dev.yml concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} diff --git a/.github/workflows/send-failure-summary-notification.yml b/.github/workflows/send-failure-summary-notification.yml new file mode 100644 index 0000000000000..5e602e70fb967 --- /dev/null +++ b/.github/workflows/send-failure-summary-notification.yml @@ -0,0 +1,34 @@ +name: Send failure summary notification + +on: + workflow_dispatch: + inputs: + # See notify_tests.py to see the format + job_failures: + description: 'Job failures' + required: true + type: string + +jobs: + send-failure-summary-team-notification: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + ref: main + - name: Setup python + uses: actions/setup-python@v5 + with: + python-version: 3.11 + cache: 'pip' + cache-dependency-path: '**/requirements*.txt' + - name: Install dependencies + run: pip install -r requirements.txt -r tasks/requirements.txt + - name: Create and send failure summary notification to team channel + id: send_failure_summary_team_notification + env: + # Too much data to be passed by arguments + JOB_FAILURES: ${{ inputs.job_failures }} + SLACK_API_TOKEN: ${{ secrets.SLACK_DATADOG_AGENT_BOT_TOKEN }} + run: inv -e notify.send-failure-summary-notification diff --git a/.gitignore b/.gitignore index 9ea55fd2e5496..3f83af55d0fb9 100644 --- a/.gitignore +++ b/.gitignore @@ -37,6 +37,10 @@ Dockerfiles/cluster-agent/nosys-seccomp # specific pre-commit hooks .pre-commit-config-*.yaml +# Utility tools +devagent +deva + # go-generated files datadog.yaml system-probe.yaml diff --git a/.gitlab/binary_build/linux.yml b/.gitlab/binary_build/linux.yml index 329eee1a568aa..d8644d63a2c9c 100644 --- a/.gitlab/binary_build/linux.yml +++ b/.gitlab/binary_build/linux.yml @@ -53,9 +53,8 @@ build_dogstatsd-binary_x64: build_dogstatsd-binary_arm64: rules: - - !reference [.on_all_builds] - - !reference [.on_packaging_change] - - !reference [.on_go-version_change] + - !reference [.except_mergequeue] + - when: on_success stage: binary_build image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX:$DATADOG_AGENT_ARMBUILDIMAGES tags: ["arch:arm64"] diff --git a/.gitlab/deploy_containers/deploy_containers.yml b/.gitlab/deploy_containers/deploy_containers.yml index dfc6b7dfd2dc4..b068839f0b6d5 100644 --- a/.gitlab/deploy_containers/deploy_containers.yml +++ b/.gitlab/deploy_containers/deploy_containers.yml @@ -1,32 +1,6 @@ --- # deploy containers stage -# Contains jobs which create child pipelines to deploy Agent 6 & 7 to staging repositories and to Dockerhub / GCR. - -# -# Agent v6 -# - -deploy_containers-a6: - stage: deploy_containers - rules: - !reference [.on_deploy] - variables: - PARENT_PIPELINE_ID: $CI_PIPELINE_ID - BUCKET_BRANCH: $BUCKET_BRANCH - trigger: - include: .gitlab/deploy_containers/deploy_containers_a6.yml - -deploy_containers-a6-on-failure: - stage: deploy_containers - rules: - !reference [.on_deploy_failure] - variables: - PARENT_PIPELINE_ID: $CI_PIPELINE_ID - BUCKET_BRANCH: $BUCKET_BRANCH - FORCE_MANUAL: "true" - trigger: - include: .gitlab/deploy_containers/deploy_containers_a6.yml - +# Contains jobs which create child pipelines to deploy Agent 7 to staging repositories and to Dockerhub / GCR. deploy_containers-a7: stage: deploy_containers diff --git a/.gitlab/deploy_containers/deploy_containers_a6.yml b/.gitlab/deploy_containers/deploy_containers_a6.yml deleted file mode 100644 index b2a4d604faf4f..0000000000000 --- a/.gitlab/deploy_containers/deploy_containers_a6.yml +++ /dev/null @@ -1,69 +0,0 @@ ---- -# deploy containers stage -# Contains jobs which deploy Agent 6 & 7 to staging repositories and to Dockerhub / GCR. - -stages: - - deploy_containers - -include: - - .gitlab/common/container_publish_job_templates.yml - - .gitlab/deploy_containers/conditions.yml - -# -# Image tagging & manifest publication -# - -# -# Agent v6 -# -.deploy_containers-a6-base: - extends: .docker_publish_job_definition - stage: deploy_containers - dependencies: [] - before_script: - - source /root/.bashrc - - if [[ "$VERSION" == "" ]]; then export VERSION="$(inv agent.version --major-version 6 --url-safe --pipeline-id $PARENT_PIPELINE_ID)"; fi - - export IMG_SOURCES="${SRC_AGENT}:v${PARENT_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6${JMX}-amd64,${SRC_AGENT}:v${PARENT_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6${JMX}-arm64" - - export IMG_DESTINATIONS="${AGENT_REPOSITORY}:${VERSION}${JMX}" - parallel: - matrix: - - JMX: - - "" - - "-jmx" - - -deploy_containers-a6: - extends: .deploy_containers-a6-base - rules: - !reference [.manual_on_deploy_auto_on_rc] - - - -deploy_containers-a6-rc: - extends: .deploy_containers-a6-base - variables: - AGENT_REPOSITORY: agent - DSD_REPOSITORY: dogstatsd - IMG_REGISTRIES: public - VERSION: 6-rc - rules: - !reference [.on_rc] - - -# -# Latest publication -# - -deploy_containers_latest-a6: - extends: .docker_publish_job_definition - stage: deploy_containers - rules: - !reference [.on_final] - dependencies: [] - parallel: - matrix: - - IMG_SOURCES: ${SRC_AGENT}:v${PARENT_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6-amd64,${SRC_AGENT}:v${PARENT_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6-arm64 - IMG_DESTINATIONS: ${AGENT_REPOSITORY}:6,${AGENT_REPOSITORY}:latest-py2 - - IMG_SOURCES: ${SRC_AGENT}:v${PARENT_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6-jmx-amd64,${SRC_AGENT}:v${PARENT_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6-jmx-arm64 - IMG_DESTINATIONS: ${AGENT_REPOSITORY}:6-jmx,${AGENT_REPOSITORY}:latest-py2-jmx - diff --git a/.gitlab/package_build/deb.yml b/.gitlab/package_build/deb.yml deleted file mode 100644 index 5283d79f23cbd..0000000000000 --- a/.gitlab/package_build/deb.yml +++ /dev/null @@ -1,66 +0,0 @@ -dogstatsd_deb-x64: - rules: - - !reference [.except_mergequeue] - - when: on_success - stage: package_build - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - needs: ["go_mod_tidy_check", "build_dogstatsd-binary_x64", "go_deps"] - variables: - DD_PKG_ARCH: "x86_64" - before_script: - - source /root/.bashrc - - !reference [.retrieve_linux_go_deps] - - !reference [.cache_omnibus_ruby_deps, setup] - script: - # remove artifacts from previous pipelines that may come from the cache - - rm -rf $OMNIBUS_PACKAGE_DIR/* - - !reference [.setup_ruby_mirror_linux] - # Artifacts and cache must live within project directory but we run omnibus in a neutral directory. - # Thus, we move the artifacts at the end in a gitlab-friendly dir. - - !reference [.setup_deb_signing_key] - # Use --skip-deps since the deps are installed by `before_script`. - - inv -e omnibus.build --release-version "$RELEASE_VERSION_7" --major-version 7 --base-dir $OMNIBUS_BASE_DIR ${USE_S3_CACHING} --skip-deps --go-mod-cache="$GOPATH/pkg/mod" --target-project="dogstatsd" - - ls -la $OMNIBUS_PACKAGE_DIR - - !reference [.lint_linux_packages] - - $S3_CP_CMD $OMNIBUS_PACKAGE_DIR/datadog-dogstatsd*_amd64.deb $S3_ARTIFACTS_URI/datadog-dogstatsd_amd64.deb - - !reference [.upload_sbom_artifacts] - artifacts: - expire_in: 2 weeks - paths: - - $OMNIBUS_PACKAGE_DIR - cache: - - !reference [.cache_omnibus_ruby_deps, cache] - -dogstatsd_deb-arm64: - rules: - - !reference [.on_all_builds] - - !reference [.on_packaging_change] - - !reference [.on_go-version_change] - stage: package_build - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX:$DATADOG_AGENT_ARMBUILDIMAGES - tags: ["arch:arm64"] - needs: ["go_mod_tidy_check", "build_dogstatsd-binary_arm64", "go_deps"] - variables: - DD_PKG_ARCH: "arm64" - before_script: - - source /root/.bashrc - - !reference [.retrieve_linux_go_deps] - script: - # remove artifacts from previous pipelines that may come from the cache - - rm -rf $OMNIBUS_PACKAGE_DIR/* - - !reference [.setup_ruby_mirror_linux] - # Artifacts and cache must live within project directory but we run omnibus in a neutral directory. - # Thus, we move the artifacts at the end in a gitlab-friendly dir. - - !reference [.setup_deb_signing_key] - # Use --skip-deps since the deps are installed by `before_script`. - - inv -e omnibus.build --release-version "$RELEASE_VERSION_7" --major-version 7 --base-dir $OMNIBUS_BASE_DIR ${USE_S3_CACHING} --skip-deps --go-mod-cache="$GOPATH/pkg/mod" --target-project="dogstatsd" - - ls -la $OMNIBUS_PACKAGE_DIR - - !reference [.lint_linux_packages] - - $S3_CP_CMD $OMNIBUS_PACKAGE_DIR/datadog-dogstatsd*_arm64.deb $S3_ARTIFACTS_URI/datadog-dogstatsd_arm64.deb - - !reference [.upload_sbom_artifacts] - artifacts: - expire_in: 2 weeks - paths: - - $OMNIBUS_PACKAGE_DIR - diff --git a/.gitlab/package_build/include.yml b/.gitlab/package_build/include.yml index 14f045683bed6..92279f4c2b88a 100644 --- a/.gitlab/package_build/include.yml +++ b/.gitlab/package_build/include.yml @@ -10,11 +10,8 @@ - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } include: - - .gitlab/package_build/deb.yml - .gitlab/package_build/heroku.yml - .gitlab/package_build/dmg.yml - .gitlab/package_build/installer.yml - - .gitlab/package_build/rpm.yml - - .gitlab/package_build/suse_rpm.yml - .gitlab/package_build/windows.yml - .gitlab/package_build/linux.yml diff --git a/.gitlab/package_build/linux.yml b/.gitlab/package_build/linux.yml index 92e91723dabf4..d6e16884241a5 100644 --- a/.gitlab/package_build/linux.yml +++ b/.gitlab/package_build/linux.yml @@ -60,41 +60,35 @@ variables: PACKAGE_ARCH: arm64 -# build Agent 6 binaries for x86_64 -datadog-agent-6-x64: - extends: [.agent_build_common, .agent_build_x86] +.agent_6_build: variables: AGENT_MAJOR_VERSION: 6 PYTHON_RUNTIMES: "2,3" before_script: - export RELEASE_VERSION=$RELEASE_VERSION_6 -# build Agent 7 binaries for x86_64 -datadog-agent-7-x64: - extends: [.agent_build_common, .agent_build_x86] +.agent_7_build: variables: AGENT_MAJOR_VERSION: 7 PYTHON_RUNTIMES: "3" before_script: - export RELEASE_VERSION=$RELEASE_VERSION_7 +# build Agent 6 binaries for x86_64 +datadog-agent-6-x64: + extends: [.agent_build_common, .agent_build_x86, .agent_6_build] + +# build Agent 7 binaries for x86_64 +datadog-agent-7-x64: + extends: [.agent_build_common, .agent_build_x86, .agent_7_build] + # build Agent 6 binaries for arm64 datadog-agent-6-arm64: - extends: [.agent_build_common, .agent_build_arm64] - variables: - AGENT_MAJOR_VERSION: 6 - PYTHON_RUNTIMES: "2,3" - before_script: - - export RELEASE_VERSION=$RELEASE_VERSION_6 + extends: [.agent_build_common, .agent_build_arm64, .agent_6_build] # build Agent 7 binaries for arm64 datadog-agent-7-arm64: - extends: [.agent_build_common, .agent_build_arm64] - variables: - AGENT_MAJOR_VERSION: 7 - PYTHON_RUNTIMES: "3" - before_script: - - export RELEASE_VERSION=$RELEASE_VERSION_7 + extends: [.agent_build_common, .agent_build_arm64, .agent_7_build] .iot-agent-common: extends: .agent_build_common @@ -134,3 +128,43 @@ iot-agent-armhf: # if we use too many compression threads or a too agressive level FORCED_PACKAGE_COMPRESSION_LEVEL: 5 +.dogstatsd_build_common: + rules: + - !reference [.except_mergequeue] + - when: on_success + stage: package_build + script: + - source /root/.bashrc + - echo "About to build for $RELEASE_VERSION" + - !reference [.setup_ruby_mirror_linux] + - !reference [.setup_python_mirror_linux] + - !reference [.retrieve_linux_go_deps] + - !reference [.cache_omnibus_ruby_deps, setup] + # remove artifacts from previous pipelines that may come from the cache + - rm -rf $OMNIBUS_PACKAGE_DIR/* + - inv -e omnibus.build --release-version $RELEASE_VERSION_7 --major-version 7 --python-runtimes 3 --base-dir $OMNIBUS_BASE_DIR ${USE_S3_CACHING} --skip-deps --go-mod-cache="$GOPATH/pkg/mod" --target-project dogstatsd + - ls -la $OMNIBUS_PACKAGE_DIR + - !reference [.upload_sbom_artifacts] + variables: + KUBERNETES_CPU_REQUEST: 16 + KUBERNETES_MEMORY_REQUEST: "32Gi" + KUBERNETES_MEMORY_LIMIT: "32Gi" + artifacts: + expire_in: 2 weeks + paths: + - $OMNIBUS_PACKAGE_DIR + cache: + - !reference [.cache_omnibus_ruby_deps, cache] + +dogstatsd-x64: + extends: .dogstatsd_build_common + needs: ["go_mod_tidy_check", "build_dogstatsd-binary_x64", "go_deps"] + image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES + tags: ["arch:amd64"] + +dogstatsd-arm64: + extends: .dogstatsd_build_common + image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX:$DATADOG_AGENT_ARMBUILDIMAGES + tags: ["arch:arm64"] + needs: ["go_mod_tidy_check", "build_dogstatsd-binary_arm64", "go_deps"] + diff --git a/.gitlab/package_build/rpm.yml b/.gitlab/package_build/rpm.yml deleted file mode 100644 index 669f4c48729bc..0000000000000 --- a/.gitlab/package_build/rpm.yml +++ /dev/null @@ -1,35 +0,0 @@ -dogstatsd_rpm-x64: - stage: package_build - rules: - - !reference [.except_mergequeue] - - when: on_success - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - needs: ["go_mod_tidy_check", "build_dogstatsd-binary_x64", "go_deps"] - variables: - DD_PKG_ARCH: "x86_64" - before_script: - - source /root/.bashrc - - !reference [.retrieve_linux_go_deps] - - !reference [.cache_omnibus_ruby_deps, setup] - script: - # remove artifacts from previous pipelines that may come from the cache - - rm -rf $OMNIBUS_PACKAGE_DIR/* - - !reference [.setup_ruby_mirror_linux] - # Artifacts and cache must live within project directory but we run omnibus - # from the GOPATH (see above). We then call `invoke` passing --base-dir, - # pointing to a gitlab-friendly location. - - RPM_GPG_KEY=$($CI_PROJECT_DIR/tools/ci/aws_ssm_get_wrapper.sh $RPM_GPG_KEY_SSM_NAME) - - printf -- "$RPM_GPG_KEY" | gpg --import --batch - - export RPM_SIGNING_PASSPHRASE=$($CI_PROJECT_DIR/tools/ci/aws_ssm_get_wrapper.sh $RPM_SIGNING_PASSPHRASE_SSM_NAME) - # Use --skip-deps since the deps are installed by `before_script`. - - inv -e omnibus.build --release-version "$RELEASE_VERSION_7" --major-version 7 --base-dir $OMNIBUS_BASE_DIR ${USE_S3_CACHING} --skip-deps --go-mod-cache="$GOPATH/pkg/mod" --target-project="dogstatsd" - - ls -la $OMNIBUS_PACKAGE_DIR - - !reference [.lint_linux_packages] - - !reference [.upload_sbom_artifacts] - artifacts: - expire_in: 2 weeks - paths: - - $OMNIBUS_PACKAGE_DIR - cache: - - !reference [.cache_omnibus_ruby_deps, cache] diff --git a/.gitlab/package_build/suse_rpm.yml b/.gitlab/package_build/suse_rpm.yml deleted file mode 100644 index 964f138254106..0000000000000 --- a/.gitlab/package_build/suse_rpm.yml +++ /dev/null @@ -1,37 +0,0 @@ -dogstatsd_suse-x64: - stage: package_build - rules: - - !reference [.except_mergequeue] - - when: on_success - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/suse_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - needs: ["go_mod_tidy_check", "build_dogstatsd-binary_x64", "go_deps"] - variables: - DD_PKG_ARCH: "x86_64" - before_script: - - source /root/.bashrc - - !reference [.retrieve_linux_go_deps] - - !reference [.cache_omnibus_ruby_deps, setup] - script: - # remove artifacts from previous pipelines that may come from the cache - - rm -rf $OMNIBUS_PACKAGE_DIR_SUSE/* - - !reference [.setup_ruby_mirror_linux] - # Artifacts and cache must live within project directory but we run omnibus - # from the GOPATH (see above). We then call `invoke` passing --base-dir, - # pointing to a gitlab-friendly location. - - RPM_GPG_KEY=$($CI_PROJECT_DIR/tools/ci/aws_ssm_get_wrapper.sh $RPM_GPG_KEY_SSM_NAME) - - printf -- "$RPM_GPG_KEY" | gpg --import --batch - - export RPM_SIGNING_PASSPHRASE=$($CI_PROJECT_DIR/tools/ci/aws_ssm_get_wrapper.sh $RPM_SIGNING_PASSPHRASE_SSM_NAME) - # Use --skip-deps since the deps are installed by `before_script`. - - inv -e omnibus.build --release-version "$RELEASE_VERSION_7" --major-version 7 --base-dir $OMNIBUS_BASE_DIR ${USE_S3_CACHING} --skip-deps --go-mod-cache="$GOPATH/pkg/mod" --target-project=dogstatsd - - ls -la $OMNIBUS_PACKAGE_DIR - - !reference [.lint_linux_packages] - # Copy to a different directory to avoid collisions if a job downloads both the RPM and SUSE RPM artifacts - - mkdir -p $OMNIBUS_PACKAGE_DIR_SUSE && cp $OMNIBUS_PACKAGE_DIR/* $OMNIBUS_PACKAGE_DIR_SUSE - - !reference [.upload_sbom_artifacts] - artifacts: - expire_in: 2 weeks - paths: - - $OMNIBUS_PACKAGE_DIR_SUSE - cache: - - !reference [.cache_omnibus_ruby_deps, cache] diff --git a/.gitlab/packaging/deb.yml b/.gitlab/packaging/deb.yml index 012725ad29f7c..9187935af85c5 100644 --- a/.gitlab/packaging/deb.yml +++ b/.gitlab/packaging/deb.yml @@ -22,108 +22,95 @@ cache: - !reference [.cache_omnibus_ruby_deps, cache] +.package_deb_x86: + image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES + tags: ["arch:amd64"] + variables: + DD_PKG_ARCH: x86_64 + PACKAGE_ARCH: amd64 + +.package_deb_arm64: + image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX:$DATADOG_AGENT_ARMBUILDIMAGES + tags: ["arch:arm64"] + variables: + PACKAGE_ARCH: arm64 + DD_PKG_ARCH: arm64 + +.package_deb_agent_6: + variables: + RELEASE_VERSION: $RELEASE_VERSION_6 + AGENT_MAJOR_VERSION: 6 + +.package_deb_agent_7: + variables: + RELEASE_VERSION: $RELEASE_VERSION_7 + AGENT_MAJOR_VERSION: 7 + agent_deb-x64-a6: - extends: .package_deb_common + extends: [.package_deb_common, .package_deb_x86, .package_deb_agent_6] rules: - !reference [.except_mergequeue] - when: on_success - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] needs: ["datadog-agent-6-x64"] variables: - PACKAGE_ARCH: amd64 DESTINATION_DEB: "datadog-agent_6_amd64.deb" - DD_PKG_ARCH: "x86_64" DD_PROJECT: "agent" - RELEASE_VERSION: $RELEASE_VERSION_6 - AGENT_MAJOR_VERSION: 6 agent_deb-x64-a7: - extends: .package_deb_common + extends: [.package_deb_common, .package_deb_x86, .package_deb_agent_7] rules: - !reference [.except_mergequeue] - when: on_success - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] needs: ["datadog-agent-7-x64"] variables: - PACKAGE_ARCH: amd64 DESTINATION_DEB: "datadog-agent_7_amd64.deb" - DD_PKG_ARCH: "x86_64" DD_PROJECT: "agent" - RELEASE_VERSION: $RELEASE_VERSION_7 - AGENT_MAJOR_VERSION: 7 agent_deb-arm64-a6: - extends: .package_deb_common + extends: [.package_deb_common, .package_deb_arm64, .package_deb_agent_6] rules: - !reference [.except_mergequeue] - when: on_success - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX:$DATADOG_AGENT_ARMBUILDIMAGES - tags: ["arch:arm64"] needs: ["datadog-agent-6-arm64"] variables: - PACKAGE_ARCH: arm64 DESTINATION_DEB: "datadog-agent_6_arm64.deb" - DD_PKG_ARCH: "arm64" DD_PROJECT: "agent" - RELEASE_VERSION: $RELEASE_VERSION_6 - AGENT_MAJOR_VERSION: 6 agent_deb-arm64-a7: - extends: .package_deb_common + extends: [.package_deb_common, .package_deb_arm64, .package_deb_agent_7] rules: - !reference [.except_mergequeue] - when: on_success - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX:$DATADOG_AGENT_ARMBUILDIMAGES - tags: ["arch:arm64"] needs: ["datadog-agent-7-arm64"] variables: - PACKAGE_ARCH: arm64 DESTINATION_DEB: "datadog-agent_7_arm64.deb" - DD_PKG_ARCH: "arm64" DD_PROJECT: "agent" - RELEASE_VERSION: $RELEASE_VERSION_7 - AGENT_MAJOR_VERSION: 7 installer_deb-amd64: - extends: .package_deb_common + extends: [.package_deb_common, .package_deb_x86, .package_deb_agent_7] rules: - !reference [.except_mergequeue] - when: on_success - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] needs: ["installer-amd64"] variables: - PACKAGE_ARCH: amd64 DESTINATION_DEB: "datadog-installer_7_amd64.deb" - DD_PKG_ARCH: "x86_64" DD_PROJECT: "installer" - RELEASE_VERSION: $RELEASE_VERSION_7 - AGENT_MAJOR_VERSION: 7 installer_deb-arm64: - extends: .package_deb_common + extends: [.package_deb_common, .package_deb_arm64, .package_deb_agent_7] rules: - !reference [.except_mergequeue] - when: on_success - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX:$DATADOG_AGENT_ARMBUILDIMAGES - tags: ["arch:arm64"] needs: ["installer-arm64"] variables: - PACKAGE_ARCH: arm64 DESTINATION_DEB: "datadog-installer_7_arm64.deb" - DD_PKG_ARCH: "arm64" DD_PROJECT: "installer" - RELEASE_VERSION: $RELEASE_VERSION_7 - AGENT_MAJOR_VERSION: 7 .package_iot_deb_common: + extends: [.package_deb_agent_7] rules: - !reference [.except_mergequeue] - when: on_success - before_script: - - export RELEASE_VERSION=$RELEASE_VERSION_7 stage: packaging script: - source /root/.bashrc @@ -147,23 +134,15 @@ installer_deb-arm64: - !reference [.cache_omnibus_ruby_deps, cache] iot_agent_deb-x64: - extends: .package_iot_deb_common - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] + extends: [.package_iot_deb_common, .package_deb_x86] needs: ["iot-agent-x64"] variables: - PACKAGE_ARCH: amd64 - DD_PKG_ARCH: "x86_64" DESTINATION_DEB: "datadog-iot-agent_7_amd64.deb" iot_agent_deb-arm64: - extends: .package_iot_deb_common - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX:$DATADOG_AGENT_ARMBUILDIMAGES - tags: ["arch:arm64"] + extends: [.package_iot_deb_common, .package_deb_arm64] needs: ["iot-agent-arm64"] variables: - PACKAGE_ARCH: arm64 - DD_PKG_ARCH: "arm64" DESTINATION_DEB: "datadog-iot-agent_7_arm64.deb" iot_agent_deb-armhf: @@ -177,4 +156,23 @@ iot_agent_deb-armhf: DD_PKG_ARCH: "arm64" FORCED_PACKAGE_COMPRESSION_LEVEL: 5 +dogstatsd_deb-x64: + extends: [.package_deb_common, .package_deb_x86, .package_deb_agent_7] + rules: + - !reference [.except_mergequeue] + - when: on_success + needs: ["dogstatsd-x64"] + variables: + DD_PROJECT: dogstatsd + DESTINATION_DEB: "datadog-dogstatsd_amd64.deb" + +dogstatsd_deb-arm64: + extends: [.package_deb_common, .package_deb_arm64, .package_deb_agent_7] + rules: + - !reference [.except_mergequeue] + - when: on_success + needs: ["dogstatsd-arm64"] + variables: + DD_PROJECT: dogstatsd + DESTINATION_DEB: "datadog-dogstatsd_arm64.deb" diff --git a/.gitlab/packaging/rpm.yml b/.gitlab/packaging/rpm.yml index 33925c2c46f2e..c09cceafa6e2e 100644 --- a/.gitlab/packaging/rpm.yml +++ b/.gitlab/packaging/rpm.yml @@ -29,6 +29,30 @@ cache: - !reference [.cache_omnibus_ruby_deps, cache] +.package_rpm_x86: + image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES + tags: ["arch:amd64"] + variables: + DD_PKG_ARCH: x86_64 + PACKAGE_ARCH: amd64 + +.package_rpm_arm64: + image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_arm64$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX:$DATADOG_AGENT_ARMBUILDIMAGES + tags: ["arch:arm64"] + variables: + PACKAGE_ARCH: arm64 + DD_PKG_ARCH: arm64 + +.package_rpm_agent_6: + variables: + RELEASE_VERSION: $RELEASE_VERSION_6 + AGENT_MAJOR_VERSION: 6 + +.package_rpm_agent_7: + variables: + RELEASE_VERSION: $RELEASE_VERSION_7 + AGENT_MAJOR_VERSION: 7 + .package_suse_rpm_common: extends: .package_rpm_common script: @@ -42,136 +66,71 @@ OMNIBUS_EXTRA_ARGS: "--host-distribution=suse" agent_rpm-x64-a6: - extends: .package_rpm_common - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] + extends: [.package_rpm_common, .package_rpm_agent_6, .package_rpm_x86] needs: ["datadog-agent-6-x64"] variables: - PACKAGE_ARCH: x86_64 - DD_PKG_ARCH: "x86_64" DD_PROJECT: agent - RELEASE_VERSION: $RELEASE_VERSION_6 - AGENT_MAJOR_VERSION: 6 agent_rpm-x64-a7: - extends: .package_rpm_common - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES + extends: [.package_rpm_common, .package_rpm_agent_7, .package_rpm_x86] tags: ["arch:amd64"] needs: ["datadog-agent-7-x64"] variables: - PACKAGE_ARCH: x86_64 - DD_PKG_ARCH: "x86_64" DD_PROJECT: agent - RELEASE_VERSION: $RELEASE_VERSION_7 - AGENT_MAJOR_VERSION: 7 agent_rpm-arm64-a6: - extends: .package_rpm_common - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_arm64$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX:$DATADOG_AGENT_ARMBUILDIMAGES - tags: ["arch:arm64"] + extends: [.package_rpm_common, .package_rpm_agent_6, .package_rpm_arm64] needs: ["datadog-agent-6-arm64"] variables: - PACKAGE_ARCH: arm64 - DD_PKG_ARCH: "arm64" DD_PROJECT: agent - RELEASE_VERSION: $RELEASE_VERSION_6 - AGENT_MAJOR_VERSION: 6 agent_rpm-arm64-a7: - extends: .package_rpm_common - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_arm64$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX:$DATADOG_AGENT_ARMBUILDIMAGES - tags: ["arch:arm64"] + extends: [.package_rpm_common, .package_rpm_agent_7, .package_rpm_arm64] needs: ["datadog-agent-7-arm64"] variables: - PACKAGE_ARCH: arm64 - DD_PKG_ARCH: "arm64" DD_PROJECT: agent - RELEASE_VERSION: $RELEASE_VERSION_7 - AGENT_MAJOR_VERSION: 7 agent_suse-x64-a6: - extends: .package_suse_rpm_common - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] + extends: [.package_suse_rpm_common, .package_rpm_agent_6, .package_rpm_x86] needs: ["datadog-agent-6-x64"] variables: - PACKAGE_ARCH: x86_64 - DD_PKG_ARCH: "x86_64" DD_PRODUCT: agent - RELEASE_VERSION: $RELEASE_VERSION_6 - AGENT_MAJOR_VERSION: 6 agent_suse-x64-a7: - extends: .package_suse_rpm_common - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] + extends: [.package_suse_rpm_common, .package_rpm_agent_7, .package_rpm_x86] needs: ["datadog-agent-7-x64"] variables: - PACKAGE_ARCH: x86_64 - DD_PKG_ARCH: "x86_64" DD_PRODUCT: agent - RELEASE_VERSION: $RELEASE_VERSION_7 - AGENT_MAJOR_VERSION: 7 agent_suse-arm64-a7: - extends: .package_suse_rpm_common - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_arm64$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX:$DATADOG_AGENT_ARMBUILDIMAGES - tags: ["arch:arm64"] + extends: [.package_suse_rpm_common, .package_rpm_agent_7, .package_rpm_arm64] needs: ["datadog-agent-7-arm64"] variables: - PACKAGE_ARCH: arm64 - DD_PKG_ARCH: "arm64" DD_PRODUCT: agent - RELEASE_VERSION: $RELEASE_VERSION_7 - AGENT_MAJOR_VERSION: 7 installer_rpm-amd64: - extends: .package_rpm_common - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] + extends: [.package_rpm_common, .package_rpm_agent_7, .package_rpm_x86] needs: ["installer-amd64"] variables: - PACKAGE_ARCH: x86_64 - DD_PKG_ARCH: "x86_64" DD_PROJECT: installer - RELEASE_VERSION: $RELEASE_VERSION_7 - AGENT_MAJOR_VERSION: 7 installer_rpm-arm64: - extends: .package_rpm_common - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_arm64$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX:$DATADOG_AGENT_ARMBUILDIMAGES - tags: ["arch:arm64"] + extends: [.package_rpm_common, .package_rpm_agent_7, .package_rpm_arm64] needs: ["installer-arm64"] variables: - PACKAGE_ARCH: aarch64 - DD_PKG_ARCH: "arm64" DD_PROJECT: installer - RELEASE_VERSION: $RELEASE_VERSION_7 - AGENT_MAJOR_VERSION: 7 installer_suse_rpm-amd64: - extends: .package_suse_rpm_common - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] + extends: [.package_suse_rpm_common, .package_rpm_agent_7, .package_rpm_x86] needs: ["installer-amd64"] variables: - PACKAGE_ARCH: x86_64 - DD_PKG_ARCH: "x86_64" DD_PROJECT: installer - RELEASE_VERSION: $RELEASE_VERSION_7 - AGENT_MAJOR_VERSION: 7 installer_suse_rpm-arm64: - extends: .package_suse_rpm_common - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_arm64$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX:$DATADOG_AGENT_ARMBUILDIMAGES - tags: ["arch:arm64"] + extends: [.package_suse_rpm_common, .package_rpm_agent_7, .package_rpm_arm64] needs: ["installer-arm64"] variables: - PACKAGE_ARCH: aarch64 - DD_PKG_ARCH: "arm64" DD_PROJECT: installer - RELEASE_VERSION: $RELEASE_VERSION_7 - AGENT_MAJOR_VERSION: 7 .package_iot_rpm_common: rules: @@ -203,24 +162,12 @@ installer_suse_rpm-arm64: - !reference [.cache_omnibus_ruby_deps, cache] iot_agent_rpm-x64: - extends: .package_iot_rpm_common - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] + extends: [.package_iot_rpm_common, .package_rpm_x86] needs: ["iot-agent-x64"] - variables: - PACKAGE_ARCH: x86_64 - DD_PKG_ARCH: "x86_64" - DD_PROJECT: agent iot_agent_rpm-arm64: - extends: .package_iot_rpm_common - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_arm64$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX:$DATADOG_AGENT_ARMBUILDIMAGES - tags: ["arch:arm64"] + extends: [.package_iot_rpm_common, .package_rpm_arm64] needs: ["iot-agent-arm64"] - variables: - PACKAGE_ARCH: aarch64 - DD_PKG_ARCH: "arm64" - DD_PROJECT: agent iot_agent_rpm-armhf: extends: .package_iot_rpm_common @@ -231,15 +178,12 @@ iot_agent_rpm-armhf: variables: PACKAGE_ARCH: armhf DD_PKG_ARCH: "arm64" - DD_PROJECT: agent before_script: # Ensures uname -m reports armv7l - export LD_PRELOAD="/usr/local/lib/libfakearmv7l.so" iot_agent_suse-x64: - extends: .package_iot_rpm_common - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] + extends: [.package_iot_rpm_common, .package_rpm_x86] needs: ["iot-agent-x64"] script: # Don't simply redefine OMNIBUS_PACKAGE_DIR since it also defines where the input @@ -252,10 +196,18 @@ iot_agent_suse-x64: paths: - $OMNIBUS_PACKAGE_DIR_SUSE variables: - PACKAGE_ARCH: x86_64 - DD_PKG_ARCH: "x86_64" - DD_PROJECT: agent OMNIBUS_EXTRA_ARGS: "--host-distribution=suse" cache: - !reference [.cache_omnibus_ruby_deps, cache] +dogstatsd_rpm-x64: + extends: [.package_rpm_common, .package_rpm_agent_7, .package_rpm_x86] + needs: ["dogstatsd-x64"] + variables: + DD_PROJECT: dogstatsd + +dogstatsd_suse-x64: + extends: [.package_suse_rpm_common, .package_rpm_agent_7, .package_rpm_x86] + needs: ["dogstatsd-x64"] + variables: + DD_PROJECT: dogstatsd diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9feb2c3fa503c..07cfdf919868c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -33,7 +33,7 @@ repos: - id: protected-branches name: protected-branches description: checks that the commit isn't created on a protected branch - entry: 'python3 -m tasks.git-hooks.protected-branches' + entry: 'inv git.check-protected-branch' language: system pass_filenames: false - id: govet @@ -46,10 +46,11 @@ repos: - id: copyright name: copyright description: copyright headers - entry: 'python3 -m tasks.git-hooks.copyright' + entry: 'inv linter.copyrights --only-staged-files' language: system require_serial: true files: \.go$ + pass_filenames: false - id: win-clang-format name: win-clang-format description: clang-format @@ -66,12 +67,13 @@ repos: exclude: '^pkg/ebpf/(c/bpf_endian|c/bpf_helpers|compiler/clang-stdarg).h$' - id: go-mod-tidy name: go-mod-tidy - description: check that go.mod files are tidy - entry: 'inv tidy --only-modified-packages' + description: check that all go.mod files are tidy + entry: 'inv tidy' language: system require_serial: true files: (\.go|^go\.mod|^go\.sum)$ pass_filenames: false + stages: [pre-push] - id: go-test name: go-test description: run go test on modified packages diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 4bc52d22e032a..5a90ce299135c 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -3085,6 +3085,7 @@ core,go.opentelemetry.io/collector/semconv/v1.13.0,Apache-2.0,Copyright The Open core,go.opentelemetry.io/collector/semconv/v1.16.0,Apache-2.0,Copyright The OpenTelemetry Authors core,go.opentelemetry.io/collector/semconv/v1.17.0,Apache-2.0,Copyright The OpenTelemetry Authors core,go.opentelemetry.io/collector/semconv/v1.18.0,Apache-2.0,Copyright The OpenTelemetry Authors +core,go.opentelemetry.io/collector/semconv/v1.21.0,Apache-2.0,Copyright The OpenTelemetry Authors core,go.opentelemetry.io/collector/semconv/v1.22.0,Apache-2.0,Copyright The OpenTelemetry Authors core,go.opentelemetry.io/collector/semconv/v1.6.1,Apache-2.0,Copyright The OpenTelemetry Authors core,go.opentelemetry.io/collector/semconv/v1.8.0,Apache-2.0,Copyright The OpenTelemetry Authors @@ -4261,6 +4262,7 @@ core,k8s.io/client-go/listers/storage/v1,Apache-2.0,Copyright 2014 The Kubernete core,k8s.io/client-go/listers/storage/v1alpha1,Apache-2.0,Copyright 2014 The Kubernetes Authors. core,k8s.io/client-go/listers/storage/v1beta1,Apache-2.0,Copyright 2014 The Kubernetes Authors. core,k8s.io/client-go/metadata,Apache-2.0,Copyright 2014 The Kubernetes Authors. +core,k8s.io/client-go/metadata/fake,Apache-2.0,Copyright 2014 The Kubernetes Authors. core,k8s.io/client-go/openapi,Apache-2.0,Copyright 2014 The Kubernetes Authors. core,k8s.io/client-go/openapi/cached,Apache-2.0,Copyright 2014 The Kubernetes Authors. core,k8s.io/client-go/pkg/apis/clientauthentication,Apache-2.0,Copyright 2014 The Kubernetes Authors. diff --git a/cmd/agent/subcommands/diagnose/command.go b/cmd/agent/subcommands/diagnose/command.go index 422bb09840518..1142645013d89 100644 --- a/cmd/agent/subcommands/diagnose/command.go +++ b/cmd/agent/subcommands/diagnose/command.go @@ -90,7 +90,8 @@ func Commands(globalParams *command.GlobalParams) []*cobra.Command { fx.Supply(cliParams), fx.Supply(core.BundleParams{ ConfigParams: config.NewAgentParams(globalParams.ConfFilePath), - LogParams: logimpl.ForOneShot("CORE", "off", true)}), + LogParams: logimpl.ForOneShot("CORE", "off", true), + }), core.Bundle(), // workloadmeta setup collectors.GetCatalog(), @@ -142,7 +143,7 @@ func Commands(globalParams *command.GlobalParams) []*cobra.Command { payloadV5Cmd := &cobra.Command{ Use: "v5", - Short: "Print the metadata payload for the agent.", + Short: "[internal] Print the metadata payload for the agent.", Long: ` This command print the V5 metadata payload for the Agent. This payload is used to populate the infra list and host map in Datadog. It's called 'V5' because it's the same payload sent since Agent V5. This payload is mandatory in order to create a new host in Datadog.`, RunE: func(cmd *cobra.Command, args []string) error { @@ -156,7 +157,7 @@ This command print the V5 metadata payload for the Agent. This payload is used t payloadGohaiCmd := &cobra.Command{ Use: "gohai", - Short: "Print the gohai payload for the agent.", + Short: "[internal] Print the gohai payload for the agent.", Long: ` This command prints the gohai data sent by the Agent, including current processes running on the machine.`, RunE: func(cmd *cobra.Command, args []string) error { @@ -170,7 +171,7 @@ This command prints the gohai data sent by the Agent, including current processe payloadInventoriesAgentCmd := &cobra.Command{ Use: "inventory-agent", - Short: "Print the Inventory agent metadata payload.", + Short: "[internal] Print the Inventory agent metadata payload.", Long: ` This command print the inventory-agent metadata payload. This payload is used by the 'inventories/sql' product.`, RunE: func(cmd *cobra.Command, args []string) error { @@ -184,7 +185,7 @@ This command print the inventory-agent metadata payload. This payload is used by payloadInventoriesHostCmd := &cobra.Command{ Use: "inventory-host", - Short: "Print the Inventory host metadata payload.", + Short: "[internal] Print the Inventory host metadata payload.", Long: ` This command print the inventory-host metadata payload. This payload is used by the 'inventories/sql' product.`, RunE: func(cmd *cobra.Command, args []string) error { @@ -198,7 +199,7 @@ This command print the inventory-host metadata payload. This payload is used by payloadInventoriesChecksCmd := &cobra.Command{ Use: "inventory-checks", - Short: "Print the Inventory checks metadata payload.", + Short: "[internal] Print the Inventory checks metadata payload.", Long: ` This command print the inventory-checks metadata payload. This payload is used by the 'inventories/sql' product.`, RunE: func(cmd *cobra.Command, args []string) error { @@ -212,7 +213,7 @@ This command print the inventory-checks metadata payload. This payload is used b payloadInventoriesPkgSigningCmd := &cobra.Command{ Use: "package-signing", - Short: "Print the Inventory package signing payload.", + Short: "[internal] Print the Inventory package signing payload.", Long: ` This command print the package-signing metadata payload. This payload is used by the 'fleet automation' product.`, RunE: func(cmd *cobra.Command, args []string) error { @@ -224,12 +225,27 @@ This command print the package-signing metadata payload. This payload is used by }, } + payloadSystemProbeCmd := &cobra.Command{ + Use: "system-probe", + Short: "[internal] Print the inventory systemprobe metadata payload.", + Long: ` +This command print the system-probe metadata payload. This payload is used by the 'fleet automation' product.`, + RunE: func(cmd *cobra.Command, args []string) error { + return fxutil.OneShot(printPayload, + fx.Supply(payloadName("system-probe")), + fx.Supply(command.GetDefaultCoreBundleParams(cliParams.GlobalParams)), + core.Bundle(), + ) + }, + } + showPayloadCommand.AddCommand(payloadV5Cmd) showPayloadCommand.AddCommand(payloadGohaiCmd) showPayloadCommand.AddCommand(payloadInventoriesAgentCmd) showPayloadCommand.AddCommand(payloadInventoriesHostCmd) showPayloadCommand.AddCommand(payloadInventoriesChecksCmd) showPayloadCommand.AddCommand(payloadInventoriesPkgSigningCmd) + showPayloadCommand.AddCommand(payloadSystemProbeCmd) diagnoseCommand.AddCommand(showPayloadCommand) return []*cobra.Command{diagnoseCommand} @@ -239,7 +255,8 @@ func cmdDiagnose(cliParams *cliParams, senderManager diagnosesendermanager.Component, wmeta optional.Option[workloadmeta.Component], ac autodiscovery.Component, - secretResolver secrets.Component) error { + secretResolver secrets.Component, +) error { diagCfg := diagnosis.Config{ Verbose: cliParams.verbose, RunLocal: cliParams.runLocal, diff --git a/cmd/agent/subcommands/diagnose/command_test.go b/cmd/agent/subcommands/diagnose/command_test.go index ffe1b4a5b94c0..c9f303f8e7745 100644 --- a/cmd/agent/subcommands/diagnose/command_test.go +++ b/cmd/agent/subcommands/diagnose/command_test.go @@ -85,3 +85,13 @@ func TestShowMetadataPkgSigningCommand(t *testing.T) { require.Equal(t, false, secretParams.Enabled) }) } + +func TestShowMetadataSystemProbeCommand(t *testing.T) { + fxutil.TestOneShotSubcommand(t, + Commands(&command.GlobalParams{}), + []string{"diagnose", "show-metadata", "system-probe"}, + printPayload, + func(coreParams core.BundleParams, secretParams secrets.Params) { + require.Equal(t, false, secretParams.Enabled) + }) +} diff --git a/cmd/agent/subcommands/run/command.go b/cmd/agent/subcommands/run/command.go index 7b5f598614080..f6d4df52d4dc7 100644 --- a/cmd/agent/subcommands/run/command.go +++ b/cmd/agent/subcommands/run/command.go @@ -83,6 +83,7 @@ import ( replay "github.com/DataDog/datadog-agent/comp/dogstatsd/replay/def" dogstatsdServer "github.com/DataDog/datadog-agent/comp/dogstatsd/server" dogstatsddebug "github.com/DataDog/datadog-agent/comp/dogstatsd/serverDebug" + "github.com/DataDog/datadog-agent/comp/dogstatsd/statsd" dogstatsdStatusimpl "github.com/DataDog/datadog-agent/comp/dogstatsd/status/statusimpl" "github.com/DataDog/datadog-agent/comp/forwarder" "github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder" @@ -101,12 +102,13 @@ import ( "github.com/DataDog/datadog-agent/comp/metadata/inventoryhost" "github.com/DataDog/datadog-agent/comp/metadata/packagesigning" "github.com/DataDog/datadog-agent/comp/metadata/runner" + systemprobemetadata "github.com/DataDog/datadog-agent/comp/metadata/systemprobe/def" "github.com/DataDog/datadog-agent/comp/ndmtmp" "github.com/DataDog/datadog-agent/comp/netflow" netflowServer "github.com/DataDog/datadog-agent/comp/netflow/server" "github.com/DataDog/datadog-agent/comp/networkpath" "github.com/DataDog/datadog-agent/comp/otelcol" - otelcollector "github.com/DataDog/datadog-agent/comp/otelcol/collector" + otelcollector "github.com/DataDog/datadog-agent/comp/otelcol/collector/def" "github.com/DataDog/datadog-agent/comp/otelcol/logsagentpipeline" processAgent "github.com/DataDog/datadog-agent/comp/process/agent" processagentStatusImpl "github.com/DataDog/datadog-agent/comp/process/status/statusimpl" @@ -219,6 +221,7 @@ func run(log log.Component, demultiplexer demultiplexer.Component, sharedSerializer serializer.MetricSerializer, logsAgent optional.Option[logsAgent.Component], + _ statsd.Component, processAgent processAgent.Component, otelcollector otelcollector.Component, _ host.Component, @@ -231,6 +234,7 @@ func run(log log.Component, _ langDetectionCl.Component, agentAPI internalAPI.Component, _ packagesigning.Component, + _ systemprobemetadata.Component, statusComponent status.Component, collector collector.Component, cloudfoundrycontainer cloudfoundrycontainer.Component, @@ -366,6 +370,7 @@ func getSharedFxOption() fx.Option { traceagentStatusImpl.Module(), processagentStatusImpl.Module(), dogstatsdStatusimpl.Module(), + statsd.Module(), statusimpl.Module(), authtokenimpl.Module(), apiimpl.Module(), diff --git a/cmd/agent/subcommands/run/command_windows.go b/cmd/agent/subcommands/run/command_windows.go index 6561b52e8ebfd..33bcde6fca29e 100644 --- a/cmd/agent/subcommands/run/command_windows.go +++ b/cmd/agent/subcommands/run/command_windows.go @@ -69,7 +69,7 @@ import ( "github.com/DataDog/datadog-agent/comp/metadata/packagesigning" "github.com/DataDog/datadog-agent/comp/metadata/runner" netflowServer "github.com/DataDog/datadog-agent/comp/netflow/server" - otelcollector "github.com/DataDog/datadog-agent/comp/otelcol/collector" + otelcollector "github.com/DataDog/datadog-agent/comp/otelcol/collector/def" processAgent "github.com/DataDog/datadog-agent/comp/process/agent" "github.com/DataDog/datadog-agent/comp/remote-config/rcclient" "github.com/DataDog/datadog-agent/pkg/serializer" diff --git a/cmd/agentless-scanner/main.go b/cmd/agentless-scanner/main.go deleted file mode 100644 index 98a166ae6932d..0000000000000 --- a/cmd/agentless-scanner/main.go +++ /dev/null @@ -1,10 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2023-present Datadog, Inc. - -// Package main implements the agentless-scanner command. -package main - -func main() { -} diff --git a/cmd/otel-agent/subcommands/run/command.go b/cmd/otel-agent/subcommands/run/command.go index 7b48903eb4a54..6d3ce3a3c663b 100644 --- a/cmd/otel-agent/subcommands/run/command.go +++ b/cmd/otel-agent/subcommands/run/command.go @@ -20,6 +20,8 @@ import ( corelogimpl "github.com/DataDog/datadog-agent/comp/core/log/logimpl" "github.com/DataDog/datadog-agent/comp/core/secrets" "github.com/DataDog/datadog-agent/comp/core/sysprobeconfig" + "github.com/DataDog/datadog-agent/comp/core/tagger" + "github.com/DataDog/datadog-agent/comp/core/tagger/taggerimpl" "github.com/DataDog/datadog-agent/comp/core/workloadmeta" "github.com/DataDog/datadog-agent/comp/forwarder" "github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder" @@ -30,7 +32,10 @@ import ( collectorfx "github.com/DataDog/datadog-agent/comp/otelcol/collector/fx" "github.com/DataDog/datadog-agent/comp/otelcol/logsagentpipeline" "github.com/DataDog/datadog-agent/comp/otelcol/logsagentpipeline/logsagentpipelineimpl" - configprovider "github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/pipeline/provider" + "go.opentelemetry.io/collector/otelcol" + + provider "github.com/DataDog/datadog-agent/comp/otelcol/provider/def" + providerfx "github.com/DataDog/datadog-agent/comp/otelcol/provider/fx" "github.com/DataDog/datadog-agent/comp/serializer/compression" "github.com/DataDog/datadog-agent/comp/serializer/compression/compressionimpl/strategy" "github.com/DataDog/datadog-agent/pkg/config/env" @@ -38,7 +43,6 @@ import ( "github.com/DataDog/datadog-agent/pkg/util/fxutil" "github.com/DataDog/datadog-agent/pkg/util/optional" "github.com/spf13/cobra" - "go.opentelemetry.io/collector/otelcol" "go.uber.org/fx" ) @@ -81,13 +85,14 @@ func runOTelAgentCommand(_ context.Context, params *subcommands.GlobalParams, op inventoryagentimpl.Module(), workloadmeta.Module(), hostnameimpl.Module(), + fx.Provide(tagger.NewTaggerParams), + taggerimpl.Module(), sysprobeconfig.NoneModule(), fetchonlyimpl.Module(), collectorfx.Module(), collectorcontribFx.Module(), - fx.Provide(configprovider.NewConfigProvider), - // For FX to provide the otelcol.ConfigProvider from the configprovider.ExtendedConfigProvider - fx.Provide(func(cp configprovider.ExtendedConfigProvider) otelcol.ConfigProvider { + providerfx.Module(), + fx.Provide(func(cp provider.Component) otelcol.ConfigProvider { return cp }), fx.Provide(func() (config.Component, error) { diff --git a/cmd/process-agent/command/main_common.go b/cmd/process-agent/command/main_common.go index 1fde7fa35abc5..005f27a65803d 100644 --- a/cmd/process-agent/command/main_common.go +++ b/cmd/process-agent/command/main_common.go @@ -54,7 +54,6 @@ import ( ddconfig "github.com/DataDog/datadog-agent/pkg/config" commonsettings "github.com/DataDog/datadog-agent/pkg/config/settings" "github.com/DataDog/datadog-agent/pkg/process/metadata/workloadmeta/collector" - "github.com/DataDog/datadog-agent/pkg/process/statsd" "github.com/DataDog/datadog-agent/pkg/process/util" ddutil "github.com/DataDog/datadog-agent/pkg/util" "github.com/DataDog/datadog-agent/pkg/util/fxutil" @@ -303,7 +302,6 @@ type miscDeps struct { Lc fx.Lifecycle Config config.Component - Statsd compstatsd.Component Syscfg sysprobeconfig.Component HostInfo hostinfo.Component WorkloadMeta workloadmeta.Component @@ -311,13 +309,9 @@ type miscDeps struct { } // initMisc initializes modules that cannot, or have not yet been componetized. -// Todo: (Components) WorkloadMeta, remoteTagger, statsd +// Todo: (Components) WorkloadMeta, remoteTagger // Todo: move metadata/workloadmeta/collector to workloadmeta func initMisc(deps miscDeps) error { - if err := statsd.Configure(ddconfig.GetBindHost(), deps.Config.GetInt("dogstatsd_port"), deps.Statsd.CreateForHostPort); err != nil { - deps.Logger.Criticalf("Error configuring statsd: %s", err) - return err - } if err := ddutil.SetupCoreDump(deps.Config); err != nil { deps.Logger.Warnf("Can't setup core dumps: %v, core dumps might not be available after a crash", err) diff --git a/cmd/serverless/dependencies_linux_amd64.txt b/cmd/serverless/dependencies_linux_amd64.txt index 8589b5f124db7..2d22c14306121 100644 --- a/cmd/serverless/dependencies_linux_amd64.txt +++ b/cmd/serverless/dependencies_linux_amd64.txt @@ -114,6 +114,7 @@ github.com/DataDog/datadog-agent/comp/otelcol/otlp github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/exporter/logsagentexporter github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/exporter/serializerexporter github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/processor/infraattributesprocessor +github.com/DataDog/datadog-agent/comp/otelcol/otlp/datatype github.com/DataDog/datadog-agent/comp/otelcol/otlp/internal/configutils github.com/DataDog/datadog-agent/comp/remote-config/rcclient/types github.com/DataDog/datadog-agent/comp/serializer/compression @@ -610,6 +611,7 @@ go.opentelemetry.io/collector/receiver/otlpreceiver/internal/trace go.opentelemetry.io/collector/receiver/receiverhelper go.opentelemetry.io/collector/semconv/v1.17.0 go.opentelemetry.io/collector/semconv/v1.18.0 +go.opentelemetry.io/collector/semconv/v1.21.0 go.opentelemetry.io/collector/semconv/v1.6.1 go.opentelemetry.io/collector/service go.opentelemetry.io/collector/service/extensions @@ -907,6 +909,7 @@ gopkg.in/DataDog/dd-trace-go.v1/internal/samplernames gopkg.in/DataDog/dd-trace-go.v1/internal/telemetry gopkg.in/DataDog/dd-trace-go.v1/internal/traceprof gopkg.in/DataDog/dd-trace-go.v1/internal/version +gopkg.in/ini.v1 gopkg.in/yaml.v2 gopkg.in/yaml.v3 hash @@ -915,32 +918,6 @@ hash/crc32 hash/fnv html html/template -internal/abi -internal/bisect -internal/bytealg -internal/coverage/rtcov -internal/cpu -internal/fmtsort -internal/goarch -internal/godebug -internal/godebugs -internal/goexperiment -internal/goos -internal/intern -internal/itoa -internal/nettrace -internal/oserror -internal/poll -internal/race -internal/reflectlite -internal/safefilepath -internal/saferio -internal/singleflight -internal/syscall/execenv -internal/syscall/unix -internal/sysinfo -internal/testlog -internal/unsafeheader io io/fs io/ioutil diff --git a/cmd/serverless/dependencies_linux_arm64.txt b/cmd/serverless/dependencies_linux_arm64.txt index 518a1dc9e0135..9158ba1484cbd 100644 --- a/cmd/serverless/dependencies_linux_arm64.txt +++ b/cmd/serverless/dependencies_linux_arm64.txt @@ -114,6 +114,7 @@ github.com/DataDog/datadog-agent/comp/otelcol/otlp github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/exporter/logsagentexporter github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/exporter/serializerexporter github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/processor/infraattributesprocessor +github.com/DataDog/datadog-agent/comp/otelcol/otlp/datatype github.com/DataDog/datadog-agent/comp/otelcol/otlp/internal/configutils github.com/DataDog/datadog-agent/comp/remote-config/rcclient/types github.com/DataDog/datadog-agent/comp/serializer/compression @@ -609,6 +610,7 @@ go.opentelemetry.io/collector/receiver/otlpreceiver/internal/trace go.opentelemetry.io/collector/receiver/receiverhelper go.opentelemetry.io/collector/semconv/v1.17.0 go.opentelemetry.io/collector/semconv/v1.18.0 +go.opentelemetry.io/collector/semconv/v1.21.0 go.opentelemetry.io/collector/semconv/v1.6.1 go.opentelemetry.io/collector/service go.opentelemetry.io/collector/service/extensions @@ -906,6 +908,7 @@ gopkg.in/DataDog/dd-trace-go.v1/internal/samplernames gopkg.in/DataDog/dd-trace-go.v1/internal/telemetry gopkg.in/DataDog/dd-trace-go.v1/internal/traceprof gopkg.in/DataDog/dd-trace-go.v1/internal/version +gopkg.in/ini.v1 gopkg.in/yaml.v2 gopkg.in/yaml.v3 hash @@ -914,32 +917,6 @@ hash/crc32 hash/fnv html html/template -internal/abi -internal/bisect -internal/bytealg -internal/coverage/rtcov -internal/cpu -internal/fmtsort -internal/goarch -internal/godebug -internal/godebugs -internal/goexperiment -internal/goos -internal/intern -internal/itoa -internal/nettrace -internal/oserror -internal/poll -internal/race -internal/reflectlite -internal/safefilepath -internal/saferio -internal/singleflight -internal/syscall/execenv -internal/syscall/unix -internal/sysinfo -internal/testlog -internal/unsafeheader io io/fs io/ioutil diff --git a/cmd/system-probe/api/config.go b/cmd/system-probe/api/config.go index 178730ae808e2..646ae49fafa8c 100644 --- a/cmd/system-probe/api/config.go +++ b/cmd/system-probe/api/config.go @@ -16,6 +16,7 @@ import ( // setupConfigHandlers adds the specific handlers for /config endpoints func setupConfigHandlers(r *mux.Router, settings settings.Component) { r.HandleFunc("/config", settings.GetFullConfig(getAggregatedNamespaces()...)).Methods("GET") + r.HandleFunc("/config/by-source", settings.GetFullConfigBySource()).Methods("GET") r.HandleFunc("/config/list-runtime", settings.ListConfigurable).Methods("GET") r.HandleFunc("/config/{setting}", settings.GetValue).Methods("GET") r.HandleFunc("/config/{setting}", settings.SetValue).Methods("POST") diff --git a/cmd/system-probe/subcommands/config/command.go b/cmd/system-probe/subcommands/config/command.go index 2bc71dde5373a..4eefd3aefa65f 100644 --- a/cmd/system-probe/subcommands/config/command.go +++ b/cmd/system-probe/subcommands/config/command.go @@ -64,6 +64,15 @@ func Commands(globalParams *command.GlobalParams) []*cobra.Command { RunE: oneShotRunE(showRuntimeConfiguration), } + cmd.AddCommand( + &cobra.Command{ + Use: "by-source", + Short: "Show the runtime configuration by source (ie: default, config file, env vars, ...)", + Long: ``, + RunE: oneShotRunE(showRuntimeConfigurationBySource), + }, + ) + cmd.AddCommand( &cobra.Command{ Use: "list-runtime", @@ -112,6 +121,22 @@ func showRuntimeConfiguration(sysprobeconfig sysprobeconfig.Component, _ *cliPar return nil } +func showRuntimeConfigurationBySource(sysprobeconfig sysprobeconfig.Component, _ *cliParams) error { + c, err := getClient(sysprobeconfig) + if err != nil { + return err + } + + config, err := c.FullConfigBySource() + if err != nil { + return err + } + + fmt.Println(config) + + return nil +} + func listRuntimeConfigurableValue(sysprobeconfig sysprobeconfig.Component, _ *cliParams) error { c, err := getClient(sysprobeconfig) if err != nil { diff --git a/cmd/system-probe/subcommands/config/command_test.go b/cmd/system-probe/subcommands/config/command_test.go index 32e6065dea601..661b5b9dcc7f5 100644 --- a/cmd/system-probe/subcommands/config/command_test.go +++ b/cmd/system-probe/subcommands/config/command_test.go @@ -44,3 +44,11 @@ func TestGetConfigValueCommand(t *testing.T) { getConfigValue, func() {}) } + +func TestShowConfigBySource(t *testing.T) { + fxutil.TestOneShotSubcommand(t, + Commands(&command.GlobalParams{}), + []string{"config", "by-source"}, + showRuntimeConfigurationBySource, + func() {}) +} diff --git a/comp/README.md b/comp/README.md index f1a2a767bdcb7..74bbcf539b1fb 100644 --- a/comp/README.md +++ b/comp/README.md @@ -309,6 +309,10 @@ Package resources implements a component to generate the 'resources' metadata pa Package runner implements a component to generate metadata payload at the right interval. +### [comp/metadata/systemprobe](https://pkg.go.dev/github.com/DataDog/datadog-agent/comp/metadata/systemprobe) + +Package def is the metadata provider for system-probe process + ## [comp/ndmtmp](https://pkg.go.dev/github.com/DataDog/datadog-agent/comp/ndmtmp) (Component Bundle) *Datadog Team*: network-device-monitoring @@ -368,6 +372,10 @@ Package collectorcontrib defines the OTel collector-contrib component Package logsagentpipeline contains logs agent pipeline component +### [comp/otelcol/provider](https://pkg.go.dev/github.com/DataDog/datadog-agent/comp/otelcol/provider) + + + ## [comp/process](https://pkg.go.dev/github.com/DataDog/datadog-agent/comp/process) (Component Bundle) *Datadog Team*: processes diff --git a/comp/core/config/go.mod b/comp/core/config/go.mod index 3fe6d96201b7a..45f927629d584 100644 --- a/comp/core/config/go.mod +++ b/comp/core/config/go.mod @@ -37,7 +37,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/fxutil v0.54.0-rc.2 github.com/DataDog/datadog-agent/pkg/util/optional v0.54.0-rc.2 github.com/DataDog/datadog-agent/pkg/util/winutil v0.54.0-rc.2 - github.com/DataDog/viper v1.13.4 + github.com/DataDog/viper v1.13.5 github.com/stretchr/testify v1.9.0 go.uber.org/fx v1.18.2 ) diff --git a/comp/core/config/go.sum b/comp/core/config/go.sum index 1144bfcc19ac1..6081ecc18017b 100644 --- a/comp/core/config/go.sum +++ b/comp/core/config/go.sum @@ -1,8 +1,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/DataDog/viper v1.13.4 h1:0SrZc3zvMAGgVKg96uP4DEJb13lK2Is9a4go7IIcFSE= -github.com/DataDog/viper v1.13.4/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= +github.com/DataDog/viper v1.13.5 h1:SZMcyMknYQN2jRY/40A16gUXexlNJOI8sDs1cWZnI64= +github.com/DataDog/viper v1.13.5/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= diff --git a/comp/core/configsync/configsyncimpl/sync_integration_test.go b/comp/core/configsync/configsyncimpl/sync_integration_test.go index 3c74d20a55d9e..808c3a14bcade 100644 --- a/comp/core/configsync/configsyncimpl/sync_integration_test.go +++ b/comp/core/configsync/configsyncimpl/sync_integration_test.go @@ -21,24 +21,6 @@ import ( ) func TestRunWithChan(t *testing.T) { - t.Run("server error", func(t *testing.T) { - var called bool - handler := func(w http.ResponseWriter, r *http.Request) { - called = true - w.WriteHeader(http.StatusInternalServerError) - } - - ctx, cancel := context.WithCancel(context.Background()) - cs := makeConfigSyncWithServer(t, ctx, handler) - - ch := make(chan time.Time, 1) - ch <- time.Now() - time.AfterFunc(100*time.Millisecond, cancel) - cs.runWithChan(ch) - - require.True(t, called) - }) - t.Run("success", func(t *testing.T) { var called bool handler := func(w http.ResponseWriter, r *http.Request) { diff --git a/comp/core/healthprobe/impl/healthprobe.go b/comp/core/healthprobe/impl/healthprobe.go index ccd6c1e95ee4a..c2a052156c190 100644 --- a/comp/core/healthprobe/impl/healthprobe.go +++ b/comp/core/healthprobe/impl/healthprobe.go @@ -111,6 +111,15 @@ func (rh readyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { healthHandler(rh.logsGoroutines, rh.log, health.GetReadyNonBlocking, w, r) } +type startupHandler struct { + logsGoroutines bool + log log.Component +} + +func (sh startupHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + healthHandler(sh.logsGoroutines, sh.log, health.GetStartupNonBlocking, w, r) +} + func buildServer(options healthprobeComponent.Options, log log.Component) *http.Server { r := mux.NewRouter() @@ -124,8 +133,14 @@ func buildServer(options healthprobeComponent.Options, log log.Component) *http. log: log, } + startupHandler := startupHandler{ + logsGoroutines: options.LogsGoroutines, + log: log, + } + r.Handle("/live", liveHandler) r.Handle("/ready", readyHandler) + r.Handle("/startup", startupHandler) // Default route for backward compatibility r.NewRoute().Handler(liveHandler) diff --git a/comp/core/log/go.mod b/comp/core/log/go.mod index 44d573bac7e97..1f9e92e37ffb2 100644 --- a/comp/core/log/go.mod +++ b/comp/core/log/go.mod @@ -68,7 +68,7 @@ require ( github.com/DataDog/go-sqllexer v0.0.9 // indirect github.com/DataDog/go-tuf v1.1.0-0.5.2 // indirect github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.14.0 // indirect - github.com/DataDog/viper v1.13.4 // indirect + github.com/DataDog/viper v1.13.5 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect diff --git a/comp/core/log/go.sum b/comp/core/log/go.sum index c33f2706ddf1b..f57da860c6479 100644 --- a/comp/core/log/go.sum +++ b/comp/core/log/go.sum @@ -9,8 +9,8 @@ github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4ti github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.14.0 h1:10TPqpTlIkmDPFWVIEZ4ZX3rWrCrx3rEoeoAooZr6LM= github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.14.0/go.mod h1:dvIWN9pA2zWNTw5rhDWZgzZnhcfpH++d+8d1SWW6xkY= -github.com/DataDog/viper v1.13.4 h1:0SrZc3zvMAGgVKg96uP4DEJb13lK2Is9a4go7IIcFSE= -github.com/DataDog/viper v1.13.4/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= +github.com/DataDog/viper v1.13.5 h1:SZMcyMknYQN2jRY/40A16gUXexlNJOI8sDs1cWZnI64= +github.com/DataDog/viper v1.13.5/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= diff --git a/comp/core/settings/component.go b/comp/core/settings/component.go index e16aef8e189fe..948323cea3947 100644 --- a/comp/core/settings/component.go +++ b/comp/core/settings/component.go @@ -53,8 +53,11 @@ type Component interface { // API related functions // Todo: (Components) Remove these functions once we can register routes using FX value groups + // GetFullConfig returns the full config GetFullConfig(namespaces ...string) http.HandlerFunc + // GetFullConfigBySource returns the full config by sources (config, default, env vars ...) + GetFullConfigBySource() http.HandlerFunc // GetValue allows to retrieve the runtime setting GetValue(w http.ResponseWriter, r *http.Request) // SetValue allows to modify the runtime setting diff --git a/comp/core/settings/settingsimpl/settings_mock.go b/comp/core/settings/settingsimpl/settings_mock.go index 0e7bd7b537a86..6b1b2743663d7 100644 --- a/comp/core/settings/settingsimpl/settings_mock.go +++ b/comp/core/settings/settingsimpl/settings_mock.go @@ -74,6 +74,11 @@ func (m mock) GetFullConfig(...string) http.HandlerFunc { return func(http.ResponseWriter, *http.Request) {} } +// GetFullConfigBySource returns the full config by sources +func (m mock) GetFullConfigBySource() http.HandlerFunc { + return func(http.ResponseWriter, *http.Request) {} +} + // GetValue allows to retrieve the runtime setting func (m mock) GetValue(http.ResponseWriter, *http.Request) {} diff --git a/comp/core/settings/settingsimpl/settingsimpl.go b/comp/core/settings/settingsimpl/settingsimpl.go index 660786b40d49a..81e50318fa72f 100644 --- a/comp/core/settings/settingsimpl/settingsimpl.go +++ b/comp/core/settings/settingsimpl/settingsimpl.go @@ -142,6 +142,32 @@ func (s *settingsRegistry) GetFullConfig(namespaces ...string) http.HandlerFunc } } +func (s *settingsRegistry) GetFullConfigBySource() http.HandlerFunc { + return func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + + settings := s.config.AllSettingsBySource() + + jsonData, err := json.Marshal(settings) + if err != nil { + s.log.Errorf("Unable to marshal config by layer: %s", err) + body, _ := json.Marshal(map[string]string{"error": err.Error()}) + http.Error(w, string(body), http.StatusInternalServerError) + return + } + + scrubbed, err := scrubber.ScrubJSON(jsonData) + if err != nil { + s.log.Errorf("Unable to scrub sensitive data from config by layer: %s", err) + body, _ := json.Marshal(map[string]string{"error": err.Error()}) + http.Error(w, string(body), http.StatusInternalServerError) + return + } + + _, _ = w.Write(scrubbed) + } +} + func (s *settingsRegistry) ListConfigurable(w http.ResponseWriter, _ *http.Request) { configurableSettings := make(map[string]settings.RuntimeSettingResponse) for name, setting := range s.RuntimeSettings() { diff --git a/comp/core/settings/settingsimpl/settingsimpl_test.go b/comp/core/settings/settingsimpl/settingsimpl_test.go index 4460c313ce1a0..4b987ad340060 100644 --- a/comp/core/settings/settingsimpl/settingsimpl_test.go +++ b/comp/core/settings/settingsimpl/settingsimpl_test.go @@ -116,6 +116,23 @@ func TestRuntimeSettings(t *testing.T) { assert.NotEqual(t, "", string(body)) }, }, + { + "GetFullConfigBySource", + func(t *testing.T, comp settings.Component) { + responseRecorder := httptest.NewRecorder() + request := httptest.NewRequest("GET", "http://agent.host/test/", nil) + + comp.GetFullConfigBySource()(responseRecorder, request) + resp := responseRecorder.Result() + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + + assert.Equal(t, 200, responseRecorder.Code) + // The full config is too big to assert against + // Ensure the response body is not empty to validate we wrote something + assert.NotEqual(t, "", string(body)) + }, + }, { "ListConfigurable", func(t *testing.T, comp settings.Component) { diff --git a/comp/core/status/statusimpl/go.mod b/comp/core/status/statusimpl/go.mod index 80ce4c5cb1102..6b0e67ad49a60 100644 --- a/comp/core/status/statusimpl/go.mod +++ b/comp/core/status/statusimpl/go.mod @@ -62,7 +62,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/system v0.54.0-rc.2 // indirect github.com/DataDog/datadog-agent/pkg/util/system/socket v0.54.0-rc.2 // indirect github.com/DataDog/datadog-agent/pkg/util/winutil v0.54.0-rc.2 // indirect - github.com/DataDog/viper v1.13.4 // indirect + github.com/DataDog/viper v1.13.5 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect github.com/davecgh/go-spew v1.1.1 // indirect diff --git a/comp/core/status/statusimpl/go.sum b/comp/core/status/statusimpl/go.sum index 48d73f03e354a..37e735d6066e9 100644 --- a/comp/core/status/statusimpl/go.sum +++ b/comp/core/status/statusimpl/go.sum @@ -1,8 +1,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/DataDog/viper v1.13.4 h1:0SrZc3zvMAGgVKg96uP4DEJb13lK2Is9a4go7IIcFSE= -github.com/DataDog/viper v1.13.4/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= +github.com/DataDog/viper v1.13.5 h1:SZMcyMknYQN2jRY/40A16gUXexlNJOI8sDs1cWZnI64= +github.com/DataDog/viper v1.13.5/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= diff --git a/comp/core/tagger/taggerimpl/collectors/workloadmeta_extract.go b/comp/core/tagger/taggerimpl/collectors/workloadmeta_extract.go index 09979ee6014a5..45570f2599e0f 100644 --- a/comp/core/tagger/taggerimpl/collectors/workloadmeta_extract.go +++ b/comp/core/tagger/taggerimpl/collectors/workloadmeta_extract.go @@ -49,6 +49,12 @@ const ( envVarVersion = "DD_VERSION" envVarService = "DD_SERVICE" + // OpenTelemetry SDK - Environment variables + // https://opentelemetry.io/docs/languages/sdk-configuration/general + // https://opentelemetry.io/docs/specs/semconv/resource/ + envVarOtelService = "OTEL_SERVICE_NAME" + envVarOtelResourceAttributes = "OTEL_RESOURCE_ATTRIBUTES" + // Docker label keys dockerLabelEnv = "com.datadoghq.tags.env" dockerLabelVersion = "com.datadoghq.tags.version" @@ -66,6 +72,16 @@ var ( envVarService: tagKeyService, } + otelStandardEnvKeys = map[string]string{ + envVarOtelService: tagKeyService, + } + + otelResourceAttributesMapping = map[string]string{ + "service.name": tagKeyService, + "service.version": tagKeyVersion, + "deployment.environment": tagKeyEnv, + } + lowCardOrchestratorEnvKeys = map[string]string{ "MARATHON_APP_ID": "marathon_app", @@ -212,6 +228,9 @@ func (c *WorkloadMetaCollector) handleContainer(ev workloadmeta.Event) []*types. // standard tags from environment c.extractFromMapWithFn(container.EnvVars, standardEnvKeys, tags.AddStandard) + // standard tags in OpenTelemetry SDK format from environment + c.addOpenTelemetryStandardTags(container, tags) + // orchestrator tags from environment c.extractFromMapWithFn(container.EnvVars, lowCardOrchestratorEnvKeys, tags.AddLow) c.extractFromMapWithFn(container.EnvVars, orchCardOrchestratorEnvKeys, tags.AddOrchestrator) @@ -650,6 +669,9 @@ func (c *WorkloadMetaCollector) extractTagsFromPodContainer(pod *workloadmeta.Ku // enrich with standard tags from environment variables c.extractFromMapWithFn(container.EnvVars, standardEnvKeys, tags.AddStandard) + // standard tags in OpenTelemetry SDK format from environment + c.addOpenTelemetryStandardTags(container, tags) + // container-specific tags provided through pod annotation annotation := fmt.Sprintf(podContainerTagsAnnotationFormat, containerName) c.extractTagsFromJSONInMap(annotation, pod.Annotations, tags) @@ -743,6 +765,18 @@ func (c *WorkloadMetaCollector) extractTagsFromJSONInMap(key string, input map[s } } +func (c *WorkloadMetaCollector) addOpenTelemetryStandardTags(container *workloadmeta.Container, tags *taglist.TagList) { + if otelResourceAttributes, ok := container.EnvVars[envVarOtelResourceAttributes]; ok { + for _, pair := range strings.Split(otelResourceAttributes, ",") { + fields := strings.SplitN(pair, "=", 2) + if tag, ok := otelResourceAttributesMapping[fields[0]]; ok { + tags.AddStandard(tag, fields[1]) + } + } + } + c.extractFromMapWithFn(container.EnvVars, otelStandardEnvKeys, tags.AddStandard) +} + func buildTaggerEntityID(entityID workloadmeta.EntityID) string { switch entityID.Kind { case workloadmeta.KindContainer: diff --git a/comp/core/tagger/taggerimpl/collectors/workloadmeta_test.go b/comp/core/tagger/taggerimpl/collectors/workloadmeta_test.go index cf05ffc883b9e..5258cb03445c8 100644 --- a/comp/core/tagger/taggerimpl/collectors/workloadmeta_test.go +++ b/comp/core/tagger/taggerimpl/collectors/workloadmeta_test.go @@ -28,6 +28,7 @@ import ( func TestHandleKubePod(t *testing.T) { const ( fullyFleshedContainerID = "foobarquux" + otelEnvContainerID = "otelcontainer" noEnvContainerID = "foobarbaz" containerName = "agent" runtimeContainerName = "k8s_datadog-agent_agent" @@ -84,6 +85,20 @@ func TestHandleKubePod(t *testing.T) { "DD_VERSION": version, }, }) + store.Set(&workloadmeta.Container{ + EntityID: workloadmeta.EntityID{ + Kind: workloadmeta.KindContainer, + ID: otelEnvContainerID, + }, + EntityMeta: workloadmeta.EntityMeta{ + Name: runtimeContainerName, + }, + Image: image, + EnvVars: map[string]string{ + "OTEL_SERVICE_NAME": svc, + "OTEL_RESOURCE_ATTRIBUTES": fmt.Sprintf("service.name=%s,service.version=%s,deployment.environment=%s", svc, version, env), + }, + }) store.Set(&workloadmeta.Container{ EntityID: workloadmeta.EntityID{ Kind: workloadmeta.KindContainer, @@ -279,6 +294,57 @@ func TestHandleKubePod(t *testing.T) { }, }, }, + { + name: "pod with fully formed container, standard tags from env with opentelemetry sdk", + pod: workloadmeta.KubernetesPod{ + EntityID: podEntityID, + EntityMeta: workloadmeta.EntityMeta{ + Name: podName, + Namespace: podNamespace, + }, + Containers: []workloadmeta.OrchestratorContainer{ + { + ID: otelEnvContainerID, + Name: containerName, + Image: image, + }, + }, + }, + expected: []*types.TagInfo{ + { + Source: podSource, + Entity: podTaggerEntityID, + HighCardTags: []string{}, + OrchestratorCardTags: []string{ + fmt.Sprintf("pod_name:%s", podName), + }, + LowCardTags: []string{ + fmt.Sprintf("kube_namespace:%s", podNamespace), + }, + StandardTags: []string{}, + }, + { + Source: podSource, + Entity: fmt.Sprintf("container_id://%s", otelEnvContainerID), + HighCardTags: []string{ + fmt.Sprintf("container_id:%s", otelEnvContainerID), + fmt.Sprintf("display_container_name:%s_%s", runtimeContainerName, podName), + }, + OrchestratorCardTags: []string{ + fmt.Sprintf("pod_name:%s", podName), + }, + LowCardTags: append([]string{ + fmt.Sprintf("kube_namespace:%s", podNamespace), + fmt.Sprintf("kube_container_name:%s", containerName), + "image_id:datadog/agent@sha256:a63d3f66fb2f69d955d4f2ca0b229385537a77872ffc04290acae65aed5317d2", + "image_name:datadog/agent", + "image_tag:latest", + "short_image:agent", + }, standardTags...), + StandardTags: standardTags, + }, + }, + }, { name: "pod with container, standard tags from labels", pod: workloadmeta.KubernetesPod{ @@ -990,6 +1056,42 @@ func TestHandleContainer(t *testing.T) { }, }, }, + { + name: "tags from environment with opentelemetry sdk", + container: workloadmeta.Container{ + EntityID: entityID, + EntityMeta: workloadmeta.EntityMeta{ + Name: containerName, + }, + EnvVars: map[string]string{ + // env as tags + "TEAM": "container-integrations", + "TIER": "node", + + // otel standard tags + "OTEL_SERVICE_NAME": svc, + "OTEL_RESOURCE_ATTRIBUTES": fmt.Sprintf("service.name=%s,service.version=%s,deployment.environment=%s", svc, version, env), + }, + }, + envAsTags: map[string]string{ + "team": "owner_team", + }, + expected: []*types.TagInfo{ + { + Source: containerSource, + Entity: taggerEntityID, + HighCardTags: []string{ + fmt.Sprintf("container_name:%s", containerName), + fmt.Sprintf("container_id:%s", entityID.ID), + }, + OrchestratorCardTags: []string{}, + LowCardTags: append([]string{ + "owner_team:container-integrations", + }, standardTags...), + StandardTags: standardTags, + }, + }, + }, { name: "tags from labels", container: workloadmeta.Container{ diff --git a/comp/core/workloadmeta/collectors/internal/containerd/containerd.go b/comp/core/workloadmeta/collectors/internal/containerd/containerd.go index cc3bd072fcf89..943214a54f7a5 100644 --- a/comp/core/workloadmeta/collectors/internal/containerd/containerd.go +++ b/comp/core/workloadmeta/collectors/internal/containerd/containerd.go @@ -75,7 +75,7 @@ var containerdTopics = []string{ } type exitInfo struct { - exitCode *uint32 + exitCode *int64 exitTS time.Time } @@ -420,7 +420,7 @@ func (c *collector) deleteExitInfo(id string) { delete(c.contToExitInfo, id) } -func (c *collector) cacheExitInfo(id string, exitCode *uint32, exitTS time.Time) { +func (c *collector) cacheExitInfo(id string, exitCode *int64, exitTS time.Time) { c.contToExitInfo[id] = &exitInfo{ exitTS: exitTS, exitCode: exitCode, diff --git a/comp/core/workloadmeta/collectors/internal/containerd/event_builder.go b/comp/core/workloadmeta/collectors/internal/containerd/event_builder.go index af4d4a312ff8c..7faaed0026f58 100644 --- a/comp/core/workloadmeta/collectors/internal/containerd/event_builder.go +++ b/comp/core/workloadmeta/collectors/internal/containerd/event_builder.go @@ -18,6 +18,7 @@ import ( "github.com/DataDog/datadog-agent/comp/core/workloadmeta" cutil "github.com/DataDog/datadog-agent/pkg/util/containerd" + "github.com/DataDog/datadog-agent/pkg/util/pointer" ) var errNoContainer = errors.New("no container") @@ -45,7 +46,7 @@ func (c *collector) buildCollectorEvent( return workloadmeta.CollectorEvent{}, err } - c.cacheExitInfo(containerID, &exited.ExitStatus, exited.ExitedAt.AsTime()) + c.cacheExitInfo(containerID, pointer.Ptr(int64(exited.ExitStatus)), exited.ExitedAt.AsTime()) return createSetEvent(container, containerdEvent.Namespace, c.containerdClient, store) case TaskDeleteTopic: @@ -54,7 +55,7 @@ func (c *collector) buildCollectorEvent( return workloadmeta.CollectorEvent{}, err } - c.cacheExitInfo(containerID, &deleted.ExitStatus, deleted.ExitedAt.AsTime()) + c.cacheExitInfo(containerID, pointer.Ptr(int64(deleted.ExitStatus)), deleted.ExitedAt.AsTime()) return createSetEvent(container, containerdEvent.Namespace, c.containerdClient, store) case TaskStartTopic, TaskOOMTopic, TaskPausedTopic, TaskResumedTopic: diff --git a/comp/core/workloadmeta/collectors/internal/containerd/event_builder_test.go b/comp/core/workloadmeta/collectors/internal/containerd/event_builder_test.go index 71dc70e76fb1e..3614df60e38b0 100644 --- a/comp/core/workloadmeta/collectors/internal/containerd/event_builder_test.go +++ b/comp/core/workloadmeta/collectors/internal/containerd/event_builder_test.go @@ -52,7 +52,7 @@ func TestBuildCollectorEvent(t *testing.T) { }, } - exitCode := uint32(137) + exitCode := int64(137) exitTime := time.Now() fakeExitInfo := &exitInfo{exitCode: &exitCode, exitTS: exitTime} diff --git a/comp/core/workloadmeta/collectors/internal/docker/docker.go b/comp/core/workloadmeta/collectors/internal/docker/docker.go index 16c5de06d4ccc..42dcd7626d175 100644 --- a/comp/core/workloadmeta/collectors/internal/docker/docker.go +++ b/comp/core/workloadmeta/collectors/internal/docker/docker.go @@ -321,13 +321,13 @@ func (c *collector) buildCollectorEvent(ctx context.Context, ev *docker.Containe } case events.ActionDie, docker.ActionDied: - var exitCode *uint32 + var exitCode *int64 if exitCodeString, found := ev.Attributes["exitCode"]; found { - exitCodeInt, err := strconv.ParseInt(exitCodeString, 10, 32) + exitCodeInt, err := strconv.ParseInt(exitCodeString, 10, 64) if err != nil { log.Debugf("Cannot convert exit code %q: %v", exitCodeString, err) } else { - exitCode = pointer.Ptr(uint32(exitCodeInt)) + exitCode = pointer.Ptr(exitCodeInt) } } diff --git a/comp/core/workloadmeta/collectors/internal/kubeapiserver/kubeapiserver.go b/comp/core/workloadmeta/collectors/internal/kubeapiserver/kubeapiserver.go index f8b7e49a9b0c9..49292be61c2b8 100644 --- a/comp/core/workloadmeta/collectors/internal/kubeapiserver/kubeapiserver.go +++ b/comp/core/workloadmeta/collectors/internal/kubeapiserver/kubeapiserver.go @@ -14,11 +14,14 @@ import ( "github.com/DataDog/datadog-agent/comp/core/workloadmeta" "github.com/DataDog/datadog-agent/pkg/config" + "github.com/DataDog/datadog-agent/pkg/config/model" "github.com/DataDog/datadog-agent/pkg/status/health" "github.com/DataDog/datadog-agent/pkg/util/kubernetes/apiserver" "github.com/DataDog/datadog-agent/pkg/util/log" "go.uber.org/fx" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/discovery" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" ) @@ -32,7 +35,7 @@ const ( // storeGenerator returns a new store specific to a given resource type storeGenerator func(context.Context, workloadmeta.Component, kubernetes.Interface) (*cache.Reflector, *reflectorStore) -func storeGenerators(cfg config.Reader) []storeGenerator { +func storeGenerators(cfg model.Reader) []storeGenerator { generators := []storeGenerator{newNodeStore} if cfg.GetBool("cluster_agent.collect_kubernetes_tags") || cfg.GetBool("autoscaling.workload.enabled") { @@ -50,6 +53,17 @@ func storeGenerators(cfg config.Reader) []storeGenerator { return generators } +func metadataCollectionGVRs(cfg model.Reader, discoveryClient discovery.DiscoveryInterface) ([]schema.GroupVersionResource, error) { + if !cfg.GetBool("cluster_agent.kube_metadata_collection.enabled") { + return []schema.GroupVersionResource{}, nil + } + + requestedResources := cfg.GetStringSlice("cluster_agent.kube_metadata_collection.resources") + + discoveredResourcesGVs, err := discoverGVRs(discoveryClient, requestedResources) + return discoveredResourcesGVs, err +} + type collector struct { id string catalog workloadmeta.AgentType @@ -79,6 +93,25 @@ func (c *collector) Start(ctx context.Context, wlmetaStore workloadmeta.Componen } client := apiserverClient.InformerCl + metadataclient, err := apiserverClient.MetadataClient() + if err != nil { + return err + } + + // Initialize metadata collection informers + // TODO(components): do not use the config.Datadog reference, use a component instead + gvrs, err := metadataCollectionGVRs(config.Datadog(), client.Discovery()) + + if err != nil { + log.Errorf("failed to discover Group and Version of requested resources: %v", err) + } else { + for _, gvr := range gvrs { + reflector, store := newMetadataStore(ctx, wlmetaStore, metadataclient, gvr) + objectStores = append(objectStores, store) + go reflector.Run(ctx.Done()) + } + } + // TODO(components): do not use the config.Datadog reference, use a component instead for _, storeBuilder := range storeGenerators(config.Datadog()) { reflector, store := storeBuilder(ctx, wlmetaStore, client) diff --git a/comp/core/workloadmeta/collectors/internal/kubeapiserver/kubeapiserver_test.go b/comp/core/workloadmeta/collectors/internal/kubeapiserver/kubeapiserver_test.go index f10b23bda0aa2..33b66cb25a5a9 100644 --- a/comp/core/workloadmeta/collectors/internal/kubeapiserver/kubeapiserver_test.go +++ b/comp/core/workloadmeta/collectors/internal/kubeapiserver/kubeapiserver_test.go @@ -8,10 +8,16 @@ package kubeapiserver import ( + "reflect" "strings" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + fakediscovery "k8s.io/client-go/discovery/fake" + fakeclientset "k8s.io/client-go/kubernetes/fake" "github.com/DataDog/datadog-agent/pkg/config" ) @@ -109,3 +115,246 @@ func collectResultStoreGenerator(funcs []storeGenerator) []*reflectorStore { } return stores } + +func Test_metadataCollectionGVRs_WithFunctionalDiscovery(t *testing.T) { + tests := []struct { + name string + apiServerResourceList []*metav1.APIResourceList + expectedGVRs []schema.GroupVersionResource + cfg map[string]interface{} + }{ + { + name: "no requested resources, no resources at all!", + apiServerResourceList: []*metav1.APIResourceList{}, + expectedGVRs: []schema.GroupVersionResource{}, + cfg: map[string]interface{}{ + "cluster_agent.kube_metadata_collection.enabled": true, + "cluster_agent.kube_metadata_collection.resources": "", + }, + }, + { + name: "requested resources, but no resources at all!", + apiServerResourceList: []*metav1.APIResourceList{}, + expectedGVRs: []schema.GroupVersionResource{}, + cfg: map[string]interface{}{ + "cluster_agent.kube_metadata_collection.enabled": true, + "cluster_agent.kube_metadata_collection.resources": "deployments", + }, + }, + { + name: "only one resource (deployments), only one version, correct resource requested", + apiServerResourceList: []*metav1.APIResourceList{ + { + GroupVersion: "apps/v1", + APIResources: []metav1.APIResource{ + { + Name: "deployments", + Kind: "Deployment", + Namespaced: true, + }, + }, + }, + }, + expectedGVRs: []schema.GroupVersionResource{{Resource: "deployments", Group: "apps", Version: "v1"}}, + cfg: map[string]interface{}{ + "cluster_agent.kube_metadata_collection.enabled": true, + "cluster_agent.kube_metadata_collection.resources": "deployments", + }, + }, + { + name: "only one resource (deployments), only one version, wrong resource requested", + apiServerResourceList: []*metav1.APIResourceList{ + { + GroupVersion: "apps/v1", + APIResources: []metav1.APIResource{ + { + Name: "deployments", + Kind: "Deployment", + Namespaced: true, + }, + }, + }, + }, + expectedGVRs: []schema.GroupVersionResource{}, + cfg: map[string]interface{}{ + "cluster_agent.kube_metadata_collection.enabled": true, + "cluster_agent.kube_metadata_collection.resources": "daemonsets", + }, + }, + { + name: "multiple resources (deployments, statefulsets), multiple versions, all resources requested", + apiServerResourceList: []*metav1.APIResourceList{ + { + GroupVersion: "apps/v1", + APIResources: []metav1.APIResource{ + { + Name: "deployments", + Kind: "Deployment", + Namespaced: true, + }, + }, + }, + { + GroupVersion: "apps/v1beta1", + APIResources: []metav1.APIResource{ + { + Name: "deployments", + Kind: "Deployment", + Namespaced: true, + }, + }, + }, + { + GroupVersion: "apps/v1", + APIResources: []metav1.APIResource{ + { + Name: "statefulsets", + Kind: "StatefulSet", + Namespaced: true, + }, + }, + }, + { + GroupVersion: "apps/v1beta1", + APIResources: []metav1.APIResource{ + { + Name: "statefulsets", + Kind: "StatefulSet", + Namespaced: true, + }, + }, + }, + }, + expectedGVRs: []schema.GroupVersionResource{ + {Resource: "deployments", Group: "apps", Version: "v1"}, + {Resource: "statefulsets", Group: "apps", Version: "v1"}, + }, + cfg: map[string]interface{}{ + "cluster_agent.kube_metadata_collection.enabled": true, + "cluster_agent.kube_metadata_collection.resources": "deployments statefulsets", + }, + }, + { + name: "multiple resources (deployments, statefulsets), multiple versions, only one resource requested", + apiServerResourceList: []*metav1.APIResourceList{ + { + GroupVersion: "apps/v1", + APIResources: []metav1.APIResource{ + { + Name: "deployments", + Kind: "Deployment", + Namespaced: true, + }, + }, + }, + { + GroupVersion: "apps/v1beta1", + APIResources: []metav1.APIResource{ + { + Name: "deployments", + Kind: "Deployment", + Namespaced: true, + }, + }, + }, + { + GroupVersion: "apps/v1", + APIResources: []metav1.APIResource{ + { + Name: "statefulsets", + Kind: "StatefulSet", + Namespaced: true, + }, + }, + }, + { + GroupVersion: "apps/v1beta1", + APIResources: []metav1.APIResource{ + { + Name: "statefulsets", + Kind: "StatefulSet", + Namespaced: true, + }, + }, + }, + }, + expectedGVRs: []schema.GroupVersionResource{{Resource: "deployments", Group: "apps", Version: "v1"}}, + cfg: map[string]interface{}{ + "cluster_agent.kube_metadata_collection.enabled": true, + "cluster_agent.kube_metadata_collection.resources": "deployments", + }, + }, + { + name: "multiple resources (deployments, statefulsets), multiple versions, two resources requested (one with a typo)", + apiServerResourceList: []*metav1.APIResourceList{ + { + GroupVersion: "apps/v1", + APIResources: []metav1.APIResource{ + { + Name: "deployments", + Kind: "Deployment", + Namespaced: true, + }, + }, + }, + { + GroupVersion: "apps/v1beta1", + APIResources: []metav1.APIResource{ + { + Name: "deployments", + Kind: "Deployment", + Namespaced: true, + }, + }, + }, + { + GroupVersion: "apps/v1", + APIResources: []metav1.APIResource{ + { + Name: "statefulsets", + Kind: "StatefulSet", + Namespaced: true, + }, + }, + }, + { + GroupVersion: "apps/v1beta1", + APIResources: []metav1.APIResource{ + { + Name: "statefulsets", + Kind: "StatefulSet", + Namespaced: true, + }, + }, + }, + }, + expectedGVRs: []schema.GroupVersionResource{ + {Resource: "deployments", Group: "apps", Version: "v1"}, + }, + cfg: map[string]interface{}{ + "cluster_agent.kube_metadata_collection.enabled": true, + "cluster_agent.kube_metadata_collection.resources": "deployments statefulsetsy", + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(tt *testing.T) { + cfg := config.NewConfig("datadog", "DD", strings.NewReplacer(".", "_")) + for k, v := range test.cfg { + cfg.SetWithoutSource(k, v) + } + + client := fakeclientset.NewSimpleClientset() + fakeDiscoveryClient, ok := client.Discovery().(*fakediscovery.FakeDiscovery) + assert.Truef(t, ok, "Failed to initialise fake discovery client") + + fakeDiscoveryClient.Resources = test.apiServerResourceList + + discoveredGVRs, err := metadataCollectionGVRs(cfg, fakeDiscoveryClient) + require.NoErrorf(t, err, "Function should not have returned an error") + + assert.Truef(t, reflect.DeepEqual(discoveredGVRs, test.expectedGVRs), "Expected %v but got %v.", test.expectedGVRs, discoveredGVRs) + }) + } +} diff --git a/comp/core/workloadmeta/collectors/internal/kubeapiserver/metadata.go b/comp/core/workloadmeta/collectors/internal/kubeapiserver/metadata.go new file mode 100644 index 0000000000000..b3a0c0aceb32c --- /dev/null +++ b/comp/core/workloadmeta/collectors/internal/kubeapiserver/metadata.go @@ -0,0 +1,82 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +//go:build kubeapiserver + +package kubeapiserver + +import ( + "context" + "fmt" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/metadata" + "k8s.io/client-go/tools/cache" + + "github.com/DataDog/datadog-agent/comp/core/workloadmeta" +) + +func newMetadataStore(ctx context.Context, wlmetaStore workloadmeta.Component, metadataclient metadata.Interface, gvr schema.GroupVersionResource) (*cache.Reflector, *reflectorStore) { + metadataListerWatcher := &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + return metadataclient.Resource(gvr).List(ctx, options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + return metadataclient.Resource(gvr).Watch(ctx, options) + }, + } + + metadataStore := &reflectorStore{ + wlmetaStore: wlmetaStore, + seen: make(map[string]workloadmeta.EntityID), + parser: newMetadataParser(gvr), + filter: nil, + } + metadataReflector := cache.NewNamedReflector( + componentName, + metadataListerWatcher, + &metav1.PartialObjectMetadata{}, + metadataStore, + noResync, + ) + return metadataReflector, metadataStore +} + +type metadataParser struct { + gvr schema.GroupVersionResource +} + +func newMetadataParser(gvr schema.GroupVersionResource) objectParser { + return metadataParser{gvr} +} + +// generateEntityID generates and returns a unique entity id for KubernetesMetadata entity +// for namespaced objects, the id will have the format {resourceType}/{namespace}/{name} (e.g. deployments/default/app ) +// for cluster scoped objects, the id will have the format {resourceType}//{name} (e.g. node//master-node) +func (p metadataParser) generateEntityID(resource, namespace, name string) string { + return fmt.Sprintf("%s/%s/%s", resource, namespace, name) +} + +func (p metadataParser) Parse(obj interface{}) workloadmeta.Entity { + partialObjectMetadata := obj.(*metav1.PartialObjectMetadata) + id := p.generateEntityID(p.gvr.Resource, partialObjectMetadata.Namespace, partialObjectMetadata.Name) + + return &workloadmeta.KubernetesMetadata{ + EntityID: workloadmeta.EntityID{ + Kind: workloadmeta.KindKubernetesMetadata, + ID: id, + }, + EntityMeta: workloadmeta.EntityMeta{ + Name: partialObjectMetadata.Name, + Namespace: partialObjectMetadata.Namespace, + Labels: partialObjectMetadata.Labels, + Annotations: partialObjectMetadata.Annotations, + }, + GVR: p.gvr, + } +} diff --git a/comp/core/workloadmeta/collectors/internal/kubeapiserver/metadata_test.go b/comp/core/workloadmeta/collectors/internal/kubeapiserver/metadata_test.go new file mode 100644 index 0000000000000..a33561cdc3f0b --- /dev/null +++ b/comp/core/workloadmeta/collectors/internal/kubeapiserver/metadata_test.go @@ -0,0 +1,154 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +//go:build kubeapiserver && test + +package kubeapiserver + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + + "github.com/DataDog/datadog-agent/comp/core/workloadmeta" + "github.com/DataDog/datadog-agent/pkg/util/kubernetes" +) + +func TestParse_ParsePartialObjectMetadata(t *testing.T) { + + testcases := []struct { + name string + gvr schema.GroupVersionResource + partialObjectMetadata *metav1.PartialObjectMetadata + expected *workloadmeta.KubernetesMetadata + }{ + { + name: "deployments [namespace scoped]", + gvr: schema.GroupVersionResource{ + Group: "apps", + Version: "v1", + Resource: "deployments", + }, + partialObjectMetadata: &metav1.PartialObjectMetadata{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-app", + Namespace: "default", + Labels: map[string]string{"l1": "v1", "l2": "v2", "l3": "v3"}, + Annotations: map[string]string{"k1": "v1", "k2": "v2", "k3": "v3"}, + }, + }, + expected: &workloadmeta.KubernetesMetadata{ + EntityID: workloadmeta.EntityID{ + Kind: workloadmeta.KindKubernetesMetadata, + ID: "deployments/default/test-app", + }, + EntityMeta: workloadmeta.EntityMeta{ + Name: "test-app", + Namespace: "default", + Labels: map[string]string{"l1": "v1", "l2": "v2", "l3": "v3"}, + Annotations: map[string]string{"k1": "v1", "k2": "v2", "k3": "v3"}, + }, + GVR: schema.GroupVersionResource{ + Group: "apps", + Version: "v1", + Resource: "deployments", + }, + }, + }, + { + name: "namespaces [cluster scoped]", + gvr: schema.GroupVersionResource{ + Group: "", + Version: "v1", + Resource: "namespaces", + }, + partialObjectMetadata: &metav1.PartialObjectMetadata{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-namespace", + Namespace: "", + Labels: map[string]string{"l1": "v1", "l2": "v2", "l3": "v3"}, + Annotations: map[string]string{"k1": "v1", "k2": "v2", "k3": "v3"}, + }, + }, + expected: &workloadmeta.KubernetesMetadata{ + EntityID: workloadmeta.EntityID{ + Kind: workloadmeta.KindKubernetesMetadata, + ID: "namespaces//test-namespace", + }, + EntityMeta: workloadmeta.EntityMeta{ + Name: "test-namespace", + Namespace: "", + Labels: map[string]string{"l1": "v1", "l2": "v2", "l3": "v3"}, + Annotations: map[string]string{"k1": "v1", "k2": "v2", "k3": "v3"}, + }, + GVR: schema.GroupVersionResource{ + Group: "", + Version: "v1", + Resource: "namespaces", + }, + }, + }, + } + + for _, test := range testcases { + t.Run(test.name, func(tt *testing.T) { + parser := newMetadataParser(test.gvr) + entity := parser.Parse(test.partialObjectMetadata) + storedMetadata, ok := entity.(*workloadmeta.KubernetesMetadata) + require.True(t, ok) + assert.Equal(t, test.expected, storedMetadata) + }) + } +} + +func Test_MetadataFakeClient(t *testing.T) { + ns := "default" + gvr := schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "deployments"} + objectMeta := metav1.ObjectMeta{ + Name: "test-app", + Namespace: ns, + Labels: map[string]string{"test-label": "test-value"}, + Annotations: map[string]string{"k": "v"}, + } + + createObjects := func() []runtime.Object { + return []runtime.Object{ + &metav1.PartialObjectMetadata{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "apps/v1", + Kind: kubernetes.DeploymentKind, + }, + ObjectMeta: objectMeta, + }, + } + } + + expected := workloadmeta.EventBundle{ + Events: []workloadmeta.Event{ + { + Type: workloadmeta.EventTypeSet, + Entity: &workloadmeta.KubernetesMetadata{ + EntityID: workloadmeta.EntityID{ + ID: "deployments/default/test-app", + Kind: workloadmeta.KindKubernetesMetadata, + }, + EntityMeta: workloadmeta.EntityMeta{ + Name: objectMeta.Name, + Namespace: "default", + Labels: objectMeta.Labels, + Annotations: objectMeta.Annotations, + }, + GVR: gvr, + }, + }, + }, + } + + testCollectMetadataEvent(t, createObjects, gvr, expected) +} diff --git a/comp/core/workloadmeta/collectors/internal/kubeapiserver/test_helpers.go b/comp/core/workloadmeta/collectors/internal/kubeapiserver/test_helpers.go index 5aa166cbfb80f..c4e4e353276d1 100644 --- a/comp/core/workloadmeta/collectors/internal/kubeapiserver/test_helpers.go +++ b/comp/core/workloadmeta/collectors/internal/kubeapiserver/test_helpers.go @@ -9,17 +9,22 @@ package kubeapiserver import ( "context" + "fmt" "testing" "time" + "github.com/stretchr/testify/assert" + "go.uber.org/fx" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/kubernetes/fake" + metafake "k8s.io/client-go/metadata/fake" + "github.com/DataDog/datadog-agent/comp/core" "github.com/DataDog/datadog-agent/comp/core/config" "github.com/DataDog/datadog-agent/comp/core/workloadmeta" "github.com/DataDog/datadog-agent/pkg/util/fxutil" - - "github.com/stretchr/testify/assert" - "go.uber.org/fx" - "k8s.io/client-go/kubernetes/fake" ) const dummySubscriber = "dummy-subscriber" @@ -88,3 +93,74 @@ func testCollectEvent(t *testing.T, createResource func(*fake.Clientset) error, close(stopStore) wlm.Unsubscribe(ch) } + +func testCollectMetadataEvent(t *testing.T, createObjects func() []runtime.Object, gvr schema.GroupVersionResource, expected workloadmeta.EventBundle) { + + // Create a resource before starting the reflector store or workloadmeta so that if the reflector calls `List()` then + // this resource can't be skipped + + // Create test scheme + testScheme := runtime.NewScheme() + // Register Metadata objects types to the test scheme + err := v1.AddMetaToScheme(testScheme) + assert.NoError(t, err) + + objects := createObjects() + + metadataclient := metafake.NewSimpleMetadataClient(testScheme, objects...) + + wlm := fxutil.Test[workloadmeta.Mock](t, fx.Options( + core.MockBundle(), + fx.Supply(context.Background()), + fx.Supply(workloadmeta.NewParams()), + workloadmeta.MockModuleV2(), + )) + ctx := context.TODO() + + // Create a fake metadata client to mock API calls. + + response, err := metadataclient.Resource(gvr).List(ctx, v1.ListOptions{}) + assert.NoError(t, err) + fmt.Println("metadata client listing: ", response.String()) + store, _ := newMetadataStore(ctx, wlm, metadataclient, gvr) + + stopStore := make(chan struct{}) + go store.Run(stopStore) + + // Subscribe to the kubeapiserver events. Two cases are possible: + // - The reflector has already populated wlm with the resource, in that case the first call to <-ch will contain the event + // - The reflector is still initializing. In that case the second call to <-ch will contain the event + + time.Sleep(5 * time.Second) + + ch := wlm.Subscribe(dummySubscriber, workloadmeta.NormalPriority, nil) + var bundle workloadmeta.EventBundle + read := assert.Eventually(t, func() bool { + select { + case bundle = <-ch: + bundle.Acknowledge() + if len(bundle.Events) == 0 { + return false + } + // If bundle finally has an event, we can return from this + return true + + default: + return false + } + }, 30*time.Second, 500*time.Millisecond) + + // Retrieving the resource in an event bundle + if !read { + bundle = <-ch + bundle.Acknowledge() + } + + // nil the bundle's Ch so we can + // deep-equal just the events later + bundle.Ch = nil + actual := bundle + assert.Equal(t, expected, actual) + close(stopStore) + wlm.Unsubscribe(ch) +} diff --git a/comp/core/workloadmeta/collectors/internal/kubeapiserver/utils.go b/comp/core/workloadmeta/collectors/internal/kubeapiserver/utils.go index 6583cdfb51631..f9fb1261e489b 100644 --- a/comp/core/workloadmeta/collectors/internal/kubeapiserver/utils.go +++ b/comp/core/workloadmeta/collectors/internal/kubeapiserver/utils.go @@ -11,7 +11,11 @@ import ( "fmt" "regexp" + "k8s.io/apimachinery/pkg/runtime/schema" utilserror "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/client-go/discovery" + + "github.com/DataDog/datadog-agent/pkg/util/log" ) func filterMapStringKey(mapInput map[string]string, keyFilters []*regexp.Regexp) map[string]string { @@ -52,3 +56,48 @@ func filterToRegex(filter string) (*regexp.Regexp, error) { } return r, nil } + +func discoverGVRs(discoveryClient discovery.DiscoveryInterface, resources []string) ([]schema.GroupVersionResource, error) { + discoveredResources, err := discoverResources(discoveryClient) + if err != nil { + return nil, err + } + + gvrs := make([]schema.GroupVersionResource, 0, len(resources)) + for _, resource := range resources { + gv, found := discoveredResources[resource] + if found { + gvrs = append(gvrs, schema.GroupVersionResource{Group: gv.Group, Version: gv.Version, Resource: resource}) + } else { + log.Errorf("failed to auto-discover group/version of resource %s,", resource) + } + } + + return gvrs, nil +} + +func discoverResources(discoveryClient discovery.DiscoveryInterface) (map[string]schema.GroupVersion, error) { + apiGroups, apiResourceLists, err := discoveryClient.ServerGroupsAndResources() + if err != nil { + return nil, err + } + + preferredGroupVersions := make(map[string]struct{}) + for _, group := range apiGroups { + preferredGroupVersions[group.PreferredVersion.GroupVersion] = struct{}{} + } + + discoveredResources := map[string]schema.GroupVersion{} + for _, resourceList := range apiResourceLists { + _, found := preferredGroupVersions[resourceList.GroupVersion] + if found { + for _, resource := range resourceList.APIResources { + // No need to handle error because we are sure it is correctly formatted + gv, _ := schema.ParseGroupVersion(resourceList.GroupVersion) + discoveredResources[resource.Name] = gv + } + } + } + + return discoveredResources, nil +} diff --git a/comp/core/workloadmeta/collectors/internal/remote/generic.go b/comp/core/workloadmeta/collectors/internal/remote/generic.go index af057d240f234..29e49cd2882c1 100644 --- a/comp/core/workloadmeta/collectors/internal/remote/generic.go +++ b/comp/core/workloadmeta/collectors/internal/remote/generic.go @@ -58,7 +58,7 @@ type StreamHandler interface { // NewClient returns a client to connect to a remote gRPC server. NewClient(cc grpc.ClientConnInterface) GrpcClient // HandleResponse handles a response from the remote gRPC server. - HandleResponse(response interface{}) ([]workloadmeta.CollectorEvent, error) + HandleResponse(store workloadmeta.Component, response interface{}) ([]workloadmeta.CollectorEvent, error) // HandleResync is called on resynchronization. HandleResync(store workloadmeta.Component, events []workloadmeta.CollectorEvent) } @@ -230,7 +230,7 @@ func (c *GenericCollector) Run() { continue } - collectorEvents, err := c.StreamHandler.HandleResponse(response) + collectorEvents, err := c.StreamHandler.HandleResponse(c.store, response) if err != nil { log.Warnf("error processing event received from remote workloadmeta: %s", err) continue diff --git a/comp/core/workloadmeta/collectors/internal/remote/processcollector/process_collector.go b/comp/core/workloadmeta/collectors/internal/remote/processcollector/process_collector.go index d50a0f0735233..65e2458b22780 100644 --- a/comp/core/workloadmeta/collectors/internal/remote/processcollector/process_collector.go +++ b/comp/core/workloadmeta/collectors/internal/remote/processcollector/process_collector.go @@ -22,13 +22,16 @@ import ( "github.com/DataDog/datadog-agent/pkg/config" "github.com/DataDog/datadog-agent/pkg/languagedetection/languagemodels" pbgo "github.com/DataDog/datadog-agent/pkg/proto/pbgo/process" + "github.com/DataDog/datadog-agent/pkg/util/containers/metrics" "github.com/DataDog/datadog-agent/pkg/util/flavor" grpcutil "github.com/DataDog/datadog-agent/pkg/util/grpc" "github.com/DataDog/datadog-agent/pkg/util/log" + "github.com/DataDog/datadog-agent/pkg/util/optional" ) const ( - collectorID = "process-collector" + collectorID = "process-collector" + cacheValidityNoRT = 2 * time.Second ) func toLanguage(proto *pbgo.Language) *languagemodels.Language { @@ -40,6 +43,37 @@ func toLanguage(proto *pbgo.Language) *languagemodels.Language { } } +type client struct { + cl pbgo.ProcessEntityStreamClient + parentCollector *streamHandler +} + +func (c *client) StreamEntities(ctx context.Context, opts ...grpc.CallOption) (remote.Stream, error) { //nolint:revive // TODO fix revive unused-parameter + log.Debug("starting a new stream") + streamcl, err := c.cl.StreamEntities( + ctx, + &pbgo.ProcessStreamEntitiesRequest{}, + ) + if err != nil { + return nil, err + } + return &stream{cl: streamcl}, nil +} + +type stream struct { + cl pbgo.ProcessEntityStream_StreamEntitiesClient +} + +func (s *stream) Recv() (interface{}, error) { + log.Trace("calling stream recv") + return s.cl.Recv() +} + +type streamHandler struct { + port int + config.Reader +} + // WorkloadmetaEventFromProcessEventSet converts the given ProcessEventSet into a workloadmeta.Event func WorkloadmetaEventFromProcessEventSet(protoEvent *pbgo.ProcessEventSet) (workloadmeta.Event, error) { if protoEvent == nil { @@ -78,37 +112,6 @@ func WorkloadmetaEventFromProcessEventUnset(protoEvent *pbgo.ProcessEventUnset) }, nil } -type client struct { - cl pbgo.ProcessEntityStreamClient - parentCollector *streamHandler -} - -func (c *client) StreamEntities(ctx context.Context, opts ...grpc.CallOption) (remote.Stream, error) { //nolint:revive // TODO fix revive unused-parameter - log.Debug("starting a new stream") - streamcl, err := c.cl.StreamEntities( - ctx, - &pbgo.ProcessStreamEntitiesRequest{}, - ) - if err != nil { - return nil, err - } - return &stream{cl: streamcl}, nil -} - -type stream struct { - cl pbgo.ProcessEntityStream_StreamEntitiesClient -} - -func (s *stream) Recv() (interface{}, error) { - log.Trace("calling stream recv") - return s.cl.Recv() -} - -type streamHandler struct { - port int - config.Reader -} - // NewCollector returns a remote process collector for workloadmeta if any func NewCollector() (workloadmeta.CollectorProvider, error) { return workloadmeta.CollectorProvider{ @@ -152,7 +155,7 @@ func (s *streamHandler) NewClient(cc grpc.ClientConnInterface) remote.GrpcClient return &client{cl: pbgo.NewProcessEntityStreamClient(cc), parentCollector: s} } -func (s *streamHandler) HandleResponse(resp interface{}) ([]workloadmeta.CollectorEvent, error) { +func (s *streamHandler) HandleResponse(store workloadmeta.Component, resp interface{}) ([]workloadmeta.CollectorEvent, error) { log.Trace("handling response") response, ok := resp.(*pbgo.ProcessStreamResponse) if !ok { @@ -160,9 +163,9 @@ func (s *streamHandler) HandleResponse(resp interface{}) ([]workloadmeta.Collect } collectorEvents := make([]workloadmeta.CollectorEvent, 0, len(response.SetEvents)+len(response.UnsetEvents)) - collectorEvents = handleEvents(collectorEvents, response.UnsetEvents, WorkloadmetaEventFromProcessEventUnset) collectorEvents = handleEvents(collectorEvents, response.SetEvents, WorkloadmetaEventFromProcessEventSet) + s.populateMissingContainerID(collectorEvents, store) log.Tracef("collected [%d] events", len(collectorEvents)) return collectorEvents, nil } @@ -200,3 +203,31 @@ func (s *streamHandler) HandleResync(store workloadmeta.Component, events []work log.Debugf("resync, handling [%d] events", len(processes)) store.ResetProcesses(processes, workloadmeta.SourceRemoteProcessCollector) } + +// populateMissingContainerID populates any missing containerID field in the entities of Set events if it is possible to get the +// container id from the shared container provider +func (s *streamHandler) populateMissingContainerID(collectorEvents []workloadmeta.CollectorEvent, store workloadmeta.Component) { + for idx, event := range collectorEvents { + if event.Type != workloadmeta.EventTypeSet { + continue + } + + processEntity := event.Entity.(*workloadmeta.Process) + pid := processEntity.GetID().ID + ctrID := processEntity.ContainerID + + if ctrID == "" { + pidAsInt, _ := strconv.Atoi(pid) + containerProvider := metrics.GetProvider(optional.NewOption(store)) + ctrIDFromProvider, err := containerProvider.GetMetaCollector().GetContainerIDForPID(pidAsInt, cacheValidityNoRT) + if err != nil { + log.Debugf("failed to get container id for process %s: %v", pid, err) + continue + } + processEntity.ContainerID = ctrIDFromProvider + } + + event.Entity = processEntity + collectorEvents[idx] = event + } +} diff --git a/comp/core/workloadmeta/collectors/internal/remote/workloadmeta/workloadmeta.go b/comp/core/workloadmeta/collectors/internal/remote/workloadmeta/workloadmeta.go index c176a52e88b5c..0e59266436733 100644 --- a/comp/core/workloadmeta/collectors/internal/remote/workloadmeta/workloadmeta.go +++ b/comp/core/workloadmeta/collectors/internal/remote/workloadmeta/workloadmeta.go @@ -139,7 +139,7 @@ func (s *streamHandler) IsEnabled() bool { return true } -func (s *streamHandler) HandleResponse(resp interface{}) ([]workloadmeta.CollectorEvent, error) { +func (s *streamHandler) HandleResponse(_ workloadmeta.Component, resp interface{}) ([]workloadmeta.CollectorEvent, error) { response, ok := resp.(*pb.WorkloadmetaStreamResponse) if !ok { return nil, fmt.Errorf("incorrect response type") diff --git a/comp/core/workloadmeta/collectors/internal/remote/workloadmeta/workloadmeta_test.go b/comp/core/workloadmeta/collectors/internal/remote/workloadmeta/workloadmeta_test.go index ef5ce553861ab..00031f3b48805 100644 --- a/comp/core/workloadmeta/collectors/internal/remote/workloadmeta/workloadmeta_test.go +++ b/comp/core/workloadmeta/collectors/internal/remote/workloadmeta/workloadmeta_test.go @@ -167,6 +167,14 @@ func TestHandleWorkloadmetaStreamResponse(t *testing.T) { }, }, } + // workloadmeta client store + mockClientStore := fxutil.Test[workloadmeta.Mock](t, fx.Options( + core.MockBundle(), + fx.Supply(workloadmeta.Params{ + AgentType: workloadmeta.Remote, + }), + workloadmeta.MockModuleV2(), + )) expectedEvent, err := proto.WorkloadmetaEventFromProtoEvent(protoWorkloadmetaEvent) require.NoError(t, err) @@ -176,7 +184,7 @@ func TestHandleWorkloadmetaStreamResponse(t *testing.T) { } streamhandler := &streamHandler{} - collectorEvents, err := streamhandler.HandleResponse(mockResponse) + collectorEvents, err := streamhandler.HandleResponse(mockClientStore, mockResponse) require.NoError(t, err) assert.Len(t, collectorEvents, 1) diff --git a/comp/core/workloadmeta/component.go b/comp/core/workloadmeta/component.go index fe5fdd9418808..7a4ac4a0002ba 100644 --- a/comp/core/workloadmeta/component.go +++ b/comp/core/workloadmeta/component.go @@ -82,6 +82,10 @@ type Component interface { // the entity with kind KindKubernetesNamespace and the given ID. GetKubernetesNamespace(id string) (*KubernetesNamespace, error) + // GetKubernetesMetadata returns metadata about a Kubernetes resource. It fetches + // the entity with kind KubernetesMetadata and the given ID. + GetKubernetesMetadata(id string) (*KubernetesMetadata, error) + // ListECSTasks returns metadata about all ECS tasks, equivalent to all // entities with kind KindECSTask. ListECSTasks() []*ECSTask diff --git a/comp/core/workloadmeta/dump.go b/comp/core/workloadmeta/dump.go index b136f64ca3f63..4da23e04ba51b 100644 --- a/comp/core/workloadmeta/dump.go +++ b/comp/core/workloadmeta/dump.go @@ -65,6 +65,8 @@ func (w *workloadmeta) Dump(verbose bool) WorkloadDumpResponse { info = e.String(verbose) case *KubernetesNamespace: info = e.String(verbose) + case *KubernetesMetadata: + info = e.String(verbose) default: return "", fmt.Errorf("unsupported type %T", e) } diff --git a/comp/core/workloadmeta/merge_test.go b/comp/core/workloadmeta/merge_test.go index 6932439307435..22ee10a53c2b9 100644 --- a/comp/core/workloadmeta/merge_test.go +++ b/comp/core/workloadmeta/merge_test.go @@ -86,7 +86,7 @@ func container2(testTime time.Time) Container { //nolint:revive // TODO fix revi CreatedAt: time.Time{}, StartedAt: time.Time{}, FinishedAt: time.Time{}, - ExitCode: pointer.Ptr(uint32(100)), + ExitCode: pointer.Ptr(int64(100)), }, CollectorTags: []string{"tag3"}, } @@ -109,7 +109,7 @@ func TestMerge(t *testing.T) { CreatedAt: testTime, StartedAt: testTime, FinishedAt: time.Time{}, - ExitCode: pointer.Ptr(uint32(100)), + ExitCode: pointer.Ptr(int64(100)), }, } diff --git a/comp/core/workloadmeta/store.go b/comp/core/workloadmeta/store.go index c2f6eb0c897ad..66fe02f9b5498 100644 --- a/comp/core/workloadmeta/store.go +++ b/comp/core/workloadmeta/store.go @@ -406,6 +406,16 @@ func (w *workloadmeta) GetImage(id string) (*ContainerImageMetadata, error) { return entity.(*ContainerImageMetadata), nil } +// GetKubernetesMetadata implements Store#GetKubernetesMetadata. +func (w *workloadmeta) GetKubernetesMetadata(id string) (*KubernetesMetadata, error) { + entity, err := w.getEntityByKind(KindKubernetesMetadata, id) + if err != nil { + return nil, err + } + + return entity.(*KubernetesMetadata), nil +} + // Notify implements Store#Notify func (w *workloadmeta) Notify(events []CollectorEvent) { if len(events) > 0 { diff --git a/comp/core/workloadmeta/store_test.go b/comp/core/workloadmeta/store_test.go index 3836f0661bc06..f049e6b10b308 100644 --- a/comp/core/workloadmeta/store_test.go +++ b/comp/core/workloadmeta/store_test.go @@ -1420,6 +1420,49 @@ func TestResetProcesses(t *testing.T) { } +func TestGetKubernetesMetadata(t *testing.T) { + deps := fxutil.Test[dependencies](t, fx.Options( + logimpl.MockModule(), + config.MockModule(), + fx.Supply(NewParams()), + )) + + s := newWorkloadmetaObject(deps) + + kubemetadata := &KubernetesMetadata{ + EntityID: EntityID{ + Kind: KindKubernetesMetadata, + ID: "deployments/default/app", + }, + } + + s.handleEvents([]CollectorEvent{ + { + Type: EventTypeSet, + Source: fooSource, + Entity: kubemetadata, + }, + }) + + retrievedMetadata, err := s.GetKubernetesMetadata("deployments/default/app") + tassert.NoError(t, err) + + if !reflect.DeepEqual(kubemetadata, retrievedMetadata) { + t.Errorf("expected metadata %q to match the one in the store", retrievedMetadata.ID) + } + + s.handleEvents([]CollectorEvent{ + { + Type: EventTypeUnset, + Source: fooSource, + Entity: kubemetadata, + }, + }) + + _, err = s.GetKubernetesMetadata("deployments/default/app") + tassert.True(t, errors.IsNotFound(err)) +} + func TestReset(t *testing.T) { fooContainer := &Container{ EntityID: EntityID{ diff --git a/comp/core/workloadmeta/types.go b/comp/core/workloadmeta/types.go index b0a5b6beb5e28..5e8d175e954d4 100644 --- a/comp/core/workloadmeta/types.go +++ b/comp/core/workloadmeta/types.go @@ -11,6 +11,8 @@ import ( "strings" "time" + "k8s.io/apimachinery/pkg/runtime/schema" + langUtil "github.com/DataDog/datadog-agent/pkg/languagedetection/util" "github.com/CycloneDX/cyclonedx-go" @@ -41,6 +43,7 @@ const ( KindContainer Kind = "container" KindKubernetesPod Kind = "kubernetes_pod" KindKubernetesNode Kind = "kubernetes_node" + KindKubernetesMetadata Kind = "kubernetes_metadata" KindKubernetesDeployment Kind = "kubernetes_deployment" KindKubernetesNamespace Kind = "kubernetes_namespace" KindECSTask Kind = "ecs_task" @@ -313,7 +316,7 @@ type ContainerState struct { CreatedAt time.Time StartedAt time.Time FinishedAt time.Time - ExitCode *uint32 + ExitCode *int64 } // String returns a string representation of ContainerState. @@ -395,7 +398,7 @@ func (c ContainerVolume) String(_ bool) string { type ContainerHealthStatus struct { Status string Since *time.Time - ExitCode *uint32 + ExitCode *int64 Output string } @@ -785,6 +788,53 @@ func (o KubernetesPodOwner) String(verbose bool) string { return sb.String() } +// KubernetesMetadata is an Entity representing kubernetes resource metadata +type KubernetesMetadata struct { + EntityID + EntityMeta + GVR schema.GroupVersionResource +} + +// GetID implements Entity#GetID. +func (m *KubernetesMetadata) GetID() EntityID { + return m.EntityID +} + +// Merge implements Entity#Merge. +func (m *KubernetesMetadata) Merge(e Entity) error { + mm, ok := e.(*KubernetesMetadata) + if !ok { + return fmt.Errorf("cannot merge KubernetesMetadata with different kind %T", e) + } + + return merge(m, mm) +} + +// DeepCopy implements Entity#DeepCopy. +func (m KubernetesMetadata) DeepCopy() Entity { + cm := deepcopy.Copy(m).(KubernetesMetadata) + return &cm +} + +// String implements Entity#String +func (m *KubernetesMetadata) String(verbose bool) string { + var sb strings.Builder + _, _ = fmt.Fprintln(&sb, "----------- Entity ID -----------") + _, _ = fmt.Fprintln(&sb, m.EntityID.String(verbose)) + + _, _ = fmt.Fprintln(&sb, "----------- Entity Meta -----------") + _, _ = fmt.Fprint(&sb, m.EntityMeta.String(verbose)) + + if verbose { + _, _ = fmt.Fprintln(&sb, "----------- Resource -----------") + _, _ = fmt.Fprint(&sb, m.GVR.String()) + } + + return sb.String() +} + +var _ Entity = &KubernetesMetadata{} + // KubernetesNode is an Entity representing a Kubernetes Node. type KubernetesNode struct { EntityID diff --git a/comp/core/workloadmeta/workloadmeta_mock.go b/comp/core/workloadmeta/workloadmeta_mock.go index 0cd07c19e0870..1b56a8295be22 100644 --- a/comp/core/workloadmeta/workloadmeta_mock.go +++ b/comp/core/workloadmeta/workloadmeta_mock.go @@ -57,6 +57,16 @@ func (w *workloadMetaMock) GetContainer(id string) (*Container, error) { return entity.(*Container), nil } +// GetKubernetesMetadata implements workloadMetaMock#GetKubernetesMetadata. +func (w *workloadMetaMock) GetKubernetesMetadata(id string) (*KubernetesMetadata, error) { + entity, err := w.getEntityByKind(KindKubernetesMetadata, id) + if err != nil { + return nil, err + } + + return entity.(*KubernetesMetadata), nil +} + // ListContainers returns metadata about all known containers. func (w *workloadMetaMock) ListContainers() []*Container { entities := w.listEntitiesByKind(KindContainer) diff --git a/comp/forwarder/defaultforwarder/go.mod b/comp/forwarder/defaultforwarder/go.mod index a695e974f0eba..38cd52436dcce 100644 --- a/comp/forwarder/defaultforwarder/go.mod +++ b/comp/forwarder/defaultforwarder/go.mod @@ -85,7 +85,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/system v0.54.0-rc.2 // indirect github.com/DataDog/datadog-agent/pkg/util/system/socket v0.54.0-rc.2 // indirect github.com/DataDog/datadog-agent/pkg/util/winutil v0.54.0-rc.2 // indirect - github.com/DataDog/viper v1.13.4 // indirect + github.com/DataDog/viper v1.13.5 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect diff --git a/comp/forwarder/defaultforwarder/go.sum b/comp/forwarder/defaultforwarder/go.sum index 85537136660db..c2359f6536dbf 100644 --- a/comp/forwarder/defaultforwarder/go.sum +++ b/comp/forwarder/defaultforwarder/go.sum @@ -1,7 +1,7 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/DataDog/viper v1.13.4 h1:0SrZc3zvMAGgVKg96uP4DEJb13lK2Is9a4go7IIcFSE= -github.com/DataDog/viper v1.13.4/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= +github.com/DataDog/viper v1.13.5 h1:SZMcyMknYQN2jRY/40A16gUXexlNJOI8sDs1cWZnI64= +github.com/DataDog/viper v1.13.5/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= diff --git a/comp/forwarder/orchestrator/orchestratorinterface/go.mod b/comp/forwarder/orchestrator/orchestratorinterface/go.mod index 5235be4c0cfcd..bd5982f5aafef 100644 --- a/comp/forwarder/orchestrator/orchestratorinterface/go.mod +++ b/comp/forwarder/orchestrator/orchestratorinterface/go.mod @@ -88,7 +88,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/system/socket v0.54.0-rc.2 // indirect github.com/DataDog/datadog-agent/pkg/util/winutil v0.54.0-rc.2 // indirect github.com/DataDog/datadog-agent/pkg/version v0.54.0-rc.2 // indirect - github.com/DataDog/viper v1.13.4 // indirect + github.com/DataDog/viper v1.13.5 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect diff --git a/comp/forwarder/orchestrator/orchestratorinterface/go.sum b/comp/forwarder/orchestrator/orchestratorinterface/go.sum index 6b63d4285311f..e77bd87853c7d 100644 --- a/comp/forwarder/orchestrator/orchestratorinterface/go.sum +++ b/comp/forwarder/orchestrator/orchestratorinterface/go.sum @@ -2,8 +2,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DataDog/cast v1.3.1-0.20190301154711-1ee8c8bd14a3 h1:SobA9WYm4K/MUtWlbKaomWTmnuYp1KhIm8Wlx3vmpsg= github.com/DataDog/cast v1.3.1-0.20190301154711-1ee8c8bd14a3/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/DataDog/viper v1.13.4 h1:0SrZc3zvMAGgVKg96uP4DEJb13lK2Is9a4go7IIcFSE= -github.com/DataDog/viper v1.13.4/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= +github.com/DataDog/viper v1.13.5 h1:SZMcyMknYQN2jRY/40A16gUXexlNJOI8sDs1cWZnI64= +github.com/DataDog/viper v1.13.5/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= diff --git a/comp/logs/agent/config/go.mod b/comp/logs/agent/config/go.mod index 654d2aad3801d..5e43dd57227f8 100644 --- a/comp/logs/agent/config/go.mod +++ b/comp/logs/agent/config/go.mod @@ -39,7 +39,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/fxutil v0.54.0-rc.2 github.com/DataDog/datadog-agent/pkg/util/log v0.54.0-rc.2 github.com/DataDog/datadog-agent/pkg/util/pointer v0.54.0-rc.2 - github.com/DataDog/viper v1.13.4 + github.com/DataDog/viper v1.13.5 github.com/stretchr/testify v1.9.0 go.uber.org/fx v1.18.2 ) diff --git a/comp/logs/agent/config/go.sum b/comp/logs/agent/config/go.sum index 4f5285c7fa353..b159185f44664 100644 --- a/comp/logs/agent/config/go.sum +++ b/comp/logs/agent/config/go.sum @@ -1,8 +1,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/DataDog/viper v1.13.4 h1:0SrZc3zvMAGgVKg96uP4DEJb13lK2Is9a4go7IIcFSE= -github.com/DataDog/viper v1.13.4/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= +github.com/DataDog/viper v1.13.5 h1:SZMcyMknYQN2jRY/40A16gUXexlNJOI8sDs1cWZnI64= +github.com/DataDog/viper v1.13.5/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= diff --git a/comp/metadata/bundle.go b/comp/metadata/bundle.go index f2896f4018116..87f4518f62c1e 100644 --- a/comp/metadata/bundle.go +++ b/comp/metadata/bundle.go @@ -15,6 +15,7 @@ import ( "github.com/DataDog/datadog-agent/comp/metadata/packagesigning/packagesigningimpl" "github.com/DataDog/datadog-agent/comp/metadata/resources/resourcesimpl" "github.com/DataDog/datadog-agent/comp/metadata/runner/runnerimpl" + systemprobe "github.com/DataDog/datadog-agent/comp/metadata/systemprobe/fx" "github.com/DataDog/datadog-agent/pkg/util/fxutil" ) @@ -30,6 +31,7 @@ func Bundle() fxutil.BundleOptions { inventoryhostimpl.Module(), inventorychecksimpl.Module(), packagesigningimpl.Module(), + systemprobe.Module(), ) } diff --git a/comp/metadata/inventoryagent/inventoryagentimpl/inventoryagent.go b/comp/metadata/inventoryagent/inventoryagentimpl/inventoryagent.go index 02ac8d644f3ca..3db2b6f534b4a 100644 --- a/comp/metadata/inventoryagent/inventoryagentimpl/inventoryagent.go +++ b/comp/metadata/inventoryagent/inventoryagentimpl/inventoryagent.go @@ -432,15 +432,15 @@ func (ia *inventoryagent) getPayload() marshaler.JSONMarshaler { } for source, conf := range layers { - if json, err := ia.marshalAndScrub(conf); err == nil { - data[layersName[source]] = json + if yaml, err := ia.marshalAndScrub(conf); err == nil { + data[layersName[source]] = yaml } } - if json, err := ia.marshalAndScrub(ia.conf.AllSettings()); err == nil { - data["full_configuration"] = json + if yaml, err := ia.marshalAndScrub(ia.conf.AllSettings()); err == nil { + data["full_configuration"] = yaml } - if json, err := ia.marshalAndScrub(ia.conf.AllSettingsWithoutDefault()); err == nil { - data["provided_configuration"] = json + if yaml, err := ia.marshalAndScrub(ia.conf.AllSettingsWithoutDefault()); err == nil { + data["provided_configuration"] = yaml } } diff --git a/comp/metadata/systemprobe/README.md b/comp/metadata/systemprobe/README.md new file mode 100644 index 0000000000000..8849646c137c5 --- /dev/null +++ b/comp/metadata/systemprobe/README.md @@ -0,0 +1,69 @@ +# System-probe metadata Payload + +This package populates some of the system-probe related fields in the `inventories` product in DataDog. More specifically the +`system-probe` table. + +This is enabled by default but can be turned off using `inventories_enabled` config. + +The payload is sent every 10min (see `inventories_max_interval` in the config). + +## System-probe Configuration + +System-probe configurations are scrubbed from any sensitive information (same logic than for the flare). +This include the following: +`full_configuration` +`provided_configuration` +`file_configuration` +`environment_variable_configuration` +`agent_runtime_configuration` +`remote_configuration` +`cli_configuration` +`source_local_configuration` + +Sending System-Probe configuration can be disabled using `inventories_configuration_enabled`. + +# Format + +The payload is a JSON dict with the following fields + +- `hostname` - **string**: the hostname of the agent as shown on the status page. +- `timestamp` - **int**: the timestamp when the payload was created. +- `system_probe_metadata` - **dict of string to JSON type**: + - `agent_version` - **string**: the version of the Agent sending this payload. + - `full_configuration` - **string**: the current System-Probe configuration scrubbed, including all the defaults, as a YAML + string. + - `provided_configuration` - **string**: the current System-Probe configuration (scrubbed), without the defaults, as a YAML + string. This includes the settings configured by the user (throuh the configuration file, the environment, CLI...). + - `file_configuration` - **string**: the System-Probe configuration specified by the configuration file (scrubbed), as a YAML string. + Only the settings written in the configuration file are included, and their value might not match what's applyed by the agent since they can be overriden by other sources. + - `environment_variable_configuration` - **string**: the System-Probe configuration specified by the environment variables (scrubbed), as a YAML string. + Only the settings written in the environment variables are included, and their value might not match what's applyed by the agent somce they can be overriden by other sources. + - `agent_runtime_configuration` - **string**: the System-Probe configuration set by the agent itself (scrubbed), as a YAML string. + Only the settings set by the agent itself are included, and their value might not match what's applyed by the agent since they can be overriden by other sources. + - `remote_configuration` - **string**: the System-Probe configuration specified by the Remote Configuration (scrubbed), as a YAML string. + Only the settings currently used by Remote Configuration are included, and their value might not match what's applyed by the agent since they can be overriden by other sources. + - `cli_configuration` - **string**: the System-Probe configuration specified by the CLI (scrubbed), as a YAML string. + Only the settings set in the CLI are included. + - `source_local_configuration` - **string**: the System-Probe configuration synchronized from the local Agent process, as a YAML string. + +("scrubbed" indicates that secrets are removed from the field value just as they are in logs) + +## Example Payload + +Here an example of an inventory payload: + +``` +{ + "system_probe_metadata": { + "agent_version": "7.55.0", + "full_configuration": "", + "provided_configuration": "system_probe_config:\n sysprobe_socket: /tmp/sysprobe.sock", + "file_configuration": "system_probe_config:\n sysprobe_socket: /tmp/sysprobe.sock", + "environment_variable_configuration": "{}", + "remote_configuration": "{}", + "cli_configuration": "{}" + } + "hostname": "my-host", + "timestamp": 1631281754507358895 +} +``` diff --git a/comp/otelcol/otlp/components/processor/infraattributesprocessor/package_test.go b/comp/metadata/systemprobe/def/component.go similarity index 59% rename from comp/otelcol/otlp/components/processor/infraattributesprocessor/package_test.go rename to comp/metadata/systemprobe/def/component.go index b95a7add774bf..be510ee393048 100644 --- a/comp/otelcol/otlp/components/processor/infraattributesprocessor/package_test.go +++ b/comp/metadata/systemprobe/def/component.go @@ -3,14 +3,10 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2024-present Datadog, Inc. -package infraattributesprocessor +// Package def is the metadata provider for system-probe process +package def -import ( - "testing" +// team: agent-shared-components - "go.uber.org/goleak" -) - -func TestMain(m *testing.M) { - goleak.VerifyTestMain(m) -} +// Component is the component type. +type Component interface{} diff --git a/comp/metadata/systemprobe/fx/fx.go b/comp/metadata/systemprobe/fx/fx.go new file mode 100644 index 0000000000000..bd3c05f63d6b5 --- /dev/null +++ b/comp/metadata/systemprobe/fx/fx.go @@ -0,0 +1,21 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024-present Datadog, Inc. + +// Package fx provides the fx module for the systemprobe metadata component +package fx + +import ( + systemprobeimpl "github.com/DataDog/datadog-agent/comp/metadata/systemprobe/impl" + "github.com/DataDog/datadog-agent/pkg/util/fxutil" +) + +// Module defines the fx options for this component +func Module() fxutil.Module { + return fxutil.Component( + fxutil.ProvideComponentConstructor( + systemprobeimpl.NewComponent, + ), + ) +} diff --git a/comp/metadata/systemprobe/impl/system_probe.go b/comp/metadata/systemprobe/impl/system_probe.go new file mode 100644 index 0000000000000..f96ddb7cec235 --- /dev/null +++ b/comp/metadata/systemprobe/impl/system_probe.go @@ -0,0 +1,181 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +// Package impl implements the systemprobe metadata providers interface +package impl + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "time" + + "github.com/DataDog/datadog-agent/comp/api/api" + "github.com/DataDog/datadog-agent/comp/api/api/utils" + "github.com/DataDog/datadog-agent/comp/api/authtoken" + "github.com/DataDog/datadog-agent/comp/core/config" + flaretypes "github.com/DataDog/datadog-agent/comp/core/flare/types" + "github.com/DataDog/datadog-agent/comp/core/log" + "github.com/DataDog/datadog-agent/comp/core/sysprobeconfig" + "github.com/DataDog/datadog-agent/comp/metadata/internal/util" + "github.com/DataDog/datadog-agent/comp/metadata/runner/runnerimpl" + "github.com/DataDog/datadog-agent/comp/metadata/systemprobe/def" + configFetcher "github.com/DataDog/datadog-agent/pkg/config/fetcher" + "github.com/DataDog/datadog-agent/pkg/config/model" + "github.com/DataDog/datadog-agent/pkg/serializer" + "github.com/DataDog/datadog-agent/pkg/serializer/marshaler" + "github.com/DataDog/datadog-agent/pkg/util/hostname" + "github.com/DataDog/datadog-agent/pkg/util/optional" + "github.com/DataDog/datadog-agent/pkg/version" + "gopkg.in/yaml.v2" +) + +var ( + // for testing + fetchSystemProbeConfig = configFetcher.SystemProbeConfig + fetchSystemProbeConfigBySource = configFetcher.SystemProbeConfigBySource +) + +// Payload handles the JSON unmarshalling of the metadata payload +type Payload struct { + Hostname string `json:"hostname"` + Timestamp int64 `json:"timestamp"` + Metadata map[string]interface{} `json:"system_probe_metadata"` +} + +// MarshalJSON serialization a Payload to JSON +func (p *Payload) MarshalJSON() ([]byte, error) { + type PayloadAlias Payload + return json.Marshal((*PayloadAlias)(p)) +} + +// SplitPayload implements marshaler.AbstractMarshaler#SplitPayload. +// +// In this case, the payload can't be split any further. +func (p *Payload) SplitPayload(_ int) ([]marshaler.AbstractMarshaler, error) { + return nil, fmt.Errorf("could not split inventories agent payload any more, payload is too big for intake") +} + +type systemprobe struct { + util.InventoryPayload + + log log.Component + conf config.Component + sysprobeConf optional.Option[sysprobeconfig.Component] + hostname string +} + +// Requires defines the dependencies for the systemprobe metadata component +type Requires struct { + Log log.Component + Config config.Component + Serializer serializer.MetricSerializer + // We need the authtoken to be created so we requires the comp. It will be used by configFetcher. + AuthToken authtoken.Component + SysProbeConfig optional.Option[sysprobeconfig.Component] +} + +// Provides defines the output of the systemprobe metadatacomponent +type Provides struct { + Comp def.Component + MetadataProvider runnerimpl.Provider + FlareProvider flaretypes.Provider + Endpoint api.AgentEndpointProvider +} + +// NewComponent creates a new systemprobe metadata Component +func NewComponent(deps Requires) Provides { + hname, _ := hostname.Get(context.Background()) + sb := &systemprobe{ + log: deps.Log, + conf: deps.Config, + hostname: hname, + sysprobeConf: deps.SysProbeConfig, + } + sb.InventoryPayload = util.CreateInventoryPayload(deps.Config, deps.Log, deps.Serializer, sb.getPayload, "system-probe.json") + + return Provides{ + Comp: sb, + MetadataProvider: sb.MetadataProvider(), + FlareProvider: sb.FlareProvider(), + Endpoint: api.NewAgentEndpointProvider(sb.writePayloadAsJSON, "/metadata/system-probe", "GET"), + } +} + +func (sb *systemprobe) writePayloadAsJSON(w http.ResponseWriter, _ *http.Request) { + // GetAsJSON calls getPayload which already scrub the data + scrubbed, err := sb.GetAsJSON() + if err != nil { + utils.SetJSONError(w, err, 500) + return + } + w.Write(scrubbed) +} + +func (sb *systemprobe) getConfigLayers() map[string]interface{} { + metadata := map[string]interface{}{ + "agent_version": version.AgentVersion, + } + + if !sb.conf.GetBool("inventories_configuration_enabled") { + return metadata + } + + sysprobeConf, isSet := sb.sysprobeConf.Get() + if !isSet { + sb.log.Debugf("system-probe config not available: disabling systemprobe metadata") + return metadata + } + + rawLayers, err := fetchSystemProbeConfigBySource(sysprobeConf) + if err != nil { + sb.log.Debugf("error fetching system-probe config layers: %s", err) + return metadata + } + + configBySources := map[string]interface{}{} + if err := json.Unmarshal([]byte(rawLayers), &configBySources); err != nil { + sb.log.Debugf("error unmarshalling systemprobe config by source: %s", err) + return metadata + } + + layersName := map[model.Source]string{ + model.SourceFile: "file_configuration", + model.SourceEnvVar: "environment_variable_configuration", + model.SourceAgentRuntime: "agent_runtime_configuration", + model.SourceLocalConfigProcess: "source_local_configuration", + model.SourceRC: "remote_configuration", + model.SourceCLI: "cli_configuration", + model.SourceProvided: "provided_configuration", + } + for source, conf := range configBySources { + if layer, ok := layersName[model.Source(source)]; ok { + if yamlStr, err := yaml.Marshal(conf); err == nil { + metadata[layer] = string(yamlStr) + } else { + sb.log.Debugf("error serializing systemprobe '%s' config layer: %s", source, err) + } + } else { + sb.log.Debugf("error unknown config layer from system-probe '%s'", source) + } + } + + if str, err := fetchSystemProbeConfig(sysprobeConf); err == nil { + metadata["full_configuration"] = str + } else { + sb.log.Debugf("error fetching system-probe config: %s", err) + } + + return metadata +} + +func (sb *systemprobe) getPayload() marshaler.JSONMarshaler { + return &Payload{ + Hostname: sb.hostname, + Timestamp: time.Now().UnixNano(), + Metadata: sb.getConfigLayers(), + } +} diff --git a/comp/metadata/systemprobe/impl/system_probe_test.go b/comp/metadata/systemprobe/impl/system_probe_test.go new file mode 100644 index 0000000000000..28e940a8a7102 --- /dev/null +++ b/comp/metadata/systemprobe/impl/system_probe_test.go @@ -0,0 +1,143 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +// Package impl implements the systemprobe metadata providers interface +package impl + +import ( + "encoding/json" + "io" + "net/http/httptest" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/fx" + + "github.com/DataDog/datadog-agent/comp/api/authtoken" + authtokenimpl "github.com/DataDog/datadog-agent/comp/api/authtoken/fetchonlyimpl" + "github.com/DataDog/datadog-agent/comp/core/config" + "github.com/DataDog/datadog-agent/comp/core/log" + "github.com/DataDog/datadog-agent/comp/core/log/logimpl" + "github.com/DataDog/datadog-agent/comp/core/sysprobeconfig" + "github.com/DataDog/datadog-agent/comp/core/sysprobeconfig/sysprobeconfigimpl" + configFetcher "github.com/DataDog/datadog-agent/pkg/config/fetcher" + "github.com/DataDog/datadog-agent/pkg/config/model" + "github.com/DataDog/datadog-agent/pkg/serializer" + "github.com/DataDog/datadog-agent/pkg/util/fxutil" + "github.com/DataDog/datadog-agent/pkg/util/optional" + "github.com/DataDog/datadog-agent/pkg/version" +) + +func setupFecther(t *testing.T) { + t.Cleanup(func() { + fetchSystemProbeConfig = configFetcher.SystemProbeConfig + fetchSystemProbeConfigBySource = configFetcher.SystemProbeConfigBySource + }) + + fetchSystemProbeConfig = func(_ model.Reader) (string, error) { return "full config", nil } + fetchSystemProbeConfigBySource = func(_ model.Reader) (string, error) { + data, err := json.Marshal(map[string]interface{}{ + string(model.SourceFile): map[string]bool{"file": true}, + string(model.SourceEnvVar): map[string]bool{"env": true}, + string(model.SourceAgentRuntime): map[string]bool{"runtime": true}, + string(model.SourceLocalConfigProcess): map[string]bool{"local": true}, + string(model.SourceRC): map[string]bool{"rc": true}, + string(model.SourceCLI): map[string]bool{"cli": true}, + string(model.SourceProvided): map[string]bool{"provided": true}, + }) + return string(data), err + } +} + +func getSystemProbeComp(t *testing.T, enableConfig bool) *systemprobe { + l := fxutil.Test[log.Component](t, logimpl.MockModule()) + + cfg := fxutil.Test[config.Component](t, config.MockModule()) + cfg.Set("inventories_configuration_enabled", enableConfig, model.SourceUnknown) + + r := Requires{ + Log: l, + Config: cfg, + Serializer: &serializer.MockSerializer{}, + AuthToken: fxutil.Test[authtoken.Component](t, + authtokenimpl.Module(), + fx.Provide(func() log.Component { return l }), + fx.Provide(func() config.Component { return cfg }), + ), + SysProbeConfig: fxutil.Test[optional.Option[sysprobeconfig.Component]](t, sysprobeconfigimpl.MockModule()), + } + + comp := NewComponent(r).Comp + return comp.(*systemprobe) +} + +func assertPayload(t *testing.T, p *Payload) { + assert.Equal(t, "test hostname", p.Hostname) + assert.True(t, p.Timestamp < time.Now().UnixNano()) + assert.Equal(t, + map[string]interface{}{ + "agent_runtime_configuration": "runtime: true\n", + "cli_configuration": "cli: true\n", + "environment_variable_configuration": "env: true\n", + "file_configuration": "file: true\n", + "full_configuration": "full config", + "provided_configuration": "provided: true\n", + "remote_configuration": "rc: true\n", + "source_local_configuration": "local: true\n", + "agent_version": version.AgentVersion, + }, + p.Metadata) +} + +func TestGetPayload(t *testing.T) { + setupFecther(t) + sb := getSystemProbeComp(t, true) + + sb.hostname = "test hostname" + + p := sb.getPayload().(*Payload) + assertPayload(t, p) +} + +func TestGetPayloadNoConfig(t *testing.T) { + setupFecther(t) + sb := getSystemProbeComp(t, false) + + sb.hostname = "test hostname" + + p := sb.getPayload().(*Payload) + assert.Equal(t, "test hostname", p.Hostname) + assert.True(t, p.Timestamp <= time.Now().UnixNano()) + assert.Equal(t, + map[string]interface{}{ + "agent_version": version.AgentVersion, + }, + p.Metadata) +} + +func TestWritePayload(t *testing.T) { + setupFecther(t) + sb := getSystemProbeComp(t, true) + + sb.hostname = "test hostname" + + req := httptest.NewRequest("GET", "http://fake_url.com", nil) + w := httptest.NewRecorder() + + sb.writePayloadAsJSON(w, req) + + resp := w.Result() + body, err := io.ReadAll(resp.Body) + resp.Body.Close() + require.NoError(t, err) + + p := Payload{} + err = json.Unmarshal(body, &p) + require.NoError(t, err) + + assertPayload(t, &p) +} diff --git a/comp/otelcol/bundle.go b/comp/otelcol/bundle.go index bf544730a9dbc..7d8623b2f4701 100644 --- a/comp/otelcol/bundle.go +++ b/comp/otelcol/bundle.go @@ -8,7 +8,7 @@ package otelcol import ( - "github.com/DataDog/datadog-agent/comp/otelcol/collector" + collectorfx "github.com/DataDog/datadog-agent/comp/otelcol/collector/fx-pipeline" "github.com/DataDog/datadog-agent/pkg/util/fxutil" ) @@ -17,5 +17,5 @@ import ( // Bundle specifies the bundle for the OTLP ingest pipeline. func Bundle() fxutil.BundleOptions { return fxutil.Bundle( - collector.Module()) + collectorfx.Module()) } diff --git a/comp/otelcol/collector/def/component.go b/comp/otelcol/collector/def/component.go index f7dcaff6c5f83..bf2b4b176b8b5 100644 --- a/comp/otelcol/collector/def/component.go +++ b/comp/otelcol/collector/def/component.go @@ -3,18 +3,16 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -//go:build otlp - // Package collector defines the OpenTelemetry Collector component. package collector import ( - "github.com/DataDog/datadog-agent/comp/otelcol/otlp" + "github.com/DataDog/datadog-agent/comp/otelcol/otlp/datatype" ) // team: opentelemetry // Component specifies the interface implemented by the collector module. type Component interface { - Status() otlp.CollectorStatus + Status() datatype.CollectorStatus } diff --git a/comp/otelcol/collector/component.go b/comp/otelcol/collector/fx-pipeline/fx.go similarity index 50% rename from comp/otelcol/collector/component.go rename to comp/otelcol/collector/fx-pipeline/fx.go index 72016fb758f63..0091ff6603037 100644 --- a/comp/otelcol/collector/component.go +++ b/comp/otelcol/collector/fx-pipeline/fx.go @@ -3,25 +3,18 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -//go:build otlp - -// Package collector implements the OpenTelemetry Collector component. -package collector +// Package fx creates the modules for fx +package fx import ( - "go.uber.org/fx" - - collectordef "github.com/DataDog/datadog-agent/comp/otelcol/collector/def" + collectorimpl "github.com/DataDog/datadog-agent/comp/otelcol/collector/impl-pipeline" "github.com/DataDog/datadog-agent/pkg/util/fxutil" ) // team: opentelemetry -// Component specifies the interface implemented by the collector module. -type Component = collectordef.Component - -// Module specifies the Collector module bundle. +// Module for OTel Agent func Module() fxutil.Module { return fxutil.Component( - fx.Provide(newPipeline)) + fxutil.ProvideComponentConstructor(collectorimpl.NewComponent)) } diff --git a/comp/otelcol/collector/fx/fx.go b/comp/otelcol/collector/fx/fx.go index 08dd50c2e1581..18f5f04598d40 100644 --- a/comp/otelcol/collector/fx/fx.go +++ b/comp/otelcol/collector/fx/fx.go @@ -18,5 +18,5 @@ import ( // Module for OTel Agent func Module() fxutil.Module { return fxutil.Component( - fxutil.ProvideComponentConstructor(collectorimpl.New)) + fxutil.ProvideComponentConstructor(collectorimpl.NewComponent)) } diff --git a/comp/otelcol/collector/impl-pipeline/no_otlp.go b/comp/otelcol/collector/impl-pipeline/no_otlp.go new file mode 100644 index 0000000000000..6c13829a106c8 --- /dev/null +++ b/comp/otelcol/collector/impl-pipeline/no_otlp.go @@ -0,0 +1,25 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +//go:build !otlp + +// Package collector contains a no-op implementation of the collector +package collector + +import ( + collector "github.com/DataDog/datadog-agent/comp/otelcol/collector/def" + "github.com/DataDog/datadog-agent/comp/otelcol/otlp/datatype" +) + +type noopImpl struct{} + +func (i *noopImpl) Status() datatype.CollectorStatus { + return datatype.CollectorStatus{} +} + +// NewComponent returns a new instance of the collector component. +func NewComponent() collector.Component { + return &noopImpl{} +} diff --git a/comp/otelcol/collector/pipeline.go b/comp/otelcol/collector/impl-pipeline/pipeline.go similarity index 55% rename from comp/otelcol/collector/pipeline.go rename to comp/otelcol/collector/impl-pipeline/pipeline.go index 9a9fd57d058c2..8420a24b02a38 100644 --- a/comp/otelcol/collector/pipeline.go +++ b/comp/otelcol/collector/impl-pipeline/pipeline.go @@ -5,19 +5,22 @@ //go:build otlp +// Package collector implements the collector component package collector import ( "context" - "go.uber.org/fx" - "github.com/DataDog/datadog-agent/comp/core/config" corelog "github.com/DataDog/datadog-agent/comp/core/log" "github.com/DataDog/datadog-agent/comp/core/status" + "github.com/DataDog/datadog-agent/comp/core/tagger" + compdef "github.com/DataDog/datadog-agent/comp/def" "github.com/DataDog/datadog-agent/comp/metadata/inventoryagent" + collector "github.com/DataDog/datadog-agent/comp/otelcol/collector/def" "github.com/DataDog/datadog-agent/comp/otelcol/logsagentpipeline" "github.com/DataDog/datadog-agent/comp/otelcol/otlp" + "github.com/DataDog/datadog-agent/comp/otelcol/otlp/datatype" "github.com/DataDog/datadog-agent/pkg/logs/message" "github.com/DataDog/datadog-agent/pkg/serializer" "github.com/DataDog/datadog-agent/pkg/util/optional" @@ -27,14 +30,12 @@ const ( otlpEnabled = "feature_otlp_enabled" ) -// dependencies specifies a list of dependencies required for the collector +// Requires specifies a list of dependencies required for the collector // to be instantiated. -type dependencies struct { - fx.In - - // Lc specifies the fx lifecycle settings, used for appending startup +type Requires struct { + // Lc specifies the lifecycle settings, used for appending startup // and shutdown hooks. - Lc fx.Lifecycle + Lc compdef.Lifecycle // Config specifies the Datadog Agent's configuration component. Config config.Component @@ -51,38 +52,45 @@ type dependencies struct { // InventoryAgent require the inventory metadata payload, allowing otelcol to add data to it. InventoryAgent inventoryagent.Component + + Tagger tagger.Component } -type provides struct { - fx.Out +// Provides specifics the types returned by the constructor +type Provides struct { + compdef.Out - Comp Component + Comp collector.Component StatusProvider status.InformationProvider } -type collector struct { - deps dependencies - col *otlp.Pipeline +type collectorImpl struct { + col *otlp.Pipeline + config config.Component + log corelog.Component + serializer serializer.MetricSerializer + logsAgent optional.Option[logsagentpipeline.Component] + inventoryAgent inventoryagent.Component + tagger tagger.Component } -func (c *collector) start(context.Context) error { - deps := c.deps - on := otlp.IsEnabled(deps.Config) - deps.InventoryAgent.Set(otlpEnabled, on) +func (c *collectorImpl) start(context.Context) error { + on := otlp.IsEnabled(c.config) + c.inventoryAgent.Set(otlpEnabled, on) if !on { return nil } var logch chan *message.Message - if v, ok := deps.LogsAgent.Get(); ok { + if v, ok := c.logsAgent.Get(); ok { if provider := v.GetPipelineProvider(); provider != nil { logch = provider.NextPipelineChan() } } var err error - col, err := otlp.NewPipelineFromAgentConfig(deps.Config, deps.Serializer, logch) + col, err := otlp.NewPipelineFromAgentConfig(c.config, c.serializer, logch, c.tagger) if err != nil { // failure to start the OTLP component shouldn't fail startup - deps.Log.Errorf("Error creating the OTLP ingest pipeline: %v", err) + c.log.Errorf("Error creating the OTLP ingest pipeline: %v", err) return nil } c.col = col @@ -91,13 +99,13 @@ func (c *collector) start(context.Context) error { ctx := context.Background() go func() { if err := col.Run(ctx); err != nil { - deps.Log.Errorf("Error running the OTLP ingest pipeline: %v", err) + c.log.Errorf("Error running the OTLP ingest pipeline: %v", err) } }() return nil } -func (c *collector) stop(context.Context) error { +func (c *collectorImpl) stop(context.Context) error { if c.col != nil { c.col.Stop() } @@ -105,22 +113,27 @@ func (c *collector) stop(context.Context) error { } // Status returns the status of the collector. -func (c *collector) Status() otlp.CollectorStatus { +func (c *collectorImpl) Status() datatype.CollectorStatus { return c.col.GetCollectorStatus() } -// newPipeline creates a new Component for this module and returns any errors on failure. -func newPipeline(deps dependencies) (provides, error) { - collector := &collector{ - deps: deps, +// NewComponent creates a new Component for this module and returns any errors on failure. +func NewComponent(reqs Requires) (Provides, error) { + collector := &collectorImpl{ + config: reqs.Config, + log: reqs.Log, + serializer: reqs.Serializer, + logsAgent: reqs.LogsAgent, + inventoryAgent: reqs.InventoryAgent, + tagger: reqs.Tagger, } - deps.Lc.Append(fx.Hook{ + reqs.Lc.Append(compdef.Hook{ OnStart: collector.start, OnStop: collector.stop, }) - return provides{ + return Provides{ Comp: collector, StatusProvider: status.NewInformationProvider(collector), }, nil diff --git a/comp/otelcol/collector/status.go b/comp/otelcol/collector/impl-pipeline/status.go similarity index 74% rename from comp/otelcol/collector/status.go rename to comp/otelcol/collector/impl-pipeline/status.go index 7b23aab22b356..c3019563ab636 100644 --- a/comp/otelcol/collector/status.go +++ b/comp/otelcol/collector/impl-pipeline/status.go @@ -18,7 +18,7 @@ import ( //go:embed status_templates var templatesFS embed.FS -func (c *collector) getStatusInfo() map[string]interface{} { +func (c *collectorImpl) getStatusInfo() map[string]interface{} { stats := make(map[string]interface{}) c.populateStatus(stats) @@ -26,9 +26,9 @@ func (c *collector) getStatusInfo() map[string]interface{} { return stats } -func (c *collector) populateStatus(stats map[string]interface{}) { +func (c *collectorImpl) populateStatus(stats map[string]interface{}) { otlpStatus := make(map[string]interface{}) - otlpIsEnabled := otlp.IsEnabled(c.deps.Config) + otlpIsEnabled := otlp.IsEnabled(c.config) var otlpCollectorStatus otlp.CollectorStatus @@ -45,28 +45,28 @@ func (c *collector) populateStatus(stats map[string]interface{}) { } // Name returns the name -func (c *collector) Name() string { +func (c *collectorImpl) Name() string { return "OTLP" } // Name returns the section -func (c *collector) Section() string { +func (c *collectorImpl) Section() string { return "OTLP" } // JSON populates the status map -func (c *collector) JSON(_ bool, stats map[string]interface{}) error { +func (c *collectorImpl) JSON(_ bool, stats map[string]interface{}) error { c.populateStatus(stats) return nil } // Text renders the text output -func (c *collector) Text(_ bool, buffer io.Writer) error { +func (c *collectorImpl) Text(_ bool, buffer io.Writer) error { return status.RenderText(templatesFS, "otlp.tmpl", buffer, c.getStatusInfo()) } // HTML renders the html output -func (c *collector) HTML(_ bool, _ io.Writer) error { +func (c *collectorImpl) HTML(_ bool, _ io.Writer) error { return nil } diff --git a/comp/otelcol/collector/status_templates/otlp.tmpl b/comp/otelcol/collector/impl-pipeline/status_templates/otlp.tmpl similarity index 100% rename from comp/otelcol/collector/status_templates/otlp.tmpl rename to comp/otelcol/collector/impl-pipeline/status_templates/otlp.tmpl diff --git a/comp/otelcol/collector/impl/collector.go b/comp/otelcol/collector/impl/collector.go index 14c53bfb25089..d9cf67eb04861 100644 --- a/comp/otelcol/collector/impl/collector.go +++ b/comp/otelcol/collector/impl/collector.go @@ -17,13 +17,14 @@ import ( flarebuilder "github.com/DataDog/datadog-agent/comp/core/flare/builder" "github.com/DataDog/datadog-agent/comp/core/hostname" corelog "github.com/DataDog/datadog-agent/comp/core/log" + "github.com/DataDog/datadog-agent/comp/core/tagger" compdef "github.com/DataDog/datadog-agent/comp/def" collectorcontrib "github.com/DataDog/datadog-agent/comp/otelcol/collector-contrib/def" collector "github.com/DataDog/datadog-agent/comp/otelcol/collector/def" "github.com/DataDog/datadog-agent/comp/otelcol/logsagentpipeline" - "github.com/DataDog/datadog-agent/comp/otelcol/otlp" "github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/exporter/datadogexporter" "github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/processor/infraattributesprocessor" + "github.com/DataDog/datadog-agent/comp/otelcol/otlp/datatype" "github.com/DataDog/datadog-agent/pkg/serializer" "github.com/DataDog/datadog-agent/pkg/util/optional" ) @@ -48,6 +49,7 @@ type Requires struct { Serializer serializer.MetricSerializer LogsAgent optional.Option[logsagentpipeline.Component] HostName hostname.Component + Tagger tagger.Component } // Provides declares the output types from the constructor @@ -58,8 +60,8 @@ type Provides struct { FlareProvider flarebuilder.Provider } -// New returns a new instance of the collector component. -func New(reqs Requires) (Provides, error) { +// NewComponent returns a new instance of the collector component. +func NewComponent(reqs Requires) (Provides, error) { set := otelcol.CollectorSettings{ BuildInfo: component.BuildInfo{ Version: "0.0.1", @@ -76,7 +78,7 @@ func New(reqs Requires) (Provides, error) { } else { factories.Exporters[datadogexporter.Type] = datadogexporter.NewFactory(reqs.Serializer, nil, reqs.HostName) } - factories.Processors[infraattributesprocessor.Type] = infraattributesprocessor.NewFactory() + factories.Processors[infraattributesprocessor.Type] = infraattributesprocessor.NewFactory(reqs.Tagger) return factories, nil }, ConfigProvider: reqs.Provider, @@ -123,8 +125,8 @@ func (c *collectorImpl) fillFlare(fb flarebuilder.FlareBuilder) error { return nil } -func (c *collectorImpl) Status() otlp.CollectorStatus { - return otlp.CollectorStatus{ +func (c *collectorImpl) Status() datatype.CollectorStatus { + return datatype.CollectorStatus{ Status: c.col.GetState().String(), } } diff --git a/comp/otelcol/collector/no_otlp.go b/comp/otelcol/collector/no_otlp.go deleted file mode 100644 index f33dc189e5776..0000000000000 --- a/comp/otelcol/collector/no_otlp.go +++ /dev/null @@ -1,38 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2016-present Datadog, Inc. - -//go:build !otlp - -// Package collector implements the OTLP Collector component for non-OTLP builds. -package collector - -import ( - "github.com/DataDog/datadog-agent/pkg/util/fxutil" - "go.uber.org/fx" -) - -// Component represents the no-op Component interface. -type Component interface { - Start() error - Stop() -} - -// Module specifies the fx module for non-OTLP builds. -func Module() fxutil.Module { - return fxutil.Component( - fx.Provide(newPipeline)) -} - -func newPipeline() (Component, error) { - return noOpComp{}, nil -} - -type noOpComp struct{} - -// Start is a no-op. -func (noOpComp) Start() error { return nil } - -// Stop is a no-op. -func (noOpComp) Stop() {} diff --git a/comp/otelcol/logsagentpipeline/go.mod b/comp/otelcol/logsagentpipeline/go.mod index 5b5790024260e..6c0ba0d794bfe 100644 --- a/comp/otelcol/logsagentpipeline/go.mod +++ b/comp/otelcol/logsagentpipeline/go.mod @@ -97,7 +97,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/winutil v0.54.0-rc.2 // indirect github.com/DataDog/datadog-agent/pkg/version v0.54.0-rc.2 // indirect github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240419161837-f1b2f553edfe // indirect - github.com/DataDog/viper v1.13.4 // indirect + github.com/DataDog/viper v1.13.5 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/benbjohnson/clock v1.3.5 // indirect github.com/beorn7/perks v1.0.1 // indirect diff --git a/comp/otelcol/logsagentpipeline/go.sum b/comp/otelcol/logsagentpipeline/go.sum index c368b404899ba..9dc78dae99954 100644 --- a/comp/otelcol/logsagentpipeline/go.sum +++ b/comp/otelcol/logsagentpipeline/go.sum @@ -43,8 +43,8 @@ github.com/DataDog/agent-payload/v5 v5.0.106 h1:A3dGX+JYoL7OJe2crpxznW7hWxLxhOk/ github.com/DataDog/agent-payload/v5 v5.0.106/go.mod h1:COngtbYYCncpIPiE5D93QlXDH/3VAKk10jDNwGHcMRE= github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240419161837-f1b2f553edfe h1:efzxujZ7VHWFxjmWjcJyUEpPrN8qdiZPYb+dBw547Wo= github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240419161837-f1b2f553edfe/go.mod h1:TX7CTOQ3LbQjfAi4SwqUoR5gY1zfUk7VRBDTuArjaDc= -github.com/DataDog/viper v1.13.4 h1:0SrZc3zvMAGgVKg96uP4DEJb13lK2Is9a4go7IIcFSE= -github.com/DataDog/viper v1.13.4/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= +github.com/DataDog/viper v1.13.5 h1:SZMcyMknYQN2jRY/40A16gUXexlNJOI8sDs1cWZnI64= +github.com/DataDog/viper v1.13.5/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= diff --git a/comp/otelcol/logsagentpipeline/logsagentpipelineimpl/go.mod b/comp/otelcol/logsagentpipeline/logsagentpipelineimpl/go.mod index 9664a5be3f4b8..79e9cb985232b 100644 --- a/comp/otelcol/logsagentpipeline/logsagentpipelineimpl/go.mod +++ b/comp/otelcol/logsagentpipeline/logsagentpipelineimpl/go.mod @@ -108,7 +108,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/winutil v0.54.0-rc.2 // indirect github.com/DataDog/datadog-agent/pkg/version v0.54.0-rc.2 // indirect github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240419161837-f1b2f553edfe // indirect - github.com/DataDog/viper v1.13.4 // indirect + github.com/DataDog/viper v1.13.5 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/benbjohnson/clock v1.3.5 // indirect github.com/beorn7/perks v1.0.1 // indirect diff --git a/comp/otelcol/logsagentpipeline/logsagentpipelineimpl/go.sum b/comp/otelcol/logsagentpipeline/logsagentpipelineimpl/go.sum index 3f8c9d86d36de..8decf55752b31 100644 --- a/comp/otelcol/logsagentpipeline/logsagentpipelineimpl/go.sum +++ b/comp/otelcol/logsagentpipeline/logsagentpipelineimpl/go.sum @@ -43,8 +43,8 @@ github.com/DataDog/agent-payload/v5 v5.0.106 h1:A3dGX+JYoL7OJe2crpxznW7hWxLxhOk/ github.com/DataDog/agent-payload/v5 v5.0.106/go.mod h1:COngtbYYCncpIPiE5D93QlXDH/3VAKk10jDNwGHcMRE= github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240419161837-f1b2f553edfe h1:efzxujZ7VHWFxjmWjcJyUEpPrN8qdiZPYb+dBw547Wo= github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240419161837-f1b2f553edfe/go.mod h1:TX7CTOQ3LbQjfAi4SwqUoR5gY1zfUk7VRBDTuArjaDc= -github.com/DataDog/viper v1.13.4 h1:0SrZc3zvMAGgVKg96uP4DEJb13lK2Is9a4go7IIcFSE= -github.com/DataDog/viper v1.13.4/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= +github.com/DataDog/viper v1.13.5 h1:SZMcyMknYQN2jRY/40A16gUXexlNJOI8sDs1cWZnI64= +github.com/DataDog/viper v1.13.5/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= diff --git a/comp/otelcol/otlp/collector.go b/comp/otelcol/otlp/collector.go index 5e8d0462bd7ba..e3896be2933e2 100644 --- a/comp/otelcol/otlp/collector.go +++ b/comp/otelcol/otlp/collector.go @@ -32,6 +32,7 @@ import ( "github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/exporter/logsagentexporter" "github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/exporter/serializerexporter" "github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/processor/infraattributesprocessor" + "github.com/DataDog/datadog-agent/comp/otelcol/otlp/datatype" "github.com/DataDog/datadog-agent/pkg/logs/message" "github.com/DataDog/datadog-agent/pkg/serializer" "github.com/DataDog/datadog-agent/pkg/util/flavor" @@ -81,7 +82,7 @@ func (t *tagEnricher) Enrich(_ context.Context, extraTags []string, dimensions * return enrichedTags } -func getComponents(s serializer.MetricSerializer, logsAgentChannel chan *message.Message) ( +func getComponents(s serializer.MetricSerializer, logsAgentChannel chan *message.Message, tagger tagger.Component) ( otelcol.Factories, error, ) { @@ -114,10 +115,11 @@ func getComponents(s serializer.MetricSerializer, logsAgentChannel chan *message errs = append(errs, err) } - processors, err := processor.MakeFactoryMap( - batchprocessor.NewFactory(), - infraattributesprocessor.NewFactory(), - ) + processorFactories := []processor.Factory{batchprocessor.NewFactory()} + if tagger != nil { + processorFactories = append(processorFactories, infraattributesprocessor.NewFactory(tagger)) + } + processors, err := processor.MakeFactoryMap(processorFactories...) if err != nil { errs = append(errs, err) } @@ -189,14 +191,11 @@ type Pipeline struct { col *otelcol.Collector } -// CollectorStatus is the status struct for an OTLP pipeline's collector -type CollectorStatus struct { - Status string - ErrorMessage string -} +// CollectorStatus is an alias to the datatype, for convenience +type CollectorStatus = datatype.CollectorStatus // NewPipeline defines a new OTLP pipeline. -func NewPipeline(cfg PipelineConfig, s serializer.MetricSerializer, logsAgentChannel chan *message.Message) (*Pipeline, error) { +func NewPipeline(cfg PipelineConfig, s serializer.MetricSerializer, logsAgentChannel chan *message.Message, tagger tagger.Component) (*Pipeline, error) { buildInfo, err := getBuildInfo() if err != nil { return nil, fmt.Errorf("failed to get build info: %w", err) @@ -216,7 +215,7 @@ func NewPipeline(cfg PipelineConfig, s serializer.MetricSerializer, logsAgentCha col, err := otelcol.NewCollector(otelcol.CollectorSettings{ Factories: func() (otelcol.Factories, error) { - return getComponents(s, logsAgentChannel) + return getComponents(s, logsAgentChannel, tagger) }, BuildInfo: buildInfo, DisableGracefulShutdown: true, @@ -253,7 +252,7 @@ func (p *Pipeline) Stop() { // NewPipelineFromAgentConfig creates a new pipeline from the given agent configuration, metric serializer and logs channel. It returns // any potential failure. -func NewPipelineFromAgentConfig(cfg config.Component, s serializer.MetricSerializer, logsAgentChannel chan *message.Message) (*Pipeline, error) { +func NewPipelineFromAgentConfig(cfg config.Component, s serializer.MetricSerializer, logsAgentChannel chan *message.Message, tagger tagger.Component) (*Pipeline, error) { pcfg, err := FromAgentConfig(cfg) if err != nil { pipelineError.Store(fmt.Errorf("config error: %w", err)) @@ -262,7 +261,7 @@ func NewPipelineFromAgentConfig(cfg config.Component, s serializer.MetricSeriali if err := checkAndUpdateCfg(cfg, pcfg, logsAgentChannel); err != nil { return nil, err } - p, err := NewPipeline(pcfg, s, logsAgentChannel) + p, err := NewPipeline(pcfg, s, logsAgentChannel, tagger) if err != nil { pipelineError.Store(fmt.Errorf("failed to build pipeline: %w", err)) return nil, pipelineError.Load() diff --git a/comp/otelcol/otlp/collector_test.go b/comp/otelcol/otlp/collector_test.go index 3c5c88d872de0..e9e746b664c34 100644 --- a/comp/otelcol/otlp/collector_test.go +++ b/comp/otelcol/otlp/collector_test.go @@ -13,6 +13,7 @@ import ( "testing" "time" + "github.com/DataDog/datadog-agent/comp/core/tagger/taggerimpl" "github.com/DataDog/datadog-agent/comp/otelcol/otlp/testutil" "github.com/DataDog/datadog-agent/pkg/config" "github.com/DataDog/datadog-agent/pkg/logs/message" @@ -23,13 +24,17 @@ import ( ) func TestGetComponents(t *testing.T) { - _, err := getComponents(&serializer.MockSerializer{}, make(chan *message.Message)) + fakeTagger := taggerimpl.SetupFakeTagger(t) + defer fakeTagger.ResetTagger() + _, err := getComponents(&serializer.MockSerializer{}, make(chan *message.Message), fakeTagger) // No duplicate component require.NoError(t, err) } func AssertSucessfulRun(t *testing.T, pcfg PipelineConfig) { - p, err := NewPipeline(pcfg, &serializer.MockSerializer{}, make(chan *message.Message)) + fakeTagger := taggerimpl.SetupFakeTagger(t) + defer fakeTagger.ResetTagger() + p, err := NewPipeline(pcfg, &serializer.MockSerializer{}, make(chan *message.Message), fakeTagger) require.NoError(t, err) ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() @@ -54,7 +59,9 @@ func AssertSucessfulRun(t *testing.T, pcfg PipelineConfig) { } func AssertFailedRun(t *testing.T, pcfg PipelineConfig, expected string) { - p, err := NewPipeline(pcfg, &serializer.MockSerializer{}, make(chan *message.Message)) + fakeTagger := taggerimpl.SetupFakeTagger(t) + defer fakeTagger.ResetTagger() + p, err := NewPipeline(pcfg, &serializer.MockSerializer{}, make(chan *message.Message), fakeTagger) require.NoError(t, err) ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() diff --git a/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod b/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod index dc68cc4f9e5e1..b5466332a7132 100644 --- a/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod +++ b/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod @@ -85,7 +85,7 @@ require ( github.com/DataDog/datadog-agent/pkg/logs/message v0.54.0-rc.2 github.com/DataDog/datadog-agent/pkg/serializer v0.54.0-rc.2 github.com/DataDog/datadog-agent/pkg/util/hostname/validate v0.54.0-rc.2 - github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.16.0 + github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.16.1 github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/metrics v0.16.0 github.com/stretchr/testify v1.9.0 go.opentelemetry.io/collector/component v0.100.0 @@ -165,7 +165,7 @@ require ( github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/logs v0.16.0 // indirect github.com/DataDog/opentelemetry-mapping-go/pkg/quantile v0.16.0 // indirect github.com/DataDog/sketches-go v1.4.4 // indirect - github.com/DataDog/viper v1.13.4 // indirect + github.com/DataDog/viper v1.13.5 // indirect github.com/DataDog/zstd v1.5.5 // indirect github.com/DataDog/zstd_0 v0.0.0-20210310093942-586c1286621f // indirect github.com/Microsoft/go-winio v0.6.1 // indirect diff --git a/comp/otelcol/otlp/components/exporter/datadogexporter/go.sum b/comp/otelcol/otlp/components/exporter/datadogexporter/go.sum index 865ca96d98bd5..47d43cbdaf7e5 100644 --- a/comp/otelcol/otlp/components/exporter/datadogexporter/go.sum +++ b/comp/otelcol/otlp/components/exporter/datadogexporter/go.sum @@ -10,8 +10,8 @@ github.com/DataDog/mmh3 v0.0.0-20210722141835-012dc69a9e49 h1:EbzDX8HPk5uE2FsJYx github.com/DataDog/mmh3 v0.0.0-20210722141835-012dc69a9e49/go.mod h1:SvsjzyJlSg0rKsqYgdcFxeEVflx3ZNAyFfkUHP0TxXg= github.com/DataDog/opentelemetry-mapping-go/pkg/internal/sketchtest v0.16.0 h1:VJT1Jjlz/ca999FEqaAS+He7S4eB14a+PJjczgRdgAY= github.com/DataDog/opentelemetry-mapping-go/pkg/internal/sketchtest v0.16.0/go.mod h1:66XlN7QpQKqIvw8e2UbCXV5X8wGnEw851nT9BjJ75dY= -github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.16.0 h1:g/ztrLYZNfkpW6Bt8kMnLed5DaKRHEtiKE0opHXLHJk= -github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.16.0/go.mod h1:dvIWN9pA2zWNTw5rhDWZgzZnhcfpH++d+8d1SWW6xkY= +github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.16.1 h1:ZI8u3CgdMXpDplrf9/gIr13+/g/tUzUcBMk2ZhXgzLE= +github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.16.1/go.mod h1:dvIWN9pA2zWNTw5rhDWZgzZnhcfpH++d+8d1SWW6xkY= github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/logs v0.16.0 h1:NbKlfbjR2joF52jEBLs3MEnT6l5zM3MCyhUFkqARZpk= github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/logs v0.16.0/go.mod h1:+LijQ2LdlocAQ4WB+7KsoIGe90bfogkRslubd9swVow= github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/metrics v0.16.0 h1:H5DzD3rwgQCX0VI3A16KgsdmC5grUCyDFflaZDpfgMc= @@ -20,8 +20,8 @@ github.com/DataDog/opentelemetry-mapping-go/pkg/quantile v0.16.0 h1:BdfRSuCoHyKa github.com/DataDog/opentelemetry-mapping-go/pkg/quantile v0.16.0/go.mod h1:6eYyd+lJYH+uRuZqhyW/u+9ykaXBWetDGj44+txz6jU= github.com/DataDog/sketches-go v1.4.4 h1:dF52vzXRFSPOj2IjXSWLvXq3jubL4CI69kwYjJ1w5Z8= github.com/DataDog/sketches-go v1.4.4/go.mod h1:XR0ns2RtEEF09mDKXiKZiQg+nfZStrq1ZuL1eezeZe0= -github.com/DataDog/viper v1.13.4 h1:0SrZc3zvMAGgVKg96uP4DEJb13lK2Is9a4go7IIcFSE= -github.com/DataDog/viper v1.13.4/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= +github.com/DataDog/viper v1.13.5 h1:SZMcyMknYQN2jRY/40A16gUXexlNJOI8sDs1cWZnI64= +github.com/DataDog/viper v1.13.5/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= github.com/DataDog/zstd v1.5.5 h1:oWf5W7GtOLgp6bciQYDmhHHjdhYkALu6S/5Ni9ZgSvQ= github.com/DataDog/zstd v1.5.5/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/DataDog/zstd_0 v0.0.0-20210310093942-586c1286621f h1:5Vuo4niPKFkfwW55jV4vY0ih3VQ9RaQqeqY67fvRn8A= diff --git a/comp/otelcol/otlp/components/exporter/logsagentexporter/go.mod b/comp/otelcol/otlp/components/exporter/logsagentexporter/go.mod index 6738710153680..83939e378bbbc 100644 --- a/comp/otelcol/otlp/components/exporter/logsagentexporter/go.mod +++ b/comp/otelcol/otlp/components/exporter/logsagentexporter/go.mod @@ -42,7 +42,7 @@ require ( github.com/DataDog/datadog-agent/pkg/logs/message v0.54.0-rc.2 github.com/DataDog/datadog-agent/pkg/logs/sources v0.54.0-rc.2 github.com/DataDog/datadog-agent/pkg/util/scrubber v0.54.0-rc.2 - github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.14.0 + github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.16.1 github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/logs v0.14.0 github.com/stormcat24/protodep v0.1.8 github.com/stretchr/testify v1.9.0 @@ -71,7 +71,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/winutil v0.54.0-rc.2 // indirect github.com/DataDog/datadog-agent/pkg/version v0.54.0-rc.2 // indirect github.com/DataDog/datadog-api-client-go/v2 v2.13.0 // indirect - github.com/DataDog/viper v1.13.4 // indirect + github.com/DataDog/viper v1.13.5 // indirect github.com/DataDog/zstd v1.5.2 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/beorn7/perks v1.0.1 // indirect diff --git a/comp/otelcol/otlp/components/exporter/logsagentexporter/go.sum b/comp/otelcol/otlp/components/exporter/logsagentexporter/go.sum index fdb527179c99c..f2273f425b08b 100644 --- a/comp/otelcol/otlp/components/exporter/logsagentexporter/go.sum +++ b/comp/otelcol/otlp/components/exporter/logsagentexporter/go.sum @@ -4,12 +4,12 @@ github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8 github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/DataDog/datadog-api-client-go/v2 v2.13.0 h1:2c1dXSyUfum2YIVoYlqnBhV5JOG1cLSW+4jB3RrKjLc= github.com/DataDog/datadog-api-client-go/v2 v2.13.0/go.mod h1:kntOqXEh1SmjwSDzW/eJkr9kS7EqttvEkelglWtJRbg= -github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.14.0 h1:10TPqpTlIkmDPFWVIEZ4ZX3rWrCrx3rEoeoAooZr6LM= -github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.14.0/go.mod h1:dvIWN9pA2zWNTw5rhDWZgzZnhcfpH++d+8d1SWW6xkY= +github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.16.1 h1:ZI8u3CgdMXpDplrf9/gIr13+/g/tUzUcBMk2ZhXgzLE= +github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.16.1/go.mod h1:dvIWN9pA2zWNTw5rhDWZgzZnhcfpH++d+8d1SWW6xkY= github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/logs v0.14.0 h1:nma5ZICTbHZ0YoMu18ziWGSLK1ICzMm6rJTv+IatJ0U= github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/logs v0.14.0/go.mod h1:xUiGj13q5uHPboc0xZ754fyusiF5C2RxNzOFdTbdZFA= -github.com/DataDog/viper v1.13.4 h1:0SrZc3zvMAGgVKg96uP4DEJb13lK2Is9a4go7IIcFSE= -github.com/DataDog/viper v1.13.4/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= +github.com/DataDog/viper v1.13.5 h1:SZMcyMknYQN2jRY/40A16gUXexlNJOI8sDs1cWZnI64= +github.com/DataDog/viper v1.13.5/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= github.com/DataDog/zstd v1.5.2 h1:vUG4lAyuPCXO0TLbXvPv7EB7cNK1QV/luu55UHLrrn8= github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= diff --git a/comp/otelcol/otlp/components/exporter/serializerexporter/go.mod b/comp/otelcol/otlp/components/exporter/serializerexporter/go.mod index 5a6904988d052..6d39cc94c30dc 100644 --- a/comp/otelcol/otlp/components/exporter/serializerexporter/go.mod +++ b/comp/otelcol/otlp/components/exporter/serializerexporter/go.mod @@ -62,7 +62,7 @@ require ( github.com/DataDog/datadog-agent/pkg/serializer v0.54.0-rc.2 github.com/DataDog/datadog-agent/pkg/tagset v0.54.0-rc.2 github.com/DataDog/datadog-agent/pkg/util/log v0.54.0-rc.2 - github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.14.0 + github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.16.1 github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/metrics v0.14.0 github.com/DataDog/opentelemetry-mapping-go/pkg/quantile v0.14.0 github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c // indirect @@ -123,7 +123,7 @@ require ( github.com/DataDog/datadog-agent/pkg/version v0.54.0-rc.2 // indirect github.com/DataDog/mmh3 v0.0.0-20210722141835-012dc69a9e49 // indirect github.com/DataDog/sketches-go v1.4.4 // indirect - github.com/DataDog/viper v1.13.4 // indirect + github.com/DataDog/viper v1.13.5 // indirect github.com/DataDog/zstd v1.5.5 // indirect github.com/DataDog/zstd_0 v0.0.0-20210310093942-586c1286621f // indirect github.com/Microsoft/go-winio v0.6.1 // indirect diff --git a/comp/otelcol/otlp/components/exporter/serializerexporter/go.sum b/comp/otelcol/otlp/components/exporter/serializerexporter/go.sum index f61fcaa4f266e..eb26676ac20a2 100644 --- a/comp/otelcol/otlp/components/exporter/serializerexporter/go.sum +++ b/comp/otelcol/otlp/components/exporter/serializerexporter/go.sum @@ -6,16 +6,16 @@ github.com/DataDog/mmh3 v0.0.0-20210722141835-012dc69a9e49 h1:EbzDX8HPk5uE2FsJYx github.com/DataDog/mmh3 v0.0.0-20210722141835-012dc69a9e49/go.mod h1:SvsjzyJlSg0rKsqYgdcFxeEVflx3ZNAyFfkUHP0TxXg= github.com/DataDog/opentelemetry-mapping-go/pkg/internal/sketchtest v0.14.0 h1:J0IEqkrB8BjtuDHofR8Q3J+Z8829Ja1Mlix9cyG8wJI= github.com/DataDog/opentelemetry-mapping-go/pkg/internal/sketchtest v0.14.0/go.mod h1:66XlN7QpQKqIvw8e2UbCXV5X8wGnEw851nT9BjJ75dY= -github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.14.0 h1:10TPqpTlIkmDPFWVIEZ4ZX3rWrCrx3rEoeoAooZr6LM= -github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.14.0/go.mod h1:dvIWN9pA2zWNTw5rhDWZgzZnhcfpH++d+8d1SWW6xkY= +github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.16.1 h1:ZI8u3CgdMXpDplrf9/gIr13+/g/tUzUcBMk2ZhXgzLE= +github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.16.1/go.mod h1:dvIWN9pA2zWNTw5rhDWZgzZnhcfpH++d+8d1SWW6xkY= github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/metrics v0.14.0 h1:2Ou9n/048KM9E70G+LpJ4svBH2kOb8NkxwEqEGAamHo= github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/metrics v0.14.0/go.mod h1:CAj5xIuF45abugAck7OzjMLH5y20ylaEj1qbQEjU1gk= github.com/DataDog/opentelemetry-mapping-go/pkg/quantile v0.14.0 h1:QHx6B/VUx3rZQqrQNZI5BfypbhhGSRzCz05viyJEQmM= github.com/DataDog/opentelemetry-mapping-go/pkg/quantile v0.14.0/go.mod h1:q4c7zbmdnIdSJNZuBsveTk5ZeRkSkS2g6b8zzFF1mE4= github.com/DataDog/sketches-go v1.4.4 h1:dF52vzXRFSPOj2IjXSWLvXq3jubL4CI69kwYjJ1w5Z8= github.com/DataDog/sketches-go v1.4.4/go.mod h1:XR0ns2RtEEF09mDKXiKZiQg+nfZStrq1ZuL1eezeZe0= -github.com/DataDog/viper v1.13.4 h1:0SrZc3zvMAGgVKg96uP4DEJb13lK2Is9a4go7IIcFSE= -github.com/DataDog/viper v1.13.4/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= +github.com/DataDog/viper v1.13.5 h1:SZMcyMknYQN2jRY/40A16gUXexlNJOI8sDs1cWZnI64= +github.com/DataDog/viper v1.13.5/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= github.com/DataDog/zstd v1.5.5 h1:oWf5W7GtOLgp6bciQYDmhHHjdhYkALu6S/5Ni9ZgSvQ= github.com/DataDog/zstd v1.5.5/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/DataDog/zstd_0 v0.0.0-20210310093942-586c1286621f h1:5Vuo4niPKFkfwW55jV4vY0ih3VQ9RaQqeqY67fvRn8A= diff --git a/comp/otelcol/otlp/components/processor/infraattributesprocessor/config.go b/comp/otelcol/otlp/components/processor/infraattributesprocessor/config.go index a24a25b8db6fa..8b1c438bdcfc2 100644 --- a/comp/otelcol/otlp/components/processor/infraattributesprocessor/config.go +++ b/comp/otelcol/otlp/components/processor/infraattributesprocessor/config.go @@ -7,6 +7,8 @@ package infraattributesprocessor import ( "go.opentelemetry.io/collector/component" + + "github.com/DataDog/datadog-agent/comp/core/tagger/types" ) // Config defines configuration for processor. @@ -14,6 +16,8 @@ type Config struct { Metrics MetricInfraAttributes `mapstructure:"metrics"` Logs LogInfraAttributes `mapstructure:"logs"` Traces TraceInfraAttributes `mapstructure:"traces"` + + Cardinality types.TagCardinality `mapstructure:"cardinality"` } // MetricInfraAttributes - configuration for metrics. diff --git a/comp/otelcol/otlp/components/processor/infraattributesprocessor/config_test.go b/comp/otelcol/otlp/components/processor/infraattributesprocessor/config_test.go index c8bf6903ec9dc..0011a3b0a8d14 100644 --- a/comp/otelcol/otlp/components/processor/infraattributesprocessor/config_test.go +++ b/comp/otelcol/otlp/components/processor/infraattributesprocessor/config_test.go @@ -13,6 +13,8 @@ import ( "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/confmap/confmaptest" + + "github.com/DataDog/datadog-agent/comp/core/tagger/taggerimpl" ) // TestLoadingConfigStrictLogs tests loading testdata/logs_strict.yaml @@ -34,8 +36,10 @@ func TestLoadingConfigStrictLogs(t *testing.T) { for _, tt := range tests { t.Run(tt.id.String(), func(t *testing.T) { - factory := NewFactory() - cfg := factory.CreateDefaultConfig() + fakeTagger := taggerimpl.SetupFakeTagger(t) + defer fakeTagger.ResetTagger() + f := NewFactory(fakeTagger) + cfg := f.CreateDefaultConfig() sub, err := cm.Sub(tt.id.String()) require.NoError(t, err) diff --git a/comp/otelcol/otlp/components/processor/infraattributesprocessor/factory.go b/comp/otelcol/otlp/components/processor/infraattributesprocessor/factory.go index b71129f5bfc16..00046a13a8296 100644 --- a/comp/otelcol/otlp/components/processor/infraattributesprocessor/factory.go +++ b/comp/otelcol/otlp/components/processor/infraattributesprocessor/factory.go @@ -8,6 +8,9 @@ package infraattributesprocessor import ( "context" + "github.com/DataDog/datadog-agent/comp/core/tagger" + "github.com/DataDog/datadog-agent/comp/core/tagger/types" + "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/consumer" "go.opentelemetry.io/collector/processor" @@ -16,28 +19,38 @@ import ( var processorCapabilities = consumer.Capabilities{MutatesData: true} +type factory struct { + tagger tagger.Component +} + // NewFactory returns a new factory for the InfraAttributes processor. -func NewFactory() processor.Factory { +func NewFactory(tagger tagger.Component) processor.Factory { + f := &factory{ + tagger: tagger, + } + return processor.NewFactory( Type, - createDefaultConfig, - processor.WithMetrics(createMetricsProcessor, MetricsStability), - processor.WithLogs(createLogsProcessor, LogsStability), - processor.WithTraces(createTracesProcessor, TracesStability), + f.createDefaultConfig, + processor.WithMetrics(f.createMetricsProcessor, MetricsStability), + processor.WithLogs(f.createLogsProcessor, LogsStability), + processor.WithTraces(f.createTracesProcessor, TracesStability), ) } -func createDefaultConfig() component.Config { - return &Config{} +func (f *factory) createDefaultConfig() component.Config { + return &Config{ + Cardinality: types.LowCardinality, + } } -func createMetricsProcessor( +func (f *factory) createMetricsProcessor( ctx context.Context, set processor.CreateSettings, cfg component.Config, nextConsumer consumer.Metrics, ) (processor.Metrics, error) { - tep, err := newInfraAttributesMetricProcessor(set, cfg.(*Config)) + iap, err := newInfraAttributesMetricProcessor(set, cfg.(*Config), f.tagger) if err != nil { return nil, err } @@ -46,17 +59,17 @@ func createMetricsProcessor( set, cfg, nextConsumer, - tep.processMetrics, + iap.processMetrics, processorhelper.WithCapabilities(processorCapabilities)) } -func createLogsProcessor( +func (f *factory) createLogsProcessor( ctx context.Context, set processor.CreateSettings, cfg component.Config, nextConsumer consumer.Logs, ) (processor.Logs, error) { - tep, err := newInfraAttributesLogsProcessor(set, cfg.(*Config)) + iap, err := newInfraAttributesLogsProcessor(set, cfg.(*Config)) if err != nil { return nil, err } @@ -65,17 +78,17 @@ func createLogsProcessor( set, cfg, nextConsumer, - tep.processLogs, + iap.processLogs, processorhelper.WithCapabilities(processorCapabilities)) } -func createTracesProcessor( +func (f *factory) createTracesProcessor( ctx context.Context, set processor.CreateSettings, cfg component.Config, nextConsumer consumer.Traces, ) (processor.Traces, error) { - tep, err := newInfraAttributesSpanProcessor(set, cfg.(*Config)) + iap, err := newInfraAttributesSpanProcessor(set, cfg.(*Config)) if err != nil { return nil, err } @@ -84,6 +97,6 @@ func createTracesProcessor( set, cfg, nextConsumer, - tep.processTraces, + iap.processTraces, processorhelper.WithCapabilities(processorCapabilities)) } diff --git a/comp/otelcol/otlp/components/processor/infraattributesprocessor/factory_test.go b/comp/otelcol/otlp/components/processor/infraattributesprocessor/factory_test.go index 0019d68fe9eef..d168a11b40922 100644 --- a/comp/otelcol/otlp/components/processor/infraattributesprocessor/factory_test.go +++ b/comp/otelcol/otlp/components/processor/infraattributesprocessor/factory_test.go @@ -18,17 +18,23 @@ import ( "go.opentelemetry.io/collector/confmap/confmaptest" "go.opentelemetry.io/collector/consumer/consumertest" "go.opentelemetry.io/collector/processor/processortest" + + "github.com/DataDog/datadog-agent/comp/core/tagger/taggerimpl" ) func TestType(t *testing.T) { - factory := NewFactory() + fakeTagger := taggerimpl.SetupFakeTagger(t) + defer fakeTagger.ResetTagger() + factory := NewFactory(fakeTagger) pType := factory.Type() assert.Equal(t, pType, Type) } func TestCreateDefaultConfig(t *testing.T) { - factory := NewFactory() + fakeTagger := taggerimpl.SetupFakeTagger(t) + defer fakeTagger.ResetTagger() + factory := NewFactory(fakeTagger) cfg := factory.CreateDefaultConfig() assert.NoError(t, componenttest.CheckConfigStruct(cfg)) } @@ -50,10 +56,12 @@ func TestCreateProcessors(t *testing.T) { t.Run(tt.configName, func(t *testing.T) { cm, err := confmaptest.LoadConf(filepath.Join("testdata", tt.configName)) require.NoError(t, err) + fakeTagger := taggerimpl.SetupFakeTagger(t) + defer fakeTagger.ResetTagger() for k := range cm.ToStringMap() { // Check if all processor variations that are defined in test config can be actually created - factory := NewFactory() + factory := NewFactory(fakeTagger) cfg := factory.CreateDefaultConfig() sub, err := cm.Sub(k) diff --git a/comp/otelcol/otlp/components/processor/infraattributesprocessor/logs.go b/comp/otelcol/otlp/components/processor/infraattributesprocessor/logs.go index 577c3ce5015ac..ad326b36c8d80 100644 --- a/comp/otelcol/otlp/components/processor/infraattributesprocessor/logs.go +++ b/comp/otelcol/otlp/components/processor/infraattributesprocessor/logs.go @@ -22,7 +22,7 @@ func newInfraAttributesLogsProcessor(set processor.CreateSettings, _ *Config) (* logger: set.Logger, } - set.Logger.Info("Logs Tag Enrichment configured") + set.Logger.Info("Logs Infra Attributes Processor configured") return telp, nil } diff --git a/comp/otelcol/otlp/components/processor/infraattributesprocessor/logs_test.go b/comp/otelcol/otlp/components/processor/infraattributesprocessor/logs_test.go index f0404fd588663..7775262b101a7 100644 --- a/comp/otelcol/otlp/components/processor/infraattributesprocessor/logs_test.go +++ b/comp/otelcol/otlp/components/processor/infraattributesprocessor/logs_test.go @@ -14,6 +14,8 @@ import ( "go.opentelemetry.io/collector/consumer/consumertest" "go.opentelemetry.io/collector/pdata/plog" "go.opentelemetry.io/collector/processor/processortest" + + "github.com/DataDog/datadog-agent/comp/core/tagger/taggerimpl" ) type logNameTest struct { @@ -40,7 +42,9 @@ func TestInfraAttributesLogProcessor(t *testing.T) { cfg := &Config{ Logs: LogInfraAttributes{}, } - factory := NewFactory() + fakeTagger := taggerimpl.SetupFakeTagger(t) + defer fakeTagger.ResetTagger() + factory := NewFactory(fakeTagger) flp, err := factory.CreateLogsProcessor( context.Background(), processortest.NewNopCreateSettings(), diff --git a/comp/otelcol/otlp/components/processor/infraattributesprocessor/metadata.go b/comp/otelcol/otlp/components/processor/infraattributesprocessor/metadata.go index bf8eba0ad92d0..18eeafea7ed2c 100644 --- a/comp/otelcol/otlp/components/processor/infraattributesprocessor/metadata.go +++ b/comp/otelcol/otlp/components/processor/infraattributesprocessor/metadata.go @@ -12,7 +12,7 @@ import ( ) var ( - // Type for tag enrichment processor. + // Type for infra attributes processor. Type = component.MustNewType("infraattributes") ) @@ -25,12 +25,12 @@ const ( LogsStability = component.StabilityLevelAlpha ) -// Meter for tag enrichement. +// Meter for infra attributes processor. func Meter(settings component.TelemetrySettings) metric.Meter { return settings.MeterProvider.Meter("otelcol/infraattributes") } -// Tracer for tag enrichment. +// Tracer for infra attributes processor. func Tracer(settings component.TelemetrySettings) trace.Tracer { return settings.TracerProvider.Tracer("otelcol/infraattributes") } diff --git a/comp/otelcol/otlp/components/processor/infraattributesprocessor/metrics.go b/comp/otelcol/otlp/components/processor/infraattributesprocessor/metrics.go index 63e09edd7d892..95a834b84f10c 100644 --- a/comp/otelcol/otlp/components/processor/infraattributesprocessor/metrics.go +++ b/comp/otelcol/otlp/components/processor/infraattributesprocessor/metrics.go @@ -7,24 +7,120 @@ package infraattributesprocessor import ( "context" + "fmt" + "strings" + "github.com/DataDog/datadog-agent/comp/core/tagger" + "github.com/DataDog/datadog-agent/comp/core/tagger/types" + + "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/pmetric" "go.opentelemetry.io/collector/processor" + conventions "go.opentelemetry.io/collector/semconv/v1.21.0" "go.uber.org/zap" ) type infraAttributesMetricProcessor struct { - logger *zap.Logger + logger *zap.Logger + tagger tagger.Component + cardinality types.TagCardinality } -func newInfraAttributesMetricProcessor(set processor.CreateSettings, _ *Config) (*infraAttributesMetricProcessor, error) { - tesp := &infraAttributesMetricProcessor{ - logger: set.Logger, +func newInfraAttributesMetricProcessor(set processor.CreateSettings, cfg *Config, tagger tagger.Component) (*infraAttributesMetricProcessor, error) { + iamp := &infraAttributesMetricProcessor{ + logger: set.Logger, + tagger: tagger, + cardinality: cfg.Cardinality, } - set.Logger.Info("Metric Tag Enrichment configured") - return tesp, nil + set.Logger.Info("Metric Infra Attributes Processor configured") + return iamp, nil } -func (temp *infraAttributesMetricProcessor) processMetrics(_ context.Context, md pmetric.Metrics) (pmetric.Metrics, error) { +// TODO: Replace OriginIDFromAttributes in opentelemetry-mapping-go with this method +// entityIDsFromAttributes gets the entity IDs from resource attributes. +// If not found, an empty string slice is returned. +func entityIDsFromAttributes(attrs pcommon.Map) []string { + entityIDs := make([]string, 0, 8) + // Prefixes come from pkg/util/kubernetes/kubelet and pkg/util/containers. + if containerID, ok := attrs.Get(conventions.AttributeContainerID); ok { + entityIDs = append(entityIDs, fmt.Sprintf("container_id://%v", containerID.AsString())) + } + if containerImageID, ok := attrs.Get(conventions.AttributeContainerImageID); ok { + splitImageID := strings.SplitN(containerImageID.AsString(), "@sha256:", 2) + if len(splitImageID) == 2 { + entityIDs = append(entityIDs, fmt.Sprintf("container_image_metadata://sha256:%v", splitImageID[1])) + } + } + if ecsTaskArn, ok := attrs.Get(conventions.AttributeAWSECSTaskARN); ok { + entityIDs = append(entityIDs, fmt.Sprintf("ecs_task://%v", ecsTaskArn.AsString())) + } + if deploymentName, ok := attrs.Get(conventions.AttributeK8SDeploymentName); ok { + namespace, namespaceOk := attrs.Get(conventions.AttributeK8SNamespaceName) + if namespaceOk { + entityIDs = append(entityIDs, fmt.Sprintf("deployment://%v/%v", namespace.AsString(), deploymentName.AsString())) + } + } + if namespace, ok := attrs.Get(conventions.AttributeK8SNamespaceName); ok { + entityIDs = append(entityIDs, fmt.Sprintf("namespace://%v", namespace.AsString())) + } + if nodeUID, ok := attrs.Get(conventions.AttributeK8SNodeUID); ok { + entityIDs = append(entityIDs, fmt.Sprintf("kubernetes_node_uid://%v", nodeUID.AsString())) + } + if podUID, ok := attrs.Get(conventions.AttributeK8SPodUID); ok { + entityIDs = append(entityIDs, fmt.Sprintf("kubernetes_pod_uid://%v", podUID.AsString())) + } + if processPid, ok := attrs.Get(conventions.AttributeProcessPID); ok { + entityIDs = append(entityIDs, fmt.Sprintf("process://%v", processPid.AsString())) + } + return entityIDs +} + +func splitTag(tag string) (key string, value string) { + split := strings.SplitN(tag, ":", 2) + if len(split) < 2 || split[0] == "" || split[1] == "" { + return "", "" + } + return split[0], split[1] +} + +func (iamp *infraAttributesMetricProcessor) processMetrics(_ context.Context, md pmetric.Metrics) (pmetric.Metrics, error) { + rms := md.ResourceMetrics() + for i := 0; i < rms.Len(); i++ { + resourceAttributes := rms.At(i).Resource().Attributes() + entityIDs := entityIDsFromAttributes(resourceAttributes) + tagMap := make(map[string]string) + + // Get all unique tags from resource attributes and global tags + for _, entityID := range entityIDs { + entityTags, err := iamp.tagger.Tag(entityID, iamp.cardinality) + if err != nil { + iamp.logger.Error("Cannot get tags for entity", zap.String("entityID", entityID), zap.Error(err)) + continue + } + for _, tag := range entityTags { + k, v := splitTag(tag) + _, hasTag := tagMap[k] + if k != "" && v != "" && !hasTag { + tagMap[k] = v + } + } + } + globalTags, err := iamp.tagger.GlobalTags(iamp.cardinality) + if err != nil { + iamp.logger.Error("Cannot get global tags", zap.Error(err)) + } + for _, tag := range globalTags { + k, v := splitTag(tag) + _, hasTag := tagMap[k] + if k != "" && v != "" && !hasTag { + tagMap[k] = v + } + } + + // Add all tags as resource attributes + for k, v := range tagMap { + resourceAttributes.PutStr(k, v) + } + } return md, nil } diff --git a/comp/otelcol/otlp/components/processor/infraattributesprocessor/metrics_test.go b/comp/otelcol/otlp/components/processor/infraattributesprocessor/metrics_test.go index c6adb8492a7aa..50892d20c94a3 100644 --- a/comp/otelcol/otlp/components/processor/infraattributesprocessor/metrics_test.go +++ b/comp/otelcol/otlp/components/processor/infraattributesprocessor/metrics_test.go @@ -11,13 +11,20 @@ import ( "github.com/stretchr/testify/assert" "go.opentelemetry.io/collector/consumer/consumertest" + "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/pmetric" "go.opentelemetry.io/collector/processor/processortest" + conventions "go.opentelemetry.io/collector/semconv/v1.21.0" + + "github.com/DataDog/datadog-agent/comp/core/tagger/taggerimpl" + "github.com/DataDog/datadog-agent/comp/core/tagger/taggerimpl/collectors" + "github.com/DataDog/datadog-agent/comp/core/tagger/types" ) type metricNameTest struct { - name string - inMetrics pmetric.Metrics + name string + inMetrics pmetric.Metrics + outResourceAttributes []map[string]any } type metricWithResource struct { @@ -32,20 +39,101 @@ var ( standardTests = []metricNameTest{ { - name: "includeFilter", - inMetrics: testResourceMetrics([]metricWithResource{{metricNames: inMetricNames}}), + name: "one tag with global", + inMetrics: testResourceMetrics([]metricWithResource{{ + metricNames: inMetricNames, + resourceAttributes: map[string]any{ + "container.id": "test", + }, + }}), + outResourceAttributes: []map[string]any{{ + "global": "tag", + "container.id": "test", + "container": "id", + }}, + }, + { + name: "two tags with global", + inMetrics: testResourceMetrics([]metricWithResource{{ + metricNames: inMetricNames, + resourceAttributes: map[string]any{ + "container.id": "test", + "k8s.namespace.name": "namespace", + "k8s.deployment.name": "deployment", + }, + }}), + outResourceAttributes: []map[string]any{{ + "global": "tag", + "container.id": "test", + "k8s.namespace.name": "namespace", + "k8s.deployment.name": "deployment", + "container": "id", + "deployment": "name", + }}, + }, + { + name: "two resource metrics, two tags with global", + inMetrics: testResourceMetrics([]metricWithResource{ + { + metricNames: inMetricNames, + resourceAttributes: map[string]any{ + "container.id": "test", + }, + }, + { + metricNames: inMetricNames, + resourceAttributes: map[string]any{ + "k8s.namespace.name": "namespace", + "k8s.deployment.name": "deployment", + }, + }}), + outResourceAttributes: []map[string]any{ + { + "global": "tag", + "container.id": "test", + "container": "id", + }, + { + "global": "tag", + "k8s.namespace.name": "namespace", + "k8s.deployment.name": "deployment", + "deployment": "name", + }, + }, }, } ) +func testResourceMetrics(mwrs []metricWithResource) pmetric.Metrics { + md := pmetric.NewMetrics() + + for _, mwr := range mwrs { + rm := md.ResourceMetrics().AppendEmpty() + //nolint:errcheck + rm.Resource().Attributes().FromRaw(mwr.resourceAttributes) + ms := rm.ScopeMetrics().AppendEmpty().Metrics() + for _, name := range mwr.metricNames { + m := ms.AppendEmpty() + m.SetName(name) + } + } + return md +} + func TestInfraAttributesMetricProcessor(t *testing.T) { for _, test := range standardTests { t.Run(test.name, func(t *testing.T) { next := new(consumertest.MetricsSink) cfg := &Config{ - Metrics: MetricInfraAttributes{}, + Metrics: MetricInfraAttributes{}, + Cardinality: types.LowCardinality, } - factory := NewFactory() + fakeTagger := taggerimpl.SetupFakeTagger(t) + defer fakeTagger.ResetTagger() + fakeTagger.SetTags("container_id://test", "test", []string{"container:id"}, nil, nil, nil) + fakeTagger.SetTags("deployment://namespace/deployment", "test", []string{"deployment:name"}, nil, nil, nil) + fakeTagger.SetTags(collectors.GlobalEntityID, "test", []string{"global:tag"}, nil, nil, nil) + factory := NewFactory(fakeTagger) fmp, err := factory.CreateMetricsProcessor( context.Background(), processortest.NewNopCreateSettings(), @@ -63,22 +151,124 @@ func TestInfraAttributesMetricProcessor(t *testing.T) { cErr := fmp.ConsumeMetrics(context.Background(), test.inMetrics) assert.Nil(t, cErr) assert.NoError(t, fmp.Shutdown(ctx)) + + assert.Len(t, next.AllMetrics(), 1) + for i, out := range test.outResourceAttributes { + rms := next.AllMetrics()[0].ResourceMetrics().At(i) + assert.NotNil(t, rms) + assert.EqualValues(t, out, rms.Resource().Attributes().AsRaw()) + } }) } } -func testResourceMetrics(mwrs []metricWithResource) pmetric.Metrics { - md := pmetric.NewMetrics() +func TestEntityIDsFromAttributes(t *testing.T) { + tests := []struct { + name string + attrs pcommon.Map + entityIDs []string + }{ + { + name: "none", + attrs: pcommon.NewMap(), + entityIDs: []string{}, + }, + { + name: "pod UID and container ID", + attrs: func() pcommon.Map { + attributes := pcommon.NewMap() + attributes.FromRaw(map[string]interface{}{ + conventions.AttributeContainerID: "container_id_goes_here", + conventions.AttributeK8SPodUID: "k8s_pod_uid_goes_here", + }) + return attributes + }(), + entityIDs: []string{"container_id://container_id_goes_here", "kubernetes_pod_uid://k8s_pod_uid_goes_here"}, + }, + { + name: "container image ID", + attrs: func() pcommon.Map { + attributes := pcommon.NewMap() + attributes.FromRaw(map[string]interface{}{ + conventions.AttributeContainerImageID: "docker.io/foo@sha256:sha_goes_here", + }) + return attributes + }(), + entityIDs: []string{"container_image_metadata://sha256:sha_goes_here"}, + }, + { + name: "ecs task arn", + attrs: func() pcommon.Map { + attributes := pcommon.NewMap() + attributes.FromRaw(map[string]interface{}{ + conventions.AttributeAWSECSTaskARN: "ecs_task_arn_goes_here", + }) + return attributes + }(), + entityIDs: []string{"ecs_task://ecs_task_arn_goes_here"}, + }, + { + name: "only deployment name without namespace", + attrs: func() pcommon.Map { + attributes := pcommon.NewMap() + attributes.FromRaw(map[string]interface{}{ + conventions.AttributeK8SDeploymentName: "k8s_deployment_name_goes_here", + }) + return attributes + }(), + entityIDs: []string{}, + }, + { + name: "deployment name and namespace", + attrs: func() pcommon.Map { + attributes := pcommon.NewMap() + attributes.FromRaw(map[string]interface{}{ + conventions.AttributeK8SDeploymentName: "k8s_deployment_name_goes_here", + conventions.AttributeK8SNamespaceName: "k8s_namespace_goes_here", + }) + return attributes + }(), + entityIDs: []string{"deployment://k8s_namespace_goes_here/k8s_deployment_name_goes_here", "namespace://k8s_namespace_goes_here"}, + }, + { + name: "only namespace name", + attrs: func() pcommon.Map { + attributes := pcommon.NewMap() + attributes.FromRaw(map[string]interface{}{ + conventions.AttributeK8SNamespaceName: "k8s_namespace_goes_here", + }) + return attributes + }(), + entityIDs: []string{"namespace://k8s_namespace_goes_here"}, + }, + { + name: "only node UID", + attrs: func() pcommon.Map { + attributes := pcommon.NewMap() + attributes.FromRaw(map[string]interface{}{ + conventions.AttributeK8SNodeUID: "k8s_node_uid_goes_here", + }) + return attributes + }(), + entityIDs: []string{"kubernetes_node_uid://k8s_node_uid_goes_here"}, + }, + { + name: "only process pid", + attrs: func() pcommon.Map { + attributes := pcommon.NewMap() + attributes.FromRaw(map[string]interface{}{ + conventions.AttributeProcessPID: "process_pid_goes_here", + }) + return attributes + }(), + entityIDs: []string{"process://process_pid_goes_here"}, + }, + } - for _, mwr := range mwrs { - rm := md.ResourceMetrics().AppendEmpty() - //nolint:errcheck - rm.Resource().Attributes().FromRaw(mwr.resourceAttributes) - ms := rm.ScopeMetrics().AppendEmpty().Metrics() - for _, name := range mwr.metricNames { - m := ms.AppendEmpty() - m.SetName(name) - } + for _, testInstance := range tests { + t.Run(testInstance.name, func(t *testing.T) { + entityIDs := entityIDsFromAttributes(testInstance.attrs) + assert.Equal(t, testInstance.entityIDs, entityIDs) + }) } - return md } diff --git a/comp/otelcol/otlp/components/processor/infraattributesprocessor/traces.go b/comp/otelcol/otlp/components/processor/infraattributesprocessor/traces.go index 6a3134833165e..857a7c482c708 100644 --- a/comp/otelcol/otlp/components/processor/infraattributesprocessor/traces.go +++ b/comp/otelcol/otlp/components/processor/infraattributesprocessor/traces.go @@ -21,7 +21,7 @@ func newInfraAttributesSpanProcessor(set processor.CreateSettings, _ *Config) (* tesp := &infraAttributesSpanProcessor{ logger: set.Logger, } - set.Logger.Info("Span Tag Enrichment configured") + set.Logger.Info("Span Infra Attributes Processor configured") return tesp, nil } diff --git a/comp/otelcol/otlp/components/processor/infraattributesprocessor/traces_test.go b/comp/otelcol/otlp/components/processor/infraattributesprocessor/traces_test.go index f52767228d5ab..aebef2f510f60 100644 --- a/comp/otelcol/otlp/components/processor/infraattributesprocessor/traces_test.go +++ b/comp/otelcol/otlp/components/processor/infraattributesprocessor/traces_test.go @@ -13,6 +13,8 @@ import ( "go.opentelemetry.io/collector/consumer/consumertest" "go.opentelemetry.io/collector/pdata/ptrace" "go.opentelemetry.io/collector/processor/processortest" + + "github.com/DataDog/datadog-agent/comp/core/tagger/taggerimpl" ) // All the data we need to test the Span @@ -51,7 +53,9 @@ func TestInfraAttributesTraceProcessor(t *testing.T) { ctx := context.Background() next := new(consumertest.TracesSink) cfg := &Config{} - factory := NewFactory() + fakeTagger := taggerimpl.SetupFakeTagger(t) + defer fakeTagger.ResetTagger() + factory := NewFactory(fakeTagger) fmp, err := factory.CreateTracesProcessor( ctx, processortest.NewNopCreateSettings(), diff --git a/comp/otelcol/otlp/components/statsprocessor/go.mod b/comp/otelcol/otlp/components/statsprocessor/go.mod index f79c5e7056aab..2769abb4681b4 100644 --- a/comp/otelcol/otlp/components/statsprocessor/go.mod +++ b/comp/otelcol/otlp/components/statsprocessor/go.mod @@ -101,6 +101,7 @@ require ( google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda // indirect google.golang.org/grpc v1.63.2 // indirect google.golang.org/protobuf v1.34.1 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/comp/otelcol/otlp/components/statsprocessor/go.sum b/comp/otelcol/otlp/components/statsprocessor/go.sum index ff32884e6938c..0ef2955aec8e3 100644 --- a/comp/otelcol/otlp/components/statsprocessor/go.sum +++ b/comp/otelcol/otlp/components/statsprocessor/go.sum @@ -286,6 +286,8 @@ google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHh gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/comp/otelcol/otlp/datatype/collector_status.go b/comp/otelcol/otlp/datatype/collector_status.go new file mode 100644 index 0000000000000..1a6cb1209b8fd --- /dev/null +++ b/comp/otelcol/otlp/datatype/collector_status.go @@ -0,0 +1,13 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2021-present Datadog, Inc. + +// Package datatype declares basic datatypes used by OTLP +package datatype + +// CollectorStatus is the status struct for an OTLP pipeline's collector +type CollectorStatus struct { + Status string + ErrorMessage string +} diff --git a/comp/otelcol/otlp/map_provider_not_serverless_test.go b/comp/otelcol/otlp/map_provider_not_serverless_test.go index 14c18fcf4f3d1..55609fb5eed90 100644 --- a/comp/otelcol/otlp/map_provider_not_serverless_test.go +++ b/comp/otelcol/otlp/map_provider_not_serverless_test.go @@ -12,13 +12,13 @@ import ( "context" "testing" - "github.com/DataDog/datadog-agent/pkg/logs/message" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/confmap" + "github.com/DataDog/datadog-agent/comp/core/tagger/taggerimpl" "github.com/DataDog/datadog-agent/comp/otelcol/otlp/testutil" + "github.com/DataDog/datadog-agent/pkg/logs/message" "github.com/DataDog/datadog-agent/pkg/serializer" ) @@ -1112,7 +1112,9 @@ func TestUnmarshal(t *testing.T) { }, }) require.NoError(t, err) - components, err := getComponents(&serializer.MockSerializer{}, make(chan *message.Message)) + fakeTagger := taggerimpl.SetupFakeTagger(t) + defer fakeTagger.ResetTagger() + components, err := getComponents(&serializer.MockSerializer{}, make(chan *message.Message), fakeTagger) require.NoError(t, err) _, err = provider.Get(context.Background(), components) diff --git a/comp/otelcol/otlp/testutil/go.mod b/comp/otelcol/otlp/testutil/go.mod index 9663cfb13a9b5..9fd6dc1454a1a 100644 --- a/comp/otelcol/otlp/testutil/go.mod +++ b/comp/otelcol/otlp/testutil/go.mod @@ -47,7 +47,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/system v0.54.0-rc.2 // indirect github.com/DataDog/datadog-agent/pkg/util/system/socket v0.54.0-rc.2 // indirect github.com/DataDog/datadog-agent/pkg/util/winutil v0.54.0-rc.2 // indirect - github.com/DataDog/viper v1.13.4 // indirect + github.com/DataDog/viper v1.13.5 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect diff --git a/comp/otelcol/otlp/testutil/go.sum b/comp/otelcol/otlp/testutil/go.sum index f9659fa496721..8b840551fac81 100644 --- a/comp/otelcol/otlp/testutil/go.sum +++ b/comp/otelcol/otlp/testutil/go.sum @@ -1,8 +1,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/DataDog/viper v1.13.4 h1:0SrZc3zvMAGgVKg96uP4DEJb13lK2Is9a4go7IIcFSE= -github.com/DataDog/viper v1.13.4/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= +github.com/DataDog/viper v1.13.5 h1:SZMcyMknYQN2jRY/40A16gUXexlNJOI8sDs1cWZnI64= +github.com/DataDog/viper v1.13.5/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= diff --git a/comp/otelcol/provider/def/component.go b/comp/otelcol/provider/def/component.go new file mode 100644 index 0000000000000..ec59df2ddb224 --- /dev/null +++ b/comp/otelcol/provider/def/component.go @@ -0,0 +1,24 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024-present Datadog, Inc. + +package provider + +import ( + "go.opentelemetry.io/collector/otelcol" +) + +// team: opentelemetry + +// Component implements the otelcol.ConfigProvider interface and +// provides extra functions to expose the provided and enhanced configs. +type Component interface { + otelcol.ConfigProvider + GetProvidedConf() string + GetEnhancedConf() string +} + +type Requires struct { + URIs []string +} diff --git a/comp/otelcol/provider/def/go.mod b/comp/otelcol/provider/def/go.mod new file mode 100644 index 0000000000000..bfca5c3d539a2 --- /dev/null +++ b/comp/otelcol/provider/def/go.mod @@ -0,0 +1,92 @@ +module github.com/DataDog/datadog-agent/comp/otelcol/provider/def + +go 1.21.0 + +require go.opentelemetry.io/collector/otelcol v0.100.0 + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect + github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 // indirect + github.com/hashicorp/go-version v1.6.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/knadh/koanf/maps v0.1.1 // indirect + github.com/knadh/koanf/providers/confmap v0.1.0 // indirect + github.com/knadh/koanf/v2 v2.1.1 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect + github.com/prometheus/client_golang v1.19.1 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.53.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect + github.com/shirou/gopsutil/v3 v3.24.4 // indirect + github.com/shoenig/go-m1cpu v0.1.6 // indirect + github.com/spf13/cobra v1.8.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/tklauser/go-sysconf v0.3.12 // indirect + github.com/tklauser/numcpus v0.6.1 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect + go.opencensus.io v0.24.0 // indirect + go.opentelemetry.io/collector v0.100.0 // indirect + go.opentelemetry.io/collector/component v0.100.0 // indirect + go.opentelemetry.io/collector/config/configtelemetry v0.101.0 // indirect + go.opentelemetry.io/collector/confmap v0.100.0 // indirect + go.opentelemetry.io/collector/confmap/converter/expandconverter v0.100.0 // indirect + go.opentelemetry.io/collector/confmap/provider/envprovider v0.100.0 // indirect + go.opentelemetry.io/collector/confmap/provider/fileprovider v0.100.0 // indirect + go.opentelemetry.io/collector/confmap/provider/httpprovider v0.100.0 // indirect + go.opentelemetry.io/collector/confmap/provider/httpsprovider v0.100.0 // indirect + go.opentelemetry.io/collector/confmap/provider/yamlprovider v0.100.0 // indirect + go.opentelemetry.io/collector/connector v0.100.0 // indirect + go.opentelemetry.io/collector/consumer v0.101.0 // indirect + go.opentelemetry.io/collector/exporter v0.100.0 // indirect + go.opentelemetry.io/collector/extension v0.100.0 // indirect + go.opentelemetry.io/collector/featuregate v1.8.0 // indirect + go.opentelemetry.io/collector/pdata v1.8.0 // indirect + go.opentelemetry.io/collector/processor v0.100.0 // indirect + go.opentelemetry.io/collector/receiver v0.100.0 // indirect + go.opentelemetry.io/collector/semconv v0.101.0 // indirect + go.opentelemetry.io/collector/service v0.100.0 // indirect + go.opentelemetry.io/contrib/config v0.6.0 // indirect + go.opentelemetry.io/contrib/propagators/b3 v1.26.0 // indirect + go.opentelemetry.io/otel v1.26.0 // indirect + go.opentelemetry.io/otel/bridge/opencensus v1.26.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.26.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.26.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.26.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.26.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.26.0 // indirect + go.opentelemetry.io/otel/exporters/prometheus v0.48.0 // indirect + go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.26.0 // indirect + go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.26.0 // indirect + go.opentelemetry.io/otel/metric v1.26.0 // indirect + go.opentelemetry.io/otel/sdk v1.26.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.26.0 // indirect + go.opentelemetry.io/otel/trace v1.26.0 // indirect + go.opentelemetry.io/proto/otlp v1.2.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.0 // indirect + golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect + golang.org/x/net v0.24.0 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/text v0.14.0 // indirect + gonum.org/v1/gonum v0.15.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda // indirect + google.golang.org/grpc v1.63.2 // indirect + google.golang.org/protobuf v1.34.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/comp/otelcol/provider/def/go.sum b/comp/otelcol/provider/def/go.sum new file mode 100644 index 0000000000000..1235540b6a15c --- /dev/null +++ b/comp/otelcol/provider/def/go.sum @@ -0,0 +1,320 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1 h1:TQcrn6Wq+sKGkpyPvppOz99zsMBaUOKXq6HSv655U1c= +github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 h1:/c3QmbOGMGTOumP2iT/rCwB7b0QDGLKzqOmktBjT+Is= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1/go.mod h1:5SN9VR2LTsRFsrEC6FHgRbTWrTHu6tqPeKxEQv15giM= +github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= +github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/knadh/koanf/maps v0.1.1 h1:G5TjmUh2D7G2YWf5SQQqSiHRJEjaicvU0KpypqB3NIs= +github.com/knadh/koanf/maps v0.1.1/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI= +github.com/knadh/koanf/providers/confmap v0.1.0 h1:gOkxhHkemwG4LezxxN8DMOFopOPghxRVp7JbIvdvqzU= +github.com/knadh/koanf/providers/confmap v0.1.0/go.mod h1:2uLhxQzJnyHKfxG927awZC7+fyHFdQkd697K4MdLnIU= +github.com/knadh/koanf/v2 v2.1.1 h1:/R8eXqasSTsmDCsAyYj+81Wteg8AqrV9CP6gvsTsOmM= +github.com/knadh/koanf/v2 v2.1.1/go.mod h1:4mnTRbZCK+ALuBXHZMjDfG9y714L7TykVnZkXbMU3Es= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= +github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= +github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= +github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.53.0 h1:U2pL9w9nmJwJDa4qqLQ3ZaePJ6ZTwt7cMD3AG3+aLCE= +github.com/prometheus/common v0.53.0/go.mod h1:BrxBKv3FWBIGXw89Mg1AeBq7FSyRzXWI3l3e7W3RN5U= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/shirou/gopsutil/v3 v3.24.4 h1:dEHgzZXt4LMNm+oYELpzl9YCqV65Yr/6SfrvgRBtXeU= +github.com/shirou/gopsutil/v3 v3.24.4/go.mod h1:lTd2mdiOspcqLgAnr9/nGi71NkeMpWKdmhuxm9GusH8= +github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= +github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= +github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= +github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= +github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= +github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/collector v0.100.0 h1:Q6IAGjMzjkZ7WepuwyCa6UytDPP0O88GemonQOUjP2s= +go.opentelemetry.io/collector v0.100.0/go.mod h1:QlVjQWlrPtBwVRm8tr+3P4FzNZSlYEfuUSaWoAwK+ko= +go.opentelemetry.io/collector/component v0.100.0 h1:3Y6dl3uDkDzilaikYrPxbZDOlzrDijrF1cIPzfyTwWA= +go.opentelemetry.io/collector/component v0.100.0/go.mod h1:HLEqEBFzPW2umagnVC3gY8yogOBhbzvuzTBFUqH54HY= +go.opentelemetry.io/collector/config/confignet v0.100.0 h1:SW8IMK+9GwFa1cNZdXw6wMkbCsqUaRtj0fgP8/yG6oI= +go.opentelemetry.io/collector/config/confignet v0.100.0/go.mod h1:3naWoPss70RhDHhYjGACi7xh4NcVRvs9itzIRVWyu1k= +go.opentelemetry.io/collector/config/configtelemetry v0.101.0 h1:G9RerNdBUm6rYW6wrJoKzleBiDsCGaCjtQx5UYr0hzw= +go.opentelemetry.io/collector/config/configtelemetry v0.101.0/go.mod h1:YV5PaOdtnU1xRomPcYqoHmyCr48tnaAREeGO96EZw8o= +go.opentelemetry.io/collector/confmap v0.100.0 h1:r70znwLWUMFRWL4LRcWLhdFfzmTvehXgbnlHFCDm0Tc= +go.opentelemetry.io/collector/confmap v0.100.0/go.mod h1:BWKPIpYeUzSG6ZgCJMjF7xsLvyrvJCfYURl57E5vhiQ= +go.opentelemetry.io/collector/confmap/converter/expandconverter v0.100.0 h1:xXPI9QzvwhefmVHNlSuq3WqmgXOrAVdaQAAdkAoMaEU= +go.opentelemetry.io/collector/confmap/converter/expandconverter v0.100.0/go.mod h1:zkADuTzmkmbsr281Xxw1jTy+YFrWr28pPIStt4nTZpo= +go.opentelemetry.io/collector/confmap/provider/envprovider v0.100.0 h1:ORj/EN7HCuTl5MD8GoirShtz7ieIbAdseGroeRdgNDM= +go.opentelemetry.io/collector/confmap/provider/envprovider v0.100.0/go.mod h1:MMrNqxsveOPN/l6xifY8KK3kqfXdFOANIJQ4hkgNxN0= +go.opentelemetry.io/collector/confmap/provider/fileprovider v0.100.0 h1:H2YtC6mOFuaY7/ht16iUDYSHS6jM2ufNqQatmKAbf7U= +go.opentelemetry.io/collector/confmap/provider/fileprovider v0.100.0/go.mod h1:JEYr74rqYYsgKc5bDtsqy8TFHmmhHnEc0Hm5ekZSj/8= +go.opentelemetry.io/collector/confmap/provider/httpprovider v0.100.0 h1:GPFfS9BmuF0bzeW/lUElvI2NzHjAKq5U1E/lXLy/7pc= +go.opentelemetry.io/collector/confmap/provider/httpprovider v0.100.0/go.mod h1:s8YCUcFOOnY4NKeQjVTup6XiA3Oj3yqs8NB10kbwuAU= +go.opentelemetry.io/collector/confmap/provider/httpsprovider v0.100.0 h1:vcOsDXQRdf/TUcD0vYTGkzvaPmoplG8Fkv8IoJ5cB58= +go.opentelemetry.io/collector/confmap/provider/httpsprovider v0.100.0/go.mod h1:GQ7LqkJKOcUFm8nxdV0P7eHPjLzVtNoBgqnwBGL4RCc= +go.opentelemetry.io/collector/confmap/provider/yamlprovider v0.100.0 h1:a7qqKfczlB8wGQNeiGiSMiwCYRwTVdlIIUG98Jn5ydY= +go.opentelemetry.io/collector/confmap/provider/yamlprovider v0.100.0/go.mod h1:IlcEhPYl3JDbl+5Bd9kWeDACHnvctdEnA7zlZCdtWk8= +go.opentelemetry.io/collector/connector v0.100.0 h1:6LN8LXhv553SjipKl7EbvlAECIZ9/v+vNxGiePp8dDQ= +go.opentelemetry.io/collector/connector v0.100.0/go.mod h1:yF4fkEtDrZOpwGOcnDTjkGjMCCQTPL2VCCBe9R43Jbg= +go.opentelemetry.io/collector/consumer v0.101.0 h1:9tDxaeHe1+Uovf3fhdx7T4pV5mo/Dc0hniH7O5H3RBA= +go.opentelemetry.io/collector/consumer v0.101.0/go.mod h1:ud5k64on9m7hHTrhjEeLhWbLkd8+Gp06rDt3p86TKNs= +go.opentelemetry.io/collector/exporter v0.100.0 h1:eyPb93tQwdft5Eboo8O5LDdaM1eXAQbtbXKBEYQlwh4= +go.opentelemetry.io/collector/exporter v0.100.0/go.mod h1:5UrDewyFp5yIQHyV7HUFAPdhHKJGbz1/uaTunm7X54I= +go.opentelemetry.io/collector/extension v0.100.0 h1:HT3h5JE+5xK3CCwF7VJKCOuZkLBMaUtm4T/BnEMpdWc= +go.opentelemetry.io/collector/extension v0.100.0/go.mod h1:B7jsEl6HAZB79NU41AdoMwLgXn4yTTO5NTlxRrsORoo= +go.opentelemetry.io/collector/extension/zpagesextension v0.100.0 h1:4bHq1NBg8hU+NSd9DBVFAl4vxi40J0tqClmU3IknrEg= +go.opentelemetry.io/collector/extension/zpagesextension v0.100.0/go.mod h1:mcLfCcjq0/yZoieuyO0H5rVL3u7Why0/MRezNV2E7AU= +go.opentelemetry.io/collector/featuregate v1.8.0 h1:p/bAuk5LiSfdYS88yFl/Jzao9bHEYqCh7YvZJ+L+IZg= +go.opentelemetry.io/collector/featuregate v1.8.0/go.mod h1:w7nUODKxEi3FLf1HslCiE6YWtMtOOrMnSwsDam8Mg9w= +go.opentelemetry.io/collector/otelcol v0.100.0 h1:5NWoo9T5tHP0oWt3OHetYpTRaQCJuef8KDDe5tLi+BA= +go.opentelemetry.io/collector/otelcol v0.100.0/go.mod h1:pdPObjfQqP2pdE70jqQiJlZdEyZ0jA1euoRdHtgZjiA= +go.opentelemetry.io/collector/pdata v1.8.0 h1:d/QQgZxB4Y+d3mqLVh2ozvzujUhloD3P/fk7X+In764= +go.opentelemetry.io/collector/pdata v1.8.0/go.mod h1:/W7clu0wFC4WSRp94Ucn6Vm36Wkrt+tmtlDb1aiNZCY= +go.opentelemetry.io/collector/pdata/testdata v0.101.0 h1:JzeUtg5RN1iIFgY8DakGlqBkGxOTJlkaYlLausnEGKY= +go.opentelemetry.io/collector/pdata/testdata v0.101.0/go.mod h1:ZGobfCus4fWo5RduZ7ENI0+HD9BewgKuO6qU2rBVnUg= +go.opentelemetry.io/collector/processor v0.100.0 h1:8Zcd3v77SCSM5mAJbGes5aR/Yof3aY1csiwFhKFpLEQ= +go.opentelemetry.io/collector/processor v0.100.0/go.mod h1:ZqUC8WWVYyPkaLUT1JXUCNpCpde8zXgSaFfJq2FXuVU= +go.opentelemetry.io/collector/receiver v0.100.0 h1:RFeOVhS7o39G562w0H0hqfh1o2QvK71ViHQuWnnfglI= +go.opentelemetry.io/collector/receiver v0.100.0/go.mod h1:Qo3xkorbUy0VXHh7WxMQyphIWiqxI3ZOG0O4YqQ2mCE= +go.opentelemetry.io/collector/semconv v0.101.0 h1:tOe9iTe9dDCnvz/bqgfNRr4w80kXG8505tQJ5h5v08Q= +go.opentelemetry.io/collector/semconv v0.101.0/go.mod h1:8ElcRZ8Cdw5JnvhTOQOdYizkJaQ10Z2fS+R6djOnj6A= +go.opentelemetry.io/collector/service v0.100.0 h1:2LFvNQNSs2NnwElyqkyhAiqaGoDdiMnTQeFPCLZNgg0= +go.opentelemetry.io/collector/service v0.100.0/go.mod h1:65NPZ6THkR/e7fd8vh+tw4Lh6iDJ1twNXVzL76a3VNk= +go.opentelemetry.io/contrib/config v0.6.0 h1:M1SRD1Z15XHPGk61tMLI1up77XT5FdrqQSRrlH0fYuk= +go.opentelemetry.io/contrib/config v0.6.0/go.mod h1:t+/kzmRWLN7J+4F/dD4fFvlYCmCO63WYwy/B00IC++c= +go.opentelemetry.io/contrib/propagators/b3 v1.26.0 h1:wgFbVA+bK2k+fGVfDOCOG4cfDAoppyr5sI2dVlh8MWM= +go.opentelemetry.io/contrib/propagators/b3 v1.26.0/go.mod h1:DDktFXxA+fyItAAM0Sbl5OBH7KOsCTjvbBdPKtoIf/k= +go.opentelemetry.io/contrib/zpages v0.51.0 h1:psVr4JTWd0qtISPj9EA6AODGJ09bvsOxWiuKqiGdSCA= +go.opentelemetry.io/contrib/zpages v0.51.0/go.mod h1:PKtp+NEp1gTTLmFHpynYgYCSkKtisPntOb9S1mQjFKg= +go.opentelemetry.io/otel v1.26.0 h1:LQwgL5s/1W7YiiRwxf03QGnWLb2HW4pLiAhaA5cZXBs= +go.opentelemetry.io/otel v1.26.0/go.mod h1:UmLkJHUAidDval2EICqBMbnAd0/m2vmpf/dAM+fvFs4= +go.opentelemetry.io/otel/bridge/opencensus v1.26.0 h1:DZzxj9QjznMVoehskOJnFP2gsTCWtDTFBDvFhPAY7nc= +go.opentelemetry.io/otel/bridge/opencensus v1.26.0/go.mod h1:rJiX0KrF5m8Tm1XE8jLczpAv5zUaDcvhKecFG0ZoFG4= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.26.0 h1:+hm+I+KigBy3M24/h1p/NHkUx/evbLH0PNcjpMyCHc4= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.26.0/go.mod h1:NjC8142mLvvNT6biDpaMjyz78kyEHIwAJlSX0N9P5KI= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.26.0 h1:HGZWGmCVRCVyAs2GQaiHQPbDHo+ObFWeUEOd+zDnp64= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.26.0/go.mod h1:SaH+v38LSCHddyk7RGlU9uZyQoRrKao6IBnJw6Kbn+c= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.26.0 h1:1u/AyyOqAWzy+SkPxDpahCNZParHV8Vid1RnI2clyDE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.26.0/go.mod h1:z46paqbJ9l7c9fIPCXTqTGwhQZ5XoTIsfeFYWboizjs= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.26.0 h1:Waw9Wfpo/IXzOI8bCB7DIk+0JZcqqsyn1JFnAc+iam8= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.26.0/go.mod h1:wnJIG4fOqyynOnnQF/eQb4/16VlX2EJAHhHgqIqWfAo= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.26.0 h1:1wp/gyxsuYtuE/JFxsQRtcCDtMrO2qMvlfXALU5wkzI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.26.0/go.mod h1:gbTHmghkGgqxMomVQQMur1Nba4M0MQ8AYThXDUjsJ38= +go.opentelemetry.io/otel/exporters/prometheus v0.48.0 h1:sBQe3VNGUjY9IKWQC6z2lNqa5iGbDSxhs60ABwK4y0s= +go.opentelemetry.io/otel/exporters/prometheus v0.48.0/go.mod h1:DtrbMzoZWwQHyrQmCfLam5DZbnmorsGbOtTbYHycU5o= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.26.0 h1:5fnmgteaar1VcAA69huatudPduNFz7guRtCmfZCooZI= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.26.0/go.mod h1:lsPccfZiz1cb1AhBPmicWM2E4F1VynFXEvD8SEBS4TM= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.26.0 h1:0W5o9SzoR15ocYHEQfvfipzcNog1lBxOLfnex91Hk6s= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.26.0/go.mod h1:zVZ8nz+VSggWmnh6tTsJqXQ7rU4xLwRtna1M4x5jq58= +go.opentelemetry.io/otel/metric v1.26.0 h1:7S39CLuY5Jgg9CrnA9HHiEjGMF/X2VHvoXGgSllRz30= +go.opentelemetry.io/otel/metric v1.26.0/go.mod h1:SY+rHOI4cEawI9a7N1A4nIg/nTQXe1ccCNWYOJUrpX4= +go.opentelemetry.io/otel/sdk v1.26.0 h1:Y7bumHf5tAiDlRYFmGqetNcLaVUZmh4iYfmGxtmz7F8= +go.opentelemetry.io/otel/sdk v1.26.0/go.mod h1:0p8MXpqLeJ0pzcszQQN4F0S5FVjBLgypeGSngLsmirs= +go.opentelemetry.io/otel/sdk/metric v1.26.0 h1:cWSks5tfriHPdWFnl+qpX3P681aAYqlZHcAyHw5aU9Y= +go.opentelemetry.io/otel/sdk/metric v1.26.0/go.mod h1:ClMFFknnThJCksebJwz7KIyEDHO+nTB6gK8obLy8RyE= +go.opentelemetry.io/otel/trace v1.26.0 h1:1ieeAUb4y0TE26jUFrCIXKpTuVK7uJGN9/Z/2LP5sQA= +go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZuWCBllGV2U2y0= +go.opentelemetry.io/proto/otlp v1.2.0 h1:pVeZGk7nXDC9O2hncA6nHldxEjm6LByfA2aN8IOkz94= +go.opentelemetry.io/proto/otlp v1.2.0/go.mod h1:gGpR8txAl5M03pDhMC79G6SdqNV26naRm/KDsgaHD8A= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= +golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.15.0 h1:2lYxjRbTYyxkJxlhC+LvJIx3SsANPdRybu1tGj9/OrQ= +gonum.org/v1/gonum v0.15.0/go.mod h1:xzZVBJBtS+Mz4q0Yl2LJTk+OxOg4jiXZ7qBoM0uISGo= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de h1:F6qOa9AZTYJXOUEr4jDysRDLrm4PHePlge4v4TGAlxY= +google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:VUhTRKeHn9wwcdrk73nvdC9gF178Tzhmt/qyaFcPLSo= +google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de h1:jFNzHPIeuzhdRwVhbZdiym9q0ory/xY3sA+v2wPg8I0= +google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:5iCWqnniDlqZHrd3neWVTOwvh/v6s3232omMecelax8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda h1:LI5DOvAxUPMv/50agcLLoo+AdWc1irS9Rzz4vPuD1V4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= +google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/comp/otelcol/provider/fx/fx.go b/comp/otelcol/provider/fx/fx.go new file mode 100644 index 0000000000000..391e7eaeb3635 --- /dev/null +++ b/comp/otelcol/provider/fx/fx.go @@ -0,0 +1,19 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024-present Datadog, Inc. + +// Package providerfx provides fx access for the provider component +package providerfx + +import ( + providerimpl "github.com/DataDog/datadog-agent/comp/otelcol/provider/impl" + "github.com/DataDog/datadog-agent/pkg/util/fxutil" +) + +// Module defines the fx options for this component. +func Module() fxutil.Module { + return fxutil.Component( + fxutil.ProvideComponentConstructor(providerimpl.NewConfigProvider), + ) +} diff --git a/comp/otelcol/otlp/components/pipeline/provider/go.mod b/comp/otelcol/provider/impl/go.mod similarity index 91% rename from comp/otelcol/otlp/components/pipeline/provider/go.mod rename to comp/otelcol/provider/impl/go.mod index b8e737ffcc0ff..bdcf7cb9ddd43 100644 --- a/comp/otelcol/otlp/components/pipeline/provider/go.mod +++ b/comp/otelcol/provider/impl/go.mod @@ -1,11 +1,13 @@ -module provider +module github.com/DataDog/datadog-agent/comp/otelcol/provider/impl go 1.21.0 +replace github.com/DataDog/datadog-agent/comp/otelcol/provider/def => ../def + require ( + github.com/DataDog/datadog-agent/comp/otelcol/provider/def v0.0.0-00010101000000-000000000000 github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datadogexporter v0.100.0 github.com/stretchr/testify v1.9.0 - go.opentelemetry.io/collector/component v0.100.0 go.opentelemetry.io/collector/confmap v0.100.0 go.opentelemetry.io/collector/confmap/converter/expandconverter v0.100.0 go.opentelemetry.io/collector/confmap/provider/envprovider v0.100.0 @@ -82,12 +84,12 @@ require ( github.com/DataDog/go-tuf v1.1.0-0.5.2 // indirect github.com/DataDog/gohai v0.0.0-20230524154621-4316413895ee // indirect github.com/DataDog/opentelemetry-mapping-go/pkg/inframetadata v0.16.0 // indirect - github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.16.0 // indirect + github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.16.1 // indirect github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/logs v0.16.0 // indirect github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/metrics v0.16.0 // indirect github.com/DataDog/opentelemetry-mapping-go/pkg/quantile v0.16.0 // indirect github.com/DataDog/sketches-go v1.4.4 // indirect - github.com/DataDog/viper v1.13.2 // indirect + github.com/DataDog/viper v1.13.4 // indirect github.com/DataDog/zstd v1.5.2 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.22.0 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect @@ -169,13 +171,13 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20220216144756-c35f1ee13d7c // indirect - github.com/prometheus/client_golang v1.19.0 // indirect + github.com/prometheus/client_golang v1.19.1 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.53.0 // indirect github.com/prometheus/procfs v0.14.0 // indirect github.com/rs/cors v1.10.1 // indirect github.com/secure-systems-lab/go-securesystemslib v0.7.0 // indirect - github.com/shirou/gopsutil/v3 v3.24.3 // indirect + github.com/shirou/gopsutil/v3 v3.24.4 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/spf13/afero v1.10.0 // indirect github.com/spf13/cast v1.5.1 // indirect @@ -190,21 +192,22 @@ require ( github.com/yusufpapurcu/wmi v1.2.4 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/collector v0.100.0 // indirect + go.opentelemetry.io/collector/component v0.100.0 // indirect go.opentelemetry.io/collector/config/configauth v0.100.0 // indirect go.opentelemetry.io/collector/config/configcompression v1.7.0 // indirect go.opentelemetry.io/collector/config/confighttp v0.100.0 // indirect - go.opentelemetry.io/collector/config/confignet v0.100.0 // indirect + go.opentelemetry.io/collector/config/confignet v0.101.0 // indirect go.opentelemetry.io/collector/config/configopaque v1.7.0 // indirect - go.opentelemetry.io/collector/config/configretry v0.100.0 // indirect - go.opentelemetry.io/collector/config/configtelemetry v0.100.0 // indirect + go.opentelemetry.io/collector/config/configretry v0.101.0 // indirect + go.opentelemetry.io/collector/config/configtelemetry v0.101.0 // indirect go.opentelemetry.io/collector/config/configtls v0.100.0 // indirect go.opentelemetry.io/collector/config/internal v0.100.0 // indirect - go.opentelemetry.io/collector/consumer v0.100.0 // indirect + go.opentelemetry.io/collector/consumer v0.101.0 // indirect go.opentelemetry.io/collector/extension/auth v0.100.0 // indirect - go.opentelemetry.io/collector/featuregate v1.7.0 // indirect - go.opentelemetry.io/collector/pdata v1.7.0 // indirect - go.opentelemetry.io/collector/pdata/testdata v0.100.0 // indirect - go.opentelemetry.io/collector/semconv v0.100.0 // indirect + go.opentelemetry.io/collector/featuregate v1.8.0 // indirect + go.opentelemetry.io/collector/pdata v1.8.0 // indirect + go.opentelemetry.io/collector/pdata/testdata v0.101.0 // indirect + go.opentelemetry.io/collector/semconv v0.101.0 // indirect go.opentelemetry.io/collector/service v0.100.0 // indirect go.opentelemetry.io/contrib/config v0.6.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0 // indirect @@ -229,20 +232,21 @@ require ( go.uber.org/fx v1.18.2 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect - golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect + golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect golang.org/x/mod v0.17.0 // indirect - golang.org/x/net v0.24.0 // indirect + golang.org/x/net v0.25.0 // indirect golang.org/x/oauth2 v0.19.0 // indirect - golang.org/x/sys v0.19.0 // indirect - golang.org/x/term v0.19.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/sync v0.7.0 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/term v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect golang.org/x/time v0.5.0 // indirect - golang.org/x/tools v0.19.0 // indirect + golang.org/x/tools v0.21.0 // indirect gonum.org/v1/gonum v0.15.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda // indirect google.golang.org/grpc v1.63.2 // indirect - google.golang.org/protobuf v1.34.0 // indirect + google.golang.org/protobuf v1.34.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/zorkian/go-datadog-api.v2 v2.30.0 // indirect diff --git a/comp/otelcol/otlp/components/pipeline/provider/go.sum b/comp/otelcol/provider/impl/go.sum similarity index 97% rename from comp/otelcol/otlp/components/pipeline/provider/go.sum rename to comp/otelcol/provider/impl/go.sum index df3e8ab7bb381..734630bf05831 100644 --- a/comp/otelcol/otlp/components/pipeline/provider/go.sum +++ b/comp/otelcol/provider/impl/go.sum @@ -193,8 +193,8 @@ github.com/DataDog/opentelemetry-mapping-go/pkg/inframetadata v0.16.0 h1:Jl7/oQQ github.com/DataDog/opentelemetry-mapping-go/pkg/inframetadata v0.16.0/go.mod h1:P/l++2cDCeeq21KSmCEdXdMH9/WMdXP7uA/vjnxhtz8= github.com/DataDog/opentelemetry-mapping-go/pkg/internal/sketchtest v0.16.0 h1:VJT1Jjlz/ca999FEqaAS+He7S4eB14a+PJjczgRdgAY= github.com/DataDog/opentelemetry-mapping-go/pkg/internal/sketchtest v0.16.0/go.mod h1:66XlN7QpQKqIvw8e2UbCXV5X8wGnEw851nT9BjJ75dY= -github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.16.0 h1:g/ztrLYZNfkpW6Bt8kMnLed5DaKRHEtiKE0opHXLHJk= -github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.16.0/go.mod h1:dvIWN9pA2zWNTw5rhDWZgzZnhcfpH++d+8d1SWW6xkY= +github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.16.1 h1:ZI8u3CgdMXpDplrf9/gIr13+/g/tUzUcBMk2ZhXgzLE= +github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.16.1/go.mod h1:dvIWN9pA2zWNTw5rhDWZgzZnhcfpH++d+8d1SWW6xkY= github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/logs v0.16.0 h1:NbKlfbjR2joF52jEBLs3MEnT6l5zM3MCyhUFkqARZpk= github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/logs v0.16.0/go.mod h1:+LijQ2LdlocAQ4WB+7KsoIGe90bfogkRslubd9swVow= github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/metrics v0.16.0 h1:H5DzD3rwgQCX0VI3A16KgsdmC5grUCyDFflaZDpfgMc= @@ -203,8 +203,8 @@ github.com/DataDog/opentelemetry-mapping-go/pkg/quantile v0.16.0 h1:BdfRSuCoHyKa github.com/DataDog/opentelemetry-mapping-go/pkg/quantile v0.16.0/go.mod h1:6eYyd+lJYH+uRuZqhyW/u+9ykaXBWetDGj44+txz6jU= github.com/DataDog/sketches-go v1.4.4 h1:dF52vzXRFSPOj2IjXSWLvXq3jubL4CI69kwYjJ1w5Z8= github.com/DataDog/sketches-go v1.4.4/go.mod h1:XR0ns2RtEEF09mDKXiKZiQg+nfZStrq1ZuL1eezeZe0= -github.com/DataDog/viper v1.13.2 h1:GrYzwGiaEoliIXA4wPkx8MHIRY5sNi8frV1Fsv7VCJU= -github.com/DataDog/viper v1.13.2/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= +github.com/DataDog/viper v1.13.4 h1:0SrZc3zvMAGgVKg96uP4DEJb13lK2Is9a4go7IIcFSE= +github.com/DataDog/viper v1.13.4/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= github.com/DataDog/zstd v1.5.2 h1:vUG4lAyuPCXO0TLbXvPv7EB7cNK1QV/luu55UHLrrn8= github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.22.0 h1:PWcDbDjrcT/ZHLn4Bc/FuglaZZVPP8bWO/YRmJBbe38= @@ -755,8 +755,8 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.4.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= -github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU= -github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k= +github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= +github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -793,8 +793,8 @@ github.com/scaleway/scaleway-sdk-go v1.0.0-beta.25/go.mod h1:fCa7OJZ/9DRTnOKmxvT github.com/secure-systems-lab/go-securesystemslib v0.7.0 h1:OwvJ5jQf9LnIAS83waAjPbcMsODrTQUpJ02eNLUoxBg= github.com/secure-systems-lab/go-securesystemslib v0.7.0/go.mod h1:/2gYnlnHVQ6xeGtfIqFy7Do03K4cdCY0A/GlJLDKLHI= github.com/shirou/gopsutil/v3 v3.22.12/go.mod h1:Xd7P1kwZcp5VW52+9XsirIKd/BROzbb2wdX3Kqlz9uI= -github.com/shirou/gopsutil/v3 v3.24.3 h1:eoUGJSmdfLzJ3mxIhmOAhgKEKgQkeOwKpz1NbhVnuPE= -github.com/shirou/gopsutil/v3 v3.24.3/go.mod h1:JpND7O217xa72ewWz9zN2eIIkPWsDN/3pl0H8Qt0uwg= +github.com/shirou/gopsutil/v3 v3.24.4 h1:dEHgzZXt4LMNm+oYELpzl9YCqV65Yr/6SfrvgRBtXeU= +github.com/shirou/gopsutil/v3 v3.24.4/go.mod h1:lTd2mdiOspcqLgAnr9/nGi71NkeMpWKdmhuxm9GusH8= github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= @@ -899,14 +899,14 @@ go.opentelemetry.io/collector/config/configgrpc v0.100.0 h1:+RuWrysXLbjaf/+I7dU9 go.opentelemetry.io/collector/config/configgrpc v0.100.0/go.mod h1:NDKPjtEVL7TJMfByR/D5MYyjveUU3D0GkM96jU0u494= go.opentelemetry.io/collector/config/confighttp v0.100.0 h1:bkB8ZkkRL+N75QofuIosf2ZzkEYaBAA5C+eQpL4fOis= go.opentelemetry.io/collector/config/confighttp v0.100.0/go.mod h1:AaugDfPoHeOmFT2BICuGNp3ja3Sq1AcTxxw4WysFZsI= -go.opentelemetry.io/collector/config/confignet v0.100.0 h1:SW8IMK+9GwFa1cNZdXw6wMkbCsqUaRtj0fgP8/yG6oI= -go.opentelemetry.io/collector/config/confignet v0.100.0/go.mod h1:3naWoPss70RhDHhYjGACi7xh4NcVRvs9itzIRVWyu1k= +go.opentelemetry.io/collector/config/confignet v0.101.0 h1:Mdb9e/EpCSac4Ccg7w4UchS/o4yY1WoIc9X5o7fTu9E= +go.opentelemetry.io/collector/config/confignet v0.101.0/go.mod h1:3naWoPss70RhDHhYjGACi7xh4NcVRvs9itzIRVWyu1k= go.opentelemetry.io/collector/config/configopaque v1.7.0 h1:nZh5Hb1ofq9xP1wHLSt4obM85pRTccSeAjV0NbrJeTc= go.opentelemetry.io/collector/config/configopaque v1.7.0/go.mod h1:vxoDKYYYUF/arrdQJxmfhlgkcsb0DpdzC9KPFP97uuE= -go.opentelemetry.io/collector/config/configretry v0.100.0 h1:jEswHFjNokqJ0U2iYSzUlDy8N6A6D+zaoHM9t1TB6yw= -go.opentelemetry.io/collector/config/configretry v0.100.0/go.mod h1:uRdmPeCkrW9Zsadh2WEbQ1AGXGYJ02vCfmmT+0g69nY= -go.opentelemetry.io/collector/config/configtelemetry v0.100.0 h1:unlhNrFFXCinxk6iPHPYwANO+eFY4S1NTb5knSxteW4= -go.opentelemetry.io/collector/config/configtelemetry v0.100.0/go.mod h1:YV5PaOdtnU1xRomPcYqoHmyCr48tnaAREeGO96EZw8o= +go.opentelemetry.io/collector/config/configretry v0.101.0 h1:5QggLq/lZiZXry1Ut52IOTbrdz1RbGoL29Io/wWdE4g= +go.opentelemetry.io/collector/config/configretry v0.101.0/go.mod h1:uRdmPeCkrW9Zsadh2WEbQ1AGXGYJ02vCfmmT+0g69nY= +go.opentelemetry.io/collector/config/configtelemetry v0.101.0 h1:G9RerNdBUm6rYW6wrJoKzleBiDsCGaCjtQx5UYr0hzw= +go.opentelemetry.io/collector/config/configtelemetry v0.101.0/go.mod h1:YV5PaOdtnU1xRomPcYqoHmyCr48tnaAREeGO96EZw8o= go.opentelemetry.io/collector/config/configtls v0.100.0 h1:qcx8EXW4u+IQvyt8ZH5ld2dEns1zp8sugyM+s7RuiKY= go.opentelemetry.io/collector/config/configtls v0.100.0/go.mod h1:f8KZu6P8hIzTfybLKG3xMIzkCmXyjxVUfDTVUp2CmhA= go.opentelemetry.io/collector/config/internal v0.100.0 h1:XSbedIpdXOxIEGnnzCZnulTmWPSGWfXTH18ZMxuqt8s= @@ -927,8 +927,8 @@ go.opentelemetry.io/collector/confmap/provider/yamlprovider v0.100.0 h1:a7qqKfcz go.opentelemetry.io/collector/confmap/provider/yamlprovider v0.100.0/go.mod h1:IlcEhPYl3JDbl+5Bd9kWeDACHnvctdEnA7zlZCdtWk8= go.opentelemetry.io/collector/connector v0.100.0 h1:6LN8LXhv553SjipKl7EbvlAECIZ9/v+vNxGiePp8dDQ= go.opentelemetry.io/collector/connector v0.100.0/go.mod h1:yF4fkEtDrZOpwGOcnDTjkGjMCCQTPL2VCCBe9R43Jbg= -go.opentelemetry.io/collector/consumer v0.100.0 h1:8sALAcWvizSyrZJCF+zTqD2RLmZAyeCuaQrNS2q6ti0= -go.opentelemetry.io/collector/consumer v0.100.0/go.mod h1:JOPOq8nSTdnQwc2xdHl4hcuYBYV8gjN2SlFqlqBe/Nc= +go.opentelemetry.io/collector/consumer v0.101.0 h1:9tDxaeHe1+Uovf3fhdx7T4pV5mo/Dc0hniH7O5H3RBA= +go.opentelemetry.io/collector/consumer v0.101.0/go.mod h1:ud5k64on9m7hHTrhjEeLhWbLkd8+Gp06rDt3p86TKNs= go.opentelemetry.io/collector/exporter v0.100.0 h1:eyPb93tQwdft5Eboo8O5LDdaM1eXAQbtbXKBEYQlwh4= go.opentelemetry.io/collector/exporter v0.100.0/go.mod h1:5UrDewyFp5yIQHyV7HUFAPdhHKJGbz1/uaTunm7X54I= go.opentelemetry.io/collector/extension v0.100.0 h1:HT3h5JE+5xK3CCwF7VJKCOuZkLBMaUtm4T/BnEMpdWc= @@ -937,16 +937,16 @@ go.opentelemetry.io/collector/extension/auth v0.100.0 h1:Z8QVtntWiORnbVSCQfOxtnO go.opentelemetry.io/collector/extension/auth v0.100.0/go.mod h1:nkqaVzUAdqqkUGdMqoIqH/xlGU0rCxRZy1Altyz0gQk= go.opentelemetry.io/collector/extension/zpagesextension v0.100.0 h1:4bHq1NBg8hU+NSd9DBVFAl4vxi40J0tqClmU3IknrEg= go.opentelemetry.io/collector/extension/zpagesextension v0.100.0/go.mod h1:mcLfCcjq0/yZoieuyO0H5rVL3u7Why0/MRezNV2E7AU= -go.opentelemetry.io/collector/featuregate v1.7.0 h1:8tNgX2VaiR9jrpZevRSvStuJrvvL6WwScT264HNLk7U= -go.opentelemetry.io/collector/featuregate v1.7.0/go.mod h1:w7nUODKxEi3FLf1HslCiE6YWtMtOOrMnSwsDam8Mg9w= +go.opentelemetry.io/collector/featuregate v1.8.0 h1:p/bAuk5LiSfdYS88yFl/Jzao9bHEYqCh7YvZJ+L+IZg= +go.opentelemetry.io/collector/featuregate v1.8.0/go.mod h1:w7nUODKxEi3FLf1HslCiE6YWtMtOOrMnSwsDam8Mg9w= go.opentelemetry.io/collector/filter v0.100.0 h1:XQyhnqJSK2sw+e9yvpkvl7y8QdJwH/gAnFoZDfEZ0dQ= go.opentelemetry.io/collector/filter v0.100.0/go.mod h1:3xGRpZo11DMJTDtMUGsDNkxKM6LMHqROGrQ/aTvskh8= go.opentelemetry.io/collector/otelcol v0.100.0 h1:5NWoo9T5tHP0oWt3OHetYpTRaQCJuef8KDDe5tLi+BA= go.opentelemetry.io/collector/otelcol v0.100.0/go.mod h1:pdPObjfQqP2pdE70jqQiJlZdEyZ0jA1euoRdHtgZjiA= -go.opentelemetry.io/collector/pdata v1.7.0 h1:/WNsBbE6KM3TTPUb9v/5B7IDqnDkgf8GyFhVJJqu7II= -go.opentelemetry.io/collector/pdata v1.7.0/go.mod h1:ehCBBA5GoFrMZkwyZAKGY/lAVSgZf6rzUt3p9mddmPU= -go.opentelemetry.io/collector/pdata/testdata v0.100.0 h1:pliojioiAv+CuLNTK+8tnCD2UgiJbKX9q8bDnpHkV1U= -go.opentelemetry.io/collector/pdata/testdata v0.100.0/go.mod h1:01BHOXvXaQaLLt5J34S093u3e+j//RhbfmEujpFJ/ME= +go.opentelemetry.io/collector/pdata v1.8.0 h1:d/QQgZxB4Y+d3mqLVh2ozvzujUhloD3P/fk7X+In764= +go.opentelemetry.io/collector/pdata v1.8.0/go.mod h1:/W7clu0wFC4WSRp94Ucn6Vm36Wkrt+tmtlDb1aiNZCY= +go.opentelemetry.io/collector/pdata/testdata v0.101.0 h1:JzeUtg5RN1iIFgY8DakGlqBkGxOTJlkaYlLausnEGKY= +go.opentelemetry.io/collector/pdata/testdata v0.101.0/go.mod h1:ZGobfCus4fWo5RduZ7ENI0+HD9BewgKuO6qU2rBVnUg= go.opentelemetry.io/collector/processor v0.100.0 h1:8Zcd3v77SCSM5mAJbGes5aR/Yof3aY1csiwFhKFpLEQ= go.opentelemetry.io/collector/processor v0.100.0/go.mod h1:ZqUC8WWVYyPkaLUT1JXUCNpCpde8zXgSaFfJq2FXuVU= go.opentelemetry.io/collector/processor/batchprocessor v0.100.0 h1:N94WWv+o9yt4HqvFeRUjL8VNEhY8mCym+2XPKUWxjmo= @@ -957,8 +957,8 @@ go.opentelemetry.io/collector/receiver v0.100.0 h1:RFeOVhS7o39G562w0H0hqfh1o2QvK go.opentelemetry.io/collector/receiver v0.100.0/go.mod h1:Qo3xkorbUy0VXHh7WxMQyphIWiqxI3ZOG0O4YqQ2mCE= go.opentelemetry.io/collector/receiver/otlpreceiver v0.100.0 h1:JuTBkWQ0xc8FargDe5aE6wCoAEk1gxLnBzfdIi8NOL8= go.opentelemetry.io/collector/receiver/otlpreceiver v0.100.0/go.mod h1:X2xzbz9K4Kz0i3c1IKa1gcwf7rpoJDidyp28A1AuHFs= -go.opentelemetry.io/collector/semconv v0.100.0 h1:QArUvWcbmsMjM4PV0zngUHRizZeUXibsPBWjDuNJXAs= -go.opentelemetry.io/collector/semconv v0.100.0/go.mod h1:8ElcRZ8Cdw5JnvhTOQOdYizkJaQ10Z2fS+R6djOnj6A= +go.opentelemetry.io/collector/semconv v0.101.0 h1:tOe9iTe9dDCnvz/bqgfNRr4w80kXG8505tQJ5h5v08Q= +go.opentelemetry.io/collector/semconv v0.101.0/go.mod h1:8ElcRZ8Cdw5JnvhTOQOdYizkJaQ10Z2fS+R6djOnj6A= go.opentelemetry.io/collector/service v0.100.0 h1:2LFvNQNSs2NnwElyqkyhAiqaGoDdiMnTQeFPCLZNgg0= go.opentelemetry.io/collector/service v0.100.0/go.mod h1:65NPZ6THkR/e7fd8vh+tw4Lh6iDJ1twNXVzL76a3VNk= go.opentelemetry.io/contrib/config v0.6.0 h1:M1SRD1Z15XHPGk61tMLI1up77XT5FdrqQSRrlH0fYuk= @@ -1032,8 +1032,8 @@ golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= -golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= +golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1044,8 +1044,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= -golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1115,8 +1115,8 @@ golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= -golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1197,14 +1197,14 @@ golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q= -golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= +golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1213,8 +1213,8 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1278,8 +1278,8 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw= -golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= +golang.org/x/tools v0.21.0 h1:qc0xYgIbsSDt9EyWz05J5wfa7LOVW0YTLOXrqdLAWIw= +golang.org/x/tools v0.21.0/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1389,8 +1389,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.34.0 h1:Qo/qEd2RZPCf2nKuorzksSknv0d3ERwp1vFG38gSmH4= -google.golang.org/protobuf v1.34.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/comp/otelcol/otlp/components/pipeline/provider/provider.go b/comp/otelcol/provider/impl/provider.go similarity index 89% rename from comp/otelcol/otlp/components/pipeline/provider/provider.go rename to comp/otelcol/provider/impl/provider.go index 0b449787e169c..f2a04036b8998 100644 --- a/comp/otelcol/otlp/components/pipeline/provider/provider.go +++ b/comp/otelcol/provider/impl/provider.go @@ -3,13 +3,14 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2024-present Datadog, Inc. -package provider +package providerimpl import ( "context" "fmt" "os" + provider "github.com/DataDog/datadog-agent/comp/otelcol/provider/def" "go.opentelemetry.io/collector/confmap" "go.opentelemetry.io/collector/confmap/converter/expandconverter" "go.opentelemetry.io/collector/confmap/provider/envprovider" @@ -21,14 +22,6 @@ import ( "gopkg.in/yaml.v3" ) -// ExtendedConfigProvider implements the otelcol.ConfigProvider interface and -// provides extra functions to expose the provided and enhanced configs. -type ExtendedConfigProvider interface { - otelcol.ConfigProvider - GetProvidedConf() string - GetEnhancedConf() string -} - type configProvider struct { base otelcol.ConfigProvider confDump confDump @@ -42,14 +35,14 @@ type confDump struct { var _ otelcol.ConfigProvider = (*configProvider)(nil) // currently only supports a single URI in the uris slice, and this URI needs to be a file path. -func NewConfigProvider(uris []string) (ExtendedConfigProvider, error) { - ocp, err := otelcol.NewConfigProvider(newDefaultConfigProviderSettings(uris)) +func NewConfigProvider(reqs provider.Requires) (provider.Component, error) { + ocp, err := otelcol.NewConfigProvider(newDefaultConfigProviderSettings(reqs.URIs)) if err != nil { return nil, fmt.Errorf("failed to create configprovider: %w", err) } // this is a hack until we are unblocked from upstream to be able to use confToString. - yamlBytes, err := os.ReadFile(uris[0]) + yamlBytes, err := os.ReadFile(reqs.URIs[0]) if err != nil { return nil, fmt.Errorf("failed to read config: %w", err) } @@ -124,8 +117,8 @@ func (cp *configProvider) addEnhancedConf(conf *otelcol.Config) error { // GetProvidedConf returns a string representing the collector configuration passed // by the user. Should not be called concurrently with Get. -// Note: the current implementation does not redact sensitive data (e.g. API Key). -// Once we are unblocked and are able to remove the hack, this will provide the config +// Note: the current implementation does not redact sensitive data (e.g. API Key). +// Once we are unblocked and are able to remove the hack, this will provide the config // with any sensitive data redacted. func (cp *configProvider) GetProvidedConf() string { return cp.confDump.provided @@ -133,7 +126,7 @@ func (cp *configProvider) GetProvidedConf() string { // GetEnhancedConf returns a string representing the ehnhanced collector configuration. // Should not be called concurrently with Get. -// Note: this is currently not supported. +// Note: this is currently not supported. func (cp *configProvider) GetEnhancedConf() string { return cp.confDump.enhanced } diff --git a/comp/otelcol/otlp/components/pipeline/provider/provider_test.go b/comp/otelcol/provider/impl/provider_test.go similarity index 87% rename from comp/otelcol/otlp/components/pipeline/provider/provider_test.go rename to comp/otelcol/provider/impl/provider_test.go index 329d1e2bfc6d2..0b8827fbad0f3 100644 --- a/comp/otelcol/otlp/components/pipeline/provider/provider_test.go +++ b/comp/otelcol/provider/impl/provider_test.go @@ -3,7 +3,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2024-present Datadog, Inc. -package provider +package providerimpl import ( "context" @@ -12,6 +12,7 @@ import ( "path/filepath" "testing" + provider "github.com/DataDog/datadog-agent/comp/otelcol/provider/def" "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datadogexporter" "github.com/stretchr/testify/assert" "go.opentelemetry.io/collector/connector" @@ -34,12 +35,16 @@ func uriFromFile(filename string) string { } func TestNewConfigProvider(t *testing.T) { - _, err := NewConfigProvider([]string{uriFromFile("nop/config.yaml")}) + _, err := NewConfigProvider(provider.Requires{ + URIs: []string{uriFromFile("nop/config.yaml")}, + }) assert.NoError(t, err) } func TestConfigProviderGet(t *testing.T) { - provider, err := NewConfigProvider([]string{uriFromFile("nop/config.yaml")}) + provider, err := NewConfigProvider(provider.Requires{ + URIs: []string{uriFromFile("nop/config.yaml")}, + }) assert.NoError(t, err) factories, err := nopFactories() @@ -67,7 +72,9 @@ func upstreamConfigProvider(file string) (otelcol.ConfigProvider, error) { } func TestConfigProviderWatch(t *testing.T) { - provider, err := NewConfigProvider([]string{uriFromFile("nop/config.yaml")}) + provider, err := NewConfigProvider(provider.Requires{ + URIs: []string{uriFromFile("nop/config.yaml")}, + }) assert.NoError(t, err) var expected <-chan error @@ -75,7 +82,9 @@ func TestConfigProviderWatch(t *testing.T) { } func TestConfigProviderShutdown(t *testing.T) { - provider, err := NewConfigProvider([]string{uriFromFile("nop/config.yaml")}) + provider, err := NewConfigProvider(provider.Requires{ + URIs: []string{uriFromFile("nop/config.yaml")}, + }) assert.NoError(t, err) err = provider.Shutdown(context.Background()) @@ -84,7 +93,9 @@ func TestConfigProviderShutdown(t *testing.T) { func TestGetConfDump(t *testing.T) { t.Run("nop", func(t *testing.T) { - provider, err := NewConfigProvider([]string{uriFromFile("nop/config.yaml")}) + provider, err := NewConfigProvider(provider.Requires{ + URIs: []string{uriFromFile("nop/config.yaml")}, + }) assert.NoError(t, err) factories, err := nopFactories() @@ -122,7 +133,9 @@ func TestGetConfDump(t *testing.T) { }) t.Run("dd", func(t *testing.T) { - provider, err := NewConfigProvider([]string{uriFromFile("dd/config-dd.yaml")}) + provider, err := NewConfigProvider(provider.Requires{ + URIs: []string{uriFromFile("dd/config-dd.yaml")}, + }) assert.NoError(t, err) factories, err := nopFactories() diff --git a/comp/otelcol/otlp/components/pipeline/provider/testdata/dd/config-dd-result.yaml b/comp/otelcol/provider/impl/testdata/dd/config-dd-result.yaml similarity index 100% rename from comp/otelcol/otlp/components/pipeline/provider/testdata/dd/config-dd-result.yaml rename to comp/otelcol/provider/impl/testdata/dd/config-dd-result.yaml diff --git a/comp/otelcol/otlp/components/pipeline/provider/testdata/dd/config-dd.yaml b/comp/otelcol/provider/impl/testdata/dd/config-dd.yaml similarity index 100% rename from comp/otelcol/otlp/components/pipeline/provider/testdata/dd/config-dd.yaml rename to comp/otelcol/provider/impl/testdata/dd/config-dd.yaml diff --git a/comp/otelcol/otlp/components/pipeline/provider/testdata/nop/config-result.yaml b/comp/otelcol/provider/impl/testdata/nop/config-result.yaml similarity index 100% rename from comp/otelcol/otlp/components/pipeline/provider/testdata/nop/config-result.yaml rename to comp/otelcol/provider/impl/testdata/nop/config-result.yaml diff --git a/comp/otelcol/otlp/components/pipeline/provider/testdata/nop/config.yaml b/comp/otelcol/provider/impl/testdata/nop/config.yaml similarity index 100% rename from comp/otelcol/otlp/components/pipeline/provider/testdata/nop/config.yaml rename to comp/otelcol/provider/impl/testdata/nop/config.yaml diff --git a/comp/process/agent/agentimpl/agent.go b/comp/process/agent/agentimpl/agent.go index 2fde222f59cfc..29e9e4787290b 100644 --- a/comp/process/agent/agentimpl/agent.go +++ b/comp/process/agent/agentimpl/agent.go @@ -14,13 +14,16 @@ import ( logComponent "github.com/DataDog/datadog-agent/comp/core/log" statusComponent "github.com/DataDog/datadog-agent/comp/core/status" "github.com/DataDog/datadog-agent/comp/core/sysprobeconfig" + statsdComp "github.com/DataDog/datadog-agent/comp/dogstatsd/statsd" "github.com/DataDog/datadog-agent/comp/process/agent" expvars "github.com/DataDog/datadog-agent/comp/process/expvars/expvarsimpl" "github.com/DataDog/datadog-agent/comp/process/hostinfo" "github.com/DataDog/datadog-agent/comp/process/runner" submitterComp "github.com/DataDog/datadog-agent/comp/process/submitter" "github.com/DataDog/datadog-agent/comp/process/types" + ddconfig "github.com/DataDog/datadog-agent/pkg/config" "github.com/DataDog/datadog-agent/pkg/process/checks" + processStatsd "github.com/DataDog/datadog-agent/pkg/process/statsd" "github.com/DataDog/datadog-agent/pkg/util/flavor" "github.com/DataDog/datadog-agent/pkg/util/fxutil" ) @@ -52,6 +55,7 @@ type dependencies struct { Submitter submitterComp.Component SysProbeConfig sysprobeconfig.Component HostInfo hostinfo.Component + Statsd statsdComp.Component } type processAgent struct { @@ -69,13 +73,13 @@ type provides struct { FlareProvider flaretypes.Provider } -func newProcessAgent(deps dependencies) provides { +func newProcessAgent(deps dependencies) (provides, error) { if !agent.Enabled(deps.Config, deps.Checks, deps.Log) { return provides{ Comp: processAgent{ enabled: false, }, - } + }, nil } enabledChecks := make([]checks.Check, 0, len(deps.Checks)) @@ -93,7 +97,16 @@ func newProcessAgent(deps dependencies) provides { Comp: processAgent{ enabled: false, }, - } + }, nil + } + + if err := processStatsd.Configure(ddconfig.GetBindHost(), deps.Config.GetInt("dogstatsd_port"), deps.Statsd.CreateForHostPort); err != nil { + deps.Log.Criticalf("Error configuring statsd for process-agent: %s", err) + return provides{ + Comp: processAgent{ + enabled: false, + }, + }, err } processAgentComponent := processAgent{ @@ -114,10 +127,10 @@ func newProcessAgent(deps dependencies) provides { Comp: processAgentComponent, StatusProvider: statusComponent.NewInformationProvider(agent.NewStatusProvider(deps.Config)), FlareProvider: flaretypes.NewProvider(processAgentComponent.flarehelper.FillFlare), - } + }, nil } - return provides{Comp: processAgentComponent} + return provides{Comp: processAgentComponent}, nil } // Enabled determines whether the process agent is enabled based on the configuration. diff --git a/comp/process/agent/agentimpl/agent_linux_test.go b/comp/process/agent/agentimpl/agent_linux_test.go index ad627fc3656bd..401c9d2c63d41 100644 --- a/comp/process/agent/agentimpl/agent_linux_test.go +++ b/comp/process/agent/agentimpl/agent_linux_test.go @@ -19,6 +19,7 @@ import ( "github.com/DataDog/datadog-agent/comp/core/tagger/taggerimpl" "github.com/DataDog/datadog-agent/comp/core/telemetry" "github.com/DataDog/datadog-agent/comp/core/telemetry/telemetryimpl" + "github.com/DataDog/datadog-agent/comp/dogstatsd/statsd" "github.com/DataDog/datadog-agent/comp/process/agent" "github.com/DataDog/datadog-agent/comp/process/hostinfo/hostinfoimpl" "github.com/DataDog/datadog-agent/comp/process/processcheck/processcheckimpl" @@ -117,6 +118,7 @@ func TestProcessAgentComponentOnLinux(t *testing.T) { submitterimpl.MockModule(), taggerimpl.MockModule(), telemetryimpl.Module(), + statsd.MockModule(), Module(), fx.Replace(configComp.MockParams{Overrides: map[string]interface{}{ @@ -178,6 +180,7 @@ func TestStatusProvider(t *testing.T) { submitterimpl.MockModule(), taggerimpl.MockModule(), telemetryimpl.Module(), + statsd.MockModule(), Module(), fx.Replace(configComp.MockParams{Overrides: map[string]interface{}{ "process_config.run_in_core_agent.enabled": true, @@ -196,8 +199,9 @@ func TestStatusProvider(t *testing.T) { } }), )) - provides := newProcessAgent(deps) + provides, err := newProcessAgent(deps) assert.IsType(t, tc.expected, provides.StatusProvider.Provider) + assert.NoError(t, err) }) } } @@ -215,6 +219,7 @@ func TestTelemetryCoreAgent(t *testing.T) { hostinfoimpl.MockModule(), submitterimpl.MockModule(), taggerimpl.MockModule(), + statsd.MockModule(), Module(), fx.Replace(configComp.MockParams{Overrides: map[string]interface{}{ "process_config.run_in_core_agent.enabled": true, @@ -234,7 +239,8 @@ func TestTelemetryCoreAgent(t *testing.T) { } }), )) - _ = newProcessAgent(deps) + _, err := newProcessAgent(deps) + assert.NoError(t, err) tel := fxutil.Test[telemetry.Mock](t, telemetryimpl.MockModule()) tel.Reset() diff --git a/comp/process/agent/agentimpl/agent_test.go b/comp/process/agent/agentimpl/agent_test.go index 0d3c73d9e4aeb..2daf319069c81 100644 --- a/comp/process/agent/agentimpl/agent_test.go +++ b/comp/process/agent/agentimpl/agent_test.go @@ -15,6 +15,7 @@ import ( "github.com/DataDog/datadog-agent/comp/core/tagger/taggerimpl" "github.com/DataDog/datadog-agent/comp/core/telemetry/telemetryimpl" + "github.com/DataDog/datadog-agent/comp/dogstatsd/statsd" "github.com/DataDog/datadog-agent/comp/process/agent" "github.com/DataDog/datadog-agent/comp/process/hostinfo/hostinfoimpl" "github.com/DataDog/datadog-agent/comp/process/processcheck/processcheckimpl" @@ -65,6 +66,7 @@ func TestProcessAgentComponent(t *testing.T) { submitterimpl.MockModule(), telemetryimpl.Module(), taggerimpl.MockModule(), + statsd.MockModule(), Module(), } diff --git a/comp/process/bundle_test.go b/comp/process/bundle_test.go index 121bba33098a5..907dcf7bb09d7 100644 --- a/comp/process/bundle_test.go +++ b/comp/process/bundle_test.go @@ -19,6 +19,7 @@ import ( "github.com/DataDog/datadog-agent/comp/core/status" "github.com/DataDog/datadog-agent/comp/core/tagger" "github.com/DataDog/datadog-agent/comp/core/workloadmeta" + "github.com/DataDog/datadog-agent/comp/dogstatsd/statsd" "github.com/DataDog/datadog-agent/comp/forwarder/eventplatform/eventplatformimpl" "github.com/DataDog/datadog-agent/comp/forwarder/eventplatformreceiver/eventplatformreceiverimpl" "github.com/DataDog/datadog-agent/comp/networkpath/npcollector/npcollectorimpl" @@ -54,6 +55,7 @@ func TestBundleDependencies(t *testing.T) { ), fx.Provide(func() context.Context { return context.TODO() }), npcollectorimpl.MockModule(), + statsd.MockModule(), ) } @@ -94,6 +96,7 @@ func TestBundleOneShot(t *testing.T) { eventplatformimpl.Module(), fx.Supply(eventplatformimpl.NewDefaultParams()), npcollectorimpl.Module(), + statsd.MockModule(), Bundle(), ) require.NoError(t, err) diff --git a/comp/serializer/compression/go.mod b/comp/serializer/compression/go.mod index cf57ae6820039..8dc326ae31660 100644 --- a/comp/serializer/compression/go.mod +++ b/comp/serializer/compression/go.mod @@ -53,7 +53,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/system v0.54.0-rc.2 // indirect github.com/DataDog/datadog-agent/pkg/util/system/socket v0.54.0-rc.2 // indirect github.com/DataDog/datadog-agent/pkg/util/winutil v0.54.0-rc.2 // indirect - github.com/DataDog/viper v1.13.4 // indirect + github.com/DataDog/viper v1.13.5 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect github.com/davecgh/go-spew v1.1.1 // indirect diff --git a/comp/serializer/compression/go.sum b/comp/serializer/compression/go.sum index 9615820c08c5e..7af49998bfb57 100644 --- a/comp/serializer/compression/go.sum +++ b/comp/serializer/compression/go.sum @@ -1,8 +1,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/DataDog/viper v1.13.4 h1:0SrZc3zvMAGgVKg96uP4DEJb13lK2Is9a4go7IIcFSE= -github.com/DataDog/viper v1.13.4/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= +github.com/DataDog/viper v1.13.5 h1:SZMcyMknYQN2jRY/40A16gUXexlNJOI8sDs1cWZnI64= +github.com/DataDog/viper v1.13.5/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= github.com/DataDog/zstd v1.5.5 h1:oWf5W7GtOLgp6bciQYDmhHHjdhYkALu6S/5Ni9ZgSvQ= github.com/DataDog/zstd v1.5.5/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= diff --git a/docs/dev/agent_build.md b/docs/dev/agent_build.md index d0ed6c0a3e72c..d4490f71e1c18 100644 --- a/docs/dev/agent_build.md +++ b/docs/dev/agent_build.md @@ -5,18 +5,18 @@ You can decide at build time which components of the Agent you want to find in the final artifact. By default, all the components are picked up, so if you want to replicate the same configuration of the Agent distributed via system packages, -all you have to do is `invoke agent.build`. +all you have to do is `deva agent.build`. To pick only certain components you have to invoke the task like this: ``` -invoke agent.build --build-include=zstd,etcd,python +deva agent.build --build-include=zstd,etcd,python ``` Conversely, if you want to exclude something: ``` -invoke agent.build --build-exclude=systemd,python +deva agent.build --build-exclude=systemd,python ``` This is the complete list of the available components: @@ -68,13 +68,13 @@ into the Agent binary. For instance, to override the defaults and bundle only th and the security Agents: ``` -invoke agent.build --bundle process-agent --bundle security-agent +deva agent.build --bundle process-agent --bundle security-agent ``` To disable bundling entirely: ``` -invoke agent.build --bundle agent +deva agent.build --bundle agent ``` One binary per Agent can still be built by using its own invoke task and passing the @@ -89,7 +89,7 @@ One binary per Agent can still be built by using its own invoke task and passing So to build the process Agent as a standalone self contained executable: ``` -invoke process-agent.build --no-bundle +deva process-agent.build --no-bundle ``` @@ -108,7 +108,7 @@ COPY agent /opt/datadog-agent/bin/agent/agent For this to work properly, two things are important: - Your change needs to be done on top of the `` tag from the DataDog repository. -- You need to run the invoke task with the proper embedded path `inv -e agent.build -e /opt/datadog-agent/embedded`. +- You need to run the invoke task with the proper embedded path `deva -e agent.build -e /opt/datadog-agent/embedded`. **Note**: This makes `invoke` install the build's artifacts in the `/opt/datadog-agent/embedded` folder. Make sure the folder exists and the current user has write permissions. diff --git a/docs/dev/agent_dev_env.md b/docs/dev/agent_dev_env.md index 70d922278b01d..933ab741a2feb 100644 --- a/docs/dev/agent_dev_env.md +++ b/docs/dev/agent_dev_env.md @@ -7,82 +7,105 @@ To build the agent on Windows, see [datadog-agent-buildimages](https://github.co ### Python -The Agent embeds a full-fledged CPython interpreter so it requires the +The Agent embeds a full-fledged CPython interpreter, so it requires the development files to be available in the dev env. The Agent can embed Python2 and/or Python3, you will need development files for all versions you want to support. -If you're on OSX/macOS, installing Python 2.7 and/or 3.9 with [Homebrew](https://brew.sh) +If you're on OSX/macOS, installing Python 2.7 and/or 3.11 with [Homebrew](https://brew.sh) brings along all the development files needed: **Please note that not using Python versions explicitly supported, you may have problems running the built Agent's Python checks, especially if using a virtualenv. -At this time, only Python 3.9 is confirmed to work as expected in the development +At this time, only Python 3.11 is confirmed to work as expected in the development environment.** ``` brew install python@2 -brew install python@3.9 +brew install python@3.11 ``` On Linux, depending on the distribution, you might need to explicitly install the development files, for example on Ubuntu: ``` sudo apt-get install python2.7-dev -sudo apt-get install python3.9-dev +sudo apt-get install python3.11-dev ``` -On Windows, install Python 2.7 and/or 3.9 via the [official installer](https://www.python.org/downloads/). +On Windows, install Python 2.7 and/or 3.11 via the [official installer](https://www.python.org/downloads/). #### Python Dependencies ##### Preface -To protect and isolate your system-wide python installation, a python virtual -environment is _highly_ recommended (though optional). It will help keep a -self-contained development environment and ensure a clean system Python. +[Invoke](http://www.pyinvoke.org) is a task runner written in Python that is extensively used in this project to orchestrate builds and test runs. To run the tasks, you need to have it installed on your machine. We offer two different ways to run our invoke tasks. + +##### `deva` (recommended) + +The `deva` CLI tool is a single binary that can be used to install and manage the development environment for the Agent, built by the Datadog team. It will install all the necessary Python dependencies for you. The development environment will be completely independent of your system Python installation. This tool leverages [PyApp](https://ofek.dev/pyapp/latest/), a wrapper for Python applications that bootstrap themselves at runtime. In our case, we wrap `invoke` itself and include the dependencies needed to work on the Agent. + +To install `deva`, you'll need to: + +1. Download the binary for your platform from the [releases page](https://github.com/DataDog/datadog-agent-devtools/releases/latest), +2. Make it executable (and optionally add it to your PATH), +3. Run the invoke command you need, using `deva` in place of `invoke` or `inv`. + +The Python environment will automatically be created on the first run. and will be reused for subsequent runs. For example: + +```shell +$ cd datadog-agent +$ curl -L -o deva https://github.com/DataDog/datadog-agent-devtools/releases/download/deva-v1.0.0/deva-aarch64-unknown-linux-gnu-1.0.0 +$ chmod +x deva +$ ./deva linter.go +``` + +Below a live demo of how the tool works: + +![deva_install](../public/assets/images/deva.gif) + +If you want to uninstall `deva`, you can simply run the `./deva self remove` command, which will remove the virtual environment from your system, and remove the binary. That's it. + +##### Manual Installation + +###### Virtual Environment + +To protect and isolate your system-wide python installation, a python virtual environment is _highly_ recommended (though optional). It will help keep a self-contained development environment and ensure a clean system Python. **Please note that due to the [way some virtual environments handle executable paths](https://bugs.python.org/issue22213) (e.g. `python -m venv`), not all virtual environment options will be able to run the built -agent correctly. At this time, the only confirmed virtual enviroment that is known for +agent correctly. At this time, the only confirmed virtual environment that is known for sure to work is `virtualenv`.** - Install the virtualenv module: -`python3 -m pip install virtualenv` + ``` + python3 -m pip install virtualenv + ``` - Create the virtual environment: -`virtualenv $GOPATH/src/github.com/DataDog/datadog-agent/venv` -- [Activate the virtualenv](https://virtualenv.pypa.io/en/latest/user_guide.html#activators) (OS-dependent). - This must be done for every new terminal before you start. - -If using virtual environments when running the built Agent, you may need to override the built -Agent's search path for Python check packages using the `PYTHONPATH` variable (your target path -must have the [pre-requisite core integration packages installed](https://datadoghq.dev/integrations-core/setup/) -though). + ``` + virtualenv $GOPATH/src/github.com/DataDog/datadog-agent/venv + ``` +- [Activate the virtualenv](https://virtualenv.pypa.io/en/latest/user_guide.html#activators) (OS-dependent). This must be done for every new terminal before you start. + +If using virtual environments when running the built Agent, you may need to override the built Agent's search path for Python check packages using the `PYTHONPATH` variable (your target path must have the [pre-requisite core integration packages installed](https://datadoghq.dev/integrations-core/setup/) though). + ```sh -PYTHONPATH="./venv/lib/python3.9/site-packages:$PYTHONPATH" ./agent run ... +PYTHONPATH="./venv/lib/python3.11/site-packages:$PYTHONPATH" ./agent run ... ``` -See also some notes in [./checks](./checks) about running custom python checks. +See also some notes in [./checks](https://github.com/DataDog/datadog-agent/tree/main/docs/dev/checks) about running custom python checks. -#### Invoke +###### Install Invoke and its dependencies -[Invoke](http://www.pyinvoke.org/) is a task runner written in Python -that is extensively used in this project to orchestrate builds and test -runs. Our invoke tasks are only compatible with Python 3, thus you will -need to use Python 3 to run them. +Our invoke tasks are only compatible with Python 3, thus you will need to use Python 3 to run them. -Though you may install invoke in a variety of way we suggest you use -the provided [requirements](https://github.com/DataDog/datadog-agent/blob/main/requirements.txt) -file and `pip`: +Though you may install invoke in a variety of way we suggest you use the provided [requirements](https://github.com/DataDog/datadog-agent/blob/main/requirements.txt) file and `pip`: ```bash -pip install -r requirements.txt +pip install -r tasks/requirements.txt ``` -This procedure ensures you not only get the correct version of `invoke`, but -also any additional python dependencies our development workflow may require, -at their expected versions. -It will also pull other handy development tools/deps (`reno`, or `docker`). +This procedure ensures you not only get the correct version of `invoke`, but also any additional python dependencies our development workflow may require, at their expected versions. It will also pull other handy development tools/deps (`reno`, or `docker`). + ### Golang @@ -97,7 +120,7 @@ binary properly.** ### Installing tooling -From the root of `datadog-agent`, run `invoke install-tools` to install go tooling. This uses `go` to install the necessary dependencies. +From the root of `datadog-agent`, run `deva install-tools` to install go tooling. This uses `go` to install the necessary dependencies. ### System or Embedded? @@ -170,7 +193,7 @@ dev environment. We use [Doxygen](http://www.doxygen.nl/) to generate the documentation for the `rtloader` part of the Agent. -To generate it (using the `invoke rtloader.generate-doc` command), you'll need to have Doxygen installed on your system and available in your `$PATH`. You can compile and install Doxygen from source with the instructions available [here](http://www.doxygen.nl/manual/install.html). +To generate it (using the `deva rtloader.generate-doc` command), you'll need to have Doxygen installed on your system and available in your `$PATH`. You can compile and install Doxygen from source with the instructions available [here](http://www.doxygen.nl/manual/install.html). Alternatively, you can use already-compiled Doxygen binaries from [here](http://www.doxygen.nl/download.html). To get the dependency graphs, you may also need to install the `dot` executable from [graphviz](http://www.graphviz.org/) and add it to your `$PATH`. @@ -192,7 +215,7 @@ The `shellcheck` pre-commit hook requires having the `shellcheck` binary install To install it, run: ```sh -inv install-shellcheck --destination +deva install-shellcheck --destination ``` (by default, the shellcheck binary is installed in `/usr/local/bin`). @@ -220,7 +243,7 @@ the dependencies needed to develop in this repository. To configure the vscode editor to use a container as remote development environment you need to: - Install the [devcontainer plugin](https://code.visualstudio.com/docs/remote/containers) and the [golang language plugin](https://code.visualstudio.com/docs/languages/go). -- Run the following invoke command `invoke vscode.setup-devcontainer --image ""`. +- Run the following invoke command `deva vscode.setup-devcontainer --image ""`. This command will create the devcontainer configuration file `./devcontainer/devcontainer.json`. - Start or restart your vscode editor. - A pop-up should show-up to propose to "reopen in container" your workspace. diff --git a/docs/dev/agent_omnibus.md b/docs/dev/agent_omnibus.md index 6a35462f447db..d5960c47f8b09 100644 --- a/docs/dev/agent_omnibus.md +++ b/docs/dev/agent_omnibus.md @@ -19,7 +19,7 @@ From the `datadog-agent` source folder, use the following command to run the `omnibus.build` task in a Docker container: ``` -docker run -v "$PWD:/go/src/github.com/DataDog/datadog-agent" -v "/tmp/omnibus:/omnibus" -v "/tmp/opt/datadog-agent:/opt/datadog-agent" -v"/tmp/gems:/gems" --workdir=/go/src/github.com/DataDog/datadog-agent datadog/agent-buildimages-deb_x64 inv -e omnibus.build --base-dir=/omnibus --gem-path=/gems +docker run -v "$PWD:/go/src/github.com/DataDog/datadog-agent" -v "/tmp/omnibus:/omnibus" -v "/tmp/opt/datadog-agent:/opt/datadog-agent" -v"/tmp/gems:/gems" --workdir=/go/src/github.com/DataDog/datadog-agent datadog/agent-buildimages-deb_x64 deva -e omnibus.build --base-dir=/omnibus --gem-path=/gems ``` The container will share 3 volumes with the host to avoid starting from scratch @@ -60,20 +60,20 @@ the filesystem without disrupting anything. To run Omnibus and build the package, make the `/opt` folder world readable and run: ``` -inv omnibus.build --base-dir=$HOME/.omnibus +deva omnibus.build --base-dir=$HOME/.omnibus ``` On Mac, you might want to skip the signing step by running: ``` -inv omnibus.build --base-dir=$HOME/.omnibus --skip-sign +deva omnibus.build --base-dir=$HOME/.omnibus --skip-sign ``` The path you pass with the `--base-dir` option will contain the sources downloaded by Omnibus in the `src` folder, the binaries cached after building those sources in the `cache` folder and the final deb/rpm/dmg artifacts in the `pkg` folder. You can fine tune an Omnibus run passing more options, see -`inv omnibus.build --help` for the list of all the available options. +`deva omnibus.build --help` for the list of all the available options. **Note:** it's strongly advised to pass `--base-dir` and point to a directory outside the Agent repo. By default Omnibus stores packages in the project folder diff --git a/docs/dev/checks/README.md b/docs/dev/checks/README.md index 7de492274f996..f5de45baf8bbc 100644 --- a/docs/dev/checks/README.md +++ b/docs/dev/checks/README.md @@ -146,7 +146,7 @@ class MyCheck(AgentCheck): 1. Place the configuration file `hello_world.yaml` in the `dev/dist/conf.d/` folder. 1. Place your Python code in the `dev/dist/` folder. -1. Run `inv agent.build` as usual. This step copies the contents +1. Run `deva agent.build` as usual. This step copies the contents of `dev/dist` into `bin/agent/dist`, which is where the Agent looks for your code. @@ -184,12 +184,12 @@ correctly. `python3` bins. 1. Activate the virtualenv (OS-dependent) 1. `python3 -m pip install '/path/to/integrations-core/datadog_checks_base[deps]'` -1. `PYTHONPATH="$PWD/venv/lib/python3.10/site-packages:$PYTHONPATH" inv agent.run` +1. `PYTHONPATH="$PWD/venv/lib/python3.10/site-packages:$PYTHONPATH" deva agent.run` #### Example for user install 1. `python3 -m pip install --user '/path/to//integrations-core/datadog_checks_base[deps]'` -1. `PYTHONPATH="$HOME/.local/lib/python3.10/site-packages:$PYTHONPATH" inv agent.run` +1. `PYTHONPATH="$HOME/.local/lib/python3.10/site-packages:$PYTHONPATH" deva agent.run` #### Getting the right `PYTHONPATH` You want the `site-packages` directory that the `datadog_checks_base` got @@ -266,7 +266,7 @@ depending on your configuration) from the `PATH` that is used to run the Agent. > in `pkg/collector/python/init.go`. ### "Could not initialize Python" -Out of the box, after an `inv agent.build`, you may see the following error on +Out of the box, after an `deva agent.build`, you may see the following error on Linux when trying to run the resulting `agent` binary: `Could not initialize Python: could not load runtime python for version 3: Unable to open three library: libdatadog-agent-three.so: cannot open shared object file: No such file or directory` diff --git a/docs/dev/checks/jmxfetch.md b/docs/dev/checks/jmxfetch.md index 674737bd891b1..34e6142fee9af 100644 --- a/docs/dev/checks/jmxfetch.md +++ b/docs/dev/checks/jmxfetch.md @@ -12,7 +12,7 @@ version of JMXFetch, follow the instructions below: 1. Download the `-jar-with-dependencies.jar` build of the latest version of JMXFetch from [`maven`](https://repo1.maven.org/maven2/com/datadoghq/jmxfetch/) 2. Copy the jar file and rename it to `$GOPATH/src/github.com/DataDog/datadog-agent/dev/dist/jmx/jmxfetch.jar`. -3. Run `inv agent.run`. +3. Run `deva agent.run`. 4. Validate that the JMXFetch section appears in `agent status`. If you have a JMX-based integration configured to run, it automatically @@ -22,7 +22,7 @@ runs in your local JMXFetch instance. ## Custom Build of JMXFetch 1. [Build JMXFetch](https://github.com/DataDog/jmxfetch/#building-from-source). 2. Copy the resulting jar into `$GOPATH/src/github.com/DataDog/datadog-agent/dev/dist/jmx/jmxfetch.jar`. -3. Run `inv agent.run`. +3. Run `deva agent.run`. 4. Validate that the JMXFetch section appears in `agent status`. If you have a JMX-based integration configured to run, it should automatically @@ -36,5 +36,5 @@ be run in your local JMXFetch instance. - An example for the above test server can be found [in the jmxfetch repo](https://github.com/DataDog/jmxfetch/blob/master/tools/misbehaving-jmx-server/misbehaving-jmxfetch-conf.yaml). - This config should live in `dev/dist/conf.d/jmx-test-server.d/conf.yaml`. -3. Run `inv agent.run`. +3. Run `deva agent.run`. 4. Validate that the check appears as scheduled in `agent status`. diff --git a/docs/dev/linters.md b/docs/dev/linters.md index f48734e8f53ed..ee1d44b488942 100644 --- a/docs/dev/linters.md +++ b/docs/dev/linters.md @@ -13,7 +13,7 @@ For Go, we're using [golangci-lint](https://golangci-lint.run/), a Go linters ag The `linters` key defines the list of linters we're using: https://github.com/DataDog/datadog-agent/blob/dffd3262934a5540b9bf8e4bd3a743732637ef37/.golangci.yml#L65-L79 -To run the linters locally, run `inv linter.go`. +To run the linters locally, run `deva linter.go`. > [!TIP] > In your code, you can ignore linter issues on a line by prepending it with [the nolint directive](https://golangci-lint.run/usage/false-positives/#nolint-directive), for example, `//nolint:linter_name`. @@ -28,7 +28,7 @@ For Python, we're using ([see invoke task](https://github.com/DataDog/datadog-ag Their configuration is defined in both the [setup.cfg](https://github.com/DataDog/datadog-agent/blob/dffd3262934a5540b9bf8e4bd3a743732637ef37/setup.cfg) and the [pyproject.toml](https://github.com/DataDog/datadog-agent/blob/dffd3262934a5540b9bf8e4bd3a743732637ef37/pyproject.toml) files. -To run the linters locally, run `inv linter.python`. +To run the linters locally, run `deva linter.python`. > [!TIP] > In your code, you can ignore linter issues on a line by prepending it with `# noqa: error_code`. @@ -45,7 +45,7 @@ A: This could have several causes: - `go version` should output the same as [the repository .go-version](https://github.com/DataDog/datadog-agent/blob/dffd3262934a5540b9bf8e4bd3a743732637ef37/.go-version). - `golangci-lint --version` should output the same as [the repository internal/tools/go.mod](https://github.com/DataDog/datadog-agent/blob/dffd3262934a5540b9bf8e4bd3a743732637ef37/internal/tools/go.mod/#L8). - You're testing OS specific code; running locally, the linters only get the results for your local OS. -- You didn't run `inv tidy-all` in the repository, making some dependencies in the remote env outdated compared to your local env. +- You didn't run `deva tidy` in the repository, making some dependencies in the remote env outdated compared to your local env. ### About the new-from-rev golangci-lint parameter diff --git a/docs/public/assets/images/deva.gif b/docs/public/assets/images/deva.gif new file mode 100644 index 0000000000000..cfa6f393c61c5 Binary files /dev/null and b/docs/public/assets/images/deva.gif differ diff --git a/docs/public/components/creating-components.md b/docs/public/components/creating-components.md index a3552cd34def9..fc4af2e358ccc 100644 --- a/docs/public/components/creating-components.md +++ b/docs/public/components/creating-components.md @@ -53,7 +53,7 @@ This file hierarchy aims to solve a few problems: ## Bootstrapping components -You can use the [invoke](../setup.md#invoke) task `inv components.new-component comp/` to generate a scaffold for your new component. +You can use the [invoke](../setup.md#preface) task `deva components.new-component comp/` to generate a scaffold for your new component. Every public variable, function, struct, and interface of your component **must** be documented. Refer to the [Documentation](#documentation) section below for details. diff --git a/docs/public/guidelines/deprecated-components-documentation/defining-bundles.md b/docs/public/guidelines/deprecated-components-documentation/defining-bundles.md index f604e71de0b50..c272bbce5fa83 100644 --- a/docs/public/guidelines/deprecated-components-documentation/defining-bundles.md +++ b/docs/public/guidelines/deprecated-components-documentation/defining-bundles.md @@ -13,7 +13,7 @@ A bundle is defined in a dedicated package named `comp/`. The packag Typically, a bundle will automatically instantiate the top-level components that represent the bundle's purpose. For example, the trace-agent bundle `comp/trace` might automatically instantiate `comp/trace/agent`. -You can use the invoke task `inv components.new-bundle comp/` to generate a pre-filled `bundle.go` file for the given bundle. +You can use the invoke task `deva components.new-bundle comp/` to generate a pre-filled `bundle.go` file for the given bundle. ## Bundle Parameters diff --git a/docs/public/guidelines/docs.md b/docs/public/guidelines/docs.md index 67d9e0115bb30..f8efe306282a1 100644 --- a/docs/public/guidelines/docs.md +++ b/docs/public/guidelines/docs.md @@ -2,7 +2,7 @@ This site is built by [MkDocs](https://github.com/mkdocs/mkdocs) and uses the [Material for MkDocs](https://squidfunk.github.io/mkdocs-material/) theme. -You can serve documentation locally with the `docs.serve` [invoke task](../setup.md#invoke). +You can serve documentation locally with the `docs.serve` [invoke task](../setup.md#preface). ## Organization diff --git a/docs/public/setup.md b/docs/public/setup.md index b874aa8e2b1ec..957c2ef7c14a0 100644 --- a/docs/public/setup.md +++ b/docs/public/setup.md @@ -33,10 +33,41 @@ On Windows, install Python 2.7 and/or 3.11 via the [official installer](https:// ##### Preface +[Invoke](http://www.pyinvoke.org) is a task runner written in Python that is extensively used in this project to orchestrate builds and test runs. To run the tasks, you need to have it installed on your machine. We offer two different ways to run our invoke tasks. + +##### `deva` (recommended) + +The `deva` CLI tool is a single binary that can be used to install and manage the development environment for the Agent, built by the Datadog team. It will install all the necessary Python dependencies for you. The development environment will be completely independent of your system Python installation. This tool leverages [PyApp](https://ofek.dev/pyapp/latest/), a wrapper for Python applications that bootstrap themselves at runtime. In our case, we wrap `invoke` itself and include the dependencies needed to work on the Agent. + +To install `deva`, you'll need to: + +1. Download the binary for your platform from the [releases page](https://github.com/DataDog/datadog-agent-devtools/releases/latest), +2. Make it executable (and optionally add it to your PATH), +3. Run the invoke command you need, using `deva` in place of `invoke` or `inv`. + +The Python environment will automatically be created on the first run. and will be reused for subsequent runs. For example: + +```shell +$ cd datadog-agent +$ curl -L -o deva https://github.com/DataDog/datadog-agent-devtools/releases/download/deva-v1.0.0/deva-aarch64-unknown-linux-gnu-1.0.0 +$ chmod +x deva +$ ./deva linter.go +``` + +Below a live demo of how the tool works: + +![deva_install](./assets/images/deva.gif) + +If you want to uninstall `deva`, you can simply run the `./deva self remove` command, which will remove the virtual environment from your system, and remove the binary. That's it. + +##### Manual Installation + +###### Virtual Environment + To protect and isolate your system-wide python installation, a python virtual environment is _highly_ recommended (though optional). It will help keep a self-contained development environment and ensure a clean system Python. !!! note - Due to the [way some virtual environments handle executable paths](https://bugs.python.org/issue22213) (e.g. `python -m venv`), not all virtual environment options will be able to run the built Agent correctly. At this time, the only confirmed virtual enviroment creator that is known for sure to work is `virtualenv`. + Due to the [way some virtual environments handle executable paths](https://bugs.python.org/issue22213) (e.g. `python -m venv`), not all virtual environment options will be able to run the built Agent correctly. At this time, the only confirmed virtual environment creator that is known for sure to work is `virtualenv`. - Install the virtualenv module: ``` @@ -56,9 +87,9 @@ PYTHONPATH="./venv/lib/python3.11/site-packages:$PYTHONPATH" ./agent run ... See also some notes in [./checks](https://github.com/DataDog/datadog-agent/tree/main/docs/dev/checks) about running custom python checks. -#### Invoke +###### Install Invoke and its dependencies -[Invoke](http://www.pyinvoke.org) is a task runner written in Python that is extensively used in this project to orchestrate builds and test runs. Our invoke tasks are only compatible with Python 3, thus you will need to use Python 3 to run them. +Our invoke tasks are only compatible with Python 3, thus you will need to use Python 3 to run them. Though you may install invoke in a variety of way we suggest you use the provided [requirements](https://github.com/DataDog/datadog-agent/blob/main/requirements.txt) file and `pip`: @@ -141,7 +172,7 @@ pre-commit install The `shellcheck` pre-commit hook requires having the `shellcheck` binary installed and in your `$PATH`. To install it, run: ```sh -inv install-shellcheck --destination +deva install-shellcheck --destination ``` (by default, the shellcheck binary is installed in `/usr/local/bin`). @@ -169,7 +200,7 @@ See `pre-commit run --help` for further options. To configure the vscode editor to use a container as remote development environment you need to: - Install the [devcontainer plugin](https://code.visualstudio.com/docs/remote/containers) and the [golang language plugin](https://code.visualstudio.com/docs/languages/go). -- Run the following invoke command `invoke vscode.setup-devcontainer --image ""`. This command will create the devcontainer configuration file `./devcontainer/devcontainer.json`. +- Run the following invoke command `deva vscode.setup-devcontainer --image ""`. This command will create the devcontainer configuration file `./devcontainer/devcontainer.json`. - Start or restart your vscode editor. - A pop-up should show-up to propose to "reopen in container" your workspace. - The first start, it might propose you to install the golang plugin dependencies/tooling. diff --git a/go.mod b/go.mod index 1881cf5256056..408e217201641 100644 --- a/go.mod +++ b/go.mod @@ -50,6 +50,8 @@ replace ( github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/pipeline/provider => ./comp/otelcol/otlp/components/pipeline/provider github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/statsprocessor => ./comp/otelcol/otlp/components/statsprocessor github.com/DataDog/datadog-agent/comp/otelcol/otlp/testutil => ./comp/otelcol/otlp/testutil + github.com/DataDog/datadog-agent/comp/otelcol/provider/def => ./comp/otelcol/provider/def + github.com/DataDog/datadog-agent/comp/otelcol/provider/impl => ./comp/otelcol/provider/impl github.com/DataDog/datadog-agent/comp/serializer/compression => ./comp/serializer/compression github.com/DataDog/datadog-agent/pkg/aggregator/ckey => ./pkg/aggregator/ckey/ github.com/DataDog/datadog-agent/pkg/api => ./pkg/api @@ -139,11 +141,11 @@ require ( github.com/DataDog/ebpf-manager v0.6.0 github.com/DataDog/gopsutil v1.2.2 github.com/DataDog/nikos v1.12.4 - github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.16.0 + github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.16.1 github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/metrics v0.16.0 github.com/DataDog/opentelemetry-mapping-go/pkg/quantile v0.16.0 github.com/DataDog/sketches-go v1.4.4 - github.com/DataDog/viper v1.13.4 + github.com/DataDog/viper v1.13.5 github.com/DataDog/watermarkpodautoscaler v0.6.1 github.com/DataDog/zstd v1.5.5 github.com/DataDog/zstd_0 v0.0.0-20210310093942-586c1286621f // indirect @@ -232,14 +234,14 @@ require ( github.com/pahanini/go-grpc-bidirectional-streaming-example v0.0.0-20211027164128-cc6111af44be github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v1.19.0 + github.com/prometheus/client_golang v1.19.1 github.com/prometheus/client_model v0.6.1 github.com/prometheus/procfs v0.14.0 github.com/richardartoul/molecule v1.0.1-0.20221107223329-32cfee06a052 // indirect github.com/robfig/cron/v3 v3.0.1 github.com/samber/lo v1.39.0 github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da - github.com/shirou/gopsutil/v3 v3.24.3 + github.com/shirou/gopsutil/v3 v3.24.4 github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 github.com/sirupsen/logrus v1.9.3 github.com/skydive-project/go-debouncer v1.0.0 @@ -271,7 +273,7 @@ require ( go.opentelemetry.io/collector/exporter v0.100.0 go.opentelemetry.io/collector/exporter/loggingexporter v0.100.0 go.opentelemetry.io/collector/exporter/otlpexporter v0.100.0 - go.opentelemetry.io/collector/pdata v1.7.0 + go.opentelemetry.io/collector/pdata v1.8.0 go.opentelemetry.io/collector/processor/batchprocessor v0.100.0 go.opentelemetry.io/collector/receiver v0.100.0 go.opentelemetry.io/collector/receiver/otlpreceiver v0.100.0 @@ -284,18 +286,18 @@ require ( go.uber.org/zap v1.27.0 go4.org/netipx v0.0.0-20220812043211-3cc044ffd68d golang.org/x/arch v0.7.0 - golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8 - golang.org/x/net v0.24.0 + golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 + golang.org/x/net v0.25.0 golang.org/x/sync v0.7.0 - golang.org/x/sys v0.19.0 + golang.org/x/sys v0.20.0 golang.org/x/text v0.15.0 golang.org/x/time v0.5.0 - golang.org/x/tools v0.20.0 + golang.org/x/tools v0.21.0 golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda // indirect google.golang.org/grpc v1.63.2 google.golang.org/grpc/examples v0.0.0-20221020162917-9127159caf5a - google.golang.org/protobuf v1.34.0 + google.golang.org/protobuf v1.34.1 gopkg.in/DataDog/dd-trace-go.v1 v1.61.0 gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 @@ -551,9 +553,9 @@ require ( go.etcd.io/etcd/client/v3 v3.6.0-alpha.0 // indirect go.etcd.io/etcd/server/v3 v3.6.0-alpha.0.0.20220522111935-c3bc4116dcd1 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/collector/consumer v0.100.0 - go.opentelemetry.io/collector/featuregate v1.7.0 - go.opentelemetry.io/collector/semconv v0.100.0 // indirect + go.opentelemetry.io/collector/consumer v0.101.0 + go.opentelemetry.io/collector/featuregate v1.8.0 + go.opentelemetry.io/collector/semconv v0.101.0 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0 // indirect go.opentelemetry.io/contrib/propagators/b3 v1.26.0 // indirect go.opentelemetry.io/otel v1.26.0 // indirect @@ -566,10 +568,10 @@ require ( go.opentelemetry.io/otel/sdk/metric v1.26.0 // indirect go.opentelemetry.io/otel/trace v1.26.0 go.opentelemetry.io/proto/otlp v1.2.0 // indirect - golang.org/x/crypto v0.22.0 // indirect + golang.org/x/crypto v0.23.0 // indirect golang.org/x/mod v0.17.0 golang.org/x/oauth2 v0.19.0 // indirect - golang.org/x/term v0.19.0 // indirect + golang.org/x/term v0.20.0 // indirect gonum.org/v1/gonum v0.15.0 // indirect google.golang.org/api v0.177.0 // indirect google.golang.org/appengine v1.6.8 // indirect @@ -625,8 +627,9 @@ require ( github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/exporter/datadogexporter v0.0.0-00010101000000-000000000000 github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/exporter/logsagentexporter v0.54.0-rc.2 github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/exporter/serializerexporter v0.54.0-rc.2 - github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/pipeline/provider v0.0.0-00010101000000-000000000000 github.com/DataDog/datadog-agent/comp/otelcol/otlp/testutil v0.54.0-rc.2 + github.com/DataDog/datadog-agent/comp/otelcol/provider/def v0.0.0-00010101000000-000000000000 + github.com/DataDog/datadog-agent/comp/otelcol/provider/impl v0.0.0-00010101000000-000000000000 github.com/DataDog/datadog-agent/comp/serializer/compression v0.54.0-rc.2 github.com/DataDog/datadog-agent/pkg/aggregator/ckey v0.54.0-rc.2 github.com/DataDog/datadog-agent/pkg/api v0.54.0-rc.2 @@ -718,7 +721,6 @@ require ( go.opentelemetry.io/collector/otelcol v0.100.0 go.opentelemetry.io/collector/processor v0.100.0 go.opentelemetry.io/collector/service v0.100.0 - go.uber.org/goleak v1.3.0 go4.org/intern v0.0.0-20230525184215-6c62f75575cb go4.org/mem v0.0.0-20220726221520-4f986261bf13 gotest.tools v2.2.0+incompatible @@ -1129,10 +1131,10 @@ require ( go.opentelemetry.io/collector/config/configcompression v1.7.0 // indirect go.opentelemetry.io/collector/config/configgrpc v0.100.0 // indirect go.opentelemetry.io/collector/config/confighttp v0.100.0 // indirect - go.opentelemetry.io/collector/config/confignet v0.100.0 // indirect + go.opentelemetry.io/collector/config/confignet v0.101.0 // indirect go.opentelemetry.io/collector/config/configopaque v1.7.0 // indirect - go.opentelemetry.io/collector/config/configretry v0.100.0 // indirect - go.opentelemetry.io/collector/config/configtelemetry v0.100.0 // indirect + go.opentelemetry.io/collector/config/configretry v0.101.0 // indirect + go.opentelemetry.io/collector/config/configtelemetry v0.101.0 // indirect go.opentelemetry.io/collector/config/configtls v0.100.0 // indirect go.opentelemetry.io/collector/config/internal v0.100.0 // indirect go.opentelemetry.io/collector/connector v0.100.0 // indirect @@ -1143,7 +1145,7 @@ require ( go.opentelemetry.io/collector/extension/ballastextension v0.100.0 // indirect go.opentelemetry.io/collector/extension/zpagesextension v0.100.0 // indirect go.opentelemetry.io/collector/filter v0.100.0 // indirect - go.opentelemetry.io/collector/pdata/testdata v0.100.0 // indirect + go.opentelemetry.io/collector/pdata/testdata v0.101.0 // indirect go.opentelemetry.io/collector/processor/memorylimiterprocessor v0.100.0 // indirect go.opentelemetry.io/collector/receiver/nopreceiver v0.100.0 // indirect go.opentelemetry.io/contrib/config v0.6.0 // indirect diff --git a/go.sum b/go.sum index 6c02f96394e00..a338104fc01bd 100644 --- a/go.sum +++ b/go.sum @@ -787,8 +787,8 @@ github.com/DataDog/opentelemetry-mapping-go/pkg/inframetadata v0.16.0 h1:Jl7/oQQ github.com/DataDog/opentelemetry-mapping-go/pkg/inframetadata v0.16.0/go.mod h1:P/l++2cDCeeq21KSmCEdXdMH9/WMdXP7uA/vjnxhtz8= github.com/DataDog/opentelemetry-mapping-go/pkg/internal/sketchtest v0.16.0 h1:VJT1Jjlz/ca999FEqaAS+He7S4eB14a+PJjczgRdgAY= github.com/DataDog/opentelemetry-mapping-go/pkg/internal/sketchtest v0.16.0/go.mod h1:66XlN7QpQKqIvw8e2UbCXV5X8wGnEw851nT9BjJ75dY= -github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.16.0 h1:g/ztrLYZNfkpW6Bt8kMnLed5DaKRHEtiKE0opHXLHJk= -github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.16.0/go.mod h1:dvIWN9pA2zWNTw5rhDWZgzZnhcfpH++d+8d1SWW6xkY= +github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.16.1 h1:ZI8u3CgdMXpDplrf9/gIr13+/g/tUzUcBMk2ZhXgzLE= +github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.16.1/go.mod h1:dvIWN9pA2zWNTw5rhDWZgzZnhcfpH++d+8d1SWW6xkY= github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/logs v0.16.0 h1:NbKlfbjR2joF52jEBLs3MEnT6l5zM3MCyhUFkqARZpk= github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/logs v0.16.0/go.mod h1:+LijQ2LdlocAQ4WB+7KsoIGe90bfogkRslubd9swVow= github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/metrics v0.16.0 h1:H5DzD3rwgQCX0VI3A16KgsdmC5grUCyDFflaZDpfgMc= @@ -799,8 +799,8 @@ github.com/DataDog/sketches-go v1.4.4 h1:dF52vzXRFSPOj2IjXSWLvXq3jubL4CI69kwYjJ1 github.com/DataDog/sketches-go v1.4.4/go.mod h1:XR0ns2RtEEF09mDKXiKZiQg+nfZStrq1ZuL1eezeZe0= github.com/DataDog/trivy v0.0.0-20240524155722-0c355f8e174c h1:Xsdjt6y5ajENVAasDmdl0d6jbFkSYwhi5HD9CXke9Rc= github.com/DataDog/trivy v0.0.0-20240524155722-0c355f8e174c/go.mod h1:xmc7xCb5KSg2mFbztyInH8ciotVbad9SOmGFClgD0cU= -github.com/DataDog/viper v1.13.4 h1:0SrZc3zvMAGgVKg96uP4DEJb13lK2Is9a4go7IIcFSE= -github.com/DataDog/viper v1.13.4/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= +github.com/DataDog/viper v1.13.5 h1:SZMcyMknYQN2jRY/40A16gUXexlNJOI8sDs1cWZnI64= +github.com/DataDog/viper v1.13.5/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= github.com/DataDog/walker v0.0.0-20230418153152-7f29bb2dc950 h1:2imDajw3V85w1iqHsuXN+hUBZQVF+r9eME8tsPq/HpA= github.com/DataDog/walker v0.0.0-20230418153152-7f29bb2dc950/go.mod h1:FU+7qU8DeQQgSZDmmThMJi93kPkLFgy0oVAcLxurjIk= github.com/DataDog/watermarkpodautoscaler v0.6.1 h1:KEj10Cm8wO/36lEOgqjgDfIMMpMPReY/+bDacWe7Adw= @@ -2809,8 +2809,8 @@ github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= github.com/shibumi/go-pathspec v1.3.0 h1:QUyMZhFo0Md5B8zV8x2tesohbb5kfbpTi9rBnKh5dkI= github.com/shibumi/go-pathspec v1.3.0/go.mod h1:Xutfslp817l2I1cZvgcfeMQJG5QnU2lh5tVaaMCl3jE= -github.com/shirou/gopsutil/v3 v3.24.3 h1:eoUGJSmdfLzJ3mxIhmOAhgKEKgQkeOwKpz1NbhVnuPE= -github.com/shirou/gopsutil/v3 v3.24.3/go.mod h1:JpND7O217xa72ewWz9zN2eIIkPWsDN/3pl0H8Qt0uwg= +github.com/shirou/gopsutil/v3 v3.24.4 h1:dEHgzZXt4LMNm+oYELpzl9YCqV65Yr/6SfrvgRBtXeU= +github.com/shirou/gopsutil/v3 v3.24.4/go.mod h1:lTd2mdiOspcqLgAnr9/nGi71NkeMpWKdmhuxm9GusH8= github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 h1:udFKJ0aHUL60LboW/A+DfgoHVedieIzIXE8uylPue0U= github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= @@ -3139,14 +3139,14 @@ go.opentelemetry.io/collector/config/configgrpc v0.100.0 h1:+RuWrysXLbjaf/+I7dU9 go.opentelemetry.io/collector/config/configgrpc v0.100.0/go.mod h1:NDKPjtEVL7TJMfByR/D5MYyjveUU3D0GkM96jU0u494= go.opentelemetry.io/collector/config/confighttp v0.100.0 h1:bkB8ZkkRL+N75QofuIosf2ZzkEYaBAA5C+eQpL4fOis= go.opentelemetry.io/collector/config/confighttp v0.100.0/go.mod h1:AaugDfPoHeOmFT2BICuGNp3ja3Sq1AcTxxw4WysFZsI= -go.opentelemetry.io/collector/config/confignet v0.100.0 h1:SW8IMK+9GwFa1cNZdXw6wMkbCsqUaRtj0fgP8/yG6oI= -go.opentelemetry.io/collector/config/confignet v0.100.0/go.mod h1:3naWoPss70RhDHhYjGACi7xh4NcVRvs9itzIRVWyu1k= +go.opentelemetry.io/collector/config/confignet v0.101.0 h1:Mdb9e/EpCSac4Ccg7w4UchS/o4yY1WoIc9X5o7fTu9E= +go.opentelemetry.io/collector/config/confignet v0.101.0/go.mod h1:3naWoPss70RhDHhYjGACi7xh4NcVRvs9itzIRVWyu1k= go.opentelemetry.io/collector/config/configopaque v1.7.0 h1:nZh5Hb1ofq9xP1wHLSt4obM85pRTccSeAjV0NbrJeTc= go.opentelemetry.io/collector/config/configopaque v1.7.0/go.mod h1:vxoDKYYYUF/arrdQJxmfhlgkcsb0DpdzC9KPFP97uuE= -go.opentelemetry.io/collector/config/configretry v0.100.0 h1:jEswHFjNokqJ0U2iYSzUlDy8N6A6D+zaoHM9t1TB6yw= -go.opentelemetry.io/collector/config/configretry v0.100.0/go.mod h1:uRdmPeCkrW9Zsadh2WEbQ1AGXGYJ02vCfmmT+0g69nY= -go.opentelemetry.io/collector/config/configtelemetry v0.100.0 h1:unlhNrFFXCinxk6iPHPYwANO+eFY4S1NTb5knSxteW4= -go.opentelemetry.io/collector/config/configtelemetry v0.100.0/go.mod h1:YV5PaOdtnU1xRomPcYqoHmyCr48tnaAREeGO96EZw8o= +go.opentelemetry.io/collector/config/configretry v0.101.0 h1:5QggLq/lZiZXry1Ut52IOTbrdz1RbGoL29Io/wWdE4g= +go.opentelemetry.io/collector/config/configretry v0.101.0/go.mod h1:uRdmPeCkrW9Zsadh2WEbQ1AGXGYJ02vCfmmT+0g69nY= +go.opentelemetry.io/collector/config/configtelemetry v0.101.0 h1:G9RerNdBUm6rYW6wrJoKzleBiDsCGaCjtQx5UYr0hzw= +go.opentelemetry.io/collector/config/configtelemetry v0.101.0/go.mod h1:YV5PaOdtnU1xRomPcYqoHmyCr48tnaAREeGO96EZw8o= go.opentelemetry.io/collector/config/configtls v0.100.0 h1:qcx8EXW4u+IQvyt8ZH5ld2dEns1zp8sugyM+s7RuiKY= go.opentelemetry.io/collector/config/configtls v0.100.0/go.mod h1:f8KZu6P8hIzTfybLKG3xMIzkCmXyjxVUfDTVUp2CmhA= go.opentelemetry.io/collector/config/internal v0.100.0 h1:XSbedIpdXOxIEGnnzCZnulTmWPSGWfXTH18ZMxuqt8s= @@ -3169,8 +3169,8 @@ go.opentelemetry.io/collector/connector v0.100.0 h1:6LN8LXhv553SjipKl7EbvlAECIZ9 go.opentelemetry.io/collector/connector v0.100.0/go.mod h1:yF4fkEtDrZOpwGOcnDTjkGjMCCQTPL2VCCBe9R43Jbg= go.opentelemetry.io/collector/connector/forwardconnector v0.100.0 h1:XDjWlanH99OcRmHsrmdGJ6Wt2Emn3ngTqYDN9hICOc0= go.opentelemetry.io/collector/connector/forwardconnector v0.100.0/go.mod h1:6CMahqUyGv/pXcOStoDPzuGR4NfQwwu8xWjp2apF0GQ= -go.opentelemetry.io/collector/consumer v0.100.0 h1:8sALAcWvizSyrZJCF+zTqD2RLmZAyeCuaQrNS2q6ti0= -go.opentelemetry.io/collector/consumer v0.100.0/go.mod h1:JOPOq8nSTdnQwc2xdHl4hcuYBYV8gjN2SlFqlqBe/Nc= +go.opentelemetry.io/collector/consumer v0.101.0 h1:9tDxaeHe1+Uovf3fhdx7T4pV5mo/Dc0hniH7O5H3RBA= +go.opentelemetry.io/collector/consumer v0.101.0/go.mod h1:ud5k64on9m7hHTrhjEeLhWbLkd8+Gp06rDt3p86TKNs= go.opentelemetry.io/collector/exporter v0.100.0 h1:eyPb93tQwdft5Eboo8O5LDdaM1eXAQbtbXKBEYQlwh4= go.opentelemetry.io/collector/exporter v0.100.0/go.mod h1:5UrDewyFp5yIQHyV7HUFAPdhHKJGbz1/uaTunm7X54I= go.opentelemetry.io/collector/exporter/debugexporter v0.100.0 h1:9y+6FxuwbjQA7osZ1ywNJ7baV+PXOBges2RPmv2Xs20= @@ -3189,16 +3189,16 @@ go.opentelemetry.io/collector/extension/ballastextension v0.100.0 h1:hQEuz3c81Kl go.opentelemetry.io/collector/extension/ballastextension v0.100.0/go.mod h1:45XovD+uwmVcPD8FAMKCIxdR1TnEbWz9cD7i4sJtFr4= go.opentelemetry.io/collector/extension/zpagesextension v0.100.0 h1:4bHq1NBg8hU+NSd9DBVFAl4vxi40J0tqClmU3IknrEg= go.opentelemetry.io/collector/extension/zpagesextension v0.100.0/go.mod h1:mcLfCcjq0/yZoieuyO0H5rVL3u7Why0/MRezNV2E7AU= -go.opentelemetry.io/collector/featuregate v1.7.0 h1:8tNgX2VaiR9jrpZevRSvStuJrvvL6WwScT264HNLk7U= -go.opentelemetry.io/collector/featuregate v1.7.0/go.mod h1:w7nUODKxEi3FLf1HslCiE6YWtMtOOrMnSwsDam8Mg9w= +go.opentelemetry.io/collector/featuregate v1.8.0 h1:p/bAuk5LiSfdYS88yFl/Jzao9bHEYqCh7YvZJ+L+IZg= +go.opentelemetry.io/collector/featuregate v1.8.0/go.mod h1:w7nUODKxEi3FLf1HslCiE6YWtMtOOrMnSwsDam8Mg9w= go.opentelemetry.io/collector/filter v0.100.0 h1:XQyhnqJSK2sw+e9yvpkvl7y8QdJwH/gAnFoZDfEZ0dQ= go.opentelemetry.io/collector/filter v0.100.0/go.mod h1:3xGRpZo11DMJTDtMUGsDNkxKM6LMHqROGrQ/aTvskh8= go.opentelemetry.io/collector/otelcol v0.100.0 h1:5NWoo9T5tHP0oWt3OHetYpTRaQCJuef8KDDe5tLi+BA= go.opentelemetry.io/collector/otelcol v0.100.0/go.mod h1:pdPObjfQqP2pdE70jqQiJlZdEyZ0jA1euoRdHtgZjiA= -go.opentelemetry.io/collector/pdata v1.7.0 h1:/WNsBbE6KM3TTPUb9v/5B7IDqnDkgf8GyFhVJJqu7II= -go.opentelemetry.io/collector/pdata v1.7.0/go.mod h1:ehCBBA5GoFrMZkwyZAKGY/lAVSgZf6rzUt3p9mddmPU= -go.opentelemetry.io/collector/pdata/testdata v0.100.0 h1:pliojioiAv+CuLNTK+8tnCD2UgiJbKX9q8bDnpHkV1U= -go.opentelemetry.io/collector/pdata/testdata v0.100.0/go.mod h1:01BHOXvXaQaLLt5J34S093u3e+j//RhbfmEujpFJ/ME= +go.opentelemetry.io/collector/pdata v1.8.0 h1:d/QQgZxB4Y+d3mqLVh2ozvzujUhloD3P/fk7X+In764= +go.opentelemetry.io/collector/pdata v1.8.0/go.mod h1:/W7clu0wFC4WSRp94Ucn6Vm36Wkrt+tmtlDb1aiNZCY= +go.opentelemetry.io/collector/pdata/testdata v0.101.0 h1:JzeUtg5RN1iIFgY8DakGlqBkGxOTJlkaYlLausnEGKY= +go.opentelemetry.io/collector/pdata/testdata v0.101.0/go.mod h1:ZGobfCus4fWo5RduZ7ENI0+HD9BewgKuO6qU2rBVnUg= go.opentelemetry.io/collector/processor v0.100.0 h1:8Zcd3v77SCSM5mAJbGes5aR/Yof3aY1csiwFhKFpLEQ= go.opentelemetry.io/collector/processor v0.100.0/go.mod h1:ZqUC8WWVYyPkaLUT1JXUCNpCpde8zXgSaFfJq2FXuVU= go.opentelemetry.io/collector/processor/batchprocessor v0.100.0 h1:N94WWv+o9yt4HqvFeRUjL8VNEhY8mCym+2XPKUWxjmo= @@ -3211,8 +3211,8 @@ go.opentelemetry.io/collector/receiver/nopreceiver v0.100.0 h1:IsLY4qMy9EVD68yXK go.opentelemetry.io/collector/receiver/nopreceiver v0.100.0/go.mod h1:kefF3fOWGppE4n3Lpr4fTP8Fuj05aKH23iDp5eXqs2M= go.opentelemetry.io/collector/receiver/otlpreceiver v0.100.0 h1:JuTBkWQ0xc8FargDe5aE6wCoAEk1gxLnBzfdIi8NOL8= go.opentelemetry.io/collector/receiver/otlpreceiver v0.100.0/go.mod h1:X2xzbz9K4Kz0i3c1IKa1gcwf7rpoJDidyp28A1AuHFs= -go.opentelemetry.io/collector/semconv v0.100.0 h1:QArUvWcbmsMjM4PV0zngUHRizZeUXibsPBWjDuNJXAs= -go.opentelemetry.io/collector/semconv v0.100.0/go.mod h1:8ElcRZ8Cdw5JnvhTOQOdYizkJaQ10Z2fS+R6djOnj6A= +go.opentelemetry.io/collector/semconv v0.101.0 h1:tOe9iTe9dDCnvz/bqgfNRr4w80kXG8505tQJ5h5v08Q= +go.opentelemetry.io/collector/semconv v0.101.0/go.mod h1:8ElcRZ8Cdw5JnvhTOQOdYizkJaQ10Z2fS+R6djOnj6A= go.opentelemetry.io/collector/service v0.100.0 h1:2LFvNQNSs2NnwElyqkyhAiqaGoDdiMnTQeFPCLZNgg0= go.opentelemetry.io/collector/service v0.100.0/go.mod h1:65NPZ6THkR/e7fd8vh+tw4Lh6iDJ1twNXVzL76a3VNk= go.opentelemetry.io/contrib/config v0.6.0 h1:M1SRD1Z15XHPGk61tMLI1up77XT5FdrqQSRrlH0fYuk= @@ -3331,8 +3331,8 @@ golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= -golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= -golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= +golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -3348,8 +3348,8 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20220827204233-334a2380cb91/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= -golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8 h1:aAcj0Da7eBAtrTp03QXWvm88pSyOt+UgdZw2BFZ+lEw= -golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8/go.mod h1:CQ1k9gNrJ50XIzaKCRR2hssIjF07kZFEiieALBM/ARQ= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= golang.org/x/exp/typeparams v0.0.0-20230307190834-24139beb5833 h1:jWGQJV4niP+CCmFW9ekjA9Zx8vYORzOUH2/Nl5WPuLQ= golang.org/x/exp/typeparams v0.0.0-20230307190834-24139beb5833/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= @@ -3488,8 +3488,8 @@ golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= -golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= -golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190130055435-99b60b757ec1/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -3677,8 +3677,9 @@ golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -3695,8 +3696,8 @@ golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= -golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q= -golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= +golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -3809,8 +3810,8 @@ golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= -golang.org/x/tools v0.20.0 h1:hz/CVckiOxybQvFw6h7b/q80NTr9IUQb4s1IIzW7KNY= -golang.org/x/tools v0.20.0/go.mod h1:WvitBU7JJf6A4jOdg4S1tviW9bhUxkgeCui/0JHctQg= +golang.org/x/tools v0.21.0 h1:qc0xYgIbsSDt9EyWz05J5wfa7LOVW0YTLOXrqdLAWIw= +golang.org/x/tools v0.21.0/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -4133,8 +4134,8 @@ google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= -google.golang.org/protobuf v1.34.0 h1:Qo/qEd2RZPCf2nKuorzksSknv0d3ERwp1vFG38gSmH4= -google.golang.org/protobuf v1.34.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/DataDog/dd-trace-go.v1 v1.61.0 h1:XKO91GwTjpIRhd56Xif/BZ2YgHkQufVTOvtkbRYSPi8= gopkg.in/DataDog/dd-trace-go.v1 v1.61.0/go.mod h1:NHKX1t9eKmajySb6H+zLdgZizCFzbt5iKvrTyxEyy8w= gopkg.in/Knetic/govaluate.v3 v3.0.0 h1:18mUyIt4ZlRlFZAAfVetz4/rzlJs9yhN+U02F4u1AOc= diff --git a/omnibus/config/projects/agentless-scanner.rb b/omnibus/config/projects/agentless-scanner.rb deleted file mode 100644 index 59f6129cc02b9..0000000000000 --- a/omnibus/config/projects/agentless-scanner.rb +++ /dev/null @@ -1,146 +0,0 @@ -# Unless explicitly stated otherwise all files in this repository are licensed -# under the Apache License Version 2.0. -# This product includes software developed at Datadog (https:#www.datadoghq.com/). -# Copyright 2016-present Datadog, Inc. - -require "./lib/ostools.rb" - -name 'agentless-scanner' -package_name 'datadog-agentless-scanner' -homepage 'http://www.datadoghq.com' -license "Apache-2.0" -license_file "../LICENSE" - -if ENV.has_key?("OMNIBUS_WORKERS_OVERRIDE") - COMPRESSION_THREADS = ENV["OMNIBUS_WORKERS_OVERRIDE"].to_i -else - COMPRESSION_THREADS = 1 -end -if ENV.has_key?("DEPLOY_AGENT") && ENV["DEPLOY_AGENT"] == "true" - COMPRESSION_LEVEL = 9 -else - COMPRESSION_LEVEL = 5 -end - -install_dir '/opt/datadog/agentless-scanner' -if redhat_target? || suse_target? - maintainer 'Datadog, Inc ' - - # NOTE: with script dependencies, we only care about preinst/postinst/posttrans, - # because these would be used in a kickstart during package installation phase. - # All of the packages that we depend on in prerm/postrm scripts always have to be - # installed on all distros that we support, so we don't have to depend on them - # explicitly. - - # postinst and posttrans scripts use a subset of preinst script deps, so we don't - # have to list them, because they'll already be there because of preinst - runtime_script_dependency :pre, "coreutils" - runtime_script_dependency :pre, "grep" - if redhat_target? - runtime_script_dependency :pre, "glibc-common" - runtime_script_dependency :pre, "shadow-utils" - else - runtime_script_dependency :pre, "glibc" - runtime_script_dependency :pre, "shadow" - end -else - maintainer 'Datadog Packages ' -end - -if debian_target? - runtime_recommended_dependency 'datadog-signing-keys (>= 1:1.3.1)' -end - -# build_version is computed by an invoke command/function. -# We can't call it directly from there, we pass it through the environment instead. -build_version ENV['PACKAGE_VERSION'] - -build_iteration 1 - -description 'Datadog Agentless Scanner - The Datadog Agentless Scanner scans your cloud environment for vulnerabilities, compliance and security issues. - . - This package installs and runs the Agentless Scanner. - . - See http://www.datadoghq.com/ for more information -' - -# ------------------------------------ -# Generic package information -# ------------------------------------ - -# .deb specific flags -package :deb do - vendor 'Datadog ' - epoch 1 - license 'Apache License Version 2.0' - section 'utils' - priority 'extra' - compression_threads COMPRESSION_THREADS - compression_level COMPRESSION_LEVEL - compression_algo "xz" - if ENV.has_key?('DEB_SIGNING_PASSPHRASE') and not ENV['DEB_SIGNING_PASSPHRASE'].empty? - signing_passphrase "#{ENV['DEB_SIGNING_PASSPHRASE']}" - if ENV.has_key?('DEB_GPG_KEY_NAME') and not ENV['DEB_GPG_KEY_NAME'].empty? - gpg_key_name "#{ENV['DEB_GPG_KEY_NAME']}" - end - end -end - -# .rpm specific flags -package :rpm do - vendor 'Datadog ' - epoch 1 - dist_tag '' - license 'Apache License Version 2.0' - category 'System Environment/Daemons' - priority 'extra' - compression_threads COMPRESSION_THREADS - compression_level COMPRESSION_LEVEL - compression_algo "xz" - if ENV.has_key?('RPM_SIGNING_PASSPHRASE') and not ENV['RPM_SIGNING_PASSPHRASE'].empty? - signing_passphrase "#{ENV['RPM_SIGNING_PASSPHRASE']}" - if ENV.has_key?('RPM_GPG_KEY_NAME') and not ENV['RPM_GPG_KEY_NAME'].empty? - gpg_key_name "#{ENV['RPM_GPG_KEY_NAME']}" - end - end -end - -# ------------------------------------ -# Dependencies -# ------------------------------------ - -# creates required build directories -dependency 'datadog-agent-prepare' - -# version manifest file -dependency 'version-manifest' - -# Agentless-scanner -dependency 'datadog-agentless-scanner' - -# this dependency puts few files out of the omnibus install dir and move them -# in the final destination. This way such files will be listed in the packages -# manifest and owned by the package manager. This is the only point in the build -# process where we operate outside the omnibus install dir, thus the need of -# the `extra_package_file` directive. -# This must be the last dependency in the project. - -dependency 'datadog-agentless-scanner-finalize' - -# package scripts -if linux_target? - if debian_target? - package_scripts_path "#{Omnibus::Config.project_root}/package-scripts/agentless-scanner-deb" - else - package_scripts_path "#{Omnibus::Config.project_root}/package-scripts/agentless-scanner-rpm" - end -end - -if linux_target? - extra_package_file '/lib/systemd/system/datadog-agentless-scanner.service' - extra_package_file '/var/log/datadog/' -end - -exclude '\.git*' -exclude 'bundler\/git' diff --git a/omnibus/config/projects/dogstatsd.rb b/omnibus/config/projects/dogstatsd.rb index 49c76c4b558d2..126f4739f6de9 100644 --- a/omnibus/config/projects/dogstatsd.rb +++ b/omnibus/config/projects/dogstatsd.rb @@ -79,8 +79,38 @@ # Generic package information # ------------------------------------ +if ENV["OMNIBUS_PACKAGE_ARTIFACT_DIR"] + dependency "package-artifact" + do_package = true +else + do_package = false + # ------------------------------------ + # Dependencies + # ------------------------------------ + + # creates required build directories + dependency 'datadog-agent-prepare' + + # version manifest file + dependency 'version-manifest' + + # Dogstatsd + dependency 'datadog-dogstatsd' + + # this dependency puts few files out of the omnibus install dir and move them + # in the final destination. This way such files will be listed in the packages + # manifest and owned by the package manager. This is the only point in the build + # process where we operate outside the omnibus install dir, thus the need of + # the `extra_package_file` directive. + # This must be the last dependency in the project. + + dependency 'datadog-dogstatsd-finalize' +end + + # .deb specific flags package :deb do + skip_packager !do_package vendor 'Datadog ' epoch 1 license 'Apache License Version 2.0' @@ -99,6 +129,7 @@ # .rpm specific flags package :rpm do + skip_packager !do_package vendor 'Datadog ' epoch 1 dist_tag '' @@ -123,7 +154,9 @@ end package :xz do - skip_packager true + skip_packager do_package + compression_threads COMPRESSION_THREADS + compression_level COMPRESSION_LEVEL end package :msi do @@ -167,34 +200,17 @@ pkg_position '10, 10' end -# ------------------------------------ -# Dependencies -# ------------------------------------ - -# creates required build directories -dependency 'datadog-agent-prepare' - -# version manifest file -dependency 'version-manifest' - -# Dogstatsd -dependency 'datadog-dogstatsd' - -# this dependency puts few files out of the omnibus install dir and move them -# in the final destination. This way such files will be listed in the packages -# manifest and owned by the package manager. This is the only point in the build -# process where we operate outside the omnibus install dir, thus the need of -# the `extra_package_file` directive. -# This must be the last dependency in the project. - -dependency 'datadog-dogstatsd-finalize' - # package scripts if linux_target? - if debian_target? - package_scripts_path "#{Omnibus::Config.project_root}/package-scripts/dogstatsd-deb" + if !do_package + extra_package_file "#{Omnibus::Config.project_root}/package-scripts/dogstatsd-deb" + extra_package_file "#{Omnibus::Config.project_root}/package-scripts/dogstatsd-rpm" else - package_scripts_path "#{Omnibus::Config.project_root}/package-scripts/dogstatsd-rpm" + if debian_target? + package_scripts_path "#{Omnibus::Config.project_root}/package-scripts/dogstatsd-deb" + else + package_scripts_path "#{Omnibus::Config.project_root}/package-scripts/dogstatsd-rpm" + end end end diff --git a/omnibus/config/software/datadog-agentless-scanner-finalize.rb b/omnibus/config/software/datadog-agentless-scanner-finalize.rb deleted file mode 100644 index b2a9049e75070..0000000000000 --- a/omnibus/config/software/datadog-agentless-scanner-finalize.rb +++ /dev/null @@ -1,35 +0,0 @@ -# Unless explicitly stated otherwise all files in this repository are licensed -# under the Apache License Version 2.0. -# This product includes software developed at Datadog (https:#www.datadoghq.com/). -# Copyright 2016-present Datadog, Inc. - -# This software definition doesn"t build anything, it"s the place where we create -# files outside the omnibus installation directory, so that we can add them to -# the package manifest using `extra_package_file` in the project definition. -require './lib/ostools.rb' - -name "datadog-agentless-scanner-finalize" -description "steps required to finalize the build" -default_version "1.0.0" - -skip_transitive_dependency_licensing true - -build do - license :project_license - - # Move system service files - mkdir "/etc/init" - move "#{install_dir}/scripts/datadog-agentless-scanner.conf", "/etc/init" - if debian_target? - # sysvinit support for debian only for now - mkdir "/etc/init.d" - move "#{install_dir}/scripts/datadog-agentless-scanner", "/etc/init.d" - end - mkdir "/lib/systemd/system" - move "#{install_dir}/scripts/datadog-agentless-scanner.service", "/lib/systemd/system" - - mkdir "/var/log/datadog" - - # cleanup clutter - delete "#{install_dir}/etc" -end diff --git a/omnibus/config/software/datadog-agentless-scanner.rb b/omnibus/config/software/datadog-agentless-scanner.rb deleted file mode 100644 index b002c7b6932d5..0000000000000 --- a/omnibus/config/software/datadog-agentless-scanner.rb +++ /dev/null @@ -1,66 +0,0 @@ -# Unless explicitly stated otherwise all files in this repository are licensed -# under the Apache License Version 2.0. -# This product includes software developed at Datadog (https:#www.datadoghq.com/). -# Copyright 2016-present Datadog, Inc. -require 'pathname' - -name 'datadog-agentless-scanner' - -skip_transitive_dependency_licensing true - -source path: '..' -relative_path 'src/github.com/DataDog/datadog-agent' - -build do - license :project_license - - # set GOPATH on the omnibus source dir for this software - gopath = Pathname.new(project_dir) + '../../../..' - etc_dir = "/etc/datadog-agent" - env = { - 'GOPATH' => gopath.to_path, - 'PATH' => "#{gopath.to_path}/bin:#{ENV['PATH']}", - } - - unless ENV["OMNIBUS_GOMODCACHE"].nil? || ENV["OMNIBUS_GOMODCACHE"].empty? - gomodcache = Pathname.new(ENV["OMNIBUS_GOMODCACHE"]) - env["GOMODCACHE"] = gomodcache.to_path - end - - # we assume the go deps are already installed before running omnibus - command "invoke agentless-scanner.build --rebuild --major-version $MAJOR_VERSION", env: env - - mkdir "#{install_dir}/etc/datadog-agent" - mkdir "#{install_dir}/run/" - mkdir "#{install_dir}/scripts/" - - # move around bin and config files - copy 'bin/agentless-scanner/agentless-scanner', "#{install_dir}/bin" - - if debian_target? - erb source: "upstart_debian.conf.erb", - dest: "#{install_dir}/scripts/datadog-agentless-scanner.conf", - mode: 0644, - vars: { install_dir: install_dir, etc_dir: etc_dir } - erb source: "sysvinit_debian.agentless-scanner.erb", - dest: "#{install_dir}/scripts/datadog-agentless-scanner", - mode: 0755, - vars: { install_dir: install_dir, etc_dir: etc_dir } - elsif redhat_target? || suse_target? - # Ship a different upstart job definition on RHEL to accommodate the old - # version of upstart (0.6.5) that RHEL 6 provides. - erb source: "upstart_redhat.conf.erb", - dest: "#{install_dir}/scripts/datadog-agentless-scanner.conf", - mode: 0644, - vars: { install_dir: install_dir, etc_dir: etc_dir } - end - erb source: "systemd.service.erb", - dest: "#{install_dir}/scripts/datadog-agentless-scanner.service", - mode: 0644, - vars: { install_dir: install_dir, etc_dir: etc_dir } - - # The file below is touched by software builds that don't put anything in the installation - # directory (libgcc right now) so that the git_cache gets updated let's remove it from the - # final package - delete "#{install_dir}/uselessfile" -end diff --git a/omnibus/config/templates/datadog-agentless-scanner/systemd.service.erb b/omnibus/config/templates/datadog-agentless-scanner/systemd.service.erb deleted file mode 100644 index 8be82b2392376..0000000000000 --- a/omnibus/config/templates/datadog-agentless-scanner/systemd.service.erb +++ /dev/null @@ -1,17 +0,0 @@ -[Unit] -Description=Datadog Agentless Scanner -After=network.target datadog-agent.service -BindsTo=datadog-agent.service -ConditionPathExists=<%= etc_dir %>/datadog.yaml -StartLimitInterval=10 -StartLimitBurst=5 - -[Service] -Type=simple -PIDFile=<%= install_dir %>/run/agentless-scanner.pid -Restart=on-failure -ExecStart=<%= install_dir %>/bin/agentless-scanner run -c <%= etc_dir %>/datadog.yaml -p <%= install_dir %>/run/agentless-scanner.pid -TimeoutStopSec=60 - -[Install] -WantedBy=multi-user.target diff --git a/omnibus/config/templates/datadog-agentless-scanner/sysvinit_debian.agentless-scanner.erb b/omnibus/config/templates/datadog-agentless-scanner/sysvinit_debian.agentless-scanner.erb deleted file mode 100644 index be34ae76c1b65..0000000000000 --- a/omnibus/config/templates/datadog-agentless-scanner/sysvinit_debian.agentless-scanner.erb +++ /dev/null @@ -1,163 +0,0 @@ -#!/bin/sh - -### BEGIN INIT INFO -# Provides: datadog-agentless-scanner -# Short-Description: Start and stop datadog agentless scanner -# Description: Datadog Agentless Scanner -# Required-Start: $remote_fs -# Required-Stop: $remote_fs -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -### END INIT INFO - -. /lib/lsb/init-functions - -ETC_DIR="<%= etc_dir %>" -INSTALL_DIR="<%= install_dir %>" -AGENTPATH="$INSTALL_DIR/bin/agentless-scanner" -PIDFILE="$INSTALL_DIR/run/agentless-scanner.pid" -AGENT_ARGS="--config=$ETC_DIR/datadog.yaml --pidfile=$PIDFILE" -AGENT_USER="root" -NAME="datadog-agentless-scanner" -DESC="Datadog Agentless Scanner" - - - -if [ ! -x $AGENTPATH ]; then - echo "$AGENTPATH not found. Exiting $NAME" - exit 0 -fi - -if [ -r "/etc/default/${NAME}" ]; then - . "/etc/default/${NAME}" -fi - - -# -# Function that starts the daemon/service -# -do_start() -{ - # Return - # 0 if daemon has been started - # 1 if daemon was already running - # 2 if daemon could not be started - start-stop-daemon --start --test --quiet --pidfile $PIDFILE --user $AGENT_USER --startas $AGENTPATH > /dev/null \ - || return 1 - start-stop-daemon --start --background --chuid $AGENT_USER --quiet --pidfile $PIDFILE --user $AGENT_USER --startas $AGENTPATH -- \ - $AGENT_ARGS \ - || return 2 - # Add code here, if necessary, that waits for the process to be ready - # to handle requests from services started subsequently which depend - # on this one. As a last resort, sleep for some time. -} - - -# -# Start the agent and wait for it to be up -# -start_and_wait() -{ - log_daemon_msg "Starting $DESC" "$NAME" - do_start - case "$?" in - 1) log_end_msg 0 ;; - 2) log_end_msg 1 ;; - *) - # check if the agent is running once per second for 5 seconds - retries=5 - while [ $retries -gt 1 ]; do - start-stop-daemon --start --test --quiet --pidfile $PIDFILE --user $AGENT_USER --startas $AGENTPATH - if [ "$?" -eq "1" ]; then - # We've started up successfully. Exit cleanly - log_end_msg 0 - return 0 - else - retries=$(($retries - 1)) - sleep 1 - fi - done - # After 5 tries the agent didn't start. Report an error - log_end_msg 1 - - ;; - esac -} - -# -# Function that stops the daemon/service -# -do_stop() -{ - # Return - # 0 if daemon has been stopped - # 1 if daemon was already stopped - # 2 if daemon could not be stopped - # other if a failure occurred - start-stop-daemon --stop --quiet --retry=30 --pidfile $PIDFILE --user $AGENT_USER --startas $AGENTPATH - RETVAL="$?" - rm -f $PIDFILE - return $RETVAL -} - -# -# Stop the agent and wait for it to be down -# -stop_and_wait() -{ - log_daemon_msg "Stopping $DESC" "$NAME" - do_stop - case "$?" in - 0|1) log_end_msg 0 ;; - *) log_end_msg 1 ;; # Failed to stop - esac -} - -# Action to take -case "$1" in - start) - if init_is_upstart; then - exit 1 - fi - if [ "$DATADOG_ENABLED" = "no" ]; then - echo "Disabled via /etc/default/$NAME. Exiting." - exit 0 - fi - - start_and_wait - - ;; - - stop) - if init_is_upstart; then - exit 0 - fi - - stop_and_wait - - ;; - - status) - status_of_proc -p $PIDFILE $AGENTPATH $NAME && exit 0 || exit $? - ;; - - restart|force-reload) - if init_is_upstart; then - exit 1 - fi - - echo "Restarting $DESC" - - stop_and_wait - start_and_wait - - ;; - - *) - N=/etc/init.d/$NAME - echo "Usage: $N {start|stop|restart|status}" - exit 1 - ;; -esac - -exit $? diff --git a/omnibus/config/templates/datadog-agentless-scanner/upstart_debian.conf.erb b/omnibus/config/templates/datadog-agentless-scanner/upstart_debian.conf.erb deleted file mode 100644 index 489981aaf7692..0000000000000 --- a/omnibus/config/templates/datadog-agentless-scanner/upstart_debian.conf.erb +++ /dev/null @@ -1,17 +0,0 @@ -description "Datadog Agentless Scanner" - -start on started networking or started network -stop on runlevel [!2345] - -respawn -respawn limit 4 25 -normal exit 0 - -console log # redirect daemon's outputs to `/var/log/upstart/agentless-scanner.log` -env DD_LOG_TO_CONSOLE=false - -script - # setuid is not available in versions of upstart before 1.4. CentOS/RHEL6 use an earlier version of upstart. - # This is the best way to set the user in the absence of setuid. - exec su -s /bin/sh -c 'exec "$0" "$@"' dd-agent -- <%= install_dir %>/bin/agentless-scanner run -c <%= etc_dir %>/datadog.yaml -p <%= install_dir %>/run/agentless-scanner.pid -end script diff --git a/omnibus/config/templates/datadog-agentless-scanner/upstart_redhat.conf.erb b/omnibus/config/templates/datadog-agentless-scanner/upstart_redhat.conf.erb deleted file mode 100644 index 06c813d73f465..0000000000000 --- a/omnibus/config/templates/datadog-agentless-scanner/upstart_redhat.conf.erb +++ /dev/null @@ -1,18 +0,0 @@ -description "Datadog Agentless Scanner" - -start on started networking or started network -stop on runlevel [!2345] - -respawn -respawn limit 4 25 -normal exit 0 - -# console log is not available before upstart 1.4. CentOS/RHEL6 use an earlier version of upstart. -console output -env DD_LOG_TO_CONSOLE=false - -script - # setuid is not available in versions of upstart before 1.4. CentOS/RHEL6 use an earlier version of upstart. - # This is the best way to set the user in the absence of setuid. - exec su -s /bin/sh -c 'exec "$0" "$@"' dd-agent -- <%= install_dir %>/bin/agentless-scanner run -c <%= etc_dir %>/datadog.yaml -p <%= install_dir %>/run/agentless-scanner.pid -end script diff --git a/omnibus/package-scripts/agent-rpm/prerm b/omnibus/package-scripts/agent-rpm/prerm index 520cb64c33e01..44a8c5398ca53 100755 --- a/omnibus/package-scripts/agent-rpm/prerm +++ b/omnibus/package-scripts/agent-rpm/prerm @@ -115,6 +115,11 @@ remove_sysprobe_secagent_files() if [ -d "$INSTALL_DIR/run" ]; then rmdir "$INSTALL_DIR/run" 2>/dev/null || true fi + + # Remove any unpacked BTF files + find "$INSTALL_DIR/embedded/share/system-probe/ebpf/co-re/btf" -name "*.btf*" -type f -delete || true + # And remove empty directories + find "$INSTALL_DIR/embedded/share/system-probe/ebpf/co-re" -type d -empty -delete || true } remove_remote_config_db() diff --git a/omnibus/package-scripts/agentless-scanner-deb/postinst b/omnibus/package-scripts/agentless-scanner-deb/postinst deleted file mode 100755 index f541b574cf578..0000000000000 --- a/omnibus/package-scripts/agentless-scanner-deb/postinst +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/sh -# -# Perform necessary datadog-agentless-scanner setup steps after package is installed. -# -# .deb: STEP 5 of 5 - -INSTALL_DIR=/opt/datadog/agentless-scanner -CONFIG_DIR=/etc/datadog-agent -SERVICE_NAME=datadog-agentless-scanner - -# If we are inside the Docker container, do nothing -if [ -n "$DOCKER_DD_AGENT" ]; then - echo "Installation from docker-dd-agent, nothing to do in postinst" - exit 0 -fi - -set -e -case "$1" in - configure) - # Create a symlink to the agent's binary - ln -sf $INSTALL_DIR/bin/agentless-scanner /usr/bin/datadog-agentless-scanner - ;; - abort-upgrade|abort-remove|abort-deconfigure) - ;; - - *) - ;; -esac - -# Enable and restart the agentless-scanner service here on Debian platforms -# On RHEL, this is done in the posttrans script -# supports systemd, upstart and sysvinit -echo "Enabling service $SERVICE_NAME" -if command -v systemctl >/dev/null 2>&1; then - systemctl enable $SERVICE_NAME || echo "[ WARNING ]\tCannot enable $SERVICE_NAME with systemctl" -elif command -v initctl >/dev/null 2>&1; then - # Nothing to do, this is defined directly in the upstart job file - : -elif command -v update-rc.d >/dev/null 2>&1; then - update-rc.d $SERVICE_NAME defaults || echo "[ WARNING ]\tCannot enable $SERVICE_NAME with update-rc.d" -else - echo "[ WARNING ]\tCannot detect a supported init system. The datadog-agentless-scanner package only provides service files for systemd, upstart and sysvinit." -fi - -# TODO: Use a configcheck command on the agent to determine if it's safe to restart, -# and avoid restarting when a check conf is invalid -if [ -f "$CONFIG_DIR/datadog.yaml" ]; then - echo "(Re)starting $SERVICE_NAME now..." - if command -v systemctl >/dev/null 2>&1; then - systemctl restart $SERVICE_NAME || true - elif command -v initctl >/dev/null 2>&1; then - initctl start $SERVICE_NAME || initctl restart $SERVICE_NAME || true - elif command -v service >/dev/null 2>&1; then - service $SERVICE_NAME restart || true - else - echo "[ WARNING ]\tCannot detect a supported init system. The datadog-agentless-scanner package only provides service files for systemd and upstart." - fi -else - # No datadog.yaml file is present. This is probably a clean install made with the - # step-by-step instructions/an automation tool, and the config file will be added next. - echo "No datadog.yaml file detected, not starting the agentless-scanner" -fi - -exit 0 diff --git a/omnibus/package-scripts/agentless-scanner-deb/postrm b/omnibus/package-scripts/agentless-scanner-deb/postrm deleted file mode 100755 index 7975f03c85e49..0000000000000 --- a/omnibus/package-scripts/agentless-scanner-deb/postrm +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/sh -# -# Perform necessary datadog-agentless-scanner removal steps after package is uninstalled. -# -# .deb: STEP 3 of 5 - -INSTALL_DIR=/opt/datadog/agentless-scanner - -# Remove the symlink to the binary. -rm -f "/usr/bin/datadog-agentless-scanner" - -set -e - -case "$1" in - purge) - echo "Force-deleting $INSTALL_DIR" - rm -rf $INSTALL_DIR - ;; - *) - ;; -esac - -exit 0 diff --git a/omnibus/package-scripts/agentless-scanner-deb/preinst b/omnibus/package-scripts/agentless-scanner-deb/preinst deleted file mode 100755 index ba1c1f7779f69..0000000000000 --- a/omnibus/package-scripts/agentless-scanner-deb/preinst +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/sh -# -# Perform necessary datadog-agentless-scanner setup steps before package is installed. -# -# .deb: STEP 2 of 5 - -SERVICE_NAME=datadog-agentless-scanner - -set -e -if [ -f "/lib/systemd/system/$SERVICE_NAME.service" ] || [ -f "/usr/lib/systemd/system/$SERVICE_NAME.service" ]; then - # Stop an already running agent - # supports systemd, upstart and sysvinit - if command -v systemctl >/dev/null 2>&1; then - systemctl stop $SERVICE_NAME || true - elif command -v initctl >/dev/null 2>&1; then - initctl stop $SERVICE_NAME || true - elif command -v service >/dev/null 2>&1; then - service $SERVICE_NAME stop || true - else - echo "[ WARNING ]\tCannot detect a supported init system. The datadog-agentless-scanner package only provides service files for systemd, upstart and sysvinit." - fi -fi - -exit 0 diff --git a/omnibus/package-scripts/agentless-scanner-deb/prerm b/omnibus/package-scripts/agentless-scanner-deb/prerm deleted file mode 100755 index 6c69a98aca5c2..0000000000000 --- a/omnibus/package-scripts/agentless-scanner-deb/prerm +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/sh -# -# Perform necessary datadog-agentless-scanner setup steps prior to remove the old package. -# -# .deb: STEP 1 of 5 - -SERVICE_NAME=datadog-agentless-scanner - -stop_agent() -{ - # Stop an already running agentless-scanner - # supports systemd, upstart and sysvinit - if command -v systemctl >/dev/null 2>&1; then - systemctl stop $SERVICE_NAME || true - elif command -v initctl >/dev/null 2>&1; then - initctl stop $SERVICE_NAME || true - elif command -v service >/dev/null 2>&1; then - service $SERVICE_NAME stop || true - else - echo "[ WARNING ]\tCannot detect a supported init system. The datadog-agentless-scanner package only provides service files for systemd and upstart." - fi -} - -deregister_agent() -{ - # Disable agentless-scanner start on system boot - # supports systemd, upstart and sysvinit - if command -v systemctl >/dev/null 2>&1; then - systemctl disable $SERVICE_NAME || true - elif command -v initctl >/dev/null 2>&1; then - # Nothing to do, this is defined directly in the upstart job file - : - elif command -v update-rc.d >/dev/null 2>&1; then - update-rc.d -f $SERVICE_NAME remove || true - else - echo "[ WARNING ]\tCannot detect a supported init system. The datadog-agentless-scanner package only provides service files for systemd, upstart and sysvinit." - fi -} - -stop_agent -deregister_agent - -exit 0 diff --git a/omnibus/package-scripts/agentless-scanner-rpm/postinst b/omnibus/package-scripts/agentless-scanner-rpm/postinst deleted file mode 100755 index f077f17e9e583..0000000000000 --- a/omnibus/package-scripts/agentless-scanner-rpm/postinst +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/sh -# -# Perform necessary datadog-agentless-scanner setup steps after package is installed. -# NOTE: part of the setup is done in the posttrans (rpm-only) script -# -# .rpm: STEP 3 of 6 - -INSTALL_DIR=/opt/datadog/agentless-scanner -LOG_DIR=/var/log/datadog -CONFIG_DIR=/etc/datadog-agent - -# Set proper rights to the dd-agent user -chown -R dd-agent:dd-agent ${CONFIG_DIR} -chown -R dd-agent:dd-agent ${LOG_DIR} -chown -R dd-agent:dd-agent ${INSTALL_DIR} - -exit 0 diff --git a/omnibus/package-scripts/agentless-scanner-rpm/postrm b/omnibus/package-scripts/agentless-scanner-rpm/postrm deleted file mode 100755 index 09afd77e196da..0000000000000 --- a/omnibus/package-scripts/agentless-scanner-rpm/postrm +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/sh -# -# Perform necessary datadog-agentless-scanner removal steps after package is uninstalled. -# -# .rpm: STEP 5 of 6 - -# Remove the symlink to the binary. -rm -f "/usr/bin/datadog-agentless-scanner" - -case "$*" in - 0) - # We're uninstalling. - # We don't delete the dd-agent user/group (see https://fedoraproject.org/wiki/Packaging:UsersAndGroups#Allocation_Strategies) - ;; - 1) - # We're upgrading. - ;; - *) - ;; -esac - -exit 0 diff --git a/omnibus/package-scripts/agentless-scanner-rpm/posttrans b/omnibus/package-scripts/agentless-scanner-rpm/posttrans deleted file mode 100755 index 6bf740508dc15..0000000000000 --- a/omnibus/package-scripts/agentless-scanner-rpm/posttrans +++ /dev/null @@ -1,43 +0,0 @@ -#! /bin/sh -# -# This script is RPM-specific -# It is run at the very end of an install/upgrade of the package -# It is NOT run on removal of the package -# -# .rpm: STEP 6 of 6 - -INSTALL_DIR=/opt/datadog/agentless-scanner -CONFIG_DIR=/etc/datadog-agent -SERVICE_NAME=datadog-agentless-scanner - -# Create a symlink to the agent's binary -ln -sf $INSTALL_DIR/bin/agentless-scanner /usr/bin/datadog-agentless-scanner - -echo "Enabling service $SERVICE_NAME" -if command -v systemctl >/dev/null 2>&1; then - systemctl enable $SERVICE_NAME || echo "[ WARNING ]\tCannot enable $SERVICE_NAME with systemctl" -elif command -v initctl >/dev/null 2>&1; then - # start/stop policy is already defined in the upstart job file - : -else - echo "[ WARNING ]\tCannot detect a supported init system. The datadog-agentless-scanner package only provides service files for systemd and upstart." -fi - -# TODO: Use a configcheck command on the agent to determine if it's safe to restart, -# and avoid restarting when a check conf is invalid -if [ -f "$CONFIG_DIR/datadog.yaml" ]; then - echo "(Re)starting $SERVICE_NAME now..." - if command -v systemctl >/dev/null 2>&1; then - systemctl restart $SERVICE_NAME || true - elif command -v initctl >/dev/null 2>&1; then - initctl start $SERVICE_NAME || initctl restart $SERVICE_NAME || true - else - echo "[ WARNING ]\tCannot detect a supported init system. The datadog-agentless-scanner package only provides service files for systemd and upstart." - fi -else - # No datadog.yaml file is present. This is probably a clean install made with the - # step-by-step instructions/an automation tool, and the config file will be added next. - echo "No datadog.yaml file detected, not starting the agent" -fi - -exit 0 diff --git a/omnibus/package-scripts/agentless-scanner-rpm/preinst b/omnibus/package-scripts/agentless-scanner-rpm/preinst deleted file mode 100755 index 112ace3695355..0000000000000 --- a/omnibus/package-scripts/agentless-scanner-rpm/preinst +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/sh -# -# Perform necessary datadog-agentless-scanner setup steps before package is installed. -# -# .rpm: STEP 2 of 6 - -SERVICE_NAME=datadog-agentless-scanner - -# Linux installation -set -e -if [ -f "/lib/systemd/system/$SERVICE_NAME.service" ] || [ -f "/usr/lib/systemd/system/$SERVICE_NAME.service" ]; then - # Stop an already running agent - # Only supports systemd and upstart - if command -v systemctl >/dev/null 2>&1; then - systemctl stop $SERVICE_NAME || true - elif command -v initctl >/dev/null 2>&1; then - initctl stop $SERVICE_NAME || true - else - echo "[ WARNING ]\tCannot detect a supported init system. The datadog-agentless-scanner package only provides service files for systemd and upstart." - fi -fi - -exit 0 diff --git a/omnibus/package-scripts/agentless-scanner-rpm/prerm b/omnibus/package-scripts/agentless-scanner-rpm/prerm deleted file mode 100755 index 5bff0e3a8521b..0000000000000 --- a/omnibus/package-scripts/agentless-scanner-rpm/prerm +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/sh -# -# Perform necessary datadog-agentless-scanner setup steps prior to remove the old package. -# -# .rpm: STEP 4 of 6 - -SERVICE_NAME=datadog-agentless-scanner - -stop_agent() -{ - # Stop an already running agentless-scanner - # Only supports systemd and upstart - if command -v systemctl >/dev/null 2>&1; then - systemctl stop $SERVICE_NAME || true - elif command -v initctl >/dev/null 2>&1; then - initctl stop $SERVICE_NAME || true - else - echo "[ WARNING ]\tCannot detect a supported init system. The datadog-agentless-scanner package only provides service files for systemd and upstart." - fi -} - -deregister_agent() -{ - # Disable agentless-scanner start on system boot - # Only supports systemd and upstart - if command -v systemctl >/dev/null 2>&1; then - systemctl disable $SERVICE_NAME || true - elif command -v initctl >/dev/null 2>&1; then - # Nothing to do, this is defined directly in the upstart job file - : - else - echo "[ WARNING ]\tCannot detect a supported init system. The datadog-agentless-scanner package only provides service files for systemd and upstart." - fi -} - -stop_agent -deregister_agent - -exit 0 diff --git a/pkg/aggregator/aggregator.go b/pkg/aggregator/aggregator.go index 9e3a58515de33..c9a8af639d688 100644 --- a/pkg/aggregator/aggregator.go +++ b/pkg/aggregator/aggregator.go @@ -281,6 +281,14 @@ func NewBufferedAggregator(s serializer.MetricSerializer, eventPlatformForwarder agentName = flavor.HerokuAgent } + if config.Datadog().GetBool("djm_config.enabled") { + AddRecurrentSeries(&metrics.Serie{ + Name: "datadog.djm.agent_host", + Points: []metrics.Point{{Value: 1.0}}, + MType: metrics.APIGaugeType, + }) + } + tagsStore := tags.NewStore(config.Datadog().GetBool("aggregator_use_tags_store"), "aggregator") aggregator := &BufferedAggregator{ diff --git a/pkg/aggregator/aggregator_test.go b/pkg/aggregator/aggregator_test.go index a8dd65ce5dd35..8cfc12672cc61 100644 --- a/pkg/aggregator/aggregator_test.go +++ b/pkg/aggregator/aggregator_test.go @@ -595,6 +595,30 @@ func TestTimeSamplerFlush(t *testing.T) { assertSeriesEqual(t, s.series, expectedSeries) } +func TestAddDJMRecurrentSeries(t *testing.T) { + // this test IS USING globals (recurrentSeries) + // - + + djmEnabled := pkgconfig.Datadog().GetBool("djm_config.enabled") + pkgconfig.Datadog().SetWithoutSource("djm_config.enabled", true) + defer pkgconfig.Datadog().SetWithoutSource("djm_config.enabled", djmEnabled) + + s := &MockSerializerIterableSerie{} + // NewBufferedAggregator with DJM enable will create a new recurrentSeries + NewBufferedAggregator(s, nil, "hostname", DefaultFlushInterval) + + expectedRecurrentSeries := metrics.Series{&metrics.Serie{ + Name: "datadog.djm.agent_host", + Points: []metrics.Point{{Value: 1.0}}, + MType: metrics.APIGaugeType, + }} + + require.EqualValues(t, expectedRecurrentSeries, recurrentSeries) + + // Reset recurrentSeries + recurrentSeries = metrics.Series{} +} + // The implementation of MockSerializer.SendIterableSeries uses `s.Called(series).Error(0)`. // It calls internaly `Printf` on each field of the real type of `IterableStreamJSONMarshaler` which is `IterableSeries`. // It can lead to a race condition, if another goruntine call `IterableSeries.Append` which modifies `series.count`. diff --git a/pkg/aggregator/demultiplexer_agent.go b/pkg/aggregator/demultiplexer_agent.go index c541d95a108a0..1512c756d86fe 100644 --- a/pkg/aggregator/demultiplexer_agent.go +++ b/pkg/aggregator/demultiplexer_agent.go @@ -46,7 +46,7 @@ type DemultiplexerWithAggregator interface { type AgentDemultiplexer struct { log log.Component - m sync.Mutex + m sync.RWMutex // stopChan completely stops the flushLoop of the Demultiplexer when receiving // a message, not doing anything else. @@ -391,8 +391,8 @@ func (d *AgentDemultiplexer) ForceFlushToSerializer(start time.Time, waitForSeri // - to have an implementation of SendIterableSeries listening on multiple sinks in parallel, or, // - to have a thread-safe implementation of the underlying `util.BufferedChan`. func (d *AgentDemultiplexer) flushToSerializer(start time.Time, waitForSerializer bool) { - d.m.Lock() - defer d.m.Unlock() + d.m.RLock() + defer d.m.RUnlock() if d.aggregator == nil { // NOTE(remy): we could consider flushing only the time samplers @@ -561,8 +561,8 @@ func (d *AgentDemultiplexer) DumpDogstatsdContexts(dest io.Writer) error { // If no error is returned here, DestroySender must be called with the same ID // once the sender is not used anymore func (d *AgentDemultiplexer) GetSender(id checkid.ID) (sender.Sender, error) { - d.m.Lock() - defer d.m.Unlock() + d.m.RLock() + defer d.m.RUnlock() if d.senders == nil { return nil, errors.New("demultiplexer is stopped") @@ -574,8 +574,8 @@ func (d *AgentDemultiplexer) GetSender(id checkid.ID) (sender.Sender, error) { // SetSender returns the passed sender with the passed ID. // This is largely for testing purposes func (d *AgentDemultiplexer) SetSender(s sender.Sender, id checkid.ID) error { - d.m.Lock() - defer d.m.Unlock() + d.m.RLock() + defer d.m.RUnlock() if d.senders == nil { return errors.New("demultiplexer is stopped") } @@ -587,8 +587,8 @@ func (d *AgentDemultiplexer) SetSender(s sender.Sender, id checkid.ID) error { // Should be called when no sender with this ID is used anymore // The metrics of this (these) sender(s) that haven't been flushed yet will be lost func (d *AgentDemultiplexer) DestroySender(id checkid.ID) { - d.m.Lock() - defer d.m.Unlock() + d.m.RLock() + defer d.m.RUnlock() if d.senders == nil { return @@ -599,8 +599,8 @@ func (d *AgentDemultiplexer) DestroySender(id checkid.ID) { // GetDefaultSender returns a default sender. func (d *AgentDemultiplexer) GetDefaultSender() (sender.Sender, error) { - d.m.Lock() - defer d.m.Unlock() + d.m.RLock() + defer d.m.RUnlock() if d.senders == nil { return nil, errors.New("demultiplexer is stopped") diff --git a/pkg/api/go.mod b/pkg/api/go.mod index 12b6088a8e173..7dae1c5aa1251 100644 --- a/pkg/api/go.mod +++ b/pkg/api/go.mod @@ -57,7 +57,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/system/socket v0.54.0-rc.2 // indirect github.com/DataDog/datadog-agent/pkg/util/winutil v0.54.0-rc.2 // indirect github.com/DataDog/datadog-agent/pkg/version v0.54.0-rc.2 // indirect - github.com/DataDog/viper v1.13.4 // indirect + github.com/DataDog/viper v1.13.5 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect github.com/davecgh/go-spew v1.1.1 // indirect diff --git a/pkg/api/go.sum b/pkg/api/go.sum index 4f5285c7fa353..b159185f44664 100644 --- a/pkg/api/go.sum +++ b/pkg/api/go.sum @@ -1,8 +1,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/DataDog/viper v1.13.4 h1:0SrZc3zvMAGgVKg96uP4DEJb13lK2Is9a4go7IIcFSE= -github.com/DataDog/viper v1.13.4/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= +github.com/DataDog/viper v1.13.5 h1:SZMcyMknYQN2jRY/40A16gUXexlNJOI8sDs1cWZnI64= +github.com/DataDog/viper v1.13.5/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= diff --git a/pkg/collector/corechecks/cluster/orchestrator/transformers/ecs/task.go b/pkg/collector/corechecks/cluster/orchestrator/transformers/ecs/task.go index 5789d716a5a3b..b8f899a7b26d7 100644 --- a/pkg/collector/corechecks/cluster/orchestrator/transformers/ecs/task.go +++ b/pkg/collector/corechecks/cluster/orchestrator/transformers/ecs/task.go @@ -140,7 +140,7 @@ func extractTimestamp(t time.Time) int64 { return t.Unix() } -func extractExitCode(exitCode *uint32) *model.ECSContainerExitCode { +func extractExitCode(exitCode *int64) *model.ECSContainerExitCode { if exitCode == nil { return nil } diff --git a/pkg/collector/corechecks/cluster/orchestrator/transformers/ecs/task_test.go b/pkg/collector/corechecks/cluster/orchestrator/transformers/ecs/task_test.go index 6c03f06b216e7..9a6c52bc81124 100644 --- a/pkg/collector/corechecks/cluster/orchestrator/transformers/ecs/task_test.go +++ b/pkg/collector/corechecks/cluster/orchestrator/transformers/ecs/task_test.go @@ -15,6 +15,7 @@ import ( model "github.com/DataDog/agent-payload/v5/process" "github.com/DataDog/datadog-agent/comp/core/workloadmeta" + "github.com/DataDog/datadog-agent/pkg/util/pointer" ) func TestExtractECSTask(t *testing.T) { @@ -74,11 +75,9 @@ func TestExtractECSTask(t *testing.T) { ECSContainer: &workloadmeta.ECSContainer{ DisplayName: "log_router_container", Health: &workloadmeta.ContainerHealthStatus{ - Status: "HEALTHY", - Since: &now, - ExitCode: func(i uint32) *uint32 { - return &i - }(2), + Status: "HEALTHY", + Since: &now, + ExitCode: pointer.Ptr(int64(2)), }, Type: "NORMAL", }, diff --git a/pkg/collector/corechecks/containerlifecycle/processor_test.go b/pkg/collector/corechecks/containerlifecycle/processor_test.go index f0f1073bd2e2e..80be6628fa2ed 100644 --- a/pkg/collector/corechecks/containerlifecycle/processor_test.go +++ b/pkg/collector/corechecks/containerlifecycle/processor_test.go @@ -121,7 +121,7 @@ func TestProcessContainer(t *testing.T) { } now := time.Now() - exitCode := uint32(1) + exitCode := int64(1) podContainer := workloadmeta.Container{ EntityID: workloadmeta.EntityID{ ID: "cont1", diff --git a/pkg/collector/corechecks/oracle/config/config.go b/pkg/collector/corechecks/oracle/config/config.go index ce1be8b857281..28695f3caf032 100644 --- a/pkg/collector/corechecks/oracle/config/config.go +++ b/pkg/collector/corechecks/oracle/config/config.go @@ -158,7 +158,7 @@ type InstanceConfig struct { UseGlobalCustomQueries string `yaml:"use_global_custom_queries"` CustomQueries []CustomQuery `yaml:"custom_queries"` MetricCollectionInterval int64 `yaml:"metric_collection_interval"` - DatabaseInstanceCollectionInterval uint64 `yaml:"database_instance_collection_interval"` + DatabaseInstanceCollectionInterval int64 `yaml:"database_instance_collection_interval"` Asm asmConfig `yaml:"asm"` ResourceManager resourceManagerConfig `yaml:"resource_manager"` Locks locksConfig `yaml:"locks"` @@ -229,7 +229,7 @@ func NewCheckConfig(rawInstance integration.Data, rawInitConfig integration.Data instance.UseGlobalCustomQueries = "true" - instance.DatabaseInstanceCollectionInterval = 1800 + instance.DatabaseInstanceCollectionInterval = 300 instance.Loader = defaultLoader initCfg.Loader = defaultLoader diff --git a/pkg/collector/corechecks/oracle/metadata.go b/pkg/collector/corechecks/oracle/metadata.go index 2f06a76d66193..d81a66f325a7d 100644 --- a/pkg/collector/corechecks/oracle/metadata.go +++ b/pkg/collector/corechecks/oracle/metadata.go @@ -28,7 +28,7 @@ type dbInstanceEvent struct { AgentVersion string `json:"agent_version"` Dbms string `json:"dbms"` Kind string `json:"kind"` - CollectionInterval uint64 `json:"collection_interval"` + CollectionInterval int64 `json:"collection_interval"` DbmsVersion string `json:"dbms_version"` Tags []string `json:"tags"` Timestamp float64 `json:"timestamp"` diff --git a/pkg/collector/corechecks/oracle/oracle.go b/pkg/collector/corechecks/oracle/oracle.go index 94779cb4a5c82..b52683c17f46a 100644 --- a/pkg/collector/corechecks/oracle/oracle.go +++ b/pkg/collector/corechecks/oracle/oracle.go @@ -182,7 +182,7 @@ func (c *Check) Run() error { c.connection = conn } - dbInstanceIntervalExpired := checkIntervalExpired(&c.dbInstanceLastRun, 1800) + dbInstanceIntervalExpired := checkIntervalExpired(&c.dbInstanceLastRun, c.config.DatabaseInstanceCollectionInterval) if dbInstanceIntervalExpired && !c.legacyIntegrationCompatibilityMode && !c.config.OnlyCustomQueries { err := sendDbInstanceMetadata(c) diff --git a/pkg/collector/corechecks/orchestrator/ecs/ecs_test.go b/pkg/collector/corechecks/orchestrator/ecs/ecs_test.go index 2e8ddf447d4f7..997cf14f746f7 100644 --- a/pkg/collector/corechecks/orchestrator/ecs/ecs_test.go +++ b/pkg/collector/corechecks/orchestrator/ecs/ecs_test.go @@ -21,6 +21,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/orchestrator" oconfig "github.com/DataDog/datadog-agent/pkg/orchestrator/config" "github.com/DataDog/datadog-agent/pkg/serializer/types" + "github.com/DataDog/datadog-agent/pkg/util/pointer" ) func TestGetRegionAndAWSAccountID(t *testing.T) { @@ -245,10 +246,8 @@ func container1(v4 bool) *workloadmeta.Container { container.ECSContainer = &workloadmeta.ECSContainer{ DisplayName: "log_router_container", Health: &workloadmeta.ContainerHealthStatus{ - Status: "HEALTHY", - ExitCode: func(i uint32) *uint32 { - return &i - }(2), + Status: "HEALTHY", + ExitCode: pointer.Ptr(int64(-2)), }, Type: "NORMAL", } @@ -327,7 +326,7 @@ func expected(v4 bool, groupID int32, ids ...string) *process.CollectorECSTask { container1.Health = &process.ECSContainerHealth{ Status: "HEALTHY", ExitCode: &process.ECSContainerExitCode{ - ExitCode: 2, + ExitCode: -2, }, } diff --git a/pkg/collector/corechecks/servicediscovery/errors.go b/pkg/collector/corechecks/servicediscovery/errors.go new file mode 100644 index 0000000000000..2611d6fbbeecd --- /dev/null +++ b/pkg/collector/corechecks/servicediscovery/errors.go @@ -0,0 +1,32 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +package servicediscovery + +import ( + "fmt" +) + +type errCode string + +const ( + errorCodeProcfs errCode = "procfs" + errorCodePortPoller errCode = "port_poller" + errorCodeRepeatedServiceName errCode = "repeated_service_name" +) + +type errWithCode struct { + err error + code errCode + svc *serviceMetadata +} + +func (e errWithCode) Error() string { + return fmt.Sprintf("%s: %s", e.code, e.err.Error()) +} + +func (e errWithCode) Code() errCode { + return e.code +} diff --git a/pkg/collector/corechecks/servicediscovery/impl_linux.go b/pkg/collector/corechecks/servicediscovery/impl_linux.go index 181295e28eaca..c80882c508c90 100644 --- a/pkg/collector/corechecks/servicediscovery/impl_linux.go +++ b/pkg/collector/corechecks/servicediscovery/impl_linux.go @@ -41,7 +41,6 @@ type linuxImpl struct { bootTime uint64 serviceDetector *serviceDetector - sender *telemetrySender ignoreCfg map[string]bool ignoreProcs map[int]bool @@ -49,7 +48,7 @@ type linuxImpl struct { potentialServices map[int]*serviceInfo } -func newLinuxImpl(sender *telemetrySender, ignoreCfg map[string]bool) (osImpl, error) { +func newLinuxImpl(ignoreCfg map[string]bool) (osImpl, error) { for _, i := range ignoreCfgLinux { ignoreCfg[i] = true } @@ -70,7 +69,6 @@ func newLinuxImpl(sender *telemetrySender, ignoreCfg map[string]bool) (osImpl, e bootTime: stat.BootTime, portPoller: poller, time: realTime{}, - sender: sender, serviceDetector: newServiceDetector(), ignoreCfg: ignoreCfg, ignoreProcs: make(map[int]bool), @@ -79,54 +77,30 @@ func newLinuxImpl(sender *telemetrySender, ignoreCfg map[string]bool) (osImpl, e }, nil } -type processEvents struct { - started []serviceInfo - stopped []serviceInfo -} - -type eventsByName map[string]*processEvents - -func (e eventsByName) addStarted(svc serviceInfo) { - events, ok := e[svc.meta.Name] - if !ok { - events = &processEvents{} - } - events.started = append(events.started, svc) - e[svc.meta.Name] = events -} - -func (e eventsByName) addStopped(svc serviceInfo) { - events, ok := e[svc.meta.Name] - if !ok { - events = &processEvents{} - } - events.stopped = append(events.stopped, svc) - e[svc.meta.Name] = events -} - -func (li *linuxImpl) DiscoverServices() error { +func (li *linuxImpl) DiscoverServices() (*discoveredServices, error) { procs, err := li.aliveProcs() if err != nil { - return fmt.Errorf("failed to get alive processes: %w", err) + return nil, errWithCode{ + err: err, + code: errorCodeProcfs, + svc: nil, + } } - ports, err := li.openPorts() + ports, err := li.portPoller.OpenPorts() if err != nil { - return fmt.Errorf("failed to get open ports: %w", err) + return nil, errWithCode{ + err: err, + code: errorCodePortPoller, + svc: nil, + } + } + portsByPID := map[int]portlist.List{} + for _, p := range ports { + portsByPID[p.Pid] = append(portsByPID[p.Pid], p) } - log.Debugf("aliveProcs: %d | ignoreProcs: %d | runningServices: %d | potentials: %d | openPorts: %+v", - len(procs), - len(li.ignoreProcs), - len(li.aliveServices), - len(li.potentialServices), - ports, - ) - - var ( - started []serviceInfo - stopped []serviceInfo - ) + events := serviceEvents{} now := li.time.Now() @@ -136,13 +110,13 @@ func (li *linuxImpl) DiscoverServices() error { if _, ok := procs[pid]; ok { svc.LastHeartbeat = now li.aliveServices[pid] = svc - started = append(started, *svc) + events.start = append(events.start, *svc) } } clear(li.potentialServices) // check open ports - these will be potential new services if they are still alive in the next iteration. - for pid := range ports { + for pid := range portsByPID { if li.ignoreProcs[pid] { continue } @@ -155,8 +129,13 @@ func (li *linuxImpl) DiscoverServices() error { continue } - svc, err := li.getServiceInfo(p, ports) + svc, err := li.getServiceInfo(p, portsByPID) if err != nil { + telemetryFromError(errWithCode{ + err: err, + code: errorCodeProcfs, + svc: nil, + }) log.Errorf("[pid: %d] failed to get process info: %v", pid, err) li.ignoreProcs[pid] = true continue @@ -175,10 +154,10 @@ func (li *linuxImpl) DiscoverServices() error { for pid, svc := range li.aliveServices { if _, ok := procs[pid]; !ok { delete(li.aliveServices, pid) - stopped = append(stopped, *svc) + events.stop = append(events.stop, *svc) } else if now.Sub(svc.LastHeartbeat).Truncate(time.Minute) >= heartbeatTime { svc.LastHeartbeat = now - li.sender.sendHeartbeatServiceEvent(*svc) + events.heartbeat = append(events.heartbeat, *svc) } } @@ -189,38 +168,14 @@ func (li *linuxImpl) DiscoverServices() error { } } - // group started and stopped processes by name - events := make(eventsByName) - for _, p := range started { - events.addStarted(p) - } - for _, p := range stopped { - events.addStopped(p) - } - - potentialNames := map[string]bool{} - for _, p := range li.potentialServices { - potentialNames[p.meta.Name] = true - } - - for name, ev := range events { - if len(ev.started) > 0 && len(ev.stopped) > 0 { - log.Warnf("found multiple started/stopped processes with the same name, ignoring end-service events (name: %q)", name) - clear(ev.stopped) - } - for _, svc := range ev.started { - li.sender.sendStartServiceEvent(svc) - } - for _, svc := range ev.stopped { - if potentialNames[name] { - log.Debugf("there is a potential service with the same name as a stopped one, skipping end-service event (name: %q)", name) - break - } - li.sender.sendEndServiceEvent(svc) - } - } - - return nil + return &discoveredServices{ + aliveProcsCount: len(procs), + openPorts: ports, + ignoreProcs: li.ignoreProcs, + potentials: li.potentialServices, + runningServices: li.aliveServices, + events: events, + }, nil } func (li *linuxImpl) aliveProcs() (map[int]proc, error) { @@ -235,18 +190,6 @@ func (li *linuxImpl) aliveProcs() (map[int]proc, error) { return procMap, nil } -func (li *linuxImpl) openPorts() (map[int]portlist.List, error) { - ports, err := li.portPoller.OpenPorts() - if err != nil { - return nil, err - } - portMap := map[int]portlist.List{} - for _, p := range ports { - portMap[p.Pid] = append(portMap[p.Pid], p) - } - return portMap, nil -} - func (li *linuxImpl) getServiceInfo(p proc, openPorts map[int]portlist.List) (*serviceInfo, error) { cmdline, err := p.CmdLine() if err != nil { diff --git a/pkg/collector/corechecks/servicediscovery/impl_linux_mock.go b/pkg/collector/corechecks/servicediscovery/impl_linux_mock.go index 7f94c1a319229..af9fd777ee97c 100644 --- a/pkg/collector/corechecks/servicediscovery/impl_linux_mock.go +++ b/pkg/collector/corechecks/servicediscovery/impl_linux_mock.go @@ -7,11 +7,6 @@ // Code generated by MockGen. DO NOT EDIT. // Source: impl_linux.go -// -// Generated by this command: -// -// mockgen -source=impl_linux.go -package=servicediscovery -destination=impl_linux_mock.go -// // Package servicediscovery is a generated GoMock package. package servicediscovery diff --git a/pkg/collector/corechecks/servicediscovery/impl_linux_test.go b/pkg/collector/corechecks/servicediscovery/impl_linux_test.go index 19912a70dbfe3..98b01dde9ad40 100644 --- a/pkg/collector/corechecks/servicediscovery/impl_linux_test.go +++ b/pkg/collector/corechecks/servicediscovery/impl_linux_test.go @@ -9,6 +9,7 @@ package servicediscovery import ( "cmp" + "errors" "testing" "time" @@ -283,8 +284,6 @@ func Test_linuxImpl(t *testing.T) { }, }, { - // TODO: ideally we would like to emit some sort of telemetry for this case. - // For now, we just test we send the correct events to EvP. name: "repeated_service_name", checkRun: []*checkRun{ { @@ -560,8 +559,8 @@ func Test_linuxImpl(t *testing.T) { check.os.(*linuxImpl).procfs = mProcFS check.os.(*linuxImpl).portPoller = mPortPoller check.os.(*linuxImpl).time = mTimer - check.os.(*linuxImpl).sender.hostname = mHostname check.os.(*linuxImpl).bootTime = bootTimeSeconds + check.sender.hostname = mHostname err = check.Run() require.NoError(t, err) @@ -578,3 +577,61 @@ func Test_linuxImpl(t *testing.T) { }) } } + +type errorProcFS struct{} + +func (errorProcFS) AllProcs() ([]proc, error) { + return nil, errors.New("procFS failure") +} + +type emptyProcFS struct{} + +func (emptyProcFS) AllProcs() ([]proc, error) { + return []proc{}, nil +} + +type errorPortPoller struct{} + +func (errorPortPoller) OpenPorts() (portlist.List, error) { + return nil, errors.New("portPoller failure") +} + +func Test_linuxImpl_errors(t *testing.T) { + // bad procFS + { + li := linuxImpl{ + procfs: errorProcFS{}, + } + ds, err := li.DiscoverServices() + if ds != nil { + t.Error("expected nil discovery service") + } + var expected errWithCode + if errors.As(err, &expected) { + if expected.Code() != errorCodeProcfs { + t.Errorf("expected error code procfs: %#v", expected) + } + } else { + t.Error("expected errWithCode, got", err) + } + } + // bad portPoller + { + li := linuxImpl{ + procfs: emptyProcFS{}, + portPoller: errorPortPoller{}, + } + ds, err := li.DiscoverServices() + if ds != nil { + t.Error("expected nil discovery service") + } + var expected errWithCode + if errors.As(err, &expected) { + if expected.Code() != errorCodePortPoller { + t.Errorf("expected error code portPoller: %#v", expected) + } + } else { + t.Error("expected errWithCode, got", err) + } + } +} diff --git a/pkg/collector/corechecks/servicediscovery/portlist/port.go b/pkg/collector/corechecks/servicediscovery/portlist/port.go index a5af1dec74546..3567962826afa 100644 --- a/pkg/collector/corechecks/servicediscovery/portlist/port.go +++ b/pkg/collector/corechecks/servicediscovery/portlist/port.go @@ -39,12 +39,11 @@ func (a *Port) lessThan(b *Port) bool { type List []Port func (pl List) String() string { - var sb strings.Builder - for _, v := range pl { - fmt.Fprintf(&sb, "%-3s %5d %#v\n", - v.Proto, v.Port, v.Process) + out := make([]string, len(pl)) + for i, v := range pl { + out[i] = fmt.Sprintf("[%s]%s:%d", v.Process, v.Proto, v.Port) } - return strings.TrimRight(sb.String(), "\n") + return strings.Join(out, ",") } // sortAndDedup sorts ps in place (by Port.lessThan) and then returns diff --git a/pkg/collector/corechecks/servicediscovery/servicediscovery.go b/pkg/collector/corechecks/servicediscovery/servicediscovery.go index 370f0e3e5a7ad..47c68851bd7bc 100644 --- a/pkg/collector/corechecks/servicediscovery/servicediscovery.go +++ b/pkg/collector/corechecks/servicediscovery/servicediscovery.go @@ -18,18 +18,14 @@ import ( "github.com/DataDog/datadog-agent/pkg/aggregator/sender" "github.com/DataDog/datadog-agent/pkg/collector/check" "github.com/DataDog/datadog-agent/pkg/collector/corechecks" + "github.com/DataDog/datadog-agent/pkg/collector/corechecks/servicediscovery/portlist" pkgconfig "github.com/DataDog/datadog-agent/pkg/config" + "github.com/DataDog/datadog-agent/pkg/util/log" "github.com/DataDog/datadog-agent/pkg/util/optional" ) //go:generate mockgen -source=$GOFILE -package=$GOPACKAGE -destination=servicediscovery_mock.go -type osImpl interface { - DiscoverServices() error -} - -var newOSImpl func(sender *telemetrySender, ignoreCfg map[string]bool) (osImpl, error) - const ( // CheckName is the name of the check. CheckName = "service_discovery" @@ -38,20 +34,16 @@ const ( heartbeatTime = 15 * time.Minute ) -type config struct { - IgnoreProcesses []string `yaml:"ignore_processes"` -} - -type procStat struct { - StartTime uint64 -} - type serviceInfo struct { process processInfo meta serviceMetadata LastHeartbeat time.Time } +type procStat struct { + StartTime uint64 +} + type processInfo struct { PID int CmdLine []string @@ -61,6 +53,35 @@ type processInfo struct { Ports []int } +type serviceEvents struct { + start []serviceInfo + stop []serviceInfo + heartbeat []serviceInfo +} + +type discoveredServices struct { + aliveProcsCount int + openPorts portlist.List + + ignoreProcs map[int]bool + potentials map[int]*serviceInfo + runningServices map[int]*serviceInfo + + events serviceEvents +} + +type osImpl interface { + DiscoverServices() (*discoveredServices, error) +} + +var ( + newOSImpl func(ignoreCfg map[string]bool) (osImpl, error) +) + +type config struct { + IgnoreProcesses []string `yaml:"ignore_processes"` +} + // Parse parses the configuration func (c *config) Parse(data []byte) error { if err := yaml.Unmarshal(data, c); err != nil { @@ -72,8 +93,10 @@ func (c *config) Parse(data []byte) error { // Check reports discovered services. type Check struct { corechecks.CheckBase - cfg *config - os osImpl + cfg *config + os osImpl + sender *telemetrySender + sentRepeatedEventPIDs map[int]bool } // Factory creates a new check factory @@ -88,8 +111,9 @@ func Factory() optional.Option[func() check.Check] { func newCheck() check.Check { return &Check{ - CheckBase: corechecks.NewCheckBase(CheckName), - cfg: &config{}, + CheckBase: corechecks.NewCheckBase(CheckName), + cfg: &config{}, + sentRepeatedEventPIDs: make(map[int]bool), } } @@ -117,8 +141,9 @@ func (c *Check) Configure(senderManager sender.SenderManager, _ uint64, instance if err != nil { return err } + c.sender = newTelemetrySender(s) - c.os, err = newOSImpl(newTelemetrySender(s), ignoreCfg) + c.os, err = newOSImpl(ignoreCfg) if err != nil { return err } @@ -128,7 +153,124 @@ func (c *Check) Configure(senderManager sender.SenderManager, _ uint64, instance // Run executes the check. func (c *Check) Run() error { - return c.os.DiscoverServices() + start := time.Now() + defer func() { + diff := time.Since(start).Seconds() + metricTimeToScan.Observe(diff) + }() + + disc, err := c.os.DiscoverServices() + if err != nil { + telemetryFromError(err) + return err + } + + log.Debugf("aliveProcs: %d | ignoreProcs: %d | runningServices: %d | potentials: %d | openPorts: %s", + disc.aliveProcsCount, + len(disc.ignoreProcs), + len(disc.runningServices), + len(disc.potentials), + disc.openPorts.String(), + ) + metricDiscoveredServices.Set(float64(len(disc.runningServices))) + + runningServicesByName := map[string][]*serviceInfo{} + for _, svc := range disc.runningServices { + runningServicesByName[svc.meta.Name] = append(runningServicesByName[svc.meta.Name], svc) + } + for _, svcs := range runningServicesByName { + if len(svcs) <= 1 { + continue + } + for _, svc := range svcs { + if c.sentRepeatedEventPIDs[svc.process.PID] { + continue + } + err := fmt.Errorf("found repeated service name: %s", svc.meta.Name) + telemetryFromError(errWithCode{ + err: err, + code: errorCodeRepeatedServiceName, + svc: &svc.meta, + }) + // track the PID, so we don't increase this counter in every run of the check. + c.sentRepeatedEventPIDs[svc.process.PID] = true + } + } + + potentialNames := map[string]bool{} + for _, p := range disc.potentials { + potentialNames[p.meta.Name] = true + } + + // group events by name in order to find repeated events for the same service name. + eventsByName := make(eventsByNameMap) + for _, p := range disc.events.start { + eventsByName.addStart(p) + } + for _, p := range disc.events.heartbeat { + eventsByName.addHeartbeat(p) + } + for _, p := range disc.events.stop { + if potentialNames[p.meta.Name] { + // we consider this situation a restart, so we skip the stop event. + log.Debugf("there is a potential service with the same name as a stopped one, skipping end-service event (name: %q)", p.meta.Name) + continue + } + eventsByName.addStop(p) + if c.sentRepeatedEventPIDs[p.process.PID] { + // delete this process from the map, so we track it if the PID gets reused + delete(c.sentRepeatedEventPIDs, p.process.PID) + } + } + + for name, ev := range eventsByName { + if len(ev.start) > 0 && len(ev.stop) > 0 || len(ev.heartbeat) > 0 && len(ev.stop) > 0 { + // this is a consequence of the possibility of generating the same service name for different processes. + // at this point, we just skip the end-service events so at least these services don't disappear in the UI. + log.Debugf("got multiple start/heartbeat/end service events for the same service name, skipping end-service events (name: %q)", name) + clear(ev.stop) + } + for _, svc := range ev.start { + c.sender.sendStartServiceEvent(svc) + } + for _, svc := range ev.heartbeat { + c.sender.sendHeartbeatServiceEvent(svc) + } + for _, svc := range ev.stop { + c.sender.sendEndServiceEvent(svc) + } + } + + return nil +} + +type eventsByNameMap map[string]*serviceEvents + +func (m eventsByNameMap) addStart(svc serviceInfo) { + events, ok := m[svc.meta.Name] + if !ok { + events = &serviceEvents{} + } + events.start = append(events.start, svc) + m[svc.meta.Name] = events +} + +func (m eventsByNameMap) addHeartbeat(svc serviceInfo) { + events, ok := m[svc.meta.Name] + if !ok { + events = &serviceEvents{} + } + events.heartbeat = append(events.heartbeat, svc) + m[svc.meta.Name] = events +} + +func (m eventsByNameMap) addStop(svc serviceInfo) { + events, ok := m[svc.meta.Name] + if !ok { + events = &serviceEvents{} + } + events.stop = append(events.stop, svc) + m[svc.meta.Name] = events } // Interval returns how often the check should run. diff --git a/pkg/collector/corechecks/servicediscovery/servicediscovery_mock.go b/pkg/collector/corechecks/servicediscovery/servicediscovery_mock.go index e06e6dbf2664d..6591e5f3b21d1 100644 --- a/pkg/collector/corechecks/servicediscovery/servicediscovery_mock.go +++ b/pkg/collector/corechecks/servicediscovery/servicediscovery_mock.go @@ -5,11 +5,6 @@ // Code generated by MockGen. DO NOT EDIT. // Source: servicediscovery.go -// -// Generated by this command: -// -// mockgen -source=servicediscovery.go -package=servicediscovery -destination=servicediscovery_mock.go -// // Package servicediscovery is a generated GoMock package. package servicediscovery @@ -45,11 +40,12 @@ func (m *MockosImpl) EXPECT() *MockosImplMockRecorder { } // DiscoverServices mocks base method. -func (m *MockosImpl) DiscoverServices() error { +func (m *MockosImpl) DiscoverServices() (*discoveredServices, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "DiscoverServices") - ret0, _ := ret[0].(error) - return ret0 + ret0, _ := ret[0].(*discoveredServices) + ret1, _ := ret[1].(error) + return ret0, ret1 } // DiscoverServices indicates an expected call of DiscoverServices. diff --git a/pkg/collector/corechecks/servicediscovery/telemetry.go b/pkg/collector/corechecks/servicediscovery/telemetry.go new file mode 100644 index 0000000000000..4f38a5e0fc434 --- /dev/null +++ b/pkg/collector/corechecks/servicediscovery/telemetry.go @@ -0,0 +1,58 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +package servicediscovery + +import ( + "errors" + "github.com/DataDog/datadog-agent/pkg/telemetry" + "github.com/DataDog/datadog-agent/pkg/util/log" + "github.com/prometheus/client_golang/prometheus" +) + +const ( + metricTagErrorCode = "error_code" + metricTagServiceName = "service_name" + metricTagServiceLanguage = "service_language" + metricTagServiceType = "service_type" +) + +var ( + metricFailureEvents = telemetry.NewCounterWithOpts( + CheckName, + "failure_events", + []string{metricTagErrorCode, metricTagServiceName, metricTagServiceLanguage, metricTagServiceType}, + "Number of times an error or an unexpected event happened.", + telemetry.Options{NoDoubleUnderscoreSep: true}, + ) + metricDiscoveredServices = telemetry.NewGaugeWithOpts( + CheckName, + "discovered_services", + []string{}, + "Number of discovered alive services.", + telemetry.Options{NoDoubleUnderscoreSep: true}, + ) + metricTimeToScan = telemetry.NewHistogramWithOpts( + CheckName, + "time_to_scan", + []string{}, + "Time it took to scan services", + prometheus.DefBuckets, + telemetry.Options{NoDoubleUnderscoreSep: true}, + ) +) + +func telemetryFromError(err error) { + var codeErr errWithCode + if errors.As(err, &codeErr) { + log.Debugf("sending telemetry for error: %v", err) + svc := serviceMetadata{} + if codeErr.svc != nil { + svc = *codeErr.svc + } + tags := []string{string(codeErr.Code()), svc.Name, svc.Language, svc.Type} + metricFailureEvents.Inc(tags...) + } +} diff --git a/pkg/config/env/go.mod b/pkg/config/env/go.mod index 6404b987b6d3d..a4e01466aa1f8 100644 --- a/pkg/config/env/go.mod +++ b/pkg/config/env/go.mod @@ -20,7 +20,7 @@ require ( require ( github.com/DataDog/datadog-agent/pkg/util/scrubber v0.54.0-rc.2 // indirect - github.com/DataDog/viper v1.13.4 // indirect + github.com/DataDog/viper v1.13.5 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect github.com/davecgh/go-spew v1.1.1 // indirect diff --git a/pkg/config/env/go.sum b/pkg/config/env/go.sum index b8cdbec3a8655..f3f8a4c5745c9 100644 --- a/pkg/config/env/go.sum +++ b/pkg/config/env/go.sum @@ -1,8 +1,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/DataDog/viper v1.13.4 h1:0SrZc3zvMAGgVKg96uP4DEJb13lK2Is9a4go7IIcFSE= -github.com/DataDog/viper v1.13.4/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= +github.com/DataDog/viper v1.13.5 h1:SZMcyMknYQN2jRY/40A16gUXexlNJOI8sDs1cWZnI64= +github.com/DataDog/viper v1.13.5/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= diff --git a/pkg/config/fetcher/from_processes.go b/pkg/config/fetcher/from_processes.go index 02066dc726c13..59d6e3ebb7250 100644 --- a/pkg/config/fetcher/from_processes.go +++ b/pkg/config/fetcher/from_processes.go @@ -95,3 +95,11 @@ func SystemProbeConfig(config config.Reader) (string, error) { c := settingshttp.NewClient(hc, "http://localhost/config", "system-probe", settingshttp.NewHTTPClientOptions(util.CloseConnection)) return c.FullConfig() } + +// SystemProbeConfigBySource fetch the all configuration layers from the system-probe process by querying its API +func SystemProbeConfigBySource(config config.Reader) (string, error) { + hc := client.Get(config.GetString("system_probe_config.sysprobe_socket")) + + c := settingshttp.NewClient(hc, "http://localhost/config", "system-probe", settingshttp.NewHTTPClientOptions(util.CloseConnection)) + return c.FullConfigBySource() +} diff --git a/pkg/config/logs/go.mod b/pkg/config/logs/go.mod index 9a599a76e19cd..36b748233c48c 100644 --- a/pkg/config/logs/go.mod +++ b/pkg/config/logs/go.mod @@ -17,7 +17,7 @@ require ( ) require ( - github.com/DataDog/viper v1.13.4 // indirect + github.com/DataDog/viper v1.13.5 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/google/go-cmp v0.6.0 // indirect diff --git a/pkg/config/logs/go.sum b/pkg/config/logs/go.sum index ffa38e4c29cd9..584c372a509b5 100644 --- a/pkg/config/logs/go.sum +++ b/pkg/config/logs/go.sum @@ -1,8 +1,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/DataDog/viper v1.13.4 h1:0SrZc3zvMAGgVKg96uP4DEJb13lK2Is9a4go7IIcFSE= -github.com/DataDog/viper v1.13.4/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= +github.com/DataDog/viper v1.13.5 h1:SZMcyMknYQN2jRY/40A16gUXexlNJOI8sDs1cWZnI64= +github.com/DataDog/viper v1.13.5/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= diff --git a/pkg/config/model/go.mod b/pkg/config/model/go.mod index 726cea6783915..47845fbec2e8b 100644 --- a/pkg/config/model/go.mod +++ b/pkg/config/model/go.mod @@ -10,7 +10,7 @@ replace ( require ( github.com/DataDog/datadog-agent/pkg/util/log v0.54.0-rc.2 - github.com/DataDog/viper v1.13.4 + github.com/DataDog/viper v1.13.5 github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 github.com/spf13/afero v1.1.2 github.com/spf13/pflag v1.0.3 diff --git a/pkg/config/model/go.sum b/pkg/config/model/go.sum index 9b3ed3de9f0a9..c9284d1b2c6c0 100644 --- a/pkg/config/model/go.sum +++ b/pkg/config/model/go.sum @@ -1,8 +1,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/DataDog/viper v1.13.4 h1:0SrZc3zvMAGgVKg96uP4DEJb13lK2Is9a4go7IIcFSE= -github.com/DataDog/viper v1.13.4/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= +github.com/DataDog/viper v1.13.5 h1:SZMcyMknYQN2jRY/40A16gUXexlNJOI8sDs1cWZnI64= +github.com/DataDog/viper v1.13.5/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= diff --git a/pkg/config/model/viper.go b/pkg/config/model/viper.go index 6f3d4942fcc8e..acb5b004f48ef 100644 --- a/pkg/config/model/viper.go +++ b/pkg/config/model/viper.go @@ -29,14 +29,28 @@ type Source string // Declare every known Source const ( - SourceDefault Source = "default" - SourceUnknown Source = "unknown" - SourceFile Source = "file" - SourceEnvVar Source = "environment-variable" - SourceAgentRuntime Source = "agent-runtime" + // SourceDefault are the values from defaults. + SourceDefault Source = "default" + // SourceUnknown are the values from unknown source. This should only be used in tests when calling + // SetWithoutSource. + SourceUnknown Source = "unknown" + // SourceFile are the values loaded from configuration file. + SourceFile Source = "file" + // SourceEnvVar are the values loaded from the environment variables. + SourceEnvVar Source = "environment-variable" + // SourceAgentRuntime are the values configured by the agent itself. The agent can dynamically compute the best + // value for some settings when not set by the user. + SourceAgentRuntime Source = "agent-runtime" + // SourceLocalConfigProcess are the values mirrored from the config process. The config process is the + // core-agent. This is used when side process like security-agent or trace-agent pull their configuration from + // the core-agent. SourceLocalConfigProcess Source = "local-config-process" - SourceRC Source = "remote-config" - SourceCLI Source = "cli" + // SourceRC are the values loaded from remote-config (aka Datadog backend) + SourceRC Source = "remote-config" + // SourceCLI are the values set by the user at runtime through the CLI. + SourceCLI Source = "cli" + // SourceProvided are all values set by any source but default. + SourceProvided Source = "provided" // everything but defaults ) // sources list the known sources, following the order of hierarchy between them @@ -644,6 +658,7 @@ func (c *safeConfig) AllSettingsBySource() map[Source]interface{} { for _, source := range sources { res[source] = c.configSources[source].AllSettingsWithoutDefault() } + res[SourceProvided] = c.AllSettingsWithoutDefault() return res } diff --git a/pkg/config/remote/go.mod b/pkg/config/remote/go.mod index 02a1d0a68066c..50c71a0af2f4c 100644 --- a/pkg/config/remote/go.mod +++ b/pkg/config/remote/go.mod @@ -58,7 +58,7 @@ require ( require ( github.com/DataDog/datadog-agent/pkg/util/scrubber v0.54.0-rc.2 // indirect github.com/DataDog/go-tuf v1.1.0-0.5.2 - github.com/DataDog/viper v1.13.4 // indirect + github.com/DataDog/viper v1.13.5 // indirect github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/fsnotify/fsnotify v1.4.7 // indirect diff --git a/pkg/config/remote/go.sum b/pkg/config/remote/go.sum index 478ea46481e13..fe4cda0b6db9a 100644 --- a/pkg/config/remote/go.sum +++ b/pkg/config/remote/go.sum @@ -46,8 +46,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= -github.com/DataDog/viper v1.13.4 h1:0SrZc3zvMAGgVKg96uP4DEJb13lK2Is9a4go7IIcFSE= -github.com/DataDog/viper v1.13.4/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= +github.com/DataDog/viper v1.13.5 h1:SZMcyMknYQN2jRY/40A16gUXexlNJOI8sDs1cWZnI64= +github.com/DataDog/viper v1.13.5/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= diff --git a/pkg/config/settings/api.go b/pkg/config/settings/api.go index 650fa8e319b62..c9e3fddf9067e 100644 --- a/pkg/config/settings/api.go +++ b/pkg/config/settings/api.go @@ -17,6 +17,7 @@ type Client interface { Set(key string, value string) (bool, error) List() (map[string]settings.RuntimeSettingResponse, error) FullConfig() (string, error) + FullConfigBySource() (string, error) } // ClientBuilder represents a function returning a runtime settings API client diff --git a/pkg/config/settings/http/client.go b/pkg/config/settings/http/client.go index 210c25b5d7878..748ad5834fbd4 100644 --- a/pkg/config/settings/http/client.go +++ b/pkg/config/settings/http/client.go @@ -30,35 +30,46 @@ func NewClient(c *http.Client, baseURL string, targetProcessName string, clientO return &runtimeSettingsHTTPClient{c, baseURL, targetProcessName, clientOptions} } -func (rc *runtimeSettingsHTTPClient) FullConfig() (string, error) { - r, err := util.DoGet(rc.c, rc.baseURL, rc.clientOptions.CloseConnection) +func (rc *runtimeSettingsHTTPClient) doGet(url string, formatError bool) (string, error) { + r, err := util.DoGet(rc.c, url, rc.clientOptions.CloseConnection) if err != nil { - var errMap = make(map[string]string) + errMap := make(map[string]string) _ = json.Unmarshal(r, &errMap) // If the error has been marshalled into a json object, check it and return it properly if e, found := errMap["error"]; found { return "", fmt.Errorf(e) } + if formatError { + return "", fmt.Errorf("Could not reach %s: %v \nMake sure the %s is running before requesting the runtime configuration and contact support if you continue having issues", rc.targetProcessName, err, rc.targetProcessName) + } + return "", err + } + return string(r), nil +} - return "", fmt.Errorf("Could not reach %s: %v \nMake sure the %s is running before requesting the runtime configuration and contact support if you continue having issues", rc.targetProcessName, err, rc.targetProcessName) +func (rc *runtimeSettingsHTTPClient) FullConfig() (string, error) { + r, err := rc.doGet(rc.baseURL, true) + if err != nil { + return "", err } + return string(r), nil +} +func (rc *runtimeSettingsHTTPClient) FullConfigBySource() (string, error) { + r, err := rc.doGet(fmt.Sprintf("%s/by-source", rc.baseURL), true) + if err != nil { + return "", err + } return string(r), nil } func (rc *runtimeSettingsHTTPClient) List() (map[string]settingsComponent.RuntimeSettingResponse, error) { - r, err := util.DoGet(rc.c, fmt.Sprintf("%s/%s", rc.baseURL, "list-runtime"), rc.clientOptions.CloseConnection) + r, err := rc.doGet(fmt.Sprintf("%s/list-runtime", rc.baseURL), false) if err != nil { - var errMap = make(map[string]string) - _ = json.Unmarshal(r, &errMap) - // If the error has been marshalled into a json object, check it and return it properly - if e, found := errMap["error"]; found { - return nil, fmt.Errorf(e) - } return nil, err } - var settingsList = make(map[string]settingsComponent.RuntimeSettingResponse) - err = json.Unmarshal(r, &settingsList) + settingsList := make(map[string]settingsComponent.RuntimeSettingResponse) + err = json.Unmarshal([]byte(r), &settingsList) if err != nil { return nil, err } @@ -67,19 +78,13 @@ func (rc *runtimeSettingsHTTPClient) List() (map[string]settingsComponent.Runtim } func (rc *runtimeSettingsHTTPClient) Get(key string) (interface{}, error) { - r, err := util.DoGet(rc.c, fmt.Sprintf("%s/%s", rc.baseURL, key), rc.clientOptions.CloseConnection) + r, err := rc.doGet(fmt.Sprintf("%s/%s", rc.baseURL, key), false) if err != nil { - var errMap = make(map[string]string) - _ = json.Unmarshal(r, &errMap) - // If the error has been marshalled into a json object, check it and return it properly - if e, found := errMap["error"]; found { - return nil, fmt.Errorf(e) - } return nil, err } - var setting = make(map[string]interface{}) - err = json.Unmarshal(r, &setting) + setting := make(map[string]interface{}) + err = json.Unmarshal([]byte(r), &setting) if err != nil { return nil, err } @@ -90,19 +95,13 @@ func (rc *runtimeSettingsHTTPClient) Get(key string) (interface{}, error) { } func (rc *runtimeSettingsHTTPClient) GetWithSources(key string) (map[string]interface{}, error) { - r, err := util.DoGet(rc.c, fmt.Sprintf("%s/%s?sources=true", rc.baseURL, key), rc.clientOptions.CloseConnection) + r, err := rc.doGet(fmt.Sprintf("%s/%s?sources=true", rc.baseURL, key), false) if err != nil { - var errMap = make(map[string]string) - _ = json.Unmarshal(r, &errMap) - // If the error has been marshalled into a json object, check it and return it properly - if e, found := errMap["error"]; found { - return nil, fmt.Errorf("%s", e) - } return nil, err } - var setting = make(map[string]interface{}) - err = json.Unmarshal(r, &setting) + setting := make(map[string]interface{}) + err = json.Unmarshal([]byte(r), &setting) if err != nil { return nil, err } @@ -127,7 +126,7 @@ func (rc *runtimeSettingsHTTPClient) Set(key string, value string) (bool, error) body := fmt.Sprintf("value=%s", html.EscapeString(value)) r, err := util.DoPost(rc.c, fmt.Sprintf("%s/%s", rc.baseURL, key), "application/x-www-form-urlencoded", bytes.NewBuffer([]byte(body))) if err != nil { - var errMap = make(map[string]string) + errMap := make(map[string]string) _ = json.Unmarshal(r, &errMap) // If the error has been marshalled into a json object, check it and return it properly if e, found := errMap["error"]; found { diff --git a/pkg/config/setup/config.go b/pkg/config/setup/config.go index bf3810d124dd5..3b4d6dbd52788 100644 --- a/pkg/config/setup/config.go +++ b/pkg/config/setup/config.go @@ -470,6 +470,8 @@ func InitConfig(config pkgconfigmodel.Config) { config.BindEnvAndSetDefault("cluster_agent.language_detection.cleanup.language_ttl", "30m") // language annotation cleanup period config.BindEnvAndSetDefault("cluster_agent.language_detection.cleanup.period", "10m") + config.BindEnvAndSetDefault("cluster_agent.kube_metadata_collection.enabled", false) + config.BindEnvAndSetDefault("cluster_agent.kube_metadata_collection.resources", []string{}) // Metadata endpoints @@ -908,6 +910,9 @@ func InitConfig(config pkgconfigmodel.Config) { config.BindEnvAndSetDefault("remote_updates", false) config.BindEnvAndSetDefault("installer.registry.url", "") config.BindEnvAndSetDefault("installer.registry.auth", "") + + // Data Jobs Monitoring config + config.BindEnvAndSetDefault("djm_config.enabled", false) } func agent(config pkgconfigmodel.Config) { diff --git a/pkg/config/setup/go.mod b/pkg/config/setup/go.mod index 53192d3d543b2..5f7c72b2ffadc 100644 --- a/pkg/config/setup/go.mod +++ b/pkg/config/setup/go.mod @@ -55,7 +55,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/pointer v0.54.0-rc.2 // indirect github.com/DataDog/datadog-agent/pkg/util/scrubber v0.54.0-rc.2 // indirect github.com/DataDog/datadog-agent/pkg/util/system/socket v0.54.0-rc.2 // indirect - github.com/DataDog/viper v1.13.4 // indirect + github.com/DataDog/viper v1.13.5 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect diff --git a/pkg/config/setup/go.sum b/pkg/config/setup/go.sum index ce57d84e6feab..ca38c4245d1f0 100644 --- a/pkg/config/setup/go.sum +++ b/pkg/config/setup/go.sum @@ -3,8 +3,8 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DataDog/cast v1.3.1-0.20190301154711-1ee8c8bd14a3 h1:SobA9WYm4K/MUtWlbKaomWTmnuYp1KhIm8Wlx3vmpsg= github.com/DataDog/cast v1.3.1-0.20190301154711-1ee8c8bd14a3/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/DataDog/viper v1.13.4 h1:0SrZc3zvMAGgVKg96uP4DEJb13lK2Is9a4go7IIcFSE= -github.com/DataDog/viper v1.13.4/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= +github.com/DataDog/viper v1.13.5 h1:SZMcyMknYQN2jRY/40A16gUXexlNJOI8sDs1cWZnI64= +github.com/DataDog/viper v1.13.5/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= diff --git a/pkg/config/utils/go.mod b/pkg/config/utils/go.mod index 8c5709d8f82d4..a39cbce46793e 100644 --- a/pkg/config/utils/go.mod +++ b/pkg/config/utils/go.mod @@ -49,7 +49,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/system v0.54.0-rc.2 // indirect github.com/DataDog/datadog-agent/pkg/util/system/socket v0.54.0-rc.2 // indirect github.com/DataDog/datadog-agent/pkg/util/winutil v0.54.0-rc.2 // indirect - github.com/DataDog/viper v1.13.4 // indirect + github.com/DataDog/viper v1.13.5 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect github.com/davecgh/go-spew v1.1.1 // indirect diff --git a/pkg/config/utils/go.sum b/pkg/config/utils/go.sum index c1503b41b0ee8..6530b0681514a 100644 --- a/pkg/config/utils/go.sum +++ b/pkg/config/utils/go.sum @@ -1,8 +1,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/DataDog/viper v1.13.4 h1:0SrZc3zvMAGgVKg96uP4DEJb13lK2Is9a4go7IIcFSE= -github.com/DataDog/viper v1.13.4/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= +github.com/DataDog/viper v1.13.5 h1:SZMcyMknYQN2jRY/40A16gUXexlNJOI8sDs1cWZnI64= +github.com/DataDog/viper v1.13.5/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= diff --git a/pkg/diagnose/runner.go b/pkg/diagnose/runner.go index 88e0d094b0b51..55da898412994 100644 --- a/pkg/diagnose/runner.go +++ b/pkg/diagnose/runner.go @@ -568,8 +568,8 @@ func RegisterConnectivityDatadogEventPlatform(catalog *diagnosis.Catalog) { // RegisterPortConflict registers the port-conflict diagnose suite. func RegisterPortConflict(catalog *diagnosis.Catalog) { - // port-conflict suite available in darwin only for now - if runtime.GOOS == "darwin" { + // port-conflict suite available in darwin and linux only for now + if runtime.GOOS == "darwin" || runtime.GOOS == "linux" { catalog.Register("port-conflict", func() []diagnosis.Diagnosis { return ports.DiagnosePortSuite() }) } } diff --git a/pkg/ebpf/c/bpf_metadata.h b/pkg/ebpf/c/bpf_metadata.h new file mode 100644 index 0000000000000..ad7162404feca --- /dev/null +++ b/pkg/ebpf/c/bpf_metadata.h @@ -0,0 +1,19 @@ +/** + * This header file is included in kconfig.h and ktypes.h, which should make + * this metadata available to all object files. + */ + +#ifndef __BPF_METADATA_H__ +#define __BPF_METADATA_H__ + +#ifndef __CGO__ // Do not include metadata in CGo builds, as it causes duplicate symbols +#if defined(__x86_64__) || defined(__TARGET_ARCH_x86) +char __dd_metadata_arch[] __attribute__((section("dd_metadata"), used)) = ""; +#elif defined(__aarch64__) || defined(__TARGET_ARCH_arm64) +char __dd_metadata_arch[] __attribute__((section("dd_metadata"), used)) = ""; +#else +char __dd_metadata_arch[] __attribute__((section("dd_metadata"), used)) = ""; +#endif +#endif + +#endif diff --git a/pkg/ebpf/c/kconfig.h b/pkg/ebpf/c/kconfig.h index c93f500eb7d09..a5a186d940453 100644 --- a/pkg/ebpf/c/kconfig.h +++ b/pkg/ebpf/c/kconfig.h @@ -1,6 +1,8 @@ #ifndef __KCONFIG_H #define __KCONFIG_H +#include "bpf_metadata.h" + #include // include asm/compiler.h to fix `error: expected string literal in 'asm'` compilation error coming from mte-kasan.h // this was fixed in https://github.com/torvalds/linux/commit/b859ebedd1e730bbda69142fca87af4e712649a1 diff --git a/pkg/ebpf/c/ktypes.h b/pkg/ebpf/c/ktypes.h index de243a94a5a9a..0daf39816bc36 100644 --- a/pkg/ebpf/c/ktypes.h +++ b/pkg/ebpf/c/ktypes.h @@ -1,6 +1,8 @@ #ifndef __KTYPES_H__ #define __KTYPES_H__ +#include "bpf_metadata.h" + #ifdef COMPILE_CORE #include "vmlinux.h" #else diff --git a/pkg/fleet/installer/service/datadog_agent.go b/pkg/fleet/installer/service/datadog_agent.go index ec65181b63467..4b49d2d8aa003 100644 --- a/pkg/fleet/installer/service/datadog_agent.go +++ b/pkg/fleet/installer/service/datadog_agent.go @@ -21,6 +21,7 @@ import ( ) const ( + pathOldAgent = "/opt/datadog-agent" agentSymlink = "/usr/bin/datadog-agent" agentUnit = "datadog-agent.service" traceAgentUnit = "datadog-agent-trace.service" @@ -62,18 +63,22 @@ func SetupAgent(ctx context.Context, _ []string) (err error) { span.Finish(tracer.WithError(err)) }() + if err = stopOldAgentUnits(ctx); err != nil { + return err + } + for _, unit := range stableUnits { if err = loadUnit(ctx, unit); err != nil { - return err + return fmt.Errorf("failed to load %s: %v", unit, err) } } for _, unit := range experimentalUnits { if err = loadUnit(ctx, unit); err != nil { - return err + return fmt.Errorf("failed to load %s: %v", unit, err) } } if err = os.MkdirAll("/etc/datadog-agent", 0755); err != nil { - return err + return fmt.Errorf("failed to create /etc/datadog-agent: %v", err) } ddAgentUID, ddAgentGID, err := getAgentIDs() if err != nil { @@ -81,31 +86,30 @@ func SetupAgent(ctx context.Context, _ []string) (err error) { } if err = os.Chown("/etc/datadog-agent", ddAgentUID, ddAgentGID); err != nil { - return err + return fmt.Errorf("failed to chown /etc/datadog-agent: %v", err) } if err = systemdReload(ctx); err != nil { - return err + return fmt.Errorf("failed to reload systemd daemon: %v", err) } - for _, unit := range stableUnits { - if err = enableUnit(ctx, unit); err != nil { - return err - } + // enabling the agentUnit only is enough as others are triggered by it + if err = enableUnit(ctx, agentUnit); err != nil { + return fmt.Errorf("failed to enable %s: %v", agentUnit, err) } if err = exec.CommandContext(ctx, "ln", "-sf", "/opt/datadog-packages/datadog-agent/stable/bin/agent/agent", agentSymlink).Run(); err != nil { - return err + return fmt.Errorf("failed to create symlink: %v", err) } // write installinfo before start, or the agent could write it // TODO: add installer version properly if err = installinfo.WriteInstallInfo("installer_package", "manual_update"); err != nil { - return err + return fmt.Errorf("failed to write install info: %v", err) } _, err = os.Stat("/etc/datadog-agent/datadog.yaml") if err != nil && !os.IsNotExist(err) { - return err + return fmt.Errorf("failed to check if /etc/datadog-agent/datadog.yaml exists: %v", err) } // this is expected during a fresh install with the install script / asible / chef / etc... // the config is populated afterwards by the install method and the agent is restarted @@ -133,20 +137,18 @@ func RemoveAgent(ctx context.Context) error { log.Warnf("Failed to stop %s: %s", unit, err) } } - // purge experimental units + + if err := disableUnit(ctx, agentUnit); err != nil { + log.Warnf("Failed to disable %s: %s", agentUnit, err) + } + + // remove units from disk for _, unit := range experimentalUnits { - if err := disableUnit(ctx, unit); err != nil { - log.Warnf("Failed to disable %s: %s", unit, err) - } if err := removeUnit(ctx, unit); err != nil { log.Warnf("Failed to remove %s: %s", unit, err) } } - // purge stable units for _, unit := range stableUnits { - if err := disableUnit(ctx, unit); err != nil { - log.Warnf("Failed to disable %s: %s", unit, err) - } if err := removeUnit(ctx, unit); err != nil { log.Warnf("Failed to remove %s: %s", unit, err) } @@ -159,6 +161,33 @@ func RemoveAgent(ctx context.Context) error { return nil } +func oldAgentInstalled() bool { + _, err := os.Stat(pathOldAgent) + return err == nil +} + +func stopOldAgentUnits(ctx context.Context) error { + if !oldAgentInstalled() { + return nil + } + span, ctx := tracer.StartSpanFromContext(ctx, "remove_old_agent_units") + defer span.Finish() + for _, unit := range stableUnits { + if err := stopUnit(ctx, unit); err != nil { + exitError, ok := err.(*exec.ExitError) + if ok && exitError.ExitCode() == 5 { + // exit code 5 means the unit is not loaded, we can continue + continue + } + return fmt.Errorf("failed to stop %s: %v", unit, err) + } + if err := disableUnit(ctx, unit); err != nil { + return fmt.Errorf("failed to disable %s: %v", unit, err) + } + } + return nil +} + // StartAgentExperiment starts the agent experiment func StartAgentExperiment(ctx context.Context) error { return startUnit(ctx, agentExp, "--no-block") diff --git a/pkg/fleet/installer/service/datadog_installer.go b/pkg/fleet/installer/service/datadog_installer.go index 929b7c4d067b1..b40a8866acc23 100644 --- a/pkg/fleet/installer/service/datadog_installer.go +++ b/pkg/fleet/installer/service/datadog_installer.go @@ -124,6 +124,11 @@ func SetupInstaller(ctx context.Context) (err error) { return nil } + err = os.MkdirAll(systemdPath, 0755) + if err != nil { + return fmt.Errorf("error creating %s: %w", systemdPath, err) + } + for _, unit := range installerUnits { if err = loadUnit(ctx, unit); err != nil { return err diff --git a/pkg/fleet/installer/service/systemd.go b/pkg/fleet/installer/service/systemd.go index b1f52449b5c47..65ae8dcba914c 100644 --- a/pkg/fleet/installer/service/systemd.go +++ b/pkg/fleet/installer/service/systemd.go @@ -19,24 +19,7 @@ import ( "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" ) -var ( - systemdPath = findSystemdPath() -) - -const ( - debSystemdPath = "/lib/systemd/system" // todo load it at build time from omnibus - rpmSystemdPath = "/usr/lib/systemd/system" -) - -// findSystemdPath todo: this is a hacky way to detect on which os family we are currently -// running and finding the correct systemd path. -// We should probably provide the correct path when we build the package -func findSystemdPath() (systemdPath string) { - if _, err := os.Stat(rpmSystemdPath); err == nil { - return rpmSystemdPath - } - return debSystemdPath -} +const systemdPath = "/etc/systemd/system" func stopUnit(ctx context.Context, unit string, args ...string) error { span, _ := tracer.StartSpanFromContext(ctx, "stop_unit") diff --git a/pkg/logs/auditor/go.mod b/pkg/logs/auditor/go.mod index 9d0b228c223a2..1c87cb9ab046a 100644 --- a/pkg/logs/auditor/go.mod +++ b/pkg/logs/auditor/go.mod @@ -64,7 +64,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/system/socket v0.54.0-rc.2 // indirect github.com/DataDog/datadog-agent/pkg/util/winutil v0.54.0-rc.2 // indirect github.com/DataDog/datadog-agent/pkg/version v0.54.0-rc.2 // indirect - github.com/DataDog/viper v1.13.4 // indirect + github.com/DataDog/viper v1.13.5 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect github.com/davecgh/go-spew v1.1.1 // indirect diff --git a/pkg/logs/auditor/go.sum b/pkg/logs/auditor/go.sum index c1503b41b0ee8..6530b0681514a 100644 --- a/pkg/logs/auditor/go.sum +++ b/pkg/logs/auditor/go.sum @@ -1,8 +1,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/DataDog/viper v1.13.4 h1:0SrZc3zvMAGgVKg96uP4DEJb13lK2Is9a4go7IIcFSE= -github.com/DataDog/viper v1.13.4/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= +github.com/DataDog/viper v1.13.5 h1:SZMcyMknYQN2jRY/40A16gUXexlNJOI8sDs1cWZnI64= +github.com/DataDog/viper v1.13.5/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= diff --git a/pkg/logs/client/go.mod b/pkg/logs/client/go.mod index c319836638833..d022834d48c23 100644 --- a/pkg/logs/client/go.mod +++ b/pkg/logs/client/go.mod @@ -79,7 +79,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/system v0.54.0-rc.2 // indirect github.com/DataDog/datadog-agent/pkg/util/system/socket v0.54.0-rc.2 // indirect github.com/DataDog/datadog-agent/pkg/util/winutil v0.54.0-rc.2 // indirect - github.com/DataDog/viper v1.13.4 // indirect + github.com/DataDog/viper v1.13.5 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect diff --git a/pkg/logs/client/go.sum b/pkg/logs/client/go.sum index 45d4b9308d579..8d8b896046108 100644 --- a/pkg/logs/client/go.sum +++ b/pkg/logs/client/go.sum @@ -39,8 +39,8 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/DataDog/viper v1.13.4 h1:0SrZc3zvMAGgVKg96uP4DEJb13lK2Is9a4go7IIcFSE= -github.com/DataDog/viper v1.13.4/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= +github.com/DataDog/viper v1.13.5 h1:SZMcyMknYQN2jRY/40A16gUXexlNJOI8sDs1cWZnI64= +github.com/DataDog/viper v1.13.5/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= diff --git a/pkg/logs/diagnostic/go.mod b/pkg/logs/diagnostic/go.mod index 4c25bf55335c3..2f75d08579d75 100644 --- a/pkg/logs/diagnostic/go.mod +++ b/pkg/logs/diagnostic/go.mod @@ -68,7 +68,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/system/socket v0.54.0-rc.2 // indirect github.com/DataDog/datadog-agent/pkg/util/winutil v0.54.0-rc.2 // indirect github.com/DataDog/datadog-agent/pkg/version v0.54.0-rc.2 // indirect - github.com/DataDog/viper v1.13.4 // indirect + github.com/DataDog/viper v1.13.5 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect github.com/davecgh/go-spew v1.1.1 // indirect diff --git a/pkg/logs/diagnostic/go.sum b/pkg/logs/diagnostic/go.sum index 4f5285c7fa353..b159185f44664 100644 --- a/pkg/logs/diagnostic/go.sum +++ b/pkg/logs/diagnostic/go.sum @@ -1,8 +1,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/DataDog/viper v1.13.4 h1:0SrZc3zvMAGgVKg96uP4DEJb13lK2Is9a4go7IIcFSE= -github.com/DataDog/viper v1.13.4/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= +github.com/DataDog/viper v1.13.5 h1:SZMcyMknYQN2jRY/40A16gUXexlNJOI8sDs1cWZnI64= +github.com/DataDog/viper v1.13.5/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= diff --git a/pkg/logs/message/go.mod b/pkg/logs/message/go.mod index a310a526504d5..77b62428d065c 100644 --- a/pkg/logs/message/go.mod +++ b/pkg/logs/message/go.mod @@ -60,7 +60,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/system/socket v0.54.0-rc.2 // indirect github.com/DataDog/datadog-agent/pkg/util/winutil v0.54.0-rc.2 // indirect github.com/DataDog/datadog-agent/pkg/version v0.54.0-rc.2 // indirect - github.com/DataDog/viper v1.13.4 // indirect + github.com/DataDog/viper v1.13.5 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect github.com/davecgh/go-spew v1.1.1 // indirect diff --git a/pkg/logs/message/go.sum b/pkg/logs/message/go.sum index c1503b41b0ee8..6530b0681514a 100644 --- a/pkg/logs/message/go.sum +++ b/pkg/logs/message/go.sum @@ -1,8 +1,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/DataDog/viper v1.13.4 h1:0SrZc3zvMAGgVKg96uP4DEJb13lK2Is9a4go7IIcFSE= -github.com/DataDog/viper v1.13.4/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= +github.com/DataDog/viper v1.13.5 h1:SZMcyMknYQN2jRY/40A16gUXexlNJOI8sDs1cWZnI64= +github.com/DataDog/viper v1.13.5/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= diff --git a/pkg/logs/pipeline/go.mod b/pkg/logs/pipeline/go.mod index be8526f758453..67fa037fe935c 100644 --- a/pkg/logs/pipeline/go.mod +++ b/pkg/logs/pipeline/go.mod @@ -98,7 +98,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/winutil v0.54.0-rc.2 // indirect github.com/DataDog/datadog-agent/pkg/version v0.54.0-rc.2 // indirect github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240419161837-f1b2f553edfe // indirect - github.com/DataDog/viper v1.13.4 // indirect + github.com/DataDog/viper v1.13.5 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/benbjohnson/clock v1.3.5 // indirect github.com/beorn7/perks v1.0.1 // indirect diff --git a/pkg/logs/pipeline/go.sum b/pkg/logs/pipeline/go.sum index 1624aff2bdd5e..01b356fe5381b 100644 --- a/pkg/logs/pipeline/go.sum +++ b/pkg/logs/pipeline/go.sum @@ -43,8 +43,8 @@ github.com/DataDog/agent-payload/v5 v5.0.106 h1:A3dGX+JYoL7OJe2crpxznW7hWxLxhOk/ github.com/DataDog/agent-payload/v5 v5.0.106/go.mod h1:COngtbYYCncpIPiE5D93QlXDH/3VAKk10jDNwGHcMRE= github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240419161837-f1b2f553edfe h1:efzxujZ7VHWFxjmWjcJyUEpPrN8qdiZPYb+dBw547Wo= github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240419161837-f1b2f553edfe/go.mod h1:TX7CTOQ3LbQjfAi4SwqUoR5gY1zfUk7VRBDTuArjaDc= -github.com/DataDog/viper v1.13.4 h1:0SrZc3zvMAGgVKg96uP4DEJb13lK2Is9a4go7IIcFSE= -github.com/DataDog/viper v1.13.4/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= +github.com/DataDog/viper v1.13.5 h1:SZMcyMknYQN2jRY/40A16gUXexlNJOI8sDs1cWZnI64= +github.com/DataDog/viper v1.13.5/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= diff --git a/pkg/logs/processor/go.mod b/pkg/logs/processor/go.mod index ad8043ebcb03c..5588f3c93a3f9 100644 --- a/pkg/logs/processor/go.mod +++ b/pkg/logs/processor/go.mod @@ -78,7 +78,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/winutil v0.54.0-rc.2 // indirect github.com/DataDog/datadog-agent/pkg/version v0.54.0-rc.2 // indirect github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240419161837-f1b2f553edfe // indirect - github.com/DataDog/viper v1.13.4 // indirect + github.com/DataDog/viper v1.13.5 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect diff --git a/pkg/logs/processor/go.sum b/pkg/logs/processor/go.sum index b3b61b6f10a75..0df28af1d9c5b 100644 --- a/pkg/logs/processor/go.sum +++ b/pkg/logs/processor/go.sum @@ -43,8 +43,8 @@ github.com/DataDog/agent-payload/v5 v5.0.106 h1:A3dGX+JYoL7OJe2crpxznW7hWxLxhOk/ github.com/DataDog/agent-payload/v5 v5.0.106/go.mod h1:COngtbYYCncpIPiE5D93QlXDH/3VAKk10jDNwGHcMRE= github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240419161837-f1b2f553edfe h1:efzxujZ7VHWFxjmWjcJyUEpPrN8qdiZPYb+dBw547Wo= github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240419161837-f1b2f553edfe/go.mod h1:TX7CTOQ3LbQjfAi4SwqUoR5gY1zfUk7VRBDTuArjaDc= -github.com/DataDog/viper v1.13.4 h1:0SrZc3zvMAGgVKg96uP4DEJb13lK2Is9a4go7IIcFSE= -github.com/DataDog/viper v1.13.4/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= +github.com/DataDog/viper v1.13.5 h1:SZMcyMknYQN2jRY/40A16gUXexlNJOI8sDs1cWZnI64= +github.com/DataDog/viper v1.13.5/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= diff --git a/pkg/logs/sds/go.mod b/pkg/logs/sds/go.mod index d8e3a1475f7a1..49a94bf7dab63 100644 --- a/pkg/logs/sds/go.mod +++ b/pkg/logs/sds/go.mod @@ -75,7 +75,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/system/socket v0.54.0-rc.2 // indirect github.com/DataDog/datadog-agent/pkg/util/winutil v0.54.0-rc.2 // indirect github.com/DataDog/datadog-agent/pkg/version v0.54.0-rc.2 // indirect - github.com/DataDog/viper v1.13.4 // indirect + github.com/DataDog/viper v1.13.5 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect diff --git a/pkg/logs/sds/go.sum b/pkg/logs/sds/go.sum index d1c188fe5103b..c8e007d714a63 100644 --- a/pkg/logs/sds/go.sum +++ b/pkg/logs/sds/go.sum @@ -41,8 +41,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240419161837-f1b2f553edfe h1:efzxujZ7VHWFxjmWjcJyUEpPrN8qdiZPYb+dBw547Wo= github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240419161837-f1b2f553edfe/go.mod h1:TX7CTOQ3LbQjfAi4SwqUoR5gY1zfUk7VRBDTuArjaDc= -github.com/DataDog/viper v1.13.4 h1:0SrZc3zvMAGgVKg96uP4DEJb13lK2Is9a4go7IIcFSE= -github.com/DataDog/viper v1.13.4/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= +github.com/DataDog/viper v1.13.5 h1:SZMcyMknYQN2jRY/40A16gUXexlNJOI8sDs1cWZnI64= +github.com/DataDog/viper v1.13.5/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= diff --git a/pkg/logs/sender/go.mod b/pkg/logs/sender/go.mod index 1c867148dcd54..12151c1fcc1a1 100644 --- a/pkg/logs/sender/go.mod +++ b/pkg/logs/sender/go.mod @@ -80,7 +80,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/system/socket v0.54.0-rc.2 // indirect github.com/DataDog/datadog-agent/pkg/util/winutil v0.54.0-rc.2 // indirect github.com/DataDog/datadog-agent/pkg/version v0.54.0-rc.2 // indirect - github.com/DataDog/viper v1.13.4 // indirect + github.com/DataDog/viper v1.13.5 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect diff --git a/pkg/logs/sender/go.sum b/pkg/logs/sender/go.sum index 94a8ba1fc2228..cdd5aa8ae77f8 100644 --- a/pkg/logs/sender/go.sum +++ b/pkg/logs/sender/go.sum @@ -39,8 +39,8 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/DataDog/viper v1.13.4 h1:0SrZc3zvMAGgVKg96uP4DEJb13lK2Is9a4go7IIcFSE= -github.com/DataDog/viper v1.13.4/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= +github.com/DataDog/viper v1.13.5 h1:SZMcyMknYQN2jRY/40A16gUXexlNJOI8sDs1cWZnI64= +github.com/DataDog/viper v1.13.5/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= diff --git a/pkg/logs/sources/go.mod b/pkg/logs/sources/go.mod index 59cd81bfa84fe..4fbdef6203254 100644 --- a/pkg/logs/sources/go.mod +++ b/pkg/logs/sources/go.mod @@ -58,7 +58,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/system/socket v0.54.0-rc.2 // indirect github.com/DataDog/datadog-agent/pkg/util/winutil v0.54.0-rc.2 // indirect github.com/DataDog/datadog-agent/pkg/version v0.54.0-rc.2 // indirect - github.com/DataDog/viper v1.13.4 // indirect + github.com/DataDog/viper v1.13.5 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect github.com/davecgh/go-spew v1.1.1 // indirect diff --git a/pkg/logs/sources/go.sum b/pkg/logs/sources/go.sum index c1503b41b0ee8..6530b0681514a 100644 --- a/pkg/logs/sources/go.sum +++ b/pkg/logs/sources/go.sum @@ -1,8 +1,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/DataDog/viper v1.13.4 h1:0SrZc3zvMAGgVKg96uP4DEJb13lK2Is9a4go7IIcFSE= -github.com/DataDog/viper v1.13.4/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= +github.com/DataDog/viper v1.13.5 h1:SZMcyMknYQN2jRY/40A16gUXexlNJOI8sDs1cWZnI64= +github.com/DataDog/viper v1.13.5/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= diff --git a/pkg/logs/util/testutils/go.mod b/pkg/logs/util/testutils/go.mod index d2fde54e51cb4..8cb33a3f9de1c 100644 --- a/pkg/logs/util/testutils/go.mod +++ b/pkg/logs/util/testutils/go.mod @@ -59,7 +59,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/system/socket v0.54.0-rc.2 // indirect github.com/DataDog/datadog-agent/pkg/util/winutil v0.54.0-rc.2 // indirect github.com/DataDog/datadog-agent/pkg/version v0.54.0-rc.2 // indirect - github.com/DataDog/viper v1.13.4 // indirect + github.com/DataDog/viper v1.13.5 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect diff --git a/pkg/logs/util/testutils/go.sum b/pkg/logs/util/testutils/go.sum index c1503b41b0ee8..6530b0681514a 100644 --- a/pkg/logs/util/testutils/go.sum +++ b/pkg/logs/util/testutils/go.sum @@ -1,8 +1,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/DataDog/viper v1.13.4 h1:0SrZc3zvMAGgVKg96uP4DEJb13lK2Is9a4go7IIcFSE= -github.com/DataDog/viper v1.13.4/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= +github.com/DataDog/viper v1.13.5 h1:SZMcyMknYQN2jRY/40A16gUXexlNJOI8sDs1cWZnI64= +github.com/DataDog/viper v1.13.5/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= diff --git a/pkg/metrics/go.mod b/pkg/metrics/go.mod index 2778615564f39..d87ae6fa6a096 100644 --- a/pkg/metrics/go.mod +++ b/pkg/metrics/go.mod @@ -38,7 +38,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/scrubber v0.54.0-rc.2 // indirect github.com/DataDog/datadog-agent/pkg/util/sort v0.54.0-rc.2 // indirect github.com/DataDog/sketches-go v1.4.4 // indirect - github.com/DataDog/viper v1.13.4 // indirect + github.com/DataDog/viper v1.13.5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect diff --git a/pkg/metrics/go.sum b/pkg/metrics/go.sum index 8fbb026336fd6..8f6e4ea435114 100644 --- a/pkg/metrics/go.sum +++ b/pkg/metrics/go.sum @@ -45,8 +45,8 @@ github.com/DataDog/opentelemetry-mapping-go/pkg/quantile v0.14.0 h1:QHx6B/VUx3rZ github.com/DataDog/opentelemetry-mapping-go/pkg/quantile v0.14.0/go.mod h1:q4c7zbmdnIdSJNZuBsveTk5ZeRkSkS2g6b8zzFF1mE4= github.com/DataDog/sketches-go v1.4.4 h1:dF52vzXRFSPOj2IjXSWLvXq3jubL4CI69kwYjJ1w5Z8= github.com/DataDog/sketches-go v1.4.4/go.mod h1:XR0ns2RtEEF09mDKXiKZiQg+nfZStrq1ZuL1eezeZe0= -github.com/DataDog/viper v1.13.4 h1:0SrZc3zvMAGgVKg96uP4DEJb13lK2Is9a4go7IIcFSE= -github.com/DataDog/viper v1.13.4/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= +github.com/DataDog/viper v1.13.5 h1:SZMcyMknYQN2jRY/40A16gUXexlNJOI8sDs1cWZnI64= +github.com/DataDog/viper v1.13.5/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= diff --git a/pkg/process/checks/container.go b/pkg/process/checks/container.go index 9cbcf499814f2..65d86a3f4bf08 100644 --- a/pkg/process/checks/container.go +++ b/pkg/process/checks/container.go @@ -7,6 +7,7 @@ package checks import ( "context" + "fmt" "sync" "time" @@ -134,7 +135,8 @@ func (c *ContainerCheck) Run(nextGroupID func() int32, options *RunOptions) (Run } numContainers := float64(len(containers)) - statsd.Client.Gauge("datadog.process.containers.host_count", numContainers, []string{}, 1) //nolint:errcheck + agentNameTag := fmt.Sprintf("agent:%s", flavor.GetFlavor()) + statsd.Client.Gauge("datadog.process.containers.host_count", numContainers, []string{agentNameTag}, 1) //nolint:errcheck log.Debugf("collected %d containers in %s", int(numContainers), time.Since(startTime)) return StandardRunResult(messages), nil } diff --git a/pkg/process/checks/process.go b/pkg/process/checks/process.go index e57798df2db24..08f81cfd4fa68 100644 --- a/pkg/process/checks/process.go +++ b/pkg/process/checks/process.go @@ -8,6 +8,7 @@ package checks import ( "context" "errors" + "fmt" "regexp" "strings" "time" @@ -340,8 +341,9 @@ func (p *ProcessCheck) run(groupID int32, collectRealTime bool) (RunResult, erro p.realtimeLastRun = p.lastRun } - statsd.Client.Gauge("datadog.process.containers.host_count", float64(totalContainers), []string{}, 1) //nolint:errcheck - statsd.Client.Gauge("datadog.process.processes.host_count", float64(totalProcs), []string{}, 1) //nolint:errcheck + agentNameTag := fmt.Sprintf("agent:%s", flavor.GetFlavor()) + statsd.Client.Gauge("datadog.process.containers.host_count", float64(totalContainers), []string{agentNameTag}, 1) //nolint:errcheck + statsd.Client.Gauge("datadog.process.processes.host_count", float64(totalProcs), []string{agentNameTag}, 1) //nolint:errcheck log.Debugf("collected processes in %s", time.Since(start)) return result, nil diff --git a/pkg/proto/datadog/workloadmeta/workloadmeta.proto b/pkg/proto/datadog/workloadmeta/workloadmeta.proto index fd3c522ebe8b8..a02ade2a01b92 100644 --- a/pkg/proto/datadog/workloadmeta/workloadmeta.proto +++ b/pkg/proto/datadog/workloadmeta/workloadmeta.proto @@ -91,7 +91,7 @@ message ContainerState { int64 createdAt = 4; int64 startedAt = 5; int64 finishedAt = 6; - uint32 exitCode = 7; + int64 exitCode = 7; } message Container { diff --git a/pkg/proto/pbgo/core/workloadmeta.pb.go b/pkg/proto/pbgo/core/workloadmeta.pb.go index d59bb72eaed8d..12dc3ba9d464a 100644 --- a/pkg/proto/pbgo/core/workloadmeta.pb.go +++ b/pkg/proto/pbgo/core/workloadmeta.pb.go @@ -778,7 +778,7 @@ type ContainerState struct { CreatedAt int64 `protobuf:"varint,4,opt,name=createdAt,proto3" json:"createdAt,omitempty"` StartedAt int64 `protobuf:"varint,5,opt,name=startedAt,proto3" json:"startedAt,omitempty"` FinishedAt int64 `protobuf:"varint,6,opt,name=finishedAt,proto3" json:"finishedAt,omitempty"` - ExitCode uint32 `protobuf:"varint,7,opt,name=exitCode,proto3" json:"exitCode,omitempty"` + ExitCode int64 `protobuf:"varint,7,opt,name=exitCode,proto3" json:"exitCode,omitempty"` } func (x *ContainerState) Reset() { @@ -855,7 +855,7 @@ func (x *ContainerState) GetFinishedAt() int64 { return 0 } -func (x *ContainerState) GetExitCode() uint32 { +func (x *ContainerState) GetExitCode() int64 { if x != nil { return x.ExitCode } @@ -1598,7 +1598,7 @@ var file_datadog_workloadmeta_workloadmeta_proto_rawDesc = []byte{ 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x66, 0x69, 0x6e, 0x69, 0x73, 0x68, 0x65, 0x64, 0x41, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x66, 0x69, 0x6e, 0x69, 0x73, 0x68, 0x65, 0x64, 0x41, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x65, 0x78, 0x69, - 0x74, 0x43, 0x6f, 0x64, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x65, 0x78, 0x69, + 0x74, 0x43, 0x6f, 0x64, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08, 0x65, 0x78, 0x69, 0x74, 0x43, 0x6f, 0x64, 0x65, 0x22, 0x89, 0x06, 0x0a, 0x09, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x12, 0x46, 0x0a, 0x08, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x49, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, diff --git a/pkg/security/ebpf/c/include/constants/custom.h b/pkg/security/ebpf/c/include/constants/custom.h index e2b8b974f1343..e7c2d4113c727 100644 --- a/pkg/security/ebpf/c/include/constants/custom.h +++ b/pkg/security/ebpf/c/include/constants/custom.h @@ -27,9 +27,9 @@ enum MONITOR_KEYS { #define MAX_PERF_STR_BUFF_LEN 256 #define MAX_STR_BUFF_LEN (1 << 15) #define MAX_ARRAY_ELEMENT_SIZE 4096 -#define MAX_ARRAY_ELEMENT_PER_TAIL 28 +#define MAX_ARRAY_ELEMENT_PER_TAIL 27 #define MAX_ARGS_ELEMENTS (MAX_ARRAY_ELEMENT_PER_TAIL * (32 / 2)) // split tailcall limit -#define MAX_ARGS_READ_PER_TAIL 208 +#define MAX_ARGS_READ_PER_TAIL 160 #define EXEC_GET_ENVS_OFFSET 0 #define EXEC_PARSE_ARGS_ENVS_SPLIT 1 diff --git a/pkg/security/ebpf/c/include/hooks/exec.h b/pkg/security/ebpf/c/include/hooks/exec.h index 22946c6426ab9..e25bdbe2d2f89 100644 --- a/pkg/security/ebpf/c/include/hooks/exec.h +++ b/pkg/security/ebpf/c/include/hooks/exec.h @@ -392,9 +392,14 @@ int tail_call_target_get_envs_offset(void *ctx) { #pragma unroll for (i = 0; i < MAX_ARGS_READ_PER_TAIL && args_count < syscall->exec.args.count; i++) { bytes_read = bpf_probe_read_str(&buff->value[0], MAX_ARRAY_ELEMENT_SIZE, (void *)(args_start + offset)); - if (bytes_read <= 0 || bytes_read == MAX_ARRAY_ELEMENT_SIZE) { + if (bytes_read < 0 || bytes_read == MAX_ARRAY_ELEMENT_SIZE) { syscall->exec.args_envs_ctx.envs_offset = 0; return 0; + } else if (buff->value[0] == '\0') { + // skip empty strings + // directly check the first character instead of bytes_read because bpf_probe_read_str() + // may return 0 or 1 depending on the kernel version when reading empty strings + bytes_read = 1; } offset += bytes_read; args_count++; @@ -440,10 +445,19 @@ void __attribute__((always_inline)) parse_args_envs(void *ctx, struct args_envs_ #pragma unroll for (i = 0; i < MAX_ARRAY_ELEMENT_PER_TAIL; i++) { - void *string_array_ptr = &(buff->value[(event.size + sizeof(bytes_read)) & (MAX_STR_BUFF_LEN - MAX_ARRAY_ELEMENT_SIZE - 1)]); + if (args_envs->counter == args_envs->count) { + break; + } + + char *string_array_ptr = &(buff->value[(event.size + sizeof(bytes_read)) & (MAX_STR_BUFF_LEN - MAX_ARRAY_ELEMENT_SIZE - 1)]); - bytes_read = bpf_probe_read_str(string_array_ptr, MAX_ARRAY_ELEMENT_SIZE, (void *)(args_start + offset)); - if (bytes_read > 0) { + bytes_read = bpf_probe_read_str((void *)string_array_ptr, MAX_ARRAY_ELEMENT_SIZE, (void *)(args_start + offset)); + // skip empty strings + // depending on the kernel version, bpf_probe_read_str() may return 0 or 1 when reading empty strings + if (bytes_read == 0 || (bytes_read == 1 && *string_array_ptr == '\0')) { + offset += 1; + args_envs->counter++; + } else if (bytes_read > 0) { bytes_read--; // remove trailing 0 // insert size before the string @@ -470,11 +484,7 @@ void __attribute__((always_inline)) parse_args_envs(void *ctx, struct args_envs_ } else { event.size += data_length; args_envs->counter++; - offset += bytes_read + 1; - } - - if (args_envs->counter == args_envs->count) { - break; + offset += bytes_read + 1; // count trailing 0 } } else { break; diff --git a/pkg/security/probe/kfilters/capabilities_windows.go b/pkg/security/probe/kfilters/capabilities_windows.go index 5ffba3ef71358..503866e04ead0 100644 --- a/pkg/security/probe/kfilters/capabilities_windows.go +++ b/pkg/security/probe/kfilters/capabilities_windows.go @@ -16,4 +16,22 @@ func init() { FieldValueTypes: eval.ScalarValueType | eval.PatternValueType, }, } + allCapabilities["rename"] = Capabilities{ + "rename.file.name": { + PolicyFlags: PolicyFlagBasename, + FieldValueTypes: eval.ScalarValueType | eval.PatternValueType, + }, + } + allCapabilities["delete"] = Capabilities{ + "delete.file.name": { + PolicyFlags: PolicyFlagBasename, + FieldValueTypes: eval.ScalarValueType | eval.PatternValueType, + }, + } + allCapabilities["write"] = Capabilities{ + "write.file.name": { + PolicyFlags: PolicyFlagBasename, + FieldValueTypes: eval.ScalarValueType | eval.PatternValueType, + }, + } } diff --git a/pkg/security/probe/probe_ebpf.go b/pkg/security/probe/probe_ebpf.go index b847f8102b7e7..49f43d90ebbb8 100644 --- a/pkg/security/probe/probe_ebpf.go +++ b/pkg/security/probe/probe_ebpf.go @@ -2051,6 +2051,9 @@ func (p *EBPFProbe) HandleActions(ctx *eval.Context, rule *rules.Rule) { p.probe.DispatchCustomEvent(rule, event) } + case action.Hash != nil: + // force the resolution as it will force the hash resolution as well + ev.ResolveFields() } } } diff --git a/pkg/security/probe/probe_ebpfless.go b/pkg/security/probe/probe_ebpfless.go index 76bb268a83a4a..5a9393a7fcec8 100644 --- a/pkg/security/probe/probe_ebpfless.go +++ b/pkg/security/probe/probe_ebpfless.go @@ -45,7 +45,6 @@ const ( type client struct { conn net.Conn probe *EBPFLessProbe - seqNum uint64 nsID uint64 containerID string containerName string @@ -81,11 +80,6 @@ type EBPFLessProbe struct { } func (p *EBPFLessProbe) handleClientMsg(cl *client, msg *ebpfless.Message) { - if cl.seqNum != msg.SeqNum { - seclog.Errorf("communication out of sync %d vs %d", cl.seqNum, msg.SeqNum) - } - cl.seqNum++ - switch msg.Type { case ebpfless.MessageTypeHello: if cl.nsID == 0 { @@ -443,6 +437,7 @@ func (p *EBPFLessProbe) handleNewClient(conn net.Conn, ch chan clientMsg) { p.Lock() delete(p.clients, conn) p.Unlock() + conn.Close() msg.Type = ebpfless.MessageTypeGoodbye ch <- msg @@ -557,6 +552,9 @@ func (p *EBPFLessProbe) HandleActions(ctx *eval.Context, rule *rules.Rule) { p.processKiller.KillAndReport(action.Kill.Scope, action.Kill.Signal, ev, func(pid uint32, sig uint32) error { return p.processKiller.KillFromUserspace(pid, sig, ev) }) + case action.Hash != nil: + // force the resolution as it will force the hash resolution as well + ev.ResolveFields() } } } diff --git a/pkg/security/probe/probe_kernel_file_windows.go b/pkg/security/probe/probe_kernel_file_windows.go index f749f77695e71..ee34f239944e9 100644 --- a/pkg/security/probe/probe_kernel_file_windows.go +++ b/pkg/security/probe/probe_kernel_file_windows.go @@ -166,7 +166,7 @@ func (wp *WindowsProbe) parseCreateHandleArgs(e *etw.DDEventRecord) (*createHand // not amazing to double compute the basename.. basename := filepath.Base(ca.fileName) - if !wp.approve("create.file.name", basename) { + if !wp.approveFimBasename(basename) { wp.discardedFileHandles.Add(fileObjectPointer(ca.fileObject), struct{}{}) wp.stats.createFileApproverRejects++ return nil, errDiscardedPath @@ -199,6 +199,9 @@ func (wp *WindowsProbe) parseCreateHandleArgs(e *etw.DDEventRecord) (*createHand if wp.filePathResolver.Add(ca.fileObject, fc) { wp.stats.fileNameCacheEvictions++ } + // if we get here, we have a new file handle. Remove it from the discarder cache in case + // we missed the close notification + wp.discardedFileHandles.Remove(fileObjectPointer(ca.fileObject)) return ca, nil } diff --git a/pkg/security/probe/probe_kernel_file_windows_test.go b/pkg/security/probe/probe_kernel_file_windows_test.go index 4c6cd15f61d05..814fb78a0d2ba 100644 --- a/pkg/security/probe/probe_kernel_file_windows_test.go +++ b/pkg/security/probe/probe_kernel_file_windows_test.go @@ -538,6 +538,7 @@ func TestETWFileNotifications(t *testing.T) { testfilerename := ex + ".testfilerename" wp, err := createTestProbe() + wp.disableApprovers = true require.NoError(t, err) // teardownTestProe calls the stop function on etw, which will diff --git a/pkg/security/probe/probe_windows.go b/pkg/security/probe/probe_windows.go index 400bd34c7afa6..8560b64f312e2 100644 --- a/pkg/security/probe/probe_windows.go +++ b/pkg/security/probe/probe_windows.go @@ -11,6 +11,7 @@ import ( "errors" "fmt" "path/filepath" + "slices" "sync" "time" @@ -102,8 +103,11 @@ type WindowsProbe struct { // channel handling. Currently configurable, but should probably be set // to false with a configurable size value blockonchannelsend bool + // approvers - approvers map[eval.Field][]approver + disableApprovers bool + currentEventTypes []string + approvers map[eval.Field][]approver } type approver interface { @@ -222,6 +226,14 @@ func (p *WindowsProbe) initEtwFIM() error { return err } + return p.reconfigureProvider() +} + +func (p *WindowsProbe) reconfigureProvider() error { + if !p.config.RuntimeSecurity.FIMEnabled { + return nil + } + pidsList := make([]uint32, 0) p.fimSession.ConfigureProvider(p.fileguid, func(cfg *etw.ProviderConfiguration) { @@ -251,16 +263,21 @@ func (p *WindowsProbe) initEtwFIM() error { idCreateNewFile, idCleanup, idClose, - idWrite, - idSetDelete, - idDeletePath, - idRename, - idRenamePath, - idRename29, + } + + if p.isWriteEnabled { + fileIds = append(fileIds, idWrite) + } + if p.isRenameEnabled { + fileIds = append(fileIds, idRename, idRenamePath, idRename29) + } + if p.isDeleteEnabled { + fileIds = append(fileIds, idSetDelete, idDeletePath) } cfg.EnabledIDs = fileIds }) + p.fimSession.ConfigureProvider(p.regguid, func(cfg *etw.ProviderConfiguration) { cfg.TraceLevel = etw.TRACE_LEVEL_VERBOSE cfg.PIDs = pidsList @@ -290,16 +307,16 @@ func (p *WindowsProbe) initEtwFIM() error { cfg.MatchAnyKeyword = 0xF7E3 }) - err = p.fimSession.EnableProvider(p.fileguid) - if err != nil { + if err := p.fimSession.EnableProvider(p.fileguid); err != nil { log.Warnf("Error enabling provider %v", err) return err } - err = p.fimSession.EnableProvider(p.regguid) - if err != nil { + + if err := p.fimSession.EnableProvider(p.regguid); err != nil { log.Warnf("Error enabling provider %v", err) return err } + return nil } @@ -319,12 +336,29 @@ func (p *WindowsProbe) Stop() { } } +func (p *WindowsProbe) approveFimBasename(value string) bool { + fields := []string{"create.file.name", "rename.file.name", "delete.file.name", "write.file.name"} + eventTypes := []string{"create", "rename", "delete", "write"} + + for i, field := range fields { + eventType := eventTypes[i] + if p.approve(field, eventType, value) { + return true + } + } + return false +} + // currently support only string base approver for now -func (p *WindowsProbe) approve(field eval.Field, value string) bool { +func (p *WindowsProbe) approve(field eval.Field, eventType string, value string) bool { + if p.disableApprovers { + return true + } + approvers, exists := p.approvers[field] if !exists { - // no approvers, so no filtering for this field - return true + // no approvers, so no filtering for this field, except if no rule for this event type + return slices.Contains(p.currentEventTypes, eventType) } for _, approver := range approvers { @@ -382,6 +416,7 @@ func (p *WindowsProbe) setupEtw(ecb etwCallback) error { p.stats.fileProcessedNotifications[e.EventHeader.EventDescriptor.ID]++ ecb(ca, e.EventHeader.ProcessID) // lru is thread safe, has its own locking + p.discardedFileHandles.Remove(ca.fileObject) p.filePathResolver.Remove(ca.fileObject) } case idFlush: @@ -1006,7 +1041,9 @@ func (p *WindowsProbe) ApplyRuleSet(rs *rules.RuleSet) (*kfilters.ApplyRuleSetRe p.isRenameEnabled = false p.isDeleteEnabled = false - for _, eventType := range rs.GetEventTypes() { + p.currentEventTypes = rs.GetEventTypes() + + for _, eventType := range p.currentEventTypes { switch eventType { case model.FileRenameEventType.String(): p.isRenameEnabled = true @@ -1022,12 +1059,19 @@ func (p *WindowsProbe) ApplyRuleSet(rs *rules.RuleSet) (*kfilters.ApplyRuleSetRe return nil, err } + // remove old approvers + clear(p.approvers) + for eventType, report := range ars.Policies { if err := p.SetApprovers(eventType, report.Approvers); err != nil { return nil, err } } + if err := p.reconfigureProvider(); err != nil { + return nil, err + } + return ars, nil } diff --git a/pkg/security/proto/ebpfless/msg.go b/pkg/security/proto/ebpfless/msg.go index ed4341b7cb726..297c9425031f0 100644 --- a/pkg/security/proto/ebpfless/msg.go +++ b/pkg/security/proto/ebpfless/msg.go @@ -332,7 +332,6 @@ type HelloMsg struct { // Message defines a message type Message struct { - SeqNum uint64 Type MessageType Hello *HelloMsg `json:",omitempty"` Syscall *SyscallMsg `json:",omitempty"` @@ -346,7 +345,6 @@ func (m Message) String() string { // Reset resets a message func (m *Message) Reset() { - m.SeqNum = 0 m.Type = MessageTypeUnknown m.Hello = nil m.Syscall = nil diff --git a/pkg/security/ptracer/cws.go b/pkg/security/ptracer/cws.go index 519cfc0ab931e..a6e7020efdb99 100644 --- a/pkg/security/ptracer/cws.go +++ b/pkg/security/ptracer/cws.go @@ -22,8 +22,6 @@ import ( "syscall" "time" - "go.uber.org/atomic" - "github.com/avast/retry-go/v4" "github.com/vmihailenco/msgpack/v5" "golang.org/x/sys/unix" @@ -169,26 +167,30 @@ func StartCWSPtracer(args []string, envs []string, probeAddr string, opts Opts) wg sync.WaitGroup ) + connectClient := func() error { + var err error + client, err = initConn(probeAddr, 600) + if err != nil { + clientReady <- false + logger.Errorf("connection to system-probe failed!") + return err + } + clientReady <- true + logger.Debugf("connection to system-probe initiated!") + return nil + } + if probeAddr != "" { logger.Debugf("connection to system-probe...") if opts.Async { go func() { - // use a local err variable to avoid race condition - var err error - client, err = initConn(probeAddr, 600) - if err != nil { - return - } - clientReady <- true - logger.Debugf("connection to system-probe initiated!") + _ = connectClient() }() } else { - client, err = initConn(probeAddr, 120) + err = connectClient() if err != nil { return err } - clientReady <- true - logger.Debugf("connection to system-probe initiated!") } } @@ -230,15 +232,10 @@ func StartCWSPtracer(args []string, envs []string, probeAddr string, opts Opts) var ( msgDataChan = make(chan []byte, 100000) - traceChan = make(chan bool) ctx, cancelFnc = context.WithCancel(context.Background()) - seq = atomic.NewUint64(0) ) send := func(msg *ebpfless.Message) { - msg.SeqNum = seq.Load() - seq.Inc() - logger.Debugf("sending message: %s", msg) if probeAddr == "" { @@ -264,47 +261,87 @@ func StartCWSPtracer(args []string, envs []string, probeAddr string, opts Opts) process := NewProcess(tracer.PID) pc.Add(tracer.PID, process) - wg.Add(1) - go func() { - defer wg.Done() - - // start tracing - traceChan <- true + if probeAddr == "" { + send(&ebpfless.Message{ + Type: ebpfless.MessageTypeHello, + Hello: &ebpfless.HelloMsg{ + NSID: getNSID(), + ContainerContext: containerCtx, + EntrypointArgs: args, + }, + }) + } else /* probeAddr != "" */ { + wg.Add(1) + go func() { + defer wg.Done() - if probeAddr != "" { - LOOP: - // wait for the client to be ready of stopped for { - select { - case <-ctx.Done(): - return - case <-clientReady: - break LOOP + LOOP: // wait for the client to be ready or stopped + for { + select { + case <-ctx.Done(): + return + case ready := <-clientReady: + if !ready { + time.Sleep(time.Second) + // re-init connection + logger.Debugf("try to reconnect to system-probe...") + go func() { + _ = connectClient() + }() + continue + } + + defer client.Close() + + // if ready, send an hello message + helloMsg := &ebpfless.Message{ + Type: ebpfless.MessageTypeHello, + Hello: &ebpfless.HelloMsg{ + NSID: getNSID(), + ContainerContext: containerCtx, + EntrypointArgs: args, + }, + } + logger.Debugf("sending message: %s", helloMsg) + data, err := msgpack.Marshal(helloMsg) + if err != nil { + logger.Errorf("unable to marshal message: %v", err) + return + } + if err = sendMsgData(client, data); err != nil { + logger.Debugf("error sending hallo msg: %v", err) + go func() { + _ = connectClient() + }() + continue + } + break LOOP + } } - } - defer client.Close() - } - for { - select { - case data := <-msgDataChan: - if err := sendMsgData(client, data); err != nil { - logger.Debugf("%v", err) + LOOP2: // unqueue and try to send messages or wait client to be stopped + for { + select { + case data := <-msgDataChan: + if err := sendMsgData(client, data); err != nil { + logger.Debugf("error sending msg: %v", err) + msgDataChan <- data + break LOOP2 + } + case <-ctx.Done(): + return + } } - case <-ctx.Done(): - return - } - } - }() - send(&ebpfless.Message{ - Type: ebpfless.MessageTypeHello, - Hello: &ebpfless.HelloMsg{ - NSID: getNSID(), - ContainerContext: containerCtx, - EntrypointArgs: args, - }, - }) + // re-init connection + logger.Debugf("try to reconnect to system-probe...") + go func() { + _ = connectClient() + }() + } + }() + } if !opts.ProcScanDisabled { every := opts.ScanProcEvery @@ -504,8 +541,6 @@ func StartCWSPtracer(args []string, envs []string, probeAddr string, opts Opts) } } - <-traceChan - defer func() { // stop client and msg chan reader cancelFnc() diff --git a/pkg/security/rules/monitor/policy_monitor.go b/pkg/security/rules/monitor/policy_monitor.go index 9690bb108dabf..96ea530109c16 100644 --- a/pkg/security/rules/monitor/policy_monitor.go +++ b/pkg/security/rules/monitor/policy_monitor.go @@ -188,6 +188,13 @@ type RuleAction struct { Filter *string `json:"filter,omitempty"` Set *RuleSetAction `json:"set,omitempty"` Kill *RuleKillAction `json:"kill,omitempty"` + Hash *HashAction `json:"hash,omitempty"` +} + +// HashAction is used to report 'hash' action +// easyjson:json +type HashAction struct { + Enabled bool `json:"enabled,omitempty"` } // RuleSetAction is used to report 'set' action @@ -267,6 +274,10 @@ func RuleStateFromDefinition(def *rules.RuleDefinition, status string, message s Append: action.Set.Append, Scope: string(action.Set.Scope), } + case action.Hash != nil: + ruleAction.Hash = &HashAction{ + Enabled: true, + } } ruleState.Actions = append(ruleState.Actions, ruleAction) } diff --git a/pkg/security/rules/monitor/policy_monitor_easyjson.go b/pkg/security/rules/monitor/policy_monitor_easyjson.go index d4db727f7676b..802f10a5b1e00 100644 --- a/pkg/security/rules/monitor/policy_monitor_easyjson.go +++ b/pkg/security/rules/monitor/policy_monitor_easyjson.go @@ -512,6 +512,16 @@ func easyjson6151911dDecodeGithubComDataDogDatadogAgentPkgSecurityRulesMonitor4( } (*out.Kill).UnmarshalEasyJSON(in) } + case "hash": + if in.IsNull() { + in.Skip() + out.Hash = nil + } else { + if out.Hash == nil { + out.Hash = new(HashAction) + } + (*out.Hash).UnmarshalEasyJSON(in) + } default: in.SkipRecursive() } @@ -552,6 +562,16 @@ func easyjson6151911dEncodeGithubComDataDogDatadogAgentPkgSecurityRulesMonitor4( } (*in.Kill).MarshalEasyJSON(out) } + if in.Hash != nil { + const prefix string = ",\"hash\":" + if first { + first = false + out.RawString(prefix[1:]) + } else { + out.RawString(prefix) + } + (*in.Hash).MarshalEasyJSON(out) + } out.RawByte('}') } @@ -761,3 +781,56 @@ func (v HeartbeatEvent) MarshalEasyJSON(w *jwriter.Writer) { func (v *HeartbeatEvent) UnmarshalEasyJSON(l *jlexer.Lexer) { easyjson6151911dDecodeGithubComDataDogDatadogAgentPkgSecurityRulesMonitor6(l, v) } +func easyjson6151911dDecodeGithubComDataDogDatadogAgentPkgSecurityRulesMonitor7(in *jlexer.Lexer, out *HashAction) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "enabled": + out.Enabled = bool(in.Bool()) + default: + in.SkipRecursive() + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} +func easyjson6151911dEncodeGithubComDataDogDatadogAgentPkgSecurityRulesMonitor7(out *jwriter.Writer, in HashAction) { + out.RawByte('{') + first := true + _ = first + if in.Enabled { + const prefix string = ",\"enabled\":" + first = false + out.RawString(prefix[1:]) + out.Bool(bool(in.Enabled)) + } + out.RawByte('}') +} + +// MarshalEasyJSON supports easyjson.Marshaler interface +func (v HashAction) MarshalEasyJSON(w *jwriter.Writer) { + easyjson6151911dEncodeGithubComDataDogDatadogAgentPkgSecurityRulesMonitor7(w, v) +} + +// UnmarshalEasyJSON supports easyjson.Unmarshaler interface +func (v *HashAction) UnmarshalEasyJSON(l *jlexer.Lexer) { + easyjson6151911dDecodeGithubComDataDogDatadogAgentPkgSecurityRulesMonitor7(l, v) +} diff --git a/pkg/security/secl/rules/actions.go b/pkg/security/secl/rules/actions.go index 42daacf763841..378ba1460db3f 100644 --- a/pkg/security/secl/rules/actions.go +++ b/pkg/security/secl/rules/actions.go @@ -29,6 +29,7 @@ type ActionDefinition struct { Set *SetDefinition `yaml:"set"` Kill *KillDefinition `yaml:"kill"` CoreDump *CoreDumpDefinition `yaml:"coredump"` + Hash *HashDefinition `yaml:"hash"` // internal InternalCallback *InternalCallbackDefinition @@ -129,3 +130,6 @@ type CoreDumpDefinition struct { Dentry bool `yaml:"dentry"` NoCompression bool `yaml:"no_compression"` } + +// HashDefinition describes the 'hash' section of a rule action +type HashDefinition struct{} diff --git a/pkg/security/tests/file_windows_test.go b/pkg/security/tests/file_windows_test.go index 8e559f8d6f475..491fa3ea6cab4 100644 --- a/pkg/security/tests/file_windows_test.go +++ b/pkg/security/tests/file_windows_test.go @@ -164,7 +164,55 @@ func TestWriteFileEvent(t *testing.T) { t.Fatal(err) } - test.Run(t, "delete", func(t *testing.T, kind wrapperType, cmdFunc func(cmd string, args []string, envs []string) *exec.Cmd) { + test.Run(t, "write", func(t *testing.T, kind wrapperType, cmdFunc func(cmd string, args []string, envs []string) *exec.Cmd) { + test.WaitSignal(t, func() error { + f, err := os.OpenFile("C:\\Temp\\test.bad", os.O_WRONLY, 0755) + if err != nil { + return err + } + if _, err := f.WriteString("test"); err != nil { + return err + } + return f.Close() + }, test.validateFileEvent(t, noWrapperType, func(event *model.Event, rule *rules.Rule) { + assertFieldEqualCaseInsensitve(t, event, "write.file.name", "test.bad", event, "write.file.name file didn't match") + })) + }) +} + +func TestWriteFileEventWithCreate(t *testing.T) { + ruleDefs := []*rules.RuleDefinition{ + { + ID: "test_create_polute", + Expression: `create.file.name =~ "*.dll"`, + }, + { + ID: "test_write_file", + Expression: `write.file.name =~ "test.bad" && write.file.path =~ "C:\Temp\**"`, + }, + } + opts := testOpts{ + enableFIM: true, + } + test, err := newTestModule(t, nil, ruleDefs, withStaticOpts(opts)) + if err != nil { + t.Fatal(err) + } + defer test.Close() + // this is kinda hokey. ETW (which is what FIM is based on) takes an indeterminant amount of time to start up. + // so wait around for it to start + time.Sleep(5 * time.Second) + + os.MkdirAll("C:\\Temp", 0755) + f, err := os.Create("C:\\Temp\\test.bad") + if err != nil { + t.Fatal(err) + } + if err := f.Close(); err != nil { + t.Fatal(err) + } + + test.Run(t, "write", func(t *testing.T, kind wrapperType, cmdFunc func(cmd string, args []string, envs []string) *exec.Cmd) { test.WaitSignal(t, func() error { f, err := os.OpenFile("C:\\Temp\\test.bad", os.O_WRONLY, 0755) if err != nil { diff --git a/pkg/security/tests/process_test.go b/pkg/security/tests/process_test.go index ca62d06d15777..aa93363f4f523 100644 --- a/pkg/security/tests/process_test.go +++ b/pkg/security/tests/process_test.go @@ -30,6 +30,7 @@ import ( sprobe "github.com/DataDog/datadog-agent/pkg/security/probe" "github.com/DataDog/datadog-agent/pkg/security/probe/constantfetch" "github.com/DataDog/datadog-agent/pkg/security/resolvers/process" + "github.com/DataDog/datadog-agent/pkg/security/utils" "github.com/avast/retry-go/v4" "github.com/oliveagle/jsonpath" @@ -39,7 +40,6 @@ import ( "github.com/DataDog/datadog-agent/pkg/security/secl/compiler/eval" "github.com/DataDog/datadog-agent/pkg/security/secl/model" "github.com/DataDog/datadog-agent/pkg/security/secl/rules" - "github.com/DataDog/datadog-agent/pkg/security/utils" ) func TestProcess(t *testing.T) { @@ -485,8 +485,8 @@ func TestProcessContext(t *testing.T) { assert.Equal(t, args[i], argv[i], "expected arg not found") } } else { - assert.Equal(t, 459, len(argv), "incorrect number of args: %s", argv) - for i := 0; i != 459; i++ { + assert.Equal(t, 439, len(argv), "incorrect number of args: %s", argv) + for i := 0; i != 439; i++ { assert.Equal(t, args[i], argv[i], "expected arg not found") } } @@ -539,8 +539,8 @@ func TestProcessContext(t *testing.T) { assert.Equal(t, expected, argv[i], "expected arg not found") } } else { - assert.Equal(t, 474, len(argv), "incorrect number of args: %s", argv) - for i := 0; i != 474; i++ { + assert.Equal(t, 457, len(argv), "incorrect number of args: %s", argv) + for i := 0; i != 457; i++ { expected := args[i] if len(expected) > model.MaxArgEnvSize { expected = args[i][:model.MaxArgEnvSize-4] + "..." // 4 is the size number of the string @@ -650,8 +650,8 @@ func TestProcessContext(t *testing.T) { assert.Equal(t, envs[i], envp[i], "expected env not found") } } else { - assert.Equal(t, 736, len(envp), "incorrect number of envs: %s", envp) - for i := 0; i != 736; i++ { + assert.Equal(t, 704, len(envp), "incorrect number of envs: %s", envp) + for i := 0; i != 704; i++ { assert.Equal(t, envs[i], envp[i], "expected env not found") } } @@ -716,8 +716,8 @@ func TestProcessContext(t *testing.T) { assert.Equal(t, expected, envp[i], "expected env not found") } } else { - assert.Equal(t, 895, len(envp), "incorrect number of envs: %s", envp) - for i := 0; i != 895; i++ { + assert.Equal(t, 863, len(envp), "incorrect number of envs: %s", envp) + for i := 0; i != 863; i++ { expected := envs[i] if len(expected) > model.MaxArgEnvSize { expected = envs[i][:model.MaxArgEnvSize-4] + "..." // 4 is the size number of the string @@ -740,6 +740,44 @@ func TestProcessContext(t *testing.T) { } }) + t.Run("args-envs-empty-strings", func(t *testing.T) { + test.WaitSignal(t, func() error { + args := []string{"-al", ""} + envs := []string{"LD_LIBRARY_PATH=/tmp/lib"} + cmd := exec.Command("ls", args...) + cmd.Env = envs + _ = cmd.Run() + return nil + }, test.validateExecEvent(t, noWrapperType, func(event *model.Event, rule *rules.Rule) { + assertTriggeredRule(t, rule, "test_rule_args_envs") + + args, err := event.GetFieldValue("exec.args") + if err != nil || len(args.(string)) == 0 { + t.Error("not able to get args") + } + assert.Contains(t, args.(string), "-al", "arg not found") + + // envs + envs, err := event.GetFieldValue("exec.envs") + if err != nil || len(envs.([]string)) == 0 { + t.Error("not able to get envs") + } + + contains := func(s string) bool { + for _, env := range envs.([]string) { + if strings.Contains(env, s) { + return true + } + } + return false + } + assert.True(t, contains("LD_LIBRARY_PATH"), "env not found") + + assert.False(t, event.Exec.ArgsTruncated, "args should not be truncated") + assert.False(t, event.Exec.EnvsTruncated, "envs should not be truncated") + })) + }) + t.Run("tty", func(t *testing.T) { testFile, _, err := test.Path("test-process-tty") if err != nil { diff --git a/pkg/serializer/go.mod b/pkg/serializer/go.mod index 19c08bd9758e2..91d6eece826f1 100644 --- a/pkg/serializer/go.mod +++ b/pkg/serializer/go.mod @@ -108,7 +108,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/winutil v0.54.0-rc.2 // indirect github.com/DataDog/mmh3 v0.0.0-20210722141835-012dc69a9e49 // indirect github.com/DataDog/sketches-go v1.4.4 // indirect - github.com/DataDog/viper v1.13.4 // indirect + github.com/DataDog/viper v1.13.5 // indirect github.com/DataDog/zstd v1.5.5 // indirect github.com/DataDog/zstd_0 v0.0.0-20210310093942-586c1286621f // indirect github.com/Microsoft/go-winio v0.6.1 // indirect diff --git a/pkg/serializer/go.sum b/pkg/serializer/go.sum index 7cee52c719d60..2241b2dba3674 100644 --- a/pkg/serializer/go.sum +++ b/pkg/serializer/go.sum @@ -10,8 +10,8 @@ github.com/DataDog/opentelemetry-mapping-go/pkg/quantile v0.14.0 h1:QHx6B/VUx3rZ github.com/DataDog/opentelemetry-mapping-go/pkg/quantile v0.14.0/go.mod h1:q4c7zbmdnIdSJNZuBsveTk5ZeRkSkS2g6b8zzFF1mE4= github.com/DataDog/sketches-go v1.4.4 h1:dF52vzXRFSPOj2IjXSWLvXq3jubL4CI69kwYjJ1w5Z8= github.com/DataDog/sketches-go v1.4.4/go.mod h1:XR0ns2RtEEF09mDKXiKZiQg+nfZStrq1ZuL1eezeZe0= -github.com/DataDog/viper v1.13.4 h1:0SrZc3zvMAGgVKg96uP4DEJb13lK2Is9a4go7IIcFSE= -github.com/DataDog/viper v1.13.4/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= +github.com/DataDog/viper v1.13.5 h1:SZMcyMknYQN2jRY/40A16gUXexlNJOI8sDs1cWZnI64= +github.com/DataDog/viper v1.13.5/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= github.com/DataDog/zstd v1.5.5 h1:oWf5W7GtOLgp6bciQYDmhHHjdhYkALu6S/5Ni9ZgSvQ= github.com/DataDog/zstd v1.5.5/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/DataDog/zstd_0 v0.0.0-20210310093942-586c1286621f h1:5Vuo4niPKFkfwW55jV4vY0ih3VQ9RaQqeqY67fvRn8A= diff --git a/pkg/serverless/otlp/otlp.go b/pkg/serverless/otlp/otlp.go index b0e11539d6408..bf03f7ca40b29 100644 --- a/pkg/serverless/otlp/otlp.go +++ b/pkg/serverless/otlp/otlp.go @@ -29,7 +29,7 @@ type ServerlessOTLPAgent struct { // NewServerlessOTLPAgent creates a new ServerlessOTLPAgent with the correct // otel pipeline. func NewServerlessOTLPAgent(serializer serializer.MetricSerializer) *ServerlessOTLPAgent { - pipeline, err := coreOtlp.NewPipelineFromAgentConfig(config.Datadog(), serializer, nil) + pipeline, err := coreOtlp.NewPipelineFromAgentConfig(config.Datadog(), serializer, nil, nil) if err != nil { log.Error("Error creating new otlp pipeline:", err) return nil diff --git a/pkg/status/health/README.md b/pkg/status/health/README.md index 245cc52b00255..05bf075e946c2 100644 --- a/pkg/status/health/README.md +++ b/pkg/status/health/README.md @@ -11,6 +11,10 @@ For more information on the context, see the `agent-healthcheck.md` proposal. receive a `*health.Handle` to keep. As soon as `Register` is called, you need to start reading the channel to be considered healthy. +- If you want to register a component for the `Startup` probe, you need to call `RegisterStartup()`. +This is useful for components that need to perform some initialization before being considered healthy. +You will need to first register the component, then do the initialization, and finally read from the channel. + - In your main goroutine, you need to read from the `handle.C` channel, at least every 15 seconds. This is accomplished by using a `select` statement in your main goroutine. If the channel is full (after two tries), your component will be considered unhealthy, which might result in the agent @@ -25,6 +29,6 @@ It depends on your component lifecycle, but the check's purpose is to check that is able to process new input and act accordingly. For components that read input from a channel, you should read the channel from this logic. -This is usually hightly unprobable, but it's exactly the scope of this system: be able to +This is usually highly unlikely, but it's exactly the scope of this system: be able to detect if a component is frozen because of a bug / race condition. This is usually the only kind of issue that could be solved by the agent restarting. diff --git a/pkg/status/health/global.go b/pkg/status/health/global.go index e6ce74c5b8555..85586d504853a 100644 --- a/pkg/status/health/global.go +++ b/pkg/status/health/global.go @@ -12,6 +12,7 @@ import ( var readinessAndLivenessCatalog = newCatalog() var readinessOnlyCatalog = newCatalog() +var startupOnlyCatalog = newCatalog() // RegisterReadiness registers a component for readiness check with the default 30 seconds timeout, returns a token func RegisterReadiness(name string) *Handle { @@ -23,12 +24,24 @@ func RegisterLiveness(name string) *Handle { return readinessAndLivenessCatalog.register(name) } +// RegisterStartup registers a component for startup check, returns a token +func RegisterStartup(name string) *Handle { + startupOnlyCatalog.startup = true + return startupOnlyCatalog.register(name) +} + // Deregister a component from the healthcheck func Deregister(handle *Handle) error { if readinessAndLivenessCatalog.deregister(handle) == nil { return nil } - return readinessOnlyCatalog.deregister(handle) + if readinessOnlyCatalog.deregister(handle) == nil { + return nil + } + if startupOnlyCatalog.deregister(handle) == nil { + return nil + } + return errors.New("component not registered") } // GetLive returns health of all components registered for liveness @@ -45,6 +58,11 @@ func GetReady() (ret Status) { return } +// GetStartup returns health of all components registered for startup +func GetStartup() Status { + return startupOnlyCatalog.getStatus() +} + // getStatusNonBlocking allows to query the health status of the agent // and is guaranteed to return under 500ms. func getStatusNonBlocking(getStatus func() Status) (Status, error) { @@ -72,3 +90,8 @@ func GetLiveNonBlocking() (Status, error) { func GetReadyNonBlocking() (Status, error) { return getStatusNonBlocking(GetReady) } + +// GetStartupNonBlocking returns the health of all components registered for startup with a 500ms timeout +func GetStartupNonBlocking() (Status, error) { + return getStatusNonBlocking(GetStartup) +} diff --git a/pkg/status/health/health.go b/pkg/status/health/health.go index 4540034547f9a..712812aa83890 100644 --- a/pkg/status/health/health.go +++ b/pkg/status/health/health.go @@ -35,12 +35,14 @@ type catalog struct { sync.RWMutex components map[*Handle]*component latestRun time.Time + startup bool } func newCatalog() *catalog { return &catalog{ components: make(map[*Handle]*component), latestRun: time.Now(), // Start healthy + startup: false, } } @@ -97,6 +99,10 @@ func (c *catalog) pingComponents(healthDeadline time.Time) bool { c.Lock() defer c.Unlock() for _, component := range c.components { + // In startup mode, we skip already healthy components. + if c.startup && component.healthy { + continue + } select { case component.healthChan <- healthDeadline: component.healthy = true diff --git a/pkg/status/health/health_test.go b/pkg/status/health/health_test.go index 52854bd11ebf8..f08c2f1bf3957 100644 --- a/pkg/status/health/health_test.go +++ b/pkg/status/health/health_test.go @@ -123,3 +123,29 @@ func TestGetHealthy(t *testing.T) { assert.Len(t, status.Healthy, 2) assert.Len(t, status.Unhealthy, 0) } + +func TestStartupCatalog(t *testing.T) { + cat := newCatalog() + cat.startup = true + token := cat.register("test1") + + // Start unhealthy + status := cat.getStatus() + assert.Len(t, status.Healthy, 1) + assert.Contains(t, status.Healthy, "healthcheck") + assert.Len(t, status.Unhealthy, 1) + assert.Contains(t, status.Unhealthy, "test1") + + // Get healthy + <-token.C + cat.pingComponents(time.Time{}) // First ping will make component healthy and fill the channel again. + status = cat.getStatus() + assert.Len(t, status.Healthy, 2) + assert.Contains(t, status.Healthy, "test1") + + // Make sure that we stay health even if we don't ping. + cat.pingComponents(time.Time{}) + status = cat.getStatus() + assert.Len(t, status.Healthy, 2) + assert.Contains(t, status.Healthy, "test1") +} diff --git a/pkg/trace/go.mod b/pkg/trace/go.mod index 5ecd607237c5b..b1c690ec3f6de 100644 --- a/pkg/trace/go.mod +++ b/pkg/trace/go.mod @@ -45,6 +45,7 @@ require ( golang.org/x/time v0.5.0 google.golang.org/grpc v1.63.2 google.golang.org/protobuf v1.34.0 + gopkg.in/ini.v1 v1.67.0 k8s.io/apimachinery v0.25.5 ) diff --git a/pkg/trace/go.sum b/pkg/trace/go.sum index 8776938637f0f..c65e5535c41f7 100644 --- a/pkg/trace/go.sum +++ b/pkg/trace/go.sum @@ -435,6 +435,8 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/trace/stats/concentrator.go b/pkg/trace/stats/concentrator.go index ef5fe2cc98c0a..138156d6b7d42 100644 --- a/pkg/trace/stats/concentrator.go +++ b/pkg/trace/stats/concentrator.go @@ -6,11 +6,14 @@ package stats import ( + _ "embed" "sort" "strings" "sync" "time" + "gopkg.in/ini.v1" + pb "github.com/DataDog/datadog-agent/pkg/proto/pbgo/trace" "github.com/DataDog/datadog-agent/pkg/trace/config" "github.com/DataDog/datadog-agent/pkg/trace/log" @@ -55,37 +58,28 @@ type Concentrator struct { statsd statsd.ClientInterface } -var defaultPeerTags = []string{ - "_dd.base_service", - "amqp.destination", - "amqp.exchange", - "amqp.queue", - "aws.queue.name", - "bucketname", - "cassandra.cluster", - "db.cassandra.contact.points", - "db.couchbase.seed.nodes", - "db.hostname", - "db.instance", - "db.name", - "db.system", - "hazelcast.instance", - "messaging.kafka.bootstrap.servers", - "mongodb.db", - "msmq.queue.path", - "net.peer.name", - "network.destination.name", - "peer.hostname", - "peer.service", - "queuename", - "rpc.service", - "rulename", - "server.address", - "statemachinename", - "streamname", - "tablename", - "topicname", -} +//go:embed peer_tags.ini +var peerTagFile []byte + +var defaultPeerTags = func() []string { + var tags []string = []string{"_dd.base_service"} + + cfg, err := ini.Load(peerTagFile) + if err != nil { + log.Error("Error loading file for peer tags: ", err) + return tags + } + keys := cfg.Section("dd.apm.peer.tags").Keys() + + for _, key := range keys { + value := strings.Split(key.Value(), ",") + tags = append(tags, value...) + } + + sort.Strings(tags) + + return tags +}() func preparePeerTags(tags ...string) []string { if len(tags) == 0 { diff --git a/pkg/trace/stats/concentrator_test.go b/pkg/trace/stats/concentrator_test.go index 1bc45cc532d42..274323c2aa7f6 100644 --- a/pkg/trace/stats/concentrator_test.go +++ b/pkg/trace/stats/concentrator_test.go @@ -1062,3 +1062,52 @@ func TestPreparePeerTags(t *testing.T) { assert.Equal(t, tc.output, preparePeerTags(tc.input...)) } } + +func TestDefaultPeerTags(t *testing.T) { + expectedListOfPeerTags := []string{ + "_dd.base_service", + "amqp.destination", + "amqp.exchange", + "amqp.queue", + "aws.queue.name", + "aws.s3.bucket", + "bucketname", + "cassandra.keyspace", + "db.cassandra.contact.points", + "db.couchbase.seed.nodes", + "db.hostname", + "db.instance", + "db.name", + "db.namespace", + "db.system", + "grpc.host", + "hostname", + "http.host", + "http.server_name", + "messaging.destination", + "messaging.destination.name", + "messaging.kafka.bootstrap.servers", + "messaging.rabbitmq.exchange", + "messaging.system", + "mongodb.db", + "msmq.queue.path", + "net.peer.name", + "network.destination.name", + "peer.hostname", + "peer.service", + "queuename", + "rpc.service", + "rpc.system", + "server.address", + "streamname", + "tablename", + "topicname", + } + actualListOfPeerTags := defaultPeerTags + + // Sort both arrays for comparison + sort.Strings(actualListOfPeerTags) + sort.Strings(expectedListOfPeerTags) + + assert.Equal(t, expectedListOfPeerTags, actualListOfPeerTags) +} diff --git a/pkg/trace/stats/oteltest/go.mod b/pkg/trace/stats/oteltest/go.mod index 6a3c378f83f54..f2e86350ceeb3 100644 --- a/pkg/trace/stats/oteltest/go.mod +++ b/pkg/trace/stats/oteltest/go.mod @@ -90,6 +90,7 @@ require ( golang.org/x/tools v0.16.1 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda // indirect google.golang.org/grpc v1.63.2 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/pkg/trace/stats/oteltest/go.sum b/pkg/trace/stats/oteltest/go.sum index c41384764df4a..37b2159b19d76 100644 --- a/pkg/trace/stats/oteltest/go.sum +++ b/pkg/trace/stats/oteltest/go.sum @@ -288,6 +288,8 @@ google.golang.org/protobuf v1.34.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHh gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/trace/stats/peer_tags.ini b/pkg/trace/stats/peer_tags.ini new file mode 100644 index 0000000000000..d1b36aacf7e5c --- /dev/null +++ b/pkg/trace/stats/peer_tags.ini @@ -0,0 +1,21 @@ +# if a peer entity tag is expected to be set at the tracer, and we want to keep it, then it must be included +# in the list. I.e., since we have at least one tracer which actually sets `peer.hostname` directly, we need +# to set `peer.hostname = "peer.hostname" else it will get stripped out +# the order of the precursors matters, as it is "first one wins". so if we expect some spans to have more than one +# precursor, and the precursors may not always have the exact same meaning, then put the higher priority one at the front +[dd.apm.peer.tags] +peer.aws.dynamodb.table = "tablename" +peer.aws.kinesis.stream = "streamname" +peer.aws.s3.bucket = "bucketname,aws.s3.bucket" +peer.aws.sqs.queue = "queuename" +peer.cassandra.contact.points = "db.cassandra.contact.points" +peer.couchbase.seed.nodes = "db.couchbase.seed.nodes" +peer.db.name = "db.name,mongodb.db,db.instance,cassandra.keyspace,db.namespace" +peer.db.system = "db.system" +peer.hostname = "peer.hostname,hostname,net.peer.name,db.hostname,network.destination.name,grpc.host,http.host,server.address,http.server_name" +peer.kafka.bootstrap.servers = "messaging.kafka.bootstrap.servers" +peer.messaging.destination = "topicname,messaging.destination,messaging.destination.name,messaging.rabbitmq.exchange,amqp.destination,amqp.queue,amqp.exchange,msmq.queue.path,aws.queue.name" +peer.messaging.system = "messaging.system" +peer.rpc.service = "rpc.service" +peer.rpc.system = "rpc.system" +peer.service = "peer.service" diff --git a/pkg/util/containers/env_vars_filter.go b/pkg/util/containers/env_vars_filter.go index bad4a1ac2f1c2..3fb4bdaa1d466 100644 --- a/pkg/util/containers/env_vars_filter.go +++ b/pkg/util/containers/env_vars_filter.go @@ -14,22 +14,23 @@ import ( var ( defaultEnvVarsIncludeList = []string{ - "DD_ENV", - "DD_VERSION", - "DD_SERVICE", "CHRONOS_JOB_NAME", "CHRONOS_JOB_OWNER", - "NOMAD_TASK_NAME", - "NOMAD_JOB_NAME", - "NOMAD_GROUP_NAME", - "NOMAD_NAMESPACE", - "NOMAD_DC", - "MESOS_TASK_ID", + "DD_ENV", + "DD_SERVICE", + "DD_VERSION", + "DOCKER_DD_AGENT", // included to be able to detect agent containers "ECS_CONTAINER_METADATA_URI", "ECS_CONTAINER_METADATA_URI_V4", - "DOCKER_DD_AGENT", // included to be able to detect agent containers - // Included to ease unit tests without requiring a mock - "TEST_ENV", + "MESOS_TASK_ID", + "NOMAD_DC", + "NOMAD_GROUP_NAME", + "NOMAD_JOB_NAME", + "NOMAD_NAMESPACE", + "NOMAD_TASK_NAME", + "OTEL_RESOURCE_ATTRIBUTES", + "OTEL_SERVICE_NAME", + "TEST_ENV", // Included to ease unit tests without requiring a mock } envFilterOnce sync.Once diff --git a/pkg/util/containers/metrics/docker/collector_test.go b/pkg/util/containers/metrics/docker/collector_test.go index af248a23b342c..cdc86b3d39da9 100644 --- a/pkg/util/containers/metrics/docker/collector_test.go +++ b/pkg/util/containers/metrics/docker/collector_test.go @@ -20,7 +20,6 @@ import ( "github.com/DataDog/datadog-agent/comp/core/config" "github.com/DataDog/datadog-agent/comp/core/log/logimpl" "github.com/DataDog/datadog-agent/comp/core/workloadmeta" - "github.com/DataDog/datadog-agent/comp/core/workloadmeta/collectors" "github.com/DataDog/datadog-agent/pkg/util/containers/metrics/provider" "github.com/DataDog/datadog-agent/pkg/util/fxutil" "github.com/DataDog/datadog-agent/pkg/util/optional" @@ -87,7 +86,6 @@ func TestGetContainerIDForPID(t *testing.T) { mockStore := fxutil.Test[workloadmeta.Mock](t, fx.Options( config.MockModule(), logimpl.MockModule(), - collectors.GetCatalog(), fx.Supply(workloadmeta.NewParams()), workloadmeta.MockModuleV2(), )) diff --git a/pkg/util/ecs/metadata/v3or4/client_test.go b/pkg/util/ecs/metadata/v3or4/client_test.go index 5805ed8a127c0..5530e1c87aaed 100644 --- a/pkg/util/ecs/metadata/v3or4/client_test.go +++ b/pkg/util/ecs/metadata/v3or4/client_test.go @@ -15,6 +15,7 @@ import ( "github.com/stretchr/testify/require" "github.com/DataDog/datadog-agent/pkg/util/ecs/metadata/testutil" + "github.com/DataDog/datadog-agent/pkg/util/pointer" ) func TestGetV4TaskWithTags(t *testing.T) { @@ -100,8 +101,9 @@ var expected = &Task{ StartedAt: "2023-11-20T12:10:44.404563253Z", Type: "NORMAL", Health: &HealthStatus{ - Status: "HEALTHY", - Since: "2023-11-20T12:11:16.383262018Z", + Status: "HEALTHY", + Since: "2023-11-20T12:11:16.383262018Z", + ExitCode: pointer.Ptr(int64(-1)), }, Volumes: []Volume{ { diff --git a/pkg/util/ecs/metadata/v3or4/testdata/task_with_tags.json b/pkg/util/ecs/metadata/v3or4/testdata/task_with_tags.json index d6930853e9fa6..e6a1798fbfd1b 100644 --- a/pkg/util/ecs/metadata/v3or4/testdata/task_with_tags.json +++ b/pkg/util/ecs/metadata/v3or4/testdata/task_with_tags.json @@ -63,7 +63,8 @@ "Type": "NORMAL", "Health": { "status": "HEALTHY", - "statusSince": "2023-11-20T12:11:16.383262018Z" + "statusSince": "2023-11-20T12:11:16.383262018Z", + "exitCode": -1 }, "Volumes": [ { diff --git a/pkg/util/ecs/metadata/v3or4/types.go b/pkg/util/ecs/metadata/v3or4/types.go index 5b6a62268322c..46a82c27f51a4 100644 --- a/pkg/util/ecs/metadata/v3or4/types.go +++ b/pkg/util/ecs/metadata/v3or4/types.go @@ -48,16 +48,16 @@ type Container struct { ContainerARN string `json:"ContainerARN,omitempty"` // present only in v4 Health *HealthStatus `json:"Health,omitempty"` Volumes []Volume `json:"Volumes,omitempty"` - ExitCode *uint32 `json:"ExitCode,omitempty"` + ExitCode *int64 `json:"ExitCode,omitempty"` Snapshotter string `json:"Snapshotter,omitempty"` } // HealthStatus represents the health status of a container type HealthStatus struct { - Status string `json:"status,omitempty"` - Since string `json:"statusSince,omitempty"` - ExitCode *uint32 `json:"exitCode,omitempty"` - Output string `json:"output,omitempty"` + Status string `json:"status,omitempty"` + Since string `json:"statusSince,omitempty"` + ExitCode *int64 `json:"exitCode,omitempty"` + Output string `json:"output,omitempty"` } // Network represents the network of a container diff --git a/pkg/util/flavor/go.mod b/pkg/util/flavor/go.mod index ee9146482184d..162df0c896fe4 100644 --- a/pkg/util/flavor/go.mod +++ b/pkg/util/flavor/go.mod @@ -47,7 +47,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/system v0.54.0-rc.2 // indirect github.com/DataDog/datadog-agent/pkg/util/system/socket v0.54.0-rc.2 // indirect github.com/DataDog/datadog-agent/pkg/util/winutil v0.54.0-rc.2 // indirect - github.com/DataDog/viper v1.13.4 // indirect + github.com/DataDog/viper v1.13.5 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect github.com/davecgh/go-spew v1.1.1 // indirect diff --git a/pkg/util/flavor/go.sum b/pkg/util/flavor/go.sum index c1503b41b0ee8..6530b0681514a 100644 --- a/pkg/util/flavor/go.sum +++ b/pkg/util/flavor/go.sum @@ -1,8 +1,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/DataDog/viper v1.13.4 h1:0SrZc3zvMAGgVKg96uP4DEJb13lK2Is9a4go7IIcFSE= -github.com/DataDog/viper v1.13.4/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= +github.com/DataDog/viper v1.13.5 h1:SZMcyMknYQN2jRY/40A16gUXexlNJOI8sDs1cWZnI64= +github.com/DataDog/viper v1.13.5/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= diff --git a/pkg/util/http/go.mod b/pkg/util/http/go.mod index ced231fc62900..df532ce677fe4 100644 --- a/pkg/util/http/go.mod +++ b/pkg/util/http/go.mod @@ -18,7 +18,7 @@ require ( require ( github.com/DataDog/datadog-agent/pkg/util/scrubber v0.54.0-rc.2 // indirect - github.com/DataDog/viper v1.13.4 // indirect + github.com/DataDog/viper v1.13.5 // indirect github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/fsnotify/fsnotify v1.4.7 // indirect diff --git a/pkg/util/http/go.sum b/pkg/util/http/go.sum index babbe9d8f6a1e..81f490d4747c6 100644 --- a/pkg/util/http/go.sum +++ b/pkg/util/http/go.sum @@ -39,8 +39,8 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/DataDog/viper v1.13.4 h1:0SrZc3zvMAGgVKg96uP4DEJb13lK2Is9a4go7IIcFSE= -github.com/DataDog/viper v1.13.4/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= +github.com/DataDog/viper v1.13.5 h1:SZMcyMknYQN2jRY/40A16gUXexlNJOI8sDs1cWZnI64= +github.com/DataDog/viper v1.13.5/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= diff --git a/pkg/util/kubernetes/apiserver/apiserver.go b/pkg/util/kubernetes/apiserver/apiserver.go index c5e18928ae327..8a0668a76d4ff 100644 --- a/pkg/util/kubernetes/apiserver/apiserver.go +++ b/pkg/util/kubernetes/apiserver/apiserver.go @@ -27,10 +27,12 @@ import ( vpa "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/clientset/versioned" "k8s.io/client-go/discovery" "k8s.io/client-go/discovery/cached/memory" + "k8s.io/client-go/dynamic" "k8s.io/client-go/dynamic/dynamicinformer" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/metadata" "k8s.io/client-go/rest" "k8s.io/client-go/restmapper" "k8s.io/client-go/scale" @@ -660,6 +662,17 @@ func (c *APIClient) RESTClient(apiPath string, groupVersion *schema.GroupVersion return rest.RESTClientFor(clientConfig) } +// MetadataClient returns a new kubernetes metadata client +func (c *APIClient) MetadataClient() (metadata.Interface, error) { + clientConfig, err := getClientConfig(c.defaultInformerTimeout) + if err != nil { + return nil, err + } + + metaclient := metadata.NewForConfigOrDie(clientConfig) + return metaclient, nil +} + // NewSPDYExecutor returns a new SPDY executor for the provided method and URL func (c *APIClient) NewSPDYExecutor(apiPath string, groupVersion *schema.GroupVersion, negotiatedSerializer runtime.NegotiatedSerializer, method string, url *url.URL) (remotecommand.Executor, error) { clientConfig, err := getClientConfig(c.defaultClientTimeout) diff --git a/pkg/util/kubernetes/apiserver/controllers/metadata_controller.go b/pkg/util/kubernetes/apiserver/controllers/metadata_controller.go index e0b7269a59e4d..5cd8a339af695 100644 --- a/pkg/util/kubernetes/apiserver/controllers/metadata_controller.go +++ b/pkg/util/kubernetes/apiserver/controllers/metadata_controller.go @@ -52,38 +52,6 @@ func newMetadataController(endpointsInformer coreinformers.EndpointsInformer, wm queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "endpoints"), } - go func() { - wmetaFilterParams := workloadmeta.FilterParams{ - Kinds: []workloadmeta.Kind{workloadmeta.KindKubernetesNode}, - Source: workloadmeta.SourceAll, - EventType: workloadmeta.EventTypeAll, - } - - wmetaEventsCh := wmeta.Subscribe( - "metadata-controller", - workloadmeta.NormalPriority, - workloadmeta.NewFilter(&wmetaFilterParams), - ) - defer wmeta.Unsubscribe(wmetaEventsCh) - - for eventBundle := range wmetaEventsCh { - eventBundle.Acknowledge() - - for _, event := range eventBundle.Events { - node := event.Entity.(*workloadmeta.KubernetesNode) - - switch event.Type { - case workloadmeta.EventTypeSet: - m.addNode(node.Name) - case workloadmeta.EventTypeUnset: - m.deleteNode(node.Name) - default: - log.Warnf("Unknown event type %v", event.Type) - } - } - } - }() - if _, err := endpointsInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: m.addEndpoints, UpdateFunc: m.updateEndpoints, @@ -110,6 +78,20 @@ func (m *metadataController) run(stopCh <-chan struct{}) { return } + wmetaFilterParams := workloadmeta.FilterParams{ + Kinds: []workloadmeta.Kind{workloadmeta.KindKubernetesNode}, + Source: workloadmeta.SourceAll, + EventType: workloadmeta.EventTypeAll, + } + + wmetaEventsCh := m.wmeta.Subscribe( + "metadata-controller", + workloadmeta.NormalPriority, + workloadmeta.NewFilter(&wmetaFilterParams), + ) + defer m.wmeta.Unsubscribe(wmetaEventsCh) + + go m.processWorkloadmetaNodeEvents(wmetaEventsCh) go wait.Until(m.worker, time.Second, stopCh) <-stopCh } @@ -134,6 +116,25 @@ func (m *metadataController) processNextWorkItem() bool { return true } +func (m *metadataController) processWorkloadmetaNodeEvents(wmetaEventsCh chan workloadmeta.EventBundle) { + for eventBundle := range wmetaEventsCh { + eventBundle.Acknowledge() + + for _, event := range eventBundle.Events { + node := event.Entity.(*workloadmeta.KubernetesNode) + + switch event.Type { + case workloadmeta.EventTypeSet: + m.addNode(node.Name) + case workloadmeta.EventTypeUnset: + m.deleteNode(node.Name) + default: + log.Warnf("Unknown event type %v", event.Type) + } + } + } +} + func (m *metadataController) addNode(name string) { bundle := m.store.getCopyOrNew(name) m.store.set(name, bundle) diff --git a/pkg/util/port/portlist/netstat_test.go b/pkg/util/port/portlist/netstat_test.go index bb9608551717f..d39801b71e66c 100644 --- a/pkg/util/port/portlist/netstat_test.go +++ b/pkg/util/port/portlist/netstat_test.go @@ -3,8 +3,8 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2014-present Datadog, Inc. -//go:build darwin && !ios -// +build darwin,!ios +//go:build darwin +// +build darwin package portlist diff --git a/pkg/util/port/portlist/poller.go b/pkg/util/port/portlist/poller.go index bd48452ca9a8a..6c6b56e1c4179 100644 --- a/pkg/util/port/portlist/poller.go +++ b/pkg/util/port/portlist/poller.go @@ -10,6 +10,7 @@ package portlist import ( "fmt" + "sort" "sync" "golang.org/x/exp/slices" @@ -93,3 +94,33 @@ func (p *Poller) getList() (List, error) { p.scratch, err = p.os.AppendListeningPorts(p.scratch[:0]) return p.scratch, err } + +func (a *Port) lessThan(b *Port) bool { + if a.Port != b.Port { + return a.Port < b.Port + } + if a.Proto != b.Proto { + return a.Proto < b.Proto + } + return a.Process < b.Process +} + +// sortAndDedup sorts ps in place (by Port.LessThan) and then returns +// a subset of it with duplicate (Proto, Port) removed. +// Multiple processes can't use the same port and protocol +// on the same port so there's no need to check for that +func sortAndDedup(ps List) List { + sort.Slice(ps, func(i, j int) bool { + return (&ps[i]).lessThan(&ps[j]) + }) + out := ps[:0] + var last Port + for _, p := range ps { + if last.Proto == p.Proto && last.Port == p.Port { + continue + } + out = append(out, p) + last = p + } + return out +} diff --git a/pkg/util/port/portlist/poller_linux.go b/pkg/util/port/portlist/poller_linux.go new file mode 100644 index 0000000000000..ee8c4cef20c33 --- /dev/null +++ b/pkg/util/port/portlist/poller_linux.go @@ -0,0 +1,459 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2014-present Datadog, Inc. + +//go:build linux + +package portlist + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "runtime" + "strings" + "syscall" + "unsafe" + + "go4.org/mem" + "golang.org/x/sys/unix" + + "github.com/DataDog/datadog-agent/pkg/util/log" +) + +func (p *Poller) init() { + p.os = newLinuxImpl(p.IncludeLocalhost) +} + +type linuxImpl struct { + procNetFiles []*os.File // seeked to start & reused between calls + readlinkPathBuf []byte + + known map[string]*portMeta // inode string => metadata + br *bufio.Reader + includeLocalhost bool +} + +type portMeta struct { + port Port + pid int + keep bool + needsProcName bool +} + +var eofReader = bytes.NewReader(nil) + +func newLinuxImplBase(includeLocalhost bool) *linuxImpl { + return &linuxImpl{ + br: bufio.NewReader(eofReader), + known: map[string]*portMeta{}, + includeLocalhost: includeLocalhost, + } +} + +func newLinuxImpl(includeLocalhost bool) osImpl { + li := newLinuxImplBase(includeLocalhost) + for _, name := range []string{ + "/proc/net/tcp", + "/proc/net/tcp6", + "/proc/net/udp", + "/proc/net/udp6", + } { + f, err := os.Open(name) + if err != nil { + if os.IsNotExist(err) { + continue + } + log.Errorf("diagnose port-conflict poller warning; ignoring: %v", err) + continue + } + li.procNetFiles = append(li.procNetFiles, f) + } + return li +} + +func (li *linuxImpl) Close() error { + for _, f := range li.procNetFiles { + f.Close() + } + li.procNetFiles = nil + return nil +} + +const ( + v6Localhost = "00000000000000000000000001000000:" + v6Any = "00000000000000000000000000000000:0000" + v4Localhost = "0100007F:" + v4Any = "00000000:0000" +) + +func (li *linuxImpl) AppendListeningPorts(base []Port) ([]Port, error) { + + br := li.br + defer br.Reset(eofReader) + + // Start by marking all previous known ports as gone. If this mark + // bit is still false later, we'll remove them. + for _, pm := range li.known { + pm.keep = false + } + + for _, f := range li.procNetFiles { + name := f.Name() + _, err := f.Seek(0, io.SeekStart) + if err != nil { + return nil, err + } + br.Reset(f) + err = li.parseProcNetFile(br, filepath.Base(name)) + if err != nil { + return nil, fmt.Errorf("parsing %q: %w", name, err) + } + } + + // Delete ports that aren't open any longer. + // And see if there are any process names we need to look for. + needProc := make(map[string]*portMeta) + for inode, pm := range li.known { + if !pm.keep { + delete(li.known, inode) + continue + } + if pm.needsProcName { + needProc[inode] = pm + } + } + err := li.findProcessNames(needProc) + if err != nil { + return nil, err + } + + ret := base + for _, pm := range li.known { + ret = append(ret, pm.port) + } + return sortAndDedup(ret), nil +} + +func (li *linuxImpl) parseProcNetFile(r *bufio.Reader, fileBase string) error { + proto := strings.TrimSuffix(fileBase, "6") + + // skip header row + _, err := r.ReadSlice('\n') + if err != nil { + return err + } + + fields := make([]mem.RO, 0, 20) // 17 current fields + some future slop + + wantRemote := mem.S(v4Any) + if strings.HasSuffix(fileBase, "6") { + wantRemote = mem.S(v6Any) + } + + // remoteIndex is the index within a line to the remote address field. + // -1 means not yet found. + remoteIndex := -1 + + // Add an upper bound on how many rows we'll attempt to read just + // to make sure this doesn't consume too much of their CPU. + // TODO(bradfitz,crawshaw): adaptively adjust polling interval as function + // of open sockets. + const maxRows = 1e6 + rows := 0 + + // Scratch buffer for making inode strings. + inoBuf := make([]byte, 0, 50) + + for { + line, err := r.ReadSlice('\n') + if err == io.EOF { + break + } + if err != nil { + return err + } + rows++ + if rows >= maxRows { + break + } + if len(line) == 0 { + continue + } + + // On the first row of output, find the index of the 3rd field (index 2), + // the remote address. All the rows are aligned, at least until 4 billion open + // TCP connections, per the Linux get_tcp4_sock's "%4d: " on an int i. + if remoteIndex == -1 { + remoteIndex = fieldIndex(line, 2) + if remoteIndex == -1 { + break + } + } + + if len(line) < remoteIndex || !mem.HasPrefix(mem.B(line).SliceFrom(remoteIndex), wantRemote) { + // Fast path for not being a listener port. + continue + } + + // sl local rem ... inode + fields = mem.AppendFields(fields[:0], mem.B(line)) + local := fields[1] + rem := fields[2] + inode := fields[9] + + if !rem.Equal(wantRemote) { + // not a "listener" port + continue + } + + // If a port is bound to localhost, ignore it. + // TODO: localhost is bigger than 1 IP, we need to ignore + // more things. + if !li.includeLocalhost && (mem.HasPrefix(local, mem.S(v4Localhost)) || mem.HasPrefix(local, mem.S(v6Localhost))) { + continue + } + + // Don't use strings.Split here, because it causes + // allocations significant enough to show up in profiles. + i := mem.IndexByte(local, ':') + if i == -1 { + return fmt.Errorf("%q unexpectedly didn't have a colon", local.StringCopy()) + } + portv, err := mem.ParseUint(local.SliceFrom(i+1), 16, 16) + if err != nil { + return fmt.Errorf("%#v: %s", local.SliceFrom(9).StringCopy(), err) + } + inoBuf = append(inoBuf[:0], "socket:["...) + inoBuf = mem.Append(inoBuf, inode) + inoBuf = append(inoBuf, ']') + + if pm, ok := li.known[string(inoBuf)]; ok { + pm.keep = true + // Rest should be unchanged. + } else { + li.known[string(inoBuf)] = &portMeta{ + needsProcName: true, + keep: true, + port: Port{ + Proto: proto, + Port: uint16(portv), + }, + } + } + } + + return nil +} + +// errDone is an internal sentinel error that we found everything we were looking for. +var errDone = errors.New("done") + +// need is keyed by inode string. +func (li *linuxImpl) findProcessNames(need map[string]*portMeta) error { + if len(need) == 0 { + return nil + } + defer func() { + // Anything we didn't find, give up on and don't try to look for it later. + for _, pm := range need { + pm.needsProcName = false + } + }() + + err := foreachPID(func(pid mem.RO) error { + var procBuf [128]byte + fdPath := mem.Append(procBuf[:0], mem.S("/proc/")) + fdPath = mem.Append(fdPath, pid) + fdPath = mem.Append(fdPath, mem.S("/fd")) + + // Android logs a bunch of audit violations in logcat + // if we try to open things we don't have access + // to. So on Android only, ask if we have permission + // rather than just trying it to determine whether we + // have permission. + if runtime.GOOS == "android" && syscall.Access(string(fdPath), unix.R_OK) != nil { + return nil + } + _ = walkShallow(mem.B(fdPath), func(fd mem.RO, de fs.DirEntry) error { + targetBuf := make([]byte, 64) // plenty big for "socket:[165614651]" + + linkPath := li.readlinkPathBuf[:0] + linkPath = fmt.Appendf(linkPath, "/proc/") + linkPath = mem.Append(linkPath, pid) + linkPath = append(linkPath, "/fd/"...) + linkPath = mem.Append(linkPath, fd) + linkPath = append(linkPath, 0) // terminating NUL + li.readlinkPathBuf = linkPath // to reuse its buffer next time + n, ok := readlink(linkPath, targetBuf) + if !ok { + // Not a symlink or no permission. + // Skip it. + return nil + } + + pe := need[string(targetBuf[:n])] // m[string([]byte)] avoids alloc + if pe == nil { + return nil + } + bs, err := os.ReadFile(fmt.Sprintf("/proc/%s/cmdline", pid.StringCopy())) + if err != nil { + // Usually shouldn't happen. One possibility is + // the process has gone away, so let's skip it. + return nil + } + + argv := strings.Split(strings.TrimSuffix(string(bs), "\x00"), "\x00") + if p, err := mem.ParseInt(pid, 10, 0); err == nil { + pe.pid = int(p) + } + pe.port.Process = argvSubject(argv...) + pid64, _ := mem.ParseInt(pid, 10, 0) + pe.port.Pid = int(pid64) + pe.needsProcName = false + delete(need, string(targetBuf[:n])) + if len(need) == 0 { + return errDone + } + return nil + }) + return nil + }) + if err == errDone { + return nil + } + return err +} + +func foreachPID(fn func(pidStr mem.RO) error) error { + err := walkShallow(mem.S("/proc"), func(name mem.RO, de fs.DirEntry) error { + if !isNumeric(name) { + return nil + } + return fn(name) + }) + if os.IsNotExist(err) { + // This can happen if the directory we're + // reading disappears during the run. No big + // deal. + return nil + } + return err +} + +func isNumeric(s mem.RO) bool { + for i, n := 0, s.Len(); i < n; i++ { + b := s.At(i) + if b < '0' || b > '9' { + return false + } + } + return s.Len() > 0 +} + +// fieldIndex returns the offset in line where the Nth field (0-based) begins, or -1 +// if there aren't that many fields. Fields are separated by 1 or more spaces. +func fieldIndex(line []byte, n int) int { + skip := 0 + for i := 0; i <= n; i++ { + // Skip spaces. + for skip < len(line) && line[skip] == ' ' { + skip++ + } + if skip == len(line) { + return -1 + } + if i == n { + break + } + // Skip non-space. + for skip < len(line) && line[skip] != ' ' { + skip++ + } + } + return skip +} + +// path must be null terminated. +func readlink(path, buf []byte) (n int, ok bool) { + if len(buf) == 0 || len(path) < 2 || path[len(path)-1] != 0 { + return 0, false + } + var dirfd int = unix.AT_FDCWD + r0, _, e1 := unix.Syscall6(unix.SYS_READLINKAT, + uintptr(dirfd), + uintptr(unsafe.Pointer(&path[0])), + uintptr(unsafe.Pointer(&buf[0])), + uintptr(len(buf)), + 0, 0) + n = int(r0) + if e1 != 0 { + return 0, false + } + return n, true +} + +// argvSubject takes a command and its flags, and returns the +// short/pretty name for the process. This is usually the basename of +// the binary being executed, but can sometimes vary (e.g. so that we +// don't report all Java programs as "java"). +func argvSubject(argv ...string) string { + if len(argv) == 0 { + return "" + } + ret := filepath.Base(argv[0]) + + // Handle special cases. + switch { + case ret == "mono" && len(argv) >= 2: + // .Net programs execute as `mono actualProgram.exe`. + ret = filepath.Base(argv[1]) + } + + // Handle space separated argv + ret, _, _ = strings.Cut(ret, " ") + + // Remove common noise. + ret = strings.TrimSpace(ret) + ret = strings.TrimSuffix(ret, ".exe") + + return ret +} + +// walkFunc is the callback type used with walkShallow. +// +// The name and de are only valid for the duration of func's call +// and should not be retained. +type walkFunc func(name mem.RO, de fs.DirEntry) error + +// walkShallow reads the entries in the named directory and calls fn for each. +// It does not recurse into subdirectories. +// +// If fn returns an error, iteration stops and walkShallow returns that value. +func walkShallow(dirName mem.RO, fn walkFunc) error { + of, err := os.Open(dirName.StringCopy()) + if err != nil { + return err + } + defer of.Close() + for { + fis, err := of.ReadDir(100) + for _, de := range fis { + if err := fn(mem.S(de.Name()), de); err != nil { + return err + } + } + if err != nil { + if err == io.EOF { + return nil + } + return err + } + } +} diff --git a/pkg/util/port/portlist/poller_linux_test.go b/pkg/util/port/portlist/poller_linux_test.go new file mode 100644 index 0000000000000..d3a96d6107130 --- /dev/null +++ b/pkg/util/port/portlist/poller_linux_test.go @@ -0,0 +1,257 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2014-present Datadog, Inc. + +package portlist + +import ( + "bufio" + "bytes" + "fmt" + "io" + "os" + "path/filepath" + "reflect" + "sort" + "testing" + + "github.com/google/go-cmp/cmp" + "go4.org/mem" +) + +func TestFieldIndex(t *testing.T) { + tests := []struct { + in string + field int + want int + }{ + {"foo", 0, 0}, + {" foo", 0, 2}, + {"foo bar", 1, 5}, + {" foo bar", 1, 6}, + {" foo bar", 2, -1}, + {" foo bar ", 2, -1}, + {" foo bar x", 2, 10}, + {" 1: 00000000:0016 00000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 34062 1 0000000000000000 100 0 0 10 0", + 2, 19}, + } + for _, tt := range tests { + if got := fieldIndex([]byte(tt.in), tt.field); got != tt.want { + t.Errorf("fieldIndex(%q, %v) = %v; want %v", tt.in, tt.field, got, tt.want) + } + } +} + +func TestParsePorts(t *testing.T) { + tests := []struct { + name string + in string + file string + want map[string]*portMeta + }{ + { + name: "empty", + in: "header line (ignored)\n", + want: map[string]*portMeta{}, + }, + { + name: "ipv4", + file: "tcp", + in: `header line + 0: 0100007F:0277 00000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 22303 1 0000000000000000 100 0 0 10 0 + 1: 00000000:0016 00000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 34062 1 0000000000000000 100 0 0 10 0 + 2: 5501A8C0:ADD4 B25E9536:01BB 01 00000000:00000000 02:00000B2B 00000000 1000 0 155276677 2 0000000000000000 22 4 30 10 -1 +`, + want: map[string]*portMeta{ + "socket:[34062]": { + port: Port{Proto: "tcp", Port: 22}, + }, + }, + }, + { + name: "ipv6", + file: "tcp6", + in: ` sl local_address remote_address st tx_queue rx_queue tr tm->when retrnsmt uid timeout inode + 0: 00000000000000000000000001000000:0277 00000000000000000000000000000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 35720 1 0000000000000000 100 0 0 10 0 + 1: 00000000000000000000000000000000:1F91 00000000000000000000000000000000:0000 0A 00000000:00000000 00:00000000 00000000 1000 0 142240557 1 0000000000000000 100 0 0 10 0 + 2: 00000000000000000000000000000000:0016 00000000000000000000000000000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 34064 1 0000000000000000 100 0 0 10 0 + 3: 69050120005716BC64906EBE009ECD4D:D506 0047062600000000000000006E171268:01BB 01 00000000:00000000 02:0000009E 00000000 1000 0 151042856 2 0000000000000000 21 4 28 10 -1 +`, + want: map[string]*portMeta{ + "socket:[142240557]": { + port: Port{Proto: "tcp", Port: 8081}, + }, + "socket:[34064]": { + port: Port{Proto: "tcp", Port: 22}, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + buf := bytes.NewBufferString(tt.in) + r := bufio.NewReader(buf) + file := "tcp" + if tt.file != "" { + file = tt.file + } + li := newLinuxImplBase(false) + err := li.parseProcNetFile(r, file) + if err != nil { + t.Fatal(err) + } + for _, pm := range tt.want { + pm.keep = true + pm.needsProcName = true + } + if diff := cmp.Diff(li.known, tt.want, cmp.AllowUnexported(Port{}), cmp.AllowUnexported(portMeta{})); diff != "" { + t.Errorf("unexpected parsed ports (-got+want):\n%s", diff) + } + }) + } +} + +func BenchmarkParsePorts(b *testing.B) { + b.ReportAllocs() + + var contents bytes.Buffer + contents.WriteString(` sl local_address remote_address st tx_queue rx_queue tr tm->when retrnsmt uid timeout inode + 0: 00000000000000000000000001000000:0277 00000000000000000000000000000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 35720 1 0000000000000000 100 0 0 10 0 + 1: 00000000000000000000000000000000:1F91 00000000000000000000000000000000:0000 0A 00000000:00000000 00:00000000 00000000 1000 0 142240557 1 0000000000000000 100 0 0 10 0 + 2: 00000000000000000000000000000000:0016 00000000000000000000000000000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 34064 1 0000000000000000 100 0 0 10 0 +`) + for i := 0; i < 50000; i++ { + contents.WriteString(" 3: 69050120005716BC64906EBE009ECD4D:D506 0047062600000000000000006E171268:01BB 01 00000000:00000000 02:0000009E 00000000 1000 0 151042856 2 0000000000000000 21 4 28 10 -1\n") + } + + li := newLinuxImplBase(false) + + r := bytes.NewReader(contents.Bytes()) + br := bufio.NewReader(&contents) + b.ResetTimer() + for i := 0; i < b.N; i++ { + r.Seek(0, io.SeekStart) + br.Reset(r) + err := li.parseProcNetFile(br, "tcp6") + if err != nil { + b.Fatal(err) + } + if len(li.known) != 2 { + b.Fatalf("wrong results; want 2 parsed got %d", len(li.known)) + } + } +} + +func BenchmarkFindProcessNames(b *testing.B) { + b.ReportAllocs() + li := &linuxImpl{} + need := map[string]*portMeta{ + "something-we'll-never-find": new(portMeta), + } + for i := 0; i < b.N; i++ { + if err := li.findProcessNames(need); err != nil { + b.Fatal(err) + } + } +} + +func TestWalkShallow(t *testing.T) { + d := t.TempDir() + + t.Run("basics", func(t *testing.T) { + if err := os.WriteFile(filepath.Join(d, "foo"), []byte("1"), 0600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(d, "bar"), []byte("22"), 0400); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(filepath.Join(d, "baz"), 0777); err != nil { + t.Fatal(err) + } + + var got []string + if err := walkShallow(mem.S(d), func(name mem.RO, de os.DirEntry) error { + var size int64 + if fi, err := de.Info(); err != nil { + t.Errorf("Info stat error on %q: %v", de.Name(), err) + } else if !fi.IsDir() { + size = fi.Size() + } + got = append(got, fmt.Sprintf("%q %q dir=%v type=%d size=%v", name.StringCopy(), de.Name(), de.IsDir(), de.Type(), size)) + return nil + }); err != nil { + t.Fatal(err) + } + sort.Strings(got) + want := []string{ + `"bar" "bar" dir=false type=0 size=2`, + `"baz" "baz" dir=true type=2147483648 size=0`, + `"foo" "foo" dir=false type=0 size=1`, + } + if !reflect.DeepEqual(got, want) { + t.Errorf("mismatch:\n got %#q\nwant %#q", got, want) + } + }) + + t.Run("err_not_exist", func(t *testing.T) { + err := walkShallow(mem.S(filepath.Join(d, "not_exist")), func(name mem.RO, de os.DirEntry) error { + return nil + }) + if !os.IsNotExist(err) { + t.Errorf("unexpected error: %v", err) + } + }) +} + +func TestArgvSubject(t *testing.T) { + tests := []struct { + in []string + want string + }{ + { + in: nil, + want: "", + }, + { + in: []string{"/usr/bin/sshd"}, + want: "sshd", + }, + { + in: []string{"/bin/mono"}, + want: "mono", + }, + { + in: []string{"/nix/store/x2cw2xjw98zdysf56bdlfzsr7cyxv0jf-mono-5.20.1.27/bin/mono", "/bin/exampleProgram.exe"}, + want: "exampleProgram", + }, + { + in: []string{"/bin/mono", "/sbin/exampleProgram.bin"}, + want: "exampleProgram.bin", + }, + { + in: []string{"/usr/bin/sshd_config [listener] 1 of 10-100 startups"}, + want: "sshd_config", + }, + { + in: []string{"/usr/bin/sshd [listener] 0 of 10-100 startups"}, + want: "sshd", + }, + { + in: []string{"/opt/aws/bin/eic_run_authorized_keys %u %f -o AuthorizedKeysCommandUser ec2-instance-connect [listener] 0 of 10-100 startups"}, + want: "eic_run_authorized_keys", + }, + { + in: []string{"/usr/bin/nginx worker"}, + want: "nginx", + }, + } + + for _, test := range tests { + got := argvSubject(test.in...) + if got != test.want { + t.Errorf("argvSubject(%v) = %q, want %q", test.in, got, test.want) + } + } +} diff --git a/pkg/util/port/portlist/poller_macos.go b/pkg/util/port/portlist/poller_macos.go index ece0a7246a6d7..cbefe70562b15 100644 --- a/pkg/util/port/portlist/poller_macos.go +++ b/pkg/util/port/portlist/poller_macos.go @@ -11,13 +11,13 @@ import ( "bufio" "bytes" "fmt" - "log" "os/exec" - "sort" "strings" "go.uber.org/atomic" "go4.org/mem" + + "github.com/DataDog/datadog-agent/pkg/util/log" ) // init initializes the Poller by ensuring it has an underlying @@ -156,14 +156,14 @@ func (im *macOSImpl) addProcesses() error { } // fails when run in a macOS sandbox, so make this non-fatal. if lsofFailed.CompareAndSwap(false, true) { - log.Printf("portlist: can't run lsof in Mac sandbox; omitting process names from service list. Error details: %v, %s", err, bytes.TrimSpace(stderr)) + log.Errorf("diagnose port-conflict poller: can't run lsof in Mac sandbox; omitting process names from service list. Error details: %v, %s", err, bytes.TrimSpace(stderr)) } return nil } defer func() { ps, err := lsofCmd.Process.Wait() if err != nil || ps.ExitCode() != 0 { - log.Printf("portlist: can't run lsof in Mac sandbox; omitting process names from service list. Error: %v, exit code %d", err, ps.ExitCode()) + log.Errorf("diagnose port-conflict poller: can't run lsof in Mac sandbox; omitting process names from service list. Error: %v, exit code %d", err, ps.ExitCode()) lsofFailed.Store(true) } }() @@ -229,16 +229,6 @@ func lsofProtoLower(p []byte) string { return strings.ToLower(string(p)) } -func (a *Port) lessThan(b *Port) bool { - if a.Port != b.Port { - return a.Port < b.Port - } - if a.Proto != b.Proto { - return a.Proto < b.Proto - } - return a.Process < b.Process -} - func (a List) String() string { var sb strings.Builder for _, v := range a { @@ -247,21 +237,3 @@ func (a List) String() string { } return strings.TrimRight(sb.String(), "\n") } - -// sortAndDedup sorts ps in place (by Port.LessThan) and then returns -// a subset of it with duplicate (Proto, Port) removed. -func sortAndDedup(ps List) List { - sort.Slice(ps, func(i, j int) bool { - return (&ps[i]).lessThan(&ps[j]) - }) - out := ps[:0] - var last Port - for _, p := range ps { - if last.Proto == p.Proto && last.Port == p.Port { - continue - } - out = append(out, p) - last = p - } - return out -} diff --git a/pkg/util/port/portlist/poller_macos_test.go b/pkg/util/port/portlist/poller_macos_test.go index 8859d4b106f6f..2f4b967eadbf5 100644 --- a/pkg/util/port/portlist/poller_macos_test.go +++ b/pkg/util/port/portlist/poller_macos_test.go @@ -8,185 +8,9 @@ package portlist import ( - "net" "testing" ) -func TestGetList(t *testing.T) { - var p Poller - pl, _, err := p.Poll() - if err != nil { - t.Fatal(err) - } - for i, p := range pl { - t.Logf("[%d] %+v", i, p) - } - t.Logf("As String: %s", List(pl)) -} - -func TestIgnoreLocallyBoundPorts(t *testing.T) { - ln, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - t.Skipf("failed to bind: %v", err) - } - defer ln.Close() - ta := ln.Addr().(*net.TCPAddr) - port := ta.Port - var p Poller - pl, _, err := p.Poll() - if err != nil { - t.Fatal(err) - } - for _, p := range pl { - if p.Proto == "tcp" && int(p.Port) == port { - t.Fatal("didn't expect to find test's localhost ephemeral port") - } - } -} - -func TestPoller(t *testing.T) { - var p Poller - p.IncludeLocalhost = true - get := func(t *testing.T) []Port { - t.Helper() - s, _, err := p.Poll() - if err != nil { - t.Fatal(err) - } - return s - } - - p1 := get(t) - ln, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - t.Skipf("failed to bind: %v", err) - } - defer ln.Close() - port := uint16(ln.Addr().(*net.TCPAddr).Port) - containsPort := func(pl List) bool { - for _, p := range pl { - if p.Proto == "tcp" && p.Port == port { - return true - } - } - return false - } - if containsPort(p1) { - t.Error("unexpectedly found ephemeral port in p1, before it was opened", port) - } - p2 := get(t) - if !containsPort(p2) { - t.Error("didn't find ephemeral port in p2", port) - } - ln.Close() - p3 := get(t) - if containsPort(p3) { - t.Error("unexpectedly found ephemeral port in p3, after it was closed", port) - } -} - -func TestEqualLessThan(t *testing.T) { - tests := []struct { - name string - a, b Port - want bool - }{ - { - "Port a < b", - Port{Proto: "tcp", Port: 100, Process: "proc1"}, - Port{Proto: "tcp", Port: 101, Process: "proc1"}, - true, - }, - { - "Port a > b", - Port{Proto: "tcp", Port: 101, Process: "proc1"}, - Port{Proto: "tcp", Port: 100, Process: "proc1"}, - false, - }, - { - "Proto a < b", - Port{Proto: "tcp", Port: 100, Process: "proc1"}, - Port{Proto: "udp", Port: 100, Process: "proc1"}, - true, - }, - { - "Proto a < b", - Port{Proto: "udp", Port: 100, Process: "proc1"}, - Port{Proto: "tcp", Port: 100, Process: "proc1"}, - false, - }, - { - "Process a < b", - Port{Proto: "tcp", Port: 100, Process: "proc1"}, - Port{Proto: "tcp", Port: 100, Process: "proc2"}, - true, - }, - { - "Process a > b", - Port{Proto: "tcp", Port: 100, Process: "proc2"}, - Port{Proto: "tcp", Port: 100, Process: "proc1"}, - false, - }, - { - "Port evaluated first", - Port{Proto: "udp", Port: 100, Process: "proc2"}, - Port{Proto: "tcp", Port: 101, Process: "proc1"}, - true, - }, - { - "Proto evaluated second", - Port{Proto: "tcp", Port: 100, Process: "proc2"}, - Port{Proto: "udp", Port: 100, Process: "proc1"}, - true, - }, - { - "Process evaluated fourth", - Port{Proto: "tcp", Port: 100, Process: "proc1"}, - Port{Proto: "tcp", Port: 100, Process: "proc2"}, - true, - }, - { - "equal", - Port{Proto: "tcp", Port: 100, Process: "proc1"}, - Port{Proto: "tcp", Port: 100, Process: "proc1"}, - false, - }, - } - - for _, tt := range tests { - got := tt.a.lessThan(&tt.b) - if got != tt.want { - t.Errorf("%s: Equal = %v; want %v", tt.name, got, tt.want) - } - lessBack := tt.b.lessThan(&tt.a) - if got && lessBack { - t.Errorf("%s: both a and b report being less than each other", tt.name) - } - wantEqual := !got && !lessBack - gotEqual := tt.a.equal(&tt.b) - if gotEqual != wantEqual { - t.Errorf("%s: equal = %v; want %v", tt.name, gotEqual, wantEqual) - } - } -} - -func TestClose(t *testing.T) { - var p Poller - err := p.Close() - if err != nil { - t.Fatal(err) - } - p = Poller{} - _, _, err = p.Poll() - if err != nil { - t.Skipf("skipping due to poll error: %v", err) - } - err = p.Close() - if err != nil { - t.Fatal(err) - } -} - func BenchmarkGetList(b *testing.B) { benchmarkGetList(b, false) } diff --git a/pkg/util/port/portlist/poller_test.go b/pkg/util/port/portlist/poller_test.go new file mode 100644 index 0000000000000..ff2df90f47e4b --- /dev/null +++ b/pkg/util/port/portlist/poller_test.go @@ -0,0 +1,256 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2014-present Datadog, Inc. + +package portlist + +import ( + "net" + "reflect" + "runtime" + "testing" +) + +func TestEqualLessThan(t *testing.T) { + tests := []struct { + name string + a, b Port + want bool + }{ + { + "Port a < b", + Port{Proto: "tcp", Port: 100, Process: "proc1"}, + Port{Proto: "tcp", Port: 101, Process: "proc1"}, + true, + }, + { + "Port a > b", + Port{Proto: "tcp", Port: 101, Process: "proc1"}, + Port{Proto: "tcp", Port: 100, Process: "proc1"}, + false, + }, + { + "Proto a < b", + Port{Proto: "tcp", Port: 100, Process: "proc1"}, + Port{Proto: "udp", Port: 100, Process: "proc1"}, + true, + }, + { + "Proto a < b", + Port{Proto: "udp", Port: 100, Process: "proc1"}, + Port{Proto: "tcp", Port: 100, Process: "proc1"}, + false, + }, + { + "Process a < b", + Port{Proto: "tcp", Port: 100, Process: "proc1"}, + Port{Proto: "tcp", Port: 100, Process: "proc2"}, + true, + }, + { + "Process a > b", + Port{Proto: "tcp", Port: 100, Process: "proc2"}, + Port{Proto: "tcp", Port: 100, Process: "proc1"}, + false, + }, + { + "Port evaluated first", + Port{Proto: "udp", Port: 100, Process: "proc2"}, + Port{Proto: "tcp", Port: 101, Process: "proc1"}, + true, + }, + { + "Proto evaluated second", + Port{Proto: "tcp", Port: 100, Process: "proc2"}, + Port{Proto: "udp", Port: 100, Process: "proc1"}, + true, + }, + { + "Process evaluated fourth", + Port{Proto: "tcp", Port: 100, Process: "proc1"}, + Port{Proto: "tcp", Port: 100, Process: "proc2"}, + true, + }, + { + "equal", + Port{Proto: "tcp", Port: 100, Process: "proc1"}, + Port{Proto: "tcp", Port: 100, Process: "proc1"}, + false, + }, + } + + for _, tt := range tests { + got := tt.a.lessThan(&tt.b) + if got != tt.want { + t.Errorf("%s: Equal = %v; want %v", tt.name, got, tt.want) + } + lessBack := tt.b.lessThan(&tt.a) + if got && lessBack { + t.Errorf("%s: both a and b report being less than each other", tt.name) + } + wantEqual := !got && !lessBack + gotEqual := tt.a.equal(&tt.b) + if gotEqual != wantEqual { + t.Errorf("%s: equal = %v; want %v", tt.name, gotEqual, wantEqual) + } + } +} + +func TestSortAndDedup(t *testing.T) { + tests := []struct { + name string + input List + expected List + }{ + { + "Simple Case", + List{ + {Port: 80, Proto: "tcp", Process: "nginx"}, + {Port: 443, Proto: "tcp", Process: "nginx"}, + {Port: 80, Proto: "tcp", Process: "apache"}, + {Port: 80, Proto: "udp", Process: "apache"}, + {Port: 443, Proto: "tcp", Process: "nginx"}, + }, + List{ + {Port: 80, Proto: "tcp", Process: "apache"}, + {Port: 80, Proto: "udp", Process: "apache"}, + {Port: 443, Proto: "tcp", Process: "nginx"}, + }, + }, + { + "Already Sorted", + List{ + {Port: 22, Proto: "tcp", Process: "ssh"}, + {Port: 80, Proto: "tcp", Process: "nginx"}, + {Port: 443, Proto: "tcp", Process: "nginx"}, + }, + List{ + {Port: 22, Proto: "tcp", Process: "ssh"}, + {Port: 80, Proto: "tcp", Process: "nginx"}, + {Port: 443, Proto: "tcp", Process: "nginx"}, + }, + }, + { + "No Duplicates", + List{ + {Port: 80, Proto: "tcp", Process: "nginx"}, + {Port: 8080, Proto: "tcp", Process: "node"}, + }, + List{ + {Port: 80, Proto: "tcp", Process: "nginx"}, + {Port: 8080, Proto: "tcp", Process: "node"}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := sortAndDedup(tt.input) + if !reflect.DeepEqual(result, tt.expected) { + t.Errorf("sortAndDedup(%v) = %v; want %v", tt.input, result, tt.expected) + } + }) + } +} + +func TestGetList(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Skipping test on Windows -- not implemented yet") + } + var p Poller + pl, _, err := p.Poll() + if err != nil { + t.Fatal(err) + } + for i, p := range pl { + t.Logf("[%d] %+v", i, p) + } +} + +func TestIgnoreLocallyBoundPorts(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Skipping test on Windows -- not implemented yet") + } + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Skipf("failed to bind: %v", err) + } + defer ln.Close() + ta := ln.Addr().(*net.TCPAddr) + port := ta.Port + var p Poller + pl, _, err := p.Poll() + if err != nil { + t.Fatal(err) + } + for _, p := range pl { + if p.Proto == "tcp" && int(p.Port) == port { + t.Fatal("didn't expect to find test's localhost ephemeral port") + } + } +} + +func TestPoller(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Skipping test on Windows -- not implemented yet") + } + var p Poller + p.IncludeLocalhost = true + get := func(t *testing.T) []Port { + t.Helper() + s, _, err := p.Poll() + if err != nil { + t.Fatal(err) + } + return s + } + + p1 := get(t) + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Skipf("failed to bind: %v", err) + } + defer ln.Close() + port := uint16(ln.Addr().(*net.TCPAddr).Port) + containsPort := func(pl List) bool { + for _, p := range pl { + if p.Proto == "tcp" && p.Port == port { + return true + } + } + return false + } + if containsPort(p1) { + t.Error("unexpectedly found ephemeral port in p1, before it was opened", port) + } + p2 := get(t) + if !containsPort(p2) { + t.Error("didn't find ephemeral port in p2", port) + } + ln.Close() + p3 := get(t) + if containsPort(p3) { + t.Error("unexpectedly found ephemeral port in p3, after it was closed", port) + } +} + +func TestClose(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Skipping test on Windows -- not implemented yet") + } + var p Poller + err := p.Close() + if err != nil { + t.Fatal(err) + } + p = Poller{} + _, _, err = p.Poll() + if err != nil { + t.Skipf("skipping due to poll error: %v", err) + } + err = p.Close() + if err != nil { + t.Fatal(err) + } +} diff --git a/pkg/util/port/portlist/poller_not_macos.go b/pkg/util/port/portlist/poller_windows.go similarity index 68% rename from pkg/util/port/portlist/poller_not_macos.go rename to pkg/util/port/portlist/poller_windows.go index e4be29de87776..ec33188b84ddf 100644 --- a/pkg/util/port/portlist/poller_not_macos.go +++ b/pkg/util/port/portlist/poller_windows.go @@ -3,11 +3,13 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -//go:build !darwin +//go:build windows package portlist -import "errors" +import ( + "errors" +) // ErrNotImplemented is the "not implemented" error given by `gopsutil` when an // OS doesn't support and API. Unfortunately it's in an internal package so @@ -16,21 +18,21 @@ var ErrNotImplemented = errors.New("not implemented yet") // init initializes the Poller by ensuring it has an underlying func (p *Poller) init() { - p.os = newOtherOSImpl(p.IncludeLocalhost) + p.os = newWindowsOSImpl(p.IncludeLocalhost) } -type otherOSImpl struct { +type windowsOSImpl struct { includeLocalhost bool } -func newOtherOSImpl(includeLocalhost bool) osImpl { - return &otherOSImpl{ +func newWindowsOSImpl(includeLocalhost bool) osImpl { + return &windowsOSImpl{ includeLocalhost: includeLocalhost, } } -func (im *otherOSImpl) AppendListeningPorts(_ []Port) ([]Port, error) { +func (im *windowsOSImpl) AppendListeningPorts(_ []Port) ([]Port, error) { return nil, ErrNotImplemented } -func (*otherOSImpl) Close() error { return ErrNotImplemented } +func (*windowsOSImpl) Close() error { return ErrNotImplemented } diff --git a/release.json b/release.json index 9bdfb7196e050..4251bc6fb9352 100644 --- a/release.json +++ b/release.json @@ -3,7 +3,7 @@ "current_milestone": "7.55.0", "last_stable": { "6": "6.53.0", - "7": "7.53.0" + "7": "7.54.0" }, "nightly": { "INTEGRATIONS_CORE_VERSION": "master", diff --git a/releasenotes/notes/add-djm-enabled-config-667dd030d0bf7d83.yaml b/releasenotes/notes/add-djm-enabled-config-667dd030d0bf7d83.yaml new file mode 100644 index 0000000000000..64d6d4a623a0a --- /dev/null +++ b/releasenotes/notes/add-djm-enabled-config-667dd030d0bf7d83.yaml @@ -0,0 +1,11 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +features: + - | + Add new parameter ``djm_config.enabled`` to enable Data Jobs Monitoring diff --git a/releasenotes/notes/opentelemetry-resource-attributes-ust-e379136c3a5c8046.yaml b/releasenotes/notes/opentelemetry-resource-attributes-ust-e379136c3a5c8046.yaml new file mode 100644 index 0000000000000..8b649dcce38ce --- /dev/null +++ b/releasenotes/notes/opentelemetry-resource-attributes-ust-e379136c3a5c8046.yaml @@ -0,0 +1,12 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +features: + - | + Implement OpenTelemetry SDK resource attributes as unified service rags. + diff --git a/releasenotes/notes/process-statsd-client-fix-66b11cc991a9f6be.yaml b/releasenotes/notes/process-statsd-client-fix-66b11cc991a9f6be.yaml new file mode 100644 index 0000000000000..922c3399d7898 --- /dev/null +++ b/releasenotes/notes/process-statsd-client-fix-66b11cc991a9f6be.yaml @@ -0,0 +1,11 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +fixes: + - | + Fix metric reporting for process checks on the core Agent. diff --git a/tasks/__init__.py b/tasks/__init__.py index 01489d7b35d2d..eb48c9bee323b 100644 --- a/tasks/__init__.py +++ b/tasks/__init__.py @@ -6,7 +6,6 @@ from tasks import ( agent, - agentless_scanner, bench, buildimages, cluster_agent, @@ -22,6 +21,7 @@ emacs, epforwarder, fakeintake, + git, github_tasks, go_deps, installer, @@ -134,7 +134,6 @@ # add namespaced tasks to the root ns.add_collection(agent) -ns.add_collection(agentless_scanner) ns.add_collection(buildimages) ns.add_collection(cluster_agent) ns.add_collection(cluster_agent_cloudfoundry) @@ -150,6 +149,7 @@ ns.add_collection(go_deps) ns.add_collection(linter) ns.add_collection(msi) +ns.add_collection(git) ns.add_collection(github_tasks, "github") ns.add_collection(package) ns.add_collection(pipeline) diff --git a/tasks/agentless_scanner.py b/tasks/agentless_scanner.py deleted file mode 100644 index 0fb6cc19ef4d9..0000000000000 --- a/tasks/agentless_scanner.py +++ /dev/null @@ -1,95 +0,0 @@ -""" -Agentless-scanner tasks -""" - -import os -import sys - -from invoke import task -from invoke.exceptions import Exit - -from tasks.build_tags import filter_incompatible_tags, get_build_tags, get_default_build_tags -from tasks.flavor import AgentFlavor -from tasks.libs.common.utils import REPO_PATH, bin_name, get_build_flags - -# constants -AGENTLESS_SCANNER_BIN_PATH = os.path.join(".", "bin", "agentless-scanner") -STATIC_BIN_PATH = os.path.join(".", "bin", "static") - - -@task -def build( - ctx, - rebuild=False, - race=False, - static=False, - build_include=None, - build_exclude=None, - major_version='7', - go_mod="mod", -): - """ - Build Agentless-scanner - """ - build_include = ( - get_default_build_tags(build="agentless-scanner", flavor=AgentFlavor.agentless_scanner) - if build_include is None - else filter_incompatible_tags(build_include.split(",")) - ) - build_exclude = [] if build_exclude is None else build_exclude.split(",") - build_tags = get_build_tags(build_include, build_exclude) - ldflags, gcflags, env = get_build_flags(ctx, static=static, major_version=major_version) - bin_path = AGENTLESS_SCANNER_BIN_PATH - - if static: - bin_path = STATIC_BIN_PATH - - # NOTE: consider stripping symbols to reduce binary size - cmd = "go build -mod={go_mod} {race_opt} {build_type} -tags \"{build_tags}\" -o {bin_name} " - cmd += "-gcflags=\"{gcflags}\" -ldflags=\"{ldflags}\" {REPO_PATH}/cmd/agentless-scanner" - args = { - "go_mod": go_mod, - "race_opt": "-race" if race else "", - "build_type": "-a" if rebuild else "", - "build_tags": " ".join(build_tags), - "bin_name": os.path.join(bin_path, bin_name("agentless-scanner")), - "gcflags": gcflags, - "ldflags": ldflags, - "REPO_PATH": REPO_PATH, - } - ctx.run(cmd.format(**args), env=env) - - # Render the configuration file template - # - # We need to remove cross compiling bits if any because go generate must - # build and execute in the native platform - env = { - "GOOS": "", - "GOARCH": "", - } - cmd = "go generate -mod={} {}/cmd/agentless-scanner" - ctx.run(cmd.format(go_mod, REPO_PATH), env=env) - - if static and sys.platform.startswith("linux"): - cmd = "file {bin_name} " - args = { - "bin_name": os.path.join(bin_path, bin_name("agentless-scanner")), - } - result = ctx.run(cmd.format(**args)) - if "statically linked" not in result.stdout: - print("agentless-scanner binary is not static, exiting...") - raise Exit(code=1) - - -@task -def clean(ctx): - """ - Remove temporary objects and binary artifacts - """ - # go clean - print("Executing go clean") - ctx.run("go clean") - - # remove the bin/agentless-scanner folder - print("Remove agentless-scanner binary folder") - ctx.run("rm -rf ./bin/agentless-scanner") diff --git a/tasks/build_tags.py b/tasks/build_tags.py index 0c5b71e4a5354..e4c50a2ed7741 100644 --- a/tasks/build_tags.py +++ b/tasks/build_tags.py @@ -99,9 +99,6 @@ } ) -# AGENTLESS_SCANNER_TAGS lists the tags needed when building the agentless-scanner -AGENTLESS_SCANNER_TAGS = {""} - # CLUSTER_AGENT_TAGS lists the tags needed when building the cluster-agent CLUSTER_AGENT_TAGS = {"clusterchecks", "datadog.no_waf", "kubeapiserver", "orchestrator", "zlib", "zstd", "ec2", "gce"} @@ -216,12 +213,6 @@ "lint": DOGSTATSD_TAGS.union(UNIT_TEST_TAGS).difference(UNIT_TEST_EXCLUDE_TAGS), "unit-tests": DOGSTATSD_TAGS.union(UNIT_TEST_TAGS).difference(UNIT_TEST_EXCLUDE_TAGS), }, - AgentFlavor.agentless_scanner: { - "dogstatsd": AGENTLESS_SCANNER_TAGS, - "system-tests": AGENT_TAGS, - "lint": AGENTLESS_SCANNER_TAGS.union(UNIT_TEST_TAGS).difference(UNIT_TEST_EXCLUDE_TAGS), - "unit-tests": AGENTLESS_SCANNER_TAGS.union(UNIT_TEST_TAGS).difference(UNIT_TEST_EXCLUDE_TAGS), - }, } diff --git a/tasks/components.py b/tasks/components.py index efad8fc60057c..3ba670bff997c 100644 --- a/tasks/components.py +++ b/tasks/components.py @@ -55,7 +55,6 @@ def has_type_component(content): "comp/metadata/inventoryagent/component.go", "comp/netflow/config/component.go", "comp/netflow/server/component.go", - "comp/otelcol/collector/component.go", "comp/remote-config/rcclient/component.go", "comp/trace/agent/component.go", "comp/trace/config/component.go", @@ -467,10 +466,6 @@ def lint_fxutil_oneshot_test(_): if str(file).endswith("_test.go"): continue - excluded_cmds = ["agentless-scanner"] - if file.parts[0] == "cmd" and file.parts[1] in excluded_cmds: - continue - one_shot_count = file.read_text().count("fxutil.OneShot(") run_count = file.read_text().count("fxutil.Run(") diff --git a/tasks/flavor.py b/tasks/flavor.py index 24e377f3f8a0e..16399f84821d3 100644 --- a/tasks/flavor.py +++ b/tasks/flavor.py @@ -7,7 +7,6 @@ class AgentFlavor(enum.Enum): iot = 2 heroku = 3 dogstatsd = 4 - agentless_scanner = 5 def is_iot(self): return self == type(self).iot diff --git a/tasks/git-hooks/copyright.py b/tasks/git-hooks/copyright.py deleted file mode 100644 index d43e759067d10..0000000000000 --- a/tasks/git-hooks/copyright.py +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env python3 - -import sys - -from tasks.libs.types.copyright import CopyrightLinter, LintFailure # noqa: E402 - -# Exclude non go files -files = [path for path in sys.argv[1:] if path.endswith(".go")] -try: - CopyrightLinter().assert_compliance(files=files) -except LintFailure: - # the linter prints useful messages on its own, so no need to print the exception - sys.exit(1) diff --git a/tasks/git-hooks/protected-branches.py b/tasks/git-hooks/protected-branches.py deleted file mode 100644 index 992b1794178c9..0000000000000 --- a/tasks/git-hooks/protected-branches.py +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/python3 - -import re -import subprocess -import sys - - -def main(): - try: - local_branch = subprocess.check_output('git rev-parse --abbrev-ref HEAD', shell=True).decode('utf-8').strip() - if local_branch == 'main': - print("You're about to commit on main, are you sure this is what you want?") - sys.exit(1) - if re.fullmatch(r'^[0-9]+\.[0-9]+\.x$', local_branch): - print("You're about to commit on a release branch, are you sure this is what you want?") - sys.exit(1) - except OSError as e: - print(e) - sys.exit(1) - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/tasks/git.py b/tasks/git.py new file mode 100644 index 0000000000000..003fb8e2f919a --- /dev/null +++ b/tasks/git.py @@ -0,0 +1,20 @@ +import re + +from invoke import task +from invoke.exceptions import Exit + +from tasks.libs.common.color import color_message +from tasks.libs.common.git import get_current_branch + + +@task +def check_protected_branch(ctx): + local_branch = get_current_branch(ctx) + + if local_branch == 'main': + print(color_message("You're about to commit on main, are you sure this is what you want?", "red")) + raise Exit(code=1) + + if re.fullmatch(r'^[0-9]+\.[0-9]+\.x$', local_branch): + print(color_message("You're about to commit on a release branch, are you sure this is what you want?", "red")) + raise Exit(code=1) diff --git a/tasks/go.py b/tasks/go.py index 17fe7df9b2a42..ef89c4bb83078 100644 --- a/tasks/go.py +++ b/tasks/go.py @@ -410,17 +410,26 @@ def tidy_all(ctx): @task -def tidy(ctx, only_modified_packages=False): - # TODO: if only_modified_packages then use `go mod tidy -diff` with golang 1.23, probably in a separate check-mod-tidy task - # https://github.com/golang/go/issues/27005 - from tasks import get_modified_packages - - # TODO: Also include packages that import them - modules = get_modified_packages(ctx) if only_modified_packages else DEFAULT_MODULES.values() - - for mod in modules: +def tidy(ctx): + if os.name != 'nt': # not windows + import resource + + # Some people might face ulimit issues, so we bump it up if needed. + # It won't change it globally, only for this process and child processes. + # TODO: if this is working fine, let's do it during the init so all tasks can benefit from it if needed. + current_ulimit = resource.getrlimit(resource.RLIMIT_NOFILE) + if current_ulimit[0] < 1024: + resource.setrlimit(resource.RLIMIT_NOFILE, (1024, current_ulimit[1])) + + # Note: It's currently faster to tidy everything than looking for exactly what we should tidy + promises = [] + for mod in DEFAULT_MODULES.values(): with ctx.cd(mod.full_path()): - ctx.run("go mod tidy") + # https://docs.pyinvoke.org/en/stable/api/runners.html#invoke.runners.Runner.run + promises.append(ctx.run("go mod tidy", asynchronous=True)) + + for promise in promises: + promise.join() @task diff --git a/tasks/go_deps.py b/tasks/go_deps.py index 70acadbd8c93a..ac83346fa8cb8 100644 --- a/tasks/go_deps.py +++ b/tasks/go_deps.py @@ -116,7 +116,7 @@ def compute_binary_dependencies_list( ) assert res - return res.stdout.strip() + return [dep for dep in res.stdout.strip().splitlines() if not dep.startswith("internal/")] @task @@ -185,14 +185,14 @@ def test_list( continue deps_file = open(filename) - deps = deps_file.read() + deps = deps_file.read().strip().splitlines() deps_file.close() list = compute_binary_dependencies_list(ctx, build, flavor, platform, arch) if list != deps: - new_dependencies_lines = len(list.splitlines()) - recorded_dependencies_lines = len(deps.splitlines()) + new_dependencies_lines = len(list) + recorded_dependencies_lines = len(deps) mismatch_binaries.add( MisMacthBinary(binary, goos, goarch, new_dependencies_lines - recorded_dependencies_lines) @@ -252,5 +252,5 @@ def generate( list = compute_binary_dependencies_list(ctx, build, flavor, platform, arch) f = open(filename, "w") - f.write(list) + f.write('\n'.join(list)) f.close() diff --git a/tasks/gotest.py b/tasks/gotest.py index 1db4175e04c91..1977d37d15819 100644 --- a/tasks/gotest.py +++ b/tasks/gotest.py @@ -28,6 +28,7 @@ from tasks.flavor import AgentFlavor from tasks.libs.common.color import color_message from tasks.libs.common.datadog_api import create_count, send_metrics +from tasks.libs.common.git import get_modified_files from tasks.libs.common.junit_upload_core import enrich_junitxml, produce_junit_tar from tasks.libs.common.utils import clean_nested_paths, collapsed_section, get_build_flags, get_distro from tasks.modules import DEFAULT_MODULES, GoModule @@ -609,14 +610,6 @@ def get_modified_packages(ctx, build_tags=None, lint=False) -> list[GoModule]: return modules_to_test.values() -def get_modified_files(ctx): - last_main_commit = ctx.run("git merge-base HEAD origin/main", hide=True).stdout - print(f"Checking diff from {last_main_commit} commit on main branch") - - modified_files = ctx.run(f"git diff --name-only --no-renames {last_main_commit}", hide=True).stdout.splitlines() - return modified_files - - @task(iterable=["extra_tag"]) def send_unit_tests_stats(_, job_name, extra_tag=None): if extra_tag is None: diff --git a/tasks/kernel_matrix_testing/tool.py b/tasks/kernel_matrix_testing/tool.py index a6d332873f5f8..69b4f3de1e7d4 100644 --- a/tasks/kernel_matrix_testing/tool.py +++ b/tasks/kernel_matrix_testing/tool.py @@ -2,6 +2,7 @@ import os import platform +import sys from typing import TYPE_CHECKING import invoke.exceptions as ie @@ -20,24 +21,28 @@ def colored(text: str, color: str | None) -> str: # noqa: U100 return text +def _logprint(msg: str): + print(msg, flush=True, file=sys.stderr) + + def ask(question: str) -> str: return input(colored(question, "blue")) def debug(msg: str): - print(colored(msg, "white")) + _logprint(colored(msg, "white")) def info(msg: str): - print(colored(msg, "green")) + _logprint(colored(msg, "green")) def warn(msg: str): - print(colored(msg, "yellow")) + _logprint(colored(msg, "yellow")) def error(msg: str): - print(colored(msg, "red")) + _logprint(colored(msg, "red")) def Exit(msg: str): diff --git a/tasks/kmt.py b/tasks/kmt.py index 0ee772f75c113..eda7805d592fe 100644 --- a/tasks/kmt.py +++ b/tasks/kmt.py @@ -667,12 +667,14 @@ def prepare( d.copy(ctx, f"{paths.arch_dir}/opt/*", "/opt/", exclude="*.ninja", verbose=verbose) d.run_cmd( ctx, - f"! [ -f {btf_dir}/minimized-btfs.tar.xz \ - && [ -d /opt/btf ] \ + f"[ -f /sys/kernel/btf/vmlinux ] \ + || [ -f {btf_dir}/minimized-btfs.tar.xz ] \ + || ([ -d /opt/btf ] \ && cd /opt/btf/ \ && tar cJf minimized-btfs.tar.xz * \ && mkdir -p {btf_dir} \ - && mv /opt/btf/minimized-btfs.tar.xz {btf_dir}/", + && mv /opt/btf/minimized-btfs.tar.xz {btf_dir}/)", + verbose=verbose, ) info(f"[+] Tests packages setup in target VM {d}") @@ -707,6 +709,37 @@ def build_object_files(ctx, fp): ctx.run(f"ninja -d explain -f {fp}") +def compute_package_dependencies(ctx: Context, packages: list[str]) -> dict[str, set[str]]: + dd_pkg_name = "github.com/DataDog/datadog-agent/" + build_tags = get_sysprobe_buildtags(False, False) + pkg_deps: dict[str, set[str]] = defaultdict(set) + + packages_list = " ".join(packages) + list_format = "{{ .ImportPath }}: {{ join .Deps \" \" }}" + res = ctx.run(f"go list -test -f '{list_format}' -tags \"{build_tags}\" {packages_list}", hide=True) + if res is None or not res.ok: + raise Exit("Failed to get dependencies for system-probe") + + for line in res.stdout.split("\n"): + if ":" not in line: + continue + + pkg, deps = line.split(":", 1) + deps = [d.strip() for d in deps.split(" ")] + dd_deps = [d[len(dd_pkg_name) :] for d in deps if d.startswith(dd_pkg_name)] + + # The import path printed by "go list" is usually path/to/pkg (e.g., pkg/ebpf/verifier). + # However, for test packages it might be either: + # - path/to/pkg.test + # - path/to/pkg [path/to/pkg.test] + # In any case all variants refer to the same variant. This code controls for that + # so that we keep the usual package name. + pkg = pkg.split(" ")[0].removeprefix(dd_pkg_name).removesuffix(".test") + pkg_deps[pkg].update(dd_deps) + + return pkg_deps + + @task def kmt_sysprobe_prepare( ctx: Context, @@ -730,7 +763,6 @@ def kmt_sysprobe_prepare( if packages: filter_pkgs = [os.path.relpath(p) for p in packages.split(",")] - target_packages = build_target_packages(filter_pkgs) kmt_paths = KMTPaths(stack, arch) nf_path = os.path.join(kmt_paths.arch_dir, "kmt-sysprobe.ninja") @@ -743,6 +775,12 @@ def kmt_sysprobe_prepare( go_path = os.path.join(go_root, "bin", "go") build_object_files(ctx, f"{kmt_paths.arch_dir}/kmt-object-files.ninja") + + info("[+] Computing Go dependencies for test packages...") + target_packages = build_target_packages(filter_pkgs) + pkg_deps = compute_package_dependencies(ctx, target_packages) + + info("[+] Generating build instructions..") with open(nf_path, 'w') as ninja_file: nw = NinjaWriter(ninja_file) @@ -760,7 +798,8 @@ def kmt_sysprobe_prepare( ninja_copy_ebpf_files(nw, "system-probe", kmt_paths) for pkg in target_packages: - target_path = os.path.join(kmt_paths.sysprobe_tests, os.path.relpath(pkg, os.getcwd())) + pkg_name = os.path.relpath(pkg, os.getcwd()) + target_path = os.path.join(kmt_paths.sysprobe_tests, pkg_name) output_path = os.path.join(target_path, "testsuite") variables = { "env": env_str, @@ -773,19 +812,22 @@ def kmt_sysprobe_prepare( if extra_arguments: variables["extra_arguments"] = extra_arguments - go_files = [os.path.abspath(i) for i in glob(f"{pkg}/*.go")] - - # We delete the output file to force ninja to rebuild the testsuite everytime - # because it cannot track go dependencies correctly. - ctx.run(f"rm -f {output_path}") - nw.build( - inputs=[pkg], - outputs=[output_path], - implicit=go_files, - rule="gotestsuite", - pool="gobuild", - variables=variables, - ) + go_files = set(glob(f"{pkg}/*.go")) + has_test_files = any(x.lower().endswith("_test.go") for x in go_files) + + # skip packages without test files + if has_test_files: + for deps in pkg_deps[pkg_name]: + go_files.update(os.path.abspath(p) for p in glob(f"{deps}/*.go")) + + nw.build( + inputs=[pkg], + outputs=[output_path], + implicit=list(go_files), + rule="gotestsuite", + pool="gobuild", + variables=variables, + ) if pkg.endswith("java"): nw.build( @@ -827,6 +869,7 @@ def kmt_sysprobe_prepare( }, ) + info("[+] Compiling tests...") ctx.run(f"ninja -d explain -v -f {nf_path}") diff --git a/tasks/libs/common/git.py b/tasks/libs/common/git.py new file mode 100644 index 0000000000000..a2954f5fbc497 --- /dev/null +++ b/tasks/libs/common/git.py @@ -0,0 +1,14 @@ +def get_staged_files(ctx, commit="HEAD") -> list[str]: + """ + Get the list of staged (to be committed) files in the repository compared to the `commit` commit. + """ + return ctx.run(f"git diff --name-only --staged {commit}", hide=True).stdout.strip().splitlines() + + +def get_modified_files(ctx) -> list[str]: + last_main_commit = ctx.run("git merge-base HEAD origin/main", hide=True).stdout + return ctx.run(f"git diff --name-only --no-renames {last_main_commit}", hide=True).stdout.splitlines() + + +def get_current_branch(ctx) -> str: + return ctx.run("git rev-parse --abbrev-ref HEAD", hide=True).stdout.strip() diff --git a/tasks/libs/common/junit_upload_core.py b/tasks/libs/common/junit_upload_core.py index 0fdb5ef7a7e8d..194223681d78f 100644 --- a/tasks/libs/common/junit_upload_core.py +++ b/tasks/libs/common/junit_upload_core.py @@ -23,6 +23,7 @@ GITHUB_JIRA_MAP, GITHUB_SLACK_MAP, ) +from tasks.modules import DEFAULT_MODULES E2E_INTERNAL_ERROR_STRING = "E2E INTERNAL ERROR" CODEOWNERS_ORG_PREFIX = "@DataDog/" @@ -104,18 +105,23 @@ def get_flaky_from_test_output(): Read the test output file generated by gotestsum which contains a list of json for each unit test. We catch all tests marked as flaky in source code: they contain a certain message in the output field. """ - TEST_OUTPUT_FILES = ["test_output.json", "test_output_fast.json"] + TEST_OUTPUT_FILE = "module_test_output.json" FLAKE_MESSAGE = "flakytest: this is a known flaky test" test_output = [] - for test_file in TEST_OUTPUT_FILES: - if os.path.isfile(test_file): - with open(test_file) as f: + flaky_tests = set() + for module in DEFAULT_MODULES: + test_file = Path(module, TEST_OUTPUT_FILE) + if test_file.is_file(): + with test_file.open(encoding="utf8") as f: for line in f.readlines(): test_output.append(json.loads(line)) - break - flaky_tests = [ - "/".join([test["Package"], test["Test"]]) for test in test_output if FLAKE_MESSAGE in test.get("Output", "") - ] + flaky_tests.update( + [ + "/".join([test["Package"], test["Test"]]) + for test in test_output + if FLAKE_MESSAGE in test.get("Output", "") + ] + ) print(f"[INFO] Found {len(flaky_tests)} flaky tests.") return flaky_tests @@ -266,6 +272,8 @@ def set_tags(owner, flavor, flag: str, additional_tags, file_name): f"jira_project:{jira_project}", "--tags", f"gitlab.pipeline_source:{pipeline.source}", + "--xpath-tag", + "test.agent_is_known_flaky=/testcase/@agent_is_known_flaky", ] if 'e2e' in flag: tags.extend(["--tags", "e2e_internal_error:true"]) diff --git a/tasks/libs/common/omnibus.py b/tasks/libs/common/omnibus.py index 86d930f92546e..d274ef87c1d8d 100644 --- a/tasks/libs/common/omnibus.py +++ b/tasks/libs/common/omnibus.py @@ -358,8 +358,6 @@ def install_dir_for_project(project): folder = 'datadog-agent' elif project == 'dogstatsd': folder = 'datadog-dogstatsd' - elif project == 'agentless-scanner': - folder = os.path.join('datadog', 'agentless-scanner') elif project == 'installer': folder = 'datadog-installer' else: diff --git a/tasks/libs/common/utils.py b/tasks/libs/common/utils.py index b2dd09d44762d..3ab39b06f57e8 100644 --- a/tasks/libs/common/utils.py +++ b/tasks/libs/common/utils.py @@ -261,6 +261,7 @@ def get_build_flags( env['CGO_LDFLAGS'] = os.environ.get('CGO_LDFLAGS', '') + ' -Wl,--allow-multiple-definition' extra_cgo_flags = " -Werror -Wno-deprecated-declarations" + extra_cgo_flags += " -D__CGO__" # Some C files include definitions that can cause duplicate symbols if included in CGo code (bpf_metadata.h) mainly if rtloader_headers: extra_cgo_flags += f" -I{rtloader_headers}" if rtloader_common_headers: @@ -742,7 +743,7 @@ def is_pr_context(branch, pr_id, test_name): @contextmanager def collapsed_section(section_name): - section_id = section_name.replace(" ", "_") + section_id = section_name.replace(" ", "_").replace("/", "_") in_ci = running_in_gitlab_ci() try: if in_ci: diff --git a/tasks/linter.py b/tasks/linter.py index 5413d3a7394d0..24e58e068ec5c 100644 --- a/tasks/linter.py +++ b/tasks/linter.py @@ -21,8 +21,9 @@ read_includes, ) from tasks.libs.common.check_tools_version import check_tools_version +from tasks.libs.common.git import get_staged_files from tasks.libs.common.utils import DEFAULT_BRANCH, GITHUB_REPO_NAME, color_message, is_pr_context, running_in_ci -from tasks.libs.types.copyright import CopyrightLinter +from tasks.libs.types.copyright import CopyrightLinter, LintFailure from tasks.modules import GoModule from tasks.test_core import ModuleLintResult, process_input_args, process_module_results, test_core from tasks.update_go import _update_go_mods, _update_references @@ -54,14 +55,23 @@ def python(ctx): @task -def copyrights(_, fix=False, dry_run=False, debug=False): +def copyrights(ctx, fix=False, dry_run=False, debug=False, only_staged_files=False): """ Checks that all Go files contain the appropriate copyright header. If '--fix' is provided as an option, it will try to fix problems as it finds them. If '--dry_run' is provided when fixing, no changes to the files will be applied. """ + files = None - CopyrightLinter(debug=debug).assert_compliance(fix=fix, dry_run=dry_run) + if only_staged_files: + staged_files = get_staged_files(ctx) + files = [path for path in staged_files if path.endswith(".go")] + + try: + CopyrightLinter(debug=debug).assert_compliance(fix=fix, dry_run=dry_run, files=files) + except LintFailure: + # the linter prints useful messages on its own, so no need to print the exception + sys.exit(1) @task diff --git a/tasks/notify.py b/tasks/notify.py index 228ba50ed196f..12e57ab2382a2 100644 --- a/tasks/notify.py +++ b/tasks/notify.py @@ -9,6 +9,7 @@ from collections import defaultdict from dataclasses import dataclass from datetime import datetime, timezone +from urllib.parse import quote from invoke import task from invoke.context import Context @@ -35,6 +36,7 @@ Please check for typos in the JOBOWNERS file and/or add them to the Github <-> Slack map. """ PROJECT_NAME = "DataDog/datadog-agent" +PROJECT_TITLE = PROJECT_NAME.removeprefix("DataDog/") AWS_S3_CP_CMD = "aws s3 cp --only-show-errors --region us-east-1 --sse AES256" S3_CI_BUCKET_URL = "s3://dd-ci-artefacts-build-stable/datadog-agent/failed_jobs" CONSECUTIVE_THRESHOLD = 3 @@ -43,6 +45,19 @@ CI_VISIBILITY_JOB_URL = 'https://app.datadoghq.com/ci/pipeline-executions?query=ci_level%3Ajob%20%40ci.pipeline.name%3ADataDog%2Fdatadog-agent%20%40git.branch%3Amain%20%40ci.job.name%3A{}&agg_m=count' +def get_ci_visibility_job_url(name: str, prefix=True) -> str: + # Escape (https://docs.datadoghq.com/logs/explorer/search_syntax/#escape-special-characters-and-spaces) + name = re.sub(r"([-+=&|> str: # Find initial PR initial_pr_sha = next(iter(self.failures.values()))[0].commit initial_pr_title = ctx.run(f'git show -s --format=%s {initial_pr_sha}', hide=True).stdout.strip() - initial_pr_info = get_pr_from_commit(initial_pr_title, PROJECT_NAME) + initial_pr_info = get_pr_from_commit(initial_pr_title, PROJECT_TITLE) if initial_pr_info: pr_id, pr_url = initial_pr_info - initial_pr = f'<{pr_url}|{pr_id}>' + initial_pr = f'<{pr_url}|#{pr_id}>' else: # Cannot find PR, display the commit sha initial_pr = initial_pr_sha[:8] @@ -443,6 +458,43 @@ def send_notification(ctx: Context, alert_jobs): send_slack_message("#agent-platform-ops", message) +@task +def send_failure_summary_notification(_, jobs: dict[str, any] | None = None, list_max_len=10): + if jobs is None: + jobs = os.environ["JOB_FAILURES"] + jobs = json.loads(jobs) + + # List of (job_name, (failure_count, total_count)) ordered by failure_count + stats = sorted( + ((name, (fail, success)) for (name, (fail, success)) in jobs.items() if fail > 0), + key=lambda x: (x[1][0], x[1][1] if x[1][1] is not None else 0), + reverse=True, + )[:list_max_len] + + # Don't send message if no failure + if len(stats) == 0: + return + + # Create message + message = ['*Daily Job Failure Report*'] + message.append('These jobs had the most failures in the last 24 hours:') + for name, (fail, total) in stats: + link = get_ci_visibility_job_url(name) + message.append(f"- <{link}|{name}>: *{fail} failures*{f' / {total} runs' if total else ''}") + + message.append( + 'Click for more details.' + ) + + # Send message + from slack_sdk import WebClient + + client = WebClient(os.environ["SLACK_API_TOKEN"]) + client.chat_postMessage(channel='#agent-platform-ops', text='\n'.join(message)) + + print('Message sent') + + @task def unit_tests(ctx, pipeline_id, pipeline_url, branch_name): from tasks.libs.ciproviders.github_api import GithubAPI diff --git a/tasks/system_probe.py b/tasks/system_probe.py index 44d96f71fc159..25a9f49b09029 100644 --- a/tasks/system_probe.py +++ b/tasks/system_probe.py @@ -787,21 +787,13 @@ def go_package_dirs(packages, build_tags): """ target_packages = [] - for pkg in packages: - dirs = ( - check_output( - f"go list -find -f \"{{{{ .Dir }}}}\" -mod=mod -tags \"{','.join(build_tags)}\" {pkg}", - shell=True, - ) - .decode('utf-8') - .strip() - .split("\n") - ) - # Some packages may not be available on all architectures, ignore them - # instead of reporting empty path names - target_packages += [dir for dir in dirs if len(dir) > 0] + format_arg = '{{ .Dir }}' + buildtags_arg = ",".join(build_tags) + packages_arg = " ".join(packages) + cmd = f"go list -find -f \"{format_arg}\" -mod=mod -tags \"{buildtags_arg}\" {packages_arg}" - return target_packages + target_packages = [p.strip() for p in check_output(cmd, shell=True, encoding='utf-8').split("\n")] + return [p for p in target_packages if len(p) > 0] BUILD_COMMIT = os.path.join(KITCHEN_ARTIFACT_DIR, "build.commit") diff --git a/tasks/testwasher.py b/tasks/testwasher.py index d6c3f28730606..3dd51c547f8e7 100644 --- a/tasks/testwasher.py +++ b/tasks/testwasher.py @@ -91,17 +91,15 @@ def process_module_results(self, module_results: list[ModuleTestResult]): """ should_succeed = True - failed_tests_string = "" + failed_tests = [] for module_result in module_results: - non_flaky_failing_tests = self.get_non_flaky_failing_tests(module_result.path) - if non_flaky_failing_tests: + if non_flaky_failing_tests := self.get_non_flaky_failing_tests(module_result.path): should_succeed = False for package, tests in non_flaky_failing_tests.items(): - for test in tests: - failed_tests_string += f"- {package} {test}\n" - if failed_tests_string: + failed_tests.extend(f"- {package} {test}" for test in tests) + if failed_tests: print("The test command failed, the following tests failed and are not supposed to be flaky:") - print(failed_tests_string) + print("\n".join(sorted(failed_tests))) return should_succeed diff --git a/tasks/unit-tests/git_tests.py b/tasks/unit-tests/git_tests.py new file mode 100644 index 0000000000000..3bbccc59e6356 --- /dev/null +++ b/tasks/unit-tests/git_tests.py @@ -0,0 +1,30 @@ +import unittest + +from invoke.context import Context +from invoke.exceptions import Exit + +from tasks.git import check_protected_branch + + +class TestGit(unittest.TestCase): + def setUp(self) -> None: + super().setUp() + self.ctx = Context() + + @unittest.mock.patch('tasks.git.get_current_branch') + def test_check_protected_branch(self, get_current_branch_mock): + get_current_branch_mock.return_value = 'my/branch' + check_protected_branch(self.ctx) + + @unittest.mock.patch('tasks.git.get_current_branch') + def test_check_protected_branch_error(self, get_current_branch_mock): + protected_branches = ( + 'main', + '7.54.x', + '6.54.x', + ) + + for branch_name in protected_branches: + with self.subTest(branch=branch_name): + get_current_branch_mock.return_value = branch_name + self.assertRaises(Exit, check_protected_branch, self.ctx) diff --git a/tasks/unit-tests/junit_tests.py b/tasks/unit-tests/junit_tests.py index 7733b5f07d700..f7173c22dc326 100644 --- a/tasks/unit-tests/junit_tests.py +++ b/tasks/unit-tests/junit_tests.py @@ -92,7 +92,7 @@ def test_default(self, mock_gitlab): mock_instance.pipelines.get.return_value = MagicMock() mock_gitlab.return_value = mock_instance tags = junit.set_tags("agent-ci-experience", "base", "", {}, "") - self.assertEqual(len(tags), 12) + self.assertEqual(len(tags), 14) self.assertIn("slack_channel:agent-developer-experience", tags) @patch.dict("os.environ", {"CI_PIPELINE_ID": "1664"}) @@ -108,7 +108,7 @@ def test_flag(self, mock_gitlab): ["upload_option.os_version_from_name"], "kitchen-rspec-win2016-azure-x86_64.xml", ) - self.assertEqual(len(tags), 16) + self.assertEqual(len(tags), 18) self.assertIn("e2e_internal_error:true", tags) self.assertIn("version:win2016", tags) self.assertNotIn("upload_option.os_version_from_name", tags) @@ -120,7 +120,7 @@ def test_additional_tags(self, mock_gitlab): mock_instance.pipelines.get.return_value = MagicMock() mock_gitlab.return_value = mock_instance tags = junit.set_tags("agent-ci-experience", "base", "", ["--tags", "simple:basique"], "") - self.assertEqual(len(tags), 14) + self.assertEqual(len(tags), 16) self.assertIn("simple:basique", tags) diff --git a/tasks/unit-tests/libs/common/git_tests.py b/tasks/unit-tests/libs/common/git_tests.py new file mode 100644 index 0000000000000..2a43395394ee3 --- /dev/null +++ b/tasks/unit-tests/libs/common/git_tests.py @@ -0,0 +1,24 @@ +import unittest +from unittest.mock import MagicMock + +from tasks.libs.common.git import get_current_branch, get_staged_files + + +class TestGit(unittest.TestCase): + def setUp(self) -> None: + super().setUp() + self.ctx_mock = MagicMock() + + def test_get_staged_files(self): + self.ctx_mock.run.return_value.stdout = "file1\nfile2\nfile3" + files = get_staged_files(self.ctx_mock) + + self.assertEqual(files, ["file1", "file2", "file3"]) + self.ctx_mock.run.assert_called_once_with("git diff --name-only --staged HEAD", hide=True) + + def test_get_current_branch(self): + self.ctx_mock.run.return_value.stdout = " main \n" + branch = get_current_branch(self.ctx_mock) + + self.assertEqual(branch, "main") + self.ctx_mock.run.assert_called_once_with("git rev-parse --abbrev-ref HEAD", hide=True) diff --git a/tasks/unit-tests/notify_tests.py b/tasks/unit-tests/notify_tests.py index 11525eae0493a..a9c01547c6159 100644 --- a/tasks/unit-tests/notify_tests.py +++ b/tasks/unit-tests/notify_tests.py @@ -400,3 +400,19 @@ def test_none(self, mock_slack): alert_jobs = {"consecutive": notify.ConsecutiveJobAlert({}), "cumulative": notify.CumulativeJobAlert({})} notify.send_notification(MagicMock(), alert_jobs) mock_slack.assert_not_called() + + +class TestSendFailureSummaryNotification(unittest.TestCase): + @patch("slack_sdk.WebClient") + @patch("os.environ", new=MagicMock()) + def test_nominal(self, mock_slack): + # jobname: [total_failures, total_runs] + jobs = { + "myjob1": [45, None], + "myjob2": [42, 45], + "myjob3": [21, None], + "myjob4": [16, 89], + } + notify.send_failure_summary_notification(MockContext(), jobs) + mock_slack.assert_called() + mock_slack.return_value.chat_postMessage.assert_called() diff --git a/tasks/unit-tests/testdata/fake_gitlab-ci.yml b/tasks/unit-tests/testdata/fake_gitlab-ci.yml index 22b8619387e90..714ed99e66363 100644 --- a/tasks/unit-tests/testdata/fake_gitlab-ci.yml +++ b/tasks/unit-tests/testdata/fake_gitlab-ci.yml @@ -113,7 +113,6 @@ variables: STATIC_BINARIES_DIR: bin/static DOGSTATSD_BINARIES_DIR: bin/dogstatsd AGENT_BINARIES_DIR: bin/agent - AGENTLESS_SCANNER_BINARIES_DIR: bin/agentless-scanner CLUSTER_AGENT_BINARIES_DIR: bin/datadog-cluster-agent CWS_INSTRUMENTATION_BINARIES_DIR: bin/cws-instrumentation CLUSTER_AGENT_CLOUDFOUNDRY_BINARIES_DIR: bin/datadog-cluster-agent-cloudfoundry diff --git a/test/new-e2e/system-probe/config/platforms.json b/test/new-e2e/system-probe/config/platforms.json index 2eabf114a7c8e..52ed9e8e168d5 100644 --- a/test/new-e2e/system-probe/config/platforms.json +++ b/test/new-e2e/system-probe/config/platforms.json @@ -7,7 +7,7 @@ "kernel": "6.0.7-301.fc37", "os_version": "37", "image": "Fedora-Cloud-Base-37.x86_64.qcow2.xz", - "image_version": "20240523_238eeb4e" + "image_version": "20240529_dd00b86a" }, "fedora_38": { "os_name": "Fedora Linux", @@ -15,7 +15,7 @@ "kernel": "6.2.9-300.fc38", "os_version": "38", "image": "Fedora-Cloud-Base-38.x86_64.qcow2.xz", - "image_version": "20240523_238eeb4e" + "image_version": "20240529_dd00b86a" }, "amazon_4.14": { "os_name": "Amazon Linux", @@ -23,7 +23,7 @@ "kernel": "4.14.314-237.533.amzn2", "os_version": "2", "image": "amzn2-x86_64-4.14.qcow2.xz", - "image_version": "20240523_238eeb4e" + "image_version": "20240529_dd00b86a" }, "amazon_5.10": { "os_name": "Amazon Linux", @@ -31,7 +31,7 @@ "kernel": "5.10.216-204.855.amzn2", "os_version": "2", "image": "amzn2-x86_64-5.10.qcow2.xz", - "image_version": "20240523_238eeb4e" + "image_version": "20240529_dd00b86a" }, "amazon_5.4": { "os_name": "Amazon Linux", @@ -39,7 +39,7 @@ "kernel": "5.4.275-189.375.amzn2", "os_version": "2", "image": "amzn2-x86_64-5.4.qcow2.xz", - "image_version": "20240523_238eeb4e" + "image_version": "20240529_dd00b86a" }, "amazon_2023": { "os_name": "Amazon Linux", @@ -47,7 +47,7 @@ "kernel": "6.1.77-99.164.amzn2023", "os_version": "2023", "image": "amzn2023-amd64-2023.qcow2.xz", - "image_version": "20240523_238eeb4e" + "image_version": "20240529_dd00b86a" }, "centos_7.9": { "os_name": "CentOS Linux", @@ -55,15 +55,15 @@ "kernel": "3.10.0-1160.80.1.el7", "os_version": "7.9.2009", "image": "centos-7.9-x86_64.qcow2.xz", - "image_version": "20240523_238eeb4e" + "image_version": "20240529_dd00b86a" }, "centos_8": { "os_name": "CentOS Stream", "os_id": "centos", - "kernel": "4.18.0-545.el8", + "kernel": "4.18.0-552.3.1.el8", "os_version": "8", "image": "centos-8-x86_64.qcow2.xz", - "image_version": "20240430_7bbb4ede" + "image_version": "20240529_dd00b86a" }, "debian_10": { "os_name": "Debian GNU/Linux", @@ -71,7 +71,7 @@ "kernel": "4.19.0-26", "os_version": "10", "image": "debian-10-generic-x86_64.qcow2.xz", - "image_version": "20240523_238eeb4e", + "image_version": "20240529_dd00b86a", "alt_version_names": [ "buster" ] @@ -82,7 +82,7 @@ "kernel": "5.10.0-28", "os_version": "11", "image": "debian-11-generic-x86_64.qcow2.xz", - "image_version": "20240523_238eeb4e", + "image_version": "20240529_dd00b86a", "alt_version_names": [ "bullseye" ] @@ -93,7 +93,7 @@ "kernel": "6.1.0-18", "os_version": "12", "image": "debian-12-generic-x86_64.qcow2.xz", - "image_version": "20240523_238eeb4e", + "image_version": "20240529_dd00b86a", "alt_version_names": [ "bookworm" ] @@ -104,7 +104,7 @@ "kernel": "3.10.0-1160.105.1.0.1.el7", "os_version": "7.9", "image": "oracle-7.9-x86_64.qcow.xz", - "image_version": "20240523_238eeb4e" + "image_version": "20240529_dd00b86a" }, "oracle_8.9": { "os_name": "Oracle Linux Server", @@ -112,7 +112,7 @@ "kernel": "5.15.0-200.131.27.el8uek", "os_version": "8.9", "image": "oracle-8.9-x86_64.qcow.xz", - "image_version": "20240523_238eeb4e" + "image_version": "20240529_dd00b86a" }, "oracle_9.3": { "os_name": "Oracle Linux Server", @@ -120,7 +120,7 @@ "kernel": "5.15.0-200.131.27.el9uek", "os_version": "9.3", "image": "oracle-9.3-x86_64.qcow.xz", - "image_version": "20240523_238eeb4e" + "image_version": "20240529_dd00b86a" }, "rocky_8.5": { "os_name": "Rocky Linux", @@ -128,7 +128,7 @@ "kernel": "4.18.0-348.el8.0.2", "os_version": "8.5", "image": "rocky-8.5-x86_64.qcow2.xz", - "image_version": "20240523_238eeb4e" + "image_version": "20240529_dd00b86a" }, "rocky_9.3": { "os_name": "Rocky Linux", @@ -136,7 +136,7 @@ "kernel": "5.14.0-362.8.1.el9_3", "os_version": "9.3", "image": "rocky-9.3-x86_64.qcow2.xz", - "image_version": "20240430_7bbb4ede" + "image_version": "20240529_dd00b86a" }, "ubuntu_16.04": { "os_name": "Ubuntu", @@ -144,7 +144,7 @@ "kernel": "4.4.0-210-generic", "os_version": "16.04", "image": "ubuntu-16.04-x86_64.qcow2.xz", - "image_version": "20240523_238eeb4e", + "image_version": "20240529_dd00b86a", "alt_version_names": [ "xenial" ] @@ -155,7 +155,7 @@ "kernel": "4.18.0-25-generic", "os_version": "18.04", "image": "ubuntu-18.04-x86_64.qcow2.xz", - "image_version": "20240523_238eeb4e", + "image_version": "20240529_dd00b86a", "alt_version_names": [ "bionic" ] @@ -166,7 +166,7 @@ "kernel": "5.4.0-167-generic", "os_version": "20.04", "image": "ubuntu-20.04-x86_64.qcow2.xz", - "image_version": "20240523_238eeb4e", + "image_version": "20240529_dd00b86a", "alt_version_names": [ "focal" ] @@ -177,7 +177,7 @@ "kernel": "5.15.0-91-generic", "os_version": "22.04", "image": "ubuntu-22.04-x86_64.qcow2.xz", - "image_version": "20240523_238eeb4e", + "image_version": "20240529_dd00b86a", "alt_version_names": [ "jammy" ] @@ -188,7 +188,7 @@ "kernel": "6.5.0-9-generic", "os_version": "23.10", "image": "ubuntu-23.10-x86_64.qcow2.xz", - "image_version": "20240523_238eeb4e", + "image_version": "20240529_dd00b86a", "alt_version_names": [ "mantic" ] @@ -199,7 +199,7 @@ "kernel": "6.8.0-31-generic", "os_version": "24.04", "image": "ubuntu-24.04-x86_64.qcow2.xz", - "image_version": "20240523_238eeb4e", + "image_version": "20240529_dd00b86a", "alt_version_names": [ "noble" ] @@ -210,15 +210,15 @@ "kernel": "5.3.18-150300.59.106.1", "os_version": "15.3", "image": "opensuse-15.3-x86_64.qcow2.xz", - "image_version": "20240523_238eeb4e" + "image_version": "20240529_dd00b86a" }, "opensuse_15.5": { "os_name": "openSUSE Leap", "os_id": "opensuse-leap", - "kernel": "5.14.21-150500.55.52.1", + "kernel": "5.14.21-150500.55.65.1", "os_version": "15.5", "image": "opensuse-15.5-x86_64.qcow2.xz", - "image_version": "20240430_7bbb4ede" + "image_version": "20240529_dd00b86a" }, "suse_12.5": { "os_name": "SLES", @@ -226,7 +226,18 @@ "kernel": "4.12.14-122.212.1", "os_version": "12.5", "image": "suse-12.5-x86_64.qcow2.xz", - "image_version": "20240523_238eeb4e" + "image_version": "20240529_dd00b86a" + }, + "debian_9": { + "os_name": "Debian GNU/Linux", + "os_id": "debian", + "kernel": "4.9.0-19", + "os_version": "9", + "image": "debian-9-generic-x86_64.qcow2.xz", + "image_version": "20240529_dd00b86a", + "alt_version_names": [ + "stretch" + ] } }, "arm64": { @@ -236,7 +247,7 @@ "kernel": "6.0.7-301.fc37", "os_version": "37", "image": "Fedora-Cloud-Base-37.arm64.qcow2.xz", - "image_version": "20240523_238eeb4e" + "image_version": "20240529_dd00b86a" }, "fedora_38": { "os_name": "Fedora Linux", @@ -244,7 +255,7 @@ "kernel": "6.2.9-300.fc38", "os_version": "38", "image": "Fedora-Cloud-Base-38.arm64.qcow2.xz", - "image_version": "20240523_238eeb4e" + "image_version": "20240529_dd00b86a" }, "amazon_4.14": { "os_name": "Amazon Linux", @@ -252,7 +263,7 @@ "kernel": "4.14.314-237.533.amzn2", "os_version": "2", "image": "amzn2-arm64-4.14.qcow2.xz", - "image_version": "20240523_238eeb4e" + "image_version": "20240529_dd00b86a" }, "amazon_5.4": { "os_name": "Amazon Linux", @@ -260,7 +271,7 @@ "kernel": "5.4.275-189.375.amzn2", "os_version": "2", "image": "amzn2-arm64-5.4.qcow2.xz", - "image_version": "20240523_238eeb4e" + "image_version": "20240529_dd00b86a" }, "centos_7.9": { "os_name": "CentOS Linux", @@ -268,15 +279,15 @@ "kernel": "4.18.0-348.20.1.el7", "os_version": "7.9.2009", "image": "centos-7.9-arm64.qcow2.xz", - "image_version": "20240523_238eeb4e" + "image_version": "20240529_dd00b86a" }, "centos_8": { "os_name": "CentOS Stream", "os_id": "centos", - "kernel": "4.18.0-545.el8", + "kernel": "4.18.0-552.3.1.el8", "os_version": "8", "image": "centos-8-arm64.qcow2.xz", - "image_version": "20240430_7bbb4ede" + "image_version": "20240529_dd00b86a" }, "debian_10": { "os_name": "Debian GNU/Linux", @@ -284,7 +295,7 @@ "kernel": "4.19.0-26", "os_version": "10", "image": "debian-10-generic-arm64.qcow2.xz", - "image_version": "20240523_238eeb4e", + "image_version": "20240529_dd00b86a", "alt_version_names": [ "buster" ] @@ -295,7 +306,7 @@ "kernel": "5.10.0-28", "os_version": "11", "image": "debian-11-generic-arm64.qcow2.xz", - "image_version": "20240523_238eeb4e", + "image_version": "20240529_dd00b86a", "alt_version_names": [ "bullseye" ] @@ -306,7 +317,7 @@ "kernel": "6.1.0-18", "os_version": "12", "image": "debian-12-generic-arm64.qcow2.xz", - "image_version": "20240523_238eeb4e", + "image_version": "20240529_dd00b86a", "alt_version_names": [ "bookworm" ] @@ -317,7 +328,7 @@ "kernel": "5.15.0-200.131.27.el8uek", "os_version": "8.9", "image": "oracle-8.9-arm64.qcow.xz", - "image_version": "20240523_238eeb4e" + "image_version": "20240529_dd00b86a" }, "oracle_9.3": { "os_name": "Oracle Linux Server", @@ -325,7 +336,7 @@ "kernel": "5.15.0-200.131.27.el9uek", "os_version": "9.3", "image": "oracle-9.3-arm64.qcow.xz", - "image_version": "20240523_238eeb4e" + "image_version": "20240529_dd00b86a" }, "rocky_8.5": { "os_name": "Rocky Linux", @@ -333,7 +344,7 @@ "kernel": "4.18.0-348.el8.0.2", "os_version": "8.5", "image": "rocky-8.5-arm64.qcow2.xz", - "image_version": "20240523_238eeb4e" + "image_version": "20240529_dd00b86a" }, "rocky_9.3": { "os_name": "Rocky Linux", @@ -341,7 +352,7 @@ "kernel": "5.14.0-362.8.1.el9_3", "os_version": "9.3", "image": "rocky-9.3-arm64.qcow2.xz", - "image_version": "20240430_7bbb4ede" + "image_version": "20240529_dd00b86a" }, "amazon_5.10": { "os_name": "Amazon Linux", @@ -349,7 +360,7 @@ "kernel": "5.10.216-204.855.amzn2", "os_version": "2", "image": "amzn2-arm64-5.10.qcow2.xz", - "image_version": "20240523_238eeb4e" + "image_version": "20240529_dd00b86a" }, "amazon_2023": { "os_name": "Amazon Linux", @@ -357,7 +368,7 @@ "kernel": "6.1.77-99.164.amzn2023", "os_version": "2023", "image": "amzn2023-arm64-2023.qcow2.xz", - "image_version": "20240523_238eeb4e" + "image_version": "20240529_dd00b86a" }, "ubuntu_18.04": { "os_name": "Ubuntu", @@ -365,7 +376,7 @@ "kernel": "4.18.0-25-generic", "os_version": "18.04", "image": "ubuntu-18.04-arm64.qcow2.xz", - "image_version": "20240523_238eeb4e", + "image_version": "20240529_dd00b86a", "alt_version_names": [ "bionic" ] @@ -376,7 +387,7 @@ "kernel": "5.4.0-167-generic", "os_version": "20.04", "image": "ubuntu-20.04-arm64.qcow2.xz", - "image_version": "20240523_238eeb4e", + "image_version": "20240529_dd00b86a", "alt_version_names": [ "focal" ] @@ -387,7 +398,7 @@ "kernel": "5.15.0-91-generic", "os_version": "22.04", "image": "ubuntu-22.04-arm64.qcow2.xz", - "image_version": "20240523_238eeb4e", + "image_version": "20240529_dd00b86a", "alt_version_names": [ "jammy" ] @@ -398,7 +409,7 @@ "kernel": "6.5.0-9-generic", "os_version": "23.10", "image": "ubuntu-23.10-arm64.qcow2.xz", - "image_version": "20240523_238eeb4e", + "image_version": "20240529_dd00b86a", "alt_version_names": [ "mantic" ] @@ -409,7 +420,7 @@ "kernel": "6.8.0-31-generic", "os_version": "24.04", "image": "ubuntu-24.04-arm64.qcow2.xz", - "image_version": "20240523_238eeb4e", + "image_version": "20240529_dd00b86a", "alt_version_names": [ "noble" ] @@ -420,15 +431,15 @@ "kernel": "5.3.18-150300.59.106.1", "os_version": "15.3", "image": "opensuse-15.3-arm64.qcow2.xz", - "image_version": "20240523_238eeb4e" + "image_version": "20240529_dd00b86a" }, "opensuse_15.5": { "os_name": "openSUSE Leap", "os_id": "opensuse-leap", - "kernel": "5.14.21-150500.55.52.1", + "kernel": "5.14.21-150500.55.65.1", "os_version": "15.5", "image": "opensuse-15.5-arm64.qcow2.xz", - "image_version": "20240430_7bbb4ede" + "image_version": "20240529_dd00b86a" }, "suse_12.5": { "os_name": "SLES", @@ -436,7 +447,7 @@ "kernel": "4.12.14-122.212.1", "os_version": "12.5", "image": "suse-12.5-arm64.qcow2.xz", - "image_version": "20240523_238eeb4e" + "image_version": "20240529_dd00b86a" } } } \ No newline at end of file diff --git a/test/new-e2e/tests/agent-platform/common/agent_behaviour.go b/test/new-e2e/tests/agent-platform/common/agent_behaviour.go index 4ea8eae34c3c9..dfb29e58a4707 100644 --- a/test/new-e2e/tests/agent-platform/common/agent_behaviour.go +++ b/test/new-e2e/tests/agent-platform/common/agent_behaviour.go @@ -15,10 +15,11 @@ import ( componentos "github.com/DataDog/test-infra-definitions/components/os" - agentclient "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclient" - boundport "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-platform/common/bound-port" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + agentclient "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclient" + boundport "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-platform/common/bound-port" ) // CheckAgentBehaviour runs test to check the agent is behaving as expected @@ -319,3 +320,53 @@ func CheckCWSBehaviour(t *testing.T, client *TestClient) { require.True(tt, result, "system-probe and security-agent should communicate") }) } + +// CheckSystemProbeBehavior runs tests to check the agent behave correctly when system-probe is enabled +func CheckSystemProbeBehavior(t *testing.T, client *TestClient) { + t.Run("enable system-probe and restarts", func(tt *testing.T) { + err := client.SetConfig(client.Helper.GetConfigFolder()+"system-probe.yaml", "system_probe_config.enabled", "true") + require.NoError(tt, err) + + err = client.SetConfig(client.Helper.GetConfigFolder()+"system-probe.yaml", "system_probe_config.enabled", "true") + require.NoError(tt, err) + + _, err = client.SvcManager.Restart(client.Helper.GetServiceName()) + require.NoError(tt, err, "datadog-agent should restart after CWS is enabled") + }) + + t.Run("system-probe is running", func(tt *testing.T) { + var err error + require.Eventually(tt, func() bool { + return AgentProcessIsRunning(client, "system-probe") + }, 1*time.Minute, 500*time.Millisecond, "system-probe should be running ", err) + }) + + t.Run("ebpf programs are unpacked and valid", func(tt *testing.T) { + ebpfPath := "/opt/datadog-agent/embedded/share/system-probe/ebpf" + output, err := client.Host.Execute(fmt.Sprintf("find %s -name '*.o'", ebpfPath)) + require.NoError(tt, err) + files := strings.Split(strings.TrimSpace(output), "\n") + require.Greater(tt, len(files), 0, "ebpf object files should be present") + + _, err = client.Host.Execute("command -v readelf") + if err != nil { + tt.Skip("readelf is not available on the host") + return + } + + hostArch, err := client.Host.Execute("uname -m") + require.NoError(tt, err) + hostArch = strings.TrimSpace(hostArch) + if hostArch == "aarch64" { + hostArch = "arm64" + } + archMetadata := fmt.Sprintf("", hostArch) + + for _, file := range files { + file = strings.TrimSpace(file) + ddMetadata, err := client.Host.Execute(fmt.Sprintf("readelf -p dd_metadata %s", file)) + require.NoError(tt, err, "readelf should not error, file is %s", file) + require.Contains(tt, ddMetadata, archMetadata, "invalid arch metadata") + } + }) +} diff --git a/test/new-e2e/tests/agent-platform/install-script/install_script_test.go b/test/new-e2e/tests/agent-platform/install-script/install_script_test.go index 89fc4a9ba9b38..56ab93a12a7c6 100644 --- a/test/new-e2e/tests/agent-platform/install-script/install_script_test.go +++ b/test/new-e2e/tests/agent-platform/install-script/install_script_test.go @@ -136,8 +136,11 @@ func (is *installScriptSuite) AgentTest(flavor string) { common.CheckAgentPython(is.T(), client, common.ExpectedPythonVersion3) common.CheckApmEnabled(is.T(), client) common.CheckApmDisabled(is.T(), client) - if flavor == "datadog-agent" && is.cwsSupported { - common.CheckCWSBehaviour(is.T(), client) + if flavor == "datadog-agent" { + common.CheckSystemProbeBehavior(is.T(), client) + if is.cwsSupported { + common.CheckCWSBehaviour(is.T(), client) + } } common.CheckInstallationInstallScript(is.T(), client) is.testUninstall(client, flavor) diff --git a/test/new-e2e/tests/agent-platform/step-by-step/step_by_step_test.go b/test/new-e2e/tests/agent-platform/step-by-step/step_by_step_test.go index 9e8a690024ed6..6a7dd091427ec 100644 --- a/test/new-e2e/tests/agent-platform/step-by-step/step_by_step_test.go +++ b/test/new-e2e/tests/agent-platform/step-by-step/step_by_step_test.go @@ -160,8 +160,11 @@ func (is *stepByStepSuite) CheckStepByStepAgentInstallation(VMclient *common.Tes } common.CheckApmEnabled(is.T(), VMclient) common.CheckApmDisabled(is.T(), VMclient) - if *flavorName == "datadog-agent" && is.cwsSupported { - common.CheckCWSBehaviour(is.T(), VMclient) + if *flavorName == "datadog-agent" { + common.CheckSystemProbeBehavior(is.T(), VMclient) + if is.cwsSupported { + common.CheckCWSBehaviour(is.T(), VMclient) + } } is.T().Run("remove the agent", func(tt *testing.T) { diff --git a/test/new-e2e/tests/agent-subcommands/diagnose/diagnose_common_test.go b/test/new-e2e/tests/agent-subcommands/diagnose/diagnose_common_test.go index bafa0a9b1d7a2..00feb0f40d510 100644 --- a/test/new-e2e/tests/agent-subcommands/diagnose/diagnose_common_test.go +++ b/test/new-e2e/tests/agent-subcommands/diagnose/diagnose_common_test.go @@ -45,7 +45,8 @@ func getDiagnoseOutput(v *baseDiagnoseSuite, commandArgs ...agentclient.AgentArg assert.NoError(c, v.Env().FakeIntake.Client().GetServerHealth()) }, 5*time.Minute, 20*time.Second, "timedout waiting for fakeintake to be healthy") - return v.Env().Agent.Client.Diagnose(commandArgs...) + diagnose := v.Env().Agent.Client.Diagnose(commandArgs...) + return diagnose } func (v *baseDiagnoseSuite) TestDiagnoseDefaultConfig() { @@ -65,10 +66,9 @@ func (v *baseDiagnoseSuite) TestDiagnoseList() { } } -func (v *baseDiagnoseSuite) TestDiagnoseInclude() { +func (v *baseDiagnoseSuite) AssertDiagnoseInclude() { diagnose := getDiagnoseOutput(v) diagnoseSummary := getDiagnoseSummary(diagnose) - for _, suite := range allSuites { diagnoseInclude := getDiagnoseOutput(v, agentclient.WithArgs([]string{"--include", suite})) resultInclude := getDiagnoseSummary(diagnoseInclude) @@ -87,7 +87,7 @@ func (v *baseDiagnoseSuite) TestDiagnoseInclude() { assert.Equal(v.T(), diagnoseIncludeEverySuiteSummary, diagnoseSummary) } -func (v *baseDiagnoseSuite) TestDiagnoseExclude() { +func (v *baseDiagnoseSuite) AssertDiagnoseExclude() { for _, suite := range allSuites { diagnoseExclude := getDiagnoseOutput(v, agentclient.WithArgs([]string{"--exclude", suite})) resultExclude := getDiagnoseSummary(diagnoseExclude) diff --git a/test/new-e2e/tests/agent-subcommands/diagnose/diagnose_nix_test.go b/test/new-e2e/tests/agent-subcommands/diagnose/diagnose_nix_test.go index ef2300c4fc200..6e5cd0fe6ee1b 100644 --- a/test/new-e2e/tests/agent-subcommands/diagnose/diagnose_nix_test.go +++ b/test/new-e2e/tests/agent-subcommands/diagnose/diagnose_nix_test.go @@ -7,13 +7,15 @@ package diagnose import ( + "slices" "testing" + "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" + "github.com/stretchr/testify/assert" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" svcmanager "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-platform/common/svc-manager" - "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" - "github.com/stretchr/testify/assert" ) type linuxDiagnoseSuite struct { @@ -42,3 +44,17 @@ func (v *linuxDiagnoseSuite) TestDiagnoseLocalFallback() { svcManager.Start("datadog-agent") } + +func (v *linuxDiagnoseSuite) TestDiagnoseInclude() { + if !slices.Contains(allSuites, "port-conflict") { + allSuites = append(allSuites, "port-conflict") + } + v.AssertDiagnoseInclude() +} + +func (v *linuxDiagnoseSuite) TestDiagnoseExclude() { + if !slices.Contains(allSuites, "port-conflict") { + allSuites = append(allSuites, "port-conflict") + } + v.AssertDiagnoseInclude() +} diff --git a/test/new-e2e/tests/agent-subcommands/diagnose/diagnose_win_test.go b/test/new-e2e/tests/agent-subcommands/diagnose/diagnose_win_test.go index 3cf5a99a0058e..f2ef75650628e 100644 --- a/test/new-e2e/tests/agent-subcommands/diagnose/diagnose_win_test.go +++ b/test/new-e2e/tests/agent-subcommands/diagnose/diagnose_win_test.go @@ -9,11 +9,12 @@ package diagnose import ( "testing" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" "github.com/DataDog/test-infra-definitions/components/os" "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" + + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" ) type windowsDiagnoseSuite struct { @@ -31,3 +32,11 @@ func (v *windowsDiagnoseSuite) TestDiagnoseOtherCmdPort() { diagnose := getDiagnoseOutput(&v.baseDiagnoseSuite) v.AssertOutputNotError(diagnose) } + +func (v *windowsDiagnoseSuite) TestDiagnoseInclude() { + v.AssertDiagnoseInclude() +} + +func (v *windowsDiagnoseSuite) TestDiagnoseExclude() { + v.AssertDiagnoseInclude() +} diff --git a/test/new-e2e/tests/installer/all_packages_test.go b/test/new-e2e/tests/installer/all_packages_test.go index 980173ed3cf13..298dee72376bf 100644 --- a/test/new-e2e/tests/installer/all_packages_test.go +++ b/test/new-e2e/tests/installer/all_packages_test.go @@ -156,6 +156,10 @@ func (s *packageBaseSuite) RunInstallScriptWithError(params ...string) error { } func (s *packageBaseSuite) RunInstallScript(params ...string) { + // bugfix for https://major.io/p/systemd-in-fedora-22-failed-to-restart-service-access-denied/ + if s.os.Flavor == e2eos.CentOS && s.os.Version == e2eos.CentOS7.Version { + s.Env().RemoteHost.MustExecute("sudo systemctl daemon-reexec") + } err := s.RunInstallScriptWithError(params...) require.NoErrorf(s.T(), err, "installer not properly installed. logs: \n%s\n%s", s.Env().RemoteHost.MustExecute("cat /tmp/datadog-installer-stdout.log"), s.Env().RemoteHost.MustExecute("cat /tmp/datadog-installer-stderr.log")) } diff --git a/test/new-e2e/tests/installer/host/host.go b/test/new-e2e/tests/installer/host/host.go index 809b22c364a2d..90e93b5dc9d25 100644 --- a/test/new-e2e/tests/installer/host/host.go +++ b/test/new-e2e/tests/installer/host/host.go @@ -31,6 +31,7 @@ type Host struct { os e2eos.Descriptor arch e2eos.Architecture systemdVersion int + pkgManager string } // Option is an option to configure a Host. @@ -49,14 +50,25 @@ func New(t *testing.T, remote *components.RemoteHost, os e2eos.Descriptor, arch } host.uploadFixtures() host.setSystemdVersion() + if _, err := host.remote.Execute("command -v dpkg-query"); err == nil { + host.pkgManager = "dpkg" + } else if _, err := host.remote.Execute("command -v rpm"); err == nil { + host.pkgManager = "rpm" + } else { + t.Fatal("no package manager found") + } return host } +// GetPkgManager returns the package manager of the host. +func (h *Host) GetPkgManager() string { + return h.pkgManager +} + func (h *Host) setSystemdVersion() { strVersion := strings.TrimSpace(h.remote.MustExecute("systemctl --version | head -n1 | awk '{print $2}'")) version, err := strconv.Atoi(strVersion) require.NoError(h.t, err) - h.t.Log("Systemd version:", version) h.systemdVersion = version } @@ -140,12 +152,13 @@ func (h *Host) AssertPackageInstalledByInstaller(pkgs ...string) { // AssertPackageInstalledByPackageManager checks if a package is installed by the package manager on the host. func (h *Host) AssertPackageInstalledByPackageManager(pkgs ...string) { for _, pkg := range pkgs { - if _, err := h.remote.Execute("command -v dpkg-query"); err == nil { + switch h.pkgManager { + case "dpkg": h.remote.MustExecute("dpkg-query -l " + pkg) - } else if _, err := h.remote.Execute("command -v rpm"); err == nil { + case "rpm": h.remote.MustExecute("rpm -q " + pkg) - } else { - h.t.Fatal("no package manager found") + default: + h.t.Fatal("unsupported package manager") } } } @@ -153,12 +166,13 @@ func (h *Host) AssertPackageInstalledByPackageManager(pkgs ...string) { // AssertPackageNotInstalledByPackageManager checks if a package is not installed by the package manager on the host. func (h *Host) AssertPackageNotInstalledByPackageManager(pkgs ...string) { for _, pkg := range pkgs { - if _, err := h.remote.Execute("command -v dpkg-query"); err == nil { + switch h.pkgManager { + case "dpkg": h.remote.MustExecute("! dpkg-query -l " + pkg) - } else if _, err := h.remote.Execute("command -v rpm"); err == nil { + case "rpm": h.remote.MustExecute("! rpm -q " + pkg) - } else { - h.t.Fatal("no package manager found") + default: + h.t.Fatal("unsupported package manager") } } } diff --git a/test/new-e2e/tests/installer/package_agent_test.go b/test/new-e2e/tests/installer/package_agent_test.go index 2603e4c045297..6bd89b73d934f 100644 --- a/test/new-e2e/tests/installer/package_agent_test.go +++ b/test/new-e2e/tests/installer/package_agent_test.go @@ -7,6 +7,7 @@ package installer import ( "fmt" + "path/filepath" awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/tests/installer/host" @@ -39,14 +40,10 @@ func testAgent(os e2eos.Descriptor, arch e2eos.Architecture) packageSuite { func (s *packageAgentSuite) TestInstall() { s.RunInstallScript(envForceInstall("datadog-agent")) defer s.Purge() - s.host.WaitForUnitActive("datadog-agent.service", "datadog-agent-trace.service", "datadog-agent-process.service") + s.host.WaitForUnitActive(agentUnit, traceUnit, processUnit) state := s.host.State() - - state.AssertUnitsLoaded("datadog-agent.service", "datadog-agent-trace.service", "datadog-agent-process.service", "datadog-agent-sysprobe.service", "datadog-agent-security.service") - state.AssertUnitsEnabled("datadog-agent.service", "datadog-agent-trace.service", "datadog-agent-process.service", "datadog-agent-sysprobe.service", "datadog-agent-security.service") - state.AssertUnitsRunning("datadog-agent.service", "datadog-agent-trace.service", "datadog-agent-process.service") - state.AssertUnitsDead("datadog-agent-sysprobe.service", "datadog-agent-security.service") + s.assertUnits(state, false) state.AssertFileExists("/etc/datadog-agent/install_info", 0644, "root", "root") state.AssertFileExists("/etc/datadog-agent/datadog.yaml", 0640, "dd-agent", "dd-agent") @@ -54,6 +51,78 @@ func (s *packageAgentSuite) TestInstall() { // state.AssertFileExists("/etc/datadog-agent/install.json", 0644, "dd-agent", "dd-agent") } +func (s *packageAgentSuite) assertUnits(state host.State, oldUnits bool) { + state.AssertUnitsLoaded(agentUnit, traceUnit, processUnit, probeUnit, securityUnit) + state.AssertUnitsEnabled(agentUnit) + state.AssertUnitsRunning(agentUnit, traceUnit, processUnit) + state.AssertUnitsDead(probeUnit, securityUnit) + + systemdPath := "/etc/systemd/system" + if oldUnits { + pkgManager := s.host.GetPkgManager() + switch pkgManager { + case "dpkg": + systemdPath = "/lib/systemd/system" + case "rpm": + systemdPath = "/usr/lib/systemd/system" + default: + s.T().Fatalf("unsupported package manager: %s", pkgManager) + } + } + + for _, unit := range []string{agentUnit, traceUnit, processUnit, probeUnit, securityUnit} { + + s.host.AssertUnitProperty(unit, "FragmentPath", filepath.Join(systemdPath, unit)) + } +} + +// TestUpgrade_AgentDebRPM_to_OCI tests the upgrade from DEB/RPM agent to the OCI one. +func (s *packageAgentSuite) TestUpgrade_AgentDebRPM_to_OCI() { + // install deb/rpm agent + s.RunInstallScript(envForceNoInstall("datadog-agent")) + s.host.AssertPackageInstalledByPackageManager("datadog-agent") + + defer s.Purge() + defer s.purgeAgentDebInstall() + + state := s.host.State() + s.assertUnits(state, true) + state.AssertDirExists("/opt/datadog-agent", 0755, "dd-agent", "dd-agent") + + // install OCI agent + s.RunInstallScript(envForceInstall("datadog-agent")) + + state = s.host.State() + s.assertUnits(state, false) + s.host.AssertPackageInstalledByInstaller("datadog-agent") + s.host.AssertPackageInstalledByPackageManager("datadog-agent") +} + +// TestUpgrade_Agent_OCI_then_DebRpm agent deb/rpm install while OCI one is installed +func (s *packageAgentSuite) TestUpgrade_Agent_OCI_then_DebRpm() { + // install OCI agent + s.RunInstallScript(envForceInstall("datadog-agent")) + defer s.Purge() + + state := s.host.State() + s.assertUnits(state, false) + state.AssertPathDoesNotExist("/opt/datadog-agent") + + // is_installed avoids a re-install of datadog-agent with the install script + s.RunInstallScript(envForceNoInstall("datadog-agent")) + state.AssertPathDoesNotExist("/opt/datadog-agent") + + // install deb/rpm manually + s.installDebRPMAgent() + defer s.purgeAgentDebInstall() + s.host.AssertPackageInstalledByPackageManager("datadog-agent") + + state = s.host.State() + s.assertUnits(state, false) + state.AssertDirExists("/opt/datadog-agent", 0755, "dd-agent", "dd-agent") + s.host.AssertPackageInstalledByInstaller("datadog-agent") +} + func (s *packageAgentSuite) TestExperimentTimeout() { s.RunInstallScript(envForceInstall("datadog-agent")) defer s.Purge() @@ -124,7 +193,7 @@ func (s *packageAgentSuite) TestExperimentIgnoringSigterm() { SetStopWithSigkill("trace-agent") for _, unit := range []string{traceUnitXP, processUnitXP, agentUnitXP} { - s.T().Logf("Testing timeoutStop of unit %s", traceUnitXP) + s.T().Logf("Testing timeoutStop of unit %s", unit) s.host.AssertUnitProperty(unit, "TimeoutStopUSec", "1min 30s") s.host.Run(fmt.Sprintf("sudo mkdir -p /etc/systemd/system/%s.d/", unit)) defer s.host.Run(fmt.Sprintf("sudo rm -rf /etc/systemd/system/%s.d/", unit)) @@ -299,3 +368,28 @@ func (s *packageAgentSuite) TestExperimentStopped() { ) } } + +func (s *packageAgentSuite) purgeAgentDebInstall() { + pkgManager := s.host.GetPkgManager() + switch pkgManager { + case "dpkg": + s.Env().RemoteHost.Execute("sudo apt-get remove -y --purge datadog-agent") + case "rpm": + s.Env().RemoteHost.Execute("sudo yum remove -y datadog-agent") + default: + s.T().Fatalf("unsupported package manager: %s", pkgManager) + } +} + +func (s *packageAgentSuite) installDebRPMAgent() { + pkgManager := s.host.GetPkgManager() + switch pkgManager { + case "dpkg": + s.Env().RemoteHost.Execute("sudo apt-get install -y --force-yes datadog-agent") + case "rpm": + s.Env().RemoteHost.Execute("sudo yum -y install datadog-agent") + default: + s.T().Fatalf("unsupported package manager: %s", pkgManager) + } + +} diff --git a/test/new-e2e/tests/windows/install-test/system_file_tester.go b/test/new-e2e/tests/windows/install-test/system_file_tester.go index 70fa8d663783c..623b11abbd24b 100644 --- a/test/new-e2e/tests/windows/install-test/system_file_tester.go +++ b/test/new-e2e/tests/windows/install-test/system_file_tester.go @@ -38,6 +38,7 @@ func SystemPaths() []string { `c:\windows\ServiceProfiles\NetworkService\AppData\`, `C:\Windows\System32\Tasks\`, `C:\Windows\System32\spp\`, + `C:\Windows\SystemTemp\`, } }