Skip to content

Commit

Permalink
Merge branch 'main' into akarpowich/debug_logs_flaky_tests
Browse files Browse the repository at this point in the history
  • Loading branch information
akarpz committed Dec 19, 2023
2 parents cfdfa8d + 2618fb9 commit 5c7d4f4
Show file tree
Hide file tree
Showing 312 changed files with 6,393 additions and 1,236 deletions.
20 changes: 18 additions & 2 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,15 @@ jobs:
command: inv -e lint-teamassignment
name: run PR check for team assignment labels

skip_qa:
<<: *job_template
steps:
- restore_cache: *restore_source
- restore_cache: *restore_deps
- run:
command: inv -e lint-skip-qa
name: run PR check for skip-qa labels

milestone:
<<: *job_template
steps:
Expand Down Expand Up @@ -208,8 +217,8 @@ jobs:
- run:
name: setting env vars for click
command: |
echo 'export LC_ALL="C.UTF-8"' >> $BASH_ENV
echo 'export LANG="C.UTF-8"' >> $BASH_ENV
echo 'export LC_ALL="C.UTF-8"' >> $BASH_ENV
echo 'export LANG="C.UTF-8"' >> $BASH_ENV
- run:
name: lint python files
command: inv -e lint-python
Expand Down Expand Up @@ -307,6 +316,13 @@ workflows:
- main
requires:
- dependencies
- skip_qa:
filters:
branches:
ignore:
- main
requires:
- dependencies
- milestone:
filters:
branches:
Expand Down
2 changes: 2 additions & 0 deletions .ddqa/config.toml
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ qa_statuses = [
]
ignored_labels = [
"qa/skip-qa",
"qa/done",
"qa/no-code-change",
]

[teams."Agent Metrics Logs"]
Expand Down
5 changes: 3 additions & 2 deletions .github/CODEOWNERS
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
# Todo: is this file still needed?
/Makefile.trace @DataDog/agent-platform

/release.json @DataDog/agent-platform @DataDog/agent-shared-components @DataDog/agent-metrics-logs @DataDog/windows-kernel-integrations
/release.json @DataDog/agent-platform @DataDog/agent-metrics-logs @DataDog/windows-kernel-integrations
/requirements.txt @DataDog/agent-platform
/pyproject.toml @DataDog/agent-platform
/setup.cfg @DataDog/agent-platform
Expand All @@ -47,6 +47,7 @@
/.github/workflows/windows-*.yml @DataDog/windows-agent
/.github/workflows/cws-btfhub-sync.yml @DataDog/agent-security
/.github/workflows/gohai.yml @DataDog/agent-shared-components
/.github/workflows/go-update-commenter.yml @DataDog/agent-shared-components

# Gitlab files
# Files containing job contents are owned by teams in charge of the jobs + agent-platform
Expand Down Expand Up @@ -339,7 +340,6 @@
/pkg/proto/datadog/trace @DataDog/agent-apm
/pkg/remoteconfig/ @DataDog/remote-config
/pkg/runtime/ @DataDog/agent-shared-components
/pkg/secrets/ @DataDog/agent-shared-components
/pkg/serializer/ @DataDog/agent-metrics-logs
/pkg/tagger/ @DataDog/container-integrations
/pkg/tagset/ @DataDog/agent-shared-components
Expand Down Expand Up @@ -507,6 +507,7 @@
/tools/ @DataDog/agent-platform
/tools/ebpf/ @DataDog/ebpf-platform
/tools/gdb/ @DataDog/agent-shared-components
/tools/go-update/ @DataDog/agent-shared-components
/tools/retry_file_dump/ @DataDog/agent-metrics-logs
/tools/windows/ @DataDog/windows-agent
/tools/windows/DatadogAgentInstaller/WixSetup/localization-en-us.wxl @DataDog/windows-agent @DataDog/documentation
Expand Down
2 changes: 1 addition & 1 deletion .github/PULL_REQUEST_TEMPLATE.md
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ Note: Adding GitHub labels is only possible for contributors with write access.
- [ ] Use the `major_change` label if your change either has a major impact on the code base, is impacting multiple teams or is changing important well-established internals of the Agent. This label will be use during QA to make sure each team pay extra attention to the changed behavior. For any customer facing change use a releasenote.
- [ ] A [release note](https://github.com/DataDog/datadog-agent/blob/main/docs/dev/contributing.md#reno) has been added or the `changelog/no-changelog` label has been applied.
- [ ] Changed code has automated tests for its functionality.
- [ ] Adequate QA/testing plan information is provided if the `qa/skip-qa` label is not applied.
- [ ] Adequate QA/testing plan information is provided. Except if the `qa/skip-qa` label, with required either `qa/done` or `qa/no-code-change` labels, are applied.
- [ ] At least one `team/..` label has been applied, indicating the team(s) that should QA this change.
- [ ] If applicable, docs team has been notified or [an issue has been opened on the documentation repo](https://github.com/DataDog/documentation/issues/new).
- [ ] If applicable, the `need-change/operator` and `need-change/helm` labels have been applied.
Expand Down
7 changes: 7 additions & 0 deletions .github/dependabot.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,7 @@ updates:
- team/agent-platform
- changelog/no-changelog
- qa/skip-qa
- qa/no-code-change
milestone: 22
schedule:
interval: monthly
Expand All @@ -139,6 +140,7 @@ updates:
- team/agent-e2e-test
- changelog/no-changelog
- qa/skip-qa
- qa/no-code-change
- dev/testing
milestone: 22
ignore:
Expand All @@ -157,6 +159,7 @@ updates:
- team/agent-e2e-test
- changelog/no-changelog
- qa/skip-qa
- qa/no-code-change
- dev/testing
milestone: 22
schedule:
Expand All @@ -170,6 +173,7 @@ updates:
- team/agent-platform
- changelog/no-changelog
- qa/skip-qa
- qa/no-code-change
- dev/tooling
milestone: 22
schedule:
Expand All @@ -183,6 +187,7 @@ updates:
- team/agent-security
- changelog/no-changelog
- qa/skip-qa
- qa/no-code-change
- dev/tooling
milestone: 22
schedule:
Expand All @@ -195,6 +200,7 @@ updates:
- team/agent-e2e-test
- changelog/no-changelog
- qa/skip-qa
- qa/no-code-change
- dev/testing
milestone: 22
schedule:
Expand All @@ -208,6 +214,7 @@ updates:
- team/agent-platform
- changelog/no-changelog
- qa/skip-qa
- qa/no-code-change
- dev/tooling
milestone: 22
schedule:
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/cws-btfhub-sync.yml
Original file line number Diff line number Diff line change
Expand Up @@ -85,5 +85,5 @@ jobs:
owner,
repo,
issue_number: result.data.number,
labels: ['changelog/no-changelog', 'qa/skip-qa', 'team/agent-security']
labels: ['changelog/no-changelog', 'qa/skip-qa', 'qa/no-code-change', 'team/agent-security']
});
65 changes: 65 additions & 0 deletions .github/workflows/go-update-commenter.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
name: "Go update commenter"

on:
pull_request:
# Only run on PR label events (in particular not on every commit)
types: [ labeled ]

jobs:
old-versions-match:
# Only run if the PR is labeled with 'go-update'
if: ${{ github.event.label.name == 'go-update' }}
runs-on: ubuntu-latest
steps:
# get the Go version of the target branch
- uses: actions/checkout@v3
with:
ref: ${{ github.base_ref }}
- name: Get former Go version
id: former_go_version
run: |
echo version="$(cat .go-version)" >> $GITHUB_OUTPUT
# get the Go version of the PR branch
- uses: actions/checkout@v3
- name: Get current Go version
id: new_go_version
run: |
echo version="$(cat .go-version)" >> $GITHUB_OUTPUT
# build the comment
- name: Build full comment
id: old_versions
run: |
set -euxo pipefail
# build the base of the Github URL to the current commit
GITHUB_HEAD_URL='${{ github.server_url }}/${{ github.repository }}/blob/${{ github.sha }}'
{
echo "matches<<EOF"
echo "Here are potential matches of the former version:"
echo ""
# this step builds a Markdown list of potential matches
# the script `detect-old-version.sh` displays each matching line as "file:line_number:line_content"
# the sed command transforms this format into a markdown list with Github permalink URL for each file/line
# note that the sed command only works properly with GNU sed (MacOS sed doesn't seem to understand \S properly)
bash -x ./tools/go-update/detect-old-version.sh "${{ steps.former_go_version.outputs.version }}" "${{ steps.new_go_version.outputs.version }}" | \
sed -E 's|^([^:]+):([^:]+):\s*(\S.*)$|- [\1:\2]('"$GITHUB_HEAD_URL"'/\1#L\2): `\3`|'
echo "EOF"
} >> $GITHUB_OUTPUT
# and display it
- uses: actions/github-script@v7
env:
# We need to store the output in an environment variable and not use it directly in the createComment,
# as it will likely not be a valid JS string (eg. if it contains a quote character)
CONTENT: ${{ steps.old_versions.outputs.matches }}
with:
script: |
github.rest.issues.createComment({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
body: process.env.CONTENT
})
2 changes: 1 addition & 1 deletion .gitlab-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,7 @@ variables:
# To use images from test-infra-definitions dev branches, set the SUFFIX variable to -dev
# and check the job creating the image to make sure you have the right SHA prefix
TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX: ""
TEST_INFRA_DEFINITIONS_BUILDIMAGES: 281b2a324002
TEST_INFRA_DEFINITIONS_BUILDIMAGES: 67d2009bcd81
DATADOG_AGENT_BUILDERS: v22276738-b36b132

DATADOG_AGENT_EMBEDDED_PATH: /opt/datadog-agent/embedded
Expand Down
18 changes: 18 additions & 0 deletions .gitlab/e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -207,13 +207,28 @@ new-e2e-agent-shared-components-main:
# Temporary, until we manage to stabilize those tests.
allow_failure: true

.agent-subcommands-tests-matrix: &agent-subcommands-tests-matrix
parallel:
matrix:
- EXTRA_PARAMS: --run TestSubcommandSuite
- EXTRA_PARAMS: --run TestAgentSecretSuite
- EXTRA_PARAMS: --run TestAgentConfigSuite
- EXTRA_PARAMS: --run TestAgentHostnameEC2Suite
- EXTRA_PARAMS: --run TestAgentDiagnoseEC2Suite
- EXTRA_PARAMS: --run TestAgentConfigCheckSuite
- EXTRA_PARAMS: --run TestLinuxFlareSuite
- EXTRA_PARAMS: --run TestWindowsFlareSuite
- EXTRA_PARAMS: --run TestLinuxSecretSuite
- EXTRA_PARAMS: --run TestWindowsSecretSuite

new-e2e-agent-subcommands-dev:
extends: .new_e2e_template
rules: !reference [.on_dev_branch_manual]
needs: []
variables:
TARGETS: ./tests/agent-subcommands
TEAM: agent-shared-components
<<: *agent-subcommands-tests-matrix

new-e2e-agent-subcommands-main:
extends: .new_e2e_template
Expand All @@ -224,6 +239,9 @@ new-e2e-agent-subcommands-main:
TEAM: agent-shared-components
# Temporary, until we manage to stabilize those tests.
allow_failure: true
<<: *agent-subcommands-tests-matrix



new-e2e-language-detection-dev:
extends: .new_e2e_template
Expand Down
17 changes: 17 additions & 0 deletions .gitlab/e2e_test_junit_upload.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,23 @@ e2e_test_junit_upload:
- new-e2e-agent-platform-install-script-centos-fips-a7-x86_64
- new-e2e-agent-platform-install-script-centos-fips-iot-agent-a7-x86_64
- new-e2e-agent-platform-install-script-centos-fips-dogstatsd-a7-x86_64
- new-e2e-agent-platform-step-by-step-debian-a7-x64
- new-e2e-agent-platform-step-by-step-debian-a7-arm64
- new-e2e-agent-platform-step-by-step-debian-a6-x86_64
- new-e2e-agent-platform-step-by-step-debian-a6-arm64
- new-e2e-agent-platform-step-by-step-ubuntu-a6-x86_64
- new-e2e-agent-platform-step-by-step-ubuntu-a6-arm64
- new-e2e-agent-platform-step-by-step-ubuntu-a7-x86_64
- new-e2e-agent-platform-step-by-step-ubuntu-a7-arm64
- new-e2e-agent-platform-step-by-step-suse-a6-x86_64
- new-e2e-agent-platform-step-by-step-suse-a7-x86_64
- new-e2e-agent-platform-step-by-step-suse-a7-arm64
- new-e2e-agent-platform-step-by-step-centos-a6-x86_64
- new-e2e-agent-platform-step-by-step-centos-a7-x86_64
- new-e2e-agent-platform-step-by-step-amazonlinux-a6-x86_64
- new-e2e-agent-platform-step-by-step-amazonlinux-a6-arm64
- new-e2e-agent-platform-step-by-step-amazonlinux-a7-x64
- new-e2e-agent-platform-step-by-step-amazonlinux-a7-arm64
- new-e2e-npm-main
- new-e2e-aml-main
- new-e2e-process-main
Expand Down
4 changes: 2 additions & 2 deletions .gitlab/functional_test/regression_detector.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@ single-machine-performance-regression_detector:
- outputs/report.html # for debugging, also on S3
when: always
variables:
SMP_VERSION: 0.11.0
LADING_VERSION: 0.20.1
SMP_VERSION: 0.12.0
LADING_VERSION: 0.20.4
CPUS: 7
MEMORY: "30g"
# At present we require two artifacts to exist for the 'baseline' and the
Expand Down
2 changes: 1 addition & 1 deletion .gitlab/kernel_version_testing/system_probe.yml
Original file line number Diff line number Diff line change
Expand Up @@ -437,7 +437,7 @@ kernel_matrix_testing_run_tests_arm64:
- INSTANCE_ID="$(jq -r '.[0][0]' < instance.json)"
- echo ${INSTANCE_ID}
- |
if [[ "${INSTANCE_ID}" != "" ]]; then
if [[ "${INSTANCE_ID}" != "" ]] && [[ "${INSTANCE_ID}" != "null" ]]; then
aws ec2 terminate-instances --instance-ids "${INSTANCE_ID}"
fi
Expand Down
32 changes: 0 additions & 32 deletions .gitlab/kitchen_testing/centos.yml
Original file line number Diff line number Diff line change
Expand Up @@ -99,38 +99,6 @@
# We only want to run step-by-step tests on deploy pipelines,
# which is why they have a different rule (if_deploy_6/7)

kitchen_centos_step_by_step_agent-a6:
extends:
- .kitchen_os_with_cws
- .kitchen_scenario_centos_without_fips_a6
- .kitchen_test_step_by_step_agent
rules:
!reference [.on_deploy_a6]

kitchen_centos_fips_step_by_step_agent-a6:
extends:
- .kitchen_os_with_cws
- .kitchen_scenario_centos_with_fips_a6
- .kitchen_test_step_by_step_agent
rules:
!reference [.on_deploy_a6]

kitchen_centos_step_by_step_agent-a7:
extends:
- .kitchen_os_with_cws
- .kitchen_scenario_centos_without_fips_a7
- .kitchen_test_step_by_step_agent
rules:
!reference [.on_deploy_a7]

kitchen_centos_fips_step_by_step_agent-a7:
extends:
- .kitchen_os_with_cws
- .kitchen_scenario_centos_with_fips_a7
- .kitchen_test_step_by_step_agent
rules:
!reference [.on_deploy_a7]

# Agent 5 RPMs won't install on CentOS/RHEL 8 in FIPS mode, so we always
# run upgrade5 on all systems with FIPS off
kitchen_centos_upgrade5_agent-a6:
Expand Down
14 changes: 0 additions & 14 deletions .gitlab/kitchen_testing/debian.yml
Original file line number Diff line number Diff line change
Expand Up @@ -85,20 +85,6 @@ kitchen_debian_install_script_heroku_agent-a6:
# We only want to run step-by-step tests on deploy pipelines,
# which is why they have a different rule (if_deploy_6/7)

kitchen_debian_step_by_step_agent-a6:
extends:
- .kitchen_os_with_cws
- .kitchen_scenario_debian_a6_x64
- .kitchen_test_step_by_step_agent
rules: !reference [.on_deploy_a6]

kitchen_debian_step_by_step_agent-a7:
extends:
- .kitchen_os_with_cws
- .kitchen_scenario_debian_a7_x64
- .kitchen_test_step_by_step_agent
rules: !reference [.on_deploy_a7]

kitchen_debian_upgrade5_agent-a6:
extends:
- .kitchen_scenario_debian_a6_x64
Expand Down
16 changes: 0 additions & 16 deletions .gitlab/kitchen_testing/suse.yml
Original file line number Diff line number Diff line change
Expand Up @@ -78,22 +78,6 @@ kitchen_suse_install_script_dogstatsd_x64-a7:
# We only want to run step-by-step tests on deploy pipelines,
# which is why they have a different rule (if_deploy_6/7)

kitchen_suse_step_by_step_agent_x64-a6:
extends:
- .kitchen_os_with_cws
- .kitchen_scenario_suse_x64_a6
- .kitchen_test_step_by_step_agent
rules:
!reference [.on_deploy_a6]

kitchen_suse_step_by_step_agent_x64-a7:
extends:
- .kitchen_os_with_cws
- .kitchen_scenario_suse_x64_a7
- .kitchen_test_step_by_step_agent
rules:
!reference [.on_deploy_a7]

kitchen_suse_install_script_agent_arm64-a7:
# Run install script test on branches, on a reduced number of platforms
rules:
Expand Down
Loading

0 comments on commit 5c7d4f4

Please sign in to comment.