Split tests across multiple VMSets

.gitlab-ci.yml

@@ -177,7 +177,7 @@ variables:
   # To use images from test-infra-definitions dev branches, set the SUFFIX variable to -dev
   # and check the job creating the image to make sure you have the right SHA prefix
   TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX: ""
-  TEST_INFRA_DEFINITIONS_BUILDIMAGES: a60850d5606e
+  TEST_INFRA_DEFINITIONS_BUILDIMAGES: 40b046c22010
   DATADOG_AGENT_BUILDERS: v22276738-b36b132
 
   DATADOG_AGENT_EMBEDDED_PATH: /opt/datadog-agent/embedded

.gitlab/junit_upload/functional_test_junit_upload_system_probe.yml

@@ -23,17 +23,12 @@ functional_test_junit_upload_system_probe:
     - set -x
     - ss=0; for f in $DD_AGENT_TESTING_DIR/kitchen-junit-*.tar.gz; do [[ -e "$f" ]] || continue; inv -e junit-upload --tgz-path $f || ((ss++)); done; exit $ss
 
-functional_test_junit_upload_system_probe_kmt:
+.functional_test_junit_upload_system_probe_kmt:
   stage: functional_test_junit_upload
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/datadog-ci-uploader$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
   tags: ["arch:amd64"]
   allow_failure: true
   when: always
-  needs:
-    - job: kernel_matrix_testing_run_tests_x64
-      optional: true
-    - job: kernel_matrix_testing_run_tests_arm64
-      optional: true
   variables:
     DD_ENV: ci
   script:
@@ -42,3 +37,19 @@ functional_test_junit_upload_system_probe_kmt:
     - export JIRA_TOKEN=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.jira_read_api_token --with-decryption --query "Parameter.Value" --out text)
     - set -x
     - ss=0; for f in $DD_AGENT_TESTING_DIR/junit-*.tar.gz; do [[ -e "$f" ]] || continue; inv -e junit-upload --tgz-path $f || ((ss++)); done; exit $ss
+
+# junit upload jobs are separted because of gitlab limit on the number of jobs
+# a single job can 'need'.
+functional_test_junit_upload_system_probe_kmt_x64:
+  extends:
+    - .functional_test_junit_upload_system_probe_kmt
+  needs:
+    - job: kernel_matrix_testing_run_tests_x64
+      optional: true
+
+functional_test_junit_upload_system_probe_kmt_arm64:
+  extends:
+    - .functional_test_junit_upload_system_probe_kmt
+  needs:
+    - job: kernel_matrix_testing_run_tests_arm64
+      optional: true

.gitlab/kernel_version_testing/system_probe.yml

@@ -154,6 +154,7 @@ upload_dependencies_arm64:
     - cp -R $KITCHEN_TESTS $SYSTEM_PROBE_TESTS
     - pushd $CI_PROJECT_DIR/test/new-e2e/system-probe/test-runner && go build -o $DEPENDENCIES/test-runner && popd
     - pushd $CI_PROJECT_DIR/test/new-e2e/system-probe/test-json-review && go build -o $DEPENDENCIES/test-json-review && popd
+    - cp $CI_PROJECT_DIR/test/new-e2e/system-probe/test-runner/files/*.json $DEPENDENCIES/
     - popd
     # package all the dependencies
     - ls -la $DEPENDENCIES
@@ -291,6 +292,8 @@ upload_minimized_btfs_arm64:
     RESOURCE_TAGS: "instance-type:${INSTANCE_TYPE},arch:${ARCH}"
     KUBERNETES_MEMORY_REQUEST: "12Gi"
     KUBERNETES_MEMORY_LIMIT: "16Gi"
+    VMCONFIG_FILE: "${CI_PROJECT_DIR}/vmconfig-${CI_PIPELINE_ID}-${ARCH}.json"
+    TEST_SETS: "no_tracersuite,only_tracersuite"
   before_script:
     - set +x
     - export DD_API_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.datadog_api_key_org2 --with-decryption --query "Parameter.Value" --out text)
@@ -301,7 +304,8 @@ upload_minimized_btfs_arm64:
   script:
     - echo "s3://dd-pulumi-state?region=us-east-1&awssdk=v2&profile=$AWS_PROFILE" > $STACK_DIR
     - pulumi login $(cat $STACK_DIR | tr -d '\n')
-    - inv -e system-probe.start-microvms --provision --vmconfig=$VMCONFIG_PATH $INSTANCE_TYPE_ARG $AMI_ID_ARG --ssh-key-name=$AWS_EC2_SSH_KEY_NAME --ssh-key-path=$AWS_EC2_SSH_KEY_FILE --infra-env=$INFRA_ENV --stack-name=kernel-matrix-testing-$ARCH-$CI_PIPELINE_ID --run-agent
+    - inv -e kmt.gen-config --ci --arch=$ARCH --output-file=$VMCONFIG_FILE --sets=$TEST_SETS
+    - inv -e system-probe.start-microvms --provision --vmconfig=$VMCONFIG_FILE $INSTANCE_TYPE_ARG $AMI_ID_ARG --ssh-key-name=$AWS_EC2_SSH_KEY_NAME --ssh-key-path=$AWS_EC2_SSH_KEY_FILE --infra-env=$INFRA_ENV --stack-name=kernel-matrix-testing-$ARCH-$CI_PIPELINE_ID --run-agent
     - cat $CI_PROJECT_DIR/stack.output
     - pulumi logout
   after_script:
@@ -317,6 +321,7 @@ upload_minimized_btfs_arm64:
     paths:
       - $CI_PROJECT_DIR/stack.output
       - $CI_PROJECT_DIR/libvirt
+      - $VMCONFIG_FILE
 
 kernel_matrix_testing_setup_env_arm64:
   extends:
@@ -327,7 +332,6 @@ kernel_matrix_testing_setup_env_arm64:
     ARCH: arm64
     AMI_ID_ARG: "--arm-ami-id=$KERNEL_MATRIX_TESTING_ARM_AMI_ID"
     LibvirtSSHKey: $CI_PROJECT_DIR/libvirt_rsa-arm
-    VMCONFIG_PATH: $CI_PROJECT_DIR/test/new-e2e/system-probe/config/vmconfig-arm64.json
 
 kernel_matrix_testing_setup_env_x64:
   extends:
@@ -338,7 +342,6 @@ kernel_matrix_testing_setup_env_x64:
     ARCH: x86_64
     AMI_ID_ARG: "--x86-ami-id=$KERNEL_MATRIX_TESTING_X86_AMI_ID"
     LibvirtSSHKey: $CI_PROJECT_DIR/libvirt_rsa-x86
-    VMCONFIG_PATH: $CI_PROJECT_DIR/test/new-e2e/system-probe/config/vmconfig-x64.json
 
 .kernel_matrix_testing_run_tests:
   stage: kernel_matrix_testing
@@ -348,6 +351,7 @@ kernel_matrix_testing_setup_env_x64:
   variables:
     AWS_EC2_SSH_KEY_FILE: $CI_PROJECT_DIR/ssh_key
     RETRY: 2
+    MICROVM_GREP_PATTERN: "$ARCH-$TAG-.*$TEST_SET.*-ddvm"
   before_script:
     - set +x
     - export DD_API_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.datadog_api_key_org2 --with-decryption --query "Parameter.Value" --out text)
@@ -367,28 +371,28 @@ kernel_matrix_testing_setup_env_x64:
         echo "These jobs do not permit retries. The go tests are retried a user-specified number of times automatically. In order to re-run the tests, you must trigger the pipeline again"
         'false'
       fi
-    - MICRO_VM_IP=$(cat $CI_PROJECT_DIR/stack.output | grep $ARCH-$TAG | cut -d ' ' -f 2)
-    - MICRO_VM_NAME=$(cat $CI_PROJECT_DIR/stack.output | grep $ARCH-$TAG | cut -d ' ' -f 1)
+    - MICRO_VM_IP=$(cat $CI_PROJECT_DIR/stack.output | grep $MICROVM_GREP_PATTERN | cut -d ' ' -f 2)
+    - MICRO_VM_NAME=$(cat $CI_PROJECT_DIR/stack.output | grep $MICROVM_GREP_PATTERN | cut -d ' ' -f 1)
     - GO_VERSION=$(inv go-version)
     - !reference [.setup_ssh_config]
     # ssh into each micro-vm and run initialization script. This script will also run the tests.
-    - scp "$DD_AGENT_TESTING_DIR/job_env.txt" "metal_instance:/home/ubuntu/job_env-${ARCH}-${TAG}.txt"
-    - ssh metal_instance "scp /home/ubuntu/job_env-${ARCH}-${TAG}.txt ${MICRO_VM_IP}:/job_env.txt"
-    - NESTED_VM_CMD="/home/ubuntu/connector -host ${MICRO_VM_IP} -user root -ssh-file /home/kernel-version-testing/ddvm_rsa -vm-cmd 'bash /root/fetch_dependencies.sh ${ARCH} && /micro-vm-init.sh ${RETRY}'"
+    - scp "$DD_AGENT_TESTING_DIR/job_env.txt" "metal_instance:/home/ubuntu/job_env-${ARCH}-${TAG}-${TEST_SET}.txt"
+    - ssh metal_instance "scp /home/ubuntu/job_env-${ARCH}-${TAG}-${TEST_SET}.txt ${MICRO_VM_IP}:/job_env.txt"
+    - NESTED_VM_CMD="/home/ubuntu/connector -host ${MICRO_VM_IP} -user root -ssh-file /home/kernel-version-testing/ddvm_rsa -vm-cmd 'bash /root/fetch_dependencies.sh ${ARCH} && /micro-vm-init.sh ${RETRY} /${TEST_SET}.json'"
     - $CI_PROJECT_DIR/connector-$ARCH -host $INSTANCE_IP -user ubuntu -ssh-file $AWS_EC2_SSH_KEY_FILE -vm-cmd "${NESTED_VM_CMD}"
     - ssh metal_instance "ssh ${MICRO_VM_IP} '/test-json-review'"
   after_script:
-    - MICRO_VM_IP=$(cat $CI_PROJECT_DIR/stack.output | grep $ARCH-$TAG | cut -d ' ' -f 2)
-    - ssh metal_instance "scp ${MICRO_VM_IP}:/ci-visibility/junit.tar.gz /home/ubuntu/junit-${ARCH}-${TAG}.tar.gz"
-    - scp "metal_instance:/home/ubuntu/junit-${ARCH}-${TAG}.tar.gz" $DD_AGENT_TESTING_DIR/
-    - ssh metal_instance "scp ${MICRO_VM_IP}:/ci-visibility/testjson.tar.gz /home/ubuntu/testjson-${ARCH}-${TAG}.tar.gz"
-    - scp "metal_instance:/home/ubuntu/testjson-${ARCH}-${TAG}.tar.gz" $DD_AGENT_TESTING_DIR/
+    - MICRO_VM_IP=$(cat $CI_PROJECT_DIR/stack.output | grep $MICROVM_GREP_PATTERN | cut -d ' ' -f 2)
+    - ssh metal_instance "scp ${MICRO_VM_IP}:/ci-visibility/junit.tar.gz /home/ubuntu/junit-${ARCH}-${TAG}-${TEST_SET}.tar.gz"
+    - scp "metal_instance:/home/ubuntu/junit-${ARCH}-${TAG}-${TEST_SET}.tar.gz" $DD_AGENT_TESTING_DIR/
+    - ssh metal_instance "scp ${MICRO_VM_IP}:/ci-visibility/testjson.tar.gz /home/ubuntu/testjson-${ARCH}-${TAG}-${TEST_SET}.tar.gz"
+    - scp "metal_instance:/home/ubuntu/testjson-${ARCH}-${TAG}-${TEST_SET}.tar.gz" $DD_AGENT_TESTING_DIR/
   artifacts:
     expire_in: 2 weeks
     when: always
     paths:
-      - $DD_AGENT_TESTING_DIR/junit-$ARCH-$TAG.tar.gz
-      - $DD_AGENT_TESTING_DIR/testjson-$ARCH-$TAG.tar.gz
+      - $DD_AGENT_TESTING_DIR/junit-$ARCH-$TAG-$TEST_SET.tar.gz
+      - $DD_AGENT_TESTING_DIR/testjson-$ARCH-$TAG-$TEST_SET.tar.gz
 
 kernel_matrix_testing_run_tests_x64:
   extends:
@@ -402,6 +406,7 @@ kernel_matrix_testing_run_tests_x64:
   parallel:
     matrix:
       - TAG: ["ubuntu_16.04", "ubuntu_18.04", "ubuntu_20.04", "ubuntu_22.04", "ubuntu_23.10", "amzn_4.14", "amzn_5.4", "amzn_5.10", "fedora_37", "fedora_38", "debian_10", "debian_11", "debian_12", "centos_79", "centos_8"]
+        TEST_SET: ["no_tracersuite", "only_tracersuite"]
 
 kernel_matrix_testing_run_tests_arm64:
   extends:
@@ -415,6 +420,7 @@ kernel_matrix_testing_run_tests_arm64:
   parallel:
     matrix:
       - TAG: ["ubuntu_18.04", "ubuntu_20.04", "ubuntu_22.04", "ubuntu_23.10", "amzn_4.14", "amzn_5.4", "amzn_5.10", "fedora_37", "fedora_38", "debian_10", "debian_11", "debian_12", "centos_79", "centos_8"]
+        TEST_SET: ["no_tracersuite", "only_tracersuite"]
 
 .kernel_matrix_testing_cleanup:
   stage: kernel_matrix_testing