From 0db2ad91820307d6c1f8b808fb05bb3554c52ab1 Mon Sep 17 00:00:00 2001
From: usamasaqib <usama.saqib@datadoghq.com>
Date: Wed, 22 Nov 2023 15:15:43 +0100
Subject: [PATCH 01/46] parallelize kmt tests

---
 .../kernel_version_testing/system_probe.yml   | 42 +++++++--
 .../system-probe/config/vmconfig-arm64.json   | 90 +++++++++++++++++-
 .../system-probe/config/vmconfig-x64.json     | 94 ++++++++++++++++++-
 3 files changed, 217 insertions(+), 9 deletions(-)

diff --git a/.gitlab/kernel_version_testing/system_probe.yml b/.gitlab/kernel_version_testing/system_probe.yml
index b5005039c0f12a..a6a0e07450df8b 100644
--- a/.gitlab/kernel_version_testing/system_probe.yml
+++ b/.gitlab/kernel_version_testing/system_probe.yml
@@ -367,14 +367,14 @@ kernel_matrix_testing_setup_env_x64:
         echo "These jobs do not permit retries. The go tests are retried a user-specified number of times automatically. In order to re-run the tests, you must trigger the pipeline again"
         'false'
       fi
-    - MICRO_VM_IP=$(cat $CI_PROJECT_DIR/stack.output | grep $ARCH-$TAG | cut -d ' ' -f 2)
-    - MICRO_VM_NAME=$(cat $CI_PROJECT_DIR/stack.output | grep $ARCH-$TAG | cut -d ' ' -f 1)
+    - MICRO_VM_IP=$(cat $CI_PROJECT_DIR/stack.output | grep $ARCH-$TAG | grep $TEST_SET | cut -d ' ' -f 2)
+    - MICRO_VM_NAME=$(cat $CI_PROJECT_DIR/stack.output | grep $ARCH-$TAG | grep $TEST_SET | cut -d ' ' -f 1)
     - GO_VERSION=$(inv go-version)
     - !reference [.setup_ssh_config]
     # ssh into each micro-vm and run initialization script. This script will also run the tests.
     - scp "$DD_AGENT_TESTING_DIR/job_env.txt" "metal_instance:/home/ubuntu/job_env-${ARCH}-${TAG}.txt"
     - ssh metal_instance "scp /home/ubuntu/job_env-${ARCH}-${TAG}.txt ${MICRO_VM_IP}:/job_env.txt"
-    - NESTED_VM_CMD="/home/ubuntu/connector -host ${MICRO_VM_IP} -user root -ssh-file /home/kernel-version-testing/ddvm_rsa -vm-cmd 'bash /root/fetch_dependencies.sh ${ARCH} && /micro-vm-init.sh ${RETRY}'"
+    - NESTED_VM_CMD="/home/ubuntu/connector -host ${MICRO_VM_IP} -user root -ssh-file /home/kernel-version-testing/ddvm_rsa -vm-cmd 'bash /root/fetch_dependencies.sh ${ARCH} && /micro-vm-init.sh ${RETRY} ${RUNNER_CMD}'"
     - $CI_PROJECT_DIR/connector-$ARCH -host $INSTANCE_IP -user ubuntu -ssh-file $AWS_EC2_SSH_KEY_FILE -vm-cmd "${NESTED_VM_CMD}"
     - ssh metal_instance "ssh ${MICRO_VM_IP} '/test-json-review'"
   after_script:
@@ -390,7 +390,7 @@ kernel_matrix_testing_setup_env_x64:
       - $DD_AGENT_TESTING_DIR/junit-$ARCH-$TAG.tar.gz
       - $DD_AGENT_TESTING_DIR/testjson-$ARCH-$TAG.tar.gz
 
-kernel_matrix_testing_run_tests_x64:
+.kmt_run_tests_x64:
   extends:
     - .kernel_matrix_testing_run_tests
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/system-probe_x64$DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_SYSPROBE_BUILDIMAGES
@@ -403,7 +403,21 @@ kernel_matrix_testing_run_tests_x64:
     matrix:
       - TAG: ["ubuntu_16.04", "ubuntu_18.04", "ubuntu_20.04", "ubuntu_22.04", "ubuntu_23.10", "amzn_4.14", "amzn_5.4", "amzn_5.10", "fedora_37", "fedora_38", "debian_10", "debian_11", "debian_12", "centos_79", "centos_8"]
 
-kernel_matrix_testing_run_tests_arm64:
+kmt_run_tests_no_tracer_x64:
+  extends:
+    - .kmt_run_tests_x64
+  variables:
+    RUNNER_CMD: "-exclude-packages pkg/network/tracer"
+    TEST_SET: "no_tracer"
+
+kmt_run_tests_only_tracer_x64:
+  extends:
+    - .kmt_run_tests_x64
+  variables:
+    RUNNER_CMD: "-include-packages pkg/network/tracer"
+    TEST_SET: "only_tracer"
+
+.kmt_run_tests_arm64:
   extends:
     - .kernel_matrix_testing_run_tests
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/system-probe_arm64$DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_SYSPROBE_BUILDIMAGES
@@ -416,6 +430,20 @@ kernel_matrix_testing_run_tests_arm64:
     matrix:
       - TAG: ["ubuntu_18.04", "ubuntu_20.04", "ubuntu_22.04", "ubuntu_23.10", "amzn_4.14", "amzn_5.4", "amzn_5.10", "fedora_37", "fedora_38", "debian_10", "debian_11", "debian_12", "centos_79", "centos_8"]
 
+kmt_run_tests_no_tracer_arm64:
+  extends:
+    - .kmt_run_tests_arm64
+  variables:
+    RUNNER_CMD: "-exclude-packages pkg/network/tracer"
+    TEST_SET: "no_tracer"
+
+kmt_run_tests_only_tracer_arm64:
+  extends:
+    - .kmt_run_tests_arm64
+  variables:
+    RUNNER_CMD: "-include-packages pkg/network/tracer"
+    TEST_SET: "only_tracer"
+
 .kernel_matrix_testing_cleanup:
   stage: kernel_matrix_testing
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/test-infra-definitions/runner$TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX:$TEST_INFRA_DEFINITIONS_BUILDIMAGES
@@ -438,7 +466,7 @@ kernel_matrix_testing_run_tests_arm64:
 kernel_matrix_testing_cleanup_arm64:
   extends:
     - .kernel_matrix_testing_cleanup
-  needs: ["kernel_matrix_testing_setup_env_arm64", "kernel_matrix_testing_run_tests_arm64"]
+  needs: ["kernel_matrix_testing_setup_env_arm64", "kmt_run_tests_no_tracer_arm64", "kmt_run_tests_only_tracer_arm64"]
   variables:
     ARCH: arm64
     INSTANCE_TYPE: "m6gd.metal"
@@ -446,7 +474,7 @@ kernel_matrix_testing_cleanup_arm64:
 kernel_matrix_testing_cleanup_x64:
   extends:
     - .kernel_matrix_testing_cleanup
-  needs: ["kernel_matrix_testing_setup_env_x64", "kernel_matrix_testing_run_tests_x64"]
+  needs: ["kernel_matrix_testing_setup_env_x64", "kmt_run_tests_no_tracer_x64", "kmt_run_tests_only_tracer_x64"]
   variables:
     ARCH: x86_64
     INSTANCE_TYPE: "m5d.metal"
diff --git a/test/new-e2e/system-probe/config/vmconfig-arm64.json b/test/new-e2e/system-probe/config/vmconfig-arm64.json
index 045a20eda7380f..2da2c8da60882a 100644
--- a/test/new-e2e/system-probe/config/vmconfig-arm64.json
+++ b/test/new-e2e/system-probe/config/vmconfig-arm64.json
@@ -1,7 +1,95 @@
 {
     "vmsets": [
         {
-            "name": "distro_arm64",
+            "name": "only_tracer_distro_arm64",
+            "recipe": "distro-arm64",
+            "arch": "arm64",
+            "console_type": "file",
+            "kernels": [
+                {
+                    "dir": "bionic-server-cloudimg-arm64.qcow2",
+                    "tag": "ubuntu_18.04",
+                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/bionic-server-cloudimg-arm64.qcow2.xz"
+                },
+                {
+                    "dir": "focal-server-cloudimg-arm64.qcow2",
+                    "tag": "ubuntu_20.04",
+                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/focal-server-cloudimg-arm64.qcow2.xz"
+                },
+                {
+                    "dir": "jammy-server-cloudimg-arm64.qcow2",
+                    "tag": "ubuntu_22.04",
+                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/jammy-server-cloudimg-arm64.qcow2.xz"
+                },
+                {
+                    "dir": "mantic-server-cloudimg-arm64.qcow2",
+                    "tag": "ubuntu_23.10",
+                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/mantic-server-cloudimg-arm64.qcow2.xz"
+                },
+                {
+                    "dir": "amzn2-kvm-2.0-arm64-4.14.qcow2",
+                    "tag": "amzn_4.14",
+                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/amzn2-kvm-2.0-arm64-4.14.qcow2.xz"
+                },
+                {
+                    "dir": "amzn2-kvm-2.0-arm64-5.4.qcow2",
+                    "tag": "amzn_5.4",
+                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/amzn2-kvm-2.0-arm64-5.4.qcow2.xz"
+                },
+                {
+                    "dir": "amzn2-kvm-2.0-arm64-5.10.qcow2",
+                    "tag": "amzn_5.10",
+                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/amzn2-kvm-2.0-arm64-5.10.qcow2.xz"
+                },
+                {
+                    "dir": "Fedora-Cloud-Base-37.arm64.qcow2",
+                    "tag": "fedora_37",
+                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/Fedora-Cloud-Base-37.arm64.qcow2.xz"
+                },
+                {
+                    "dir": "Fedora-Cloud-Base-38.arm64.qcow2",
+                    "tag": "fedora_38",
+                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/Fedora-Cloud-Base-38.arm64.qcow2.xz"
+                },
+                {
+                    "dir": "debian-10-generic-arm64.qcow2",
+                    "tag": "debian_10",
+                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/debian-10-generic-arm64.qcow2.xz-test-only"
+                },
+                {
+                    "dir": "debian-11-generic-arm64.qcow2",
+                    "tag": "debian_11",
+                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/debian-11-generic-arm64.qcow2.xz-test-only"
+                },
+                {
+                    "dir": "debian-12-generic-arm64.qcow2",
+                    "tag": "debian_12",
+                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/debian-12-generic-arm64.qcow2.xz"
+                },
+                {
+                    "dir": "CentOS-7-aarch64-GenericCloud-2211.qcow2",
+                    "tag": "centos_79",
+                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/CentOS-7-aarch64-GenericCloud-2211.qcow2.xz"
+                }
+            ],
+            "machine": "virt",
+            "vcpu": [
+                4
+            ],
+            "memory": [
+                8192
+            ],
+            "disks": [
+                {
+                    "mount_point": "/mnt/docker",
+                    "source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/docker-arm64.qcow2",
+                    "target": "/home/kernel-version-testing/docker-arm64.qcow2",
+                    "type": "default"
+                }
+            ]
+        },
+        {
+            "name": "no_tracer_distro_arm64",
             "recipe": "distro-arm64",
             "arch": "arm64",
             "console_type": "file",
diff --git a/test/new-e2e/system-probe/config/vmconfig-x64.json b/test/new-e2e/system-probe/config/vmconfig-x64.json
index 39ac353d570724..e52f31a14fe96f 100644
--- a/test/new-e2e/system-probe/config/vmconfig-x64.json
+++ b/test/new-e2e/system-probe/config/vmconfig-x64.json
@@ -1,7 +1,99 @@
 {
     "vmsets": [
         {
-            "name": "distro_x86_64",
+            "name": "no_tracer_distro_x86_64",
+            "recipe": "distro-x86_64",
+            "arch": "x86_64",
+            "console_type": "file",
+            "kernels": [
+                {
+                    "dir": "xenial-server-cloudimg-amd64.qcow2",
+                    "tag": "ubuntu_16.04",
+                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/xenial-server-cloudimg-amd64.qcow2.xz"
+                },
+                {
+                    "dir": "bionic-server-cloudimg-amd64.qcow2",
+                    "tag": "ubuntu_18.04",
+                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/bionic-server-cloudimg-amd64.qcow2.xz"
+                },
+                {
+                    "dir": "focal-server-cloudimg-amd64.qcow2",
+                    "tag": "ubuntu_20.04",
+                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/focal-server-cloudimg-amd64.qcow2.xz"
+                },
+                {
+                    "dir": "jammy-server-cloudimg-amd64.qcow2",
+                    "tag": "ubuntu_22.04",
+                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/jammy-server-cloudimg-amd64.qcow2.xz"
+                },
+                {
+                    "dir": "mantic-server-cloudimg-amd64.qcow2",
+                    "tag": "ubuntu_23.10",
+                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/mantic-server-cloudimg-amd64.qcow2.xz"
+                },
+                {
+                    "dir": "amzn2-kvm-2.0-amd64-4.14.qcow2",
+                    "tag": "amzn_4.14",
+                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/amzn2-kvm-2.0-amd64-4.14.qcow2.xz"
+                },
+                {
+                    "dir": "amzn2-kvm-2.0-amd64-5.4.qcow2",
+                    "tag": "amzn_5.4",
+                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/amzn2-kvm-2.0-amd64-5.4.qcow2.xz"
+                },
+                {
+                    "dir": "amzn2-kvm-2.0-amd64-5.10.qcow2",
+                    "tag": "amzn_5.10",
+                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/amzn2-kvm-2.0-amd64-5.10.qcow2.xz"
+                },
+                {
+                    "dir": "Fedora-Cloud-Base-37.amd64.qcow2",
+                    "tag": "fedora_37",
+                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/Fedora-Cloud-Base-37.amd64.qcow2.xz"
+                },
+                {
+                    "dir": "Fedora-Cloud-Base-38.amd64.qcow2",
+                    "tag": "fedora_38",
+                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/Fedora-Cloud-Base-38.amd64.qcow2.xz"
+                },
+                {
+                    "dir": "debian-10-generic-amd64.qcow2",
+                    "tag": "debian_10",
+                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/debian-10-generic-amd64.qcow2.xz"
+                },
+                {
+                    "dir": "debian-11-generic-amd64.qcow2",
+                    "tag": "debian_11",
+                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/debian-11-generic-amd64.qcow2.xz"
+                },
+                {
+                    "dir": "debian-12-generic-amd64.qcow2",
+                    "tag": "debian_12",
+                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/debian-12-generic-amd64.qcow2.xz"
+                },
+                {
+                    "dir": "CentOS-7-x86_64-GenericCloud-2211.qcow2",
+                    "tag": "centos_79",
+                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/CentOS-7-x86_64-GenericCloud-2211.qcow2.xz"
+                }
+            ],
+            "vcpu": [
+                4
+            ],
+            "memory": [
+                8192
+            ],
+            "disks": [
+                {
+                    "mount_point": "/mnt/docker",
+                    "source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/docker-amd64.qcow2",
+                    "target": "/home/kernel-version-testing/docker-amd64.qcow2",
+                    "type": "default"
+                }
+            ]
+        },
+        {
+            "name": "only_tracer_distro_x86_64",
             "recipe": "distro-x86_64",
             "arch": "x86_64",
             "console_type": "file",

From 8104cd685533a1107d5cc0f48fca71fdfa8c064d Mon Sep 17 00:00:00 2001
From: usamasaqib <usama.saqib@datadoghq.com>
Date: Thu, 23 Nov 2023 12:08:18 +0100
Subject: [PATCH 02/46] shorten URNs

---
 test/new-e2e/go.mod | 4 +---
 test/new-e2e/go.sum | 6 ++----
 2 files changed, 3 insertions(+), 7 deletions(-)

diff --git a/test/new-e2e/go.mod b/test/new-e2e/go.mod
index f7ec0801f565b6..dd5d5684e4b260 100644
--- a/test/new-e2e/go.mod
+++ b/test/new-e2e/go.mod
@@ -22,7 +22,7 @@ require (
 	// `TEST_INFRA_DEFINITIONS_BUILDIMAGES` matches the commit sha in the module version
 	// Example: 	github.com/DataDog/test-infra-definitions v0.0.0-YYYYMMDDHHmmSS-0123456789AB
 	// => TEST_INFRA_DEFINITIONS_BUILDIMAGES: 0123456789AB
-	github.com/DataDog/test-infra-definitions v0.0.0-20231129092443-a60850d5606e
+	github.com/DataDog/test-infra-definitions v0.0.0-20231123110544-d505ce3abecf
 	github.com/aws/aws-sdk-go-v2 v1.22.1
 	github.com/aws/aws-sdk-go-v2/config v1.18.40
 	github.com/aws/aws-sdk-go-v2/service/ec2 v1.130.0
@@ -222,5 +222,3 @@ require (
 	sigs.k8s.io/yaml v1.3.0 // indirect
 	sourcegraph.com/sourcegraph/appdash v0.0.0-20211028080628-e2786a622600 // indirect
 )
-
-require github.com/pulumi/pulumi-tls/sdk/v4 v4.10.0 // indirect
diff --git a/test/new-e2e/go.sum b/test/new-e2e/go.sum
index c5f52da5f7155f..b67d12f34fd666 100644
--- a/test/new-e2e/go.sum
+++ b/test/new-e2e/go.sum
@@ -12,8 +12,8 @@ github.com/DataDog/datadog-api-client-go/v2 v2.15.0 h1:5UVON1xs6Lul4d6R5TmLDqqSJ
 github.com/DataDog/datadog-api-client-go/v2 v2.15.0/go.mod h1:ZG8wS+y2rUmkRDJZQq7Og7EAPFPage+7vXcmuah2I9o=
 github.com/DataDog/mmh3 v0.0.0-20200805151601-30884ca2197a h1:m9REhmyaWD5YJ0P53ygRHxKKo+KM+nw+zz0hEdKztMo=
 github.com/DataDog/mmh3 v0.0.0-20200805151601-30884ca2197a/go.mod h1:SvsjzyJlSg0rKsqYgdcFxeEVflx3ZNAyFfkUHP0TxXg=
-github.com/DataDog/test-infra-definitions v0.0.0-20231129092443-a60850d5606e h1:yVjfUFnwtxqppiAnlQbQcGt6wQ9hqaRoI72+HP690OE=
-github.com/DataDog/test-infra-definitions v0.0.0-20231129092443-a60850d5606e/go.mod h1:P6FDz6Iyki8kX1WoDln5U+3kVLC+QzRi7FIzauuZ6jM=
+github.com/DataDog/test-infra-definitions v0.0.0-20231123110544-d505ce3abecf h1:Kx0KlmdWcfLqHjh/SRCEx9auxfCrDtDNoWJWwtdXb5c=
+github.com/DataDog/test-infra-definitions v0.0.0-20231123110544-d505ce3abecf/go.mod h1:eyJ09i8Y8Ofu1gsfNJzL6VtIbLY0MR1atqxsculv4Lo=
 github.com/DataDog/zstd v1.5.2 h1:vUG4lAyuPCXO0TLbXvPv7EB7cNK1QV/luu55UHLrrn8=
 github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
 github.com/DataDog/zstd_0 v0.0.0-20210310093942-586c1286621f h1:5Vuo4niPKFkfwW55jV4vY0ih3VQ9RaQqeqY67fvRn8A=
@@ -453,8 +453,6 @@ github.com/pulumi/pulumi-libvirt/sdk v0.4.0 h1:wq1Ox8FRKQ1kc2DPq3m5DGQgZEhE7kp4m
 github.com/pulumi/pulumi-libvirt/sdk v0.4.0/go.mod h1:tjjyDajp6Pb1pRCdaIugknIfzxw3Prev3o/k2nade+I=
 github.com/pulumi/pulumi-random/sdk/v4 v4.13.4 h1:g3jdktE5L5IDrOw4OiB+yhgxSw0okRPJnyV6PlIzTEQ=
 github.com/pulumi/pulumi-random/sdk/v4 v4.13.4/go.mod h1:cFlJw0eQnqN+62QpITEF9M08gVyzNCeXrKRsuJptFak=
-github.com/pulumi/pulumi-tls/sdk/v4 v4.10.0 h1:4MC0GyEomAjEZJPXEzBZpZ4+TOUg5WE77k38tMDIvS0=
-github.com/pulumi/pulumi-tls/sdk/v4 v4.10.0/go.mod h1:tNXsM/+RsiVVmBdzJMOOp6gMoi3sPko5u0FKdiei+cE=
 github.com/pulumi/pulumi/sdk/v3 v3.16.0/go.mod h1:252ou/zAU1g6E8iTwe2Y9ht7pb5BDl2fJlOuAgZCHiA=
 github.com/pulumi/pulumi/sdk/v3 v3.50.1/go.mod h1:tqQ4z9ocyM/UI2VQ7ZReWR3w6dF5ffEozoHipOMcDh4=
 github.com/pulumi/pulumi/sdk/v3 v3.84.0 h1:/vCRj6ATGVZw4pFmG7pZgjlKUcnbbnb9vmlqd+OpdXo=

From 5a8b9fc6193cfde1c7d9467501635942572cbbfb Mon Sep 17 00:00:00 2001
From: usamasaqib <usama.saqib@datadoghq.com>
Date: Thu, 23 Nov 2023 13:26:55 +0100
Subject: [PATCH 03/46] add test set suffix to CI visibility related files

---
 .gitlab/kernel_version_testing/system_probe.yml | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/.gitlab/kernel_version_testing/system_probe.yml b/.gitlab/kernel_version_testing/system_probe.yml
index a6a0e07450df8b..48e55520625474 100644
--- a/.gitlab/kernel_version_testing/system_probe.yml
+++ b/.gitlab/kernel_version_testing/system_probe.yml
@@ -372,17 +372,17 @@ kernel_matrix_testing_setup_env_x64:
     - GO_VERSION=$(inv go-version)
     - !reference [.setup_ssh_config]
     # ssh into each micro-vm and run initialization script. This script will also run the tests.
-    - scp "$DD_AGENT_TESTING_DIR/job_env.txt" "metal_instance:/home/ubuntu/job_env-${ARCH}-${TAG}.txt"
-    - ssh metal_instance "scp /home/ubuntu/job_env-${ARCH}-${TAG}.txt ${MICRO_VM_IP}:/job_env.txt"
+    - scp "$DD_AGENT_TESTING_DIR/job_env.txt" "metal_instance:/home/ubuntu/job_env-${ARCH}-${TAG}-${TEST_SET}.txt"
+    - ssh metal_instance "scp /home/ubuntu/job_env-${ARCH}-${TAG}-${TEST_SET}.txt ${MICRO_VM_IP}:/job_env.txt"
     - NESTED_VM_CMD="/home/ubuntu/connector -host ${MICRO_VM_IP} -user root -ssh-file /home/kernel-version-testing/ddvm_rsa -vm-cmd 'bash /root/fetch_dependencies.sh ${ARCH} && /micro-vm-init.sh ${RETRY} ${RUNNER_CMD}'"
     - $CI_PROJECT_DIR/connector-$ARCH -host $INSTANCE_IP -user ubuntu -ssh-file $AWS_EC2_SSH_KEY_FILE -vm-cmd "${NESTED_VM_CMD}"
     - ssh metal_instance "ssh ${MICRO_VM_IP} '/test-json-review'"
   after_script:
-    - MICRO_VM_IP=$(cat $CI_PROJECT_DIR/stack.output | grep $ARCH-$TAG | cut -d ' ' -f 2)
-    - ssh metal_instance "scp ${MICRO_VM_IP}:/ci-visibility/junit.tar.gz /home/ubuntu/junit-${ARCH}-${TAG}.tar.gz"
-    - scp "metal_instance:/home/ubuntu/junit-${ARCH}-${TAG}.tar.gz" $DD_AGENT_TESTING_DIR/
-    - ssh metal_instance "scp ${MICRO_VM_IP}:/ci-visibility/testjson.tar.gz /home/ubuntu/testjson-${ARCH}-${TAG}.tar.gz"
-    - scp "metal_instance:/home/ubuntu/testjson-${ARCH}-${TAG}.tar.gz" $DD_AGENT_TESTING_DIR/
+    - MICRO_VM_IP=$(cat $CI_PROJECT_DIR/stack.output | grep $ARCH-$TAG | grep $TEST_SET | cut -d ' ' -f 2)
+    - ssh metal_instance "scp ${MICRO_VM_IP}:/ci-visibility/junit.tar.gz /home/ubuntu/junit-${ARCH}-${TAG}-${TEST_SET}.tar.gz"
+    - scp "metal_instance:/home/ubuntu/junit-${ARCH}-${TAG}-${TEST_SET}.tar.gz" $DD_AGENT_TESTING_DIR/
+    - ssh metal_instance "scp ${MICRO_VM_IP}:/ci-visibility/testjson.tar.gz /home/ubuntu/testjson-${ARCH}-${TAG}-${TEST_SET}.tar.gz"
+    - scp "metal_instance:/home/ubuntu/testjson-${ARCH}-${TAG}-${TEST_SET}.tar.gz" $DD_AGENT_TESTING_DIR/
   artifacts:
     expire_in: 2 weeks
     when: always

From 413b925e7cbbaf7c9ddb14f2e1d802fa56362e04 Mon Sep 17 00:00:00 2001
From: usamasaqib <usama.saqib@datadoghq.com>
Date: Thu, 23 Nov 2023 14:17:27 +0100
Subject: [PATCH 04/46] add test set suffix to artifacts

---
 .gitlab/kernel_version_testing/system_probe.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.gitlab/kernel_version_testing/system_probe.yml b/.gitlab/kernel_version_testing/system_probe.yml
index 48e55520625474..e274d714dfe558 100644
--- a/.gitlab/kernel_version_testing/system_probe.yml
+++ b/.gitlab/kernel_version_testing/system_probe.yml
@@ -387,8 +387,8 @@ kernel_matrix_testing_setup_env_x64:
     expire_in: 2 weeks
     when: always
     paths:
-      - $DD_AGENT_TESTING_DIR/junit-$ARCH-$TAG.tar.gz
-      - $DD_AGENT_TESTING_DIR/testjson-$ARCH-$TAG.tar.gz
+      - $DD_AGENT_TESTING_DIR/junit-$ARCH-$TAG-$TEST_SET.tar.gz
+      - $DD_AGENT_TESTING_DIR/testjson-$ARCH-$TAG-$TEST_SET.tar.gz
 
 .kmt_run_tests_x64:
   extends:

From 92444dc5eac739dfdedf9b5cd9145361da3daf18 Mon Sep 17 00:00:00 2001
From: usamasaqib <usama.saqib@datadoghq.com>
Date: Thu, 23 Nov 2023 17:31:27 +0100
Subject: [PATCH 05/46] pass tests to run as configuration

---
 test/new-e2e/system-probe/test-runner/main.go | 77 +++++++++++--------
 1 file changed, 45 insertions(+), 32 deletions(-)

diff --git a/test/new-e2e/system-probe/test-runner/main.go b/test/new-e2e/system-probe/test-runner/main.go
index 3fb45e420dc6b1..06df12a6ad8060 100644
--- a/test/new-e2e/system-probe/test-runner/main.go
+++ b/test/new-e2e/system-probe/test-runner/main.go
@@ -10,6 +10,7 @@ package main
 import (
 	"bufio"
 	"bytes"
+	"encoding/json"
 	"flag"
 	"fmt"
 	"io"
@@ -26,15 +27,21 @@ import (
 	"golang.org/x/sys/unix"
 )
 
+const matchAllPackages = "*"
+
 func init() {
 	color.NoColor = false
 }
 
+type packageRunConfiguration struct {
+	Run     string
+	Skip    string
+	Exclude bool
+}
+
 type testConfig struct {
-	retryCount      int
-	includePackages []string
-	excludePackages []string
-	runTests        string
+	retryCount        int
+	packagesRunConfig map[string]packageRunConfiguration
 }
 
 const (
@@ -115,8 +122,12 @@ func buildCommandArgs(pkg string, xmlpath string, jsonpath string, file string,
 		"/go/bin/test2json", "-t", "-p", pkg, file, "-test.v", "-test.count=1", "-test.timeout=" + getTimeout(pkg).String(),
 	}
 
-	if testConfig.runTests != "" {
-		args = append(args, "-test.run", testConfig.runTests)
+	packagesRunConfig := testConfig.packagesRunConfig
+	if config, ok := packagesRunConfig[pkg]; ok && config.Run != "" {
+		args = append(args, "-test.run", config.Run)
+	}
+	if config, ok := packagesRunConfig[pkg]; ok && config.Skip != "" {
+		args = append(args, "-test.skip", config.Skip)
 	}
 
 	return args
@@ -161,19 +172,21 @@ func createDir(d string) error {
 func testPass(testConfig *testConfig, props map[string]string) error {
 	testsuites, err := glob(testDirRoot, "testsuite", func(path string) bool {
 		dir := pathToPackage(path)
-		for _, p := range testConfig.excludePackages {
-			if dir == p {
+
+		if config, ok := testConfig.packagesRunConfig[dir]; ok {
+			if config.Exclude {
 				return false
 			}
+
+			return true
 		}
-		if len(testConfig.includePackages) != 0 {
-			for _, p := range testConfig.includePackages {
-				if dir == p {
-					return true
-				}
+
+		if config, ok := testConfig.packagesRunConfig[matchAllPackages]; ok {
+			if config.Exclude {
+				return false
 			}
-			return false
 		}
+
 		return true
 	})
 	if err != nil {
@@ -234,30 +247,26 @@ func fixAssetPermissions() error {
 	return nil
 }
 
-func buildTestConfiguration() *testConfig {
+func buildTestConfiguration() (*testConfig, error) {
 	retryPtr := flag.Int("retry", 2, "number of times to retry testing pass")
-	packagesPtr := flag.String("include-packages", "", "Comma separated list of packages to test")
-	excludePackagesPtr := flag.String("exclude-packages", "", "Comma separated list of packages to exclude")
-	runTestsPtr := flag.String("run-tests", "", "Regex for running specific tests")
+	packageRunConfigPtr := flag.String("packages-run-config", "", "Configuration for controlling which tests run in a package")
 
 	flag.Parse()
 
-	var packagesLs []string
-	var excludeLs []string
-
-	if *packagesPtr != "" {
-		packagesLs = strings.Split(*packagesPtr, ",")
-	}
-	if *excludePackagesPtr != "" {
-		excludeLs = strings.Split(*excludePackagesPtr, ",")
+	breakdown := make(map[string]packageRunConfiguration)
+	if *packageRunConfigPtr != "" {
+		dec := json.NewDecoder(strings.NewReader(*packageRunConfigPtr))
+		dec.DisallowUnknownFields()
+		if err := dec.Decode(&breakdown); err != nil {
+			return nil, err
+		}
 	}
 
+	fmt.Println(breakdown)
 	return &testConfig{
-		retryCount:      *retryPtr,
-		includePackages: packagesLs,
-		excludePackages: excludeLs,
-		runTests:        *runTestsPtr,
-	}
+		retryCount:        *retryPtr,
+		packagesRunConfig: breakdown,
+	}, nil
 }
 
 func readOSRelease() (map[string]string, error) {
@@ -309,7 +318,11 @@ func run() error {
 		return fmt.Errorf("props: %s", err)
 	}
 
-	testConfig := buildTestConfiguration()
+	testConfig, err := buildTestConfiguration()
+	if err != nil {
+		return fmt.Errorf("failed to build test configuration: %w", err)
+	}
+
 	if err := fixAssetPermissions(); err != nil {
 		return fmt.Errorf("asset perms: %s", err)
 	}

From 8bc22af358c26d2f7494e12eb32a5f5e12cc5093 Mon Sep 17 00:00:00 2001
From: usamasaqib <usama.saqib@datadoghq.com>
Date: Thu, 23 Nov 2023 17:38:26 +0100
Subject: [PATCH 06/46] run tests with new configuration

---
 .gitlab/kernel_version_testing/system_probe.yml        | 3 +--
 test/new-e2e/system-probe/test-runner/no_tracer.json   | 5 +++++
 test/new-e2e/system-probe/test-runner/only_tracer.json | 8 ++++++++
 3 files changed, 14 insertions(+), 2 deletions(-)
 create mode 100644 test/new-e2e/system-probe/test-runner/no_tracer.json
 create mode 100644 test/new-e2e/system-probe/test-runner/only_tracer.json

diff --git a/.gitlab/kernel_version_testing/system_probe.yml b/.gitlab/kernel_version_testing/system_probe.yml
index e274d714dfe558..ce877340e0de6f 100644
--- a/.gitlab/kernel_version_testing/system_probe.yml
+++ b/.gitlab/kernel_version_testing/system_probe.yml
@@ -374,6 +374,7 @@ kernel_matrix_testing_setup_env_x64:
     # ssh into each micro-vm and run initialization script. This script will also run the tests.
     - scp "$DD_AGENT_TESTING_DIR/job_env.txt" "metal_instance:/home/ubuntu/job_env-${ARCH}-${TAG}-${TEST_SET}.txt"
     - ssh metal_instance "scp /home/ubuntu/job_env-${ARCH}-${TAG}-${TEST_SET}.txt ${MICRO_VM_IP}:/job_env.txt"
+    - RUNNER_CMD="-packages-run-config $(cat $CI_PROJECT_DIR/test/new-e2e/system-probe/test-runner/$TEST_SET.json | tr -d '\n')"
     - NESTED_VM_CMD="/home/ubuntu/connector -host ${MICRO_VM_IP} -user root -ssh-file /home/kernel-version-testing/ddvm_rsa -vm-cmd 'bash /root/fetch_dependencies.sh ${ARCH} && /micro-vm-init.sh ${RETRY} ${RUNNER_CMD}'"
     - $CI_PROJECT_DIR/connector-$ARCH -host $INSTANCE_IP -user ubuntu -ssh-file $AWS_EC2_SSH_KEY_FILE -vm-cmd "${NESTED_VM_CMD}"
     - ssh metal_instance "ssh ${MICRO_VM_IP} '/test-json-review'"
@@ -434,14 +435,12 @@ kmt_run_tests_no_tracer_arm64:
   extends:
     - .kmt_run_tests_arm64
   variables:
-    RUNNER_CMD: "-exclude-packages pkg/network/tracer"
     TEST_SET: "no_tracer"
 
 kmt_run_tests_only_tracer_arm64:
   extends:
     - .kmt_run_tests_arm64
   variables:
-    RUNNER_CMD: "-include-packages pkg/network/tracer"
     TEST_SET: "only_tracer"
 
 .kernel_matrix_testing_cleanup:
diff --git a/test/new-e2e/system-probe/test-runner/no_tracer.json b/test/new-e2e/system-probe/test-runner/no_tracer.json
new file mode 100644
index 00000000000000..6798ed19245bc0
--- /dev/null
+++ b/test/new-e2e/system-probe/test-runner/no_tracer.json
@@ -0,0 +1,5 @@
+{
+    "pkg/network/tracer": {
+        "run": "TestUSMSuite"
+    }
+}
diff --git a/test/new-e2e/system-probe/test-runner/only_tracer.json b/test/new-e2e/system-probe/test-runner/only_tracer.json
new file mode 100644
index 00000000000000..2eb1de8bdf26e0
--- /dev/null
+++ b/test/new-e2e/system-probe/test-runner/only_tracer.json
@@ -0,0 +1,8 @@
+{
+    "pkg/network/tracer": {
+        "skip": "TestUSMSuite"
+    },
+    "*": {
+        "exclude": true
+    }
+}

From 672e2d7d255e7dd2ed7c01173f6cb183f78521ae Mon Sep 17 00:00:00 2001
From: usamasaqib <usama.saqib@datadoghq.com>
Date: Thu, 23 Nov 2023 18:13:23 +0100
Subject: [PATCH 07/46] simplify

---
 test/new-e2e/system-probe/test-runner/main.go | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/test/new-e2e/system-probe/test-runner/main.go b/test/new-e2e/system-probe/test-runner/main.go
index 06df12a6ad8060..6dac6b140c1399 100644
--- a/test/new-e2e/system-probe/test-runner/main.go
+++ b/test/new-e2e/system-probe/test-runner/main.go
@@ -174,17 +174,11 @@ func testPass(testConfig *testConfig, props map[string]string) error {
 		dir := pathToPackage(path)
 
 		if config, ok := testConfig.packagesRunConfig[dir]; ok {
-			if config.Exclude {
-				return false
-			}
-
-			return true
+			return !config.Exclude
 		}
 
 		if config, ok := testConfig.packagesRunConfig[matchAllPackages]; ok {
-			if config.Exclude {
-				return false
-			}
+			return !config.Exclude
 		}
 
 		return true

From 671d13b2693c1fee527d1a8f29bc0126d30cbb4a Mon Sep 17 00:00:00 2001
From: usamasaqib <usama.saqib@datadoghq.com>
Date: Thu, 23 Nov 2023 18:13:30 +0100
Subject: [PATCH 08/46] fix runner_CMD

---
 .gitlab/kernel_version_testing/system_probe.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab/kernel_version_testing/system_probe.yml b/.gitlab/kernel_version_testing/system_probe.yml
index ce877340e0de6f..ba5e9ae7c068d9 100644
--- a/.gitlab/kernel_version_testing/system_probe.yml
+++ b/.gitlab/kernel_version_testing/system_probe.yml
@@ -374,7 +374,7 @@ kernel_matrix_testing_setup_env_x64:
     # ssh into each micro-vm and run initialization script. This script will also run the tests.
     - scp "$DD_AGENT_TESTING_DIR/job_env.txt" "metal_instance:/home/ubuntu/job_env-${ARCH}-${TAG}-${TEST_SET}.txt"
     - ssh metal_instance "scp /home/ubuntu/job_env-${ARCH}-${TAG}-${TEST_SET}.txt ${MICRO_VM_IP}:/job_env.txt"
-    - RUNNER_CMD="-packages-run-config $(cat $CI_PROJECT_DIR/test/new-e2e/system-probe/test-runner/$TEST_SET.json | tr -d '\n')"
+    - RUNNER_CMD="-packages-run-config '$(cat $CI_PROJECT_DIR/test/new-e2e/system-probe/test-runner/$TEST_SET.json | tr -d '\n')'"
     - NESTED_VM_CMD="/home/ubuntu/connector -host ${MICRO_VM_IP} -user root -ssh-file /home/kernel-version-testing/ddvm_rsa -vm-cmd 'bash /root/fetch_dependencies.sh ${ARCH} && /micro-vm-init.sh ${RETRY} ${RUNNER_CMD}'"
     - $CI_PROJECT_DIR/connector-$ARCH -host $INSTANCE_IP -user ubuntu -ssh-file $AWS_EC2_SSH_KEY_FILE -vm-cmd "${NESTED_VM_CMD}"
     - ssh metal_instance "ssh ${MICRO_VM_IP} '/test-json-review'"

From d441d12cda2cebaca342a2c8e68be41a882c0069 Mon Sep 17 00:00:00 2001
From: usamasaqib <usama.saqib@datadoghq.com>
Date: Thu, 23 Nov 2023 18:48:44 +0100
Subject: [PATCH 09/46] read config data from file

---
 .gitlab/kernel_version_testing/system_probe.yml | 6 ++----
 test/new-e2e/system-probe/test-runner/main.go   | 8 ++++++--
 test/new-e2e/system-probe/test/micro-vm-init.sh | 5 +++--
 3 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/.gitlab/kernel_version_testing/system_probe.yml b/.gitlab/kernel_version_testing/system_probe.yml
index ba5e9ae7c068d9..277088a14fe040 100644
--- a/.gitlab/kernel_version_testing/system_probe.yml
+++ b/.gitlab/kernel_version_testing/system_probe.yml
@@ -154,6 +154,7 @@ upload_dependencies_arm64:
     - cp -R $KITCHEN_TESTS $SYSTEM_PROBE_TESTS
     - pushd $CI_PROJECT_DIR/test/new-e2e/system-probe/test-runner && go build -o $DEPENDENCIES/test-runner && popd
     - pushd $CI_PROJECT_DIR/test/new-e2e/system-probe/test-json-review && go build -o $DEPENDENCIES/test-json-review && popd
+    - cp $CI_PROJECT_DIR/test/new-e2e/system-probe/test-runner/*.json $DEPENDENCIES/
     - popd
     # package all the dependencies
     - ls -la $DEPENDENCIES
@@ -374,8 +375,7 @@ kernel_matrix_testing_setup_env_x64:
     # ssh into each micro-vm and run initialization script. This script will also run the tests.
     - scp "$DD_AGENT_TESTING_DIR/job_env.txt" "metal_instance:/home/ubuntu/job_env-${ARCH}-${TAG}-${TEST_SET}.txt"
     - ssh metal_instance "scp /home/ubuntu/job_env-${ARCH}-${TAG}-${TEST_SET}.txt ${MICRO_VM_IP}:/job_env.txt"
-    - RUNNER_CMD="-packages-run-config '$(cat $CI_PROJECT_DIR/test/new-e2e/system-probe/test-runner/$TEST_SET.json | tr -d '\n')'"
-    - NESTED_VM_CMD="/home/ubuntu/connector -host ${MICRO_VM_IP} -user root -ssh-file /home/kernel-version-testing/ddvm_rsa -vm-cmd 'bash /root/fetch_dependencies.sh ${ARCH} && /micro-vm-init.sh ${RETRY} ${RUNNER_CMD}'"
+    - NESTED_VM_CMD="/home/ubuntu/connector -host ${MICRO_VM_IP} -user root -ssh-file /home/kernel-version-testing/ddvm_rsa -vm-cmd 'bash /root/fetch_dependencies.sh ${ARCH} && /micro-vm-init.sh ${RETRY} /${TEST_SET}.json'"
     - $CI_PROJECT_DIR/connector-$ARCH -host $INSTANCE_IP -user ubuntu -ssh-file $AWS_EC2_SSH_KEY_FILE -vm-cmd "${NESTED_VM_CMD}"
     - ssh metal_instance "ssh ${MICRO_VM_IP} '/test-json-review'"
   after_script:
@@ -408,14 +408,12 @@ kmt_run_tests_no_tracer_x64:
   extends:
     - .kmt_run_tests_x64
   variables:
-    RUNNER_CMD: "-exclude-packages pkg/network/tracer"
     TEST_SET: "no_tracer"
 
 kmt_run_tests_only_tracer_x64:
   extends:
     - .kmt_run_tests_x64
   variables:
-    RUNNER_CMD: "-include-packages pkg/network/tracer"
     TEST_SET: "only_tracer"
 
 .kmt_run_tests_arm64:
diff --git a/test/new-e2e/system-probe/test-runner/main.go b/test/new-e2e/system-probe/test-runner/main.go
index 6dac6b140c1399..b1d7d777a93990 100644
--- a/test/new-e2e/system-probe/test-runner/main.go
+++ b/test/new-e2e/system-probe/test-runner/main.go
@@ -249,14 +249,18 @@ func buildTestConfiguration() (*testConfig, error) {
 
 	breakdown := make(map[string]packageRunConfiguration)
 	if *packageRunConfigPtr != "" {
-		dec := json.NewDecoder(strings.NewReader(*packageRunConfigPtr))
+		configData, err := os.ReadFile(*packageRunConfigPtr)
+		if err != nil {
+			return nil, fmt.Errorf("failed to read config file: %w", err)
+		}
+
+		dec := json.NewDecoder(bytes.NewReader(configData))
 		dec.DisallowUnknownFields()
 		if err := dec.Decode(&breakdown); err != nil {
 			return nil, err
 		}
 	}
 
-	fmt.Println(breakdown)
 	return &testConfig{
 		retryCount:        *retryPtr,
 		packagesRunConfig: breakdown,
diff --git a/test/new-e2e/system-probe/test/micro-vm-init.sh b/test/new-e2e/system-probe/test/micro-vm-init.sh
index 5f92459eb6fd06..5d5ce558079873 100755
--- a/test/new-e2e/system-probe/test/micro-vm-init.sh
+++ b/test/new-e2e/system-probe/test/micro-vm-init.sh
@@ -1,7 +1,8 @@
 #!/bin/bash
 set -eEuxo pipefail
 
-retry_count=$1; shift
+retry_count=$1
+pkgs_run_config_file=$2
 docker_dir=/kmt-docker
 
 # Add provisioning steps here !
@@ -15,7 +16,7 @@ fi
 
 # Start tests
 code=0
-/test-runner -retry "${retry_count}" "${@}" || code=$?
+/test-runner -retry "${retry_count}" -packages-run-config "${pkgs_run_config_file}" || code=$?
 
 cp /job_env.txt /ci-visibility/junit/
 tar -C /ci-visibility/testjson -czvf /ci-visibility/testjson.tar.gz .

From 873a6f8b153a0ba1c8cfff7651701d0c9f507ad4 Mon Sep 17 00:00:00 2001
From: usamasaqib <usama.saqib@datadoghq.com>
Date: Thu, 23 Nov 2023 19:18:04 +0100
Subject: [PATCH 10/46] organize files

---
 .gitlab/kernel_version_testing/system_probe.yml                 | 2 +-
 .../new-e2e/system-probe/test-runner/{ => files}/no_tracer.json | 0
 .../system-probe/test-runner/{ => files}/only_tracer.json       | 0
 test/new-e2e/system-probe/test-runner/main.go                   | 2 ++
 4 files changed, 3 insertions(+), 1 deletion(-)
 rename test/new-e2e/system-probe/test-runner/{ => files}/no_tracer.json (100%)
 rename test/new-e2e/system-probe/test-runner/{ => files}/only_tracer.json (100%)

diff --git a/.gitlab/kernel_version_testing/system_probe.yml b/.gitlab/kernel_version_testing/system_probe.yml
index 277088a14fe040..6c8c716107dacf 100644
--- a/.gitlab/kernel_version_testing/system_probe.yml
+++ b/.gitlab/kernel_version_testing/system_probe.yml
@@ -154,7 +154,7 @@ upload_dependencies_arm64:
     - cp -R $KITCHEN_TESTS $SYSTEM_PROBE_TESTS
     - pushd $CI_PROJECT_DIR/test/new-e2e/system-probe/test-runner && go build -o $DEPENDENCIES/test-runner && popd
     - pushd $CI_PROJECT_DIR/test/new-e2e/system-probe/test-json-review && go build -o $DEPENDENCIES/test-json-review && popd
-    - cp $CI_PROJECT_DIR/test/new-e2e/system-probe/test-runner/*.json $DEPENDENCIES/
+    - cp $CI_PROJECT_DIR/test/new-e2e/system-probe/test-runner/files/*.json $DEPENDENCIES/
     - popd
     # package all the dependencies
     - ls -la $DEPENDENCIES
diff --git a/test/new-e2e/system-probe/test-runner/no_tracer.json b/test/new-e2e/system-probe/test-runner/files/no_tracer.json
similarity index 100%
rename from test/new-e2e/system-probe/test-runner/no_tracer.json
rename to test/new-e2e/system-probe/test-runner/files/no_tracer.json
diff --git a/test/new-e2e/system-probe/test-runner/only_tracer.json b/test/new-e2e/system-probe/test-runner/files/only_tracer.json
similarity index 100%
rename from test/new-e2e/system-probe/test-runner/only_tracer.json
rename to test/new-e2e/system-probe/test-runner/files/only_tracer.json
diff --git a/test/new-e2e/system-probe/test-runner/main.go b/test/new-e2e/system-probe/test-runner/main.go
index b1d7d777a93990..4d0f6bca356052 100644
--- a/test/new-e2e/system-probe/test-runner/main.go
+++ b/test/new-e2e/system-probe/test-runner/main.go
@@ -249,6 +249,8 @@ func buildTestConfiguration() (*testConfig, error) {
 
 	breakdown := make(map[string]packageRunConfiguration)
 	if *packageRunConfigPtr != "" {
+		fmt.Printf("Runner under configuration: %v\n", data)
+
 		configData, err := os.ReadFile(*packageRunConfigPtr)
 		if err != nil {
 			return nil, fmt.Errorf("failed to read config file: %w", err)

From 0e6937b570bf3f022269c925cbd7ab36db362654 Mon Sep 17 00:00:00 2001
From: usamasaqib <usama.saqib@datadoghq.com>
Date: Thu, 23 Nov 2023 19:18:53 +0100
Subject: [PATCH 11/46] prints testnames

---
 test/new-e2e/system-probe/test-runner/main.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/new-e2e/system-probe/test-runner/main.go b/test/new-e2e/system-probe/test-runner/main.go
index 4d0f6bca356052..d79f07de15cbfc 100644
--- a/test/new-e2e/system-probe/test-runner/main.go
+++ b/test/new-e2e/system-probe/test-runner/main.go
@@ -113,7 +113,7 @@ func pathToPackage(path string) string {
 
 func buildCommandArgs(pkg string, xmlpath string, jsonpath string, file string, testConfig *testConfig) []string {
 	args := []string{
-		"--format", "dots",
+		"--format", "testname",
 		"--junitfile", xmlpath,
 		"--jsonfile", jsonpath,
 		fmt.Sprintf("--rerun-fails=%d", testConfig.retryCount),

From 0c70eea9925cae6733a22a31c4e81c3bbc1914cc Mon Sep 17 00:00:00 2001
From: usamasaqib <usama.saqib@datadoghq.com>
Date: Thu, 23 Nov 2023 20:21:52 +0100
Subject: [PATCH 12/46] fix compilation

---
 test/new-e2e/system-probe/test-runner/main.go | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/test/new-e2e/system-probe/test-runner/main.go b/test/new-e2e/system-probe/test-runner/main.go
index d79f07de15cbfc..0601092d47c40d 100644
--- a/test/new-e2e/system-probe/test-runner/main.go
+++ b/test/new-e2e/system-probe/test-runner/main.go
@@ -249,12 +249,11 @@ func buildTestConfiguration() (*testConfig, error) {
 
 	breakdown := make(map[string]packageRunConfiguration)
 	if *packageRunConfigPtr != "" {
-		fmt.Printf("Runner under configuration: %v\n", data)
-
 		configData, err := os.ReadFile(*packageRunConfigPtr)
 		if err != nil {
 			return nil, fmt.Errorf("failed to read config file: %w", err)
 		}
+		fmt.Printf("Runner under configuration: %s\n", configData)
 
 		dec := json.NewDecoder(bytes.NewReader(configData))
 		dec.DisallowUnknownFields()

From cd5d4f5989ff800c5fd6e079358afb16eb78930b Mon Sep 17 00:00:00 2001
From: usamasaqib <usama.saqib@datadoghq.com>
Date: Fri, 24 Nov 2023 10:53:48 +0100
Subject: [PATCH 13/46] update test-infra runner

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f515728354853f..48ae6088598cd5 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -177,7 +177,7 @@ variables:
   # To use images from test-infra-definitions dev branches, set the SUFFIX variable to -dev
   # and check the job creating the image to make sure you have the right SHA prefix
   TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX: ""
-  TEST_INFRA_DEFINITIONS_BUILDIMAGES: a60850d5606e
+  TEST_INFRA_DEFINITIONS_BUILDIMAGES: d505ce3abecf
   DATADOG_AGENT_BUILDERS: v22276738-b36b132
 
   DATADOG_AGENT_EMBEDDED_PATH: /opt/datadog-agent/embedded

From ab15e0bd834dbd2a80dcda178bfdb30c1fa96c0c Mon Sep 17 00:00:00 2001
From: usamasaqib <usama.saqib@datadoghq.com>
Date: Tue, 28 Nov 2023 13:48:58 +0100
Subject: [PATCH 14/46] add to task to generate vmconfig.json at runtime

---
 tasks/kernel_matrix_testing/vmconfig.py       | 468 +++++++++---------
 tasks/kmt.py                                  |   4 +-
 .../system-probe/config/vmconfig-arm64.json   |  90 +---
 .../system-probe/config/vmconfig-x64.json     |  96 +---
 .../new-e2e/system-probe/config/vmconfig.json | 184 +++++++
 5 files changed, 429 insertions(+), 413 deletions(-)
 create mode 100644 test/new-e2e/system-probe/config/vmconfig.json

diff --git a/tasks/kernel_matrix_testing/vmconfig.py b/tasks/kernel_matrix_testing/vmconfig.py
index c827374529147a..21d3269da5eff6 100644
--- a/tasks/kernel_matrix_testing/vmconfig.py
+++ b/tasks/kernel_matrix_testing/vmconfig.py
@@ -1,14 +1,20 @@
+import copy
 import itertools
 import json
 import math
 import os
 import platform
 
-from .download import archs_mapping, karch_mapping, url_base
-from .init_kmt import KMT_STACKS_DIR, VMCONFIG, check_and_get_stack
+from .download import url_base
+from .init_kmt import KMT_STACKS_DIR, VMCONFIG, KMT_ROOTFS_DIR, check_and_get_stack
 from .stacks import ARM_INSTANCE_TYPE, X86_INSTANCE_TYPE, create_stack, stack_exists
 from .tool import Exit, ask, info, warn
 
+vm_recipe = "recipe"
+vm_architecture = "arch"
+vm_version = "version"
+local_arch = "local"
+
 try:
     from thefuzz import fuzz, process
 except ImportError:
@@ -55,81 +61,36 @@
     "4.20",
 ]
 distributions = {
-    "ubuntu_18": "bionic",
-    "ubuntu_20": "focal",
-    "ubuntu_22": "jammy",
-    "jammy": "jammy",
-    "focal": "focal",
-    "bionic": "bionic",
+    # Ubuntu mappings
+    "ubuntu_16": "ubuntu_16.04",
+    "ubuntu_18": "ubuntu_18.04",
+    "ubuntu_20": "ubuntu_20.04",
+    "ubuntu_22": "ubuntu_22.04",
+    "ubuntu_23": "ubuntu_23.10",
+    "xenial": "ubuntu_16.04",
+    "bionic": "ubuntu_18.04",
+    "focal": "ubuntu_20.04",
+    "jammy": "ubuntu_22.04",
+    "mantic": "ubuntu_23.10",
+    # Amazon Linux mappings
     "amazon_4.14": "amzn_4.14",
     "amazon_5.4": "amzn_5.4",
     "amazon_5.10": "amzn_5.10",
-    "amazon_5.15": "amzn_5.15",
     "amzn_4.14": "amzn_4.14",
     "amzn_5.4": "amzn_5.4",
     "amzn_5.10": "amzn_5.10",
-    "amzn_5.15": "amzn_5.15",
-    "fedora_35": "fedora_35",
-    "fedora_36": "fedora_36",
+    # Fedora mappings
     "fedora_37": "fedora_37",
     "fedora_38": "fedora_38",
+    # Debian mappings
     "debian_10": "debian_10",
     "debian_11": "debian_11",
-}
-distro_arch_mapping = {"x86_64": "amd64", "arm64": "arm64"}
-images_path_local = {
-    "bionic": "file:///home/kernel-version-testing/rootfs/bionic-server-cloudimg-{arch}.qcow2",
-    "focal": "file:///home/kernel-version-testing/rootfs/focal-server-cloudimg-{arch}.qcow2",
-    "jammy": "file:///home/kernel-version-testing/rootfs/jammy-server-cloudimg-{arch}.qcow2",
-    "bullseye": "file:///home/kernel-version-testing/rootfs/custom-bullseye.{arch}.qcow2",
-    "buster": "file:///home/kernel-version-testing/rootfs/custom-buster.{arch}.qcow2",
-    "amzn_4.14": "file:///home/kernel-version-testing/rootfs/amzn2-kvm-2.0-{arch}-4.14.qcow2",
-    "amzn_5.4": "file:///home/kernel-version-testing/rootfs/amzn2-kvm-2.0-{arch}-5.4.qcow2",
-    "amzn_5.10": "file:///home/kernel-version-testing/rootfs/amzn2-kvm-2.0-{arch}-5.10.qcow2",
-    "amzn_5.15": "file:///home/kernel-version-testing/rootfs/amzn2-kvm-2.0-{arch}-5.15.qcow2",
-    "fedora_35": "file:///home/kernel-version-testing/rootfs/Fedora-Cloud-Base-35.{arch}.qcow2",
-    "fedora_36": "file:///home/kernel-version-testing/rootfs/Fedora-Cloud-Base-36.{arch}.qcow2",
-    "fedora_37": "file:///home/kernel-version-testing/rootfs/Fedora-Cloud-Base-37.{arch}.qcow2",
-    "fedora_38": "file:///home/kernel-version-testing/rootfs/Fedora-Cloud-Base-38.{arch}.qcow2",
-    "debian_10": "file:///home/kernel-version-testing/rootfs/debian-10-generic-{arch}.qcow2",
-    "debian_11": "file:///home/kernel-version-testing/rootfs/debian-11-generic-{arch}.qcow2",
-}
-
-images_path_s3 = {
-    "bionic": "{url_base}bionic-server-cloudimg-{arch}.qcow2",
-    "focal": "{url_base}focal-server-cloudimg-{arch}.qcow2",
-    "jammy": "{url_base}jammy-server-cloudimg-{arch}.qcow2",
-    "bullseye": "{url_base}custom-bullseye.{arch}.qcow2",
-    "buster": "{url_base}custom-buster.{arch}.qcow2",
-    "amzn_4.14": "{url_base}amzn2-kvm-2.0-{arch}-4.14.qcow2",
-    "amzn_5.4": "{url_base}amzn2-kvm-2.0-{arch}-5.4.qcow2",
-    "amzn_5.10": "{url_base}amzn2-kvm-2.0-{arch}-5.10.qcow2",
-    "amzn_5.15": "{url_base}amzn2-kvm-2.0-{arch}-5.15.qcow2",
-    "fedora_35": "{url_base}Fedora-Cloud-Base-35.{arch}.qcow2",
-    "fedora_36": "{url_base}Fedora-Cloud-Base-36.{arch}.qcow2",
-    "fedora_37": "{url_base}Fedora-Cloud-Base-37.{arch}.qcow2",
-    "fedora_38": "{url_base}Fedora-Cloud-Base-38.{arch}.qcow2",
-    "debian_10": "{url_base}debian-10-generic-{arch}.qcow2",
-    "debian_11": "{url_base}debian-11-generic-{arch}.qcow2",
+    "debian_12": "debian_12",
+    # CentOS mappings
+    "centos_79": "centos_79",
 }
 
-images_name = {
-    "bionic": "bionic-server-cloudimg-{arch}.qcow2",
-    "focal": "focal-server-cloudimg-{arch}.qcow2",
-    "jammy": "jammy-server-cloudimg-{arch}.qcow2",
-    "bullseye": "custom-bullseye.{arch}.qcow2",
-    "buster": "custom-buster.{arch}.qcow2",
-    "amzn_4.14": "amzn2-kvm-2.0-{arch}-4.14.qcow2",
-    "amzn_5.4": "amzn2-kvm-2.0-{arch}-5.4.qcow2",
-    "amzn_5.10": "amzn2-kvm-2.0-{arch}-5.10.qcow2",
-    "amzn_5.15": "amzn2-kvm-2.0-{arch}-5.15.qcow2",
-    "fedora_35": "Fedora-Cloud-Base-35.{arch}.qcow2",
-    "fedora_36": "Fedora-Cloud-Base-36.{arch}.qcow2",
-    "fedora_37": "Fedora-Cloud-Base-37.{arch}.qcow2",
-    "fedora_38": "Fedora-Cloud-Base-38.{arch}.qcow2",
-    "debian_10": "debian-10-generic-{arch}.qcow2",
-    "debian_11": "debian-11-generic-{arch}.qcow2",
-}
+arch_mapping = {"amd64": "x86_64", "x86": "x86_64", "x86_64": "x86_64", "arm64": "arm64", "arm": "arm64", "aarch64": "arm64"}
 
 TICK = "\u2713"
 CROSS = "\u2718"
@@ -152,13 +113,9 @@
 
 consoles = {"x86_64": "ttyS0", "arm64": "ttyAMA0"}
 
-
-def get_image_path(img, arch, local):
-    if local:
-        return images_path_local[img].format(arch=arch)
-
-    return images_path_s3[img].format(arch=arch, url_base=url_base)
-
+def lte_414(version):
+    major, minor = version.split('.')
+    return (int(major) <= 4) and (int(minor) <= 14)
 
 def get_image_list(distro, custom):
     custom_kernels = list()
@@ -207,7 +164,8 @@ def empty_config(file_path):
 
 def list_possible():
     distros = list(distributions.keys())
-    archs = list(archs_mapping.keys())
+    archs = list(arch_mapping.keys())
+    archs.append(local_arch)
 
     result = list()
     possible = list(itertools.product(["custom"], kernels, archs)) + list(itertools.product(["distro"], distros, archs))
@@ -216,7 +174,6 @@ def list_possible():
 
     return result
 
-
 # normalize_vm_def converts the detected user provider vm-def
 # to a standard form with consisten values for
 # recipe: [custom, distro]
@@ -225,221 +182,274 @@ def list_possible():
 # Each normalized_vm_def output corresponds to each VM
 # requested by the user
 def normalize_vm_def(possible, vm):
-    # attempt to fuzzy match user provided vm-def with the possible list.
+    # atempt to fuzzy match user provided vm-def with the possible list.
     vm_def, _ = process.extractOne(vm, possible, scorer=fuzz.token_sort_ratio)
     recipe, version, arch = vm_def.split('-')
 
-    arch = archs_mapping[arch]
+    if arch != local_arch:
+        arch = arch_mapping[arch]
+
     if recipe == "distro":
         version = distributions[version]
 
     return recipe, version, arch
 
+def get_custom_kernel_config(template, recipe, version, arch):
+    if arch == local_arch:
+        arch = arch_mapping[platform.machine()]
 
-def vmset_name_from_id(set_id):
-    recipe, arch, id_tag = set_id
-
-    return f"{recipe}_{id_tag}_{arch}"
-
+    if arch == "x86_64":
+        console = "ttyS0"
+    else:
+        console = "ttyAMA0"
 
-# Set id uniquely categorizes each requested
-# VM into particular sets.
-# Each set id will contain 1 or more of the VMs requested
-# by the user.
-def vmset_id(recipe, version, arch):
-    if recipe == "custom":
-        if lte_414(version):
-            return (recipe, arch, "lte_414")
-        else:
-            return (recipe, arch, "gt_414")
+    if lte_414(version):
+        extra_params = {
+            "console": console,
+            "systemd.unified_cgroup_hierarchy": "0"
+        }
     else:
-        return recipe, arch, "distro"
+        extra_params = {
+            "console": console,
+        }
 
+    return {
+        "dir": f"kernel-{version}.{arch}.pkg",
+        "tag": version,
+        "extra_params": extra_params,
+    }
 
 # This function derives the configuration for each
 # unique kernel or distribution from the normalized vm-def.
 # For more details on the generated configuration element, refer
 # to the micro-vms scenario in test-infra-definitions
-def get_kernel_config(recipe, version, arch):
+def get_kernel_config(template, recipe, version, arch):
     if recipe == "custom":
-        return get_custom_kernel_config(version, arch)
-    elif recipe == "distro":
-        return get_distro_image_config(version, arch)
+        return get_custom_kernel_config(template, recipe, version, arch)
+
+    if arch == "local":
+        arch = arch_mapping[platform.machine()]
 
-    raise Exit(f"Invalid recipe {recipe}")
+    setname = f"{recipe}_{arch}"
 
+    for vmset in template["vmsets"]:
+        if vmset["name"] != setname:
+            continue
 
-def lte_414(version):
-    major, minor = version.split('.')
-    return (int(major) <= 4) and (int(minor) <= 14)
+        for kernel in vmset["kernels"]:
+            if kernel["tag"] == version:
+                return copy.deepcopy(kernel)
 
+    raise Exit(f"No kernel {version} in set {setname}")
 
-def get_custom_kernel_config(version, arch):
-    if arch == "local":
-        arch = archs_mapping[platform.machine()]
 
-    kernel = {
-        "dir": f"kernel-v{version}.{karch_mapping[arch]}.pkg",
-        "tag": version,
-        "extra_params": {"console": consoles[arch]},
+def vmset_exists(vm_config, setname):
+    vmsets = vm_config["vmsets"]
+
+    for vmset in vmsets:
+        if vmset["name"] == setname:
+            return True
+
+    return False
+
+
+def kernel_in_vmset(vmset, kernel):
+    vmset_kernels = vmset["kernels"]
+    for k in vmset_kernels:
+        if k["tag"] == kernel["tag"]:
+            return True
+
+    return False
+
+def get_vmconfig_file():
+    return "test/new-e2e/system-probe/config/vmconfig.json"
+
+def vmset_name(arch, recipe, setprefix):
+    name = f"{recipe}_{arch}"
+    if setprefix != "":
+        return f"{setprefix}_{name}"
+
+    return name
+
+def add_custom_vmset(vmset, vm_config):
+    arch = vmset.arch
+    if arch == local_arch:
+        arch = arch_mapping[platform.machine()]
+
+    lte = False
+    for vm in vmset.vms:
+        if lte_414(vm.version):
+            lte = True
+            break
+
+    image_path = f"custom-bullseye.{arch}.qcow2"
+    if lte:
+        image_path = f"custom-buster.{arch}.qcow2"
+
+    if vmset_exists(vm_config, vmset.name):
+        return
+
+    new_set = {
+        "name": vmset.name,
+        "recipe": f"{vmset.recipe}-{vmset.arch}",
+        "arch": vmset.arch,
+        "kernels": list(),
+        "image": {
+            "image_path": image_path,
+            "image_source": f"https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/{image_path}"
+        }
     }
 
-    if lte_414(version):
-        kernel["extra_params"]["systemd.unified_cgroup_hierarchy"] = "0"
+    vm_config["vmsets"].append(new_set)
 
-    return kernel
 
+def add_vmset(vmset, vm_config):
+    if vmset_exists(vm_config, vmset.name):
+        return
 
-def get_distro_image_config(version, arch):
-    local = False
-    if arch == "local":
-        local = True
-        arch = archs_mapping[platform.machine()]
+    if vmset.recipe == "custom":
+        return add_custom_vmset(vmset, vm_config)
 
-    return {
-        "dir": images_name[version].format(arch=distro_arch_mapping[arch]),
-        "tag": version,
-        "image_source": get_image_path(version, distro_arch_mapping[arch], local),
+    new_set = {
+        "name": vmset.name,
+        "recipe": f"{vmset.recipe}-{vmset.arch}",
+        "arch": vmset.arch,
+        "kernels": list(),
     }
 
+    vm_config["vmsets"].append(new_set)
 
-# This function generates new VMSets. Refer to the documentation
-# of the micro-vm scenario in test-infra-definitions to see what
-# a VMSet is.
-def build_new_vmset(set_id, kernels):
-    recipe, arch, version = set_id
-    vmset = dict()
 
-    local = False
-    if arch == "local":
-        local = True
-        platform_arch = archs_mapping[platform.machine()]
-    else:
-        platform_arch = arch
+def add_kernel(vm_config, kernel, setname):
+    for vmset in vm_config["vmsets"]:
+        if vmset["name"] != setname:
+            continue
 
-    if recipe == "custom":
-        vmset = {"name": vmset_name_from_id(set_id), "recipe": f"custom-{arch}", "arch": arch, "kernels": kernels}
-        if version == "lte_414":
-            vmset["image"] = {
-                "image_path": f"custom-buster.{distro_arch_mapping[platform_arch]}.qcow2",
-                "image_source": get_image_path("buster", distro_arch_mapping[platform_arch], local),
-            }
-        else:
-            vmset["image"] = {
-                "image_path": f"custom-bullseye.{distro_arch_mapping[platform_arch]}.qcow2",
-                "image_source": get_image_path("bullseye", distro_arch_mapping[platform_arch], local),
-            }
-    elif recipe == "distro":
-        vmset = {"name": vmset_name_from_id(set_id), "recipe": f"distro-{arch}", "arch": arch, "kernels": kernels}
-    else:
-        raise Exit(f"Invalid recipe {recipe}")
+        if not kernel_in_vmset(vmset, kernel):
+            vmset["kernels"].append(kernel)
+            return
 
-    if arch == "arm64":
-        vmset["machine"] = "virt"
+    raise Exit(f"Unable to find vmset with name {setname}")
 
-    return vmset
+def add_vcpu(vmset, vcpu):
+    vmset["vcpu"] = vcpu
 
+def add_memory(vmset, memory):
+    vmset["memory"] = memory
 
-def vmset_exists(vm_config, set_name):
-    vmsets = vm_config["vmsets"]
+def add_docker_disk(vmset):
+    arch = vmset['arch']
+    vmset["disks"] = {
+        "mount_point": "/mnt/docker",
+        "source": f"https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/docker-{arch}.qcow2",
+        "target": f"/home/kernel-version-testing/docker-{arch}.qcow2",
+        "type": "default",
+    }
 
-    for vmset in vmsets:
-        if vmset["name"] == set_name:
-            return True
+def add_console(vm_config):
+    vmset["console_type"] = "file"
 
-    return False
+from urllib.parse import urlparse
+def url_to_fspath(url):
+    source = urlparse(url)
+    if os.path.basename(source.path).endswith(".xz"):
+        filename = os.path.basename(source.path)[:-len(".xz")]
+    else:
+        filename = os.path.basename(source.path)
 
+    return f"file://{os.path.join(KMT_ROOTFS_DIR,filename)}"
 
-def kernel_in_vmset(vmset, kernel):
-    vmset_kernels = vmset["kernels"]
-    for k in vmset_kernels:
-        if k["tag"] == kernel["tag"]:
-            return True
+def image_source_to_path(vmset):
+    if vmset["recipe"] == f"custom-{vmset['arch']}":
+        vmset["image"]["image_source"] = url_to_fspath(vmset["image"]["image_source"])
+        return
 
-    return False
+    for kernel in vmset["kernels"]:
+        kernel["image_source"] = url_to_fspath(kernel["image_source"])
 
+class VM:
+    def __init__(self, version):
+        self.version = version
 
-def add_kernels_to_vmset(vmset, set_name, kernels):
-    for k in kernels:
-        if kernel_in_vmset(vmset, k):
-            continue
-        if vmset["name"] == set_name:
-            vmset["kernels"].append(k)
+class VMSet:
+    def __init__(self, arch, recipe, name):
+        self.arch = arch
+        self.recipe = recipe
+        self.name = name
+        self.vms = list()
 
+    def __eq__(self, other):
+        return self.name == other.name
 
-# Each vmset is uniquely identified by its name, which
-# can be derived from the set_id. If a vmset exists,
-# and we have data to add, this function modifies the appropriate
-# vmset.
-def modify_existing_vmsets(vm_config, set_id, kernels):
-    set_name = vmset_name_from_id(set_id)
+    def __hash__(self):
+        return hash(self.name)
 
-    if not vmset_exists(vm_config, set_name):
-        return False
+    def __repr__(self):
+        vm_str = list()
+        for vm in self.vms:
+            vm_str.append(vm.version)
+        return f"<VMSet> name={self.name} arch={self.arch} vms={','.join(vm_str)}"
 
-    vmsets = vm_config["vmsets"]
-    for vmset in vmsets:
-        add_kernels_to_vmset(vmset, set_name, kernels)
+    def add_vm_if_belongs(self, recipe, version, arch):
+        if recipe == "custom":
+            expected_prefix = custom_version_prefix(version)
+            if not self.name.startswith(expected_prefix):
+                return
 
-    return True
+        if self.recipe == recipe and self.arch == arch:
+            self.vms.append(VM(version))
 
+def custom_version_prefix(version):
+    return "lte_414" if lte_414(version) else "gt_414"
 
-def generate_vm_config(vm_config, vms, vcpu, memory):
+def generate_vmconfig(vm_config, user_vm_ls, vcpu, memory, sets, ci):
     # get all possible (recipe, version, arch) combinations we can support.
     possible = list_possible()
 
-    kernels = dict()
-    for vm in vms:
-        normalized_vm_def = normalize_vm_def(possible, vm)
-        set_id = vmset_id(*normalized_vm_def)
-        # generate kernel configuration for each vm-def
-        if set_id not in kernels:
-            kernels[set_id] = [get_kernel_config(*normalized_vm_def)]
-        else:
-            kernels[set_id].append(get_kernel_config(*normalized_vm_def))
+    with open(get_vmconfig_file()) as f:
+        vmconfig_template = json.load(f)
 
-    keys_to_remove = list()
-    # detect if the requested VM falls in an already existing vmset
-    for set_id in kernels:
-        if modify_existing_vmsets(vm_config, set_id, kernels[set_id]):
-            keys_to_remove.append(set_id)
+    # generate all vmsets
+    vmsets = set()
+    for vm in user_vm_ls:
+        recipe, version, arch = normalize_vm_def(possible, vm)
 
-    # delete kernels already added
-    for key in keys_to_remove:
-        del kernels[key]
+        if recipe == "custom":
+            sets.append(custom_version_prefix(version))
 
-    # this loop generates vmsets which do not already exist
-    for set_id in kernels:
-        vm_config["vmsets"].append(build_new_vmset(set_id, kernels[set_id]))
+        # duplicate vm if multiple sets provided by user
+        for s in sets:
+            vmsets.add(VMSet(arch, recipe, vmset_name(arch, recipe, s)))
 
-    # Modify the vcpu and memory configuration of all sets.
-    for vmset in vm_config["vmsets"]:
-        vmset["vcpu"] = vcpu
-        vmset["memory"] = memory
+        if len(sets) == 0:
+            vmsets.add(VMSet(arch, recipe, vmset_name(arch, recipe, "")))
 
-    local_cnt = 0
-    remote_cnt = 0
-    amd64_ec2 = False
-    arm64_ec2 = False
-    for vmset in vm_config["vmsets"]:
-        if vmset["arch"] == "local":
-            local_cnt += len(vmset["kernels"])
-        if vmset["arch"] != "local":
-            remote_cnt += len(vmset["kernels"])
-        if vmset["arch"] == "x86_64":
-            amd64_ec2 = True
-        if vmset["arch"] == "arm64":
-            arm64_ec2 = True
+    # map vms to vmsets
+    for vm in user_vm_ls:
+        recipe, version, arch = normalize_vm_def(possible, vm)
+        for vmset in vmsets:
+            vmset.add_vm_if_belongs(recipe, version, arch)
 
-    print()
-    warn("[!] Please review configuration")
-    if arm64_ec2:
-        info(f"[*] Configuration will launch 1 arm64 {ARM_INSTANCE_TYPE} EC2 instance")
-    if amd64_ec2:
-        info(f"[*] Configuration will launch 1 x86_64 {X86_INSTANCE_TYPE} EC2 instance")
+    # add new vmsets to new vm_config
+    for vmset in vmsets:
+        add_vmset(vmset, vm_config)
 
-    info(f"[*] Configuration launches {local_cnt} VMs locally, and {remote_cnt} VMs on remote instances")
+    # add vm configurations to vmsets.
+    for vmset in vmsets:
+        for vm in vmset.vms:
+            add_kernel(vm_config, get_kernel_config(vmconfig_template, vmset.recipe, vm.version, vmset.arch), vmset.name)
+
+    for vmset in vm_config["vmsets"]:
+        add_vcpu(vmset, vcpu)
+        add_memory(vmset, memory)
+
+        if vmset["arch"] == local_arch:
+            image_source_to_path(vmset)
+        else:
+            add_docker_disk(vmset)
 
+        if ci:
+            add_console(vmset)
 
 def ls_to_int(ls):
     int_ls = list()
@@ -449,7 +459,7 @@ def ls_to_int(ls):
     return int_ls
 
 
-def gen_config(ctx, stack=None, vms="", init_stack=False, vcpu="4", memory="8192", new=False):
+def gen_config(ctx, stack=None, vms="", sets="", init_stack=False, vcpu="4", memory="8192", new=False, ci=False):
     stack = check_and_get_stack(stack)
     if not stack_exists(stack) and not init_stack:
         raise Exit(
@@ -472,7 +482,6 @@ def gen_config(ctx, stack=None, vms="", init_stack=False, vcpu="4", memory="8192
     mem_to_pow_of_2(memory_ls)
 
     vmconfig_file = f"{KMT_STACKS_DIR}/{stack}/{VMCONFIG}"
-    # vmconfig_file = "/tmp/vm-config.json"
     if new or not os.path.exists(vmconfig_file):
         empty_config(vmconfig_file)
 
@@ -480,7 +489,10 @@ def gen_config(ctx, stack=None, vms="", init_stack=False, vcpu="4", memory="8192
         orig_vm_config = f.read()
     vm_config = json.loads(orig_vm_config)
 
-    generate_vm_config(vm_config, vm_types, ls_to_int(vcpu_ls), ls_to_int(memory_ls))
+    set_ls = list()
+    if sets != "":
+        set_ls = sets.split(",")
+    generate_vmconfig(vm_config, vm_types, ls_to_int(vcpu_ls), ls_to_int(memory_ls), set_ls, ci)
     vm_config_str = json.dumps(vm_config, indent=4)
 
     tmpfile = "/tmp/vm.json"
diff --git a/tasks/kmt.py b/tasks/kmt.py
index a3d5aed1369e59..6496d87c32eb7c 100644
--- a/tasks/kmt.py
+++ b/tasks/kmt.py
@@ -45,8 +45,8 @@ def create_stack(ctx, stack=None):
         "init-stack": "Automatically initialize stack if not present. Equivalent to calling 'inv -e kmt.create-stack [--stack=<stack>]'",
     }
 )
-def gen_config(ctx, stack=None, vms="", init_stack=False, vcpu="4", memory="8192", new=False):
-    vmconfig.gen_config(ctx, stack, vms, init_stack, vcpu, memory, new)
+def gen_config(ctx, stack=None, vms="*", sets="", init_stack=False, vcpu="4", memory="8192", new=False, ci=False):
+    vmconfig.gen_config(ctx, stack, vms, sets, init_stack, vcpu, memory, new, ci)
 
 
 @task
diff --git a/test/new-e2e/system-probe/config/vmconfig-arm64.json b/test/new-e2e/system-probe/config/vmconfig-arm64.json
index 2da2c8da60882a..045a20eda7380f 100644
--- a/test/new-e2e/system-probe/config/vmconfig-arm64.json
+++ b/test/new-e2e/system-probe/config/vmconfig-arm64.json
@@ -1,95 +1,7 @@
 {
     "vmsets": [
         {
-            "name": "only_tracer_distro_arm64",
-            "recipe": "distro-arm64",
-            "arch": "arm64",
-            "console_type": "file",
-            "kernels": [
-                {
-                    "dir": "bionic-server-cloudimg-arm64.qcow2",
-                    "tag": "ubuntu_18.04",
-                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/bionic-server-cloudimg-arm64.qcow2.xz"
-                },
-                {
-                    "dir": "focal-server-cloudimg-arm64.qcow2",
-                    "tag": "ubuntu_20.04",
-                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/focal-server-cloudimg-arm64.qcow2.xz"
-                },
-                {
-                    "dir": "jammy-server-cloudimg-arm64.qcow2",
-                    "tag": "ubuntu_22.04",
-                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/jammy-server-cloudimg-arm64.qcow2.xz"
-                },
-                {
-                    "dir": "mantic-server-cloudimg-arm64.qcow2",
-                    "tag": "ubuntu_23.10",
-                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/mantic-server-cloudimg-arm64.qcow2.xz"
-                },
-                {
-                    "dir": "amzn2-kvm-2.0-arm64-4.14.qcow2",
-                    "tag": "amzn_4.14",
-                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/amzn2-kvm-2.0-arm64-4.14.qcow2.xz"
-                },
-                {
-                    "dir": "amzn2-kvm-2.0-arm64-5.4.qcow2",
-                    "tag": "amzn_5.4",
-                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/amzn2-kvm-2.0-arm64-5.4.qcow2.xz"
-                },
-                {
-                    "dir": "amzn2-kvm-2.0-arm64-5.10.qcow2",
-                    "tag": "amzn_5.10",
-                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/amzn2-kvm-2.0-arm64-5.10.qcow2.xz"
-                },
-                {
-                    "dir": "Fedora-Cloud-Base-37.arm64.qcow2",
-                    "tag": "fedora_37",
-                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/Fedora-Cloud-Base-37.arm64.qcow2.xz"
-                },
-                {
-                    "dir": "Fedora-Cloud-Base-38.arm64.qcow2",
-                    "tag": "fedora_38",
-                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/Fedora-Cloud-Base-38.arm64.qcow2.xz"
-                },
-                {
-                    "dir": "debian-10-generic-arm64.qcow2",
-                    "tag": "debian_10",
-                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/debian-10-generic-arm64.qcow2.xz-test-only"
-                },
-                {
-                    "dir": "debian-11-generic-arm64.qcow2",
-                    "tag": "debian_11",
-                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/debian-11-generic-arm64.qcow2.xz-test-only"
-                },
-                {
-                    "dir": "debian-12-generic-arm64.qcow2",
-                    "tag": "debian_12",
-                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/debian-12-generic-arm64.qcow2.xz"
-                },
-                {
-                    "dir": "CentOS-7-aarch64-GenericCloud-2211.qcow2",
-                    "tag": "centos_79",
-                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/CentOS-7-aarch64-GenericCloud-2211.qcow2.xz"
-                }
-            ],
-            "machine": "virt",
-            "vcpu": [
-                4
-            ],
-            "memory": [
-                8192
-            ],
-            "disks": [
-                {
-                    "mount_point": "/mnt/docker",
-                    "source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/docker-arm64.qcow2",
-                    "target": "/home/kernel-version-testing/docker-arm64.qcow2",
-                    "type": "default"
-                }
-            ]
-        },
-        {
-            "name": "no_tracer_distro_arm64",
+            "name": "distro_arm64",
             "recipe": "distro-arm64",
             "arch": "arm64",
             "console_type": "file",
diff --git a/test/new-e2e/system-probe/config/vmconfig-x64.json b/test/new-e2e/system-probe/config/vmconfig-x64.json
index e52f31a14fe96f..ed380f7d0cc9a0 100644
--- a/test/new-e2e/system-probe/config/vmconfig-x64.json
+++ b/test/new-e2e/system-probe/config/vmconfig-x64.json
@@ -1,99 +1,7 @@
-{
+{r
     "vmsets": [
         {
-            "name": "no_tracer_distro_x86_64",
-            "recipe": "distro-x86_64",
-            "arch": "x86_64",
-            "console_type": "file",
-            "kernels": [
-                {
-                    "dir": "xenial-server-cloudimg-amd64.qcow2",
-                    "tag": "ubuntu_16.04",
-                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/xenial-server-cloudimg-amd64.qcow2.xz"
-                },
-                {
-                    "dir": "bionic-server-cloudimg-amd64.qcow2",
-                    "tag": "ubuntu_18.04",
-                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/bionic-server-cloudimg-amd64.qcow2.xz"
-                },
-                {
-                    "dir": "focal-server-cloudimg-amd64.qcow2",
-                    "tag": "ubuntu_20.04",
-                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/focal-server-cloudimg-amd64.qcow2.xz"
-                },
-                {
-                    "dir": "jammy-server-cloudimg-amd64.qcow2",
-                    "tag": "ubuntu_22.04",
-                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/jammy-server-cloudimg-amd64.qcow2.xz"
-                },
-                {
-                    "dir": "mantic-server-cloudimg-amd64.qcow2",
-                    "tag": "ubuntu_23.10",
-                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/mantic-server-cloudimg-amd64.qcow2.xz"
-                },
-                {
-                    "dir": "amzn2-kvm-2.0-amd64-4.14.qcow2",
-                    "tag": "amzn_4.14",
-                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/amzn2-kvm-2.0-amd64-4.14.qcow2.xz"
-                },
-                {
-                    "dir": "amzn2-kvm-2.0-amd64-5.4.qcow2",
-                    "tag": "amzn_5.4",
-                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/amzn2-kvm-2.0-amd64-5.4.qcow2.xz"
-                },
-                {
-                    "dir": "amzn2-kvm-2.0-amd64-5.10.qcow2",
-                    "tag": "amzn_5.10",
-                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/amzn2-kvm-2.0-amd64-5.10.qcow2.xz"
-                },
-                {
-                    "dir": "Fedora-Cloud-Base-37.amd64.qcow2",
-                    "tag": "fedora_37",
-                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/Fedora-Cloud-Base-37.amd64.qcow2.xz"
-                },
-                {
-                    "dir": "Fedora-Cloud-Base-38.amd64.qcow2",
-                    "tag": "fedora_38",
-                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/Fedora-Cloud-Base-38.amd64.qcow2.xz"
-                },
-                {
-                    "dir": "debian-10-generic-amd64.qcow2",
-                    "tag": "debian_10",
-                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/debian-10-generic-amd64.qcow2.xz"
-                },
-                {
-                    "dir": "debian-11-generic-amd64.qcow2",
-                    "tag": "debian_11",
-                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/debian-11-generic-amd64.qcow2.xz"
-                },
-                {
-                    "dir": "debian-12-generic-amd64.qcow2",
-                    "tag": "debian_12",
-                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/debian-12-generic-amd64.qcow2.xz"
-                },
-                {
-                    "dir": "CentOS-7-x86_64-GenericCloud-2211.qcow2",
-                    "tag": "centos_79",
-                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/CentOS-7-x86_64-GenericCloud-2211.qcow2.xz"
-                }
-            ],
-            "vcpu": [
-                4
-            ],
-            "memory": [
-                8192
-            ],
-            "disks": [
-                {
-                    "mount_point": "/mnt/docker",
-                    "source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/docker-amd64.qcow2",
-                    "target": "/home/kernel-version-testing/docker-amd64.qcow2",
-                    "type": "default"
-                }
-            ]
-        },
-        {
-            "name": "only_tracer_distro_x86_64",
+            "name": "distro_x86_64",
             "recipe": "distro-x86_64",
             "arch": "x86_64",
             "console_type": "file",
diff --git a/test/new-e2e/system-probe/config/vmconfig.json b/test/new-e2e/system-probe/config/vmconfig.json
new file mode 100644
index 00000000000000..545f33668ab34f
--- /dev/null
+++ b/test/new-e2e/system-probe/config/vmconfig.json
@@ -0,0 +1,184 @@
+{
+    "vmsets": [
+        {
+            "name": "distro_x86_64",
+            "recipe": "distro-x86_64",
+            "arch": "x86_64",
+            "console_type": "file",
+            "kernels": [
+                {
+                    "dir": "xenial-server-cloudimg-x86_64.qcow2",
+                    "tag": "ubuntu_16.04",
+                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/xenial-server-cloudimg-x86_64.qcow2.xz"
+                },
+                {
+                    "dir": "bionic-server-cloudimg-x86_64.qcow2",
+                    "tag": "ubuntu_18.04",
+                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/bionic-server-cloudimg-x86_64.qcow2.xz"
+                },
+                {
+                    "dir": "focal-server-cloudimg-x86_64.qcow2",
+                    "tag": "ubuntu_20.04",
+                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/focal-server-cloudimg-x86_64.qcow2.xz"
+                },
+                {
+                    "dir": "jammy-server-cloudimg-x86_64.qcow2",
+                    "tag": "ubuntu_22.04",
+                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/jammy-server-cloudimg-x86_64.qcow2.xz"
+                },
+                {
+                    "dir": "mantic-server-cloudimg-x86_64.qcow2",
+                    "tag": "ubuntu_23.10",
+                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/mantic-server-cloudimg-x86_64.qcow2.xz"
+                },
+                {
+                    "dir": "amzn2-kvm-2.0-x86_64-4.14.qcow2",
+                    "tag": "amzn_4.14",
+                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/amzn2-kvm-2.0-x86_64-4.14.qcow2.xz"
+                },
+                {
+                    "dir": "amzn2-kvm-2.0-x86_64-5.4.qcow2",
+                    "tag": "amzn_5.4",
+                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/amzn2-kvm-2.0-x86_64-5.4.qcow2.xz"
+                },
+                {
+                    "dir": "amzn2-kvm-2.0-x86_64-5.10.qcow2",
+                    "tag": "amzn_5.10",
+                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/amzn2-kvm-2.0-x86_64-5.10.qcow2.xz"
+                },
+                {
+                    "dir": "Fedora-Cloud-Base-37.x86_64.qcow2",
+                    "tag": "fedora_37",
+                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/Fedora-Cloud-Base-37.x86_64.qcow2.xz"
+                },
+                {
+                    "dir": "Fedora-Cloud-Base-38.x86_64.qcow2",
+                    "tag": "fedora_38",
+                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/Fedora-Cloud-Base-38.x86_64.qcow2.xz"
+                },
+                {
+                    "dir": "debian-10-generic-x86_64.qcow2",
+                    "tag": "debian_10",
+                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/debian-10-generic-x86_64.qcow2.xz"
+                },
+                {
+                    "dir": "debian-11-generic-x86_64.qcow2",
+                    "tag": "debian_11",
+                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/debian-11-generic-x86_64.qcow2.xz"
+                },
+                {
+                    "dir": "debian-12-generic-x86_64.qcow2",
+                    "tag": "debian_12",
+                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/debian-12-generic-x86_64.qcow2.xz"
+                },
+                {
+                    "dir": "CentOS-7-x86_64-GenericCloud-2211.qcow2",
+                    "tag": "centos_79",
+                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/CentOS-7-x86_64-GenericCloud-2211.qcow2.xz"
+                }
+            ],
+            "vcpu": [
+                4
+            ],
+            "memory": [
+                8192
+            ],
+            "disks": [
+                {
+                    "mount_point": "/mnt/docker",
+                    "source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/docker-x86_64.qcow2",
+                    "target": "/home/kernel-version-testing/docker-x86_64.qcow2",
+                    "type": "default"
+                }
+            ]
+        },
+        {
+            "name": "distro_arm64",
+            "recipe": "distro-arm64",
+            "arch": "arm64",
+            "console_type": "file",
+            "kernels": [
+                {
+                    "dir": "bionic-server-cloudimg-arm64.qcow2",
+                    "tag": "ubuntu_18.04",
+                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/bionic-server-cloudimg-arm64.qcow2.xz"
+                },
+                {
+                    "dir": "focal-server-cloudimg-arm64.qcow2",
+                    "tag": "ubuntu_20.04",
+                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/focal-server-cloudimg-arm64.qcow2.xz"
+                },
+                {
+                    "dir": "jammy-server-cloudimg-arm64.qcow2",
+                    "tag": "ubuntu_22.04",
+                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/jammy-server-cloudimg-arm64.qcow2.xz"
+                },
+                {
+                    "dir": "mantic-server-cloudimg-arm64.qcow2",
+                    "tag": "ubuntu_23.10",
+                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/mantic-server-cloudimg-arm64.qcow2.xz"
+                },
+                {
+                    "dir": "amzn2-kvm-2.0-arm64-4.14.qcow2",
+                    "tag": "amzn_4.14",
+                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/amzn2-kvm-2.0-arm64-4.14.qcow2.xz"
+                },
+                {
+                    "dir": "amzn2-kvm-2.0-arm64-5.4.qcow2",
+                    "tag": "amzn_5.4",
+                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/amzn2-kvm-2.0-arm64-5.4.qcow2.xz"
+                },
+                {
+                    "dir": "amzn2-kvm-2.0-arm64-5.10.qcow2",
+                    "tag": "amzn_5.10",
+                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/amzn2-kvm-2.0-arm64-5.10.qcow2.xz"
+                },
+                {
+                    "dir": "Fedora-Cloud-Base-37.arm64.qcow2",
+                    "tag": "fedora_37",
+                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/Fedora-Cloud-Base-37.arm64.qcow2.xz"
+                },
+                {
+                    "dir": "Fedora-Cloud-Base-38.arm64.qcow2",
+                    "tag": "fedora_38",
+                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/Fedora-Cloud-Base-38.arm64.qcow2.xz"
+                },
+                {
+                    "dir": "debian-10-generic-arm64.qcow2",
+                    "tag": "debian_10",
+                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/debian-10-generic-arm64.qcow2.xz-test-only"
+                },
+                {
+                    "dir": "debian-11-generic-arm64.qcow2",
+                    "tag": "debian_11",
+                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/debian-11-generic-arm64.qcow2.xz-test-only"
+                },
+                {
+                    "dir": "debian-12-generic-arm64.qcow2",
+                    "tag": "debian_12",
+                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/debian-12-generic-arm64.qcow2.xz"
+                },
+                {
+                    "dir": "CentOS-7-aarch64-GenericCloud-2211.qcow2",
+                    "tag": "centos_79",
+                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/CentOS-7-aarch64-GenericCloud-2211.qcow2.xz"
+                }
+            ],
+            "machine": "virt",
+            "vcpu": [
+                4
+            ],
+            "memory": [
+                8192
+            ],
+            "disks": [
+                {
+                    "mount_point": "/mnt/docker",
+                    "source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/docker-arm64.qcow2",
+                    "target": "/home/kernel-version-testing/docker-arm64.qcow2",
+                    "type": "default"
+                }
+            ]
+        }
+    ]
+}

From 33b7f028c7990e057527cf83d25d4f369fba895f Mon Sep 17 00:00:00 2001
From: usamasaqib <usama.saqib@datadoghq.com>
Date: Tue, 28 Nov 2023 14:32:27 +0100
Subject: [PATCH 15/46] generate configuration file

---
 .../kernel_version_testing/system_probe.yml   |  6 +-
 tasks/kernel_matrix_testing/vmconfig.py       | 62 +++++++++++++++----
 tasks/kmt.py                                  |  4 +-
 3 files changed, 55 insertions(+), 17 deletions(-)

diff --git a/.gitlab/kernel_version_testing/system_probe.yml b/.gitlab/kernel_version_testing/system_probe.yml
index 6c8c716107dacf..5a5ddd0496db4f 100644
--- a/.gitlab/kernel_version_testing/system_probe.yml
+++ b/.gitlab/kernel_version_testing/system_probe.yml
@@ -302,7 +302,8 @@ upload_minimized_btfs_arm64:
   script:
     - echo "s3://dd-pulumi-state?region=us-east-1&awssdk=v2&profile=$AWS_PROFILE" > $STACK_DIR
     - pulumi login $(cat $STACK_DIR | tr -d '\n')
-    - inv -e system-probe.start-microvms --provision --vmconfig=$VMCONFIG_PATH $INSTANCE_TYPE_ARG $AMI_ID_ARG --ssh-key-name=$AWS_EC2_SSH_KEY_NAME --ssh-key-path=$AWS_EC2_SSH_KEY_FILE --infra-env=$INFRA_ENV --stack-name=kernel-matrix-testing-$ARCH-$CI_PIPELINE_ID --run-agent
+    - inv -e kmt.gen-config --ci --arch=$ARCH
+    - inv -e system-probe.start-microvms --provision --vmconfig=vmconfig-$CI_PIPELINE_ID.json $INSTANCE_TYPE_ARG $AMI_ID_ARG --ssh-key-name=$AWS_EC2_SSH_KEY_NAME --ssh-key-path=$AWS_EC2_SSH_KEY_FILE --infra-env=$INFRA_ENV --stack-name=kernel-matrix-testing-$ARCH-$CI_PIPELINE_ID --run-agent
     - cat $CI_PROJECT_DIR/stack.output
     - pulumi logout
   after_script:
@@ -318,6 +319,7 @@ upload_minimized_btfs_arm64:
     paths:
       - $CI_PROJECT_DIR/stack.output
       - $CI_PROJECT_DIR/libvirt
+      - vmconfig-$CI_PIPELINE_ID.json
 
 kernel_matrix_testing_setup_env_arm64:
   extends:
@@ -328,7 +330,6 @@ kernel_matrix_testing_setup_env_arm64:
     ARCH: arm64
     AMI_ID_ARG: "--arm-ami-id=$KERNEL_MATRIX_TESTING_ARM_AMI_ID"
     LibvirtSSHKey: $CI_PROJECT_DIR/libvirt_rsa-arm
-    VMCONFIG_PATH: $CI_PROJECT_DIR/test/new-e2e/system-probe/config/vmconfig-arm64.json
 
 kernel_matrix_testing_setup_env_x64:
   extends:
@@ -339,7 +340,6 @@ kernel_matrix_testing_setup_env_x64:
     ARCH: x86_64
     AMI_ID_ARG: "--x86-ami-id=$KERNEL_MATRIX_TESTING_X86_AMI_ID"
     LibvirtSSHKey: $CI_PROJECT_DIR/libvirt_rsa-x86
-    VMCONFIG_PATH: $CI_PROJECT_DIR/test/new-e2e/system-probe/config/vmconfig-x64.json
 
 .kernel_matrix_testing_run_tests:
   stage: kernel_matrix_testing
diff --git a/tasks/kernel_matrix_testing/vmconfig.py b/tasks/kernel_matrix_testing/vmconfig.py
index 21d3269da5eff6..72743d301e389b 100644
--- a/tasks/kernel_matrix_testing/vmconfig.py
+++ b/tasks/kernel_matrix_testing/vmconfig.py
@@ -346,7 +346,7 @@ def add_docker_disk(vmset):
         "type": "default",
     }
 
-def add_console(vm_config):
+def add_console(vmset):
     vmset["console_type"] = "file"
 
 from urllib.parse import urlparse
@@ -451,6 +451,8 @@ def generate_vmconfig(vm_config, user_vm_ls, vcpu, memory, sets, ci):
         if ci:
             add_console(vmset)
 
+    return vm_config
+
 def ls_to_int(ls):
     int_ls = list()
     for elem in ls:
@@ -459,7 +461,7 @@ def ls_to_int(ls):
     return int_ls
 
 
-def gen_config(ctx, stack=None, vms="", sets="", init_stack=False, vcpu="4", memory="8192", new=False, ci=False):
+def gen_config_for_stack(ctx, stack, vms, sets, init_stack, vcpu, memory, new, ci):
     stack = check_and_get_stack(stack)
     if not stack_exists(stack) and not init_stack:
         raise Exit(
@@ -475,12 +477,6 @@ def gen_config(ctx, stack=None, vms="", sets="", init_stack=False, vcpu="4", mem
     if len(vm_types) == 0:
         raise Exit("No VMs to boot provided")
 
-    vcpu_ls = vcpu.split(',')
-    memory_ls = memory.split(',')
-
-    check_memory_and_vcpus(memory_ls, vcpu_ls)
-    mem_to_pow_of_2(memory_ls)
-
     vmconfig_file = f"{KMT_STACKS_DIR}/{stack}/{VMCONFIG}"
     if new or not os.path.exists(vmconfig_file):
         empty_config(vmconfig_file)
@@ -489,10 +485,7 @@ def gen_config(ctx, stack=None, vms="", sets="", init_stack=False, vcpu="4", mem
         orig_vm_config = f.read()
     vm_config = json.loads(orig_vm_config)
 
-    set_ls = list()
-    if sets != "":
-        set_ls = sets.split(",")
-    generate_vmconfig(vm_config, vm_types, ls_to_int(vcpu_ls), ls_to_int(memory_ls), set_ls, ci)
+    vm_config = generate_vmconfig(vm_config, vm_types, vcpu, memory, sets, ci)
     vm_config_str = json.dumps(vm_config, indent=4)
 
     tmpfile = "/tmp/vm.json"
@@ -513,3 +506,48 @@ def gen_config(ctx, stack=None, vms="", sets="", init_stack=False, vcpu="4", mem
         f.write(vm_config_str)
 
     info(f"[+] vmconfig @ {vmconfig_file}")
+
+def list_all_distro_vms(archs):
+    with open(get_vmconfig_file()) as f:
+        vmconfig = json.load(f)
+
+    vms = list()
+    for arch in archs:
+        distributions = list()
+        for vmset in vmconfig["vmsets"]:
+            if vmset["arch"] not in arch:
+                continue
+
+            for kernel in vmset["kernels"]:
+                distributions.append(kernel["tag"])
+
+        for distro in distributions:
+            vms.append(f"{distro}-{arch}-distro")
+
+    return vms
+
+
+def gen_config(ctx, stack, vms, sets, init_stack, vcpu, memory, new, ci, arch):
+    vcpu_ls = vcpu.split(',')
+    memory_ls = memory.split(',')
+
+    check_memory_and_vcpus(memory_ls, vcpu_ls)
+    mem_to_pow_of_2(memory_ls)
+    set_ls = list()
+    if sets != "":
+        set_ls = sets.split(",")
+
+    if not ci:
+        return gen_config_for_stack(ctx, stack, vms, set_ls, init_stack, ls_to_int(vcpu_ls), ls_to_int(memory_ls), new, ci)
+
+    arch_ls = ["x86_64", "arm64"]
+    if arch != "":
+        arch_ls = [arch_mapping[arch]]
+
+    vms_to_generate = list_all_distro_vms(arch_ls)
+
+    vm_config = generate_vmconfig({"vmsets":[]}, vms_to_generate, ls_to_int(vcpu_ls), ls_to_int(memory_ls), set_ls, ci)
+
+    pipeline_id = os.environ["CI_PIPELINE_ID"]
+    with open(f"vmconfig-{pipeline_id}.json", "rw") as f:
+        f.write(vm_config)
diff --git a/tasks/kmt.py b/tasks/kmt.py
index 6496d87c32eb7c..882fd8c97cfac6 100644
--- a/tasks/kmt.py
+++ b/tasks/kmt.py
@@ -45,8 +45,8 @@ def create_stack(ctx, stack=None):
         "init-stack": "Automatically initialize stack if not present. Equivalent to calling 'inv -e kmt.create-stack [--stack=<stack>]'",
     }
 )
-def gen_config(ctx, stack=None, vms="*", sets="", init_stack=False, vcpu="4", memory="8192", new=False, ci=False):
-    vmconfig.gen_config(ctx, stack, vms, sets, init_stack, vcpu, memory, new, ci)
+def gen_config(ctx, stack=None, vms="", sets="", init_stack=False, vcpu="4", memory="8192", new=False, ci=False, arch=""):
+    vmconfig.gen_config(ctx, stack, vms, sets, init_stack, vcpu, memory, new, ci, arch)
 
 
 @task

From a56cf13050f105c3dffa6a099019d571f8ac7e4a Mon Sep 17 00:00:00 2001
From: usamasaqib <usama.saqib@datadoghq.com>
Date: Tue, 28 Nov 2023 14:54:22 +0100
Subject: [PATCH 16/46] avoid use of uninstalled library in ci

---
 tasks/kernel_matrix_testing/vmconfig.py | 29 +++++++++++++------------
 1 file changed, 15 insertions(+), 14 deletions(-)

diff --git a/tasks/kernel_matrix_testing/vmconfig.py b/tasks/kernel_matrix_testing/vmconfig.py
index 72743d301e389b..0d411719df122b 100644
--- a/tasks/kernel_matrix_testing/vmconfig.py
+++ b/tasks/kernel_matrix_testing/vmconfig.py
@@ -402,18 +402,13 @@ def add_vm_if_belongs(self, recipe, version, arch):
 def custom_version_prefix(version):
     return "lte_414" if lte_414(version) else "gt_414"
 
-def generate_vmconfig(vm_config, user_vm_ls, vcpu, memory, sets, ci):
-    # get all possible (recipe, version, arch) combinations we can support.
-    possible = list_possible()
-
+def generate_vmconfig(vm_config, normalized_vm_defs, vcpu, memory, sets, ci):
     with open(get_vmconfig_file()) as f:
         vmconfig_template = json.load(f)
 
     # generate all vmsets
     vmsets = set()
-    for vm in user_vm_ls:
-        recipe, version, arch = normalize_vm_def(possible, vm)
-
+    for recipe, version, arch in normalized_vm_defs:
         if recipe == "custom":
             sets.append(custom_version_prefix(version))
 
@@ -425,8 +420,7 @@ def generate_vmconfig(vm_config, user_vm_ls, vcpu, memory, sets, ci):
             vmsets.add(VMSet(arch, recipe, vmset_name(arch, recipe, "")))
 
     # map vms to vmsets
-    for vm in user_vm_ls:
-        recipe, version, arch = normalize_vm_def(possible, vm)
+    for recipe, version, arch in normalized_vm_defs:
         for vmset in vmsets:
             vmset.add_vm_if_belongs(recipe, version, arch)
 
@@ -473,10 +467,17 @@ def gen_config_for_stack(ctx, stack, vms, sets, init_stack, vcpu, memory, new, c
 
     info(f"[+] Select stack {stack}")
 
+    # get normalized vm definitions
     vm_types = vms.split(',')
     if len(vm_types) == 0:
         raise Exit("No VMs to boot provided")
 
+    ## get all possible (recipe, version, arch) combinations we can support.
+    possible = list_possible()
+    normalized_vms = list()
+    for vm in vm_types:
+        normalized_vms.append(normalize_vm_def(possible, vm))
+
     vmconfig_file = f"{KMT_STACKS_DIR}/{stack}/{VMCONFIG}"
     if new or not os.path.exists(vmconfig_file):
         empty_config(vmconfig_file)
@@ -485,7 +486,7 @@ def gen_config_for_stack(ctx, stack, vms, sets, init_stack, vcpu, memory, new, c
         orig_vm_config = f.read()
     vm_config = json.loads(orig_vm_config)
 
-    vm_config = generate_vmconfig(vm_config, vm_types, vcpu, memory, sets, ci)
+    vm_config = generate_vmconfig(vm_config, normalized_vms, vcpu, memory, sets, ci)
     vm_config_str = json.dumps(vm_config, indent=4)
 
     tmpfile = "/tmp/vm.json"
@@ -507,7 +508,7 @@ def gen_config_for_stack(ctx, stack, vms, sets, init_stack, vcpu, memory, new, c
 
     info(f"[+] vmconfig @ {vmconfig_file}")
 
-def list_all_distro_vms(archs):
+def list_all_distro_normalized_vms(archs):
     with open(get_vmconfig_file()) as f:
         vmconfig = json.load(f)
 
@@ -521,8 +522,8 @@ def list_all_distro_vms(archs):
             for kernel in vmset["kernels"]:
                 distributions.append(kernel["tag"])
 
-        for distro in distributions:
-            vms.append(f"{distro}-{arch}-distro")
+        for distro_version in distributions:
+            vms.append(("distro", distro_version, arch))
 
     return vms
 
@@ -544,7 +545,7 @@ def gen_config(ctx, stack, vms, sets, init_stack, vcpu, memory, new, ci, arch):
     if arch != "":
         arch_ls = [arch_mapping[arch]]
 
-    vms_to_generate = list_all_distro_vms(arch_ls)
+    vms_to_generate = list_all_distro_normalized_vms(arch_ls)
 
     vm_config = generate_vmconfig({"vmsets":[]}, vms_to_generate, ls_to_int(vcpu_ls), ls_to_int(memory_ls), set_ls, ci)
 

From af4e34e2365207b7d722de27080b42ea6240032b Mon Sep 17 00:00:00 2001
From: usamasaqib <usama.saqib@datadoghq.com>
Date: Tue, 28 Nov 2023 15:09:56 +0100
Subject: [PATCH 17/46] fix file open

---
 tasks/kernel_matrix_testing/vmconfig.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/tasks/kernel_matrix_testing/vmconfig.py b/tasks/kernel_matrix_testing/vmconfig.py
index 0d411719df122b..52e4444dba14d8 100644
--- a/tasks/kernel_matrix_testing/vmconfig.py
+++ b/tasks/kernel_matrix_testing/vmconfig.py
@@ -5,9 +5,9 @@
 import os
 import platform
 
-from .download import url_base
 from .init_kmt import KMT_STACKS_DIR, VMCONFIG, KMT_ROOTFS_DIR, check_and_get_stack
-from .stacks import ARM_INSTANCE_TYPE, X86_INSTANCE_TYPE, create_stack, stack_exists
+from .stacks import create_stack, stack_exists
+from urllib.parse import urlparse
 from .tool import Exit, ask, info, warn
 
 vm_recipe = "recipe"
@@ -349,7 +349,6 @@ def add_docker_disk(vmset):
 def add_console(vmset):
     vmset["console_type"] = "file"
 
-from urllib.parse import urlparse
 def url_to_fspath(url):
     source = urlparse(url)
     if os.path.basename(source.path).endswith(".xz"):
@@ -550,5 +549,5 @@ def gen_config(ctx, stack, vms, sets, init_stack, vcpu, memory, new, ci, arch):
     vm_config = generate_vmconfig({"vmsets":[]}, vms_to_generate, ls_to_int(vcpu_ls), ls_to_int(memory_ls), set_ls, ci)
 
     pipeline_id = os.environ["CI_PIPELINE_ID"]
-    with open(f"vmconfig-{pipeline_id}.json", "rw") as f:
+    with open(f"vmconfig-{pipeline_id}.json", "w") as f:
         f.write(vm_config)

From cc21f899f3b855e0384f36c277fe82be265b5c60 Mon Sep 17 00:00:00 2001
From: usamasaqib <usama.saqib@datadoghq.com>
Date: Tue, 28 Nov 2023 15:33:25 +0100
Subject: [PATCH 18/46] pass vmconfig file in the ci

---
 .gitlab/kernel_version_testing/system_probe.yml | 5 +++--
 tasks/kernel_matrix_testing/vmconfig.py         | 7 +++----
 tasks/kmt.py                                    | 4 ++--
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/.gitlab/kernel_version_testing/system_probe.yml b/.gitlab/kernel_version_testing/system_probe.yml
index 5a5ddd0496db4f..cd36fe651121aa 100644
--- a/.gitlab/kernel_version_testing/system_probe.yml
+++ b/.gitlab/kernel_version_testing/system_probe.yml
@@ -292,6 +292,7 @@ upload_minimized_btfs_arm64:
     RESOURCE_TAGS: "instance-type:${INSTANCE_TYPE},arch:${ARCH}"
     KUBERNETES_MEMORY_REQUEST: "12Gi"
     KUBERNETES_MEMORY_LIMIT: "16Gi"
+    VMCONFIG_FILE: "vmconfig-${CI_PIPELINE_ID}-${ARCH}.json"
   before_script:
     - set +x
     - export DD_API_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.datadog_api_key_org2 --with-decryption --query "Parameter.Value" --out text)
@@ -303,7 +304,7 @@ upload_minimized_btfs_arm64:
     - echo "s3://dd-pulumi-state?region=us-east-1&awssdk=v2&profile=$AWS_PROFILE" > $STACK_DIR
     - pulumi login $(cat $STACK_DIR | tr -d '\n')
     - inv -e kmt.gen-config --ci --arch=$ARCH
-    - inv -e system-probe.start-microvms --provision --vmconfig=vmconfig-$CI_PIPELINE_ID.json $INSTANCE_TYPE_ARG $AMI_ID_ARG --ssh-key-name=$AWS_EC2_SSH_KEY_NAME --ssh-key-path=$AWS_EC2_SSH_KEY_FILE --infra-env=$INFRA_ENV --stack-name=kernel-matrix-testing-$ARCH-$CI_PIPELINE_ID --run-agent
+    - inv -e system-probe.start-microvms --provision --vmconfig=$VMCONFIG_FILE $INSTANCE_TYPE_ARG $AMI_ID_ARG --ssh-key-name=$AWS_EC2_SSH_KEY_NAME --ssh-key-path=$AWS_EC2_SSH_KEY_FILE --infra-env=$INFRA_ENV --stack-name=kernel-matrix-testing-$ARCH-$CI_PIPELINE_ID --run-agent
     - cat $CI_PROJECT_DIR/stack.output
     - pulumi logout
   after_script:
@@ -319,7 +320,7 @@ upload_minimized_btfs_arm64:
     paths:
       - $CI_PROJECT_DIR/stack.output
       - $CI_PROJECT_DIR/libvirt
-      - vmconfig-$CI_PIPELINE_ID.json
+      - $VMCONFIG_FILE
 
 kernel_matrix_testing_setup_env_arm64:
   extends:
diff --git a/tasks/kernel_matrix_testing/vmconfig.py b/tasks/kernel_matrix_testing/vmconfig.py
index 52e4444dba14d8..16dcabb4f19a75 100644
--- a/tasks/kernel_matrix_testing/vmconfig.py
+++ b/tasks/kernel_matrix_testing/vmconfig.py
@@ -527,7 +527,7 @@ def list_all_distro_normalized_vms(archs):
     return vms
 
 
-def gen_config(ctx, stack, vms, sets, init_stack, vcpu, memory, new, ci, arch):
+def gen_config(ctx, stack, vms, sets, init_stack, vcpu, memory, new, ci, arch, output_file):
     vcpu_ls = vcpu.split(',')
     memory_ls = memory.split(',')
 
@@ -548,6 +548,5 @@ def gen_config(ctx, stack, vms, sets, init_stack, vcpu, memory, new, ci, arch):
 
     vm_config = generate_vmconfig({"vmsets":[]}, vms_to_generate, ls_to_int(vcpu_ls), ls_to_int(memory_ls), set_ls, ci)
 
-    pipeline_id = os.environ["CI_PIPELINE_ID"]
-    with open(f"vmconfig-{pipeline_id}.json", "w") as f:
-        f.write(vm_config)
+    with open(output_file, "w") as f:
+        f.write(json.dumps(vm_config, indent=4))
diff --git a/tasks/kmt.py b/tasks/kmt.py
index 882fd8c97cfac6..1681c6b18eaf0c 100644
--- a/tasks/kmt.py
+++ b/tasks/kmt.py
@@ -45,8 +45,8 @@ def create_stack(ctx, stack=None):
         "init-stack": "Automatically initialize stack if not present. Equivalent to calling 'inv -e kmt.create-stack [--stack=<stack>]'",
     }
 )
-def gen_config(ctx, stack=None, vms="", sets="", init_stack=False, vcpu="4", memory="8192", new=False, ci=False, arch=""):
-    vmconfig.gen_config(ctx, stack, vms, sets, init_stack, vcpu, memory, new, ci, arch)
+def gen_config(ctx, stack=None, vms="", sets="", init_stack=False, vcpu="4", memory="8192", new=False, ci=False, arch="", output_file="vmconfig.json"):
+    vmconfig.gen_config(ctx, stack, vms, sets, init_stack, vcpu, memory, new, ci, arch, output_file)
 
 
 @task

From fd2be6ede0ebb4be545578bc84ac67d0a7c766ac Mon Sep 17 00:00:00 2001
From: usamasaqib <usama.saqib@datadoghq.com>
Date: Tue, 28 Nov 2023 15:34:25 +0100
Subject: [PATCH 19/46] add sets into vmconfig file

---
 .gitlab/kernel_version_testing/system_probe.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitlab/kernel_version_testing/system_probe.yml b/.gitlab/kernel_version_testing/system_probe.yml
index cd36fe651121aa..f569bbf724095d 100644
--- a/.gitlab/kernel_version_testing/system_probe.yml
+++ b/.gitlab/kernel_version_testing/system_probe.yml
@@ -293,6 +293,7 @@ upload_minimized_btfs_arm64:
     KUBERNETES_MEMORY_REQUEST: "12Gi"
     KUBERNETES_MEMORY_LIMIT: "16Gi"
     VMCONFIG_FILE: "vmconfig-${CI_PIPELINE_ID}-${ARCH}.json"
+    TEST_SETS: "no_tracer,only_tracer"
   before_script:
     - set +x
     - export DD_API_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.datadog_api_key_org2 --with-decryption --query "Parameter.Value" --out text)

From 6f24f96c510fc5d9d8463d65d11daa29e80cb6f4 Mon Sep 17 00:00:00 2001
From: usamasaqib <usama.saqib@datadoghq.com>
Date: Tue, 28 Nov 2023 15:47:19 +0100
Subject: [PATCH 20/46] full path to vmconfig file

---
 .gitlab/kernel_version_testing/system_probe.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab/kernel_version_testing/system_probe.yml b/.gitlab/kernel_version_testing/system_probe.yml
index f569bbf724095d..a092650488671b 100644
--- a/.gitlab/kernel_version_testing/system_probe.yml
+++ b/.gitlab/kernel_version_testing/system_probe.yml
@@ -292,7 +292,7 @@ upload_minimized_btfs_arm64:
     RESOURCE_TAGS: "instance-type:${INSTANCE_TYPE},arch:${ARCH}"
     KUBERNETES_MEMORY_REQUEST: "12Gi"
     KUBERNETES_MEMORY_LIMIT: "16Gi"
-    VMCONFIG_FILE: "vmconfig-${CI_PIPELINE_ID}-${ARCH}.json"
+    VMCONFIG_FILE: "${CI_PROJECT_DIR}/vmconfig-${CI_PIPELINE_ID}-${ARCH}.json"
     TEST_SETS: "no_tracer,only_tracer"
   before_script:
     - set +x

From 56a78a315886f93b97391f7fc6c833f9dc69211a Mon Sep 17 00:00:00 2001
From: usamasaqib <usama.saqib@datadoghq.com>
Date: Tue, 28 Nov 2023 16:13:14 +0100
Subject: [PATCH 21/46] pass output file parameter

---
 .gitlab/kernel_version_testing/system_probe.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab/kernel_version_testing/system_probe.yml b/.gitlab/kernel_version_testing/system_probe.yml
index a092650488671b..28adf84e74c982 100644
--- a/.gitlab/kernel_version_testing/system_probe.yml
+++ b/.gitlab/kernel_version_testing/system_probe.yml
@@ -304,7 +304,7 @@ upload_minimized_btfs_arm64:
   script:
     - echo "s3://dd-pulumi-state?region=us-east-1&awssdk=v2&profile=$AWS_PROFILE" > $STACK_DIR
     - pulumi login $(cat $STACK_DIR | tr -d '\n')
-    - inv -e kmt.gen-config --ci --arch=$ARCH
+    - inv -e kmt.gen-config --ci --arch=$ARCH --output_file=$VMCONFIG_FILE
     - inv -e system-probe.start-microvms --provision --vmconfig=$VMCONFIG_FILE $INSTANCE_TYPE_ARG $AMI_ID_ARG --ssh-key-name=$AWS_EC2_SSH_KEY_NAME --ssh-key-path=$AWS_EC2_SSH_KEY_FILE --infra-env=$INFRA_ENV --stack-name=kernel-matrix-testing-$ARCH-$CI_PIPELINE_ID --run-agent
     - cat $CI_PROJECT_DIR/stack.output
     - pulumi logout

From c03ac0997933e221f6ae0664857c5fefbb43259a Mon Sep 17 00:00:00 2001
From: usamasaqib <usama.saqib@datadoghq.com>
Date: Tue, 28 Nov 2023 16:13:36 +0100
Subject: [PATCH 22/46] fix parameter name

---
 .gitlab/kernel_version_testing/system_probe.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab/kernel_version_testing/system_probe.yml b/.gitlab/kernel_version_testing/system_probe.yml
index 28adf84e74c982..f8c1e6a559316e 100644
--- a/.gitlab/kernel_version_testing/system_probe.yml
+++ b/.gitlab/kernel_version_testing/system_probe.yml
@@ -304,7 +304,7 @@ upload_minimized_btfs_arm64:
   script:
     - echo "s3://dd-pulumi-state?region=us-east-1&awssdk=v2&profile=$AWS_PROFILE" > $STACK_DIR
     - pulumi login $(cat $STACK_DIR | tr -d '\n')
-    - inv -e kmt.gen-config --ci --arch=$ARCH --output_file=$VMCONFIG_FILE
+    - inv -e kmt.gen-config --ci --arch=$ARCH --output-file=$VMCONFIG_FILE
     - inv -e system-probe.start-microvms --provision --vmconfig=$VMCONFIG_FILE $INSTANCE_TYPE_ARG $AMI_ID_ARG --ssh-key-name=$AWS_EC2_SSH_KEY_NAME --ssh-key-path=$AWS_EC2_SSH_KEY_FILE --infra-env=$INFRA_ENV --stack-name=kernel-matrix-testing-$ARCH-$CI_PIPELINE_ID --run-agent
     - cat $CI_PROJECT_DIR/stack.output
     - pulumi logout

From fe2ed9a0552905a448953f748b00b10e72d000fb Mon Sep 17 00:00:00 2001
From: usamasaqib <usama.saqib@datadoghq.com>
Date: Tue, 28 Nov 2023 16:40:57 +0100
Subject: [PATCH 23/46] make disk into list

---
 tasks/kernel_matrix_testing/vmconfig.py | 31 +++++++++++++++++--------
 1 file changed, 21 insertions(+), 10 deletions(-)

diff --git a/tasks/kernel_matrix_testing/vmconfig.py b/tasks/kernel_matrix_testing/vmconfig.py
index 16dcabb4f19a75..ad1435ec2b358e 100644
--- a/tasks/kernel_matrix_testing/vmconfig.py
+++ b/tasks/kernel_matrix_testing/vmconfig.py
@@ -337,14 +337,19 @@ def add_vcpu(vmset, vcpu):
 def add_memory(vmset, memory):
     vmset["memory"] = memory
 
-def add_docker_disk(vmset):
-    arch = vmset['arch']
-    vmset["disks"] = {
-        "mount_point": "/mnt/docker",
-        "source": f"https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/docker-{arch}.qcow2",
-        "target": f"/home/kernel-version-testing/docker-{arch}.qcow2",
-        "type": "default",
-    }
+def template_name(arch, recipe):
+    if arch == local_arch:
+        arch = arch_mapping[platform.machine()]
+
+    recipe_without_arch = recipe.split("-")[0]
+    return f"{recipe_without_arch}_{arch}"
+
+def add_disks(vmconfig_template, vmset):
+    tname = template_name(vmset["arch"], vmset["recipe"])
+
+    for template in vmconfig_template["vmsets"]:
+        if template["name"] == tname:
+            vmset["disks"] = copy.deepcopy(template["disks"])
 
 def add_console(vmset):
     vmset["console_type"] = "file"
@@ -366,6 +371,10 @@ def image_source_to_path(vmset):
     for kernel in vmset["kernels"]:
         kernel["image_source"] = url_to_fspath(kernel["image_source"])
 
+    if "disks" in vmset:
+        for disk in vmset["disks"]:
+            disk["source"] = url_to_fspath(disk["source"])
+
 class VM:
     def __init__(self, version):
         self.version = version
@@ -436,10 +445,12 @@ def generate_vmconfig(vm_config, normalized_vm_defs, vcpu, memory, sets, ci):
         add_vcpu(vmset, vcpu)
         add_memory(vmset, memory)
 
+        if vmset["recipe"] != "custom":
+            add_disks(vmconfig_template, vmset)
+
+        # For local VMs we want to read images from the filesystem
         if vmset["arch"] == local_arch:
             image_source_to_path(vmset)
-        else:
-            add_docker_disk(vmset)
 
         if ci:
             add_console(vmset)

From b0167aae0f973e140ecb46b3bccb9176a507bb9c Mon Sep 17 00:00:00 2001
From: usamasaqib <usama.saqib@datadoghq.com>
Date: Tue, 28 Nov 2023 16:48:55 +0100
Subject: [PATCH 24/46] lint

---
 tasks/kernel_matrix_testing/vmconfig.py | 48 +++++++++++++++++++------
 tasks/kmt.py                            | 14 +++++++-
 2 files changed, 51 insertions(+), 11 deletions(-)

diff --git a/tasks/kernel_matrix_testing/vmconfig.py b/tasks/kernel_matrix_testing/vmconfig.py
index ad1435ec2b358e..c5c539f0387e12 100644
--- a/tasks/kernel_matrix_testing/vmconfig.py
+++ b/tasks/kernel_matrix_testing/vmconfig.py
@@ -90,7 +90,14 @@
     "centos_79": "centos_79",
 }
 
-arch_mapping = {"amd64": "x86_64", "x86": "x86_64", "x86_64": "x86_64", "arm64": "arm64", "arm": "arm64", "aarch64": "arm64"}
+arch_mapping = {
+    "amd64": "x86_64",
+    "x86": "x86_64",
+    "x86_64": "x86_64",
+    "arm64": "arm64",
+    "arm": "arm64",
+    "aarch64": "arm64"
+}
 
 TICK = "\u2713"
 CROSS = "\u2718"
@@ -113,10 +120,12 @@
 
 consoles = {"x86_64": "ttyS0", "arm64": "ttyAMA0"}
 
+
 def lte_414(version):
     major, minor = version.split('.')
     return (int(major) <= 4) and (int(minor) <= 14)
 
+
 def get_image_list(distro, custom):
     custom_kernels = list()
     for k in kernels:
@@ -194,6 +203,7 @@ def normalize_vm_def(possible, vm):
 
     return recipe, version, arch
 
+
 def get_custom_kernel_config(template, recipe, version, arch):
     if arch == local_arch:
         arch = arch_mapping[platform.machine()]
@@ -204,10 +214,7 @@ def get_custom_kernel_config(template, recipe, version, arch):
         console = "ttyAMA0"
 
     if lte_414(version):
-        extra_params = {
-            "console": console,
-            "systemd.unified_cgroup_hierarchy": "0"
-        }
+        extra_params = {"console": console, "systemd.unified_cgroup_hierarchy": "0"}
     else:
         extra_params = {
             "console": console,
@@ -219,6 +226,7 @@ def get_custom_kernel_config(template, recipe, version, arch):
         "extra_params": extra_params,
     }
 
+
 # This function derives the configuration for each
 # unique kernel or distribution from the normalized vm-def.
 # For more details on the generated configuration element, refer
@@ -261,9 +269,11 @@ def kernel_in_vmset(vmset, kernel):
 
     return False
 
+
 def get_vmconfig_file():
     return "test/new-e2e/system-probe/config/vmconfig.json"
 
+
 def vmset_name(arch, recipe, setprefix):
     name = f"{recipe}_{arch}"
     if setprefix != "":
@@ -271,6 +281,7 @@ def vmset_name(arch, recipe, setprefix):
 
     return name
 
+
 def add_custom_vmset(vmset, vm_config):
     arch = vmset.arch
     if arch == local_arch:
@@ -296,8 +307,8 @@ def add_custom_vmset(vmset, vm_config):
         "kernels": list(),
         "image": {
             "image_path": image_path,
-            "image_source": f"https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/{image_path}"
-        }
+            "image_source": f"https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/{image_path}",
+        },
     }
 
     vm_config["vmsets"].append(new_set)
@@ -331,12 +342,15 @@ def add_kernel(vm_config, kernel, setname):
 
     raise Exit(f"Unable to find vmset with name {setname}")
 
+
 def add_vcpu(vmset, vcpu):
     vmset["vcpu"] = vcpu
 
+
 def add_memory(vmset, memory):
     vmset["memory"] = memory
 
+
 def template_name(arch, recipe):
     if arch == local_arch:
         arch = arch_mapping[platform.machine()]
@@ -344,6 +358,7 @@ def template_name(arch, recipe):
     recipe_without_arch = recipe.split("-")[0]
     return f"{recipe_without_arch}_{arch}"
 
+
 def add_disks(vmconfig_template, vmset):
     tname = template_name(vmset["arch"], vmset["recipe"])
 
@@ -351,9 +366,11 @@ def add_disks(vmconfig_template, vmset):
         if template["name"] == tname:
             vmset["disks"] = copy.deepcopy(template["disks"])
 
+
 def add_console(vmset):
     vmset["console_type"] = "file"
 
+
 def url_to_fspath(url):
     source = urlparse(url)
     if os.path.basename(source.path).endswith(".xz"):
@@ -363,6 +380,7 @@ def url_to_fspath(url):
 
     return f"file://{os.path.join(KMT_ROOTFS_DIR,filename)}"
 
+
 def image_source_to_path(vmset):
     if vmset["recipe"] == f"custom-{vmset['arch']}":
         vmset["image"]["image_source"] = url_to_fspath(vmset["image"]["image_source"])
@@ -375,10 +393,12 @@ def image_source_to_path(vmset):
         for disk in vmset["disks"]:
             disk["source"] = url_to_fspath(disk["source"])
 
+
 class VM:
     def __init__(self, version):
         self.version = version
 
+
 class VMSet:
     def __init__(self, arch, recipe, name):
         self.arch = arch
@@ -407,9 +427,11 @@ def add_vm_if_belongs(self, recipe, version, arch):
         if self.recipe == recipe and self.arch == arch:
             self.vms.append(VM(version))
 
+
 def custom_version_prefix(version):
     return "lte_414" if lte_414(version) else "gt_414"
 
+
 def generate_vmconfig(vm_config, normalized_vm_defs, vcpu, memory, sets, ci):
     with open(get_vmconfig_file()) as f:
         vmconfig_template = json.load(f)
@@ -439,7 +461,9 @@ def generate_vmconfig(vm_config, normalized_vm_defs, vcpu, memory, sets, ci):
     # add vm configurations to vmsets.
     for vmset in vmsets:
         for vm in vmset.vms:
-            add_kernel(vm_config, get_kernel_config(vmconfig_template, vmset.recipe, vm.version, vmset.arch), vmset.name)
+            add_kernel(
+                vm_config, get_kernel_config(vmconfig_template, vmset.recipe, vm.version, vmset.arch), vmset.name
+            )
 
     for vmset in vm_config["vmsets"]:
         add_vcpu(vmset, vcpu)
@@ -457,6 +481,7 @@ def generate_vmconfig(vm_config, normalized_vm_defs, vcpu, memory, sets, ci):
 
     return vm_config
 
+
 def ls_to_int(ls):
     int_ls = list()
     for elem in ls:
@@ -518,6 +543,7 @@ def gen_config_for_stack(ctx, stack, vms, sets, init_stack, vcpu, memory, new, c
 
     info(f"[+] vmconfig @ {vmconfig_file}")
 
+
 def list_all_distro_normalized_vms(archs):
     with open(get_vmconfig_file()) as f:
         vmconfig = json.load(f)
@@ -549,7 +575,9 @@ def gen_config(ctx, stack, vms, sets, init_stack, vcpu, memory, new, ci, arch, o
         set_ls = sets.split(",")
 
     if not ci:
-        return gen_config_for_stack(ctx, stack, vms, set_ls, init_stack, ls_to_int(vcpu_ls), ls_to_int(memory_ls), new, ci)
+        return gen_config_for_stack(
+            ctx, stack, vms, set_ls, init_stack, ls_to_int(vcpu_ls), ls_to_int(memory_ls), new, ci
+        )
 
     arch_ls = ["x86_64", "arm64"]
     if arch != "":
@@ -557,7 +585,7 @@ def gen_config(ctx, stack, vms, sets, init_stack, vcpu, memory, new, ci, arch, o
 
     vms_to_generate = list_all_distro_normalized_vms(arch_ls)
 
-    vm_config = generate_vmconfig({"vmsets":[]}, vms_to_generate, ls_to_int(vcpu_ls), ls_to_int(memory_ls), set_ls, ci)
+    vm_config = generate_vmconfig({"vmsets": []}, vms_to_generate, ls_to_int(vcpu_ls), ls_to_int(memory_ls), set_ls, ci)
 
     with open(output_file, "w") as f:
         f.write(json.dumps(vm_config, indent=4))
diff --git a/tasks/kmt.py b/tasks/kmt.py
index 1681c6b18eaf0c..d2189ae803bfa3 100644
--- a/tasks/kmt.py
+++ b/tasks/kmt.py
@@ -45,7 +45,19 @@ def create_stack(ctx, stack=None):
         "init-stack": "Automatically initialize stack if not present. Equivalent to calling 'inv -e kmt.create-stack [--stack=<stack>]'",
     }
 )
-def gen_config(ctx, stack=None, vms="", sets="", init_stack=False, vcpu="4", memory="8192", new=False, ci=False, arch="", output_file="vmconfig.json"):
+def gen_config(
+    ctx,
+    stack=None,
+    vms="",
+    sets="",
+    init_stack=False,
+    vcpu="4",
+    memory="8192",
+    new=False,
+    ci=False,
+    arch="",
+    output_file="vmconfig.json"
+):
     vmconfig.gen_config(ctx, stack, vms, sets, init_stack, vcpu, memory, new, ci, arch, output_file)
 
 

From 3e4ab8036d810ab4927c8c2c2c98701bd985c5eb Mon Sep 17 00:00:00 2001
From: usamasaqib <usama.saqib@datadoghq.com>
Date: Tue, 28 Nov 2023 16:49:59 +0100
Subject: [PATCH 25/46] pass test sets

---
 .gitlab/kernel_version_testing/system_probe.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab/kernel_version_testing/system_probe.yml b/.gitlab/kernel_version_testing/system_probe.yml
index f8c1e6a559316e..096e4a8b4c5367 100644
--- a/.gitlab/kernel_version_testing/system_probe.yml
+++ b/.gitlab/kernel_version_testing/system_probe.yml
@@ -304,7 +304,7 @@ upload_minimized_btfs_arm64:
   script:
     - echo "s3://dd-pulumi-state?region=us-east-1&awssdk=v2&profile=$AWS_PROFILE" > $STACK_DIR
     - pulumi login $(cat $STACK_DIR | tr -d '\n')
-    - inv -e kmt.gen-config --ci --arch=$ARCH --output-file=$VMCONFIG_FILE
+    - inv -e kmt.gen-config --ci --arch=$ARCH --output-file=$VMCONFIG_FILE --sets=$TEST_SETS
     - inv -e system-probe.start-microvms --provision --vmconfig=$VMCONFIG_FILE $INSTANCE_TYPE_ARG $AMI_ID_ARG --ssh-key-name=$AWS_EC2_SSH_KEY_NAME --ssh-key-path=$AWS_EC2_SSH_KEY_FILE --infra-env=$INFRA_ENV --stack-name=kernel-matrix-testing-$ARCH-$CI_PIPELINE_ID --run-agent
     - cat $CI_PROJECT_DIR/stack.output
     - pulumi logout

From c470acacd6a5a1c72442fb6d6a0df33172efa9ec Mon Sep 17 00:00:00 2001
From: usamasaqib <usama.saqib@datadoghq.com>
Date: Tue, 28 Nov 2023 16:53:21 +0100
Subject: [PATCH 26/46] make vmconfig file global

---
 tasks/kernel_matrix_testing/vmconfig.py | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/tasks/kernel_matrix_testing/vmconfig.py b/tasks/kernel_matrix_testing/vmconfig.py
index c5c539f0387e12..4accdcde72efec 100644
--- a/tasks/kernel_matrix_testing/vmconfig.py
+++ b/tasks/kernel_matrix_testing/vmconfig.py
@@ -14,6 +14,7 @@
 vm_architecture = "arch"
 vm_version = "version"
 local_arch = "local"
+vmconfig_file = "test/new-e2e/system-probe/config/vmconfig.json"
 
 try:
     from thefuzz import fuzz, process
@@ -270,10 +271,6 @@ def kernel_in_vmset(vmset, kernel):
     return False
 
 
-def get_vmconfig_file():
-    return "test/new-e2e/system-probe/config/vmconfig.json"
-
-
 def vmset_name(arch, recipe, setprefix):
     name = f"{recipe}_{arch}"
     if setprefix != "":
@@ -433,7 +430,7 @@ def custom_version_prefix(version):
 
 
 def generate_vmconfig(vm_config, normalized_vm_defs, vcpu, memory, sets, ci):
-    with open(get_vmconfig_file()) as f:
+    with open(vmconfig_file) as f:
         vmconfig_template = json.load(f)
 
     # generate all vmsets
@@ -545,7 +542,7 @@ def gen_config_for_stack(ctx, stack, vms, sets, init_stack, vcpu, memory, new, c
 
 
 def list_all_distro_normalized_vms(archs):
-    with open(get_vmconfig_file()) as f:
+    with open(vmconfig_file) as f:
         vmconfig = json.load(f)
 
     vms = list()

From 01945bf1162b6688c93e01db0243534dc043777c Mon Sep 17 00:00:00 2001
From: usamasaqib <usama.saqib@datadoghq.com>
Date: Tue, 28 Nov 2023 17:32:56 +0100
Subject: [PATCH 27/46] remove arch specific file

---
 .../system-probe/config/vmconfig-x64.json     | 101 ------------------
 .../new-e2e/system-probe/config/vmconfig.json |   9 +-
 2 files changed, 7 insertions(+), 103 deletions(-)
 delete mode 100644 test/new-e2e/system-probe/config/vmconfig-x64.json

diff --git a/test/new-e2e/system-probe/config/vmconfig-x64.json b/test/new-e2e/system-probe/config/vmconfig-x64.json
deleted file mode 100644
index ed380f7d0cc9a0..00000000000000
--- a/test/new-e2e/system-probe/config/vmconfig-x64.json
+++ /dev/null
@@ -1,101 +0,0 @@
-{r
-    "vmsets": [
-        {
-            "name": "distro_x86_64",
-            "recipe": "distro-x86_64",
-            "arch": "x86_64",
-            "console_type": "file",
-            "kernels": [
-                {
-                    "dir": "xenial-server-cloudimg-amd64.qcow2",
-                    "tag": "ubuntu_16.04",
-                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/xenial-server-cloudimg-amd64.qcow2.xz"
-                },
-                {
-                    "dir": "bionic-server-cloudimg-amd64.qcow2",
-                    "tag": "ubuntu_18.04",
-                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/bionic-server-cloudimg-amd64.qcow2.xz"
-                },
-                {
-                    "dir": "focal-server-cloudimg-amd64.qcow2",
-                    "tag": "ubuntu_20.04",
-                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/focal-server-cloudimg-amd64.qcow2.xz"
-                },
-                {
-                    "dir": "jammy-server-cloudimg-amd64.qcow2",
-                    "tag": "ubuntu_22.04",
-                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/jammy-server-cloudimg-amd64.qcow2.xz"
-                },
-                {
-                    "dir": "mantic-server-cloudimg-amd64.qcow2",
-                    "tag": "ubuntu_23.10",
-                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/mantic-server-cloudimg-amd64.qcow2.xz"
-                },
-                {
-                    "dir": "amzn2-kvm-2.0-amd64-4.14.qcow2",
-                    "tag": "amzn_4.14",
-                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/amzn2-kvm-2.0-amd64-4.14.qcow2.xz"
-                },
-                {
-                    "dir": "amzn2-kvm-2.0-amd64-5.4.qcow2",
-                    "tag": "amzn_5.4",
-                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/amzn2-kvm-2.0-amd64-5.4.qcow2.xz"
-                },
-                {
-                    "dir": "amzn2-kvm-2.0-amd64-5.10.qcow2",
-                    "tag": "amzn_5.10",
-                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/amzn2-kvm-2.0-amd64-5.10.qcow2.xz"
-                },
-                {
-                    "dir": "Fedora-Cloud-Base-37.amd64.qcow2",
-                    "tag": "fedora_37",
-                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/Fedora-Cloud-Base-37.amd64.qcow2.xz"
-                },
-                {
-                    "dir": "Fedora-Cloud-Base-38.amd64.qcow2",
-                    "tag": "fedora_38",
-                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/Fedora-Cloud-Base-38.amd64.qcow2.xz"
-                },
-                {
-                    "dir": "debian-10-generic-amd64.qcow2",
-                    "tag": "debian_10",
-                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/debian-10-generic-amd64.qcow2.xz"
-                },
-                {
-                    "dir": "debian-11-generic-amd64.qcow2",
-                    "tag": "debian_11",
-                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/debian-11-generic-amd64.qcow2.xz"
-                },
-                {
-                    "dir": "debian-12-generic-amd64.qcow2",
-                    "tag": "debian_12",
-                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/debian-12-generic-amd64.qcow2.xz"
-                },
-                {
-                    "dir": "centos-7.9-x86_64.qcow2",
-                    "tag": "centos_79",
-                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/centos-7.9-x86_64.qcow2.xz"
-                },
-                {
-                    "dir": "centos-8-x86_64.qcow2",
-                    "tag": "centos_8",
-                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/centos-8-x86_64.qcow2.xz"
-                }
-            ],
-            "vcpu": [
-                4
-            ],
-            "memory": [
-                8192
-            ],
-            "disks": [
-                {
-                    "mount_point": "/mnt/docker",
-                    "source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/docker-amd64.qcow2",
-                    "target": "/home/kernel-version-testing/docker-amd64.qcow2",
-                    "type": "default"
-                }
-            ]
-        }
-    ]
-}
diff --git a/test/new-e2e/system-probe/config/vmconfig.json b/test/new-e2e/system-probe/config/vmconfig.json
index 545f33668ab34f..b7e1ae9037b36e 100644
--- a/test/new-e2e/system-probe/config/vmconfig.json
+++ b/test/new-e2e/system-probe/config/vmconfig.json
@@ -72,9 +72,14 @@
                     "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/debian-12-generic-x86_64.qcow2.xz"
                 },
                 {
-                    "dir": "CentOS-7-x86_64-GenericCloud-2211.qcow2",
+                    "dir": "centos-7.9-x86_64.qcow2",
                     "tag": "centos_79",
-                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/CentOS-7-x86_64-GenericCloud-2211.qcow2.xz"
+                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/centos-7.9-x86_64.qcow2.xz"
+                },
+                {
+                    "dir": "centos-8-x86_64.qcow2",
+                    "tag": "centos_8",
+                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/centos-8-x86_64.qcow2.xz"
                 }
             ],
             "vcpu": [

From b3b275319ded929c64a3ce3f5d15b8525acab565 Mon Sep 17 00:00:00 2001
From: usamasaqib <usama.saqib@datadoghq.com>
Date: Wed, 29 Nov 2023 11:50:39 +0100
Subject: [PATCH 28/46] use tags in vmconfig file instead of names

---
 tasks/kernel_matrix_testing/vmconfig.py       | 58 ++++++++++---------
 test/new-e2e/go.mod                           |  4 +-
 test/new-e2e/go.sum                           |  6 +-
 .../new-e2e/system-probe/config/vmconfig.json |  8 ++-
 4 files changed, 44 insertions(+), 32 deletions(-)

diff --git a/tasks/kernel_matrix_testing/vmconfig.py b/tasks/kernel_matrix_testing/vmconfig.py
index 4accdcde72efec..b3d286648ac6cd 100644
--- a/tasks/kernel_matrix_testing/vmconfig.py
+++ b/tasks/kernel_matrix_testing/vmconfig.py
@@ -242,7 +242,7 @@ def get_kernel_config(template, recipe, version, arch):
     setname = f"{recipe}_{arch}"
 
     for vmset in template["vmsets"]:
-        if vmset["name"] != setname:
+        if setname not in vmset["tags"]:
             continue
 
         for kernel in vmset["kernels"]:
@@ -252,11 +252,11 @@ def get_kernel_config(template, recipe, version, arch):
     raise Exit(f"No kernel {version} in set {setname}")
 
 
-def vmset_exists(vm_config, setname):
+def vmset_exists(vm_config, tags):
     vmsets = vm_config["vmsets"]
 
     for vmset in vmsets:
-        if vmset["name"] == setname:
+        if set(vmset["tags"]) == tags:
             return True
 
     return False
@@ -271,12 +271,8 @@ def kernel_in_vmset(vmset, kernel):
     return False
 
 
-def vmset_name(arch, recipe, setprefix):
-    name = f"{recipe}_{arch}"
-    if setprefix != "":
-        return f"{setprefix}_{name}"
-
-    return name
+def vmset_name(arch, recipe):
+    return f"{recipe}_{arch}"
 
 
 def add_custom_vmset(vmset, vm_config):
@@ -294,11 +290,11 @@ def add_custom_vmset(vmset, vm_config):
     if lte:
         image_path = f"custom-buster.{arch}.qcow2"
 
-    if vmset_exists(vm_config, vmset.name):
+    if vmset_exists(vm_config, vmset.tags):
         return
 
     new_set = {
-        "name": vmset.name,
+        "tags": list(vmset.tags),
         "recipe": f"{vmset.recipe}-{vmset.arch}",
         "arch": vmset.arch,
         "kernels": list(),
@@ -312,14 +308,14 @@ def add_custom_vmset(vmset, vm_config):
 
 
 def add_vmset(vmset, vm_config):
-    if vmset_exists(vm_config, vmset.name):
+    if vmset_exists(vm_config, vmset.tags):
         return
 
     if vmset.recipe == "custom":
         return add_custom_vmset(vmset, vm_config)
 
     new_set = {
-        "name": vmset.name,
+        "tags": list(vmset.tags),
         "recipe": f"{vmset.recipe}-{vmset.arch}",
         "arch": vmset.arch,
         "kernels": list(),
@@ -328,16 +324,16 @@ def add_vmset(vmset, vm_config):
     vm_config["vmsets"].append(new_set)
 
 
-def add_kernel(vm_config, kernel, setname):
+def add_kernel(vm_config, kernel, tags):
     for vmset in vm_config["vmsets"]:
-        if vmset["name"] != setname:
+        if set(vmset["tags"]) != tags:
             continue
 
         if not kernel_in_vmset(vmset, kernel):
             vmset["kernels"].append(kernel)
             return
 
-    raise Exit(f"Unable to find vmset with name {setname}")
+    raise Exit(f"Unable to find vmset with tags {tags}")
 
 
 def add_vcpu(vmset, vcpu):
@@ -360,7 +356,7 @@ def add_disks(vmconfig_template, vmset):
     tname = template_name(vmset["arch"], vmset["recipe"])
 
     for template in vmconfig_template["vmsets"]:
-        if template["name"] == tname:
+        if tname in template["tags"]:
             vmset["disks"] = copy.deepcopy(template["disks"])
 
 
@@ -397,28 +393,36 @@ def __init__(self, version):
 
 
 class VMSet:
-    def __init__(self, arch, recipe, name):
+    def __init__(self, arch, recipe, tags):
         self.arch = arch
         self.recipe = recipe
-        self.name = name
+        self.tags = tags
         self.vms = list()
 
     def __eq__(self, other):
-        return self.name == other.name
+        for tag in self.tags:
+            if tag not in other.tags:
+                return False
+        return True
 
     def __hash__(self):
-        return hash(self.name)
+        return hash('-'.join(self.tags))
 
     def __repr__(self):
         vm_str = list()
         for vm in self.vms:
             vm_str.append(vm.version)
-        return f"<VMSet> name={self.name} arch={self.arch} vms={','.join(vm_str)}"
+        return f"<VMSet> tags={'-'.join(self.tags)} arch={self.arch} vms={','.join(vm_str)}"
 
     def add_vm_if_belongs(self, recipe, version, arch):
         if recipe == "custom":
-            expected_prefix = custom_version_prefix(version)
-            if not self.name.startswith(expected_prefix):
+            expected_tag = custom_version_prefix(version)
+            found = False
+            for tag in self.tags:
+                if tag == expected_tag:
+                    found = True
+
+            if not found:
                 return
 
         if self.recipe == recipe and self.arch == arch:
@@ -441,10 +445,10 @@ def generate_vmconfig(vm_config, normalized_vm_defs, vcpu, memory, sets, ci):
 
         # duplicate vm if multiple sets provided by user
         for s in sets:
-            vmsets.add(VMSet(arch, recipe, vmset_name(arch, recipe, s)))
+            vmsets.add(VMSet(arch, recipe, set([vmset_name(arch, recipe), s])))
 
         if len(sets) == 0:
-            vmsets.add(VMSet(arch, recipe, vmset_name(arch, recipe, "")))
+            vmsets.add(VMSet(arch, recipe, set([vmset_name(arch, recipe)])))
 
     # map vms to vmsets
     for recipe, version, arch in normalized_vm_defs:
@@ -459,7 +463,7 @@ def generate_vmconfig(vm_config, normalized_vm_defs, vcpu, memory, sets, ci):
     for vmset in vmsets:
         for vm in vmset.vms:
             add_kernel(
-                vm_config, get_kernel_config(vmconfig_template, vmset.recipe, vm.version, vmset.arch), vmset.name
+                vm_config, get_kernel_config(vmconfig_template, vmset.recipe, vm.version, vmset.arch), vmset.tags,
             )
 
     for vmset in vm_config["vmsets"]:
diff --git a/test/new-e2e/go.mod b/test/new-e2e/go.mod
index dd5d5684e4b260..7b1270ab560903 100644
--- a/test/new-e2e/go.mod
+++ b/test/new-e2e/go.mod
@@ -22,7 +22,7 @@ require (
 	// `TEST_INFRA_DEFINITIONS_BUILDIMAGES` matches the commit sha in the module version
 	// Example: 	github.com/DataDog/test-infra-definitions v0.0.0-YYYYMMDDHHmmSS-0123456789AB
 	// => TEST_INFRA_DEFINITIONS_BUILDIMAGES: 0123456789AB
-	github.com/DataDog/test-infra-definitions v0.0.0-20231123110544-d505ce3abecf
+	github.com/DataDog/test-infra-definitions v0.0.0-20231129104320-d5c2750c39f5
 	github.com/aws/aws-sdk-go-v2 v1.22.1
 	github.com/aws/aws-sdk-go-v2/config v1.18.40
 	github.com/aws/aws-sdk-go-v2/service/ec2 v1.130.0
@@ -222,3 +222,5 @@ require (
 	sigs.k8s.io/yaml v1.3.0 // indirect
 	sourcegraph.com/sourcegraph/appdash v0.0.0-20211028080628-e2786a622600 // indirect
 )
+
+require github.com/pulumi/pulumi-tls/sdk/v4 v4.10.0 // indirect
diff --git a/test/new-e2e/go.sum b/test/new-e2e/go.sum
index b67d12f34fd666..22d5fedd80e6cd 100644
--- a/test/new-e2e/go.sum
+++ b/test/new-e2e/go.sum
@@ -12,8 +12,8 @@ github.com/DataDog/datadog-api-client-go/v2 v2.15.0 h1:5UVON1xs6Lul4d6R5TmLDqqSJ
 github.com/DataDog/datadog-api-client-go/v2 v2.15.0/go.mod h1:ZG8wS+y2rUmkRDJZQq7Og7EAPFPage+7vXcmuah2I9o=
 github.com/DataDog/mmh3 v0.0.0-20200805151601-30884ca2197a h1:m9REhmyaWD5YJ0P53ygRHxKKo+KM+nw+zz0hEdKztMo=
 github.com/DataDog/mmh3 v0.0.0-20200805151601-30884ca2197a/go.mod h1:SvsjzyJlSg0rKsqYgdcFxeEVflx3ZNAyFfkUHP0TxXg=
-github.com/DataDog/test-infra-definitions v0.0.0-20231123110544-d505ce3abecf h1:Kx0KlmdWcfLqHjh/SRCEx9auxfCrDtDNoWJWwtdXb5c=
-github.com/DataDog/test-infra-definitions v0.0.0-20231123110544-d505ce3abecf/go.mod h1:eyJ09i8Y8Ofu1gsfNJzL6VtIbLY0MR1atqxsculv4Lo=
+github.com/DataDog/test-infra-definitions v0.0.0-20231129104320-d5c2750c39f5 h1:4Yao9E7HwDFccroVqfSAhvlr4QhOMr0O7HE0uKwPq8E=
+github.com/DataDog/test-infra-definitions v0.0.0-20231129104320-d5c2750c39f5/go.mod h1:P6FDz6Iyki8kX1WoDln5U+3kVLC+QzRi7FIzauuZ6jM=
 github.com/DataDog/zstd v1.5.2 h1:vUG4lAyuPCXO0TLbXvPv7EB7cNK1QV/luu55UHLrrn8=
 github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
 github.com/DataDog/zstd_0 v0.0.0-20210310093942-586c1286621f h1:5Vuo4niPKFkfwW55jV4vY0ih3VQ9RaQqeqY67fvRn8A=
@@ -453,6 +453,8 @@ github.com/pulumi/pulumi-libvirt/sdk v0.4.0 h1:wq1Ox8FRKQ1kc2DPq3m5DGQgZEhE7kp4m
 github.com/pulumi/pulumi-libvirt/sdk v0.4.0/go.mod h1:tjjyDajp6Pb1pRCdaIugknIfzxw3Prev3o/k2nade+I=
 github.com/pulumi/pulumi-random/sdk/v4 v4.13.4 h1:g3jdktE5L5IDrOw4OiB+yhgxSw0okRPJnyV6PlIzTEQ=
 github.com/pulumi/pulumi-random/sdk/v4 v4.13.4/go.mod h1:cFlJw0eQnqN+62QpITEF9M08gVyzNCeXrKRsuJptFak=
+github.com/pulumi/pulumi-tls/sdk/v4 v4.10.0 h1:4MC0GyEomAjEZJPXEzBZpZ4+TOUg5WE77k38tMDIvS0=
+github.com/pulumi/pulumi-tls/sdk/v4 v4.10.0/go.mod h1:tNXsM/+RsiVVmBdzJMOOp6gMoi3sPko5u0FKdiei+cE=
 github.com/pulumi/pulumi/sdk/v3 v3.16.0/go.mod h1:252ou/zAU1g6E8iTwe2Y9ht7pb5BDl2fJlOuAgZCHiA=
 github.com/pulumi/pulumi/sdk/v3 v3.50.1/go.mod h1:tqQ4z9ocyM/UI2VQ7ZReWR3w6dF5ffEozoHipOMcDh4=
 github.com/pulumi/pulumi/sdk/v3 v3.84.0 h1:/vCRj6ATGVZw4pFmG7pZgjlKUcnbbnb9vmlqd+OpdXo=
diff --git a/test/new-e2e/system-probe/config/vmconfig.json b/test/new-e2e/system-probe/config/vmconfig.json
index b7e1ae9037b36e..39169717e0676a 100644
--- a/test/new-e2e/system-probe/config/vmconfig.json
+++ b/test/new-e2e/system-probe/config/vmconfig.json
@@ -1,7 +1,9 @@
 {
     "vmsets": [
         {
-            "name": "distro_x86_64",
+            "tags": [
+                "distro_x86_64"
+            ],
             "recipe": "distro-x86_64",
             "arch": "x86_64",
             "console_type": "file",
@@ -98,7 +100,9 @@
             ]
         },
         {
-            "name": "distro_arm64",
+            "tags": [
+                "distro_arm64"
+            ],
             "recipe": "distro-arm64",
             "arch": "arm64",
             "console_type": "file",

From e77a527be0c3ee04153b827a235339965a824b5c Mon Sep 17 00:00:00 2001
From: usamasaqib <usama.saqib@datadoghq.com>
Date: Wed, 29 Nov 2023 12:08:41 +0100
Subject: [PATCH 29/46] exact match on domain name

---
 .gitlab/kernel_version_testing/system_probe.yml | 4 ++--
 test/new-e2e/go.mod                             | 2 +-
 test/new-e2e/go.sum                             | 4 ++--
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.gitlab/kernel_version_testing/system_probe.yml b/.gitlab/kernel_version_testing/system_probe.yml
index 096e4a8b4c5367..a21e6b1562ca6e 100644
--- a/.gitlab/kernel_version_testing/system_probe.yml
+++ b/.gitlab/kernel_version_testing/system_probe.yml
@@ -370,8 +370,8 @@ kernel_matrix_testing_setup_env_x64:
         echo "These jobs do not permit retries. The go tests are retried a user-specified number of times automatically. In order to re-run the tests, you must trigger the pipeline again"
         'false'
       fi
-    - MICRO_VM_IP=$(cat $CI_PROJECT_DIR/stack.output | grep $ARCH-$TAG | grep $TEST_SET | cut -d ' ' -f 2)
-    - MICRO_VM_NAME=$(cat $CI_PROJECT_DIR/stack.output | grep $ARCH-$TAG | grep $TEST_SET | cut -d ' ' -f 1)
+    - MICRO_VM_IP=$(cat $CI_PROJECT_DIR/stack.output | grep "$ARCH-$TAG-.*$TEST_SET.*-ddvm" | cut -d ' ' -f 2)
+    - MICRO_VM_NAME=$(cat $CI_PROJECT_DIR/stack.output | grep "$ARCH-$TAG-.*$TEST_SET.*-ddvm" | cut -d ' ' -f 1)
     - GO_VERSION=$(inv go-version)
     - !reference [.setup_ssh_config]
     # ssh into each micro-vm and run initialization script. This script will also run the tests.
diff --git a/test/new-e2e/go.mod b/test/new-e2e/go.mod
index 7b1270ab560903..75a9f53ad4fb82 100644
--- a/test/new-e2e/go.mod
+++ b/test/new-e2e/go.mod
@@ -22,7 +22,7 @@ require (
 	// `TEST_INFRA_DEFINITIONS_BUILDIMAGES` matches the commit sha in the module version
 	// Example: 	github.com/DataDog/test-infra-definitions v0.0.0-YYYYMMDDHHmmSS-0123456789AB
 	// => TEST_INFRA_DEFINITIONS_BUILDIMAGES: 0123456789AB
-	github.com/DataDog/test-infra-definitions v0.0.0-20231129104320-d5c2750c39f5
+	github.com/DataDog/test-infra-definitions v0.0.0-20231129110704-121fcc5993ec
 	github.com/aws/aws-sdk-go-v2 v1.22.1
 	github.com/aws/aws-sdk-go-v2/config v1.18.40
 	github.com/aws/aws-sdk-go-v2/service/ec2 v1.130.0
diff --git a/test/new-e2e/go.sum b/test/new-e2e/go.sum
index 22d5fedd80e6cd..26e032477c7d8a 100644
--- a/test/new-e2e/go.sum
+++ b/test/new-e2e/go.sum
@@ -12,8 +12,8 @@ github.com/DataDog/datadog-api-client-go/v2 v2.15.0 h1:5UVON1xs6Lul4d6R5TmLDqqSJ
 github.com/DataDog/datadog-api-client-go/v2 v2.15.0/go.mod h1:ZG8wS+y2rUmkRDJZQq7Og7EAPFPage+7vXcmuah2I9o=
 github.com/DataDog/mmh3 v0.0.0-20200805151601-30884ca2197a h1:m9REhmyaWD5YJ0P53ygRHxKKo+KM+nw+zz0hEdKztMo=
 github.com/DataDog/mmh3 v0.0.0-20200805151601-30884ca2197a/go.mod h1:SvsjzyJlSg0rKsqYgdcFxeEVflx3ZNAyFfkUHP0TxXg=
-github.com/DataDog/test-infra-definitions v0.0.0-20231129104320-d5c2750c39f5 h1:4Yao9E7HwDFccroVqfSAhvlr4QhOMr0O7HE0uKwPq8E=
-github.com/DataDog/test-infra-definitions v0.0.0-20231129104320-d5c2750c39f5/go.mod h1:P6FDz6Iyki8kX1WoDln5U+3kVLC+QzRi7FIzauuZ6jM=
+github.com/DataDog/test-infra-definitions v0.0.0-20231129110704-121fcc5993ec h1:APP1Bv8ZSo99BG+yhmROpcDBu9We7zC3Io8DfTovQpw=
+github.com/DataDog/test-infra-definitions v0.0.0-20231129110704-121fcc5993ec/go.mod h1:P6FDz6Iyki8kX1WoDln5U+3kVLC+QzRi7FIzauuZ6jM=
 github.com/DataDog/zstd v1.5.2 h1:vUG4lAyuPCXO0TLbXvPv7EB7cNK1QV/luu55UHLrrn8=
 github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
 github.com/DataDog/zstd_0 v0.0.0-20210310093942-586c1286621f h1:5Vuo4niPKFkfwW55jV4vY0ih3VQ9RaQqeqY67fvRn8A=

From b7578fa247a2347a73f248e70a375ffe48b25a33 Mon Sep 17 00:00:00 2001
From: usamasaqib <usama.saqib@datadoghq.com>
Date: Wed, 29 Nov 2023 12:50:59 +0100
Subject: [PATCH 30/46] make include explicit

---
 test/new-e2e/system-probe/test-runner/files/no_tracer.json | 3 +++
 test/new-e2e/system-probe/test-runner/main.go              | 1 +
 2 files changed, 4 insertions(+)

diff --git a/test/new-e2e/system-probe/test-runner/files/no_tracer.json b/test/new-e2e/system-probe/test-runner/files/no_tracer.json
index 6798ed19245bc0..25e592ca78d4e0 100644
--- a/test/new-e2e/system-probe/test-runner/files/no_tracer.json
+++ b/test/new-e2e/system-probe/test-runner/files/no_tracer.json
@@ -1,5 +1,8 @@
 {
     "pkg/network/tracer": {
         "run": "TestUSMSuite"
+    },
+    "*": {
+        "exclude": false
     }
 }
diff --git a/test/new-e2e/system-probe/test-runner/main.go b/test/new-e2e/system-probe/test-runner/main.go
index 0601092d47c40d..5dd18f122b4b9e 100644
--- a/test/new-e2e/system-probe/test-runner/main.go
+++ b/test/new-e2e/system-probe/test-runner/main.go
@@ -182,6 +182,7 @@ func testPass(testConfig *testConfig, props map[string]string) error {
 		}
 
 		return true
+		return false
 	})
 	if err != nil {
 		return fmt.Errorf("test glob: %s", err)

From 0f10aac4967402c33cc777d5c2e135afbc0eb9ca Mon Sep 17 00:00:00 2001
From: usamasaqib <usama.saqib@datadoghq.com>
Date: Wed, 29 Nov 2023 12:51:16 +0100
Subject: [PATCH 31/46] run and skip are lists

---
 .../system-probe/test-runner/files/no_tracer.json   |  2 +-
 .../system-probe/test-runner/files/only_tracer.json |  2 +-
 test/new-e2e/system-probe/test-runner/main.go       | 13 ++++++-------
 3 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/test/new-e2e/system-probe/test-runner/files/no_tracer.json b/test/new-e2e/system-probe/test-runner/files/no_tracer.json
index 25e592ca78d4e0..fad878e4e0a2af 100644
--- a/test/new-e2e/system-probe/test-runner/files/no_tracer.json
+++ b/test/new-e2e/system-probe/test-runner/files/no_tracer.json
@@ -1,6 +1,6 @@
 {
     "pkg/network/tracer": {
-        "run": "TestUSMSuite"
+        "run-only": ["^TestUSMSuite$"]
     },
     "*": {
         "exclude": false
diff --git a/test/new-e2e/system-probe/test-runner/files/only_tracer.json b/test/new-e2e/system-probe/test-runner/files/only_tracer.json
index 2eb1de8bdf26e0..cba2cb80d54861 100644
--- a/test/new-e2e/system-probe/test-runner/files/only_tracer.json
+++ b/test/new-e2e/system-probe/test-runner/files/only_tracer.json
@@ -1,6 +1,6 @@
 {
     "pkg/network/tracer": {
-        "skip": "TestUSMSuite"
+        "skip": ["^TestUSMSuite$"]
     },
     "*": {
         "exclude": true
diff --git a/test/new-e2e/system-probe/test-runner/main.go b/test/new-e2e/system-probe/test-runner/main.go
index 5dd18f122b4b9e..3479f10f79daca 100644
--- a/test/new-e2e/system-probe/test-runner/main.go
+++ b/test/new-e2e/system-probe/test-runner/main.go
@@ -34,8 +34,8 @@ func init() {
 }
 
 type packageRunConfiguration struct {
-	Run     string
-	Skip    string
+	RunOnly []string `json:"run-only"`
+	Skip    []string
 	Exclude bool
 }
 
@@ -123,11 +123,11 @@ func buildCommandArgs(pkg string, xmlpath string, jsonpath string, file string,
 	}
 
 	packagesRunConfig := testConfig.packagesRunConfig
-	if config, ok := packagesRunConfig[pkg]; ok && config.Run != "" {
-		args = append(args, "-test.run", config.Run)
+	if config, ok := packagesRunConfig[pkg]; ok && config.RunOnly != nil {
+		args = append(args, "-test.run", strings.Join(config.RunOnly, "|"))
 	}
-	if config, ok := packagesRunConfig[pkg]; ok && config.Skip != "" {
-		args = append(args, "-test.skip", config.Skip)
+	if config, ok := packagesRunConfig[pkg]; ok && config.Skip != nil {
+		args = append(args, "-test.skip", strings.Join(config.Skip, "|"))
 	}
 
 	return args
@@ -181,7 +181,6 @@ func testPass(testConfig *testConfig, props map[string]string) error {
 			return !config.Exclude
 		}
 
-		return true
 		return false
 	})
 	if err != nil {

From 579e105a993a648c06af152a1e217168822237a0 Mon Sep 17 00:00:00 2001
From: usamasaqib <usama.saqib@datadoghq.com>
Date: Wed, 29 Nov 2023 12:54:24 +0100
Subject: [PATCH 32/46] use wildcard to only signify includes

---
 test/new-e2e/system-probe/test-runner/files/only_tracer.json | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/test/new-e2e/system-probe/test-runner/files/only_tracer.json b/test/new-e2e/system-probe/test-runner/files/only_tracer.json
index cba2cb80d54861..13e10a3ce5eb3d 100644
--- a/test/new-e2e/system-probe/test-runner/files/only_tracer.json
+++ b/test/new-e2e/system-probe/test-runner/files/only_tracer.json
@@ -1,8 +1,5 @@
 {
     "pkg/network/tracer": {
         "skip": ["^TestUSMSuite$"]
-    },
-    "*": {
-        "exclude": true
     }
 }

From 73027f84782b59fe5e35e3074e8b7f1821cae106 Mon Sep 17 00:00:00 2001
From: usamasaqib <usama.saqib@datadoghq.com>
Date: Wed, 29 Nov 2023 13:21:01 +0100
Subject: [PATCH 33/46] keep single test job per arch

---
 .../kernel_version_testing/system_probe.yml   | 32 +++----------------
 .../new-e2e/system-probe/config/vmconfig.json |  9 ++++--
 2 files changed, 12 insertions(+), 29 deletions(-)

diff --git a/.gitlab/kernel_version_testing/system_probe.yml b/.gitlab/kernel_version_testing/system_probe.yml
index a21e6b1562ca6e..74d855442dcd87 100644
--- a/.gitlab/kernel_version_testing/system_probe.yml
+++ b/.gitlab/kernel_version_testing/system_probe.yml
@@ -293,7 +293,7 @@ upload_minimized_btfs_arm64:
     KUBERNETES_MEMORY_REQUEST: "12Gi"
     KUBERNETES_MEMORY_LIMIT: "16Gi"
     VMCONFIG_FILE: "${CI_PROJECT_DIR}/vmconfig-${CI_PIPELINE_ID}-${ARCH}.json"
-    TEST_SETS: "no_tracer,only_tracer"
+    TEST_SETS: "no_tracersuite,only_tracersuite"
   before_script:
     - set +x
     - export DD_API_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.datadog_api_key_org2 --with-decryption --query "Parameter.Value" --out text)
@@ -393,7 +393,7 @@ kernel_matrix_testing_setup_env_x64:
       - $DD_AGENT_TESTING_DIR/junit-$ARCH-$TAG-$TEST_SET.tar.gz
       - $DD_AGENT_TESTING_DIR/testjson-$ARCH-$TAG-$TEST_SET.tar.gz
 
-.kmt_run_tests_x64:
+kernel_matrix_testing_run_tests_x64:
   extends:
     - .kernel_matrix_testing_run_tests
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/system-probe_x64$DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_SYSPROBE_BUILDIMAGES
@@ -405,20 +405,9 @@ kernel_matrix_testing_setup_env_x64:
   parallel:
     matrix:
       - TAG: ["ubuntu_16.04", "ubuntu_18.04", "ubuntu_20.04", "ubuntu_22.04", "ubuntu_23.10", "amzn_4.14", "amzn_5.4", "amzn_5.10", "fedora_37", "fedora_38", "debian_10", "debian_11", "debian_12", "centos_79", "centos_8"]
+        TEST_SET: ["no_tracersuite", "only_tracersuite"]
 
-kmt_run_tests_no_tracer_x64:
-  extends:
-    - .kmt_run_tests_x64
-  variables:
-    TEST_SET: "no_tracer"
-
-kmt_run_tests_only_tracer_x64:
-  extends:
-    - .kmt_run_tests_x64
-  variables:
-    TEST_SET: "only_tracer"
-
-.kmt_run_tests_arm64:
+kernel_matrix_testing_run_tests_arm64:
   extends:
     - .kernel_matrix_testing_run_tests
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/system-probe_arm64$DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_SYSPROBE_BUILDIMAGES
@@ -430,18 +419,7 @@ kmt_run_tests_only_tracer_x64:
   parallel:
     matrix:
       - TAG: ["ubuntu_18.04", "ubuntu_20.04", "ubuntu_22.04", "ubuntu_23.10", "amzn_4.14", "amzn_5.4", "amzn_5.10", "fedora_37", "fedora_38", "debian_10", "debian_11", "debian_12", "centos_79", "centos_8"]
-
-kmt_run_tests_no_tracer_arm64:
-  extends:
-    - .kmt_run_tests_arm64
-  variables:
-    TEST_SET: "no_tracer"
-
-kmt_run_tests_only_tracer_arm64:
-  extends:
-    - .kmt_run_tests_arm64
-  variables:
-    TEST_SET: "only_tracer"
+        TEST_SET: ["no_tracersuite", "only_tracersuite"]
 
 .kernel_matrix_testing_cleanup:
   stage: kernel_matrix_testing
diff --git a/test/new-e2e/system-probe/config/vmconfig.json b/test/new-e2e/system-probe/config/vmconfig.json
index 39169717e0676a..df4644b78f2834 100644
--- a/test/new-e2e/system-probe/config/vmconfig.json
+++ b/test/new-e2e/system-probe/config/vmconfig.json
@@ -168,9 +168,14 @@
                     "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/debian-12-generic-arm64.qcow2.xz"
                 },
                 {
-                    "dir": "CentOS-7-aarch64-GenericCloud-2211.qcow2",
+                    "dir": "centos-7.9-arm64.qcow2",
                     "tag": "centos_79",
-                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/CentOS-7-aarch64-GenericCloud-2211.qcow2.xz"
+                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/centos-7.9-arm64.qcow2.xz"
+                },
+                {
+                    "dir": "centos-8-arm64.qcow2",
+                    "tag": "centos_8",
+                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/centos-8-arm64.qcow2.xz"
                 }
             ],
             "machine": "virt",

From 14c580b14e4d0d8e2be94d89157713a2b87d69af Mon Sep 17 00:00:00 2001
From: usamasaqib <usama.saqib@datadoghq.com>
Date: Wed, 29 Nov 2023 13:23:09 +0100
Subject: [PATCH 34/46] use grep pattern everywhere

---
 .gitlab/kernel_version_testing/system_probe.yml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/.gitlab/kernel_version_testing/system_probe.yml b/.gitlab/kernel_version_testing/system_probe.yml
index 74d855442dcd87..6f8662f121c6a3 100644
--- a/.gitlab/kernel_version_testing/system_probe.yml
+++ b/.gitlab/kernel_version_testing/system_probe.yml
@@ -351,6 +351,7 @@ kernel_matrix_testing_setup_env_x64:
   variables:
     AWS_EC2_SSH_KEY_FILE: $CI_PROJECT_DIR/ssh_key
     RETRY: 2
+    MICROVM_GREP_PATTERN: "$ARCH-$TAG-.*$TEST_SET.*-ddvm"
   before_script:
     - set +x
     - export DD_API_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.datadog_api_key_org2 --with-decryption --query "Parameter.Value" --out text)
@@ -370,8 +371,8 @@ kernel_matrix_testing_setup_env_x64:
         echo "These jobs do not permit retries. The go tests are retried a user-specified number of times automatically. In order to re-run the tests, you must trigger the pipeline again"
         'false'
       fi
-    - MICRO_VM_IP=$(cat $CI_PROJECT_DIR/stack.output | grep "$ARCH-$TAG-.*$TEST_SET.*-ddvm" | cut -d ' ' -f 2)
-    - MICRO_VM_NAME=$(cat $CI_PROJECT_DIR/stack.output | grep "$ARCH-$TAG-.*$TEST_SET.*-ddvm" | cut -d ' ' -f 1)
+    - MICRO_VM_IP=$(cat $CI_PROJECT_DIR/stack.output | grep $MICROVM_GREP_PATTERN | cut -d ' ' -f 2)
+    - MICRO_VM_NAME=$(cat $CI_PROJECT_DIR/stack.output | grep $MICROVM_GREP_PATTERN | cut -d ' ' -f 1)
     - GO_VERSION=$(inv go-version)
     - !reference [.setup_ssh_config]
     # ssh into each micro-vm and run initialization script. This script will also run the tests.
@@ -381,7 +382,7 @@ kernel_matrix_testing_setup_env_x64:
     - $CI_PROJECT_DIR/connector-$ARCH -host $INSTANCE_IP -user ubuntu -ssh-file $AWS_EC2_SSH_KEY_FILE -vm-cmd "${NESTED_VM_CMD}"
     - ssh metal_instance "ssh ${MICRO_VM_IP} '/test-json-review'"
   after_script:
-    - MICRO_VM_IP=$(cat $CI_PROJECT_DIR/stack.output | grep $ARCH-$TAG | grep $TEST_SET | cut -d ' ' -f 2)
+    - MICRO_VM_IP=$(cat $CI_PROJECT_DIR/stack.output | grep $MICROVM_GREP_PATTERN | cut -d ' ' -f 2)
     - ssh metal_instance "scp ${MICRO_VM_IP}:/ci-visibility/junit.tar.gz /home/ubuntu/junit-${ARCH}-${TAG}-${TEST_SET}.tar.gz"
     - scp "metal_instance:/home/ubuntu/junit-${ARCH}-${TAG}-${TEST_SET}.tar.gz" $DD_AGENT_TESTING_DIR/
     - ssh metal_instance "scp ${MICRO_VM_IP}:/ci-visibility/testjson.tar.gz /home/ubuntu/testjson-${ARCH}-${TAG}-${TEST_SET}.tar.gz"

From ff153239c5a0508b368d6d904cd14ce1e965967c Mon Sep 17 00:00:00 2001
From: usamasaqib <usama.saqib@datadoghq.com>
Date: Wed, 29 Nov 2023 13:24:17 +0100
Subject: [PATCH 35/46] fix needs for cleanup jobs

---
 .gitlab/kernel_version_testing/system_probe.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.gitlab/kernel_version_testing/system_probe.yml b/.gitlab/kernel_version_testing/system_probe.yml
index 6f8662f121c6a3..c7745d7a56d64e 100644
--- a/.gitlab/kernel_version_testing/system_probe.yml
+++ b/.gitlab/kernel_version_testing/system_probe.yml
@@ -444,7 +444,7 @@ kernel_matrix_testing_run_tests_arm64:
 kernel_matrix_testing_cleanup_arm64:
   extends:
     - .kernel_matrix_testing_cleanup
-  needs: ["kernel_matrix_testing_setup_env_arm64", "kmt_run_tests_no_tracer_arm64", "kmt_run_tests_only_tracer_arm64"]
+  needs: ["kernel_matrix_testing_setup_env_arm64", "kernel_matrix_testing_run_tests_arm64"]
   variables:
     ARCH: arm64
     INSTANCE_TYPE: "m6gd.metal"
@@ -452,7 +452,7 @@ kernel_matrix_testing_cleanup_arm64:
 kernel_matrix_testing_cleanup_x64:
   extends:
     - .kernel_matrix_testing_cleanup
-  needs: ["kernel_matrix_testing_setup_env_x64", "kmt_run_tests_no_tracer_x64", "kmt_run_tests_only_tracer_x64"]
+  needs: ["kernel_matrix_testing_setup_env_x64", "kernel_matrix_testing_run_tests_x64"]
   variables:
     ARCH: x86_64
     INSTANCE_TYPE: "m5d.metal"

From ce7ba11bc8a956bac70a87b74b79825f1e76ccd9 Mon Sep 17 00:00:00 2001
From: usamasaqib <usama.saqib@datadoghq.com>
Date: Wed, 29 Nov 2023 13:30:50 +0100
Subject: [PATCH 36/46] separte junit upload jobs by architecture

---
 ...ctional_test_junit_upload_system_probe.yml | 29 ++++++++++++++-----
 1 file changed, 22 insertions(+), 7 deletions(-)

diff --git a/.gitlab/junit_upload/functional_test_junit_upload_system_probe.yml b/.gitlab/junit_upload/functional_test_junit_upload_system_probe.yml
index d362441aab6e5e..7ff79215716476 100644
--- a/.gitlab/junit_upload/functional_test_junit_upload_system_probe.yml
+++ b/.gitlab/junit_upload/functional_test_junit_upload_system_probe.yml
@@ -23,17 +23,12 @@ functional_test_junit_upload_system_probe:
     - set -x
     - ss=0; for f in $DD_AGENT_TESTING_DIR/kitchen-junit-*.tar.gz; do [[ -e "$f" ]] || continue; inv -e junit-upload --tgz-path $f || ((ss++)); done; exit $ss
 
-functional_test_junit_upload_system_probe_kmt:
+.functional_test_junit_upload_system_probe_kmt:
   stage: functional_test_junit_upload
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/datadog-ci-uploader$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
   tags: ["arch:amd64"]
   allow_failure: true
   when: always
-  needs:
-    - job: kernel_matrix_testing_run_tests_x64
-      optional: true
-    - job: kernel_matrix_testing_run_tests_arm64
-      optional: true
   variables:
     DD_ENV: ci
   script:
@@ -41,4 +36,24 @@ functional_test_junit_upload_system_probe_kmt:
     - export DATADOG_API_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.datadog_api_key_org2 --with-decryption --query "Parameter.Value" --out text)
     - export JIRA_TOKEN=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.jira_read_api_token --with-decryption --query "Parameter.Value" --out text)
     - set -x
-    - ss=0; for f in $DD_AGENT_TESTING_DIR/junit-*.tar.gz; do [[ -e "$f" ]] || continue; inv -e junit-upload --tgz-path $f || ((ss++)); done; exit $ss
+    - ss=0; for f in $DD_AGENT_TESTING_DIR/junit-$ARCH*.tar.gz; do [[ -e "$f" ]] || continue; inv -e junit-upload --tgz-path $f || ((ss++)); done; exit $ss
+
+# junit upload jobs are separted because of gitlab limit on the number of jobs
+# a single job can 'need'.
+functional_test_junit_upload_system_probe_kmt_x64:
+  extends:
+    - .functional_test_junit_upload_system_probe_kmt
+  needs:
+    - job: kernel_matrix_testing_run_tests_x64
+      optional: true
+  variables:
+    ARCH: "x86_64"
+
+functional_test_junit_upload_system_probe_kmt_arm64:
+  extends:
+    - .functional_test_junit_upload_system_probe_kmt
+  needs:
+    - job: kernel_matrix_testing_run_tests_arm64
+      optional: true
+  variables:
+    ARCH: "arm64"

From 946559a5cb5e43d32a780d3086e67064ac0092aa Mon Sep 17 00:00:00 2001
From: usamasaqib <usama.saqib@datadoghq.com>
Date: Wed, 29 Nov 2023 14:06:03 +0100
Subject: [PATCH 37/46] fix test-runner configuration file names

---
 .../test-runner/files/{no_tracer.json => no_tracersuite.json}     | 0
 .../test-runner/files/{only_tracer.json => only_tracersuite.json} | 0
 2 files changed, 0 insertions(+), 0 deletions(-)
 rename test/new-e2e/system-probe/test-runner/files/{no_tracer.json => no_tracersuite.json} (100%)
 rename test/new-e2e/system-probe/test-runner/files/{only_tracer.json => only_tracersuite.json} (100%)

diff --git a/test/new-e2e/system-probe/test-runner/files/no_tracer.json b/test/new-e2e/system-probe/test-runner/files/no_tracersuite.json
similarity index 100%
rename from test/new-e2e/system-probe/test-runner/files/no_tracer.json
rename to test/new-e2e/system-probe/test-runner/files/no_tracersuite.json
diff --git a/test/new-e2e/system-probe/test-runner/files/only_tracer.json b/test/new-e2e/system-probe/test-runner/files/only_tracersuite.json
similarity index 100%
rename from test/new-e2e/system-probe/test-runner/files/only_tracer.json
rename to test/new-e2e/system-probe/test-runner/files/only_tracersuite.json

From 7df6446688586e1e8da58b25ff900f9aa30b8cbc Mon Sep 17 00:00:00 2001
From: usamasaqib <usama.saqib@datadoghq.com>
Date: Wed, 29 Nov 2023 14:33:20 +0100
Subject: [PATCH 38/46] python-linting

---
 tasks/kernel_matrix_testing/vmconfig.py | 17 ++++++++++-------
 tasks/kmt.py                            |  2 +-
 2 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/tasks/kernel_matrix_testing/vmconfig.py b/tasks/kernel_matrix_testing/vmconfig.py
index b3d286648ac6cd..00ec9012145911 100644
--- a/tasks/kernel_matrix_testing/vmconfig.py
+++ b/tasks/kernel_matrix_testing/vmconfig.py
@@ -97,7 +97,7 @@
     "x86_64": "x86_64",
     "arm64": "arm64",
     "arm": "arm64",
-    "aarch64": "arm64"
+    "aarch64": "arm64",
 }
 
 TICK = "\u2713"
@@ -184,6 +184,7 @@ def list_possible():
 
     return result
 
+
 # normalize_vm_def converts the detected user provider vm-def
 # to a standard form with consisten values for
 # recipe: [custom, distro]
@@ -205,7 +206,7 @@ def normalize_vm_def(possible, vm):
     return recipe, version, arch
 
 
-def get_custom_kernel_config(template, recipe, version, arch):
+def get_custom_kernel_config(version, arch):
     if arch == local_arch:
         arch = arch_mapping[platform.machine()]
 
@@ -234,7 +235,7 @@ def get_custom_kernel_config(template, recipe, version, arch):
 # to the micro-vms scenario in test-infra-definitions
 def get_kernel_config(template, recipe, version, arch):
     if recipe == "custom":
-        return get_custom_kernel_config(template, recipe, version, arch)
+        return get_custom_kernel_config(version, arch)
 
     if arch == "local":
         arch = arch_mapping[platform.machine()]
@@ -367,7 +368,7 @@ def add_console(vmset):
 def url_to_fspath(url):
     source = urlparse(url)
     if os.path.basename(source.path).endswith(".xz"):
-        filename = os.path.basename(source.path)[:-len(".xz")]
+        filename = os.path.basename(source.path)[: -len(".xz")]
     else:
         filename = os.path.basename(source.path)
 
@@ -445,10 +446,10 @@ def generate_vmconfig(vm_config, normalized_vm_defs, vcpu, memory, sets, ci):
 
         # duplicate vm if multiple sets provided by user
         for s in sets:
-            vmsets.add(VMSet(arch, recipe, set([vmset_name(arch, recipe), s])))
+            vmsets.add(VMSet(arch, recipe, {vmset_name(arch, recipe), s}))
 
         if len(sets) == 0:
-            vmsets.add(VMSet(arch, recipe, set([vmset_name(arch, recipe)])))
+            vmsets.add(VMSet(arch, recipe, {vmset_name(arch, recipe)}))
 
     # map vms to vmsets
     for recipe, version, arch in normalized_vm_defs:
@@ -463,7 +464,9 @@ def generate_vmconfig(vm_config, normalized_vm_defs, vcpu, memory, sets, ci):
     for vmset in vmsets:
         for vm in vmset.vms:
             add_kernel(
-                vm_config, get_kernel_config(vmconfig_template, vmset.recipe, vm.version, vmset.arch), vmset.tags,
+                vm_config,
+                get_kernel_config(vmconfig_template, vmset.recipe, vm.version, vmset.arch),
+                vmset.tags,
             )
 
     for vmset in vm_config["vmsets"]:
diff --git a/tasks/kmt.py b/tasks/kmt.py
index d2189ae803bfa3..626f5c87619bdc 100644
--- a/tasks/kmt.py
+++ b/tasks/kmt.py
@@ -56,7 +56,7 @@ def gen_config(
     new=False,
     ci=False,
     arch="",
-    output_file="vmconfig.json"
+    output_file="vmconfig.json",
 ):
     vmconfig.gen_config(ctx, stack, vms, sets, init_stack, vcpu, memory, new, ci, arch, output_file)
 

From 577d9af32631f2476bcafbf919f412d87307056d Mon Sep 17 00:00:00 2001
From: usamasaqib <usama.saqib@datadoghq.com>
Date: Wed, 29 Nov 2023 16:11:49 +0100
Subject: [PATCH 39/46] fix sorting

---
 tasks/kernel_matrix_testing/vmconfig.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tasks/kernel_matrix_testing/vmconfig.py b/tasks/kernel_matrix_testing/vmconfig.py
index 00ec9012145911..cce128d99e7d24 100644
--- a/tasks/kernel_matrix_testing/vmconfig.py
+++ b/tasks/kernel_matrix_testing/vmconfig.py
@@ -4,10 +4,10 @@
 import math
 import os
 import platform
+from urllib.parse import urlparse
 
 from .init_kmt import KMT_STACKS_DIR, VMCONFIG, KMT_ROOTFS_DIR, check_and_get_stack
 from .stacks import create_stack, stack_exists
-from urllib.parse import urlparse
 from .tool import Exit, ask, info, warn
 
 vm_recipe = "recipe"

From 233575ef331c8649099c0b7fffb600718acd4bed Mon Sep 17 00:00:00 2001
From: usamasaqib <usama.saqib@datadoghq.com>
Date: Wed, 29 Nov 2023 19:58:36 +0100
Subject: [PATCH 40/46] update test-infra

---
 test/new-e2e/go.mod | 2 +-
 test/new-e2e/go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/test/new-e2e/go.mod b/test/new-e2e/go.mod
index 75a9f53ad4fb82..00742efa25ee7f 100644
--- a/test/new-e2e/go.mod
+++ b/test/new-e2e/go.mod
@@ -22,7 +22,7 @@ require (
 	// `TEST_INFRA_DEFINITIONS_BUILDIMAGES` matches the commit sha in the module version
 	// Example: 	github.com/DataDog/test-infra-definitions v0.0.0-YYYYMMDDHHmmSS-0123456789AB
 	// => TEST_INFRA_DEFINITIONS_BUILDIMAGES: 0123456789AB
-	github.com/DataDog/test-infra-definitions v0.0.0-20231129110704-121fcc5993ec
+	github.com/DataDog/test-infra-definitions v0.0.0-20231129185556-40b046c22010
 	github.com/aws/aws-sdk-go-v2 v1.22.1
 	github.com/aws/aws-sdk-go-v2/config v1.18.40
 	github.com/aws/aws-sdk-go-v2/service/ec2 v1.130.0
diff --git a/test/new-e2e/go.sum b/test/new-e2e/go.sum
index 26e032477c7d8a..230f4082ffc85d 100644
--- a/test/new-e2e/go.sum
+++ b/test/new-e2e/go.sum
@@ -12,8 +12,8 @@ github.com/DataDog/datadog-api-client-go/v2 v2.15.0 h1:5UVON1xs6Lul4d6R5TmLDqqSJ
 github.com/DataDog/datadog-api-client-go/v2 v2.15.0/go.mod h1:ZG8wS+y2rUmkRDJZQq7Og7EAPFPage+7vXcmuah2I9o=
 github.com/DataDog/mmh3 v0.0.0-20200805151601-30884ca2197a h1:m9REhmyaWD5YJ0P53ygRHxKKo+KM+nw+zz0hEdKztMo=
 github.com/DataDog/mmh3 v0.0.0-20200805151601-30884ca2197a/go.mod h1:SvsjzyJlSg0rKsqYgdcFxeEVflx3ZNAyFfkUHP0TxXg=
-github.com/DataDog/test-infra-definitions v0.0.0-20231129110704-121fcc5993ec h1:APP1Bv8ZSo99BG+yhmROpcDBu9We7zC3Io8DfTovQpw=
-github.com/DataDog/test-infra-definitions v0.0.0-20231129110704-121fcc5993ec/go.mod h1:P6FDz6Iyki8kX1WoDln5U+3kVLC+QzRi7FIzauuZ6jM=
+github.com/DataDog/test-infra-definitions v0.0.0-20231129185556-40b046c22010 h1:v8ChDpXZVMYEUEyyfYsS3rzDqqkRm2BeR0FlUeaWEXs=
+github.com/DataDog/test-infra-definitions v0.0.0-20231129185556-40b046c22010/go.mod h1:P6FDz6Iyki8kX1WoDln5U+3kVLC+QzRi7FIzauuZ6jM=
 github.com/DataDog/zstd v1.5.2 h1:vUG4lAyuPCXO0TLbXvPv7EB7cNK1QV/luu55UHLrrn8=
 github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
 github.com/DataDog/zstd_0 v0.0.0-20210310093942-586c1286621f h1:5Vuo4niPKFkfwW55jV4vY0ih3VQ9RaQqeqY67fvRn8A=

From 282dc495c3cea152ed1d7f0414b3f1f7250add0f Mon Sep 17 00:00:00 2001
From: usamasaqib <usama.saqib@datadoghq.com>
Date: Wed, 29 Nov 2023 20:09:20 +0100
Subject: [PATCH 41/46] update test-infra runner image

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 48ae6088598cd5..962c6103a109ae 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -177,7 +177,7 @@ variables:
   # To use images from test-infra-definitions dev branches, set the SUFFIX variable to -dev
   # and check the job creating the image to make sure you have the right SHA prefix
   TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX: ""
-  TEST_INFRA_DEFINITIONS_BUILDIMAGES: d505ce3abecf
+  TEST_INFRA_DEFINITIONS_BUILDIMAGES: 40b046c22010
   DATADOG_AGENT_BUILDERS: v22276738-b36b132
 
   DATADOG_AGENT_EMBEDDED_PATH: /opt/datadog-agent/embedded

From f315f3c3567e268c594561473e97fc4c388eb926 Mon Sep 17 00:00:00 2001
From: usamasaqib <usama.saqib@datadoghq.com>
Date: Wed, 29 Nov 2023 20:31:53 +0100
Subject: [PATCH 42/46] fix import sorting

---
 tasks/kernel_matrix_testing/vmconfig.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tasks/kernel_matrix_testing/vmconfig.py b/tasks/kernel_matrix_testing/vmconfig.py
index cce128d99e7d24..fd82067c2d0b5c 100644
--- a/tasks/kernel_matrix_testing/vmconfig.py
+++ b/tasks/kernel_matrix_testing/vmconfig.py
@@ -6,7 +6,7 @@
 import platform
 from urllib.parse import urlparse
 
-from .init_kmt import KMT_STACKS_DIR, VMCONFIG, KMT_ROOTFS_DIR, check_and_get_stack
+from .init_kmt import KMT_ROOTFS_DIR, KMT_STACKS_DIR, VMCONFIG, check_and_get_stack
 from .stacks import create_stack, stack_exists
 from .tool import Exit, ask, info, warn
 

From 6b591be268bd2243609f65cc975589b70b0d5001 Mon Sep 17 00:00:00 2001
From: usamasaqib <usama.saqib@datadoghq.com>
Date: Wed, 29 Nov 2023 20:49:30 +0100
Subject: [PATCH 43/46] remove unused variables

---
 tasks/kernel_matrix_testing/vmconfig.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/tasks/kernel_matrix_testing/vmconfig.py b/tasks/kernel_matrix_testing/vmconfig.py
index fd82067c2d0b5c..919487f0bf5ee1 100644
--- a/tasks/kernel_matrix_testing/vmconfig.py
+++ b/tasks/kernel_matrix_testing/vmconfig.py
@@ -10,9 +10,6 @@
 from .stacks import create_stack, stack_exists
 from .tool import Exit, ask, info, warn
 
-vm_recipe = "recipe"
-vm_architecture = "arch"
-vm_version = "version"
 local_arch = "local"
 vmconfig_file = "test/new-e2e/system-probe/config/vmconfig.json"
 
@@ -119,8 +116,6 @@
     ["debian 11 - v5.10.0", TICK, TICK],
 ]
 
-consoles = {"x86_64": "ttyS0", "arm64": "ttyAMA0"}
-
 
 def lte_414(version):
     major, minor = version.split('.')

From 98e45119f8e15a456987d0ca50e56f6845947f02 Mon Sep 17 00:00:00 2001
From: usamasaqib <usama.saqib@datadoghq.com>
Date: Thu, 30 Nov 2023 11:55:55 +0100
Subject: [PATCH 44/46] remote unused vmconfig file

---
 .../system-probe/config/vmconfig-arm64.json   | 97 -------------------
 1 file changed, 97 deletions(-)
 delete mode 100644 test/new-e2e/system-probe/config/vmconfig-arm64.json

diff --git a/test/new-e2e/system-probe/config/vmconfig-arm64.json b/test/new-e2e/system-probe/config/vmconfig-arm64.json
deleted file mode 100644
index 045a20eda7380f..00000000000000
--- a/test/new-e2e/system-probe/config/vmconfig-arm64.json
+++ /dev/null
@@ -1,97 +0,0 @@
-{
-    "vmsets": [
-        {
-            "name": "distro_arm64",
-            "recipe": "distro-arm64",
-            "arch": "arm64",
-            "console_type": "file",
-            "kernels": [
-                {
-                    "dir": "bionic-server-cloudimg-arm64.qcow2",
-                    "tag": "ubuntu_18.04",
-                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/bionic-server-cloudimg-arm64.qcow2.xz"
-                },
-                {
-                    "dir": "focal-server-cloudimg-arm64.qcow2",
-                    "tag": "ubuntu_20.04",
-                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/focal-server-cloudimg-arm64.qcow2.xz"
-                },
-                {
-                    "dir": "jammy-server-cloudimg-arm64.qcow2",
-                    "tag": "ubuntu_22.04",
-                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/jammy-server-cloudimg-arm64.qcow2.xz"
-                },
-                {
-                    "dir": "mantic-server-cloudimg-arm64.qcow2",
-                    "tag": "ubuntu_23.10",
-                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/mantic-server-cloudimg-arm64.qcow2.xz"
-                },
-                {
-                    "dir": "amzn2-kvm-2.0-arm64-4.14.qcow2",
-                    "tag": "amzn_4.14",
-                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/amzn2-kvm-2.0-arm64-4.14.qcow2.xz"
-                },
-                {
-                    "dir": "amzn2-kvm-2.0-arm64-5.4.qcow2",
-                    "tag": "amzn_5.4",
-                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/amzn2-kvm-2.0-arm64-5.4.qcow2.xz"
-                },
-                {
-                    "dir": "amzn2-kvm-2.0-arm64-5.10.qcow2",
-                    "tag": "amzn_5.10",
-                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/amzn2-kvm-2.0-arm64-5.10.qcow2.xz"
-                },
-                {
-                    "dir": "Fedora-Cloud-Base-37.arm64.qcow2",
-                    "tag": "fedora_37",
-                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/Fedora-Cloud-Base-37.arm64.qcow2.xz"
-                },
-                {
-                    "dir": "Fedora-Cloud-Base-38.arm64.qcow2",
-                    "tag": "fedora_38",
-                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/Fedora-Cloud-Base-38.arm64.qcow2.xz"
-                },
-                {
-                    "dir": "debian-10-generic-arm64.qcow2",
-                    "tag": "debian_10",
-                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/debian-10-generic-arm64.qcow2.xz-test-only"
-                },
-                {
-                    "dir": "debian-11-generic-arm64.qcow2",
-                    "tag": "debian_11",
-                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/debian-11-generic-arm64.qcow2.xz-test-only"
-                },
-                {
-                    "dir": "debian-12-generic-arm64.qcow2",
-                    "tag": "debian_12",
-                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/debian-12-generic-arm64.qcow2.xz"
-                },
-                {
-                    "dir": "centos-7.9-arm64.qcow2",
-                    "tag": "centos_79",
-                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/centos-7.9-arm64.qcow2.xz"
-                },
-                {
-                    "dir": "centos-8-arm64.qcow2",
-                    "tag": "centos_8",
-                    "image_source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/centos-8-arm64.qcow2.xz"
-                }
-            ],
-            "machine": "virt",
-            "vcpu": [
-                4
-            ],
-            "memory": [
-                8192
-            ],
-            "disks": [
-                {
-                    "mount_point": "/mnt/docker",
-                    "source": "https://dd-agent-omnibus.s3.amazonaws.com/kernel-version-testing/rootfs/master/docker-arm64.qcow2",
-                    "target": "/home/kernel-version-testing/docker-arm64.qcow2",
-                    "type": "default"
-                }
-            ]
-        }
-    ]
-}

From 7bb6b75be37af8de91827dd3f0d001038f39c173 Mon Sep 17 00:00:00 2001
From: Usama Saqib <usamasaqib.96@live.com>
Date: Fri, 1 Dec 2023 11:00:17 +0100
Subject: [PATCH 45/46] Update tasks/kernel_matrix_testing/vmconfig.py

Co-authored-by: Bryce Kahle <bryce.kahle@datadoghq.com>
---
 tasks/kernel_matrix_testing/vmconfig.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tasks/kernel_matrix_testing/vmconfig.py b/tasks/kernel_matrix_testing/vmconfig.py
index 919487f0bf5ee1..3a8a25dff47f7c 100644
--- a/tasks/kernel_matrix_testing/vmconfig.py
+++ b/tasks/kernel_matrix_testing/vmconfig.py
@@ -188,7 +188,7 @@ def list_possible():
 # Each normalized_vm_def output corresponds to each VM
 # requested by the user
 def normalize_vm_def(possible, vm):
-    # atempt to fuzzy match user provided vm-def with the possible list.
+    # attempt to fuzzy match user provided vm-def with the possible list.
     vm_def, _ = process.extractOne(vm, possible, scorer=fuzz.token_sort_ratio)
     recipe, version, arch = vm_def.split('-')
 

From 683b7e211ce138521fd4751cbadf24cc9a1ffdf5 Mon Sep 17 00:00:00 2001
From: usamasaqib <usama.saqib@datadoghq.com>
Date: Fri, 1 Dec 2023 11:03:25 +0100
Subject: [PATCH 46/46] remove arch mapping

---
 .../functional_test_junit_upload_system_probe.yml           | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/.gitlab/junit_upload/functional_test_junit_upload_system_probe.yml b/.gitlab/junit_upload/functional_test_junit_upload_system_probe.yml
index 7ff79215716476..c151297b07e80d 100644
--- a/.gitlab/junit_upload/functional_test_junit_upload_system_probe.yml
+++ b/.gitlab/junit_upload/functional_test_junit_upload_system_probe.yml
@@ -36,7 +36,7 @@ functional_test_junit_upload_system_probe:
     - export DATADOG_API_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.datadog_api_key_org2 --with-decryption --query "Parameter.Value" --out text)
     - export JIRA_TOKEN=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.jira_read_api_token --with-decryption --query "Parameter.Value" --out text)
     - set -x
-    - ss=0; for f in $DD_AGENT_TESTING_DIR/junit-$ARCH*.tar.gz; do [[ -e "$f" ]] || continue; inv -e junit-upload --tgz-path $f || ((ss++)); done; exit $ss
+    - ss=0; for f in $DD_AGENT_TESTING_DIR/junit-*.tar.gz; do [[ -e "$f" ]] || continue; inv -e junit-upload --tgz-path $f || ((ss++)); done; exit $ss
 
 # junit upload jobs are separted because of gitlab limit on the number of jobs
 # a single job can 'need'.
@@ -46,8 +46,6 @@ functional_test_junit_upload_system_probe_kmt_x64:
   needs:
     - job: kernel_matrix_testing_run_tests_x64
       optional: true
-  variables:
-    ARCH: "x86_64"
 
 functional_test_junit_upload_system_probe_kmt_arm64:
   extends:
@@ -55,5 +53,3 @@ functional_test_junit_upload_system_probe_kmt_arm64:
   needs:
     - job: kernel_matrix_testing_run_tests_arm64
       optional: true
-  variables:
-    ARCH: "arm64"