add ci visibility reporting for KMT (#20881)

.gitlab/junit_upload/functional_test_junit_upload_system_probe.yml

@@ -19,3 +19,23 @@ functional_test_junit_upload_system_probe:
     - export DATADOG_API_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.datadog_api_key_org2 --with-decryption --query "Parameter.Value" --out text)
     - set -x
     - ss=0; for f in $DD_AGENT_TESTING_DIR/kitchen-junit-*.tar.gz; do [[ -e "$f" ]] || continue; inv -e junit-upload --tgz-path $f || ((ss++)); done; exit $ss
+
+functional_test_junit_upload_system_probe_kmt:
+  stage: functional_test_junit_upload
+  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/datadog-ci-uploader$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
+  tags: ["arch:amd64"]
+  allow_failure: true
+  when: always
+  needs:
+    - job: kernel_matrix_testing_run_tests_x64
+      optional: true
+    - job: kernel_matrix_testing_run_tests_arm64
+      optional: true
+  variables:
+    DD_ENV: ci
+  script:
+    - set +x
+    - export DATADOG_API_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.datadog_api_key_org2 --with-decryption --query "Parameter.Value" --out text)
+    - export JIRA_TOKEN=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.jira_read_api_token --with-decryption --query "Parameter.Value" --out text)
+    - set -x
+    - ss=0; for f in $DD_AGENT_TESTING_DIR/junit-*.tar.gz; do [[ -e "$f" ]] || continue; inv -e junit-upload --tgz-path $f || ((ss++)); done; exit $ss

.gitlab/kernel_version_testing/system_probe.yml

@@ -10,14 +10,14 @@
     - DOCKER_REGISTRY_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.$DOCKER_REGISTRY_LOGIN_SSM_KEY --with-decryption --query "Parameter.Value" --out text)
     - aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.$DOCKER_REGISTRY_PWD_SSM_KEY --with-decryption --query "Parameter.Value" --out text | crane auth login --username "$DOCKER_REGISTRY_LOGIN" --password-stdin "$DOCKER_REGISTRY_URL"
     # Pull base images
-    - mkdir $KITCHEN_DOCKERS
-    - inv -e system-probe.save-test-dockers --use-crane --output-dir $KITCHEN_DOCKERS --arch $ARCH
+    - mkdir $KMT_DOCKERS
+    - inv -e system-probe.save-test-dockers --use-crane --output-dir $KMT_DOCKERS --arch $ARCH
   artifacts:
     expire_in: 1 day
     paths:
-      - $KITCHEN_DOCKERS
+      - $KMT_DOCKERS
   variables:
-    KITCHEN_DOCKERS: $DD_AGENT_TESTING_DIR/kitchen-dockers-$ARCH
+    KMT_DOCKERS: $DD_AGENT_TESTING_DIR/kmt-dockers-$ARCH
 
 pull_test_dockers_x64:
   extends: .pull_test_dockers
@@ -90,7 +90,7 @@ pull_test_dockers_arm64:
     # copy micro-vm-init.sh
     - cp $CI_PROJECT_DIR/test/new-e2e/system-probe/test/micro-vm-init.sh $DEPENDENCIES
     # copy over docker images
-    - cp -R $KITCHEN_DOCKERS $DEPENDENCIES/kitchen-docker
+    - cp -R $KMT_DOCKERS $DEPENDENCIES/kmt-docker
     # package all the dependencies
     - ls -la $DEPENDENCIES
     - pushd $DD_AGENT_TESTING_DIR/$ARCH
@@ -103,7 +103,7 @@ pull_test_dockers_arm64:
     - scp $DD_AGENT_TESTING_DIR/$ARCHIVE_NAME metal_instance:/opt/kernel-version-testing/
   variables:
     DEPENDENCIES: $DD_AGENT_TESTING_DIR/$ARCH/dependencies
-    KITCHEN_DOCKERS: $DD_AGENT_TESTING_DIR/kitchen-dockers-$ARCH
+    KMT_DOCKERS: $DD_AGENT_TESTING_DIR/kmt-dockers-$ARCH
     AWS_EC2_SSH_KEY_FILE: $CI_PROJECT_DIR/ssh_key
 
 upload_dependencies_x64:
@@ -152,8 +152,8 @@ upload_dependencies_arm64:
     # copy system probe tests
     - mkdir -p $SYSTEM_PROBE_TESTS
     - cp -R $KITCHEN_TESTS $SYSTEM_PROBE_TESTS
-    - GOOS=linux GOARCH=$GOARCH go build -o $DEPENDENCIES/test-runner $CI_PROJECT_DIR/test/new-e2e/system-probe/test-runner/main.go
-    - GOOS=linux GOARCH=$GOARCH go build -o $DEPENDENCIES/test-json-review $CI_PROJECT_DIR/test/new-e2e/system-probe/test-json-review/main.go
+    - pushd $CI_PROJECT_DIR/test/new-e2e/system-probe/test-runner && go build -o $DEPENDENCIES/test-runner && popd
+    - pushd $CI_PROJECT_DIR/test/new-e2e/system-probe/test-json-review && go build -o $DEPENDENCIES/test-json-review && popd
     - popd
     # package all the dependencies
     - ls -la $DEPENDENCIES
@@ -354,6 +354,10 @@ kernel_matrix_testing_setup_env_x64:
     - set -x
     - !reference [.kernel_matrix_testing_new_profile]
     - !reference [.write_ssh_key_file]
+    - echo "CI_JOB_URL=${CI_JOB_URL}" >> $DD_AGENT_TESTING_DIR/job_env.txt
+    - echo "CI_JOB_ID=${CI_JOB_ID}" >> $DD_AGENT_TESTING_DIR/job_env.txt
+    - echo "CI_JOB_NAME=${CI_JOB_NAME}" >> $DD_AGENT_TESTING_DIR/job_env.txt
+    - echo "CI_JOB_STAGE=${CI_JOB_STAGE}" >> $DD_AGENT_TESTING_DIR/job_env.txt
   script:
     - INSTANCE_IP=$(grep "$ARCH-instance-ip" $CI_PROJECT_DIR/stack.output | cut -d ' ' -f 2 | tr -d '\n')
     - !reference [.shared_filters_and_queries]
@@ -368,13 +372,23 @@ kernel_matrix_testing_setup_env_x64:
     - GO_VERSION=$(inv go-version)
     - !reference [.setup_ssh_config]
     # ssh into each micro-vm and run initialization script. This script will also run the tests.
-    - NESTED_VM_CMD="/home/ubuntu/connector -host ${MICRO_VM_IP} -user root -ssh-file /home/kernel-version-testing/ddvm_rsa -vm-cmd 'bash /root/fetch_dependencies.sh ${ARCH} && /micro-vm-init.sh ${RETRY} ${ARCH}'"
+    - scp "$DD_AGENT_TESTING_DIR/job_env.txt" "metal_instance:/home/ubuntu/job_env-${ARCH}-${TAG}.txt"
+    - ssh metal_instance "scp /home/ubuntu/job_env-${ARCH}-${TAG}.txt ${MICRO_VM_IP}:/job_env.txt"
+    - NESTED_VM_CMD="/home/ubuntu/connector -host ${MICRO_VM_IP} -user root -ssh-file /home/kernel-version-testing/ddvm_rsa -vm-cmd 'bash /root/fetch_dependencies.sh ${ARCH} && /micro-vm-init.sh ${RETRY}'"
     - $CI_PROJECT_DIR/connector-$ARCH -host $INSTANCE_IP -user ubuntu -ssh-file $AWS_EC2_SSH_KEY_FILE -vm-cmd "${NESTED_VM_CMD}"
-    - ssh metal_instance "scp ${MICRO_VM_IP}:/ci-visibility/junit.tar.gz /home/ubuntu/junit.tar.gz"
-    - ssh metal_instance "scp ${MICRO_VM_IP}:/ci-visibility/testjson.tar.gz /home/ubuntu/testjson.tar.gz"
-    - scp metal_instance:/home/ubuntu/testjson.tar.gz $DD_AGENT_TESTING_DIR/
-    - scp metal_instance:/home/ubuntu/junit.tar.gz $DD_AGENT_TESTING_DIR/
     - ssh metal_instance "ssh ${MICRO_VM_IP} '/test-json-review'"
+  after_script:
+    - MICRO_VM_IP=$(cat $CI_PROJECT_DIR/stack.output | grep $ARCH-$TAG | cut -d ' ' -f 2)
+    - ssh metal_instance "scp ${MICRO_VM_IP}:/ci-visibility/junit.tar.gz /home/ubuntu/junit-${ARCH}-${TAG}.tar.gz"
+    - scp "metal_instance:/home/ubuntu/junit-${ARCH}-${TAG}.tar.gz" $DD_AGENT_TESTING_DIR/
+    - ssh metal_instance "scp ${MICRO_VM_IP}:/ci-visibility/testjson.tar.gz /home/ubuntu/testjson-${ARCH}-${TAG}.tar.gz"
+    - scp "metal_instance:/home/ubuntu/testjson-${ARCH}-${TAG}.tar.gz" $DD_AGENT_TESTING_DIR/
+  artifacts:
+    expire_in: 2 weeks
+    when: always
+    paths:
+      - $DD_AGENT_TESTING_DIR/junit-$ARCH-$TAG.tar.gz
+      - $DD_AGENT_TESTING_DIR/testjson-$ARCH-$TAG.tar.gz
 
 kernel_matrix_testing_run_tests_x64:
   extends:

tasks/kmt.py

@@ -374,7 +374,7 @@ def test(ctx, vms, stack=None, packages="", run=None, retry=2, rebuild_deps=Fals
     run_cmd_vms(
         ctx,
         stack,
-        f"bash /micro-vm-init.sh {retry} {platform.machine()} {' '.join(args)}",
+        f"bash /micro-vm-init.sh {retry} {' '.join(args)}",
         target_vms,
         "",
         allow_fail=True,

test/new-e2e/system-probe/test-json-review/main.go

@@ -3,7 +3,7 @@
 // This product includes software developed at Datadog (https://www.datadoghq.com/).
 // Copyright 2016-present Datadog, Inc.
 
-//go:build !windows
+//go:build linux
 
 package main
 
@@ -19,14 +19,12 @@ import (
 	"github.com/fatih/color"
 )
 
-const TestJSONOut = "/ci-visibility/testjson/out.json"
-
 func init() {
 	color.NoColor = false
 }
 
 func main() {
-	failedTests, err := reviewTests(TestJSONOut)
+	failedTests, err := reviewTests("/ci-visibility/testjson/out.json")
 	if err != nil {
 		log.Fatal(err)
 	}