Add testLog metrics to assert logs in the containers new-e2e test…

…s. (DataDog#21045)

.gitlab-ci.yml

@@ -176,7 +176,7 @@ variables:
   # To use images from test-infra-definitions dev branches, set the SUFFIX variable to -dev
   # and check the job creating the image to make sure you have the right SHA prefix
   TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX: ""
-  TEST_INFRA_DEFINITIONS_BUILDIMAGES: 58b54f06414b
+  TEST_INFRA_DEFINITIONS_BUILDIMAGES: 2edaabddf28b
   DATADOG_AGENT_BUILDERS: v22276738-b36b132
 
   DATADOG_AGENT_EMBEDDED_PATH: /opt/datadog-agent/embedded

test/new-e2e/go.mod

@@ -22,7 +22,7 @@ require (
 	// `TEST_INFRA_DEFINITIONS_BUILDIMAGES` matches the commit sha in the module version
 	// Example: 	github.com/DataDog/test-infra-definitions v0.0.0-YYYYMMDDHHmmSS-0123456789AB
 	// => TEST_INFRA_DEFINITIONS_BUILDIMAGES: 0123456789AB
-	github.com/DataDog/test-infra-definitions v0.0.0-20231117170844-58b54f06414b
+	github.com/DataDog/test-infra-definitions v0.0.0-20231122132756-2edaabddf28b
 	github.com/aws/aws-sdk-go-v2 v1.22.1
 	github.com/aws/aws-sdk-go-v2/config v1.18.40
 	github.com/aws/aws-sdk-go-v2/service/ec2 v1.130.0

test/new-e2e/tests/containers/base_test.go

@@ -86,7 +86,7 @@ func (mc *myCollectT) Errorf(format string, args ...interface{}) {
 func (suite *baseSuite) testMetric(args *testMetricArgs) {
 	prettyMetricQuery := fmt.Sprintf("%s{%s}", args.Filter.Name, strings.Join(args.Filter.Tags, ","))
 
-	suite.Run(prettyMetricQuery, func() {
+	suite.Run("metric   "+prettyMetricQuery, func() {
 		var expectedTags []*regexp.Regexp
 		if args.Expect.Tags != nil {
 			expectedTags = lo.Map(*args.Expect.Tags, func(tag string, _ int) *regexp.Regexp { return regexp.MustCompile(tag) })
@@ -129,7 +129,7 @@ func (suite *baseSuite) testMetric(args *testMetricArgs) {
 
 		defer func() {
 			if suite.T().Failed() {
-				sendEvent("error", fmt.Sprintf("Failed finding %s with proper tags", prettyMetricQuery))
+				sendEvent("error", fmt.Sprintf("Failed finding %s with proper tags and value", prettyMetricQuery))
 			} else {
 				sendEvent("success", "All good!")
 			}
@@ -187,3 +187,123 @@ func (suite *baseSuite) testMetric(args *testMetricArgs) {
 		}, 2*time.Minute, 10*time.Second, "Failed finding `%s` with proper tags and value", prettyMetricQuery)
 	})
 }
+
+type testLogArgs struct {
+	Filter testLogFilterArgs
+	Expect testLogExpectArgs
+}
+
+type testLogFilterArgs struct {
+	Service string
+	Tags    []string
+}
+
+type testLogExpectArgs struct {
+	Tags    *[]string
+	Message string
+}
+
+func (suite *baseSuite) testLog(args *testLogArgs) {
+	prettyLogQuery := fmt.Sprintf("%s{%s}", args.Filter.Service, strings.Join(args.Filter.Tags, ","))
+
+	suite.Run("log   "+prettyLogQuery, func() {
+		var expectedTags []*regexp.Regexp
+		if args.Expect.Tags != nil {
+			expectedTags = lo.Map(*args.Expect.Tags, func(tag string, _ int) *regexp.Regexp { return regexp.MustCompile(tag) })
+		}
+
+		var expectedMessage *regexp.Regexp
+		if args.Expect.Message != "" {
+			expectedMessage = regexp.MustCompile(args.Expect.Message)
+		}
+
+		sendEvent := func(alertType, text string) {
+			formattedArgs, err := yaml.Marshal(args)
+			suite.Require().NoError(err)
+
+			tags := lo.Map(args.Filter.Tags, func(tag string, _ int) string {
+				return "filter_tag_" + tag
+			})
+
+			if _, err := suite.datadogClient.PostEvent(&datadog.Event{
+				Title: pointer.Ptr(fmt.Sprintf("testLog %s", prettyLogQuery)),
+				Text: pointer.Ptr(fmt.Sprintf(`%%%%%%
+### Result
+
+`+"```"+`
+%s
+`+"```"+`
+
+### Query
+
+`+"```"+`
+%s
+`+"```"+`
+ %%%%%%`, text, formattedArgs)),
+				AlertType: &alertType,
+				Tags: append([]string{
+					"app:agent-new-e2e-tests-containers",
+					"cluster_name:" + suite.clusterName,
+					"log_service:" + args.Filter.Service,
+					"test:" + suite.T().Name(),
+				}, tags...),
+			}); err != nil {
+				suite.T().Logf("Failed to post event: %s", err)
+			}
+		}
+
+		defer func() {
+			if suite.T().Failed() {
+				sendEvent("error", fmt.Sprintf("Failed finding %s with proper tags and message", prettyLogQuery))
+			} else {
+				sendEvent("success", "All good!")
+			}
+		}()
+
+		suite.EventuallyWithTf(func(collect *assert.CollectT) {
+			c := &myCollectT{
+				CollectT: collect,
+				errors:   []error{},
+			}
+			// To enforce the use of myCollectT instead
+			collect = nil //nolint:ineffassign
+
+			defer func() {
+				if len(c.errors) == 0 {
+					sendEvent("success", "All good!")
+				} else {
+					sendEvent("warning", errors.Join(c.errors...).Error())
+				}
+			}()
+
+			logs, err := suite.Fakeintake.FilterLogs(
+				args.Filter.Service,
+				fakeintake.WithTags[*aggregator.Log](args.Filter.Tags),
+			)
+			// Can be replaced by require.NoErrorf(…) once https://github.com/stretchr/testify/pull/1481 is merged
+			if !assert.NoErrorf(c, err, "Failed to query fake intake") {
+				return
+			}
+			// Can be replaced by require.NoEmptyf(…) once https://github.com/stretchr/testify/pull/1481 is merged
+			if !assert.NotEmptyf(c, logs, "No `%s` logs yet", prettyLogQuery) {
+				return
+			}
+
+			// Check tags
+			if expectedTags != nil {
+				err := assertTags(logs[len(logs)-1].GetTags(), expectedTags)
+				assert.NoErrorf(c, err, "Tags mismatch on `%s`", prettyLogQuery)
+			}
+
+			// Check message
+			if args.Expect.Message != "" {
+				assert.NotEmptyf(c, lo.Filter(logs, func(m *aggregator.Log, _ int) bool {
+					return expectedMessage.MatchString(m.Message)
+				}), "No log of `%s` is matching %q",
+					prettyLogQuery,
+					args.Expect.Message,
+				)
+			}
+		}, 2*time.Minute, 10*time.Second, "Failed finding `%s` with proper tags and message", prettyLogQuery)
+	})
+}

test/new-e2e/tests/containers/ecs_test.go

@@ -196,6 +196,15 @@ func (suite *ecsSuite) TestNginx() {
 			},
 		},
 	})
+
+	suite.testLog(&testLogArgs{
+		Filter: testLogFilterArgs{
+			Service: "apps-nginx-server",
+		},
+		Expect: testLogExpectArgs{
+			Message: `GET / HTTP/1\.1`,
+		},
+	})
 }
 
 func (suite *ecsSuite) TestRedis() {
@@ -227,6 +236,15 @@ func (suite *ecsSuite) TestRedis() {
 			},
 		},
 	})
+
+	suite.testLog(&testLogArgs{
+		Filter: testLogFilterArgs{
+			Service: "redis",
+		},
+		Expect: testLogExpectArgs{
+			Message: `oO0OoO0OoO0Oo Redis is starting oO0OoO0OoO0Oo`,
+		},
+	})
 }
 
 func (suite *ecsSuite) TestCPU() {

test/new-e2e/tests/containers/k8s_test.go

@@ -282,6 +282,16 @@ func (suite *k8sSuite) TestNginx() {
 		},
 	})
 
+	// Test Nginx logs
+	suite.testLog(&testLogArgs{
+		Filter: testLogFilterArgs{
+			Service: "apps-nginx-server",
+		},
+		Expect: testLogExpectArgs{
+			Message: `GET / HTTP/1\.1`,
+		},
+	})
+
 	// Check HPA is properly scaling up and down
 	// This indirectly tests the cluster-agent external metrics server
 	suite.testHPA("workload-nginx", "nginx")
@@ -342,6 +352,16 @@ func (suite *k8sSuite) TestRedis() {
 		},
 	})
 
+	// Test Redis logs
+	suite.testLog(&testLogArgs{
+		Filter: testLogFilterArgs{
+			Service: "redis",
+		},
+		Expect: testLogExpectArgs{
+			Message: `oO0OoO0OoO0Oo Redis is starting oO0OoO0OoO0Oo`,
+		},
+	})
+
 	// Check HPA is properly scaling up and down
 	// This indirectly tests the cluster-agent external metrics server
 	suite.testHPA("workload-redis", "redis")
@@ -602,7 +622,7 @@ func (suite *k8sSuite) TestPrometheus() {
 }
 
 func (suite *k8sSuite) testHPA(namespace, deployment string) {
-	suite.Run(fmt.Sprintf("kubernetes_state.deployment.replicas_available{kube_namespace:%s,kube_deployment:%s}", namespace, deployment), func() {
+	suite.Run(fmt.Sprintf("hpa   kubernetes_state.deployment.replicas_available{kube_namespace:%s,kube_deployment:%s}", namespace, deployment), func() {
 		sendEvent := func(alertType, text string, time *int) {
 			if _, err := suite.datadogClient.PostEvent(&datadog.Event{
 				Title: pointer.Ptr(fmt.Sprintf("testHPA %s/%s", namespace, deployment)),