[HA Agent] Add initial e2e test for HA Agent feature (DataDog#31610)

.github/CODEOWNERS

@@ -611,6 +611,7 @@
 /test/new-e2e/tests/agent-subcommands         @DataDog/agent-shared-components
 /test/new-e2e/tests/containers                @DataDog/container-integrations @DataDog/container-platform
 /test/new-e2e/tests/discovery                 @DataDog/universal-service-monitoring
+/test/new-e2e/tests/ha-agent                  @DataDog/ndm-core
 /test/new-e2e/tests/language-detection        @DataDog/processes
 /test/new-e2e/tests/ndm                       @DataDog/ndm-core
 /test/new-e2e/tests/ndm/netflow               @DataDog/ndm-integrations

.gitlab-ci.yml

@@ -998,6 +998,17 @@ workflow:
   - when: manual
     allow_failure: true
 
+.on_ha_agent_or_e2e_changes:
+  - !reference [.on_e2e_main_release_or_rc]
+  - changes:
+      paths:
+        - comp/haagent/**/*
+        - pkg/aggregator/**/*
+        - test/new-e2e/tests/ha-agent/**/*
+        - test/new-e2e/go.mod
+      compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916
+    when: on_success
+
 .on_otel_or_e2e_changes:
   - !reference [.on_e2e_main_release_or_rc]
   - changes:

.gitlab/e2e/e2e.yml

@@ -471,6 +471,15 @@ new-e2e-ndm-snmp:
     TARGETS: ./tests/ndm/snmp
     TEAM: network-device-monitoring
 
+new-e2e-ha-agent:
+  extends: .new_e2e_template_needs_deb_x64
+  rules:
+    - !reference [.on_ha_agent_or_e2e_changes]
+    - !reference [.manual]
+  variables:
+    TARGETS: ./tests/ha-agent
+    TEAM: ndm-core
+
 new-e2e-windows-systemprobe:
   extends: .new_e2e_template
   rules:

test/new-e2e/tests/ha-agent/haagent_test.go

@@ -0,0 +1,74 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2016-present Datadog, Inc.
+
+// Package haagent contains e2e tests for HA Agent feature
+package haagent
+
+import (
+	_ "embed"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/DataDog/test-infra-definitions/components/datadog/agentparams"
+
+	"github.com/DataDog/datadog-agent/test/fakeintake/aggregator"
+	fakeintakeclient "github.com/DataDog/datadog-agent/test/fakeintake/client"
+	"github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e"
+	"github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments"
+	awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host"
+)
+
+type haAgentTestSuite struct {
+	e2e.BaseSuite[environments.Host]
+}
+
+// TestHaAgentSuite runs the HA Agent e2e suite
+func TestHaAgentSuite(t *testing.T) {
+	// language=yaml
+	agentConfig := `
+ha_agent:
+    enabled: true
+    group: test-group01
+log_level: debug
+`
+	e2e.Run(t, &haAgentTestSuite{}, e2e.WithProvisioner(awshost.Provisioner(
+		awshost.WithAgentOptions(agentparams.WithAgentConfig(agentConfig))),
+	))
+}
+
+func (s *haAgentTestSuite) TestHaAgentGroupTagPresentOnDatadogAgentRunningMetric() {
+	fakeClient := s.Env().FakeIntake.Client()
+	s.EventuallyWithT(func(c *assert.CollectT) {
+		s.T().Log("try assert datadog.agent.running metric")
+		metrics, err := fakeClient.FilterMetrics("datadog.agent.running")
+		require.NoError(c, err)
+		assert.NotEmpty(c, metrics)
+		for _, metric := range metrics {
+			s.T().Logf("    datadog.agent.running metric tags: %+v", metric.Tags)
+		}
+
+		tags := []string{"agent_group:test-group01"}
+		metrics, err = fakeClient.FilterMetrics("datadog.agent.running", fakeintakeclient.WithTags[*aggregator.MetricSeries](tags))
+		require.NoError(c, err)
+		assert.NotEmpty(c, metrics)
+	}, 5*time.Minute, 3*time.Second)
+}
+
+func (s *haAgentTestSuite) TestHaAgentAddedToRCListeners() {
+	s.EventuallyWithT(func(c *assert.CollectT) {
+		s.T().Log("try assert HA Agent added to RCListeners in agent.log")
+		output, err := s.Env().RemoteHost.Execute("cat /var/log/datadog/agent.log")
+		require.NoError(c, err)
+
+		assert.Contains(c, output, "Add onHaAgentUpdate RCListener")
+	}, 5*time.Minute, 3*time.Second)
+}
+
+// TODO: Add test for Agent behaviour when receiving RC HA_AGENT messages
+//       - Agent receiving message to become leader
+//       - Agent receiving message to become follower