[ha-agent] Reset Agent State on empty RC updates

comp/haagent/impl/haagent.go

@@ -66,6 +66,10 @@ func (h *haAgentImpl) SetLeader(leaderAgentHostname string) {
 	}
 }
 
+func (h *haAgentImpl) resetAgentState() {
+	h.state.Store(string(haagent.Unknown))
+}
+
 // ShouldRunIntegration return true if the agent integrations should to run.
 // When ha-agent is disabled, the agent behave as standalone agent (non HA) and will always run all integrations.
 func (h *haAgentImpl) ShouldRunIntegration(integrationName string) bool {
@@ -78,6 +82,15 @@ func (h *haAgentImpl) ShouldRunIntegration(integrationName string) bool {
 func (h *haAgentImpl) onHaAgentUpdate(updates map[string]state.RawConfig, applyStateCallback func(string, state.ApplyStatus)) {
 	h.log.Debugf("Updates received: count=%d", len(updates))
 
+	// New updates arrived, but if the list of updates is empty,
+	// it means we don't have any updates applying to this agent anymore.
+	// In this case, reset HA Agent setting to default states.
+	if len(updates) == 0 {
+		h.log.Warn("Empty update received. Resetting Agent State to Unknown.")
+		h.resetAgentState()
+		return
+	}
+
 	for configPath, rawConfig := range updates {
 		h.log.Debugf("Received config %s: %s", configPath, string(rawConfig.Config))
 		haAgentMsg := haAgentConfig{}

comp/haagent/impl/haagent_test.go

@@ -14,6 +14,7 @@ import (
 	"github.com/DataDog/datadog-agent/pkg/remoteconfig/state"
 	"github.com/DataDog/datadog-agent/pkg/util/fxutil"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	"go.uber.org/fx"
 )
 
@@ -106,25 +107,41 @@ func Test_RCListener(t *testing.T) {
 }
 
 func Test_haAgentImpl_onHaAgentUpdate(t *testing.T) {
-
 	tests := []struct {
 		name                string
+		initialState        haagent.State
 		updates             map[string]state.RawConfig
 		expectedApplyID     string
 		expectedApplyStatus state.ApplyStatus
+		expectedAgentState  haagent.State
 	}{
 		{
-			name: "successful update",
+			name:         "successful update with leader matching current agent",
+			initialState: haagent.Unknown,
+			updates: map[string]state.RawConfig{
+				testConfigID: {Config: []byte(`{"group":"testGroup01","leader":"my-agent-hostname"}`)},
+			},
+			expectedApplyID: testConfigID,
+			expectedApplyStatus: state.ApplyStatus{
+				State: state.ApplyStateAcknowledged,
+			},
+			expectedAgentState: haagent.Active,
+		},
+		{
+			name:         "successful update with leader NOT matching current agent",
+			initialState: haagent.Unknown,
 			updates: map[string]state.RawConfig{
-				testConfigID: {Config: []byte(`{"group":"testGroup01","leader":"ha-agent1"}`)},
+				testConfigID: {Config: []byte(`{"group":"testGroup01","leader":"another-agent-hostname"}`)},
 			},
 			expectedApplyID: testConfigID,
 			expectedApplyStatus: state.ApplyStatus{
 				State: state.ApplyStateAcknowledged,
 			},
+			expectedAgentState: haagent.Standby,
 		},
 		{
-			name: "invalid payload",
+			name:         "invalid payload",
+			initialState: haagent.Unknown,
 			updates: map[string]state.RawConfig{
 				testConfigID: {Config: []byte(`invalid-json`)},
 			},
@@ -133,17 +150,28 @@ func Test_haAgentImpl_onHaAgentUpdate(t *testing.T) {
 				State: state.ApplyStateError,
 				Error: "error unmarshalling payload",
 			},
+			expectedAgentState: haagent.Unknown,
 		},
 		{
-			name: "invalid group",
+			name:         "invalid group",
+			initialState: haagent.Unknown,
 			updates: map[string]state.RawConfig{
-				testConfigID: {Config: []byte(`{"group":"invalidGroup","leader":"ha-agent1"}`)},
+				testConfigID: {Config: []byte(`{"group":"invalidGroup","leader":"another-agent-hostname"}`)},
 			},
 			expectedApplyID: testConfigID,
 			expectedApplyStatus: state.ApplyStatus{
 				State: state.ApplyStateError,
 				Error: "group does not match",
 			},
+			expectedAgentState: haagent.Unknown,
+		},
+		{
+			name:                "empty update",
+			initialState:        haagent.Active,
+			updates:             map[string]state.RawConfig{},
+			expectedApplyID:     "",
+			expectedApplyStatus: state.ApplyStatus{},
+			expectedAgentState:  haagent.Unknown,
 		},
 	}
 	for _, tt := range tests {
@@ -160,6 +188,10 @@ func Test_haAgentImpl_onHaAgentUpdate(t *testing.T) {
 
 			h := newHaAgentImpl(logmock.New(t), newHaAgentConfigs(agentConfigComponent))
 
+			if tt.initialState != "" {
+				h.state.Store(string(tt.initialState))
+			}
+
 			var applyID string
 			var applyStatus state.ApplyStatus
 			applyFunc := func(id string, status state.ApplyStatus) {
@@ -169,6 +201,7 @@ func Test_haAgentImpl_onHaAgentUpdate(t *testing.T) {
 			h.onHaAgentUpdate(tt.updates, applyFunc)
 			assert.Equal(t, tt.expectedApplyID, applyID)
 			assert.Equal(t, tt.expectedApplyStatus, applyStatus)
+			assert.Equal(t, tt.expectedAgentState, h.GetState())
 		})
 	}
 }
@@ -249,3 +282,17 @@ func Test_haAgentImpl_ShouldRunIntegration(t *testing.T) {
 		})
 	}
 }
+
+func Test_haAgentImpl_resetAgentState(t *testing.T) {
+	// GIVEN
+	haAgent := newTestHaAgentComponent(t, nil)
+	haAgentComp := haAgent.Comp.(*haAgentImpl)
+	haAgentComp.state.Store(string(haagent.Active))
+	require.Equal(t, haagent.Active, haAgentComp.GetState())
+
+	// WHEN
+	haAgentComp.resetAgentState()
+
+	// THEN
+	assert.Equal(t, haagent.Unknown, haAgentComp.GetState())
+}