[Heartbeat] Decreases the ES default timeout to 10 for the load monit…

…or state requests (#37028)

* [Heartbeat] decrease ES timeout for monitor state requests

* Add changelog

* Fix typo

* ensure config change is not global

* fix lint issues

CHANGELOG.next.asciidoc

@@ -27,7 +27,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Switch types of `log.file.device`, `log.file.inode`, `log.file.idxhi`, `log.file.idxlo` and `log.file.vol` fields to strings to better align with ECS and integrations. {pull}36697[36697]
 
 *Heartbeat*
-
+- Decreases the ES default timeout to 10 for the load monitor state requests
 
 *Metricbeat*
 

heartbeat/beater/heartbeat.go

@@ -323,6 +323,7 @@ func (bt *Heartbeat) Stop() {
 	bt.stopOnce.Do(func() { close(bt.done) })
 }
 
+// makeESClient establishes an ES connection meant to load monitors' state
 func makeESClient(cfg *conf.C, attempts int, wait time.Duration) (*eslegclient.Connection, error) {
 	var (
 		esClient *eslegclient.Connection
@@ -336,8 +337,23 @@ func makeESClient(cfg *conf.C, attempts int, wait time.Duration) (*eslegclient.C
 		wait,
 	)
 
+	// Overriding the default ES request timeout:
+	// Higher values of timeouts cannot be applied on the SAAS Service
+	// where we are running in tight loops and want the next successive check to be run for a given monitor
+	// within the next scheduled interval which could be 1m or 3m
+
+	// Clone original config since we don't want this change to be global
+	newCfg, err := conf.NewConfigFrom(cfg)
+	if err != nil {
+		return nil, fmt.Errorf("error cloning config: %w", err)
+	}
+	timeout := int64((10 * time.Second).Seconds())
+	if err := newCfg.SetInt("timeout", -1, timeout); err != nil {
+		return nil, fmt.Errorf("error setting the ES timeout in config: %w", err)
+	}
+
 	for i := 0; i < attempts; i++ {
-		esClient, err = eslegclient.NewConnectedClient(cfg, "Heartbeat")
+		esClient, err = eslegclient.NewConnectedClient(newCfg, "Heartbeat")
 		if err == nil {
 			connectDelay.Reset()
 			return esClient, nil

heartbeat/beater/heartbeat_test.go

@@ -0,0 +1,48 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package beater
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	conf "github.com/elastic/elastic-agent-libs/config"
+)
+
+func TestMakeESClient(t *testing.T) {
+	t.Run("should not modify the timeout setting from original config", func(t *testing.T) {
+		origTimeout := 90
+		origCfg, _ := conf.NewConfigFrom(map[interface{}]interface{}{
+			"hosts":    []string{"http://localhost:9200"},
+			"username": "anyuser",
+			"password": "anypwd",
+			"timeout":  origTimeout,
+		})
+		anyAttempt := 1
+		anyDuration := 1 * time.Second
+
+		_, _ = makeESClient(origCfg, anyAttempt, anyDuration)
+
+		timeout, err := origCfg.Int("timeout", -1)
+		require.NoError(t, err)
+		assert.EqualValues(t, origTimeout, timeout)
+	})
+}