Cherry-pick #20898 to 7.9: Avoid generating incomplete configurations in autodiscover

CHANGELOG.next.asciidoc

@@ -12,6 +12,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 - Update to Golang 1.12.1. {pull}11330[11330]
 - Disable Alibaba Cloud and Tencent Cloud metadata providers by default. {pull}13812[12812]
+- Autodiscover doesn't generate any configuration when a variable is missing. Previously it generated an incomplete configuration. {pull}20898[20898]
 
 *Auditbeat*
 
@@ -50,6 +51,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix `setup.dashboards.index` setting not working. {pull}17749[17749]
 - Fix Elasticsearch license endpoint URL referenced in error message. {issue}17880[17880] {pull}18030[18030]
 - Change `decode_json_fields` processor, to merge parsed json objects with existing objects in the event instead of fully replacing them. {pull}17958[17958]
+- Explicitly detect missing variables in autodiscover configuration, log them at the debug level. {issue}20568[20568] {pull}20898[20898]
 
 *Auditbeat*
 

libbeat/autodiscover/template/config.go

@@ -18,7 +18,10 @@
 package template
 
 import (
+	"fmt"
+
 	"github.com/elastic/go-ucfg"
+	"github.com/elastic/go-ucfg/parse"
 
 	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/libbeat/common/bus"
@@ -123,7 +126,16 @@ func ApplyConfigTemplate(event bus.Event, configs []*common.Config, options ...u
 	if err != nil {
 		logp.Err("Error building config: %v", err)
 	}
+
 	opts := []ucfg.Option{
+		// Catch-all resolve function to log fields not resolved in any other way,
+		// it needs to be the first resolver added, so it is executed the last one.
+		// Being the last one, its returned error will be the one returned by `Unpack`,
+		// this is important to give better feedback in case of failure.
+		ucfg.Resolve(func(name string) (string, parse.Config, error) {
+			return "", parse.Config{}, fmt.Errorf("field '%s' not available in event or environment", name)
+		}),
+
 		ucfg.PathSep("."),
 		ucfg.Env(vars),
 		ucfg.ResolveEnv,
@@ -139,9 +151,9 @@ func ApplyConfigTemplate(event bus.Event, configs []*common.Config, options ...u
 		}
 		// Unpack config to process any vars in the template:
 		var unpacked map[string]interface{}
-		c.Unpack(&unpacked, opts...)
+		err = c.Unpack(&unpacked, opts...)
 		if err != nil {
-			logp.Err("Error unpacking config: %v", err)
+			logp.Debug("autodiscover", "Configuration template cannot be resolved: %v", err)
 			continue
 		}
 		// Repack again:

libbeat/autodiscover/template/config_test.go

@@ -28,9 +28,12 @@ import (
 	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/libbeat/common/bus"
 	"github.com/elastic/beats/v7/libbeat/keystore"
+	"github.com/elastic/beats/v7/libbeat/logp"
 )
 
 func TestConfigsMapping(t *testing.T) {
+	logp.TestingSetup()
+
 	config, _ := common.NewConfigFrom(map[string]interface{}{
 		"correct": "config",
 	})
@@ -40,6 +43,13 @@ func TestConfigsMapping(t *testing.T) {
 		"hosts":   [1]string{"1.2.3.4:8080"},
 	})
 
+	const envValue = "valuefromenv"
+	configFromEnv, _ := common.NewConfigFrom(map[string]interface{}{
+		"correct": envValue,
+	})
+
+	os.Setenv("CONFIGS_MAPPING_TESTENV", envValue)
+
 	tests := []struct {
 		mapping  string
 		event    bus.Event
@@ -79,6 +89,16 @@ func TestConfigsMapping(t *testing.T) {
 			},
 			expected: []*common.Config{config},
 		},
+		// No condition, value from environment
+		{
+			mapping: `
+- config:
+    - correct: ${CONFIGS_MAPPING_TESTENV}`,
+			event: bus.Event{
+				"foo": 3,
+			},
+			expected: []*common.Config{configFromEnv},
+		},
 		// Match config and replace data.host and data.ports.<name> properly
 		{
 			mapping: `
@@ -111,6 +131,17 @@ func TestConfigsMapping(t *testing.T) {
 			},
 			expected: []*common.Config{configPorts},
 		},
+		// Missing variable, config is not generated
+		{
+			mapping: `
+- config:
+  - module: something
+    hosts: ["${not.exists.host}"]`,
+			event: bus.Event{
+				"host": "1.2.3.4",
+			},
+			expected: nil,
+		},
 	}
 
 	for _, test := range tests {