Upgrade go-ucfg to v0.8.1 release (#15937) (#16259)

* Update go-ucfg to v0.8.0.

* Update go-ucfg to 0.8.1.

* Fix tests with latest go-ucfg.

* Fix filebeat x-pack httpjson internal not to be required when the default value is zero.

* Fix system test_no_es_connection to ensure that it times out quickly within the wait_until range.

* Fix filebeat UDP input config to allow zero in the read buffer, allowing the default buffer to be used.

* Fix test_shutdown_wait_timeout to set a low timeout for logstash output.

* Fix test_shutdown_wait_timeout to fail instantly by trying to connect to localhost.

* Run govendor github.com/elastic/[email protected]

* Run make notice.

* Add to CHANGELOG.next.asciidoc.

* Perform recursive govendor on go-ucfg v0.8.1 and make notice.

(cherry picked from commit 73b8518)

CHANGELOG.next.asciidoc

@@ -21,7 +21,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Refactor metadata generator to support adding metadata across resources {pull}14875[14875]
 - Update to ECS 1.4.0. {pull}14844[14844]
 - The document id fields has been renamed from @metadata.id to @metadata._id {pull}15859[15859]
-
+- Variable substitution from environment variables is not longer supported. {pull}15937{15937}
 
 *Auditbeat*
 
@@ -74,6 +74,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix issue where TLS settings would be ignored when a forward proxy was in use. {pull}15516{15516}
 - Remove superfluous use of number_of_routing_shards setting from the default template. {pull}16038[16038]
 - Fix index names for indexing not always guaranteed to be lower case. {pull}16081[16081]
+- Upgrade go-ucfg to latest v0.8.1. {pull}15937{15937}
 
 *Auditbeat*
 

NOTICE.txt

@@ -1518,8 +1518,8 @@ Apache License 2.0
 
 --------------------------------------------------------------------
 Dependency: github.com/elastic/go-ucfg
-Version: v0.7.0
-Revision: 0539807037ce820e147797f051ff32b05f4f9288
+Version: v0.8.1
+Revision: 093a6898c440d2e5e93abf09850068c60428b2cd
 License type (autodetected): Apache-2.0
 ./vendor/github.com/elastic/go-ucfg/LICENSE:
 --------------------------------------------------------------------
@@ -6611,6 +6611,34 @@ License type (autodetected): Apache-2.0
 Apache License 2.0
 
 
+--------------------------------------------------------------------
+Dependency: gopkg.in/hjson/hjson-go.v3
+Revision: 8f44e1182a33737b57cb6855fe656ece998e9b3c
+License type (autodetected): MIT
+./vendor/gopkg.in/hjson/hjson-go.v3/LICENSE:
+--------------------------------------------------------------------
+MIT License
+
+Copyright (c) 2016, 2017 Christian Zangl
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
 --------------------------------------------------------------------
 Dependency: gopkg.in/inf.v0
 Revision: 3887ee99ecf07df5b447e9b00d9c0b2adaa9f3e4

filebeat/fileset/modules_test.go

@@ -162,6 +162,7 @@ func TestApplyOverrides(t *testing.T) {
 					"a":   "test",
 					"b.c": "test",
 				},
+				Input: map[string]interface{}{},
 			},
 			module:  "nginx",
 			fileset: "access",
@@ -177,6 +178,7 @@ func TestApplyOverrides(t *testing.T) {
 					"a": "test1",
 					"b": map[string]interface{}{"c": "test2"},
 				},
+				Input: map[string]interface{}{},
 			},
 		},
 		{
@@ -186,6 +188,7 @@ func TestApplyOverrides(t *testing.T) {
 				Var: map[string]interface{}{
 					"paths": []string{"/var/log/nginx"},
 				},
+				Input: map[string]interface{}{},
 			},
 			module:  "nginx",
 			fileset: "access",
@@ -201,6 +204,7 @@ func TestApplyOverrides(t *testing.T) {
 				Var: map[string]interface{}{
 					"paths": []interface{}{"/var/local/nginx/log"},
 				},
+				Input: map[string]interface{}{},
 			},
 		},
 		{
@@ -219,14 +223,17 @@ func TestApplyOverrides(t *testing.T) {
 				Input: map[string]interface{}{
 					"close_eof": true,
 				},
+				Var: map[string]interface{}{},
 			},
 		},
 	}
 
 	for _, test := range tests {
-		result, err := applyOverrides(&test.fcfg, test.module, test.fileset, test.overrides)
-		assert.NoError(t, err)
-		assert.Equal(t, &test.expected, result, test.name)
+		t.Run(test.name, func(t *testing.T) {
+			result, err := applyOverrides(&test.fcfg, test.module, test.fileset, test.overrides)
+			assert.NoError(t, err)
+			assert.Equal(t, &test.expected, result)
+		})
 	}
 }
 
@@ -314,9 +321,11 @@ func TestAppendWithoutDuplicates(t *testing.T) {
 	}
 
 	for _, test := range tests {
-		result, err := appendWithoutDuplicates(test.configs, test.modules)
-		assert.NoError(t, err, test.name)
-		assert.Equal(t, test.expected, result, test.name)
+		t.Run(test.name, func(t *testing.T) {
+			result, err := appendWithoutDuplicates(test.configs, test.modules)
+			assert.NoError(t, err)
+			assert.Equal(t, test.expected, result)
+		})
 	}
 }
 
@@ -338,6 +347,8 @@ func TestMcfgFromConfig(t *testing.T) {
 				Filesets: map[string]*FilesetConfig{
 					"error": {
 						Enabled: &falseVar,
+						Var:     map[string]interface{}{},
+						Input:   map[string]interface{}{},
 					},
 				},
 			},
@@ -355,20 +366,23 @@ func TestMcfgFromConfig(t *testing.T) {
 						Var: map[string]interface{}{
 							"test": false,
 						},
+						Input: map[string]interface{}{},
 					},
 				},
 			},
 		},
 	}
 
 	for _, test := range tests {
-		result, err := mcfgFromConfig(test.config)
-		assert.NoError(t, err, test.name)
-		assert.Equal(t, test.expected.Module, result.Module)
-		assert.Equal(t, len(test.expected.Filesets), len(result.Filesets))
-		for name, fileset := range test.expected.Filesets {
-			assert.Equal(t, fileset, result.Filesets[name])
-		}
+		t.Run(test.name, func(t *testing.T) {
+			result, err := mcfgFromConfig(test.config)
+			assert.NoError(t, err)
+			assert.Equal(t, test.expected.Module, result.Module)
+			assert.Equal(t, len(test.expected.Filesets), len(result.Filesets))
+			for name, fileset := range test.expected.Filesets {
+				assert.Equal(t, fileset, result.Filesets[name])
+			}
+		})
 	}
 }
 
@@ -428,7 +442,9 @@ func TestInterpretError(t *testing.T) {
 	}
 
 	for _, test := range tests {
-		errResult := interpretError(errors.New("test"), []byte(test.Input))
-		assert.Equal(t, errResult.Error(), test.Output, test.Test)
+		t.Run(test.Test, func(t *testing.T) {
+			errResult := interpretError(errors.New("test"), []byte(test.Input))
+			assert.Equal(t, errResult.Error(), test.Output, test.Test)
+		})
 	}
 }

filebeat/inputsource/udp/config.go

@@ -28,5 +28,5 @@ type Config struct {
 	Host           string           `config:"host"`
 	MaxMessageSize cfgtype.ByteSize `config:"max_message_size" validate:"positive,nonzero"`
 	Timeout        time.Duration    `config:"timeout"`
-	ReadBuffer     cfgtype.ByteSize `config:"read_buffer" validate:"positive,nonzero"`
+	ReadBuffer     cfgtype.ByteSize `config:"read_buffer" validate:"positive"`
 }

filebeat/tests/system/test_reload_modules.py

@@ -103,7 +103,7 @@ def test_no_es_connection(self):
             reload_path=self.working_dir + "/configs/*.yml",
             reload_type="modules",
             inputs=False,
-            elasticsearch={"host": 'errorhost:9201'}
+            elasticsearch={"host": 'errorhost:9201', "timeout": '1'}
         )
 
         proc = self.start_beat()

filebeat/tests/system/test_shutdown.py

@@ -81,8 +81,9 @@ def test_shutdown_wait_timeout(self):
 
         self.nasa_logs()
 
+        # Use 'localhost' so connection is refused instantly
         self.render_config_template(
-            logstash={"host": "does.not.exist:12345"},
+            logstash={"host": "localhost:12345", "timeout": 1},
             path=os.path.abspath(self.working_dir) + "/log/*",
             ignore_older="1h",
             shutdown_timeout="1s",

journalbeat/input/input_test.go

@@ -59,44 +59,46 @@ func TestProcessorsForInput(t *testing.T) {
 		},
 	}
 	for description, test := range testCases {
-		if test.event.Fields == nil {
-			test.event.Fields = common.MapStr{}
-		}
-		config, err := inputConfigFromString(test.configStr)
-		if err != nil {
-			t.Errorf("[%s] %v", description, err)
-			continue
-		}
-		processors, err := processorsForInput(test.beatInfo, config)
-		if err != nil {
-			t.Errorf("[%s] %v", description, err)
-			continue
-		}
-		processedEvent, err := processors.Run(&test.event)
-		// We don't check if err != nil, because we are testing the final outcome
-		// of running the processors, including when some of them fail.
-		if processedEvent == nil {
-			t.Errorf("[%s] Unexpected fatal error running processors: %v\n",
-				description, err)
-		}
-		for key, value := range test.expectedFields {
-			field, err := processedEvent.GetValue(key)
+		t.Run(description, func(t *testing.T) {
+			if test.event.Fields == nil {
+				test.event.Fields = common.MapStr{}
+			}
+			config, err := inputConfigFromString(test.configStr)
+			if err != nil {
+				t.Errorf("[%s] %v", description, err)
+				return
+			}
+			processors, err := processorsForInput(test.beatInfo, config)
 			if err != nil {
-				t.Errorf("[%s] Couldn't get field %s from event: %v", description, key, err)
-				continue
+				t.Errorf("[%s] %v", description, err)
+				return
 			}
-			assert.Equal(t, field, value)
-			fieldStr, ok := field.(string)
-			if !ok {
-				// Note that requiring a string here is just to simplify the test setup,
-				// not a requirement of the underlying api.
-				t.Errorf("[%s] Field [%s] should be a string", description, key)
-				continue
+			processedEvent, err := processors.Run(&test.event)
+			// We don't check if err != nil, because we are testing the final outcome
+			// of running the processors, including when some of them fail.
+			if processedEvent == nil {
+				t.Errorf("[%s] Unexpected fatal error running processors: %v\n",
+					description, err)
 			}
-			if fieldStr != value {
-				t.Errorf("[%s] Event field [%s]: expected [%s], got [%s]", description, key, value, fieldStr)
+			for key, value := range test.expectedFields {
+				field, err := processedEvent.GetValue(key)
+				if err != nil {
+					t.Errorf("[%s] Couldn't get field %s from event: %v", description, key, err)
+					return
+				}
+				assert.Equal(t, field, value)
+				fieldStr, ok := field.(string)
+				if !ok {
+					// Note that requiring a string here is just to simplify the test setup,
+					// not a requirement of the underlying api.
+					t.Errorf("[%s] Field [%s] should be a string", description, key)
+					return
+				}
+				if fieldStr != value {
+					t.Errorf("[%s] Event field [%s]: expected [%s], got [%s]", description, key, value, fieldStr)
+				}
 			}
-		}
+		})
 	}
 }
 
@@ -147,7 +149,7 @@ func (p *setRawIndex) String() string {
 // Helper function to convert from YML input string to an unpacked
 // Config
 func inputConfigFromString(s string) (Config, error) {
-	config := Config{}
+	config := DefaultConfig
 	cfg, err := common.NewConfigFrom(s)
 	if err != nil {
 		return config, err

libbeat/common/flags_test.go

@@ -91,7 +91,7 @@ func TestSettingsFlag(t *testing.T) {
 		in       []string
 		expected map[string]interface{}
 	}{
-		{nil, nil},
+		{nil, map[string]interface{}{}},
 		{[]string{"a=1"}, map[string]interface{}{"a": uint64(1)}},
 		{[]string{"a=1", "b=false"}, map[string]interface{}{"a": uint64(1), "b": false}},
 		{[]string{"a=1", "b"}, map[string]interface{}{"a": uint64(1), "b": true}},

libbeat/conditions/conditions.go

@@ -72,7 +72,7 @@ func NewCondition(config *Config) (Condition, error) {
 		condition, err = NewRangeCondition(config.Range.fields)
 	case config.HasFields != nil:
 		condition = NewHasFieldsCondition(config.HasFields)
-	case config.Network != nil:
+	case config.Network != nil && len(config.Network) > 0:
 		condition, err = NewNetworkCondition(config.Network)
 	case len(config.OR) > 0:
 		var conditionsList []Condition

libbeat/idxmgmt/std_test.go

@@ -288,7 +288,7 @@ func TestIndexManager_Setup(t *testing.T) {
 		if c.Settings.Index != nil {
 			c.Settings.Index = (map[string]interface{})(common.MapStr(c.Settings.Index).Clone())
 		}
-		if c.Settings.Index != nil {
+		if c.Settings.Source != nil {
 			c.Settings.Source = (map[string]interface{})(common.MapStr(c.Settings.Source).Clone())
 		}
 		return c
@@ -302,6 +302,12 @@ func TestIndexManager_Setup(t *testing.T) {
 			if err != nil {
 				panic(err)
 			}
+			if s.Settings.Index != nil && len(s.Settings.Index) == 0 {
+				s.Settings.Index = nil
+			}
+			if s.Settings.Source != nil && len(s.Settings.Source) == 0 {
+				s.Settings.Source = nil
+			}
 		}
 		return &s
 	}

libbeat/keystore/keystore.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/elastic/beats/libbeat/common"
 	ucfg "github.com/elastic/go-ucfg"
+	"github.com/elastic/go-ucfg/parse"
 )
 
 var (
@@ -91,7 +92,7 @@ func Factory(cfg *common.Config, defaultPath string) (Keystore, error) {
 }
 
 // ResolverFromConfig create a resolver from a configuration.
-func ResolverFromConfig(cfg *common.Config, dataPath string) (func(string) (string, error), error) {
+func ResolverFromConfig(cfg *common.Config, dataPath string) (func(string) (string, parse.Config, error), error) {
 	keystore, err := Factory(cfg, dataPath)
 
 	if err != nil {
@@ -102,24 +103,24 @@ func ResolverFromConfig(cfg *common.Config, dataPath string) (func(string) (stri
 }
 
 // ResolverWrap wrap a config resolver around an existing keystore.
-func ResolverWrap(keystore Keystore) func(string) (string, error) {
-	return func(keyName string) (string, error) {
+func ResolverWrap(keystore Keystore) func(string) (string, parse.Config, error) {
+	return func(keyName string) (string, parse.Config, error) {
 		key, err := keystore.Retrieve(keyName)
 
 		if err != nil {
 			// If we cannot find the key, its a non fatal error
 			// and we pass to other resolver.
 			if err == ErrKeyDoesntExists {
-				return "", ucfg.ErrMissing
+				return "", parse.DefaultConfig, ucfg.ErrMissing
 			}
-			return "", err
+			return "", parse.DefaultConfig, err
 		}
 
 		v, err := key.Get()
 		if err != nil {
-			return "", err
+			return "", parse.DefaultConfig, err
 		}
 
-		return string(v), nil
+		return string(v), parse.DefaultConfig, nil
 	}
 }