Upgrade elastic-agent-libs to pick up fix to keystore package (#31694)

Fixes loading values from the keystore that include commas. * Update test to include comma in keystore value. * Upgrade elastic-agent-libs and go-ucfg. * Fix heartbeat unit tests broken in go-ucfg 0.8.4+ * Fix metricbeat unit tests broken in go-ucfg 0.8.4+ * Fix metricbeat lint errors in autodiscover test.
elastic · May 20, 2022 · 0022ea4 · 0022ea4
1 parent 9605c28
commit 0022ea4
Show file tree

Hide file tree

Showing 7 changed files with 48 additions and 20 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -39,6 +39,7 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff]
 - Fix group write permissions on runtime directories. {pull}30869[30869]
 - Store syslog version as string. {pull}31446[31446]
 - Accept XML that declares non-UTF-8 encoding to allow decode_xml and decode_xml_wineventlog decoding of incorrectly annotated documents. {issue}31395[31395] {pull}31546[31546]
+- Allow loading secrets that contain commas from the keystore {pull}31694{pull}.
 
 *Auditbeat*
 

diff --git a/NOTICE.txt b/NOTICE.txt
@@ -6281,11 +6281,11 @@ SOFTWARE
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/elastic-agent-libs
-Version: v0.2.3
+Version: v0.2.4
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].3/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].4/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004
@@ -8033,11 +8033,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected]
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/go-ucfg
-Version: v0.8.3
+Version: v0.8.5
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].3/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].5/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004

diff --git a/go.mod b/go.mod
@@ -61,7 +61,7 @@ require (
 	github.com/elastic/go-seccomp-bpf v1.2.0
 	github.com/elastic/go-structform v0.0.9
 	github.com/elastic/go-sysinfo v1.7.1
-	github.com/elastic/go-ucfg v0.8.4
+	github.com/elastic/go-ucfg v0.8.5
 	github.com/elastic/go-windows v1.0.1
 	github.com/elastic/gosigar v0.14.2
 	github.com/fatih/color v1.13.0
@@ -161,7 +161,7 @@ require (
 require (
 	github.com/elastic/bayeux v1.0.5
 	github.com/elastic/elastic-agent-autodiscover v0.1.1
-	github.com/elastic/elastic-agent-libs v0.2.3
+	github.com/elastic/elastic-agent-libs v0.2.4
 	github.com/shirou/gopsutil/v3 v3.21.12
 	go.elastic.co/apm/module/apmelasticsearch/v2 v2.0.0
 	go.elastic.co/apm/module/apmhttp/v2 v2.0.0
@@ -309,7 +309,6 @@ replace (
 	github.com/docker/go-plugins-helpers => github.com/elastic/go-plugins-helpers v0.0.0-20200207104224-bdf17607b79f
 	github.com/dop251/goja => github.com/andrewkroh/goja v0.0.0-20190128172624-dd2ac4456e20
 	github.com/dop251/goja_nodejs => github.com/dop251/goja_nodejs v0.0.0-20171011081505-adff31b136e6
-	github.com/elastic/go-ucfg => github.com/elastic/go-ucfg v0.8.3 // starting with v0.8.4 we need to re-write a lot of tests
 	github.com/fsnotify/fsevents => github.com/elastic/fsevents v0.0.0-20181029231046-e1d381a4d270
 	github.com/fsnotify/fsnotify => github.com/adriansr/fsnotify v1.4.8-0.20211018144411-a81f2b630e7c
 	github.com/godror/godror => github.com/godror/godror v0.10.4 // updating to v0.24.2 caused a breaking change

diff --git a/go.sum b/go.sum
@@ -536,8 +536,8 @@ github.com/elastic/elastic-agent-autodiscover v0.1.1/go.mod h1:Gg1fsQI+rVms9FJ2D
 github.com/elastic/elastic-agent-client/v7 v7.0.0-20210727140539-f0905d9377f6 h1:nFvXHBjYK3e9+xF0WKDeAKK4aOO51uC28s+L9rBmilo=
 github.com/elastic/elastic-agent-client/v7 v7.0.0-20210727140539-f0905d9377f6/go.mod h1:uh/Gj9a0XEbYoM4NYz4LvaBVARz3QXLmlNjsrKY9fTc=
 github.com/elastic/elastic-agent-libs v0.0.0-20220303160015-5b4e674da3dd/go.mod h1://82M1l73IHx0wDbS2Tzkq6Fx9fkmytS1KgkIyzvNTM=
-github.com/elastic/elastic-agent-libs v0.2.3 h1:GY8M0fxOs/GBY2nIB+JOB91aoD72S87iEcm2qVGFUqI=
-github.com/elastic/elastic-agent-libs v0.2.3/go.mod h1:1xDLBhIqBIjhJ7lr2s+xRFFkQHpitSp8q2zzv1Dqg+s=
+github.com/elastic/elastic-agent-libs v0.2.4 h1:TOy+vild5MSkn/eTOwrnffAeAntq4GiLpkvWe+uNVms=
+github.com/elastic/elastic-agent-libs v0.2.4/go.mod h1:eUiaofWIVdxVOAR4ICGxn2wbFByhrnmR6/kppwYq3qI=
 github.com/elastic/fsevents v0.0.0-20181029231046-e1d381a4d270 h1:cWPqxlPtir4RoQVCpGSRXmLqjEHpJKbR60rxh1nQZY4=
 github.com/elastic/fsevents v0.0.0-20181029231046-e1d381a4d270/go.mod h1:Msl1pdboCbArMF/nSCDUXgQuWTeoMmE/z8607X+k7ng=
 github.com/elastic/glog v1.0.1-0.20210831205241-7d8b5c89dfc4 h1:ViJxdtOsHeO+SWVekzM82fYHH1xnvZ8CvGPXZj+G4YI=
@@ -562,8 +562,10 @@ github.com/elastic/go-structform v0.0.9 h1:HpcS7xljL4kSyUfDJ8cXTJC6rU5ChL1wYb6cx
 github.com/elastic/go-structform v0.0.9/go.mod h1:CZWf9aIRYY5SuKSmOhtXScE5uQiLZNqAFnwKR4OrIM4=
 github.com/elastic/go-sysinfo v1.7.1 h1:Wx4DSARcKLllpKT2TnFVdSUJOsybqMYCNQZq1/wO+s0=
 github.com/elastic/go-sysinfo v1.7.1/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0=
-github.com/elastic/go-ucfg v0.8.3 h1:leywnFjzr2QneZZWhE6uWd+QN/UpP0sdJRHYyuFvkeo=
-github.com/elastic/go-ucfg v0.8.3/go.mod h1:iaiY0NBIYeasNgycLyTvhJftQlQEUO2hpF+FX0JKxzo=
+github.com/elastic/go-ucfg v0.7.0/go.mod h1:iaiY0NBIYeasNgycLyTvhJftQlQEUO2hpF+FX0JKxzo=
+github.com/elastic/go-ucfg v0.8.4/go.mod h1:4E8mPOLSUV9hQ7sgLEJ4bvt0KhMuDJa8joDT2QGAEKA=
+github.com/elastic/go-ucfg v0.8.5 h1:4GB/rMpuh7qTcSFaxJUk97a/JyvFzhi6t+kaskTTLdM=
+github.com/elastic/go-ucfg v0.8.5/go.mod h1:4E8mPOLSUV9hQ7sgLEJ4bvt0KhMuDJa8joDT2QGAEKA=
 github.com/elastic/go-windows v1.0.0/go.mod h1:TsU0Nrp7/y3+VwE82FoZF8gC/XFg/Elz6CcloAxnPgU=
 github.com/elastic/go-windows v1.0.1 h1:AlYZOldA+UJ0/2nBuqWdo90GFCgG9xuyw9SYzGUtJm0=
 github.com/elastic/go-windows v1.0.1/go.mod h1:FoVvqWSun28vaDQPbj2Elfc0JahhPB7WQEGa3c814Ss=
@@ -2389,6 +2391,7 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy
 gopkg.in/fsnotify/fsnotify.v1 v1.4.7/go.mod h1:Fyux9zXlo4rWoMSIzpn9fDAYjalPqJ/K1qJ27s+7ltE=
 gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o=
 gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
+gopkg.in/hjson/hjson-go.v3 v3.0.1/go.mod h1:X6zrTSVeImfwfZLfgQdInl9mWjqPqgH90jom9nym/lw=
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=

diff --git a/heartbeat/autodiscover/builder/hints/monitors_test.go b/heartbeat/autodiscover/builder/hints/monitors_test.go
@@ -82,6 +82,7 @@ func TestGenerateHints(t *testing.T) {
 			result: mapstr.M{
 				"schedule": "@every 5s",
 				"type":     "icmp",
+				"hosts":    []interface{}{},
 			},
 		},
 		{

diff --git a/libbeat/tests/system/test_keystore.py b/libbeat/tests/system/test_keystore.py
@@ -23,7 +23,8 @@ def test_keystore_with_present_key(self):
         """
 
         key = "mysecretpath"
-        secret = path.join(self.working_dir, "thisisultrasecretpath")
+        # Include a comma in the path, regression test for https://github.com/elastic/beats/issues/29789
+        secret = path.join(self.working_dir, "thisisultra,secretpath")
 
         self.render_config_template("mockbeat",
                                     keystore_path=self.keystore_path,

diff --git a/metricbeat/autodiscover/builder/hints/metrics_test.go b/metricbeat/autodiscover/builder/hints/metrics_test.go
@@ -108,6 +108,7 @@ func TestGenerateHints(t *testing.T) {
 					"period":     "1m",
 					"enabled":    true,
 					"hosts":      []interface{}{"1.2.3.4:9090"},
+					"processors": []interface{}{},
 				},
 			},
 		},
@@ -132,6 +133,7 @@ func TestGenerateHints(t *testing.T) {
 					"period":     "1m",
 					"enabled":    true,
 					"hosts":      []interface{}{"1.2.3.4:9090"},
+					"processors": []interface{}{},
 				},
 			},
 		},
@@ -153,6 +155,7 @@ func TestGenerateHints(t *testing.T) {
 					"timeout":    "3s",
 					"period":     "1m",
 					"enabled":    true,
+					"processors": []interface{}{},
 				},
 			},
 		},
@@ -175,6 +178,7 @@ func TestGenerateHints(t *testing.T) {
 					"timeout":    "3s",
 					"period":     "1m",
 					"enabled":    true,
+					"processors": []interface{}{},
 				},
 			},
 		},
@@ -196,6 +200,7 @@ func TestGenerateHints(t *testing.T) {
 					"timeout":    "3s",
 					"period":     "1m",
 					"enabled":    true,
+					"processors": []interface{}{},
 				},
 			},
 		},
@@ -243,6 +248,7 @@ func TestGenerateHints(t *testing.T) {
 					"period":     "1m",
 					"enabled":    true,
 					"hosts":      []interface{}{"1.2.3.4:9090"},
+					"processors": []interface{}{},
 				},
 			},
 		},
@@ -321,6 +327,7 @@ func TestGenerateHints(t *testing.T) {
 					"timeout":    "3s",
 					"period":     "1m",
 					"enabled":    true,
+					"processors": []interface{}{},
 				},
 			},
 		},
@@ -365,6 +372,7 @@ func TestGenerateHints(t *testing.T) {
 					"timeout":    "3s",
 					"period":     "1m",
 					"enabled":    true,
+					"processors": []interface{}{},
 				},
 			},
 		},
@@ -391,6 +399,7 @@ func TestGenerateHints(t *testing.T) {
 					"timeout":    "3s",
 					"period":     "1m",
 					"enabled":    true,
+					"processors": []interface{}{},
 				},
 			},
 		},
@@ -417,6 +426,7 @@ func TestGenerateHints(t *testing.T) {
 					"timeout":    "3s",
 					"period":     "1m",
 					"enabled":    true,
+					"processors": []interface{}{},
 				},
 			},
 		},
@@ -444,11 +454,12 @@ func TestGenerateHints(t *testing.T) {
 					"timeout":    "3s",
 					"period":     "1m",
 					"enabled":    true,
+					"processors": []interface{}{},
 				},
 			},
 		},
 		{
-			message: "Module with mutliple sets of hints must return the right configs",
+			message: "Module with multiple sets of hints must return the right configs",
 			event: bus.Event{
 				"host": "1.2.3.4",
 				"hints": mapstr.M{
@@ -476,6 +487,7 @@ func TestGenerateHints(t *testing.T) {
 					"period":     "1m",
 					"enabled":    true,
 					"hosts":      []interface{}{"1.2.3.4:9090"},
+					"processors": []interface{}{},
 				},
 				{
 					"module":     "mockmoduledefaults",
@@ -485,6 +497,7 @@ func TestGenerateHints(t *testing.T) {
 					"period":     "1m",
 					"enabled":    true,
 					"hosts":      []interface{}{"1.2.3.4:9090/fake"},
+					"processors": []interface{}{},
 				},
 			},
 		},
@@ -510,6 +523,7 @@ func TestGenerateHints(t *testing.T) {
 					"period":     "1m",
 					"enabled":    true,
 					"hosts":      []interface{}{"1.2.3.4:9090"},
+					"processors": []interface{}{},
 				},
 				{
 					"module":     "mockmoduledefaults",
@@ -519,6 +533,7 @@ func TestGenerateHints(t *testing.T) {
 					"period":     "1m",
 					"enabled":    true,
 					"hosts":      []interface{}{"1.2.3.4:9091"},
+					"processors": []interface{}{},
 				},
 			},
 		},
@@ -545,6 +560,7 @@ func TestGenerateHints(t *testing.T) {
 					"period":     "1m",
 					"enabled":    true,
 					"hosts":      []interface{}{"1.2.3.4:9091"},
+					"processors": []interface{}{},
 				},
 			},
 		},
@@ -574,6 +590,7 @@ func TestGenerateHints(t *testing.T) {
 					"period":     "1m",
 					"enabled":    true,
 					"hosts":      []interface{}{"1.2.3.4:9090"},
+					"processors": []interface{}{},
 					"metrics_filters": map[string]interface{}{
 						"exclude": []interface{}{"foo", "bar"},
 						"include": []interface{}{"xxx", "yyy"},
@@ -615,7 +632,9 @@ func TestGenerateHints(t *testing.T) {
 				if msets, ok := v.([]interface{}); ok {
 					metricsets := make([]string, len(msets))
 					for i, v := range msets {
-						metricsets[i] = v.(string)
+						var ok bool
+						metricsets[i], ok = v.(string)
+						assert.Truef(t, ok, "Failed to convert metricset: %d=%v", i, metricsets[i])
 					}
 					sort.Strings(metricsets)
 					config["metricsets"] = metricsets
@@ -632,7 +651,7 @@ func TestGenerateHintsDoesNotAccessGlobalKeystore(t *testing.T) {
 	path := getTemporaryKeystoreFile()
 	defer os.Remove(path)
 	// store the secret
-	keystore := createAnExistingKeystore(path, "stored_secret")
+	keystore := createAnExistingKeystore(t, path, "stored_secret")
 	os.Setenv("PASSWORD", "env_secret")
 
 	tests := []struct {
@@ -664,6 +683,7 @@ func TestGenerateHintsDoesNotAccessGlobalKeystore(t *testing.T) {
 				"period":     "1m",
 				"enabled":    true,
 				"password":   "env_secret",
+				"processors": []interface{}{},
 			},
 		},
 	}
@@ -688,7 +708,9 @@ func TestGenerateHintsDoesNotAccessGlobalKeystore(t *testing.T) {
 				if msets, ok := v.([]interface{}); ok {
 					metricsets := make([]string, len(msets))
 					for i, v := range msets {
-						metricsets[i] = v.(string)
+						var ok bool
+						metricsets[i], ok = v.(string)
+						assert.Truef(t, ok, "Failed to convert metricset: %d=%v", i, metricsets[i])
 					}
 					sort.Strings(metricsets)
 					config["metricsets"] = metricsets
@@ -709,7 +731,7 @@ func NewMockMetricSet(base mb.BaseMetricSet) (mb.MetricSet, error) {
 	return &MockMetricSet{}, nil
 }
 
-func (ms *MockMetricSet) Fetch(report mb.Reporter) {
+func (ms *MockMetricSet) Fetch(report mb.ReporterV2) {
 
 }
 
@@ -723,7 +745,8 @@ func NewMockPrometheus(base mb.BaseMetricSet) (mb.MetricSet, error) {
 
 // create a keystore with an existing key
 // `PASSWORD` with the value of `secret` variable.
-func createAnExistingKeystore(path string, secret string) keystore.Keystore {
+func createAnExistingKeystore(t *testing.T, path string, secret string) keystore.Keystore {
+	t.Helper()
 	keyStore, err := keystore.NewFileKeystore(path)
 	// Fail fast in the test suite
 	if err != nil {
@@ -735,8 +758,8 @@ func createAnExistingKeystore(path string, secret string) keystore.Keystore {
 		panic(err)
 	}
 
-	writableKeystore.Store("PASSWORD", []byte(secret))
-	writableKeystore.Save()
+	assert.NoError(t, writableKeystore.Store("PASSWORD", []byte(secret)))
+	assert.NoError(t, writableKeystore.Save())
 	return keyStore
 }