Report FQDN on host.name enabled by a feature flag (#34456)

* wip

* wip

* wip

* something is broken

* debug logs

* avoid panics

* wip broken

* wip - perhaps fixed double configuring features

* using triggers to know what changed

* it works, but the agent-client is sending feature changes every time

* debug logs

* more debug logs

* wip

* it works

* remove debug logs

* fix after rebase

* update elastic-agent-client and make notice

* license headres

* Add TODO for Winlogbeat FQDN implementation

* Removing unnecesary code

* Use RWMutex for greater read efficiency

* Move mutex inside struct

* Better error handling

* Temporarily updating dependency

* Always store FQDN in Beat info

* Set hostname in monitoring registries after FQDN config has been evaluated

* Pass down FQDN flag from add_host_metadata processor

* Better error handling

* Add temporary notice override

* Changing log level

* Update dependency on elastic-agent-system-metrics

* Running go mod tidy

* Update elastic-agent-system-metrics dependency version

* Add feature flags configuration section to reference yml

* Bump up version of elastic-agent-client depdendency

* Updating reference file

* Updating reference files

* Move feature flags setting into same method where log level setting is handled

* Updating docs

* Adding CHANGELOG entry

* Remove replace directive for go-sysinfo

* Running go mod tidy

* Remove replace directive for elastic-agent-client dependency

* Remove personal repos from overrides.json

* Updating NOTICE

* Bump up version for elastic-agent-client dependency

* Update NOTICE

* Removing noisy debug logging

* Removing commented out code that's not needed

* Fixing error message

* Make default flag setting more explicit

* Adding config with no features to unit test

* Add helper method to beat.Info for returning FQDN-aware hostname

* Adding license header to test file

* Remove use of errors package

* Check error from data.Put

* Expire add_host_metadata cache immediately on FQDN feature flag value change

* Fix casing in error message

* Have callback registration function accept ID

* Fixing go formatting

* Account for feature flags in TestManagerV2 test

* Update logging for unit changes

* Remove TODO from wineventlog code

* Updating dependency

* Include shared file in docs

* Bumping dependency on elastic-agent-client

* Include features config in diagnostics output

* Update comment

* Updating NOTICE

* Re-formatting

* Running make update

---------

Co-authored-by: Shaunak Kashyap <[email protected]>

CHANGELOG.next.asciidoc

@@ -181,7 +181,7 @@ automatic splitting at root level, if root level element is an array. {pull}3415
 *Affecting all Beats*
 
 - Added append Processor which will append concrete values or values from a field to target. {issue}29934[29934] {pull}33364[33364]
-
+- Allow users to enable features via configuration, starting with the FQDN reporting feature. {issue}1070[1070] {pull}34456[34456]
 
 *Auditbeat*
 

auditbeat/auditbeat.reference.yml

@@ -1740,3 +1740,10 @@ logging.files:
 # This allows to enable 6.7 migration aliases
 #migration.6_to_7.enabled: false
 
+# =============================== Feature Flags ================================
+
+# Enable and configure feature flags.
+#features:
+#  fqdn:
+#    enabled: true
+

auditbeat/docs/configuring-howto.asciidoc

@@ -25,6 +25,7 @@ include::{libbeat-dir}/shared/configuring-intro.asciidoc[]
 * <<http-endpoint>>
 * <<regexp-support>>
 * <<configuration-instrumentation>>
+* <<configuration-feature-flags>>
 * <<{beatname_lc}-reference-yml>>
 
 After changing configuration settings, you need to restart {beatname_uc} to
@@ -64,4 +65,6 @@ include::{libbeat-dir}/regexp.asciidoc[]
 
 include::{libbeat-dir}/shared-instrumentation.asciidoc[]
 
+include::{libbeat-dir}/shared-feature-flags.asciidoc[]
+
 include::{libbeat-dir}/reference-yml.asciidoc[]

dev-tools/notice/overrides.json

@@ -1,4 +1,5 @@
 {"name": "github.com/elastic/elastic-agent-client/v7", "licenceType": "Elastic"}
+{"name": "github.com/ycombinator/elastic-agent-client/v7", "licenceType": "Elastic"}
 {"name": "github.com/elastic/elastic-agent-shipper-client", "licenceType": "Elastic"}
 {"name": "github.com/gorhill/cronexpr", "licenceType": "Apache-2.0", "licenceFile":"APLv2"}
 {"name": "github.com/hashicorp/cronexpr", "licenceType": "Apache-2.0", "licenceFile":"APLv2"}

filebeat/beater/filebeat.go

@@ -190,9 +190,9 @@ func (fb *Filebeat) setupPipelineLoaderCallback(b *beat.Beat) error {
 		modulesFactory := fileset.NewSetupFactory(b.Info, pipelineLoaderFactory, enableAllFilesets)
 		if fb.config.ConfigModules.Enabled() {
 			if enableAllFilesets {
-				//All module configs need to be loaded to enable all the filesets
-				//contained in the modules.  The default glob just loads the enabled
-				//ones.  Switching the glob pattern from *.yml to * achieves this.
+				// All module configs need to be loaded to enable all the filesets
+				// contained in the modules.  The default glob just loads the enabled
+				// ones.  Switching the glob pattern from *.yml to * achieves this.
 				origPath, _ := fb.config.ConfigModules.String("path", -1)
 				newPath := strings.TrimSuffix(origPath, ".yml")
 				_ = fb.config.ConfigModules.SetString("path", -1, newPath)

filebeat/config/config.go

@@ -119,7 +119,7 @@ func mergeConfigFiles(configFiles []string, config *Config) error {
 	return nil
 }
 
-// Fetches and merges all config files given by configDir. All are put into one config object
+// FetchConfigs fetches and merges all config files given by configDir. All are put into one config object
 func (config *Config) FetchConfigs() error {
 	configDir := config.ConfigDir
 

filebeat/docs/configuring-howto.asciidoc

@@ -28,6 +28,7 @@ include::{libbeat-dir}/shared/configuring-intro.asciidoc[]
 * <<http-endpoint>>
 * <<regexp-support>>
 * <<configuration-instrumentation>>
+* <<configuration-feature-flags>>
 * <<{beatname_lc}-reference-yml>>
 
 --
@@ -73,4 +74,6 @@ include::{libbeat-dir}/regexp.asciidoc[]
 
 include::{libbeat-dir}/shared-instrumentation.asciidoc[]
 
+include::{libbeat-dir}/shared-feature-flags.asciidoc[]
+
 include::{libbeat-dir}/reference-yml.asciidoc[]

filebeat/filebeat.reference.yml

@@ -2823,3 +2823,10 @@ logging.files:
 # This allows to enable 6.7 migration aliases
 #migration.6_to_7.enabled: false
 
+# =============================== Feature Flags ================================
+
+# Enable and configure feature flags.
+#features:
+#  fqdn:
+#    enabled: true
+

go.mod

@@ -17,7 +17,7 @@ require (
 	github.com/Azure/go-autorest/autorest/azure/auth v0.4.2
 	github.com/Azure/go-autorest/autorest/date v0.3.0
 	github.com/Masterminds/semver v1.5.0
-	github.com/Microsoft/go-winio v0.5.2
+	github.com/Microsoft/go-winio v0.6.0
 	github.com/PaesslerAG/gval v1.0.0
 	github.com/PaesslerAG/jsonpath v0.1.1
 	github.com/Shopify/sarama v1.27.0
@@ -60,17 +60,17 @@ require (
 	github.com/devigned/tab v0.1.2-0.20190607222403-0c15cf42f9a2 // indirect
 	github.com/dgraph-io/badger/v3 v3.2103.1
 	github.com/digitalocean/go-libvirt v0.0.0-20180301200012-6075ea3c39a1
-	github.com/docker/docker v20.10.12+incompatible
+	github.com/docker/docker v20.10.22+incompatible
 	github.com/docker/go-connections v0.4.0
 	github.com/docker/go-plugins-helpers v0.0.0-20181025120712-1e6269c305b8
-	github.com/docker/go-units v0.4.0
+	github.com/docker/go-units v0.5.0
 	github.com/dolmen-go/contextio v0.0.0-20200217195037-68fc5150bcd5
 	github.com/dop251/goja v0.0.0-20200831102558-9af81ddcf0e1
 	github.com/dop251/goja_nodejs v0.0.0-20171011081505-adff31b136e6
 	github.com/dustin/go-humanize v1.0.0
 	github.com/eapache/go-resiliency v1.2.0
 	github.com/eclipse/paho.mqtt.golang v1.3.5
-	github.com/elastic/elastic-agent-client/v7 v7.0.3
+	github.com/elastic/elastic-agent-client/v7 v7.0.3-0.20230315204017-166fd1fd746f
 	github.com/elastic/go-concert v0.2.0
 	github.com/elastic/go-libaudit/v2 v2.3.2
 	github.com/elastic/go-licenser v0.4.0
@@ -79,7 +79,7 @@ require (
 	github.com/elastic/go-perf v0.0.0-20191212140718-9c656876f595
 	github.com/elastic/go-seccomp-bpf v1.3.0
 	github.com/elastic/go-structform v0.0.10
-	github.com/elastic/go-sysinfo v1.9.0
+	github.com/elastic/go-sysinfo v1.9.1-0.20230215152520-f544eca983fb
 	github.com/elastic/go-ucfg v0.8.6
 	github.com/elastic/gosigar v0.14.2
 	github.com/fatih/color v1.13.0
@@ -133,7 +133,7 @@ require (
 	github.com/pmezard/go-difflib v1.0.0
 	github.com/prometheus/client_model v0.2.0
 	github.com/prometheus/common v0.30.0
-	github.com/prometheus/procfs v0.8.0
+	github.com/prometheus/procfs v0.9.0
 	github.com/prometheus/prometheus v1.8.2-0.20210701133801-b0944590a1c9
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475
 	github.com/samuel/go-parser v0.0.0-20130731160455-ca8abbf65d0e // indirect
@@ -160,9 +160,9 @@ require (
 	golang.org/x/net v0.2.0
 	golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783
 	golang.org/x/sync v0.1.0
-	golang.org/x/sys v0.2.0
+	golang.org/x/sys v0.6.0
 	golang.org/x/text v0.4.0
-	golang.org/x/time v0.0.0-20220609170525-579cf78fd858
+	golang.org/x/time v0.3.0
 	golang.org/x/tools v0.1.12
 	google.golang.org/api v0.100.0
 	google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c
@@ -195,7 +195,7 @@ require (
 	github.com/elastic/elastic-agent-autodiscover v0.5.0
 	github.com/elastic/elastic-agent-libs v0.3.3
 	github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3
-	github.com/elastic/elastic-agent-system-metrics v0.4.6
+	github.com/elastic/elastic-agent-system-metrics v0.4.6-0.20230308003052-ba171438211e
 	github.com/elastic/go-elasticsearch/v8 v8.2.0
 	github.com/elastic/mito v0.0.0-20230302005114-1dda06e81678
 	github.com/elastic/toutoumomoma v0.0.0-20221026030040-594ef30cb640
@@ -246,7 +246,6 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/sso v1.11.10 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash v1.1.0 // indirect
-	github.com/containerd/containerd v1.5.13 // indirect
 	github.com/cyphar/filepath-securejoin v0.2.3 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dgraph-io/ristretto v0.1.0 // indirect
@@ -316,7 +315,7 @@ require (
 	github.com/santhosh-tekuri/jsonschema v1.2.4 // indirect
 	github.com/sergi/go-diff v1.3.1 // indirect
 	github.com/shirou/gopsutil v3.21.11+incompatible // indirect
-	github.com/sirupsen/logrus v1.8.1 // indirect
+	github.com/sirupsen/logrus v1.9.0 // indirect
 	github.com/stoewer/go-strcase v1.2.0 // indirect
 	github.com/stretchr/objx v0.5.0 // indirect
 	github.com/tklauser/go-sysconf v0.3.9 // indirect
@@ -351,7 +350,7 @@ require (
 	github.com/docker/distribution v2.8.1+incompatible // indirect
 	github.com/godbus/dbus/v5 v5.0.6 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
-	github.com/moby/term v0.0.0-20210610120745-9d4ed1856297 // indirect
+	github.com/moby/term v0.0.0-20221205130635-1aeaba878587 // indirect
 	github.com/yuin/gopher-lua v0.0.0-20170403160031-b402f3114ec7 // indirect
 	gopkg.in/jcmturner/rpc.v1 v1.1.0 // indirect
 )