From 5e6e3587a436a16ff5899c6458d3bd5750a83cc6 Mon Sep 17 00:00:00 2001 From: michel-laterman Date: Wed, 18 Aug 2021 15:13:40 -0700 Subject: [PATCH 01/11] Add a header round tripper option to httpcommon Add a new TransportOption to add a set of headers to each HTTP request through a custom http.RoundTripper. It will set the passed headers to each request if the header key is not present. Use the new transport option to add User-Agent headers to heartbeat, metricbeat, and filebeat. --- CHANGELOG.next.asciidoc | 1 + heartbeat/monitors/active/http/http.go | 4 +++ heartbeat/monitors/active/http/task.go | 6 ----- .../common/transport/httpcommon/httpcommon.go | 26 +++++++++++++++++++ metricbeat/helper/http.go | 4 +++ metricbeat/helper/http_test.go | 25 ++++++++++++++++++ x-pack/filebeat/input/httpjson/input.go | 1 + x-pack/filebeat/input/httpjson/requester.go | 1 - 8 files changed, 61 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index b83958387c3..26f153a7cd1 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -103,6 +103,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add state_job metricset to Kubernetes module{pull}26479[26479] - Bump AWS SDK version to v0.24.0 for WebIdentity authentication flow {issue}19393[19393] {pull}27126[27126] - Add Linux pressure metricset {pull}27355[27355] +- Add User-Agent header to HTTP requests. {issue}18160[18160] *Packetbeat* diff --git a/heartbeat/monitors/active/http/http.go b/heartbeat/monitors/active/http/http.go index c02ae25943e..2532a3d1c7a 100644 --- a/heartbeat/monitors/active/http/http.go +++ b/heartbeat/monitors/active/http/http.go @@ -29,6 +29,7 @@ import ( "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/transport/httpcommon" "github.com/elastic/beats/v7/libbeat/common/transport/tlscommon" + "github.com/elastic/beats/v7/libbeat/common/useragent" "github.com/elastic/beats/v7/libbeat/logp" ) @@ -38,6 +39,8 @@ func init() { var debugf = logp.MakeDebug("http") +var userAgent = useragent.UserAgent("Heartbeat") + // Create makes a new HTTP monitor func create( name string, @@ -128,5 +131,6 @@ func newRoundTripper(config *Config) (http.RoundTripper, error) { httpcommon.WithKeepaliveSettings{ Disable: true, }, + httpcommon.WithHeaderRoundTripper(map[string]string{"User-Agent": userAgent}), ) } diff --git a/heartbeat/monitors/active/http/task.go b/heartbeat/monitors/active/http/task.go index 630283c8bab..8648b489feb 100644 --- a/heartbeat/monitors/active/http/task.go +++ b/heartbeat/monitors/active/http/task.go @@ -42,11 +42,8 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/transport/tlscommon" - "github.com/elastic/beats/v7/libbeat/common/useragent" ) -var userAgent = useragent.UserAgent("Heartbeat") - func newHTTPMonitorHostJob( addr string, config *Config, @@ -206,9 +203,6 @@ func buildRequest(addr string, config *Config, enc contentEncoder) (*http.Reques request.Header.Add(k, v) } - if ua := request.Header.Get("User-Agent"); ua == "" { - request.Header.Set("User-Agent", userAgent) - } if enc != nil { enc.AddHeaders(&request.Header) diff --git a/libbeat/common/transport/httpcommon/httpcommon.go b/libbeat/common/transport/httpcommon/httpcommon.go index 7d4e9fb595a..8dfeb104a9a 100644 --- a/libbeat/common/transport/httpcommon/httpcommon.go +++ b/libbeat/common/transport/httpcommon/httpcommon.go @@ -134,6 +134,20 @@ type extraOptionFunc func(*extraSettings) func (extraOptionFunc) sealTransportOption() {} func (fn extraOptionFunc) applyExtra(s *extraSettings) { fn(s) } +type headerRoundTripper struct { + headers map[string]string + rt http.RoundTripper +} + +func (rt *headerRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + for k, v := range rt.headers { + if len(req.Header.Get(k)) == 0 { + req.Header.Set(k, v) + } + } + return rt.rt.RoundTrip(req) +} + // DefaultHTTPTransportSettings returns the default HTTP transport setting. func DefaultHTTPTransportSettings() HTTPTransportSettings { return HTTPTransportSettings{ @@ -373,6 +387,18 @@ func WithAPMHTTPInstrumentation() TransportOption { return withAPMHTTPRountTripper } +func withHeaderRoundTripper(rt http.RoundTripper, headers map[string]string) http.RoundTripper { + return &headerRoundTripper{headers, rt} +} + +// WithHeaderRoundTripper instuments the HTTP client via a custom http.RoundTripper. +// This RoundTripper will add headers to each request if the key is not present. +func WithHeaderRoundTripper(headers map[string]string) TransportOption { + return WithModRoundtripper(func(rt http.RoundTripper) http.RoundTripper { + return withHeaderRoundTripper(rt, headers) + }) +} + // WithLogger sets the internal logger that will be used to log dial or TCP level errors. // Logging at the connection level will only happen if the logger has been set. func WithLogger(logger *logp.Logger) TransportOption { diff --git a/metricbeat/helper/http.go b/metricbeat/helper/http.go index ba53a0e42ea..9fbbf696a33 100644 --- a/metricbeat/helper/http.go +++ b/metricbeat/helper/http.go @@ -29,10 +29,13 @@ import ( "github.com/pkg/errors" "github.com/elastic/beats/v7/libbeat/common/transport/httpcommon" + "github.com/elastic/beats/v7/libbeat/common/useragent" "github.com/elastic/beats/v7/metricbeat/helper/dialer" "github.com/elastic/beats/v7/metricbeat/mb" ) +var userAgent = useragent.UserAgent("Metricbeat") + // HTTP is a custom HTTP Client that handle the complexity of connection and retrieving information // from HTTP endpoint. type HTTP struct { @@ -87,6 +90,7 @@ func NewHTTPFromConfig(config Config, hostData mb.HostData) (*HTTP, error) { client, err := config.Transport.Client( httpcommon.WithBaseDialer(dialer), httpcommon.WithAPMHTTPInstrumentation(), + httpcommon.WithHeaderRoundTripper(map[string]string{"User-Agent": userAgent}), ) if err != nil { return nil, err diff --git a/metricbeat/helper/http_test.go b/metricbeat/helper/http_test.go index a88ae4796fa..2fbfea0d1ad 100644 --- a/metricbeat/helper/http_test.go +++ b/metricbeat/helper/http_test.go @@ -269,6 +269,31 @@ func TestOverUnixSocket(t *testing.T) { } } +func TestUserAgentCheck(t *testing.T) { + ua := "" + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ua = r.Header.Get("User-Agent") + w.WriteHeader(http.StatusOK) + })) + defer ts.Close() + + cfg := defaultConfig() + hostData := mb.HostData{ + URI: ts.URL, + SanitizedURI: ts.URL, + } + + h, err := NewHTTPFromConfig(cfg, hostData) + require.NoError(t, err) + + res, err := h.FetchResponse() + require.NoError(t, err) + res.Body.Close() + + assert.Equal(t, http.StatusOK, res.StatusCode) + assert.Contains(t, ua, "Metricbeat") +} + func checkTimeout(t *testing.T, h *HTTP) { t.Helper() diff --git a/x-pack/filebeat/input/httpjson/input.go b/x-pack/filebeat/input/httpjson/input.go index 0411fa65d41..9c6a2b4e71b 100644 --- a/x-pack/filebeat/input/httpjson/input.go +++ b/x-pack/filebeat/input/httpjson/input.go @@ -162,6 +162,7 @@ func newHTTPClient(ctx context.Context, config config) (*http.Client, error) { config.Transport.Client( httpcommon.WithAPMHTTPInstrumentation(), httpcommon.WithKeepaliveSettings{Disable: true}, + httpcommon.WithHeaderRoundTripper(map[string]string{"User-Agent": userAgent}), ) if err != nil { return nil, err diff --git a/x-pack/filebeat/input/httpjson/requester.go b/x-pack/filebeat/input/httpjson/requester.go index bf9abff19ee..9e0e26edf2d 100644 --- a/x-pack/filebeat/input/httpjson/requester.go +++ b/x-pack/filebeat/input/httpjson/requester.go @@ -194,7 +194,6 @@ func (r *requester) createHTTPRequest(ctx context.Context, ri *requestInfo) (*ht req = req.WithContext(ctx) req.Header.Set("Accept", "application/json") req.Header.Set("Content-Type", "application/json") - req.Header.Set("User-Agent", userAgent) if r.apiKey != "" { if r.authScheme != "" { req.Header.Set("Authorization", r.authScheme+" "+r.apiKey) From 6d6d5a6ca9ab3e4e229beea0034be762ae3e2ff2 Mon Sep 17 00:00:00 2001 From: michel-laterman Date: Thu, 19 Aug 2021 13:03:09 -0700 Subject: [PATCH 02/11] Fix useragent injection in heartbeat IPMonitor jobs --- heartbeat/monitors/active/http/http_test.go | 26 +++++++++++++ heartbeat/monitors/active/http/task.go | 38 ++++++++++--------- heartbeat/monitors/active/http/task_test.go | 9 ----- .../common/transport/httpcommon/httpcommon.go | 5 ++- 4 files changed, 49 insertions(+), 29 deletions(-) diff --git a/heartbeat/monitors/active/http/http_test.go b/heartbeat/monitors/active/http/http_test.go index 0bc74655a26..4e6f67dec97 100644 --- a/heartbeat/monitors/active/http/http_test.go +++ b/heartbeat/monitors/active/http/http_test.go @@ -36,6 +36,7 @@ import ( "testing" "time" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/elastic/beats/v7/heartbeat/hbtest" @@ -674,3 +675,28 @@ func mustParseURL(t *testing.T, url string) *url.URL { } return parsed } + +func TestUserAgentInject(t *testing.T) { + ua := "" + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ua = r.Header.Get("User-Agent") + w.WriteHeader(http.StatusOK) + })) + defer ts.Close() + + cfg, err := common.NewConfigFrom(map[string]interface{}{ + "urls": ts.URL, + }) + require.NoError(t, err) + + p, err := create("ua", cfg) + require.NoError(t, err) + + sched, _ := schedule.Parse("@every 1s") + job := wrappers.WrapCommon(p.Jobs, stdfields.StdMonitorFields{ID: "test", Type: "http", Schedule: sched, Timeout: 1})[0] + + event := &beat.Event{} + _, err = job(event) + require.NoError(t, err) + assert.Contains(t, ua, "Heartbeat") +} diff --git a/heartbeat/monitors/active/http/task.go b/heartbeat/monitors/active/http/task.go index 8648b489feb..f273b0a4cc7 100644 --- a/heartbeat/monitors/active/http/task.go +++ b/heartbeat/monitors/active/http/task.go @@ -41,6 +41,7 @@ import ( "github.com/elastic/beats/v7/heartbeat/reason" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/transport/httpcommon" "github.com/elastic/beats/v7/libbeat/common/transport/tlscommon" ) @@ -138,27 +139,28 @@ func createPingFactory( // prevents following redirects in this case, we know that // config.MaxRedirects must be zero to even be here checkRedirect := makeCheckRedirect(0, nil) + transport := &SimpleTransport{ + Dialer: dialer, + OnStartWrite: func() { + cbMutex.Lock() + writeStart = time.Now() + cbMutex.Unlock() + }, + OnEndWrite: func() { + cbMutex.Lock() + writeEnd = time.Now() + cbMutex.Unlock() + }, + OnStartRead: func() { + cbMutex.Lock() + readStart = time.Now() + cbMutex.Unlock() + }, + } client := &http.Client{ CheckRedirect: checkRedirect, Timeout: timeout, - Transport: &SimpleTransport{ - Dialer: dialer, - OnStartWrite: func() { - cbMutex.Lock() - writeStart = time.Now() - cbMutex.Unlock() - }, - OnEndWrite: func() { - cbMutex.Lock() - writeEnd = time.Now() - cbMutex.Unlock() - }, - OnStartRead: func() { - cbMutex.Lock() - readStart = time.Now() - cbMutex.Unlock() - }, - }, + Transport: httpcommon.HeaderRoundTripper(transport, map[string]string{"User-Agent": userAgent}), } _, end, err := execPing(event, client, request, body, timeout, validator, config.Response) diff --git a/heartbeat/monitors/active/http/task_test.go b/heartbeat/monitors/active/http/task_test.go index 7fd3948d0b1..358a4a6ec2b 100644 --- a/heartbeat/monitors/active/http/task_test.go +++ b/heartbeat/monitors/active/http/task_test.go @@ -24,8 +24,6 @@ import ( "reflect" "testing" - "github.com/elastic/beats/v7/libbeat/common/useragent" - "github.com/stretchr/testify/require" "github.com/stretchr/testify/assert" @@ -174,13 +172,6 @@ func TestRequestBuildingWithCustomHost(t *testing.T) { } } -func TestRequestBuildingWithNoUserAgent(t *testing.T) { - request, err := buildRequest("localhost", &Config{}, nilEncoder{}) - - require.NoError(t, err) - assert.Equal(t, useragent.UserAgent("Heartbeat"), request.Header.Get("User-Agent")) -} - func TestRequestBuildingWithExplicitUserAgent(t *testing.T) { expectedUserAgent := "some-user-agent" diff --git a/libbeat/common/transport/httpcommon/httpcommon.go b/libbeat/common/transport/httpcommon/httpcommon.go index 8dfeb104a9a..ac751f97771 100644 --- a/libbeat/common/transport/httpcommon/httpcommon.go +++ b/libbeat/common/transport/httpcommon/httpcommon.go @@ -387,7 +387,8 @@ func WithAPMHTTPInstrumentation() TransportOption { return withAPMHTTPRountTripper } -func withHeaderRoundTripper(rt http.RoundTripper, headers map[string]string) http.RoundTripper { +// HeaderRoundTripper will return a RoundTripper that sets header KVs if the key is not present. +func HeaderRoundTripper(rt http.RoundTripper, headers map[string]string) http.RoundTripper { return &headerRoundTripper{headers, rt} } @@ -395,7 +396,7 @@ func withHeaderRoundTripper(rt http.RoundTripper, headers map[string]string) htt // This RoundTripper will add headers to each request if the key is not present. func WithHeaderRoundTripper(headers map[string]string) TransportOption { return WithModRoundtripper(func(rt http.RoundTripper) http.RoundTripper { - return withHeaderRoundTripper(rt, headers) + return HeaderRoundTripper(rt, headers) }) } From 8f61c1c3a06bbdd9e82d1bf4a578e9b26c348ef5 Mon Sep 17 00:00:00 2001 From: michel-laterman Date: Fri, 20 Aug 2021 09:22:45 -0700 Subject: [PATCH 03/11] Add User-Agent header to kibana and eslegclient Add the User-Agent header with the RoundTripper to requests made by the Kibana and eslegclient. The value for the User-Agent will be constructed from what is returned by the os executable name. --- libbeat/esleg/eslegclient/connection.go | 12 ++++++++++++ libbeat/kibana/client.go | 14 +++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/libbeat/esleg/eslegclient/connection.go b/libbeat/esleg/eslegclient/connection.go index 3ebf96210bf..293a3f2384a 100644 --- a/libbeat/esleg/eslegclient/connection.go +++ b/libbeat/esleg/eslegclient/connection.go @@ -25,6 +25,8 @@ import ( "io/ioutil" "net/http" "net/url" + "os" + "strings" "time" "go.elastic.co/apm/module/apmelasticsearch" @@ -34,6 +36,7 @@ import ( "github.com/elastic/beats/v7/libbeat/common/transport/httpcommon" "github.com/elastic/beats/v7/libbeat/common/transport/kerberos" "github.com/elastic/beats/v7/libbeat/common/transport/tlscommon" + "github.com/elastic/beats/v7/libbeat/common/useragent" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/testing" ) @@ -110,6 +113,14 @@ func NewConnection(s ConnectionSettings) (*Connection, error) { } } + name, err := os.Executable() + if err != nil { + name = "ESLegClient" + } else { + name = strings.Title(name) + } + userAgent = useragent.UserAgent(name) + httpClient, err := s.Transport.Client( httpcommon.WithLogger(logger), httpcommon.WithIOStats(s.Observer), @@ -119,6 +130,7 @@ func NewConnection(s ConnectionSettings) (*Connection, error) { // eg, like in https://github.com/elastic/apm-server/blob/7.7/elasticsearch/client.go return apmelasticsearch.WrapRoundTripper(rt) }), + httpcommon.WithHeaderRoundTripper(map[string]string{"User-Agent": userAgent}), ) if err != nil { return nil, err diff --git a/libbeat/kibana/client.go b/libbeat/kibana/client.go index 2d2b9dc313a..189d9351ddc 100644 --- a/libbeat/kibana/client.go +++ b/libbeat/kibana/client.go @@ -29,12 +29,15 @@ import ( "net/http" "net/textproto" "net/url" + "os" "path" "strings" "github.com/joeshaw/multierror" "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/transport/httpcommon" + "github.com/elastic/beats/v7/libbeat/common/useragent" "github.com/elastic/beats/v7/libbeat/logp" ) @@ -143,7 +146,16 @@ func NewClientWithConfigDefault(config *ClientConfig, defaultPort int) (*Client, headers.Set(k, v) } - rt, err := config.Transport.Client() + name, err := os.Executable() + if err != nil { + logp.Errorf("Unable to get running executable name: %v", err) + name = "KibanaClient" + } else { + name = strings.Title(name) + } + userAgent = useragent.UserAgent(name) + + rt, err := config.Transport.Client(httpcommon.WithHeaderRoundTripper(map[string]string{"User-Agent": userAgent})) if err != nil { return nil, err } From 04762fd7d46fce4389b94fd94a7629cc635f1557 Mon Sep 17 00:00:00 2001 From: michel-laterman Date: Fri, 20 Aug 2021 11:40:07 -0700 Subject: [PATCH 04/11] Fix syntax errors --- libbeat/esleg/eslegclient/connection.go | 2 +- libbeat/kibana/client.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/libbeat/esleg/eslegclient/connection.go b/libbeat/esleg/eslegclient/connection.go index 293a3f2384a..ecaded1079c 100644 --- a/libbeat/esleg/eslegclient/connection.go +++ b/libbeat/esleg/eslegclient/connection.go @@ -119,7 +119,7 @@ func NewConnection(s ConnectionSettings) (*Connection, error) { } else { name = strings.Title(name) } - userAgent = useragent.UserAgent(name) + userAgent := useragent.UserAgent(name) httpClient, err := s.Transport.Client( httpcommon.WithLogger(logger), diff --git a/libbeat/kibana/client.go b/libbeat/kibana/client.go index 189d9351ddc..1cad68a06dd 100644 --- a/libbeat/kibana/client.go +++ b/libbeat/kibana/client.go @@ -148,12 +148,12 @@ func NewClientWithConfigDefault(config *ClientConfig, defaultPort int) (*Client, name, err := os.Executable() if err != nil { - logp.Errorf("Unable to get running executable name: %v", err) + log.Errorf("Unable to get running executable name: %v", err) name = "KibanaClient" } else { name = strings.Title(name) } - userAgent = useragent.UserAgent(name) + userAgent := useragent.UserAgent(name) rt, err := config.Transport.Client(httpcommon.WithHeaderRoundTripper(map[string]string{"User-Agent": userAgent})) if err != nil { From e5c7902d5f04b7444e231d08fd794a9292bd9246 Mon Sep 17 00:00:00 2001 From: michel-laterman Date: Tue, 24 Aug 2021 10:42:58 -0700 Subject: [PATCH 05/11] Get user agent from existing ingo Get user-agent values for the Kibana and eslegclient from existing config settings that get passed into the clients. --- dev-tools/cmd/dashboards/export_dashboards.go | 2 +- filebeat/beater/filebeat.go | 4 ++-- libbeat/cmd/export/dashboard.go | 2 +- libbeat/cmd/instance/beat.go | 4 ++-- libbeat/dashboards/dashboards.go | 6 +++--- libbeat/dashboards/kibana_loader.go | 8 +++---- libbeat/esleg/eslegclient/connection.go | 20 +++++++----------- libbeat/kibana/client.go | 21 ++++++------------- libbeat/kibana/client_test.go | 4 ++-- .../report/elasticsearch/elasticsearch.go | 5 +++-- libbeat/outputs/elasticsearch/client.go | 2 ++ .../outputs/elasticsearch/elasticsearch.go | 1 + .../elastic-agent/pkg/agent/cmd/container.go | 2 +- 13 files changed, 35 insertions(+), 46 deletions(-) diff --git a/dev-tools/cmd/dashboards/export_dashboards.go b/dev-tools/cmd/dashboards/export_dashboards.go index b13351678f2..211f854b71e 100644 --- a/dev-tools/cmd/dashboards/export_dashboards.go +++ b/dev-tools/cmd/dashboards/export_dashboards.go @@ -77,7 +77,7 @@ func main() { Path: u.Path, SpaceID: *spaceID, Transport: transport, - }) + }, "beat") if err != nil { log.Fatalf("Error while connecting to Kibana: %v", err) } diff --git a/filebeat/beater/filebeat.go b/filebeat/beater/filebeat.go index 41b15b1543c..a66a674b525 100644 --- a/filebeat/beater/filebeat.go +++ b/filebeat/beater/filebeat.go @@ -174,7 +174,7 @@ func (fb *Filebeat) setupPipelineLoaderCallback(b *beat.Beat) error { overwritePipelines := true b.OverwritePipelinesCallback = func(esConfig *common.Config) error { - esClient, err := eslegclient.NewConnectedClient(esConfig) + esClient, err := eslegclient.NewConnectedClient(esConfig, "Filebeat") if err != nil { return err } @@ -428,7 +428,7 @@ func (fb *Filebeat) Stop() { // Create a new pipeline loader (es client) factory func newPipelineLoaderFactory(esConfig *common.Config) fileset.PipelineLoaderFactory { pipelineLoaderFactory := func() (fileset.PipelineLoader, error) { - esClient, err := eslegclient.NewConnectedClient(esConfig) + esClient, err := eslegclient.NewConnectedClient(esConfig, "Filebeat") if err != nil { return nil, errors.Wrap(err, "Error creating Elasticsearch client") } diff --git a/libbeat/cmd/export/dashboard.go b/libbeat/cmd/export/dashboard.go index 10fc358d76a..780ea78ea0d 100644 --- a/libbeat/cmd/export/dashboard.go +++ b/libbeat/cmd/export/dashboard.go @@ -55,7 +55,7 @@ func GenDashboardCmd(settings instance.Settings) *cobra.Command { // part of the initialization. initConfig := instance.InitKibanaConfig(b.Config) - client, err := kibana.NewKibanaClient(initConfig) + client, err := kibana.NewKibanaClient(initConfig, b.Info.Name) if err != nil { fatalf("Error creating Kibana client: %+v.\n", err) } diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go index ade05d78f8b..8bb96bba225 100644 --- a/libbeat/cmd/instance/beat.go +++ b/libbeat/cmd/instance/beat.go @@ -531,7 +531,7 @@ func (b *Beat) Setup(settings Settings, bt beat.Creator, setup SetupSettings) er if outCfg.Name() != "elasticsearch" { return fmt.Errorf("Index management requested but the Elasticsearch output is not configured/enabled") } - esClient, err := eslegclient.NewConnectedClient(outCfg.Config()) + esClient, err := eslegclient.NewConnectedClient(outCfg.Config(), settings.Name) if err != nil { return err } @@ -808,7 +808,7 @@ func (b *Beat) loadDashboards(ctx context.Context, force bool) error { // initKibanaConfig will attach the username and password into kibana config as a part of the initialization. kibanaConfig := InitKibanaConfig(b.Config) - client, err := kibana.NewKibanaClient(kibanaConfig) + client, err := kibana.NewKibanaClient(kibanaConfig, b.Info.Name) if err != nil { return fmt.Errorf("error connecting to Kibana: %v", err) } diff --git a/libbeat/dashboards/dashboards.go b/libbeat/dashboards/dashboards.go index afb655fa67b..0e00bf04711 100644 --- a/libbeat/dashboards/dashboards.go +++ b/libbeat/dashboards/dashboards.go @@ -54,13 +54,13 @@ func ImportDashboards( return errors.New("kibana configuration missing for loading dashboards") } - return setupAndImportDashboardsViaKibana(ctx, beatInfo.Hostname, kibanaConfig, &dashConfig, msgOutputter, pattern) + return setupAndImportDashboardsViaKibana(ctx, beatInfo.Hostname, beatInfo.Name, kibanaConfig, &dashConfig, msgOutputter, pattern) } -func setupAndImportDashboardsViaKibana(ctx context.Context, hostname string, kibanaConfig *common.Config, +func setupAndImportDashboardsViaKibana(ctx context.Context, hostname, beatname string, kibanaConfig *common.Config, dashboardsConfig *Config, msgOutputter MessageOutputter, fields common.MapStr) error { - kibanaLoader, err := NewKibanaLoader(ctx, kibanaConfig, dashboardsConfig, hostname, msgOutputter) + kibanaLoader, err := NewKibanaLoader(ctx, kibanaConfig, dashboardsConfig, hostname, msgOutputter, beatname) if err != nil { return fmt.Errorf("fail to create the Kibana loader: %v", err) } diff --git a/libbeat/dashboards/kibana_loader.go b/libbeat/dashboards/kibana_loader.go index e3c8d3ee17b..9cf4b9c62f4 100644 --- a/libbeat/dashboards/kibana_loader.go +++ b/libbeat/dashboards/kibana_loader.go @@ -47,13 +47,13 @@ type KibanaLoader struct { } // NewKibanaLoader creates a new loader to load Kibana files -func NewKibanaLoader(ctx context.Context, cfg *common.Config, dashboardsConfig *Config, hostname string, msgOutputter MessageOutputter) (*KibanaLoader, error) { +func NewKibanaLoader(ctx context.Context, cfg *common.Config, dashboardsConfig *Config, hostname string, msgOutputter MessageOutputter, beatname string) (*KibanaLoader, error) { if cfg == nil || !cfg.Enabled() { return nil, fmt.Errorf("Kibana is not configured or enabled") } - client, err := getKibanaClient(ctx, cfg, dashboardsConfig.Retry, 0) + client, err := getKibanaClient(ctx, cfg, dashboardsConfig.Retry, 0, beatname) if err != nil { return nil, fmt.Errorf("Error creating Kibana client: %v", err) } @@ -73,8 +73,8 @@ func NewKibanaLoader(ctx context.Context, cfg *common.Config, dashboardsConfig * return &loader, nil } -func getKibanaClient(ctx context.Context, cfg *common.Config, retryCfg *Retry, retryAttempt uint) (*kibana.Client, error) { - client, err := kibana.NewKibanaClient(cfg) +func getKibanaClient(ctx context.Context, cfg *common.Config, retryCfg *Retry, retryAttempt uint, beatname string) (*kibana.Client, error) { + client, err := kibana.NewKibanaClient(cfg, beatname) if err != nil { if retryCfg.Enabled && (retryCfg.Maximum == 0 || retryCfg.Maximum > retryAttempt) { select { diff --git a/libbeat/esleg/eslegclient/connection.go b/libbeat/esleg/eslegclient/connection.go index ecaded1079c..47d55167d1b 100644 --- a/libbeat/esleg/eslegclient/connection.go +++ b/libbeat/esleg/eslegclient/connection.go @@ -25,8 +25,6 @@ import ( "io/ioutil" "net/http" "net/url" - "os" - "strings" "time" "go.elastic.co/apm/module/apmelasticsearch" @@ -60,7 +58,8 @@ type Connection struct { // ConnectionSettings are the settings needed for a Connection type ConnectionSettings struct { - URL string + URL string + Beatname string Username string Password string @@ -113,13 +112,7 @@ func NewConnection(s ConnectionSettings) (*Connection, error) { } } - name, err := os.Executable() - if err != nil { - name = "ESLegClient" - } else { - name = strings.Title(name) - } - userAgent := useragent.UserAgent(name) + userAgent := useragent.UserAgent(s.Beatname) httpClient, err := s.Transport.Client( httpcommon.WithLogger(logger), @@ -172,7 +165,7 @@ func settingsWithDefaults(s ConnectionSettings) ConnectionSettings { // configuration. It accepts the same configuration parameters as the Elasticsearch // output, except for the output specific configuration options. If multiple hosts // are defined in the configuration, a client is returned for each of them. -func NewClients(cfg *common.Config) ([]Connection, error) { +func NewClients(cfg *common.Config, beatname string) ([]Connection, error) { config := defaultConfig() if err := cfg.Unpack(&config); err != nil { return nil, err @@ -197,6 +190,7 @@ func NewClients(cfg *common.Config) ([]Connection, error) { client, err := NewConnection(ConnectionSettings{ URL: esURL, + Beatname: beatname, Kerberos: config.Kerberos, Username: config.Username, Password: config.Password, @@ -217,8 +211,8 @@ func NewClients(cfg *common.Config) ([]Connection, error) { return clients, nil } -func NewConnectedClient(cfg *common.Config) (*Connection, error) { - clients, err := NewClients(cfg) +func NewConnectedClient(cfg *common.Config, beatname string) (*Connection, error) { + clients, err := NewClients(cfg, beatname) if err != nil { return nil, err } diff --git a/libbeat/kibana/client.go b/libbeat/kibana/client.go index 1cad68a06dd..d892a88ddf3 100644 --- a/libbeat/kibana/client.go +++ b/libbeat/kibana/client.go @@ -29,7 +29,6 @@ import ( "net/http" "net/textproto" "net/url" - "os" "path" "strings" @@ -96,22 +95,22 @@ func extractError(result []byte) error { } // NewKibanaClient builds and returns a new Kibana client -func NewKibanaClient(cfg *common.Config) (*Client, error) { +func NewKibanaClient(cfg *common.Config, beatname string) (*Client, error) { config := DefaultClientConfig() if err := cfg.Unpack(&config); err != nil { return nil, err } - return NewClientWithConfig(&config) + return NewClientWithConfig(&config, beatname) } // NewClientWithConfig creates and returns a kibana client using the given config -func NewClientWithConfig(config *ClientConfig) (*Client, error) { - return NewClientWithConfigDefault(config, 5601) +func NewClientWithConfig(config *ClientConfig, beatname string) (*Client, error) { + return NewClientWithConfigDefault(config, 5601, beatname) } // NewClientWithConfig creates and returns a kibana client using the given config -func NewClientWithConfigDefault(config *ClientConfig, defaultPort int) (*Client, error) { +func NewClientWithConfigDefault(config *ClientConfig, defaultPort int, beatname string) (*Client, error) { p := config.Path if config.SpaceID != "" { p = path.Join(p, "s", config.SpaceID) @@ -146,15 +145,7 @@ func NewClientWithConfigDefault(config *ClientConfig, defaultPort int) (*Client, headers.Set(k, v) } - name, err := os.Executable() - if err != nil { - log.Errorf("Unable to get running executable name: %v", err) - name = "KibanaClient" - } else { - name = strings.Title(name) - } - userAgent := useragent.UserAgent(name) - + userAgent := useragent.UserAgent(beatname) rt, err := config.Transport.Client(httpcommon.WithHeaderRoundTripper(map[string]string{"User-Agent": userAgent})) if err != nil { return nil, err diff --git a/libbeat/kibana/client_test.go b/libbeat/kibana/client_test.go index c69592b8ef5..d14be6e2a22 100644 --- a/libbeat/kibana/client_test.go +++ b/libbeat/kibana/client_test.go @@ -97,7 +97,7 @@ headers: content-type: text/plain accept: text/plain kbn-xsrf: 0 -`, kibanaTs.Listener.Addr().String()))) +`, kibanaTs.Listener.Addr().String())), "Testbeat") require.NoError(t, err) require.NotNil(t, client) @@ -137,7 +137,7 @@ headers: content-type: multipart/form-data; boundary=46bea21be603a2c2ea6f51571a5e1baf5ea3be8ebd7101199320607b36ff accept: text/plain kbn-xsrf: 0 -`, kibanaTs.Listener.Addr().String()))) +`, kibanaTs.Listener.Addr().String())), "Testbeat") require.NoError(t, err) require.NotNil(t, client) diff --git a/libbeat/monitoring/report/elasticsearch/elasticsearch.go b/libbeat/monitoring/report/elasticsearch/elasticsearch.go index dddb2c53a00..5eb537989c9 100644 --- a/libbeat/monitoring/report/elasticsearch/elasticsearch.go +++ b/libbeat/monitoring/report/elasticsearch/elasticsearch.go @@ -127,7 +127,7 @@ func makeReporter(beat beat.Info, settings report.Settings, cfg *common.Config) var clients []outputs.NetworkClient for _, host := range hosts { - client, err := makeClient(host, params, &config) + client, err := makeClient(host, params, &config, beat.Name) if err != nil { return nil, err } @@ -291,7 +291,7 @@ func (r *reporter) snapshotLoop(namespace, prefix string, period time.Duration, } } -func makeClient(host string, params map[string]string, config *config) (outputs.NetworkClient, error) { +func makeClient(host string, params map[string]string, config *config, beatname string) (outputs.NetworkClient, error) { url, err := common.MakeURL(config.Protocol, "", host, 9200) if err != nil { return nil, err @@ -299,6 +299,7 @@ func makeClient(host string, params map[string]string, config *config) (outputs. esClient, err := eslegclient.NewConnection(eslegclient.ConnectionSettings{ URL: url, + Beatname: beatname, Username: config.Username, Password: config.Password, APIKey: config.APIKey, diff --git a/libbeat/outputs/elasticsearch/client.go b/libbeat/outputs/elasticsearch/client.go index 318678007c4..deab29c3dcd 100644 --- a/libbeat/outputs/elasticsearch/client.go +++ b/libbeat/outputs/elasticsearch/client.go @@ -84,6 +84,7 @@ func NewClient( conn, err := eslegclient.NewConnection(eslegclient.ConnectionSettings{ URL: s.URL, + Beatname: s.Beatname, Username: s.Username, Password: s.Password, APIKey: s.APIKey, @@ -145,6 +146,7 @@ func (client *Client) Clone() *Client { // create install a template, we don't want these to be included in the clone. connection := eslegclient.ConnectionSettings{ URL: client.conn.URL, + Beatname: client.conn.Beatname, Kerberos: client.conn.Kerberos, Username: client.conn.Username, Password: client.conn.Password, diff --git a/libbeat/outputs/elasticsearch/elasticsearch.go b/libbeat/outputs/elasticsearch/elasticsearch.go index 417043eff83..fffdbd68e29 100644 --- a/libbeat/outputs/elasticsearch/elasticsearch.go +++ b/libbeat/outputs/elasticsearch/elasticsearch.go @@ -92,6 +92,7 @@ func makeES( client, err = NewClient(ClientSettings{ ConnectionSettings: eslegclient.ConnectionSettings{ URL: esURL, + Beatname: beat.Name, Kerberos: config.Kerberos, Username: config.Username, Password: config.Password, diff --git a/x-pack/elastic-agent/pkg/agent/cmd/container.go b/x-pack/elastic-agent/pkg/agent/cmd/container.go index bce71fa874a..b86eb4fe673 100644 --- a/x-pack/elastic-agent/pkg/agent/cmd/container.go +++ b/x-pack/elastic-agent/pkg/agent/cmd/container.go @@ -474,7 +474,7 @@ func kibanaClient(cfg kibanaConfig, headers map[string]string) (*kibana.Client, IgnoreVersion: true, Transport: transport, Headers: headers, - }, 0) + }, 0, "Elastic-agent") } func findPolicy(cfg setupConfig, policies []kibanaPolicy) (*kibanaPolicy, error) { From 27c93e49bf0b2db688064753052ee78454bcd68a Mon Sep 17 00:00:00 2001 From: michel-laterman Date: Tue, 24 Aug 2021 10:51:14 -0700 Subject: [PATCH 06/11] change from settings.Name to b.Info.Name --- libbeat/cmd/instance/beat.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go index 8bb96bba225..0ccd6692926 100644 --- a/libbeat/cmd/instance/beat.go +++ b/libbeat/cmd/instance/beat.go @@ -531,7 +531,7 @@ func (b *Beat) Setup(settings Settings, bt beat.Creator, setup SetupSettings) er if outCfg.Name() != "elasticsearch" { return fmt.Errorf("Index management requested but the Elasticsearch output is not configured/enabled") } - esClient, err := eslegclient.NewConnectedClient(outCfg.Config(), settings.Name) + esClient, err := eslegclient.NewConnectedClient(outCfg.Config(), b.Info.Name) if err != nil { return err } From 46ec023c46ad847cad6f9b2f1ad6a762e0e8d5bf Mon Sep 17 00:00:00 2001 From: michel-laterman Date: Tue, 24 Aug 2021 11:43:56 -0700 Subject: [PATCH 07/11] Fix missing param --- libbeat/dashboards/kibana_loader.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libbeat/dashboards/kibana_loader.go b/libbeat/dashboards/kibana_loader.go index 9cf4b9c62f4..3efdc62c274 100644 --- a/libbeat/dashboards/kibana_loader.go +++ b/libbeat/dashboards/kibana_loader.go @@ -81,7 +81,7 @@ func getKibanaClient(ctx context.Context, cfg *common.Config, retryCfg *Retry, r case <-ctx.Done(): return nil, err case <-time.After(retryCfg.Interval): - return getKibanaClient(ctx, cfg, retryCfg, retryAttempt+1) + return getKibanaClient(ctx, cfg, retryCfg, retryAttempt+1, beatname) } } return nil, fmt.Errorf("Error creating Kibana client: %v", err) From d6e9027c171c48004c8ce641216da03ebd8195bb Mon Sep 17 00:00:00 2001 From: michel-laterman Date: Mon, 30 Aug 2021 10:32:24 -0700 Subject: [PATCH 08/11] Rename export dashboard useragent value --- dev-tools/cmd/dashboards/export_dashboards.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-tools/cmd/dashboards/export_dashboards.go b/dev-tools/cmd/dashboards/export_dashboards.go index 211f854b71e..6d036c9f7f8 100644 --- a/dev-tools/cmd/dashboards/export_dashboards.go +++ b/dev-tools/cmd/dashboards/export_dashboards.go @@ -77,7 +77,7 @@ func main() { Path: u.Path, SpaceID: *spaceID, Transport: transport, - }, "beat") + }, "Beat Development Tools") if err != nil { log.Fatalf("Error while connecting to Kibana: %v", err) } From ab53cb444631c272b38b4e6cc263125c0ca08bf0 Mon Sep 17 00:00:00 2001 From: michel-laterman Date: Fri, 3 Sep 2021 16:37:52 -0700 Subject: [PATCH 09/11] Review feedback --- libbeat/esleg/eslegclient/connection.go | 3 +++ libbeat/kibana/client.go | 3 +++ x-pack/elastic-agent/pkg/agent/cmd/container.go | 2 +- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/libbeat/esleg/eslegclient/connection.go b/libbeat/esleg/eslegclient/connection.go index 47d55167d1b..57a4534c87a 100644 --- a/libbeat/esleg/eslegclient/connection.go +++ b/libbeat/esleg/eslegclient/connection.go @@ -112,6 +112,9 @@ func NewConnection(s ConnectionSettings) (*Connection, error) { } } + if s.Beatname == "" { + s.Beatname == "Libbeat" + } userAgent := useragent.UserAgent(s.Beatname) httpClient, err := s.Transport.Client( diff --git a/libbeat/kibana/client.go b/libbeat/kibana/client.go index f4335488bc4..a90fb9c5d78 100644 --- a/libbeat/kibana/client.go +++ b/libbeat/kibana/client.go @@ -155,6 +155,9 @@ func NewClientWithConfigDefault(config *ClientConfig, defaultPort int, beatname headers.Set(k, v) } + if beatname == "" { + beatname = "Libbeat" + } userAgent := useragent.UserAgent(beatname) rt, err := config.Transport.Client(httpcommon.WithHeaderRoundTripper(map[string]string{"User-Agent": userAgent})) if err != nil { diff --git a/x-pack/elastic-agent/pkg/agent/cmd/container.go b/x-pack/elastic-agent/pkg/agent/cmd/container.go index b86eb4fe673..5faba8548c4 100644 --- a/x-pack/elastic-agent/pkg/agent/cmd/container.go +++ b/x-pack/elastic-agent/pkg/agent/cmd/container.go @@ -474,7 +474,7 @@ func kibanaClient(cfg kibanaConfig, headers map[string]string) (*kibana.Client, IgnoreVersion: true, Transport: transport, Headers: headers, - }, 0, "Elastic-agent") + }, 0, "Elastic-Agent") } func findPolicy(cfg setupConfig, policies []kibanaPolicy) (*kibanaPolicy, error) { From 5590c5c8ac4ba8a226dc04b1e4c37b61b67f9d36 Mon Sep 17 00:00:00 2001 From: michel-laterman Date: Tue, 7 Sep 2021 09:20:37 -0700 Subject: [PATCH 10/11] Change beat.Info.Name to beat.Info.Beat --- CHANGELOG.next.asciidoc | 2 +- libbeat/cmd/export/dashboard.go | 2 +- libbeat/cmd/instance/beat.go | 4 ++-- libbeat/dashboards/dashboards.go | 2 +- libbeat/monitoring/report/elasticsearch/elasticsearch.go | 2 +- libbeat/outputs/elasticsearch/elasticsearch.go | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 04efb39786c..ed3bdf9ffd8 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -107,7 +107,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Bump AWS SDK version to v0.24.0 for WebIdentity authentication flow {issue}19393[19393] {pull}27126[27126] - Add Linux pressure metricset {pull}27355[27355] - Add support for kube-state-metrics v2.0.0 {pull}27552[27552] -- Add User-Agent header to HTTP requests. {issue}18160[18160] +- Add User-Agent header to HTTP requests. {issue}18160[18160] {pull}27509[27509] *Packetbeat* diff --git a/libbeat/cmd/export/dashboard.go b/libbeat/cmd/export/dashboard.go index fd26fdac558..070afc251be 100644 --- a/libbeat/cmd/export/dashboard.go +++ b/libbeat/cmd/export/dashboard.go @@ -58,7 +58,7 @@ func GenDashboardCmd(settings instance.Settings) *cobra.Command { // part of the initialization. initConfig := instance.InitKibanaConfig(b.Config) - client, err := kibana.NewKibanaClient(initConfig, b.Info.Name) + client, err := kibana.NewKibanaClient(initConfig, b.Info.Beat) if err != nil { fatalf("Error creating Kibana client: %+v.\n", err) } diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go index 374222a6e77..00e5f89a1c9 100644 --- a/libbeat/cmd/instance/beat.go +++ b/libbeat/cmd/instance/beat.go @@ -531,7 +531,7 @@ func (b *Beat) Setup(settings Settings, bt beat.Creator, setup SetupSettings) er if outCfg.Name() != "elasticsearch" { return fmt.Errorf("Index management requested but the Elasticsearch output is not configured/enabled") } - esClient, err := eslegclient.NewConnectedClient(outCfg.Config(), b.Info.Name) + esClient, err := eslegclient.NewConnectedClient(outCfg.Config(), b.Info.Beat) if err != nil { return err } @@ -808,7 +808,7 @@ func (b *Beat) loadDashboards(ctx context.Context, force bool) error { // initKibanaConfig will attach the username and password into kibana config as a part of the initialization. kibanaConfig := InitKibanaConfig(b.Config) - client, err := kibana.NewKibanaClient(kibanaConfig, b.Info.Name) + client, err := kibana.NewKibanaClient(kibanaConfig, b.Info.Beat) if err != nil { return fmt.Errorf("error connecting to Kibana: %v", err) } diff --git a/libbeat/dashboards/dashboards.go b/libbeat/dashboards/dashboards.go index 0e00bf04711..d1209235a54 100644 --- a/libbeat/dashboards/dashboards.go +++ b/libbeat/dashboards/dashboards.go @@ -54,7 +54,7 @@ func ImportDashboards( return errors.New("kibana configuration missing for loading dashboards") } - return setupAndImportDashboardsViaKibana(ctx, beatInfo.Hostname, beatInfo.Name, kibanaConfig, &dashConfig, msgOutputter, pattern) + return setupAndImportDashboardsViaKibana(ctx, beatInfo.Hostname, beatInfo.Beat, kibanaConfig, &dashConfig, msgOutputter, pattern) } func setupAndImportDashboardsViaKibana(ctx context.Context, hostname, beatname string, kibanaConfig *common.Config, diff --git a/libbeat/monitoring/report/elasticsearch/elasticsearch.go b/libbeat/monitoring/report/elasticsearch/elasticsearch.go index 5eb537989c9..96210c56c54 100644 --- a/libbeat/monitoring/report/elasticsearch/elasticsearch.go +++ b/libbeat/monitoring/report/elasticsearch/elasticsearch.go @@ -127,7 +127,7 @@ func makeReporter(beat beat.Info, settings report.Settings, cfg *common.Config) var clients []outputs.NetworkClient for _, host := range hosts { - client, err := makeClient(host, params, &config, beat.Name) + client, err := makeClient(host, params, &config, beat.Beat) if err != nil { return nil, err } diff --git a/libbeat/outputs/elasticsearch/elasticsearch.go b/libbeat/outputs/elasticsearch/elasticsearch.go index fffdbd68e29..8ed2b2cbb91 100644 --- a/libbeat/outputs/elasticsearch/elasticsearch.go +++ b/libbeat/outputs/elasticsearch/elasticsearch.go @@ -92,7 +92,7 @@ func makeES( client, err = NewClient(ClientSettings{ ConnectionSettings: eslegclient.ConnectionSettings{ URL: esURL, - Beatname: beat.Name, + Beatname: beat.Beat, Kerberos: config.Kerberos, Username: config.Username, Password: config.Password, From 42128196affee2e1dbff04e7fbd4011a89174e9e Mon Sep 17 00:00:00 2001 From: michel-laterman Date: Tue, 7 Sep 2021 09:46:34 -0700 Subject: [PATCH 11/11] Fix typo --- libbeat/esleg/eslegclient/connection.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libbeat/esleg/eslegclient/connection.go b/libbeat/esleg/eslegclient/connection.go index 57a4534c87a..8db37a9b0ef 100644 --- a/libbeat/esleg/eslegclient/connection.go +++ b/libbeat/esleg/eslegclient/connection.go @@ -113,7 +113,7 @@ func NewConnection(s ConnectionSettings) (*Connection, error) { } if s.Beatname == "" { - s.Beatname == "Libbeat" + s.Beatname = "Libbeat" } userAgent := useragent.UserAgent(s.Beatname)