From 2714baf5b4cd5877710669a794234e1642edddfb Mon Sep 17 00:00:00 2001 From: Michel Laterman <82832767+michel-laterman@users.noreply.github.com> Date: Tue, 7 Sep 2021 14:50:20 -0700 Subject: [PATCH 1/3] Add a header round tripper option to httpcommon (#27509) * Add a header round tripper option to httpcommon Add a new TransportOption to add a set of headers to each HTTP request through a custom http.RoundTripper. It will set the passed headers to each request if the header key is not present. Use the new transport option to add User-Agent headers to heartbeat, metricbeat, and filebeat. * Fix useragent injection in heartbeat IPMonitor jobs * Add User-Agent header to kibana and eslegclient Add the User-Agent header with the RoundTripper to requests made by the Kibana and eslegclient. The value for the User-Agent will be constructed from what is returned by the os executable name. * Fix syntax errors * Get user agent from existing ingo Get user-agent values for the Kibana and eslegclient from existing config settings that get passed into the clients. * change from settings.Name to b.Info.Name * Fix missing param * Rename export dashboard useragent value * Review feedback * Change beat.Info.Name to beat.Info.Beat * Fix typo (cherry picked from commit 8a5dac67f9c9a50c66cc62835c7ca94da05ba030) # Conflicts: # heartbeat/monitors/active/http/task.go # heartbeat/monitors/active/http/task_test.go # libbeat/cmd/export/dashboard.go --- CHANGELOG.next.asciidoc | 1 + dev-tools/cmd/dashboards/export_dashboards.go | 2 +- filebeat/beater/filebeat.go | 4 +- heartbeat/monitors/active/http/http.go | 4 ++ heartbeat/monitors/active/http/http_test.go | 26 +++++++++++ heartbeat/monitors/active/http/task.go | 45 ++++++++++--------- heartbeat/monitors/active/http/task_test.go | 5 ++- libbeat/cmd/export/dashboard.go | 10 +++++ libbeat/cmd/instance/beat.go | 4 +- .../common/transport/httpcommon/httpcommon.go | 27 +++++++++++ libbeat/dashboards/dashboards.go | 6 +-- libbeat/dashboards/kibana_loader.go | 10 ++--- libbeat/esleg/eslegclient/connection.go | 17 +++++-- libbeat/kibana/client.go | 18 +++++--- libbeat/kibana/client_test.go | 4 +- .../report/elasticsearch/elasticsearch.go | 5 ++- libbeat/outputs/elasticsearch/client.go | 2 + .../outputs/elasticsearch/elasticsearch.go | 1 + metricbeat/helper/http.go | 4 ++ metricbeat/helper/http_test.go | 25 +++++++++++ .../elastic-agent/pkg/agent/cmd/container.go | 2 +- x-pack/filebeat/input/httpjson/input.go | 1 + x-pack/filebeat/input/httpjson/requester.go | 1 - 23 files changed, 171 insertions(+), 53 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 0ff18ccb77ac..185f621bce71 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -73,6 +73,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Bump AWS SDK version to v0.24.0 for WebIdentity authentication flow {issue}19393[19393] {pull}27126[27126] - Add Linux pressure metricset {pull}27355[27355] - Add support for kube-state-metrics v2.0.0 {pull}27552[27552] +- Add User-Agent header to HTTP requests. {issue}18160[18160] {pull}27509[27509] *Packetbeat* diff --git a/dev-tools/cmd/dashboards/export_dashboards.go b/dev-tools/cmd/dashboards/export_dashboards.go index fb375436ebb8..364fae9e0f5e 100644 --- a/dev-tools/cmd/dashboards/export_dashboards.go +++ b/dev-tools/cmd/dashboards/export_dashboards.go @@ -78,7 +78,7 @@ func main() { Path: u.Path, SpaceID: *spaceID, Transport: transport, - }) + }, "Beat Development Tools") if err != nil { log.Fatalf("Error while connecting to Kibana: %v", err) } diff --git a/filebeat/beater/filebeat.go b/filebeat/beater/filebeat.go index 7b6e8d39ffac..2e097d319de6 100644 --- a/filebeat/beater/filebeat.go +++ b/filebeat/beater/filebeat.go @@ -181,7 +181,7 @@ func (fb *Filebeat) setupPipelineLoaderCallback(b *beat.Beat) error { overwritePipelines := true b.OverwritePipelinesCallback = func(esConfig *common.Config) error { - esClient, err := eslegclient.NewConnectedClient(esConfig) + esClient, err := eslegclient.NewConnectedClient(esConfig, "Filebeat") if err != nil { return err } @@ -521,7 +521,7 @@ func (fb *Filebeat) Stop() { // Create a new pipeline loader (es client) factory func newPipelineLoaderFactory(esConfig *common.Config) fileset.PipelineLoaderFactory { pipelineLoaderFactory := func() (fileset.PipelineLoader, error) { - esClient, err := eslegclient.NewConnectedClient(esConfig) + esClient, err := eslegclient.NewConnectedClient(esConfig, "Filebeat") if err != nil { return nil, errors.Wrap(err, "Error creating Elasticsearch client") } diff --git a/heartbeat/monitors/active/http/http.go b/heartbeat/monitors/active/http/http.go index c02ae25943e0..2532a3d1c7a0 100644 --- a/heartbeat/monitors/active/http/http.go +++ b/heartbeat/monitors/active/http/http.go @@ -29,6 +29,7 @@ import ( "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/transport/httpcommon" "github.com/elastic/beats/v7/libbeat/common/transport/tlscommon" + "github.com/elastic/beats/v7/libbeat/common/useragent" "github.com/elastic/beats/v7/libbeat/logp" ) @@ -38,6 +39,8 @@ func init() { var debugf = logp.MakeDebug("http") +var userAgent = useragent.UserAgent("Heartbeat") + // Create makes a new HTTP monitor func create( name string, @@ -128,5 +131,6 @@ func newRoundTripper(config *Config) (http.RoundTripper, error) { httpcommon.WithKeepaliveSettings{ Disable: true, }, + httpcommon.WithHeaderRoundTripper(map[string]string{"User-Agent": userAgent}), ) } diff --git a/heartbeat/monitors/active/http/http_test.go b/heartbeat/monitors/active/http/http_test.go index 0bc74655a263..4e6f67dec973 100644 --- a/heartbeat/monitors/active/http/http_test.go +++ b/heartbeat/monitors/active/http/http_test.go @@ -36,6 +36,7 @@ import ( "testing" "time" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/elastic/beats/v7/heartbeat/hbtest" @@ -674,3 +675,28 @@ func mustParseURL(t *testing.T, url string) *url.URL { } return parsed } + +func TestUserAgentInject(t *testing.T) { + ua := "" + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ua = r.Header.Get("User-Agent") + w.WriteHeader(http.StatusOK) + })) + defer ts.Close() + + cfg, err := common.NewConfigFrom(map[string]interface{}{ + "urls": ts.URL, + }) + require.NoError(t, err) + + p, err := create("ua", cfg) + require.NoError(t, err) + + sched, _ := schedule.Parse("@every 1s") + job := wrappers.WrapCommon(p.Jobs, stdfields.StdMonitorFields{ID: "test", Type: "http", Schedule: sched, Timeout: 1})[0] + + event := &beat.Event{} + _, err = job(event) + require.NoError(t, err) + assert.Contains(t, ua, "Heartbeat") +} diff --git a/heartbeat/monitors/active/http/task.go b/heartbeat/monitors/active/http/task.go index 87c0a089fd8b..a8ba9753b601 100644 --- a/heartbeat/monitors/active/http/task.go +++ b/heartbeat/monitors/active/http/task.go @@ -41,12 +41,15 @@ import ( "github.com/elastic/beats/v7/heartbeat/reason" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/transport/httpcommon" "github.com/elastic/beats/v7/libbeat/common/transport/tlscommon" - "github.com/elastic/beats/v7/libbeat/common/useragent" ) +<<<<<<< HEAD var userAgent = useragent.UserAgent("Heartbeat", true) +======= +>>>>>>> 8a5dac67f9 (Add a header round tripper option to httpcommon (#27509)) func newHTTPMonitorHostJob( addr string, config *Config, @@ -141,27 +144,28 @@ func createPingFactory( // prevents following redirects in this case, we know that // config.MaxRedirects must be zero to even be here checkRedirect := makeCheckRedirect(0, nil) + transport := &SimpleTransport{ + Dialer: dialer, + OnStartWrite: func() { + cbMutex.Lock() + writeStart = time.Now() + cbMutex.Unlock() + }, + OnEndWrite: func() { + cbMutex.Lock() + writeEnd = time.Now() + cbMutex.Unlock() + }, + OnStartRead: func() { + cbMutex.Lock() + readStart = time.Now() + cbMutex.Unlock() + }, + } client := &http.Client{ CheckRedirect: checkRedirect, Timeout: timeout, - Transport: &SimpleTransport{ - Dialer: dialer, - OnStartWrite: func() { - cbMutex.Lock() - writeStart = time.Now() - cbMutex.Unlock() - }, - OnEndWrite: func() { - cbMutex.Lock() - writeEnd = time.Now() - cbMutex.Unlock() - }, - OnStartRead: func() { - cbMutex.Lock() - readStart = time.Now() - cbMutex.Unlock() - }, - }, + Transport: httpcommon.HeaderRoundTripper(transport, map[string]string{"User-Agent": userAgent}), } _, end, err := execPing(event, client, request, body, timeout, validator, config.Response) @@ -206,9 +210,6 @@ func buildRequest(addr string, config *Config, enc contentEncoder) (*http.Reques request.Header.Add(k, v) } - if ua := request.Header.Get("User-Agent"); ua == "" { - request.Header.Set("User-Agent", userAgent) - } if enc != nil { enc.AddHeaders(&request.Header) diff --git a/heartbeat/monitors/active/http/task_test.go b/heartbeat/monitors/active/http/task_test.go index 01f1f6ac1dac..ac4691491b32 100644 --- a/heartbeat/monitors/active/http/task_test.go +++ b/heartbeat/monitors/active/http/task_test.go @@ -24,8 +24,6 @@ import ( "reflect" "testing" - "github.com/elastic/beats/v7/libbeat/common/useragent" - "github.com/stretchr/testify/require" "github.com/stretchr/testify/assert" @@ -174,6 +172,7 @@ func TestRequestBuildingWithCustomHost(t *testing.T) { } } +<<<<<<< HEAD func TestRequestBuildingWithNoUserAgent(t *testing.T) { request, err := buildRequest("localhost", &Config{}, nilEncoder{}) @@ -181,6 +180,8 @@ func TestRequestBuildingWithNoUserAgent(t *testing.T) { assert.Equal(t, useragent.UserAgent("Heartbeat", true), request.Header.Get("User-Agent")) } +======= +>>>>>>> 8a5dac67f9 (Add a header round tripper option to httpcommon (#27509)) func TestRequestBuildingWithExplicitUserAgent(t *testing.T) { expectedUserAgent := "some-user-agent" diff --git a/libbeat/cmd/export/dashboard.go b/libbeat/cmd/export/dashboard.go index e0cb74545ded..c41e57738119 100644 --- a/libbeat/cmd/export/dashboard.go +++ b/libbeat/cmd/export/dashboard.go @@ -52,7 +52,17 @@ func GenDashboardCmd(settings instance.Settings) *cobra.Command { b.Config.Kibana = common.NewConfig() } +<<<<<<< HEAD client, err := kibana.NewKibanaClient(b.Config.Kibana) +======= + // Initialize kibana config. If username and password is set in + // elasticsearch output config but not in kibana, initKibanaConfig + // will attach the username and password into kibana config as a + // part of the initialization. + initConfig := instance.InitKibanaConfig(b.Config) + + client, err := kibana.NewKibanaClient(initConfig, b.Info.Beat) +>>>>>>> 8a5dac67f9 (Add a header round tripper option to httpcommon (#27509)) if err != nil { fatalf("Error creating Kibana client: %+v.\n", err) } diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go index b65263add523..06beaaefea98 100644 --- a/libbeat/cmd/instance/beat.go +++ b/libbeat/cmd/instance/beat.go @@ -532,7 +532,7 @@ func (b *Beat) Setup(settings Settings, bt beat.Creator, setup SetupSettings) er if outCfg.Name() != "elasticsearch" { return fmt.Errorf("Index management requested but the Elasticsearch output is not configured/enabled") } - esClient, err := eslegclient.NewConnectedClient(outCfg.Config()) + esClient, err := eslegclient.NewConnectedClient(outCfg.Config(), b.Info.Beat) if err != nil { return err } @@ -822,7 +822,7 @@ func (b *Beat) loadDashboards(ctx context.Context, force bool) error { return fmt.Errorf("error initKibanaConfig: %v", err) } - client, err := kibana.NewKibanaClient(kibanaConfig) + client, err := kibana.NewKibanaClient(kibanaConfig, b.Info.Beat) if err != nil { return fmt.Errorf("error connecting to Kibana: %v", err) } diff --git a/libbeat/common/transport/httpcommon/httpcommon.go b/libbeat/common/transport/httpcommon/httpcommon.go index 7d4e9fb595a8..ac751f97771f 100644 --- a/libbeat/common/transport/httpcommon/httpcommon.go +++ b/libbeat/common/transport/httpcommon/httpcommon.go @@ -134,6 +134,20 @@ type extraOptionFunc func(*extraSettings) func (extraOptionFunc) sealTransportOption() {} func (fn extraOptionFunc) applyExtra(s *extraSettings) { fn(s) } +type headerRoundTripper struct { + headers map[string]string + rt http.RoundTripper +} + +func (rt *headerRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + for k, v := range rt.headers { + if len(req.Header.Get(k)) == 0 { + req.Header.Set(k, v) + } + } + return rt.rt.RoundTrip(req) +} + // DefaultHTTPTransportSettings returns the default HTTP transport setting. func DefaultHTTPTransportSettings() HTTPTransportSettings { return HTTPTransportSettings{ @@ -373,6 +387,19 @@ func WithAPMHTTPInstrumentation() TransportOption { return withAPMHTTPRountTripper } +// HeaderRoundTripper will return a RoundTripper that sets header KVs if the key is not present. +func HeaderRoundTripper(rt http.RoundTripper, headers map[string]string) http.RoundTripper { + return &headerRoundTripper{headers, rt} +} + +// WithHeaderRoundTripper instuments the HTTP client via a custom http.RoundTripper. +// This RoundTripper will add headers to each request if the key is not present. +func WithHeaderRoundTripper(headers map[string]string) TransportOption { + return WithModRoundtripper(func(rt http.RoundTripper) http.RoundTripper { + return HeaderRoundTripper(rt, headers) + }) +} + // WithLogger sets the internal logger that will be used to log dial or TCP level errors. // Logging at the connection level will only happen if the logger has been set. func WithLogger(logger *logp.Logger) TransportOption { diff --git a/libbeat/dashboards/dashboards.go b/libbeat/dashboards/dashboards.go index 8049d114dac8..1272cf05b7de 100644 --- a/libbeat/dashboards/dashboards.go +++ b/libbeat/dashboards/dashboards.go @@ -54,13 +54,13 @@ func ImportDashboards( return errors.New("kibana configuration missing for loading dashboards.") } - return setupAndImportDashboardsViaKibana(ctx, beatInfo.Hostname, kibanaConfig, &dashConfig, msgOutputter, pattern) + return setupAndImportDashboardsViaKibana(ctx, beatInfo.Hostname, beatInfo.Beat, kibanaConfig, &dashConfig, msgOutputter, pattern) } -func setupAndImportDashboardsViaKibana(ctx context.Context, hostname string, kibanaConfig *common.Config, +func setupAndImportDashboardsViaKibana(ctx context.Context, hostname, beatname string, kibanaConfig *common.Config, dashboardsConfig *Config, msgOutputter MessageOutputter, fields common.MapStr) error { - kibanaLoader, err := NewKibanaLoader(ctx, kibanaConfig, dashboardsConfig, hostname, msgOutputter) + kibanaLoader, err := NewKibanaLoader(ctx, kibanaConfig, dashboardsConfig, hostname, msgOutputter, beatname) if err != nil { return fmt.Errorf("fail to create the Kibana loader: %v", err) } diff --git a/libbeat/dashboards/kibana_loader.go b/libbeat/dashboards/kibana_loader.go index 719ac7f29526..edc624a6d0df 100644 --- a/libbeat/dashboards/kibana_loader.go +++ b/libbeat/dashboards/kibana_loader.go @@ -47,13 +47,13 @@ type KibanaLoader struct { } // NewKibanaLoader creates a new loader to load Kibana files -func NewKibanaLoader(ctx context.Context, cfg *common.Config, dashboardsConfig *Config, hostname string, msgOutputter MessageOutputter) (*KibanaLoader, error) { +func NewKibanaLoader(ctx context.Context, cfg *common.Config, dashboardsConfig *Config, hostname string, msgOutputter MessageOutputter, beatname string) (*KibanaLoader, error) { if cfg == nil || !cfg.Enabled() { return nil, fmt.Errorf("Kibana is not configured or enabled") } - client, err := getKibanaClient(ctx, cfg, dashboardsConfig.Retry, 0) + client, err := getKibanaClient(ctx, cfg, dashboardsConfig.Retry, 0, beatname) if err != nil { return nil, fmt.Errorf("Error creating Kibana client: %v", err) } @@ -73,15 +73,15 @@ func NewKibanaLoader(ctx context.Context, cfg *common.Config, dashboardsConfig * return &loader, nil } -func getKibanaClient(ctx context.Context, cfg *common.Config, retryCfg *Retry, retryAttempt uint) (*kibana.Client, error) { - client, err := kibana.NewKibanaClient(cfg) +func getKibanaClient(ctx context.Context, cfg *common.Config, retryCfg *Retry, retryAttempt uint, beatname string) (*kibana.Client, error) { + client, err := kibana.NewKibanaClient(cfg, beatname) if err != nil { if retryCfg.Enabled && (retryCfg.Maximum == 0 || retryCfg.Maximum > retryAttempt) { select { case <-ctx.Done(): return nil, err case <-time.After(retryCfg.Interval): - return getKibanaClient(ctx, cfg, retryCfg, retryAttempt+1) + return getKibanaClient(ctx, cfg, retryCfg, retryAttempt+1, beatname) } } return nil, fmt.Errorf("Error creating Kibana client: %v", err) diff --git a/libbeat/esleg/eslegclient/connection.go b/libbeat/esleg/eslegclient/connection.go index 3ebf96210bf4..8db37a9b0efd 100644 --- a/libbeat/esleg/eslegclient/connection.go +++ b/libbeat/esleg/eslegclient/connection.go @@ -34,6 +34,7 @@ import ( "github.com/elastic/beats/v7/libbeat/common/transport/httpcommon" "github.com/elastic/beats/v7/libbeat/common/transport/kerberos" "github.com/elastic/beats/v7/libbeat/common/transport/tlscommon" + "github.com/elastic/beats/v7/libbeat/common/useragent" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/testing" ) @@ -57,7 +58,8 @@ type Connection struct { // ConnectionSettings are the settings needed for a Connection type ConnectionSettings struct { - URL string + URL string + Beatname string Username string Password string @@ -110,6 +112,11 @@ func NewConnection(s ConnectionSettings) (*Connection, error) { } } + if s.Beatname == "" { + s.Beatname = "Libbeat" + } + userAgent := useragent.UserAgent(s.Beatname) + httpClient, err := s.Transport.Client( httpcommon.WithLogger(logger), httpcommon.WithIOStats(s.Observer), @@ -119,6 +126,7 @@ func NewConnection(s ConnectionSettings) (*Connection, error) { // eg, like in https://github.com/elastic/apm-server/blob/7.7/elasticsearch/client.go return apmelasticsearch.WrapRoundTripper(rt) }), + httpcommon.WithHeaderRoundTripper(map[string]string{"User-Agent": userAgent}), ) if err != nil { return nil, err @@ -160,7 +168,7 @@ func settingsWithDefaults(s ConnectionSettings) ConnectionSettings { // configuration. It accepts the same configuration parameters as the Elasticsearch // output, except for the output specific configuration options. If multiple hosts // are defined in the configuration, a client is returned for each of them. -func NewClients(cfg *common.Config) ([]Connection, error) { +func NewClients(cfg *common.Config, beatname string) ([]Connection, error) { config := defaultConfig() if err := cfg.Unpack(&config); err != nil { return nil, err @@ -185,6 +193,7 @@ func NewClients(cfg *common.Config) ([]Connection, error) { client, err := NewConnection(ConnectionSettings{ URL: esURL, + Beatname: beatname, Kerberos: config.Kerberos, Username: config.Username, Password: config.Password, @@ -205,8 +214,8 @@ func NewClients(cfg *common.Config) ([]Connection, error) { return clients, nil } -func NewConnectedClient(cfg *common.Config) (*Connection, error) { - clients, err := NewClients(cfg) +func NewConnectedClient(cfg *common.Config, beatname string) (*Connection, error) { + clients, err := NewClients(cfg, beatname) if err != nil { return nil, err } diff --git a/libbeat/kibana/client.go b/libbeat/kibana/client.go index b1414759fc0a..c4f22c376af0 100644 --- a/libbeat/kibana/client.go +++ b/libbeat/kibana/client.go @@ -36,6 +36,8 @@ import ( "github.com/joeshaw/multierror" "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/transport/httpcommon" + "github.com/elastic/beats/v7/libbeat/common/useragent" "github.com/elastic/beats/v7/libbeat/logp" ) @@ -124,22 +126,22 @@ func extractMessage(result []byte) error { } // NewKibanaClient builds and returns a new Kibana client -func NewKibanaClient(cfg *common.Config) (*Client, error) { +func NewKibanaClient(cfg *common.Config, beatname string) (*Client, error) { config := DefaultClientConfig() if err := cfg.Unpack(&config); err != nil { return nil, err } - return NewClientWithConfig(&config) + return NewClientWithConfig(&config, beatname) } // NewClientWithConfig creates and returns a kibana client using the given config -func NewClientWithConfig(config *ClientConfig) (*Client, error) { - return NewClientWithConfigDefault(config, 5601) +func NewClientWithConfig(config *ClientConfig, beatname string) (*Client, error) { + return NewClientWithConfigDefault(config, 5601, beatname) } // NewClientWithConfig creates and returns a kibana client using the given config -func NewClientWithConfigDefault(config *ClientConfig, defaultPort int) (*Client, error) { +func NewClientWithConfigDefault(config *ClientConfig, defaultPort int, beatname string) (*Client, error) { if err := config.Validate(); err != nil { return nil, err } @@ -181,7 +183,11 @@ func NewClientWithConfigDefault(config *ClientConfig, defaultPort int) (*Client, headers.Set(k, v) } - rt, err := config.Transport.Client() + if beatname == "" { + beatname = "Libbeat" + } + userAgent := useragent.UserAgent(beatname) + rt, err := config.Transport.Client(httpcommon.WithHeaderRoundTripper(map[string]string{"User-Agent": userAgent})) if err != nil { return nil, err } diff --git a/libbeat/kibana/client_test.go b/libbeat/kibana/client_test.go index 9cb5d0d47aba..27dfc53f3701 100644 --- a/libbeat/kibana/client_test.go +++ b/libbeat/kibana/client_test.go @@ -114,7 +114,7 @@ headers: content-type: text/plain accept: text/plain kbn-xsrf: 0 -`, kibanaTs.Listener.Addr().String()))) +`, kibanaTs.Listener.Addr().String())), "Testbeat") require.NoError(t, err) require.NotNil(t, client) @@ -154,7 +154,7 @@ headers: content-type: multipart/form-data; boundary=46bea21be603a2c2ea6f51571a5e1baf5ea3be8ebd7101199320607b36ff accept: text/plain kbn-xsrf: 0 -`, kibanaTs.Listener.Addr().String()))) +`, kibanaTs.Listener.Addr().String())), "Testbeat") require.NoError(t, err) require.NotNil(t, client) diff --git a/libbeat/monitoring/report/elasticsearch/elasticsearch.go b/libbeat/monitoring/report/elasticsearch/elasticsearch.go index f4e108395d75..2f1c08d9a07e 100644 --- a/libbeat/monitoring/report/elasticsearch/elasticsearch.go +++ b/libbeat/monitoring/report/elasticsearch/elasticsearch.go @@ -139,7 +139,7 @@ func makeReporter(beat beat.Info, settings report.Settings, cfg *common.Config) var clients []outputs.NetworkClient for _, host := range hosts { - client, err := makeClient(host, params, &config) + client, err := makeClient(host, params, &config, beat.Beat) if err != nil { return nil, err } @@ -303,7 +303,7 @@ func (r *reporter) snapshotLoop(namespace, prefix string, period time.Duration, } } -func makeClient(host string, params map[string]string, config *config) (outputs.NetworkClient, error) { +func makeClient(host string, params map[string]string, config *config, beatname string) (outputs.NetworkClient, error) { url, err := common.MakeURL(config.Protocol, "", host, 9200) if err != nil { return nil, err @@ -311,6 +311,7 @@ func makeClient(host string, params map[string]string, config *config) (outputs. esClient, err := eslegclient.NewConnection(eslegclient.ConnectionSettings{ URL: url, + Beatname: beatname, Username: config.Username, Password: config.Password, APIKey: config.APIKey, diff --git a/libbeat/outputs/elasticsearch/client.go b/libbeat/outputs/elasticsearch/client.go index 318678007c4b..deab29c3dcd8 100644 --- a/libbeat/outputs/elasticsearch/client.go +++ b/libbeat/outputs/elasticsearch/client.go @@ -84,6 +84,7 @@ func NewClient( conn, err := eslegclient.NewConnection(eslegclient.ConnectionSettings{ URL: s.URL, + Beatname: s.Beatname, Username: s.Username, Password: s.Password, APIKey: s.APIKey, @@ -145,6 +146,7 @@ func (client *Client) Clone() *Client { // create install a template, we don't want these to be included in the clone. connection := eslegclient.ConnectionSettings{ URL: client.conn.URL, + Beatname: client.conn.Beatname, Kerberos: client.conn.Kerberos, Username: client.conn.Username, Password: client.conn.Password, diff --git a/libbeat/outputs/elasticsearch/elasticsearch.go b/libbeat/outputs/elasticsearch/elasticsearch.go index 417043eff831..8ed2b2cbb912 100644 --- a/libbeat/outputs/elasticsearch/elasticsearch.go +++ b/libbeat/outputs/elasticsearch/elasticsearch.go @@ -92,6 +92,7 @@ func makeES( client, err = NewClient(ClientSettings{ ConnectionSettings: eslegclient.ConnectionSettings{ URL: esURL, + Beatname: beat.Beat, Kerberos: config.Kerberos, Username: config.Username, Password: config.Password, diff --git a/metricbeat/helper/http.go b/metricbeat/helper/http.go index 9dd6e6689bfe..aa08902cb855 100644 --- a/metricbeat/helper/http.go +++ b/metricbeat/helper/http.go @@ -29,10 +29,13 @@ import ( "github.com/pkg/errors" "github.com/elastic/beats/v7/libbeat/common/transport/httpcommon" + "github.com/elastic/beats/v7/libbeat/common/useragent" "github.com/elastic/beats/v7/metricbeat/helper/dialer" "github.com/elastic/beats/v7/metricbeat/mb" ) +var userAgent = useragent.UserAgent("Metricbeat") + // HTTP is a custom HTTP Client that handle the complexity of connection and retrieving information // from HTTP endpoint. type HTTP struct { @@ -87,6 +90,7 @@ func newHTTPFromConfig(config Config, name string, hostData mb.HostData) (*HTTP, client, err := config.Transport.Client( httpcommon.WithBaseDialer(dialer), httpcommon.WithAPMHTTPInstrumentation(), + httpcommon.WithHeaderRoundTripper(map[string]string{"User-Agent": userAgent}), ) if err != nil { return nil, err diff --git a/metricbeat/helper/http_test.go b/metricbeat/helper/http_test.go index 2f05085117eb..3382ce44b322 100644 --- a/metricbeat/helper/http_test.go +++ b/metricbeat/helper/http_test.go @@ -269,6 +269,31 @@ func TestOverUnixSocket(t *testing.T) { } } +func TestUserAgentCheck(t *testing.T) { + ua := "" + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ua = r.Header.Get("User-Agent") + w.WriteHeader(http.StatusOK) + })) + defer ts.Close() + + cfg := defaultConfig() + hostData := mb.HostData{ + URI: ts.URL, + SanitizedURI: ts.URL, + } + + h, err := NewHTTPFromConfig(cfg, hostData) + require.NoError(t, err) + + res, err := h.FetchResponse() + require.NoError(t, err) + res.Body.Close() + + assert.Equal(t, http.StatusOK, res.StatusCode) + assert.Contains(t, ua, "Metricbeat") +} + func checkTimeout(t *testing.T, h *HTTP) { t.Helper() diff --git a/x-pack/elastic-agent/pkg/agent/cmd/container.go b/x-pack/elastic-agent/pkg/agent/cmd/container.go index bce71fa874a9..5faba8548c4a 100644 --- a/x-pack/elastic-agent/pkg/agent/cmd/container.go +++ b/x-pack/elastic-agent/pkg/agent/cmd/container.go @@ -474,7 +474,7 @@ func kibanaClient(cfg kibanaConfig, headers map[string]string) (*kibana.Client, IgnoreVersion: true, Transport: transport, Headers: headers, - }, 0) + }, 0, "Elastic-Agent") } func findPolicy(cfg setupConfig, policies []kibanaPolicy) (*kibanaPolicy, error) { diff --git a/x-pack/filebeat/input/httpjson/input.go b/x-pack/filebeat/input/httpjson/input.go index c2e03e9744e9..f4a824b69d29 100644 --- a/x-pack/filebeat/input/httpjson/input.go +++ b/x-pack/filebeat/input/httpjson/input.go @@ -162,6 +162,7 @@ func newHTTPClient(ctx context.Context, config config) (*http.Client, error) { config.Transport.Client( httpcommon.WithAPMHTTPInstrumentation(), httpcommon.WithKeepaliveSettings{Disable: true}, + httpcommon.WithHeaderRoundTripper(map[string]string{"User-Agent": userAgent}), ) if err != nil { return nil, err diff --git a/x-pack/filebeat/input/httpjson/requester.go b/x-pack/filebeat/input/httpjson/requester.go index bf9abff19ee0..9e0e26edf2d2 100644 --- a/x-pack/filebeat/input/httpjson/requester.go +++ b/x-pack/filebeat/input/httpjson/requester.go @@ -194,7 +194,6 @@ func (r *requester) createHTTPRequest(ctx context.Context, ri *requestInfo) (*ht req = req.WithContext(ctx) req.Header.Set("Accept", "application/json") req.Header.Set("Content-Type", "application/json") - req.Header.Set("User-Agent", userAgent) if r.apiKey != "" { if r.authScheme != "" { req.Header.Set("Authorization", r.authScheme+" "+r.apiKey) From bbb4851ba4f1c2bd4f1b09b7e9756074c165472b Mon Sep 17 00:00:00 2001 From: michel-laterman Date: Wed, 8 Sep 2021 09:25:48 -0700 Subject: [PATCH 2/3] Cleanup merge --- heartbeat/monitors/active/http/http.go | 2 +- heartbeat/monitors/active/http/task.go | 5 ----- heartbeat/monitors/active/http/task_test.go | 10 ---------- libbeat/cmd/export/dashboard.go | 12 +----------- libbeat/esleg/eslegclient/connection.go | 2 +- libbeat/kibana/client.go | 2 +- metricbeat/helper/http.go | 2 +- metricbeat/helper/http_test.go | 2 +- 8 files changed, 6 insertions(+), 31 deletions(-) diff --git a/heartbeat/monitors/active/http/http.go b/heartbeat/monitors/active/http/http.go index 2532a3d1c7a0..315870e25c53 100644 --- a/heartbeat/monitors/active/http/http.go +++ b/heartbeat/monitors/active/http/http.go @@ -39,7 +39,7 @@ func init() { var debugf = logp.MakeDebug("http") -var userAgent = useragent.UserAgent("Heartbeat") +var userAgent = useragent.UserAgent("Heartbeat", true) // Create makes a new HTTP monitor func create( diff --git a/heartbeat/monitors/active/http/task.go b/heartbeat/monitors/active/http/task.go index a8ba9753b601..f273b0a4cc72 100644 --- a/heartbeat/monitors/active/http/task.go +++ b/heartbeat/monitors/active/http/task.go @@ -45,11 +45,6 @@ import ( "github.com/elastic/beats/v7/libbeat/common/transport/tlscommon" ) -<<<<<<< HEAD -var userAgent = useragent.UserAgent("Heartbeat", true) - -======= ->>>>>>> 8a5dac67f9 (Add a header round tripper option to httpcommon (#27509)) func newHTTPMonitorHostJob( addr string, config *Config, diff --git a/heartbeat/monitors/active/http/task_test.go b/heartbeat/monitors/active/http/task_test.go index ac4691491b32..342dfdec1adc 100644 --- a/heartbeat/monitors/active/http/task_test.go +++ b/heartbeat/monitors/active/http/task_test.go @@ -172,16 +172,6 @@ func TestRequestBuildingWithCustomHost(t *testing.T) { } } -<<<<<<< HEAD -func TestRequestBuildingWithNoUserAgent(t *testing.T) { - request, err := buildRequest("localhost", &Config{}, nilEncoder{}) - - require.Nil(t, err) - assert.Equal(t, useragent.UserAgent("Heartbeat", true), request.Header.Get("User-Agent")) -} - -======= ->>>>>>> 8a5dac67f9 (Add a header round tripper option to httpcommon (#27509)) func TestRequestBuildingWithExplicitUserAgent(t *testing.T) { expectedUserAgent := "some-user-agent" diff --git a/libbeat/cmd/export/dashboard.go b/libbeat/cmd/export/dashboard.go index c41e57738119..7cbe30c70bc2 100644 --- a/libbeat/cmd/export/dashboard.go +++ b/libbeat/cmd/export/dashboard.go @@ -52,17 +52,7 @@ func GenDashboardCmd(settings instance.Settings) *cobra.Command { b.Config.Kibana = common.NewConfig() } -<<<<<<< HEAD - client, err := kibana.NewKibanaClient(b.Config.Kibana) -======= - // Initialize kibana config. If username and password is set in - // elasticsearch output config but not in kibana, initKibanaConfig - // will attach the username and password into kibana config as a - // part of the initialization. - initConfig := instance.InitKibanaConfig(b.Config) - - client, err := kibana.NewKibanaClient(initConfig, b.Info.Beat) ->>>>>>> 8a5dac67f9 (Add a header round tripper option to httpcommon (#27509)) + client, err := kibana.NewKibanaClient(b.Config.Kibana, b.Info.Beat) if err != nil { fatalf("Error creating Kibana client: %+v.\n", err) } diff --git a/libbeat/esleg/eslegclient/connection.go b/libbeat/esleg/eslegclient/connection.go index 8db37a9b0efd..bc85baaf6d3a 100644 --- a/libbeat/esleg/eslegclient/connection.go +++ b/libbeat/esleg/eslegclient/connection.go @@ -115,7 +115,7 @@ func NewConnection(s ConnectionSettings) (*Connection, error) { if s.Beatname == "" { s.Beatname = "Libbeat" } - userAgent := useragent.UserAgent(s.Beatname) + userAgent := useragent.UserAgent(s.Beatname, true) httpClient, err := s.Transport.Client( httpcommon.WithLogger(logger), diff --git a/libbeat/kibana/client.go b/libbeat/kibana/client.go index c4f22c376af0..725225d79090 100644 --- a/libbeat/kibana/client.go +++ b/libbeat/kibana/client.go @@ -186,7 +186,7 @@ func NewClientWithConfigDefault(config *ClientConfig, defaultPort int, beatname if beatname == "" { beatname = "Libbeat" } - userAgent := useragent.UserAgent(beatname) + userAgent := useragent.UserAgent(beatname, true) rt, err := config.Transport.Client(httpcommon.WithHeaderRoundTripper(map[string]string{"User-Agent": userAgent})) if err != nil { return nil, err diff --git a/metricbeat/helper/http.go b/metricbeat/helper/http.go index aa08902cb855..a351f5421357 100644 --- a/metricbeat/helper/http.go +++ b/metricbeat/helper/http.go @@ -34,7 +34,7 @@ import ( "github.com/elastic/beats/v7/metricbeat/mb" ) -var userAgent = useragent.UserAgent("Metricbeat") +var userAgent = useragent.UserAgent("Metricbeat", true) // HTTP is a custom HTTP Client that handle the complexity of connection and retrieving information // from HTTP endpoint. diff --git a/metricbeat/helper/http_test.go b/metricbeat/helper/http_test.go index 3382ce44b322..ab9b63deb92a 100644 --- a/metricbeat/helper/http_test.go +++ b/metricbeat/helper/http_test.go @@ -283,7 +283,7 @@ func TestUserAgentCheck(t *testing.T) { SanitizedURI: ts.URL, } - h, err := NewHTTPFromConfig(cfg, hostData) + h, err := newHTTPFromConfig(cfg, "test", hostData) require.NoError(t, err) res, err := h.FetchResponse() From 9a3003958dd9de0fd6e9f1ee3e01ced7ff4f8e73 Mon Sep 17 00:00:00 2001 From: michel-laterman Date: Wed, 8 Sep 2021 11:29:58 -0700 Subject: [PATCH 3/3] Fix for Filebeat --- filebeat/beater/filebeat.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/filebeat/beater/filebeat.go b/filebeat/beater/filebeat.go index 2e097d319de6..214d819ab850 100644 --- a/filebeat/beater/filebeat.go +++ b/filebeat/beater/filebeat.go @@ -235,7 +235,7 @@ func (fb *Filebeat) loadModulesML(b *beat.Beat, kibanaConfig *common.Config) err } esConfig := b.Config.Output.Config() - esClient, err := eslegclient.NewConnectedClient(esConfig) + esClient, err := eslegclient.NewConnectedClient(esConfig, "Filebeat") if err != nil { return errors.Errorf("Error creating Elasticsearch client: %v", err) } @@ -256,7 +256,7 @@ func (fb *Filebeat) loadModulesML(b *beat.Beat, kibanaConfig *common.Config) err } } - kibanaClient, err := kibana.NewKibanaClient(kibanaConfig) + kibanaClient, err := kibana.NewKibanaClient(kibanaConfig, "Filebeat") if err != nil { return errors.Errorf("Error creating Kibana client: %v", err) }