From 88cd8ab6bfcb480828e0a9c4d1c0c2c2754a3eda Mon Sep 17 00:00:00 2001 From: Craig MacKenzie Date: Thu, 20 Jan 2022 20:11:51 -0500 Subject: [PATCH 1/5] Set the beats product origin header by default. --- libbeat/esleg/eslegclient/connection.go | 28 +++++++++++++------- libbeat/esleg/eslegclient/connection_test.go | 19 +++++++------ 2 files changed, 29 insertions(+), 18 deletions(-) diff --git a/libbeat/esleg/eslegclient/connection.go b/libbeat/esleg/eslegclient/connection.go index d14ba8544f06..22eecff926f9 100644 --- a/libbeat/esleg/eslegclient/connection.go +++ b/libbeat/esleg/eslegclient/connection.go @@ -39,6 +39,13 @@ import ( "github.com/elastic/beats/v7/libbeat/testing" ) +const ( + // Identifies the request as originating from an Elastic product. Has the side effect of + // suppressing Elasticsearch API deprecation warnings in Kibana when set. + ProductOriginHeader = "X-Elastic-Product-Origin" + BeatsProductOrigin = "beats" +) + type esHTTPClient interface { Do(req *http.Request) (resp *http.Response, err error) CloseIdleConnections() @@ -84,7 +91,9 @@ type ConnectionSettings struct { func NewConnection(s ConnectionSettings) (*Connection, error) { logger := logp.NewLogger("esclientleg") - s = settingsWithDefaults(s) + if s.IdleConnTimeout == 0 { + s.IdleConnTimeout = 1 * time.Minute + } u, err := url.Parse(s.URL) if err != nil { @@ -117,6 +126,14 @@ func NewConnection(s ConnectionSettings) (*Connection, error) { } userAgent := useragent.UserAgent(s.Beatname) + // Default the product origin header to beats if it wasn't already set. + if _, ok := s.Headers[ProductOriginHeader]; !ok { + if s.Headers == nil { + s.Headers = make(map[string]string) + } + s.Headers[ProductOriginHeader] = BeatsProductOrigin + } + httpClient, err := s.Transport.Client( httpcommon.WithLogger(logger), httpcommon.WithIOStats(s.Observer), @@ -155,15 +172,6 @@ func NewConnection(s ConnectionSettings) (*Connection, error) { return &conn, nil } -func settingsWithDefaults(s ConnectionSettings) ConnectionSettings { - settings := s - if settings.IdleConnTimeout == 0 { - settings.IdleConnTimeout = 1 * time.Minute - } - - return settings -} - // NewClients returns a list of Elasticsearch clients based on the given // configuration. It accepts the same configuration parameters as the Elasticsearch // output, except for the output specific configuration options. If multiple hosts diff --git a/libbeat/esleg/eslegclient/connection_test.go b/libbeat/esleg/eslegclient/connection_test.go index e0735ebe992d..f91e9e50f18e 100644 --- a/libbeat/esleg/eslegclient/connection_test.go +++ b/libbeat/esleg/eslegclient/connection_test.go @@ -71,18 +71,21 @@ func TestHeaders(t *testing.T) { expected map[string][]string }{ {input: map[string]string{ - "Accept": "application/vnd.elasticsearch+json;compatible-with=7", - "Content-Type": "application/vnd.elasticsearch+json;compatible-with=7", - "X-My-Header": "true"}, + "Accept": "application/vnd.elasticsearch+json;compatible-with=7", + "Content-Type": "application/vnd.elasticsearch+json;compatible-with=7", + ProductOriginHeader: "elastic-product", + "X-My-Header": "true"}, expected: map[string][]string{ - "Accept": {"application/vnd.elasticsearch+json;compatible-with=7"}, - "Content-Type": {"application/vnd.elasticsearch+json;compatible-with=7"}, - "X-My-Header": {"true"}}}, + "Accept": {"application/vnd.elasticsearch+json;compatible-with=7"}, + "Content-Type": {"application/vnd.elasticsearch+json;compatible-with=7"}, + ProductOriginHeader: {"elastic-product"}, + "X-My-Header": {"true"}}}, {input: map[string]string{ "X-My-Header": "true"}, expected: map[string][]string{ - "Accept": {"application/json"}, - "X-My-Header": {"true"}}}, + "Accept": {"application/json"}, + ProductOriginHeader: {BeatsProductOrigin}, + "X-My-Header": {"true"}}}, } { conn, err := NewConnection(ConnectionSettings{ Headers: td.input, From 147a4a420d747d2f0d2fb696c60f60fbd1279783 Mon Sep 17 00:00:00 2001 From: Craig MacKenzie Date: Mon, 24 Jan 2022 09:46:30 -0500 Subject: [PATCH 2/5] Move product origin definitions to a package. --- libbeat/common/productorigin/productorigin.go | 30 +++++++++++++++++++ libbeat/esleg/eslegclient/connection.go | 12 ++------ libbeat/esleg/eslegclient/connection_test.go | 23 +++++++------- 3 files changed, 45 insertions(+), 20 deletions(-) create mode 100644 libbeat/common/productorigin/productorigin.go diff --git a/libbeat/common/productorigin/productorigin.go b/libbeat/common/productorigin/productorigin.go new file mode 100644 index 000000000000..d967c85f4367 --- /dev/null +++ b/libbeat/common/productorigin/productorigin.go @@ -0,0 +1,30 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Package productorigin defines the Elastic product origin header. +package productorigin + +const ( + // Identifies a request as originating from an Elastic product. Has the side effect of + // suppressing Elasticsearch API deprecation warnings in Kibana when set. + Header = "X-Elastic-Product-Origin" + + // Applicable values from https://github.com/elastic/kibana/blob/main/x-pack/plugins/upgrade_assistant/common/constants.ts#L50 + Observability = "observability" + Beats = "beats" + Fleet = "fleet" +) diff --git a/libbeat/esleg/eslegclient/connection.go b/libbeat/esleg/eslegclient/connection.go index 22eecff926f9..6f2f13aa6b2b 100644 --- a/libbeat/esleg/eslegclient/connection.go +++ b/libbeat/esleg/eslegclient/connection.go @@ -30,6 +30,7 @@ import ( "go.elastic.co/apm/module/apmelasticsearch" "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/productorigin" "github.com/elastic/beats/v7/libbeat/common/transport" "github.com/elastic/beats/v7/libbeat/common/transport/httpcommon" "github.com/elastic/beats/v7/libbeat/common/transport/kerberos" @@ -39,13 +40,6 @@ import ( "github.com/elastic/beats/v7/libbeat/testing" ) -const ( - // Identifies the request as originating from an Elastic product. Has the side effect of - // suppressing Elasticsearch API deprecation warnings in Kibana when set. - ProductOriginHeader = "X-Elastic-Product-Origin" - BeatsProductOrigin = "beats" -) - type esHTTPClient interface { Do(req *http.Request) (resp *http.Response, err error) CloseIdleConnections() @@ -127,11 +121,11 @@ func NewConnection(s ConnectionSettings) (*Connection, error) { userAgent := useragent.UserAgent(s.Beatname) // Default the product origin header to beats if it wasn't already set. - if _, ok := s.Headers[ProductOriginHeader]; !ok { + if _, ok := s.Headers[productorigin.Header]; !ok { if s.Headers == nil { s.Headers = make(map[string]string) } - s.Headers[ProductOriginHeader] = BeatsProductOrigin + s.Headers[productorigin.Header] = productorigin.Beats } httpClient, err := s.Transport.Client( diff --git a/libbeat/esleg/eslegclient/connection_test.go b/libbeat/esleg/eslegclient/connection_test.go index f91e9e50f18e..06c95021e4ea 100644 --- a/libbeat/esleg/eslegclient/connection_test.go +++ b/libbeat/esleg/eslegclient/connection_test.go @@ -24,6 +24,7 @@ import ( "net/http" "testing" + "github.com/elastic/beats/v7/libbeat/common/productorigin" "github.com/stretchr/testify/require" ) @@ -71,21 +72,21 @@ func TestHeaders(t *testing.T) { expected map[string][]string }{ {input: map[string]string{ - "Accept": "application/vnd.elasticsearch+json;compatible-with=7", - "Content-Type": "application/vnd.elasticsearch+json;compatible-with=7", - ProductOriginHeader: "elastic-product", - "X-My-Header": "true"}, + "Accept": "application/vnd.elasticsearch+json;compatible-with=7", + "Content-Type": "application/vnd.elasticsearch+json;compatible-with=7", + productorigin.Header: "elastic-product", + "X-My-Header": "true"}, expected: map[string][]string{ - "Accept": {"application/vnd.elasticsearch+json;compatible-with=7"}, - "Content-Type": {"application/vnd.elasticsearch+json;compatible-with=7"}, - ProductOriginHeader: {"elastic-product"}, - "X-My-Header": {"true"}}}, + "Accept": {"application/vnd.elasticsearch+json;compatible-with=7"}, + "Content-Type": {"application/vnd.elasticsearch+json;compatible-with=7"}, + productorigin.Header: {"elastic-product"}, + "X-My-Header": {"true"}}}, {input: map[string]string{ "X-My-Header": "true"}, expected: map[string][]string{ - "Accept": {"application/json"}, - ProductOriginHeader: {BeatsProductOrigin}, - "X-My-Header": {"true"}}}, + "Accept": {"application/json"}, + productorigin.Header: {productorigin.Beats}, + "X-My-Header": {"true"}}}, } { conn, err := NewConnection(ConnectionSettings{ Headers: td.input, From 8d58031e08c52347be60e0e40bf0fecc773d2b0e Mon Sep 17 00:00:00 2001 From: Craig MacKenzie Date: Mon, 24 Jan 2022 09:49:52 -0500 Subject: [PATCH 3/5] Add product origin header to ES and Kibana modules --- metricbeat/module/elasticsearch/metricset.go | 3 +++ metricbeat/module/kibana/settings/settings.go | 3 +++ metricbeat/module/kibana/stats/stats.go | 3 +++ metricbeat/module/kibana/status/status.go | 3 +++ 4 files changed, 12 insertions(+) diff --git a/metricbeat/module/elasticsearch/metricset.go b/metricbeat/module/elasticsearch/metricset.go index 7a7b3d863f54..3dcb456b3574 100644 --- a/metricbeat/module/elasticsearch/metricset.go +++ b/metricbeat/module/elasticsearch/metricset.go @@ -23,6 +23,7 @@ import ( "github.com/pkg/errors" + "github.com/elastic/beats/v7/libbeat/common/productorigin" "github.com/elastic/beats/v7/metricbeat/helper" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/mb/parse" @@ -90,6 +91,8 @@ func NewMetricSet(base mb.BaseMetricSet, servicePath string) (*MetricSet, error) return nil, err } + http.SetHeaderDefault(productorigin.Header, productorigin.Beats) + config := struct { Scope Scope `config:"scope"` XPackEnabled bool `config:"xpack.enabled"` diff --git a/metricbeat/module/kibana/settings/settings.go b/metricbeat/module/kibana/settings/settings.go index d0cea670b5f7..b2468bfa461c 100644 --- a/metricbeat/module/kibana/settings/settings.go +++ b/metricbeat/module/kibana/settings/settings.go @@ -20,6 +20,7 @@ package settings import ( "fmt" + "github.com/elastic/beats/v7/libbeat/common/productorigin" "github.com/elastic/beats/v7/metricbeat/helper" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/mb/parse" @@ -77,6 +78,8 @@ func (m *MetricSet) init() (err error) { return err } + httpHelper.SetHeaderDefault(productorigin.Header, productorigin.Beats) + kibanaVersion, err := kibana.GetVersion(httpHelper, kibana.SettingsPath) if err != nil { return err diff --git a/metricbeat/module/kibana/stats/stats.go b/metricbeat/module/kibana/stats/stats.go index 5551e57823f7..2f6ba8faa2f8 100644 --- a/metricbeat/module/kibana/stats/stats.go +++ b/metricbeat/module/kibana/stats/stats.go @@ -22,6 +22,7 @@ import ( "github.com/pkg/errors" + "github.com/elastic/beats/v7/libbeat/common/productorigin" "github.com/elastic/beats/v7/metricbeat/helper" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/mb/parse" @@ -84,6 +85,8 @@ func (m *MetricSet) init() error { return err } + statsHTTP.SetHeaderDefault(productorigin.Header, productorigin.Beats) + kibanaVersion, err := kibana.GetVersion(statsHTTP, kibana.StatsPath) if err != nil { return err diff --git a/metricbeat/module/kibana/status/status.go b/metricbeat/module/kibana/status/status.go index 32b38723324b..7715ca48cb91 100644 --- a/metricbeat/module/kibana/status/status.go +++ b/metricbeat/module/kibana/status/status.go @@ -18,6 +18,7 @@ package status import ( + "github.com/elastic/beats/v7/libbeat/common/productorigin" "github.com/elastic/beats/v7/metricbeat/helper" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/mb/parse" @@ -59,6 +60,8 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return nil, err } + http.SetHeaderDefault(productorigin.Header, productorigin.Beats) + return &MetricSet{ ms, http, From 8686e8ea06da9d2e5bad82bac61421e50d6a5190 Mon Sep 17 00:00:00 2001 From: Craig MacKenzie Date: Mon, 24 Jan 2022 10:31:50 -0500 Subject: [PATCH 4/5] Adjust import formatting to satisfy linter. --- libbeat/esleg/eslegclient/connection_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libbeat/esleg/eslegclient/connection_test.go b/libbeat/esleg/eslegclient/connection_test.go index 06c95021e4ea..af553d71c099 100644 --- a/libbeat/esleg/eslegclient/connection_test.go +++ b/libbeat/esleg/eslegclient/connection_test.go @@ -24,8 +24,9 @@ import ( "net/http" "testing" - "github.com/elastic/beats/v7/libbeat/common/productorigin" "github.com/stretchr/testify/require" + + "github.com/elastic/beats/v7/libbeat/common/productorigin" ) func TestAPIKeyEncoding(t *testing.T) { From 498c2b29b53d0c78c1e788d27d98bf09403dc95c Mon Sep 17 00:00:00 2001 From: Craig MacKenzie Date: Tue, 25 Jan 2022 10:52:15 -0500 Subject: [PATCH 5/5] Remove fleet as available product origin. --- libbeat/common/productorigin/productorigin.go | 1 - 1 file changed, 1 deletion(-) diff --git a/libbeat/common/productorigin/productorigin.go b/libbeat/common/productorigin/productorigin.go index d967c85f4367..133442fae908 100644 --- a/libbeat/common/productorigin/productorigin.go +++ b/libbeat/common/productorigin/productorigin.go @@ -26,5 +26,4 @@ const ( // Applicable values from https://github.com/elastic/kibana/blob/main/x-pack/plugins/upgrade_assistant/common/constants.ts#L50 Observability = "observability" Beats = "beats" - Fleet = "fleet" )