From 25540cb4b5d7dc0b9f1ce4385a76f5d5b94b3fb7 Mon Sep 17 00:00:00 2001 From: Craig MacKenzie Date: Tue, 25 Jan 2022 12:53:38 -0500 Subject: [PATCH] Add the Elastic product origin header when talking to Elasticsearch or Kibana. (#29966) Set the beats product origin header by default when communicating with Elasticsearch or Kibana. (cherry picked from commit 5f3dd3e39deb2f062a5b051d92203d2444366825) # Conflicts: # metricbeat/module/kibana/settings/settings.go # metricbeat/module/kibana/stats/stats.go # metricbeat/module/kibana/status/status.go --- libbeat/common/productorigin/productorigin.go | 29 ++++++ libbeat/esleg/eslegclient/connection.go | 22 +++-- libbeat/esleg/eslegclient/connection_test.go | 21 ++-- metricbeat/module/elasticsearch/metricset.go | 3 + metricbeat/module/kibana/settings/settings.go | 97 +++++++++++++++++++ metricbeat/module/kibana/stats/stats.go | 7 ++ metricbeat/module/kibana/status/status.go | 6 ++ 7 files changed, 167 insertions(+), 18 deletions(-) create mode 100644 libbeat/common/productorigin/productorigin.go create mode 100644 metricbeat/module/kibana/settings/settings.go diff --git a/libbeat/common/productorigin/productorigin.go b/libbeat/common/productorigin/productorigin.go new file mode 100644 index 000000000000..133442fae908 --- /dev/null +++ b/libbeat/common/productorigin/productorigin.go @@ -0,0 +1,29 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Package productorigin defines the Elastic product origin header. +package productorigin + +const ( + // Identifies a request as originating from an Elastic product. Has the side effect of + // suppressing Elasticsearch API deprecation warnings in Kibana when set. + Header = "X-Elastic-Product-Origin" + + // Applicable values from https://github.com/elastic/kibana/blob/main/x-pack/plugins/upgrade_assistant/common/constants.ts#L50 + Observability = "observability" + Beats = "beats" +) diff --git a/libbeat/esleg/eslegclient/connection.go b/libbeat/esleg/eslegclient/connection.go index fdba8b277442..9cd4b94f5630 100644 --- a/libbeat/esleg/eslegclient/connection.go +++ b/libbeat/esleg/eslegclient/connection.go @@ -30,6 +30,7 @@ import ( "go.elastic.co/apm/module/apmelasticsearch" "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/productorigin" "github.com/elastic/beats/v7/libbeat/common/transport" "github.com/elastic/beats/v7/libbeat/common/transport/httpcommon" "github.com/elastic/beats/v7/libbeat/common/transport/kerberos" @@ -84,7 +85,9 @@ type ConnectionSettings struct { func NewConnection(s ConnectionSettings) (*Connection, error) { logger := logp.NewLogger("esclientleg") - s = settingsWithDefaults(s) + if s.IdleConnTimeout == 0 { + s.IdleConnTimeout = 1 * time.Minute + } u, err := url.Parse(s.URL) if err != nil { @@ -117,6 +120,14 @@ func NewConnection(s ConnectionSettings) (*Connection, error) { } userAgent := useragent.UserAgent(s.Beatname, true) + // Default the product origin header to beats if it wasn't already set. + if _, ok := s.Headers[productorigin.Header]; !ok { + if s.Headers == nil { + s.Headers = make(map[string]string) + } + s.Headers[productorigin.Header] = productorigin.Beats + } + httpClient, err := s.Transport.Client( httpcommon.WithLogger(logger), httpcommon.WithIOStats(s.Observer), @@ -155,15 +166,6 @@ func NewConnection(s ConnectionSettings) (*Connection, error) { return &conn, nil } -func settingsWithDefaults(s ConnectionSettings) ConnectionSettings { - settings := s - if settings.IdleConnTimeout == 0 { - settings.IdleConnTimeout = 1 * time.Minute - } - - return settings -} - // NewClients returns a list of Elasticsearch clients based on the given // configuration. It accepts the same configuration parameters as the Elasticsearch // output, except for the output specific configuration options. If multiple hosts diff --git a/libbeat/esleg/eslegclient/connection_test.go b/libbeat/esleg/eslegclient/connection_test.go index e0735ebe992d..af553d71c099 100644 --- a/libbeat/esleg/eslegclient/connection_test.go +++ b/libbeat/esleg/eslegclient/connection_test.go @@ -25,6 +25,8 @@ import ( "testing" "github.com/stretchr/testify/require" + + "github.com/elastic/beats/v7/libbeat/common/productorigin" ) func TestAPIKeyEncoding(t *testing.T) { @@ -71,18 +73,21 @@ func TestHeaders(t *testing.T) { expected map[string][]string }{ {input: map[string]string{ - "Accept": "application/vnd.elasticsearch+json;compatible-with=7", - "Content-Type": "application/vnd.elasticsearch+json;compatible-with=7", - "X-My-Header": "true"}, + "Accept": "application/vnd.elasticsearch+json;compatible-with=7", + "Content-Type": "application/vnd.elasticsearch+json;compatible-with=7", + productorigin.Header: "elastic-product", + "X-My-Header": "true"}, expected: map[string][]string{ - "Accept": {"application/vnd.elasticsearch+json;compatible-with=7"}, - "Content-Type": {"application/vnd.elasticsearch+json;compatible-with=7"}, - "X-My-Header": {"true"}}}, + "Accept": {"application/vnd.elasticsearch+json;compatible-with=7"}, + "Content-Type": {"application/vnd.elasticsearch+json;compatible-with=7"}, + productorigin.Header: {"elastic-product"}, + "X-My-Header": {"true"}}}, {input: map[string]string{ "X-My-Header": "true"}, expected: map[string][]string{ - "Accept": {"application/json"}, - "X-My-Header": {"true"}}}, + "Accept": {"application/json"}, + productorigin.Header: {productorigin.Beats}, + "X-My-Header": {"true"}}}, } { conn, err := NewConnection(ConnectionSettings{ Headers: td.input, diff --git a/metricbeat/module/elasticsearch/metricset.go b/metricbeat/module/elasticsearch/metricset.go index 22b4b2c6c499..79e1143df63a 100644 --- a/metricbeat/module/elasticsearch/metricset.go +++ b/metricbeat/module/elasticsearch/metricset.go @@ -22,6 +22,7 @@ import ( "github.com/pkg/errors" + "github.com/elastic/beats/v7/libbeat/common/productorigin" "github.com/elastic/beats/v7/metricbeat/helper" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/mb/parse" @@ -83,6 +84,8 @@ func NewMetricSet(base mb.BaseMetricSet, servicePath string) (*MetricSet, error) return nil, err } + http.SetHeaderDefault(productorigin.Header, productorigin.Beats) + config := struct { XPack bool `config:"xpack.enabled"` Scope Scope `config:"scope"` diff --git a/metricbeat/module/kibana/settings/settings.go b/metricbeat/module/kibana/settings/settings.go new file mode 100644 index 000000000000..b2468bfa461c --- /dev/null +++ b/metricbeat/module/kibana/settings/settings.go @@ -0,0 +1,97 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package settings + +import ( + "fmt" + + "github.com/elastic/beats/v7/libbeat/common/productorigin" + "github.com/elastic/beats/v7/metricbeat/helper" + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/mb/parse" + "github.com/elastic/beats/v7/metricbeat/module/kibana" +) + +// init registers the MetricSet with the central registry. +// The New method will be called after the setup of the module and before starting to fetch data +func init() { + mb.Registry.MustAddMetricSet(kibana.ModuleName, "settings", New, + mb.WithHostParser(hostParser), + ) +} + +var ( + hostParser = parse.URLHostParserBuilder{ + DefaultScheme: "http", + DefaultPath: kibana.SettingsPath, + QueryParams: "extended=true", // make Kibana fetch the cluster_uuid + }.Build() +) + +// MetricSet type defines all fields of the MetricSet +type MetricSet struct { + mb.BaseMetricSet + settingsHTTP *helper.HTTP +} + +// New create a new instance of the MetricSet +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + return &MetricSet{ + BaseMetricSet: base, + }, nil +} + +// Fetch methods implements the data gathering and data conversion to the right format +// It returns the event which is then forward to the output. In case of an error, a +// descriptive error must be returned. +func (m *MetricSet) Fetch(r mb.ReporterV2) (err error) { + if err = m.init(); err != nil { + return + } + + content, err := m.settingsHTTP.FetchContent() + if err != nil { + return + } + + return eventMapping(r, content) +} + +func (m *MetricSet) init() (err error) { + httpHelper, err := helper.NewHTTP(m.BaseMetricSet) + if err != nil { + return err + } + + httpHelper.SetHeaderDefault(productorigin.Header, productorigin.Beats) + + kibanaVersion, err := kibana.GetVersion(httpHelper, kibana.SettingsPath) + if err != nil { + return err + } + + isSettingsAPIAvailable := kibana.IsSettingsAPIAvailable(kibanaVersion) + if !isSettingsAPIAvailable { + const errorMsg = "the %v metricset is only supported with Kibana >= %v. You are currently running Kibana %v" + return fmt.Errorf(errorMsg, m.FullyQualifiedName(), kibana.SettingsAPIAvailableVersion, kibanaVersion) + } + + m.settingsHTTP, err = helper.NewHTTP(m.BaseMetricSet) + + return +} diff --git a/metricbeat/module/kibana/stats/stats.go b/metricbeat/module/kibana/stats/stats.go index 0335e814fd4a..31c8ff86232f 100644 --- a/metricbeat/module/kibana/stats/stats.go +++ b/metricbeat/module/kibana/stats/stats.go @@ -22,6 +22,7 @@ import ( "strings" "time" + "github.com/elastic/beats/v7/libbeat/common/productorigin" "github.com/elastic/beats/v7/metricbeat/helper" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/mb/parse" @@ -106,7 +107,13 @@ func (m *MetricSet) init() error { return err } +<<<<<<< HEAD kibanaVersion, err := kibana.GetVersion(statsHTTP, statsPath) +======= + statsHTTP.SetHeaderDefault(productorigin.Header, productorigin.Beats) + + kibanaVersion, err := kibana.GetVersion(statsHTTP, kibana.StatsPath) +>>>>>>> 5f3dd3e39d (Add the Elastic product origin header when talking to Elasticsearch or Kibana. (#29966)) if err != nil { return err } diff --git a/metricbeat/module/kibana/status/status.go b/metricbeat/module/kibana/status/status.go index c386ccb00101..8f9c86d4560d 100644 --- a/metricbeat/module/kibana/status/status.go +++ b/metricbeat/module/kibana/status/status.go @@ -18,8 +18,12 @@ package status import ( +<<<<<<< HEAD "fmt" +======= + "github.com/elastic/beats/v7/libbeat/common/productorigin" +>>>>>>> 5f3dd3e39d (Add the Elastic product origin header when talking to Elasticsearch or Kibana. (#29966)) "github.com/elastic/beats/v7/metricbeat/helper" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/mb/parse" @@ -65,6 +69,8 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return nil, err } + http.SetHeaderDefault(productorigin.Header, productorigin.Beats) + return &MetricSet{ ms, http,