From 507f1eda6c643571f37367f3d67b9b93b55da7be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Thu, 12 Jul 2018 12:34:32 +0200 Subject: [PATCH] Cherry-pick #7527 to 6.3: Add `bearer_token_file` paramter to HTTP helper (#7577) * Add `bearer_token_file` paramter to HTTP helper (#7527) This change allows to load bearer tokens from files in modules using the HTTP helper. This is especially useful for Kubernetes and Prometheus, as some deployments enforce SSL access (like OpenShift): ``` - module: kubernetes metricsets: - pod bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token ssl.certificate_authorities: - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt ``` Closes #7518 (cherry picked from commit 7b9083662189b457a6f0f2b41f66e8e35cf91a11) * Update CHANGELOG.asciidoc --- CHANGELOG.asciidoc | 2 + .../appender/kubernetes/token/token.go | 2 +- metricbeat/docs/metricbeat-options.asciidoc | 6 ++ metricbeat/docs/modules/kubernetes.asciidoc | 4 +- metricbeat/docs/modules/prometheus.asciidoc | 5 ++ metricbeat/helper/http.go | 37 +++++++++- metricbeat/helper/http_test.go | 71 +++++++++++++++++++ metricbeat/metricbeat.reference.yml | 9 ++- .../kubernetes/_meta/config.reference.yml | 4 +- metricbeat/module/kubernetes/_meta/config.yml | 6 +- .../prometheus/_meta/config.reference.yml | 5 ++ metricbeat/module/prometheus/_meta/config.yml | 5 ++ metricbeat/modules.d/kubernetes.yml.disabled | 6 +- metricbeat/modules.d/prometheus.yml.disabled | 5 ++ 14 files changed, 154 insertions(+), 13 deletions(-) create mode 100644 metricbeat/helper/http_test.go diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 2ab9fcdd052..515adf8a7b0 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -57,6 +57,8 @@ https://github.com/elastic/beats/compare/v6.3.1...6.3[Check the HEAD diff] *Metricbeat* +- Add support for bearer token files to HTTP helper. {pull}7527[7527] + *Packetbeat* - Updated the TLS protocol parser with new cipher suites added to TLS 1.3. {issue}7455[7455] diff --git a/metricbeat/autodiscover/appender/kubernetes/token/token.go b/metricbeat/autodiscover/appender/kubernetes/token/token.go index 4ceaa7ad333..acc81010671 100644 --- a/metricbeat/autodiscover/appender/kubernetes/token/token.go +++ b/metricbeat/autodiscover/appender/kubernetes/token/token.go @@ -24,7 +24,7 @@ type tokenAppender struct { // NewTokenAppender creates a token appender that can append a bearer token required to authenticate with // protected endpoints func NewTokenAppender(cfg *common.Config) (autodiscover.Appender, error) { - cfgwarn.Beta("The token appender is beta") + cfgwarn.Deprecate("7.0.0", "token appender is deprecated in favor of bearer_token_file config parameter") conf := defaultConfig() err := cfg.Unpack(&conf) diff --git a/metricbeat/docs/metricbeat-options.asciidoc b/metricbeat/docs/metricbeat-options.asciidoc index 73ffd1c7ef6..fffd55d38c8 100644 --- a/metricbeat/docs/metricbeat-options.asciidoc +++ b/metricbeat/docs/metricbeat-options.asciidoc @@ -216,3 +216,9 @@ The username to use for basic authentication. ==== `password` The password to use for basic authentication. + +[float] +==== `bearer_token_file` + +If defined, Metricbeat will read the contents of the file once at initialization +and then use the value in an HTTP Authorization header. \ No newline at end of file diff --git a/metricbeat/docs/modules/kubernetes.asciidoc b/metricbeat/docs/modules/kubernetes.asciidoc index 22d08b1b76e..de0c5508608 100644 --- a/metricbeat/docs/modules/kubernetes.asciidoc +++ b/metricbeat/docs/modules/kubernetes.asciidoc @@ -35,7 +35,9 @@ metricbeat.modules: period: 10s hosts: ["localhost:10255"] enabled: true - #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + #bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + #ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt #ssl.certificate: "/etc/pki/client/cert.pem" #ssl.key: "/etc/pki/client/cert.key" diff --git a/metricbeat/docs/modules/prometheus.asciidoc b/metricbeat/docs/modules/prometheus.asciidoc index d25d1618726..cada7468f0d 100644 --- a/metricbeat/docs/modules/prometheus.asciidoc +++ b/metricbeat/docs/modules/prometheus.asciidoc @@ -37,6 +37,11 @@ metricbeat.modules: hosts: ["localhost:9090"] #metrics_path: /metrics #namespace: example + + # This can be used for service account based authorization: + # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + #ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt ---- This module supports TLS connection when using `ssl` config field, as described in <>. It also supports the options described in <>. diff --git a/metricbeat/helper/http.go b/metricbeat/helper/http.go index e5fbf250ceb..70c066f5eb3 100644 --- a/metricbeat/helper/http.go +++ b/metricbeat/helper/http.go @@ -11,6 +11,9 @@ import ( "time" "github.com/elastic/beats/libbeat/outputs" + + "github.com/pkg/errors" + "github.com/elastic/beats/libbeat/outputs/transport" "github.com/elastic/beats/metricbeat/mb" ) @@ -27,9 +30,10 @@ type HTTP struct { // NewHTTP creates new http helper func NewHTTP(base mb.BaseMetricSet) (*HTTP, error) { config := struct { - TLS *outputs.TLSConfig `config:"ssl"` - Timeout time.Duration `config:"timeout"` - Headers map[string]string `config:"headers"` + TLS *outputs.TLSConfig `config:"ssl"` + Timeout time.Duration `config:"timeout"` + Headers map[string]string `config:"headers"` + BearerTokenFile string `config:"bearer_token_file"` }{} if err := base.Module().UnpackConfig(&config); err != nil { return nil, err @@ -39,6 +43,14 @@ func NewHTTP(base mb.BaseMetricSet) (*HTTP, error) { config.Headers = map[string]string{} } + if config.BearerTokenFile != "" { + header, err := getAuthHeaderFromToken(config.BearerTokenFile) + if err != nil { + return nil, err + } + config.Headers["Authorization"] = header + } + tlsConfig, err := outputs.LoadTLSConfig(config.TLS) if err != nil { return nil, err @@ -153,3 +165,22 @@ func (h *HTTP) FetchJSON() (map[string]interface{}, error) { return data, nil } + +// getAuthHeaderFromToken reads a bearer authorizaiton token from the given file +func getAuthHeaderFromToken(path string) (string, error) { + var token string + + b, err := ioutil.ReadFile(path) + if err != nil { + return "", errors.Wrap(err, "reading bearer token file") + } + + if len(b) != 0 { + if b[len(b)-1] == '\n' { + b = b[0 : len(b)-1] + } + token = fmt.Sprintf("Bearer %s", string(b)) + } + + return token, nil +} diff --git a/metricbeat/helper/http_test.go b/metricbeat/helper/http_test.go new file mode 100644 index 00000000000..a3e5d615e4d --- /dev/null +++ b/metricbeat/helper/http_test.go @@ -0,0 +1,71 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package helper + +import ( + "io/ioutil" + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetAuthHeaderFromToken(t *testing.T) { + tests := []struct { + Name, Content, Expected string + }{ + { + "Test a token is read", + "testtoken", + "Bearer testtoken", + }, + { + "Test a token is trimmed", + "testtoken\n", + "Bearer testtoken", + }, + } + + for _, test := range tests { + t.Run(test.Name, func(t *testing.T) { + content := []byte(test.Content) + tmpfile, err := ioutil.TempFile("", "token") + if err != nil { + t.Fatal(err) + } + defer os.Remove(tmpfile.Name()) + + if _, err := tmpfile.Write(content); err != nil { + t.Fatal(err) + } + if err := tmpfile.Close(); err != nil { + t.Fatal(err) + } + + header, err := getAuthHeaderFromToken(tmpfile.Name()) + assert.NoError(t, err) + assert.Equal(t, test.Expected, header) + }) + } +} + +func TestGetAuthHeaderFromTokenNoFile(t *testing.T) { + header, err := getAuthHeaderFromToken("nonexistingfile") + assert.Equal(t, "", header) + assert.Error(t, err) +} diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 8e297217eb4..8de1f7f1efd 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -347,7 +347,9 @@ metricbeat.modules: period: 10s hosts: ["localhost:10255"] enabled: true - #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + #bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + #ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt #ssl.certificate: "/etc/pki/client/cert.pem" #ssl.key: "/etc/pki/client/cert.key" @@ -504,6 +506,11 @@ metricbeat.modules: #metrics_path: /metrics #namespace: example + # This can be used for service account based authorization: + # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + #ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt + #------------------------------ RabbitMQ Module ------------------------------ - module: rabbitmq metricsets: ["node", "queue", "connection"] diff --git a/metricbeat/module/kubernetes/_meta/config.reference.yml b/metricbeat/module/kubernetes/_meta/config.reference.yml index 584cc73507e..4f38fa02745 100644 --- a/metricbeat/module/kubernetes/_meta/config.reference.yml +++ b/metricbeat/module/kubernetes/_meta/config.reference.yml @@ -9,7 +9,9 @@ period: 10s hosts: ["localhost:10255"] enabled: true - #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + #bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + #ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt #ssl.certificate: "/etc/pki/client/cert.pem" #ssl.key: "/etc/pki/client/cert.key" diff --git a/metricbeat/module/kubernetes/_meta/config.yml b/metricbeat/module/kubernetes/_meta/config.yml index f12530d27f0..48a0bfdc432 100644 --- a/metricbeat/module/kubernetes/_meta/config.yml +++ b/metricbeat/module/kubernetes/_meta/config.yml @@ -6,8 +6,8 @@ - system - volume hosts: ["localhost:10255"] - #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] - #ssl.certificate: "/etc/pki/client/cert.pem" - #ssl.key: "/etc/pki/client/cert.key" + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + ssl.certificate_authorities: + - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt #username: "user" #password: "secret" diff --git a/metricbeat/module/prometheus/_meta/config.reference.yml b/metricbeat/module/prometheus/_meta/config.reference.yml index dda648a4243..59680f2a901 100644 --- a/metricbeat/module/prometheus/_meta/config.reference.yml +++ b/metricbeat/module/prometheus/_meta/config.reference.yml @@ -13,3 +13,8 @@ hosts: ["localhost:9090"] #metrics_path: /metrics #namespace: example + + # This can be used for service account based authorization: + # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + #ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt diff --git a/metricbeat/module/prometheus/_meta/config.yml b/metricbeat/module/prometheus/_meta/config.yml index bcb50ffd37d..1d5f9437374 100644 --- a/metricbeat/module/prometheus/_meta/config.yml +++ b/metricbeat/module/prometheus/_meta/config.yml @@ -4,3 +4,8 @@ #namespace: example #username: "user" #password: "secret" + + # This can be used for service account based authorization: + # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + #ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt diff --git a/metricbeat/modules.d/kubernetes.yml.disabled b/metricbeat/modules.d/kubernetes.yml.disabled index f12530d27f0..48a0bfdc432 100644 --- a/metricbeat/modules.d/kubernetes.yml.disabled +++ b/metricbeat/modules.d/kubernetes.yml.disabled @@ -6,8 +6,8 @@ - system - volume hosts: ["localhost:10255"] - #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] - #ssl.certificate: "/etc/pki/client/cert.pem" - #ssl.key: "/etc/pki/client/cert.key" + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + ssl.certificate_authorities: + - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt #username: "user" #password: "secret" diff --git a/metricbeat/modules.d/prometheus.yml.disabled b/metricbeat/modules.d/prometheus.yml.disabled index bcb50ffd37d..1d5f9437374 100644 --- a/metricbeat/modules.d/prometheus.yml.disabled +++ b/metricbeat/modules.d/prometheus.yml.disabled @@ -4,3 +4,8 @@ #namespace: example #username: "user" #password: "secret" + + # This can be used for service account based authorization: + # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + #ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt