Cherry-pick elastic#7527 to 6.3: Add bearer_token_file paramter to …

…HTTP helper (elastic#7577)

* Add `bearer_token_file` paramter to HTTP helper (elastic#7527)

This change allows to load bearer tokens from files in modules using
the HTTP helper. This is especially useful for Kubernetes and Prometheus, as some deployments enforce SSL access (like OpenShift):

```
- module: kubernetes
  metricsets:
    - pod
  bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
  ssl.certificate_authorities:
    - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt
```

Closes elastic#7518

(cherry picked from commit 7b90836)

* Update CHANGELOG.asciidoc

CHANGELOG.asciidoc

@@ -57,6 +57,8 @@ https://github.com/elastic/beats/compare/v6.3.1...6.3[Check the HEAD diff]
 
 *Metricbeat*
 
+- Add support for bearer token files to HTTP helper. {pull}7527[7527]
+
 *Packetbeat*
 
 - Updated the TLS protocol parser with new cipher suites added to TLS 1.3. {issue}7455[7455]

metricbeat/autodiscover/appender/kubernetes/token/token.go

@@ -24,7 +24,7 @@ type tokenAppender struct {
 // NewTokenAppender creates a token appender that can append a bearer token required to authenticate with
 // protected endpoints
 func NewTokenAppender(cfg *common.Config) (autodiscover.Appender, error) {
-	cfgwarn.Beta("The token appender is beta")
+	cfgwarn.Deprecate("7.0.0", "token appender is deprecated in favor of bearer_token_file config parameter")
 	conf := defaultConfig()
 
 	err := cfg.Unpack(&conf)

metricbeat/docs/metricbeat-options.asciidoc

@@ -216,3 +216,9 @@ The username to use for basic authentication.
 ==== `password`
 
 The password to use for basic authentication.
+
+[float]
+==== `bearer_token_file`
+
+If defined, Metricbeat will read the contents of the file once at initialization
+and then use the value in an HTTP Authorization header.

metricbeat/docs/modules/kubernetes.asciidoc

@@ -35,7 +35,9 @@ metricbeat.modules:
   period: 10s
   hosts: ["localhost:10255"]
   enabled: true
-  #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]
+  #bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+  #ssl.certificate_authorities:
+  #  - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt
   #ssl.certificate: "/etc/pki/client/cert.pem"
   #ssl.key: "/etc/pki/client/cert.key"
 

metricbeat/docs/modules/prometheus.asciidoc

@@ -37,6 +37,11 @@ metricbeat.modules:
   hosts: ["localhost:9090"]
   #metrics_path: /metrics
   #namespace: example
+
+  # This can be used for service account based authorization:
+  #  bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+  #ssl.certificate_authorities:
+  #  - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt
 ----
 
 This module supports TLS connection when using `ssl` config field, as described in <<configuration-ssl>>. It also supports the options described in <<module-http-config-options>>.

metricbeat/helper/http.go

@@ -11,6 +11,9 @@ import (
 	"time"
 
 	"github.com/elastic/beats/libbeat/outputs"
+
+	"github.com/pkg/errors"
+
 	"github.com/elastic/beats/libbeat/outputs/transport"
 	"github.com/elastic/beats/metricbeat/mb"
 )
@@ -27,9 +30,10 @@ type HTTP struct {
 // NewHTTP creates new http helper
 func NewHTTP(base mb.BaseMetricSet) (*HTTP, error) {
 	config := struct {
-		TLS     *outputs.TLSConfig `config:"ssl"`
-		Timeout time.Duration      `config:"timeout"`
-		Headers map[string]string  `config:"headers"`
+		TLS             *outputs.TLSConfig `config:"ssl"`
+		Timeout         time.Duration      `config:"timeout"`
+		Headers         map[string]string  `config:"headers"`
+		BearerTokenFile string             `config:"bearer_token_file"`
 	}{}
 	if err := base.Module().UnpackConfig(&config); err != nil {
 		return nil, err
@@ -39,6 +43,14 @@ func NewHTTP(base mb.BaseMetricSet) (*HTTP, error) {
 		config.Headers = map[string]string{}
 	}
 
+	if config.BearerTokenFile != "" {
+		header, err := getAuthHeaderFromToken(config.BearerTokenFile)
+		if err != nil {
+			return nil, err
+		}
+		config.Headers["Authorization"] = header
+	}
+
 	tlsConfig, err := outputs.LoadTLSConfig(config.TLS)
 	if err != nil {
 		return nil, err
@@ -153,3 +165,22 @@ func (h *HTTP) FetchJSON() (map[string]interface{}, error) {
 
 	return data, nil
 }
+
+// getAuthHeaderFromToken reads a bearer authorizaiton token from the given file
+func getAuthHeaderFromToken(path string) (string, error) {
+	var token string
+
+	b, err := ioutil.ReadFile(path)
+	if err != nil {
+		return "", errors.Wrap(err, "reading bearer token file")
+	}
+
+	if len(b) != 0 {
+		if b[len(b)-1] == '\n' {
+			b = b[0 : len(b)-1]
+		}
+		token = fmt.Sprintf("Bearer %s", string(b))
+	}
+
+	return token, nil
+}

metricbeat/helper/http_test.go

@@ -0,0 +1,71 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package helper
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetAuthHeaderFromToken(t *testing.T) {
+	tests := []struct {
+		Name, Content, Expected string
+	}{
+		{
+			"Test a token is read",
+			"testtoken",
+			"Bearer testtoken",
+		},
+		{
+			"Test a token is trimmed",
+			"testtoken\n",
+			"Bearer testtoken",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.Name, func(t *testing.T) {
+			content := []byte(test.Content)
+			tmpfile, err := ioutil.TempFile("", "token")
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer os.Remove(tmpfile.Name())
+
+			if _, err := tmpfile.Write(content); err != nil {
+				t.Fatal(err)
+			}
+			if err := tmpfile.Close(); err != nil {
+				t.Fatal(err)
+			}
+
+			header, err := getAuthHeaderFromToken(tmpfile.Name())
+			assert.NoError(t, err)
+			assert.Equal(t, test.Expected, header)
+		})
+	}
+}
+
+func TestGetAuthHeaderFromTokenNoFile(t *testing.T) {
+	header, err := getAuthHeaderFromToken("nonexistingfile")
+	assert.Equal(t, "", header)
+	assert.Error(t, err)
+}

metricbeat/metricbeat.reference.yml

@@ -347,7 +347,9 @@ metricbeat.modules:
   period: 10s
   hosts: ["localhost:10255"]
   enabled: true
-  #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]
+  #bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+  #ssl.certificate_authorities:
+  #  - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt
   #ssl.certificate: "/etc/pki/client/cert.pem"
   #ssl.key: "/etc/pki/client/cert.key"
 
@@ -504,6 +506,11 @@ metricbeat.modules:
   #metrics_path: /metrics
   #namespace: example
 
+  # This can be used for service account based authorization:
+  #  bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+  #ssl.certificate_authorities:
+  #  - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt
+
 #------------------------------ RabbitMQ Module ------------------------------
 - module: rabbitmq
   metricsets: ["node", "queue", "connection"]

metricbeat/module/kubernetes/_meta/config.reference.yml

@@ -9,7 +9,9 @@
   period: 10s
   hosts: ["localhost:10255"]
   enabled: true
-  #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]
+  #bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+  #ssl.certificate_authorities:
+  #  - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt
   #ssl.certificate: "/etc/pki/client/cert.pem"
   #ssl.key: "/etc/pki/client/cert.key"
 

metricbeat/module/kubernetes/_meta/config.yml

@@ -6,8 +6,8 @@
     - system
     - volume
   hosts: ["localhost:10255"]
-  #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]
-  #ssl.certificate: "/etc/pki/client/cert.pem"
-  #ssl.key: "/etc/pki/client/cert.key"
+  bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+  ssl.certificate_authorities:
+    - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt
   #username: "user"
   #password: "secret"

metricbeat/module/prometheus/_meta/config.reference.yml

@@ -13,3 +13,8 @@
   hosts: ["localhost:9090"]
   #metrics_path: /metrics
   #namespace: example
+
+  # This can be used for service account based authorization:
+  #  bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+  #ssl.certificate_authorities:
+  #  - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt

metricbeat/module/prometheus/_meta/config.yml

@@ -4,3 +4,8 @@
   #namespace: example
   #username: "user"
   #password: "secret"
+
+  # This can be used for service account based authorization:
+  #  bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+  #ssl.certificate_authorities:
+  #  - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt

metricbeat/modules.d/kubernetes.yml.disabled

@@ -6,8 +6,8 @@
     - system
     - volume
   hosts: ["localhost:10255"]
-  #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]
-  #ssl.certificate: "/etc/pki/client/cert.pem"
-  #ssl.key: "/etc/pki/client/cert.key"
+  bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+  ssl.certificate_authorities:
+    - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt
   #username: "user"
   #password: "secret"

metricbeat/modules.d/prometheus.yml.disabled

@@ -4,3 +4,8 @@
   #namespace: example
   #username: "user"
   #password: "secret"
+
+  # This can be used for service account based authorization:
+  #  bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+  #ssl.certificate_authorities:
+  #  - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt