Merge pull request #8 from mxinden/merge-directxman12

Merge tag directxman12/v0.4.1 into openshift/master

README.md

@@ -1,13 +1,13 @@
-Kubernetes Custom Metrics Adapter for Prometheus
-================================================
+# Prometheus Adapter for Kubernetes Metrics APIs
 
 [![Build Status](https://travis-ci.org/DirectXMan12/k8s-prometheus-adapter.svg?branch=master)](https://travis-ci.org/DirectXMan12/k8s-prometheus-adapter)
 
-This repository contains an implementation of the Kubernetes custom
-metrics API
-([custom.metrics.k8s.io/v1beta1](https://github.com/kubernetes/metrics/tree/master/pkg/apis/custom_metrics)),
-suitable for use with the autoscaling/v2 Horizontal Pod Autoscaler in
-Kubernetes 1.6+.
+This repository contains an implementation of the Kubernetes
+[resource metrics](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/instrumentation/resource-metrics-api.md) API and
+[custom metrics](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/instrumentation/custom-metrics-api.md) API.
+
+This adapter is therefore suitable for use with the autoscaling/v2 Horizontal Pod Autoscaler in Kubernetes 1.6+.  
+It can also replace the [metrics server](https://github.com/kubernetes-incubator/metrics-server) on clusters that already run Prometheus and collect the appropriate metrics.
 
 Quick Links
 -----------
@@ -62,7 +62,7 @@ Presentation
 ------------
 
 The adapter gathers the names of available metrics from Prometheus
-a regular interval (see [Configuration](#configuration) above), and then
+at a regular interval (see [Configuration](#configuration) above), and then
 only exposes metrics that follow specific forms.
 
 The rules governing this discovery are specified in a [configuration file](docs/config.md).

cmd/adapter/adapter.go

@@ -17,8 +17,11 @@ limitations under the License.
 package main
 
 import (
+	"crypto/tls"
+	"crypto/x509"
 	"flag"
 	"fmt"
+	"io/ioutil"
 	"net/http"
 	"net/url"
 	"os"
@@ -32,6 +35,7 @@ import (
 	"k8s.io/apiserver/pkg/util/logs"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/client-go/transport"
 
 	prom "github.com/directxman12/k8s-prometheus-adapter/pkg/client"
 	mprom "github.com/directxman12/k8s-prometheus-adapter/pkg/client/metrics"
@@ -49,10 +53,16 @@ type PrometheusAdapter struct {
 	PrometheusAuthInCluster bool
 	// PrometheusAuthConf is the kubeconfig file that contains auth details used to connect to Prometheus
 	PrometheusAuthConf string
+	// PrometheusCAFile points to the file containing the ca-root for connecting with Prometheus
+	PrometheusCAFile string
+	// PrometheusTokenFile points to the file that contains the bearer token when connecting with Prometheus
+	PrometheusTokenFile string
 	// AdapterConfigFile points to the file containing the metrics discovery configuration.
 	AdapterConfigFile string
 	// MetricsRelistInterval is the interval at which to relist the set of available metrics
 	MetricsRelistInterval time.Duration
+	// MetricsMaxAge is the period to query available metrics for
+	MetricsMaxAge time.Duration
 
 	metricsConfig *adaptercfg.MetricsDiscoveryConfig
 }
@@ -62,11 +72,34 @@ func (cmd *PrometheusAdapter) makePromClient() (prom.Client, error) {
 	if err != nil {
 		return nil, fmt.Errorf("invalid Prometheus URL %q: %v", baseURL, err)
 	}
-	promHTTPClient, err := makeHTTPClient(cmd.PrometheusAuthInCluster, cmd.PrometheusAuthConf)
-	if err != nil {
-		return nil, err
+
+	var httpClient *http.Client
+
+	if cmd.PrometheusCAFile != "" {
+		prometheusCAClient, err := makePrometheusCAClient(cmd.PrometheusCAFile)
+		if err != nil {
+			return nil, err
+		}
+		httpClient = prometheusCAClient
+		glog.Info("successfully loaded ca from file")
+	} else {
+		kubeconfigHTTPClient, err := makeKubeconfigHTTPClient(cmd.PrometheusAuthInCluster, cmd.PrometheusAuthConf)
+		if err != nil {
+			return nil, err
+		}
+		httpClient = kubeconfigHTTPClient
+		glog.Info("successfully using in-cluster auth")
+	}
+
+	if cmd.PrometheusTokenFile != "" {
+		data, err := ioutil.ReadFile(cmd.PrometheusTokenFile)
+		if err != nil {
+			return nil, fmt.Errorf("failed to read prometheus-token-file: %v", err)
+		}
+		httpClient.Transport = transport.NewBearerAuthRoundTripper(string(data), httpClient.Transport)
 	}
-	genericPromClient := prom.NewGenericAPIClient(promHTTPClient, baseURL)
+
+	genericPromClient := prom.NewGenericAPIClient(httpClient, baseURL)
 	instrumentedGenericPromClient := mprom.InstrumentGenericAPIClient(genericPromClient, baseURL.String())
 	return prom.NewClientForAPI(instrumentedGenericPromClient), nil
 }
@@ -78,11 +111,17 @@ func (cmd *PrometheusAdapter) addFlags() {
 		"use auth details from the in-cluster kubeconfig when connecting to prometheus.")
 	cmd.Flags().StringVar(&cmd.PrometheusAuthConf, "prometheus-auth-config", cmd.PrometheusAuthConf,
 		"kubeconfig file used to configure auth when connecting to Prometheus.")
+	cmd.Flags().StringVar(&cmd.PrometheusCAFile, "prometheus-ca-file", cmd.PrometheusCAFile,
+		"Optional CA file to use when connecting with Prometheus")
+	cmd.Flags().StringVar(&cmd.PrometheusTokenFile, "prometheus-token-file", cmd.PrometheusTokenFile,
+		"Optional file containing the bearer token to use when connecting with Prometheus")
 	cmd.Flags().StringVar(&cmd.AdapterConfigFile, "config", cmd.AdapterConfigFile,
 		"Configuration file containing details of how to transform between Prometheus metrics "+
 			"and custom metrics API resources")
 	cmd.Flags().DurationVar(&cmd.MetricsRelistInterval, "metrics-relist-interval", cmd.MetricsRelistInterval, ""+
 		"interval at which to re-list the set of all available metrics from Prometheus")
+	cmd.Flags().DurationVar(&cmd.MetricsMaxAge, "metrics-max-age", cmd.MetricsMaxAge, ""+
+		"period for which to query the set of available metrics from Prometheus")
 }
 
 func (cmd *PrometheusAdapter) loadConfig() error {
@@ -105,6 +144,10 @@ func (cmd *PrometheusAdapter) makeProvider(promClient prom.Client, stopCh <-chan
 		return nil, nil
 	}
 
+	if cmd.MetricsMaxAge < cmd.MetricsRelistInterval {
+		return nil, fmt.Errorf("max age must not be less than relist interval")
+	}
+
 	// grab the mapper and dynamic client
 	mapper, err := cmd.RESTMapper()
 	if err != nil {
@@ -122,7 +165,7 @@ func (cmd *PrometheusAdapter) makeProvider(promClient prom.Client, stopCh <-chan
 	}
 
 	// construct the provider and start it
-	cmProvider, runner := cmprov.NewPrometheusProvider(mapper, dynClient, promClient, namers, cmd.MetricsRelistInterval)
+	cmProvider, runner := cmprov.NewPrometheusProvider(mapper, dynClient, promClient, namers, cmd.MetricsRelistInterval, cmd.MetricsMaxAge)
 	runner.RunUntil(stopCh)
 
 	return cmProvider, nil
@@ -173,11 +216,14 @@ func main() {
 	cmd := &PrometheusAdapter{
 		PrometheusURL:         "https://localhost",
 		MetricsRelistInterval: 10 * time.Minute,
+		MetricsMaxAge:         20 * time.Minute,
 	}
 	cmd.Name = "prometheus-metrics-adapter"
 	cmd.addFlags()
 	cmd.Flags().AddGoFlagSet(flag.CommandLine) // make sure we get the glog flags
-	cmd.Flags().Parse(os.Args)
+	if err := cmd.Flags().Parse(os.Args); err != nil {
+		glog.Fatalf("unable to parse flags: %v", err)
+	}
 
 	// make the prometheus client
 	promClient, err := cmd.makePromClient()
@@ -212,8 +258,8 @@ func main() {
 	}
 }
 
-// makeHTTPClient constructs an HTTP for connecting with the given auth options.
-func makeHTTPClient(inClusterAuth bool, kubeConfigPath string) (*http.Client, error) {
+// makeKubeconfigHTTPClient constructs an HTTP for connecting with the given auth options.
+func makeKubeconfigHTTPClient(inClusterAuth bool, kubeConfigPath string) (*http.Client, error) {
 	// make sure we're not trying to use two different sources of auth
 	if inClusterAuth && kubeConfigPath != "" {
 		return nil, fmt.Errorf("may not use both in-cluster auth and an explicit kubeconfig at the same time")
@@ -246,3 +292,23 @@ func makeHTTPClient(inClusterAuth bool, kubeConfigPath string) (*http.Client, er
 	}
 	return &http.Client{Transport: tr}, nil
 }
+
+func makePrometheusCAClient(caFilename string) (*http.Client, error) {
+	data, err := ioutil.ReadFile(caFilename)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read prometheus-ca-file: %v", err)
+	}
+
+	pool := x509.NewCertPool()
+	if !pool.AppendCertsFromPEM(data) {
+		return nil, fmt.Errorf("no certs found in prometheus-ca-file")
+	}
+
+	return &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{
+				RootCAs: pool,
+			},
+		},
+	}, nil
+}

docs/config-walkthrough.md

@@ -9,18 +9,18 @@ Per-pod HTTP Requests
 
 ### Background
 
-*The [full walkthrough](/docs/walkthrough.md) sets up a the background for
+*The [full walkthrough](/docs/walkthrough.md) sets up the background for
 something like this*
 
-Suppose we have some frontend webserver, and we're trying to write an
-configuration for the Promtheus adapter so that we can autoscale it based
+Suppose we have some frontend webserver, and we're trying to write a
+configuration for the Prometheus adapter so that we can autoscale it based
 on the HTTP requests per second that it receives.
 
 Before starting, we've gone and instrumented our frontend server with
 a metric, `http_requests_total`.  It is exposed with a single label,
 `method`, breaking down the requests by HTTP verb.
 
-We've configured our Prometheus to collect the metric, and our promethues
+We've configured our Prometheus to collect the metric, and it
 adds the `kubernetes_namespace` and `kubernetes_pod_name` labels,
 representing namespace and pod, respectively.
 

docs/config.md

@@ -179,7 +179,7 @@ template:
   group-resource, plus the label for namespace, if the group-resource is
   namespaced.
 - `GroupBy`: a comma-separated list of labels to group by.  Currently,
-  this contains the group-resoure label used in `LabelMarchers`.
+  this contains the group-resource label used in `LabelMatchers`.
 
 For instance, suppose we had a series `http_requests_total` (exposed as
 `http_requests_per_second` in the API) with labels `service`, `pod`,

docs/walkthrough.md

@@ -20,7 +20,7 @@ Detailed instructions can be found in the Kubernetes documentation under
 [Horizontal Pod
 Autoscaling](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-custom-metrics).
 
-Make sure that you've properly configured metrics-server (as is default in
+Make sure that you've properly configured metrics-server (as default in
 Kubernetes 1.9+), or enabling custom metrics autoscaling support will
 disable CPU autoscaling support.
 
@@ -34,15 +34,14 @@ significantly different.
 In order to follow this walkthrough, you'll need container images for
 Prometheus and the custom metrics adapter.
 
-It's easiest to deploy Prometheus with the [Prometheus
-Operator](https://coreos.com/operators/prometheus/docs/latest/), which
+The [Prometheus Operator](https://coreos.com/operators/prometheus/docs/latest/),
 makes it easy to get up and running with Prometheus.  This walkthrough
 will assume you're planning on doing that -- if you've deployed it by hand
 instead, you'll need to make a few adjustments to the way you expose
 metrics to Prometheus.
 
-The adapter has different images for each arch, and can be found at
-`directxman12/k8s-prometheus-adapter-${ARCH}`.  For instance, if you're on
+The adapter has different images for each arch, which can be found at
+`directxman12/k8s-prometheus-adapter-${ARCH}`. For instance, if you're on
 an x86_64 machine, use the `directxman12/k8s-prometheus-adapter-amd64`
 image.
 
@@ -172,7 +171,7 @@ for the Operator to deploy a copy of Prometheus.
 
 This walkthrough assumes that Prometheus is deployed in the `prom`
 namespace. Most of the sample commands and files are namespace-agnostic,
-but there are a few commands or pieces of configuration that rely on
+but there are a few commands or pieces of configuration that rely on that
 namespace.  If you're using a different namespace, simply substitute that
 in for `prom` when it appears.
 

pkg/client/helpers.go

@@ -22,14 +22,14 @@ import (
 
 // LabelNeq produces a not-equal label selector expression.
 // Label is passed verbatim, and value is double-quote escaped
-// using Go's escaping is used on value (as per the PromQL rules).
+// using Go's escaping (as per the PromQL rules).
 func LabelNeq(label string, value string) string {
 	return fmt.Sprintf("%s!=%q", label, value)
 }
 
 // LabelEq produces a equal label selector expression.
 // Label is passed verbatim, and value is double-quote escaped
-// using Go's escaping is used on value (as per the PromQL rules).
+// using Go's escaping (as per the PromQL rules).
 func LabelEq(label string, value string) string {
 	return fmt.Sprintf("%s=%q", label, value)
 }
@@ -52,7 +52,7 @@ func NameMatches(expr string) string {
 }
 
 // NameNotMatches produces a label selector expression that checks that the series name doesn't matches the given expression.
-// It's a convinience wrapper around LabelNotMatches.
+// It's a convenience wrapper around LabelNotMatches.
 func NameNotMatches(expr string) string {
 	return LabelNotMatches("__name__", expr)
 }

pkg/client/interfaces.go

@@ -27,7 +27,7 @@ import (
 
 // NB: the official prometheus API client at https://github.com/prometheus/client_golang
 // is rather lackluster -- as of the time of writing of this file, it lacked support
-// for querying the series metadata, which we need for the adapter.  Instead, we use
+// for querying the series metadata, which we need for the adapter. Instead, we use
 // this client.
 
 // Selector represents a series selector

pkg/config/config.go

@@ -7,13 +7,13 @@ import (
 type MetricsDiscoveryConfig struct {
 	// Rules specifies how to discover and map Prometheus metrics to
 	// custom metrics API resources.  The rules are applied independently,
-	// and thus must be mutually exclusive.  Rules will the same SeriesQuery
+	// and thus must be mutually exclusive.  Rules with the same SeriesQuery
 	// will make only a single API call.
 	Rules         []DiscoveryRule `yaml:"rules"`
 	ResourceRules *ResourceRules  `yaml:"resourceRules,omitempty"`
 }
 
-// DiscoveryRule describes on set of rules for transforming Prometheus metrics to/from
+// DiscoveryRule describes a set of rules for transforming Prometheus metrics to/from
 // custom metrics API resources.
 type DiscoveryRule struct {
 	// SeriesQuery specifies which metrics this rule should consider via a Prometheus query

pkg/custom-provider/metric_namer.go

@@ -23,11 +23,11 @@ var groupNameSanitizer = strings.NewReplacer(".", "_", "-", "_")
 // themselves be normalized.
 type MetricNamer interface {
 	// Selector produces the appropriate Prometheus series selector to match all
-	// series handlable by this namer.
+	// series handable by this namer.
 	Selector() prom.Selector
 	// FilterSeries checks to see which of the given series match any additional
-	// constrains beyond the series query.  It's assumed that the series given
-	// already matche the series query.
+	// constraints beyond the series query.  It's assumed that the series given
+	// already match the series query.
 	FilterSeries(series []prom.Series) []prom.Series
 	// MetricNameForSeries returns the name (as presented in the API) for a given series.
 	MetricNameForSeries(series prom.Series) (string, error)

pkg/custom-provider/provider.go

@@ -55,9 +55,10 @@ type prometheusProvider struct {
 	SeriesRegistry
 }
 
-func NewPrometheusProvider(mapper apimeta.RESTMapper, kubeClient dynamic.Interface, promClient prom.Client, namers []MetricNamer, updateInterval time.Duration) (provider.CustomMetricsProvider, Runnable) {
+func NewPrometheusProvider(mapper apimeta.RESTMapper, kubeClient dynamic.Interface, promClient prom.Client, namers []MetricNamer, updateInterval time.Duration, maxAge time.Duration) (provider.CustomMetricsProvider, Runnable) {
 	lister := &cachingMetricsLister{
 		updateInterval: updateInterval,
+		maxAge:         maxAge,
 		promClient:     promClient,
 		namers:         namers,
 
@@ -191,6 +192,7 @@ type cachingMetricsLister struct {
 
 	promClient     prom.Client
 	updateInterval time.Duration
+	maxAge         time.Duration
 	namers         []MetricNamer
 }
 
@@ -212,7 +214,7 @@ type selectorSeries struct {
 }
 
 func (l *cachingMetricsLister) updateMetrics() error {
-	startTime := pmodel.Now().Add(-1 * l.updateInterval)
+	startTime := pmodel.Now().Add(-1 * l.maxAge)
 
 	// don't do duplicate queries when it's just the matchers that change
 	seriesCacheByQuery := make(map[prom.Selector][]prom.Series)

pkg/custom-provider/provider_test.go

@@ -32,6 +32,7 @@ import (
 )
 
 const fakeProviderUpdateInterval = 2 * time.Second
+const fakeProviderStartDuration = 2 * time.Second
 
 func setupPrometheusProvider() (provider.CustomMetricsProvider, *fakeprom.FakePrometheusClient) {
 	fakeProm := &fakeprom.FakePrometheusClient{}
@@ -41,7 +42,7 @@ func setupPrometheusProvider() (provider.CustomMetricsProvider, *fakeprom.FakePr
 	namers, err := NamersFromConfig(cfg, restMapper())
 	Expect(err).NotTo(HaveOccurred())
 
-	prov, _ := NewPrometheusProvider(restMapper(), fakeKubeClient, fakeProm, namers, fakeProviderUpdateInterval)
+	prov, _ := NewPrometheusProvider(restMapper(), fakeKubeClient, fakeProm, namers, fakeProviderUpdateInterval, fakeProviderStartDuration)
 
 	containerSel := prom.MatchSeries("", prom.NameMatches("^container_.*"), prom.LabelNeq("container_name", "POD"), prom.LabelNeq("namespace", ""), prom.LabelNeq("pod_name", ""))
 	namespacedSel := prom.MatchSeries("", prom.LabelNeq("namespace", ""), prom.NameNotMatches("^container_.*"))

pkg/resourceprovider/provider.go

@@ -118,9 +118,10 @@ type nsQueryResults struct {
 	err       error
 }
 
+// GetContainerMetrics implements the provider.MetricsProvider interface. It may return nil, nil, nil.
 func (p *resourceProvider) GetContainerMetrics(pods ...apitypes.NamespacedName) ([]provider.TimeInfo, [][]metrics.ContainerMetrics, error) {
 	if len(pods) == 0 {
-		return nil, nil, fmt.Errorf("no pods to fetch metrics for")
+		return nil, nil, nil
 	}
 
 	// TODO(directxman12): figure out how well this scales if we go to list 1000+ pods
@@ -238,9 +239,10 @@ func (p *resourceProvider) assignForPod(pod apitypes.NamespacedName, resultsByNs
 	*resMetrics = containerMetricsList
 }
 
+// GetNodeMetrics implements the provider.MetricsProvider interface. It may return nil, nil, nil.
 func (p *resourceProvider) GetNodeMetrics(nodes ...string) ([]provider.TimeInfo, []corev1.ResourceList, error) {
 	if len(nodes) == 0 {
-		return nil, nil, fmt.Errorf("no nodes to fetch metrics for")
+		return nil, nil, nil
 	}
 
 	now := pmodel.Now()