Support ValueType override. (#191)

README.md

@@ -78,10 +78,14 @@ metric_renames:
 static_metadata:
   - metric: some_metric_name
     type: counter # or gauge/histogram
+    value_type: double # or int64
     help: an arbitrary help string
 # - ...
 ```
 
+  * All `static_metadata` entries must have `type` specified. This specifies the Stackdriver metric type and overrides the metric type chosen by the Prometheus client.
+  * If `value_type` is specified, it will override the default value type for counters and gauges. All Prometheus metrics have a default type of double.
+
 #### Counter Aggregator
 
 Counter Aggregator is an advanced feature of the sidecar that can be used to export a sum of multiple Prometheus counters to Stackdriver as a single CUMULATIVE metric.

cmd/stackdriver-prometheus-sidecar/main.go

@@ -53,11 +53,11 @@ import (
 	"github.com/prometheus/prometheus/pkg/labels"
 	"github.com/prometheus/prometheus/pkg/textparse"
 	"github.com/prometheus/prometheus/promql"
-	"github.com/prometheus/prometheus/scrape"
 	"go.opencensus.io/plugin/ocgrpc"
 	"go.opencensus.io/plugin/ochttp"
 	"go.opencensus.io/stats/view"
 	"go.opencensus.io/tag"
+	metric_pb "google.golang.org/genproto/googleapis/api/metric"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/resolver/manual"
 	kingpin "gopkg.in/alecthomas/kingpin.v2"
@@ -154,23 +154,28 @@ type genericConfig struct {
 	Namespace string
 }
 
+type metricRenamesConfig struct {
+	From string `json:"from"`
+	To   string `json:"to"`
+}
+
+type staticMetadataConfig struct {
+	Metric    string `json:"metric"`
+	Type      string `json:"type"`
+	ValueType string `json:"value_type"`
+	Help      string `json:"help"`
+}
+
+type aggregatedCountersConfig struct {
+	Metric  string   `json:"metric"`
+	Filters []string `json:"filters"`
+	Help    string   `json:"help"`
+}
+
 type fileConfig struct {
-	MetricRenames []struct {
-		From string `json:"from"`
-		To   string `json:"to"`
-	} `json:"metric_renames"`
-
-	StaticMetadata []struct {
-		Metric string `json:"metric"`
-		Type   string `json:"type"`
-		Help   string `json:"help"`
-	} `json:"static_metadata"`
-
-	AggregatedCounters []struct {
-		Metric  string   `json:"metric"`
-		Filters []string `json:"filters"`
-		Help    string   `json:"help"`
-	} `json:"aggregated_counters"`
+	MetricRenames      []metricRenamesConfig      `json:"metric_renames"`
+	StaticMetadata     []staticMetadataConfig     `json:"static_metadata"`
+	AggregatedCounters []aggregatedCountersConfig `json:"aggregated_counters"`
 }
 
 // Note: When adding a new config field, consider adding it to
@@ -192,7 +197,7 @@ type mainConfig struct {
 	Filtersets            []string
 	Aggregations          retrieval.CounterAggregatorConfig
 	MetricRenames         map[string]string
-	StaticMetadata        []scrape.MetricMetadata
+	StaticMetadata        []*metadata.Entry
 	UseRestrictedIPs      bool
 	manualResolver        *manual.Resolver
 	MonitoringBackends    []string
@@ -728,7 +733,7 @@ func fillMetadata(staticConfig *map[string]string) {
 	}
 }
 
-func parseConfigFile(filename string) (map[string]string, []scrape.MetricMetadata, retrieval.CounterAggregatorConfig, error) {
+func parseConfigFile(filename string) (map[string]string, []*metadata.Entry, retrieval.CounterAggregatorConfig, error) {
 	b, err := ioutil.ReadFile(filename)
 	if err != nil {
 		return nil, nil, nil, errors.Wrap(err, "reading file")
@@ -737,11 +742,15 @@ func parseConfigFile(filename string) (map[string]string, []scrape.MetricMetadat
 	if err := yaml.Unmarshal(b, &fc); err != nil {
 		return nil, nil, nil, errors.Wrap(err, "invalid YAML")
 	}
+	return processFileConfig(fc)
+}
+
+func processFileConfig(fc fileConfig) (map[string]string, []*metadata.Entry, retrieval.CounterAggregatorConfig, error) {
 	renameMapping := map[string]string{}
 	for _, r := range fc.MetricRenames {
 		renameMapping[r.From] = r.To
 	}
-	var staticMetadata []scrape.MetricMetadata
+	staticMetadata := []*metadata.Entry{}
 	for _, sm := range fc.StaticMetadata {
 		switch sm.Type {
 		case metadata.MetricTypeUntyped:
@@ -752,11 +761,19 @@ func parseConfigFile(filename string) (map[string]string, []scrape.MetricMetadat
 		default:
 			return nil, nil, nil, errors.Errorf("invalid metric type %q", sm.Type)
 		}
-		staticMetadata = append(staticMetadata, scrape.MetricMetadata{
-			Metric: sm.Metric,
-			Type:   textparse.MetricType(sm.Type),
-			Help:   sm.Help,
-		})
+		var valueType metric_pb.MetricDescriptor_ValueType
+		switch sm.ValueType {
+		case "double":
+			valueType = metric_pb.MetricDescriptor_DOUBLE
+		case "int64":
+			valueType = metric_pb.MetricDescriptor_INT64
+		case "":
+			valueType = metric_pb.MetricDescriptor_VALUE_TYPE_UNSPECIFIED
+		default:
+			return nil, nil, nil, errors.Errorf("invalid value type %q", sm.ValueType)
+		}
+		staticMetadata = append(staticMetadata,
+			&metadata.Entry{Metric: sm.Metric, MetricType: textparse.MetricType(sm.Type), ValueType: valueType, Help: sm.Help})
 	}
 
 	aggregations := make(retrieval.CounterAggregatorConfig)

cmd/stackdriver-prometheus-sidecar/main_test.go

@@ -15,6 +15,7 @@ package main
 
 import (
 	"bytes"
+	"errors"
 	"flag"
 	"fmt"
 	"net/http"
@@ -24,7 +25,14 @@ import (
 	"testing"
 	"time"
 
+	"github.com/Stackdriver/stackdriver-prometheus-sidecar/metadata"
+	"github.com/Stackdriver/stackdriver-prometheus-sidecar/retrieval"
 	"github.com/go-kit/kit/log"
+	"github.com/google/go-cmp/cmp"
+	"github.com/prometheus/prometheus/pkg/labels"
+	"github.com/prometheus/prometheus/pkg/textparse"
+	"github.com/prometheus/prometheus/promql"
+	metric_pb "google.golang.org/genproto/googleapis/api/metric"
 )
 
 var promPath string
@@ -171,3 +179,91 @@ func TestParseWhitelists(t *testing.T) {
 		})
 	}
 }
+
+func TestProcessFileConfig(t *testing.T) {
+	mustParseMetricSelector := func(input string) []*labels.Matcher {
+		m, err := promql.ParseMetricSelector(input)
+		if err != nil {
+			t.Fatalf("bad test input %v: %v", input, err)
+		}
+		return m
+	}
+	for _, tt := range []struct {
+		name           string
+		config         fileConfig
+		renameMappings map[string]string
+		staticMetadata []*metadata.Entry
+		aggregations   retrieval.CounterAggregatorConfig
+		err            error
+	}{
+		{
+			"empty",
+			fileConfig{},
+			map[string]string{},
+			[]*metadata.Entry{},
+			retrieval.CounterAggregatorConfig{},
+			nil,
+		},
+		{
+			"smoke",
+			fileConfig{
+				MetricRenames: []metricRenamesConfig{
+					{From: "from", To: "to"},
+				},
+				StaticMetadata: []staticMetadataConfig{
+					{Metric: "int64_counter", Type: "counter", ValueType: "int64", Help: "help1"},
+					{Metric: "double_gauge", Type: "gauge", ValueType: "double", Help: "help2"},
+					{Metric: "default_gauge", Type: "gauge"},
+				},
+				AggregatedCounters: []aggregatedCountersConfig{
+					{
+						Metric:  "network_transmit_bytes",
+						Help:    "total number of bytes sent over eth0",
+						Filters: []string{"filter1", "filter2"},
+					},
+				},
+			},
+			map[string]string{"from": "to"},
+			[]*metadata.Entry{
+				&metadata.Entry{Metric: "int64_counter", MetricType: textparse.MetricTypeCounter, ValueType: metric_pb.MetricDescriptor_INT64, Help: "help1"},
+				&metadata.Entry{Metric: "double_gauge", MetricType: textparse.MetricTypeGauge, ValueType: metric_pb.MetricDescriptor_DOUBLE, Help: "help2"},
+				&metadata.Entry{Metric: "default_gauge", MetricType: textparse.MetricTypeGauge},
+			},
+			retrieval.CounterAggregatorConfig{
+				"network_transmit_bytes": &retrieval.CounterAggregatorMetricConfig{
+					Matchers: [][]*labels.Matcher{
+						mustParseMetricSelector("filter1"),
+						mustParseMetricSelector("filter2"),
+					},
+					Help: "total number of bytes sent over eth0",
+				},
+			},
+			nil,
+		},
+		{
+			"missing_metric_type",
+			fileConfig{
+				StaticMetadata: []staticMetadataConfig{{Metric: "int64_default", ValueType: "int64"}},
+			},
+			nil, nil, nil,
+			errors.New("invalid metric type \"\""),
+		},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			renameMappings, staticMetadata, aggregations, err := processFileConfig(tt.config)
+			if diff := cmp.Diff(tt.renameMappings, renameMappings); diff != "" {
+				t.Errorf("renameMappings mismatch: %v", diff)
+			}
+			if diff := cmp.Diff(tt.staticMetadata, staticMetadata); diff != "" {
+				t.Errorf("staticMetadata mismatch: %v", diff)
+			}
+			if diff := cmp.Diff(tt.aggregations, aggregations); diff != "" {
+				t.Errorf("aggregations mismatch: %v", diff)
+			}
+			if (tt.err != nil && err != nil && tt.err.Error() != err.Error()) ||
+				(tt.err == nil && err != nil) || (tt.err != nil && err == nil) {
+				t.Errorf("error mismatch: got %v, expected %v", err, tt.err)
+			}
+		})
+	}
+}

cmd/stackdriver-prometheus-sidecar/statusz-tmpl.html

@@ -111,7 +111,7 @@ <h2>Static metadata</h2>
     <table>
       <tr><th>metric</th><th>type</th><th>unit</th></tr>
       {{range .StaticMetadata}}
-      <tr><td>{{.Metric}}</td><td>{{.Type}}</td><td>{{.Unit}}</td></tr>
+      <tr><td>{{.Metric}}</td><td>{{.MetricType}}</td><td>{{.ValueType}}</td></tr>
       {{end}}
     </table>
     {{else}}

cmd/stackdriver-prometheus-sidecar/statusz_test.go

@@ -22,12 +22,13 @@ import (
 	"regexp"
 	"testing"
 
+	"github.com/Stackdriver/stackdriver-prometheus-sidecar/metadata"
 	"github.com/Stackdriver/stackdriver-prometheus-sidecar/retrieval"
 	"github.com/go-kit/kit/log"
 	"github.com/prometheus/common/promlog"
 	"github.com/prometheus/prometheus/pkg/labels"
 	"github.com/prometheus/prometheus/pkg/textparse"
-	"github.com/prometheus/prometheus/scrape"
+	metric_pb "google.golang.org/genproto/googleapis/api/metric"
 )
 
 func scrapeStatusz(handler *statuszHandler) ([]byte, error) {
@@ -88,9 +89,9 @@ func TestStatuszHandler(t *testing.T) {
 			ProjectIDResource:  "my-project",
 			PrometheusURL:      mustParseURL(t, "http://127.0.0.1:9090/"),
 			StackdriverAddress: mustParseURL(t, "https://monitoring.googleapis.com:443/"),
-			StaticMetadata: []scrape.MetricMetadata{
-				{"metric1", textparse.MetricType("type1"), "", "unit1"},
-				{"metric2", textparse.MetricType("type2"), "", "unit2"},
+			StaticMetadata: []*metadata.Entry{
+				&metadata.Entry{Metric: "metric1", MetricType: textparse.MetricType("type1"), ValueType: metric_pb.MetricDescriptor_INT64},
+				&metadata.Entry{Metric: "metric2", MetricType: textparse.MetricType("type2"), ValueType: metric_pb.MetricDescriptor_DOUBLE},
 			},
 			StoreInFilesDirectory: "/my/files/directory",
 			UseGKEResource:        true,
@@ -136,8 +137,8 @@ func TestStatuszHandler(t *testing.T) {
 		regexp.MustCompile(`<tr><td>from2</td><td>to2</td></tr>`),
 		// for static metadata
 		regexp.MustCompile(`<h2>Static metadata</h2>`),
-		regexp.MustCompile(`<tr><td>metric1</td><td>type1</td><td>unit1</td></tr>`),
-		regexp.MustCompile(`<tr><td>metric2</td><td>type2</td><td>unit2</td></tr>`),
+		regexp.MustCompile(`<tr><td>metric1</td><td>type1</td><td>INT64</td></tr>`),
+		regexp.MustCompile(`<tr><td>metric2</td><td>type2</td><td>DOUBLE</td></tr>`),
 		// for aggregations
 		regexp.MustCompile(`<h2>Aggregations</h2>`),
 		regexp.MustCompile(`<tr><td>aggmetric1</td><td>\[\[k=.*v.*\]\]</td></tr>`),