diff --git a/go.mod b/go.mod index 5f8f1775..02ee6a89 100644 --- a/go.mod +++ b/go.mod @@ -22,6 +22,7 @@ require ( github.com/gogo/protobuf v1.2.2-0.20190730201129-28a6bbf47e48 // indirect github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 // indirect github.com/golang/protobuf v1.3.2 + github.com/google/go-cmp v0.3.0 github.com/googleapis/gnostic v0.3.0 // indirect github.com/gophercloud/gophercloud v0.3.0 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 diff --git a/retrieval/resource_map.go b/retrieval/resource_map.go index 28287093..baed5095 100644 --- a/retrieval/resource_map.go +++ b/retrieval/resource_map.go @@ -42,6 +42,8 @@ func constValue(labelName string) labelTranslation { type ResourceMap struct { // The name of the Stackdriver MonitoredResource. Type string + // MatchLabel must exist in the set of Prometheus labels in order for this map to match. Ignored if empty. + MatchLabel string // Mapping from Prometheus to Stackdriver labels LabelMap map[string]labelTranslation } @@ -88,8 +90,37 @@ var GKEResourceMap = ResourceMap{ }, } -// TODO(jkohen): ensure these are sorted from more specific to less specific. -var ResourceMappings = []ResourceMap{ +var DevappResourceMap = ResourceMap{ + Type: "devapp", + MatchLabel: "__meta_kubernetes_pod_label_type_devapp", + LabelMap: map[string]labelTranslation{ + ProjectIDLabel: constValue("resource_container"), + KubernetesLocationLabel: constValue("location"), + "__meta_kubernetes_pod_label_org": constValue("org"), + "__meta_kubernetes_pod_label_env": constValue("env"), + "api_product_name": constValue("api_product_name"), + }, +} + +var ProxyResourceMap = ResourceMap{ + Type: "proxy", + MatchLabel: "__meta_kubernetes_pod_label_type_proxy", + LabelMap: map[string]labelTranslation{ + ProjectIDLabel: constValue("resource_container"), + KubernetesLocationLabel: constValue("location"), + "__meta_kubernetes_pod_label_org": constValue("org"), + "__meta_kubernetes_pod_label_env": constValue("env"), + "proxy_name": constValue("proxy_name"), + "revision": constValue("revision"), + }, +} + +type ResourceMapList []ResourceMap + +// When you add new elements, you also probably want to update TestResourceMappingsOrder. +var ResourceMappings = ResourceMapList{ + ProxyResourceMap, + DevappResourceMap, { Type: "k8s_container", LabelMap: map[string]labelTranslation{ @@ -134,40 +165,57 @@ var ResourceMappings = []ResourceMap{ }, } -func (m *ResourceMap) Translate(discovered, final labels.Labels) map[string]string { - stackdriverLabels := m.tryTranslate(discovered, final) +// Translate translates labels to a monitored resource and entry labels, if +// possible. Returns the resource and the modified entry labels. +// +// The labels in `discovered` and `entryLabels` are used as input. If a label +// exists in both sets, the one in `entryLabels` takes precedence. Whenever a +// label from `entryLabels` is used, it is removed from the set that is +// returned. +func (m *ResourceMap) Translate(discovered, entryLabels labels.Labels) (map[string]string, labels.Labels) { + stackdriverLabels, entryLabels := m.tryTranslate(discovered, entryLabels) if len(m.LabelMap) == len(stackdriverLabels) { - return stackdriverLabels + return stackdriverLabels, entryLabels } - return nil + return nil, nil } // BestEffortTranslate translates labels to resource with best effort. If the resource label // cannot be filled, use empty string instead. -func (m *ResourceMap) BestEffortTranslate(discovered, final labels.Labels) map[string]string { - stackdriverLabels := m.tryTranslate(discovered, final) +func (m *ResourceMap) BestEffortTranslate(discovered, entryLabels labels.Labels) (map[string]string, labels.Labels) { + stackdriverLabels, entryLabels := m.tryTranslate(discovered, entryLabels) for _, t := range m.LabelMap { if _, ok := stackdriverLabels[t.stackdriverLabelName]; !ok { stackdriverLabels[t.stackdriverLabelName] = "" } } - return stackdriverLabels + return stackdriverLabels, entryLabels } -func (m *ResourceMap) tryTranslate(discovered, final labels.Labels) map[string]string { +func (m *ResourceMap) tryTranslate(discovered, entryLabels labels.Labels) (map[string]string, labels.Labels) { + matched := false stackdriverLabels := make(map[string]string, len(m.LabelMap)) for _, l := range discovered { + if l.Name == m.MatchLabel { + matched = true + } if translator, ok := m.LabelMap[l.Name]; ok { stackdriverLabels[translator.stackdriverLabelName] = translator.convert(l.Value) } } - // The final labels are applied second so they overwrite mappings from discovered labels. + // The entryLabels labels are applied second so they overwrite mappings from discovered labels. // This ensures, that the Prometheus's relabeling rules are respected for labels that // appear in both label sets, e.g. the "job" label for generic resources. - for _, l := range final { + var finalLabels labels.Labels + for _, l := range entryLabels { if translator, ok := m.LabelMap[l.Name]; ok { stackdriverLabels[translator.stackdriverLabelName] = translator.convert(l.Value) + } else { + finalLabels = append(finalLabels, l) } } - return stackdriverLabels + if len(m.MatchLabel) > 0 && !matched { + return nil, finalLabels + } + return stackdriverLabels, finalLabels } diff --git a/retrieval/resource_map_test.go b/retrieval/resource_map_test.go index ce31c3b6..bc5ea525 100644 --- a/retrieval/resource_map_test.go +++ b/retrieval/resource_map_test.go @@ -17,12 +17,14 @@ import ( "reflect" "testing" + "github.com/google/go-cmp/cmp" "github.com/prometheus/prometheus/pkg/labels" ) func TestTranslate(t *testing.T) { r := ResourceMap{ - Type: "my_type", + Type: "my_type", + MatchLabel: "__match_type", LabelMap: map[string]labelTranslation{ "__target1": constValue("sdt1"), "__target2": constValue("sdt2"), @@ -33,29 +35,40 @@ func TestTranslate(t *testing.T) { noMatchTarget := labels.Labels{ {"ignored", "x"}, {"__target2", "y"}, + {"__match_type", "true"}, } - if labels := r.Translate(noMatchTarget, nil); labels != nil { + if labels, _ := r.Translate(noMatchTarget, nil); labels != nil { t.Errorf("Expected no match, matched %v", labels) } matchTargetDiscovered := labels.Labels{ {"ignored", "x"}, {"__target2", "y"}, {"__target1", "z"}, + {"__match_type", "true"}, } matchTargetFinal := labels.Labels{ {"__target1", "z2"}, {"__target3", "v"}, + {"__match_type", "true"}, } expectedLabels := map[string]string{ "sdt1": "z2", "sdt2": "y", "sdt3": "v", } - if labels := r.Translate(matchTargetDiscovered, matchTargetFinal); labels == nil { + if labels, _ := r.Translate(matchTargetDiscovered, matchTargetFinal); labels == nil { t.Errorf("Expected %v, actual nil", expectedLabels) } else if !reflect.DeepEqual(labels, expectedLabels) { t.Errorf("Expected %v, actual %v", expectedLabels, labels) } + missingType := labels.Labels{ + {"__target1", "x"}, + {"__target2", "y"}, + {"__target3", "z"}, + } + if labels, _ := r.Translate(missingType, nil); labels != nil { + t.Errorf("Expected no match, matched %v", labels) + } } func TestTranslateEc2Instance(t *testing.T) { @@ -71,7 +84,7 @@ func TestTranslateEc2Instance(t *testing.T) { "region": "aws:us-east-1b", "aws_account": "12345678", } - if labels := EC2ResourceMap.Translate(target, nil); labels == nil { + if labels, _ := EC2ResourceMap.Translate(target, nil); labels == nil { t.Errorf("Expected %v, actual nil", expectedLabels) } else if !reflect.DeepEqual(labels, expectedLabels) { t.Errorf("Expected %v, actual %v", expectedLabels, labels) @@ -89,7 +102,7 @@ func TestTranslateGceInstance(t *testing.T) { "zone": "us-central1-a", "instance_id": "1234110975759588", } - if labels := GCEResourceMap.Translate(target, nil); labels == nil { + if labels, _ := GCEResourceMap.Translate(target, nil); labels == nil { t.Errorf("Expected %v, actual nil", expectedLabels) } else if !reflect.DeepEqual(labels, expectedLabels) { t.Errorf("Expected %v, actual %v", expectedLabels, labels) @@ -111,13 +124,152 @@ func TestBestEffortTranslate(t *testing.T) { "pod_id": "", "container_name": "", } - if labels := GKEResourceMap.BestEffortTranslate(target, nil); labels == nil { + if labels, _ := GKEResourceMap.BestEffortTranslate(target, nil); labels == nil { t.Errorf("Expected %v, actual nil", expectedLabels) } else if !reflect.DeepEqual(labels, expectedLabels) { t.Errorf("Expected %v, actual %v", expectedLabels, labels) } } +func TestTranslateDevapp(t *testing.T) { + discoveredLabels := labels.Labels{ + {"__meta_kubernetes_pod_label_type_devapp", "true"}, + {ProjectIDLabel, "my-project"}, + {KubernetesLocationLabel, "us-central1-a"}, + {"__meta_kubernetes_pod_label_org", "my-org"}, + {"__meta_kubernetes_pod_label_env", "my-env"}, + } + metricLabels := labels.Labels{ + {"api_product_name", "my-name"}, + {"extra_label", "my-label"}, + } + expectedLabels := map[string]string{ + "resource_container": "my-project", + "location": "us-central1-a", + "org": "my-org", + "env": "my-env", + "api_product_name": "my-name", + } + expectedFinalLabels := labels.Labels{ + {"extra_label", "my-label"}, + } + if labels, finalLabels := DevappResourceMap.Translate(discoveredLabels, metricLabels); labels == nil { + t.Errorf("Expected %v, actual nil", expectedLabels) + } else { + if diff := cmp.Diff(expectedLabels, labels); len(diff) > 0 { + t.Error(diff) + } + if diff := cmp.Diff(expectedFinalLabels, finalLabels); len(diff) > 0 { + t.Error(diff) + } + } +} + +func TestTranslateProxy(t *testing.T) { + discoveredLabels := labels.Labels{ + {"__meta_kubernetes_pod_label_type_proxy", "true"}, + {ProjectIDLabel, "my-project"}, + {KubernetesLocationLabel, "us-central1-a"}, + {"__meta_kubernetes_pod_label_org", "my-org"}, + {"__meta_kubernetes_pod_label_env", "my-env"}, + } + metricLabels := labels.Labels{ + {"proxy_name", "my-name"}, + {"revision", "my-revision"}, + {"extra_label", "my-label"}, + } + expectedLabels := map[string]string{ + "resource_container": "my-project", + "location": "us-central1-a", + "org": "my-org", + "env": "my-env", + "proxy_name": "my-name", + "revision": "my-revision", + } + expectedFinalLabels := labels.Labels{ + {"extra_label", "my-label"}, + } + if labels, finalLabels := ProxyResourceMap.Translate(discoveredLabels, metricLabels); labels == nil { + t.Errorf("Expected %v, actual nil", expectedLabels) + } else { + if diff := cmp.Diff(expectedLabels, labels); len(diff) > 0 { + t.Error(diff) + } + if diff := cmp.Diff(expectedFinalLabels, finalLabels); len(diff) > 0 { + t.Error(diff) + } + } +} + +func (m *ResourceMapList) getByType(t string) (*ResourceMap, bool) { + for _, m := range *m { + if m.Type == t { + return &m, true + } + } + return nil, false +} + +func (m *ResourceMapList) matchType(matchLabels labels.Labels) string { + for _, m := range *m { + if lset, _ := m.Translate(matchLabels, nil); lset != nil { + return m.Type + } + } + return "" +} + +func TestResourceMappingsOrder(t *testing.T) { + // For each pair of resource types on the input, ensure that the first + // one is picked if there are labels that match both. This guarantees + // that more specific resource types are picked, e.g. k8s_container before + // k8s_pod, and k8s_node before gce_instance. + cases := []struct { + first string // Higher priority. + second string // Lower priority. + }{ + {"k8s_container", "k8s_pod"}, + {"k8s_pod", "k8s_node"}, + {"k8s_node", "gce_instance"}, + {"k8s_node", "aws_ec2_instance"}, + {"proxy", "k8s_container"}, + {"devapp", "k8s_container"}, + } + for _, c := range cases { + var ( + first, second *ResourceMap + ok bool + ) + if first, ok = ResourceMappings.getByType(c.first); !ok { + t.Fatalf("invalid test case, missing %v", c.first) + } + if second, ok = ResourceMappings.getByType(c.second); !ok { + t.Fatalf("invalid test case, missing %v", c.second) + } + // The values are uninteresting for this test. + combinedKeys := make(map[string]string) + for k, _ := range first.LabelMap { + combinedKeys[k] = "" + } + if len(first.MatchLabel) > 0 { + combinedKeys[first.MatchLabel] = "" + } + for k, _ := range second.LabelMap { + combinedKeys[k] = "" + } + if len(second.MatchLabel) > 0 { + combinedKeys[second.MatchLabel] = "" + } + combinedLabels := labels.FromMap(combinedKeys) + if match := ResourceMappings.matchType(combinedLabels); match != c.first { + t.Errorf("expected to match %v, got %v", c.first, match) + } + if match := ResourceMappings.matchType(combinedLabels); match == c.second { + t.Errorf("unexpected match %v", match) + } + } +} + func BenchmarkTranslate(b *testing.B) { r := ResourceMap{ Type: "gke_container", @@ -149,7 +301,7 @@ func BenchmarkTranslate(b *testing.B) { b.ReportAllocs() for i := 0; i < b.N; i++ { - if labels := r.Translate(discoveredLabels, finalLabels); labels == nil { + if labels, _ := r.Translate(discoveredLabels, finalLabels); labels == nil { b.Fail() } } diff --git a/retrieval/series_cache.go b/retrieval/series_cache.go index d09f7045..e12ca541 100644 --- a/retrieval/series_cache.go +++ b/retrieval/series_cache.go @@ -338,12 +338,13 @@ func (c *seriesCache) refresh(ctx context.Context, ref uint64) error { c.mtx.Unlock() entry.lastRefresh = time.Now() + entryLabels := pkgLabels(entry.lset) // Probe for the target, its applicable resource, and the series metadata. // They will be used subsequently for all other Prometheus series that map to the same complex // Stackdriver series. // If either of those pieces of data is missing, the series will be skipped. - target, err := c.targets.Get(ctx, pkgLabels(entry.lset)) + target, err := c.targets.Get(ctx, entryLabels) if err != nil { return errors.Wrap(err, "retrieving target failed") } @@ -353,11 +354,19 @@ func (c *seriesCache) refresh(ctx context.Context, ref uint64) error { level.Debug(c.logger).Log("msg", "target not found", "labels", entry.lset) return nil } + resource, entryLabels, ok := c.getResource(target.DiscoveredLabels, entryLabels) + if !ok { + ctx, _ = tag.New(ctx, tag.Insert(keyReason, "unknown_resource")) + stats.Record(ctx, droppedSeries.M(1)) + level.Debug(c.logger).Log("msg", "unknown resource", "labels", target.Labels, "discovered_labels", target.DiscoveredLabels) + return nil + } + // Remove target labels and __name__ label. // Stackdriver only accepts a limited amount of labels, so we choose to economize aggressively here. This should be OK // because we expect that the target.Labels will be redundant with the Stackdriver MonitoredResource, which is derived // from the target Labels and DiscoveredLabels. - finalLabels := targets.DropTargetLabels(pkgLabels(entry.lset), target.Labels) + finalLabels := targets.DropTargetLabels(entryLabels, target.Labels) for i, l := range finalLabels { if l.Name == "__name__" { finalLabels = append(finalLabels[:i], finalLabels[i+1:]...) @@ -372,13 +381,6 @@ func (c *seriesCache) refresh(ctx context.Context, ref uint64) error { return nil } - resource, ok := c.getResource(target.DiscoveredLabels, target.Labels) - if !ok { - ctx, _ = tag.New(ctx, tag.Insert(keyReason, "unknown_resource")) - stats.Record(ctx, droppedSeries.M(1)) - level.Debug(c.logger).Log("msg", "unknown resource", "labels", target.Labels, "discovered_labels", target.DiscoveredLabels) - return nil - } var ( metricName = entry.lset.Get("__name__") baseMetricName string @@ -472,24 +474,26 @@ func (c *seriesCache) getMetricType(prefix, name string) string { return getMetricType(prefix, name) } -func (c *seriesCache) getResource(discovered, final promlabels.Labels) (*monitoredres_pb.MonitoredResource, bool) { +// getResource returns the monitored resource, the entry labels, and whether the operation succeeded. +// The returned entry labels are a subset of `entryLabels` without the labels that were used as resource labels. +func (c *seriesCache) getResource(discovered, entryLabels promlabels.Labels) (*monitoredres_pb.MonitoredResource, promlabels.Labels, bool) { if c.useGkeResource { - if lset := GKEResourceMap.BestEffortTranslate(discovered, final); lset != nil { + if lset, finalLabels := GKEResourceMap.BestEffortTranslate(discovered, entryLabels); lset != nil { return &monitoredres_pb.MonitoredResource{ Type: GKEResourceMap.Type, Labels: lset, - }, true + }, finalLabels, true } } for _, m := range c.resourceMaps { - if lset := m.Translate(discovered, final); lset != nil { + if lset, finalLabels := m.Translate(discovered, entryLabels); lset != nil { return &monitoredres_pb.MonitoredResource{ Type: m.Type, Labels: lset, - }, true + }, finalLabels, true } } - return nil, false + return nil, entryLabels, false } // matchFiltersets checks whether any of the supplied filtersets passes. diff --git a/retrieval/transform_test.go b/retrieval/transform_test.go index ab53627f..a2c5a89c 100644 --- a/retrieval/transform_test.go +++ b/retrieval/transform_test.go @@ -16,12 +16,12 @@ package retrieval import ( "context" "math" - "reflect" "testing" "github.com/Stackdriver/stackdriver-prometheus-sidecar/targets" "github.com/go-kit/kit/log" timestamp_pb "github.com/golang/protobuf/ptypes/timestamp" + "github.com/google/go-cmp/cmp" promlabels "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/textparse" "github.com/prometheus/prometheus/scrape" @@ -65,6 +65,11 @@ func TestSampleBuilder(t *testing.T) { LabelMap: map[string]labelTranslation{ "__resource_a": constValue("resource_a"), }, + }, { + Type: "resource3", + LabelMap: map[string]labelTranslation{ + "metric_label": constValue("resource_a"), + }, }, } cases := []struct { @@ -86,18 +91,24 @@ func TestSampleBuilder(t *testing.T) { "a", "1", "b", "2", "c", "3", "d", "4", "e", "5", "f", "6", "g", "7", "h", "8", "i", "9", "j", "10"), 4: labels.FromStrings("job", "job1", "instance", "instance1", "__name__", "labelnum_bad", "a", "1", "b", "2", "c", "3", "d", "4", "e", "5", "f", "6", "g", "7", "h", "8", "i", "9", "j", "10", "k", "11"), + 5: labels.FromStrings("job", "job2", "instance", "instance1", "__name__", "resource_from_metric", "metric_label", "resource3_a", "a", "1"), }, targets: targetMap{ "job1/instance1": &targets.Target{ Labels: promlabels.FromStrings("job", "job1", "instance", "instance1"), DiscoveredLabels: promlabels.FromStrings("__resource_a", "resource2_a"), }, + "job2/instance1": &targets.Target{ + Labels: promlabels.FromStrings("job", "job2", "instance", "instance1"), + DiscoveredLabels: promlabels.FromStrings("__unused", "xxx"), + }, }, metadata: metadataMap{ - "job1/instance1/metric1": &scrape.MetricMetadata{Type: textparse.MetricTypeGauge, Metric: "metric1"}, - "job1/instance1/metric2": &scrape.MetricMetadata{Type: textparse.MetricTypeCounter, Metric: "metric2"}, - "job1/instance1/labelnum_ok": &scrape.MetricMetadata{Type: textparse.MetricTypeUnknown, Metric: "labelnum_ok"}, - "job1/instance1/labelnum_bad": &scrape.MetricMetadata{Type: textparse.MetricTypeGauge, Metric: "labelnum_bad"}, + "job1/instance1/metric1": &scrape.MetricMetadata{Type: textparse.MetricTypeGauge, Metric: "metric1"}, + "job1/instance1/metric2": &scrape.MetricMetadata{Type: textparse.MetricTypeCounter, Metric: "metric2"}, + "job1/instance1/labelnum_ok": &scrape.MetricMetadata{Type: textparse.MetricTypeUnknown, Metric: "labelnum_ok"}, + "job1/instance1/labelnum_bad": &scrape.MetricMetadata{Type: textparse.MetricTypeGauge, Metric: "labelnum_bad"}, + "job2/instance1/resource_from_metric": &scrape.MetricMetadata{Type: textparse.MetricTypeGauge, Metric: "resource_from_metric"}, }, input: []tsdb.RefSample{ {Ref: 2, T: 2000, V: 5.5}, @@ -107,6 +118,7 @@ func TestSampleBuilder(t *testing.T) { {Ref: 1, T: 1000, V: 200}, {Ref: 3, T: 3000, V: 1}, {Ref: 4, T: 4000, V: 2}, + {Ref: 5, T: 1000, V: 200}, }, result: []*monitoring_pb.TimeSeries{ nil, // Skipped by reset timestamp handling. @@ -216,6 +228,28 @@ func TestSampleBuilder(t *testing.T) { }}, }, nil, // 6: Dropped sample with too many labels. + { // 7 + Resource: &monitoredres_pb.MonitoredResource{ + Type: "resource3", + Labels: map[string]string{"resource_a": "resource3_a"}, + }, + Metric: &metric_pb.Metric{ + Type: "external.googleapis.com/prometheus/resource_from_metric", + Labels: map[string]string{ + "a": "1", + }, + }, + MetricKind: metric_pb.MetricDescriptor_GAUGE, + ValueType: metric_pb.MetricDescriptor_DOUBLE, + Points: []*monitoring_pb.Point{{ + Interval: &monitoring_pb.TimeInterval{ + EndTime: ×tamp_pb.Timestamp{Seconds: 1}, + }, + Value: &monitoring_pb.TypedValue{ + Value: &monitoring_pb.TypedValue_DoubleValue{200}, + }, + }}, + }, }, }, // Various cases where we drop series due to absence of additional information. @@ -865,26 +899,24 @@ func TestSampleBuilder(t *testing.T) { hashes = append(hashes, h) } if err == nil && c.fail { - t.Fatal("expected error but got none") + t.Error("expected error but got none") } if err != nil && !c.fail { - t.Fatalf("unexpected error: %s", err) + t.Errorf("unexpected error: %s", err) + } + if diff := cmp.Diff(c.result, result); len(diff) > 0 { + t.Errorf("unexpected result:\n%v", diff) } if len(result) != len(c.result) { - t.Fatalf("mismatching count %d of received samples, want %d", len(result), len(c.result)) + t.Errorf("mismatching count %d of received samples, want %d", len(result), len(c.result)) } - for k, res := range result { - if !reflect.DeepEqual(res, c.result[k]) { - t.Logf("gotres %v", result) - t.Logf("expres %v", c.result) - t.Fatalf("unexpected sample %d: got\n\t%v\nwant\n\t%v", k, res, c.result[k]) - } + for k, hash := range hashes { expectedHash := uint64(0) if c.result[k] != nil { expectedHash = hashSeries(c.result[k]) } - if hashes[k] != expectedHash { - t.Fatalf("unexpected hash %v; want %v", hashes[k], expectedHash) + if hash != expectedHash { + t.Errorf("unexpected hash %v; want %v", hash, expectedHash) } } }