From f7dc83809213fc0812af01299c15bb39be9ffb4a Mon Sep 17 00:00:00 2001 From: Jan Fajerski Date: Fri, 31 May 2024 10:29:07 +0200 Subject: [PATCH] support native histograms Signed-off-by: Jan Fajerski --- .circleci/config.yml | 2 +- .promu.yml | 2 +- README.md | 37 ++++++++ go.mod | 11 +-- go.sum | 7 ++ histogram/prometheus_model.go | 153 ++++++++++++++++++++++++++++++++++ prom2json.go | 32 +++++-- prom2json_test.go | 82 +++++++++++++++++- 8 files changed, 309 insertions(+), 17 deletions(-) create mode 100644 histogram/prometheus_model.go diff --git a/.circleci/config.yml b/.circleci/config.yml index a643bfb8..521d7c02 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,7 +6,7 @@ executors: # Whenever the Go version is updated here, .promu.yml should also be updated. golang: docker: - - image: cimg/go:1.20 + - image: cimg/go:1.21 jobs: test: executor: golang diff --git a/.promu.yml b/.promu.yml index 996c6ae3..ce610375 100644 --- a/.promu.yml +++ b/.promu.yml @@ -2,7 +2,7 @@ go: # Whenever the Go version is updated here # .circle/config.yml should also be updated. - version: 1.20 + version: 1.21 repository: path: github.com/prometheus/prom2json build: diff --git a/README.md b/README.md index 3fea4a0e..93f717e9 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,9 @@ Example input from stdin: Note that all numbers are encoded as strings. Some parsers want it that way. Also, Prometheus allows sample values like `NaN` or `+Inf`, which cannot be encoded as JSON numbers. +Native histograms are formated the same way as [the query +API](https://prometheus.io/docs/prometheus/latest/querying/api/#native-histograms) +would return. ```json [ @@ -129,6 +132,40 @@ which cannot be encoded as JSON numbers. "value": "1063110" } ] + }, + { + "name": "http_request_duration_seconds", + "type": "HISTOGRAM", + "help": "More HTTP request latencies in seconds.", + "metrics": [ + { + "labels": { + "method": "GET", + }, + "buckets": [ + [ + 0, + "17.448123722644123", + "19.027313840043536", + "139" + ], + [ + 0, + "19.027313840043536", + "20.749432874416154", + "85" + ], + [ + 0, + "20.749432874416154", + "22.62741699796952", + "70" + ], + ], + "count": "1000", + "sum": "29969.50000000001" + } + ] } ] ``` diff --git a/go.mod b/go.mod index 16964b9b..6a26019d 100644 --- a/go.mod +++ b/go.mod @@ -1,15 +1,16 @@ module github.com/prometheus/prom2json -go 1.17 +go 1.21 require ( - github.com/davecgh/go-spew v1.1.1 + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc github.com/matttproud/golang_protobuf_extensions v1.0.4 github.com/prometheus/client_model v0.6.1 - github.com/prometheus/common v0.53.0 + github.com/prometheus/common v0.54.0 + github.com/prometheus/prometheus v0.53.0 ) require ( - github.com/golang/protobuf v1.5.3 // indirect - google.golang.org/protobuf v1.33.0 // indirect + github.com/golang/protobuf v1.5.4 // indirect + google.golang.org/protobuf v1.34.1 // indirect ) diff --git a/go.sum b/go.sum index 074089ae..8cf092cd 100644 --- a/go.sum +++ b/go.sum @@ -652,6 +652,7 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -732,6 +733,7 @@ github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -899,6 +901,7 @@ github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGy github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= github.com/prometheus/common v0.53.0 h1:U2pL9w9nmJwJDa4qqLQ3ZaePJ6ZTwt7cMD3AG3+aLCE= github.com/prometheus/common v0.53.0/go.mod h1:BrxBKv3FWBIGXw89Mg1AeBq7FSyRzXWI3l3e7W3RN5U= +github.com/prometheus/common v0.54.0/go.mod h1:/TQgMJP5CuVYveyT7n/0Ix8yLNNXy9yRSkhnLTHPDIQ= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= @@ -908,6 +911,8 @@ github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0ua github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/prometheus/prometheus v0.53.0 h1:vOnhpUKrDv954jnVBvhG/ZQJ3kqscnKI+Hbdwo2tAhc= +github.com/prometheus/prometheus v0.53.0/go.mod h1:RZDkzs+ShMBDkAPQkLEaLBXpjmDcjhNxU2drUVPgKUU= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -1641,6 +1646,8 @@ google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/histogram/prometheus_model.go b/histogram/prometheus_model.go new file mode 100644 index 00000000..c35ac843 --- /dev/null +++ b/histogram/prometheus_model.go @@ -0,0 +1,153 @@ +// Copyright 2020 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package histogram + +import ( + "fmt" + + dto "github.com/prometheus/client_model/go" + model "github.com/prometheus/prometheus/model/histogram" +) + +type APIBucket[BC model.BucketCount] struct { + Boundaries uint64 + Lower, Upper float64 + Count BC +} + +func NewModelHistogram(ch *dto.Histogram) (*model.Histogram, *model.FloatHistogram) { + if ch.GetSampleCountFloat() > 0 || ch.GetZeroCountFloat() > 0 { + // It is a float histogram. + fh := model.FloatHistogram{ + Count: ch.GetSampleCountFloat(), + Sum: ch.GetSampleSum(), + ZeroThreshold: ch.GetZeroThreshold(), + ZeroCount: ch.GetZeroCountFloat(), + Schema: ch.GetSchema(), + PositiveSpans: make([]model.Span, len(ch.GetPositiveSpan())), + PositiveBuckets: ch.GetPositiveCount(), + NegativeSpans: make([]model.Span, len(ch.GetNegativeSpan())), + NegativeBuckets: ch.GetNegativeCount(), + } + for i, span := range ch.GetPositiveSpan() { + fh.PositiveSpans[i].Offset = span.GetOffset() + fh.PositiveSpans[i].Length = span.GetLength() + } + for i, span := range ch.GetNegativeSpan() { + fh.NegativeSpans[i].Offset = span.GetOffset() + fh.NegativeSpans[i].Length = span.GetLength() + } + return nil, &fh + } + h := model.Histogram{ + Count: ch.GetSampleCount(), + Sum: ch.GetSampleSum(), + ZeroThreshold: ch.GetZeroThreshold(), + ZeroCount: ch.GetZeroCount(), + Schema: ch.GetSchema(), + PositiveSpans: make([]model.Span, len(ch.GetPositiveSpan())), + PositiveBuckets: ch.GetPositiveDelta(), + NegativeSpans: make([]model.Span, len(ch.GetNegativeSpan())), + NegativeBuckets: ch.GetNegativeDelta(), + } + for i, span := range ch.GetPositiveSpan() { + h.PositiveSpans[i].Offset = span.GetOffset() + h.PositiveSpans[i].Length = span.GetLength() + } + for i, span := range ch.GetNegativeSpan() { + h.NegativeSpans[i].Offset = span.GetOffset() + h.NegativeSpans[i].Length = span.GetLength() + } + return &h, nil +} + +func BucketsAsJson[BC model.BucketCount](buckets []APIBucket[BC]) [][]interface{} { + ret := make([][]interface{}, len(buckets)) + for i, b := range buckets { + ret[i] = []interface{}{b.Boundaries, fmt.Sprintf("%v", b.Lower), fmt.Sprintf("%v", b.Upper), fmt.Sprintf("%v", b.Count)} + } + return ret +} + +func GetAPIBuckets(h *model.Histogram) []APIBucket[uint64] { + var apiBuckets []APIBucket[uint64] + var nBuckets []model.Bucket[uint64] + for it := h.NegativeBucketIterator(); it.Next(); { + bucket := it.At() + if bucket.Count != 0 { + nBuckets = append(nBuckets, it.At()) + } + } + for i := len(nBuckets) - 1; i >= 0; i-- { + apiBuckets = append(apiBuckets, makeBucket[uint64](nBuckets[i])) + } + + if h.ZeroCount != 0 { + apiBuckets = append(apiBuckets, makeBucket[uint64](h.ZeroBucket())) + } + + for it := h.PositiveBucketIterator(); it.Next(); { + bucket := it.At() + if bucket.Count != 0 { + apiBuckets = append(apiBuckets, makeBucket[uint64](bucket)) + } + } + return apiBuckets +} + +func GetAPIFloatBuckets(h *model.FloatHistogram) []APIBucket[float64] { + var apiBuckets []APIBucket[float64] + var nBuckets []model.Bucket[float64] + for it := h.NegativeBucketIterator(); it.Next(); { + bucket := it.At() + if bucket.Count != 0 { + nBuckets = append(nBuckets, it.At()) + } + } + for i := len(nBuckets) - 1; i >= 0; i-- { + apiBuckets = append(apiBuckets, makeBucket[float64](nBuckets[i])) + } + + if h.ZeroCount != 0 { + apiBuckets = append(apiBuckets, makeBucket[float64](h.ZeroBucket())) + } + + for it := h.PositiveBucketIterator(); it.Next(); { + bucket := it.At() + if bucket.Count != 0 { + apiBuckets = append(apiBuckets, makeBucket[float64](bucket)) + } + } + return apiBuckets +} + +func makeBucket[BC model.BucketCount](bucket model.Bucket[BC]) APIBucket[BC] { + boundaries := uint64(2) // () Exclusive on both sides AKA open interval. + if bucket.LowerInclusive { + if bucket.UpperInclusive { + boundaries = 3 // [] Inclusive on both sides AKA closed interval. + } else { + boundaries = 1 // [) Inclusive only on lower end AKA right open. + } + } else { + if bucket.UpperInclusive { + boundaries = 0 // (] Inclusive only on upper end AKA left open. + } + } + return APIBucket[BC]{ + Boundaries: boundaries, + Lower: bucket.Lower, + Upper: bucket.Upper, + Count: bucket.Count, + } +} diff --git a/prom2json.go b/prom2json.go index 191b83fa..62cbc6db 100644 --- a/prom2json.go +++ b/prom2json.go @@ -23,6 +23,7 @@ import ( "github.com/prometheus/common/expfmt" dto "github.com/prometheus/client_model/go" + "github.com/prometheus/prom2json/histogram" ) const acceptHeader = `application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.7,text/plain;version=0.0.4;q=0.3` @@ -56,7 +57,7 @@ type Summary struct { type Histogram struct { Labels map[string]string `json:"labels,omitempty"` TimestampMs string `json:"timestamp_ms,omitempty"` - Buckets map[string]string `json:"buckets,omitempty"` + Buckets interface{} `json:"buckets,omitempty"` Count string `json:"count"` Sum string `json:"sum"` } @@ -81,13 +82,7 @@ func NewFamily(dtoMF *dto.MetricFamily) *Family { Sum: fmt.Sprint(m.GetSummary().GetSampleSum()), } case dto.MetricType_HISTOGRAM: - mf.Metrics[i] = Histogram{ - Labels: makeLabels(m), - TimestampMs: makeTimestamp(m), - Buckets: makeBuckets(m), - Count: fmt.Sprint(m.GetHistogram().GetSampleCount()), - Sum: fmt.Sprint(m.GetHistogram().GetSampleSum()), - } + mf.Metrics[i] = makeHistogram(m) default: mf.Metrics[i] = Metric{ Labels: makeLabels(m), @@ -112,6 +107,27 @@ func getValue(m *dto.Metric) float64 { } } +func makeHistogram(m *dto.Metric) Histogram { + hist := Histogram{ + Labels: makeLabels(m), + TimestampMs: makeTimestamp(m), + Count: fmt.Sprint(m.GetHistogram().GetSampleCount()), + Sum: fmt.Sprint(m.GetHistogram().GetSampleSum()), + } + if b := makeBuckets(m); len(b) > 0 { + hist.Buckets = b + } else { + h, fh := histogram.NewModelHistogram(m.GetHistogram()) + if h == nil { + // float histogram + hist.Buckets = histogram.BucketsAsJson[float64](histogram.GetAPIFloatBuckets(fh)) + } else { + hist.Buckets = histogram.BucketsAsJson[uint64](histogram.GetAPIBuckets(h)) + } + } + return hist +} + func makeLabels(m *dto.Metric) map[string]string { result := map[string]string{} for _, lp := range m.Label { diff --git a/prom2json_test.go b/prom2json_test.go index f4d1dcca..f45ee62d 100644 --- a/prom2json_test.go +++ b/prom2json_test.go @@ -51,7 +51,7 @@ var tcs = []testCase{ createLabelPair("tag1", "foo"), createLabelPair("tag2", "bar"), }, - TimestampMs: intPtr(123456), + TimestampMs: int64Ptr(123456), Counter: &dto.Counter{ Value: floatPtr(42), }, @@ -212,6 +212,73 @@ var tcs = []testCase{ }, }, }, + testCase{ + name: "test native histograms", + mFamily: &dto.MetricFamily{ + Name: strPtr("histogram2"), + Type: metricTypePtr(dto.MetricType_HISTOGRAM), + Metric: []*dto.Metric{ + &dto.Metric{ + // Test summary with NaN + Label: []*dto.LabelPair{ + createLabelPair("tag1", "abc"), + createLabelPair("tag2", "def"), + }, + Histogram: &dto.Histogram{ + SampleCount: uintPtr(10), + SampleSum: floatPtr(123.45), + Schema: int32Ptr(1), + PositiveSpan: []*dto.BucketSpan{ + createBucketSpan(0, 3), + createBucketSpan(1, 1), + }, + PositiveDelta: []int64{1, 2, 3, 4}, + }, + }, + }, + }, + output: &Family{ + Name: "histogram2", + Help: "", + Type: "HISTOGRAM", + Metrics: []interface{}{ + Histogram{ + Labels: map[string]string{ + "tag1": "abc", + "tag2": "def", + }, + Buckets: [][]interface{}{ + { + uint64(0), + "0.7071067811865475", + "1", + "1", + }, + { + uint64(0), + "1", + "1.414213562373095", + "3", + }, + { + uint64(0), + "1.414213562373095", + "2", + "6", + }, + { + uint64(0), + "2.82842712474619", + "4", + "10", + }, + }, + Count: "10", + Sum: "123.45", + }, + }, + }, + }, } func TestConvertToMetricFamily(t *testing.T) { @@ -240,7 +307,11 @@ func uintPtr(u uint64) *uint64 { return &u } -func intPtr(i int64) *int64 { +func int32Ptr(i int32) *int32 { + return &i +} + +func int64Ptr(i int64) *int64 { return &i } @@ -264,3 +335,10 @@ func createBucket(bound float64, count uint64) *dto.Bucket { CumulativeCount: &count, } } + +func createBucketSpan(offset int32, length uint32) *dto.BucketSpan { + return &dto.BucketSpan{ + Offset: &offset, + Length: &length, + } +}