From 3b6032bf20dceb6c35467b007982e1603de80173 Mon Sep 17 00:00:00 2001 From: Jeanette Tan Date: Thu, 8 Dec 2022 15:49:22 +0800 Subject: [PATCH] Add histograms to model Signed-off-by: Jeanette Tan --- model/value.go | 70 +++++++---- model/value_histogram.go | 173 +++++++++++++++++++++++++++ model/value_histogram_test.go | 217 ++++++++++++++++++++++++++++++++++ model/value_test.go | 146 ++++++++++++++++++++++- 4 files changed, 582 insertions(+), 24 deletions(-) create mode 100644 model/value_histogram.go create mode 100644 model/value_histogram_test.go diff --git a/model/value.go b/model/value.go index c9d8fb1a..017faacc 100644 --- a/model/value.go +++ b/model/value.go @@ -111,9 +111,10 @@ func (s SamplePair) String() string { // Sample is a sample pair associated with a metric. type Sample struct { - Metric Metric `json:"metric"` - Value SampleValue `json:"value"` - Timestamp Time `json:"timestamp"` + Metric Metric `json:"metric"` + Value SampleValue `json:"value"` + Timestamp Time `json:"timestamp"` + Histogram SampleHistogram `json:"histogram"` } // Equal compares first the metrics, then the timestamp, then the value. The @@ -129,8 +130,11 @@ func (s *Sample) Equal(o *Sample) bool { if !s.Timestamp.Equal(o.Timestamp) { return false } + if !s.Value.Equal(o.Value) { + return false + } - return s.Value.Equal(o.Value) + return s.Histogram.Equal(&o.Histogram) } func (s Sample) String() string { @@ -142,31 +146,49 @@ func (s Sample) String() string { // MarshalJSON implements json.Marshaler. func (s Sample) MarshalJSON() ([]byte, error) { - v := struct { - Metric Metric `json:"metric"` - Value SamplePair `json:"value"` - }{ - Metric: s.Metric, - Value: SamplePair{ - Timestamp: s.Timestamp, - Value: s.Value, - }, + if s.Histogram.Count != 0 { + v := struct { + Metric Metric `json:"metric"` + Histogram SampleHistogramPair `json:"histogram"` + }{ + Metric: s.Metric, + Histogram: SampleHistogramPair{ + Timestamp: s.Timestamp, + Histogram: s.Histogram, + }, + } + return json.Marshal(&v) + } else { + v := struct { + Metric Metric `json:"metric"` + Value SamplePair `json:"value"` + }{ + Metric: s.Metric, + Value: SamplePair{ + Timestamp: s.Timestamp, + Value: s.Value, + }, + } + return json.Marshal(&v) } - - return json.Marshal(&v) } // UnmarshalJSON implements json.Unmarshaler. func (s *Sample) UnmarshalJSON(b []byte) error { v := struct { - Metric Metric `json:"metric"` - Value SamplePair `json:"value"` + Metric Metric `json:"metric"` + Value SamplePair `json:"value"` + Histogram SampleHistogramPair `json:"histogram"` }{ Metric: s.Metric, Value: SamplePair{ Timestamp: s.Timestamp, Value: s.Value, }, + Histogram: SampleHistogramPair{ + Timestamp: s.Timestamp, + Histogram: s.Histogram, + }, } if err := json.Unmarshal(b, &v); err != nil { @@ -174,8 +196,13 @@ func (s *Sample) UnmarshalJSON(b []byte) error { } s.Metric = v.Metric - s.Timestamp = v.Value.Timestamp - s.Value = v.Value.Value + if v.Histogram.Timestamp != 0 { + s.Timestamp = v.Histogram.Timestamp + s.Histogram = v.Histogram.Histogram + } else { + s.Timestamp = v.Value.Timestamp + s.Value = v.Value.Value + } return nil } @@ -221,8 +248,9 @@ func (s Samples) Equal(o Samples) bool { // SampleStream is a stream of Values belonging to an attached COWMetric. type SampleStream struct { - Metric Metric `json:"metric"` - Values []SamplePair `json:"values"` + Metric Metric `json:"metric"` + Values []SamplePair `json:"values"` + Histograms []SampleHistogramPair `json:"histograms"` } func (ss SampleStream) String() string { diff --git a/model/value_histogram.go b/model/value_histogram.go new file mode 100644 index 00000000..5c0926df --- /dev/null +++ b/model/value_histogram.go @@ -0,0 +1,173 @@ +// Copyright 2013 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package model + +import ( + "encoding/json" + "fmt" + "strconv" +) + +type IntString int64 + +func (v IntString) String() string { + return fmt.Sprintf("%d", v) +} + +func (v IntString) MarshalJSON() ([]byte, error) { + return json.Marshal(v.String()) +} + +func (v *IntString) UnmarshalJSON(b []byte) error { + if len(b) < 2 || b[0] != '"' || b[len(b)-1] != '"' { + return fmt.Errorf("int value must be a quoted string") + } + i, err := strconv.ParseInt(string(b[1:len(b)-1]), 10, 64) + if err != nil { + return err + } + *v = IntString(i) + return nil +} + +func (v IntString) Equal(o IntString) bool { + return v == o +} + +type FloatString float64 + +func (v FloatString) String() string { + return strconv.FormatFloat(float64(v), 'f', -1, 64) +} + +func (v FloatString) MarshalJSON() ([]byte, error) { + return json.Marshal(v.String()) +} + +func (v *FloatString) UnmarshalJSON(b []byte) error { + if len(b) < 2 || b[0] != '"' || b[len(b)-1] != '"' { + return fmt.Errorf("float value must be a quoted string") + } + f, err := strconv.ParseFloat(string(b[1:len(b)-1]), 64) + if err != nil { + return err + } + *v = FloatString(f) + return nil +} + +func (v FloatString) Equal(o FloatString) bool { + return v == o +} + +type HistogramBucket struct { + Boundaries int + Lower FloatString + Upper FloatString + Count IntString +} + +func (s HistogramBucket) MarshalJSON() ([]byte, error) { + b, err := json.Marshal(s.Boundaries) + if err != nil { + return nil, err + } + l, err := json.Marshal(s.Lower) + if err != nil { + return nil, err + } + u, err := json.Marshal(s.Upper) + if err != nil { + return nil, err + } + c, err := json.Marshal(s.Count) + if err != nil { + return nil, err + } + return []byte(fmt.Sprintf("[%s,%s,%s,%s]", b, l, u, c)), nil +} + +func (s *HistogramBucket) UnmarshalJSON(buf []byte) error { + tmp := []interface{}{&s.Boundaries, &s.Lower, &s.Upper, &s.Count} + wantLen := len(tmp) + if err := json.Unmarshal(buf, &tmp); err != nil { + return err + } + if g, e := len(tmp), wantLen; g != e { + return fmt.Errorf("wrong number of fields: %d != %d", g, e) + } + return nil +} + +func (s *HistogramBucket) Equal(o *HistogramBucket) bool { + return s == o || (s.Boundaries == o.Boundaries && s.Lower.Equal(o.Lower) && s.Upper.Equal(o.Upper) && s.Count.Equal(o.Count)) +} + +type HistogramBuckets []*HistogramBucket + +func (s HistogramBuckets) Equal(o HistogramBuckets) bool { + if len(s) != len(o) { + return false + } + + for i, bucket := range s { + if !bucket.Equal(o[i]) { + return false + } + } + return true +} + +type SampleHistogram struct { + Count IntString `json:"count"` + Sum FloatString `json:"sum"` + Buckets HistogramBuckets `json:"buckets"` +} + +func (s *SampleHistogram) Equal(o *SampleHistogram) bool { + return s == o || (s.Count.Equal(o.Count) && s.Sum.Equal(o.Sum) && s.Buckets.Equal(o.Buckets)) +} + +type SampleHistogramPair struct { + Timestamp Time + Histogram SampleHistogram +} + +func (s SampleHistogramPair) MarshalJSON() ([]byte, error) { + t, err := json.Marshal(s.Timestamp) + if err != nil { + return nil, err + } + v, err := json.Marshal(s.Histogram) + if err != nil { + return nil, err + } + return []byte(fmt.Sprintf("[%s,%s]", t, v)), nil +} + +func (s *SampleHistogramPair) UnmarshalJSON(buf []byte) error { + tmp := []interface{}{&s.Timestamp, &s.Histogram} + wantLen := len(tmp) + if err := json.Unmarshal(buf, &tmp); err != nil { + return err + } + if g, e := len(tmp), wantLen; g != e { + return fmt.Errorf("wrong number of fields: %d != %d", g, e) + } + return nil +} + +func (s *SampleHistogramPair) Equal(o *SampleHistogramPair) bool { + return s == o || (s.Histogram.Equal(&o.Histogram) && s.Timestamp.Equal(o.Timestamp)) +} diff --git a/model/value_histogram_test.go b/model/value_histogram_test.go new file mode 100644 index 00000000..d5b3033c --- /dev/null +++ b/model/value_histogram_test.go @@ -0,0 +1,217 @@ +// Copyright 2013 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package model + +import ( + "encoding/json" + "reflect" + "testing" +) + +func TestSampleHistogramPairJSON(t *testing.T) { + input := []struct { + plain string + value SampleHistogramPair + }{ + { + plain: `[1234.567,{"count":"1","sum":"4500","buckets":[[0,"4466.7196729968955","4870.992343051145","1"]]}]`, + value: SampleHistogramPair{ + Histogram: SampleHistogram{ + Count: 1, + Sum: 4500, + Buckets: HistogramBuckets{ + { + Boundaries: 0, + Lower: 4466.7196729968955, + Upper: 4870.992343051145, + Count: 1, + }, + }, + }, + Timestamp: 1234567, + }, + }, + } + + for _, test := range input { + b, err := json.Marshal(test.value) + if err != nil { + t.Error(err) + continue + } + + if string(b) != test.plain { + t.Errorf("encoding error: expected %q, got %q", test.plain, b) + continue + } + + var sp SampleHistogramPair + err = json.Unmarshal(b, &sp) + if err != nil { + t.Error(err) + continue + } + + if !sp.Equal(&test.value) { + t.Errorf("decoding error: expected %v, got %v", test.value, sp) + } + } +} + +func TestSampleHistogramJSON(t *testing.T) { + input := []struct { + plain string + value Sample + }{ + { + plain: `{"metric":{"__name__":"test_metric"},"histogram":[1234.567,{"count":"1","sum":"4500","buckets":[[0,"4466.7196729968955","4870.992343051145","1"]]}]}`, + value: Sample{ + Metric: Metric{ + MetricNameLabel: "test_metric", + }, + Histogram: SampleHistogram{ + Count: 1, + Sum: 4500, + Buckets: HistogramBuckets{ + { + Boundaries: 0, + Lower: 4466.7196729968955, + Upper: 4870.992343051145, + Count: 1, + }, + }, + }, + Timestamp: 1234567, + }, + }, + } + + for _, test := range input { + b, err := json.Marshal(test.value) + if err != nil { + t.Error(err) + continue + } + + if string(b) != test.plain { + t.Errorf("encoding error: expected %q, got %q", test.plain, b) + continue + } + + var sv Sample + err = json.Unmarshal(b, &sv) + if err != nil { + t.Error(err) + continue + } + + if !reflect.DeepEqual(sv, test.value) { + t.Errorf("decoding error: expected %v, got %v", test.value, sv) + } + } +} + +func TestVectorHistogramJSON(t *testing.T) { + input := []struct { + plain string + value Vector + }{ + { + plain: `[{"metric":{"__name__":"test_metric"},"histogram":[1234.567,{"count":"1","sum":"4500","buckets":[[0,"4466.7196729968955","4870.992343051145","1"]]}]}]`, + value: Vector{&Sample{ + Metric: Metric{ + MetricNameLabel: "test_metric", + }, + Histogram: SampleHistogram{ + Count: 1, + Sum: 4500, + Buckets: HistogramBuckets{ + { + Boundaries: 0, + Lower: 4466.7196729968955, + Upper: 4870.992343051145, + Count: 1, + }, + }, + }, + Timestamp: 1234567, + }}, + }, + { + plain: `[{"metric":{"__name__":"test_metric"},"histogram":[1234.567,{"count":"1","sum":"4500","buckets":[[0,"4466.7196729968955","4870.992343051145","1"]]}]},{"metric":{"foo":"bar"},"histogram":[1.234,{"count":"1","sum":"4500","buckets":[[0,"4466.7196729968955","4870.992343051145","1"]]}]}]`, + value: Vector{ + &Sample{ + Metric: Metric{ + MetricNameLabel: "test_metric", + }, + Histogram: SampleHistogram{ + Count: 1, + Sum: 4500, + Buckets: HistogramBuckets{ + { + Boundaries: 0, + Lower: 4466.7196729968955, + Upper: 4870.992343051145, + Count: 1, + }, + }, + }, + Timestamp: 1234567, + }, + &Sample{ + Metric: Metric{ + "foo": "bar", + }, + Histogram: SampleHistogram{ + Count: 1, + Sum: 4500, + Buckets: HistogramBuckets{ + { + Boundaries: 0, + Lower: 4466.7196729968955, + Upper: 4870.992343051145, + Count: 1, + }, + }, + }, + Timestamp: 1234, + }, + }, + }, + } + + for _, test := range input { + b, err := json.Marshal(test.value) + if err != nil { + t.Error(err) + continue + } + + if string(b) != test.plain { + t.Errorf("encoding error: expected %q, got %q", test.plain, b) + continue + } + + var vec Vector + err = json.Unmarshal(b, &vec) + if err != nil { + t.Error(err) + continue + } + + if !reflect.DeepEqual(vec, test.value) { + t.Errorf("decoding error: expected %v, got %v", test.value, vec) + } + } +} diff --git a/model/value_test.go b/model/value_test.go index 7936a06e..cb97e37d 100644 --- a/model/value_test.go +++ b/model/value_test.go @@ -36,17 +36,17 @@ func TestEqualValues(t *testing.T) { in2: 3.1415, want: false, }, - "positive inifinities": { + "positive infinities": { in1: SampleValue(math.Inf(+1)), in2: SampleValue(math.Inf(+1)), want: true, }, - "negative inifinities": { + "negative infinities": { in1: SampleValue(math.Inf(-1)), in2: SampleValue(math.Inf(-1)), want: true, }, - "different inifinities": { + "different infinities": { in1: SampleValue(math.Inf(+1)), in2: SampleValue(math.Inf(-1)), want: false, @@ -116,6 +116,146 @@ func TestEqualSamples(t *testing.T) { }, want: true, }, + "equal histograms": { + in1: &Sample{ + Metric: Metric{"foo": "bar"}, + Timestamp: 0, + Histogram: SampleHistogram{ + Count: 1, + Sum: 4500, + Buckets: HistogramBuckets{ + { + Boundaries: 0, + Lower: 4466.7196729968955, + Upper: 4870.992343051145, + Count: 1, + }, + }, + }, + }, + in2: &Sample{ + Metric: Metric{"foo": "bar"}, + Timestamp: 0, + Histogram: SampleHistogram{ + Count: 1, + Sum: 4500, + Buckets: HistogramBuckets{ + { + Boundaries: 0, + Lower: 4466.7196729968955, + Upper: 4870.992343051145, + Count: 1, + }, + }, + }, + }, + want: true, + }, + "different histogram counts": { + in1: &Sample{ + Metric: Metric{"foo": "bar"}, + Timestamp: 0, + Histogram: SampleHistogram{ + Count: 2, + Sum: 4500, + Buckets: HistogramBuckets{ + { + Boundaries: 0, + Lower: 4466.7196729968955, + Upper: 4870.992343051145, + Count: 1, + }, + }, + }, + }, + in2: &Sample{ + Metric: Metric{"foo": "bar"}, + Timestamp: 0, + Histogram: SampleHistogram{ + Count: 1, + Sum: 4500, + Buckets: HistogramBuckets{ + { + Boundaries: 0, + Lower: 4466.7196729968955, + Upper: 4870.992343051145, + Count: 1, + }, + }, + }, + }, + want: false, + }, + "different histogram sums": { + in1: &Sample{ + Metric: Metric{"foo": "bar"}, + Timestamp: 0, + Histogram: SampleHistogram{ + Count: 1, + Sum: 4500.01, + Buckets: HistogramBuckets{ + { + Boundaries: 0, + Lower: 4466.7196729968955, + Upper: 4870.992343051145, + Count: 1, + }, + }, + }, + }, + in2: &Sample{ + Metric: Metric{"foo": "bar"}, + Timestamp: 0, + Histogram: SampleHistogram{ + Count: 1, + Sum: 4500, + Buckets: HistogramBuckets{ + { + Boundaries: 0, + Lower: 4466.7196729968955, + Upper: 4870.992343051145, + Count: 1, + }, + }, + }, + }, + want: false, + }, + "different histogram inner counts": { + in1: &Sample{ + Metric: Metric{"foo": "bar"}, + Timestamp: 0, + Histogram: SampleHistogram{ + Count: 1, + Sum: 4500, + Buckets: HistogramBuckets{ + { + Boundaries: 0, + Lower: 4466.7196729968955, + Upper: 4870.992343051145, + Count: 2, + }, + }, + }, + }, + in2: &Sample{ + Metric: Metric{"foo": "bar"}, + Timestamp: 0, + Histogram: SampleHistogram{ + Count: 1, + Sum: 4500, + Buckets: HistogramBuckets{ + { + Boundaries: 0, + Lower: 4466.7196729968955, + Upper: 4870.992343051145, + Count: 1, + }, + }, + }, + }, + want: false, + }, } for name, test := range tests {