diff --git a/src/dbnode/ts/downsample/counter_resets_downsampler.go b/src/dbnode/ts/downsample/counter_resets_downsampler.go deleted file mode 100644 index 5fb1523f2b..0000000000 --- a/src/dbnode/ts/downsample/counter_resets_downsampler.go +++ /dev/null @@ -1,133 +0,0 @@ -// Copyright (c) 2020 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package downsample - -import ( - "math" -) - -// Value represents a datapoint value after downsampling. -type Value struct { - // FrameIndex is the index to the original datapoint within source frame (before downsampling). - FrameIndex int - // Value is a downsampled datapoint value. - Value float64 -} - -// DownsampleCounterResets downsamples datapoints in a way that preserves counter invariants: -// The last (and the first, if necessary) values stay the same as original (to transfer counter state from tile to tile). -// Also, after applying counter reset adjustment logics, all the values would be the same as -// after applying this logics to the original values. -// As an optimization (to reduce the amount of datapoints), prevFrameLastValue can be passed in (if available), -// and then in some cases the first value of this tile may be omitted. -// If the value for prevFrameLastValue is not available, pass math.NaN() instead. -// Pass a slice of capacity 4 as results to avoid potential allocations. -// -// Examples: -// -// 1. Frame without resets, prevFrameLastValue not provided (x): -// ... x | 5 6 7 8 9 | ... -// downsamples to: -// ... x | 5 9 | ... -// - we always keep the last value (9), and also need to keep the first value (5) because there might have been -// a reset between the previous frame and the current one (x -> 5). -// -// 2. Frame without resets, prevFrameLastValue provided (2): -// ... 2 | 5 6 7 8 9 | ... -// downsamples to: -// ... 2 | 9 | ... -// - we always keep the last value (9), and we can drop the first value (5) because we know there was no reset -// between the previous frame and the current one (2 -> 5). -// -// 3. Frame with one reset (6 -> 1): -// ... 2 | 5 6 1 3 9 | ... -// downsamples to: -// ... 2 | 6 1 9 | ... -// - we always keep the last value (9), and also include two datapoints around the reset (6 -> 1). -// -// 4. Frame with two resets (5 -> 1, 3 -> 0): -// ... 2 | 5 1 3 0 9 | ... -// downsamples to: -// ... 2 | 8 0 9 | ... -// - we always keep the last value (9), and also include two datapoints around the last reset (8 -> 0), -// where 8 is the value accumulated from all resets within the frame (5 + 3 = 8). - -func DownsampleCounterResets( - prevFrameLastValue float64, - frameValues []float64, - results []Value, -) []Value { - results = results[:0] - - if len(frameValues) == 0 { - return results - } - - firstValue := frameValues[0] - if math.IsNaN(prevFrameLastValue) || prevFrameLastValue > firstValue { - // include the first original datapoint to handle resets right before this frame - results = append(results, Value{0, firstValue}) - } - - var ( - previous = firstValue - accumulated = previous - lastResetPosition = math.MinInt64 - adjustedValueBeforeLastReset = 0.0 - ) - - for i := 1; i < len(frameValues); i++ { - current := frameValues[i] - delta := current - previous - previous = current - if delta >= 0 { - // Monotonic non-decrease. - accumulated += delta - continue - } - // A reset. - adjustedValueBeforeLastReset = accumulated - lastResetPosition = i - 1 - accumulated += current - } - - if lastResetPosition >= 0 && (len(results) == 0 || results[0].Value != adjustedValueBeforeLastReset) { - // include the adjusted value right before the last reset (if it is not equal to the included first value) - results = append(results, Value{lastResetPosition, adjustedValueBeforeLastReset}) - } - - lastPosition := len(frameValues) - 1 - lastValue := frameValues[lastPosition] - - positionAfterLastReset := lastResetPosition + 1 - if lastResetPosition >= 0 && adjustedValueBeforeLastReset <= lastValue { - // include the original value right after the last reset (unless it is the last value, which is always included) - results = append(results, Value{positionAfterLastReset, frameValues[positionAfterLastReset]}) - } - - if len(results) == 1 && results[0].Value == lastValue { - // if only the first value was included until now, and it's equal to the last value, it can be discarded - results = results[:0] - } - - // always include the last original datapoint - return append(results, Value{lastPosition, lastValue}) -} diff --git a/src/dbnode/ts/downsample/counter_resets_downsampler_prop_test.go b/src/dbnode/ts/downsample/counter_resets_downsampler_prop_test.go deleted file mode 100644 index 086e024b6d..0000000000 --- a/src/dbnode/ts/downsample/counter_resets_downsampler_prop_test.go +++ /dev/null @@ -1,179 +0,0 @@ -// +build big -// -// Copyright (c) 2020 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package downsample - -import ( - "math" - "testing" - "time" - - "github.com/m3db/m3/src/x/instrument" - - "github.com/leanovate/gopter" - "github.com/leanovate/gopter/gen" - "github.com/leanovate/gopter/prop" - "go.uber.org/zap" -) - -func TestDownsampleCounterResetsInvariants(t *testing.T) { - testDownsampleCounterResetsInvariants(t, false) -} - -func TestDownsampleCounterResetsInvariantsWithPrevFrameLastValue(t *testing.T) { - testDownsampleCounterResetsInvariants(t, true) -} - -func testDownsampleCounterResetsInvariants(t *testing.T, usePrevFrameLastValue bool) { - seed := time.Now().UnixNano() - params := gopter.DefaultTestParameters() - params.MinSize = 1 - if usePrevFrameLastValue { - params.MinSize++ - } - params.MinSuccessfulTests = 10000 - params.MaxShrinkCount = 0 - params.Rng.Seed(seed) - properties := gopter.NewProperties(params) - generator := gen.SliceOf(gen.Float64Range(0, 100)) - - epsilon := 0.00001 - - // NB: capture seed to be able to replicate failed runs. - logger := instrument.NewTestOptions(t).Logger() - logger.Info("Running tests", zap.Int64("seed", seed), zap.Bool("usePrevFrameLastValue", usePrevFrameLastValue)) - - properties.Property("return consistent indices", prop.ForAll( - func(v []float64) bool { - results := downsampleFromSlice(v, usePrevFrameLastValue) - - for i := 1; i < len(results); i++ { - if results[i].FrameIndex <= results[i-1].FrameIndex { - return false - } - } - - return true - }, - generator, - )) - - properties.Property("shrink the result", prop.ForAll( - func(v []float64) bool { - maxResultLength := len(v) - if maxResultLength > 4 { - maxResultLength = 4 - } - - results := downsampleFromSlice(v, usePrevFrameLastValue) - - return len(results) <= maxResultLength - }, - generator, - )) - - properties.Property("preserve the last value", prop.ForAll( - func(v []float64) bool { - results := downsampleFromSlice(v, usePrevFrameLastValue) - - lastIndex := len(v) - 1 - lastFrameValue := v[lastIndex] - if usePrevFrameLastValue { - lastIndex-- - } - - lastResult := results[len(results)-1] - - return lastResult.Value == lastFrameValue && lastResult.FrameIndex == lastIndex - }, - generator, - )) - - properties.Property("preserve values after adjusting for counter resets", prop.ForAll( - func(v []float64) bool { - results := downsampleFromSlice(v, usePrevFrameLastValue) - - downsampledValues := make([]float64, 0, len(results)) - for _, result := range results { - downsampledValues = append(downsampledValues, result.Value) - } - - prevFrameLastValue := 0.0 - input := v - if usePrevFrameLastValue { - prevFrameLastValue = v[0] - input = input[1:] - } - - adjustedInput := applyCounterResetAdjustment(prevFrameLastValue, input) - adjustedResults := applyCounterResetAdjustment(prevFrameLastValue, downsampledValues) - - for i, result := range results { - if math.Abs(adjustedResults[i]-adjustedInput[result.FrameIndex]) > epsilon { - return false - } - } - - return true - }, - generator, - )) - - properties.TestingRun(t) -} - -func downsampleFromSlice(vals []float64, usePrevFrameLastValue bool) []Value { - prevFrameLastValue := math.NaN() - - if usePrevFrameLastValue { - prevFrameLastValue = vals[0] - vals = vals[1:] - } - - return downsample(prevFrameLastValue, vals) -} - -func applyCounterResetAdjustment(previousVal float64, vals []float64) []float64 { - transformed := make([]float64, 0, len(vals)) - if len(vals) == 0 { - return transformed - } - - var ( - previous = previousVal - accumulated float64 - ) - - for i := 0; i < len(vals); i++ { - current := vals[i] - delta := current - previous - if delta >= 0 { - accumulated += delta - } else { // a reset - accumulated += current - } - transformed = append(transformed, accumulated) - previous = current - } - - return transformed -} diff --git a/src/dbnode/ts/downsample/counter_resets_downsampler_test.go b/src/dbnode/ts/downsample/counter_resets_downsampler_test.go deleted file mode 100644 index cf004f1d19..0000000000 --- a/src/dbnode/ts/downsample/counter_resets_downsampler_test.go +++ /dev/null @@ -1,251 +0,0 @@ -// Copyright (c) 2020 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package downsample - -import ( - "math" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestDownsampleCounterResets(t *testing.T) { - tests := []struct { - name string - given []float64 - want []Value - }{ - { - name: "empty", - given: []float64{}, - want: []Value{}, - }, - { - name: "one value", - given: []float64{3.14}, - want: []Value{{0, 3.14}}, - }, - { - name: "two different values", - given: []float64{3, 5}, - want: []Value{{0, 3}, {1, 5}}, - }, - { - name: "two identical values", - given: []float64{5, 5}, - want: []Value{{1, 5}}, - }, - { - name: "two values with reset", - given: []float64{3, 1}, - want: []Value{{0, 3}, {1, 1}}, - }, - { - name: "second value equal to first, then reset", - given: []float64{2, 2, 1}, - want: []Value{{0, 2}, {2, 1}}, - }, - { - name: "three values, no reset", - given: []float64{3, 4, 5}, - want: []Value{{0, 3}, {2, 5}}, - }, - { - name: "three identical values", - given: []float64{5, 5, 5}, - want: []Value{{2, 5}}, - }, - { - name: "three values, reset after first", - given: []float64{3, 1, 5}, - want: []Value{{0, 3}, {1, 1}, {2, 5}}, - }, - { - name: "three values, reset after second", - given: []float64{3, 5, 1}, - want: []Value{{0, 3}, {1, 5}, {2, 1}}, - }, - { - name: "three values, two resets", - given: []float64{5, 3, 2}, - want: []Value{{0, 5}, {1, 8}, {2, 2}}, - }, - { - name: "four values, reset after first", - given: []float64{3, 1, 4, 5}, - want: []Value{{0, 3}, {1, 1}, {3, 5}}, - }, - { - name: "four values, reset after second (A)", - given: []float64{3, 4, 1, 5}, - want: []Value{{0, 3}, {1, 4}, {2, 1}, {3, 5}}, - }, - { - name: "four values, reset after second (B)", - given: []float64{3, 4, 1, 4}, - want: []Value{{0, 3}, {1, 4}, {2, 1}, {3, 4}}, - }, - { - name: "four values, reset after second (C)", - given: []float64{3, 4, 1, 2}, - want: []Value{{0, 3}, {1, 4}, {3, 2}}, - }, - { - name: "four values, reset after third", - given: []float64{3, 4, 5, 1}, - want: []Value{{0, 3}, {2, 5}, {3, 1}}, - }, - { - name: "four values, two resets (A)", - given: []float64{3, 1, 5, 4}, - want: []Value{{0, 3}, {2, 8}, {3, 4}}, - }, - { - name: "four values, two resets (B)", - given: []float64{5, 2, 1, 4}, - want: []Value{{0, 5}, {1, 7}, {3, 4}}, - }, - { - name: "four values, two resets (C)", - given: []float64{5, 2, 2, 1}, - want: []Value{{0, 5}, {2, 7}, {3, 1}}, - }, - { - name: "four values, two resets (D)", - given: []float64{3, 5, 2, 1}, - want: []Value{{0, 3}, {2, 7}, {3, 1}}, - }, - { - name: "reset between two equal values", - given: []float64{4, 3, 4}, - want: []Value{{0, 4}, {1, 3}, {2, 4}}, - }, - { - name: "four values, three resets", - given: []float64{9, 7, 4, 1}, - want: []Value{{0, 9}, {2, 20}, {3, 1}}, - }, - { - name: "five values, two resets (A)", - given: []float64{3, 1, 2, 5, 4}, - want: []Value{{0, 3}, {3, 8}, {4, 4}}, - }, - { - name: "five equal values", - given: []float64{1, 1, 1, 1, 1}, - want: []Value{{4, 1}}, - }, - { - name: "five increasing values", - given: []float64{1, 2, 3, 4, 5}, - want: []Value{{0, 1}, {4, 5}}, - }, - { - name: "five decreasing values", - given: []float64{5, 4, 3, 2, 1}, - want: []Value{{0, 5}, {3, 5 + 4 + 3 + 2}, {4, 1}}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - results := downsample(math.NaN(), tt.given) - assert.Equal(t, tt.want, results) - }) - } -} - -func TestDownsampleCounterResetsWithPrevFrameLastValue(t *testing.T) { - tests := []struct { - name string - givenPrevFrameLastValue float64 - givenFrame []float64 - want []Value - }{ - { - name: "empty", - givenPrevFrameLastValue: 3, - givenFrame: []float64{}, - want: []Value{}, - }, - { - name: "one value equal to prev frame last", - givenPrevFrameLastValue: 2, - givenFrame: []float64{2}, - want: []Value{{0, 2}}, - }, - { - name: "one value less than prev frame last", - givenPrevFrameLastValue: 3, - givenFrame: []float64{2}, - want: []Value{{0, 2}}, - }, - { - name: "one value more than prev frame last", - givenPrevFrameLastValue: 3, - givenFrame: []float64{4}, - want: []Value{{0, 4}}, - }, - { - name: "two values, increasing", - givenPrevFrameLastValue: 3, - givenFrame: []float64{4, 5}, - want: []Value{{1, 5}}, - }, - { - name: "reset between frames", - givenPrevFrameLastValue: 3, - givenFrame: []float64{2, 5}, - want: []Value{{0, 2}, {1, 5}}, - }, - { - name: "reset between frames and within frame", - givenPrevFrameLastValue: 4, - givenFrame: []float64{2, 1}, - want: []Value{{0, 2}, {1, 1}}, - }, - { - name: "reset within frame", - givenPrevFrameLastValue: 1, - givenFrame: []float64{4, 3}, - want: []Value{{0, 4}, {1, 3}}, - }, - { - name: "all equal", - givenPrevFrameLastValue: 1, - givenFrame: []float64{1, 1}, - want: []Value{{1, 1}}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - results := downsample(tt.givenPrevFrameLastValue, tt.givenFrame) - assert.Equal(t, tt.want, results) - }) - } -} - -func downsample(prevFrameLastValue float64, vals []float64) []Value { - results := make([]Value, 0, 4) - - return DownsampleCounterResets(prevFrameLastValue, vals, results) -}