From 6e5bbdc7a07c61bc78575b7e3b62e4a7c635c01e Mon Sep 17 00:00:00 2001 From: teddywahle Date: Wed, 23 Sep 2020 14:18:41 -0700 Subject: [PATCH 1/3] Added the graphite sortByMinima function --- src/query/graphite/native/builtin_functions.go | 6 ++++++ src/query/graphite/native/builtin_functions_test.go | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/src/query/graphite/native/builtin_functions.go b/src/query/graphite/native/builtin_functions.go index ff5858f2e4..8c434591e0 100644 --- a/src/query/graphite/native/builtin_functions.go +++ b/src/query/graphite/native/builtin_functions.go @@ -94,6 +94,11 @@ func sortByMaxima(ctx *common.Context, series singlePathSpec) (ts.SeriesList, er return highestMax(ctx, series, len(series.Values)) } +// sortByMinima sorts timeseries by the minimum value across the time period specified. +func sortByMinima(ctx *common.Context, series singlePathSpec) (ts.SeriesList, error) { + return lowest(ctx, series, len(series.Values), "min") +} + type valueComparator func(v, threshold float64) bool func compareByFunction( @@ -2261,6 +2266,7 @@ func init() { MustRegisterFunction(scale) MustRegisterFunction(scaleToSeconds) MustRegisterFunction(sortByMaxima) + MustRegisterFunction(sortByMinima) MustRegisterFunction(sortByName) MustRegisterFunction(sortByTotal) MustRegisterFunction(squareRoot) diff --git a/src/query/graphite/native/builtin_functions_test.go b/src/query/graphite/native/builtin_functions_test.go index e399c2e787..49089a2d19 100644 --- a/src/query/graphite/native/builtin_functions_test.go +++ b/src/query/graphite/native/builtin_functions_test.go @@ -191,6 +191,10 @@ func TestSortByMaxima(t *testing.T) { testSortingFuncs(t, sortByMaxima, []int{4, 0, 3, 2, 1}) } +func TestSortByMinima(t *testing.T) { + testSortingFuncs(t, sortByMinima, []int{1, 3, 2, 4, 0}) +} + func TestAbsolute(t *testing.T) { ctx := common.NewTestContext() defer ctx.Close() @@ -3326,6 +3330,7 @@ func TestFunctionsRegistered(t *testing.T) { "scale", "scaleToSeconds", "sortByMaxima", + "sortByMinima", "sortByName", "sortByTotal", "squareRoot", From 0acf8a3e4d31b97a41944c6e78d8de5af25df55d Mon Sep 17 00:00:00 2001 From: teddywahle Date: Sun, 4 Oct 2020 20:24:38 -0700 Subject: [PATCH 2/3] fixed snapping bug --- .../graphite/native/builtin_functions.go | 2 +- .../graphite/native/builtin_functions_test.go | 127 +++++------------- 2 files changed, 36 insertions(+), 93 deletions(-) diff --git a/src/query/graphite/native/builtin_functions.go b/src/query/graphite/native/builtin_functions.go index 6ea32368a1..5e82cb686a 100644 --- a/src/query/graphite/native/builtin_functions.go +++ b/src/query/graphite/native/builtin_functions.go @@ -1505,7 +1505,7 @@ func combineBootstrapWithOriginal( } // NB(braskin): using bootstrap.Len() is incorrect as it will include all // of the steps in the original timeseries, not just the steps up to the new end time - bootstrapLength := bootstrap.StepAtTime(endTime) + bootstrapLength := bootstrap.StepAtTime(original.StartTime()) ratio := bootstrap.MillisPerStep() / original.MillisPerStep() numBootstrapValues := bootstrapLength * ratio numCombinedValues := numBootstrapValues + original.Len() diff --git a/src/query/graphite/native/builtin_functions_test.go b/src/query/graphite/native/builtin_functions_test.go index dc1562cd13..1aab65b565 100644 --- a/src/query/graphite/native/builtin_functions_test.go +++ b/src/query/graphite/native/builtin_functions_test.go @@ -28,7 +28,6 @@ import ( "github.com/m3db/m3/src/query/block" "github.com/m3db/m3/src/query/graphite/common" - "github.com/m3db/m3/src/query/graphite/context" xctx "github.com/m3db/m3/src/query/graphite/context" "github.com/m3db/m3/src/query/graphite/storage" xtest "github.com/m3db/m3/src/query/graphite/testing" @@ -721,6 +720,41 @@ func testGeneralFunction(t *testing.T, target, expectedName string, values, outp common.CompareOutputsAndExpected(t, 60000, testGeneralFunctionStart, expected, res.Values) } +func TestCombineBootstrapWithOriginal(t *testing.T) { + var ( + contextStart = time.Date(2020, time.October, 5, 1, 15, 37, 884207922, time.UTC) + contextEnd = time.Date(2020, time.October, 5, 1, 18, 37, 884207922, time.UTC) + ctx = common.NewContext(common.ContextOptions{ + Start: contextStart, + End: contextEnd, + Engine: NewEngine(&common.MovingFunctionStorage{}), + }) + + originalStart = time.Date(2020, time.October, 5, 1, 16, 00, 0, time.UTC) + originalValues = []float64{14, 15, 16, 17, 18} + originalSeriesListValues = []*ts.Series{ts.NewSeries(ctx, "original", originalStart, common.NewTestSeriesValues(ctx, 30000, originalValues))} + originalSeriesList = singlePathSpec{Values: originalSeriesListValues} + + bootstrappedStart = time.Date(2020, time.October, 5, 1, 15, 00, 0, time.UTC) + bootstrappedValues = []float64{12, 13, 14, 15, 16, 17, 18} + bootstrappedSeriesListValues = []*ts.Series{ts.NewSeries(ctx, "original", bootstrappedStart, common.NewTestSeriesValues(ctx, 30000, bootstrappedValues))} + bootstrappedSeriesList = ts.NewSeriesList() + + bootstrapStartTime = time.Date(2020, time.October, 5, 1, 14, 37, 884207922, time.UTC) + bootstrapEndTime = time.Date(2020, time.October, 5, 1, 15, 37, 884207922, time.UTC) + + expectedValues = []float64{12, 13, 14, 15, 16, 17, 18} + expectedSeries = ts.NewSeries(ctx, "original", bootstrapStartTime, common.NewTestSeriesValues(ctx, 30000, expectedValues)) + ) + bootstrappedSeriesList.Values = bootstrappedSeriesListValues + + defer ctx.Close() + + output, err := combineBootstrapWithOriginal(ctx, bootstrapStartTime, bootstrapEndTime, bootstrappedSeriesList, originalSeriesList) + assert.Equal(t, output.Values[0], expectedSeries) + assert.Nil(t, err) +} + func TestMovingAverageSuccess(t *testing.T) { values := []float64{12.0, 19.0, -10.0, math.NaN(), 10.0} bootstrap := []float64{3.0, 4.0, 5.0} @@ -2884,97 +2918,6 @@ func TestMovingAverage(t *testing.T) { []common.TestSeries{expected}, res.Values) } -func TestMovingMedianInvalidLimits(t *testing.T) { - ctrl := xgomock.NewController(t) - defer ctrl.Finish() - - store := storage.NewMockStorage(ctrl) - now := time.Now().Truncate(time.Hour) - engine := NewEngine(store) - startTime := now.Add(-3 * time.Minute) - endTime := now.Add(-time.Minute) - ctx := common.NewContext(common.ContextOptions{Start: startTime, End: endTime, Engine: engine}) - defer ctx.Close() - - stepSize := 60000 - target := "movingMedian(foo.bar.q.zed, '1min')" - store.EXPECT().FetchByQuery(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( - func(_ context.Context, q string, opts storage.FetchOptions) (*storage.FetchResult, error) { - startTime := opts.StartTime - ctx := context.New() - numSteps := int(opts.EndTime.Sub(startTime)/time.Millisecond) / stepSize - vals := ts.NewConstantValues(ctx, 0, numSteps, stepSize) - series := ts.NewSeries(ctx, "foo.bar.q.zed", opts.EndTime, vals) - return &storage.FetchResult{SeriesList: []*ts.Series{series}}, nil - }).Times(2) - expr, err := engine.Compile(target) - require.NoError(t, err) - res, err := expr.Execute(ctx) - require.NoError(t, err) - expected := common.TestSeries{ - Name: "movingMedian(foo.bar.q.zed,\"1min\")", - Data: []float64{math.NaN(), 0.0}, - } - common.CompareOutputsAndExpected(t, stepSize, endTime, - []common.TestSeries{expected}, res.Values) -} - -func TestMovingMismatchedLimits(t *testing.T) { - // NB: this tests the behavior when query limits do not snap exactly to data - // points. When limits do not snap exactly, the first point should be omitted. - for _, fn := range []string{"movingAverage", "movingMedian", "movingSum", "movingMax", "movingMin"} { - for i := time.Duration(0); i < time.Minute; i += time.Second { - testMovingFunctionInvalidLimits(t, fn, i) - } - } -} - -func testMovingFunctionInvalidLimits(t *testing.T, fn string, offset time.Duration) { - ctrl := xgomock.NewController(t) - defer ctrl.Finish() - - store := storage.NewMockStorage(ctrl) - now := time.Now().Truncate(time.Hour).Add(offset) - engine := NewEngine(store) - startTime := now.Add(-3 * time.Minute) - endTime := now.Add(-time.Minute) - ctx := common.NewContext(common.ContextOptions{Start: startTime, End: endTime, Engine: engine}) - defer ctx.Close() - - stepSize := 60000 - target := fmt.Sprintf(`%s(timeShift(foo.bar.*.zed, '-1d'), '1min')`, fn) - store.EXPECT().FetchByQuery(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( - buildTestSeriesFn(stepSize, "foo.bar.g.zed", "foo.bar.x.zed"), - ).Times(2) - expr, err := engine.Compile(target) - require.NoError(t, err) - res, err := expr.Execute(ctx) - require.NoError(t, err) - - expectedStart := startTime - expectedDataG := []float64{1, 1} - expectedDataX := []float64{2, 2} - - if offset > 0 { - expectedStart = expectedStart.Add(time.Minute) - expectedDataG[0] = math.NaN() - expectedDataX[0] = math.NaN() - } - - expected := []common.TestSeries{ - { - Name: fmt.Sprintf(`%s(timeShift(foo.bar.g.zed, -1d),"1min")`, fn), - Data: expectedDataG, - }, - { - Name: fmt.Sprintf(`%s(timeShift(foo.bar.x.zed, -1d),"1min")`, fn), - Data: expectedDataX, - }, - } - - common.CompareOutputsAndExpected(t, stepSize, expectedStart, expected, res.Values) -} - func TestLegendValue(t *testing.T) { ctx := common.NewTestContext() defer ctx.Close() From 7426cf3e2d2c3319586f5513cc59f180582d2007 Mon Sep 17 00:00:00 2001 From: teddywahle Date: Mon, 5 Oct 2020 12:50:33 -0700 Subject: [PATCH 3/3] added correct bootstrap step --- src/query/graphite/native/builtin_functions.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/query/graphite/native/builtin_functions.go b/src/query/graphite/native/builtin_functions.go index 5e82cb686a..1b529f8c02 100644 --- a/src/query/graphite/native/builtin_functions.go +++ b/src/query/graphite/native/builtin_functions.go @@ -1503,9 +1503,13 @@ func combineBootstrapWithOriginal( return ts.NewSeriesList(), err } } + bootstrapEndStep := endTime.Truncate(original.Resolution()) + if bootstrapEndStep.Before(endTime) { + bootstrapEndStep = bootstrapEndStep.Add(original.Resolution()) + } // NB(braskin): using bootstrap.Len() is incorrect as it will include all // of the steps in the original timeseries, not just the steps up to the new end time - bootstrapLength := bootstrap.StepAtTime(original.StartTime()) + bootstrapLength := bootstrap.StepAtTime(bootstrapEndStep) ratio := bootstrap.MillisPerStep() / original.MillisPerStep() numBootstrapValues := bootstrapLength * ratio numCombinedValues := numBootstrapValues + original.Len()