Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[query] Implemented the Graphite interpolate function #2650

Merged
merged 8 commits into from
Sep 30, 2020
Merged

[query] Implemented the Graphite interpolate function #2650

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
78e126f
Added Graphite's interpolate function
teddywahle Sep 21, 2020
5283b93
Ran go fmt
teddywahle Sep 21, 2020
49b56ee
Merge branch 'master' into graphite-interpolate
teddywahle Sep 21, 2020
6b184e1
Merge branch 'master' into graphite-interpolate
teddywahle Sep 23, 2020
1f9ce3a
Update src/query/graphite/native/builtin_functions.go
teddywahle Sep 30, 2020
db59583
Added i=0 test case
teddywahle Sep 30, 2020
c7a0a1d
added preceding NaNs test case
teddywahle Sep 30, 2020
c6d8c1b
Merge branch 'master' into graphite-interpolate
teddywahle Sep 30, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 48 additions & 0 deletions src/query/graphite/native/builtin_functions.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1047,6 +1047,51 @@ func logarithm(ctx *common.Context, input singlePathSpec, base int) (ts.SeriesLi
return r, nil
}

// Takes one metric or a wildcard seriesList, and optionally a limit to the number of ‘None’ values
// to skip over. Continues the line with the last received value when gaps (‘None’ values)
// appear in your data, rather than breaking your line.
func interpolate(ctx *common.Context, input singlePathSpec, limit int) (ts.SeriesList, error) {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: can we explicitly call out that it won't interpolate at the end or the start? Graphite isn't very clear on this in their docs but would be useful to have here

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

added

output := make([]*ts.Series, 0, len(input.Values))
for _, series := range input.Values {
consecutiveNaNs := 0
numSteps := series.Len()
vals := ts.NewValues(ctx, series.MillisPerStep(), numSteps)

for i := 0; i < numSteps; i++ {
value := series.ValueAt(i)
vals.SetValueAt(i, value)

if i == 0 {
continue
} else if math.IsNaN(value) {
consecutiveNaNs++
} else if consecutiveNaNs == 0 { // have a value but no need to interpolate
continue
} else if math.IsNaN(series.ValueAt(i - consecutiveNaNs - 1)) { //have a value but can't interpolate: reset count
consecutiveNaNs = 0
continue
} else {
if limit == -1 || consecutiveNaNs <= limit {
for index := i - consecutiveNaNs; index < i; index++ {
lastGoodVal := series.ValueAt(i - consecutiveNaNs - 1)
v := lastGoodVal + float64(index-(i-consecutiveNaNs-1))*(value-lastGoodVal)/float64(consecutiveNaNs+1)
vals.SetValueAt(index, v)

}
consecutiveNaNs = 0
}
}
}

name := fmt.Sprintf("interpolate(%s)", series.Name())
newSeries := ts.NewSeries(ctx, name, series.StartTime(), vals)
output = append(output, newSeries)
}
teddywahle marked this conversation as resolved.
Show resolved Hide resolved
r := ts.SeriesList(input)
r.Values = output
return r, nil
}

// group takes an arbitrary number of pathspecs and adds them to a single timeseries array.
// This function is used to pass multiple pathspecs to a function which only takes one
func group(_ *common.Context, input multiplePathSpecs) (ts.SeriesList, error) {
Expand Down Expand Up @@ -2211,6 +2256,9 @@ func init() {
MustRegisterFunction(identity)
MustRegisterFunction(integral)
MustRegisterFunction(integralByInterval)
MustRegisterFunction(interpolate).WithDefaultParams(map[uint8]interface{}{
2: -1, // limit
})
MustRegisterFunction(isNonNull)
MustRegisterFunction(keepLastValue).WithDefaultParams(map[uint8]interface{}{
2: -1, // limit
Expand Down
51 changes: 47 additions & 4 deletions src/query/graphite/native/builtin_functions_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1941,10 +1941,52 @@ func TestIntegral(t *testing.T) {
}
}

/*
seriesList = self._gen_series_list_with_data(key='test',start=0,end=600,step=60,data=[None, 1, 2, 3, 4, 5, None, 6, 7, 8])
expected = [TimeSeries("integralByInterval(test,'2min')", 0, 600, 60, [0, 1, 2, 5, 4, 9, 0, 6, 7, 15])]
*/
func TestInterpolate(t *testing.T) {
ctx := common.NewTestContext()
defer ctx.Close()

tests := []struct {
values []float64
output []float64
}{
{
Copy link
Collaborator

@arnikola arnikola Sep 27, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: mind adding a test where there's more nans than limit, and maybe one with a bunch of preceeding nans (to flex the i==0 case?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

added both

[]float64{1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0},
[]float64{1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0},
},
{
[]float64{math.NaN(), 2.0, math.NaN(), 4.0, math.NaN(), 6.0, math.NaN(), 8.0, math.NaN(), 10.0, math.NaN(), 12.0, math.NaN(), 14.0, math.NaN(), 16.0, math.NaN(), 18.0, math.NaN(), 20.0},
[]float64{math.NaN(), 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0},
},
{
[]float64{1.0, 2.0, math.NaN(), math.NaN(), math.NaN(), 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, math.NaN(), math.NaN(), math.NaN()},
[]float64{1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, math.NaN(), math.NaN(), math.NaN()},
},
{
[]float64{1.0, 2.0, 3.0, 4.0, math.NaN(), 6.0, math.NaN(), math.NaN(), 9.0, 10.0, 11.0, math.NaN(), 13.0, math.NaN(), math.NaN(), math.NaN(), math.NaN(), 18.0, 19.0, 20.0},
[]float64{1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0},
},
{
[]float64{1.0, 2.0, math.NaN(), math.NaN(), math.NaN(), 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, math.NaN(), math.NaN()},
[]float64{1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, math.NaN(), math.NaN()},
},
}

start := time.Now()
step := 100
for _, test := range tests {
input := []common.TestSeries{{"foo", test.values}}
expected := []common.TestSeries{{"interpolate(foo)", test.output}}
timeSeries := generateSeriesList(ctx, start, input, step)
output, err := interpolate(ctx, singlePathSpec{
Values: timeSeries,
},
-1)
require.NoError(t, err)
common.CompareOutputsAndExpected(t, step, start,
expected, output.Values)
}
}

func TestIntegralByInterval(t *testing.T) {
ctx := common.NewTestContext()
defer ctx.Close()
Expand Down Expand Up @@ -3288,6 +3330,7 @@ func TestFunctionsRegistered(t *testing.T) {
"identity",
"integral",
"integralByInterval",
"interpolate",
"isNonNull",
"keepLastValue",
"legendValue",
Expand Down