Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[query] Implemented the Graphite timeSlice function #2581

Merged
merged 34 commits into from
Sep 8, 2020
Merged
Show file tree
Hide file tree
Changes from 33 commits
Commits
Show all changes
34 commits
Select commit Hold shift + click to select a range
4e901d7
Added the moving movingMin function
teddywahle Aug 27, 2020
f42685a
Merge branch 'master' into twahle-moving-min
teddywahle Aug 27, 2020
f0df4ce
worked on moving min
teddywahle Aug 27, 2020
bef6377
Merge branch 'twahle-moving-min' of https://github.com/teddywahle/m3 …
teddywahle Aug 27, 2020
8ce6dc3
more work on the moving min func
teddywahle Aug 27, 2020
30ef946
Merge branch 'master' into twahle-moving-min
teddywahle Aug 28, 2020
cf9288f
updated movingMedian
teddywahle Aug 28, 2020
50b0ae9
Merge branch 'twahle-moving-min' of https://github.com/teddywahle/m3 …
teddywahle Aug 28, 2020
1ebe629
Apply suggestions from code review
teddywahle Aug 28, 2020
ebaafce
testMovingFunction
teddywahle Aug 28, 2020
2a0643c
wrote test movingSum function
teddywahle Aug 28, 2020
53222b9
Merge branch 'master' into twahle-moving-min
teddywahle Aug 31, 2020
b2d895c
Merge branch 'master' into graphite-timeslice
teddywahle Sep 2, 2020
88260f7
Apply suggestions from code review
teddywahle Sep 3, 2020
73e279f
Apply suggestions from code review
teddywahle Sep 3, 2020
007e83e
Update src/query/graphite/native/builtin_functions.go
teddywahle Sep 3, 2020
dafe31b
added graphite timeslice function
teddywahle Sep 3, 2020
0c9f44e
completed a merge
teddywahle Sep 3, 2020
17c42bd
Apply suggestions from code review
teddywahle Sep 3, 2020
f246f82
Finished the timeSlice function
teddywahle Sep 3, 2020
8f60dca
Update src/query/graphite/native/builtin_functions_test.go
teddywahle Sep 3, 2020
ca19d91
fixed up the timeSlice code
teddywahle Sep 3, 2020
9f0e2c7
Apply suggestions from code review
teddywahle Sep 3, 2020
5ebab5a
Merge branch 'master' into graphite-timeslice
teddywahle Sep 3, 2020
c164a5f
Merge branch 'master' into graphite-timeslice
teddywahle Sep 4, 2020
902accc
Update src/query/graphite/native/builtin_functions.go
teddywahle Sep 4, 2020
381dcec
added method description
teddywahle Sep 4, 2020
a6f915c
Merge branch 'graphite-timeslice' of https://github.com/teddywahle/m3…
teddywahle Sep 4, 2020
89a9600
Update src/query/graphite/native/builtin_functions.go
teddywahle Sep 4, 2020
2d7c62a
Merge branch 'graphite-timeslice' of https://github.com/teddywahle/m3…
teddywahle Sep 4, 2020
2659565
Added suggestions from code review
teddywahle Sep 4, 2020
9e558a8
fixed indexing bug
teddywahle Sep 4, 2020
5d2c66b
Merge branch 'master' into graphite-timeslice
teddywahle Sep 7, 2020
fbe5d13
Merge branch 'master' into graphite-timeslice
robskillington Sep 8, 2020
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 48 additions & 0 deletions src/query/graphite/native/builtin_functions.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import (

"github.com/m3db/m3/src/query/graphite/common"
"github.com/m3db/m3/src/query/graphite/errors"
"github.com/m3db/m3/src/query/graphite/graphite"
"github.com/m3db/m3/src/query/graphite/ts"
"github.com/m3db/m3/src/query/util"
)
Expand Down Expand Up @@ -242,6 +243,50 @@ func timeShift(
}, nil
}

// timeSlice takes one metric or a wildcard metric, followed by a quoted string with the time to start the line and
// another quoted string with the time to end the line. The start and end times are inclusive.
// Useful for filtering out a part of a series of data from a wider range of data.
func timeSlice(ctx *common.Context, inputPath singlePathSpec, start string, end string) (ts.SeriesList, error) {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: Can you add a description for this function, maybe something similar to the docs for the function?

var (
now = time.Now()
tzOffsetForAbsoluteTime time.Duration
)
startTime, err := graphite.ParseTime(start, now, tzOffsetForAbsoluteTime)
if err != nil {
return ts.NewSeriesList(), err
}
endTime, err := graphite.ParseTime(end, now, tzOffsetForAbsoluteTime)
if err != nil {
return ts.NewSeriesList(), err
}

input := ts.SeriesList(inputPath)
output := make([]*ts.Series, 0, input.Len())

for _, series := range input.Values {
stepDuration := time.Duration(series.MillisPerStep()) * time.Millisecond
truncatedValues := ts.NewValues(ctx, series.MillisPerStep(), series.Len())

currentTime := series.StartTime()
for i := 0; i < series.Len(); i++ {
equalOrAfterStart := currentTime.Equal(startTime) || currentTime.After(startTime)
beforeOrEqualEnd := currentTime.Before(endTime) || currentTime.Equal(endTime)
if equalOrAfterStart && beforeOrEqualEnd {
truncatedValues.SetValueAt(i, series.ValueAtTime(currentTime))
}
currentTime = currentTime.Add(stepDuration)
}

slicedSeries := ts.NewSeries(ctx, series.Name(), series.StartTime(), truncatedValues)
renamedSlicedSeries := slicedSeries.RenamedTo(fmt.Sprintf("timeSlice(%s, %s, %s)", slicedSeries.Name(), start, end))
output = append(output, renamedSlicedSeries)
}
input.Values = output
return input, nil
}



// absolute returns the absolute value of each element in the series.
func absolute(ctx *common.Context, input singlePathSpec) (ts.SeriesList, error) {
return transform(ctx, input,
Expand Down Expand Up @@ -1974,6 +2019,9 @@ func init() {
MustRegisterFunction(timeShift).WithDefaultParams(map[uint8]interface{}{
3: true, // resetEnd
})
MustRegisterFunction(timeSlice).WithDefaultParams(map[uint8]interface{}{
3: "now", // endTime
})
MustRegisterFunction(transformNull).WithDefaultParams(map[uint8]interface{}{
2: 0.0, // defaultValue
})
Expand Down
45 changes: 45 additions & 0 deletions src/query/graphite/native/builtin_functions_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -642,6 +642,43 @@ func testMovingFunction(t *testing.T, target, expectedName string, values, boots
expected, res.Values)
}

var (
testGeneralFunctionStart = time.Now().Add(time.Minute * -11).Truncate(time.Minute)
testGeneralFunctionEnd = time.Now().Add(time.Minute * -3).Truncate(time.Minute)
)

// testGeneralFunction is a copy of testMovingFunction but without any logic for bootstrapping values
func testGeneralFunction(t *testing.T, target, expectedName string, values, output []float64) {
ctx := common.NewTestContext()
defer ctx.Close()

engine := NewEngine(
&common.MovingFunctionStorage{
StepMillis: 60000,
Values: values,
},
)
phonyContext := common.NewContext(common.ContextOptions{
Start: testGeneralFunctionStart,
End: testGeneralFunctionEnd,
Engine: engine,
})

expr, err := phonyContext.Engine.(*Engine).Compile(target)
require.NoError(t, err)
res, err := expr.Execute(phonyContext)
require.NoError(t, err)
var expected []common.TestSeries
if output != nil {
expectedSeries := common.TestSeries{
Name: expectedName,
Data: output,
}
expected = append(expected, expectedSeries)
}
common.CompareOutputsAndExpected(t, 60000, testGeneralFunctionStart, expected, res.Values)
}

func TestMovingAverageSuccess(t *testing.T) {
values := []float64{12.0, 19.0, -10.0, math.NaN(), 10.0}
bootstrap := []float64{3.0, 4.0, 5.0}
Expand Down Expand Up @@ -2840,6 +2877,13 @@ func TestTimeShift(t *testing.T) {
[]common.TestSeries{expected}, res.Values)
}

func TestTimeSlice(t *testing.T) {
values := []float64{math.NaN(),1.0,2.0,3.0,math.NaN(),5.0,6.0,math.NaN(),7.0,8.0,9.0}
expected := []float64{math.NaN(),math.NaN(),math.NaN(),3.0,math.NaN(),5.0,6.0,math.NaN(),7.0,math.NaN(),math.NaN()}

testGeneralFunction(t, "timeSlice(foo.bar.baz, '-9min','-3min')", "timeSlice(foo.bar.baz, -9min, -3min)", values, expected)
}

func TestDashed(t *testing.T) {
ctx := common.NewTestContext()
defer ctx.Close()
Expand Down Expand Up @@ -2990,6 +3034,7 @@ func TestFunctionsRegistered(t *testing.T) {
"time",
"timeFunction",
"timeShift",
"timeSlice",
"transformNull",
"weightedAverage",
}
Expand Down