-
Notifications
You must be signed in to change notification settings - Fork 454
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
[query] Implemented movingSum, movingMax, movingMin (graphite functions) #2570
Changes from 25 commits
4e901d7
f42685a
f0df4ce
bef6377
8ce6dc3
30ef946
cf9288f
50b0ae9
1ebe629
ebaafce
2a0643c
53222b9
8c2b6bf
01792e7
27f5d53
b9e6a92
91b2778
c024c85
a2698c2
7c1129d
9c070b8
a0807a4
9877de0
86e9b6a
3dbc68e
2124c31
3a201d6
1847586
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -71,6 +71,53 @@ func SafeSort(input []float64) int { | |
return nans | ||
} | ||
|
||
// SafeSum returns the sum of the input slice the number of NaNs in the input. | ||
func SafeSum(input []float64) (float64, int) { | ||
nans := 0 | ||
sum := 0.0 | ||
for _, v := range input { | ||
if !math.IsNaN(v) { | ||
sum += v | ||
} else { | ||
nans += 1 | ||
} | ||
} | ||
return sum, nans | ||
} | ||
|
||
// SafeMax returns the maximum value of the input slice the number of NaNs in the input. | ||
func SafeMax(input []float64) (float64, int) { | ||
nans := 0 | ||
max := -math.MaxFloat64 | ||
for _, v := range input { | ||
if !math.IsNaN(v) { | ||
if v > max { | ||
max = v | ||
} | ||
} else { | ||
nans += 1 | ||
} | ||
} | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. nit: Go prefers to limit indentation wherever possible (since it's relative to complexity of the statement). Can simplify loop by following's advice of Effective Go "In the Go libraries, you'll find that when an if statement doesn't flow into the next statement—that is, the body ends in break, continue, goto, or return—the unnecessary else is omitted.": e.g. for _, v := range input {
if math.IsNaN(v) {
nans++
continue
}
if v > max {
max = v
}
} There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. done! |
||
return max, nans | ||
} | ||
|
||
// SafeMin returns the minimum value of the input slice the number of NaNs in the input. | ||
func SafeMin(input []float64) (float64, int) { | ||
nans := 0 | ||
min := math.MaxFloat64 | ||
for _, v := range input { | ||
if !math.IsNaN(v) { | ||
if v < min { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. nit: Go prefers to limit indentation wherever possible (since it's relative to complexity of the statement). Can simplify loop by following's advice of Effective Go "In the Go libraries, you'll find that when an if statement doesn't flow into the next statement—that is, the body ends in break, continue, goto, or return—the unnecessary else is omitted.": e.g. for _, v := range input {
if math.IsNaN(v) {
nans++
continue
}
if v < min {
min = v
}
} There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. done! |
||
min = v | ||
} | ||
} else { | ||
nans += 1 | ||
} | ||
} | ||
return min, nans | ||
} | ||
|
||
|
||
// GetPercentile computes the percentile cut off for an array of floats | ||
func GetPercentile(input []float64, percentile float64, interpolate bool) float64 { | ||
nans := SafeSort(input) | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -1591,16 +1591,70 @@ func changed(ctx *common.Context, seriesList singlePathSpec) (ts.SeriesList, err | |
}) | ||
} | ||
|
||
// movingMedian takes one metric or a wildcard seriesList followed by a a quoted string | ||
// with a length of time like '1hour' or '5min'. Graphs the median of the preceding | ||
// datapoints for each point on the graph. All previous datapoints are set to None at | ||
// the beginning of the graph. | ||
func movingMedian(ctx *common.Context, _ singlePathSpec, windowSize string) (*binaryContextShifter, error) { | ||
interval, err := common.ParseInterval(windowSize) | ||
// windowPointsLength calculates the number of window points in a interval | ||
func windowPointsLength(series *ts.Series, interval time.Duration) int { | ||
return int(interval / (time.Duration(series.MillisPerStep()) * time.Millisecond)) | ||
} | ||
|
||
type movingImplementationFn func(window []float64, values ts.MutableValues, windowPoints int, i int) | ||
|
||
// movingMedianHelper given a slice of floats, calculates the median and assigns it into vals as index i | ||
func movingMedianHelper(window []float64, vals ts.MutableValues, windowPoints int, i int) { | ||
nans := common.SafeSort(window) | ||
|
||
if nans < windowPoints { | ||
index := (windowPoints - nans) / 2 | ||
median := window[nans+index] | ||
vals.SetValueAt(i, median) | ||
} | ||
} | ||
|
||
// movingSumHelper given a slice of floats, calculates the sum and assigns it into vals as index i | ||
func movingSumHelper(window []float64, vals ts.MutableValues, windowPoints int, i int) { | ||
sum, nans := common.SafeSum(window) | ||
|
||
if nans < windowPoints { | ||
vals.SetValueAt(i, sum) | ||
} | ||
} | ||
|
||
// movingMaxHelper given a slice of floats, finds the max and assigns it into vals as index i | ||
func movingMaxHelper(window []float64, vals ts.MutableValues, windowPoints int, i int) { | ||
max, nans := common.SafeMax(window) | ||
|
||
if nans < windowPoints { | ||
vals.SetValueAt(i, max) | ||
} | ||
} | ||
|
||
// movingMinHelper given a slice of floats, finds the min and assigns it into vals as index i | ||
func movingMinHelper(window []float64, vals ts.MutableValues, windowPoints int, i int) { | ||
min, nans := common.SafeMin(window) | ||
|
||
if nans < windowPoints { | ||
vals.SetValueAt(i, min) | ||
} | ||
} | ||
|
||
|
||
|
||
func newMovingBinaryTransform( | ||
ctx *common.Context, | ||
input singlePathSpec, | ||
windowSizeValue genericInterface, | ||
movingFunctionName string, | ||
impl movingImplementationFn, | ||
) (*binaryContextShifter, error) { | ||
if len(input.Values) == 0 { | ||
return nil, nil | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Can you add some explicit tests to make sure this case is handled ok? Would add a test case to I just spent 10-20mins finding if it was or not and not 100% confident about it. I see this which implies it should be: // context shifter ptr is nil, nothing to do here, return empty series.
if result.IsNil() {
return reflect.ValueOf(ts.NewSeriesList()), nil
} There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. OK. So you said you want me to add "a test case to i added the test case to but i dont quite understand your wording on the second part. were you saying that you also wanted me to modify There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. resolved offline. Rob said this test was enough. |
||
} | ||
|
||
windowSize, err := parseWindowSize(windowSizeValue, input) | ||
if err != nil { | ||
return nil, err | ||
} | ||
|
||
interval := windowSize.deltaValue | ||
if interval <= 0 { | ||
return nil, common.ErrInvalidIntervalFormat | ||
} | ||
|
@@ -1613,64 +1667,88 @@ func movingMedian(ctx *common.Context, _ singlePathSpec, windowSize string) (*bi | |
} | ||
|
||
bootstrapStartTime, bootstrapEndTime := ctx.StartTime.Add(-interval), ctx.StartTime | ||
transformerFn := func(bootstrapped, original ts.SeriesList) (ts.SeriesList, error) { | ||
bootstrapList, err := combineBootstrapWithOriginal(ctx, | ||
bootstrapStartTime, bootstrapEndTime, | ||
bootstrapped, singlePathSpec(original)) | ||
if err != nil { | ||
return ts.NewSeriesList(), err | ||
} | ||
|
||
results := make([]*ts.Series, 0, original.Len()) | ||
for i, bootstrap := range bootstrapList.Values { | ||
series := original.Values[i] | ||
windowPoints := int(interval / (time.Duration(series.MillisPerStep()) * time.Millisecond)) | ||
if windowPoints <= 0 { | ||
err := errors.NewInvalidParamsError(fmt.Errorf( | ||
"non positive window points, windowSize=%s, stepSize=%d", | ||
windowSize, series.MillisPerStep())) | ||
return &binaryContextShifter{ | ||
ContextShiftFunc: contextShiftingFn, | ||
BinaryTransformer: func(bootstrapped, original ts.SeriesList) (ts.SeriesList, error) { | ||
bootstrapList, err := combineBootstrapWithOriginal(ctx, | ||
bootstrapStartTime, bootstrapEndTime, | ||
bootstrapped, singlePathSpec(original)) | ||
if err != nil { | ||
return ts.NewSeriesList(), err | ||
} | ||
window := make([]float64, windowPoints) | ||
util.Memset(window, math.NaN()) | ||
numSteps := series.Len() | ||
offset := bootstrap.Len() - numSteps | ||
vals := ts.NewValues(ctx, series.MillisPerStep(), numSteps) | ||
for i := 0; i < numSteps; i++ { | ||
for j := i + offset - windowPoints; j < i+offset; j++ { | ||
if j < 0 || j >= bootstrap.Len() { | ||
continue | ||
} | ||
|
||
idx := j - i - offset + windowPoints | ||
if idx < 0 || idx > len(window)-1 { | ||
continue | ||
} | ||
|
||
window[idx] = bootstrap.ValueAt(j) | ||
results := make([]*ts.Series, 0, original.Len()) | ||
maxWindowPoints := 0 | ||
for i, _ := range bootstrapList.Values { | ||
series := original.Values[i] | ||
windowPoints := windowPointsLength(series, interval) | ||
if windowPoints <= 0 { | ||
err := errors.NewInvalidParamsError(fmt.Errorf( | ||
"non positive window points, windowSize=%s, stepSize=%d", | ||
windowSize.stringValue, series.MillisPerStep())) | ||
return ts.NewSeriesList(), err | ||
} | ||
nans := common.SafeSort(window) | ||
if nans < windowPoints { | ||
index := (windowPoints - nans) / 2 | ||
median := window[nans+index] | ||
vals.SetValueAt(i, median) | ||
if windowPoints > maxWindowPoints { | ||
maxWindowPoints = windowPoints | ||
} | ||
} | ||
name := fmt.Sprintf("movingMedian(%s,%q)", series.Name(), windowSize) | ||
newSeries := ts.NewSeries(ctx, name, series.StartTime(), vals) | ||
results = append(results, newSeries) | ||
} | ||
|
||
original.Values = results | ||
return original, nil | ||
} | ||
windowPoints := make([]float64, maxWindowPoints) | ||
for i, bootstrap := range bootstrapList.Values { | ||
series := original.Values[i] | ||
currWindowPoints := windowPointsLength(series, interval) | ||
window := windowPoints[:currWindowPoints] | ||
util.Memset(window, math.NaN()) | ||
numSteps := series.Len() | ||
offset := bootstrap.Len() - numSteps | ||
vals := ts.NewValues(ctx, series.MillisPerStep(), numSteps) | ||
for i := 0; i < numSteps; i++ { | ||
for j := i + offset - currWindowPoints; j < i+offset; j++ { | ||
if j < 0 || j >= bootstrap.Len() { | ||
continue | ||
} | ||
|
||
return &binaryContextShifter{ | ||
ContextShiftFunc: contextShiftingFn, | ||
BinaryTransformer: transformerFn, | ||
idx := j - i - offset + currWindowPoints | ||
if idx < 0 || idx > len(window)-1 { | ||
continue | ||
} | ||
|
||
window[idx] = bootstrap.ValueAt(j) | ||
} | ||
impl(window, vals, currWindowPoints, i) | ||
} | ||
name := fmt.Sprintf("%s(%s,%s)", movingFunctionName, series.Name(), windowSize.stringValue) | ||
newSeries := ts.NewSeries(ctx, name, series.StartTime(), vals) | ||
results = append(results, newSeries) | ||
} | ||
|
||
original.Values = results | ||
return original, nil | ||
}, | ||
}, nil | ||
} | ||
|
||
// movingMedian calculates the moving median of a metric (or metrics) over a time interval. | ||
func movingMedian(ctx *common.Context, input singlePathSpec, windowSize genericInterface) (*binaryContextShifter, error) { | ||
return newMovingBinaryTransform(ctx, input, windowSize, "movingMedian", movingMedianHelper) | ||
} | ||
|
||
// movingSum calculates the moving sum of a metric (or metrics) over a time interval. | ||
func movingSum(ctx *common.Context, input singlePathSpec, windowSize genericInterface) (*binaryContextShifter, error) { | ||
return newMovingBinaryTransform(ctx, input, windowSize, "movingSum", movingSumHelper) | ||
} | ||
|
||
// movingMax calculates the moving maximum of a metric (or metrics) over a time interval. | ||
func movingMax(ctx *common.Context, input singlePathSpec, windowSize genericInterface) (*binaryContextShifter, error) { | ||
return newMovingBinaryTransform(ctx, input, windowSize, "movingMax", movingMaxHelper) | ||
} | ||
|
||
// movingMin calculates the moving minimum of a metric (or metrics) over a time interval. | ||
func movingMin(ctx *common.Context, input singlePathSpec, windowSize genericInterface) (*binaryContextShifter, error) { | ||
return newMovingBinaryTransform(ctx, input, windowSize, "movingMin", movingMinHelper) | ||
} | ||
|
||
|
||
// legendValue takes one metric or a wildcard seriesList and a string in quotes. | ||
// Appends a value to the metric name in the legend. Currently one or several of: | ||
// "last", "avg", "total", "min", "max". | ||
|
@@ -1920,6 +1998,9 @@ func init() { | |
MustRegisterFunction(mostDeviant) | ||
MustRegisterFunction(movingAverage) | ||
MustRegisterFunction(movingMedian) | ||
MustRegisterFunction(movingSum) | ||
MustRegisterFunction(movingMax) | ||
MustRegisterFunction(movingMin) | ||
MustRegisterFunction(multiplySeries) | ||
MustRegisterFunction(nonNegativeDerivative).WithDefaultParams(map[uint8]interface{}{ | ||
2: math.NaN(), // maxValue | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Indentation of inner of "if" contents is missing - needs further indenting. Looks like this file is not saved with "gofmt". Suprised linter hasn't caught this... Can you run "gofmt" on all these files?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
If you use a standard Go editor, it should run it for you on save. (i.e. vimgo, vscode w/ Go extensions, Go Land, etc)
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Yeah, I do edit with GoLand, which always catches stuff like this. The only reason this mis-indentation was included was because I manually did it on Github. A few days ago, I tried to make a trivial change, just so I could force the CI to re-run.