Skip to content

Commit

Permalink
feat: update Prometheus API to query metrics over a range (#1587)
Browse files Browse the repository at this point in the history
Signed-off-by: Rakshit Gondwal <rakshitgondwal3@gmail.com>
Signed-off-by: Rakshit Gondwal <98955085+rakshitgondwal@users.noreply.github.com>
Co-authored-by: Florian Bacher <florian.bacher@dynatrace.com>
  • Loading branch information
rakshitgondwal and bacherfl authored Jun 30, 2023
1 parent d6f4307 commit 47a3e06
Showing 2 changed files with 182 additions and 37 deletions.
Original file line number Diff line number Diff line change
@@ -23,34 +23,116 @@ func (r *KeptnPrometheusProvider) EvaluateQuery(ctx context.Context, metric metr
ctx, cancel := context.WithTimeout(ctx, 20*time.Second)
defer cancel()

queryTime := time.Now().UTC()
r.Log.Info("Running query: /api/v1/query?query=" + metric.Spec.Query + "&time=" + queryTime.String())
client, err := promapi.NewClient(promapi.Config{Address: provider.Spec.TargetServer, Client: &r.HttpClient})
if err != nil {
return "", nil, err
}

api := prometheus.NewAPI(client)
result, w, err := api.Query(
if metric.Spec.Range != nil {
result, warnings, err := evaluateQueryWithRange(ctx, metric, r, api)
if err != nil {
return "", nil, err
}
if len(warnings) != 0 {
r.Log.Info("Prometheus API returned warnings: " + warnings[0])
}
return getResultForMatrix(result, r)
} else {
result, warnings, err := evaluateQueryWithoutRange(ctx, metric, r, api)
if err != nil {
return "", nil, err
}
if len(warnings) != 0 {
r.Log.Info("Prometheus API returned warnings: " + warnings[0])
}
return getResultForVector(result, r)
}
}

func evaluateQueryWithRange(ctx context.Context, metric metricsapi.KeptnMetric, r *KeptnPrometheusProvider, api prometheus.API) (model.Value, prometheus.Warnings, error) {
queryTime := time.Now().UTC()
// Get the duration
queryInterval, err := time.ParseDuration(metric.Spec.Range.Interval)
if err != nil {
return nil, nil, err
}
// Convert type Duration to type Time
startTime := queryTime.Add(-queryInterval).UTC()
r.Log.Info(fmt.Sprintf(
"Running query: /api/v1/query_range?query=%s&start=%d&end=%d&step=%v",
metric.Spec.Query,
startTime.Unix(), queryTime.Unix(),
queryInterval,
))
queryRange := prometheus.Range{
Start: startTime,
End: queryTime,
Step: queryInterval,
}
result, warnings, err := api.QueryRange(
ctx,
metric.Spec.Query,
queryTime,
queryRange,
[]prometheus.Option{}...,
)
if err != nil {
return nil, nil, err
}
return result, warnings, nil
}

func evaluateQueryWithoutRange(ctx context.Context, metric metricsapi.KeptnMetric, r *KeptnPrometheusProvider, api prometheus.API) (model.Value, prometheus.Warnings, error) {
queryTime := time.Now().UTC()
r.Log.Info(fmt.Sprintf(
"Running query: /api/v1/query?query=%s&time=%d",
metric.Spec.Query,
queryTime.Unix(),
))
result, warnings, err := api.Query(
ctx,
metric.Spec.Query,
queryTime,
[]prometheus.Option{}...,
)
if err != nil {
return "", nil, err
return nil, nil, err
}
return result, warnings, nil
}

if len(w) != 0 {
r.Log.Info("Prometheus API returned warnings: " + w[0])
func getResultForMatrix(result model.Value, r *KeptnPrometheusProvider) (string, []byte, error) {
// check if we can cast the result to a matrix
resultMatrix, ok := result.(model.Matrix)
if !ok {
return "", nil, fmt.Errorf("could not cast result")
}
// We are only allowed to return one value, if not the query may be malformed
// we are using two different errors to give the user more information about the result
// There can be more than 1 values in the matrixResults but we are defining the step
// parameter as the interval itself, hence there can only be one value.
// This logic should be changed, once we work onto the aggregation functions.
if len(resultMatrix) == 0 {
r.Log.Info("No values in query result")
return "", nil, fmt.Errorf("no values in query result")
} else if len(resultMatrix) > 1 {
r.Log.Info("Too many values in the query result")
return "", nil, fmt.Errorf("too many values in the query result")
}
value := resultMatrix[0].Values[0].Value.String()
b, err := resultMatrix[0].Values[0].Value.MarshalJSON()
if err != nil {
return "", nil, err
}
return value, b, nil
}

// check if we can cast the result to a vector, it might be another data struct which we can't process
func getResultForVector(result model.Value, r *KeptnPrometheusProvider) (string, []byte, error) {
// check if we can cast the result to a vector
resultVector, ok := result.(model.Vector)
if !ok {
return "", nil, fmt.Errorf("could not cast result")
}

// We are only allowed to return one value, if not the query may be malformed
// we are using two different errors to give the user more information about the result
if len(resultVector) == 0 {
Original file line number Diff line number Diff line change
@@ -12,11 +12,17 @@ import (
ctrl "sigs.k8s.io/controller-runtime"
)

const promWarnPayload = "{\"status\":\"success\",\"warnings\":[\"awarning\"],\"data\":{\"resultType\":\"vector\",\"result\":[{\"metric\":{\"__name__\":\"kube_pod_info\",\"container\":\"kube-rbac-proxy-main\",\"created_by_kind\":\"DaemonSet\",\"created_by_name\":\"kindnet\",\"host_ip\":\"172.18.0.2\",\"host_network\":\"true\",\"instance\":\"10.244.0.24:8443\",\"job\":\"kube-state-metrics\",\"namespace\":\"kube-system\",\"node\":\"kind-control-plane\",\"pod\":\"kindnet-llt85\",\"pod_ip\":\"172.18.0.2\",\"uid\":\"0bb9d9db-2658-439f-aed9-ab3e8502397d\"},\"value\":[1669714193.275,\"1\"]}]}}"
const promPayload = "{\"status\":\"success\",\"data\":{\"resultType\":\"vector\",\"result\":[{\"metric\":{\"__name__\":\"kube_pod_info\",\"container\":\"kube-rbac-proxy-main\",\"created_by_kind\":\"DaemonSet\",\"created_by_name\":\"kindnet\",\"host_ip\":\"172.18.0.2\",\"host_network\":\"true\",\"instance\":\"10.244.0.24:8443\",\"job\":\"kube-state-metrics\",\"namespace\":\"kube-system\",\"node\":\"kind-control-plane\",\"pod\":\"kindnet-llt85\",\"pod_ip\":\"172.18.0.2\",\"uid\":\"0bb9d9db-2658-439f-aed9-ab3e8502397d\"},\"value\":[1669714193.275,\"1\"]}]}}"
const promEmptyDataPayload = "{\"status\":\"success\",\"data\":{\"resultType\":\"vector\",\"result\":[]}}"
const promMatrixPayload = "{\"status\":\"success\",\"data\":{\"resultType\":\"matrix\",\"result\":[]}}"
const promMultiPointPayload = "{\"status\":\"success\",\"data\":{\"resultType\":\"vector\",\"result\":[{\"metric\":{\"__name__\":\"kube_pod_info\",\"container\":\"kube-rbac-proxy-main\",\"created_by_kind\":\"DaemonSet\",\"created_by_name\":\"kindnet\",\"host_ip\":\"172.18.0.2\",\"host_network\":\"true\",\"instance\":\"10.244.0.24:8443\",\"job\":\"kube-state-metrics\",\"namespace\":\"kube-system\",\"node\":\"kind-control-plane\",\"pod\":\"kindnet-llt85\",\"pod_ip\":\"172.18.0.2\",\"uid\":\"0bb9d9db-2658-439f-aed9-ab3e8502397d\"},\"value\":[1669714193.275,\"1\"]},{\"metric\":{\"__name__\":\"kube_pod_info\",\"container\":\"kube-rbac-proxy-main\",\"created_by_kind\":\"DaemonSet\",\"created_by_name\":\"kube-proxy\",\"host_ip\":\"172.18.0.2\",\"host_network\":\"true\",\"instance\":\"10.244.0.24:8443\",\"job\":\"kube-state-metrics\",\"namespace\":\"kube-system\",\"node\":\"kind-control-plane\",\"pod\":\"kube-proxy-dlq7m\",\"pod_ip\":\"172.18.0.2\",\"priority_class\":\"system-node-critical\",\"uid\":\"31240e57-5286-4bc6-ad69-80b68bf806d0\"},\"value\":[1669714193.275,\"1\"]},{\"metric\":{\"__name__\":\"kube_pod_info\",\"container\":\"kube-rbac-proxy-main\",\"created_by_kind\":\"DaemonSet\",\"created_by_name\":\"node-exporter\",\"host_ip\":\"172.18.0.2\",\"host_network\":\"true\",\"instance\":\"10.244.0.24:8443\",\"job\":\"kube-state-metrics\",\"namespace\":\"monitoring\",\"node\":\"kind-control-plane\",\"pod\":\"node-exporter-dv6nr\",\"pod_ip\":\"172.18.0.2\",\"priority_class\":\"system-cluster-critical\",\"uid\":\"cf7baf10-ac9a-4b7d-9510-a6502d7ed271\"},\"value\":[1669714193.275,\"1\"]}]}}"
const promWarnPayloadWithNoRange = "{\"status\":\"success\",\"warnings\":[\"awarning\"],\"data\":{\"resultType\":\"vector\",\"result\":[{\"metric\":{\"__name__\":\"kube_pod_info\",\"container\":\"kube-rbac-proxy-main\",\"created_by_kind\":\"DaemonSet\",\"created_by_name\":\"kindnet\",\"host_ip\":\"172.18.0.2\",\"host_network\":\"true\",\"instance\":\"10.244.0.24:8443\",\"job\":\"kube-state-metrics\",\"namespace\":\"kube-system\",\"node\":\"kind-control-plane\",\"pod\":\"kindnet-llt85\",\"pod_ip\":\"172.18.0.2\",\"uid\":\"0bb9d9db-2658-439f-aed9-ab3e8502397d\"},\"value\":[1669714193.275,\"1\"]}]}}"
const promPayloadWithNoRange = "{\"status\":\"success\",\"data\":{\"resultType\":\"vector\",\"result\":[{\"metric\":{\"__name__\":\"kube_pod_info\",\"container\":\"kube-rbac-proxy-main\",\"created_by_kind\":\"DaemonSet\",\"created_by_name\":\"kindnet\",\"host_ip\":\"172.18.0.2\",\"host_network\":\"true\",\"instance\":\"10.244.0.24:8443\",\"job\":\"kube-state-metrics\",\"namespace\":\"kube-system\",\"node\":\"kind-control-plane\",\"pod\":\"kindnet-llt85\",\"pod_ip\":\"172.18.0.2\",\"uid\":\"0bb9d9db-2658-439f-aed9-ab3e8502397d\"},\"value\":[1669714193.275,\"1\"]}]}}"
const promEmptyDataPayloadWithNoRange = "{\"status\":\"success\",\"data\":{\"resultType\":\"vector\",\"result\":[]}}"
const promMatrixPayloadWithNoRange = "{\"status\":\"success\",\"data\":{\"resultType\":\"matrix\",\"result\":[]}}"
const promMultiPointPayloadWithNoRange = "{\"status\":\"success\",\"data\":{\"resultType\":\"vector\",\"result\":[{\"metric\":{\"__name__\":\"kube_pod_info\",\"container\":\"kube-rbac-proxy-main\",\"created_by_kind\":\"DaemonSet\",\"created_by_name\":\"kindnet\",\"host_ip\":\"172.18.0.2\",\"host_network\":\"true\",\"instance\":\"10.244.0.24:8443\",\"job\":\"kube-state-metrics\",\"namespace\":\"kube-system\",\"node\":\"kind-control-plane\",\"pod\":\"kindnet-llt85\",\"pod_ip\":\"172.18.0.2\",\"uid\":\"0bb9d9db-2658-439f-aed9-ab3e8502397d\"},\"value\":[1669714193.275,\"1\"]},{\"metric\":{\"__name__\":\"kube_pod_info\",\"container\":\"kube-rbac-proxy-main\",\"created_by_kind\":\"DaemonSet\",\"created_by_name\":\"kube-proxy\",\"host_ip\":\"172.18.0.2\",\"host_network\":\"true\",\"instance\":\"10.244.0.24:8443\",\"job\":\"kube-state-metrics\",\"namespace\":\"kube-system\",\"node\":\"kind-control-plane\",\"pod\":\"kube-proxy-dlq7m\",\"pod_ip\":\"172.18.0.2\",\"priority_class\":\"system-node-critical\",\"uid\":\"31240e57-5286-4bc6-ad69-80b68bf806d0\"},\"value\":[1669714193.275,\"1\"]},{\"metric\":{\"__name__\":\"kube_pod_info\",\"container\":\"kube-rbac-proxy-main\",\"created_by_kind\":\"DaemonSet\",\"created_by_name\":\"node-exporter\",\"host_ip\":\"172.18.0.2\",\"host_network\":\"true\",\"instance\":\"10.244.0.24:8443\",\"job\":\"kube-state-metrics\",\"namespace\":\"monitoring\",\"node\":\"kind-control-plane\",\"pod\":\"node-exporter-dv6nr\",\"pod_ip\":\"172.18.0.2\",\"priority_class\":\"system-cluster-critical\",\"uid\":\"cf7baf10-ac9a-4b7d-9510-a6502d7ed271\"},\"value\":[1669714193.275,\"1\"]}]}}"

const promWarnPayloadWithRange = "{\"status\":\"success\",\"warnings\":[\"awarning\"],\"data\":{\"resultType\":\"matrix\",\"result\":[{\"metric\":{\"__name__\":\"kube_pod_info\",\"container\":\"kube-rbac-proxy-main\",\"created_by_kind\":\"DaemonSet\",\"created_by_name\":\"kindnet\",\"host_ip\":\"172.18.0.2\",\"host_network\":\"true\",\"instance\":\"10.244.0.24:8443\",\"job\":\"kube-state-metrics\",\"namespace\":\"kube-system\",\"node\":\"kind-control-plane\",\"pod\":\"kindnet-llt85\",\"pod_ip\":\"172.18.0.2\",\"uid\":\"0bb9d9db-2658-439f-aed9-ab3e8502397d\"},\"values\":[[1669714193.275,\"1\"]]}]}}"
const promPayloadWithRange = "{\"status\":\"success\",\"data\":{\"resultType\":\"matrix\",\"result\":[{\"metric\":{\"__name__\":\"kube_pod_info\",\"container\":\"kube-rbac-proxy-main\",\"created_by_kind\":\"DaemonSet\",\"created_by_name\":\"kindnet\",\"host_ip\":\"172.18.0.2\",\"host_network\":\"true\",\"instance\":\"10.244.0.24:8443\",\"job\":\"kube-state-metrics\",\"namespace\":\"kube-system\",\"node\":\"kind-control-plane\",\"pod\":\"kindnet-llt85\",\"pod_ip\":\"172.18.0.2\",\"uid\":\"0bb9d9db-2658-439f-aed9-ab3e8502397d\"},\"values\":[[1669714193.275,\"1\"]]}]}}"
const promEmptyDataPayloadWithRange = "{\"status\":\"success\",\"data\":{\"resultType\":\"matrix\",\"result\":[[]]}}"
const promVectorPayloadWithRange = "{\"status\":\"success\",\"data\":{\"resultType\":\"vector\",\"result\":[[]]}}"
const promMultiPointPayloadWithRange = "{\"status\":\"success\",\"data\":{\"resultType\":\"matrix\",\"result\":[{\"metric\":{\"__name__\":\"kube_pod_info\",\"container\":\"kube-rbac-proxy-main\",\"created_by_kind\":\"DaemonSet\",\"created_by_name\":\"kindnet\",\"host_ip\":\"172.18.0.2\",\"host_network\":\"true\",\"instance\":\"10.244.0.24:8443\",\"job\":\"kube-state-metrics\",\"namespace\":\"kube-system\",\"node\":\"kind-control-plane\",\"pod\":\"kindnet-llt85\",\"pod_ip\":\"172.18.0.2\",\"uid\":\"0bb9d9db-2658-439f-aed9-ab3e8502397d\"},\"values\":[[1669714193.275,\"1\"]]},{\"metric\":{\"__name__\":\"kube_pod_info\",\"container\":\"kube-rbac-proxy-main\",\"created_by_kind\":\"DaemonSet\",\"created_by_name\":\"kindnet\",\"host_ip\":\"172.18.0.2\",\"host_network\":\"true\",\"instance\":\"10.244.0.24:8443\",\"job\":\"kube-state-metrics\",\"namespace\":\"kube-system\",\"node\":\"kind-control-plane\",\"pod\":\"kindnet-llt85\",\"pod_ip\":\"172.18.0.2\",\"uid\":\"0bb9d9db-2658-439f-aed9-ab3e8502397d\"},\"values\":[[1669714193.275,\"1\"]]}]}}"

func Test_prometheus(t *testing.T) {
tests := []struct {
@@ -25,44 +31,87 @@ func Test_prometheus(t *testing.T) {
out string
outraw []byte
wantError bool
hasRange bool
}{
{
name: "wrong data",
name: "wrong data with no range",
in: "garbage",
out: "",
wantError: true,
},
{
name: "warnings",
in: promWarnPayload,
name: "warnings with no range",
in: promWarnPayloadWithNoRange,
out: "1",
outraw: []byte("\"1\""),
wantError: false,
hasRange: false,
},
{
name: "multiple datapoint",
in: promMultiPointPayload,
name: "multiple datapoint with no range",
in: promMultiPointPayloadWithNoRange,
out: "",
wantError: true,
hasRange: false,
},
{
name: "empty datapoint",
in: promEmptyDataPayload,
name: "empty datapoint with no range",
in: promEmptyDataPayloadWithNoRange,
out: "",
wantError: true,
hasRange: false,
},
{
name: "unsupported answer type",
in: promMatrixPayload,
name: "unsupported answer type with no range",
in: promMatrixPayloadWithNoRange,
out: "",
wantError: true,
hasRange: false,
},
{
name: "happy path with no range",
in: promPayloadWithNoRange,
out: "1",
outraw: []byte("\"1\""),
wantError: false,
hasRange: false,
},
{
name: "happy path",
in: promPayload,
name: "warnings with range",
in: promWarnPayloadWithRange,
out: "1",
outraw: []byte("\"1\""),
wantError: false,
hasRange: true,
},
{
name: "multiple datapoint with range",
in: promMultiPointPayloadWithRange,
out: "",
wantError: true,
hasRange: true,
},
{
name: "empty datapoint with range",
in: promEmptyDataPayloadWithRange,
out: "",
wantError: true,
hasRange: true,
},
{
name: "unsupported answer type with range",
in: promVectorPayloadWithRange,
out: "",
wantError: true,
hasRange: false,
},
{
name: "happy path with range",
in: promPayloadWithRange,
out: "1",
outraw: []byte("\"1\""),
wantError: false,
hasRange: true,
},
}

@@ -78,11 +127,6 @@ func Test_prometheus(t *testing.T) {
HttpClient: http.Client{},
Log: ctrl.Log.WithName("testytest"),
}
obj := metricsapi.KeptnMetric{
Spec: metricsapi.KeptnMetricSpec{
Query: "my-query",
},
}
p := metricsapi.KeptnMetricsProvider{
Spec: metricsapi.KeptnMetricsProviderSpec{
SecretKeyRef: v1.SecretKeySelector{
@@ -94,14 +138,33 @@ func Test_prometheus(t *testing.T) {
TargetServer: svr.URL,
},
}
r, raw, e := kpp.EvaluateQuery(context.TODO(), obj, p)
require.Equal(t, tt.out, r)
require.Equal(t, tt.outraw, raw)
if tt.wantError != (e != nil) {
t.Errorf("want error: %t, got: %v", tt.wantError, e)
switch tt.hasRange {
case false:
obj := metricsapi.KeptnMetric{
Spec: metricsapi.KeptnMetricSpec{
Query: "my-query",
},
}
r, raw, e := kpp.EvaluateQuery(context.TODO(), obj, p)
require.Equal(t, tt.out, r)
require.Equal(t, tt.outraw, raw)
if tt.wantError != (e != nil) {
t.Errorf("want error: %t, got: %v", tt.wantError, e)
}
case true:
obj := metricsapi.KeptnMetric{
Spec: metricsapi.KeptnMetricSpec{
Query: "my-query",
Range: &metricsapi.RangeSpec{Interval: "5m"},
},
}
r, raw, e := kpp.EvaluateQuery(context.TODO(), obj, p)
require.Equal(t, tt.out, r)
require.Equal(t, tt.outraw, raw)
if tt.wantError != (e != nil) {
t.Errorf("want error: %t, got: %v", tt.wantError, e)
}
}

})

}
}

0 comments on commit 47a3e06

Please sign in to comment.