Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update prometheus_metrics_test.go #4437

Merged
merged 6 commits into from
Apr 25, 2023
Merged

Update prometheus_metrics_test.go #4437

Changes from 2 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
3540bdf
Update prometheus_metrics_test.go
Parthiba-Hazra Apr 5, 2023
ebda240
Merge branch 'kedacore:main' into test-prometheus
Parthiba-Hazra Apr 6, 2023
d1ac303
Merge branch 'main' into test-prometheus
Parthiba-Hazra Apr 14, 2023
9bc66d9
Update prometheus_metrics_test.go
Parthiba-Hazra Apr 14, 2023
40f40d5
Update CHANGELOG.md
Parthiba-Hazra Apr 25, 2023
0a4411e
Merge branch 'main' into test-prometheus
Parthiba-Hazra Apr 25, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
145 changes: 145 additions & 0 deletions tests/sequential/prometheus_metrics/prometheus_metrics_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"fmt"
"strings"
"testing"
"time"

promModel "github.com/prometheus/client_model/go"
"github.com/prometheus/common/expfmt"
Expand All @@ -29,6 +30,7 @@ var (
deploymentName = fmt.Sprintf("%s-deployment", testName)
monitoredDeploymentName = fmt.Sprintf("%s-monitored", testName)
scaledObjectName = fmt.Sprintf("%s-so", testName)
wrongScaledObjectName = fmt.Sprintf("%s-wrong", testName)
cronScaledJobName = fmt.Sprintf("%s-cron-sj", testName)
clientName = fmt.Sprintf("%s-client", testName)
kedaOperatorPrometheusURL = "http://keda-operator.keda.svc.cluster.local:8080/metrics"
Expand All @@ -41,6 +43,7 @@ type templateData struct {
TestNamespace string
DeploymentName string
ScaledObjectName string
WrongScaledObjectName string
CronScaledJobName string
MonitoredDeploymentName string
ClientName string
Expand Down Expand Up @@ -114,6 +117,29 @@ spec:
value: '1'
`

wrongScaledObjectTemplate = `
apiVersion: keda.sh/v1alpha1
kind: ScaledObject
metadata:
name: {{.WrongScaledObjectName}}
namespace: {{.TestNamespace}}
spec:
scaleTargetRef:
name: {{.DeploymentName}}
pollingInterval: 2
idleReplicaCount: 0
minReplicaCount: 1
maxReplicaCount: 2
cooldownPeriod: 10
triggers:
- type: prometheus
metadata:
serverAddress: http://keda-prometheus.keda.svc.cluster.local:8080
metricName: keda_scaler_errors_total
threshold: '1'
query: 'keda_scaler_errors_total{namespace="{{.TestNamespace}}",scaledObject="{{.WrongScaledObjectName}}"}'
`

cronScaledJobTemplate = `
apiVersion: keda.sh/v1alpha1
kind: ScaledJob
Expand Down Expand Up @@ -230,6 +256,9 @@ func TestPrometheusMetrics(t *testing.T) {
testScalerMetricValue(t)
testScalerMetricLatency(t)
testScalerActiveMetric(t)
testScaledObjectErrors(t, data)
testScalerErrors(t, data)
testScalerErrorsTotal(t, data)
testMetricsServerScalerMetricValue(t)
testOperatorMetrics(t, kc, data)
testWebhookMetrics(t, data)
Expand All @@ -244,6 +273,7 @@ func getTemplateData() (templateData, []Template) {
TestNamespace: testNamespace,
DeploymentName: deploymentName,
ScaledObjectName: scaledObjectName,
WrongScaledObjectName: wrongScaledObjectName,
MonitoredDeploymentName: monitoredDeploymentName,
ClientName: clientName,
CronScaledJobName: cronScaledJobName,
Expand Down Expand Up @@ -292,6 +322,121 @@ func testScalerMetricValue(t *testing.T) {
}
}

func testScaledObjectErrors(t *testing.T, data templateData) {
t.Log("--- testing scaled object errors ---")

KubectlDeleteWithTemplate(t, data, "scaledObjectTemplate", scaledObjectTemplate)
KubectlApplyWithTemplate(t, data, "wrongScaledObjectTemplate", wrongScaledObjectTemplate)

family := fetchAndParsePrometheusMetrics(t, fmt.Sprintf("curl --insecure %s", kedaOperatorPrometheusURL))
if val, ok := family["keda_scaled_object_errors"]; ok {
errCounterVal1 := getErrorMetricsValue(val)

// wait for 2 seconds as pollinginterval is 2
time.Sleep(2 * time.Second)

family = fetchAndParsePrometheusMetrics(t, fmt.Sprintf("curl --insecure %s", kedaOperatorPrometheusURL))
if val, ok := family["keda_scaled_object_errors"]; ok {
errCounterVal2 := getErrorMetricsValue(val)
assert.Greater(t, errCounterVal2, errCounterVal1)
JorTurFer marked this conversation as resolved.
Show resolved Hide resolved
JorTurFer marked this conversation as resolved.
Show resolved Hide resolved
} else {
t.Errorf("metric not available")
}
} else {
t.Errorf("metric not available")
}

KubectlDeleteWithTemplate(t, data, "wrongScaledObjectTemplate", wrongScaledObjectTemplate)
KubectlApplyWithTemplate(t, data, "scaledObjectTemplate", scaledObjectTemplate)
}

func testScalerErrors(t *testing.T, data templateData) {
t.Log("--- testing scaler errors ---")

KubectlDeleteWithTemplate(t, data, "scaledObjectTemplate", scaledObjectTemplate)
KubectlApplyWithTemplate(t, data, "wrongScaledObjectTemplate", wrongScaledObjectTemplate)

family := fetchAndParsePrometheusMetrics(t, fmt.Sprintf("curl --insecure %s", kedaOperatorPrometheusURL))
if val, ok := family["keda_scaler_errors"]; ok {
errCounterVal1 := getErrorMetricsValue(val)

// wait for 2 seconds as pollinginterval is 2
time.Sleep(2 * time.Second)

Comment on lines +365 to +366
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe we need to increase this time to ensure that the metric is exposed

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should I increase it from 2 to 4?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Or even bigger xD, we need to ensure that the test sin't flaky. I mean, in this case, waiting 2,4,10 seconds doesn't change the test because it's checking > 0, never mind if 1 or 100. If you think that 10 is safer, go with 10 :)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Actually its confusing, cause in my local cluster it's worked well. At least it should pass after 4 seconds, I will change it to 4.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should I create a new PR on this?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, please 🙏

family = fetchAndParsePrometheusMetrics(t, fmt.Sprintf("curl --insecure %s", kedaOperatorPrometheusURL))
if val, ok := family["keda_scaler_errors"]; ok {
errCounterVal2 := getErrorMetricsValue(val)
assert.Greater(t, errCounterVal2, errCounterVal1)
JorTurFer marked this conversation as resolved.
Show resolved Hide resolved
} else {
t.Errorf("metric not available")
}
} else {
t.Errorf("metric not available")
}

KubectlDeleteWithTemplate(t, data, "wrongScaledObjectTemplate", wrongScaledObjectTemplate)
KubectlApplyWithTemplate(t, data, "scaledObjectTemplate", scaledObjectTemplate)
}

func testScalerErrorsTotal(t *testing.T, data templateData) {
t.Log("--- testing scaler errors total ---")

KubectlDeleteWithTemplate(t, data, "scaledObjectTemplate", scaledObjectTemplate)
KubectlApplyWithTemplate(t, data, "wrongScaledObjectTemplate", wrongScaledObjectTemplate)

family := fetchAndParsePrometheusMetrics(t, fmt.Sprintf("curl --insecure %s", kedaOperatorPrometheusURL))
if val, ok := family["keda_scaler_errors_total"]; ok {
errCounterVal1 := getErrorMetricsValue(val)

// wait for 2 seconds as pollinginterval is 2
time.Sleep(2 * time.Second)

family = fetchAndParsePrometheusMetrics(t, fmt.Sprintf("curl --insecure %s", kedaOperatorPrometheusURL))
if val, ok := family["keda_scaler_errors_total"]; ok {
errCounterVal2 := getErrorMetricsValue(val)
assert.Greater(t, errCounterVal2, errCounterVal1)
JorTurFer marked this conversation as resolved.
Show resolved Hide resolved
} else {
t.Errorf("metric not available")
}
} else {
t.Errorf("metric not available")
}

KubectlDeleteWithTemplate(t, data, "wrongScaledObjectTemplate", wrongScaledObjectTemplate)
KubectlApplyWithTemplate(t, data, "scaledObjectTemplate", scaledObjectTemplate)
}

func getErrorMetricsValue(val *promModel.MetricFamily) float64 {
switch val.GetName() {
case "keda_scaler_errors_total":
metrics := val.GetMetric()
for _, metric := range metrics {
return metric.GetCounter().GetValue()
}
case "keda_scaled_object_errors":
metrics := val.GetMetric()
for _, metric := range metrics {
labels := metric.GetLabel()
for _, label := range labels {
if *label.Name == "scaledObject" && *label.Value == wrongScaledObjectName {
return *metric.Counter.Value
}
}
}
case "keda_scaler_errors":
metrics := val.GetMetric()
for _, metric := range metrics {
labels := metric.GetLabel()
for _, label := range labels {
if *label.Name == "scaler" && *label.Value == "prometheusScaler" {
return *metric.Counter.Value
}
}
}
}
return 0
}

func testScalerMetricLatency(t *testing.T) {
t.Log("--- testing scaler metric latency ---")

Expand Down