diff --git a/internal/test/envtest/environment.go b/internal/test/envtest/environment.go index fb915b143410..1c6dd9dd0252 100644 --- a/internal/test/envtest/environment.go +++ b/internal/test/envtest/environment.go @@ -48,6 +48,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/metrics" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/webhook" @@ -176,10 +177,21 @@ func Run(ctx context.Context, input RunInput) int { return code } + var errs []error + + if err := verifyPanicMetrics(); err != nil { + errs = append(errs, errors.Wrapf(err, "panics occurred during tests")) + } + // Tearing down the test environment if err := env.stop(); err != nil { - panic(fmt.Sprintf("Failed to stop the test environment: %v", err)) + errs = append(errs, errors.Wrapf(err, "failed to stop the test environment")) } + + if len(errs) > 0 { + panic(kerrors.NewAggregate(errs)) + } + return code } @@ -527,3 +539,41 @@ func (e *Environment) CreateNamespace(ctx context.Context, generateName string) return ns, nil } + +func verifyPanicMetrics() error { + metricFamilies, err := metrics.Registry.Gather() + if err != nil { + return err + } + + var errs []error + for _, metricFamily := range metricFamilies { + if metricFamily.GetName() == "controller_runtime_reconcile_panics_total" { + for _, controllerPanicMetric := range metricFamily.Metric { + if controllerPanicMetric.Counter != nil && controllerPanicMetric.Counter.Value != nil && *controllerPanicMetric.Counter.Value > 0 { + controllerName := "unknown" + for _, label := range controllerPanicMetric.Label { + if *label.Name == "controller" { + controllerName = *label.Value + } + } + errs = append(errs, fmt.Errorf("%.0f panics occurred in %q controller (check logs for more details)", *controllerPanicMetric.Counter.Value, controllerName)) + } + } + } + + if metricFamily.GetName() == "controller_runtime_webhook_panics_total" { + for _, webhookPanicMetric := range metricFamily.Metric { + if webhookPanicMetric.Counter != nil && webhookPanicMetric.Counter.Value != nil && *webhookPanicMetric.Counter.Value > 0 { + errs = append(errs, fmt.Errorf("%.0f panics occurred in webhooks (check logs for more details)", *webhookPanicMetric.Counter.Value)) + } + } + } + } + + if len(errs) > 0 { + return kerrors.NewAggregate(errs) + } + + return nil +} diff --git a/test/framework/deployment_helpers.go b/test/framework/deployment_helpers.go index 93d4d7186efa..f31a03eed7a5 100644 --- a/test/framework/deployment_helpers.go +++ b/test/framework/deployment_helpers.go @@ -436,7 +436,7 @@ func verifyMetrics(data []byte) error { controllerName = *label.Value } } - errs = append(errs, fmt.Errorf("panic occurred in %q controller", controllerName)) + errs = append(errs, fmt.Errorf("%.0f panics occurred in %q controller (check logs for more details)", *controllerPanicMetric.Counter.Value, controllerName)) } } } @@ -444,7 +444,7 @@ func verifyMetrics(data []byte) error { if metric == "controller_runtime_webhook_panics_total" { for _, webhookPanicMetric := range metricFamily.Metric { if webhookPanicMetric.Counter != nil && webhookPanicMetric.Counter.Value != nil && *webhookPanicMetric.Counter.Value > 0 { - errs = append(errs, fmt.Errorf("panic occurred in webhook")) + errs = append(errs, fmt.Errorf("%.0f panics occurred in webhooks (check logs for more details)", *webhookPanicMetric.Counter.Value)) } } }