Use current operator SDK-style metrics setup

Instead of starting our own HTTP listener, use the server provided by
the operator library.

The current operator SDK produces service and service monitor
templates for use with Kubebuilder. We still need to support setting
up metrics without Kubebuilder, so we still take care of creating the
metrics service and service monitor; however, instead of using the old
SDK-inspired code, which needs obsolete SDK packages, use the metrics
setup code we have for our own managed applications. The major
difference between the two is that the old SDK-inspired code set the
service's owner reference appropriately, whereas the new code doesn't;
but the Kubebuilder-based setup also does *not* set the owner
reference, so this doesn't seem too bad.

The Kubebuilder-based setup also supports an authenticating proxy,
which this doesn't; using this also requires adding permissions for
Prometheus, and should be handled as a separate change.

Signed-off-by: Stephen Kitt <[email protected]>

controllers/helpers/helpers.go

@@ -192,9 +192,11 @@ func ReconcileService(owner metav1.Object, service *corev1.Service, reqLogger lo
 ) (*corev1.Service, error) {
 	var err error
 
-	// Set the owner and controller
-	if err := controllerutil.SetControllerReference(owner, service, scheme); err != nil {
-		return nil, errors.Wrapf(err, "error setting owner reference for Service %s/%s", service.Namespace, service.Name)
+	if owner != nil {
+		// Set the owner and controller
+		if err := controllerutil.SetControllerReference(owner, service, scheme); err != nil {
+			return nil, errors.Wrapf(err, "error setting owner reference for Service %s/%s", service.Namespace, service.Name)
+		}
 	}
 
 	err = retry.RetryOnConflict(retry.DefaultRetry, func() error {
@@ -222,8 +224,11 @@ func ReconcileService(owner metav1.Object, service *corev1.Service, reqLogger lo
 			for k, v := range service.Annotations {
 				toUpdate.Annotations[k] = v
 			}
-			// Set the owner and controller
-			return controllerutil.SetControllerReference(owner, toUpdate, scheme) // nolint:wrapcheck // No need to wrap here
+			if owner != nil {
+				// Set the owner and controller
+				return controllerutil.SetControllerReference(owner, toUpdate, scheme) // nolint:wrapcheck // No need to wrap here
+			}
+			return nil
 		})
 		if err != nil {
 			return err // nolint:wrapcheck // No need to wrap here

controllers/metrics/metrics.go

@@ -38,12 +38,20 @@ func Setup(namespace string, owner metav1.Object, labels map[string]string, port
 	client controllerClient.Client, config *rest.Config, scheme *runtime.Scheme,
 	reqLogger logr.Logger,
 ) error {
-	app, ok := labels["app"]
+	applicationKey := "app"
+	applicationName, ok := labels[applicationKey]
+
 	if !ok {
-		return fmt.Errorf("no app label in the provided labels, %v", labels)
+		applicationKey := "name"
+		applicationName, ok = labels[applicationKey]
+
+		if !ok {
+			return fmt.Errorf("no app or name label in the provided labels, %v", labels)
+		}
 	}
 
-	metricsService, err := helpers.ReconcileService(owner, newMetricsService(namespace, app, port), reqLogger, client, scheme)
+	metricsService, err := helpers.ReconcileService(owner, newMetricsService(namespace, applicationKey, applicationName, port), reqLogger,
+		client, scheme)
 	if err != nil {
 		return err // nolint:wrapcheck // No need to wrap here
 	}
@@ -67,11 +75,10 @@ func Setup(namespace string, owner metav1.Object, labels map[string]string, port
 }
 
 // newMetricsService populates a Service providing access to metrics for the given application.
-// It is assumed that the application's resources are labeled with "app=" the given app name.
 // The Service is named after the application name, suffixed with "-metrics".
-func newMetricsService(namespace, app string, port int32) *corev1.Service {
+func newMetricsService(namespace, appKey, appName string, port int32) *corev1.Service {
 	labels := map[string]string{
-		"app": app,
+		appKey: appName,
 	}
 
 	servicePorts := []corev1.ServicePort{
@@ -85,7 +92,7 @@ func newMetricsService(namespace, app string, port int32) *corev1.Service {
 		ObjectMeta: metav1.ObjectMeta{
 			Labels:    labels,
 			Namespace: namespace,
-			Name:      fmt.Sprintf("%s-metrics", app),
+			Name:      fmt.Sprintf("%s-metrics", appName),
 		},
 		Spec: corev1.ServiceSpec{
 			Ports:    servicePorts,

go.mod

@@ -129,7 +129,6 @@ require (
 	k8s.io/klog v1.0.0 // indirect
 	k8s.io/klog/v2 v2.70.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20220627174259-011e075b9cb8 // indirect
-	k8s.io/kube-state-metrics v1.7.2 // indirect
 	sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
 	sigs.k8s.io/kustomize/api v0.8.0 // indirect
 	sigs.k8s.io/kustomize/cmd/config v0.9.11 // indirect

go.sum

@@ -2151,7 +2151,6 @@ k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2R
 k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42/go.mod h1:Z/45zLw8lUo4wdiUkI+v/ImEGAvu3WatcZl3lPMR4Rk=
 k8s.io/kube-openapi v0.0.0-20220627174259-011e075b9cb8 h1:yEQKdMCjzAOvGeiTwG4hO/hNVNtDOuUFvMUZ0OlaIzs=
 k8s.io/kube-openapi v0.0.0-20220627174259-011e075b9cb8/go.mod h1:mbJ+NSUoAhuR14N0S63bPkh8MGVSo3VYSGZtH/mfMe0=
-k8s.io/kube-state-metrics v1.7.2 h1:6vdtgXrrRRMSgnyDmgua+qvgCYv954JNfxXAtDkeLVQ=
 k8s.io/kube-state-metrics v1.7.2/go.mod h1:U2Y6DRi07sS85rmVPmBFlmv+2peBcL8IWGjM+IjYA/E=
 k8s.io/kubectl v0.18.0/go.mod h1:LOkWx9Z5DXMEg5KtOjHhRiC1fqJPLyCr3KtQgEolCkU=
 k8s.io/kubectl v0.18.2/go.mod h1:OdgFa3AlsPKRpFFYE7ICTwulXOcMGXHTc+UKhHKvrb4=

main.go

@@ -20,38 +20,32 @@ package main
 
 import (
 	"context"
-	goerrors "errors"
 	"flag"
 	"fmt"
 	"os"
 	"runtime"
 
 	operatorclient "github.com/openshift/cluster-dns-operator/pkg/operator/client"
 	"github.com/operator-framework/operator-lib/leader"
-	"github.com/operator-framework/operator-sdk/pkg/k8sutil"
-	kubemetrics "github.com/operator-framework/operator-sdk/pkg/kube-metrics"
 	sdkVersion "github.com/operator-framework/operator-sdk/version"
-	"github.com/pkg/errors"
 	"github.com/submariner-io/admiral/pkg/log/kzerolog"
 	"github.com/submariner-io/submariner-operator/api/v1alpha1"
+	"github.com/submariner-io/submariner-operator/controllers/metrics"
 	"github.com/submariner-io/submariner-operator/controllers/servicediscovery"
 	"github.com/submariner-io/submariner-operator/controllers/submariner"
 	"github.com/submariner-io/submariner-operator/pkg/crd"
 	"github.com/submariner-io/submariner-operator/pkg/gateway"
 	"github.com/submariner-io/submariner-operator/pkg/lighthouse"
-	"github.com/submariner-io/submariner-operator/pkg/metrics"
 	"github.com/submariner-io/submariner-operator/pkg/version"
 	submv1 "github.com/submariner-io/submariner/pkg/apis/submariner.io/v1"
-	v1 "k8s.io/api/core/v1"
 	apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	apiruntime "k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/util/intstr"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/kubernetes"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
-	"k8s.io/client-go/rest"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
 	"sigs.k8s.io/controller-runtime/pkg/client/config"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
@@ -61,9 +55,8 @@ import (
 
 // Change below variables to serve metrics on different host or port.
 var (
-	metricsHost               = "0.0.0.0"
-	metricsPort         int32 = 8383
-	operatorMetricsPort int32 = 8686
+	metricsHost       = "0.0.0.0"
+	metricsPort int32 = 8383
 )
 
 var (
@@ -160,23 +153,22 @@ func main() {
 		os.Exit(1)
 	}
 
-	if err = serveCRMetrics(cfg); err != nil {
-		log.Info("Could not generate and serve custom resource metrics", "error", err.Error())
-	}
+	log.Info("Setting up metrics services and monitors")
+
+	// Setup the metrics services and service monitors
+	labels := map[string]string{"name": os.Getenv("OPERATOR_NAME")}
 
-	// Add to the below struct any other metrics ports you want to expose.
-	servicePorts := []v1.ServicePort{
-		{Port: metricsPort, Name: metrics.OperatorPortName, Protocol: v1.ProtocolTCP, TargetPort: intstr.IntOrString{
-			Type:   intstr.Int,
-			IntVal: metricsPort,
-		}},
-		{Port: operatorMetricsPort, Name: metrics.CRPortName, Protocol: v1.ProtocolTCP, TargetPort: intstr.IntOrString{
-			Type:   intstr.Int,
-			IntVal: operatorMetricsPort,
-		}},
+	// We need a new client using the manager's rest.Config because
+	// the manager's caches haven't started yet and it won't allow
+	// modifications until then
+	metricsClient, err := client.New(cfg, client.Options{})
+	if err != nil {
+		log.Error(err, "Error obtaining a Kubernetes client")
 	}
 
-	createServiceMonitors(ctx, cfg, servicePorts, namespace)
+	if err := metrics.Setup(namespace, nil, labels, metricsPort, metricsClient, cfg, scheme, log); err != nil {
+		log.Error(err, "Error setting up metrics services and monitors")
+	}
 
 	log.Info("Registering Components.")
 
@@ -223,60 +215,6 @@ func main() {
 	}
 }
 
-func createServiceMonitors(ctx context.Context, cfg *rest.Config, servicePorts []v1.ServicePort, namespace string) {
-	// Create Service object to expose the metrics port(s).
-	service, ok, err := metrics.CreateMetricsService(ctx, cfg, servicePorts)
-	if err != nil {
-		log.Info("Could not create metrics Service", "error", err.Error())
-		return
-	}
-
-	if !ok {
-		return
-	}
-
-	// CreateServiceMonitors will automatically create the prometheus-operator ServiceMonitor resources
-	// necessary to configure Prometheus to scrape metrics from this operator.
-	services := []*v1.Service{service}
-
-	serviceMonitors, err := metrics.CreateServiceMonitors(cfg, namespace, services)
-	if err != nil {
-		log.Info("Could not create ServiceMonitor object", "error", err.Error())
-		// If this operator is deployed to a cluster without the prometheus-operator running, it will return
-		// ErrServiceMonitorNotPresent, which can be used to safely skip ServiceMonitor creation.
-		if goerrors.Is(err, metrics.ErrServiceMonitorNotPresent) {
-			log.Info("Install prometheus-operator in your cluster to create ServiceMonitor objects", "error", err.Error())
-		}
-	} else {
-		log.Info("Created service monitors", "service monitors", serviceMonitors)
-	}
-}
-
-// serveCRMetrics gets the Operator/CustomResource GVKs and generates metrics based on those types.
-// It serves those metrics on "http://metricsHost:operatorMetricsPort".
-func serveCRMetrics(cfg *rest.Config) error {
-	// Below function returns filtered operator/CustomResource specific GVKs.
-	// For more control override the below GVK list with your own custom logic.
-	filteredGVK, err := k8sutil.GetGVKsFromAddToScheme(v1alpha1.AddToScheme)
-	if err != nil {
-		return errors.Wrap(err, "error getting GVKs")
-	}
-	// Get the namespace the operator is currently deployed in.
-	operatorNs, err := k8sutil.GetOperatorNamespace()
-	if err != nil {
-		return errors.Wrap(err, "error getting operator namespace")
-	}
-	// To generate metrics in other namespaces, add the values below.
-	ns := []string{operatorNs}
-	// Generate and serve custom resource specific metrics.
-	err = kubemetrics.GenerateAndServeCRMetrics(cfg, ns, filteredGVK, metricsHost, operatorMetricsPort)
-	if err != nil {
-		return errors.Wrap(err, "error initializing metrics")
-	}
-
-	return nil
-}
-
 // getWatchNamespace returns the Namespace the operator should be watching for changes.
 func getWatchNamespace() (string, error) {
 	// WatchNamespaceEnvVar is the constant for env variable WATCH_NAMESPACE