diff --git a/controllers/helpers/helpers.go b/controllers/helpers/helpers.go index d73126645..30c3358bd 100644 --- a/controllers/helpers/helpers.go +++ b/controllers/helpers/helpers.go @@ -192,9 +192,11 @@ func ReconcileService(owner metav1.Object, service *corev1.Service, reqLogger lo ) (*corev1.Service, error) { var err error - // Set the owner and controller - if err := controllerutil.SetControllerReference(owner, service, scheme); err != nil { - return nil, errors.Wrapf(err, "error setting owner reference for Service %s/%s", service.Namespace, service.Name) + if owner != nil { + // Set the owner and controller + if err := controllerutil.SetControllerReference(owner, service, scheme); err != nil { + return nil, errors.Wrapf(err, "error setting owner reference for Service %s/%s", service.Namespace, service.Name) + } } err = retry.RetryOnConflict(retry.DefaultRetry, func() error { @@ -222,8 +224,11 @@ func ReconcileService(owner metav1.Object, service *corev1.Service, reqLogger lo for k, v := range service.Annotations { toUpdate.Annotations[k] = v } - // Set the owner and controller - return controllerutil.SetControllerReference(owner, toUpdate, scheme) // nolint:wrapcheck // No need to wrap here + if owner != nil { + // Set the owner and controller + return controllerutil.SetControllerReference(owner, toUpdate, scheme) // nolint:wrapcheck // No need to wrap here + } + return nil }) if err != nil { return err // nolint:wrapcheck // No need to wrap here diff --git a/controllers/metrics/metrics.go b/controllers/metrics/metrics.go index 43b55ffb5..d9ed22c04 100644 --- a/controllers/metrics/metrics.go +++ b/controllers/metrics/metrics.go @@ -38,12 +38,20 @@ func Setup(namespace string, owner metav1.Object, labels map[string]string, port client controllerClient.Client, config *rest.Config, scheme *runtime.Scheme, reqLogger logr.Logger, ) error { - app, ok := labels["app"] + applicationKey := "app" + applicationName, ok := labels[applicationKey] + if !ok { - return fmt.Errorf("no app label in the provided labels, %v", labels) + applicationKey := "name" + applicationName, ok = labels[applicationKey] + + if !ok { + return fmt.Errorf("no app or name label in the provided labels, %v", labels) + } } - metricsService, err := helpers.ReconcileService(owner, newMetricsService(namespace, app, port), reqLogger, client, scheme) + metricsService, err := helpers.ReconcileService(owner, newMetricsService(namespace, applicationKey, applicationName, port), reqLogger, + client, scheme) if err != nil { return err // nolint:wrapcheck // No need to wrap here } @@ -67,11 +75,10 @@ func Setup(namespace string, owner metav1.Object, labels map[string]string, port } // newMetricsService populates a Service providing access to metrics for the given application. -// It is assumed that the application's resources are labeled with "app=" the given app name. // The Service is named after the application name, suffixed with "-metrics". -func newMetricsService(namespace, app string, port int32) *corev1.Service { +func newMetricsService(namespace, appKey, appName string, port int32) *corev1.Service { labels := map[string]string{ - "app": app, + appKey: appName, } servicePorts := []corev1.ServicePort{ @@ -85,7 +92,7 @@ func newMetricsService(namespace, app string, port int32) *corev1.Service { ObjectMeta: metav1.ObjectMeta{ Labels: labels, Namespace: namespace, - Name: fmt.Sprintf("%s-metrics", app), + Name: fmt.Sprintf("%s-metrics", appName), }, Spec: corev1.ServiceSpec{ Ports: servicePorts, diff --git a/go.mod b/go.mod index b36cfb456..859567193 100644 --- a/go.mod +++ b/go.mod @@ -129,7 +129,6 @@ require ( k8s.io/klog v1.0.0 // indirect k8s.io/klog/v2 v2.70.1 // indirect k8s.io/kube-openapi v0.0.0-20220627174259-011e075b9cb8 // indirect - k8s.io/kube-state-metrics v1.7.2 // indirect sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect sigs.k8s.io/kustomize/api v0.8.0 // indirect sigs.k8s.io/kustomize/cmd/config v0.9.11 // indirect diff --git a/go.sum b/go.sum index 23dd29589..c975bcdf7 100644 --- a/go.sum +++ b/go.sum @@ -2151,7 +2151,6 @@ k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2R k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42/go.mod h1:Z/45zLw8lUo4wdiUkI+v/ImEGAvu3WatcZl3lPMR4Rk= k8s.io/kube-openapi v0.0.0-20220627174259-011e075b9cb8 h1:yEQKdMCjzAOvGeiTwG4hO/hNVNtDOuUFvMUZ0OlaIzs= k8s.io/kube-openapi v0.0.0-20220627174259-011e075b9cb8/go.mod h1:mbJ+NSUoAhuR14N0S63bPkh8MGVSo3VYSGZtH/mfMe0= -k8s.io/kube-state-metrics v1.7.2 h1:6vdtgXrrRRMSgnyDmgua+qvgCYv954JNfxXAtDkeLVQ= k8s.io/kube-state-metrics v1.7.2/go.mod h1:U2Y6DRi07sS85rmVPmBFlmv+2peBcL8IWGjM+IjYA/E= k8s.io/kubectl v0.18.0/go.mod h1:LOkWx9Z5DXMEg5KtOjHhRiC1fqJPLyCr3KtQgEolCkU= k8s.io/kubectl v0.18.2/go.mod h1:OdgFa3AlsPKRpFFYE7ICTwulXOcMGXHTc+UKhHKvrb4= diff --git a/main.go b/main.go index bcd468bce..18d589525 100644 --- a/main.go +++ b/main.go @@ -20,7 +20,6 @@ package main import ( "context" - goerrors "errors" "flag" "fmt" "os" @@ -28,30 +27,25 @@ import ( operatorclient "github.com/openshift/cluster-dns-operator/pkg/operator/client" "github.com/operator-framework/operator-lib/leader" - "github.com/operator-framework/operator-sdk/pkg/k8sutil" - kubemetrics "github.com/operator-framework/operator-sdk/pkg/kube-metrics" sdkVersion "github.com/operator-framework/operator-sdk/version" - "github.com/pkg/errors" "github.com/submariner-io/admiral/pkg/log/kzerolog" "github.com/submariner-io/submariner-operator/api/v1alpha1" + "github.com/submariner-io/submariner-operator/controllers/metrics" "github.com/submariner-io/submariner-operator/controllers/servicediscovery" "github.com/submariner-io/submariner-operator/controllers/submariner" "github.com/submariner-io/submariner-operator/pkg/crd" "github.com/submariner-io/submariner-operator/pkg/gateway" "github.com/submariner-io/submariner-operator/pkg/lighthouse" - "github.com/submariner-io/submariner-operator/pkg/metrics" "github.com/submariner-io/submariner-operator/pkg/version" submv1 "github.com/submariner-io/submariner/pkg/apis/submariner.io/v1" - v1 "k8s.io/api/core/v1" apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apiruntime "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/intstr" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" clientgoscheme "k8s.io/client-go/kubernetes/scheme" _ "k8s.io/client-go/plugin/pkg/client/auth" - "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" "sigs.k8s.io/controller-runtime/pkg/client/config" logf "sigs.k8s.io/controller-runtime/pkg/log" @@ -61,9 +55,8 @@ import ( // Change below variables to serve metrics on different host or port. var ( - metricsHost = "0.0.0.0" - metricsPort int32 = 8383 - operatorMetricsPort int32 = 8686 + metricsHost = "0.0.0.0" + metricsPort int32 = 8383 ) var ( @@ -160,23 +153,22 @@ func main() { os.Exit(1) } - if err = serveCRMetrics(cfg); err != nil { - log.Info("Could not generate and serve custom resource metrics", "error", err.Error()) - } + log.Info("Setting up metrics services and monitors") + + // Setup the metrics services and service monitors + labels := map[string]string{"name": os.Getenv("OPERATOR_NAME")} - // Add to the below struct any other metrics ports you want to expose. - servicePorts := []v1.ServicePort{ - {Port: metricsPort, Name: metrics.OperatorPortName, Protocol: v1.ProtocolTCP, TargetPort: intstr.IntOrString{ - Type: intstr.Int, - IntVal: metricsPort, - }}, - {Port: operatorMetricsPort, Name: metrics.CRPortName, Protocol: v1.ProtocolTCP, TargetPort: intstr.IntOrString{ - Type: intstr.Int, - IntVal: operatorMetricsPort, - }}, + // We need a new client using the manager's rest.Config because + // the manager's caches haven't started yet and it won't allow + // modifications until then + metricsClient, err := client.New(cfg, client.Options{}) + if err != nil { + log.Error(err, "Error obtaining a Kubernetes client") } - createServiceMonitors(ctx, cfg, servicePorts, namespace) + if err := metrics.Setup(namespace, nil, labels, metricsPort, metricsClient, cfg, scheme, log); err != nil { + log.Error(err, "Error setting up metrics services and monitors") + } log.Info("Registering Components.") @@ -223,60 +215,6 @@ func main() { } } -func createServiceMonitors(ctx context.Context, cfg *rest.Config, servicePorts []v1.ServicePort, namespace string) { - // Create Service object to expose the metrics port(s). - service, ok, err := metrics.CreateMetricsService(ctx, cfg, servicePorts) - if err != nil { - log.Info("Could not create metrics Service", "error", err.Error()) - return - } - - if !ok { - return - } - - // CreateServiceMonitors will automatically create the prometheus-operator ServiceMonitor resources - // necessary to configure Prometheus to scrape metrics from this operator. - services := []*v1.Service{service} - - serviceMonitors, err := metrics.CreateServiceMonitors(cfg, namespace, services) - if err != nil { - log.Info("Could not create ServiceMonitor object", "error", err.Error()) - // If this operator is deployed to a cluster without the prometheus-operator running, it will return - // ErrServiceMonitorNotPresent, which can be used to safely skip ServiceMonitor creation. - if goerrors.Is(err, metrics.ErrServiceMonitorNotPresent) { - log.Info("Install prometheus-operator in your cluster to create ServiceMonitor objects", "error", err.Error()) - } - } else { - log.Info("Created service monitors", "service monitors", serviceMonitors) - } -} - -// serveCRMetrics gets the Operator/CustomResource GVKs and generates metrics based on those types. -// It serves those metrics on "http://metricsHost:operatorMetricsPort". -func serveCRMetrics(cfg *rest.Config) error { - // Below function returns filtered operator/CustomResource specific GVKs. - // For more control override the below GVK list with your own custom logic. - filteredGVK, err := k8sutil.GetGVKsFromAddToScheme(v1alpha1.AddToScheme) - if err != nil { - return errors.Wrap(err, "error getting GVKs") - } - // Get the namespace the operator is currently deployed in. - operatorNs, err := k8sutil.GetOperatorNamespace() - if err != nil { - return errors.Wrap(err, "error getting operator namespace") - } - // To generate metrics in other namespaces, add the values below. - ns := []string{operatorNs} - // Generate and serve custom resource specific metrics. - err = kubemetrics.GenerateAndServeCRMetrics(cfg, ns, filteredGVK, metricsHost, operatorMetricsPort) - if err != nil { - return errors.Wrap(err, "error initializing metrics") - } - - return nil -} - // getWatchNamespace returns the Namespace the operator should be watching for changes. func getWatchNamespace() (string, error) { // WatchNamespaceEnvVar is the constant for env variable WATCH_NAMESPACE diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go deleted file mode 100644 index 6890e93ce..000000000 --- a/pkg/metrics/metrics.go +++ /dev/null @@ -1,181 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// -// Copyright Contributors to the Submariner project. -// Copyright 2018 The Operator-SDK Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package metrics - -import ( - "context" - goerrors "errors" - "fmt" - - "github.com/operator-framework/operator-sdk/pkg/k8sutil" - "github.com/pkg/errors" - "github.com/submariner-io/admiral/pkg/resource" - "github.com/submariner-io/admiral/pkg/util" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - crclient "sigs.k8s.io/controller-runtime/pkg/client" - logf "sigs.k8s.io/controller-runtime/pkg/log" -) - -var log = logf.Log.WithName("metrics") - -const ( - // OperatorPortName defines the default operator metrics port name used in the metrics Service. - OperatorPortName = "http-metrics" - // CRPortName defines the custom resource specific metrics' port name used in the metrics Service. - CRPortName = "cr-metrics" -) - -// CreateMetricsService creates a Kubernetes Service to expose the passed metrics -// port(s) with the given name(s). -func CreateMetricsService(ctx context.Context, cfg *rest.Config, servicePorts []v1.ServicePort) (*v1.Service, bool, error) { - if len(servicePorts) < 1 { - return nil, false, fmt.Errorf("failed to create metrics Serice; service ports were empty") - } - - client, err := crclient.New(cfg, crclient.Options{}) - if err != nil { - return nil, false, fmt.Errorf("failed to create new client: %w", err) - } - - clientSet, err := kubernetes.NewForConfig(cfg) - if err != nil { - return nil, false, fmt.Errorf("failed to create clientset: %w", err) - } - - s, err := initOperatorService(ctx, client, servicePorts) - if err != nil { - if goerrors.Is(err, k8sutil.ErrNoNamespace) || goerrors.Is(err, k8sutil.ErrRunLocal) { - log.Info("Skipping metrics Service creation; not running in a cluster.") - return nil, false, nil - } - - return nil, false, fmt.Errorf("failed to initialize service object for metrics: %w", err) - } - - _, err = util.CreateOrUpdate(ctx, resource.ForService(clientSet, s.Namespace), - s, func(existing runtime.Object) (runtime.Object, error) { - existingService := existing.(*v1.Service) - if existingService.Spec.Type == v1.ServiceTypeClusterIP { - s.Spec.ClusterIP = existingService.Spec.ClusterIP - } - return s, nil - }) - - if err != nil { - return nil, false, errors.Wrapf(err, "error creating or updating Service %s/%s", s.Namespace, s.Name) - } - - s, err = clientSet.CoreV1().Services(s.Namespace).Get(ctx, s.Name, metav1.GetOptions{}) - - return s, true, errors.Wrapf(err, "error retrieving Service %s/%s", s.Namespace, s.Name) -} - -// initOperatorService returns the static service which exposes specified port(s). -func initOperatorService(ctx context.Context, client crclient.Client, sp []v1.ServicePort) (*v1.Service, error) { - operatorName, err := k8sutil.GetOperatorName() - if err != nil { - return nil, err // nolint:wrapcheck // No need to wrap here - } - - namespace, err := k8sutil.GetOperatorNamespace() - if err != nil { - return nil, err // nolint:wrapcheck // No need to wrap here - } - - label := map[string]string{"name": operatorName} - - service := &v1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-metrics", operatorName), - Namespace: namespace, - Labels: label, - }, - Spec: v1.ServiceSpec{ - Ports: sp, - Selector: label, - }, - } - - ownRef, err := getPodOwnerRef(ctx, client, namespace) - if err != nil { - return nil, err - } - - service.SetOwnerReferences([]metav1.OwnerReference{*ownRef}) - - return service, nil -} - -func getPodOwnerRef(ctx context.Context, client crclient.Client, ns string) (*metav1.OwnerReference, error) { - // Get current Pod the operator is running in - pod, err := k8sutil.GetPod(ctx, client, ns) - if err != nil { - return nil, err // nolint:wrapcheck // No need to wrap here - } - - podOwnerRefs := metav1.NewControllerRef(pod, pod.GroupVersionKind()) - - // Get Owner that the Pod belongs to - ownerRef := metav1.GetControllerOf(pod) - - finalOwnerRef, found, err := findFinalOwnerRef(ctx, client, ns, ownerRef) - if err != nil { - return nil, err - } - - if found { - return finalOwnerRef, nil - } - - // Default to returning Pod as the Owner - return podOwnerRefs, nil -} - -// findFinalOwnerRef tries to locate the final controller/owner based on the owner reference provided. -func findFinalOwnerRef(ctx context.Context, client crclient.Client, ns string, - ownerRef *metav1.OwnerReference, -) (*metav1.OwnerReference, bool, error) { - if ownerRef == nil { - return nil, false, nil - } - - obj := &unstructured.Unstructured{} - obj.SetAPIVersion(ownerRef.APIVersion) - obj.SetKind(ownerRef.Kind) - - err := client.Get(ctx, types.NamespacedName{Namespace: ns, Name: ownerRef.Name}, obj) - if err != nil { - return nil, false, errors.Wrapf(err, "error retrieving owner reference %s/%s", ns, ownerRef.Name) - } - - newOwnerRef := metav1.GetControllerOf(obj) - if newOwnerRef != nil { - return findFinalOwnerRef(ctx, client, ns, newOwnerRef) - } - - log.V(1).Info("Pods owner found", "Kind", ownerRef.Kind, "Name", - ownerRef.Name, "Namespace", ns) - - return ownerRef, true, nil -} diff --git a/pkg/metrics/service-monitor.go b/pkg/metrics/service-monitor.go index 7069526c8..ca2503c08 100644 --- a/pkg/metrics/service-monitor.go +++ b/pkg/metrics/service-monitor.go @@ -31,9 +31,13 @@ import ( "k8s.io/client-go/discovery" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" + logf "sigs.k8s.io/controller-runtime/pkg/log" ) -var ErrServiceMonitorNotPresent = fmt.Errorf("no ServiceMonitor registered with the API") +var ( + log = logf.Log.WithName("metrics") + ErrServiceMonitorNotPresent = fmt.Errorf("no ServiceMonitor registered with the API") +) const openshiftMonitoringNS = "openshift-monitoring"