[FEAT] service monitor creation

cmd/controller/main.go

@@ -26,6 +26,9 @@ import (
 	"github.com/sap/cap-operator/pkg/client/clientset/versioned"
 	istio "istio.io/client-go/pkg/clientset/versioned"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	promop "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned"
+	apiext "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 )
 
 const (
@@ -52,6 +55,16 @@ func main() {
 		klog.Fatal("could not create client for custom resources: ", err.Error())
 	}
 
+	apiExtClient, err := apiext.NewForConfig(config)
+	if err != nil {
+		klog.Fatal("could not create client for api-extensions: ", err.Error())
+	}
+
+	promClient, err := promop.NewForConfig(config)
+	if err != nil {
+		klog.Fatal("could not create client for prometheus-operator resources: ", err.Error())
+	}
+
 	istioClient, err := istio.NewForConfig(config)
 	if err != nil {
 		klog.Fatal("could not create client for istio resources: ", err.Error())
@@ -107,7 +120,7 @@ func main() {
 		Callbacks: leaderelection.LeaderCallbacks{
 			OnStartedLeading: func(ctx context.Context) {
 				klog.InfoS("Started leading: ", LeaseLockName, leaseLockId)
-				c := controller.NewController(coreClient, crdClient, istioClient, certClient, certManagerClient, dnsClient)
+				c := controller.NewController(coreClient, crdClient, istioClient, certClient, certManagerClient, dnsClient, apiExtClient, promClient)
 				go c.Start(ctx)
 			},
 			OnStoppedLeading: func() {

crds/sme.sap.com_capapplicationversions.yaml

@@ -1410,6 +1410,21 @@ spec:
                                     type: object
                                   type: array
                               type: object
+                            scrapeConfig:
+                              properties:
+                                interval:
+                                  pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$
+                                  type: string
+                                path:
+                                  type: string
+                                port:
+                                  type: string
+                                scrapeTimeout:
+                                  pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$
+                                  type: string
+                              required:
+                              - port
+                              type: object
                           type: object
                         nodeName:
                           type: string

go.mod

@@ -13,14 +13,18 @@ require (
 	github.com/google/go-cmp v0.6.0
 	github.com/google/uuid v1.6.0
 	github.com/lestrrat-go/jwx/v2 v2.1.1
+	github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.76.2
+	github.com/prometheus-operator/prometheus-operator/pkg/client v0.76.2
 	github.com/prometheus/client_golang v1.20.2
 	github.com/prometheus/common v0.55.0
 	go.uber.org/zap v1.27.0
+	golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e
 	golang.org/x/mod v0.21.0
 	google.golang.org/protobuf v1.34.2
 	istio.io/api v1.23.0
 	istio.io/client-go v1.23.0
 	k8s.io/api v0.31.0
+	k8s.io/apiextensions-apiserver v0.31.0
 	k8s.io/apimachinery v0.31.0
 	k8s.io/client-go v0.31.0
 	k8s.io/code-generator v0.31.0
@@ -61,7 +65,6 @@ require (
 	github.com/x448/float16 v0.8.4 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/crypto v0.27.0 // indirect
-	golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e
 	golang.org/x/net v0.29.0 // indirect
 	golang.org/x/oauth2 v0.23.0 // indirect
 	golang.org/x/sync v0.8.0 // indirect
@@ -75,10 +78,10 @@ require (
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/apiextensions-apiserver v0.31.0 // indirect
 	k8s.io/gengo/v2 v2.0.0-20240826214909-a7b603a56eb7 // indirect
 	k8s.io/kube-openapi v0.0.0-20240903163716-9e1beecbcb38 // indirect
 	k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3 // indirect
+	sigs.k8s.io/controller-runtime v0.18.5 // indirect
 	sigs.k8s.io/gateway-api v1.1.0 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/yaml v1.4.0 // indirect

go.sum

@@ -30,6 +30,7 @@ github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF
 github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4=
 github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE=
 github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ=
+github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
 github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
 github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
 github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
@@ -100,6 +101,10 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.76.2 h1:BpGDC87A2SaxbKgONsFLEX3kRcRJee2aLQbjXsuz0hA=
+github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.76.2/go.mod h1:Rd8YnCqz+2FYsiGmE2DMlaLjQRB4v2jFNnzCt9YY4IM=
+github.com/prometheus-operator/prometheus-operator/pkg/client v0.76.2 h1:yncs8NglhE3hB+viNsabCAF9TBBDOBljHUyxHC5fSGY=
+github.com/prometheus-operator/prometheus-operator/pkg/client v0.76.2/go.mod h1:AfbzyEUFxJmSoTiMcgNHHjDKcorBVd9TIwx0viURgEw=
 github.com/prometheus/client_golang v1.20.2 h1:5ctymQzZlyOON1666svgwn3s6IKWgfbjsejTMiXIyjg=
 github.com/prometheus/client_golang v1.20.2/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
 github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
@@ -216,6 +221,8 @@ k8s.io/kube-openapi v0.0.0-20240903163716-9e1beecbcb38 h1:1dWzkmJrrprYvjGwh9kEUx
 k8s.io/kube-openapi v0.0.0-20240903163716-9e1beecbcb38/go.mod h1:coRQXBK9NxO98XUv3ZD6AK3xzHCxV6+b7lrquKwaKzA=
 k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3 h1:b2FmK8YH+QEwq/Sy2uAEhmqL5nPfGYbJOcaqjeYYZoA=
 k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
+sigs.k8s.io/controller-runtime v0.18.5 h1:nTHio/W+Q4aBlQMgbnC5hZb4IjIidyrizMai9P6n4Rk=
+sigs.k8s.io/controller-runtime v0.18.5/go.mod h1:TVoGrfdpbA9VRFaRnKgk9P5/atA0pMwq+f+msb9M8Sg=
 sigs.k8s.io/gateway-api v1.1.0 h1:DsLDXCi6jR+Xz8/xd0Z1PYl2Pn0TyaFMOPPZIj4inDM=
 sigs.k8s.io/gateway-api v1.1.0/go.mod h1:ZH4lHrL2sDi0FHZ9jjneb8kKnGzFWyrTya35sWUTrRs=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=

internal/controller/common_test.go

@@ -30,6 +30,7 @@ import (
 	gardenerdnsscheme "github.com/gardener/external-dns-management/pkg/client/dns/clientset/versioned/scheme"
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
+	promopFake "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned/fake"
 	"github.com/sap/cap-operator/pkg/apis/sme.sap.com/v1alpha1"
 	copfake "github.com/sap/cap-operator/pkg/client/clientset/versioned/fake"
 	smeScheme "github.com/sap/cap-operator/pkg/client/clientset/versioned/scheme"
@@ -42,6 +43,7 @@ import (
 	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
 	networkingv1 "k8s.io/api/networking/v1"
+	apiextFake "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/fake"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -217,6 +219,8 @@ func initializeControllerForReconciliationTests(t *testing.T, items []ResourceAc
 	gardenerCertClient := gardenercertfake.NewSimpleClientset()
 	gardenerDNSClient := gardenerdnsfake.NewSimpleClientset()
 	certManagerClient := certManagerFake.NewSimpleClientset()
+	apiExtClient := apiextFake.NewSimpleClientset()
+	promopClient := promopFake.NewSimpleClientset()
 
 	copClient.PrependReactor("create", "*", generateNameCreateHandler)
 	copClient.PrependReactor("update", "*", removeStatusTimestampHandler)
@@ -239,7 +243,7 @@ func initializeControllerForReconciliationTests(t *testing.T, items []ResourceAc
 	gardenerCertClient.PrependReactor("*", "*", getErrorReactorWithResources(t, items))
 	gardenerCertClient.PrependReactor("delete-collection", "*", getDeleteCollectionHandler(t, gardenerDNSClient))
 
-	c := NewController(coreClient, copClient, istioClient, gardenerCertClient, certManagerClient, gardenerDNSClient)
+	c := NewController(coreClient, copClient, istioClient, gardenerCertClient, certManagerClient, gardenerDNSClient, apiExtClient, promopClient)
 	c.eventRecorder = events.NewFakeRecorder(10)
 	return c
 }

internal/controller/controller.go

@@ -33,6 +33,9 @@ import (
 	"k8s.io/client-go/tools/events"
 	"k8s.io/client-go/util/workqueue"
 	"k8s.io/klog/v2"
+
+	promop "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned"
+	apiext "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 )
 
 type Controller struct {
@@ -42,6 +45,8 @@ type Controller struct {
 	gardenerCertificateClient    gardenerCert.Interface
 	certManagerCertificateClient certManager.Interface
 	gardenerDNSClient            gardenerDNS.Interface
+	apiExtClient                 apiext.Interface
+	promClient                   promop.Interface
 	kubeInformerFactory          informers.SharedInformerFactory
 	crdInformerFactory           crdInformers.SharedInformerFactory
 	istioInformerFactory         istioInformers.SharedInformerFactory
@@ -53,7 +58,7 @@ type Controller struct {
 	eventRecorder                events.EventRecorder
 }
 
-func NewController(client kubernetes.Interface, crdClient versioned.Interface, istioClient istio.Interface, gardenerCertificateClient gardenerCert.Interface, certManagerCertificateClient certManager.Interface, gardenerDNSClient gardenerDNS.Interface) *Controller {
+func NewController(client kubernetes.Interface, crdClient versioned.Interface, istioClient istio.Interface, gardenerCertificateClient gardenerCert.Interface, certManagerCertificateClient certManager.Interface, gardenerDNSClient gardenerDNS.Interface, apiExtClient apiext.Interface, promClient promop.Interface) *Controller {
 	queues := map[int]workqueue.RateLimitingInterface{
 		ResourceCAPApplication:        workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()),
 		ResourceCAPApplicationVersion: workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()),
@@ -101,6 +106,8 @@ func NewController(client kubernetes.Interface, crdClient versioned.Interface, i
 		gardenerCertificateClient:    gardenerCertificateClient,
 		certManagerCertificateClient: certManagerCertificateClient,
 		gardenerDNSClient:            gardenerDNSClient,
+		apiExtClient:                 apiExtClient,
+		promClient:                   promClient,
 		kubeInformerFactory:          kubeInformerFactory,
 		crdInformerFactory:           crdInformerFactory,
 		istioInformerFactory:         istioInformerFactory,

internal/controller/reconcile-capapplicationversion.go

@@ -31,8 +31,9 @@ const (
 )
 
 const (
-	CategoryWorkload = "Workload"
-	CategoryService  = "Service"
+	CategoryWorkload       = "Workload"
+	CategoryService        = "Service"
+	CategoryServiceMonitor = "ServiceMonitor"
 )
 
 const (
@@ -397,7 +398,9 @@ func (c *Controller) updateServices(ca *v1alpha1.CAPApplication, cav *v1alpha1.C
 			return err
 		}
 	}
-	return nil
+
+	// attempt to reconcile service monitors
+	return c.updateServiceMonitors(context.TODO(), ca, cav, workloadServicePortInfos)
 }
 
 // newService creates a new Service for a CAV resource. It also sets the appropriate OwnerReferences.

internal/controller/reconcile_test.go

@@ -27,10 +27,12 @@ import (
 	certfake "github.com/gardener/cert-management/pkg/client/cert/clientset/versioned/fake"
 	dnsv1alpha1 "github.com/gardener/external-dns-management/pkg/apis/dns/v1alpha1"
 	dnsfake "github.com/gardener/external-dns-management/pkg/client/dns/clientset/versioned/fake"
+	promopFake "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned/fake"
 	"github.com/sap/cap-operator/pkg/apis/sme.sap.com/v1alpha1"
 	"github.com/sap/cap-operator/pkg/client/clientset/versioned/fake"
 	istionwv1beta1 "istio.io/client-go/pkg/apis/networking/v1beta1"
 	istiofake "istio.io/client-go/pkg/clientset/versioned/fake"
+	apiextFake "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/fake"
 )
 
 const (
@@ -331,6 +333,10 @@ func getTestController(resources testResources) *Controller {
 
 	crdClient := fake.NewSimpleClientset(crdObjects...)
 
+	apiExtClient := apiextFake.NewSimpleClientset()
+
+	promopClient := promopFake.NewSimpleClientset()
+
 	istioClient := istiofake.NewSimpleClientset(istioObjects...)
 
 	certClient := certfake.NewSimpleClientset(gardenerCertObjects...)
@@ -339,7 +345,7 @@ func getTestController(resources testResources) *Controller {
 
 	dnsClient := dnsfake.NewSimpleClientset(dnsObjects...)
 
-	c := NewController(coreClient, crdClient, istioClient, certClient, certManagerCertClient, dnsClient)
+	c := NewController(coreClient, crdClient, istioClient, certClient, certManagerCertClient, dnsClient, apiExtClient, promopClient)
 
 	for _, ca := range resources.cas {
 		if ca != nil {

internal/controller/version-cleanup.go

@@ -1,3 +1,8 @@
+/*
+SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and cap-operator contributors
+SPDX-License-Identifier: Apache-2.0
+*/
+
 package controller
 
 import (
@@ -10,13 +15,19 @@ import (
 	promapi "github.com/prometheus/client_golang/api"
 	promv1 "github.com/prometheus/client_golang/api/prometheus/v1"
 	prommodel "github.com/prometheus/common/model"
+	"github.com/sap/cap-operator/internal/util"
 	"github.com/sap/cap-operator/pkg/apis/sme.sap.com/v1alpha1"
 	"golang.org/x/mod/semver"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apiextensions-apiserver/pkg/apihelpers"
+	k8sErrors "k8s.io/apimachinery/pkg/api/errors"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/client-go/util/workqueue"
 	"k8s.io/klog/v2"
+
+	monv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
+	apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 )
 
 const (
@@ -374,3 +385,98 @@ func evaluateMetric(ctx context.Context, rule *v1alpha1.MetricRule, job, ns stri
 		return true, nil
 	}
 }
+
+/**************************************************************************************
+ * ServiceMonitor creation
+ **************************************************************************************/
+
+func (c *Controller) checkServiceMonitorCapability(ctx context.Context) error {
+	crdName := "servicemonitors.monitoring.coreos.com"
+	crd, err := c.apiExtClient.ApiextensionsV1().CustomResourceDefinitions().Get(ctx, crdName, v1.GetOptions{})
+	if err != nil {
+		return fmt.Errorf("could not get custom resource definition %s: %v", crdName, err)
+	}
+	requiredVersion := "v1"
+	if !apihelpers.HasVersionServed(crd, requiredVersion) {
+		return fmt.Errorf("version %s of custom resource %s is not served", requiredVersion, crdName)
+	}
+	if !apihelpers.IsCRDConditionTrue(crd, apiextv1.Established) {
+		return fmt.Errorf("custom resource %s condition %s not true", crdName, apiextv1.Established)
+	}
+	return nil
+}
+
+func (c *Controller) updateServiceMonitors(ctx context.Context, ca *v1alpha1.CAPApplication, cav *v1alpha1.CAPApplicationVersion, workloadServicePortInfos []servicePortInfo) error {
+	if err := c.checkServiceMonitorCapability(ctx); err != nil {
+		klog.ErrorS(err, "could not confirm availability of service monitor resource; service monitors will not be reconciled")
+		return nil
+	}
+
+	for i := range cav.Spec.Workloads {
+		wl := cav.Spec.Workloads[i]
+		if wl.DeploymentDefinition == nil || wl.DeploymentDefinition.Monitoring == nil || wl.DeploymentDefinition.Monitoring.ScrapeConfig == nil {
+			continue // do not reconcile service monitors
+		}
+
+		var wlPortInfos *servicePortInfo
+		for j := range workloadServicePortInfos {
+			item := workloadServicePortInfos[j]
+			if item.WorkloadName == getWorkloadName(cav.Name, wl.Name) {
+				wlPortInfos = &item
+				break
+			}
+		}
+		if wlPortInfos == nil {
+			return fmt.Errorf("could not identify workload port information for workload %s in version %s", wl.Name, cav.Name)
+		}
+
+		portVerified := false
+		for j := range wlPortInfos.Ports {
+			if wlPortInfos.Ports[j].Name == wl.DeploymentDefinition.Monitoring.ScrapeConfig.WorkloadPort {
+				portVerified = true
+				break
+			}
+		}
+		if !portVerified {
+			return fmt.Errorf("invalid port reference in workload %s monitoring config of version %s", wl.Name, cav.Name)
+		}
+
+		sm, err := c.promClient.MonitoringV1().ServiceMonitors(cav.Namespace).Get(ctx, wlPortInfos.WorkloadName+SecretSuffix, v1.GetOptions{})
+		if err != nil {
+			if k8sErrors.IsNotFound(err) {
+				sm, err = c.promClient.MonitoringV1().ServiceMonitors(cav.Namespace).Create(ctx, newServiceMonitor(ca, cav, &wl, wlPortInfos), v1.CreateOptions{})
+				if err == nil {
+					util.LogInfo("ServiceMonitor created successfully", string(Processing), cav, sm, "version", cav.Spec.Version)
+				}
+			}
+		}
+		err = doChecks(err, sm, cav, wlPortInfos.WorkloadName+ServiceSuffix)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func newServiceMonitor(ca *v1alpha1.CAPApplication, cav *v1alpha1.CAPApplicationVersion, wl *v1alpha1.WorkloadDetails, wlPortInfos *servicePortInfo) *monv1.ServiceMonitor {
+	config := wl.DeploymentDefinition.Monitoring.ScrapeConfig
+	return &monv1.ServiceMonitor{
+		ObjectMeta: v1.ObjectMeta{
+			Name:      wlPortInfos.WorkloadName + ServiceSuffix,
+			Namespace: cav.Namespace,
+			Labels:    copyMaps(wl.Labels, getLabels(ca, cav, CategoryServiceMonitor, string(wl.DeploymentDefinition.Type), wlPortInfos.WorkloadName+ServiceSuffix, true)),
+		},
+		Spec: monv1.ServiceMonitorSpec{
+			Endpoints: []monv1.Endpoint{{
+				Port:          config.WorkloadPort,
+				Interval:      monv1.Duration(config.ScrapeInterval),
+				ScrapeTimeout: monv1.Duration(config.Timeout),
+				Path:          config.Path,
+			}},
+			Selector: v1.LabelSelector{
+				MatchLabels: copyMaps(wl.Labels, getLabels(ca, cav, CategoryService, wlPortInfos.DeploymentType, wlPortInfos.WorkloadName+ServiceSuffix, false)),
+			},
+		},
+	}
+}