Create ServiceMonitors to gather metrics from the OTEL Operands

.chloggen/1768-gather-metrics-from-collectors.yaml

@@ -0,0 +1,16 @@
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. operator, target allocator, github action)
+component: operator
+
+# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Add the ability to the operator to create Service Monitors for the OpenTelemetry Collectors in order to gather the metrics they are generating
+
+# One or more tracking issues related to the change
+issues: [1768]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:

.github/workflows/e2e.yaml

@@ -29,10 +29,10 @@ jobs:
        - "1.27"
        group:
          - e2e e2e-upgrade
+         - e2e-prometheuscr
          - e2e-autoscale
 
     steps:
-
     - name: Set up Go
       uses: actions/setup-go@v4
       with:

Makefile

@@ -176,6 +176,11 @@ generate: controller-gen api-docs
 e2e:
 	$(KUTTL) test
 
+# end-to-end-test for PrometheusCR E2E tests
+.PHONY: e2e-prometheuscr
+e2e-prometheuscr:
+	$(KUTTL) test --config kuttl-test-prometheuscr.yaml
+
 # end-to-end-test for testing upgrading
 .PHONY: e2e-upgrade
 e2e-upgrade: undeploy
@@ -200,6 +205,11 @@ e2e-log-operator:
 prepare-e2e: kuttl set-image-controller container container-target-allocator container-operator-opamp-bridge start-kind cert-manager install-metrics-server load-image-all deploy
 	TARGETALLOCATOR_IMG=$(TARGETALLOCATOR_IMG) SED_BIN="$(SED)" ./hack/modify-test-images.sh
 
+.PHONY: enable-prometheus-feature-flag
+enable-prometheus-feature-flag:
+	$(SED) -i "s#--feature-gates=+operator.autoinstrumentation.go#--feature-gates=+operator.autoinstrumentation.go,+operator.observability.prometheus#g" config/default/manager_auth_proxy_patch.yaml
+
+
 .PHONY: scorecard-tests
 scorecard-tests: operator-sdk
 	$(OPERATOR_SDK) scorecard -w=5m bundle || (echo "scorecard test failed" && exit 1)
@@ -238,6 +248,10 @@ endif
 install-metrics-server:
 	./hack/install-metrics-server.sh
 
+.PHONY: install-prometheus-operator
+install-prometheus-operator:
+	./hack/install-prometheus-operator.sh
+
 .PHONY: load-image-all
 load-image-all: load-image-operator load-image-target-allocator load-image-operator-opamp-bridge
 

apis/v1alpha1/opentelemetrycollector_types.go

@@ -190,6 +190,14 @@ type OpenTelemetryCollectorSpec struct {
 	// https://kubernetes.io/docs/concepts/workloads/pods/init-containers/
 	// +optional
 	InitContainers []v1.Container `json:"initContainers,omitempty"`
+
+	// ObservabilitySpec defines how telemetry data gets handled.
+	//
+	// +optional
+	// +kubebuilder:validation:Optional
+	// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Observability"
+	Observability ObservabilitySpec `json:"observability,omitempty"`
+
 	// TopologySpreadConstraints embedded kubernetes pod configuration option,
 	// controls how pods are spread across your cluster among failure-domains
 	// such as regions, zones, nodes, and other user-defined topology domains
@@ -371,6 +379,27 @@ type AutoscalerSpec struct {
 	TargetMemoryUtilization *int32 `json:"targetMemoryUtilization,omitempty"`
 }
 
+// MetricsConfigSpec defines a metrics config.
+type MetricsConfigSpec struct {
+	// EnableMetrics specifies if ServiceMonitors should be created for the OpenTelemetry Collector.
+	// The operator.observability.prometheus feature gate must be enabled to use this feature.
+	//
+	// +optional
+	// +kubebuilder:validation:Optional
+	// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Create ServiceMonitors for OpenTelemetry Collector"
+	EnableMetrics bool `json:"enableMetrics,omitempty"`
+}
+
+// ObservabilitySpec defines how telemetry data gets handled.
+type ObservabilitySpec struct {
+	// Metrics defines the metrics configuration for operands.
+	//
+	// +optional
+	// +kubebuilder:validation:Optional
+	// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Metrics Config"
+	Metrics MetricsConfigSpec `json:"metrics,omitempty"`
+}
+
 // Probe defines the OpenTelemetry's pod probe config. Only Liveness probe is supported currently.
 type Probe struct {
 	// Number of seconds after the container has started before liveness probes are initiated.

bundle/manifests/opentelemetry-operator.clusterserviceversion.yaml

@@ -76,6 +76,18 @@ spec:
       - kind: StatefulSets
         name: ""
         version: apps/v1
+      specDescriptors:
+      - description: ObservabilitySpec defines how telemetry data gets handled.
+        displayName: Observability
+        path: observability
+      - description: Metrics defines the metrics configuration for operands.
+        displayName: Metrics Config
+        path: observability.metrics
+      - description: EnableMetrics specifies if ServiceMonitors should be created
+          for the OpenTelemetry Collector. The operator.observability.prometheus feature
+          gate must be enabled to use this feature.
+        displayName: Create ServiceMonitors for OpenTelemetry Collector
+        path: observability.metrics.enableMetrics
       version: v1alpha1
   description: |-
     OpenTelemetry is a collection of tools, APIs, and SDKs. You use it to instrument, generate, collect, and export telemetry data (metrics, logs, and traces) for analysis in order to understand your software's performance and behavior.
@@ -209,6 +221,18 @@ spec:
           - get
           - list
           - update
+        - apiGroups:
+          - monitoring.coreos.com
+          resources:
+          - servicemonitors
+          verbs:
+          - create
+          - delete
+          - get
+          - list
+          - patch
+          - update
+          - watch
         - apiGroups:
           - networking.k8s.io
           resources:

bundle/manifests/opentelemetry.io_opentelemetrycollectors.yaml

@@ -2949,6 +2949,19 @@ spec:
                   This is only relevant to daemonset, statefulset, and deployment
                   mode
                 type: object
+              observability:
+                description: ObservabilitySpec defines how telemetry data gets handled.
+                properties:
+                  metrics:
+                    description: Metrics defines the metrics configuration for operands.
+                    properties:
+                      enableMetrics:
+                        description: EnableMetrics specifies if ServiceMonitors should
+                          be created for the OpenTelemetry Collector. The operator.observability.prometheus
+                          feature gate must be enabled to use this feature.
+                        type: boolean
+                    type: object
+                type: object
               podAnnotations:
                 additionalProperties:
                   type: string

config/crd/bases/opentelemetry.io_opentelemetrycollectors.yaml

@@ -2946,6 +2946,19 @@ spec:
                   This is only relevant to daemonset, statefulset, and deployment
                   mode
                 type: object
+              observability:
+                description: ObservabilitySpec defines how telemetry data gets handled.
+                properties:
+                  metrics:
+                    description: Metrics defines the metrics configuration for operands.
+                    properties:
+                      enableMetrics:
+                        description: EnableMetrics specifies if ServiceMonitors should
+                          be created for the OpenTelemetry Collector. The operator.observability.prometheus
+                          feature gate must be enabled to use this feature.
+                        type: boolean
+                    type: object
+                type: object
               podAnnotations:
                 additionalProperties:
                   type: string

config/manifests/bases/opentelemetry-operator.clusterserviceversion.yaml

@@ -56,6 +56,18 @@ spec:
       - kind: StatefulSets
         name: ""
         version: apps/v1
+      specDescriptors:
+      - description: ObservabilitySpec defines how telemetry data gets handled.
+        displayName: Observability
+        path: observability
+      - description: Metrics defines the metrics configuration for operands.
+        displayName: Metrics Config
+        path: observability.metrics
+      - description: EnableMetrics specifies if ServiceMonitors should be created
+          for the OpenTelemetry Collector. The operator.observability.prometheus feature
+          gate must be enabled to use this feature.
+        displayName: Create ServiceMonitors for OpenTelemetry Collector
+        path: observability.metrics.enableMetrics
       version: v1alpha1
   description: |-
     OpenTelemetry is a collection of tools, APIs, and SDKs. You use it to instrument, generate, collect, and export telemetry data (metrics, logs, and traces) for analysis in order to understand your software's performance and behavior.

config/rbac/role.yaml

@@ -119,6 +119,18 @@ rules:
   - get
   - list
   - update
+- apiGroups:
+  - monitoring.coreos.com
+  resources:
+  - servicemonitors
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - networking.k8s.io
   resources:

controllers/opentelemetrycollector_controller.go

@@ -21,6 +21,7 @@ import (
 	"sync"
 
 	"github.com/go-logr/logr"
+	monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
 	appsv1 "k8s.io/api/apps/v1"
 	autoscalingv2 "k8s.io/api/autoscaling/v2"
 	autoscalingv2beta2 "k8s.io/api/autoscaling/v2beta2"
@@ -35,6 +36,7 @@ import (
 	"github.com/open-telemetry/opentelemetry-operator/internal/config"
 	"github.com/open-telemetry/opentelemetry-operator/pkg/autodetect"
 	"github.com/open-telemetry/opentelemetry-operator/pkg/collector/reconcile"
+	"github.com/open-telemetry/opentelemetry-operator/pkg/featuregate"
 )
 
 // OpenTelemetryCollectorReconciler reconciles a OpenTelemetryCollector object.
@@ -164,6 +166,11 @@ func NewReconciler(p Params) *OpenTelemetryCollectorReconciler {
 				"ingresses",
 				true,
 			},
+			{
+				reconcile.ServiceMonitors,
+				"service monitors",
+				true,
+			},
 			{
 				reconcile.Self,
 				"opentelemetry",
@@ -251,6 +258,10 @@ func (r *OpenTelemetryCollectorReconciler) SetupWithManager(mgr ctrl.Manager) er
 		Owns(&appsv1.DaemonSet{}).
 		Owns(&appsv1.StatefulSet{})
 
+	if featuregate.PrometheusOperatorIsAvailable.IsEnabled() {
+		builder.Owns(&monitoringv1.ServiceMonitor{})
+	}
+
 	autoscalingVersion := r.config.AutoscalingVersion()
 	if autoscalingVersion == autodetect.AutoscalingVersionV2 {
 		builder = builder.Owns(&autoscalingv2.HorizontalPodAutoscaler{})

docs/api.md

@@ -3736,6 +3736,13 @@ OpenTelemetryCollectorSpec defines the desired state of OpenTelemetryCollector.
           NodeSelector to schedule OpenTelemetry Collector pods. This is only relevant to daemonset, statefulset, and deployment mode<br/>
         </td>
         <td>false</td>
+      </tr><tr>
+        <td><b><a href="#opentelemetrycollectorspecobservability">observability</a></b></td>
+        <td>object</td>
+        <td>
+          ObservabilitySpec defines how telemetry data gets handled.<br/>
+        </td>
+        <td>false</td>
       </tr><tr>
         <td><b>podAnnotations</b></td>
         <td>map[string]string</td>
@@ -8985,6 +8992,60 @@ Liveness config for the OpenTelemetry Collector except the probe handler which i
 </table>
 
 
+### OpenTelemetryCollector.spec.observability
+<sup><sup>[↩ Parent](#opentelemetrycollectorspec)</sup></sup>
+
+
+
+ObservabilitySpec defines how telemetry data gets handled.
+
+<table>
+    <thead>
+        <tr>
+            <th>Name</th>
+            <th>Type</th>
+            <th>Description</th>
+            <th>Required</th>
+        </tr>
+    </thead>
+    <tbody><tr>
+        <td><b><a href="#opentelemetrycollectorspecobservabilitymetrics">metrics</a></b></td>
+        <td>object</td>
+        <td>
+          Metrics defines the metrics configuration for operands.<br/>
+        </td>
+        <td>false</td>
+      </tr></tbody>
+</table>
+
+
+### OpenTelemetryCollector.spec.observability.metrics
+<sup><sup>[↩ Parent](#opentelemetrycollectorspecobservability)</sup></sup>
+
+
+
+Metrics defines the metrics configuration for operands.
+
+<table>
+    <thead>
+        <tr>
+            <th>Name</th>
+            <th>Type</th>
+            <th>Description</th>
+            <th>Required</th>
+        </tr>
+    </thead>
+    <tbody><tr>
+        <td><b>enableMetrics</b></td>
+        <td>boolean</td>
+        <td>
+          EnableMetrics specifies if ServiceMonitors should be created for the OpenTelemetry Collector. The operator.observability.prometheus feature gate must be enabled to use this feature.<br/>
+        </td>
+        <td>false</td>
+      </tr></tbody>
+</table>
+
+
 ### OpenTelemetryCollector.spec.podSecurityContext
 <sup><sup>[↩ Parent](#opentelemetrycollectorspec)</sup></sup>