Enable metrics via prometheus operator

Expose metrics via prometheus.monitoring.coreos.com/v1 The exposed metrics are | Metric | Type | Meaning | | --------------- | ---------------- | ---------------- | | `nfd_master_build_info` | Gauge | Version from which nfd-master was built. | | `nfd_worker_build_info` | Gauge | Version from which nfd-worker was built. | | `nfd_updated_nodes` | Counter | Time taken to label a node | | `nfd_crd_processing_time` | Gauge | Time taken to process a NodeFeatureRule CRD | | `nfd_feature_discovery_duration_seconds` | HistogramVec | Time taken to discover features on a node | Signed-off-by: Carlos Eduardo Arango Gutierrez <[email protected]>
kubernetes-sigs · Jun 9, 2023 · bf7757f · bf7757f
1 parent b9fa66b
commit bf7757f
Show file tree

Hide file tree

Showing 22 changed files with 414 additions and 25 deletions.
diff --git a/cmd/nfd-master/main.go b/cmd/nfd-master/main.go
@@ -124,6 +124,8 @@ func initFlags(flagset *flag.FlagSet) (*master.Args, *master.ConfigOverrideArgs)
 		"Enable NFD CRD API controller for processing NodeFeature and NodeFeatureRule objects.")
 	flagset.IntVar(&args.Port, "port", 8080,
 		"Port on which to listen for connections.")
+	flagset.IntVar(&args.MetricsPort, "metrics", 8081,
+		"Port on which to expose metrics.")
 	flagset.BoolVar(&args.Prune, "prune", false,
 		"Prune all NFD related attributes from all nodes of the cluaster and exit.")
 	flagset.BoolVar(&args.VerifyNodeName, "verify-node-name", false,

diff --git a/cmd/nfd-worker/main.go b/cmd/nfd-worker/main.go
@@ -109,6 +109,8 @@ func initFlags(flagset *flag.FlagSet) (*worker.Args, *worker.ConfigOverrideArgs)
 		"Kubeconfig to use")
 	flagset.BoolVar(&args.Oneshot, "oneshot", false,
 		"Do not publish feature labels")
+	flagset.IntVar(&args.MetricsPort, "metrics", 8081,
+		"Port on which to expose metrics.")
 	flagset.StringVar(&args.Options, "options", "",
 		"Specify config options from command line. Config options are specified "+
 			"in the same format as in the config file (i.e. json or yaml). These options")

diff --git a/deployment/base/master/master-deployment.yaml b/deployment/base/master/master-deployment.yaml
@@ -35,5 +35,7 @@ spec:
           command:
             - "nfd-master"
           ports:
+            - name: metrics
+              containerPort: 8081
             - name: grpc
-              containerPort: 8080
+              containerPort: 8080
diff --git a/deployment/base/worker-daemonset/worker-daemonset.yaml b/deployment/base/worker-daemonset/worker-daemonset.yaml
@@ -23,3 +23,6 @@ spec:
             - "nfd-worker"
           args:
             - "-server=nfd-master:8080"
+          ports:
+            - name: metrics
+              containerPort: 8081
diff --git a/deployment/components/prometheus/kustomization.yaml b/deployment/components/prometheus/kustomization.yaml
@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1alpha1
+kind: Component
+
+namespace: node-feature-discovery
+
+resources:
+- monitor.yaml
+- rbac.yaml
diff --git a/deployment/components/prometheus/monitor.yaml b/deployment/components/prometheus/monitor.yaml
@@ -0,0 +1,20 @@
+# Prometheus Monitor Service (Metrics)
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: nfd-metrics
+  labels:
+    app: nfd
+spec:
+  podMetricsEndpoints:
+    - honorLabels: true
+      interval: 10s
+      path: /metrics
+      port: metrics
+      scheme: http
+  namespaceSelector:
+    matchNames:
+    - node-feature-discovery      
+  selector:
+    matchExpressions:
+    - {key: app, operator: In, values: ["nfd-master", "nfd-worker"]}
diff --git a/deployment/components/prometheus/rbac.yaml b/deployment/components/prometheus/rbac.yaml
@@ -0,0 +1,31 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: prometheus-k8s
+  namespace: node-feature-discovery
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: prometheus-k8s
+  namespace: node-feature-discovery
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: prometheus-k8s
+subjects:
+- kind: ServiceAccount
+  name: prometheus-k8s
+  namespace: monitoring
+- kind: ServiceAccount
+  name: prometheus-operator
+  namespace: monitoring
diff --git a/deployment/helm/node-feature-discovery/templates/master.yaml b/deployment/helm/node-feature-discovery/templates/master.yaml
@@ -118,6 +118,18 @@ spec:
             - "-key-file=/etc/kubernetes/node-feature-discovery/certs/tls.key"
             - "-cert-file=/etc/kubernetes/node-feature-discovery/certs/tls.crt"
             {{- end }}
+            {{- if .Values.metrics.prometheus.enable }}
+            - "-metrics={{ .Values.metrics.port }}"       
+            {{- end }}
+          {{- if .Values.metrics.enable }}
+          ports:
+            - name: metrics
+              containerPort: {{ .Values.metrics.port }}
+          {{- else }}   
+          ports:
+            - name: metrics
+              containerPort: 0
+          {{- end }}
           volumeMounts:
             {{- if .Values.tls.enable }}
             - name: nfd-master-cert
@@ -139,7 +151,6 @@ spec:
             items:
               - key: nfd-master.conf
                 path: nfd-master.conf
-
     {{- with .Values.master.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}

diff --git a/deployment/helm/node-feature-discovery/templates/prometheus.yaml b/deployment/helm/node-feature-discovery/templates/prometheus.yaml
@@ -0,0 +1,55 @@
+{{- if .Values.metrics.prometheus.enable }}
+# Prometheus Monitor Service (Metrics)
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: nfd-metrics
+  labels:
+    app: nfd
+spec:
+  podMetricsEndpoints:
+    - honorLabels: true
+      interval: 10s
+      path: /metrics
+      port: metrics
+      scheme: http
+  namespaceSelector:
+    matchNames:
+    - node-feature-discovery      
+  selector:
+    matchExpressions:
+    - {key: app, operator: Exists}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: prometheus-k8s
+  namespace: node-feature-discovery
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: prometheus-k8s
+  namespace: node-feature-discovery
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: prometheus-k8s
+subjects:
+- kind: ServiceAccount
+  name: prometheus-k8s
+  namespace: monitoring
+- kind: ServiceAccount
+  name: prometheus-operator
+  namespace: monitoring
+
+{{- end }}
diff --git a/deployment/helm/node-feature-discovery/templates/worker.yaml b/deployment/helm/node-feature-discovery/templates/worker.yaml
@@ -54,6 +54,14 @@ spec:
         - "-key-file=/etc/kubernetes/node-feature-discovery/certs/tls.key"
         - "-cert-file=/etc/kubernetes/node-feature-discovery/certs/tls.crt"
 {{- end }}
+        {{- if .Values.metrics.enable }}
+        - "-metrics={{ .Values.metrics.port }}"       
+        {{- end }}
+      {{- if .Values.metrics.enable }}
+        ports:
+          - name: metrics
+            containerPort: {{ .Values.metrics.port }}   
+      {{- end }}
         volumeMounts:
         - name: host-boot
           mountPath: "/host-boot"

diff --git a/deployment/helm/node-feature-discovery/values.yaml b/deployment/helm/node-feature-discovery/values.yaml
@@ -493,3 +493,8 @@ topologyGC:
 tls:
   enable: false
   certManager: false
+
+metrics:
+  prometheus:
+    enable: false
+    port: 8081
diff --git a/deployment/overlays/default/kustomization.yaml b/deployment/overlays/default/kustomization.yaml
@@ -16,3 +16,4 @@ components:
 - ../../components/worker-config
 - ../../components/common
 - ../../components/master-config
+- ../../components/prometheus
diff --git a/deployment/overlays/prometheus/kustomization.yaml b/deployment/overlays/prometheus/kustomization.yaml
@@ -0,0 +1,10 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: node-feature-discovery
+
+resources:
+- namespace.yaml
+
+components:
+- ../../components/prometheus
diff --git a/deployment/overlays/prometheus/namespace.yaml b/deployment/overlays/prometheus/namespace.yaml
@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: node-feature-discovery
diff --git a/docs/deployment/helm.md b/docs/deployment/helm.md
@@ -102,6 +102,12 @@ We have introduced the following Chart parameters.
 | `tls.enable` | bool | false | Specifies whether to use TLS for communications between components |
 | `tls.certManager` | bool | false | If enabled, requires [cert-manager](https://cert-manager.io/docs/) to be installed and will automatically create the required TLS certificates |
 | `enableNodeFeatureApi` | bool  | false | Enable the [NodeFeature](../usage/custom-resources.md#nodefeature) CRD API for communicating node features. This will automatically disable the gRPC communication.
+| `metrics.prometheus.enable` | bool | false | Specifies whether to expose mtrics using prometheus.Operator |
+| `metrics.prometheus.port` | integer | 8081 | Port on which to expose metrics from components to prometheus.Operator |
+
+Metrics are configured to be exposed using prometheus.Operator API's by 
+default. If you want to expose metrics using the prometheus.Operator 
+API's you need to install the prometheus.Operator in your cluster.
 
 ### Master pod parameters
 

diff --git a/docs/deployment/kustomize.md b/docs/deployment/kustomize.md
@@ -57,6 +57,8 @@ scenarios under
   see [Master Worker Topologyupdater](#master-worker-topologyupdater) below
 - [`topologyupdater`](https://github.com/kubernetes-sigs/node-feature-discovery/blob/{{site.release}}/deployment/overlays/topologyupdater):
   see [Topology Updater](#topologyupdater) below
+- [`Metrics`](https://github.com/kubernetes-sigs/node-feature-discovery/blob/{{site.release}}/deployment/overlays/prometheus):
+  see [Metrics](#metrics) below
 - [`prune`](https://github.com/kubernetes-sigs/node-feature-discovery/blob/{{site.release}}/deployment/overlays/prune):
   clean up the cluster after uninstallation, see
   [Removing feature labels](uninstallation.md#removing-feature-labels)
@@ -137,6 +139,28 @@ kubectl apply -k https://github.com/kubernetes-sigs/node-feature-discovery/deplo
 
 ```
 
+# Metrics
+
+To Add metrics scraping for prometheus-operator,
+To allow [prometheus-operator][prometheus-operator]
+to scrape metrics from node-feature-discovery,
+run the following command:
+
+```bash
+kubectl apply -k https://github.com/kubernetes-sigs/node-feature-discovery/deployment/overlays/default?ref={{ site.release }}
+kubectl apply -k https://github.com/kubernetes-sigs/node-feature-discovery/deployment/overlays/prometheus?ref={{ site.release }}
+```
+
+The exposed metrics are
+
+| Metric        | Type | Meaning |
+| --------------- | ---------------- | ---------------- |
+|  `nfd_master_build_info`           | Gauge | Version from which nfd-master was built. |
+|  `nfd_worker_build_info`           | Gauge | Version from which nfd-worker was built. |
+|  `nfd_updated_nodes`           |  Counter | Time taken to label a node |
+|  `nfd_crd_processing_time`          |  Gauge | Time taken to process a NodeFeatureRule CRD |
+| `nfd_feature_discovery_duration_seconds` |  HistogramVec | Time taken to discover features on a node |
+
 ## Uninstallation
 
 Simplest way is to invoke `kubectl delete` on the overlay that was used for
@@ -162,3 +186,4 @@ kubectl delete clusterrolebinding nfd-master
 
 <!-- Links -->
 [kustomize]: https://github.com/kubernetes-sigs/kustomize
+[prometheus-operator]: https://github.com/prometheus-operator/prometheus-operator
diff --git a/go.mod b/go.mod
@@ -15,6 +15,7 @@ require (
 	github.com/onsi/ginkgo/v2 v2.9.1
 	github.com/onsi/gomega v1.27.4
 	github.com/opencontainers/runc v1.1.6
+	github.com/prometheus/client_golang v1.15.1
 	github.com/smartystreets/assertions v1.2.0
 	github.com/smartystreets/goconvey v1.6.4
 	github.com/stretchr/testify v1.8.1
@@ -62,7 +63,7 @@ require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver/v4 v4.0.0 // indirect
 	github.com/cenkalti/backoff/v4 v4.1.3 // indirect
-	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/checkpoint-restore/go-criu/v5 v5.3.0 // indirect
 	github.com/cilium/ebpf v0.10.0 // indirect
 	github.com/container-storage-interface/spec v1.7.0 // indirect
@@ -111,7 +112,7 @@ require (
 	github.com/libopenstorage/openstorage v1.0.0 // indirect
 	github.com/lithammer/dedent v1.1.0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mitchellh/mapstructure v1.4.1 // indirect
@@ -129,10 +130,9 @@ require (
 	github.com/opencontainers/selinux v1.11.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_golang v1.14.0 // indirect
 	github.com/prometheus/client_model v0.3.0 // indirect
-	github.com/prometheus/common v0.37.0 // indirect
-	github.com/prometheus/procfs v0.8.0 // indirect
+	github.com/prometheus/common v0.42.0 // indirect
+	github.com/prometheus/procfs v0.9.0 // indirect
 	github.com/rubiojr/go-vhd v0.0.0-20200706105327-02e210299021 // indirect
 	github.com/seccomp/libseccomp-golang v0.10.0 // indirect
 	github.com/sirupsen/logrus v1.9.0 // indirect
@@ -161,14 +161,14 @@ require (
 	go.opentelemetry.io/proto/otlp v0.19.0 // indirect
 	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
-	go.uber.org/zap v1.19.0 // indirect
+	go.uber.org/zap v1.24.0 // indirect
 	golang.org/x/crypto v0.1.0 // indirect
-	golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b // indirect
+	golang.org/x/oauth2 v0.5.0 // indirect
 	golang.org/x/sync v0.1.0 // indirect
 	golang.org/x/sys v0.7.0 // indirect
 	golang.org/x/term v0.7.0 // indirect
 	golang.org/x/text v0.9.0 // indirect
-	golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect
+	golang.org/x/time v0.3.0 // indirect
 	golang.org/x/tools v0.7.0 // indirect
 	google.golang.org/api v0.60.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect