Add metrics to machine and cluster controller

Signed-off-by: Warren Fernandes <[email protected]>
kubernetes-sigs · Nov 8, 2019 · c821586 · c821586
1 parent 86c8608
commit c821586
Show file tree

Hide file tree

Showing 9 changed files with 422 additions and 123 deletions.
diff --git a/controllers/cluster_controller.go b/controllers/cluster_controller.go
@@ -25,6 +25,7 @@ import (
 
 	"github.com/go-logr/logr"
 	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -40,6 +41,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/handler"
+	"sigs.k8s.io/controller-runtime/pkg/metrics"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	"sigs.k8s.io/controller-runtime/pkg/source"
 )
@@ -50,6 +52,38 @@ const (
 	deleteRequeueAfter = 5 * time.Second
 )
 
+var (
+	clusterControlPlaneReady = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "capi_cluster_control_plane_ready",
+			Help: "Cluster control plane is ready if set to 1 and not if 0.",
+		},
+		[]string{"cluster", "namespace"},
+	)
+	clusterInfrastructureReady = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "capi_cluster_infrastructure_ready",
+			Help: "Cluster infrastructure is ready if set to 1 and not if 0.",
+		},
+		[]string{"cluster", "namespace"},
+	)
+	clusterKubeconfigReady = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "capi_cluster_kubeconfig_ready",
+			Help: "Cluster kubeconfig is ready if set to 1 and not if 0.",
+		},
+		[]string{"cluster", "namespace"},
+	)
+)
+
+func init() {
+	metrics.Registry.MustRegister(
+		clusterControlPlaneReady,
+		clusterInfrastructureReady,
+		clusterKubeconfigReady,
+	)
+}
+
 // +kubebuilder:rbac:groups=core,resources=events,verbs=get;list;watch;create;patch
 // +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch;create;patch
 // +kubebuilder:rbac:groups=core,resources=nodes,verbs=get;list;watch;create;update;patch;delete
@@ -338,7 +372,14 @@ func splitMachineList(list *clusterv1.MachineList) (*clusterv1.MachineList, *clu
 	return controlplanes, nodes
 }
 
-func (r *ClusterReconciler) reconcileControlPlaneInitialized(ctx context.Context, cluster *clusterv1.Cluster) error {
+func (r *ClusterReconciler) reconcileControlPlaneInitialized(ctx context.Context, cluster *clusterv1.Cluster) (err error) {
+	defer func() {
+		if err != nil || !cluster.Status.ControlPlaneInitialized {
+			clusterControlPlaneReady.WithLabelValues(cluster.Name, cluster.Namespace).Set(0)
+		} else {
+			clusterControlPlaneReady.WithLabelValues(cluster.Name, cluster.Namespace).Set(1)
+		}
+	}()
 	logger := r.Log.WithValues("cluster", cluster.Name, "namespace", cluster.Namespace)
 
 	if cluster.Status.ControlPlaneInitialized {

diff --git a/controllers/cluster_controller_phases.go b/controllers/cluster_controller_phases.go
@@ -143,7 +143,14 @@ func (r *ClusterReconciler) reconcileExternal(ctx context.Context, cluster *clus
 }
 
 // reconcileInfrastructure reconciles the Spec.InfrastructureRef object on a Cluster.
-func (r *ClusterReconciler) reconcileInfrastructure(ctx context.Context, cluster *clusterv1.Cluster) error {
+func (r *ClusterReconciler) reconcileInfrastructure(ctx context.Context, cluster *clusterv1.Cluster) (err error) {
+	defer func() {
+		if err != nil || !cluster.Status.InfrastructureReady {
+			clusterInfrastructureReady.WithLabelValues(cluster.Name, cluster.Namespace).Set(0)
+		} else {
+			clusterInfrastructureReady.WithLabelValues(cluster.Name, cluster.Namespace).Set(1)
+		}
+	}()
 	logger := r.Log.WithValues("cluster", cluster.Name, "namespace", cluster.Namespace)
 
 	if cluster.Spec.InfrastructureRef == nil {
@@ -187,7 +194,15 @@ func (r *ClusterReconciler) reconcileInfrastructure(ctx context.Context, cluster
 	return nil
 }
 
-func (r *ClusterReconciler) reconcileKubeconfig(ctx context.Context, cluster *clusterv1.Cluster) error {
+func (r *ClusterReconciler) reconcileKubeconfig(ctx context.Context, cluster *clusterv1.Cluster) (rerr error) {
+	defer func() {
+		if rerr != nil || len(cluster.Status.APIEndpoints) == 0 {
+			clusterKubeconfigReady.WithLabelValues(cluster.Name, cluster.Namespace).Set(0)
+		} else {
+			clusterKubeconfigReady.WithLabelValues(cluster.Name, cluster.Namespace).Set(1)
+		}
+	}()
+
 	if len(cluster.Status.APIEndpoints) == 0 {
 		return nil
 	}