ROX-21530: Updating Cert Expiry Metrics

fleetshard/main.go

@@ -130,9 +130,10 @@ func main() {
 	k8sInterface := k8s.CreateInterfaceOrDie()
 	informerFactory := informers.NewSharedInformerFactory(k8sInterface, time.Minute)
 	secretInformer := informerFactory.Core().V1().Secrets().Informer()
+	namespaceInformer := informerFactory.Core().V1().Namespaces().Informer()
 	namespaceLister := informerFactory.Core().V1().Namespaces().Lister()
 
-	monitor := certmonitor.NewCertMonitor(certmonitorConfig, informerFactory, secretInformer, namespaceLister)
+	monitor := certmonitor.NewCertMonitor(certmonitorConfig, informerFactory, secretInformer, namespaceInformer, namespaceLister)
 	if err := monitor.Start(); err != nil {
 		glog.Fatalf("Error starting certmonitor: %v", err)
 	}

fleetshard/pkg/fleetshardmetrics/metrics.go

@@ -130,11 +130,15 @@ func (m *Metrics) SetCertKeyExpiryMetric(namespace, name, key string, expiry flo
 }
 
 func (m *Metrics) DeleteCertMetric(namespace, name string) {
-	m.CertificatesExpiry.Delete(prometheus.Labels{"namespace": namespace, "secret": name}) // pragma: allowlist secret
+	m.CertificatesExpiry.DeletePartialMatch(prometheus.Labels{"namespace": namespace, "secret": name}) // pragma: allowlist secret
+}
+
+func (m *Metrics) DeleteCertNamespaceMetric(namespace string) {
+	m.CertificatesExpiry.DeletePartialMatch(prometheus.Labels{"namespace": namespace}) // pragma: allowlist secret
 }
 
 func (m *Metrics) DeleteKeyCertMetric(namespace, name, key string) {
-	m.CertificatesExpiry.Delete(prometheus.Labels{"namespace": namespace, "secret": name, "data_key": key}) // pragma: allowlist secret
+	m.CertificatesExpiry.DeletePartialMatch(prometheus.Labels{"namespace": namespace, "secret": name, "data_key": key}) // pragma: allowlist secret
 }
 
 // MetricsInstance return the global Singleton instance for Metrics

internal/certmonitor/certmonitor.go

@@ -41,25 +41,36 @@ type NamespaceGetter interface {
 
 // certMonitor is the Certificate Monitor. It watches Kubernetes secrets containing certificates, and populates prometheus metrics with the expiration time of those certificates.
 type certMonitor struct {
-	informerfactory informers.SharedInformerFactory
-	secretInformer  cache.SharedIndexInformer
-	config          *Config
-	namespaceGetter NamespaceGetter
-	metrics         *fleetshardmetrics.Metrics
-	stopCh          chan struct{}
+	informerfactory   informers.SharedInformerFactory
+	secretInformer    cache.SharedIndexInformer
+	config            *Config
+	namespaceInformer cache.SharedIndexInformer
+	namespaceGetter   NamespaceGetter
+	metrics           *fleetshardmetrics.Metrics
+	stopCh            chan struct{}
 }
 
 // Start the certificate monitor
 func (c *certMonitor) Start() error {
+	var err error
 	if c.stopCh != nil {
 		return errors.New("already started")
 	}
 	c.stopCh = make(chan struct{})
-	c.secretInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
+	_, err = c.secretInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
 		AddFunc:    c.handleSecretCreation,
 		UpdateFunc: c.handleSecretUpdate,
 		DeleteFunc: c.handleSecretDeletion,
 	})
+	if err != nil {
+		return err
+	}
+	_, err = c.namespaceInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
+		DeleteFunc: c.handleNamespaceDeletion,
+	})
+	if err != nil {
+		return err
+	}
 	c.informerfactory.Start(c.stopCh)
 	if !cache.WaitForCacheSync(c.stopCh) {
 		return fmt.Errorf("timed out waiting for caches to sync")
@@ -76,13 +87,14 @@ func (c *certMonitor) Stop() error {
 }
 
 // NewCertMonitor creates new instance of certMonitor
-func NewCertMonitor(config *Config, informerFactory informers.SharedInformerFactory, secretInformer cache.SharedIndexInformer, namespaceGetter NamespaceGetter) *certMonitor {
+func NewCertMonitor(config *Config, informerFactory informers.SharedInformerFactory, secretInformer cache.SharedIndexInformer, namespaceInformer cache.SharedIndexInformer, namespaceGetter NamespaceGetter) *certMonitor {
 	return &certMonitor{
-		informerfactory: informerFactory,
-		secretInformer:  secretInformer, // pragma: allowlist secret
-		config:          config,
-		namespaceGetter: namespaceGetter,
-		metrics:         fleetshardmetrics.MetricsInstance(),
+		informerfactory:   informerFactory,
+		secretInformer:    secretInformer, // pragma: allowlist secret
+		config:            config,
+		namespaceInformer: namespaceInformer, // pragma: allowlist secret
+		namespaceGetter:   namespaceGetter,
+		metrics:           fleetshardmetrics.MetricsInstance(),
 	}
 }
 
@@ -158,6 +170,15 @@ func (c *certMonitor) handleSecretDeletion(obj interface{}) {
 	c.metrics.DeleteCertMetric(secret.Namespace, secret.Name)
 }
 
+func (c *certMonitor) handleNamespaceDeletion(obj interface{}) {
+	namespace, ok := obj.(*corev1.Namespace)
+	if !ok {
+		return
+	}
+
+	c.metrics.DeleteCertNamespaceMetric(namespace.Name)
+}
+
 func (c *certMonitor) shouldProcessSecret(s *corev1.Secret) bool {
 	for _, monitor := range c.config.Monitors {
 		if c.secretMatches(s, monitor) {

internal/certmonitor/certmonitor_test.go

@@ -290,8 +290,8 @@ func TestCertMonitor_secretMatches(t *testing.T) {
 	}
 }
 
-// TestCertMonitor func tests certificates event handlers + prometheus metrics handling
-func TestCertMonitor(t *testing.T) {
+// TestCertMonitor_Secret tests that secret event handlers correctly populate prometheus metrics
+func TestCertMonitor_Secret(t *testing.T) {
 	fleetshardmetrics.MetricsInstance().CertificatesExpiry.Reset()
 
 	namespaces := []v1.Namespace{
@@ -343,6 +343,53 @@ func TestCertMonitor(t *testing.T) {
 
 	certMonitor.handleSecretDeletion(secretUpdated)
 	verifyPrometheusMetricDelete(t, "namespace-1", "secret-1", "tls.crt")
+}
+
+// TestCertMonitor_Namespace tests that namespace secret handlers remove prometheus metrics when the namespace is deleted
+func TestCertMonitor_Namespace(t *testing.T) {
+	fleetshardmetrics.MetricsInstance().CertificatesExpiry.Reset()
+
+	namespaces := []v1.Namespace{
+		{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "namespace-1",
+				Labels: map[string]string{
+					"foo": "bar"},
+			},
+		},
+	}
+	certMonitor := &certMonitor{
+		namespaceGetter: newFakeNamespaceGetter(namespaces),
+		metrics:         fleetshardmetrics.MetricsInstance(),
+		config: &Config{
+			Monitors: []MonitorConfig{
+				{
+					Namespace: SelectorConfig{
+						Name: "namespace-1",
+					},
+					Secret: SelectorConfig{ // pragma: allowlist secret
+						Name: "secret-1",
+					},
+				},
+			},
+		},
+	}
+	now1 := time.Now().UTC()
+	expirytime := now1.Add(1 * time.Hour)
+
+	mockNamespace := &v1.Namespace{ // pragma: allowlist secret
+		ObjectMeta: metav1.ObjectMeta{Name: "namespace-1"},
+	}
+	secret := &v1.Secret{ // pragma: allowlist secret
+		ObjectMeta: metav1.ObjectMeta{Namespace: mockNamespace.Name, Name: "secret-1"},
+		Data:       map[string][]byte{"tls.crt": generateCertWithExpiration(t, expirytime)},
+	}
+	expirationUnix := float64(expirytime.Unix())
+	certMonitor.handleSecretCreation(secret)
+	verifyPrometheusMetric(t, "namespace-1", "secret-1", "tls.crt", expirationUnix)
+
+	certMonitor.handleNamespaceDeletion(mockNamespace)
+	verifyPrometheusMetricDelete(t, "namespace-1", "secret-1", "tls.crt")
 
 }