Report metrics from custom-plugin-monitor

kubernetes · Jul 24, 2019 · b1d33da · b1d33da
1 parent fbebcf3
commit b1d33da
Show file tree

Hide file tree

Showing 7 changed files with 98 additions and 0 deletions.
diff --git a/config/custom-plugin-monitor.json b/config/custom-plugin-monitor.json
@@ -8,6 +8,7 @@
     "enable_message_change_based_condition_update": false
   },
   "source": "ntp-custom-plugin-monitor",
+  "metricsReporting": true,
   "conditions": [
     {
       "type": "NTPProblem",

diff --git a/config/kernel-monitor-counter.json b/config/kernel-monitor-counter.json
@@ -7,6 +7,7 @@
     "concurrency": 1
   },
   "source": "kernel-monitor",
+  "metricsReporting": true,
   "conditions": [
     {
       "type": "FrequentUnregisterNetDevice",

diff --git a/config/network-problem-monitor.json b/config/network-problem-monitor.json
@@ -7,6 +7,7 @@
     "concurrency": 3
   },
   "source": "network-custom-plugin-monitor",
+  "metricsReporting": true,
   "conditions": [],
   "rules": [
     {

diff --git a/config/systemd-monitor-counter.json b/config/systemd-monitor-counter.json
@@ -7,6 +7,7 @@
     "concurrency": 1
   },
   "source": "systemd-monitor",
+  "metricsReporting": true,
   "conditions": [
     {
       "type": "FrequentKubeletRestart",

diff --git a/pkg/custompluginmonitor/custom_plugin_monitor.go b/pkg/custompluginmonitor/custom_plugin_monitor.go
@@ -26,6 +26,7 @@ import (
 	"k8s.io/node-problem-detector/pkg/custompluginmonitor/plugin"
 	cpmtypes "k8s.io/node-problem-detector/pkg/custompluginmonitor/types"
 	"k8s.io/node-problem-detector/pkg/problemdaemon"
+	"k8s.io/node-problem-detector/pkg/problemmetrics"
 	"k8s.io/node-problem-detector/pkg/types"
 	"k8s.io/node-problem-detector/pkg/util"
 	"k8s.io/node-problem-detector/pkg/util/tomb"
@@ -80,9 +81,31 @@ func NewCustomPluginMonitorOrDie(configPath string) types.Monitor {
 	c.plugin = plugin.NewPlugin(c.config)
 	// A 1000 size channel should be big enough.
 	c.statusChan = make(chan *types.Status, 1000)
+
+	if *c.config.EnableMetricsReporting {
+		initializeProblemMetricsOrDie(c.config.Rules)
+	}
 	return c
 }
 
+// initializeProblemMetricsOrDie creates problem metrics for all problems and set the value to 0,
+// panic if error occurs.
+func initializeProblemMetricsOrDie(rules []*cpmtypes.CustomRule) {
+	for _, rule := range rules {
+		if rule.Type == types.Perm {
+			err := problemmetrics.GlobalProblemMetricsManager.SetProblemGauge(rule.Condition, rule.Reason, false)
+			if err != nil {
+				glog.Fatalf("Failed to initialize problem gauge metrics for problem %q, reason %q: %v",
+					rule.Condition, rule.Reason, err)
+			}
+		}
+		err := problemmetrics.GlobalProblemMetricsManager.IncrementProblemCounter(rule.Reason, 0)
+		if err != nil {
+			glog.Fatalf("Failed to initialize problem counter metrics for %q: %v", rule.Reason, err)
+		}
+	}
+}
+
 func (c *customPluginMonitor) Start() (<-chan *types.Status, error) {
 	glog.Info("Start custom plugin monitor")
 	go c.plugin.Run()
@@ -130,6 +153,13 @@ func (c *customPluginMonitor) generateStatus(result cpmtypes.Result) *types.Stat
 				Reason:    result.Rule.Reason,
 				Message:   result.Message,
 			})
+
+			if *c.config.EnableMetricsReporting {
+				err := problemmetrics.GlobalProblemMetricsManager.IncrementProblemCounter(result.Rule.Reason, 1)
+				if err != nil {
+					glog.Errorf("Failed to update problem counter metrics for %q: %v", result.Rule.Reason, err)
+				}
+			}
 		}
 	} else {
 		// For permanent error changes the condition
@@ -161,6 +191,14 @@ func (c *customPluginMonitor) generateStatus(result cpmtypes.Result) *types.Stat
 					condition.Status = status
 					condition.Message = defaultConditionMessage
 					condition.Reason = defaultConditionReason
+
+					if *c.config.EnableMetricsReporting {
+						err := problemmetrics.GlobalProblemMetricsManager.SetProblemGauge(result.Rule.Condition, result.Rule.Reason, false)
+						if err != nil {
+							glog.Errorf("Failed to update problem gauge metrics for problem %q, reason %q: %v",
+								result.Rule.Condition, result.Rule.Reason, err)
+						}
+					}
 				} else if condition.Status != types.True && status == types.True {
 					// change 2: Condition status change from False/Unknown to True
 					condition.Transition = timestamp
@@ -174,6 +212,18 @@ func (c *customPluginMonitor) generateStatus(result cpmtypes.Result) *types.Stat
 
 					condition.Status = status
 					condition.Reason = result.Rule.Reason
+
+					if *c.config.EnableMetricsReporting {
+						err := problemmetrics.GlobalProblemMetricsManager.IncrementProblemCounter(result.Rule.Reason, 1)
+						if err != nil {
+							glog.Errorf("Failed to update problem counter metrics for %q: %v", result.Rule.Reason, err)
+						}
+						err = problemmetrics.GlobalProblemMetricsManager.SetProblemGauge(result.Rule.Condition, result.Rule.Reason, true)
+						if err != nil {
+							glog.Errorf("Failed to update problem gauge metrics for problem %q, reason %q: %v",
+								result.Rule.Condition, result.Rule.Reason, err)
+						}
+					}
 				} else if condition.Status != status {
 					// change 3: Condition status change from False to Unknown or vice versa
 					condition.Transition = timestamp
@@ -202,6 +252,19 @@ func (c *customPluginMonitor) generateStatus(result cpmtypes.Result) *types.Stat
 						condition.Reason,
 						timestamp,
 					))
+
+					// update metrics only when condition reason changes.
+					if condition.Reason != result.Rule.Reason && *c.config.EnableMetricsReporting {
+						err := problemmetrics.GlobalProblemMetricsManager.IncrementProblemCounter(result.Rule.Reason, 1)
+						if err != nil {
+							glog.Errorf("Failed to update problem counter metrics for %q: %v", result.Rule.Reason, err)
+						}
+						err = problemmetrics.GlobalProblemMetricsManager.SetProblemGauge(result.Rule.Condition, result.Rule.Reason, true)
+						if err != nil {
+							glog.Errorf("Failed to update problem gauge metrics for problem %q, reason %q: %v",
+								result.Rule.Condition, result.Rule.Reason, err)
+						}
+					}
 				}
 
 				break

diff --git a/pkg/custompluginmonitor/types/config.go b/pkg/custompluginmonitor/types/config.go
@@ -32,6 +32,7 @@ var (
 	defaultMaxOutputLength                   = 80
 	defaultConcurrency                       = 3
 	defaultMessageChangeBasedConditionUpdate = false
+	defaultEnableMetricsReporting            = true
 
 	customPluginName = "custom"
 )
@@ -66,6 +67,8 @@ type CustomPluginConfig struct {
 	DefaultConditions []types.Condition `json:"conditions"`
 	// Rules are the rules custom plugin monitor will follow to parse and invoke plugins.
 	Rules []*CustomRule `json:"rules"`
+	// EnableMetricsReporting describes whether to report problems as metrics or not.
+	EnableMetricsReporting *bool `json:"metricsReporting,omitempty"`
 }
 
 // ApplyConfiguration applies default configurations.
@@ -112,6 +115,10 @@ func (cpc *CustomPluginConfig) ApplyConfiguration() error {
 		}
 	}
 
+	if cpc.EnableMetricsReporting == nil {
+		cpc.EnableMetricsReporting = &defaultEnableMetricsReporting
+	}
+
 	return nil
 }
 

diff --git a/pkg/custompluginmonitor/types/config_test.go b/pkg/custompluginmonitor/types/config_test.go
@@ -30,6 +30,7 @@ func TestCustomPluginConfigApplyConfiguration(t *testing.T) {
 	maxOutputLength := 79
 	concurrency := 2
 	messageChangeBasedConditionUpdate := true
+	disableMetricsReporting := false
 
 	ruleTimeout := 1 * time.Second
 	ruleTimeoutString := ruleTimeout.String()
@@ -60,6 +61,7 @@ func TestCustomPluginConfigApplyConfiguration(t *testing.T) {
 					Concurrency:                             &defaultConcurrency,
 					EnableMessageChangeBasedConditionUpdate: &defaultMessageChangeBasedConditionUpdate,
 				},
+				EnableMetricsReporting: &defaultEnableMetricsReporting,
 				Rules: []*CustomRule{
 					{
 						Path: "../plugin/test-data/ok.sh",
@@ -88,6 +90,7 @@ func TestCustomPluginConfigApplyConfiguration(t *testing.T) {
 					Concurrency:                             &defaultConcurrency,
 					EnableMessageChangeBasedConditionUpdate: &defaultMessageChangeBasedConditionUpdate,
 				},
+				EnableMetricsReporting: &defaultEnableMetricsReporting,
 			},
 		},
 		"custom default timeout": {
@@ -106,6 +109,7 @@ func TestCustomPluginConfigApplyConfiguration(t *testing.T) {
 					Concurrency:                             &defaultConcurrency,
 					EnableMessageChangeBasedConditionUpdate: &defaultMessageChangeBasedConditionUpdate,
 				},
+				EnableMetricsReporting: &defaultEnableMetricsReporting,
 			},
 		},
 		"custom max output length": {
@@ -124,6 +128,7 @@ func TestCustomPluginConfigApplyConfiguration(t *testing.T) {
 					Concurrency:                             &defaultConcurrency,
 					EnableMessageChangeBasedConditionUpdate: &defaultMessageChangeBasedConditionUpdate,
 				},
+				EnableMetricsReporting: &defaultEnableMetricsReporting,
 			},
 		},
 		"custom concurrency": {
@@ -142,6 +147,7 @@ func TestCustomPluginConfigApplyConfiguration(t *testing.T) {
 					Concurrency:                             &concurrency,
 					EnableMessageChangeBasedConditionUpdate: &defaultMessageChangeBasedConditionUpdate,
 				},
+				EnableMetricsReporting: &defaultEnableMetricsReporting,
 			},
 		},
 		"custom message change based condition update": {
@@ -160,6 +166,24 @@ func TestCustomPluginConfigApplyConfiguration(t *testing.T) {
 					Concurrency:                             &defaultConcurrency,
 					EnableMessageChangeBasedConditionUpdate: &messageChangeBasedConditionUpdate,
 				},
+				EnableMetricsReporting: &defaultEnableMetricsReporting,
+			},
+		},
+		"disable metrics reporting": {
+			Orig: CustomPluginConfig{
+				EnableMetricsReporting: &disableMetricsReporting,
+			},
+			Wanted: CustomPluginConfig{
+				PluginGlobalConfig: pluginGlobalConfig{
+					InvokeIntervalString:                    &defaultInvokeIntervalString,
+					InvokeInterval:                          &defaultInvokeInterval,
+					TimeoutString:                           &defaultGlobalTimeoutString,
+					Timeout:                                 &defaultGlobalTimeout,
+					MaxOutputLength:                         &defaultMaxOutputLength,
+					Concurrency:                             &defaultConcurrency,
+					EnableMessageChangeBasedConditionUpdate: &defaultMessageChangeBasedConditionUpdate,
+				},
+				EnableMetricsReporting: &disableMetricsReporting,
 			},
 		},
 	}