Update to go1.16 and rework for config files

.gitignore

@@ -2,3 +2,4 @@ outputs
 id_rsa
 cisco_exporter
 dist/
+config.yml

README.md

@@ -6,16 +6,16 @@ The basic structure is based on https://github.com/czerwonk/junos_exporter
 # flags
 Name     | Description | Default
 ---------|-------------|---------
-version | Print version information. |
-web.listen-address | Address on which to expose metrics and web interface. | :9362
-web.telemetry-path | Path under which to expose metrics. | /metrics
+version | Print version information. |
+web.listen-address | Address on which to expose metrics and web interface. | :9362
+web.telemetry-path | Path under which to expose metrics. | /metrics
 ssh.targets | Comma seperated list of hosts to scrape |
 ssh.user | Username to use for SSH connection | cisco_exporter
-ssh.keyfile | Key file to use for SSH connection | cisco_exporter
-ssh.timeout | Timeout in seconds to use for SSH connection | 5
-ssh.batch-size | The SSH response batch size | 10000
-debug | Show verbose debug output | false
+ssh.keyfile | Key file to use for SSH connection | cisco_exporter
+ssh.timeout | Timeout in seconds to use for SSH connection | 5
+debug | Show verbose debug output | false
 legacy.ciphers | Allow insecure legacy ciphers: aes128-cbc 3des-cbc aes192-cbc aes256-cbc | false
+config.file | Path to config file |
 
 # metrics
 
@@ -24,21 +24,61 @@ All metrics are enabled by default. To disable something pass a flag `--<name>.e
 Name     | Description | OS
 ---------|-------------|----
 bgp | BGP (message count, prefix counts per peer, session state) | IOS XE/NX-OS
-environment | Environment (temperatures, state of power supply) | NX-OS/IOS XE/IOS
+environment | Environment (temperatures, state of power supply) | NX-OS/IOS XE/IOS
 facts | System informations (OS Version, memory: total/used/free, cpu: 5s/1m/5m/interrupts) | IOS XE/IOS
-interface | Interfaces (transmitted/received: bytes/errors/drops, admin/oper state) | NX-OS (*_drops is always 0)/IOS XE/IOS
-optics | Optical signals (tx/rx) | NX-OS/IOS XE/IOS
+interfaces | Interfaces (transmitted/received: bytes/errors/drops, admin/oper state) | NX-OS (*_drops is always 0)/IOS XE/IOS
+optics | Optical signals (tx/rx) | NX-OS/IOS XE/IOS
 
 ## Install
 ```bash
 go get -u github.com/lwlcom/cisco_exporter
 ```
 
 ## Usage
+
+### Binary
 ```bash
 ./cisco_exporter -ssh.targets="host1.example.com,host2.example.com:2233,172.16.0.1" -ssh.keyfile=cisco_exporter
 ```
 
+```bash
+./cisco_exporter -config.file=config.yml
+```
+
+## Config file
+The exporter can be configured with a YAML based config file:
+
+```yaml
+---
+debug: false
+legacy_ciphers: false
+# default values
+timeout: 5
+batch_size: 10000
+username: default-username
+password: default-password
+key_file: /path/to/key
+
+devices:
+  - host: host1.example.com
+    key_file: /path/to/key
+    timeout: 5
+    batch_size: 10000
+    features: # enable/disable per host
+      bgp: false
+  - host: host2.example.com:2233
+    username: exporter
+    password: secret
+
+features:
+  bgp: true
+  environment: true
+  facts: true
+  interfaces: true
+  optics: true
+
+```
+
 ## Third Party Components
 This software uses components of the following projects
 * Prometheus Go client library (https://github.com/prometheus/client_golang)

bgp/bgp_collector.go

@@ -34,6 +34,11 @@ func NewCollector() collector.RPCCollector {
 	return &bgpCollector{}
 }
 
+// Name returns the name of the collector
+func (*bgpCollector) Name() string {
+	return "BGP"
+}
+
 // Describe describes the metrics
 func (*bgpCollector) Describe(ch chan<- *prometheus.Desc) {
 	ch <- upDesc

cisco_collector.go

@@ -1,18 +1,11 @@
 package main
 
 import (
-	"strings"
 	"time"
 
 	"sync"
 
-	"github.com/lwlcom/cisco_exporter/bgp"
-	"github.com/lwlcom/cisco_exporter/collector"
 	"github.com/lwlcom/cisco_exporter/connector"
-	"github.com/lwlcom/cisco_exporter/environment"
-	"github.com/lwlcom/cisco_exporter/facts"
-	"github.com/lwlcom/cisco_exporter/interfaces"
-	"github.com/lwlcom/cisco_exporter/optics"
 	"github.com/lwlcom/cisco_exporter/rpc"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/log"
@@ -21,85 +14,63 @@ import (
 const prefix = "cisco_"
 
 var (
-	scrapeDurationDesc *prometheus.Desc
-	upDesc             *prometheus.Desc
+	scrapeCollectorDurationDesc *prometheus.Desc
+	scrapeDurationDesc          *prometheus.Desc
+	upDesc                      *prometheus.Desc
 )
 
 func init() {
 	upDesc = prometheus.NewDesc(prefix+"up", "Scrape of target was successful", []string{"target"}, nil)
 	scrapeDurationDesc = prometheus.NewDesc(prefix+"collector_duration_seconds", "Duration of a collector scrape for one target", []string{"target"}, nil)
+	scrapeCollectorDurationDesc = prometheus.NewDesc(prefix+"collect_duration_seconds", "Duration of a scrape by collector and target", []string{"target", "collector"}, nil)
 }
 
 type ciscoCollector struct {
-	targets    []string
-	collectors map[string]collector.RPCCollector
+	devices    []*connector.Device
+	collectors *collectors
 }
 
-func newCiscoCollector(targets []string) *ciscoCollector {
-	collectors := collectors()
-	return &ciscoCollector{targets, collectors}
-}
-
-func collectors() map[string]collector.RPCCollector {
-	m := map[string]collector.RPCCollector{}
-
-	if *bgpEnabled == true {
-		m["bgp"] = bgp.NewCollector()
-	}
-
-	if *environmetEnabled == true {
-		m["environment"] = environment.NewCollector()
-	}
-
-	if *factsEnabled == true {
-		m["facts"] = facts.NewCollector()
+func newCiscoCollector(devices []*connector.Device) *ciscoCollector {
+	return &ciscoCollector{
+		devices:    devices,
+		collectors: collectorsForDevices(devices, cfg),
 	}
-
-	if *interfacesEnabled == true {
-		m["interfaces"] = interfaces.NewCollector()
-	}
-
-	if *opticsEnabled == true {
-		m["optics"] = optics.NewCollector()
-	}
-
-	return m
 }
 
 // Describe implements prometheus.Collector interface
 func (c *ciscoCollector) Describe(ch chan<- *prometheus.Desc) {
 	ch <- upDesc
 	ch <- scrapeDurationDesc
+	ch <- scrapeCollectorDurationDesc
 
-	for _, col := range c.collectors {
+	for _, col := range c.collectors.allEnabledCollectors() {
 		col.Describe(ch)
 	}
 }
 
 // Collect implements prometheus.Collector interface
 func (c *ciscoCollector) Collect(ch chan<- prometheus.Metric) {
-	hosts := c.targets
 	wg := &sync.WaitGroup{}
 
-	wg.Add(len(hosts))
-	for _, h := range hosts {
-		go c.collectForHost(strings.Trim(h, " "), ch, wg)
+	wg.Add(len(c.devices))
+	for _, d := range c.devices {
+		go c.collectForHost(d, ch, wg)
 	}
 
 	wg.Wait()
 }
 
-func (c *ciscoCollector) collectForHost(host string, ch chan<- prometheus.Metric, wg *sync.WaitGroup) {
+func (c *ciscoCollector) collectForHost(device *connector.Device, ch chan<- prometheus.Metric, wg *sync.WaitGroup) {
 	defer wg.Done()
 
-	l := []string{host}
+	l := []string{device.Host}
 
 	t := time.Now()
 	defer func() {
 		ch <- prometheus.MustNewConstMetric(scrapeDurationDesc, prometheus.GaugeValue, time.Since(t).Seconds(), l...)
 	}()
 
-	conn, err := connector.NewSSSHConnection(host, *sshUsername, *sshKeyFile, *legacyCiphers, *sshTimeout, *sshBatchSize)
+	conn, err := connector.NewSSSHConnection(device, cfg)
 	if err != nil {
 		log.Errorln(err)
 		ch <- prometheus.MustNewConstMetric(upDesc, prometheus.GaugeValue, 0, l...)
@@ -109,17 +80,21 @@ func (c *ciscoCollector) collectForHost(host string, ch chan<- prometheus.Metric
 
 	ch <- prometheus.MustNewConstMetric(upDesc, prometheus.GaugeValue, 1, l...)
 
-	rpc := rpc.NewClient(conn, *debug)
-	err = rpc.Identify()
+	client := rpc.NewClient(conn, cfg.Debug)
+	err = client.Identify()
 	if err != nil {
-		log.Errorln(host + ": " + err.Error())
+		log.Errorln(device.Host + ": " + err.Error())
 		return
 	}
 
-	for k, col := range c.collectors {
-		err = col.Collect(rpc, ch, l)
+	for _, col := range c.collectors.collectorsForDevice(device) {
+		ct := time.Now()
+		err := col.Collect(client, ch, l)
+
 		if err != nil && err.Error() != "EOF" {
-			log.Errorln(k + ": " + err.Error())
+			log.Errorln(col.Name() + ": " + err.Error())
 		}
+
+		ch <- prometheus.MustNewConstMetric(scrapeCollectorDurationDesc, prometheus.GaugeValue, time.Since(ct).Seconds(), append(l, col.Name())...)
 	}
 }

collector/rpc_collector.go

@@ -8,6 +8,8 @@ import (
 
 // RPCCollector collects metrics from Cisco using rpc.Client
 type RPCCollector interface {
+	// Name returns an human readable name for logging and debugging purposes
+	Name() string
 
 	// Describe describes the metrics
 	Describe(ch chan<- *prometheus.Desc)

collectors.go

@@ -0,0 +1,79 @@
+package main
+
+import (
+	"github.com/lwlcom/cisco_exporter/bgp"
+	"github.com/lwlcom/cisco_exporter/collector"
+	"github.com/lwlcom/cisco_exporter/config"
+	"github.com/lwlcom/cisco_exporter/connector"
+	"github.com/lwlcom/cisco_exporter/environment"
+	"github.com/lwlcom/cisco_exporter/facts"
+	"github.com/lwlcom/cisco_exporter/interfaces"
+	"github.com/lwlcom/cisco_exporter/optics"
+)
+
+type collectors struct {
+	collectors map[string]collector.RPCCollector
+	devices    map[string][]collector.RPCCollector
+	cfg        *config.Config
+}
+
+func collectorsForDevices(devices []*connector.Device, cfg *config.Config) *collectors {
+	c := &collectors{
+		collectors: make(map[string]collector.RPCCollector),
+		devices:    make(map[string][]collector.RPCCollector),
+		cfg:        cfg,
+	}
+
+	for _, d := range devices {
+		c.initCollectorsForDevice(d)
+	}
+
+	return c
+}
+
+func (c *collectors) initCollectorsForDevice(device *connector.Device) {
+	f := c.cfg.FeaturesForDevice(device.Host)
+
+	c.devices[device.Host] = make([]collector.RPCCollector, 0)
+	c.addCollectorIfEnabledForDevice(device, "bgp", f.BGP, bgp.NewCollector)
+	c.addCollectorIfEnabledForDevice(device, "environment", f.Environment, environment.NewCollector)
+	c.addCollectorIfEnabledForDevice(device, "facts", f.Facts, facts.NewCollector)
+	c.addCollectorIfEnabledForDevice(device, "interfaces", f.Interfaces, interfaces.NewCollector)
+	c.addCollectorIfEnabledForDevice(device, "optics", f.Optics, optics.NewCollector)
+
+}
+
+func (c *collectors) addCollectorIfEnabledForDevice(device *connector.Device, key string, enabled *bool, newCollector func() collector.RPCCollector) {
+	if !*enabled {
+		return
+	}
+
+	col, found := c.collectors[key]
+	if !found {
+		col = newCollector()
+		c.collectors[key] = col
+	}
+
+	c.devices[device.Host] = append(c.devices[device.Host], col)
+}
+
+func (c *collectors) allEnabledCollectors() []collector.RPCCollector {
+	collectors := make([]collector.RPCCollector, len(c.collectors))
+
+	i := 0
+	for _, collector := range c.collectors {
+		collectors[i] = collector
+		i++
+	}
+
+	return collectors
+}
+
+func (c *collectors) collectorsForDevice(device *connector.Device) []collector.RPCCollector {
+	cols, found := c.devices[device.Host]
+	if !found {
+		return []collector.RPCCollector{}
+	}
+
+	return cols
+}

config.yml.example

@@ -0,0 +1,27 @@
+---
+debug: false
+legacy_ciphers: false
+# default values
+timeout: 5
+batch_size: 10000
+username: default-username
+password: default-password
+key_file: /path/to/key
+
+devices:
+  - host: host1.example.com
+    key_file: /path/to/key
+    timeout: 5
+    batch_size: 10000
+    features:
+      bgp: false
+  - host: host2.example.com:2233
+    username: exporter
+    password: secret
+
+features:
+  bgp: true
+  environment: true
+  facts: true
+  interfaces: true
+  optics: true