From 2f8273eec3092387f7c3bbffd25edf35536a97d3 Mon Sep 17 00:00:00 2001 From: Kakuya Ando Date: Fri, 9 Feb 2024 17:02:36 +0900 Subject: [PATCH 1/8] Add SNMPScraper Interface for easier maintenance Signed-off-by: Kakuya Ando --- scraper/gosnmp.go | 108 +++++++++++++++++++++++++++++++++++++++++++++ scraper/scraper.go | 14 ++++++ 2 files changed, 122 insertions(+) create mode 100644 scraper/gosnmp.go create mode 100644 scraper/scraper.go diff --git a/scraper/gosnmp.go b/scraper/gosnmp.go new file mode 100644 index 00000000..9ceb4b40 --- /dev/null +++ b/scraper/gosnmp.go @@ -0,0 +1,108 @@ +package scraper + +import ( + "context" + "fmt" + "net" + "strconv" + "strings" + "time" + + "github.com/go-kit/log" + "github.com/go-kit/log/level" + "github.com/gosnmp/gosnmp" +) + +type GoSNMPWrapper struct { + c *gosnmp.GoSNMP + logger log.Logger +} + +func NewGoSNMP(logger log.Logger, target, srcAddress string) (*GoSNMPWrapper, error) { + transport := "udp" + if s := strings.SplitN(target, "://", 2); len(s) == 2 { + transport = s[0] + target = s[1] + } + port := uint16(161) + if host, _port, err := net.SplitHostPort(target); err == nil { + target = host + p, err := strconv.Atoi(_port) + if err != nil { + return nil, fmt.Errorf("error converting port number to int for target %q: %w", target, err) + } + port = uint16(p) + } + g := &gosnmp.GoSNMP{ + Transport: transport, + Target: target, + Port: port, + LocalAddr: srcAddress, + } + return &GoSNMPWrapper{c: g, logger: logger}, nil +} + +func (g *GoSNMPWrapper) SetOptions(fns ...func(*gosnmp.GoSNMP)) { + for _, fn := range fns { + fn(g.c) + } +} + +func (g *GoSNMPWrapper) GetVersion() gosnmp.SnmpVersion { + return g.c.Version +} + +func (g *GoSNMPWrapper) Connect() error { + st := time.Now() + err := g.c.Connect() + if err != nil { + if err == context.Canceled { + return fmt.Errorf("scrape cancelled after %s (possible timeout) connecting to target %s", + time.Since(st), g.c.Target) + } + return fmt.Errorf("error connecting to target %s: %s", g.c.Target, err) + } + return nil +} + +func (g *GoSNMPWrapper) Close() error { + return g.c.Conn.Close() +} + +func (g *GoSNMPWrapper) Get(oids []string) (results *gosnmp.SnmpPacket, err error) { + level.Debug(g.logger).Log("msg", "Getting OIDs", "oids", oids) + st := time.Now() + results, err = g.c.Get(oids) + if err != nil { + if err == context.Canceled { + err = fmt.Errorf("scrape cancelled after %s (possible timeout) getting target %s", + time.Since(st), g.c.Target) + } else { + err = fmt.Errorf("error getting target %s: %s", g.c.Target, err) + } + return + } + level.Debug(g.logger).Log("msg", "Get of OIDs completed", "oids", oids, "duration_seconds", time.Since(st)) + return +} + +func (g *GoSNMPWrapper) WalkAll(oid string) (results []gosnmp.SnmpPDU, err error) { + level.Debug(g.logger).Log("msg", "Walking subtree", "oid", oid) + st := time.Now() + if g.c.Version == gosnmp.Version1 { + results, err = g.c.WalkAll(oid) + } else { + results, err = g.c.BulkWalkAll(oid) + } + if err != nil { + if err == context.Canceled { + err = fmt.Errorf("scrape canceled after %s (possible timeout) walking target %s", + time.Since(st), g.c.Target) + } else { + err = fmt.Errorf("error walking target %s: %s", g.c.Target, err) + } + return + } + level.Debug(g.logger).Log("msg", "Walk of subtree completed", "oid", oid, "duration_seconds", time.Since(st)) + return +} diff --git a/scraper/scraper.go b/scraper/scraper.go new file mode 100644 index 00000000..17af653c --- /dev/null +++ b/scraper/scraper.go @@ -0,0 +1,14 @@ +package scraper + +import ( + "github.com/gosnmp/gosnmp" +) + +type SNMPScraper interface { + Get([]string) (*gosnmp.SnmpPacket, error) + WalkAll(string) ([]gosnmp.SnmpPDU, error) + Connect() error + Close() error + GetVersion() gosnmp.SnmpVersion + SetOptions(...func(*gosnmp.GoSNMP)) +} From 0bfbc47e41f5969b0d61f5ac5e7175efd60a4b3f Mon Sep 17 00:00:00 2001 From: Kakuya Ando Date: Fri, 9 Feb 2024 17:02:49 +0900 Subject: [PATCH 2/8] Moved SNMP-related operations to the scraper side and simplified the ScrapeTarget function Signed-off-by: Kakuya Ando --- collector/collector.go | 169 ++++++++++++++++++----------------------- 1 file changed, 73 insertions(+), 96 deletions(-) diff --git a/collector/collector.go b/collector/collector.go index 00f5b01f..669bcb41 100644 --- a/collector/collector.go +++ b/collector/collector.go @@ -31,6 +31,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/snmp_exporter/config" + "github.com/prometheus/snmp_exporter/scraper" ) var ( @@ -77,74 +78,17 @@ func listToOid(l []int) string { } type ScrapeResults struct { - pdus []gosnmp.SnmpPDU - packets uint64 - retries uint64 + pdus []gosnmp.SnmpPDU } -func ScrapeTarget(ctx context.Context, target string, auth *config.Auth, module *config.Module, logger log.Logger, metrics Metrics) (ScrapeResults, error) { +func ScrapeTarget(snmp scraper.SNMPScraper, target string, module *config.Module, logger log.Logger, metrics Metrics) (ScrapeResults, error) { results := ScrapeResults{} - // Set the options. - snmp := gosnmp.GoSNMP{} - snmp.Context = ctx - snmp.MaxRepetitions = module.WalkParams.MaxRepetitions - snmp.Retries = *module.WalkParams.Retries - snmp.Timeout = module.WalkParams.Timeout - snmp.UseUnconnectedUDPSocket = module.WalkParams.UseUnconnectedUDPSocket - snmp.LocalAddr = *srcAddress - - // Allow a set of OIDs that aren't in a strictly increasing order - if module.WalkParams.AllowNonIncreasingOIDs { - snmp.AppOpts = make(map[string]interface{}) - snmp.AppOpts["c"] = true - } - - var sent time.Time - snmp.OnSent = func(x *gosnmp.GoSNMP) { - sent = time.Now() - metrics.SNMPPackets.Inc() - results.packets++ - } - snmp.OnRecv = func(x *gosnmp.GoSNMP) { - metrics.SNMPDuration.Observe(time.Since(sent).Seconds()) - } - snmp.OnRetry = func(x *gosnmp.GoSNMP) { - metrics.SNMPRetries.Inc() - results.retries++ - } - - // Configure target. - if err := configureTarget(&snmp, target); err != nil { - return results, err - } - - // Configure auth. - auth.ConfigureSNMP(&snmp) - - // Do the actual walk. - getInitialStart := time.Now() - err := snmp.Connect() - if err != nil { - if err == context.Canceled { - return results, fmt.Errorf("scrape cancelled after %s (possible timeout) connecting to target %s", - time.Since(getInitialStart), snmp.Target) - } - return results, fmt.Errorf("error connecting to target %s: %s", target, err) - } - defer snmp.Conn.Close() - // Evaluate rules. newGet := module.Get newWalk := module.Walk for _, filter := range module.Filters { - var pdus []gosnmp.SnmpPDU allowedList := []string{} - - if snmp.Version == gosnmp.Version1 { - pdus, err = snmp.WalkAll(filter.Oid) - } else { - pdus, err = snmp.BulkWalkAll(filter.Oid) - } + pdus, err := snmp.WalkAll(filter.Oid) // Do not try to filter anything if we had errors. if err != nil { level.Info(logger).Log("msg", "Error getting OID, won't do any filter on this oid", "oid", filter.Oid) @@ -165,10 +109,11 @@ func ScrapeTarget(ctx context.Context, target string, auth *config.Auth, module newGet = newCfg } + version := snmp.GetVersion() getOids := newGet maxOids := int(module.WalkParams.MaxRepetitions) // Max Repetition can be 0, maxOids cannot. SNMPv1 can only report one OID error per call. - if maxOids == 0 || snmp.Version == gosnmp.Version1 { + if maxOids == 0 || version == gosnmp.Version1 { maxOids = 1 } for len(getOids) > 0 { @@ -177,19 +122,12 @@ func ScrapeTarget(ctx context.Context, target string, auth *config.Auth, module oids = maxOids } - level.Debug(logger).Log("msg", "Getting OIDs", "oids", oids) - getStart := time.Now() packet, err := snmp.Get(getOids[:oids]) if err != nil { - if err == context.Canceled { - return results, fmt.Errorf("scrape cancelled after %s (possible timeout) getting target %s", - time.Since(getInitialStart), snmp.Target) - } - return results, fmt.Errorf("error getting target %s: %s", snmp.Target, err) + return results, err } - level.Debug(logger).Log("msg", "Get of OIDs completed", "oids", oids, "duration_seconds", time.Since(getStart)) // SNMPv1 will return packet error for unsupported OIDs. - if packet.Error == gosnmp.NoSuchName && snmp.Version == gosnmp.Version1 { + if packet.Error == gosnmp.NoSuchName && version == gosnmp.Version1 { level.Debug(logger).Log("msg", "OID not supported by target", "oids", getOids[0]) getOids = getOids[oids:] continue @@ -197,7 +135,7 @@ func ScrapeTarget(ctx context.Context, target string, auth *config.Auth, module // Response received with errors. // TODO: "stringify" gosnmp errors instead of showing error code. if packet.Error != gosnmp.NoError { - return results, fmt.Errorf("error reported by target %s: Error Status %d", snmp.Target, packet.Error) + return results, fmt.Errorf("error reported by target %s: Error Status %d", target, packet.Error) } for _, v := range packet.Variables { if v.Type == gosnmp.NoSuchObject || v.Type == gosnmp.NoSuchInstance { @@ -210,23 +148,10 @@ func ScrapeTarget(ctx context.Context, target string, auth *config.Auth, module } for _, subtree := range newWalk { - var pdus []gosnmp.SnmpPDU - level.Debug(logger).Log("msg", "Walking subtree", "oid", subtree) - walkStart := time.Now() - if snmp.Version == gosnmp.Version1 { - pdus, err = snmp.WalkAll(subtree) - } else { - pdus, err = snmp.BulkWalkAll(subtree) - } + pdus, err := snmp.WalkAll(subtree) if err != nil { - if err == context.Canceled { - return results, fmt.Errorf("scrape canceled after %s (possible timeout) walking target %s", - time.Since(getInitialStart), snmp.Target) - } - return results, fmt.Errorf("error walking target %s: %s", snmp.Target, err) + return results, err } - level.Debug(logger).Log("msg", "Walk of subtree completed", "oid", subtree, "duration_seconds", time.Since(walkStart)) - results.pdus = append(results.pdus, pdus...) } return results, nil @@ -384,11 +309,44 @@ func (c Collector) Describe(ch chan<- *prometheus.Desc) { ch <- prometheus.NewDesc("dummy", "dummy", nil, nil) } -func (c Collector) collect(ch chan<- prometheus.Metric, module *NamedModule) { - logger := log.With(c.logger, "module", module.name) +func (c Collector) collect(ch chan<- prometheus.Metric, logger log.Logger, client scraper.SNMPScraper, module *NamedModule) { + var ( + packets uint64 + retries uint64 + ) + client.SetOptions( + // Set the metrics options. + func(g *gosnmp.GoSNMP) { + var sent time.Time + g.OnSent = func(x *gosnmp.GoSNMP) { + sent = time.Now() + c.metrics.SNMPPackets.Inc() + packets++ + } + g.OnRecv = func(x *gosnmp.GoSNMP) { + c.metrics.SNMPDuration.Observe(time.Since(sent).Seconds()) + } + g.OnRetry = func(x *gosnmp.GoSNMP) { + c.metrics.SNMPRetries.Inc() + retries++ + } + }, + // Set the Walk options. + func(g *gosnmp.GoSNMP) { + g.Retries = *module.WalkParams.Retries + g.Timeout = module.WalkParams.Timeout + g.MaxRepetitions = module.WalkParams.MaxRepetitions + g.UseUnconnectedUDPSocket = module.WalkParams.UseUnconnectedUDPSocket + if module.WalkParams.AllowNonIncreasingOIDs { + g.AppOpts = map[string]interface{}{ + "c": true, + } + } + }, + ) start := time.Now() - results, err := ScrapeTarget(c.ctx, c.target, c.auth, module.Module, logger, c.metrics) moduleLabel := prometheus.Labels{"module": module.name} + results, err := ScrapeTarget(client, c.target, module.Module, logger, c.metrics) if err != nil { level.Info(logger).Log("msg", "Error scraping target", "err", err) ch <- prometheus.NewInvalidMetric(prometheus.NewDesc("snmp_error", "Error scraping target", nil, moduleLabel), err) @@ -401,15 +359,16 @@ func (c Collector) collect(ch chan<- prometheus.Metric, module *NamedModule) { ch <- prometheus.MustNewConstMetric( prometheus.NewDesc("snmp_scrape_packets_sent", "Packets sent for get, bulkget, and walk; including retries.", nil, moduleLabel), prometheus.GaugeValue, - float64(results.packets)) + float64(packets)) ch <- prometheus.MustNewConstMetric( prometheus.NewDesc("snmp_scrape_packets_retried", "Packets retried for get, bulkget, and walk.", nil, moduleLabel), prometheus.GaugeValue, - float64(results.retries)) + float64(retries)) ch <- prometheus.MustNewConstMetric( prometheus.NewDesc("snmp_scrape_pdus_returned", "PDUs returned from get, bulkget, and walk.", nil, moduleLabel), prometheus.GaugeValue, float64(len(results.pdus))) + oidToPdu := make(map[string]gosnmp.SnmpPDU, len(results.pdus)) for _, pdu := range results.pdus { oidToPdu[pdu.Name[1:]] = pdu @@ -417,7 +376,6 @@ func (c Collector) collect(ch chan<- prometheus.Metric, module *NamedModule) { metricTree := buildMetricTree(module.Metrics) // Look for metrics that match each pdu. -PduLoop: for oid, pdu := range oidToPdu { head := metricTree oidList := oidToList(oid) @@ -425,7 +383,7 @@ PduLoop: var ok bool head, ok = head.children[o] if !ok { - continue PduLoop + break } if head.metric != nil { // Found a match. @@ -453,18 +411,37 @@ func (c Collector) Collect(ch chan<- prometheus.Metric) { workerChan := make(chan *NamedModule) for i := 0; i < workerCount; i++ { wg.Add(1) - go func() { + go func(i int) { + logger := log.With(c.logger, "worker", i) + client, err := scraper.NewGoSNMP(logger, c.target, *srcAddress) + if err != nil { + level.Info(logger).Log("msg", "Error creating scraper", "err", err) + ch <- prometheus.NewInvalidMetric(prometheus.NewDesc("snmp_error", "Error creating GoSNMPWrapper", nil, nil), err) + return + } + // Set the options. + client.SetOptions(func(g *gosnmp.GoSNMP) { + g.Context = c.ctx + c.auth.ConfigureSNMP(g) + }) + err = client.Connect() + if err != nil { + level.Info(logger).Log("msg", "Error connecting to target", "err", err) + ch <- prometheus.NewInvalidMetric(prometheus.NewDesc("snmp_error", "Error connecting to target", nil, nil), err) + return + } + defer client.Close() defer wg.Done() for m := range workerChan { logger := log.With(c.logger, "module", m.name) level.Debug(logger).Log("msg", "Starting scrape") start := time.Now() - c.collect(ch, m) + c.collect(ch, logger, client, m) duration := time.Since(start).Seconds() level.Debug(logger).Log("msg", "Finished scrape", "duration_seconds", duration) c.metrics.SNMPCollectionDuration.WithLabelValues(m.name).Observe(duration) } - }() + }(i) } for _, module := range c.modules { From 209741790920f5ea3fb99beaa897055ce2ac4fe9 Mon Sep 17 00:00:00 2001 From: Kakuya Ando Date: Mon, 12 Feb 2024 14:24:15 +0900 Subject: [PATCH 3/8] Remove version retrieval from scraper and update to fetch from auth information Signed-off-by: Kakuya Ando --- collector/collector.go | 10 +++++----- scraper/gosnmp.go | 4 ---- scraper/scraper.go | 1 - 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/collector/collector.go b/collector/collector.go index 669bcb41..bef63a1c 100644 --- a/collector/collector.go +++ b/collector/collector.go @@ -81,7 +81,7 @@ type ScrapeResults struct { pdus []gosnmp.SnmpPDU } -func ScrapeTarget(snmp scraper.SNMPScraper, target string, module *config.Module, logger log.Logger, metrics Metrics) (ScrapeResults, error) { +func ScrapeTarget(snmp scraper.SNMPScraper, target string, auth *config.Auth, module *config.Module, logger log.Logger, metrics Metrics) (ScrapeResults, error) { results := ScrapeResults{} // Evaluate rules. newGet := module.Get @@ -109,11 +109,11 @@ func ScrapeTarget(snmp scraper.SNMPScraper, target string, module *config.Module newGet = newCfg } - version := snmp.GetVersion() + version := auth.Version getOids := newGet maxOids := int(module.WalkParams.MaxRepetitions) // Max Repetition can be 0, maxOids cannot. SNMPv1 can only report one OID error per call. - if maxOids == 0 || version == gosnmp.Version1 { + if maxOids == 0 || version == 1 { maxOids = 1 } for len(getOids) > 0 { @@ -127,7 +127,7 @@ func ScrapeTarget(snmp scraper.SNMPScraper, target string, module *config.Module return results, err } // SNMPv1 will return packet error for unsupported OIDs. - if packet.Error == gosnmp.NoSuchName && version == gosnmp.Version1 { + if packet.Error == gosnmp.NoSuchName && version == 1 { level.Debug(logger).Log("msg", "OID not supported by target", "oids", getOids[0]) getOids = getOids[oids:] continue @@ -346,7 +346,7 @@ func (c Collector) collect(ch chan<- prometheus.Metric, logger log.Logger, clien ) start := time.Now() moduleLabel := prometheus.Labels{"module": module.name} - results, err := ScrapeTarget(client, c.target, module.Module, logger, c.metrics) + results, err := ScrapeTarget(client, c.target, c.auth, module.Module, logger, c.metrics) if err != nil { level.Info(logger).Log("msg", "Error scraping target", "err", err) ch <- prometheus.NewInvalidMetric(prometheus.NewDesc("snmp_error", "Error scraping target", nil, moduleLabel), err) diff --git a/scraper/gosnmp.go b/scraper/gosnmp.go index 9ceb4b40..859bad5f 100644 --- a/scraper/gosnmp.go +++ b/scraper/gosnmp.go @@ -48,10 +48,6 @@ func (g *GoSNMPWrapper) SetOptions(fns ...func(*gosnmp.GoSNMP)) { } } -func (g *GoSNMPWrapper) GetVersion() gosnmp.SnmpVersion { - return g.c.Version -} - func (g *GoSNMPWrapper) Connect() error { st := time.Now() err := g.c.Connect() diff --git a/scraper/scraper.go b/scraper/scraper.go index 17af653c..eee05ecd 100644 --- a/scraper/scraper.go +++ b/scraper/scraper.go @@ -9,6 +9,5 @@ type SNMPScraper interface { WalkAll(string) ([]gosnmp.SnmpPDU, error) Connect() error Close() error - GetVersion() gosnmp.SnmpVersion SetOptions(...func(*gosnmp.GoSNMP)) } From e77bfbc769ad4ad086179064d3e1a42a3a4bed17 Mon Sep 17 00:00:00 2001 From: Kakuya Ando Date: Mon, 12 Feb 2024 16:33:12 +0900 Subject: [PATCH 4/8] Add error handling for scraper used in worker Signed-off-by: Kakuya Ando --- collector/collector.go | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/collector/collector.go b/collector/collector.go index bef63a1c..cb1faed0 100644 --- a/collector/collector.go +++ b/collector/collector.go @@ -408,44 +408,57 @@ func (c Collector) Collect(ch chan<- prometheus.Metric) { if workerCount < 1 { workerCount = 1 } + ctx, cancel := context.WithCancel(c.ctx) + defer cancel() workerChan := make(chan *NamedModule) for i := 0; i < workerCount; i++ { wg.Add(1) go func(i int) { + defer wg.Done() logger := log.With(c.logger, "worker", i) client, err := scraper.NewGoSNMP(logger, c.target, *srcAddress) if err != nil { - level.Info(logger).Log("msg", "Error creating scraper", "err", err) - ch <- prometheus.NewInvalidMetric(prometheus.NewDesc("snmp_error", "Error creating GoSNMPWrapper", nil, nil), err) + level.Info(logger).Log("msg", err) + cancel() + ch <- prometheus.NewInvalidMetric(prometheus.NewDesc("snmp_error", "Error during initialisation of the Worker", nil, nil), err) return } // Set the options. client.SetOptions(func(g *gosnmp.GoSNMP) { - g.Context = c.ctx + g.Context = ctx c.auth.ConfigureSNMP(g) }) - err = client.Connect() - if err != nil { + if err = client.Connect(); err != nil { level.Info(logger).Log("msg", "Error connecting to target", "err", err) ch <- prometheus.NewInvalidMetric(prometheus.NewDesc("snmp_error", "Error connecting to target", nil, nil), err) + cancel() return } defer client.Close() - defer wg.Done() for m := range workerChan { - logger := log.With(c.logger, "module", m.name) - level.Debug(logger).Log("msg", "Starting scrape") + _logger := log.With(logger, "module", m.name) + level.Debug(_logger).Log("msg", "Starting scrape") start := time.Now() - c.collect(ch, logger, client, m) + c.collect(ch, _logger, client, m) duration := time.Since(start).Seconds() - level.Debug(logger).Log("msg", "Finished scrape", "duration_seconds", duration) + level.Debug(_logger).Log("msg", "Finished scrape", "duration_seconds", duration) c.metrics.SNMPCollectionDuration.WithLabelValues(m.name).Observe(duration) } }(i) } + done := false for _, module := range c.modules { - workerChan <- module + if done { + break + } + select { + case <-ctx.Done(): + done = true + level.Debug(c.logger).Log("msg", "Context canceled", "err", ctx.Err(), "module", module.name) + case workerChan <- module: + level.Debug(c.logger).Log("msg", "Sent module to worker", "module", module.name) + } } close(workerChan) wg.Wait() From ccec5469b035cafba4b5c5bb9cf40fb9ad5a9abd Mon Sep 17 00:00:00 2001 From: Kakuya Ando Date: Mon, 12 Feb 2024 16:38:24 +0900 Subject: [PATCH 5/8] Add LICENSE to newly added files Signed-off-by: Kakuya Ando --- scraper/gosnmp.go | 13 +++++++++++++ scraper/scraper.go | 13 +++++++++++++ 2 files changed, 26 insertions(+) diff --git a/scraper/gosnmp.go b/scraper/gosnmp.go index 859bad5f..581381ec 100644 --- a/scraper/gosnmp.go +++ b/scraper/gosnmp.go @@ -1,3 +1,16 @@ +// Copyright 2018 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package scraper import ( diff --git a/scraper/scraper.go b/scraper/scraper.go index eee05ecd..ea112339 100644 --- a/scraper/scraper.go +++ b/scraper/scraper.go @@ -1,3 +1,16 @@ +// Copyright 2018 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package scraper import ( From d4da0de199aabee1214621e3a3b8e63943d64783 Mon Sep 17 00:00:00 2001 From: Kakuya Ando Date: Mon, 12 Feb 2024 22:42:40 +0900 Subject: [PATCH 6/8] Add unit test for ScrapeTarget function using a mock SNMP scraper Signed-off-by: Kakuya Ando --- collector/collector_test.go | 37 ++++++++++++++++++++ scraper/mock.go | 68 +++++++++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+) create mode 100644 scraper/mock.go diff --git a/collector/collector_test.go b/collector/collector_test.go index 2f0fa377..93eae9be 100644 --- a/collector/collector_test.go +++ b/collector/collector_test.go @@ -26,6 +26,7 @@ import ( io_prometheus_client "github.com/prometheus/client_model/go" "github.com/prometheus/snmp_exporter/config" + "github.com/prometheus/snmp_exporter/scraper" ) func TestPduToSample(t *testing.T) { @@ -1401,3 +1402,39 @@ func TestAddAllowedIndices(t *testing.T) { } } } + +func TestScrapeTarget(t *testing.T) { + mock := &scraper.MockSNMPScraper{ + GetResponses: map[string]*gosnmp.SnmpPDU{ + "1.3.6.1.2.1.1.1.0": {Type: gosnmp.OctetString, Value: "Test Device"}, // sysDescr + // sysUpTime undefined + }, + WalkAllResponses: map[string]gosnmp.SnmpPDU{ + "1.3.6.1.2.1.2.2.1.2.1": {Type: gosnmp.OctetString, Value: "Interface 1"}, // ifDescr + "1.3.6.1.2.1.2.2.1.2.2": {Type: gosnmp.OctetString, Value: "Interface 2"}, // ifDescr + // ifType undefined + }, + } + + auth := &config.Auth{ + Version: 2, + } + module := &config.Module{ + Get: []string{ + "1.3.6.1.2.1.1.1.0", // sysDescr + "1.3.6.1.2.1.1.3.0", // sysUpTime + }, + Walk: []string{ + "1.3.6.1.2.1.2.2.1.2", // ifDescr + "1.3.6.1.2.1.2.2.1.3", // ifType + }, + } + results, err := ScrapeTarget(mock, "someTarget", auth, module, log.NewNopLogger(), Metrics{}) + if err != nil { + t.Errorf("ScrapeTarget returned an error: %v", err) + } + expectedPdusLen := 3 + if len(results.pdus) != expectedPdusLen { + t.Errorf("Expected %d PDUs, got %d", expectedPdusLen, len(results.pdus)) + } +} diff --git a/scraper/mock.go b/scraper/mock.go new file mode 100644 index 00000000..c9ce18f7 --- /dev/null +++ b/scraper/mock.go @@ -0,0 +1,68 @@ +// Copyright 2018 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scraper + +import ( + "strings" + + "github.com/gosnmp/gosnmp" +) + +type MockSNMPScraper struct { + GetResponses map[string]*gosnmp.SnmpPDU + WalkAllResponses map[string]gosnmp.SnmpPDU + ConnectError error + CloseError error +} + +func (m *MockSNMPScraper) Get(oids []string) (*gosnmp.SnmpPacket, error) { + pdus := make([]gosnmp.SnmpPDU, 0, len(oids)) + for _, oid := range oids { + if response, exists := m.GetResponses[oid]; exists { + pdus = append(pdus, *response) + } else { + pdus = append(pdus, gosnmp.SnmpPDU{ + Name: oid, + Type: gosnmp.NoSuchObject, + Value: nil, + }) + } + } + return &gosnmp.SnmpPacket{ + Variables: pdus, + Error: gosnmp.NoError, + }, nil +} + +func (m *MockSNMPScraper) WalkAll(baseOID string) ([]gosnmp.SnmpPDU, error) { + var pdus []gosnmp.SnmpPDU + for k, v := range m.WalkAllResponses { + if strings.HasPrefix(k, baseOID) { + pdus = append(pdus, v) + } + } + return pdus, nil + +} + +func (m *MockSNMPScraper) Connect() error { + return m.ConnectError +} + +func (m *MockSNMPScraper) Close() error { + return m.CloseError +} + +func (m *MockSNMPScraper) SetOptions(...func(*gosnmp.GoSNMP)) { +} From 2476912f27bd842579e51ca0d31a9205a2aecae5 Mon Sep 17 00:00:00 2001 From: Kakuya Ando Date: Thu, 15 Feb 2024 20:15:14 +0900 Subject: [PATCH 7/8] Update copyright year to 2024 Signed-off-by: Kakuya Ando --- scraper/gosnmp.go | 2 +- scraper/mock.go | 2 +- scraper/scraper.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scraper/gosnmp.go b/scraper/gosnmp.go index 581381ec..83166898 100644 --- a/scraper/gosnmp.go +++ b/scraper/gosnmp.go @@ -1,4 +1,4 @@ -// Copyright 2018 The Prometheus Authors +// Copyright 2024 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at diff --git a/scraper/mock.go b/scraper/mock.go index c9ce18f7..ebf7b1d8 100644 --- a/scraper/mock.go +++ b/scraper/mock.go @@ -1,4 +1,4 @@ -// Copyright 2018 The Prometheus Authors +// Copyright 2024 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at diff --git a/scraper/scraper.go b/scraper/scraper.go index ea112339..0092023b 100644 --- a/scraper/scraper.go +++ b/scraper/scraper.go @@ -1,4 +1,4 @@ -// Copyright 2018 The Prometheus Authors +// Copyright 2024 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at From 029885c0018c5d09c1c08c7adbb37f503542c135 Mon Sep 17 00:00:00 2001 From: Kakuya Ando Date: Sun, 18 Feb 2024 17:53:17 +0900 Subject: [PATCH 8/8] Add test cases for scraping targets with different configurations Signed-off-by: Kakuya Ando --- collector/collector_test.go | 129 ++++++++++++++++++++++++++++-------- scraper/mock.go | 56 ++++++++++------ 2 files changed, 136 insertions(+), 49 deletions(-) diff --git a/collector/collector_test.go b/collector/collector_test.go index 93eae9be..5582ad4e 100644 --- a/collector/collector_test.go +++ b/collector/collector_test.go @@ -1404,37 +1404,108 @@ func TestAddAllowedIndices(t *testing.T) { } func TestScrapeTarget(t *testing.T) { - mock := &scraper.MockSNMPScraper{ - GetResponses: map[string]*gosnmp.SnmpPDU{ - "1.3.6.1.2.1.1.1.0": {Type: gosnmp.OctetString, Value: "Test Device"}, // sysDescr - // sysUpTime undefined - }, - WalkAllResponses: map[string]gosnmp.SnmpPDU{ - "1.3.6.1.2.1.2.2.1.2.1": {Type: gosnmp.OctetString, Value: "Interface 1"}, // ifDescr - "1.3.6.1.2.1.2.2.1.2.2": {Type: gosnmp.OctetString, Value: "Interface 2"}, // ifDescr - // ifType undefined + cases := []struct { + name string + module *config.Module + getResponse map[string]gosnmp.SnmpPDU + walkResponses map[string][]gosnmp.SnmpPDU + expectPdus []gosnmp.SnmpPDU + getCall []string + walkCall []string + }{ + { + name: "basic", + module: &config.Module{ + Get: []string{"1.3.6.1.2.1.1.1.0"}, + Walk: []string{"1.3.6.1.2.1.2.2.1.2", "1.3.6.1.2.1.31.1.1.1.18"}, + }, + getResponse: map[string]gosnmp.SnmpPDU{ + "1.3.6.1.2.1.1.1.0": {Type: gosnmp.OctetString, Name: "1.3.6.1.2.1.1.1.0", Value: "Test Device"}, // sysDescr + }, + walkResponses: map[string][]gosnmp.SnmpPDU{ + "1.3.6.1.2.1.2.2.1.2": { + {Type: gosnmp.OctetString, Name: ".1.3.6.1.2.1.2.2.1.2.1", Value: "lo"}, + {Type: gosnmp.OctetString, Name: ".1.3.6.1.2.1.2.2.1.2.2", Value: "Intel Corporation 82540EM Gigabit Ethernet Controller"}, + {Type: gosnmp.OctetString, Name: ".1.3.6.1.2.1.2.2.1.2.3", Value: "Intel Corporation 82540EM Gigabit Ethernet Controller"}, + }, + "1.3.6.1.2.1.31.1.1.1.18": { + {Type: gosnmp.OctetString, Name: ".1.3.6.1.2.1.31.1.1.1.18.1", Value: "lo"}, + {Type: gosnmp.OctetString, Name: ".1.3.6.1.2.1.31.1.1.1.18.2", Value: "eth0"}, + {Type: gosnmp.OctetString, Name: ".1.3.6.1.2.1.31.1.1.1.18.3", Value: "swp1"}, + }, + }, + expectPdus: []gosnmp.SnmpPDU{ + {Type: gosnmp.OctetString, Name: "1.3.6.1.2.1.1.1.0", Value: "Test Device"}, + {Type: gosnmp.OctetString, Name: ".1.3.6.1.2.1.2.2.1.2.1", Value: "lo"}, + {Type: gosnmp.OctetString, Name: ".1.3.6.1.2.1.2.2.1.2.2", Value: "Intel Corporation 82540EM Gigabit Ethernet Controller"}, + {Type: gosnmp.OctetString, Name: ".1.3.6.1.2.1.2.2.1.2.3", Value: "Intel Corporation 82540EM Gigabit Ethernet Controller"}, + {Type: gosnmp.OctetString, Name: ".1.3.6.1.2.1.31.1.1.1.18.1", Value: "lo"}, + {Type: gosnmp.OctetString, Name: ".1.3.6.1.2.1.31.1.1.1.18.2", Value: "eth0"}, + {Type: gosnmp.OctetString, Name: ".1.3.6.1.2.1.31.1.1.1.18.3", Value: "swp1"}, + }, + getCall: []string{"1.3.6.1.2.1.1.1.0"}, + walkCall: []string{"1.3.6.1.2.1.2.2.1.2", "1.3.6.1.2.1.31.1.1.1.18"}, + }, + { + name: "dynamic filter", + module: &config.Module{ + Get: []string{}, + Walk: []string{"1.3.6.1.2.1.31.1.1.1.18"}, + Filters: []config.DynamicFilter{ + { + Oid: "1.3.6.1.2.1.2.2.1.2", + Targets: []string{ + "1.3.6.1.2.1.31.1.1.1.18", + }, + Values: []string{"Intel Corporation 82540EM Gigabit Ethernet Controller"}, + }, + }, + }, + getResponse: map[string]gosnmp.SnmpPDU{ + "1.3.6.1.2.1.31.1.1.1.18.2": {Type: gosnmp.OctetString, Name: ".1.3.6.1.2.1.31.1.1.1.18.2", Value: "eth0"}, + "1.3.6.1.2.1.31.1.1.1.18.3": {Type: gosnmp.OctetString, Name: ".1.3.6.1.2.1.31.1.1.1.18.3", Value: "swp1"}, + }, + walkResponses: map[string][]gosnmp.SnmpPDU{ + "1.3.6.1.2.1.2.2.1.2": { + {Type: gosnmp.OctetString, Name: ".1.3.6.1.2.1.2.2.1.2.1", Value: "lo"}, + {Type: gosnmp.OctetString, Name: ".1.3.6.1.2.1.2.2.1.2.2", Value: "Intel Corporation 82540EM Gigabit Ethernet Controller"}, + {Type: gosnmp.OctetString, Name: ".1.3.6.1.2.1.2.2.1.2.3", Value: "Intel Corporation 82540EM Gigabit Ethernet Controller"}, + }, + }, + expectPdus: []gosnmp.SnmpPDU{ + {Type: gosnmp.OctetString, Name: ".1.3.6.1.2.1.31.1.1.1.18.2", Value: "eth0"}, + {Type: gosnmp.OctetString, Name: ".1.3.6.1.2.1.31.1.1.1.18.3", Value: "swp1"}, + }, + getCall: []string{"1.3.6.1.2.1.31.1.1.1.18.2", "1.3.6.1.2.1.31.1.1.1.18.3"}, + walkCall: []string{"1.3.6.1.2.1.2.2.1.2"}, }, } - auth := &config.Auth{ - Version: 2, - } - module := &config.Module{ - Get: []string{ - "1.3.6.1.2.1.1.1.0", // sysDescr - "1.3.6.1.2.1.1.3.0", // sysUpTime - }, - Walk: []string{ - "1.3.6.1.2.1.2.2.1.2", // ifDescr - "1.3.6.1.2.1.2.2.1.3", // ifType - }, - } - results, err := ScrapeTarget(mock, "someTarget", auth, module, log.NewNopLogger(), Metrics{}) - if err != nil { - t.Errorf("ScrapeTarget returned an error: %v", err) - } - expectedPdusLen := 3 - if len(results.pdus) != expectedPdusLen { - t.Errorf("Expected %d PDUs, got %d", expectedPdusLen, len(results.pdus)) + auth := &config.Auth{Version: 2} + for _, c := range cases { + tt := c + t.Run(tt.name, func(t *testing.T) { + mock := scraper.NewMockSNMPScraper(tt.getResponse, tt.walkResponses) + results, err := ScrapeTarget(mock, "someTarget", auth, tt.module, log.NewNopLogger(), Metrics{}) + if err != nil { + t.Errorf("ScrapeTarget returned an error: %v", err) + } + if !reflect.DeepEqual(mock.CallGet(), tt.getCall) { + t.Errorf("Expected get call %v, got %v", tt.getCall, mock.CallGet()) + } + if !reflect.DeepEqual(mock.CallWalk(), tt.walkCall) { + t.Errorf("Expected walk call %v, got %v", tt.walkCall, mock.CallWalk()) + } + expectedPdusLen := len(tt.expectPdus) + if len(results.pdus) != expectedPdusLen { + t.Fatalf("Expected %d PDUs, got %d", expectedPdusLen, len(results.pdus)) + } + + for i, pdu := range tt.expectPdus { + if !reflect.DeepEqual(pdu, results.pdus[i]) { + t.Errorf("Expected %v, got %v", pdu, results.pdus[i]) + } + } + }) } } diff --git a/scraper/mock.go b/scraper/mock.go index ebf7b1d8..a2be8595 100644 --- a/scraper/mock.go +++ b/scraper/mock.go @@ -14,23 +14,41 @@ package scraper import ( - "strings" - "github.com/gosnmp/gosnmp" ) -type MockSNMPScraper struct { - GetResponses map[string]*gosnmp.SnmpPDU - WalkAllResponses map[string]gosnmp.SnmpPDU - ConnectError error - CloseError error +func NewMockSNMPScraper(get map[string]gosnmp.SnmpPDU, walk map[string][]gosnmp.SnmpPDU) *mockSNMPScraper { + return &mockSNMPScraper{ + GetResponses: get, + WalkResponses: walk, + callGet: make([]string, 0), + callWalk: make([]string, 0), + } +} + +type mockSNMPScraper struct { + GetResponses map[string]gosnmp.SnmpPDU + WalkResponses map[string][]gosnmp.SnmpPDU + ConnectError error + CloseError error + + callGet []string + callWalk []string } -func (m *MockSNMPScraper) Get(oids []string) (*gosnmp.SnmpPacket, error) { +func (m *mockSNMPScraper) CallGet() []string { + return m.callGet +} + +func (m *mockSNMPScraper) CallWalk() []string { + return m.callWalk +} + +func (m *mockSNMPScraper) Get(oids []string) (*gosnmp.SnmpPacket, error) { pdus := make([]gosnmp.SnmpPDU, 0, len(oids)) for _, oid := range oids { if response, exists := m.GetResponses[oid]; exists { - pdus = append(pdus, *response) + pdus = append(pdus, response) } else { pdus = append(pdus, gosnmp.SnmpPDU{ Name: oid, @@ -38,6 +56,7 @@ func (m *MockSNMPScraper) Get(oids []string) (*gosnmp.SnmpPacket, error) { Value: nil, }) } + m.callGet = append(m.callGet, oid) } return &gosnmp.SnmpPacket{ Variables: pdus, @@ -45,24 +64,21 @@ func (m *MockSNMPScraper) Get(oids []string) (*gosnmp.SnmpPacket, error) { }, nil } -func (m *MockSNMPScraper) WalkAll(baseOID string) ([]gosnmp.SnmpPDU, error) { - var pdus []gosnmp.SnmpPDU - for k, v := range m.WalkAllResponses { - if strings.HasPrefix(k, baseOID) { - pdus = append(pdus, v) - } +func (m *mockSNMPScraper) WalkAll(baseOID string) ([]gosnmp.SnmpPDU, error) { + m.callWalk = append(m.callWalk, baseOID) + if pdus, exists := m.WalkResponses[baseOID]; exists { + return pdus, nil } - return pdus, nil - + return nil, nil } -func (m *MockSNMPScraper) Connect() error { +func (m *mockSNMPScraper) Connect() error { return m.ConnectError } -func (m *MockSNMPScraper) Close() error { +func (m *mockSNMPScraper) Close() error { return m.CloseError } -func (m *MockSNMPScraper) SetOptions(...func(*gosnmp.GoSNMP)) { +func (m *mockSNMPScraper) SetOptions(...func(*gosnmp.GoSNMP)) { }