From d1de18507cd40712d205db037804652c4d0953e0 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Mon, 21 Aug 2023 10:54:04 -0400 Subject: [PATCH] dns processor - Add A, AAAA, and TXT query support The dns processor previously supported only reverse DNS lookups. This adds support for performing A, AAAA, and TXT record queries. The response.ptr.histogram metric was renamed to request_duration.histogram. This naming allows the metric to represent the duration of the DNS request for all query types. Closes #11416 --- libbeat/processors/dns/cache.go | 68 +++++++------- libbeat/processors/dns/cache_test.go | 52 +++++------ libbeat/processors/dns/config.go | 80 +++++++++++------ libbeat/processors/dns/dns.go | 45 ++++++++-- libbeat/processors/dns/dns_test.go | 14 +-- libbeat/processors/dns/doc.go | 2 +- libbeat/processors/dns/docs/dns.asciidoc | 38 +++++--- libbeat/processors/dns/resolver.go | 107 +++++++++++++++-------- libbeat/processors/dns/resolver_test.go | 16 ++-- 9 files changed, 265 insertions(+), 157 deletions(-) diff --git a/libbeat/processors/dns/cache.go b/libbeat/processors/dns/cache.go index 6e77af028880..b1b9c35cfac4 100644 --- a/libbeat/processors/dns/cache.go +++ b/libbeat/processors/dns/cache.go @@ -24,23 +24,23 @@ import ( "github.com/elastic/elastic-agent-libs/monitoring" ) -type ptrRecord struct { - host string +type successRecord struct { + data []string expires time.Time } -func (r ptrRecord) IsExpired(now time.Time) bool { +func (r successRecord) IsExpired(now time.Time) bool { return now.After(r.expires) } -type ptrCache struct { +type successCache struct { sync.RWMutex - data map[string]ptrRecord + data map[string]successRecord maxSize int minSuccessTTL time.Duration } -func (c *ptrCache) set(now time.Time, key string, ptr *PTR) { +func (c *successCache) set(now time.Time, key string, result *result) { c.Lock() defer c.Unlock() @@ -48,14 +48,14 @@ func (c *ptrCache) set(now time.Time, key string, ptr *PTR) { c.evict() } - c.data[key] = ptrRecord{ - host: ptr.Host, - expires: now.Add(time.Duration(ptr.TTL) * time.Second), + c.data[key] = successRecord{ + data: result.Data, + expires: now.Add(time.Duration(result.TTL) * time.Second), } } // evict removes a single random key from the cache. -func (c *ptrCache) evict() { +func (c *successCache) evict() { var key string for k := range c.data { key = k @@ -64,13 +64,13 @@ func (c *ptrCache) evict() { delete(c.data, key) } -func (c *ptrCache) get(now time.Time, key string) *PTR { +func (c *successCache) get(now time.Time, key string) *result { c.RLock() defer c.RUnlock() r, found := c.data[key] if found && !r.IsExpired(now) { - return &PTR{r.host, uint32(r.expires.Sub(now) / time.Second)} + return &result{r.data, uint32(r.expires.Sub(now) / time.Second)} } return nil } @@ -132,13 +132,13 @@ type cachedError struct { func (ce *cachedError) Error() string { return ce.err.Error() + " (from failure cache)" } func (ce *cachedError) Cause() error { return ce.err } -// PTRLookupCache is a cache for storing and retrieving the results of -// reverse DNS queries. It caches the results of queries regardless of their +// LookupCache is a cache for storing and retrieving the results of +// DNS queries. It caches the results of queries regardless of their // outcome (success or failure). -type PTRLookupCache struct { - success *ptrCache +type LookupCache struct { + success *successCache failure *failureCache - resolver PTRResolver + resolver resolver stats cacheStats } @@ -147,15 +147,15 @@ type cacheStats struct { Miss *monitoring.Int } -// NewPTRLookupCache returns a new cache. -func NewPTRLookupCache(reg *monitoring.Registry, conf CacheConfig, resolver PTRResolver) (*PTRLookupCache, error) { +// NewLookupCache returns a new cache. +func NewLookupCache(reg *monitoring.Registry, conf CacheConfig, resolver resolver) (*LookupCache, error) { if err := conf.Validate(); err != nil { return nil, err } - c := &PTRLookupCache{ - success: &ptrCache{ - data: make(map[string]ptrRecord, conf.SuccessCache.InitialCapacity), + c := &LookupCache{ + success: &successCache{ + data: make(map[string]successRecord, conf.SuccessCache.InitialCapacity), maxSize: conf.SuccessCache.MaxCapacity, minSuccessTTL: conf.SuccessCache.MinTTL, }, @@ -174,36 +174,36 @@ func NewPTRLookupCache(reg *monitoring.Registry, conf CacheConfig, resolver PTRR return c, nil } -// LookupPTR performs a reverse lookup on the given IP address. A cached result +// Lookup performs a lookup on the given query string. A cached result // will be returned if it is contained in the cache, otherwise a lookup is // performed. -func (c PTRLookupCache) LookupPTR(ip string) (*PTR, error) { +func (c LookupCache) Lookup(q string, qt queryType) (*result, error) { now := time.Now() - ptr := c.success.get(now, ip) - if ptr != nil { + r := c.success.get(now, q) + if r != nil { c.stats.Hit.Inc() - return ptr, nil + return r, nil } - err := c.failure.get(now, ip) + err := c.failure.get(now, q) if err != nil { c.stats.Hit.Inc() return nil, err } c.stats.Miss.Inc() - ptr, err = c.resolver.LookupPTR(ip) + r, err = c.resolver.Lookup(q, qt) if err != nil { - c.failure.set(now, ip, &cachedError{err}) + c.failure.set(now, q, &cachedError{err}) return nil, err } - // We set the ptr.TTL to the minimum TTL in case it is less than that. - ptr.TTL = max(ptr.TTL, uint32(c.success.minSuccessTTL/time.Second)) + // We set the result TTL to the minimum TTL in case it is less than that. + r.TTL = max(r.TTL, uint32(c.success.minSuccessTTL/time.Second)) - c.success.set(now, ip, ptr) - return ptr, nil + c.success.set(now, q, r) + return r, nil } func max(a, b uint32) uint32 { diff --git a/libbeat/processors/dns/cache_test.go b/libbeat/processors/dns/cache_test.go index fdc531c54fb7..16e151ed0d7c 100644 --- a/libbeat/processors/dns/cache_test.go +++ b/libbeat/processors/dns/cache_test.go @@ -29,85 +29,85 @@ import ( type stubResolver struct{} -func (r *stubResolver) LookupPTR(ip string) (*PTR, error) { +func (r *stubResolver) Lookup(ip string, _ queryType) (*result, error) { switch ip { case gatewayIP: - return &PTR{Host: gatewayName, TTL: gatewayTTL}, nil + return &result{Data: []string{gatewayName}, TTL: gatewayTTL}, nil case gatewayIP + "1": return nil, io.ErrUnexpectedEOF case gatewayIP + "2": - return &PTR{Host: gatewayName, TTL: 0}, nil + return &result{Data: []string{gatewayName}, TTL: 0}, nil } return nil, &dnsError{"fake lookup returned NXDOMAIN"} } func TestCache(t *testing.T) { - c, err := NewPTRLookupCache( + c, err := NewLookupCache( monitoring.NewRegistry(), - defaultConfig.CacheConfig, + defaultConfig().CacheConfig, &stubResolver{}) if err != nil { t.Fatal(err) } // Initial success query. - ptr, err := c.LookupPTR(gatewayIP) + r, err := c.Lookup(gatewayIP, typePTR) if assert.NoError(t, err) { - assert.EqualValues(t, gatewayName, ptr.Host) - assert.EqualValues(t, gatewayTTL, ptr.TTL) + assert.EqualValues(t, []string{gatewayName}, r.Data) + assert.EqualValues(t, gatewayTTL, r.TTL) assert.EqualValues(t, 0, c.stats.Hit.Get()) assert.EqualValues(t, 1, c.stats.Miss.Get()) } // Cached success query. - ptr, err = c.LookupPTR(gatewayIP) + r, err = c.Lookup(gatewayIP, typePTR) if assert.NoError(t, err) { - assert.EqualValues(t, gatewayName, ptr.Host) + assert.EqualValues(t, []string{gatewayName}, r.Data) // TTL counts down while in cache. - assert.InDelta(t, gatewayTTL, ptr.TTL, 1) + assert.InDelta(t, gatewayTTL, r.TTL, 1) assert.EqualValues(t, 1, c.stats.Hit.Get()) assert.EqualValues(t, 1, c.stats.Miss.Get()) } // Initial failure query (like a dns error response code). - ptr, err = c.LookupPTR(gatewayIP + "0") + r, err = c.Lookup(gatewayIP+"0", typePTR) if assert.Error(t, err) { - assert.Nil(t, ptr) + assert.Nil(t, r) assert.EqualValues(t, 1, c.stats.Hit.Get()) assert.EqualValues(t, 2, c.stats.Miss.Get()) } // Cached failure query. - ptr, err = c.LookupPTR(gatewayIP + "0") + r, err = c.Lookup(gatewayIP+"0", typePTR) if assert.Error(t, err) { - assert.Nil(t, ptr) + assert.Nil(t, r) assert.EqualValues(t, 2, c.stats.Hit.Get()) assert.EqualValues(t, 2, c.stats.Miss.Get()) } // Initial network failure (like I/O timeout). - ptr, err = c.LookupPTR(gatewayIP + "1") + r, err = c.Lookup(gatewayIP+"1", typePTR) if assert.Error(t, err) { - assert.Nil(t, ptr) + assert.Nil(t, r) assert.EqualValues(t, 2, c.stats.Hit.Get()) assert.EqualValues(t, 3, c.stats.Miss.Get()) } // Check for a cache hit for the network failure. - ptr, err = c.LookupPTR(gatewayIP + "1") + r, err = c.Lookup(gatewayIP+"1", typePTR) if assert.Error(t, err) { - assert.Nil(t, ptr) + assert.Nil(t, r) assert.EqualValues(t, 3, c.stats.Hit.Get()) assert.EqualValues(t, 3, c.stats.Miss.Get()) // Cache miss. } - minTTL := defaultConfig.CacheConfig.SuccessCache.MinTTL + minTTL := defaultConfig().CacheConfig.SuccessCache.MinTTL // Initial success returned TTL=0 with MinTTL. - ptr, err = c.LookupPTR(gatewayIP + "2") + r, err = c.Lookup(gatewayIP+"2", typePTR) if assert.NoError(t, err) { - assert.EqualValues(t, gatewayName, ptr.Host) + assert.EqualValues(t, []string{gatewayName}, r.Data) - assert.EqualValues(t, minTTL/time.Second, ptr.TTL) + assert.EqualValues(t, minTTL/time.Second, r.TTL) assert.EqualValues(t, 3, c.stats.Hit.Get()) assert.EqualValues(t, 4, c.stats.Miss.Get()) @@ -117,11 +117,11 @@ func TestCache(t *testing.T) { } // Cached success from a previous TTL=0 response. - ptr, err = c.LookupPTR(gatewayIP + "2") + r, err = c.Lookup(gatewayIP+"2", typePTR) if assert.NoError(t, err) { - assert.EqualValues(t, gatewayName, ptr.Host) + assert.EqualValues(t, []string{gatewayName}, r.Data) // TTL counts down while in cache. - assert.InDelta(t, minTTL/time.Second, ptr.TTL, 1) + assert.InDelta(t, minTTL/time.Second, r.TTL, 1) assert.EqualValues(t, 4, c.stats.Hit.Get()) assert.EqualValues(t, 4, c.stats.Miss.Get()) } diff --git a/libbeat/processors/dns/config.go b/libbeat/processors/dns/config.go index fb8a13eaf216..928cd46cee0a 100644 --- a/libbeat/processors/dns/config.go +++ b/libbeat/processors/dns/config.go @@ -23,6 +23,8 @@ import ( "strings" "time" + "github.com/miekg/dns" + "github.com/elastic/elastic-agent-libs/mapstr" ) @@ -31,7 +33,7 @@ type Config struct { CacheConfig `config:",inline"` Nameservers []string `config:"nameservers"` // Required on Windows. /etc/resolv.conf is used if none are given. Timeout time.Duration `config:"timeout"` // Per request timeout (with 2 nameservers the total timeout would be 2x). - Type string `config:"type" validate:"required"` // Reverse is the only supported type currently. + Type queryType `config:"type" validate:"required"` // Reverse is the only supported type currently. Action FieldAction `config:"action"` // Append or replace (defaults to append) when target exists. TagOnFailure []string `config:"tag_on_failure"` // Tags to append when a failure occurs. Fields mapstr.M `config:"fields"` // Mapping of source fields to target fields. @@ -75,6 +77,41 @@ func (fa *FieldAction) Unpack(v string) error { return nil } +// queryType represents a DNS query type. +type queryType uint16 + +const ( + typePTR = queryType(dns.TypePTR) + typeA = queryType(dns.TypeA) + typeAAAA = queryType(dns.TypeAAAA) + typeTXT = queryType(dns.TypeTXT) +) + +func (qt queryType) String() string { + if name := dns.TypeToString[uint16(qt)]; name != "" { + return name + } + return strconv.FormatUint(uint64(qt), 10) +} + +// Unpack unpacks a string to a queryType. +func (qt *queryType) Unpack(v string) error { + switch strings.ToLower(v) { + case "a": + *qt = typeA + case "aaaa": + *qt = typeAAAA + case "reverse", "ptr": + *qt = typePTR + case "txt": + *qt = typeTXT + default: + return fmt.Errorf("invalid dns lookup type '%v' specified in "+ + "config (valid values are: A, AAAA, PTR, reverse, TXT)", v) + } + return nil +} + // CacheConfig defines the success and failure caching parameters. type CacheConfig struct { SuccessCache CacheSettings `config:"success_cache"` @@ -100,15 +137,6 @@ type CacheSettings struct { // Validate validates the data contained in the config. func (c *Config) Validate() error { - // Validate lookup type. - c.Type = strings.ToLower(c.Type) - switch c.Type { - case "reverse": - default: - return fmt.Errorf("invalid dns lookup type '%v' specified in "+ - "config (valid values are: reverse)", c.Type) - } - // Flatten the mapping of source fields to target fields. c.reverseFlat = map[string]string{} for k, v := range c.Fields.Flatten() { @@ -157,20 +185,22 @@ func (c *CacheConfig) Validate() error { return nil } -var defaultConfig = Config{ - CacheConfig: CacheConfig{ - SuccessCache: CacheSettings{ - MinTTL: time.Minute, - InitialCapacity: 1000, - MaxCapacity: 10000, +func defaultConfig() Config { + return Config{ + CacheConfig: CacheConfig{ + SuccessCache: CacheSettings{ + MinTTL: time.Minute, + InitialCapacity: 1000, + MaxCapacity: 10000, + }, + FailureCache: CacheSettings{ + MinTTL: time.Minute, + TTL: time.Minute, + InitialCapacity: 1000, + MaxCapacity: 10000, + }, }, - FailureCache: CacheSettings{ - MinTTL: time.Minute, - TTL: time.Minute, - InitialCapacity: 1000, - MaxCapacity: 10000, - }, - }, - Transport: "udp", - Timeout: 500 * time.Millisecond, + Transport: "udp", + Timeout: 500 * time.Millisecond, + } } diff --git a/libbeat/processors/dns/dns.go b/libbeat/processors/dns/dns.go index ee8dd918ebc3..154ab3349a49 100644 --- a/libbeat/processors/dns/dns.go +++ b/libbeat/processors/dns/dns.go @@ -45,13 +45,13 @@ func init() { type processor struct { Config - resolver PTRResolver + resolver resolver log *logp.Logger } // New constructs a new DNS processor. func New(cfg *config.C) (beat.Processor, error) { - c := defaultConfig + c := defaultConfig() if err := cfg.Unpack(&c); err != nil { return nil, fmt.Errorf("fail to unpack the dns configuration: %w", err) } @@ -69,7 +69,7 @@ func New(cfg *config.C) (beat.Processor, error) { return nil, err } - cache, err := NewPTRLookupCache(metrics.NewRegistry("cache"), c.CacheConfig, resolver) + cache, err := NewLookupCache(metrics.NewRegistry("cache"), c.CacheConfig, resolver) if err != nil { return nil, err } @@ -95,17 +95,21 @@ func (p *processor) processField(source, target string, action FieldAction, even return nil } - maybeIP, ok := v.(string) + strVal, ok := v.(string) if !ok { return nil } - ptrRecord, err := p.resolver.LookupPTR(maybeIP) + result, err := p.resolver.Lookup(strVal, p.Type) if err != nil { - return fmt.Errorf("reverse lookup of %v value '%v' failed: %w", source, maybeIP, err) + return fmt.Errorf("dns lookup (%v) of %v value '%v' failed: %w", p.Type, source, strVal, err) } - return setFieldValue(action, event, target, ptrRecord.Host) + // PTR lookups return a scalar. All other lookup types return a string slice. + if p.Type == typePTR { + return setFieldValue(action, event, target, result.Data[0]) + } + return setFieldSliceValue(action, event, target, result.Data) } func setFieldValue(action FieldAction, event *beat.Event, key string, value string) error { @@ -129,7 +133,32 @@ func setFieldValue(action FieldAction, event *beat.Event, key string, value stri } return err default: - panic(fmt.Errorf("Unexpected dns field action value encountered: %v", action)) + panic(fmt.Errorf("unexpected dns field action value encountered: %v", action)) + } +} + +func setFieldSliceValue(action FieldAction, event *beat.Event, key string, value []string) error { + switch action { + case ActionReplace: + _, err := event.PutValue(key, value) + return err + case ActionAppend: + old, err := event.PutValue(key, value) + if err != nil { + return err + } + + if old != nil { + switch v := old.(type) { + case string: + _, err = event.PutValue(key, append([]string{v}, value...)) + case []string: + _, err = event.PutValue(key, append(v, value...)) + } + } + return err + default: + panic(fmt.Errorf("unexpected dns field action value encountered: %v", action)) } } diff --git a/libbeat/processors/dns/dns_test.go b/libbeat/processors/dns/dns_test.go index fa3b2d67f41b..f0b0dcf72346 100644 --- a/libbeat/processors/dns/dns_test.go +++ b/libbeat/processors/dns/dns_test.go @@ -32,8 +32,10 @@ import ( ) func TestDNSProcessorRun(t *testing.T) { + c := defaultConfig() + c.Type = typePTR p := &processor{ - Config: defaultConfig, + Config: c, resolver: &stubResolver{}, log: logp.NewLogger(logName), } @@ -94,7 +96,8 @@ func TestDNSProcessorRun(t *testing.T) { }) t.Run("metadata target", func(t *testing.T) { - config := defaultConfig + config := defaultConfig() + config.Type = typePTR config.reverseFlat = map[string]string{ "@metadata.ip": "@metadata.domain", } @@ -121,12 +124,11 @@ func TestDNSProcessorRun(t *testing.T) { assert.Equal(t, expMeta, newEvent.Meta) assert.Equal(t, event.Fields, newEvent.Fields) }) - } func TestDNSProcessorTagOnFailure(t *testing.T) { p := &processor{ - Config: defaultConfig, + Config: defaultConfig(), resolver: &stubResolver{}, log: logp.NewLogger(logName), } @@ -157,9 +159,9 @@ func TestDNSProcessorRunInParallel(t *testing.T) { // This is a simple smoke test to make sure that there are no concurrency // issues. It is most effective when run with the race detector. - conf := defaultConfig + conf := defaultConfig() reg := monitoring.NewRegistry() - cache, err := NewPTRLookupCache(reg, conf.CacheConfig, &stubResolver{}) + cache, err := NewLookupCache(reg, conf.CacheConfig, &stubResolver{}) if err != nil { t.Fatal(err) } diff --git a/libbeat/processors/dns/doc.go b/libbeat/processors/dns/doc.go index 8c895b268000..781d7e5284dd 100644 --- a/libbeat/processors/dns/doc.go +++ b/libbeat/processors/dns/doc.go @@ -16,7 +16,7 @@ // under the License. // Package dns implements a processor that can perform DNS lookups by sending -// a DNS request over UDP to a recursive nameserver. Each instance of the +// a DNS request over UDP or TLS to a recursive nameserver. Each instance of the // processor is independent (no shared cache) so it's best to only define one // instance of the processor. // diff --git a/libbeat/processors/dns/docs/dns.asciidoc b/libbeat/processors/dns/docs/dns.asciidoc index 8d03e8b4c0a4..9350b109a858 100644 --- a/libbeat/processors/dns/docs/dns.asciidoc +++ b/libbeat/processors/dns/docs/dns.asciidoc @@ -5,10 +5,10 @@ dns ++++ -The `dns` processor performs reverse DNS lookups of IP addresses. It caches the -responses that it receives in accordance to the time-to-live (TTL) value -contained in the response. It also caches failures that occur during lookups. -Each instance of this processor maintains its own independent cache. +The `dns` processor performs DNS queries. It caches the responses that it +receives in accordance to the time-to-live (TTL) value contained in the +response. It also caches failures that occur during lookups. Each instance +of this processor maintains its own independent cache. The processor uses its own DNS resolver to send requests to nameservers and does not use the operating system's resolver. It does not read any values contained @@ -24,6 +24,16 @@ throughput you can achieve is 500 events per second (1000 milliseconds / 2 milliseconds). If you have a high cache hit ratio then your throughput can be higher. +The processor can send the following query types: + +- `A` - IPv4 addresses +- `AAAA` - IPv6 addresses +- `TXT` - arbitrary human-readable text data +- `PTR` - reverse IP address lookups + +The output value is a list of strings for all query types except `PTR`. For +`PTR` queries the output value is a string. + This is a minimal configuration example that resolves the IP addresses contained in two fields. @@ -33,8 +43,8 @@ processors: - dns: type: reverse fields: - source.ip: source.hostname - destination.ip: destination.hostname + source.ip: source.domain + destination.ip: destination.domain ---- Next is a configuration example showing all options. @@ -47,8 +57,8 @@ processors: action: append transport: tls fields: - server.ip: server.hostname - client.ip: client.hostname + server.ip: server.domain + client.ip: client.domain success_cache: capacity.initial: 1000 capacity.max: 10000 @@ -64,8 +74,8 @@ processors: The `dns` processor has the following configuration settings: -`type`:: The type of DNS lookup to perform. The only supported type is -`reverse` which queries for a PTR record. +`type`:: The type of DNS query to perform. The supported types are `A`, `AAAA`, +`PTR` (or `reverse`), and `TXT`. `action`:: This defines the behavior of the processor when the target field already exists in the event. The options are `append` (default) and `replace`. @@ -82,8 +92,10 @@ the memory for this number of items. Default value is `1000`. cache can hold. When the maximum capacity is reached a random item is evicted. Default value is `10000`. -`success_cache.min_ttl`:: The duration of the minimum alternative cache TTL for successful DNS responses. Ensures that `TTL=0` successful reverse DNS responses can be cached. -Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". Default value is `1m`. +`success_cache.min_ttl`:: The duration of the minimum alternative cache TTL for +successful DNS responses. Ensures that `TTL=0` successful reverse DNS responses +can be cached. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". +Default value is `1m`. `failure_cache.capacity.initial`:: The initial number of items that the failure cache will be allocated to hold. When initialized the processor will allocate @@ -107,7 +119,7 @@ for each DNS request so if you have 2 nameservers then the total timeout will be "h". Default value is `500ms`. `tag_on_failure`:: A list of tags to add to the event when any lookup fails. The -tags are only added once even if multiple lookups fail. By default no tags are +tags are only added once even if multiple lookups fail. By default, no tags are added upon failure. `transport`:: The type of transport connection that should be used can either be diff --git a/libbeat/processors/dns/resolver.go b/libbeat/processors/dns/resolver.go index 7e0f160315cb..f8a9d93ce639 100644 --- a/libbeat/processors/dns/resolver.go +++ b/libbeat/processors/dns/resolver.go @@ -19,6 +19,8 @@ package dns import ( "errors" + "golang.org/x/exp/constraints" + "math" "net" "strconv" "strings" @@ -34,18 +36,18 @@ import ( const etcResolvConf = "/etc/resolv.conf" -// PTR represents a DNS pointer record (IP to hostname). -type PTR struct { - Host string // Hostname. - TTL uint32 // Time to live in seconds. +// result represents a DNS lookup result. +type result struct { + Data []string // Hostname. + TTL uint32 // Time to live in seconds. } -// PTRResolver performs PTR record lookups. -type PTRResolver interface { - LookupPTR(ip string) (*PTR, error) +// resolver performs result record lookups. +type resolver interface { + Lookup(q string, qt queryType) (*result, error) } -// MiekgResolver is a PTRResolver that is implemented using github.com/miekg/dns +// MiekgResolver is a resolver that is implemented using github.com/miekg/dns // to send requests to DNS servers. It does not use the Go resolver. type MiekgResolver struct { client *dns.Client @@ -57,9 +59,9 @@ type MiekgResolver struct { } type nameserverStats struct { - success *monitoring.Int // Number of responses from server. - failure *monitoring.Int // Number of failures (e.g. I/O timeout) (not NXDOMAIN). - ptrResponse metrics.Sample // Histogram of response times. + success *monitoring.Int // Number of responses from server. + failure *monitoring.Int // Number of failures (e.g. I/O timeout) (not NXDOMAIN). + responseTime metrics.Sample // Histogram of response times. } // NewMiekgResolver returns a new MiekgResolver. It returns an error if no @@ -94,7 +96,7 @@ func NewMiekgResolver(reg *monitoring.Registry, timeout time.Duration, transport } if timeout == 0 { - timeout = defaultConfig.Timeout + timeout = defaultConfig().Timeout } var clientTransferType string @@ -129,37 +131,42 @@ func (e *dnsError) Error() string { return "dns: " + e.err } -// LookupPTR performs a reverse lookup on the given IP address. -func (res *MiekgResolver) LookupPTR(ip string) (*PTR, error) { +// Lookup performs a DNS query. +func (res *MiekgResolver) Lookup(q string, qt queryType) (*result, error) { if len(res.servers) == 0 { return nil, errors.New("no dns servers configured") } - // Create PTR (reverse) DNS request. + // Create DNS request. m := new(dns.Msg) - arpa, err := dns.ReverseAddr(ip) - if err != nil { - return nil, err + switch qt { + case typePTR: + arpa, err := dns.ReverseAddr(q) + if err != nil { + return nil, err + } + m.SetQuestion(arpa, dns.TypePTR) + case typeA, typeAAAA, typeTXT: + m.SetQuestion(dns.Fqdn(q), uint16(qt)) } - m.SetQuestion(arpa, dns.TypePTR) m.RecursionDesired = true // Try the nameservers until we get a response. - var rtnErr error + var nameserverErr error for _, server := range res.servers { stats := res.getOrCreateNameserverStats(server) r, rtt, err := res.client.Exchange(m, server) if err != nil { - // Try next server if any. Otherwise return retErr. - rtnErr = err + // Try next server if any. Otherwise, return retErr. + nameserverErr = err stats.failure.Inc() continue } // We got a response. stats.success.Inc() - stats.ptrResponse.Update(int64(rtt)) + stats.responseTime.Update(int64(rtt)) if r.Rcode != dns.RcodeSuccess { name, found := dns.RcodeToString[r.Rcode] if !found { @@ -168,24 +175,45 @@ func (res *MiekgResolver) LookupPTR(ip string) (*PTR, error) { return nil, &dnsError{"nameserver " + server + " returned " + name} } + var rtn result + rtn.TTL = math.MaxUint32 for _, a := range r.Answer { - if ptr, ok := a.(*dns.PTR); ok { - return &PTR{ - Host: strings.TrimSuffix(ptr.Ptr, "."), - TTL: ptr.Hdr.Ttl, + // Ignore records that don't match the query type. + if a.Header().Rrtype != uint16(qt) { + continue + } + + switch rr := a.(type) { + case *dns.PTR: + return &result{ + Data: []string{strings.TrimSuffix(rr.Ptr, ".")}, + TTL: rr.Hdr.Ttl, }, nil + case *dns.A: + rtn.Data = append(rtn.Data, rr.A.String()) + rtn.TTL = min(rtn.TTL, rr.Hdr.Ttl) + case *dns.AAAA: + rtn.Data = append(rtn.Data, rr.AAAA.String()) + rtn.TTL = min(rtn.TTL, rr.Hdr.Ttl) + case *dns.TXT: + rtn.Data = append(rtn.Data, rr.Txt...) + rtn.TTL = min(rtn.TTL, rr.Hdr.Ttl) } } - return nil, &dnsError{"no PTR record was found in the response"} + if len(rtn.Data) == 0 { + return nil, &dnsError{"no " + qt.String() + " resource records were found in the response"} + } + + return &rtn, nil } - if rtnErr != nil { - return nil, rtnErr + if nameserverErr != nil { + return nil, nameserverErr } // This should never get here. - panic("LookupPTR should have returned a response.") + panic("dns resolver Lookup() should have returned a response.") } func (res *MiekgResolver) getOrCreateNameserverStats(ns string) *nameserverStats { @@ -212,13 +240,20 @@ func (res *MiekgResolver) getOrCreateNameserverStats(ns string) *nameserverStats // Create stats for the nameserver. reg := res.registry.NewRegistry(strings.Replace(ns, ".", "_", -1)) stats = &nameserverStats{ - success: monitoring.NewInt(reg, "success"), - failure: monitoring.NewInt(reg, "failure"), - ptrResponse: metrics.NewUniformSample(1028), + success: monitoring.NewInt(reg, "success"), + failure: monitoring.NewInt(reg, "failure"), + responseTime: metrics.NewUniformSample(1028), } - adapter.NewGoMetrics(reg, "response.ptr", adapter.Accept). - Register("histogram", metrics.NewHistogram(stats.ptrResponse)) + adapter.NewGoMetrics(reg, "request_duration", adapter.Accept). + Register("histogram", metrics.NewHistogram(stats.responseTime)) res.nsStats[ns] = stats return stats } + +func min[T constraints.Ordered](a, b T) T { + if a < b { + return a + } + return b +} diff --git a/libbeat/processors/dns/resolver_test.go b/libbeat/processors/dns/resolver_test.go index 1e2e56b86282..21c597a0ae82 100644 --- a/libbeat/processors/dns/resolver_test.go +++ b/libbeat/processors/dns/resolver_test.go @@ -29,7 +29,7 @@ import ( "github.com/elastic/elastic-agent-libs/monitoring" ) -var _ PTRResolver = (*MiekgResolver)(nil) +var _ resolver = (*MiekgResolver)(nil) func TestMiekgResolverLookupPTR(t *testing.T) { stop, addr, err := ServeDNS(FakeDNSHandler) @@ -45,15 +45,15 @@ func TestMiekgResolverLookupPTR(t *testing.T) { } // Success - ptr, err := res.LookupPTR("8.8.8.8") + ptr, err := res.Lookup("8.8.8.8", typePTR) if err != nil { t.Fatal(err) } - assert.EqualValues(t, "google-public-dns-a.google.com", ptr.Host) + assert.EqualValues(t, "google-public-dns-a.google.com", ptr.Data[0]) assert.EqualValues(t, 19273, ptr.TTL) // NXDOMAIN - _, err = res.LookupPTR("1.1.1.1") + _, err = res.Lookup("1.1.1.1", typePTR) if assert.Error(t, err) { assert.Contains(t, err.Error(), "NXDOMAIN") } @@ -91,21 +91,21 @@ func TestMiekgResolverLookupPTRTLS(t *testing.T) { if err != nil { t.Fatal(err) } - // we use a self signed certificate for localhost + // we use a self-signed certificate for localhost // we have to pass InsecureSSL to the DNS resolver res.client.TLSConfig = &tls.Config{ InsecureSkipVerify: true, } // Success - ptr, err := res.LookupPTR("8.8.8.8") + ptr, err := res.Lookup("8.8.8.8", typePTR) if err != nil { t.Fatal(err) } - assert.EqualValues(t, "google-public-dns-a.google.com", ptr.Host) + assert.EqualValues(t, "google-public-dns-a.google.com", ptr.Data[0]) assert.EqualValues(t, 19273, ptr.TTL) // NXDOMAIN - _, err = res.LookupPTR("1.1.1.1") + _, err = res.Lookup("1.1.1.1", typePTR) if assert.Error(t, err) { assert.Contains(t, err.Error(), "NXDOMAIN") }