[Ruler][DNS] Don't propagate no such host error if using default resolver

CHANGELOG.md

@@ -11,7 +11,10 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re
 
 ## Unreleased
 
+
+### Fixed
 - [#3261](https://github.com/thanos-io/thanos/pull/3261) Thanos Store: Use segment files specified in meta.json file, if present. If not present, Store does the LIST operation as before.
+- [#3257](https://github.com/thanos-io/thanos/pull/3257) Ruler: Prevent Ruler from crashing when using default DNS to lookup hosts that results in "No such hosts" errors.
 
 ## [v0.16.0](https://github.com/thanos-io/thanos/releases) - Release in progress
 

pkg/discovery/dns/provider.go

@@ -55,9 +55,10 @@ func (t ResolverType) ToResolver(logger log.Logger) ipLookupResolver {
 
 // NewProvider returns a new empty provider with a given resolver type.
 // If empty resolver type is net.DefaultResolver.
+// TODO(OGKevin): Refactor code to use new method and eventually rename NewProviderWithReturnOnErrorIfNotFound back to NewProvider.
 func NewProvider(logger log.Logger, reg prometheus.Registerer, resolverType ResolverType) *Provider {
 	p := &Provider{
-		resolver: NewResolver(resolverType.ToResolver(logger)),
+		resolver: NewResolver(resolverType.ToResolver(logger), logger),
 		resolved: make(map[string][]string),
 		logger:   logger,
 		resolverAddrs: extprom.NewTxGaugeVec(reg, prometheus.GaugeOpts{

pkg/discovery/dns/resolver.go

@@ -9,6 +9,9 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/go-kit/kit/log"
+	"github.com/go-kit/kit/log/level"
+
 	"github.com/pkg/errors"
 )
 
@@ -38,11 +41,12 @@ type ipLookupResolver interface {
 
 type dnsSD struct {
 	resolver ipLookupResolver
+	logger   log.Logger
 }
 
 // NewResolver creates a resolver with given underlying resolver.
-func NewResolver(resolver ipLookupResolver) Resolver {
-	return &dnsSD{resolver: resolver}
+func NewResolver(resolver ipLookupResolver, logger log.Logger) Resolver {
+	return &dnsSD{resolver: resolver, logger: logger}
 }
 
 func (s *dnsSD) Resolve(ctx context.Context, name string, qtype QType) ([]string, error) {
@@ -71,7 +75,15 @@ func (s *dnsSD) Resolve(ctx context.Context, name string, qtype QType) ([]string
 		}
 		ips, err := s.resolver.LookupIPAddr(ctx, host)
 		if err != nil {
-			return nil, errors.Wrapf(err, "lookup IP addresses %q", host)
+			// We exclude error from std Golang resolver for the case of the domain (e.g `NXDOMAIN`) not being found by DNS
+			// server. Since `miekg` does not consider this as an error,  when the host cannot be found, empty slice will be
+			// returned.
+			if dnsErr, ok := err.(*net.DNSError); !ok || !dnsErr.IsNotFound {
+				return nil, errors.Wrapf(err, "lookup IP addresses %q", host)
+			}
+			if ips == nil {
+				level.Error(s.logger).Log("msg", "failed to lookup IP addresses", "host", host, "err", err)
+			}
 		}
 		for _, ip := range ips {
 			res = append(res, appendScheme(scheme, net.JoinHostPort(ip.String(), port)))
@@ -106,6 +118,10 @@ func (s *dnsSD) Resolve(ctx context.Context, name string, qtype QType) ([]string
 		return nil, errors.Errorf("invalid lookup scheme %q", qtype)
 	}
 
+	if res == nil && err == nil {
+		level.Warn(s.logger).Log("msg", "IP address lookup yielded no results. No host found or no addresses found", "host", host)
+	}
+
 	return res, nil
 }
 

pkg/discovery/dns/resolver_test.go

@@ -9,6 +9,8 @@ import (
 	"sort"
 	"testing"
 
+	"github.com/go-kit/kit/log"
+
 	"github.com/pkg/errors"
 	"github.com/thanos-io/thanos/pkg/testutil"
 )
@@ -184,7 +186,7 @@ func TestDnsSD_Resolve(t *testing.T) {
 
 func testDnsSd(t *testing.T, tt DNSSDTest) {
 	ctx := context.TODO()
-	dnsSD := dnsSD{tt.resolver}
+	dnsSD := dnsSD{tt.resolver, log.NewNopLogger()}
 
 	result, err := dnsSD.Resolve(ctx, tt.addr, tt.qtype)
 	if tt.expectedErr != nil {