Add flag to ensure new behaviour only impacts ruler AM resolution.

Signed-off-by: Kevin Hellemun <[email protected]>

cmd/thanos/query.go

@@ -265,6 +265,7 @@ func runQuery(
 		logger,
 		extprom.WrapRegistererWithPrefix("thanos_querier_store_apis_", reg),
 		dns.ResolverType(dnsSDResolver),
+		true,
 	)
 
 	for _, store := range strictStores {
@@ -277,6 +278,7 @@ func runQuery(
 		logger,
 		extprom.WrapRegistererWithPrefix("thanos_querier_rule_apis_", reg),
 		dns.ResolverType(dnsSDResolver),
+		true,
 	)
 
 	var (

cmd/thanos/rule.go

@@ -338,6 +338,7 @@ func runRule(
 		logger,
 		extprom.WrapRegistererWithPrefix("thanos_ruler_query_apis_", reg),
 		dns.ResolverType(dnsSDResolver),
+		true,
 	)
 	var queryClients []*http_util.Client
 	for _, cfg := range queryCfg {
@@ -401,6 +402,7 @@ func runRule(
 		logger,
 		extprom.WrapRegistererWithPrefix("thanos_ruler_alertmanagers_", reg),
 		dns.ResolverType(dnsSDResolver),
+		false,
 	)
 	var alertmgrs []*alert.Alertmanager
 	for _, cfg := range alertingCfg.Alertmanagers {

pkg/cacheutil/memcached_client.go

@@ -219,6 +219,7 @@ func newMemcachedClient(
 		logger,
 		extprom.WrapRegistererWithPrefix("thanos_memcached_", reg),
 		dns.GolangResolverType,
+		true,
 	)
 
 	c := &memcachedClient{

pkg/discovery/dns/provider.go

@@ -37,7 +37,6 @@ type ResolverType string
 const (
 	GolangResolverType   ResolverType = "golang"
 	MiekgdnsResolverType ResolverType = "miekgdns"
-	MockdnsResolverType  ResolverType = "mockdns"
 )
 
 func (t ResolverType) ToResolver(logger log.Logger) ipLookupResolver {
@@ -56,9 +55,9 @@ func (t ResolverType) ToResolver(logger log.Logger) ipLookupResolver {
 
 // NewProvider returns a new empty provider with a given resolver type.
 // If empty resolver type is net.DefaultResolver.
-func NewProvider(logger log.Logger, reg prometheus.Registerer, resolverType ResolverType) *Provider {
+func NewProvider(logger log.Logger, reg prometheus.Registerer, resolverType ResolverType, returnErrOnNotFound bool) *Provider {
 	p := &Provider{
-		resolver: NewResolver(resolverType.ToResolver(logger), resolverType),
+		resolver: NewResolver(resolverType.ToResolver(logger), logger, returnErrOnNotFound),
 		resolved: make(map[string][]string),
 		logger:   logger,
 		resolverAddrs: extprom.NewTxGaugeVec(reg, prometheus.GaugeOpts{

pkg/discovery/dns/provider_test.go

@@ -22,7 +22,7 @@ func TestProvider(t *testing.T) {
 		"127.0.0.5:19095",
 	}
 
-	prv := NewProvider(log.NewNopLogger(), nil, "")
+	prv := NewProvider(log.NewNopLogger(), nil, "", true)
 	prv.resolver = &mockResolver{
 		res: map[string][]string{
 			"a": ips[:2],

pkg/discovery/dns/resolver.go

@@ -5,6 +5,8 @@ package dns
 
 import (
 	"context"
+	"github.com/go-kit/kit/log"
+	"github.com/go-kit/kit/log/level"
 	"net"
 	"strconv"
 	"strings"
@@ -38,12 +40,15 @@ type ipLookupResolver interface {
 
 type dnsSD struct {
 	resolver     ipLookupResolver
-	resolverType ResolverType
+	logger log.Logger
+	// https://github.com/thanos-io/thanos/issues/3186
+	// This flag is used to prevent components from crashing if hosts are not found.
+	returnErrOnNotFound bool
 }
 
 // NewResolver creates a resolver with given underlying resolver.
-func NewResolver(resolver ipLookupResolver, resolverType ResolverType) Resolver {
-	return &dnsSD{resolver: resolver, resolverType: resolverType}
+func NewResolver(resolver ipLookupResolver, logger log.Logger, returnErrOnNotFound bool) Resolver {
+	return &dnsSD{resolver: resolver, logger: logger, returnErrOnNotFound: returnErrOnNotFound}
 }
 
 func (s *dnsSD) Resolve(ctx context.Context, name string, qtype QType) ([]string, error) {
@@ -73,9 +78,13 @@ func (s *dnsSD) Resolve(ctx context.Context, name string, qtype QType) ([]string
 		ips, err := s.resolver.LookupIPAddr(ctx, host)
 		if err != nil {
 			dnsErr, ok := err.(*net.DNSError)
-			if !(s.resolverType == GolangResolverType && ok && dnsErr.IsNotFound) {
+			// https://github.com/thanos-io/thanos/issues/3186
+			// Default DNS resolver can make thanos components crash if DSN resolutions results in EAI_NONAME.
+			// the flag returnErrOnNotFound can be used to prevent such crash.
+			if !(!s.returnErrOnNotFound && ok && dnsErr.IsNotFound) {
 				return nil, errors.Wrapf(err, "lookup IP addresses %q", host)
 			}
+			level.Error(s.logger).Log("msg", "failed to lookup IP addresses", "host", host, "err", err)
 		}
 		for _, ip := range ips {
 			res = append(res, appendScheme(scheme, net.JoinHostPort(ip.String(), port)))
@@ -110,6 +119,12 @@ func (s *dnsSD) Resolve(ctx context.Context, name string, qtype QType) ([]string
 		return nil, errors.Errorf("invalid lookup scheme %q", qtype)
 	}
 
+	// https://github.com/thanos-io/thanos/issues/3186
+	// This happens when miekg is used as resolver. When the host cannot be found, nothing is returned.
+	if res == nil && err == nil {
+		level.Warn(s.logger).Log("msg", "IP address lookup yielded no results nor errors", "host", host)
+	}
+
 	return res, nil
 }
 

pkg/discovery/dns/resolver_test.go

@@ -5,6 +5,7 @@ package dns
 
 import (
 	"context"
+	"github.com/go-kit/kit/log"
 	"net"
 	"sort"
 	"testing"
@@ -184,7 +185,7 @@ func TestDnsSD_Resolve(t *testing.T) {
 
 func testDnsSd(t *testing.T, tt DNSSDTest) {
 	ctx := context.TODO()
-	dnsSD := dnsSD{tt.resolver, MockdnsResolverType}
+	dnsSD := dnsSD{tt.resolver, log.NewNopLogger(), false}
 
 	result, err := dnsSD.Resolve(ctx, tt.addr, tt.qtype)
 	if tt.expectedErr != nil {