diff --git a/cmd/thanos/query.go b/cmd/thanos/query.go index 4ce74da4955..374265db16f 100644 --- a/cmd/thanos/query.go +++ b/cmd/thanos/query.go @@ -265,6 +265,7 @@ func runQuery( logger, extprom.WrapRegistererWithPrefix("thanos_querier_store_apis_", reg), dns.ResolverType(dnsSDResolver), + true, ) for _, store := range strictStores { @@ -277,6 +278,7 @@ func runQuery( logger, extprom.WrapRegistererWithPrefix("thanos_querier_rule_apis_", reg), dns.ResolverType(dnsSDResolver), + true, ) var ( diff --git a/cmd/thanos/rule.go b/cmd/thanos/rule.go index f72bbedfd22..17079cb9436 100644 --- a/cmd/thanos/rule.go +++ b/cmd/thanos/rule.go @@ -338,6 +338,7 @@ func runRule( logger, extprom.WrapRegistererWithPrefix("thanos_ruler_query_apis_", reg), dns.ResolverType(dnsSDResolver), + true, ) var queryClients []*http_util.Client for _, cfg := range queryCfg { @@ -401,6 +402,7 @@ func runRule( logger, extprom.WrapRegistererWithPrefix("thanos_ruler_alertmanagers_", reg), dns.ResolverType(dnsSDResolver), + false, ) var alertmgrs []*alert.Alertmanager for _, cfg := range alertingCfg.Alertmanagers { diff --git a/pkg/cacheutil/memcached_client.go b/pkg/cacheutil/memcached_client.go index 15751109dc8..3becb31e79a 100644 --- a/pkg/cacheutil/memcached_client.go +++ b/pkg/cacheutil/memcached_client.go @@ -219,6 +219,7 @@ func newMemcachedClient( logger, extprom.WrapRegistererWithPrefix("thanos_memcached_", reg), dns.GolangResolverType, + true, ) c := &memcachedClient{ diff --git a/pkg/discovery/dns/provider.go b/pkg/discovery/dns/provider.go index 4df7c07f054..11867345ac0 100644 --- a/pkg/discovery/dns/provider.go +++ b/pkg/discovery/dns/provider.go @@ -37,7 +37,6 @@ type ResolverType string const ( GolangResolverType ResolverType = "golang" MiekgdnsResolverType ResolverType = "miekgdns" - MockdnsResolverType ResolverType = "mockdns" ) func (t ResolverType) ToResolver(logger log.Logger) ipLookupResolver { @@ -56,9 +55,9 @@ func (t ResolverType) ToResolver(logger log.Logger) ipLookupResolver { // NewProvider returns a new empty provider with a given resolver type. // If empty resolver type is net.DefaultResolver. -func NewProvider(logger log.Logger, reg prometheus.Registerer, resolverType ResolverType) *Provider { +func NewProvider(logger log.Logger, reg prometheus.Registerer, resolverType ResolverType, returnErrOnNotFound bool) *Provider { p := &Provider{ - resolver: NewResolver(resolverType.ToResolver(logger), resolverType), + resolver: NewResolver(resolverType.ToResolver(logger), logger, returnErrOnNotFound), resolved: make(map[string][]string), logger: logger, resolverAddrs: extprom.NewTxGaugeVec(reg, prometheus.GaugeOpts{ diff --git a/pkg/discovery/dns/provider_test.go b/pkg/discovery/dns/provider_test.go index 668d5a65c84..87277c3d222 100644 --- a/pkg/discovery/dns/provider_test.go +++ b/pkg/discovery/dns/provider_test.go @@ -22,7 +22,7 @@ func TestProvider(t *testing.T) { "127.0.0.5:19095", } - prv := NewProvider(log.NewNopLogger(), nil, "") + prv := NewProvider(log.NewNopLogger(), nil, "", true) prv.resolver = &mockResolver{ res: map[string][]string{ "a": ips[:2], diff --git a/pkg/discovery/dns/resolver.go b/pkg/discovery/dns/resolver.go index d9be8978290..cff1a5f940a 100644 --- a/pkg/discovery/dns/resolver.go +++ b/pkg/discovery/dns/resolver.go @@ -5,6 +5,8 @@ package dns import ( "context" + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" "net" "strconv" "strings" @@ -38,12 +40,15 @@ type ipLookupResolver interface { type dnsSD struct { resolver ipLookupResolver - resolverType ResolverType + logger log.Logger + // https://github.com/thanos-io/thanos/issues/3186 + // This flag is used to prevent components from crashing if hosts are not found. + returnErrOnNotFound bool } // NewResolver creates a resolver with given underlying resolver. -func NewResolver(resolver ipLookupResolver, resolverType ResolverType) Resolver { - return &dnsSD{resolver: resolver, resolverType: resolverType} +func NewResolver(resolver ipLookupResolver, logger log.Logger, returnErrOnNotFound bool) Resolver { + return &dnsSD{resolver: resolver, logger: logger, returnErrOnNotFound: returnErrOnNotFound} } func (s *dnsSD) Resolve(ctx context.Context, name string, qtype QType) ([]string, error) { @@ -73,9 +78,13 @@ func (s *dnsSD) Resolve(ctx context.Context, name string, qtype QType) ([]string ips, err := s.resolver.LookupIPAddr(ctx, host) if err != nil { dnsErr, ok := err.(*net.DNSError) - if !(s.resolverType == GolangResolverType && ok && dnsErr.IsNotFound) { + // https://github.com/thanos-io/thanos/issues/3186 + // Default DNS resolver can make thanos components crash if DSN resolutions results in EAI_NONAME. + // the flag returnErrOnNotFound can be used to prevent such crash. + if !(!s.returnErrOnNotFound && ok && dnsErr.IsNotFound) { return nil, errors.Wrapf(err, "lookup IP addresses %q", host) } + level.Error(s.logger).Log("msg", "failed to lookup IP addresses", "host", host, "err", err) } for _, ip := range ips { res = append(res, appendScheme(scheme, net.JoinHostPort(ip.String(), port))) @@ -110,6 +119,12 @@ func (s *dnsSD) Resolve(ctx context.Context, name string, qtype QType) ([]string return nil, errors.Errorf("invalid lookup scheme %q", qtype) } + // https://github.com/thanos-io/thanos/issues/3186 + // This happens when miekg is used as resolver. When the host cannot be found, nothing is returned. + if res == nil && err == nil { + level.Warn(s.logger).Log("msg", "IP address lookup yielded no results nor errors", "host", host) + } + return res, nil } diff --git a/pkg/discovery/dns/resolver_test.go b/pkg/discovery/dns/resolver_test.go index b6635c8fe3a..3034496c686 100644 --- a/pkg/discovery/dns/resolver_test.go +++ b/pkg/discovery/dns/resolver_test.go @@ -5,6 +5,7 @@ package dns import ( "context" + "github.com/go-kit/kit/log" "net" "sort" "testing" @@ -184,7 +185,7 @@ func TestDnsSD_Resolve(t *testing.T) { func testDnsSd(t *testing.T, tt DNSSDTest) { ctx := context.TODO() - dnsSD := dnsSD{tt.resolver, MockdnsResolverType} + dnsSD := dnsSD{tt.resolver, log.NewNopLogger(), false} result, err := dnsSD.Resolve(ctx, tt.addr, tt.qtype) if tt.expectedErr != nil {