Adding a metric for hosts not being found in resolver

cmd/server/cadence/server.go

@@ -182,6 +182,7 @@ func (s *server) startService() common.Daemon {
 	params.MembershipResolver, err = membership.NewResolver(
 		peerProvider,
 		params.Logger,
+		params.MetricsClient,
 	)
 	if err != nil {
 		log.Fatalf("error creating membership monitor: %v", err)

common/membership/resolver.go

@@ -31,6 +31,7 @@ import (
 	"github.com/uber/cadence/common"
 	"github.com/uber/cadence/common/log"
 	"github.com/uber/cadence/common/log/tag"
+	"github.com/uber/cadence/common/metrics"
 	"github.com/uber/cadence/common/service"
 )
 
@@ -80,7 +81,8 @@ type (
 
 // MultiringResolver uses ring-per-service for membership information
 type MultiringResolver struct {
-	status int32
+	metrics metrics.Client
+	status  int32
 
 	provider PeerProvider
 	rings    map[string]*ring
@@ -92,20 +94,23 @@ var _ Resolver = (*MultiringResolver)(nil)
 func NewResolver(
 	provider PeerProvider,
 	logger log.Logger,
+	metrics metrics.Client,
 ) (*MultiringResolver, error) {
-	return NewMultiringResolver(service.List, provider, logger.WithTags(tag.ComponentServiceResolver)), nil
+	return NewMultiringResolver(service.List, provider, logger.WithTags(tag.ComponentServiceResolver), metrics), nil
 }
 
 // NewMultiringResolver creates hashrings for all services
 func NewMultiringResolver(
 	services []string,
 	provider PeerProvider,
 	logger log.Logger,
+	metrics metrics.Client,
 ) *MultiringResolver {
 	rpo := &MultiringResolver{
 		status:   common.DaemonStatusInitialized,
 		provider: provider,
 		rings:    make(map[string]*ring),
+		metrics:  metrics,
 	}
 
 	for _, s := range services {
@@ -208,6 +213,7 @@ func (rpo *MultiringResolver) LookupByAddress(service, address string) (HostInfo
 			return m, nil
 		}
 	}
+	rpo.metrics.Scope(metrics.ResolverHostNotFoundScope).IncCounter(1)
 	return HostInfo{}, errors.New("host not found")
 }
 

common/membership/resolver_test.go

@@ -30,6 +30,7 @@ import (
 
 	"github.com/uber/cadence/common"
 	"github.com/uber/cadence/common/log"
+	"github.com/uber/cadence/common/metrics"
 )
 
 var testServices = []string{"test-worker", "test-services"}
@@ -136,5 +137,6 @@ func newTestResolver(t *testing.T) (*MultiringResolver, *MockPeerProvider) {
 		testServices,
 		pp,
 		log.NewNoop(),
+		metrics.NewNoopMetricsClient(),
 	), pp
 }

common/metrics/defs.go

@@ -305,6 +305,9 @@ const (
 	PersistenceUpdateDynamicConfigScope
 	// PersistenceShardRequestCountScope tracks number of persistence calls made to each shard
 	PersistenceShardRequestCountScope
+
+	// ResolverHostNotFoundScope is a simple low level error indicating a lookup failed in the membership resolver
+	ResolverHostNotFoundScope
 	// HistoryClientStartWorkflowExecutionScope tracks RPC calls to history service
 	HistoryClientStartWorkflowExecutionScope
 	// HistoryClientDescribeHistoryHostScope tracks RPC calls to history service
@@ -1392,6 +1395,7 @@ var ScopeDefs = map[ServiceIdx]map[int]scopeDefinition{
 		PersistenceFetchDynamicConfigScope:                             {operation: "FetchDynamicConfig"},
 		PersistenceUpdateDynamicConfigScope:                            {operation: "UpdateDynamicConfig"},
 		PersistenceShardRequestCountScope:                              {operation: "ShardIdPersistenceRequest"},
+		ResolverHostNotFoundScope:                                      {operation: "ResolverHostNotFound"},
 
 		ClusterMetadataArchivalConfigScope: {operation: "ArchivalConfig"},