[aggregator] Fix metrics registration issues

DEVELOPER.md

@@ -46,7 +46,7 @@ make m3dbnode
 
 ## Running the M3 stack locally
 
-Follow the instructions in `./scripts/development/m3_stack/README.md`
+Follow the instructions in [this README](./scripts/development/m3_stack/README.md).
 
 ## Testing Changes
 

src/aggregator/aggregator/forwarded_writer.go

@@ -91,32 +91,38 @@ type forwardedWriterMetrics struct {
 	unregisterAggregationNotFound tally.Counter
 	prepare                       tally.Counter
 	flushSuccess                  tally.Counter
-	flushErrors                   tally.Counter
+	flushErrorsClient             tally.Counter
 }
 
 func newForwardedWriterMetrics(scope tally.Scope) forwardedWriterMetrics {
+	const (
+		errorsName = "errors"
+		reasonTag  = "reason"
+	)
 	registerScope := scope.Tagged(map[string]string{"action": "register"})
 	unregisterScope := scope.Tagged(map[string]string{"action": "unregister"})
 	prepareScope := scope.Tagged(map[string]string{"action": "prepare"})
 	flushScope := scope.Tagged(map[string]string{"action": "flush"})
 	return forwardedWriterMetrics{
 		registerSuccess: registerScope.Counter("success"),
 		registerWriterClosed: registerScope.Tagged(map[string]string{
-			"reason": "writer-closed",
-		}).Counter("errors"),
+			reasonTag: "writer-closed",
+		}).Counter(errorsName),
 		unregisterSuccess: unregisterScope.Counter("success"),
 		unregisterWriterClosed: unregisterScope.Tagged(map[string]string{
-			"reason": "writer-closed",
-		}).Counter("errors"),
+			reasonTag: "writer-closed",
+		}).Counter(errorsName),
 		unregisterMetricNotFound: unregisterScope.Tagged(map[string]string{
-			"reason": "metric-not-found",
-		}).Counter("errors"),
+			reasonTag: "metric-not-found",
+		}).Counter(errorsName),
 		unregisterAggregationNotFound: unregisterScope.Tagged(map[string]string{
-			"reason": "aggregation-not-found",
-		}).Counter("errors"),
+			reasonTag: "aggregation-not-found",
+		}).Counter(errorsName),
 		prepare:      prepareScope.Counter("prepare"),
 		flushSuccess: flushScope.Counter("success"),
-		flushErrors:  flushScope.Counter("errors"),
+		flushErrorsClient: flushScope.Tagged(map[string]string{
+			reasonTag: "client-flush-error",
+		}).Counter(errorsName),
 	}
 }
 
@@ -214,7 +220,7 @@ func (w *forwardedWriter) Prepare() {
 
 func (w *forwardedWriter) Flush() error {
 	if err := w.client.Flush(); err != nil {
-		w.metrics.flushErrors.Inc(1)
+		w.metrics.flushErrorsClient.Inc(1)
 		return err
 	}
 	w.metrics.flushSuccess.Inc(1)

src/cmd/services/m3aggregator/config/aggregator.go

@@ -653,7 +653,7 @@ func (c flushTimesManagerConfiguration) NewFlushTimesManager(
 		return nil, err
 	}
 	scope := instrumentOpts.MetricsScope()
-	retrier := c.FlushTimesPersistRetrier.NewRetrier(scope.SubScope("flush-times-persist"))
+	retrier := c.FlushTimesPersistRetrier.NewRetrier(scope.SubScope("flush-times-persist-retrier"))
 	flushTimesManagerOpts := aggregator.NewFlushTimesManagerOptions().
 		SetInstrumentOptions(instrumentOpts).
 		SetFlushTimesKeyFmt(c.FlushTimesKeyFmt).
@@ -707,9 +707,9 @@ func (c electionManagerConfiguration) NewElectionManager(
 	}
 	campaignOpts = campaignOpts.SetLeaderValue(leaderValue)
 	scope := instrumentOpts.MetricsScope()
-	campaignRetryOpts := c.CampaignRetrier.NewOptions(scope.SubScope("campaign"))
-	changeRetryOpts := c.ChangeRetrier.NewOptions(scope.SubScope("change"))
-	resignRetryOpts := c.ResignRetrier.NewOptions(scope.SubScope("resign"))
+	campaignRetryOpts := c.CampaignRetrier.NewOptions(scope.SubScope("campaign-retrier"))
+	changeRetryOpts := c.ChangeRetrier.NewOptions(scope.SubScope("change-retrier"))
+	resignRetryOpts := c.ResignRetrier.NewOptions(scope.SubScope("resign-retrier"))
 	opts := aggregator.NewElectionManagerOptions().
 		SetInstrumentOptions(instrumentOpts).
 		SetElectionOptions(electionOpts).

src/msg/producer/writer/writer.go

@@ -53,7 +53,7 @@ func newWriterMetrics(scope tally.Scope) writerMetrics {
 		topicUpdateError:   scope.Counter("topic-update-error"),
 		invalidTopicUpdate: scope.Counter("invalid-topic"),
 		invalidShard: scope.Tagged(map[string]string{"reason": "invalid-shard"}).
-			Counter("invalid-write"),
+			Counter("invalid-shard-write"),
 		numConsumerServices: scope.Gauge("num-consumer-services"),
 	}
 }