Merge pull request #3161 from oasisprotocol/ptrus/fix/runtimes-metric

go/registry/metrics: Runtimes metrics: query for runtimes

.changelog/3161.bugfix.md

@@ -0,0 +1,5 @@
+go/registry/metrics: Fix `oasis_registry_runtimes` metric
+
+Metric was counting runtime events, which does not correctly take into account
+the case where runtime is suspended and resumed.
+The metric is now computed by querying the registry.

go/consensus/tendermint/full/full.go

@@ -161,15 +161,14 @@ type fullService struct { // nolint: maligned
 
 	stateDb tmdb.DB
 
-	beacon          beaconAPI.Backend
-	epochtime       epochtimeAPI.Backend
-	keymanager      keymanagerAPI.Backend
-	registry        registryAPI.Backend
-	registryMetrics *registry.MetricsUpdater
-	roothash        roothashAPI.Backend
-	staking         stakingAPI.Backend
-	scheduler       schedulerAPI.Backend
-	submissionMgr   consensusAPI.SubmissionManager
+	beacon        beaconAPI.Backend
+	epochtime     epochtimeAPI.Backend
+	keymanager    keymanagerAPI.Backend
+	registry      registryAPI.Backend
+	roothash      roothashAPI.Backend
+	staking       stakingAPI.Backend
+	scheduler     schedulerAPI.Backend
+	submissionMgr consensusAPI.SubmissionManager
 
 	serviceClients   []api.ServiceClient
 	serviceClientsWg sync.WaitGroup
@@ -229,7 +228,7 @@ func (t *fullService) Start() error {
 		// Start block notifier.
 		go t.blockNotifierWorker()
 		// Optionally start metrics updater.
-		if viper.GetString(cmmetrics.CfgMetricsMode) != cmmetrics.MetricsModeNone {
+		if cmmetrics.Enabled() {
 			go t.metrics()
 		}
 	case false:
@@ -922,10 +921,11 @@ func (t *fullService) initialize() error {
 		return err
 	}
 	t.registry = scRegistry
-	t.registryMetrics = registry.NewMetricsUpdater(t.ctx, t.registry)
+	if cmmetrics.Enabled() {
+		t.svcMgr.RegisterCleanupOnly(registry.NewMetricsUpdater(t.ctx, t.registry), "registry metrics updater")
+	}
 	t.serviceClients = append(t.serviceClients, scRegistry)
 	t.svcMgr.RegisterCleanupOnly(t.registry, "registry backend")
-	t.svcMgr.RegisterCleanupOnly(t.registryMetrics, "registry metrics updater")
 
 	var scStaking tmstaking.ServiceClient
 	if scStaking, err = tmstaking.New(t.ctx, t); err != nil {

go/oasis-node/cmd/common/metrics/metrics.go

@@ -265,6 +265,11 @@ func New(ctx context.Context) (service.BackgroundService, error) {
 	}
 }
 
+// Enabled returns if metrics are enabled.
+func Enabled() bool {
+	return viper.GetString(CfgMetricsMode) != MetricsModeNone
+}
+
 // EscapeLabelCharacters replaces invalid prometheus label name characters with "_".
 func EscapeLabelCharacters(l string) string {
 	return strings.Replace(l, ".", "_", -1)

go/registry/metrics.go

@@ -12,7 +12,7 @@ import (
 	"github.com/oasisprotocol/oasis-core/go/registry/api"
 )
 
-const metricsUpdateInterval = 10 * time.Second
+const metricsUpdateInterval = 60 * time.Second
 
 var (
 	registryNodes = prometheus.NewGauge(
@@ -67,22 +67,10 @@ func (m *MetricsUpdater) worker(ctx context.Context) {
 	t := time.NewTicker(metricsUpdateInterval)
 	defer t.Stop()
 
-	runtimeCh, sub, err := m.backend.WatchRuntimes(ctx)
-	if err != nil {
-		m.logger.Error("failed to watch runtimes, metrics will not be updated",
-			"err", err,
-		)
-		return
-	}
-	defer sub.Close()
-
 	for {
 		select {
 		case <-m.closeCh:
 			return
-		case <-runtimeCh:
-			registryRuntimes.Inc()
-			continue
 		case <-t.C:
 		}
 
@@ -100,6 +88,11 @@ func (m *MetricsUpdater) updatePeriodicMetrics(ctx context.Context) {
 	if err == nil {
 		registryEntities.Set(float64(len(entities)))
 	}
+
+	runtimes, err := m.backend.GetRuntimes(ctx, &api.GetRuntimesQuery{Height: consensus.HeightLatest, IncludeSuspended: false})
+	if err == nil {
+		registryRuntimes.Set(float64(len(runtimes)))
+	}
 }
 
 // NewMetricsUpdater creates a new registry metrics updater.

go/runtime/host/protocol/connection.go

@@ -11,7 +11,6 @@ import (
 	"github.com/opentracing/opentracing-go"
 	opentracingExt "github.com/opentracing/opentracing-go/ext"
 	"github.com/prometheus/client_golang/prometheus"
-	"github.com/spf13/viper"
 
 	"github.com/oasisprotocol/oasis-core/go/common"
 	"github.com/oasisprotocol/oasis-core/go/common/cbor"
@@ -232,7 +231,7 @@ func (c *connection) Call(ctx context.Context, body *Body) (*Body, error) {
 func (c *connection) call(ctx context.Context, body *Body) (result *Body, err error) {
 	start := time.Now()
 	defer func() {
-		if viper.GetString(metrics.CfgMetricsMode) != metrics.MetricsModeNone {
+		if metrics.Enabled() {
 			rhpLatency.With(prometheus.Labels{"call": body.Type()}).Observe(time.Since(start).Seconds())
 			if err != nil {
 				rhpCallFailures.With(prometheus.Labels{"call": body.Type()}).Inc()