Deprecate expvar metrics backend (#4740)

## Which problem is this PR solving?
- Step 1 of #4722

## Description of the changes
- add deprecation notice for `expvar` as metrics backend
- change HotROD default backend to `prometheus`

## How was this change tested?
- run binaries locally to observe the messages
```
$ go run ./cmd/all-in-one help
      --metrics-backend string                                    Defines which metrics backend to use for metrics reporting: prometheus, none, or expvar (deprecated, will be removed after 2024-01-01 or in release v1.53.0, whichever is later)  (default "prometheus")

$ go run ./cmd/all-in-one --metrics-backend=expvar
2023/09/09 19:09:01 using expvar as metrics backend (deprecated, will be removed after 2024-01-01 or in release v1.53.0, whichever is later)
```

---------

Signed-off-by: Yuri Shkuro <[email protected]>

examples/hotrod/cmd/flags.go

@@ -38,9 +38,11 @@ var (
 	jaegerUI string
 )
 
+const expvarDepr = "(deprecated, will be removed after 2024-01-01 or in release v1.53.0, whichever is later) "
+
 // used by root command
 func addFlags(cmd *cobra.Command) {
-	cmd.PersistentFlags().StringVarP(&metricsBackend, "metrics", "m", "expvar", "Metrics backend (expvar|prometheus)")
+	cmd.PersistentFlags().StringVarP(&metricsBackend, "metrics", "m", "prometheus", expvarDepr+"Metrics backend (expvar|prometheus). ")
 	cmd.PersistentFlags().StringVarP(&otelExporter, "otel-exporter", "x", "otlp", "OpenTelemetry exporter (otlp|stdout)")
 
 	cmd.PersistentFlags().DurationVarP(&fixDBConnDelay, "fix-db-query-delay", "D", 300*time.Millisecond, "Average latency of MySQL DB query")

examples/hotrod/cmd/root.go

@@ -73,7 +73,7 @@ func onInitialize() {
 	switch metricsBackend {
 	case "expvar":
 		metricsFactory = expvar.NewFactory(10) // 10 buckets for histograms
-		logger.Info("Using expvar as metrics backend")
+		logger.Info("*** Using expvar as metrics backend " + expvarDepr)
 	case "prometheus":
 		metricsFactory = prometheus.New().Namespace(metrics.NSOptions{Name: "hotrod", Tags: nil})
 		logger.Info("Using Prometheus as metrics backend")

internal/metrics/metricsbuilder/builder.go

@@ -19,6 +19,7 @@ import (
 	"errors"
 	"expvar"
 	"flag"
+	"log"
 	"net/http"
 
 	"github.com/prometheus/client_golang/prometheus"
@@ -46,12 +47,14 @@ type Builder struct {
 	handler   http.Handler
 }
 
+const expvarDepr = "(deprecated, will be removed after 2024-01-01 or in release v1.53.0, whichever is later) "
+
 // AddFlags adds flags for Builder.
 func AddFlags(flags *flag.FlagSet) {
 	flags.String(
 		metricsBackend,
 		defaultMetricsBackend,
-		"Defines which metrics backend to use for metrics reporting: expvar, prometheus, none")
+		"Defines which metrics backend to use for metrics reporting: prometheus, none, or expvar "+expvarDepr)
 	flags.String(
 		metricsHTTPRoute,
 		defaultMetricsRoute,
@@ -77,6 +80,7 @@ func (b *Builder) CreateMetricsFactory(namespace string) (metrics.Factory, error
 	if b.Backend == "expvar" {
 		metricsFactory := jexpvar.NewFactory(10).Namespace(metrics.NSOptions{Name: namespace, Tags: nil})
 		b.handler = expvar.Handler()
+		log.Printf("using expvar as metrics backend " + expvarDepr)
 		return metricsFactory, nil
 	}
 	if b.Backend == "none" || b.Backend == "" {