roachtest: mixedversion test steps are annotated in grafana

Now that we have the ability to create grafana annotations
in roachtests, we can annotate each mixed version step
in grafana.

build/bazelutil/distdir_files.bzl

@@ -1046,7 +1046,7 @@ DISTDIR_FILES = {
     "https://storage.googleapis.com/cockroach-godeps/gomod/go.etcd.io/etcd/client/pkg/v3/io_etcd_go_etcd_client_pkg_v3-v3.5.0.zip": "c0ca209767c5734c6ed023888ba5be02aab5bd3c4d018999467f2bfa8bf65ee3",
     "https://storage.googleapis.com/cockroach-godeps/gomod/go.etcd.io/etcd/client/v2/io_etcd_go_etcd_client_v2-v2.305.0.zip": "91fcb507fe8c193844b56bfb6c8741aaeb6ffa11ee9043de2af0f141173679f3",
     "https://storage.googleapis.com/cockroach-godeps/gomod/go.etcd.io/etcd/io_etcd_go_etcd-v0.5.0-alpha.5.0.20200910180754-dd1b699fc489.zip": "d982ee501979b41b68625693bad77d15e4ae79ab9d0eae5f6028205f96a74e49",
-    "https://storage.googleapis.com/cockroach-godeps/gomod/go.mongodb.org/mongo-driver/org_mongodb_go_mongo_driver-v1.5.1.zip": "446cff132e82c64af7ffcf48e268eb16ec81f694914aa6baecb06cbbae1be0d7",
+    "https://storage.googleapis.com/cockroach-godeps/gomod/go.mongodb.org/mongo-driver/org_mongodb_go_mongo_driver-v1.13.1.zip": "72d6d482c70104374d8d5ac91653b46aec4c7c1e610e0fd4a82d5d88b4a65b3e",
     "https://storage.googleapis.com/cockroach-godeps/gomod/go.mozilla.org/pkcs7/org_mozilla_go_pkcs7-v0.0.0-20200128120323-432b2356ecb1.zip": "3c4c1667907ff3127e371d44696326bad9e965216d4257917ae28e8b82a9e08d",
     "https://storage.googleapis.com/cockroach-godeps/gomod/go.opencensus.io/io_opencensus_go-v0.24.0.zip": "203a767d7f8e7c1ebe5588220ad168d1e15b14ae70a636de7ca9a4a88a7e0d0c",
     "https://storage.googleapis.com/cockroach-godeps/gomod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/io_opentelemetry_go_contrib_instrumentation_google_golang_org_grpc_otelgrpc-v0.28.0.zip": "056ab084a48e44c5fb5ba4e1970880570de71eac6133d23e4b55d009e71be2c3",

go.mod

@@ -230,7 +230,7 @@ require (
 	github.com/xdg-go/stringprep v1.0.4
 	github.com/zabawaba99/go-gitignore v0.0.0-20200117185801-39e6bddfb292
 	gitlab.com/golang-commonmark/markdown v0.0.0-20211110145824-bf3e522c626a
-	go.etcd.io/raft/v3 v3.0.0-20240110140044-f1c02c9909c0
+	go.opentelemetry.io/otel v1.17.0
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.3.0
 	go.opentelemetry.io/otel/exporters/zipkin v1.0.0-RC3
 	go.opentelemetry.io/otel/sdk v1.17.0

go.sum

@@ -2393,7 +2393,6 @@ go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3C
 go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
 go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
 go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ=
-go.etcd.io/raft/v3 v3.0.0-20240110140044-f1c02c9909c0/go.mod h1:QpxpKeYmocQQFHP75LxNrdJTukZmqQig9lotwYLsUJY=
 go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
 go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
 go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=

pkg/cmd/roachtest/roachtestutil/mixedversion/BUILD.bazel

@@ -13,6 +13,7 @@ go_library(
     importpath = "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/roachtestutil/mixedversion",
     visibility = ["//visibility:public"],
     deps = [
+        "//pkg/cmd/roachprod/grafana",
         "//pkg/cmd/roachtest/cluster",
         "//pkg/cmd/roachtest/option",
         "//pkg/cmd/roachtest/roachtestutil",

pkg/cmd/roachtest/roachtestutil/mixedversion/runner.go

@@ -26,6 +26,7 @@ import (
 	"text/tabwriter"
 	"time"
 
+	"github.com/cockroachdb/cockroach/pkg/cmd/roachprod/grafana"
 	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster"
 	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option"
 	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/roachtestutil/clusterupgrade"
@@ -103,6 +104,11 @@ type (
 			mu    syncutil.Mutex
 			cache []*gosql.DB
 		}
+
+		// the following are test-only fields, allowing tests to simulate
+		// cluster properties without passing a cluster.Cluster
+		// implementation.
+		_addAnnotation func() error
 	}
 )
 
@@ -242,6 +248,13 @@ func (tr *testRunner) runSingleStep(ctx context.Context, ss *singleStep, l *logg
 	defer func() {
 		prefix := fmt.Sprintf("FINISHED [%s]", timeutil.Since(start))
 		tr.logStep(prefix, ss, l)
+		annotation := fmt.Sprintf("(%d): %s", ss.ID, ss.impl.Description())
+		err := tr.addGrafanaAnnotation(tr.ctx, tr.logger, grafana.AddAnnotationRequest{
+			Text: annotation, StartTime: start.UnixMilli(), EndTime: timeutil.Now().UnixMilli(),
+		})
+		if err != nil {
+			l.Printf("WARN: Adding Grafana annotation failed: %s", err)
+		}
 	}()
 
 	if err := panicAsError(l, func() error {
@@ -505,6 +518,16 @@ func (tr *testRunner) closeConnections() {
 	}
 }
 
+func (tr *testRunner) addGrafanaAnnotation(
+	ctx context.Context, l *logger.Logger, req grafana.AddAnnotationRequest,
+) error {
+	if tr._addAnnotation != nil {
+		return tr._addAnnotation() // test-only
+	}
+
+	return tr.cluster.AddGrafanaAnnotation(ctx, l, req)
+}
+
 func newCRDBMonitor(
 	ctx context.Context, c cluster.Cluster, crdbNodes option.NodeListOption,
 ) *crdbMonitor {

pkg/cmd/roachtest/roachtestutil/mixedversion/runner_test.go

@@ -54,15 +54,20 @@ func Test_runSingleStep(t *testing.T) {
 	require.Contains(t, err.Error(), "panic (stack trace above): runtime error: index out of range [0] with length 0")
 }
 
+func testAddAnnotation() error {
+	return nil
+}
+
 func testTestRunner() *testRunner {
 	runnerCtx, cancel := context.WithCancel(ctx)
 	return &testRunner{
-		ctx:        runnerCtx,
-		cancel:     cancel,
-		logger:     nilLogger,
-		crdbNodes:  nodes,
-		background: newBackgroundRunner(runnerCtx, nilLogger),
-		seed:       seed,
+		ctx:            runnerCtx,
+		cancel:         cancel,
+		logger:         nilLogger,
+		crdbNodes:      nodes,
+		background:     newBackgroundRunner(runnerCtx, nilLogger),
+		seed:           seed,
+		_addAnnotation: testAddAnnotation,
 	}
 }