integrate with roachtests

Release note (<category, see below>): <what> <show> <why>

pkg/cmd/roachprod/flags.go

@@ -41,6 +41,7 @@ var (
 	wipePreserveCerts     bool
 	grafanaConfig         string
 	grafanaurlOpen        bool
+	grafanaDumpDir        string
 	listDetails           bool
 	listJSON              bool
 	listMine              bool
@@ -242,6 +243,9 @@ func initFlags() {
 	grafanaURLCmd.Flags().BoolVar(&grafanaurlOpen,
 		"open", false, "open the grafana dashboard url on the browser")
 
+	grafanaStopCmd.Flags().StringVar(&grafanaDumpDir, "dump-dir", "",
+		"the absolute path to dump prometheus data to. If empty, no data will get dumped")
+
 	for _, cmd := range []*cobra.Command{createCmd, destroyCmd, extendCmd, logsCmd} {
 		cmd.Flags().StringVarP(&username, "username", "u", os.Getenv("ROACHPROD_USER"),
 			"Username to run under, detect if blank")

pkg/cmd/roachprod/main.go

@@ -893,18 +893,18 @@ and will scrape from all nodes in the cluster`,
 	Args: cobra.ExactArgs(1),
 	Run: wrap(func(cmd *cobra.Command, args []string) error {
 		return roachprod.StartGrafana(context.Background(), roachprodLibraryLogger, args[0],
-			grafanaConfig)
+			grafanaConfig, nil)
 	}),
 }
 
 var grafanaStopCmd = &cobra.Command{
-	Use:   `prom-stop <cluster>`,
+	Use:   `grafana-stop <cluster>`,
 	Short: `spins down prometheus and grafana instances on the last node in the cluster`,
-	Long: fmt.Sprintf(`spins down the prometheus and grafana instances on provided roachprod node and
-dumps the prometheus data into %s`, config.ClustersDir),
+	Long: `spins down the prometheus and grafana instances on provided roachprod node and
+dumps the prometheus data at the root of the last node in the cluster`,
 	Args: cobra.ExactArgs(1),
 	Run: wrap(func(cmd *cobra.Command, args []string) error {
-		return roachprod.StopGrafana(context.Background(), roachprodLibraryLogger, args[0])
+		return roachprod.StopGrafana(context.Background(), roachprodLibraryLogger, args[0], "")
 	}),
 }
 
@@ -921,6 +921,7 @@ var grafanaURLCmd = &cobra.Command{
 		for _, url := range urls {
 			fmt.Println(url)
 		}
+		fmt.Println("username: admin; pwd: admin")
 		return nil
 	}),
 }

pkg/cmd/roachtest/BUILD.bazel

@@ -28,6 +28,7 @@ go_library(
         "//pkg/roachprod/config",
         "//pkg/roachprod/install",
         "//pkg/roachprod/logger",
+        "//pkg/roachprod/prometheus",
         "//pkg/roachprod/vm",
         "//pkg/testutils/skip",
         "//pkg/util/contextutil",

pkg/cmd/roachtest/cluster.go

@@ -41,6 +41,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/roachprod"
 	"github.com/cockroachdb/cockroach/pkg/roachprod/install"
 	"github.com/cockroachdb/cockroach/pkg/roachprod/logger"
+	"github.com/cockroachdb/cockroach/pkg/roachprod/prometheus"
 	"github.com/cockroachdb/cockroach/pkg/roachprod/vm"
 	"github.com/cockroachdb/cockroach/pkg/util/contextutil"
 	"github.com/cockroachdb/cockroach/pkg/util/log"
@@ -2384,3 +2385,13 @@ func (c *clusterImpl) Extend(ctx context.Context, d time.Duration, l *logger.Log
 func (c *clusterImpl) NewMonitor(ctx context.Context, opts ...option.Option) cluster.Monitor {
 	return newMonitor(ctx, c.t, c, opts...)
 }
+
+func (c *clusterImpl) StartGrafana(
+	ctx context.Context, l *logger.Logger, promCfg *prometheus.Config,
+) error {
+	return roachprod.StartGrafana(ctx, l, c.name, "", promCfg)
+}
+
+func (c *clusterImpl) StopGrafana(ctx context.Context, l *logger.Logger, dumpDir string) error {
+	return roachprod.StopGrafana(ctx, l, c.name, dumpDir)
+}

pkg/cmd/roachtest/cluster/BUILD.bazel

@@ -15,6 +15,7 @@ go_library(
         "//pkg/cmd/roachtest/test",
         "//pkg/roachprod/install",
         "//pkg/roachprod/logger",
+        "//pkg/roachprod/prometheus",
         "@com_github_cockroachdb_errors//:errors",
     ],
 )

pkg/cmd/roachtest/cluster/cluster_interface.go

@@ -20,6 +20,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test"
 	"github.com/cockroachdb/cockroach/pkg/roachprod/install"
 	"github.com/cockroachdb/cockroach/pkg/roachprod/logger"
+	"github.com/cockroachdb/cockroach/pkg/roachprod/prometheus"
 )
 
 // Cluster is the interface through which a given roachtest interacts with the
@@ -131,4 +132,7 @@ type Cluster interface {
 
 	FetchTimeseriesData(ctx context.Context, t test.Test) error
 	RefetchCertsFromNode(ctx context.Context, node int) error
+
+	StartGrafana(ctx context.Context, l *logger.Logger, promCfg *prometheus.Config) error
+	StopGrafana(ctx context.Context, l *logger.Logger, dumpDir string) error
 }

pkg/cmd/roachtest/option/node_list_option.go

@@ -16,6 +16,8 @@ import (
 	"math/rand"
 	"sort"
 	"strconv"
+
+	"github.com/cockroachdb/cockroach/pkg/roachprod/install"
 )
 
 // A NodeListOption is a slice of roachprod node identifiers. The first node is
@@ -116,3 +118,12 @@ func (n NodeListOption) String() string {
 	}
 	return buf.String()
 }
+
+// InstallNodes converts the NodeListOption to install.Nodes
+func (n NodeListOption) InstallNodes() install.Nodes {
+	installNodes := make(install.Nodes, 0, len(n))
+	for _, i := range n {
+		installNodes = append(installNodes, install.Node(i))
+	}
+	return installNodes
+}

pkg/cmd/roachtest/tests/tpcc.go

@@ -99,7 +99,6 @@ type workloadInstance struct {
 }
 
 const workloadPProfStartPort = 33333
-const workloadPrometheusPort = 2112
 
 // tpccImportCmd generates the command string to load tpcc data for the
 // specified warehouse count into a cluster.
@@ -183,8 +182,7 @@ func runTPCC(ctx context.Context, t test.Test, c cluster.Cluster, opts tpccOptio
 		workloadInstances = append(
 			workloadInstances,
 			workloadInstance{
-				nodes:          c.Range(1, c.Spec().NodeCount-1),
-				prometheusPort: workloadPrometheusPort,
+				nodes: c.Range(1, c.Spec().NodeCount-1),
 			},
 		)
 	}
@@ -202,7 +200,7 @@ func runTPCC(ctx context.Context, t test.Test, c cluster.Cluster, opts tpccOptio
 			return
 		}
 		cep, err := opts.ChaosEventsProcessor(
-			promCfg.PrometheusNode,
+			c.Nodes(int(promCfg.PrometheusNode[0])),
 			workloadInstances,
 		)
 		if err != nil {
@@ -1401,13 +1399,13 @@ func registerTPCCBench(r registry.Registry) {
 
 // makeWorkloadScrapeNodes creates a ScrapeNode for every workloadInstance.
 func makeWorkloadScrapeNodes(
-	workloadNode option.NodeListOption, workloadInstances []workloadInstance,
+	workloadNode install.Node, workloadInstances []workloadInstance,
 ) []prometheus.ScrapeNode {
 	workloadScrapeNodes := make([]prometheus.ScrapeNode, len(workloadInstances))
 	for i, workloadInstance := range workloadInstances {
 		workloadScrapeNodes[i] = prometheus.ScrapeNode{
-			Nodes: workloadNode,
-			Port:  workloadInstance.prometheusPort,
+			Node: workloadNode,
+			Port: workloadInstance.prometheusPort,
 		}
 	}
 	return workloadScrapeNodes
@@ -1435,14 +1433,13 @@ func setupPrometheusForTPCC(
 		if opts.DisablePrometheus {
 			return nil, func() {}
 		}
-		workloadNode := c.Node(c.Spec().NodeCount)
-		cfg = &prometheus.Config{
-			PrometheusNode: workloadNode,
-			// Scrape each CockroachDB node and the workload node.
-			ScrapeConfigs: append(prometheus.MakeInsecureCockroachScrapeConfig(c.Range(1, c.Spec().NodeCount-1)),
-				prometheus.MakeWorkloadScrapeConfig("workload", makeWorkloadScrapeNodes(workloadNode, workloadInstances)),
-			),
-		}
+		cfg = &prometheus.Config{}
+		workloadNode := c.Node(c.Spec().NodeCount).InstallNodes()[0]
+		cfg.WithPrometheusNode(workloadNode)
+		cfg.WithCluster(c.Range(1, c.Spec().NodeCount-1).InstallNodes())
+		cfg.ScrapeConfigs = append(cfg.ScrapeConfigs, prometheus.MakeWorkloadScrapeConfig("workload",
+			"/", makeWorkloadScrapeNodes(workloadNode, workloadInstances)))
+
 	}
 	if opts.DisablePrometheus {
 		t.Fatal("test has PrometheusConfig but DisablePrometheus was on")
@@ -1451,15 +1448,14 @@ func setupPrometheusForTPCC(
 		t.Skip("skipping test as prometheus is needed, but prometheus does not yet work locally")
 		return nil, func() {}
 	}
-	_, saveSnap, err := prometheus.Init(
-		ctx,
-		*cfg,
-		c,
-		t.L(),
-		repeatRunner{C: c, T: t}.repeatRunE,
-	)
-	if err != nil {
+
+	if err := c.StartGrafana(ctx, t.L(), cfg); err != nil {
 		t.Fatal(err)
 	}
-	return cfg, func() { saveSnap(t.ArtifactsDir()) }
+	cleanupFunc := func() {
+		if err := c.StopGrafana(ctx, t.L(), t.ArtifactsDir()); err != nil {
+			t.Fatal()
+		}
+	}
+	return cfg, cleanupFunc
 }

pkg/gen/gomock.bzl

@@ -6,7 +6,6 @@ GOMOCK_SRCS = [
   "//pkg/kv/kvclient/rangecache/rangecachemock:mocks_generated.go",
   "//pkg/kv/kvclient/rangefeed:mocks_generated_test.go",
   "//pkg/roachpb/roachpbmock:mocks_generated.go",
-  "//pkg/roachprod/prometheus:mocks_generated_test.go",
   "//pkg/security/certmgr:mocks_generated_test.go",
   "//pkg/sql/schemachanger/scexec:mocks_generated_test.go",
   "//pkg/util/log:mocks_generated_test.go",